Commits

Anonymous committed b5c02af

[model,lib,controllers] fix home page, search, indexing and the loader

Comments (0)

Files changed (7)

wdmmg/config/environment.py

 import wdmmg.lib.app_globals as app_globals
 import wdmmg.lib.helpers
 from wdmmg.config.routing import make_map
-from wdmmg.model import init_model, init_mongo
+from wdmmg.model import init_mongo
 
 def load_environment(global_conf, app_conf):
     """Configure the Pylons environment via the ``pylons.config``
     # Create the Genshi TemplateLoader
     config['pylons.app_globals'].genshi_loader = TemplateLoader(
         paths['templates'], auto_reload=True)
-
-    # Setup the SQLAlchemy database engine
-    engine = engine_from_config(config, 'sqlalchemy.')
-    init_model(engine)
     
     init_mongo(config)
 

wdmmg/controllers/home.py

 class HomeController(BaseController):
 
     def index(self):
-        c.dataset_count = model.Session.query(model.Dataset).count()
-        c.entry_count = model.Session.query(model.Entry).count()
-        c.key_count = model.Session.query(model.Key).count()
+        c.q = ''
+        c.dataset_count = model.Dataset.find().count()
+        c.entry_count = model.Entry.find().count() 
         return render('home/index.html')
 
     def govspending(self):

wdmmg/controllers/search.py

         c.q = request.params.get('q', None)
 
         c.dataset_options = [ ('*', 'All') ] +  [
-            (x.name,x.name) for x in model.Session.query(model.Dataset).all()]
+            (x.name,x.name) for x in model.Dataset.find()]
         c.dataset = request.params.get('dataset', '*')
 
         c.facet_1_options = [ ('from','from'), ('to','to'), (None, '') ]
         query = app_globals.solr.query(ourq, fq='dataset:%s' % c.dataset, rows=c.items_per_page,
                 start=start, sort='score desc, amount desc', **kwargs)
         c.results = query.results
+        c.amounts = []
         if c.q and query.numFound > 0:
             c.amounts = self._get_totals()
         if c.facet_1:

wdmmg/lib/helpers.py

 
 def convert_search_result(result):
     entry_id = result.get('id', '')
-    entry = (model.Session.query(model.Entry)
-        .filter_by(id=entry_id)
-        ).first()
-    # HACK: to allow us to display remove search results
-    # just got data from solr with no record in local db
+    entry = model.Entry.c.find_one({'_id': entry_id})
     if entry is None:
         return result
-    return entry.as_big_dict()
+    return entry
 
 def format_number(number):
     '''Format a number with m,b,k etc.

wdmmg/lib/loader.py

 from wdmmg import model
-from wdmmg.model import Dataset, Entry, Entity
+from wdmmg.model import Dataset, Entry, Entity, Classifier
 
 
 # TODO: Rethink Loader API.
     avoid database traffic.
     '''
     
-    def __init__(self, dataset_name, notes=u'', metadata=None, \
+    def __init__(self, dataset_name, description=u'', metadata=None, \
                  dataset_long_name=None, currency=u'gbp'):
         '''
         Constructs a Loader for a new Dataset `dataset_name`. (Raises an exception
         entity = Entity.find_one({'name': name})
         kwargs.update({'name': name, 
                        'label': label,
-                       'description': notes})
+                       'description': description})
         if entity is None:
             entity = Entity(**kwargs)
         else:
             entity.update(kwargs)
         Entity.c.update({'name': entity.name}, entity, upsert=True)
         return Entity.find_one({'name': name})
+
+    
+    def create_classifier(self, name, label=u'', description=u'', **kwargs):
+        classifier = Classifier.find_one({'name': name})
+        kwargs.update({'name': name, 
+                      'label': label,
+                      'description': description})
+        if classifier is None:
+            classifier = Classifier(**kwargs)
+        else:
+            classifier.update(kwargs)
+        Classifier.c.update({'name': classifier.name}, classifier, upsert=True)
+        return Classifier.find_one({'name': name})
     
     
     def compute_aggregates(self):

wdmmg/lib/solrhelp.py

 SOLR_CORE_FIELDS = ['id', 'dataset', 'amount', 'time', 'location', 'from',
 'to', 'notes' ]
 
-def build_index(
-    dataset_name=None,
-    solr=None
-):
+def dict_to_indexable(orig, sep='.'):
+    from pymongo.dbref import DBRef
+    """ Flatten down a dictionary with some smartness. """
+    flat = {}
+    for k, v in orig.items():
+        if isinstance(v, DBRef):
+            v = v.as_doc().to_dict()
+        k = k.replace('$', '')
+        if k == '_id': 
+            k = 'id'
+        if isinstance(v, dict):
+            if 'name' in v.keys():
+                flat[k] = v['name']
+                del v['name']
+            for sk, sv in dict_to_indexable(v, sep=sep).items():
+                flat[k + sep + sk] = sv
+        #elif isinstance(v, dict):
+        else:
+            flat[k] = unicode(v)
+    return flat
+
+
+def build_index(dataset_name=None, solr=None):
     dataset_name = dataset_name or unicode(config.get('default_dataset', u'cra'))
     solr = solr or solr_connection()
-    dataset_ = model.Session.query(model.Dataset).filter_by(name=dataset_name).one()
+    dataset_ = model.Dataset.find_one({'name': dataset_name})
+    assert dataset_, "No such dataset: %s" % dataset_name
     dataset_name = dataset_.name
-    model.Session.remove()
-    q = model.Session.query(model.Entry).filter_by(dataset_=dataset_)
+    cur = model.Entry.find({'dataset.name': dataset_name})
     buf = []
     total = 0
     increment = 500
-    for entry in q.yield_per(increment):
-        ourdata = entry.as_big_dict()
+    for entry in cur:
+        ourdata = dict_to_indexable(entry)
+        from pprint import pprint
+        #pprint(ourdata)
         buf.append(ourdata)
         if len(buf) == increment:
             print 'Writing %d records...' % len(buf)

wdmmg/model/__init__.py

 """The application's model objects"""
-import sqlalchemy as sa
-from sqlalchemy import orm
-
-from wdmmg.model import meta
-from meta import Session
-#from atp import Dataset, Entry
-from keyvalue import Key, EnumerationValue, KeyValue
-from classification import ClassificationItem
-
-import meta 
-
 import mongo
 
 from wdmmg.model.mongo import Base, Explanation
     port = config.get('mongodb.port', 27017)
     mongo.make_connection(host, port) 
     mongo.db_name = config.get('mongodb.database', 'wdmmg')
-
-
-def init_model(engine):
-    '''Call me before using any of the tables or classes in the model'''
-    meta.Session.configure(bind=engine)
-    meta.metadata.bind = engine
-    meta.engine = engine
-
-
-class Repository(object):
-    def create_db(self):
-        meta.metadata.create_all()
-    
-    def init_db(self):
-        self.create_db()
-    
-    def clean_db(self):
-        meta.metadata.drop_all()
-
-    def rebuild_db(self):
-        self.clean_db()
-        self.init_db()
-    
-    def delete_all(self):
-        for obj in [ ClassificationItem, Entry, Dataset,
-            KeyValue, EnumerationValue,  Key]:
-            Session.query(obj).delete()
-        Session.commit()
-        Session.remove()
-
-repo = Repository()
-