Commits

pudo  committed af56315

[model,tests] remove old dependencies and model files

  • Participants
  • Parent commits 80ffc3c
  • Branches mongo

Comments (0)

Files changed (15)

     url='',
     install_requires=[
         "Pylons>=1.0",
-        "SQLAlchemy>=0.5,<=0.5.99",
         "Genshi>=0.5,<=0.5.99",
         "gdata>=2.0.10",
         "PyMongo>=1.9",
 
 # Add additional test specific configuration options as necessary.
 
-sqlalchemy.url = sqlite:///test.db
-# Following is an in-memory database.
-# sqlalchemy.url = sqlite://
+mongodb.database = wdmmg_test
 
 beaker.cache.data_dir = /tmp/wdmmg-test-cache
 

File wdmmg/lib/base.py

         try:
             return WSGIController.__call__(self, environ, start_response)
         finally:
-            pass
+            model.mongo.connection.end_request()
     
     def __before__(self, action, **params):
         c.q = ''

File wdmmg/lib/cli.py

         import pkg_resources, json
         from wdmmgext.load import cofog
         from wdmmgext.load import cra
-        model.repo.delete_all()
-        model.Session.remove()
+        from wdmmgext.load import util
+        model.mongo.drop_db()
         cofog.load_file(pkg_resources.resource_stream('wdmmg', 'tests/COFOG_english_structure_short.txt'))
-        cofog_mapper = cra.CofogMapper(json.load(pkg_resources.resource_stream('wdmmg', 'tests/cofog_map_short.json')))
+        cofog_mapper = util.CofogMapper(json.load(pkg_resources.resource_stream('wdmmg', 'tests/cofog_map_short.json')))
         cra.load_file(pkg_resources.resource_stream('wdmmg', 'tests/cra_2009_db_short.csv'), cofog_mapper)
         cra.load_population(pkg_resources.resource_stream('wdmmg', 'tests/nuts1_population_2006.csv'))
-        model.Session.commit()
-        model.Session.remove()
-        self.dataset_ = (model.Session.query(model.Dataset)
-            .filter_by(name=u'cra')
-            ).one()
-        self.from_ = model.Session.query(model.Key).filter_by(name=u'from').one()
-        self.time = model.Session.query(model.Key).filter_by(name=u'time').one()
-        self.cap_or_cur = model.Session.query(model.Key).filter_by(name=u'cap_or_cur').one()
-        self.pog = model.Session.query(model.Key).filter_by(name=u'pog').one()
-        self.region = model.Session.query(model.Key).filter_by(name=u'region').one()
-        self.cofog1 = model.Session.query(model.Key).filter_by(name=u'cofog1').one()
-        self.cofog2 = model.Session.query(model.Key).filter_by(name=u'cofog2').one()
-        self.cofog3 = model.Session.query(model.Key).filter_by(name=u'cofog3').one()
+        self.dataset_ = model.Dataset.find_one({'name': u'cra'})
     
     @classmethod
     def teardown(self):
         from wdmmg import model
-        model.repo.delete_all()
-        model.Session.remove()
+        model.mongo.drop_db()
 
 
 class Load(WdmmgCommand):
             return res
         check_page('/', 'Welcome')
         check_page('/entry', 'Entries')
-        check_page('/key', 'keys')
-        check_page('/key/time', 'time')
+        check_page('/entity', 'Entities')
+        check_page('/classifier', 'Classifiers')
         check_page('/api/rest/dataset/cra', 'cra')
         check_page('/search?q=cra', 'results')
         check_page('/api/search?q=cra', 'numFound')

File wdmmg/model/atp.py

-from sqlalchemy import Table, Column, ForeignKey
-from sqlalchemy.types import *
-from sqlalchemy.orm import mapper, relation, backref
-import swiss.date
-
-import meta
-from base import DomainObject, PublishedDomainObject
-from keyvalue import add_keyvalues
-
-class Dataset(PublishedDomainObject):
-    def as_dict(self):
-        return {
-            'id': self.id,
-            'name': self.name,
-            'currency': self.currency,
-        }
-
-    def as_big_dict(self):
-        return {
-            'id': self.id,
-            'name': self.name,
-            'currency': self.currency,
-            'metadata': self.metadata,
-            'notes': self.notes,
-        }
-
-class Entry(PublishedDomainObject):
-    # TODO: methods for retrieving 'from' and 'to' account names, and 'time'.
-
-    def as_dict(self):
-        dict_ = self.classification_as_dict()
-        dict_.update(self.keyvalues_as_dict())
-        dict_.update({
-            'id': self.id,
-            'dataset_id': self.dataset_id,
-            'amount': self.amount,
-            'currency': self.currency,
-        })
-        return dict_
-
-    def as_big_dict(self):
-        out = self.as_dict()
-        cis = dict([(ci.value.key.name, ci.value)
-            for ci in self.classificationitems])
-        out.update({
-            'dataset': self.dataset_.name,
-            'notes': self.notes,
-        })
-        if 'region' in cis:
-            out['location'] = cis['region'].name
-            out['location_code'] = cis['region'].code
-        if 'year' in cis and 'time' not in out:
-            out['time'] = cis['year']
-        if 'from' in cis:
-            # HACK: got to be simpler but just for the moment
-            out['from'] = cis['from'].name
-            out['from_code'] = cis['from'].code
-        if 'to' in cis:
-            # HACK: got to be simpler but just for the moment
-            out['to'] = cis['to'].name
-            out['to_code'] = cis['to'].code
-        if 'time' in out:
-            try: # ignore bad dates ... (ugly)
-                ourdt = swiss.date.parse(out['time']).as_datetime()
-                # add Z as solr requires this
-                out['time_norm'] = ourdt.isoformat() + 'Z'
-            except:
-                pass
-        return out
-
-    def classification_as_dict(self):
-        '''
-        Returns all ClassificationItems of this Entry in the form of a
-        dict from Key.names to EnumerationValue.codes.
-        '''
-        return dict([(item.value.key.name, item.value.code)
-            for item in self.classificationitems])
-    
-    def summarise(self):
-        '''
-        Returns a (unicode, dict) pair describing this Entry. The unicode
-        string is a human-readable summary of the key facts about the
-        Entry. The dict is from Key.names to EnumerationValues.
-        '''
-        cis = dict([(ci.value.key.name, ci.value)
-            for ci in self.classificationitems])
-        def value(name):
-            ev = cis[name]
-            if not ev:
-                return u'unknown'
-            elif ev.code == ev.name:
-                return ev.code
-            else:
-                return u'%s (%s)' % (ev.code, ev.name)
-        ans = []
-        currency = self.currency or self.dataset_.currency or ''
-        currency = currency.upper()
-        ans.append(u'%s %.2f was paid' % (currency, self.amount))
-        if 'from' in cis:
-            ans.append(u'\n  from %s' % value('from'))
-        if 'to' in cis:
-            ans.append(u'\n  to %s' % value('to'))
-        if 'time' in cis:
-            ans.append(u'\n  in period %s' % value('time'))
-        ans.append(u'.')
-        ans.append(u'\n\nOther classifiers:')
-        for name in sorted(cis.keys()):
-            if name not in ['from', 'to', 'time']:
-                ans.append(u'\n  %s: %s' % (name, value(name)))
-        if False: # self.is_aggregate == u'Y':
-            ans.append(u'\nThis is an aggregate formed by summing over all other classifiers.')
-        else:
-            ans.append(u'\nAll other classifiers are unknown for this entry.')
-        ans.append(u' Recorded in data set: %s' % self.dataset_.name)
-        return u''.join(ans), cis
-
-table_dataset = Table('dataset', meta.metadata,
-    Column('id', Integer, primary_key=True),
-    Column('name', UnicodeText(), unique=True),
-    Column('long_name', UnicodeText(), unique=True),
-    Column('currency', UnicodeText()),
-    Column('metadata', UnicodeText(),nullable=True),
-    Column('notes', UnicodeText()),
-    )
-
-table_entry = Table('entry', meta.metadata,
-    Column('id', Integer, primary_key=True),
-    Column('dataset_id', Integer, ForeignKey('dataset.id')),
-    Column('amount', Float()),
-    Column('currency', UnicodeText(),nullable=True),
-    Column('notes', UnicodeText()),
-    )
-
-mapper(Dataset, table_dataset,
-    order_by=table_dataset.c.name
-    )
-
-mapper(Entry, table_entry, properties={
-        'dataset_': relation(Dataset, backref=backref('entries')),
-        },
-#    order_by=table_entry.c.timestamp
-    )
-
-add_keyvalues(Entry)
-

File wdmmg/model/base.py

-try:
-    import json
-except:
-    import simplejson as json
-from sqlalchemy.orm import class_mapper
-import sqlalchemy.types as types
-
-
-class DomainObject(object):
-    def __init__(self, **kwargs):
-        for k,v in kwargs.items():
-            setattr(self, k, v)
-    
-    def as_dict(self):
-        '''
-        Returns a summary of this DomainObject suitable for inclusion in a
-        JSON list of search results. The summary should include most fields
-        of the domain object, but may omit potentially large fields such as
-        `notes`. Foreign keys should be included as ids only.
-        
-        Link tables may be treated as part of one of the tables that they link.
-        In particular, KeyValues attached to this DomainObject may be
-        considered part of it, and therefore included in the result returned by
-        this method.
-        
-        The default implementation returns a dict with only the key 'id'.
-        '''
-        return {'id': self.id}
-
-    def __unicode__(self):
-        fields = u', '.join([u'%s=%r' % (k, v) for k, v in self.as_dict().items()])
-        return u'%s(%s, ...)' % (self.__class__.__name__, fields)
-
-    def __str__(self):
-        return self.__unicode__().encode('utf8')
-
-
-class PublishedDomainObject(DomainObject):
-    def keyvalues_as_dict(self):
-        return dict([(key.name, value)
-            for key, value in self.keyvalues.items()])
-
-    def as_big_dict(self):
-        '''
-        Returns detailed information about this DomainObject in a form suitable
-        for converting to JSON. All fields should be included, and foreign keys
-        should generally be replaced by the `as_dict()` representation of their
-        referrent.
-        
-        The default implementation returns `self.as_dict()`.
-        '''
-        return self.as_dict()
-
-
-class JsonType(types.TypeDecorator):
-    '''Store data as JSON serializing on save and unserializing on use.
-    '''
-    impl = types.UnicodeText
-
-    def process_bind_param(self, value, engine):
-        if value is None or value == {}: # ensure we stores nulls in db not json "null"
-            return None
-        else:
-            # ensure_ascii=False => allow unicode but still need to convert
-            return unicode(json.dumps(value, ensure_ascii=False))
-
-    def process_result_value(self, value, engine):
-        if value is None:
-            return None
-        else:
-            return json.loads(value)
-
-    def copy(self):
-        return JsonType(self.impl.length)
-

File wdmmg/model/classification.py

-from sqlalchemy import Table, Column, ForeignKey
-from sqlalchemy.types import *
-from sqlalchemy.orm import mapper, relation, backref
-
-import meta
-from base import DomainObject, PublishedDomainObject
-from atp import Entry
-from keyvalue import EnumerationValue, add_keyvalues
-
-
-class ClassificationItem(DomainObject):
-    pass
-
-table_classification_item = Table('classification_item', meta.metadata,
-    Column('id', Integer, primary_key=True),
-    Column('entry_id', Integer, ForeignKey('entry.id'), index=True),
-    Column('value_id', Integer, ForeignKey('enumeration_value.id'), index=True)
-)
-
-mapper(ClassificationItem, table_classification_item, properties={
-        'entry': relation(Entry, backref=backref('classificationitems')),
-        'value': relation(EnumerationValue, backref=backref('classificationitems')),
-    },
-    order_by=table_classification_item.c.entry_id
-)
-
-
-# add_keyvalues(Classification)
-
-# TODO: Use an attribute_mapped_collection for Classification.items.
-

File wdmmg/model/keyvalue.py

-from sqlalchemy import Table, Column, ForeignKey, and_
-from sqlalchemy.types import *
-from sqlalchemy.orm import mapper, relation, backref, class_mapper
-
-import meta
-from base import PublishedDomainObject, DomainObject
-
-class Key(PublishedDomainObject):
-    def as_dict(self):
-        return {
-            'id': self.id,
-            'name': self.name,
-            'keyvalues': self.keyvalues_as_dict(),
-        }
-    
-    def as_big_dict(self):
-        return {
-            'id': self.id,
-            'name': self.name,
-            'notes': self.notes,
-            'enumeration_values': dict([(ev.code, ev.as_dict())
-                for ev in self.enumeration_values]),
-            'keyvalues': self.keyvalues_as_dict(),
-        }
-
-class EnumerationValue(PublishedDomainObject):
-    def as_dict(self):
-        return {
-            'id': self.id,
-            'key_id': self.key_id,
-            'code': self.code,
-            'name': self.name,
-            'keyvalues': self.keyvalues_as_dict(),
-        }
-    
-    def as_big_dict(self):
-        return {
-            'id': self.id,
-            'key': self.key.as_dict(),
-            'code': self.code,
-            'name': self.name,
-            'notes': self.notes,
-            'keyvalues': self.keyvalues_as_dict(),
-        }
-
-class KeyValue(DomainObject):
-    pass
-
-def add_keyvalues(domain_object, proxy_name='keyvalues',
-        sqlalchemy_property_name='_keyvalues'):
-    '''Add an attribute `proxy_name` and `sqlalchemy_property_name` to 
-    `domain_object`.
-
-    E.g.::
-        add_keyvalues(Entry, 'myproxyname', 'myproperty')
-        # now ...
-        acc = Entry()
-        acc.myproxyname[Key] = u'value'
-        acc.myproperty[Key] = KeyValue(...)
-        # also on KeyValue
-        kv = KeyValue(entry=...)
-        kv.entry 
-    '''
-    mapper = class_mapper(domain_object)
-    table = mapper.local_table
-    table_name = unicode(table.name)
-    primaryjoin = and_(
-        table_key_value.c.ns == table_name,
-        table_key_value.c.object_id== list(table.primary_key)[0]
-        )
-    foreign_keys = [table_key_value.c.object_id]
-    from sqlalchemy.orm.collections import attribute_mapped_collection
-    mapper.add_property(sqlalchemy_property_name, relation(
-        KeyValue,
-        primaryjoin=primaryjoin,
-        foreign_keys=foreign_keys,
-        collection_class=attribute_mapped_collection('key'),
-        backref='ns_'+domain_object.__name__.lower(),
-        cascade='all, delete, delete-orphan'
-        )
-    )
-    from sqlalchemy.ext.associationproxy import association_proxy
-    def _create_keyvalue(key, value):
-        return KeyValue(ns=table_name, key=key, value=value)
-    setattr(domain_object, proxy_name,
-            association_proxy(sqlalchemy_property_name, 'value', creator=_create_keyvalue)
-            )
-
-table_key = Table('key', meta.metadata,
-    Column('id', Integer, primary_key=True),
-    Column('name', UnicodeText(), unique=True),
-    Column('notes', UnicodeText()),
-    )
-
-table_enumeration_value = Table('enumeration_value', meta.metadata,
-    Column('id', Integer, primary_key=True),
-    Column('key_id', Integer, ForeignKey('key.id'), index=True),
-    Column('code', UnicodeText(), index=True),
-    Column('name', UnicodeText()),
-    Column('notes', UnicodeText()),
-    )
-
-table_key_value = Table('key_value', meta.metadata,
-    Column('id', Integer, primary_key=True),
-    Column('ns', UnicodeText()),
-    Column('object_id', Integer, index=True),
-    Column('key_id', Integer, ForeignKey('key.id'), index=True),
-    Column('value', UnicodeText()),
-    )
-
-mapper(Key, table_key,
-    order_by=table_key.c.id
-    )
-
-mapper(EnumerationValue, table_enumeration_value, properties={
-        'key': relation(Key, backref=backref(
-            'enumeration_values',
-            cascade='all, delete, delete-orphan'
-        )),
-    },
-    order_by=[table_enumeration_value.c.key_id,table_enumeration_value.c.code]
-    )
-
-mapper(KeyValue, table_key_value, properties={
-        'key': relation(Key, backref=backref('key_values', cascade='all, delete, delete-orphan')),
-        'enumeration_value': relation(
-            EnumerationValue,
-            primaryjoin=and_(
-                table_key_value.c.key_id == table_enumeration_value.c.key_id,
-                table_key_value.c.value == table_enumeration_value.c.code
-            ),
-            foreign_keys=[table_enumeration_value.c.key_id, table_enumeration_value.c.code],
-            uselist=False
-        ),
-        # Relations on `object_id` can be created by `add_keyvalues()`.
-    },
-    order_by=table_key_value.c.id
-    )
-
-add_keyvalues(Key)
-add_keyvalues(EnumerationValue)
-

File wdmmg/model/meta.py

-"""SQLAlchemy Metadata and Session object"""
-from sqlalchemy import MetaData
-from sqlalchemy.orm import scoped_session, sessionmaker
-
-__all__ = ['Session', 'engine', 'metadata']
-
-# SQLAlchemy database engine. Updated by model.init_model()
-engine = None
-
-# SQLAlchemy session manager. Updated by model.init_model()
-Session = scoped_session(sessionmaker())
-
-# Global metadata. If you have multiple databases with overlapping table
-# names, you'll need a metadata for each database
-metadata = MetaData()

File wdmmg/model/mongo.py

 def db():
     return Database(connection, db_name)
     
+def drop_db():
+    connection.drop_database(db_name)
+    
 
  
 

File wdmmg/tests/__init__.py

 
 # Does not work at the moment! (metadata not bound it seems)
 import wdmmg.model as model
-model.repo.rebuild_db()
+model.mongo.drop_db()
 # Just duplicates rebuild_db
 # Invoke websetup with the current config file
 # SetupCommand('setup-app').run([config['__file__']])

File wdmmg/tests/functional/test_enumeration_value.py

-from wdmmg.tests import *
-from wdmmg import model
-
-class TestEnumerationValueController(TestController):
-
-    @classmethod
-    def setup_class(self):
-        Fixtures.setup()
-    
-    @classmethod
-    def teardown_class(self):
-        Fixtures.teardown()
-
-    def test_view(self):
-        response = self.app.get(url(controller='enumeration_value', action='view',
-            name_or_id=Fixtures.cofog1.name, code=u'07'))
-        assert '''cofog1''' in response, response
-        assert '''07''' in response, response
-        assert '''Health''' in response, response
-

File wdmmg/tests/functional/test_key.py

-from wdmmg.tests import *
-
-class TestKeyController(TestController):
-
-    @classmethod
-    def setup_class(self):
-        Fixtures.setup()
-    
-    @classmethod
-    def teardown_class(self):
-        Fixtures.teardown()
-
-    def test_index(self):
-        response = self.app.get(url(controller='key', action='index'))
-        assert '''The database contains the following keys:''' in response
-        assert '''region''' in response
-
-    def test_view(self):
-        response = self.app.get(url(controller='key', action='view', name_or_id=Fixtures.region.name))
-        assert '''region''' in response
-        assert '''ENGLAND_London''' in response
-    
-    def test_view_paginate(self):
-        response = self.app.get(url(controller='key', action='view', name_or_id=Fixtures.pog.name, items_per_page=3))
-        assert '''Next''' in response
-

File wdmmg/tests/test_aggregator.py

             axes=[Fixtures.cofog1, Fixtures.region],
         )
         print ans
-        key_population = (model.Session.query(model.Key)
-            .filter_by(name=u'population2006')
-            ).one()
-        ans.divide_by_statistic(Fixtures.region, key_population)
+        ans.divide_by_statistic('region', 'population2006')
         print ans
-        assert ans.axes == [u'cofog1', u'region'], ans.axes
+        assert ans.axes == [u'cofog.1.name', u'region'], ans.axes
         index = dict([(coords, amount[-2]) for (coords, amount) in ans.matrix.items()])
         for k, v in index.items():
             print k, v

File wdmmg/tests/test_loader.py

     @classmethod
     def setup_class(self):
         print 'setting up TestLoader'
-        key_colour = model.Key(name=u'colour')
-        key_smell = model.Key(name=u'smell')
-        model.Session.add_all([key_colour, key_smell])
-        model.Session.commit()
         # Load an example data set.
-        cache_colour = loader.ValueCache(key_colour)
-        cache_smell = loader.ValueCache(key_smell)
-        my_loader = loader.Loader(u'test', [key_colour, key_smell], commit_every=2)
-        for (colour, smell), amount in test_data.items():
-            my_loader.create_entry(amount, [
-                cache_colour.get_value_id(colour),
-                cache_smell.get_value_id(smell),
-            ])
-        model.Session.commit()
-        model.Session.remove()
+        my_loader = loader.Loader(u'test', 'Test Dataset')
 
     @classmethod
     def teardown_class(self):
         # Clean up.
-        model.repo.delete_all()
-        model.Session.remove()
+        model.mongo.drop_db()
 
     def test_loader(self):
         print 'test_loader'