Commits

Chad Dombrova committed 36426d1

Rough draft of abstracting out the orm from DocumentCollection.

Still need to improve orm API and work on backends and querysets.

Comments (0)

Files changed (4)

denormalize/models.py

 from django.db import models
-from .orms.base import inspector, MultipleValues, ModelInstancePair, Skip
+from .orms.base import ModelInspector, inspector, Skip
 import logging
 
 log = logging.getLogger(__name__)
         return model_info
 
     def queryset(self, prefetch=True):
-        # FIXME:
-        qs = self.model.model.objects
+        # FIXME: django-specific
+        qs = self.model._model.objects
         if prefetch and self.select_related:
             qs = qs.select_related(*self.select_related)
         if prefetch and self.prefetch_related:
         return self.dump(self.model.objects.get(pk=root_pk))
 
     def dump(self, root_obj):
-        if not isinstance(root_obj, self.model.model):
+        if not self.model.is_compatible(root_obj):
             raise ValueError("root_obj is not an instance of self.model")
         return self.dump_obj(self.model, root_obj, path=[])
 
         # TODO: Add follow_extra=[] param
         # TODO: Refactor this code. I'm not happy with how relations are
         #       handled here.
-
         if obj is None:
             return None
 
-        model = inspector(model)
-        if not isinstance(obj, model.model):
-            raise ValueError("obj is not an instance of model")
+        if not isinstance(model, ModelInspector):
+            raise ValueError("%r is not an instance of ModelInspector" % model)
 
-        # FIXME: obj.pk
+        if not model.is_compatible(obj):
+            raise ValueError("%r is not an instance of %s" %
+                             (obj, model._model))
+
+        # FIXME: obj.pk is django-specific
         log.debug("dump_obj for %s %s, path '%s'",
-            model.name, obj.pk, '__'.join(path))
-
-        meta = obj._meta
-
-        # Create a meta field mapping
-        fieldmap = self.model.fieldmap(obj)
+                  model.name, obj.pk, '__'.join(path))
 
         # Filter the fields based on user preferences
         excluded = self._excluded_for_path(path)
-        selected_fields = set(fieldmap.keys()) - excluded
-        selected_fields = self.filter_fields(model, selected_fields)
+
+        # print " " * len(path), model.name, path
+        # print " " * len(path), "rawfields", [f.name for f in meta.fields]
+        # print " " * len(path), "fields   ", list(model.iter_fields(obj))
+        # print " " * len(path), "fkeys    ", list(model.iter_foreign_keys(obj))
+        # print " " * len(path), "one2one  ", list(model.iter_relations(obj, ['one-to-one']))
+        # print " " * len(path), "many2one ", list(model.iter_relations(obj, ['many-to-one']))
+        # print " " * len(path), "one2many ", list(model.iter_relations(obj, ['one-to-many']))
+        # print " " * len(path), "many2many", list(model.iter_relations(obj, ['many-to-many']))
+        # print
 
         # Build the output dictionary
         data = {}
 
-        for fieldname in model.iter_fields():
-            if fieldname in selected_fields:
-                try:
-                    value = model.get_value(obj, fieldname)
-                except Skip:
-                    pass
+        # Class name
+        if self.add_model_name:
+            # FIXME: app_label is django-specific
+            data['_model'] = model._model._meta.app_label + ':' + model.name
 
         fields_to_follow = self._fk_fields_to_follow(path)
-
-        for fieldname in selected_fields:
-            # FIXME: obj.pk
-            log.debug("  value_for_field %s %s %s, path '%s'",
-                      model.name, obj.pk, fieldname, '__'.join(path))
-            try:
-                value = model.field_value(obj, fieldname, fields_to_follow)
-            except Skip:
-                pass
-            else:
-                if isinstance(value, MultipleValues):
-                    # This allows the method to rename the field or
-                    # set multiple fields in the output dict.
-                    for key, val in value.items():
-                        data[key] = val
-                elif isinstance(value, ModelInstancePair):
-                    value = self.dump_obj(value.model, value.instance,
-                                          path + [fieldname])
-                else:
-                    # Simply set the value
-                    data[fieldname] = value
-
-        # Class name
-        if self.add_model_name:
-            # FIXME:
-            data['_model'] = model.model._meta.app_label + ':' + model.name
-
-        # Find reverse OneToOneField relations (they do not appear in fields)
-        # TODO: merge this with the next block
-        for fieldname in fields_to_follow:
-            if fieldname in data:
-                # Probably a ForeignKey that was already dumped
-                continue
-            try:
-                field = getattr(obj, fieldname)
-                if isinstance(field, models.Model):
-                    reldoc = self.dump_obj(field.__class__, field,
-                        path + [fieldname])
-                    data[fieldname] = reldoc
-            except models.ObjectDoesNotExist:
-                # It's probably a OneToOneField, but the relation does not
-                # exist. Ignoring this seems nicer than crashing.
-                # TODO: decide whether crashing is the desired behavior here
-                pass
-
         # Follow allowed relations
         included_relations = set(x[0] for x in
             self._split_selectors(self.prefetch_related, path))
         log.debug("  included_relations = %r", included_relations)
-        m2one = dict((x[0].var_name, x[0]) for x in meta.get_all_related_objects_with_model())
-        m2m = dict((x[0].var_name, x[0]) for x in meta.get_all_related_m2m_objects_with_model())
 
-        print "m2one", m2one.keys()
-        print "m2m", m2m.keys()
-        print "fields", fieldmap.keys()
+        # fields
+        for fieldname in model.iter_fields(obj):
+            if fieldname not in excluded:
+                try:
+                    data[fieldname] = model.get_value(obj, fieldname)
+                except Skip:
+                    pass
 
-        for relname in included_relations:
-            # If the related_name is not set, the path will contain 'chapter' and
-            # the accessor attribute will be 'chapter_set', so we cannot assume
-            # these are the same.
-            accessor = relname
-            if relname in m2one:
-                accessor = m2one[relname].get_accessor_name()
-            if relname in m2m:
-                accessor = m2m[relname].get_accessor_name()
+        # *-to-one relationships
+        for relname, field_id in model.iter_relations(obj, ['one-to-one',
+                                                            'many-to-one']):
+            if relname not in excluded:
+                if relname in fields_to_follow:
+                    try:
+                        value = model.get_value(obj, relname)
+                    except Skip:
+                        pass
+                    else:
+                        data[relname] = self.dump_obj(value.model,
+                                                      value.instance,
+                                                      path + [relname])
+                elif field_id is not None:
+                    # field_id is set if the foreign key is local to obj
+                    try:
+                        data[field_id] = model.get_value(obj, field_id)
+                    except Skip:
+                        pass
 
-            if accessor in data:
-                # Probably a ForeignKey that was already dumped
-                continue
-            doclist = []
-            rel = getattr(obj, accessor)
-            for relobj in rel.all():
+        # *-to-many relationships
+        for relname, _ in model.iter_relations(obj, ['one-to-many',
+                                                     'many-to-many']):
+            if relname in included_relations:
                 try:
-                    reldoc = self.dump_obj(relobj.__class__, relobj, path + [relname])
+                    values = model.get_value(obj, relname)
                 except Skip:
                     pass
                 else:
-                    doclist.append(reldoc)
-            data[relname] = doclist
+                    doclist = []
+                    for value in values:
+                        reldoc = self.dump_obj(value.model, value.instance,
+                                               path + [relname])
+                        doclist.append(reldoc)
+                    data[relname] = doclist
 
         # Allow custom processing on the output dict
-        data = self.transform_data(model.model, obj, data, path)
+        # FIXME: passing django model for backward compatibility
+        data = self.transform_data(model._model, obj, data, path)
 
         return data
 

denormalize/orms/base.py

+
 
 class Skip(Exception):
     """Skip serializing a value"""
         self.model = model
         self.instance = instance
 
-class RelationProperty(object):
-    def __init__(self, model, cardinality):
-        self.model = model
-        self.cardinality = cardinality
 
-class Inspector(object):
+# class Property(object):
+#     def __init__(self, model, obj):
+#         self.name = model
+#         self.obj = obj
+
+# class FieldProperty(Property):
+#     pass
+
+# class RelationProperty(Property):
+#     def __init__(self, model, obj, cardinality):
+#         super(RelationProperty, self).__init__(model, obj)
+#         self.cardinality = cardinality
+
+# class OneToOneRelation(RelationProperty):
+#     pass
+
+class ModelInspector(object):
     def __init__(self, model):
-        self.model = model
+        self._model = model
 
     def __repr__(self):
-        return "{0}({1})".format(self.__class__.__name__, repr(self.model))
+        return "{0}({1})".format(self.__class__.__name__, repr(self._model))
 
     @property
     def name(self):
-        return self.model.__name__
+        return self._model.__name__
+
+    def is_compatible(self, obj):
+        return isinstance(obj, self._model)
+
+    # TODO: move over base methods from DjangoModelInspector when locked down
 
 def inspector(model):
     # FIXME: register inspectors/orms
     from .django import DjangoModelInspector
-    if isinstance(model, Inspector):
+    if isinstance(model, ModelInspector):
         return model
     return DjangoModelInspector(model)

denormalize/orms/django.py

 from __future__ import absolute_import
-from .base import RelationProperty, Inspector, MultipleValues, ModelInstancePair
+from .base import ModelInspector, ModelInstancePair
 from django.db import models
 
 import logging
 
 log = logging.getLogger(__name__)
 
-class DjangoRelationProperty(RelationProperty):
-    pass
+# class DjangoField(FieldProperty):
+#     @property
+#     def name(self):
+#         return self.obj.name
 
-class DjangoModelInspector(Inspector):
+#     @property
+#     def accessor(self):
+#         return self.obj.name
+
+#     def get_value(self, model_instance):
+#         return getattr(model_instance, self.accessor)
+
+# class DjangoRelation(RelationProperty):
+#     @property
+#     def name(self):
+#         return self.obj.name
+
+# class DjangoOnetoOneRelation():
+#     def get_value(self, model_instance):
+#         fk_obj = getattr(model_instance, self.name)
+#         fk_model = self._model.__class__(fk_obj.__class__)
+#         return ModelInstancePair(fk_model, fk_obj)
+
+
+class DjangoModelInspector(ModelInspector):
+    def __init__(self, model):
+        super(DjangoModelInspector, self).__init__(model)
+        self._field_info_cache = {}
+
     @property
     def table_name(self):
-        self.model._meta.db_table
+        self._model._meta.db_table
 
     def get_field_info(self, accessor):
+        if accessor in self._field_info_cache:
+            return self._field_info_cache[accessor]
+
         # Most of the time these are equal
         fieldname = accessor
         info = {}
         #       is causing it.
         # We need to be careful, though: MPTTModel does not have this
         # attribute for some reason.
-        if not getattr(self.model._meta, '_related_objects_cache', True):
-            log.debug("Calling Meta cache fill methods for %s", self.model)
-            self.model._meta._fill_related_objects_cache()
-            self.model._meta._fill_related_many_to_many_cache()
+        if not getattr(self._model._meta, '_related_objects_cache', True):
+            log.debug("Calling Meta cache fill methods for %s", self._model)
+            self._model._meta._fill_related_objects_cache()
+            self._model._meta._fill_related_many_to_many_cache()
 
         # Get field info
         # TODO: show list of valid names if the field is invalid
         try:
             field, field_model, direct, m2m = \
-                self.model._meta.get_field_by_name(fieldname)
+                self._model._meta.get_field_by_name(fieldname)
         except models.FieldDoesNotExist:
             if accessor.endswith('_set'):
                 # If no related_name is specified, the field name will
                 # will be the same.
                 fieldname = accessor[:-4] # strip '_set'
                 field, field_model, direct, m2m = \
-                    self.model._meta.get_field_by_name(fieldname)
+                    self._model._meta.get_field_by_name(fieldname)
             else:
                 raise
 
         info['model'] = self.__class__(model)
         info['m2m'] = m2m
         info['direct'] = direct
+        self._field_info_cache[accessor] = fieldname, info
         return fieldname, info
 
-    def iter_fields(self, obj=None, include_fk_ids=False):
-        # print self.name
-        # print "  object", type(obj._meta)
-        # for field in obj._meta.fields:
-        #     print "   ", field
+    # -- fields
 
-        # print "  model ", type(self.model._meta)
-        # for field in self.model._meta.fields:
-        #     print "   ", field
-
-        # assert obj._meta.fields == self.model._meta.fields
-        # FIXME: can this info can be determined from the model alone?
+    def _iter_fields(self, obj=None):
         if obj is not None:
             meta = obj._meta
         else:
-            meta = self.model._meta
+            meta = self._model._meta
 
         for field in meta.fields:
-            yield field
+            yield field.name, field
 
-    def fieldmap(self, obj=None):
-        # FIXME: cache this
-        return dict([(field.name, field) for field in self.iter_fields(obj)])
+    def iter_fields(self, obj=None):
+        for name, field in self._iter_fields(obj):
+            if not isinstance(field, models.ForeignKey):
+                yield name
 
-    def field_value(self, obj, fieldname, fields_to_follow):
-        field = self.fieldmap(obj)[fieldname]
-        # TODO: Maybe move this check to dump_obj()?
-        if isinstance(field, models.ForeignKey):
-            # (Note that this also matches OneToOneField, which is a subclass)
-            if field.name not in fields_to_follow:
-                # Not following the ForeignKey, only the corresponding
-                # fieldname_id field will be set
-                id_field = '{0}_id'.format(field.name)
-                fields_to_set = {id_field: getattr(obj, id_field)}
-                return MultipleValues(fields_to_set)
-            else:
-                fk_obj = getattr(obj, field.name)
-                fk_model = self.__class__(fk_obj.__class__)
-                return ModelInstancePair(fk_model, fk_obj)
+    # -- relationships
 
-        value = getattr(obj, field.name)
-        return value
+    def _iter_foreign_keys(self, obj=None):
+        for name, field in self._iter_fields(obj):
+            if isinstance(field, models.ForeignKey):
+                field_id = '{0}_id'.format(name)
+                yield name, field_id, field
 
-    # --
+    def iter_foreign_keys(self, obj=None):
+        for name, field_id, field in self._iter_foreign_keys(obj):
+            yield name, field_id
 
-    # def iter_relations(self, obj):
-    #     for rel in obj._meta.get_all_related_objects_with_model():
-    #         relation = x[0]
-    #         name = relation.var_name
-    #         accessor = relation.get_accessor_name()
+    def _iter_relations(self, obj, cardinality, source):
+        # FIXME: split cardinality into 2 args? local_has_many, remote_has_many?
+        if isinstance(cardinality, basestring):
+            cardinality = [cardinality]
+        for c in cardinality:
+            assert c in ('one-to-one', 'one-to-many',
+                                   'many-to-one', 'many-to-many')
+        assert source in ('local', 'remote', 'any')
 
-    #     m2one = dict((x[0].var_name, x[0]) for x in )
-    #     m2m = dict((x[0].var_name, x[0]) for x in meta.get_all_related_m2m_objects_with_model())
+        if obj is not None:
+            meta = obj._meta
+        else:
+            meta = self._model._meta
 
-    #     m2one = dict((x[0].var_name, x[0]) for x in meta.get_all_related_objects_with_model())
-    #     m2m = dict((x[0].var_name, x[0]) for x in meta.get_all_related_m2m_objects_with_model())
-    #     for relname in included_relations:
-    #         # If the related_name is not set, the path will contain 'chapter' and
-    #         # the accessor attribute will be 'chapter_set', so we cannot assume
-    #         # these are the same.
-    #         accessor = relname
-    #         if relname in m2one:
-    #             accessor = m2one[relname].get_accessor_name()
-    #         if relname in m2m:
-    #             accessor = m2m[relname].get_accessor_name()
+        # FIXME: look into combining these loops
+        if 'one-to-one' in cardinality:
+            if source in ('any', 'remote'):
+                for rel, model in meta.get_all_related_objects_with_model():
+                    if isinstance(rel.field, models.OneToOneField):
+                        yield rel.get_accessor_name(), None, rel
 
-    #         if accessor in data:
-    #             # Probably a ForeignKey that was already dumped
-    #             continue
-    #         doclist = []
-    #         rel = getattr(obj, accessor)
-    #         for relobj in rel.all():
-    #             try:
-    #                 reldoc = self.dump_obj(relobj.__class__, relobj, path + [relname])
-    #             except Skip:
-    #                 pass
-    #             else:
-    #                 doclist.append(reldoc)
+            if source in ('any', 'local'):
+                for rel, field_id, field in self._iter_foreign_keys(obj):
+                    if isinstance(field, models.OneToOneField):
+                        yield rel, field_id, field
+
+        if 'many-to-one' in cardinality:
+            if source in ('any', 'local'):
+                for rel, field_id, field in self._iter_foreign_keys(obj):
+                    if not isinstance(field, models.OneToOneField):
+                        yield rel, field_id, field
+
+        if 'one-to-many' in cardinality:
+            if source in ('any', 'remote'):
+                for rel, model in meta.get_all_related_objects_with_model():
+                    if not isinstance(rel.field, models.OneToOneField):
+                        yield rel.get_accessor_name(), None, rel
+
+        if 'many-to-many' in cardinality:
+            if source in ('any', 'remote'):
+                for rel, model in meta.get_all_related_m2m_objects_with_model():
+                    yield rel.get_accessor_name(), None, rel
+            if source in ('any', 'local'):
+                for rel, model in meta.get_m2m_with_model():
+                    yield rel.name, None, rel
+
+    def iter_relations(self, obj, cardinality, source='any'):
+        for relname, field_id, _ in self._iter_relations(obj, cardinality,
+                                                         source):
+            yield relname, field_id
+
+    def get_value(self, obj, accessor):
+        value = getattr(obj, accessor)
+        if isinstance(value, models.Model):
+            model = self.__class__(value.__class__)
+            return ModelInstancePair(model, value)
+        elif isinstance(value, models.Manager):
+            result = []
+            for res in value.all():
+                model = self.__class__(res.__class__)
+                result.append(ModelInstancePair(model, res))
+            return result
+        else:
+            return value

denormalize/tests/test_collections.py

 
     def dump_obj(self, model, obj, path):
         # Customization: for tags, use the name string as the representation
-        if model is Tag:
+        if model.name == 'Tag':
             return obj.name
         # Here only the url
-        if model is PublisherLink:
+        if model.name == 'PublisherLink':
             return obj.url
         return super(BookCollection, self).dump_obj(model, obj, path)
 
                 rel = 'category'
             self.assertTrue(rel in deps, "Relation {0} not found!".format(rel))
         pprint(deps)
-        self.assertIs(deps['publisher']['model'].model, Publisher)
-        self.assertIs(deps['extra_info']['model'].model, ExtraBookInfo)
-        self.assertIs(deps['chapter']['model'].model, Chapter)
-        self.assertIs(deps['authors']['model'].model, Author)
-        self.assertIs(deps['tags']['model'].model, Tag)
-        self.assertIs(deps['publisher__links']['model'].model, PublisherLink)
+        self.assertIs(deps['publisher']['model']._model, Publisher)
+        self.assertIs(deps['extra_info']['model']._model, ExtraBookInfo)
+        self.assertIs(deps['chapter']['model']._model, Chapter)
+        self.assertIs(deps['authors']['model']._model, Author)
+        self.assertIs(deps['tags']['model']._model, Tag)
+        self.assertIs(deps['publisher__links']['model']._model, PublisherLink)