Commits

Chad Dombrova committed d5b1dfd Merge

Merged orms into external_triggers

Comments (0)

Files changed (5)

denormalize/models.py

 from .orms.base import ModelInspector, inspector, Skip
-from .utils import matches
+from .selectors import ModelGlobber, matches, to_path, from_path
 import logging
 from collections import defaultdict
+from posixpath import join, isabs
 
 log = logging.getLogger(__name__)
 
 
-def to_path(property_path):
-    return '/'.join(x.accessor for x in property_path)
-
-
 class DocumentCollectionBase(object):
     # The root model
     model = None
         # for storing the cache:
         self._relations = None
         self._fields = None
-        # do the caching:
-        self._cache_members()
+        self._globber = ModelGlobber(self.model)
 
     # -- filters
 
     @classmethod
-    def _matches(cls, path, patterns):
-        """Return True if `path` matches any pattern in `patterns`
-
-        :param path: slash (/) separated path
-        :type path: str
-        :param patterns: slash (/) separated path patterns
-        :type patterns: list of str
-        :rtype: bool
-        """
-        return any(x for x in patterns if matches(path, x))
-
-    @classmethod
     def _include(cls, path, includes, excludes):
         """Return True if `path` should be included, based on the
         patterns provided by `includes` and `excludes`
         :rtype: bool
         """
         # must be included and not excluded to pass
-        return cls._matches(path, includes) \
-            and not cls._matches(path, excludes)
+        return any(x for x in includes if matches(path, x)) \
+            and not any(x for x in excludes if matches(path, x))
 
     def include_relation(self, parent_path, relation):
         """Return True if the relationship should be included.
         :type relation: denormalize.orms.RelationProperty
         :rtype: bool
         """
-        path = to_path(parent_path + (relation,))
-        return self._include(path,
-                             self.included_relations,
-                             self.excluded_relations)
+        return True
 
     def include_field(self, parent_path, field):
         """Return True if the field should be included.
         :type field: denormalize.orms.FieldProperty
         :rtype: bool
         """
-        path = to_path(parent_path + (field,))
-        return self._include(path,
-                             self.included_fields,
-                             self.excluded_fields)
+        return True
 
-    def _cache_members(self):
+    @classmethod
+    def _process_selectors(cls):
+        if isinstance(cls.included_relations, dict):
+            def graft(path, subs):
+                # absolute paths are considered private, and thus not grafted
+                return [join(path, s) for s in subs if not isabs(s)]
+
+            included_relations = []
+            excluded_relations = cls.excluded_relations[:]
+            included_fields = cls.included_fields[:]
+            excluded_fields = cls.excluded_fields[:]
+
+            for path, collection in cls.included_relations.iteritems():
+                inrels, exrels, infields, exfields = collection._process_selectors()
+                included_relations.append(path)
+                included_relations += graft(path, inrels)
+                excluded_relations += graft(path, exrels)
+                included_fields += graft(path, infields)
+                excluded_fields += graft(path, exfields)
+
+            return (included_relations, excluded_relations,
+                    included_fields, excluded_fields)
+        else:
+            return (cls.included_relations, cls.excluded_relations,
+                    cls.included_fields, cls.excluded_fields)
+
+    def get_members(self):
         """Inspect the model and use the collection's filters to find the
         fields and relations that will be exported
         """
         if self._relations is None:
-            self._relations = defaultdict(set)
-            self._fields = defaultdict(set)
-            # set of models for finding fields, starting with the current model
-            models = set([((), self.model)])
-            for relation_path in self.model.walk_relations():
+            inrels, exrels, infields, exfields = self._process_selectors()
+            # get relation paths as strings, using a set to get rid of dupes
+            paths = set([])
+            for selector in inrels:
+                for path in self._globber.glob(selector):
+                    if not any(matches(path, pat) for pat in exrels):
+                        paths.add(path)
+
+            # convert path strings to relation tuples and cache them
+            models = {}
+            relations = defaultdict(list)
+            for path in paths:
+                relation_path = tuple(from_path(self.model, path))
                 parent_path = relation_path[:-1]
                 relation = relation_path[-1]
                 if self.include_relation(parent_path, relation):
-                    self._relations[parent_path].add(relation)
-                elif relation.foreign_key_field is not None:
-                    # if 'publisher' relationship is excluded, provide
-                    # 'publisher_id' foreign key field instead
-                    # FIXME: there should be a way to exclude these ids
-                    fk = relation.foreign_key_field
-                    field = relation.model.field_map(foreign_keys=True)[fk]
-                    self._fields[parent_path].add(field)
+                    relations[parent_path].append(relation)
+                    models[relation_path] = relation.related_model
 
-                # different relation paths might lead to the same model
-                models.add((relation_path, relation.related_model))
+            # add the current model
+            models[()] = self.model
+            # cache fields
+            fields = defaultdict(list)
+            for relation_path, model in models.iteritems():
+                for field in model.fields(foreign_keys=True):
+                    # if the foreign key is already represented as a relation,
+                    # then skip it
+                    if field.foreign_key_relation and \
+                            field.foreign_key_relation in \
+                            [r.accessor for r in relations[relation_path]]:
+                        continue
 
-            for parent_path, model in models:
-                for field in model.fields():
-                    if self.include_field(parent_path, field):
-                        self._fields[parent_path].add(field)
-        return self._relations, self._fields
+                    path = to_path(relation_path + (field,))
+                    if self._include(path, infields, exfields) and \
+                            self.include_field(relation_path, field):
+                        fields[relation_path].append(field)
+            self._relations = dict(relations)
+            self._fields = dict(fields)
+        return self._fields, self._relations
 
     def get_related_models(self, include_self=True):
         """Find the models on which the document depends.
         :rtype: iterator of (tuple of denormalize.orms.ModelInspector,
             list of denormalize.orms.RelationProperty)
         """
+        relations = self.get_members()[1]
         models = defaultdict(list)
-        for parent_path, rels in self._relations.iteritems():
+        for parent_path, rels in relations.iteritems():
             for rel in rels:
                 models[rel.related_model].append(parent_path + (rel,))
         if include_self:
         log.debug("dump_obj for %s %s, path '%s'",
                   model.name, model.get_primary_key_value(obj), to_path(path))
 
+        fields, relations = self.get_members()
+
         # Build the output dictionary
         data = {}
 
         # fields
-        for field in self._fields.get(path, []):
+        for field in fields.get(path, []):
             try:
                 data[field.accessor] = model.get_value(obj, field.accessor)
             except Skip:
                 pass
 
         # relations
-        for relation in self._relations.get(path, []):
+        for relation in relations.get(path, []):
             relname = relation.accessor
             try:
                 value = model.get_value(obj, relname)

denormalize/orms/base.py

         self.primary_key = primary_key
         self.foreign_key_relation = foreign_key_relation
 
+    def is_field(self):
+        return True
+
+    def is_relation(self):
+        return False
+
 
 class RelationProperty(Property):
     """Intermediate representation of a relationship between two models
     def has_many(self):
         return self.cardinality.endswith('-to-many')
 
+    def is_field(self):
+        return False
+
+    def is_relation(self):
+        return True
+
 class ModelInspector(object):
     """Provides a common interface for inspecting an ORM model class"""
     _inspectors = []
         the inspector instance"""
         raise NotImplementedError()
 
+    def properties(self):
+        return self.fields() + \
+            self.relations()
+
+    def properity_map(self, by_accessor=True):
+        """dictionary of field name to `FieldProperty`
+
+        :rtype: dict
+        """
+        props = self.properties()
+        if by_accessor:
+            return dict((prop.accessor, prop) for prop in props)
+        else:
+            return dict((prop.name, prop) for prop in props)
+
     def iter_fields(self):
         """iterate over the fields (aka columns) on the model
 

denormalize/selectors.py

+from glob2 import Globber
+import re
+
+_MAXCACHE = 100
+_pattern_cache = {}
+
+# _translate is adapted from stlib fnmatch.translate
+def _translate(pat):
+    """Translate a shell PATTERN to a regular expression.
+
+    There is no way to quote meta-characters.
+    """
+
+    i, n = 0, len(pat)
+    res = ''
+    while i < n:
+        c = pat[i]
+        i = i+1
+        if c == '*':
+            if i < n and pat[i] == '*':
+                i = i+1
+                res = res + '.*'
+            else:
+                res = res + '[^/]*'
+        elif c == '?':
+            res = res + '[^/]'
+        elif c == '[':
+            j = i
+            if j < n and pat[j] == '!':
+                j = j+1
+            if j < n and pat[j] == ']':
+                j = j+1
+            while j < n and pat[j] != ']':
+                j = j+1
+            if j >= n:
+                res = res + '\\['
+            else:
+                stuff = pat[i:j].replace('\\', '\\\\')
+                i = j+1
+                if stuff[0] == '!':
+                    stuff = '^' + stuff[1:]
+                elif stuff[0] == '^':
+                    stuff = '\\' + stuff
+                res = '%s[%s]' % (res, stuff)
+        else:
+            res = res + re.escape(c)
+    return res + '\Z(?ms)'
+
+def matches(path, pattern):
+    if not pattern in _pattern_cache:
+        res = _translate(pattern)
+        if len(_pattern_cache) >= _MAXCACHE:
+            _pattern_cache.clear()
+        _pattern_cache[pattern] = re.compile(res)
+    return _pattern_cache[pattern].match(path) is not None
+
+def from_path(model, path):
+    if path == '/':
+        return
+
+    if path.startswith('/'):
+        path = path[1:]
+
+    parts = path.split('/')
+    # if parts[0] == '':
+    #     parts = parts[1:]
+
+    # if parts[0] == '.':
+    #     parts = parts[1:]
+    result = []
+    n = len(parts)
+    for i, part in enumerate(parts):
+        prop = model.properity_map()[part]
+        try:
+            model = prop.related_model
+        except AttributeError:
+            if i != (n - 1):
+                raise
+        result.append(prop)
+    return result
+
+def to_path(props):
+    return '/'.join([x.accessor for x in props])
+
+
+class ModelGlobber(Globber):
+    def __init__(self, model):
+        self.model = model
+
+    def listdir(self, path):
+        import os
+        prop_path = from_path(self.model, path)
+        if not prop_path:
+            model = self.model
+        else:
+            prop = prop_path[-1]
+            if not prop.is_relation():
+                # Must raise os.error if path is not a directory
+                raise os.error
+            model = prop.related_model
+        props = model.properties()
+        return [p.accessor for p in props]
+
+    def exists(self, path):
+        try:
+            from_path(self.model, path)
+        except (AttributeError, KeyError):
+            return False
+        return True
+
+    def isdir(self, path):
+        # Used only for trailing slash syntax (``foo/``).
+        prop = from_path(self.model, path)
+        return prop.is_relation()
+
+    def islink(self, path):
+        # Used only for recursive glob (``**``).
+        # stop cycles
+        prop_path = from_path(self.model, path)
+        if not prop_path[-1].is_relation():
+            return False
+        models = [self.model]
+        for p in prop_path:
+            model = p.related_model
+            if model in models:
+                return True
+            models.append(model)
+        return False
+
+    def iglob(self, path, with_matches=False):
+        # self.model is the root of all paths, so they can all be treated
+        # as absolute.  this makes the Globber easier to work with.
+        if not path.startswith('/'):
+            path = '/' + path
+        for result in super(ModelGlobber, self).iglob(path, with_matches):
+            yield result[1:]

denormalize/utils.py

 import decimal
 import json
 import datetime
-import re
-
-_MAXCACHE = 100
-_pattern_cache = {}
 
 def to_json(data, indent=2, **kwargs):
     """Serialize data to JSON
     else:
         raise TypeError, 'Object of type %s with value of %s is not ' \
                          'JSON serializable' % (type(obj), repr(obj))
-
-# _translate is adapted from stlib fnmatch.translate
-def _translate(pat):
-    """Translate a shell PATTERN to a regular expression.
-
-    There is no way to quote meta-characters.
-    """
-
-    i, n = 0, len(pat)
-    res = ''
-    while i < n:
-        c = pat[i]
-        i = i+1
-        if c == '*':
-            if i < n and pat[i] == '*':
-                i = i+1
-                res = res + '.*'
-            else:
-                res = res + '[^/]*'
-        elif c == '?':
-            res = res + '[^/]'
-        elif c == '[':
-            j = i
-            if j < n and pat[j] == '!':
-                j = j+1
-            if j < n and pat[j] == ']':
-                j = j+1
-            while j < n and pat[j] != ']':
-                j = j+1
-            if j >= n:
-                res = res + '\\['
-            else:
-                stuff = pat[i:j].replace('\\', '\\\\')
-                i = j+1
-                if stuff[0] == '!':
-                    stuff = '^' + stuff[1:]
-                elif stuff[0] == '^':
-                    stuff = '\\' + stuff
-                res = '%s[%s]' % (res, stuff)
-        else:
-            res = res + re.escape(c)
-    return res + '\Z(?ms)'
-
-def matches(path, pattern):
-    if not pattern in _pattern_cache:
-        res = _translate(pattern)
-        if len(_pattern_cache) >= _MAXCACHE:
-            _pattern_cache.clear()
-        _pattern_cache[pattern] = re.compile(res)
-    return _pattern_cache[pattern].match(path) is not None
       include_package_data=True,
       zip_safe=False,
       install_requires=[
-        'Django >= 1.4' # It might work on 1.3 too, but I have not tested this. Django 1.5 works.
+        'Django >= 1.4', # It might work on 1.3 too, but I have not tested this. Django 1.5 works.
+        'glob2'
       ],
       entry_points="""
       """,