Commits

Mike Bayer committed 477511e

merged eager loading overhaul rev 1001:1009
this includes:
sql.Alias object keeps track of the immediate thing it aliased as well
as the ultimate non-aliased (usually a Table) object, so that proxied columns can have
a "parent" attribute
some cleanup to SelectBaseMixin.order_by_clause to allow easier access, needs more cleanup
engine has been making two ResultProxies all this time, added "return_raw" quickie flag to
disable that
some cleanup to _get_col_by_original so that it also works for oid columns, new eager load stuff
more aggressively aliaseses orderby's so this was needed
EagerLoader now makes "chains" of unique aliased eager loaders in all cases. no need for
use_alias/selectalias anymore since it aliases every time.
properly detects recursive eager loads and terminates them with a lazyloader, instead of
raising an exception. totally simplified setup() and init() is more straightforward and has
a single codepath now instead of two or three.

Comments (0)

Files changed (6)

lib/sqlalchemy/ansisql.py

         return self.engine.bindtemplate % name
         
     def visit_alias(self, alias):
-        self.froms[alias] = self.get_from_text(alias.selectable) + " AS " + alias.name
-        self.strings[alias] = self.get_str(alias.selectable)
+        self.froms[alias] = self.get_from_text(alias.original) + " AS " + alias.name
+        self.strings[alias] = self.get_str(alias.original)
 
     def visit_select(self, select):
         

lib/sqlalchemy/engine.py

                 else:
                     parameters = parameters.values()
 
-            self.execute(statement, parameters, connection=connection, cursor=cursor)        
+            self.execute(statement, parameters, connection=connection, cursor=cursor, return_raw=True)        
             return cursor
 
         self.pre_exec(proxy, compiled, parameters, **kwargs)
         self.post_exec(proxy, compiled, parameters, **kwargs)
         return ResultProxy(cursor, self, typemap=compiled.typemap)
 
-    def execute(self, statement, parameters, connection=None, cursor=None, echo=None, typemap=None, commit=False, **kwargs):
+    def execute(self, statement, parameters, connection=None, cursor=None, echo=None, typemap=None, commit=False, return_raw=False, **kwargs):
         """executes the given string-based SQL statement with the given parameters.  
 
         The parameters can be a dictionary or a list, or a list of dictionaries or lists, depending
         except:
             self.do_rollback(connection)
             raise
-        return ResultProxy(cursor, self, typemap=typemap)
+        if return_raw:
+            return cursor
+        else:
+            return ResultProxy(cursor, self, typemap=typemap)
 
     def _execute(self, c, statement, parameters):
         try:

lib/sqlalchemy/mapping/properties.py

 
     """describes an object property that holds a single item or list of items that correspond
     to a related database table."""
-    def __init__(self, argument, secondary, primaryjoin, secondaryjoin, foreignkey=None, uselist=None, private=False, association=None, use_alias=False, selectalias=None, order_by=False, attributeext=None, backref=None, is_backref=False, post_update=False):
+    def __init__(self, argument, secondary, primaryjoin, secondaryjoin, foreignkey=None, uselist=None, private=False, association=None, use_alias=None, selectalias=None, order_by=False, attributeext=None, backref=None, is_backref=False, post_update=False):
         self.uselist = uselist
         self.argument = argument
         self.secondary = secondary
             
         self.private = private
         self.association = association
-        if isinstance(selectalias, str):
-            print "'selectalias' argument to property is deprecated.  please use 'use_alias=True'"
-            self.use_alias = True
-        else:
-            self.use_alias = use_alias
+        if selectalias is not None:
+            print "'selectalias' argument to relation() is deprecated.  eager loads automatically alias-ize tables now."
+        if use_alias is not None:
+            print "'use_alias' argument to relation() is deprecated.  eager loads automatically alias-ize tables now."
         self.order_by = order_by
         self.attributeext=attributeext
         self.backref = backref
         elif self.association is not None:
             c = self.mapper._get_criterion(key, value) & self.primaryjoin
             return c.copy_container()
-
         return None
 
     def register_deleted(self, obj, uow):
 
         if recursion_stack is None:
             recursion_stack = {}
+    
+        self.eagertarget = self.target.alias()
+        if self.secondary:
+            self.eagersecondary = self.secondary.alias()
+            self.aliasizer = Aliasizer(self.target, self.secondary, aliases={
+                    self.target:self.eagertarget,
+                    self.secondary:self.eagersecondary
+                    })
+            self.eagersecondaryjoin = self.secondaryjoin.copy_container()
+            self.eagersecondaryjoin.accept_visitor(self.aliasizer)
+            self.eagerprimary = self.primaryjoin.copy_container()
+            self.eagerprimary.accept_visitor(self.aliasizer)
+        else:
+            self.aliasizer = Aliasizer(self.target, aliases={self.target:self.eagertarget})
+            self.eagerprimary = self.primaryjoin.copy_container()
+            self.eagerprimary.accept_visitor(self.aliasizer)
+        
+        if self.order_by:
+            self.eager_order_by = self._aliasize_orderby(self.order_by)
+        else:
+            self.eager_order_by = None
 
-        if self.use_alias:
-            pass
-
-        # figure out tables in the various join clauses we have, because user-defined
-        # whereclauses that reference the same tables will be converted to use
-        # aliases of those tables
-        self.to_alias = util.HashSet()
-        [self.to_alias.append(f) for f in self.primaryjoin._get_from_objects()]
-        if self.secondaryjoin is not None:
-            [self.to_alias.append(f) for f in self.secondaryjoin._get_from_objects()]
-        try:
-            del self.to_alias[parent.table]
-        except KeyError:
-            pass
-    
-        # if this eagermapper is to select using an "alias" to isolate it from other
-        # eager mappers against the same table, we have to redefine our secondary
-        # or primary join condition to reference the aliased table (and the order_by).  
-        # else we set up the target clause objects as what they are defined in the 
-        # superclass.
-        if self.use_alias:
-            self.eagertarget = self.target.alias()
-            aliasizer = Aliasizer(self.target, aliases={self.target:self.eagertarget})
-            if self.secondaryjoin is not None:
-                self.eagersecondary = self.secondaryjoin.copy_container()
-                self.eagersecondary.accept_visitor(aliasizer)
-                self.eagerprimary = self.primaryjoin.copy_container()
-                self.eagerprimary.accept_visitor(aliasizer)
+        eagerprops = []
+        # create a new "eager chain", starting from this eager loader and descending downwards
+        # through all sub-eagerloaders.  this will copy all those eagerloaders and have them set up
+        # aliases distinct to this eager chain.  if a recursive relationship to any of the tables is detected,
+        # the chain will terminate by copying that eager loader into a lazy loader.
+        for key, prop in self.mapper.props.iteritems():
+            if isinstance(prop, EagerLoader):
+                eagerprops.append(prop)
+        if len(eagerprops):
+            recursion_stack[self.parent.table] = True
+            self.mapper = self.mapper.copy()
+            try:
+                for prop in eagerprops:
+                    if recursion_stack.has_key(prop.target):
+                        # recursion - set the relationship as a LazyLoader
+                        p = EagerLazyOption(None, False).create_prop(self.mapper, prop.key)
+                        continue
+                    p = prop.copy()
+                    self.mapper.props[prop.key] = p
+                    #print "we are:", id(self), self.target.name, (self.secondary and self.secondary.name or "None"), self.parent.table.name
+                    #print "prop is",id(prop), prop.target.name, (prop.secondary and prop.secondary.name or "None"), prop.parent.table.name
+                    p.do_init_subclass(prop.key, prop.parent, recursion_stack)
+                    p.eagerprimary = p.eagerprimary.copy_container()
+                    aliasizer = Aliasizer(p.parent.table, aliases={p.parent.table:self.eagertarget})
+                    p.eagerprimary.accept_visitor(aliasizer)
+                    #print "new eagertqarget", p.eagertarget.name, (p.secondary and p.secondary.name or "none"), p.parent.table.name
+            finally:
+                del recursion_stack[self.parent.table]
+                
+    def _aliasize_orderby(self, orderby, copy=True):
+        if copy:
+            orderby = [o.copy_container() for o in util.to_list(orderby)]
+        else:
+            orderby = util.to_list(orderby)
+        for i in range(0, len(orderby)):
+            if isinstance(orderby[i], schema.Column):
+                orderby[i] = self.eagertarget._get_col_by_original(orderby[i])
             else:
-                self.eagerprimary = self.primaryjoin.copy_container()
-                self.eagerprimary.accept_visitor(aliasizer)
-            if self.order_by:
-                self.eager_order_by = [o.copy_container() for o in util.to_list(self.order_by)]
-                for i in range(0, len(self.eager_order_by)):
-                    if isinstance(self.eager_order_by[i], schema.Column):
-                        self.eager_order_by[i] = self.eagertarget._get_col_by_original(self.eager_order_by[i])
-                    else:
-                        self.eager_order_by[i].accept_visitor(aliasizer)
-            else:
-                self.eager_order_by = self.order_by
-
-            # we have to propigate the "use_alias" fact into 
-            # any sub-mappers that are also eagerloading so that they create a unique tablename
-            # as well.  this copies our child mapper and replaces any eager properties on the 
-            # new mapper with an equivalent eager property, just containing use_alias=True
-            eagerprops = []
-            for key, prop in self.mapper.props.iteritems():
-                if isinstance(prop, EagerLoader) and not prop.use_alias:
-                    eagerprops.append(prop)
-            if len(eagerprops):
-                recursion_stack[self] = True
-                self.mapper = self.mapper.copy()
-                try:
-                    for prop in eagerprops:
-                        p = prop.copy()
-                        p.use_alias=True
-
-                        self.mapper.props[prop.key] = p
-
-                        if recursion_stack.has_key(prop):
-                            raise ArgumentError("Circular eager load relationship detected on " + str(self.mapper) + " " + key + repr(self.mapper.props))
-
-                        p.do_init_subclass(prop.key, prop.parent, recursion_stack)
-
-                        p.eagerprimary = p.eagerprimary.copy_container()
-                        aliasizer = Aliasizer(p.parent.table, aliases={p.parent.table:self.eagertarget})
-                        p.eagerprimary.accept_visitor(aliasizer)
-                finally:
-                    del recursion_stack[self]
-
-        else:
-            self.eagertarget = self.target
-            self.eagerprimary = self.primaryjoin
-            self.eagersecondary = self.secondaryjoin
-            self.eager_order_by = self.order_by
-            
-    def setup(self, key, statement, recursion_stack = None, eagertable=None, **options):
+                orderby[i].accept_visitor(self.aliasizer)
+        return orderby
+        
+    def setup(self, key, statement, eagertable=None, **options):
         """add a left outer join to the statement thats being constructed"""
 
-        if recursion_stack is None:
-            recursion_stack = {}
-        
-        if statement.whereclause is not None:
-            # "aliasize" the tables referenced in the user-defined whereclause to not 
-            # collide with the tables used by the eager load
-            # note that we arent affecting the mapper's table, nor our own primary or secondary joins
-            aliasizer = Aliasizer(*self.to_alias)
-            statement.whereclause.accept_visitor(aliasizer)
-            for alias in aliasizer.aliases.values():
-                statement.append_from(alias)
-
         if hasattr(statement, '_outerjoin'):
             towrap = statement._outerjoin
         else:
             towrap = self.parent.table
 
         if self.secondaryjoin is not None:
-            statement._outerjoin = sql.outerjoin(towrap, self.secondary, self.eagerprimary).outerjoin(self.eagertarget, self.eagersecondary)
+            statement._outerjoin = sql.outerjoin(towrap, self.eagersecondary, self.eagerprimary).outerjoin(self.eagertarget, self.eagersecondaryjoin)
             if self.order_by is False and self.secondary.default_order_by() is not None:
-                statement.order_by(*self.secondary.default_order_by())
+                statement.order_by(*self.eagersecondary.default_order_by())
         else:
             statement._outerjoin = towrap.outerjoin(self.eagertarget, self.eagerprimary)
             if self.order_by is False and self.eagertarget.default_order_by() is not None:
 
         if self.eager_order_by:
             statement.order_by(*util.to_list(self.eager_order_by))
-            
+        elif getattr(statement, 'order_by_clause', None):
+            self._aliasize_orderby(statement.order_by_clause, False)
+                
         statement.append_from(statement._outerjoin)
-        recursion_stack[self] = True
-        try:
-            for key, value in self.mapper.props.iteritems():
-                if recursion_stack.has_key(value):
-                    raise InvalidRequestError("Circular eager load relationship detected on " + str(self.mapper) + " " + key + repr(self.mapper.props))
-                value.setup(key, statement, recursion_stack=recursion_stack, eagertable=self.eagertarget)
-        finally:
-            del recursion_stack[self]
+        for key, value in self.mapper.props.iteritems():
+            value.setup(key, statement, eagertable=self.eagertarget)
             
     def execute(self, instance, row, identitykey, imap, isnew):
         """receive a row.  tell our mapper to look for a new object instance in the row, and attach
 
     def _instance(self, row, imap, result_list=None):
         """gets an instance from a row, via this EagerLoader's mapper."""
-        # if we have an alias for our mapper's table via the use_alias
-        # parameter, we need to translate the 
-        # aliased columns from the incoming row into a new row that maps
-        # the values against the columns of the mapper's original non-aliased table.
-        if self.use_alias:
-            fakerow = {}
-            fakerow = util.DictDecorator(row)
-            for c in self.eagertarget.c:
-                fakerow[c.original] = row[c]
-            row = fakerow
+        fakerow = util.DictDecorator(row)
+        for c in self.eagertarget.c:
+            fakerow[c.parent] = row[c]
+        row = fakerow
         return self.mapper._instance(row, imap, result_list)
 
 class GenericOption(MapperOption):
     def create_prop(self, mapper, key):
         kwargs = util.constructor_args(oldprop)
         mapper.set_property(key, class_(**kwargs ))
+
             
 class EagerLazyOption(GenericOption):
     """an option that switches a PropertyLoader to be an EagerLoader or LazyLoader"""
         newprop = class_.__new__(class_)
         newprop.__dict__.update(oldprop.__dict__)
         newprop.do_init_subclass(key, mapper)
-        if self.kwargs.get('selectalias', None):
-            newprop.use_alias = True
-        elif self.kwargs.get('use_alias', None) is not None:
-            newprop.use_alias = self.kwargs['use_alias']
         mapper.set_property(key, newprop)
 
 class DeferredOption(GenericOption):
         for t in tables:
             self.tables[t] = t
         self.binary = None
-        self.match = False
         self.aliases = kwargs.get('aliases', {})
-
     def get_alias(self, table):
         try:
             return self.aliases[table]
         except:
             return self.aliases.setdefault(table, sql.alias(table))
-
     def visit_compound(self, compound):
-        for i in range(0, len(compound.clauses)):
-            if isinstance(compound.clauses[i], schema.Column) and self.tables.has_key(compound.clauses[i].table):
-                compound.clauses[i] = self.get_alias(compound.clauses[i].table)._get_col_by_original(compound.clauses[i])
-                self.match = True
-
+        self.visit_clauselist(compound)
+    def visit_clauselist(self, clist):
+        for i in range(0, len(clist.clauses)):
+            if isinstance(clist.clauses[i], schema.Column) and self.tables.has_key(clist.clauses[i].table):
+                orig = clist.clauses[i]
+                clist.clauses[i] = self.get_alias(clist.clauses[i].table)._get_col_by_original(clist.clauses[i])
+                if clist.clauses[i] is None:
+                    raise "cant get orig for " + str(orig) + " against table " + orig.table.name + " " + self.get_alias(orig.table).name
     def visit_binary(self, binary):
         if isinstance(binary.left, schema.Column) and self.tables.has_key(binary.left.table):
             binary.left = self.get_alias(binary.left.table)._get_col_by_original(binary.left)
-            self.match = True
         if isinstance(binary.right, schema.Column) and self.tables.has_key(binary.right.table):
             binary.right = self.get_alias(binary.right.table)._get_col_by_original(binary.right)
-            self.match = True
 
 class BinaryVisitor(sql.ClauseVisitor):
     def __init__(self, func):

lib/sqlalchemy/schema.py

         self.default = kwargs.pop('default', None)
         self.foreign_key = None
         self._orig = None
+        self._parent = None
         if len(kwargs):
             raise ArgumentError("Unknown arguments passed to Column: " + repr(kwargs.keys()))
         
     original = property(lambda s: s._orig or s)
+    parent = property(lambda s:s._parent or s)
     engine = property(lambda s: s.table.engine)
      
     def __repr__(self):
         c = Column(name or self.name, self.type, fk, self.default, key = name or self.key, primary_key = self.primary_key, nullable = self.nullable, hidden = self.hidden)
         c.table = selectable
         c._orig = self.original
+        c._parent = self
         if not c.hidden:
             selectable.columns[c.key] = c
             if self.primary_key:
         
     def references(self, table):
         """returns True if the given table is referenced by this ForeignKey."""
-        return table._get_col_by_original(self.column) is not None
+        return table._get_col_by_original(self.column, False) is not None
         
     def _init_column(self):
         # ForeignKey inits its remote column as late as possible, so tables can

lib/sqlalchemy/sql.py

     primary_key = property(lambda self:getattr(self, '_primary_key', False))
     foreign_key = property(lambda self:getattr(self, '_foreign_key', False))
     original = property(lambda self:getattr(self, '_original', self))
+    parent = property(lambda self:getattr(self, '_parent', self))
     columns = property(lambda self:[self])
     def _make_proxy(self, selectable, name=None):
         """creates a new ColumnElement representing this ColumnElement as it appears in the select list
         return Join(self, right, isouter = True, *args, **kwargs)
     def alias(self, name=None):
         return Alias(self, name)
-    def _get_col_by_original(self, column):
+    def _get_col_by_original(self, column, raiseerr=True):
         """given a column which is a schema.Column object attached to a schema.Table object
         (i.e. an "original" column), return the Column object from this 
         Selectable which corresponds to that original Column, or None if this Selectable
         does not contain the column."""
-        return self.original_columns.get(column.original, None)
+        try:
+            return self.original_columns[column.original]
+        except KeyError:
+            if not raiseerr:
+                return None
+            else:
+                raise InvalidRequestError("cant get orig for " + str(column) + " with table " + column.table.id + " from table " + self.id)
+                
     def _get_exported_attribute(self, name):
         try:
             return getattr(self, name)
                 for co in column.columns:
                     cp = self._proxy_column(co)
                     self._orig_cols[co.original] = cp
+        if getattr(self, 'oid_column', None):
+            self._orig_cols[self.oid_column.original] = self.oid_column
     def _exportable_columns(self):
         return []
     def _proxy_column(self, column):
         self.clauses.append(clause)
     def accept_visitor(self, visitor):
         for c in self.clauses:
+            if c is None:
+                raise "oh weird" + repr(self.clauses)
             c.accept_visitor(visitor)
         visitor.visit_clauselist(self)
     def _get_from_objects(self):
         
 class Alias(FromClause):
     def __init__(self, selectable, alias = None):
-        while isinstance(selectable, Alias):
-            selectable = selectable.selectable
+        baseselectable = selectable
+        while isinstance(baseselectable, Alias):
+            baseselectable = baseselectable.selectable
+        self.original = baseselectable
         self.selectable = selectable
         if alias is None:
-            n = getattr(selectable, 'name')
+            n = getattr(self.original, 'name')
             if n is None:
                 n = 'anon'
+            elif len(n) > 15:
+                n = n[0:15]
             alias = n + "_" + hex(random.randint(0, 65535))[2:]
         self.name = alias
         self.id = self.name
     key = property(lambda s: s.name)
     _label = property(lambda s: s.name)
     original = property(lambda s:s.obj.original)
+    parent = property(lambda s:s.obj.parent)
     def accept_visitor(self, visitor):
         self.obj.accept_visitor(visitor)
         visitor.visit_label(self)
 
     engine = property(lambda s: s.column.engine)
     default_label = property(lambda s:s._label)
-    original = property(lambda self:self.column)
+    original = property(lambda self:self.column.original)
+    parent = property(lambda self:self.column.parent)
     columns = property(lambda self:[self.column])
     
     def label(self, name):
             self._orig_cols= {}
             for c in self.columns:
                 self._orig_cols[c.original] = c
+            oid = self.oid_column
+            if oid is not None:
+                self._orig_cols[oid.original] = oid
             return self._orig_cols
             
     oid_column = property(_oid_col)
         if not hasattr(self, attribute):
             l = ClauseList(*clauses)
             setattr(self, attribute, l)
-            self.append_clause(prefix, l)
         else:
             getattr(self, attribute).clauses  += clauses
-    def append_clause(self, keyword, clause):
-        if type(clause) == str:
-            clause = TextClause(clause)
-        self.clauses.append((keyword, clause))
+    def _get_clauses(self):
+        # TODO: this is a little stupid.  make ORDER BY/GROUP BY keywords handled by 
+        # the compiler, make group_by_clause/order_by_clause regular attributes
+        x =[]
+        if getattr(self, 'group_by_clause', None):
+            x.append(("GROUP BY", self.group_by_clause))
+        if getattr(self, 'order_by_clause', None):
+            x.append(("ORDER BY", self.order_by_clause))
+        return x
+    clauses = property(_get_clauses)
     def select(self, whereclauses = None, **params):
         return select([self], whereclauses, **params)
     def _get_from_objects(self):
         for s in self.selects:
             s.group_by(None)
             s.order_by(None)
-        self.clauses = []
         group_by = kwargs.get('group_by', None)
         if group_by:
             self.group_by(*group_by)
         # indicates if this select statement is a subquery as a criterion
         # inside of a WHERE clause
         self.is_where = False
-        self.clauses = []
 
         self.distinct = distinct
         self._text = None
         self.assert_result(l, User, *user_all_result)
         objectstore.clear()
         m = mapper(Item, orderitems, is_primary=True, properties = dict(
-                keywords = relation(mapper(Keyword, keywords), itemkeywords, lazy = False),
+                keywords = relation(mapper(Keyword, keywords), itemkeywords, lazy = False, order_by=[keywords.c.keyword_id]),
             ))
         l = m.select((Item.c.item_name=='item 2') | (Item.c.item_name=='item 5') | (Item.c.item_name=='item 3'), order_by=[Item.c.item_id], limit=2)        
         self.assert_result(l, Item, *[item_keyword_result[1], item_keyword_result[2]])
         """tests eager loading with two relations simulatneously, from the same table.  """
         openorders = alias(orders, 'openorders')
         closedorders = alias(orders, 'closedorders')
+        ordermapper = mapper(Order, orders)
         m = mapper(User, users, properties = dict(
             addresses = relation(mapper(Address, addresses), lazy = False),
             open_orders = relation(mapper(Order, openorders), primaryjoin = and_(openorders.c.isopen == 1, users.c.user_id==openorders.c.user_id), lazy = False),
         items = orderitems
         
         m = mapper(Item, items, properties = dict(
-                keywords = relation(mapper(Keyword, keywords), itemkeywords, lazy=False),
+                keywords = relation(mapper(Keyword, keywords), itemkeywords, lazy=False, order_by=[keywords.c.keyword_id]),
             ))
         l = m.select()
         self.assert_result(l, Item, *item_keyword_result)
 
         m = mapper(Item, items, 
             properties = dict(
-                keywords = relation(mapper(Keyword, keywords), itemkeywords, lazy = False),
+                keywords = relation(mapper(Keyword, keywords), itemkeywords, lazy = False, order_by=[keywords.c.keyword_id]),
             ))
 
         m = mapper(Order, orders, properties = dict(