create resultproxy._getter to remove overhead of column lookups

Issue #3175 resolved
Mike Bayer repo owner created an issue

proof of concept, see if we can get into the C code to speed this up

note also it looks like collections.namedtuple() might be faster in the majority of cases, tricky one though as it is slow on the init

diff --git a/lib/sqlalchemy/engine/result.py b/lib/sqlalchemy/engine/result.py
index 06a81aa..8e43709 100644
--- a/lib/sqlalchemy/engine/result.py
+++ b/lib/sqlalchemy/engine/result.py
@@ -268,6 +268,19 @@ class ResultMetaData(object):
         # high precedence keymap.
         keymap.update(primary_keymap)

+    def _getter(self, key):
+        try:
+            processor, obj, index = self._keymap[key]
+        except KeyError:
+            processor, obj, index = self._parent._key_fallback(key)
+
+        if index is None:
+            raise exc.InvalidRequestError(
+                "Ambiguous column name '%s' in result set! "
+                "try 'use_labels' option on select statement." % key)
+
+        return operator.itemgetter(index)
+
     @util.pending_deprecation("0.8", "sqlite dialect uses "
                               "_translate_colname() now")
     def _set_keymap_synonym(self, name, origname):
@@ -517,6 +530,9 @@ class ResultProxy(object):
         """
         return self.context.isinsert

+    def _getter(self, key):
+        return self._metadata._getter(key)
+
     def _cursor_description(self):
         """May be overridden by subclasses."""

diff --git a/lib/sqlalchemy/orm/loading.py b/lib/sqlalchemy/orm/loading.py
index 232eb89..098bf73 100644
--- a/lib/sqlalchemy/orm/loading.py
+++ b/lib/sqlalchemy/orm/loading.py
@@ -12,7 +12,7 @@ the functions here are called primarily by Query, Mapper,
 as well as some of the attribute loading strategies.

 """
-
+from __future__ import absolute_import

 from .. import util
 from . import attributes, exc as orm_exc, state as statelib
@@ -20,6 +20,7 @@ from .interfaces import EXT_CONTINUE
 from ..sql import util as sql_util
 from .util import _none_set, state_str
 from .. import exc as sa_exc
+import collections

 _new_runid = util.counter()

@@ -50,10 +51,13 @@ def instances(query, cursor, context):
     (process, labels) = \
         list(zip(*[
             query_entity.row_processor(query,
-                                       context, custom_rows)
+                                       context, custom_rows, cursor)
             for query_entity in query._entities
         ]))

+    if not custom_rows and not single_entity:
+        keyed_tuple = collections.namedtuple('result', labels)
+
     while True:
         context.progress = {}
         context.partials = {}
@@ -72,8 +76,9 @@ def instances(query, cursor, context):
         elif single_entity:
             rows = [process[0](row, None) for row in fetch]
         else:
-            rows = [util.KeyedTuple([proc(row, None) for proc in process],
-                                    labels) for row in fetch]
+            rows = [
+                keyed_tuple(*[proc(row, None) for proc in process])
+                for row in fetch]

         if filtered:
             rows = util.unique_list(rows, filter_fn)
diff --git a/lib/sqlalchemy/orm/query.py b/lib/sqlalchemy/orm/query.py
index 12e11b2..10097d0 100644
--- a/lib/sqlalchemy/orm/query.py
+++ b/lib/sqlalchemy/orm/query.py
@@ -3082,7 +3082,7 @@ class _MapperEntity(_QueryEntity):

         return ret

-    def row_processor(self, query, context, custom_rows):
+    def row_processor(self, query, context, custom_rows, result):
         adapter = self._get_entity_clauses(query, context)

         if context.adapter and adapter:
@@ -3344,7 +3344,7 @@ class _BundleEntity(_QueryEntity):
         for ent in self._entities:
             ent.setup_context(query, context)

-    def row_processor(self, query, context, custom_rows):
+    def row_processor(self, query, context, custom_rows, result):
         procs, labels = zip(
             *[ent.row_processor(query, context, custom_rows)
               for ent in self._entities]
@@ -3473,15 +3473,17 @@ class _ColumnEntity(_QueryEntity):
     def _resolve_expr_against_query_aliases(self, query, expr, context):
         return query._adapt_clause(expr, False, True)

-    def row_processor(self, query, context, custom_rows):
+    def row_processor(self, query, context, custom_rows, result):
         column = self._resolve_expr_against_query_aliases(
             query, self.column, context)

         if context.adapter:
             column = context.adapter.columns[column]

+        getter = result._getter(column)
+
         def proc(row, result):
-            return row[column]
+            return getter(row)

         return proc, self._label_name

diff --git a/lib/sqlalchemy/orm/strategies.py b/lib/sqlalchemy/orm/strategies.py
index c3edbf6..27dcce5 100644
--- a/lib/sqlalchemy/orm/strategies.py
+++ b/lib/sqlalchemy/orm/strategies.py
@@ -165,8 +165,10 @@ class ColumnLoader(LoaderStrategy):
             if adapter:
                 col = adapter.columns[col]
             if col is not None and col in row:
+                getter = row._parent._getter(col)
+
                 def fetch_col(state, dict_, row):
-                    dict_[key] = row[col]
+                    dict_[key] = getter(row)
                 return fetch_col, None, None
         else:
             def expire_for_non_present_col(state, dict_, row):

Comments (4)

  1. Mike Bayer reporter
    • major refactoring/inlining to loader.instances(), though not really any speed improvements :(. code is in a much better place to be run into C, however
    • The proc() callable passed to the create_row_processor() method of custom :class:.Bundle classes now accepts only a single "row" argument.
    • Deprecated event hooks removed: populate_instance, create_instance, translate_row, append_result
    • the getter() idea is somewhat restored; see ref #3175

    → <<cset c192e447f3a5>>

  2. Log in to comment