create resultproxy._getter to remove overhead of column lookups
Issue #3175
resolved
proof of concept, see if we can get into the C code to speed this up
note also it looks like collections.namedtuple() might be faster in the majority of cases, tricky one though as it is slow on the init
diff --git a/lib/sqlalchemy/engine/result.py b/lib/sqlalchemy/engine/result.py
index 06a81aa..8e43709 100644
--- a/lib/sqlalchemy/engine/result.py
+++ b/lib/sqlalchemy/engine/result.py
@@ -268,6 +268,19 @@ class ResultMetaData(object):
# high precedence keymap.
keymap.update(primary_keymap)
+ def _getter(self, key):
+ try:
+ processor, obj, index = self._keymap[key]
+ except KeyError:
+ processor, obj, index = self._parent._key_fallback(key)
+
+ if index is None:
+ raise exc.InvalidRequestError(
+ "Ambiguous column name '%s' in result set! "
+ "try 'use_labels' option on select statement." % key)
+
+ return operator.itemgetter(index)
+
@util.pending_deprecation("0.8", "sqlite dialect uses "
"_translate_colname() now")
def _set_keymap_synonym(self, name, origname):
@@ -517,6 +530,9 @@ class ResultProxy(object):
"""
return self.context.isinsert
+ def _getter(self, key):
+ return self._metadata._getter(key)
+
def _cursor_description(self):
"""May be overridden by subclasses."""
diff --git a/lib/sqlalchemy/orm/loading.py b/lib/sqlalchemy/orm/loading.py
index 232eb89..098bf73 100644
--- a/lib/sqlalchemy/orm/loading.py
+++ b/lib/sqlalchemy/orm/loading.py
@@ -12,7 +12,7 @@ the functions here are called primarily by Query, Mapper,
as well as some of the attribute loading strategies.
"""
-
+from __future__ import absolute_import
from .. import util
from . import attributes, exc as orm_exc, state as statelib
@@ -20,6 +20,7 @@ from .interfaces import EXT_CONTINUE
from ..sql import util as sql_util
from .util import _none_set, state_str
from .. import exc as sa_exc
+import collections
_new_runid = util.counter()
@@ -50,10 +51,13 @@ def instances(query, cursor, context):
(process, labels) = \
list(zip(*[
query_entity.row_processor(query,
- context, custom_rows)
+ context, custom_rows, cursor)
for query_entity in query._entities
]))
+ if not custom_rows and not single_entity:
+ keyed_tuple = collections.namedtuple('result', labels)
+
while True:
context.progress = {}
context.partials = {}
@@ -72,8 +76,9 @@ def instances(query, cursor, context):
elif single_entity:
rows = [process[0](row, None) for row in fetch]
else:
- rows = [util.KeyedTuple([proc(row, None) for proc in process],
- labels) for row in fetch]
+ rows = [
+ keyed_tuple(*[proc(row, None) for proc in process])
+ for row in fetch]
if filtered:
rows = util.unique_list(rows, filter_fn)
diff --git a/lib/sqlalchemy/orm/query.py b/lib/sqlalchemy/orm/query.py
index 12e11b2..10097d0 100644
--- a/lib/sqlalchemy/orm/query.py
+++ b/lib/sqlalchemy/orm/query.py
@@ -3082,7 +3082,7 @@ class _MapperEntity(_QueryEntity):
return ret
- def row_processor(self, query, context, custom_rows):
+ def row_processor(self, query, context, custom_rows, result):
adapter = self._get_entity_clauses(query, context)
if context.adapter and adapter:
@@ -3344,7 +3344,7 @@ class _BundleEntity(_QueryEntity):
for ent in self._entities:
ent.setup_context(query, context)
- def row_processor(self, query, context, custom_rows):
+ def row_processor(self, query, context, custom_rows, result):
procs, labels = zip(
*[ent.row_processor(query, context, custom_rows)
for ent in self._entities]
@@ -3473,15 +3473,17 @@ class _ColumnEntity(_QueryEntity):
def _resolve_expr_against_query_aliases(self, query, expr, context):
return query._adapt_clause(expr, False, True)
- def row_processor(self, query, context, custom_rows):
+ def row_processor(self, query, context, custom_rows, result):
column = self._resolve_expr_against_query_aliases(
query, self.column, context)
if context.adapter:
column = context.adapter.columns[column]
+ getter = result._getter(column)
+
def proc(row, result):
- return row[column]
+ return getter(row)
return proc, self._label_name
diff --git a/lib/sqlalchemy/orm/strategies.py b/lib/sqlalchemy/orm/strategies.py
index c3edbf6..27dcce5 100644
--- a/lib/sqlalchemy/orm/strategies.py
+++ b/lib/sqlalchemy/orm/strategies.py
@@ -165,8 +165,10 @@ class ColumnLoader(LoaderStrategy):
if adapter:
col = adapter.columns[col]
if col is not None and col in row:
+ getter = row._parent._getter(col)
+
def fetch_col(state, dict_, row):
- dict_[key] = row[col]
+ dict_[key] = getter(row)
return fetch_col, None, None
else:
def expire_for_non_present_col(state, dict_, row):
Comments (4)
-
reporter -
reporter OK it's not the getter, its just the hash on annotated columns. will fix that. namedtuple is
#3176. -
reporter - changed status to resolved
→ <<cset becd78503f09>>
-
reporter - major refactoring/inlining to loader.instances(), though not really any speed improvements :(. code is in a much better place to be run into C, however
- The
proc()
callable passed to thecreate_row_processor()
method of custom :class:.Bundle
classes now accepts only a single "row" argument. - Deprecated event hooks removed:
populate_instance
,create_instance
,translate_row
,append_result
- the getter() idea is somewhat restored; see ref
#3175
→ <<cset c192e447f3a5>>
- Log in to comment