Commits

Vladimir Mihailenco  committed ceafe4c Merge

Merge with upstream

  • Participants
  • Parent commits e38a0a7, 48589f5

Comments (0)

Files changed (8)

File dbindexer/api.py

-from django.db import models
-from djangotoolbox.fields import ListField
-from copy import deepcopy
+from lookups import LookupDoesNotExist, ExtraFieldLookup
+import lookups as lookups_module
+from resolver import resolver
+import inspect
 
-import re
-regex = type(re.compile(''))
-
-FIELD_INDEXES = {}
-COLUMN_TO_NAME = {}
-
-def get_index_name(field_name, lookup_type):
-    if lookup_type in ('iexact', 'istartswith'):
-        index_name = 'idxf_%s_l_icase' % (field_name)
-    elif isinstance(lookup_type, regex) or lookup_type in ('regex', 'iregex'):
-        index_name = 'idxf_%s_l_%s' % (field_name, 'regex')
-    else:
-        index_name = 'idxf_%s_l_%s' % (field_name, lookup_type)
-
-    return index_name
-
-def get_denormalization_info(start_model, name):
-    denormalized_model = start_model
-    for value in name.split('__')[:-1]:
-        denormalized_model = denormalized_model._meta.get_field(value).rel.to
-
-    return denormalized_model, denormalized_model._meta.get_field(name.split('__')[-1])
-
-def get_column_name(start_model, name):
-    # returns the column name for name or a chain of column names if name contains
-    # '__' in the case of JOINs
-    denormalized_model = start_model
-    column_name = ''
-    for value in name.split('__')[:-1]:
-        column_name += denormalized_model._meta.get_field(value).column + '__'
-        denormalized_model = denormalized_model._meta.get_field(value).rel.to
-
-    return column_name + denormalized_model._meta.get_field(name.split('__')[-1]).column
+# TODO: add possibility to add lookup modules
+def create_lookup(lookup_def):
+    for _, cls in inspect.getmembers(lookups_module):
+        if inspect.isclass(cls) and issubclass(cls, ExtraFieldLookup) and \
+                cls.matches_lookup_def(lookup_def):
+            return cls()
+    raise LookupDoesNotExist('No Lookup found for %s .' % lookup_def)
 
 def register_index(model, mapping):
-    for name, lookup_types in mapping.items():
-        regex_index = False
-        if isinstance(lookup_types, basestring):
-            lookup_types = (lookup_types,)
-
-        denormalized_model = None
-        if len(name.split('__', 1)) > 1:
-            # foreign key case
-            denormalized_model, field = get_denormalization_info(model, name)
-        else:
-            field = model._meta.get_field(name)
-
-        column_name = get_column_name(model, name)
-        COLUMN_TO_NAME[column_name] = name
-        name = column_name
-        new_lookup_types = list(lookup_types)
-        for lookup_type in lookup_types:
-            # TODO: for denormalization the index_name should not contain '__'
-            index_name = get_index_name(name, lookup_type)
-            if lookup_type in ('month', 'day', 'year', 'week_day'):
-                index_field = models.IntegerField(editable=False, null=True)
-            elif isinstance(lookup_type, regex):
-                lookup_type = re.compile(lookup_type.pattern, re.S | re.U |
-                    (lookup_type.flags & re.I))
-                # add (i)regex lookup type to map for later conversations
-                if not lookup_type.flags & re.I and 'regex' not in new_lookup_types:
-                    new_lookup_types.append('regex')
-                elif lookup_type.flags & re.I and 'iregex' not in new_lookup_types:
-                    new_lookup_types.append('iregex')
-                # for each indexed field only add one list field shared by all
-                # regexes
-                if regex_index:
-                    continue
-                index_field = ListField(models.CharField(
-                    max_length=256), editable=False, null=True)
-                regex_index = True
-            elif lookup_type == 'contains':
-                # in the case of foreignkey we do not know which max_length to use
-                # so use 500
-                index_field = ListField(models.CharField(
-                    max_length=field.max_length or 500), editable=False, null=True)
-            elif lookup_type == '$default':
-                # TODO: rename $default because it will be used for the field name
-                # and not every database allows to use the sign $
-                index_field = deepcopy(field)
-                if isinstance(index_field, (models.DateTimeField,
-                        models.DateField, models.TimeField)):
-                    index_field.auto_now_add = index_field.auto_now = False
-            else:
-                # in the case of foreignkey we do not know which max_length to use
-                # so use 500
-                index_field = models.CharField(max_length=field.max_length or 500,
-                    editable=False, null=True)
-            model.add_to_class(index_name, index_field)
-
-        if denormalized_model:
-            # denormalization case, for denormalized fields we append
-            # 'denormalized__' to the lookup_type so dbindexer knows that it
-            # should denormalize when saving an entity of the denormalized model
-            denormalization_lookup_types = []
-            for lookup_type in new_lookup_types:
-                denormalization_lookup_types.append('denormalized__%s' % lookup_type)
-            FIELD_INDEXES.setdefault(denormalized_model, {})
-            if field.column not in FIELD_INDEXES[denormalized_model]:
-                FIELD_INDEXES[denormalized_model][field.column] = denormalization_lookup_types
-            else:
-                FIELD_INDEXES[denormalized_model][field.column].extend(denormalization_lookup_types)
-
-        # set new lookup_types (can be different because of regex lookups)
-        FIELD_INDEXES.setdefault(model, {})[name] = new_lookup_types
-
-    # debug info
-#    print COLUMN_TO_NAME
-#    print FIELD_INDEXES
-#    print [(field.name, field.__class__.__name__) for field in model._meta.fields]
+    for field_name, lookups in mapping.items():
+        if not isinstance(lookups, (list, tuple)):
+            lookups = (lookups, )
+            
+        # create indexes and add model and field_name to lookups
+        # create ExtraFieldLookup instances on the fly if needed
+        for lookup in lookups:
+            lookup_def = None
+            if not isinstance(lookup, ExtraFieldLookup):
+                lookup_def = lookup
+                lookup = create_lookup(lookup_def)
+            lookup.contribute(model, field_name, lookup_def)
+            resolver.create_index(lookup)

File dbindexer/backends.py

+from django.db import models
+from django.db.models.fields import FieldDoesNotExist
+from django.db.models.sql.constants import JOIN_TYPE, LHS_ALIAS, LHS_JOIN_COL, \
+    TABLE_NAME, RHS_JOIN_COL
+from django.utils.tree import Node
+from djangotoolbox.fields import ListField
+from dbindexer.lookups import StandardLookup
+
+OR = 'OR'
+
+# TODO: optimize code
+class BaseResolver(object):
+    def __init__(self):
+        # mapping from lookups to indexes
+        self.index_map = {}
+        # mapping from column names to field names
+        self.column_to_name = {}
+        
+    ''' API called by resolver''' 
+    
+    def create_index(self, lookup):
+        field_to_index = self.get_field_to_index(lookup.model, lookup.field_name)
+        
+        # backend doesn't now how to handle this index definition
+        if not field_to_index:
+            return 
+        
+        index_field = lookup.get_field_to_add(field_to_index)        
+        config_field = index_field.item_field if \
+            isinstance(index_field, ListField) else index_field  
+        if hasattr(field_to_index, 'max_length') and \
+                isinstance(config_field, models.CharField):
+            config_field.max_length = field_to_index.max_length
+            
+        # don't install a field if it already exists
+        try:
+            lookup.model._meta.get_field(self.index_name(lookup))
+        except:
+            lookup.model.add_to_class(self.index_name(lookup), index_field)
+            self.index_map[lookup] = index_field
+            self.add_column_to_name(lookup.model, lookup.field_name)
+        else:
+            if lookup not in self.index_map:
+                self.index_map[lookup] = lookup.model._meta.get_field(
+                    self.index_name(lookup))
+                self.add_column_to_name(lookup.model, lookup.field_name)
+                
+        
+
+    def convert_query(self, query):
+        '''Converts a database saving query.'''
+        
+        for lookup in self.index_map.keys():
+            if not lookup.model == query.model:
+                continue
+            
+            position = self.get_query_position(query, lookup)
+            if position is None:
+                return
+            
+            value = self.get_value(lookup.model, lookup.field_name, query)
+            value = lookup.convert_value(value)
+            query.values[position] = (self.get_index(lookup), value)
+    
+    def convert_filters(self, query):
+        self._convert_filters(query, query.where)
+
+    ''' helper methods '''
+    
+    def _convert_filters(self, query, filters):
+        for index, child in enumerate(filters.children[:]):
+            if isinstance(child, Node):
+                self._convert_filters(query, child)
+                continue
+
+            self.convert_filter(query, filters, child, index)
+
+    def convert_filter(self, query, filters, child, index):
+        constraint, lookup_type, annotation, value = child
+        
+        if constraint.field is None:
+            return
+        
+        field_name = self.column_to_name.get(constraint.field.column)
+        if field_name and constraint.alias == \
+                query.table_map[query.model._meta.db_table][0]:
+            for lookup in self.index_map.keys():
+                if lookup.matches_filter(query.model, field_name, lookup_type,
+                                         value):
+                    new_lookup_type, new_value = lookup.convert_lookup(value,
+                                                                       lookup_type)
+                    index_name = self.index_name(lookup)
+                    self._convert_filter(query, filters, child, index,
+                                         new_lookup_type, new_value, index_name)
+        
+    def _convert_filter(self, query, filters, child, index, new_lookup_type,
+                        new_value, index_name):
+        constraint, lookup_type, annotation, value = child
+        lookup_type, value = new_lookup_type, new_value
+        constraint.field = query.get_meta().get_field(index_name)
+        constraint.col = constraint.field.column
+        child = constraint, lookup_type, annotation, value
+        filters.children[index] = child
+    
+    def index_name(self, lookup):
+        return lookup.index_name
+    
+    def get_field_to_index(self, model, field_name):
+        try:
+            return model._meta.get_field(field_name)
+        except:
+            return None
+    
+    def get_value(self, model, field_name, query):
+        field_to_index = self.get_field_to_index(model, field_name)
+        for query_field, value in query.values[:]:
+            if field_to_index == query_field:
+                return value
+        raise FieldDoesNotExist('Cannot find field in query.')
+    
+    def add_column_to_name(self, model, field_name):
+        column_name = model._meta.get_field(field_name).column
+        self.column_to_name[column_name] = field_name
+    
+    def get_index(self, lookup):
+        return self.index_map[lookup]
+    
+    def get_query_position(self, query, lookup):
+        for index, (field, query_value) in enumerate(query.values[:]):
+            if field is self.get_index(lookup):
+                return index
+        return None
+
+def unref_alias(query, alias):
+    table_name = query.alias_map[alias][TABLE_NAME]
+    query.alias_refcount[alias] -= 1
+    if query.alias_refcount[alias] < 1:
+        # Remove all information about the join
+        del query.alias_refcount[alias]
+        del query.join_map[query.rev_join_map[alias]]
+        del query.rev_join_map[alias]
+        del query.alias_map[alias]
+        query.table_map[table_name].remove(alias)
+        if len(query.table_map[table_name]) == 0:
+            del query.table_map[table_name]
+        query.used_aliases.discard(alias)
+
+class FKNullFix(BaseResolver):
+    '''
+        Django doesn't generate correct code for ForeignKey__isnull.
+        It becomes a JOIN with pk__isnull which won't work on nonrel DBs,
+        so we rewrite the JOIN here.
+    '''
+     
+    def create_index(self, lookup):
+        pass
+    
+    def convert_query(self, query):
+        pass
+    
+    def convert_filter(self, query, filters, child, index):
+        constraint, lookup_type, annotation, value = child
+        if constraint.field is not None and lookup_type == 'isnull' and \
+                        isinstance(constraint.field, models.ForeignKey):
+            self.fix_fk_null_filter(query, constraint)
+            
+    def unref_alias(self, query, alias):
+        unref_alias(query, alias)
+            
+    def fix_fk_null_filter(self, query, constraint):
+        alias = constraint.alias
+        table_name = query.alias_map[alias][TABLE_NAME]
+        lhs_join_col = query.alias_map[alias][LHS_JOIN_COL]
+        rhs_join_col = query.alias_map[alias][RHS_JOIN_COL]
+        if table_name != constraint.field.rel.to._meta.db_table or \
+                rhs_join_col != constraint.field.rel.to._meta.pk.column or \
+                lhs_join_col != constraint.field.column:
+            return
+        next_alias = query.alias_map[alias][LHS_ALIAS]
+        if not next_alias:
+            return
+        self.unref_alias(query, alias)
+        alias = next_alias
+        constraint.col = constraint.field.column
+        constraint.alias = alias
+
+class ConstantFieldJOINResolver(BaseResolver):
+    def create_index(self, lookup):
+        if '__' in lookup.field_name:
+            BaseResolver.create_index(self, lookup)
+    
+    def convert_query(self, query):
+        for lookup in self.index_map.keys():
+            if lookup.model == query.model and '__' in lookup.field_name:
+                BaseResolver.convert_query(self, query)
+    
+    def convert_filter(self, query, filters, child, index):
+        constraint, lookup_type, annotation, value = child
+        field_chain = self.get_field_chain(query, constraint)
+
+        if field_chain is None:
+            return
+        
+        for lookup in self.index_map.keys():
+            if lookup.matches_filter(query.model, field_chain, lookup_type,
+                                     value):
+                self.resolve_join(query, child)
+                new_lookup_type, new_value = lookup.convert_lookup(value,
+                                                                   lookup_type)
+                index_name = self.index_name(lookup)
+                self._convert_filter(query, filters, child, index,
+                                     new_lookup_type, new_value, index_name)
+    
+    def get_field_to_index(self, model, field_name):
+        model = self.get_model_chain(model, field_name)[-1]
+        field_name = field_name.split('__')[-1]
+        return BaseResolver.get_field_to_index(self, model, field_name)
+    
+    def get_value(self, model, field_name, query):
+        value = BaseResolver.get_value(self, model, field_name.split('__')[0],
+                                    query)
+        if value is not None:
+            value = self.get_target_value(model, field_name, value)
+        return value        
+
+    def get_field_chain(self, query, constraint):
+        if constraint.field is None:
+            return
+
+        column_index = self.get_column_index(query, constraint)
+        return self.column_to_name.get(column_index)
+
+    def get_model_chain(self, model, field_chain):
+        model_chain = [model, ]
+        for value in field_chain.split('__')[:-1]:
+            model = model._meta.get_field(value).rel.to
+            model_chain.append(model)
+        return model_chain
+       
+    def get_target_value(self, start_model, field_chain, pk):
+        fields = field_chain.split('__')
+        foreign_key = start_model._meta.get_field(fields[0])
+        
+        if not foreign_key.rel:
+            # field isn't a related one, so return the value itself
+            return pk
+        
+        target_model = foreign_key.rel.to
+        foreignkey = target_model.objects.all().get(pk=pk)
+        for value in fields[1:-1]:
+            foreignkey = getattr(foreignkey, value)
+        return getattr(foreignkey, fields[-1])
+    
+    def add_column_to_name(self, model, field_name):
+        model_chain = self.get_model_chain(model, field_name)
+        column_chain = ''
+        field_names = field_name.split('__')
+        for model, name in zip(model_chain, field_names):
+            column_chain += model._meta.get_field(name).column + '__'
+        self.column_to_name[column_chain[:-2]] = field_name
+        
+    def unref_alias(self, query, alias):
+        unref_alias(query, alias)
+        
+    def get_column_index(self, query, constraint):
+        if constraint.field:
+            column_chain = constraint.field.column
+            alias = constraint.alias
+            while alias:
+                join = query.alias_map.get(alias)
+                if join and join[JOIN_TYPE] == 'INNER JOIN':
+                    column_chain += '__' + join[LHS_JOIN_COL]
+                    alias = query.alias_map[alias][LHS_ALIAS]
+                else:
+                    alias = None
+        return '__'.join(reversed(column_chain.split('__')))
+
+    def resolve_join(self, query, child):
+        constraint, lookup_type, annotation, value = child
+        if not constraint.field:
+            return
+
+        alias = constraint.alias
+        while True:
+            next_alias = query.alias_map[alias][LHS_ALIAS]
+            if not next_alias:
+                break
+            self.unref_alias(query, alias)
+            alias = next_alias
+        
+        constraint.alias = alias
+
+# TODO: distinguish in memory joins from standard joins somehow
+class InMemoryJOINResolver(ConstantFieldJOINResolver):
+    def __init__(self):
+        self.field_chains = []
+        ConstantFieldJOINResolver.__init__(self)
+
+    def create_index(self, lookup):
+        if '__' in lookup.field_name:
+            field_to_index = self.get_field_to_index(lookup.model, lookup.field_name)
+        
+            if not field_to_index:
+                return 
+            
+            # save old column_to_name so we can make in memory queries later on 
+            self.add_column_to_name(lookup.model, lookup.field_name)
+            
+            # don't add an extra field for standard lookups!
+            if isinstance(lookup, StandardLookup):
+                return 
+             
+            # install lookup on target model
+            model = self.get_model_chain(lookup.model, lookup.field_name)[-1]
+            lookup.model = model
+            lookup.field_name = lookup.field_name.split('__')[-1]
+            BaseResolver.create_index(self, lookup)
+
+    def convert_query(self, query):
+        BaseResolver.convert_query(self, query)
+        
+    def _convert_filters(self, query, filters):
+        # or queries are not supported for in-memory-JOINs
+        if self.contains_OR(query.where, OR):
+            return
+        
+        # start with the deepest JOIN level filter!
+        all_filters = self.get_all_filters(filters)
+        all_filters.sort(key=lambda item: self.get_field_chain(query, item[1][0]) and \
+                         -len(self.get_field_chain(query, item[1][0])) or 0)
+        
+        for filters, child, index in all_filters:
+            # check if convert_filter removed a given child from the where-tree
+            if not self.contains_child(query.where, child):
+                continue
+            self.convert_filter(query, filters, child, index)
+    
+    def convert_filter(self, query, filters, child, index):
+        constraint, lookup_type, annotation, value = child
+        field_chain = self.get_field_chain(query, constraint)
+
+        if field_chain is None:
+            return
+        
+        if '__' not in field_chain:
+            return BaseResolver.convert_filter(self, query, filters, child, index)
+        
+        pks = self.get_pks(query, field_chain, lookup_type, value)
+        self.resolve_join(query, child)
+        self._convert_filter(query, filters, child, index, 'in',
+                             (pk for pk in pks), field_chain.split('__')[0])
+        
+    def tree_contains(self, filters, to_find, func):
+        result = False
+        for child in filters.children[:]:
+            if func(child, to_find):
+                result = True
+                break
+            if isinstance(child, Node):
+                result = self.tree_contains(child, to_find, func)
+                if result:
+                    break
+        return result
+    
+    def contains_OR(self, filters, or_):
+        return self.tree_contains(filters, or_,
+            lambda c, f: isinstance(c, Node) and c.connector == f)
+
+    def contains_child(self, filters, to_find):
+        return self.tree_contains(filters, to_find, lambda c, f: c is f)
+    
+    def get_all_filters(self, filters):
+        all_filters = []
+        for index, child in enumerate(filters.children[:]):
+            if isinstance(child, Node):
+                all_filters.extend(self.get_all_filters(child))
+                continue
+
+            all_filters.append((filters, child, index))
+        return all_filters
+    
+    def index_name(self, lookup):
+        # use another index_name to avoid conflicts with lookups defined on the
+        # target model which are handled by the BaseBackend
+        return lookup.index_name + '_in_memory_join'
+    
+    def get_pks(self, query, field_chain, lookup_type, value):
+        model_chain = self.get_model_chain(query.model, field_chain)
+                
+        first_lookup = {'%s__%s' %(field_chain.rsplit('__', 1)[-1],
+                                   lookup_type): value}
+        self.combine_with_same_level_filter(first_lookup, query, field_chain)
+        pks = model_chain[-1].objects.all().filter(**first_lookup).values_list(
+            'id', flat=True)
+
+        chains = [field_chain.rsplit('__', i+1)[0]
+                  for i in range(field_chain.count('__'))]
+        lookup = {}
+        for model, chain in reversed(zip(model_chain[1:-1], chains[:-1])):
+            lookup.update({'%s__%s' %(chain.rsplit('__', 1)[-1], 'in'):
+                           (pk for pk in pks)})
+            self.combine_with_same_level_filter(lookup, query, chain)
+            pks = model.objects.all().filter(**lookup).values_list('id', flat=True)
+        return pks
+    
+    def combine_with_same_level_filter(self, lookup, query, field_chain):
+        lookup_updates = {}
+        field_chains = self.get_all_field_chains(query, query.where)
+
+        for chain, child in field_chains.items():
+            if chain == field_chain:
+                continue
+            if field_chain.rsplit('__', 1)[0] == chain.rsplit('__', 1)[0]:
+                lookup_updates ['%s__%s' %(chain.rsplit('__', 1)[1], child[1])] \
+                    = child[3]
+                
+                self.remove_child(query.where, child)
+                self.resolve_join(query, child)
+                # TODO: update query.alias_refcount correctly!
+        lookup.update(lookup_updates)
+                
+    def remove_child(self, filters, to_remove):
+        ''' Removes a child object from filters. If filters doesn't contain
+            children afterwoods, filters will be removed from its parent. '''
+            
+        for child in filters.children[:]:
+            if child is to_remove:
+                self._remove_child(filters, to_remove)
+                return
+            elif isinstance(child, Node):
+                self.remove_child(child, to_remove)
+            
+            if hasattr(child, 'children') and not child.children:
+                self.remove_child(filters, child)
+    
+    def _remove_child(self, filters, to_remove):
+        result = []
+        for child in filters.children[:]:
+            if child is to_remove:
+                continue
+            result.append(child)
+        filters.children = result
+    
+    def get_all_field_chains(self, query, filters):
+        ''' Returns a dict mapping from field_chains to the corresponding child.'''
+
+        field_chains = {}
+        all_filters = self.get_all_filters(filters)
+        for filters, child, index in all_filters:
+            field_chain = self.get_field_chain(query, child[0])
+            # field_chain can be None if the user didn't specified an index for it
+            if field_chain:
+                field_chains[field_chain] = child
+        return field_chains

File dbindexer/compiler.py

-from .api import FIELD_INDEXES, COLUMN_TO_NAME, get_index_name, get_column_name, regex
-from django.db import models
-from django.db.models.sql import aggregates as sqlaggregates
-from django.db.models.sql.constants import LOOKUP_SEP, MULTI, SINGLE, LHS_ALIAS,\
-    JOIN_TYPE, LHS_JOIN_COL, TABLE_NAME, RHS_JOIN_COL
-from django.db.models.sql.where import AND, OR
-from django.db.utils import DatabaseError, IntegrityError
+from resolver import resolver
 from django.utils.importlib import import_module
-from django.utils.tree import Node
-import re
-
-def contains_indexer(value):
-    # In indexing mode we add all postfixes ('o', 'lo', ..., 'hello')
-    result = []
-    if value:
-        result.extend([value[count:] for count in range(len(value))])
-    return result
-
-LOOKUP_TYPE_CONVERSION = {
-    'iexact': lambda value, _: ('exact', value.lower()),
-    'istartswith': lambda value, _: ('startswith', value.lower()),
-    'iendswith': lambda value, _: ('startswith', value[::-1].lower()),
-    'endswith': lambda value, _: ('startswith', value[::-1]),
-    'year': lambda value, _: ('exact', value),
-    'month': lambda value, _: ('exact', value),
-    'day': lambda value, _: ('exact', value),
-    'week_day': lambda value, _: ('exact', value),
-    'contains': lambda value, _: ('startswith', value),
-    'icontains': lambda value, _: ('startswith', value.lower()),
-    'regex': lambda value, _: ('exact', ':' + value),
-    'iregex': lambda value, _: ('exact', 'i:' + value),
-}
-
-# value conversion for (i)regex works via special code
-VALUE_CONVERSION = {
-    'iexact': lambda value: value.lower(),
-    'istartswith': lambda value: value.lower(),
-    'endswith': lambda value: value[::-1],
-    'iendswith': lambda value: value[::-1].lower(),
-    'year': lambda value: value.year,
-    'month': lambda value: value.month,
-    'day': lambda value: value.day,
-    'week_day': lambda value: value.isoweekday(),
-    'contains': lambda value: contains_indexer(value),
-    'icontains': lambda value: [val.lower() for val in contains_indexer(value)],
-    # TODO: clean $default case
-    '$default': lambda value: value,
-}
-
-def get_denormalization_value(start_model, index_key, foreignkey_pk):
-    denormalized_model = start_model._meta.get_field(index_key.split('__')[0]).rel.to
-    foreignkey = denormalized_model.objects.all().get(pk=foreignkey_pk)
-    for value in index_key.split('__')[1:-1]:
-        foreignkey = getattr(foreignkey, value)
-    return getattr(foreignkey, index_key.split('__')[-1])
-
 
 def __repr__(self):
     return '<%s, %s, %s, %s>' % (self.alias, self.col, self.field.name,
 Constraint.__repr__ = __repr__
 
 # TODO: manipulate a copy of the query instead of the query itself. This has to
-# be done because the query can be reused afterwoods by the user so that a
+# be done because the query can be reused afterwards by the user so that a
 # manipulated query can result in strange behavior for these cases!
+#TODO: Add watching layer which gives suggestions for indexes via query inspection
+# at runtime
 
 class BaseCompiler(object):
-    def get_column_index(self, constraint):
-        if constraint.field:
-            column_chain = constraint.field.column
-            alias = constraint.alias
-            while alias:
-                join = self.query.alias_map[alias]
-                if join[JOIN_TYPE] == 'INNER JOIN':
-                    column_chain += '__' + join[LHS_JOIN_COL]
-                    alias = self.query.alias_map[alias][LHS_ALIAS]
-                else:
-                    alias = None
-        return '__'.join(reversed(column_chain.split('__')))
-
-    def resolve_join(self, constraint, lookup_type, annotation, value, filters,
-            index, column_index):
-        if not constraint.field:
-            return
-
-        alias = constraint.alias
-        while True:
-            next_alias = self.query.alias_map[alias][LHS_ALIAS]
-            if not next_alias:
-                break
-            self.unref_alias(alias)
-            alias = next_alias
-
-        index_name = get_index_name(column_index, lookup_type)
-        lookup_type, value = LOOKUP_TYPE_CONVERSION[lookup_type](value,
-            annotation)
-        constraint.alias = alias
-        constraint.field = self.query.get_meta().get_field(index_name)
-        constraint.col = constraint.field.column
-        child = (constraint, lookup_type, annotation, value)
-        filters.children[index] = child
-
-    def unref_alias(self, alias):
-        table_name = self.query.alias_map[alias][TABLE_NAME]
-        self.query.alias_refcount[alias] -= 1
-        if self.query.alias_refcount[alias] < 1:
-            # Remove all information about the join
-            del self.query.alias_refcount[alias]
-            del self.query.join_map[self.query.rev_join_map[alias]]
-            del self.query.rev_join_map[alias]
-            del self.query.alias_map[alias]
-            self.query.table_map[table_name].remove(alias)
-            if len(self.query.table_map[table_name]) == 0:
-                del self.query.table_map[table_name]
-            self.query.used_aliases.discard(alias)
-
-    def convert_filters(self, filters):
-        model = self.query.model
-        for index, child in enumerate(filters.children[:]):
-            if isinstance(child, Node):
-                self.convert_filters(child)
-                continue
-
-            constraint, lookup_type, annotation, value = child
-            if model in FIELD_INDEXES and constraint.field is not None:
-                if lookup_type == 'isnull' and \
-                        isinstance(constraint.field, models.ForeignKey):
-                    self.fix_fk_null_filter(constraint)
-                if constraint.alias == self.query.table_map[model._meta.db_table][0]:
-                    if lookup_type not in FIELD_INDEXES[model].get(constraint.field.column, ()):
-                        continue
-                    index_name = get_index_name(constraint.field.column, lookup_type)
-                    lookup_type, value = LOOKUP_TYPE_CONVERSION[lookup_type](value,
-                        annotation)
-                    constraint.field = self.query.get_meta().get_field(index_name)
-                    constraint.col = constraint.field.column
-                    child = (constraint, lookup_type, annotation, value)
-                    filters.children[index] = child
-                else:
-                    # check for joins
-                    column_index = self.get_column_index(constraint)
-                    if lookup_type in FIELD_INDEXES[model].get(column_index, ()):
-                        self.resolve_join(constraint, lookup_type, annotation,
-                            value, filters, index, column_index)
-
-    def fix_fk_null_filter(self, constraint):
-        # Django doesn't generate correct code for ForeignKey__isnull.
-        # It becomes a JOIN with pk__isnull which won't work on nonrel DBs,
-        # so we rewrite the JOIN here.
-        alias = constraint.alias
-        table_name = self.query.alias_map[alias][TABLE_NAME]
-        lhs_join_col = self.query.alias_map[alias][LHS_JOIN_COL]
-        rhs_join_col = self.query.alias_map[alias][RHS_JOIN_COL]
-        if table_name != constraint.field.rel.to._meta.db_table or \
-                rhs_join_col != constraint.field.rel.to._meta.pk.column or \
-                lhs_join_col != constraint.field.column:
-            return
-        next_alias = self.query.alias_map[alias][LHS_ALIAS]
-        if not next_alias:
-            return
-        self.unref_alias(alias)
-        alias = next_alias
-        constraint.col = constraint.field.column
-        constraint.alias = alias
+    def convert_filters(self):
+        resolver.convert_filters(self.query)
 
 class SQLCompiler(BaseCompiler):
     def execute_sql(self, *args, **kwargs):
-        self.convert_filters(self.query.where)
+        self.convert_filters()
         return super(SQLCompiler, self).execute_sql(*args, **kwargs)
 
     def results_iter(self):
-        self.convert_filters(self.query.where)
+        self.convert_filters()
         return super(SQLCompiler, self).results_iter()
 
+
 class SQLInsertCompiler(BaseCompiler):
     def execute_sql(self, return_id=False):
-        position = {}
-        for index, (field, value) in enumerate(self.query.values[:]):
-            position[field.name] = index
-
-        model = self.query.model
-        # TODO: we should reverse this iteration, instead of iterating through all
-        # fields and values and then looking up their index definitions we should
-        # iterate through all index definitions first and getting the
-        # corresponding fields and values or in general we need to refactore the
-        # whole dbindexer ;)
-        for field, value in self.query.values[:]:
-            regex_values = []
-            index_keys = []
-
-            if field is None or model not in FIELD_INDEXES:
-                continue
-            if field.column not in FIELD_INDEXES[model]:
-                # check for denormalization indexes on the left table, if none
-                # exist continue with next field
-                denormalization_indexes = [field_index.split('__', 1)[0]
-                    for field_index in FIELD_INDEXES[model].keys()]
-                if field.column not in denormalization_indexes:
-                    continue
-                else:
-                    for field_index in FIELD_INDEXES[model].keys():
-                        # check against field column + '__' to avoid name clashes
-                        # caused by startswith() i.e. field.column = foreignkey
-                        # and field2.column = foreignkey2
-                        if field_index.startswith(field.column + '__'):
-                            index_keys.append(field_index)
-            else:
-                start_background_tasks = [lookup_type.startswith('denormalized__')
-                    for lookup_type in FIELD_INDEXES[model][field.column]
-                    if not isinstance(lookup_type, regex)]
-                if True in start_background_tasks:
-                    # TODO: we should push background tasks here, background tasks
-                    # have to use model.update to ensure transactional behavior
-                    continue
-                index_keys.append(field.column)
-                # check for denormalization indexes here too
-                for field_index in FIELD_INDEXES[model].keys():
-                    # check against field name + '__' to avoid name clashes
-                    # i.e. field.name = foreignkey and field2.name = foreignkey2
-                    if field_index.startswith(field.column + '__'):
-                        index_keys.append(field_index)
-
-            foreign_key_pk = value
-            for index_key in index_keys:
-                for lookup_type in FIELD_INDEXES[model][index_key]:
-                    if len(index_key.split('__', 1)) > 1:
-                        # TODO: this has to be done in background too so that it's
-                        # possible to use transactions, background tasks
-                        # have to use model.update to ensure transactional behavior
-                        # denormalization case
-                        value = get_denormalization_value(model,
-                            COLUMN_TO_NAME[index_key], foreign_key_pk)
-                    if lookup_type in ['regex', 'iregex']:
-                        continue
-                    index_name = get_index_name(index_key, lookup_type)
-                    index_field = model._meta.get_field(index_name)
-                    if isinstance(lookup_type, regex):
-                        if lookup_type.match(value):
-                            val = ('i:' if lookup_type.flags & re.I else ':') + \
-                                lookup_type.pattern
-                            regex_values.append(val)
-                        self.query.values[position[index_name]] = (index_field,
-                            regex_values)
-                    else:
-                        if isinstance(field, models.ForeignKey) and \
-                                len(index_key.split('__', 1)) <= 1:
-                            # case for which we try to convert the foreign key
-                            # value itself
-                            value = unicode(value)
-                        self.query.values[position[index_name]] = (index_field,
-                            VALUE_CONVERSION[lookup_type](value))
-        # debug info
-#        print dict((field.column, value) for field, value in self.query.values)
+        resolver.convert_query(self.query)
         return super(SQLInsertCompiler, self).execute_sql(return_id=return_id)
 
 class SQLUpdateCompiler(BaseCompiler):

File dbindexer/filter.py

-from django.db import models
-from djangotoolbox.fields import ListField
-from copy import deepcopy
-
-class ExtraFieldLookup():
-    def __init__(self, model=None, field_name=None, lookup_type=None,
-            field_to_add=models.CharField(max_length=500, editable=False, null=True)):
-        self.model = model
-        self.field_name = field_name
-
-        self.column_name = None
-        if model and field_name:
-            self.column_name = model._meta.get_field(self.field_name).column
-            
-        self.field_to_add = field_to_add
-        self.lookup_type = lookup_type
-
-    def contribute(self, model, field_name):
-        self.model = model
-        self.field_name = field_name
-        if model and field_name:
-            self.column_name = model._meta.get_field(self.field_name).column
-
-    @property
-    def index_name(self):
-        return 'idxf_%s_l_%s' % (self.column_name, self.lookup_type)
-
-    def create_index(self, model):
-        index_field = deepcopy(self.field_to_add)
-        model.add_to_class(self.index_name, index_field)
-
-    def convert_lookup(self, value, annotation):
-        pass
-
-    def convert_value(self, value):
-        pass
-
-class DateLookup(ExtraFieldLookup):
-    def __init__(self, model, field_name, lookup_type):
-        super(ExtraFieldLookup, self).__init__(model, field_name,
-            lookup_type, models.IntegerField(editable=False, null=True))
-
-    def convert_lookup(self, value, annotation):
-        return 'exact', value
-
-class Day(DateLookup):
-    def __init__(self, model, field_name):
-        super(DateLookup, self).__init__(model, field_name, 'day')
-
-    def convert_value(self, value):
-        return value.day
-
-class Month(DateLookup):
-    def __init__(self, model, field_name):
-        super(DateLookup, self).__init__(model, field_name, 'month')
-
-    def convert_value(self, value):
-        return value.month
-
-class Year(DateLookup):
-    def __init__(self, model, field_name):
-        super(DateLookup, self).__init__(model, field_name, 'year')
-
-    def convert_value(self, value):
-        return value.year
-
-class Weekday(DateLookup):
-    def __init__(self, model, field_name):
-        super(DateLookup, self).__init__(model, field_name, 'week_day')
-
-    def convert_value(self, value):
-        return value.isoweekday()
-
-class RegexFilter(ExtraFieldLookup):
-    def __init__(self, model, field_name, regex):
-        self.regex = re.compile(regex.pattern, re.S | re.U | (regex.flags & re.I))
-        lookup_type = self.regex.flags & re.I and 'iregex' or 'regex'
-        super(ExtraFieldLookup, self).__init__(model, field_name,
-            lookup_type, ListField(models.CharField(max_length=256),
-            editable=False, null=True))
-
-    def create_index(self, model):
-        # TODO: only create one list field for all regexes for a given model
-        super(ExtraFieldLookup, self).create_index(model)
-
-    def convert_lookup(self, value, annotation):
-        return self.lookup_type == 'regex' and ('exact', ':' + value) or \
-            ('exact', 'i:' + value)
-
-    def convert_value(self, value):
-        return
-
-class Contains(ExtraFieldLookup):
-    def __init__(self, model, field_name):
-        super(ExtraFieldLookup, self).__init__(model, field_name,
-            'contains', ListField(models.CharField(500), editable=False, null=True))
-
-    def convert_lookup(self, value, annotation):
-        return 'startswith', value
-
-    def convert_value(self, value):
-        return self.contains_indexer(value)
-
-    @classmethod
-    def contains_indexer(cls, value):
-        # In indexing mode we add all postfixes ('o', 'lo', ..., 'hello')
-        result = []
-        if value:
-            result.extend([value[count:] for count in range(len(value))])
-        return result
-
-class Icontains(Contains):
-    def __init__(self, model, field_name):
-        super(Contains, self).__init__(model, field_name)
-        self.lookup_type = 'icontains'
-
-    def convert_lookup(self, value, annotation):
-        return 'startswith', value.lower()
-
-    def convert_value(self, value):
-        return [val.lower() for val in super(Contains, self).convert_value(value)]
-
-class Iexact(ExtraFieldLookup):
-    def convert_lookup(self, value, annotation):
-        return 'exact', value.lower()
-
-    def convert_value(self, value):
-        return value.lower()
-
-class Istartswith(ExtraFieldLookup):
-    def convert_lookup(self, value, annotation):
-        return 'startswith', value.lower()
-
-    def convert_value(self, value):
-        return value.lower()
-
-class Endswith(ExtraFieldLookup):
-    def convert_lookup(self, value, annotation):
-        return 'startswith', value[::-1]
-
-    def convert_value(self, value):
-        return value[::-1]
-
-class Iendswith(ExtraFieldLookup):
-    def convert_lookup(self, value, annotation):
-        return 'startswith', value[::-1].lower()
-
-    def convert_value(self, value):
-        return value[::-1].lower()
-    

File dbindexer/lookups.py

+from django.db import models
+from djangotoolbox.fields import ListField
+from copy import deepcopy 
+
+import re
+regex = type(re.compile(''))
+
+class LookupDoesNotExist(Exception):
+    pass
+
+class LookupBase(type):
+    def __new__(cls, name, bases, attrs):
+        new_cls = type.__new__(cls, name, bases, attrs)
+        if not isinstance(new_cls.lookup_types, (list, tuple)):
+            new_cls.lookup_types = (new_cls.lookup_types, )
+        return new_cls 
+
+class ExtraFieldLookup(object):
+    '''Default is to behave like an exact filter on an ExtraField.'''
+    __metaclass__ = LookupBase
+    lookup_types = 'exact'
+    
+    def __init__(self, model=None, field_name=None, lookup_def=None,
+                 new_lookup='exact', field_to_add=models.CharField(
+                 max_length=500, editable=False, null=True)):
+        self.field_to_add = field_to_add
+        self.new_lookup = new_lookup
+        self.contribute(model, field_name, lookup_def)
+        
+    def contribute(self, model, field_name, lookup_def):
+        self.model = model
+        self.field_name = field_name
+        self.lookup_def = lookup_def
+            
+    @property
+    def index_name(self):
+        return 'idxf_%s_l_%s' % (self.field_name, self.lookup_types[0])
+    
+    def convert_lookup(self, value, lookup_type):
+        # TODO: can value be a list or tuple? (in case of in yes)
+        if isinstance(value, (tuple, list)):
+            value = [self._convert_lookup(val, lookup_type)[1] for val in value]
+        else:
+            _, value = self._convert_lookup(value, lookup_type)
+        return self.new_lookup, value
+    
+    def _convert_lookup(self, value, lookup_type):
+        return lookup_type, value
+    
+    def convert_value(self, value):
+        if isinstance(value, (tuple, list)):
+            value = [self._convert_value(val) for val in value]
+        else:
+            value = self._convert_value(value)
+        return value
+    
+    def _convert_value(self, value):
+        return value
+        
+    def matches_filter(self, model, field_name, lookup_type, value):
+        return self.model == model and lookup_type in self.lookup_types \
+            and field_name == self.field_name
+    
+    @classmethod
+    def matches_lookup_def(cls, lookup_def):
+        if lookup_def in cls.lookup_types:
+            return True
+        return False
+    
+    def get_field_to_add(self, field_to_index):
+        field_to_add = deepcopy(self.field_to_add)
+        if isinstance(field_to_index, ListField):
+            field_to_add = ListField(field_to_add, editable=False, null=True)
+        return field_to_add
+
+class DateLookup(ExtraFieldLookup):
+    def __init__(self, *args, **kwargs):
+        defaults = {'new_lookup': 'exact',
+                    'field_to_add': models.IntegerField(editable=False, null=True)}
+        defaults.update(kwargs)
+        ExtraFieldLookup.__init__(self, *args, **defaults)
+    
+    def _convert_lookup(self, value, lookup_type):
+        return self.new_lookup, value
+
+class Day(DateLookup):
+    lookup_types = 'day'
+    
+    def _convert_value(self, value):
+        return value.day
+
+class Month(DateLookup):
+    lookup_types = 'month'
+    
+    def _convert_value(self, value):
+        return value.month
+
+class Year(DateLookup):
+    lookup_types = 'year'
+
+    def _convert_value(self, value):
+        return value.year
+
+class Weekday(DateLookup):
+    lookup_types = 'week_day'
+    
+    def _convert_value(self, value):
+        return value.isoweekday()
+
+class Contains(ExtraFieldLookup):
+    lookup_types = 'contains'
+
+    def __init__(self, *args, **kwargs):
+        defaults = {'new_lookup': 'startswith',
+                    'field_to_add': ListField(models.CharField(500),
+                                              editable=False, null=True)
+        }
+        defaults.update(kwargs)
+        ExtraFieldLookup.__init__(self, *args, **defaults)
+    
+    def get_field_to_add(self, field_to_index):
+        # always return a ListField of CharFields even in the case of
+        # field_to_index being a ListField itself!
+        return deepcopy(self.field_to_add)
+    
+    def convert_value(self, value):
+        new_value = []
+        if isinstance(value, (tuple, list)):
+            for val in value:
+                new_value.extend(self.contains_indexer(val))
+        else:
+            new_value = self.contains_indexer(value)
+        return new_value
+     
+    def _convert_lookup(self, value, lookup_type):
+        return self.new_lookup, value
+
+    def contains_indexer(self, value):
+        # In indexing mode we add all postfixes ('o', 'lo', ..., 'hello')
+        result = []
+        if value:
+            result.extend([value[count:] for count in range(len(value))])
+        return result
+
+class Icontains(Contains):
+    lookup_types = 'icontains'
+    
+    def convert_value(self, value):
+        return [val.lower() for val in Contains.convert_value(self, value)]
+    
+    def _convert_lookup(self, value, lookup_type):
+        return self.new_lookup, value.lower()
+
+class Iexact(ExtraFieldLookup):
+    lookup_types = 'iexact'
+        
+    def _convert_lookup(self, value, lookup_type):
+        return self.new_lookup, value.lower()
+    
+    def _convert_value(self, value):
+        return value.lower()
+
+class Istartswith(ExtraFieldLookup):
+    lookup_types = 'istartswith'
+    
+    def __init__(self, *args, **kwargs):
+        defaults = {'new_lookup': 'startswith'}
+        defaults.update(kwargs)
+        ExtraFieldLookup.__init__(self, *args, **defaults)
+    
+    def _convert_lookup(self, value, lookup_type):
+        return self.new_lookup, value.lower()
+
+    def _convert_value(self, value):
+        return value.lower()
+
+class Endswith(ExtraFieldLookup):
+    lookup_types = 'endswith'
+    
+    def __init__(self, *args, **kwargs):
+        defaults = {'new_lookup': 'startswith'}
+        defaults.update(kwargs)
+        ExtraFieldLookup.__init__(self, *args, **defaults)
+    
+    def _convert_lookup(self, value, lookup_type):
+        return self.new_lookup, value[::-1]
+
+    def _convert_value(self, value):
+        return value[::-1]
+
+class Iendswith(Endswith):
+    lookup_types = 'iendswith'
+    
+    def _convert_lookup(self, value, lookup_type):
+        return self.new_lookup, value[::-1].lower()
+
+    def _convert_value(self, value):
+        return value[::-1].lower()
+
+class RegexLookup(ExtraFieldLookup):
+    lookup_types = ('regex', 'iregex')
+    
+    def __init__(self, *args, **kwargs):
+        defaults = {'field_to_add': models.NullBooleanField(editable=False,
+                                                            null=True) 
+        }
+        defaults.update(kwargs)
+        ExtraFieldLookup.__init__(self, *args, **defaults)        
+    
+    def contribute(self, model, field_name, lookup_def):
+        ExtraFieldLookup.contribute(self, model, field_name, lookup_def)
+        if isinstance(lookup_def, regex):
+            self.lookup_def = re.compile(lookup_def.pattern, re.S | re.U |
+                                         (lookup_def.flags & re.I))
+    
+    @property
+    def index_name(self):
+        return 'idxf_%s_l_%s' % (self.field_name,
+                                 self.lookup_def.pattern.encode('hex'))
+
+    def is_icase(self):
+        return self.lookup_def.flags & re.I
+    
+    def _convert_lookup(self, value, lookup_type):
+        return self.new_lookup, True
+
+    def _convert_value(self, value):
+        if self.lookup_def.match(value):
+            return True
+        return False
+        
+    def matches_filter(self, model, field_name, lookup_type, value):
+        return self.model == model and lookup_type == \
+                '%sregex' % ('i' if self.is_icase() else '') and \
+                value == self.lookup_def.pattern and field_name == self.field_name
+    
+    @classmethod
+    def matches_lookup_def(cls, lookup_def):
+        if isinstance(lookup_def, regex):
+            return True
+        return False 
+
+class StandardLookup(ExtraFieldLookup):
+    ''' Creates a copy of the field_to_index in order to allow querying for 
+        standard lookup_types on a JOINed property. '''
+    # TODO: database backend can specify standardLookups
+    lookup_types = ('exact', 'gt', 'gte', 'lt', 'lte', 'in', 'range', 'isnull')
+    
+    @property
+    def index_name(self):
+        return 'idxf_%s_l_%s' % (self.field_name, 'standard')
+    
+    def convert_lookup(self, value, lookup_type):
+        return lookup_type, value
+    
+    def get_field_to_add(self, field_to_index):
+        field_to_add = deepcopy(field_to_index)
+        if isinstance(field_to_add, (models.DateTimeField,
+                                    models.DateField, models.TimeField)):
+            field_to_add.auto_now_add = field_to_add.auto_now = False
+        return field_to_add

File dbindexer/resolver.py

+from django.conf import settings
+from django.utils.importlib import import_module
+from django.core.exceptions import ImproperlyConfigured
+
+class Resolver(object):
+    def __init__(self):
+        self.backends = []
+        self.load_backends(getattr(settings, 'DBINDEXER_BACKENDS',
+                               ('dbindexer.backends.BaseResolver',
+                                'dbindexer.backends.FKNullFix')))
+
+    def load_backends(self, backend_paths):
+        for backend in backend_paths:
+                self.backends.append(self.load_backend(backend))
+    
+    def load_backend(self, path):
+        module_name, attr_name = path.rsplit('.', 1)
+        try:
+            mod = import_module(module_name)
+        except (ImportError, ValueError), e:
+            raise ImproperlyConfigured('Error importing backend module %s: "%s"'
+                % (module_name, e))
+        try:
+            return getattr(mod, attr_name)()
+        except AttributeError:
+            raise ImproperlyConfigured('Module "%s" does not define a "%s" backend'
+                % (module_name, attr_name))
+
+    def convert_filters(self, query):
+        for backend in self.backends:
+            backend.convert_filters(query)
+
+    def create_index(self, lookup):
+        for backend in self.backends:
+            backend.create_index(lookup)
+
+    def convert_query(self, query):
+        for backend in self.backends:
+            backend.convert_query(query)
+
+resolver = Resolver()

File dbindexer/tests.py

-from dbindexer.api import register_index
 from django.db import models
 from django.test import TestCase
+from dbindexer.api import register_index
+from dbindexer.lookups import StandardLookup
+from dbindexer.resolver import resolver 
+from djangotoolbox.fields import ListField
 from datetime import datetime
 import re
 
 class ForeignIndexed2(models.Model):
     name_fi2 = models.CharField(max_length=500)
+    age = models.IntegerField()
     
 class ForeignIndexed(models.Model):
     title = models.CharField(max_length=500)
     published = models.DateTimeField(auto_now_add=True)
     foreignkey = models.ForeignKey(ForeignIndexed, null=True)
     foreignkey2 = models.ForeignKey(ForeignIndexed2, related_name='idx_set', null=True)
+    tags = ListField(models.CharField(max_length=500, null=True))
 
-register_index(Indexed, {
-    'name': ('iexact', 'endswith', 'istartswith', 'iendswith', 'contains',
-        'icontains', re.compile('^i+', re.I), re.compile('^I+'),
-        re.compile('^i\d*i$', re.I)),
-    'published': ('month', 'day', 'week_day'),
-    'foreignkey': 'iexact',
-    'foreignkey__title': 'iexact',
-    'foreignkey__fk__name_fi2': 'iexact',
-    'foreignkey__name_fi': 'iexact',
-    'foreignkey2__name_fi2': '$default'
-})
-
+# TODO: add test for foreign key with multiple filters via different and equal paths
+# to do so we have to create some entities matching equal paths but not matching
+# different paths
 class TestIndexed(TestCase):
     def setUp(self):
-        juubi = ForeignIndexed2(name_fi2='Juubi')
+        self.backends = list(resolver.backends)
+        resolver.backends = []
+        resolver.load_backends(('dbindexer.backends.BaseResolver',
+                      'dbindexer.backends.FKNullFix',
+                      'dbindexer.backends.InMemoryJOINResolver',
+#                      'dbindexer.backends.ConstantFieldJOINResolver',
+        ))
+        self.register_indexex()
+        for backend in resolver.backends:
+            print backend.index_map
+        
+        juubi = ForeignIndexed2(name_fi2='Juubi', age=2)
         juubi.save()
+        rikudo = ForeignIndexed2(name_fi2='Rikudo', age=200)
+        rikudo.save()
+        
         kyuubi = ForeignIndexed(name_fi='Kyuubi', title='Bijuu', fk=juubi)
+        hachibi= ForeignIndexed(name_fi='Hachibi', title='Bijuu', fk=rikudo)
         kyuubi.save()
-        Indexed(name='ItAchi', foreignkey=kyuubi, foreignkey2=juubi).save()
-        Indexed(name='YondAimE', foreignkey=kyuubi, foreignkey2=juubi).save()
-        Indexed(name='I1038593i', foreignkey=kyuubi, foreignkey2=juubi).save()
-
+        hachibi.save()
+                
+        Indexed(name='ItAchi', tags=('Sasuke', 'Madara'), foreignkey=kyuubi,
+                foreignkey2=juubi).save()
+        Indexed(name='YondAimE', tags=('Naruto', 'Jiraya'), foreignkey=kyuubi,
+                foreignkey2=juubi).save()
+        Indexed(name='Neji', tags=('Hinata'), foreignkey=hachibi,
+                foreignkey2=juubi).save()
+        Indexed(name='I1038593i', tags=('Sharingan'), foreignkey=hachibi,
+                foreignkey2=rikudo).save()
+    
+    def tearDown(self):
+        resolver.backends = self.backends
+        
+    def register_indexex(self):
+        register_index(Indexed, {
+            'name': ('iexact', 'endswith', 'istartswith', 'iendswith', 'contains',
+                     'icontains', re.compile('^i+', re.I), re.compile('^I+'),
+                     re.compile('^i\d*i$', re.I)),
+            'published': ('month', 'day', 'year', 'week_day'),
+            'tags': ('iexact', 'icontains', StandardLookup() ),
+        #    'foreignkey': 'iexact',
+            'foreignkey__title': 'iexact',
+            'foreignkey__name_fi': 'iexact',
+            'foreignkey__fk__name_fi2': ('iexact', 'endswith'),
+            'foreignkey2__name_fi2': (StandardLookup(), 'iexact'),
+            'foreignkey2__age': (StandardLookup())
+        })
+        
+        register_index(ForeignIndexed, {
+            'title': 'iexact',
+            'name_fi': ('iexact', 'icontains'),
+            'fk__name_fi2': ('iexact', 'endswith'),
+            'fk__age': (StandardLookup()),
+        })
+        
+    # TODO: add tests for created indexes for all backends!
+#    def test_model_fields(self):
+#        field_list = [(item[0], item[0].column) 
+#                       for item in Indexed._meta.get_fields_with_model()]
+#        print field_list
+#        x()
+        # in-memory JOIN backend shouldn't create multiple indexes on the foreignkey side
+        # for different paths or not even for index definition on different models. Test this!
+        # standard JOIN backend should always add extra fields to registered model. Test this!
+    
     def test_joins(self):
+        self.assertEqual(2, len(Indexed.objects.all().filter(
+            foreignkey__fk__name_fi2__iexact='juuBi',
+            foreignkey__title__iexact='biJuu')))
+        
+        self.assertEqual(0, len(Indexed.objects.all().filter(
+            foreignkey__fk__name_fi2__iexact='juuBi',
+            foreignkey2__name_fi2__iexact='Rikudo')))
+        
+        self.assertEqual(1, len(Indexed.objects.all().filter(
+            foreignkey__fk__name_fi2__endswith='udo',
+            foreignkey2__name_fi2__iexact='Rikudo')))
+        
+        self.assertEqual(2, len(Indexed.objects.all().filter(
+            foreignkey__title__iexact='biJuu',
+            foreignkey__name_fi__iexact='kyuuBi')))
+        
+        self.assertEqual(2, len(Indexed.objects.all().filter(
+            foreignkey__title__iexact='biJuu',
+            foreignkey__name_fi__iexact='Hachibi')))
+                
+        self.assertEqual(1, len(Indexed.objects.all().filter(
+            foreignkey__title__iexact='biJuu', name__iendswith='iMe')))
+        
+        # JOINs on one field only
+        self.assertEqual(4, len(Indexed.objects.all().filter(
+            foreignkey__title__iexact='biJuu')))
         self.assertEqual(3, len(Indexed.objects.all().filter(
+           foreignkey2__name_fi2='Juubi')))
+        
+        # text endswith instead iexact all the time :)
+        self.assertEqual(2, len(Indexed.objects.all().filter(
+            foreignkey__fk__name_fi2__endswith='bi')))
+        
+        # test JOINs via different paths targeting the same field
+        self.assertEqual(2, len(Indexed.objects.all().filter(
             foreignkey__fk__name_fi2__iexact='juuBi')))
         self.assertEqual(3, len(Indexed.objects.all().filter(
-            foreignkey__fk__name_fi2__iexact='juuBi',
-            foreignkey__title__iexact='biJuu')))
+           foreignkey2__name_fi2__iexact='Juubi')))
+        
+        # test standard lookups for foreign_keys
         self.assertEqual(3, len(Indexed.objects.all().filter(
-            foreignkey__name_fi__iexact='kyuuBi', foreignkey__title__iexact='biJuu')))
-        self.assertEqual(3, len(Indexed.objects.all().filter(
-            foreignkey__title__iexact='biJuu')))
-        self.assertEqual(1, len(Indexed.objects.all().filter(
-            foreignkey__title__iexact='biJuu', name__iendswith='iMe')))
+            foreignkey2__age=2)))
+        self.assertEqual(4, len(Indexed.objects.all().filter(
+            foreignkey2__age__lt=201)))
+        
+        # test JOINs on different model
+        # standard lookups JOINs
+        self.assertEqual(1, len(ForeignIndexed.objects.all().filter(
+            fk__age=2)))
+        self.assertEqual(2, len(ForeignIndexed.objects.all().filter(
+            fk__age__lt=210)))
+        
+        # other JOINs
+        self.assertEqual(1, len(ForeignIndexed.objects.all().filter(
+            fk__name_fi2__iexact='juUBI')))
+        self.assertEqual(1, len(ForeignIndexed.objects.all().filter(
+            fk__name_fi2__endswith='bi')))
 
     def test_fix_fk_isnull(self):
         self.assertEqual(0, len(Indexed.objects.filter(foreignkey=None)))
-        self.assertEqual(3, len(Indexed.objects.exclude(foreignkey=None)))
+        self.assertEqual(4, len(Indexed.objects.exclude(foreignkey=None)))
 
     def test_iexact(self):
         self.assertEqual(1, len(Indexed.objects.filter(name__iexact='itaChi')))
         self.assertEqual(1, Indexed.objects.filter(name__iexact='itaChi').count())
-
+        
+        self.assertEqual(2, ForeignIndexed.objects.filter(title__iexact='BIJUU').count())
+        self.assertEqual(1, ForeignIndexed.objects.filter(name_fi__iexact='KYuubi').count())
+        
+        # test on list field
+        self.assertEqual(1, Indexed.objects.filter(tags__iexact='SasuKE').count())
+    
+    def test_standard_lookups(self):
+        self.assertEqual(1, Indexed.objects.filter(tags__exact='Naruto').count())
+    
     def test_delete(self):
         Indexed.objects.get(name__iexact='itaChi').delete()
         self.assertEqual(0, Indexed.objects.all().filter(name__iexact='itaChi').count())
     def test_regex(self):
         self.assertEqual(2, len(Indexed.objects.all().filter(name__iregex='^i+')))
         self.assertEqual(2, len(Indexed.objects.all().filter(name__regex='^I+')))
-        self.assertEqual(0, len(Indexed.objects.all().filter(name__regex='^i+')))
         self.assertEqual(1, len(Indexed.objects.all().filter(name__iregex='^i\d*i$')))
 
     def test_date_filters(self):
         now = datetime.now()
-        self.assertEqual(3, len(Indexed.objects.all().filter(published__month=now.month)))
-        self.assertEqual(3, len(Indexed.objects.all().filter(published__day=now.day)))
-        self.assertEqual(3, len(Indexed.objects.all().filter(
+        self.assertEqual(4, len(Indexed.objects.all().filter(published__month=now.month)))
+        self.assertEqual(4, len(Indexed.objects.all().filter(published__day=now.day)))
+        self.assertEqual(4, len(Indexed.objects.all().filter(published__year=now.year)))
+        self.assertEqual(4, len(Indexed.objects.all().filter(
             published__week_day=now.isoweekday())))
 
 #    def test_contains(self):
 #        # passes on production but not on gae-sdk (development)
 #        self.assertEqual(1, len(Indexed.objects.all().filter(name__contains='Aim')))
-#        self.assertEqual(1, len(Indexed.objects.all().filter(name__icontains='aim')))
+#        self.assertEqual(1, len(Indexed.objects.all().filter(name__icontains='aim')))
+#
+#        self.assertEqual(1, ForeignIndexed.objects.filter(name_fi__icontains='Yu').count())
+#
+#        # test icontains on a list
+#        self.assertEqual(2, len(Indexed.objects.all().filter(tags__icontains='RA')))
 from setuptools import setup, find_packages
 
-DESCRIPTION = 'Emulate SQL features on NoSQL/non-relational DBs'
+DESCRIPTION = 'Expressive NoSQL'
 LONG_DESCRIPTION = None
 try:
     LONG_DESCRIPTION = open('README.rst').read()
     pass
 
 setup(name='django-dbindexer',
-      version='0.2',
+      version='0.3',
       packages=find_packages(),
       author='Waldemar Kornewald, Thomas Wanschik',
       author_email='team@allbuttonspressed.com',