Commits

Andy Mikhailenko committed b985142

Refactoring. Documentation. Tests.

Comments (0)

Files changed (24)

    :maxdepth: 2
     
    document
-   fields
+   ext_fields
    backend_base
     ext_tokyo_tyrant
     ext_mongodb
 
+Convenience abstractions
+------------------------
+
+.. toctree::
+    :maxdepth: 2
+
+    ext_fields
+
 Integration with other libraries
 --------------------------------
 
+.. automodule:: doqu.ext.fields
+   :members:

doc/fields.rst

-.. automodule:: doqu.fields
-   :members:
     validators
     utils
     ext
-    fields
     api
 
 Indices and tables
     
     At this point you may ask why are the definitions so verbose. Why not Field
     classes à la Django? Well, they *can* be added on top of what's described
-    here. Actually Docu ships with :doc:`fields` so you can easily write::
+    here. Actually Docu ships with :doc:`ext_fields` so you can easily write::
 
         class Person(Document):
             name = Field(unicode, required=True)
 """
 
 import logging
+import warnings
 
 import document_base
 
 
 
 class BaseStorageAdapter(object):
+    """Abstract adapter class for storage backends.
+
+    .. note:: Backends policy
+
+        If a public method `foo()` internally uses a private method `_foo()`,
+        then subclasses should only overload only the private attribute. This
+        ensures that docstring and signature are always correct. However, if
+        the backend introduces some deviations in behaviour or extends the
+        signature, the public method can (and should) be overloaded at least to
+        provide documentation.
+
     """
-    Abstract adapter class for storage backends.
-    """
-
     # these must be defined by the backend subclass
     converter_manager = NotImplemented
     lookup_manager = NotImplemented
         raise NotImplementedError
 
     def __nonzero__(self):
-        if self.connection is not None:
-            return True
+        return self.connection is not None
 
-    #----------------------+
-    #  Private attributes  |
-    #----------------------+
+    #----------------+
+    #  Internal API  |
+    #----------------+
 
     @classmethod
     def _assert_implemented(cls, *attrs):
                 'Backend {cls.__module__} must define '
                 '{cls.__name__}.{attr}'.format(**locals()))
 
-    def _decorate(self, doc_class, key, data):
-        """
-        Populates a document class instance with given data. If the class has
-        the method `from_storage(storage, key, data)`, it is used to produce
-        the result. If this method is not present, a tuple of key and data
-        dictionary is returned.
+    def _clear(self):
+        raise NotImplementedError # pragma: nocover
+
+    def _connect(self):
+        raise NotImplementedError
+
+    def _decorate(self, key, data, doc_class=dict):
+        """Populates a document class instance with given data. If the class
+        has the method `from_storage(storage, key, data)`, it is used to
+        produce the result. If this method is not present, a tuple of key and
+        data dictionary is returned.
         """
         if hasattr(doc_class, 'from_storage'):
             return doc_class.from_storage(storage=self, key=key, data=data)
         else:
             return key, doc_class(**data)
 
+    def _delete(self, key):
+        raise NotImplementedError
 
-    def _fetch(self, primary_key):
-        """
-        Returns a dictionary representing the record with given primary key.
-        """
+    def _get(self, key):
+        "Returns a dictionary representing the record with given primary key."
+        raise NotImplementedError # pragma: nocover
+
+    def _get_many(self, keys):
+        # return (or yield) key/data pairs (order doesn't matter)
+        return ((pk, self._get(pk)) for pk in keys)
+
+    def _disconnect(self):
+        # typical implementation:
+        #   self.connection.close()
+        #   self.connection = None
+        raise NotImplementedError # pragma: nocover
+
+    def _prepare_data_for_saving(self, data):
+        # some backends (e.g. MongoDB) need to skip certain fields so they
+        # would overload this method
+        return dict((k, self.value_to_db(v)) for k,v in data.iteritems())
+
+    def _save(self, key, data):
+        # NOTE: must return the key (given or a new one if none given)
+        # NOTE: `key` can be `None`.
+        raise NotImplementedError # pragma: nocover
+
+    def _sync(self):
+        # a backend should only overload this if it supports the operation
         raise NotImplementedError # pragma: nocover
 
     #--------------+
     #--------------+
 
     def clear(self):
+        """Clears the whole storage from data, resets autoincrement counters.
         """
-        Clears the whole storage from data, resets autoincrement counters.
-        """
-        raise NotImplementedError # pragma: nocover
+        if self.connection is None:  # pragma: nocover
+            raise RuntimeError('Cannot clear storage: no connection.')
+        self._clear()
 
     def connect(self):
+        """Connects to the database. Raises RuntimeError if the connection is
+        not closed yet. Use :meth:`reconnect` to explicitly close the
+        connection and open it again.
         """
-        Connects to the database. Raises RuntimeError if the connection is not
-        closed yet. Use :meth:`reconnect` to explicitly close the connection
-        and open it again.
-        """
-        raise NotImplementedError
+        if self.connection is not None:  # pragma: nocover
+            raise RuntimeError('already connected')
+        self._connect()
 
     def delete(self, key):
+        """Deletes record with given primary key.
         """
-        Deletes record with given primary key.
-        """
-        raise NotImplementedError # pragma: nocover
+        if self.connection is None:
+            raise RuntimeError('Cannot delete key: no connection.')
+        self._delete(key)
 
     def disconnect(self):
+        """Closes internal store and removes the reference to it. If the
+        backend works with a file, then all pending changes are saved now.
         """
-        Closes internal store and removes the reference to it.
+        if self.connection is None:
+            raise RuntimeError('Cannot disconnect: no connection.')
+        self._disconnect()
+
+    def get(self, key, doc_class=dict):
+        """Returns document instance for given document class and primary key.
+        Raises KeyError if there is no item with given key in the database.
+
+        :param key:
+            a numeric or string primary key (as supported by the backend).
+        :param doc_class:
+            a document class to wrap the data into. Default is `dict`.
         """
-        # typical implementation:
-        #   self.connection.close()
-        #   self.connection = None
-        raise NotImplementedError # pragma: nocover
+        if self.connection is None:  # pragma: nocover
+            raise RuntimeError('Cannot fetch item: no connection.')
 
-    def get(self, doc_class, primary_key):
-        """
-        Returns document instance for given document class and primary key.
-        Raises KeyError if there is no item with given key in the database.
-        """
-        log.debug('fetching record "%s"' % primary_key)
-        data = self._fetch(primary_key)
-        return self._decorate(doc_class, primary_key, data)
+        if not isinstance(key, (int,basestring)):
+            warnings.warn('db.get(doc_class, key) is deprecated; use '
+                          'db.get(key, doc_class) instead', DeprecationWarning)
+            key, doc_class = doc_class, key
 
-    def get_many(self, doc_class, primary_keys):
-        """
-        Returns a list of documents with primary keys from given list.
+        #log.debug('fetching record "%s"' % key)
+        data = self._get(key)
+        #return self._decorate(doc_class, primary_key, data)
+
+        return self._decorate(key, data, doc_class)
+
+#        # FIXME HACK this should use some nice simple API (maybe "require_key")
+#        return self._decorate(key, data, doc_class)
+#        if hasattr(doc_class, 'from_storage'):
+#            return result
+#        else:
+#            return result#[1]  # HACK!!!
+
+    def get_many(self, keys, doc_class=dict):
+        """Returns a list of documents with primary keys from given list.
         Basically this is just a simple wrapper around
         :meth:`~BaseStorageAdapter.get` but some backends can reimplement the
         method in a much more efficient way.
         """
-        return [self.get(doc_class, pk) for pk in primary_keys]
+        if self.connection is None:  # pragma: nocover
+            raise RuntimeError('Cannot fetch items: no connection.')
 
-    def get_or_create(self, doc_class, **kwargs):
-        """
-        Queries the database for records associated with given document class
-        and conforming to given extra condtions. If such records exist, picks
-        the first one (the order may be random depending on the database). If
-        there are no such records, creates one.
+        if not hasattr(keys, '__iter__') or \
+           not isinstance(keys[0], (int,basestring)):
+            warnings.warn('db.get_many(doc_class, keys) is deprecated; use '
+                          'db.get_many(keys, doc_class) instead', DeprecationWarning)
+            keys, doc_class = doc_class, keys
+
+        return [self._decorate(key, data)
+                           for key, data in self._get_many(keys)]
+
+    def get_or_create(self, doc_class=dict, **conditions):
+        """Queries the database for records associated with given document
+        class and conforming to given extra conditions. If such records exist,
+        picks the first one (the order may be random depending on the
+        database). If there are no such records, creates one.
 
         Returns the document instance and a boolean value "created".
         """
         assert kwargs
-        query = self.find(doc_class).where(**kwargs)
+
+        if self.connection is None:  # pragma: nocover
+            raise RuntimeError('Cannot fetch items: no connection.')
+
+        query = self.find(doc_class).where(**conditions)
         if query.count():
             return query[0], False
         else:
-            obj = doc_class(**kwargs)
+            obj = doc_class(**conditions)
             obj.save(self)
             return obj, True
 
     def find(self, doc_class=dict, **conditions):
-        """
-        Returns instances of given class, optionally filtered by given
+        """Returns instances of given class, optionally filtered by given
         conditions.
 
         :param doc_class:
             `Document` object is returned per item.
 
         """
-        query = self.query_adapter(storage=self, model=doc_class)
+        if self.connection is None:  # pragma: nocover
+            raise RuntimeError('Cannot fetch items: no connection.')
+
+        query = self.query_adapter(storage=self, doc_class=doc_class)
         if hasattr(doc_class, 'contribute_to_query'):
             query = doc_class.contribute_to_query(query)
         return query.where(**conditions)
         import warnings
         warnings.warn('StorageAdapter.get_query() is deprecated, use '
                       'StorageAdapter.find() instead.', DeprecationWarning)
-        return self.find(model)
-
+        return self.find(doc_class=model)
 
     def reconnect(self):
-        """
-        Gracefully closes current connection (if it's not broken) and connects
-        again to the database (e.g. reopens the file).
+        """Gracefully closes current connection (if it's not broken) and
+        connects again to the database (e.g. reopens the file).
         """
         self.disconnect()
         self.connect()
 
-    def save(self, model, data, primary_key=None):
+    def save(self, key, data): #, doc_class=dict):
+        """Saves given data with given primary key into the storage. Returns
+        the primary key.
+
+        :param key:
+
+            the primary key for given object; if `None`, will be generated.
+
+        :param data:
+
+            a `dict` containing all properties to be saved.
+
+        Note that you must provide current primary key for a record which is
+        already in the database in order to update it instead of copying it.
         """
-        Saves given model instance into the storage. Returns
-        primary key.
+        if self.connection is None:  # pragma: nocover
+            raise RuntimeError('Cannot fetch items: no connection.')
 
-        :param model: model class
-        :param data: dict containing all properties
-            to be saved
-        :param primary_key: the key for given object; if undefined, will be
-            generated
+        if key is not None and not isinstance(key, (int,basestring)):
+            warnings.warn('db.save(data, key) is deprecated; use '
+                          'db.get(key, data) instead',
+                          DeprecationWarning)
+            key, data = data, key
 
-        Note that you must provide current primary key for a model instance
-        which is already in the database in order to update it instead of
-        copying it.
+        outgoing = self._prepare_data_for_saving(data)
+
+        resulting_key = self._save(key, outgoing)
+        assert resulting_key, 'Backend-specific _save() must return a key'
+        return resulting_key
+
+    def sync(self):
+        """Synchronizes the storage to disk immediately if the backend supports
+        this operation. Normally the data is synchronized either on
+        :meth:`save`, or on timeout, or on :meth:`disconnect`. This is strictly
+        backend-specific. If a backend does not support the operation,
+        `NotImplementedError` is raised.
         """
-        raise NotImplementedError # pragma: nocover
+        if self.connection is None:  # pragma: nocover
+            raise RuntimeError('Cannot fetch items: no connection.')
+
+        self._sync()
 
     def value_from_db(self, datatype, value):
         return self.converter_manager.from_db(datatype, value)
     def __getitem__(self, key):
         raise NotImplementedError # pragma: nocover
 
-    def __init__(self, storage, model):
+    def __init__(self, storage, doc_class):
         self.storage = storage
-        self.model = model
+        self.doc_class = doc_class
         self._init()
 
     def __iter__(self):
     def __len__(self):
         return len(self[:])
 
+    def __nonzero__(self):
+        # __len__ would be enough for a simple iterable but it would fetch the
+        # whole set of results on `bool(query)`. This method makes sure that if
+        # the results are fetched in chunks, only the first chunk is fetched.
+        try:
+            self[0]
+        except IndexError:
+            return False
+        else:
+            return True
+
     def __or__(self, other):
         raise NotImplementedError # pragma: nocover
 
     def __sub__(self, other):
         raise NotImplementedError # pragma: nocover
 
-    #----------------------+
-    #  Private attributes  |
-    #----------------------+
+    #----------------+
+    #  Internal API  |
+    #----------------+
 
     def _get_native_conditions(self, conditions, negate=False):
         """
     def _init(self):
         pass
 
+    def _delete(self):
+        raise NotImplementedError # pragma: nocover
+
+    def _order_by(self, names, reverse=False):
+        raise NotImplementedError # pragma: nocover
+
+    def _values(self, **conditions):
+        raise NotImplementedError # pragma: nocover
+
+    def _where(self, **conditions):
+        raise NotImplementedError # pragma: nocover
+
+    def _where_not(self, **conditions):
+        raise NotImplementedError # pragma: nocover
+
     #--------------+
     #  Public API  |
     #--------------+
 
     def count(self):
-        """
-        Returns the number of records that match given query. The result of
+        """Returns the number of records that match given query. The result of
         `q.count()` is exactly equivalent to the result of `len(q)`. The
         implementation details do not differ by default, but it is recommended
         that the backends stick to the following convention:
         return len(self)    # may be inefficient, override if possible
 
     def delete(self):
+        """Deletes all records that match current query.
         """
-        Deletes all records that match current query.
-        """
-        raise NotImplementedError # pragma: nocover
+        self._delete()
 
-    def order_by(self, name):
-        """
-        Returns a query object with same conditions but with results sorted by
-        given column.
+    def order_by(self, names, reverse=False):
+        """Returns a query object with same conditions but with results sorted
+        by given field. By default the direction of sorting is ascending.
 
-        :param name: string: name of the column by which results should be
-            sorted. If the name begins with a ``-``, results will come in
-            reversed order.
+        :param names:
+
+            list of strings: names of fields by which results should be sorted.
+            Some backends may only support a single field for sorting.
+
+        :param reverse:
+
+            `bool`: if `True`, the direction of sorting is reversed
+            and becomes descending. Default is `False`.
 
         """
-        raise NotImplementedError # pragma: nocover
+        return self._order_by(names, reverse=reverse)
 
     def values(self, name):
+        """Returns a list of unique values for given field name.
+
+        :param name:
+            the field name.
+
         """
-        Returns a list of unique values for given column name.
-        """
-        raise NotImplementedError # pragma: nocover
+        return self._values(name)
 
     def where(self, **conditions):
-        """
-        Returns Query instance filtered by given conditions.
+        """Returns Query instance filtered by given conditions.
         The conditions are specified by backend's underlying API.
         """
-        raise NotImplementedError # pragma: nocover
+        return self._where(**conditions)
 
     def where_not(self, **conditions):
+        """Returns Query instance. Inverted version of `where()`.
         """
-        Returns Query instance. Inverted version of `where()`.
-        """
-        raise NotImplementedError # pragma: nocover
+        return self._where_not(**conditions)
 
 
 #--- PROCESSORS
 
 
 class ProcessorDoesNotExist(Exception):
-    """
-    This exception is raised when given backend does not have a processor
+    """This exception is raised when given backend does not have a processor
     suitable for given value. Usually you will need to catch a subclass of this
     exception.
     """
 
 
 class ProcessorManager(object):
-    """
-    Abstract manager of named functions or classes that process data.
+    """Abstract manager of named functions or classes that process data.
     """
     exception_class = ProcessorDoesNotExist
 
         self.default = None
 
     def register(self, key, default=False):
-        """
-        Registers given processor class with given datatype. Decorator. Usage::
+        """Registers given processor class with given datatype. Decorator.
+        Usage::
 
             converter_manager = ConverterManager()
 
         return _inner
 
     def unregister(self, key):
-        """
-        Unregisters and returns a previously registered processor for given
-        value or raises :class:`ProcessorDoesNotExist` is none was
-        registered.
+        """Unregisters and returns a previously registered processor for given
+        value or raises :class:`ProcessorDoesNotExist` is none was registered.
         """
         try:
             processor = self.processors[key]
             return processor
 
     def get_processor(self, value):
-        """
-        Returns processor for given value.
+        """Returns processor for given value.
 
         Raises :class:`DataProcessorDoesNotExist` if no suitable processor is
         defined by the backend.
 
 
 class LookupProcessorDoesNotExist(ProcessorDoesNotExist):
-    """
-    This exception is raised when given backend does not support the requested
-    lookup.
+    """This exception is raised when given backend does not support the
+    requested lookup.
     """
     pass
 
 
 class LookupManager(ProcessorManager):
-    """
-    Usage::
+    """Usage::
 
         lookup_manager = LookupManager()
 
     # (beware: 1. endless recursion, and 2. possible logic trees)
 
     def resolve(self, name, operation, value):
-        """
-        Returns a set of backend-specific conditions for given backend-agnostic
-        triplet, e.g.::
+        """Returns a set of backend-specific conditions for given
+        backend-agnostic triplet, e.g.::
 
             ('age', 'gt', 90)
 
 
 
 class DataProcessorDoesNotExist(ProcessorDoesNotExist):
-    """
-    This exception is raised when given backend does not have a datatype
+    """This exception is raised when given backend does not have a datatype
     processor suitable for given value.
     """
     pass
 
 
 class ConverterManager(ProcessorManager):
-    """
-    An instance of this class can manage property processors for given backend.
-    Processor classes must be registered against Python types or classes. The
-    processor manager allows encoding and decoding data between a model
-    instance and a database record. Each backend supports only a certain subset
-    of Python datatypes and has its own rules in regard to how `None` values
-    are interpreted, how complex data structures are serialized and so on.
-    Moreover, there's no way to guess how a custom class should be processed.
-    Therefore, each combination of data type + backend has to be explicitly
-    defined as a set of processing methods (to and from).
+    """An instance of this class can manage property processors for given
+    backend. Processor classes must be registered against Python types or
+    classes. The processor manager allows encoding and decoding data between a
+    document class instance and a database record. Each backend supports only a
+    certain subset of Python datatypes and has its own rules in regard to how
+    `None` values are interpreted, how complex data structures are serialized
+    and so on. Moreover, there's no way to guess how a custom class should be
+    processed. Therefore, each combination of data type + backend has to be
+    explicitly defined as a set of processing methods (to and from).
     """
     exception_class = DataProcessorDoesNotExist
 
 
 
     def from_db(self, datatype, value):
-        """
-        Converts given value to given Python datatype. The value must be
+        """Converts given value to given Python datatype. The value must be
         correctly pre-encoded by the symmetrical :meth:`PropertyManager.to_db`
         method before saving it to the database.
 
         return p.from_db(value)
 
     def to_db(self, value, storage):
-        """
-        Prepares given value and returns it in a form ready for storing in the
-        database.
+        """Prepares given value and returns it in a form ready for storing in
+        the database.
 
         Raises :class:`DataProcessorDoesNotExist` if no suitable processor is
         defined by the backend.
         datatype = type(value)
         p = self._pick_processor(datatype)
         return p.to_db(value, storage)
-
-

doqu/document_base.py

 
 
 class Document(ReprMixin, DotDict):
-
+    """A document/query object. Dict-like representation of a document stored
+    in a database. Includes schema declaration, bi-directional validation
+    (outgoing and query), handles relations and has the notion of the saved
+    state, i.e. knows the storage and primary key of the corresponding record.
+    """
     __metaclass__ = DocumentMetaclass
     supports_nested_data = False
 
     # XXX validation-related, move to a mixin within doqu.validation?
     @classmethod
     def contribute_to_query(cls, query):
-        """
-        Returns a query for records stored in given storage and associated with
-        given document class. Usage::
-
-            events = Event.objects(db)
-
-        :param storage:
-            a :class:`~doqu.backend_base.BaseStorageAdapter` subclass (see
-            :doc:`ext`).
-
+        """Returns given query filtered by schema and validators defined for
+        this document.
         """
         # use validators to filter the results to only yield records that
         # belong to this schema
                         raise
                 pythonized_data[name] = value
         else:
-            # if the structure is unknown, just populate the document as is
+            # if the structure is unknown, just populate the document as is.
+            # copy.deepcopy is safer than dict.copy but more than 10× slower.
             pythonized_data = data.copy()
 
         instance = cls(**pythonized_data)
             primary_key = None
         # let the storage backend prepare data and save it to the actual storage
         key = storage.save(
+            key = primary_key,
+            data = data,
             #doc_class = type(self),
-            data = data,
-            primary_key = primary_key,
         )
         assert key, 'storage must return primary key of saved item'
         # okay, update our internal representation of the record with what have
                     value = value()
             yield name, value
 
-
-
 def _get_document_by_ref(doc, field, value):
     if not value:
         return value
 def _validate_value_type(cls, key, value):
     if value is None:
         return
+
     datatype = cls.meta.structure.get(key)
+
     if isinstance(datatype, basestring):
         # A text reference, i.e. "cls" or document class name.
         return
+
     if issubclass(datatype, Document) and isinstance(value, basestring):
         # A class reference; value is the PK, not the document object.
         # This is a normal situation when a document instance is being
         # created from a database record. The reference will be resolved
         # later on __getitem__ call. We just skip it for now.
         return
+
     if isinstance(datatype, OneToManyRelation):
         if not hasattr(value, '__iter__'):
             msg = u'{cls}.{field}: expected list of documents, got {value}'
             raise validators.ValidationError(msg.format(
                 cls=type(cls).__name__, field=key, value=repr(value)))
         return
+
     if datatype and not isinstance(value, datatype):
         msg = u'{cls}.{field}: expected a {datatype} instance, got {value}'
         raise validators.ValidationError(msg.format(

doqu/ext/mongodb/__init__.py

     involving complex queries. Patches, improvements, rewrites are welcome.
 
 """
+import warnings
 
 from doqu import dist
 dist.check_dependencies(__name__)
 
     # ...
 
-    #----------------------+
-    #  Private attributes  |
-    #----------------------+
+    #----------------+
+    #  Internal API  |
+    #----------------+
 
     def _do_search(self, **kwargs):
         # TODO: slicing? MongoDB supports it since 1.5.1
         spec = self.storage.lookup_manager.combine_conditions(self._conditions)
         if self._ordering:
             kwargs.setdefault('sort',  self._ordering)
+            # may be undesirable but prevents "pymongo.errors.OperationFailure:
+            # database error: too much data for sort() with no index"
+            self.storage.connection.ensure_index(self._ordering)
         cursor = self.storage.connection.find(spec, **kwargs)
         self._cursor = cursor  # used in count()    XXX that's a mess
         return iter(cursor) if cursor is not None else []
 
-    def _init(self, storage, model, conditions=None, ordering=None):
+    def _init(self, storage, doc_class=dict, conditions=None, ordering=None):
         self.storage = storage
-        self.model = model
+        self.doc_class = doc_class
         self._conditions = conditions or []
         self._ordering = ordering
         #self._query = self.storage.connection.find()
     def _clone(self, extra_conditions=None, extra_ordering=None):
         return self.__class__(
             self.storage,
-            self.model,
+            self.doc_class,
             conditions = self._conditions + (extra_conditions or []),
             ordering = extra_ordering or self._ordering,
         )
             self._iter = self._do_search()
 
     def _prepare_item(self, raw_data):
-        return self.storage._decorate(self.model, None, raw_data)
+        return self.storage._decorate(None, raw_data, self.doc_class)
 
-    def _where(self, lookups, negate=False):
+    def __where(self, lookups, negate=False):
         conditions = list(self._get_native_conditions(lookups, negate))
         return self._clone(extra_conditions=conditions)
 
-    #--------------+
-    #  Public API  |
-    #--------------+
+    def _where(self, **conditions):
+        return self.__where(conditions, negate=False)
 
-    def count(self):
-        return self._cursor.count()
-
-    def where(self, **conditions):
-        """
-        Returns Query instance filtered by given conditions.
-        The conditions are defined exactly as in Pyrant's high-level query API.
-        See pyrant.query.Query.filter documentation for details.
-        """
-        return self._where(conditions, negate=False)
-
-    def where_not(self, **conditions):
-        """
-        Returns Query instance. Inverted version of
-        :meth:`~doqu.backends.tokyo_cabinet.Query.where`.
-        """
-        return self._where(conditions, negate=True)
+    def _where_not(self, **conditions):
+        return self.__where(conditions, negate=True)
 
 #        # FIXME PyMongo conditions API propagates; we would like to unify all
 #        # APIs but let user specify backend-specific stuff.
 #        q = self.storage.connection.find(combined_conds)
 #        return self._clone(q)
 
-#    def where_not(self, **conditions):
-#        raise NotImplementedError
-
-#    def count(self):
-#        return self._query.count()
-
-    def order_by(self, names, reverse=False):
+    def _order_by(self, names, reverse=False):
         # TODO: MongoDB supports per-key directions. Support them somehow?
+        # E.g.   names=[('foo', True)]   ==   names=['foo'],reverse=True
         direction = pymongo.DESCENDING if reverse else pymongo.ASCENDING
         if isinstance(names, basestring):
             names = [names]
         ordering = [(name, direction) for name in names]
         return self._clone(extra_ordering=ordering)
 
+    #--------------+
+    #  Public API  |
+    #--------------+
+
+    def count(self):
+        """Returns the number of records that match given query. The result of
+        `q.count()` is exactly equivalent to the result of `len(q)` but does
+        not involve fetching of the records.
+        """
+        return self._cursor.count()
+
     def values(self, name):
-        """
-        Returns distinct values for given field.
+        """Returns a list of unique values for given field name.
 
         :param name:
             the field name.
             values.add(d.get(name))
         return values
 
-#    def delete(self):
-#        """
-#        Deletes all records that match current query.
-#        """
-#        raise NotImplementedError
-
 
 class StorageAdapter(BaseStorageAdapter):
     """
     :param database:
     :param collection:
     """
-    supports_nested_data = True
+    #supports_nested_data = True
 
     converter_manager = converter_manager
     lookup_manager = lookup_manager
         """
         Yields all keys available for this connection.
         """
-        return iter(self.connection.find(spec={}, fields={'_id': 1}))
+        return (x['_id'] for x in self.connection.find(spec={}, fields={'_id': 1}))
 
     def __len__(self):
         return self.connection.count()
 
-    #----------------------+
-    #  Private attributes  |
-    #----------------------+
+    #----------------+
+    #  Internal API  |
+    #----------------+
 
-    def _decorate(self, model, primary_key, raw_data):
-        data = dict(raw_data)
-        key = data.pop('_id')
+    def _decorate(self, key, data, doc_class=dict):
+        clean_data = dict(data)
+        raw_key = clean_data.pop('_id')
         # this case is for queries where we don't know the PKs in advance;
         # however, we do know them when fetching a certain document by PK
-        if primary_key is None:
-            primary_key = self._object_id_to_string(key)
-        return super(StorageAdapter, self)._decorate(model, primary_key, data)
+        if key is None:
+            key = self._object_id_to_string(raw_key)
+        return super(StorageAdapter, self)._decorate(key, clean_data, doc_class)
 
-    def _object_id_to_string(self, pk):
-        if isinstance(pk, ObjectId):
-            return u'x-objectid-{0}'.format(pk)
-        return pk
+    def _object_id_to_string(self, key):
+        if isinstance(key, ObjectId):
+            return u'x-objectid-{0}'.format(key)
+        return key
 
-    def _string_to_object_id(self, pk):
+    def _string_to_object_id(self, key):
         # XXX check consistency
         # MongoDB will *not* find items by the str/unicode representation of
         # ObjectId so we must wrap them; however, it *will* find items if their
         # likely be not accepted by ObjectId as arguments.
         # Also check StorageAdapter.__contains__, same try/catch there.
         #print 'TESTING GET', model.__name__, primary_key
-        assert isinstance(pk, basestring)
-        if pk.startswith('x-objectid-'):
-            return ObjectId(pk.split('x-objectid-')[1])
-        return pk
+        assert isinstance(key, basestring)
+        if key.startswith('x-objectid-'):
+            return ObjectId(key.split('x-objectid-')[1])
+        return key
 
-    #--------------+
-    #  Public API  |
-    #--------------+
-
-    def clear(self):
+    def _clear(self):
         """
         Clears the whole storage from data.
         """
         self.connection.remove()
 
-    def connect(self):
+    def _connect(self):
         host = self._connection_options.get('host', '127.0.0.1')
         port = self._connection_options.get('port', 27017)
         database_name = self._connection_options.get('database', 'default')
         self._mongo_collection = self._mongo_database[collection_name]
         self.connection = self._mongo_collection
 
-    def delete(self, primary_key):
-        """
-        Permanently deletes the record with given primary key from the database.
-        """
-        primary_key = self._string_to_object_id(primary_key)
-        self.connection.remove({'_id': primary_key})
+    def _delete(self, key):
+        key = self._string_to_object_id(key)
+        self.connection.remove({'_id': key})
 
-    def disconnect(self):
+    def _disconnect(self):
         self._mongo_connection.disconnect()
         self._mongo_connection = None
         self._mongo_database = None
         self._mongo_collection = None
         self.connection = None
 
-    def get(self, model, primary_key):
-        """
-        Returns model instance for given model and primary key.
-        Raises KeyError if there is no item with given key in the database.
-        """
-        obj_id = self._string_to_object_id(primary_key)
+    def _get(self, key):
+        obj_id = self._string_to_object_id(key)
         data = self.connection.find_one({'_id': obj_id})
         if data:
-            return self._decorate(model, str(primary_key), data)
+            return data
         raise KeyError('collection "{collection}" of database "{database}" '
                        'does not contain key "{key}"'.format(
                            database = self._mongo_database.name,
                            collection = self._mongo_collection.name,
-                           key = str(primary_key)
+                           key = str(key)
                        ))
 
-    def get_many(self, doc_class, primary_keys):
-        """
-        Returns a list of documents with primary keys from given list. More
-        efficient than calling :meth:`~StorageAdapter.get` multiple times.
-        """
-        obj_ids = [self._string_to_object_id(pk) for pk in primary_keys]
+    def _get_many(self, keys):
+        obj_ids = [self._string_to_object_id(key) for key in keys]
         results = self.connection.find({'_id': {'$in': obj_ids}}) or []
-        assert len(results) <= len(primary_keys), '_id must be unique'
+
+
+
+        print dir(results)
+
+
+        found_keys = []
+        for data in results:
+            key = str(self._object_id_to_string(data['_id']))
+            found_keys.append(key)
+            yield key, data
+
+        if len(found_keys) < len(keys):
+            missing_keys = [k for k in keys if k not in found_keys]
+            raise KeyError('collection "{collection}" of database "{database}"'
+                           ' does not contain keys "{keys}"'.format(
+                               database = self._mongo_database.name,
+                               collection = self._mongo_collection.name,
+                               keys = ', '.join(missing_keys)))
+        '''
+        assert len(results) <= len(keys), '_id must be unique'
         _get_obj_pk = lambda obj: str(self._object_id_to_string(data['_id']))
-        if len(data) == len(primary_keys):
-            return [self._decorate(model, _get_obj_pk(obj), data)
-                    for data in results]
-        keys = [_get_obj_pk(obj) for obj in results]
-        missing_keys = [pk for pk in keys if pk not in primary_keys]
+        if len(data) == len(keys):
+            return ((_get_obj_pk(data), data) for data in results)
+#            return [self._decorate(doc_class, _get_obj_pk(obj), data)
+#                    for data in results]
+
+        # not all keys were found; raise an informative exception
+        _keys = [_get_obj_pk(obj) for obj in results]
+        missing_keys = [pk for pk in _keys if pk not in keys]
         raise KeyError('collection "{collection}" of database "{database}" '
                        'does not contain keys "{keys}"'.format(
                            database = self._mongo_database.name,
                            collection = self._mongo_collection.name,
                            keys = ', '.join(missing_keys)))
+        '''
 
-    def save(self, data, primary_key=None):
-        """
-        Saves given model instance into the storage. Returns primary key.
+    def _prepare_data_for_saving(self, data):
+        # the "_id" field should be excluded from the resulting dictionary
+        # because a) ObjectId is not handled correctly by converters, and
+        # b) the _save() method will anyway add the correct "_id"
+        clean = dict((k,v) for k,v in data.iteritems() if k != '_id')
+        return super(StorageAdapter, self)._prepare_data_for_saving(clean)
 
-        :param data:
-            dict containing all properties to be saved
-        :param primary_key:
-            the key for given object; if undefined, will be generated
-
-        Note that you must provide current primary key for a model instance which
-        is already in the database in order to update it instead of copying it.
-        """
-        outgoing = data.copy()
-        if primary_key:
-            outgoing.update({'_id': self._string_to_object_id(primary_key)})
-#        print outgoing
-        obj_id = self.connection.save(outgoing)
-        return self._object_id_to_string(obj_id) or primary_key
-#        return unicode(self.connection.save(outgoing) or primary_key)
+    def _save(self, key, data):
+        if key:
+            data = dict(data, _id=self._string_to_object_id(key))
+        obj_id = self.connection.save(data)
+        return self._object_id_to_string(obj_id) or key

doqu/ext/shelve_db/__init__.py

             def make_sort_key(pk):
                 data = self.storage.connection[pk]
                 return [data[name] for name in self._ordering['names']
-                        if data.get(name) is not None]
+                           if data.get(name) is not None]
 
             return iter(LazySorted(
                 data = finder(),
             ))
         return finder()
 
-    def _init(self, storage, model, conditions=None, ordering=None):
+    def _init(self, storage, doc_class, conditions=None, ordering=None):
         self.storage = storage
-        self.model = model
+        self.doc_class = doc_class
         self._conditions = conditions or []
         self._ordering = ordering or {}
         # this is safe because the adapter is instantiated already with final
             self._iter = self._do_search()
 
     def _prepare_item(self, key):
-        return self.storage.get(self.model, key)
+        x = self.storage.get(key, self.doc_class)
+        return self.storage.get(key, self.doc_class)
 
-    def _where(self, lookups, negate=False):
+    def __where(self, lookups, negate=False):
         """
         Returns Query instance filtered by given conditions.
         The conditions are defined exactly as in Pyrant's high-level query API.
     def _clone(self, extra_conditions=None, extra_ordering=None):
         return self.__class__(
             self.storage,
-            self.model,
+            self.doc_class,
             conditions = self._conditions + (extra_conditions or []),
             ordering = extra_ordering or self._ordering,
         )
 
-    #--------------+
-    #  Public API  |
-    #--------------+
+    def _delete(self):
+        """
+        Deletes all records that match current query. Iterates the whole set of
+        records.
+        """
+        for pk in self._do_search():
+            self.storage.delete(pk)
 
-    def where(self, **conditions):
+    def _where(self, **conditions):
         """
         Returns Query instance filtered by given conditions.
         The conditions are defined exactly as in Pyrant's high-level query API.
         See pyrant.query.Query.filter documentation for details.
         """
-        return self._where(conditions)
+        return self.__where(conditions)
 
-    def where_not(self, **conditions):
+    def _where_not(self, **conditions):
         """
         Returns Query instance. Inverted version of
         :meth:`~doqu.backends.tokyo_cabinet.Query.where`.
         """
         return self._where(conditions, negate=True)
 
+    #--------------+
+    #  Public API  |
+    #--------------+
+
     def count(self):
         """
         Same as ``__len__`` but a bit faster.
         return len(list(self._do_search()))
 
     def values(self, name):
-        """
-        Returns an iterator that yields distinct values for given column name.
+        """Returns an iterator that yields distinct values for given column
+        name.
 
         Supports date parts (i.e. `date__month=7`).
 
             # foo__bar__baz --> foo.bar.baz
             attrs = name.split('__') if '__' in name else [name]
             field = attrs.pop(0)
+
+            # FIXME this seems to be a HACK because we are guessing what
+            # storage._decorate() returned
+            if isinstance(doc, tuple):
+                doc = doc[1]
+
             value = doc.get(field)
             if value is None:
                 return
                 known_values[value] = 1
                 yield value
 
-    def delete(self):
-        """
-        Deletes all records that match current query. Iterates the whole set of
-        records.
-        """
-        for pk in self._do_search():
-            self.storage.delete(pk)
-
     def order_by(self, names, reverse=False):
         """
         Defines order in which results should be retrieved.
 
 
 class StorageAdapter(BaseStorageAdapter):
-    """
+    """Provides unified Doqu API for MongoDB (see
+    :class:`doqu.backend_base.BaseStorageAdapter`).
+
     :param path:
         relative or absolute path to the database file (e.g. `test.db`)
 
     """
-    supports_nested_data = True
+    #supports_nested_data = True
     converter_manager = converter_manager
     lookup_manager = lookup_manager
     query_adapter = QueryAdapter
     def __len__(self):
         return len(self.connection)
 
-    def _generate_uid(self):
-        key = str(uuid.uuid4())
-        assert key not in self
-        return key
+    #----------------+
+    #  Internal API  |
+    #----------------+
 
-    #--------------+
-    #  Public API  |
-    #--------------+
-
-    def clear(self):
-        """
-        Clears the whole storage from data.
-        """
+    def _clear(self):
         self.connection.clear()
 
-    def connect(self):
-        """
-        Connects to the database. Raises RuntimeError if the connection is not
-        closed yet. Use :meth:`StorageAdapter.reconnect` to explicitly close
-        the connection and open it again.
-        """
-        if self.connection is not None:
-            raise RuntimeError('already connected')
-
+    def _connect(self):
         path = self._connection_options['path']
         self.connection = shelve.open(path)
 
         atexit.register(lambda: self.connection is not None and
                                 self.connection.close())
 
-    def disconnect(self):
-        """
-        Writes the data into the file, closes the file and deletes the
-        connection.
-        """
-        self.connection.close()
+    def _disconnect(self):
+        self.connection.close()  # writes data to the file and closes the file
         self.connection = None
 
-    def delete(self, primary_key):
-        """
-        Permanently deletes the record with given primary key from the database.
-        """
-        del self.connection[primary_key]
+    def _delete(self, key):
+        del self.connection[key]
 
-    def get(self, model, primary_key):
-        """
-        Returns model instance for given model and primary key.
-        """
-        primary_key = str(primary_key)
-        data = self.connection[primary_key]
-        return self._decorate(model, primary_key, data)
+    def _get(self, key):
+        key = str(key)
+        return self.connection[key]
 
-    def save(self, data, primary_key=None, sync=False):
-        """
-        Saves given model instance into the storage. Returns primary key.
+    def _generate_uid(self):
+        key = str(uuid.uuid4())
+        assert key not in self
+        return key
 
-        :param data:
-            dict containing all properties to be saved
-        :param primary_key:
-            the key for given object; if undefined, will be generated
-        :param sync:
-            if `True`, the storage is synchronized to disk immediately. This
-            slows down bulk operations but ensures that the data is stored no
-            matter what happens. Normally the data is synchronized on exit.
-
-        Note that you must provide current primary key for a model instance which
-        is already in the database in order to update it instead of copying it.
-        """
+    def _save(self, key, data):
         assert isinstance(data, dict)
-
-        primary_key = str(primary_key or self._generate_uid())
-
-        self.connection[primary_key] = data
-
-        if sync:
-            self.connection.sync()
-
-        return primary_key
+        key = str(key or self._generate_uid())
+        self.connection[key] = data
+        return key

doqu/ext/shove_db/__init__.py

 from doqu.ext import shelve_db
 
 
-__all__ = ['StorageAdapter']
+__all__ = ['StorageAdapter', 'QueryAdapter']
+
+
+QueryAdapter = shelve_db.QueryAdapter   # for generated documentation
 
 
 class StorageAdapter(shelve_db.StorageAdapter):
     .. _SQLAlchemy's: http://www.sqlalchemy.org/docs/04/dbengine.html#dbengine_establishing
 
     """
-    #--------------------+
-    #  Magic attributes  |
-    #--------------------+
+    #----------------+
+    #  Internal API  |
+    #----------------+
 
-    def connect(self):
-        """
-        Connects to the database. Raises RuntimeError if the connection is not
-        closed yet. Use :meth:`StorageAdapter.reconnect` to explicitly close
-        the connection and open it again.
-        """
+    def _connect(self):
         if self.connection is not None:
             raise RuntimeError('already connected')
 

doqu/ext/tokyo_cabinet/__init__.py

 import tokyo.cabinet as tc
 
 from doqu.backend_base import BaseStorageAdapter, BaseQueryAdapter
+from doqu import utils
 from doqu.utils.data_structures import CachedIterator
 
 from converters import converter_manager
     #  Private attributes  |
     #----------------------+
 
-    def _init(self, storage, model, conditions=None, ordering=None):
+    def _init(self, storage, doc_class, conditions=None, ordering=None):
         self.storage = storage
-        self.model = model
+        self.doc_class = doc_class
         self._conditions = conditions or []
         self._ordering = ordering
         # TODO: make this closer to the Pyrant's internal mechanism so that
             self._iter = iter(self._query.search())
 
     def _prepare_item(self, key):
-        return self.storage.get(self.model, key)
+        return self.storage.get(key, self.doc_class)
 
-    def _where(self, lookups, negate=False):
+    def __where(self, lookups, negate=False):
         """
         Returns Query instance filtered by given conditions.
         The conditions are defined exactly as in Pyrant's high-level query API.
     def _clone(self, extra_conditions=None, extra_ordering=None):
         return self.__class__(
             self.storage,
-            self.model,
+            self.doc_class,
             conditions = self._conditions + (extra_conditions or []),
             ordering = extra_ordering or self._ordering,
         )
 
-    #--------------+
-    #  Public API  |
-    #--------------+
-
-    def where(self, **conditions):
+    def _where(self, **conditions):
         """
         Returns Query instance filtered by given conditions.
         The conditions are defined exactly as in Pyrant's high-level query API.
         See pyrant.query.Query.filter documentation for details.
         """
-        return self._where(conditions)
+        return self.__where(conditions)
 
-    def where_not(self, **conditions):
+    def _where_not(self, **conditions):
         """
         Returns Query instance. Inverted version of
         :meth:`~doqu.backends.tokyo_cabinet.Query.where`.
         """
-        return self._where(conditions, negate=True)
+        return self.__where(conditions, negate=True)
+
+    def _order_by(self, names, reverse=False):
+        if not isinstance(names, basestring):
+            raise ValueError('This backend only supports sorting by a single '
+                             'field')
+        name = names
+        direction = Ordering.DESC if reverse else Ordering.ASC
+
+        # introspect document class and use numeric sorting if appropriate
+        # FIXME declare this API somewhere?
+        if (hasattr(self.doc_class, 'meta') and
+            hasattr(self.doc_class.meta, 'structure')):
+            field_type = self.doc_class.meta.structure.get(name)
+            numeric = field_type in (int, float)
+        else:
+            numeric = False
+
+        ordering = Ordering(name, direction, numeric)
+
+        return self._clone(extra_ordering=ordering)
+
+
+    #----------------+
+    #  Internal API  |
+    #----------------+
 
     def count(self):
         """
         """
         return self._query.count()
 
+    '''
     def order_by(self, names, reverse=False):
         """
         Defines order in which results should be retrieved.
         name = names
         direction = Ordering.DESC if reverse else Ordering.ASC
 
-        # introspect model and use numeric sorting if appropriate
+        # FIXME this implies doqu.Document but we must not rely on that
+        # introspect document class and use numeric sorting if appropriate
         numeric = False
-        datatype = self.model.meta.structure.get(name)
+        datatype = self.doc_class.meta.structure.get(name)
         if datatype and isinstance(datatype, (int, float, long, Decimal)):
             numeric = True
 
         ordering = Ordering(name, direction, numeric)
 
         return self._clone(extra_ordering=ordering)
+    '''
 
     def values(self, name):
         """
         """
         known_values = {}
         for d in self:
+
+
+            # FIXME this seems to be a HACK because we are guessing what
+            # storage._decorate() returned
+            if isinstance(d, tuple):
+                d = d[1]
+
+
+            # note that the value is already converted to the desired type if
+            # the document class declares it
             value = d.get(name)
             if value and value not in known_values:
                 known_values[value] = 1
         do not provide query mechanisms other than access by primary key.
 
     """
-
-    supports_nested_data = False
+    #supports_nested_data = False
     converter_manager = converter_manager
     lookup_manager = lookup_manager
     query_adapter = QueryAdapter
     def __len__(self):
         return len(self.connection)
 
-    #--------------+
-    #  Public API  |
-    #--------------+
+    #----------------+
+    #  Internal API  |
+    #----------------+
 
-    def clear(self):
-        """
-        Clears the whole storage from data, resets autoincrement counters.
-        """
+    def _clear(self):
         self.connection.clear()
 
-    def connect(self):
-        """
-        Connects to the database. Raises RuntimeError if the connection is not
-        closed yet. Use :meth:`StorageAdapter.reconnect` to explicitly close
-        the connection and open it again.
-        """
-        if self.connection is not None:
-            raise RuntimeError('already connected')
-
+    def _connect(self):
         path = self._connection_options['path']
         self.connection = tc.TDB()
         self.connection.open(path, tc.TDBOWRITER | tc.TDBOCREAT)
 
-    def delete(self, primary_key):
-        """
-        Permanently deletes the record with given primary key from the database.
-        """
+    # NOTE: commented out for performance reasons. Unit tests weren't
+    # affected. Please add a unit test if you decide to uncomment this:
+    #def _decorate(self, key, data, doc_class=dict):
+    #    udata = dict((utils.safe_unicode(k),
+    #                  utils.safe_unicode(v)) for k,v in data.iteritems())
+    #    return super(StorageAdapter, self)._decorate(key, udata, doc_class)
+
+    def _delete(self, primary_key):
         del self.connection[primary_key]
 
-    def disconnect(self):
-        """
-        Closes internal store and removes the reference to it.
-        """
+    def _disconnect(self):
         self.connection.close()
         self.connection = None
 
-    def get(self, doc_class, primary_key):
-        """
-        Returns document object for given document class and primary key.
-        """
-        data = self.connection[primary_key]
-        return self._decorate(doc_class, primary_key, data)
+    def _get(self, key):
+        return self.connection[key]
 
-    def save(self, data, primary_key=None):
-        """
-        Saves given model instance into the storage. Returns primary key.
-
-        :param data:
-            dict containing all properties to be saved
-        :param primary_key:
-            the key for given object; if undefined, will be generated
-
-        Note that you must provide current primary key for a document object
-        which is already in the database in order to update it instead of
-        copying it.
-        """
+    def _save(self, key, data):
         # sanitize data for Tokyo Cabinet:
         # None-->'None' is wrong, force None-->''
-        for key in data:
-            if data[key] is None:
-                data[key] = ''
+        for field in data:
+            if data[field] is None:
+                data[field] = ''
             try:
-                data[key] = str(data[key])
+                data[field] = str(data[field])
             except UnicodeEncodeError:
-                data[key] = unicode(data[key]).encode('UTF-8')
+                data[field] = unicode(data[field]).encode('UTF-8')
 
-        primary_key = primary_key or unicode(self.connection.uid())
+        key = key or unicode(self.connection.uid())
 
-        self.connection[primary_key] = data
+        self.connection[key] = data
 
-        return primary_key
+        return key

doqu/ext/tokyo_tyrant/query.py

     def __getitem__(self, k):
         result = self._query[k]
         if isinstance(k, slice):
-            return [self.storage._decorate(self.model, key, data)
-                                                   for key, data in result]
+            return [self.storage._decorate(key, data, self.doc_class)
+                                       for key, data in result]
         else:
             key, data = result
-            return self.storage._decorate(self.model, key, data)
+            return self.storage._decorate(key, data, self.doc_class)
 
     def __iter__(self):
         for key, data in self._query:
-            yield self.storage._decorate(self.model, key, data)
+            yield self.storage._decorate(key, data, self.doc_class)
 
     def __or__(self, other):
         assert isinstance(other, self.__class__)
     def _init(self):
         self._query = self.storage.connection.query
     #    # by default only fetch columns specified in the Model
-    #    col_names = self.model._meta.props.keys()
+    #    col_names = self.doc_class._meta.props.keys()
     #    self._query = self.storage.connection.query.columns(*col_names)
 
     def _clone(self, inner_query=None):
-        clone = self.__class__(self.storage, self.model)
+        clone = self.__class__(self.storage, self.doc_class)
         clone._query = self._query if inner_query is None else inner_query
         return clone
 
         """
         return self._query.count()
 
-    def delete(self):
+    def _delete(self):
         """
         Deletes all records that match current query.
         """
         self._query.delete()
 
-    def order_by(self, name):
-        # introspect model and use numeric sorting if appropriate
-        attr_name = name[1:] if name.startswith('-') else name
-        numeric = self.model.meta.structure.get(attr_name) in (int, float)
+    def _order_by(self, names, reverse=False):
+        if not isinstance(names, basestring):
+            raise ValueError('This backend only supports sorting by a single '
+                             'field')
+        name = names
+        # introspect document class and use numeric sorting if appropriate
+        # FIXME declare this API somewhere?
+        if (hasattr(self.doc_class, 'meta') and
+            hasattr(self.doc_class.meta, 'structure')):
+            field_type = self.doc_class.meta.structure.get(name)
+            numeric = field_type in (int, float)
+        else:
+            numeric = False
+
+        if reverse:
+            name = '-{0}'.format(name)
 
         q = self._query.order_by(name, numeric)
         return self._clone(q)
         """
         Returns a list of unique values for given column name.
         """
+        # note that we get raw values (never through a Document instances) so
+        # we need to convert them if possibble
         values = self._query.values(name)
-        datatype = self.model.meta.structure.get(name, unicode)
+        if (hasattr(self.doc_class, 'meta') and
+            hasattr(self.doc_class.meta, 'structure')):
+            datatype = self.doc_class.meta.structure.get(name, unicode)
+        else:
+            datatype = unicode
+        # FIXME are these unique? :) nopes.
         return [self.storage.value_from_db(datatype, v) for v in values]
 
     def where(self, **conditions):

doqu/ext/tokyo_tyrant/storage.py

     def __len__(self):
         return len(self.connection)
 
-    #----------------------+
-    #  Private attributes  |
-    #----------------------+
+    #----------------+
+    #  Internal API  |
+    #----------------+
 
-    def _fetch(self, primary_key):
-        """
-        Returns model instance for given model and primary key.
-        Raises KeyError if there is no item with given key in the database.
-        """
-        return self.connection[primary_key] or {}
-
-    #--------------+
-    #  Public API  |
-    #--------------+
-
-    def clear(self):
-        """
-        Clears the whole storage from data, resets autoincrement counters.
-        """
+    def _clear(self):
         self.connection.clear()
 
-    def connect(self):
-        """
-        Connects to the database. Raises RuntimeError if the connection is not
-        closed yet. Use :meth:`StorageAdapter.reconnect` to explicitly close
-        the connection and open it again.
-        """
+    def _connect(self):
         # TODO: sockets, etc.
         host = self._connection_options.get('host', DEFAULT_HOST)
         port = self._connection_options.get('port', DEFAULT_PORT)
         self.connection = Tyrant(host=host, port=port)
 
-    def delete(self, key):
-        """
-        Permanently deletes the record with given primary key from the database.
-        """
+    def _delete(self, key):
         del self.connection[key]
 
-    def disconnect(self):
+    def _disconnect(self):
         self.connection = None
 
-    def save(self, data, primary_key=None):
-        """
-        Saves given model instance into the storage. Returns primary key.
+    def _get(self, primary_key):
+        return self.connection[primary_key] or {}
 
-        :param data: dict containing all properties to be saved
-        :param primary_key: the key for given object; if undefined, will be
-            generated
-
-        Note that you must provide current primary key for a model instance which
-        is already in the database in order to update it instead of copying it.
-        """
-        primary_key = primary_key or self.connection.generate_key()
-
-        self.connection[primary_key] = data
-
-        return primary_key
+    def _save(self, key, data):
+        key = key or self.connection.generate_key()
+        self.connection[key] = data
+        return key

doqu/utils/__init__.py

         return False
     else:
         return True
+
+def safe_unicode(value, force_string=False):
+    """Returns a Unicode version of given string. If given value is not a
+    string, it is returned as is.
+
+    :param force_string:
+
+        if `True`, non-string values are coerced to strings. Default is
+        `False`.
+
+    """
+    if isinstance(value, unicode):
+        return value
+    elif isinstance(value, basestring):
+        return value.decode('utf-8', 'replace')
+    else:
+        if force_string:
+            return unicode(value)
+        else:
+            return value
 [upload_sphinx]
 upload-dir = build/sphinx/html
+
 [nosetests]
-with-coverage=1
-cover-html=1
-cover-package=doqu
-cover-erase=1
+with-coverage = 1
+cover-html = 1
+cover-package = doqu
+cover-erase = 1
+rednose = 1
 
 Not intended to be run itself.
 """
+from datetime import date
 import os
 import unittest2 as unittest
 
 
 class BaseQueryTestCase(unittest.TestCase):
     FIXTURE = {
-        'john-id': {'name': u'John', 'age': 30},
-        'mary-id': {'name': u'Mary', 'age': 25}
+        'john-id': {'name': u'John', 'last_name': u'Connor', 'age': 30,
+                    'birth_date': date(1985,2,28)},
+        'mary-id': {'name': u'Mary', 'last_name': u'Connor', 'age': 25}
     }
     TMP_FILENAME_EXTENSION = 'tmp'  # matters for Tokyo Cabinet
 
 
     def setUp(self):
         self.db = self.get_connection()
+        self.db.clear()
         for pk, data in self.FIXTURE.items():
-            self.db.save(data, primary_key=pk)
+            self.db.save(pk, data)
 
     def tearDown(self):
-        self.db.disconnect()
+        if self.db:
+            self.db.clear()
+            self.db.disconnect()
         if os.path.exists(self._tmp_filename):
             os.unlink(self._tmp_filename)
 
         "Items can be found by simple value comparison"
         self.assert_finds('John', name='John')
         self.assert_finds('Mary', age=25)
+        self.assert_finds('John', 'Mary', last_name='Connor')
 
     def test_op_exists(self):
         self.assert_finds('John', 'Mary', name__exists=True)
         self.assert_finds('John', 'Mary', whatchamacallit__exists=False)
 
     def test_op_gt(self):
+        # int
         self.assert_finds('John', age__gt=25)
+        # date
+        self.assert_finds('John', birth_date__gt=date(1985,2,1))
+        self.assert_finds_nobody(birth_date__gt=date(1985,3,1))
 
     def test_op_gte(self):
+        # int
         self.assert_finds('John', 'Mary', age__gte=25)
+        # date
+        self.assert_finds('John', birth_date__gte=date(1985,2,28))
+        self.assert_finds_nobody(birth_date__gte=date(1985,3,1))
 
     def test_op_in(self):
         self.assert_finds('John', age__in=[29,30])
         self.assert_finds('John', 'Mary', age__in=[25,29,30])
 
     def test_op_lt(self):
+        # int
         self.assert_finds('Mary', age__lt=30)
+        # date
+        self.assert_finds('John', birth_date__lt=date(1985,3,1))
+        self.assert_finds_nobody(birth_date__lt=date(1985,2,1))
 
     def test_op_lte(self):
+        # int
         self.assert_finds('John', 'Mary', age__lte=30)
+        # date
+        self.assert_finds('John', birth_date__lte=date(1985,2,28))
+        self.assert_finds_nobody(birth_date__lte=date(1985,2,27))
 
     def test_op_matches(self):
         self.assert_finds('Mary', name__matches='M.ry')
         self.assert_finds('John', name__startswith='J')
         self.assert_finds('Mary', name__startswith='M')
 
-    @unittest.expectedFailure
     def test_op_year(self):
-        raise NotImplementedError
+        self.assert_finds('John', birth_date__year=1985)
+        self.assert_finds_nobody(birth_date__year=1777)
 
-    @unittest.expectedFailure
     def test_op_month(self):
-        raise NotImplementedError
+        self.assert_finds('John', birth_date__month=2)
+        self.assert_finds_nobody(birth_date__month=3)
 
-    @unittest.expectedFailure
     def test_op_day(self):
-        raise NotImplementedError
+        self.assert_finds('John', birth_date__day=28)
+        self.assert_finds_nobody(birth_date__day=29)
+
+    def test_sorting(self):
+        # string
+        keys = [key for key, data in self.db.find().order_by('name')]
+        self.assertEquals(keys, ['john-id', 'mary-id'])
+
+        # numeric
+        keys = [key for key, data in self.db.find().order_by('age')]
+        self.assertEquals(keys, ['mary-id', 'john-id'])
+
+    def test_sorting_reversed(self):
+        # string
+        keys = [key for key, data in
+                self.db.find().order_by('name', reverse=True)]
+        self.assertEquals(keys, ['mary-id', 'john-id'])
+
+        # numeric
+        keys = [key for key, data in
+                self.db.find().order_by('age', reverse=True)]
+        self.assertEquals(keys, ['john-id', 'mary-id'])
+
+    def test_values(self):
+        # TODO: check uniqueness!
+        # TODO: optionally unwrap lists (see failing Dark tests)
+
+        # string
+        self.assertEquals(['John', 'Mary'],
+                          sorted(self.db.find().values('name')))
+        # string (distinct)
+        self.assertEquals(['Connor'], list(self.db.find().values('last_name')))
+
+        # numeric
+        #   some backends (TC, TT) cannot restore the original datatype without
+        #   metadata so we'll check if both dicts and Document instances
+        #   behave correctly
+        self.assertEquals(['25', '30'],
+                          sorted(str(x) for x in
+                                 self.db.find().values('age')))
+        class Person(Document): structure = {'age': int}
+        self.assertEquals([25, 30],
+                          sorted(self.db.find(Person).values('age')))
 
 
 if __name__ == '__main__':

tests/test_document.py

         note = Note(text=u'foo')
         note.save(self.db1)
         note.text = u'quux'
-        note_retrieved = self.db1.get(Note, note.pk)
+        note_retrieved = self.db1.get(note.pk, Note)
         self.assertEqual(note, note_retrieved)
 
     def test_saved_different_pk(self):
             structure = {'text':unicode}
         note1 = Note(text=u'foo')
         note1.save(self.db1)
-        self.db2.save(note1._saved_state.data, note1.pk)
-        note2 = self.db2.get(Note, note1.pk)
+        self.db2.save(note1.pk, note1._saved_state.data)
+        note2 = self.db2.get(note1.pk, Note)
         self.assertEqual(note1.pk, note2.pk)
         self.assertNotEqual(note1, note2)
 

tests/test_ext_fields.py

         d = D(foo=u'hello')
         self.assertEquals(d['foo'], u'hello')
         d.save(self.db)
-        d2 = self.db.get(D, d.pk)
+        d2 = self.db.get(d.pk, D)
         self.assertEquals(d.foo, d2.foo)
 
     def test_int_field(self):