Mike Bayer avatar Mike Bayer committed b323546

- Added new methods :meth:`.CacheRegion.get_or_create_multi`
and :meth:`.CacheRegion.cache_multi_on_arguments`, which
make use of the :meth:`.CacheRegion.get_multi` and similar
functions to store and retrieve multiple keys at once while
maintaining dogpile semantics for each. #33
- fix some py3k identifiers
- remove erroneous docstring regarding creator returning NO_VALUE,
that change was reverted

Comments (0)

Files changed (9)

docs/build/changelog.rst

 Changelog
 ==============
 .. changelog::
-    :version: 0.4.4
+    :version: 0.5.0
+
+    .. change::
+        :tags: feature
+        :tickets: 33
+
+      Added new methods :meth:`.CacheRegion.get_or_create_multi`
+      and :meth:`.CacheRegion.cache_multi_on_arguments`, which
+      make use of the :meth:`.CacheRegion.get_multi` and similar
+      functions to store and retrieve multiple keys at once while
+      maintaining dogpile semantics for each.
 
     .. change::
         :tags: feature

dogpile/cache/__init__.py

-__version__ = '0.4.4'
+__version__ = '0.5.0'
 
 from .region import CacheRegion, register_backend, make_region

dogpile/cache/backends/redis.py

                 values[key] = pickle.loads(values[key])
             else:
                 values[key] = NO_VALUE
-        return values        
+        return values
 
     def set(self, key, value):
         if self.redis_expiration_time:
         pipe = self.client.pipeline()
         for key in keys:
             pipe.delete(key)
-        pipe.execute()        
+        pipe.execute()
 
 class RedisLock(object):
     """Simple distributed lock using Redis.

dogpile/cache/compat.py

 
     import configparser
     import io
+    import _thread as thread
 else:
     string_types = basestring,
     text_type = unicode
     import StringIO as io
 
     callable = callable
+    import thread
+
 
 if py3k or jython:
     import pickle

dogpile/cache/region.py

 from dogpile.core import Lock, NeedRegenerationException
 from dogpile.core.nameregistry import NameRegistry
 from .util import function_key_generator, PluginLoader, \
-    memoized_property, coerce_string_conf
+    memoized_property, coerce_string_conf, function_multi_key_generator
 from .api import NO_VALUE, CachedValue
 from .proxy import ProxyBackend
 from . import compat
         creation function may or may not be used to recreate the value
         and persist the newly generated value in the cache.
 
-        If the creation function returns :const:`NO_VALUE`, nothing is cached.
-        Note that if the returns `None`, `None` will be cached.
-
         Whether or not the function is used depends on if the
         *dogpile lock* can be acquired or not.  If it can't, it means
         a different thread or process is already running a creation
          function, if present.
 
         :param creator: function which creates a new value.
+
         :param expiration_time: optional expiration time which will overide
          the expiration time already configured on this :class:`.CacheRegion`
          if not None.   To set no expiration, use the value -1.
 
          .. versionadded:: 0.4.3
 
-        See also:
+        .. seealso::
 
-        :meth:`.CacheRegion.cache_on_arguments` - applies :meth:`.get_or_create`
-        to any function using a decorator.
+            :meth:`.CacheRegion.cache_on_arguments` - applies
+            :meth:`.get_or_create` to any function using a decorator.
+
+            :meth:`.CacheRegion.get_or_create_multi` - multiple key/value version
 
         """
         if self.key_mangler:
                 async_creator) as value:
             return value
 
+    def get_or_create_multi(self, keys, creator, expiration_time=None,
+                                should_cache_fn=None):
+        """Return a sequence of cached values based on a sequence of keys.
+
+        The behavior for generation of values based on keys corresponds
+        to that of :meth:`.Region.get_or_create`, with the exception that
+        the ``creator()`` function may be asked to generate any subset of
+        the given keys.   The list of keys to be generated is passed to
+        ``creator()``, and ``creator()`` should return the generated values
+        as a sequence corresponding to the order of the keys.
+
+        The method uses the same approach as :meth:`.Region.get_multi`
+        and :meth:`.Region.set_multi` to get and set values from the
+        backend.
+
+        :param keys: Sequence of keys to be retrieved.
+
+        :param creator: function which accepts a sequence of keys and
+         returns a sequence of new values.
+
+        :param expiration_time: optional expiration time which will overide
+         the expiration time already configured on this :class:`.CacheRegion`
+         if not None.   To set no expiration, use the value -1.
+
+        :param should_cache_fn: optional callable function which will receive
+         each value returned by the "creator", and will then return True or False,
+         indicating if the value should actually be cached or not.  If it
+         returns False, the value is still returned, but isn't cached.
+
+        .. versionadded:: 0.5.0
+
+        .. seealso::
+
+
+            :meth:`.CacheRegion.cache_multi_on_arguments`
+
+            :meth:`.CacheRegion.get_or_create`
+
+        """
+
+        def get_value(key):
+            value = values.get(key, NO_VALUE)
+            if value is NO_VALUE or \
+                value.metadata['v'] != value_version or \
+                    (self._invalidated and
+                    value.metadata["ct"] < self._invalidated):
+                return value.payload, 0
+            else:
+                return value.payload, value.metadata["ct"]
+
+        def gen_value():
+            raise NotImplementedError()
+
+        def async_creator(key, mutex):
+            mutexes[key] = mutex
+
+        if expiration_time is None:
+            expiration_time = self.expiration_time
+
+        mutexes = {}
+
+        sorted_unique_keys = sorted(set(keys))
+
+        if self.key_mangler:
+            mangled_keys = [self.key_mangler(k) for k in sorted_unique_keys]
+        else:
+            mangled_keys = sorted_unique_keys
+
+        orig_to_mangled = dict(zip(sorted_unique_keys, mangled_keys))
+
+        values = self.backend.get_multi(mangled_keys)
+
+        for orig_key, mangled_key in orig_to_mangled.items():
+            with Lock(
+                    self._mutex(mangled_key),
+                    gen_value,
+                    lambda: get_value(mangled_key),
+                    expiration_time,
+                    async_creator=lambda mutex:
+                            async_creator(orig_key, mutex)):
+                pass
+        try:
+            if mutexes:
+                # sort the keys, the idea is to prevent deadlocks.
+                # though haven't been able to simulate one anyway.
+                keys_to_get = sorted(mutexes)
+                new_values = creator(*keys_to_get)
+
+                values_w_created = dict(
+                    (orig_to_mangled[k], self._value(v))
+                    for k, v in zip(keys_to_get, new_values)
+                )
+
+                if not should_cache_fn:
+                    self.backend.set_multi(values_w_created)
+                else:
+                    self.backend.set_multi(dict(
+                        (k, v)
+                        for k, v in values_w_created.items()
+                        if should_cache_fn(v[0])
+                    ))
+
+                values.update(values_w_created)
+            return [values[orig_to_mangled[k]].payload for k in keys]
+        finally:
+            for mutex in mutexes.values():
+                mutex.release()
+
+
     def _value(self, value):
         """Return a :class:`.CachedValue` given a value."""
         return CachedValue(value, {
          associated with classes - note that the decorator itself
          can't see the parent class on a function as the class is
          being declared.
+
         :param expiration_time: if not None, will override the normal
          expiration time.
 
 
           .. versionadded:: 0.4.3
 
+        .. seealso::
+
+            :meth:`.CacheRegion.cache_multi_on_arguments`
+
+            :meth:`.CacheRegion.get_or_create`
+
         """
         expiration_time_is_callable = compat.callable(expiration_time)
         def decorator(fn):
             return decorate
         return decorator
 
+    def cache_multi_on_arguments(self, namespace=None, expiration_time=None,
+                                        should_cache_fn=None):
+        """A function decorator that will cache multiple return
+        values from the function using a sequence of keys derived from the
+        function itself and the arguments passed to it.
+
+        This method is the "multiple key" analogue to the
+        :meth:`.CacheRegion.cache_on_arguments` method.
+
+        Example::
+
+            @someregion.cache_multi_on_arguments()
+            def generate_something(*keys):
+                return [
+                    somedatabase.query(key)
+                    for key in keys
+                ]
+
+        The decorated function can be called normally.  The decorator
+        will produce a list of cache keys using a mechanism similar to that
+        of :meth:`.CacheRegion.cache_on_arguments`, combining the name
+        of the function with the optional namespace and with the string form
+        of each key.  It will then consult the cache using the same mechanism as that
+        of :meth:`.CacheRegion.get_multi` to retrieve all current values;
+        the originally passed keys corresponding to those values which
+        aren't generated or need regeneration will
+        be assembled into a new argument list, and the decorated function
+        is then called with that subset of arguments.
+
+        The returned result is a list::
+
+            result = generate_something("key1", "key2", "key3")
+
+        The decorator internally makes use of the
+        :meth:`.CacheRegion.get_or_create_multi` method to access the
+        cache and conditionally call the function.  See that
+        method for additional behavioral details.
+
+        Unlike the :meth:`.CacheRegion.cache_on_arguments`
+        method, :meth:`.CacheRegion.cache_multi_on_arguments` works only
+        with a single function signature, one which takes a simple list of keys as
+        arguments.
+
+        Like :meth:`.CacheRegion.cache_on_arguments`, the decorated function
+        is also provided with a ``set()`` method, which here accepts a
+        mapping of keys and values to set in the cache::
+
+            generate_something.set({"k1": "value1",
+                                    "k2": "value2", "k3": "value3"})
+
+        as well as an ``invalidate()`` method, which has the effect of deleting
+        the given sequence of keys using the same mechanism as that of
+        :meth:`.CacheRegion.delete_multi`::
+
+            generate_something.invalidate("k1", "k2", "k3")
+
+        Parameters passed to :meth:`.CacheRegion.cache_multi_on_arguments`
+        have the same meaning as those passed to
+        :meth:`.CacheRegion.cache_on_arguments`.
+
+        :param namespace: optional string argument which will be
+         established as part of each cache key.
+
+        :param expiration_time: if not None, will override the normal
+         expiration time.  May be passed as an integer or a
+         callable.
+
+        :param should_cache_fn: passed to :meth:`.CacheRegion.get_or_create_multi`.
+
+        .. versionadded:: 0.5.0
+
+        .. seealso::
+
+            :meth:`.CacheRegion.cache_on_arguments`
+
+            :meth:`.CacheRegion.get_or_create_multi`
+
+        """
+        expiration_time_is_callable = compat.callable(expiration_time)
+        def decorator(fn):
+            key_generator = function_multi_key_generator(namespace, fn)
+            @wraps(fn)
+            def decorate(*arg, **kw):
+                cache_keys = arg
+                keys = key_generator(*arg, **kw)
+                key_lookup = dict(zip(keys, cache_keys))
+                @wraps(fn)
+                def creator(*keys_to_create):
+                    return fn(*[key_lookup[k] for k in keys_to_create])
+
+                timeout = expiration_time() if expiration_time_is_callable \
+                            else expiration_time
+                return self.get_or_create_multi(keys, creator, timeout,
+                                          should_cache_fn)
+
+            def invalidate(*arg):
+                keys = key_generator(*arg)
+                self.delete_multi(keys)
+
+            def set_(mapping):
+                keys = list(mapping)
+                gen_keys = key_generator(*keys)
+                self.set_multi(dict(
+                        (gen_key, mapping[key])
+                        for gen_key, key
+                        in zip(gen_keys, keys))
+                    )
+
+            decorate.set = set_
+            decorate.invalidate = invalidate
+
+            return decorate
+        return decorator
+
+
+
 def make_region(*arg, **kw):
     """Instantiate a new :class:`.CacheRegion`.
 

dogpile/cache/util.py

         return namespace + "|" + " ".join(map(compat.string_type, args))
     return generate_key
 
+def function_multi_key_generator(namespace, fn):
+
+    if namespace is None:
+        namespace = '%s:%s' % (fn.__module__, fn.__name__)
+    else:
+        namespace = '%s:%s|%s' % (fn.__module__, fn.__name__, namespace)
+
+    args = inspect.getargspec(fn)
+    has_self = args[0] and args[0][0] in ('self', 'cls')
+    def generate_keys(*args, **kw):
+        if kw:
+            raise ValueError(
+                    "dogpile.cache's default key creation "
+                    "function does not accept keyword arguments.")
+        if has_self:
+            args = args[1:]
+        return [namespace + "|" + key for key in map(compat.string_type, args)]
+    return generate_keys
+
 def sha1_mangle_key(key):
     """a SHA1 key mangler."""
 
 cover-package = dogpile.cache
 with-coverage = 1
 cover-erase = 1
+nologcapture = 1

tests/cache/_fixtures.py

 import time
 from nose import SkipTest
 from threading import Thread, Lock
+from dogpile.cache.compat import thread
 from unittest import TestCase
+import random
+import collections
 
 class _GenericBackendFixture(object):
     @classmethod
         assert len(canary) > 3
         assert False not in canary
 
+    def test_threaded_get_multi(self):
+        reg = self._region(config_args={"expiration_time": .25})
+        locks = dict((str(i), Lock()) for i in range(11))
+
+        canary = collections.defaultdict(list)
+        def creator(*keys):
+            assert keys
+            ack = [locks[key].acquire(False) for key in keys]
+
+            #print(
+            #        ("%s " % thread.get_ident()) + \
+            #        ", ".join(sorted("%s=%s" % (key, acq)
+            #                    for acq, key in zip(ack, keys)))
+            #    )
+
+            for acq, key in zip(ack, keys):
+                canary[key].append(acq)
+
+            time.sleep(.5)
+
+            for acq, key in zip(ack, keys):
+                if acq:
+                    locks[key].release()
+            return ["some value %s" % k for k in keys]
+        def f():
+            for x in range(5):
+                reg.get_or_create_multi(
+                    [str(random.randint(1, 10))
+                        for i in range(random.randint(1, 5))],
+                            creator)
+                time.sleep(.5)
+        f()
+        return
+        threads = [Thread(target=f) for i in range(5)]
+        for t in threads:
+            t.start()
+        for t in threads:
+            t.join()
+
+        assert sum([len(v) for v in canary.values()]) > 10
+        for l in canary.values():
+            assert False not in l
+
+
     def test_region_delete(self):
         reg = self._region()
         reg.set("some key", "some value")

tests/cache/test_region.py

     def test_datetime_expiration_time(self):
         my_region = make_region()
         my_region.configure(
-            backend='mock', 
+            backend='mock',
             expiration_time=datetime.timedelta(days=1, hours=8)
         )
         eq_(my_region.expiration_time, 32*60*60)
 
     def test_reject_invalid_expiration_time(self):
         my_region = make_region()
-        
+
         assert_raises_message(
             Exception,
             "expiration_time is not a number or timedelta.",
             return "some value"
         eq_(reg.get_or_create("some key", creator), "some value")
 
+    def test_multi_creator(self):
+        reg = self._region()
+        def creator(*keys):
+            return ["some value %s" % key for key in keys]
+        eq_(reg.get_or_create_multi(["k3", "k2", "k5"], creator),
+                    ['some value k3', 'some value k2', 'some value k5'])
+
     def test_remove(self):
         reg = self._region()
         reg.set("some key", "some value")
         eq_(reg.get_or_create("some key", creator), "some value 2")
         eq_(reg.get("some key"), "some value 2")
 
+    def test_expire_multi(self):
+        reg = self._region(config_args={"expiration_time":1})
+        counter = itertools.count(1)
+        def creator(*keys):
+            return ["some value %s %d" % (key, next(counter)) for key in keys]
+        eq_(reg.get_or_create_multi(["k3", "k2", "k5"], creator),
+                    ['some value k3 2', 'some value k2 1', 'some value k5 3'])
+        time.sleep(2)
+        is_(reg.get("k2"), NO_VALUE)
+        eq_(reg.get("k2", ignore_expiration=True), "some value k2 1")
+        eq_(reg.get_or_create_multi(["k3", "k2"], creator),
+                    ['some value k3 5', 'some value k2 4'])
+        eq_(reg.get("k2"), "some value k2 4")
+
     def test_expire_on_get(self):
         reg = self._region(config_args={"expiration_time":.5})
         reg.set("some key", "some value")
         eq_(ret, 3)
         eq_(reg.backend._cache['some key'][0], 3)
 
+    def test_should_cache_fn_multi(self):
+        reg = self._region()
+        values = [1, 2, 3]
+        def creator(*keys):
+            v = values.pop(0)
+            return [v for k in keys]
+        should_cache_fn = lambda val: val in (1, 3)
+        ret = reg.get_or_create_multi(
+                    [1, 2], creator,
+                    should_cache_fn=should_cache_fn)
+        eq_(ret, [1, 1])
+        eq_(reg.backend._cache[1][0], 1)
+        reg.invalidate()
+        ret = reg.get_or_create_multi(
+                    [1, 2], creator,
+                    should_cache_fn=should_cache_fn)
+        eq_(ret, [2, 2])
+        eq_(reg.backend._cache[1][0], 1)
+        reg.invalidate()
+        ret = reg.get_or_create_multi(
+                    [1, 2], creator,
+                    should_cache_fn=should_cache_fn)
+        eq_(ret, [3, 3])
+        eq_(reg.backend._cache[1][0], 3)
+
     def test_should_set_multiple_values(self):
         reg = self._region()
         values = {'key1': 'value1', 'key2': 'value2', 'key3': 'value3'}
 
         eq_(Bar.generate(1, 2), 4)
 
+    def test_multi(self):
+        reg = self._region()
+
+        counter = itertools.count(1)
+
+        @reg.cache_multi_on_arguments()
+        def generate(*args):
+            return ["%d %d" % (arg, next(counter)) for arg in args]
+
+        eq_(generate(2, 8, 10), ['2 2', '8 3', '10 1'])
+        eq_(generate(2, 9, 10), ['2 2', '9 4', '10 1'])
+
+        generate.invalidate(2)
+        eq_(generate(2, 7, 10), ['2 5', '7 6', '10 1'])
+
+        generate.set({7: 18, 10: 15})
+        eq_(generate(2, 7, 10), ['2 5', 18, 15])
+
+    def test_multi_namespace(self):
+        reg = self._region()
+
+        counter = itertools.count(1)
+
+        @reg.cache_multi_on_arguments(namespace="foo")
+        def generate(*args):
+            return ["%d %d" % (arg, next(counter)) for arg in args]
+
+        eq_(generate(2, 8, 10), ['2 2', '8 3', '10 1'])
+        eq_(generate(2, 9, 10), ['2 2', '9 4', '10 1'])
+
+        eq_(
+            sorted(list(reg.backend._cache)),
+            [
+            'tests.cache.test_region:generate|foo|10',
+            'tests.cache.test_region:generate|foo|2',
+            'tests.cache.test_region:generate|foo|8',
+            'tests.cache.test_region:generate|foo|9']
+        )
+        generate.invalidate(2)
+        eq_(generate(2, 7, 10), ['2 5', '7 6', '10 1'])
+
+        generate.set({7: 18, 10: 15})
+        eq_(generate(2, 7, 10), ['2 5', 18, 15])
+
+
 
 class ProxyRegionTest(RegionTest):
     ''' This is exactly the same as the region test above, but it goes through
         proxy_abc = ProxyBackendTest.UsedKeysProxy(5)
         reg_num = self._region(config_args={"wrap": [proxy_num]})
         reg_abc = self._region(config_args={"wrap": [proxy_abc]})
-        for i in xrange(10):
+        for i in range(10):
             reg_num.set(i, True)
             reg_abc.set(chr(ord('a') + i), True)
 
-        for i in xrange(5):
+        for i in range(5):
             reg_num.set(i, True)
             reg_abc.set(chr(ord('a') + i), True)
 
         # Test that we can pass an argument to Proxy on creation
         proxy = ProxyBackendTest.NeverSetProxy(5)
         reg = self._region(config_args={"wrap": [proxy]})
-        for i in xrange(10):
+        for i in range(10):
             reg.set(i, True)
 
         # make sure 1 was set, but 5 was not
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.