Commits

Mikhail Korobov committed 0adb40c

Initial import

  • Participants

Comments (0)

Files changed (11)

+syntax: glob
+
+#IDE files
+.settings/*
+.project
+.pydevproject
+.cache/*
+
+#temp files
+*.pyc
+*.pyo
+*.orig
+*~
+
+#misc files
+pip-log.txt
+
+#os files
+.DS_Store
+Thumbs.db
+
+#setup files
+build/
+dist/
+.build/
+MANIFEST
+Copyright (c) 2010, Mikhail Korobov
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of the tastypie nor the
+      names of its contributors may be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL MATT CROYDON BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+include *.txt
+include *.rst
+==================
+django-cache-utils
+==================
+
+django-cache-utils provides some utils for make cache-related work easier:
+
+* django memcached cache backend with group O(1)
+  invalidation ability, dog-pile effect prevention using MintCache algorythm
+  and project version support to allow gracefull updates and multiple django
+  projects on same memcached instance.
+  Long keys (>250) are truncated and appended with md5 hash.
+* ``cached`` decorator. Can be applied to function, method or classmethod.
+  Supports bulk O(1) cache invalidation and meaningful cache keys.
+  Takes function's arguments and full name into account while
+  constructing cache key.
+
+Installation
+============
+
+::
+
+    pip install django-cache-utils
+
+and then::
+
+    # settings.py
+    CACHE_BACKEND = 'cache_utils.group_backend://localhost:11211/'
+
+Usage
+=====
+
+::
+
+    from django.db import models
+    from cache_utils.decorators import cached
+
+    class CityManager(models.Manager):
+
+        # cache a method result. 'self' parameter is ignored
+        @cached(60*60*24, 'cities')
+        def default(self):
+            return self.active()[0]
+
+        # cache a method result. 'self' parameter is ignored, args and
+        # kwargs are used to construct cache key
+        @cached(60*60*24, 'cities')
+        def get(self, *args, **kwargs):
+            return super(CityManager, self).get(*args, **kwargs)
+
+
+    class City(models.Model):
+        # ... field declarations
+
+        objects = CityManager()
+
+        # an example how to cache django model methods by instance id
+        def has_offers(self):
+            @cached(30)
+            def offer_count(pk):
+                return self.offer_set.count()
+            return history_count(self.pk) > 0
+
+    # cache the function result based on passed parameter
+    @cached(60*60*24, 'cities')
+    def get_cities(slug)
+        return City.objects.get(slug=slug)
+
+
+    # cache for 'cities' group can be invalidated at once
+    def invalidate_city(sender, **kwargs):
+        cache.invalidate_group('cities')
+    pre_delete.connect(invalidate_city, City)
+    post_save.connect(invalidate_city, City)
+
+Notes
+=====
+
+django-cache-utils use 2 reads from memcached to get a value if 'group'
+argument is passed to 'cached' decorator::
+
+    @cached(60)
+    def my_func(param)
+        return ..
+
+    @cached(60, 'my_group')
+    def my_func2(param)
+        return ..
+
+    # 1 read from memcached
+    value1 = my_func(1)
+
+    # 2 reads from memcached + ability to invalidate all values at once
+    value2 = my_func2(2)
+
+Running tests
+=============
+
+Add ``'cache_utils'`` to ``INSTALLED_APPS`` and run ``./manage test cache_utils``.

cache_utils/__init__.py

+

cache_utils/decorators.py

+#coding: utf-8
+from django.core.cache import cache
+from django.utils.functional import wraps
+from cache_utils.utils import get_args_string
+
+def cached(timeout, group=None):
+    """ Caching decorator. Can be applied to function, method or classmethod.
+        Supports bulk O(1) cache invalidation and meaningful cache keys.
+        Takes function's arguments and full name into account while
+        constructing cache key.
+    """
+
+    def _cached(func):
+
+        # check if decorator is applied to function, method or classmethod
+        argnames = func.func_code.co_varnames[:func.func_code.co_argcount]
+        static = method = False
+        if len(argnames) > 0:
+            if argnames[0] == 'self' or argnames[0] == 'cls':
+                method = True
+                if argnames[0] == 'cls':
+                    static = True
+
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+
+            # introspect function's or method's full name
+            if method:
+                if static:
+                    class_name = args[0].__name__
+                else:
+                    class_name = args[0].__class__.__name__
+                func_name = ".".join([func.__module__, class_name, func.__name__])
+                key_args = args[1:]
+            else:
+                func_name = ".".join([func.__module__, func.__name__])
+                key_args = args
+
+            # construct the key using function's (method's) full name and
+            # passed parameters
+            key = '[cached]%s(%s)' % (func_name, get_args_string(key_args, kwargs))
+
+            # try to get the value from cache
+            value = cache.get(key, group=group)
+
+            # in case of cache miss recalculate the value and put it to the cache
+            if value is None:
+                value = func(*args, **kwargs)
+                cache.set(key, value, timeout, group=group)
+            return value
+        return wrapper
+    return _cached

cache_utils/group_backend.py

+"""
+Memcached cache backend with group O(1) invalidation ability, dog-pile
+effect prevention using MintCache algorythm and project version support to allow
+gracefull updates and multiple django projects on same memcached instance.
+Long keys (>250) are truncated and appended with md5 hash.
+"""
+
+import uuid
+import logging
+import sys
+import time
+from django.core.cache.backends.memcached import CacheClass as MemcachedCacheClass
+from django.conf import settings
+from cache_utils.utils import sanitize_memcached_key
+
+# This prefix is appended to the group name to prevent cache key clashes.
+_VERSION_PREFIX = getattr(settings, 'VERSION', "")
+_KEY_PREFIX = "_group::"
+
+# MINT_DELAY is an upper bound on how long any value should take to
+# be generated (in seconds)
+MINT_DELAY = 30
+
+class CacheClass(MemcachedCacheClass):
+
+    def add(self, key, value, timeout=0, group=None):
+        key = self._make_key(group, key)
+
+        refresh_time = timeout + time.time()
+        real_timeout = timeout + MINT_DELAY
+        packed_value = (value, refresh_time, False)
+
+        return super(CacheClass, self).add(key, packed_value, real_timeout)
+
+    def get(self, key, default=None, group=None):
+        key = self._make_key(group, key)
+        packed_value = super(CacheClass, self).get(key, default)
+        if packed_value is None:
+            return default
+        value, refresh_time, refreshed = packed_value
+        if (time.time() > refresh_time) and not refreshed:
+            # Store the stale value while the cache revalidates for another
+            # MINT_DELAY seconds.
+            self.set(key, value, timeout=MINT_DELAY, group=group, refreshed=True)
+            return default
+        return value
+
+    def set(self, key, value, timeout=0, group=None, refreshed=False):
+        key = self._make_key(group, key)
+
+        refresh_time = timeout + time.time()
+        real_timeout = timeout + MINT_DELAY
+        packed_value = (value, refresh_time, refreshed)
+        return super(CacheClass, self).set(key, packed_value, real_timeout)
+
+    def delete(self, key, group=None):
+        key = self._make_key(group, key)
+        return super(CacheClass, self).delete(key)
+
+    def invalidate_group(self, group):
+        """ Invalidates all cache keys belonging to group """
+        key = "%s%s%s" % (_VERSION_PREFIX, _KEY_PREFIX, group)
+        super(CacheClass, self).delete(key)
+
+    def _make_key(self, group, key, hashkey=None):
+        """ Generates a new cache key which belongs to a group, has
+            _VERSION_PREFIX prepended and is shorter than memcached key length
+            limit.
+        """
+        key = _VERSION_PREFIX + key
+        if group:
+            if not hashkey:
+                hashkey = self._get_hashkey(group)
+            key = "%s:%s-%s" % (group, key, hashkey)
+        return sanitize_memcached_key(key)
+
+    def _get_hashkey(self, group):
+        """ This can be useful sometimes if you're doing a very large number
+            of operations and you want to avoid all of the extra cache hits.
+        """
+        key = "%s%s%s" % (_VERSION_PREFIX, _KEY_PREFIX, group)
+        hashkey = super(CacheClass, self).get(key)
+        if hashkey is None:
+            hashkey = str(uuid.uuid4())
+            super(CacheClass, self).set(key, hashkey)
+        return hashkey
+
+    def clear(self):
+        self._cache.flush_all()
+
+# ======================================
+# I didn't implement methods below to work with MintCache so raise
+# NotImplementedError for them.
+
+    def incr(self, key, delta=1, group=None):
+#        if group:
+#            key = self._make_key(group, key)
+#        return super(CacheClass, self).incr(key, delta)
+        raise NotImplementedError
+
+    def decr(self, key, delta=1, group=None):
+#        if group:
+#            key = self._make_key(group, key)
+#        return super(CacheClass, self).decr(key, delta)
+        raise NotImplementedError
+
+    def get_many(self, keys, group=None):
+#        hashkey = self._get_hashkey(group)
+#        keys = [self._make_key(group, k, hashkey) for k in keys]
+#        return super(CacheClass, self).get_many(keys)
+        raise NotImplementedError

cache_utils/models.py

+# Hello, testrunner!

cache_utils/tests.py

+#coding: utf-8
+
+from unittest import TestCase
+
+from django.core.cache import cache
+from cache_utils.decorators import cached
+from cache_utils.utils import sanitize_memcached_key
+
+class ClearMemcachedTest(TestCase):
+    def tearDown(self):
+        cache._cache.flush_all()
+
+    def setUp(self):
+        cache._cache.flush_all()
+
+
+class InvalidationTest(ClearMemcachedTest):
+
+    def testInvalidation(self):
+        cache.set('vasia', 'foo', 60, group='names')
+        cache.set('petya', 'bar', 60, group='names')
+        cache.set('red', 'good', 60, group='colors')
+
+        self.assertEqual(cache.get('vasia', group='names'), 'foo')
+        self.assertEqual(cache.get('petya', group='names'), 'bar')
+        self.assertEqual(cache.get('red', group='colors'), 'good')
+
+        cache.invalidate_group('names')
+        self.assertEqual(cache.get('petya', group='names'), None)
+        self.assertEqual(cache.get('vasia', group='names'), None)
+        self.assertEqual(cache.get('red', group='colors'), 'good')
+
+        cache.set('vasia', 'foo', 60, group='names')
+        self.assertEqual(cache.get('vasia', group='names'), 'foo')
+
+
+class UtilsTest(ClearMemcachedTest):
+
+    def testSanitizeKeys(self):
+        key = u"12345678901234567890123456789012345678901234567890"
+        self.assertTrue(len(key) >= 40)
+        key = sanitize_memcached_key(key, 40)
+        self.assertTrue(len(key) <= 40)
+
+    def testDecorator(self):
+        self._x = 0
+
+        @cached(60, group='test-group')
+        def my_func(params=""):
+            self._x = self._x + 1
+            return u"%d%s" % (self._x, params)
+
+        self.assertEqual(my_func(), "1")
+        self.assertEqual(my_func(), "1")
+
+        self.assertEqual(my_func("x"), u"2x")
+        self.assertEqual(my_func("x"), u"2x")
+
+        self.assertEqual(my_func(u"Василий"), u"3Василий")
+        self.assertEqual(my_func(u"Василий"), u"3Василий")
+
+        self.assertEqual(my_func(u"й"*240), u"4"+u"й"*240)
+        self.assertEqual(my_func(u"й"*240), u"4"+u"й"*240)
+
+        self.assertEqual(my_func(u"Ы"*500), u"5"+u"Ы"*500)
+        self.assertEqual(my_func(u"Ы"*500), u"5"+u"Ы"*500)

cache_utils/utils.py

+from hashlib import md5
+
+CONTROL_CHARACTERS = set([chr(i) for i in range(0,33)])
+CONTROL_CHARACTERS.add(chr(127))
+
+def sanitize_memcached_key(key, max_length=250):
+    """ Removes control characters and ensures that key will
+        not hit the memcached key length limit by replacing
+        the key tail with md5 hash if key is too long.
+    """
+    key = ''.join([c for c in key if c not in CONTROL_CHARACTERS])
+    if len(key) > max_length:
+        hash = md5(key).hexdigest()
+        key = key[:max_length-33]+'-'+hash
+    return key
+
+def get_args_string(args, kwargs):
+    key = ""
+    if args:
+        key += unicode(args)
+    if kwargs:
+        key += unicode(kwargs)
+    return key
+
+
+#!/usr/bin/env python
+from distutils.core import setup
+
+version='0.5.0'
+
+setup(
+    name='django-cache-utils',
+    version=version,
+    author='Mikhail Korobov',
+    author_email='kmike84@gmail.com',
+
+    packages=['cache_utils'],
+
+    url='http://bitbucket.org/kmike/django-cache-utils/',
+    download_url = 'http://bitbucket.org/kmike/django-cache-utils/get/tip.zip',
+    license = 'MIT license',
+    description = """ Caching decorator and django cache backend with advanced invalidation ability and dog-pile effect prevention """,
+
+    long_description = open('README.rst').read(),
+    requires = ['django', 'memcached'],
+
+    classifiers=(
+        'Development Status :: 4 - Beta',
+        'Environment :: Web Environment',
+        'Framework :: Django',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: MIT License',
+        'Programming Language :: Python',
+        'Topic :: Software Development :: Libraries :: Python Modules',
+    ),
+)