Commits

Mike Bayer committed b9f7ff4

- add dbm backend plus file lock
- improve memcached tests to check that memcached is running

Comments (0)

Files changed (8)

docs/build/api.rst

 .. automodule:: dogpile.cache.backends.memcached
     :members:
 
+.. automodule:: dogpile.cache.backends.file
+    :members:
+
 Plugins
 ========
 

dogpile/cache/backends/__init__.py

 from dogpile.cache.region import register_backend
 
-register_backend("dogpile.cache.dbm", "dogpile.cache.backends.dbm", "DBMBackend")
+register_backend("dogpile.cache.dbm", "dogpile.cache.backends.file", "DBMBackend")
 register_backend("dogpile.cache.pylibmc", "dogpile.cache.backends.memcached", "PylibmcBackend")
 register_backend("dogpile.cache.bmemcached", "dogpile.cache.backends.memcached", "BMemcachedBackend")
 register_backend("dogpile.cache.memcached", "dogpile.cache.backends.memcached", "MemcachedBackend")

dogpile/cache/backends/dbm.py

-from dogpile.cache.api import CacheBackend, CachedValue, NO_VALUE
-
-class DBMBackend(CacheBackend):
-    pass

dogpile/cache/backends/file.py

+"""
+File Backends
+------------------
+
+Provides backends that deal with local filesystem access.
+
+"""
+
+from dogpile.cache.api import CacheBackend, NO_VALUE
+from contextlib import contextmanager
+from dogpile.cache import util
+import os
+import fcntl
+
+__all__ = 'DBMBackend', 'FileLock'
+
+class DBMBackend(CacheBackend):
+    """A file-backend using a dbm file to store keys.
+
+    Basic usage::
+
+        from dogpile.cache import make_region
+
+        region = make_region().configure(
+            'dogpile.cache.dbm',
+            expiration_time = 3600,
+            arguments = {
+                "filename":"/path/to/cachefile.dbm"
+            }
+        )
+
+    DBM access is provided using the Python ``anydbm`` module,
+    which selects a platform-specific dbm module to use.
+    This may be made to be more configurable in a future 
+    release.
+    
+    Note that different dbm modules have different behaviors.
+    Some dbm implementations handle their own locking, while 
+    others don't.  The :class:`.DBMBackend` uses a read/write
+    lockfile by default, which is compatible even with those
+    DBM implementations for which this is unnecessary,
+    though the behavior can be disabled.
+
+    The DBM backend by default makes use of two lockfiles.
+    One is in order to protect the DBM file itself from 
+    concurrent writes, the other is to coordinate
+    value creation (i.e. the dogpile lock).  By default,
+    these lockfiles use the ``flock()`` system call 
+    for locking; this is only available on Unix 
+    platforms.
+    
+    Currently, the dogpile lock is against the entire
+    DBM file, not per key.   This means there can
+    only be one "creator" job running at a time
+    per dbm file.
+    
+    A future improvement might be to have the dogpile lock 
+    using a filename that's based on a modulus of the key. 
+    Locking on a filename that uniquely corresponds to the 
+    key is problematic, since it's not generally safe to
+    delete lockfiles as the application runs, implying an 
+    unlimited number of key-based files would need to be 
+    created and never deleted.
+    
+    Parameters to the ``arguments`` dictionary are 
+    below.
+
+    :param filename: path of the filename in which to 
+     create the DBM file.  Note that some dbm backends
+     will change this name to have additional suffixes.
+    :param rw_lockfile: the name of the file to use for
+     read/write locking.  If omitted, a default name
+     is used by appending the suffix ".rw.lock" to the 
+     DBM filename.  If False, then no lock is used.
+    :param dogpile_lockfile: the name of the file to use
+     for value creation, i.e. the dogpile lock.  If 
+     omitted, a default name is used by appending the 
+     suffix ".dogpile.lock" to the DBM filename. If 
+     False, then dogpile.cache uses the default dogpile
+     lock, a plain thread-based mutex.
+
+
+    """
+    def __init__(self, arguments):
+        self.filename = os.path.abspath(
+                            os.path.normpath(arguments['filename'])
+                        )
+        dir_, filename = os.path.split(self.filename)
+
+        self._rw_lock = self._init_lock(
+                                arguments.get('rw_lockfile'), 
+                                ".rw.lock", dir_, filename)
+        self._dogpile_lock = self._init_lock(
+                                arguments.get('dogpile_lockfile'), 
+                                ".dogpile.lock", 
+                                dir_, filename)
+
+        # TODO: make this configurable
+        import anydbm
+        self.dbmmodule = anydbm
+        self._init_dbm_file()
+
+    def _init_lock(self, argument, suffix, basedir, basefile):
+        if argument is None:
+            return FileLock(os.path.join(basedir, basefile + suffix))
+        elif argument is not False:
+            return FileLock(
+                        os.path.abspath(
+                            os.path.normpath(argument)
+                        ))
+        else:
+            return None
+
+    def _init_dbm_file(self):
+        exists = os.access(self.filename, os.F_OK)
+        if not exists:
+            for ext in ('db', 'dat', 'pag', 'dir'):
+                if os.access(self.filename + os.extsep + ext, os.F_OK):
+                    exists = True
+                    break
+        if not exists:
+            fh = self.dbmmodule.open(self.filename, 'c')
+            fh.close()
+
+    def get_mutex(self, key):
+        # using one dogpile for the whole file.   Other ways
+        # to do this might be using a set of files keyed to a 
+        # hash/modulus of the key.   the issue is it's never
+        # really safe to delete a lockfile as this can 
+        # break other processes trying to get at the file
+        # at the same time - so handling unlimited keys
+        # can't imply unlimited filenames
+        return self._dogpile_lock
+
+    @contextmanager
+    def _use_rw_lock(self, write):
+        if self._rw_lock is None:
+            yield
+        elif write:
+            with self._rw_lock.write():
+                yield
+        else:
+            with self._rw_lock.read():
+                yield
+
+    @contextmanager
+    def _dbm_file(self, write):
+        with self._use_rw_lock(write):
+            dbm = self.dbmmodule.open(self.filename, 
+                                "w" if write else "r")
+            yield dbm
+            dbm.close()
+
+    def get(self, key):
+        with self._dbm_file('r') as dbm:
+            value = dbm.get(key, NO_VALUE)
+            if value is not NO_VALUE:
+                value = util.pickle.loads(value)
+            return value
+
+    def set(self, key, value):
+        with self._dbm_file('w') as dbm:
+            dbm[key] = util.pickle.dumps(value)
+
+    def delete(self, key):
+        with self._dbm_file('w') as dbm:
+            dbm.pop(key, None)
+
+class FileLock(object):
+    """Use lockfiles to coordinate read/write access to a file."""
+
+    def __init__(self, filename):
+        self._filedescriptor = util.threading.local()
+        self.filename = filename
+
+    def acquire(self, wait=True):
+        return self.acquire_write_lock(wait)
+
+    def release(self):
+        self.release_write_lock()
+
+    @property
+    def is_open(self):
+        return hasattr(self._filedescriptor, 'fileno')
+
+    @contextmanager
+    def read(self):
+        self.acquire_read_lock(True)
+        yield
+        self.release_read_lock()
+
+    @contextmanager
+    def write(self):
+        self.acquire_write_lock(True)
+        yield
+        self.release_write_lock()
+
+    def acquire_read_lock(self, wait):
+        return self._acquire(wait, os.O_RDONLY, fcntl.LOCK_SH)
+
+    def acquire_write_lock(self, wait):
+        return self._acquire(wait, os.O_WRONLY, fcntl.LOCK_EX)
+
+    def release_read_lock(self):
+        self._release()
+
+    def release_write_lock(self):
+        self._release()
+
+    def _acquire(self, wait, wrflag, lockflag):
+        wrflag |= os.O_CREAT
+        fileno = os.open(self.filename, wrflag)
+        try:
+            if not wait:
+                lockflag |= fcntl.LOCK_NB
+            fcntl.flock(fileno, lockflag)
+        except IOError:
+            os.close(fileno)
+            if not wait:
+                # this is typically 
+                # "[Errno 35] Resource temporarily unavailable",
+                # because of LOCK_NB
+                return False
+            else:
+                raise
+        else:
+            self._filedescriptor.fileno = fileno
+            return True
+
+    def _release(self):
+        try:
+            fileno = self._filedescriptor.fileno
+        except AttributeError:
+            return
+        else:
+            fcntl.flock(fileno, fcntl.LOCK_UN)
+            os.close(fileno)
+            del self._filedescriptor.fileno

dogpile/cache/util.py

 from hashlib import sha1
 import inspect
+import sys
 
 try:
     import threading
     import dummy_threading as threading
     import dummy_thread as thread
 
+py3k = getattr(sys, 'py3kwarning', False) or sys.version_info >= (3, 0)
+jython = sys.platform.startswith('java')
+
+if py3k or jython:
+    import pickle
+else:
+    import cPickle as pickle
+
 class PluginLoader(object):
     def __init__(self, group):
         self.group = group

tests/_fixtures.py

     def setup_class(cls):
         try:
             backend_cls = _backend_loader.load(cls.backend)
-            backend_cls(cls.config_args.get('arguments', {}))
+            backend = backend_cls(cls.config_args.get('arguments', {}))
         except ImportError:
             raise SkipTest("Backend %s not installed" % cls.backend)
+        cls._check_backend_available(backend)
+
+    @classmethod
+    def _check_backend_available(cls, backend):
+        pass
 
     region_args = {}
     config_args = {}

tests/test_dbm_backend.py

+from tests._fixtures import _GenericBackendTest, _GenericMutexTest
+from tests import eq_
+from unittest import TestCase
+from threading import Thread
+import time
+import os
+from nose import SkipTest
+
+try:
+    import fcntl
+except ImportError:
+    raise SkipTest("fcntl not available")
+
+class DBMBackendTest(_GenericBackendTest):
+    backend = "dogpile.cache.dbm"
+
+    config_args = {
+        "arguments":{
+            "filename":"test.dbm"
+        }
+    }
+
+class DBMBackendNoLockTest(_GenericBackendTest):
+    backend = "dogpile.cache.dbm"
+
+    config_args = {
+        "arguments":{
+            "filename":"test.dbm",
+            "rw_lockfile":False,
+            "dogpile_lockfile":False,
+        }
+    }
+
+
+class DBMMutexTest(_GenericMutexTest):
+    backend = "dogpile.cache.dbm"
+
+    config_args = {
+        "arguments":{
+            "filename":"test.dbm"
+        }
+    }
+
+
+def teardown():
+    for fname in os.listdir(os.curdir):
+        if fname.startswith("test.dbm"):
+            os.unlink(fname)

tests/test_memcached_backend.py

 from unittest import TestCase
 from threading import Thread
 import time
+from nose import SkipTest
 
-class _NonDistributedMemcachedTest(_GenericBackendTest):
+class _TestMemcachedConn(object):
+    @classmethod
+    def _check_backend_available(cls, backend):
+        try:
+            client = backend._create_client()
+            client.set("x", "y")
+            assert client.get("x") == "y"
+        except:
+            raise SkipTest(
+                "memcached is not running or "
+                "otherwise not functioning correctly")
+
+class _NonDistributedMemcachedTest(_TestMemcachedConn, _GenericBackendTest):
     region_args = {
         "key_mangler":lambda x: x.replace(" ", "_")
     }
         }
     }
 
-class _DistributedMemcachedTest(_GenericBackendTest):
+class _DistributedMemcachedTest(_TestMemcachedConn, _GenericBackendTest):
     region_args = {
         "key_mangler":lambda x: x.replace(" ", "_")
     }
         }
     }
 
-class _DistributedMemcachedMutexTest(_GenericMutexTest):
+class _DistributedMemcachedMutexTest(_TestMemcachedConn, _GenericMutexTest):
     config_args = {
         "arguments":{
             "url":"127.0.0.1:11211",
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.