Commits

Anonymous committed 1900ccf

- reorganize the memcached imports again so that
a specific module name can be specified
- add 'memcache_module' param for memcached backend,
allows specification of specific library
- Sphinx 1.0 seems to use pathto("py-modindex") and
not pathto("modindex")
- update copyright in sphinx docs
- add docstrings and such to NamespaceManager and others,
though this is an uphill battle...this is a really old API

  • Participants
  • Parent commits cf8c227

Comments (0)

Files changed (12)

 * Defer running of pkg_resources to look for external cache modules
   until requested. #66
 * memcached backend uses pylibmc.ThreadMappedPool to ensure thread-local
-  usage of pylibmc when that library is installed.
+  usage of pylibmc when that library is in use.
+* memcached backend also has ``memcache_module`` string argument, allows
+  direct specification of the name of which memcache backend to use.
 
 Release 1.5.4 (6/16/2010)
 =========================

beaker/container.py

 class NamespaceManager(object):
     """Handles dictionary operations and locking for a namespace of
     values.
+    
+    :class:`.NamespaceManager` provides a dictionary-like interface,
+    implementing ``__getitem__()``, ``__setitem__()``, and 
+    ``__contains__()``, as well as functions related to lock
+    acquisition.
 
     The implementation for setting and retrieving the namespace data is
     handled by subclasses.
 
-    NamespaceManager may be used alone, or may be privately accessed by
-    one or more Container objects.  Container objects provide per-key
+    NamespaceManager may be used alone, or may be accessed by
+    one or more :class:`.Value` objects.  :class:`.Value` objects provide per-key
     services like expiration times and automatic recreation of values.
 
     Multiple NamespaceManagers created with a particular name will all
 
     @classmethod
     def _init_dependencies(cls):
-        pass
+        """Initialize module-level dependent libraries required
+        by this :class:`.NamespaceManager`."""
+
 
     def __init__(self, namespace):
         self._init_dependencies()
         self.namespace = namespace
 
     def get_creation_lock(self, key):
+        """Return a locking object that is used to synchronize
+        multiple threads or processes which wish to generate a new
+        cache value.
+        
+        This function is typically an instance of
+        :class:`.FileSynchronizer`, :class:`.ConditionSynchronizer`,
+        or :class:`.null_synchronizer`.   
+        
+        The creation lock is only used when a requested value
+        does not exist, or has been expired, and is only used
+        by the :class:`.Value` key-management object in conjunction
+        with a "createfunc" value-creation function.
+        
+        """
         raise NotImplementedError()
 
     def do_remove(self):
+        """Implement removal of the entire contents of this 
+        :class:`.NamespaceManager`.
+        
+        e.g. for a file-based namespace, this would remove
+        all the files.
+        
+        The front-end to this method is the
+        :meth:`.NamespaceManager.remove` method.
+        
+        """
         raise NotImplementedError()
 
     def acquire_read_lock(self):
-        pass
+        """Establish a read lock.
+        
+        This operation is called before a key is read.    By
+        default the function does nothing.
+
+        """
 
     def release_read_lock(self):
-        pass
+        """Release a read lock.
+        
+        This operation is called after a key is read.    By
+        default the function does nothing.
+
+        """
 
     def acquire_write_lock(self, wait=True):
+        """Establish a write lock.
+        
+        This operation is called before a key is written.   
+        A return value of ``True`` indicates the lock has 
+        been acquired.
+        
+        By default the function returns ``True`` unconditionally.
+        
+        """
         return True
 
     def release_write_lock(self):
-        pass
+        """Release a write lock.
+        
+        This operation is called after a new value is written.
+        By default this function does nothing.
+        
+        """
 
     def has_key(self, key):
+        """Return ``True`` if the given key is present in this 
+        :class:`.Namespace`.
+        """
         return self.__contains__(key)
 
     def __getitem__(self, key):
         raise NotImplementedError()
 
     def set_value(self, key, value, expiretime=None):
-        """Optional set_value() method called by Value.
-
-        Allows an expiretime to be passed, for namespace
-        implementations which can prune their collections
-        using expiretime.
+        """Sets a value in this :class:`.NamespaceManager`.
+        
+        This is the same as ``__setitem__()``, but
+        also allows an expiration time to be passed
+        at the same time.
 
         """
         self[key] = value
         raise NotImplementedError()
 
     def keys(self):
+        """Return the list of all keys.
+        
+        This method may not be supported by all
+        :class:`.NamespaceManager` implementations.
+        
+        """
         raise NotImplementedError()
 
     def remove(self):
+        """Remove the entire contents of this 
+        :class:`.NamespaceManager`.
+        
+        e.g. for a file-based namespace, this would remove
+        all the files.
+        """
         self.do_remove()
 
 
             self.access_lock.release_write_lock()
 
 class Value(object):
+    """Implements synchronization, expiration, and value-creation logic
+    for a single value stored in a :class:`.NamespaceManager`.
+
+    """
+
     __slots__ = 'key', 'createfunc', 'expiretime', 'expire_argument', 'starttime', 'storedtime',\
                 'namespace'
 
 
     def get_creation_lock(self, key):
         return NameLock(
-            identifier="memorynamespace/funclock/%s/%s" % (self.namespace, key),
+            identifier="memorynamespace/funclock/%s/%s" % 
+                        (self.namespace, key),
             reentrant=True
         )
 
         return self.dictionary.keys()
 
 class MemoryNamespaceManager(AbstractDictionaryNSManager):
+    """:class:`.NamespaceManager` that uses a Python dictionary for storage."""
+
     namespaces = util.SyncDict()
 
     def __init__(self, namespace, **kwargs):
         AbstractDictionaryNSManager.__init__(self, namespace)
-        self.dictionary = MemoryNamespaceManager.namespaces.get(self.namespace,
-                                                                dict)
+        self.dictionary = MemoryNamespaceManager.\
+                                namespaces.get(self.namespace, dict)
 
 class DBMNamespaceManager(OpenResourceNamespaceManager):
+    """:class:`.NamespaceManager` that uses ``dbm`` files for storage."""
+
     def __init__(self, namespace, dbmmodule=None, data_dir=None, 
-            dbm_dir=None, lock_dir=None, digest_filenames=True, **kwargs):
+            dbm_dir=None, lock_dir=None, 
+            digest_filenames=True, **kwargs):
         self.digest_filenames = digest_filenames
 
         if not dbm_dir and not data_dir:
 
 
 class FileNamespaceManager(OpenResourceNamespaceManager):
+    """:class:`.NamespaceManager` that uses binary files for storage.
+    
+    Each namespace is implemented as a single file storing a 
+    dictionary of key/value pairs, serialized using the Python
+    ``pickle`` module.
+    
+    """
     def __init__(self, namespace, data_dir=None, file_dir=None, lock_dir=None,
                  digest_filenames=True, **kwargs):
         self.digest_filenames = digest_filenames
                      expiretime=expiretime, starttime=starttime)
 
 class Container(object):
+    """Implements synchronization and value-creation logic
+    for a 'value' stored in a :class:`.NamespaceManager`.
+    
+    :class:`.Container` and its subclasses are deprecated.   The
+    :class:`.Value` class is now used for this purpose.
+    
+    """
     __metaclass__ = ContainerMeta
     namespace_class = NamespaceManager
 

beaker/docs/_templates/index.html

     </td><td width="50%">
       <p class="biglink"><a class="biglink" href="{{ pathto("genindex") }}">General Index</a><br/>
          <span class="linkdescr">all functions, classes, terms</span></p>
-      <p class="biglink"><a class="biglink" href="{{ pathto("modindex") }}">Module Index</a><br/>
+      <p class="biglink"><a class="biglink" href="{{ pathto("py-modindex") }}">Module Index</a><br/>
          <span class="linkdescr">quick access to all documented modules</span></p>
     </td></tr>
   </table>

beaker/docs/conf.py

 # If your extensions are in another directory, add it here. If the directory
 # is relative to the documentation root, use os.path.abspath to make it
 # absolute, like shown here.
-sys.path.append(os.path.abspath('../..'))
+sys.path.insert(0, os.path.abspath('../..'))
 
 # General configuration
 # ---------------------
 
 # General information about the project.
 project = u'Beaker'
-copyright = u'2008-2010, Ben Bangert, Mike Bayer'
+copyright = u'2008-2011, Ben Bangert, Mike Bayer'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

beaker/docs/configuration.rst

     file is used to ensure that multiple processes/threads aren't attempting
     to re-create the same value at the same time (The :term:`Dog-Pile Effect`)
 
+memcache_module (**optional**, string)
+    One of the names ``memcache``, ``cmemcache``, ``pylibmc``, or ``auto``.
+    Default is ``auto``.  Specifies which memcached client library should
+    be imported when using the ext:memcached backend.  If left at its
+    default of ``auto``, ``pylibmc`` is favored first, then ``cmemcache``,
+    then ``memcache``.  New in 1.5.5.
+
 type (**required**, string)
     The name of the back-end to use for storing the sessions or cache objects.
 
 
     Some of these back-ends require the url option as listed below.
 
-webtest_varname (**optionall**, string)
+webtest_varname (**optional**, string)
     The name of the attribute to use when stashing the session object into
     the environ for use with WebTest. The name provided here is where the 
     session object will be attached to the WebTest TestApp return value.

beaker/docs/modules/container.rst

 Module Contents
 ---------------
 
+.. autoclass:: DBMNamespaceManager
+    :show-inheritance:
+.. autoclass:: FileNamespaceManager
+    :show-inheritance:
+.. autoclass:: MemoryNamespaceManager
+    :show-inheritance:
+.. autoclass:: NamespaceManager
+    :members:
+.. autoclass:: OpenResourceNamespaceManager
+    :show-inheritance:
+.. autoclass:: Value
+   :members:
+   :undoc-members:
+
+Deprecated Classes
+------------------
 .. autoclass:: Container
 .. autoclass:: ContainerMeta
+    :show-inheritance:
 .. autoclass:: DBMContainer
-.. autoclass:: DBMNamespaceManager
+    :show-inheritance:
 .. autoclass:: FileContainer
-.. autoclass:: FileNamespaceManager
+    :show-inheritance:
 .. autoclass:: MemoryContainer
-.. autoclass:: MemoryNamespaceManager
-.. autoclass:: NamespaceManager
-   :members: set_value, get_creation_lock, do_remove, has_key
-.. autoclass:: OpenResourceNamespaceManager
-.. autoclass:: Value
-   :members: set_value, has_value, can_have_value, has_current_value, get_value, clear_value
+    :show-inheritance:

beaker/docs/modules/memcached.rst

 ---------------
 
 .. autoclass:: MemcachedContainer
+    :show-inheritance:
 .. autoclass:: MemcachedNamespaceManager
+    :show-inheritance:
+.. autoclass:: PyLibMCNamespaceManager
+    :show-inheritance:

beaker/docs/modules/synchronization.rst

 .. autoclass:: ConditionSynchronizer
 .. autoclass:: FileSynchronizer
 .. autoclass:: NameLock
+.. autoclass:: null_synchronizer
 .. autoclass:: SynchronizerImpl
+    :members:

beaker/ext/memcached.py

 from beaker.util import verify_directory, SyncDict
 import warnings
 
-memcache = pylibmc = None
+_client_libs = {}
+def _load_client(name='auto'):
+    if name in _client_libs:
+        return _client_libs[name]
+
+    def _pylibmc():
+        global pylibmc
+        import pylibmc
+        return pylibmc
+
+    def _cmemcache():
+        global cmemcache
+        import cmemcache
+        warnings.warn("cmemcache is known to have serious "
+                    "concurrency issues; consider using 'memcache' "
+                    "or 'pylibmc'")
+        return cmemcache
+
+    def _memcache():
+        global memcache
+        import memcache
+        return memcache
+
+    def _auto():
+        for _client in (_pylibmc, _cmemcache, _memcache):
+            try:
+                return _client()
+            except ImportError:
+                pass
+        else:
+            raise InvalidCacheBackendError(
+                    "Memcached cache backend requires one "
+                    "of: 'pylibmc' or 'memcache' to be installed.")
+
+    clients = {
+        'pylibmc':_pylibmc,
+        'cmemcache':_cmemcache,
+        'memcache':_memcache,
+        'auto':_auto
+    }
+    _client_libs[name] = clib = clients[name]()
+    return clib
 
 class MemcachedNamespaceManager(NamespaceManager):
+    """Provides the :class:`.NamespaceManager` API over a memcache client library."""
+
     clients = SyncDict()
 
-    @classmethod
-    def _init_dependencies(cls):
-        global memcache
-        if memcache is not None:
-            return
+    def __new__(cls, *args, **kw):
+        memcache_module = kw.pop('memcache_module', 'auto')
 
-        try:
-            import pylibmc
-            memcache = pylibmc
-        except ImportError:
-            try:
-                import cmemcache as memcache
-                warnings.warn("cmemcache is known to have serious "
-                            "concurrency issues; consider using 'memcache' or 'pylibmc'")
-            except ImportError:
-                try:
-                    import memcache
-                except ImportError:
-                    raise InvalidCacheBackendError(
-                            "Memcached cache backend requires one "
-                            "of: 'pylibmc' or 'memcache' to be installed.")
+        memcache_client = _load_client(memcache_module)
 
-    def __new__(cls, *args, **kwargs):
-        cls._init_dependencies()
-        if pylibmc is not None:
+        if memcache_module == 'pylibmc' or \
+            memcache_client.__name__.startswith('pylibmc'):
             return object.__new__(PyLibMCNamespaceManager)
         else:
             return object.__new__(MemcachedNamespaceManager)
 
-    def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, **params):
+    def __init__(self, namespace, url, 
+                        memcache_module='auto', 
+                        data_dir=None, lock_dir=None, 
+                        **kw):
         NamespaceManager.__init__(self, namespace)
 
+        _memcache_module = _client_libs[memcache_module]
+
         if not url:
             raise MissingCacheParameter("url is required") 
 
         if self.lock_dir:
             verify_directory(self.lock_dir)
 
-        self.mc = MemcachedNamespaceManager.clients.get(url, memcache.Client, url.split(';'))
+        self.mc = MemcachedNamespaceManager.clients.get(
+                        (memcache_module, url), 
+                        _memcache_module.Client, 
+                        url.split(';'))
 
     def get_creation_lock(self, key):
         return file_synchronizer(
         self.mc.flush_all()
 
     def keys(self):
-        raise NotImplementedError("Memcache caching does not support iteration of all cache keys")
+        raise NotImplementedError(
+                "Memcache caching does not "
+                "support iteration of all cache keys")
 
 class PyLibMCNamespaceManager(MemcachedNamespaceManager):
     """Provide thread-local support for pylibmc."""
             mc.flush_all()
 
 class MemcachedContainer(Container):
+    """Container class which invokes :class:`.MemcacheNamespaceManager`."""
     namespace_class = MemcachedNamespaceManager

beaker/session.py

     def load(self):
         "Loads the data from this session from persistent storage"
         self.namespace = self.namespace_class(self.id,
-            data_dir=self.data_dir, digest_filenames=False,
+            data_dir=self.data_dir, 
+            digest_filenames=False,
             **self.namespace_args)
         now = time.time()
         if self.use_cookies:

beaker/synchronization.py

 
 
 class null_synchronizer(object):
+    """A 'null' synchronizer, which provides the :class:`.SynchronizerImpl` interface
+    without any locking.
+    
+    """
     def acquire_write_lock(self, wait=True):
         return True
     def acquire_read_lock(self):
 
 
 class SynchronizerImpl(object):
+    """Base class for a synchronization object that allows
+    multiple readers, single writers.
+    
+    """
     def __init__(self):
         self._state = util.ThreadLocal()
 
 
 
 class FileSynchronizer(SynchronizerImpl):
-    """a synchronizer which locks using flock().
-
-    Adapted for Python/multithreads from Apache::Session::Lock::File,
-    http://search.cpan.org/src/CWEST/Apache-Session-1.81/Session/Lock/File.pm
-
-    This module does not unlink temporary files, 
-    because it interferes with proper locking.  This can cause 
-    problems on certain systems (Linux) whose file systems (ext2) do not 
-    perform well with lots of files in one directory.  To prevent this
-    you should use a script to clean out old files from your lock directory.
+    """A synchronizer which locks using flock().
 
     """
     def __init__(self, identifier, lock_dir):

tests/test_memcached.py

 import unittest
 
 try:
-    clsmap['ext:memcached']._init_dependencies()
+    from beaker.ext import memcached
+    client = memcached._load_client()
 except InvalidCacheBackendError:
     raise SkipTest("an appropriate memcached backend is not installed")
 
 mc_url = '127.0.0.1:11211'
 
-from beaker.ext.memcached import memcache
-c = memcache.Client([mc_url])
+c =client.Client([mc_url])
 c.set('x', 'y')
 if not c.get('x'):
     raise SkipTest("Memcached is not running at %s" % mc_url)
         from beaker.ext import memcached
         try:
             import pylibmc as memcache
-            memcached.pylibmc = memcache
         except:
             import memcache
-            memcached.pylibmc = memcache
+            memcached._client_libs['pylibmc'] = memcached.pylibmc = memcache
             from contextlib import contextmanager
             class ThreadMappedPool(dict):
                 "a mock of pylibmc's ThreadMappedPool"
                 @contextmanager
                 def reserve(self):
                     yield self.master
-            memcached.pylibmc.ThreadMappedPool = ThreadMappedPool
+            memcache.ThreadMappedPool = ThreadMappedPool
 
     def tearDown(self):
         from beaker.ext import memcached
 
     def test_uses_pylibmc_client(self):
         from beaker.ext import memcached
-        cache = Cache('test', data_dir='./cache', url=mc_url, type="ext:memcached")
+        cache = Cache('test', data_dir='./cache', 
+                            memcache_module='pylibmc', 
+                            url=mc_url, type="ext:memcached")
         assert isinstance(cache.namespace, memcached.PyLibMCNamespaceManager)
 
     def test_dont_use_pylibmc_client(self):