Commits

Lynn Rees  committed 856dc95

[svn]

  • Participants
  • Parent commits d5a80d7

Comments (0)

Files changed (2)

File trunk/multishove/__init__.py

-# Copyright (c) 2001-2006 Python Software Foundation
-# Copyright (c) 2005, the Lawrence Journal-World
 # Copyright (c) 2006 L. C. Rees
 # All rights reserved.
 #
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-'''Common object storage frontend.'''
+'''Common frontend for multiple object stores.''' 
 
-import os
-import zlib
-import urllib
-try:
-    import cPickle as pickle
-except ImportError:
-    import pickle
-try:
-    # Import store and cache entry points if setuptools installed
-    import pkg_resources
-    stores = dict((_store.name, _store) for _store in
-        pkg_resources.iter_entry_points('shove.stores'))
-    caches = dict((_cache.name, _cache) for _cache in
-        pkg_resources.iter_entry_points('shove.caches'))
-    # Pass if nothing loaded
-    if not stores and not caches: raise ImportError()
-except ImportError:
-    # Static store backend registry
-    stores = dict(
-        simple='shove.store.simple:SimpleStore',
-        memory='shove.store.memory:MemoryStore',
-        file='shove.store.file:FileStore',
-        dbm='shove.store.dbm:DbmStore',
-        bsddb='shove.store.bsdb:BsdStore',
-        sqlite='shove.store.db:DbStore',
-        postgres='shove.store.db:DbStore',
-        mysql='shove.store.db:DbStore',
-        oracle='shove.store.db:DbStore',
-        firebird='shove.store.db:DbStore',
-        mssql='shove.store.db:DbStore',
-        svn='shove.store.svn:SvnStore',
-        s3='shove.store.s3:S3Store',
-        ftp='shove.store.ftp:FtpStore',
-        zodb='shove.store.zodb:ZodbStore',
-        durus='shove.store.durusdb:DurusStore')
-    # Static cache backend registry
-    caches = dict(
-        simple='shove.cache.simple:SimpleCache',
-        memory='shove.cache.memory:MemoryCache',
-        file='shove.cache.file:FileCache',
-        mssql='shove.cache.db:DbCache',
-        sqlite='shove.cache.db:DbCache',
-        postgres='shove.cache.db:DbCache',
-        firebird='shove.cache.db:DbCache',
-        mysql='shove.cache.db:DbCache',
-        oracle='shove.cache.db:DbCache',
-        memcache='shove.cache.memcached:MemCached',
-        bsddb='shove.cache.bsdb:BsdCache')
+from shove import Shove, getbackend 
 
-__all__ = ['Shove', 'storage', 'cache']
+__all__ = ['MultiShove']
     
-def synchronized(func):
-    '''Decorator to lock and unlock a method (Phillip J. Eby).
+  
+class MultiShove(Shove):
 
-    @param func Method to decorate
-    '''
-    def wrapper(self, *__args, **__kw):
-        self._lock.acquire()
-        try:
-            return func(self, *__args, **__kw)
-        finally:
-            self._lock.release()
-    wrapper.__name__ = func.__name__
-    wrapper.__dict__ = func.__dict__
-    wrapper.__doc__ = func.__doc__
-    return wrapper
-
-def getbackend(uri, engines, **kw):
-    '''Loads the right backend based on a URI.
-
-    @param uri Instance or name string
-    @param engines A dictionary of scheme/class pairs
-    @param kw Keywords'''
-    if isinstance(uri, basestring):
-        mod = engines[uri.split('://', 1)[0]]
-        # Load module if setuptools not present
-        if isinstance(mod, basestring): 
-            # Isolate classname from dot path
-            module, klass = mod.split(':')
-            # Load module
-            mod = getattr(__import__(module, '', '', ['']), klass)
-        # Load appropriate class from setuptools entry point
-        else:
-            mod = mod.load()
-        # Return instance
-        return mod(uri, **kw)
-    # No-op for existing instances
-    return uri
-
-
-class Base(object):
-
-    '''Base Mapping class.'''
+    '''Common frontend for multiple object stores.'''
     
-    def __init__(self, engine, **kw):
-        super(Base, self).__init__()
-        self._compress = kw.get('compress', False)
-    
-    def __getitem__(self, key):
-        raise NotImplementedError()
-
-    def __setitem__(self, key, value):
-        raise NotImplementedError()
-
-    def __delitem__(self, key):
-        raise NotImplementedError()
-
-    def __contains__(self, key):
-        try:
-            value = self[key]
-        except KeyError:
-            return False
-        return True
-
-    def get(self, key, default=None):
-        '''Fetch a given key from the mapping. If the key does not exist,
-        return the default.
-
-        @param key Keyword of item in mapping.
-        @param default Default value (default: None)
-        '''
-        try:
-            return self[key]
-        except KeyError:
-            return default
-    
-    def dumps(self, value):
-        '''Optionally serializes and  compresses an object.'''
-        # Serialize everything but ASCII strings
-        if not isinstance(value, str): value = pickle.dumps(value)
-        # Apply maximum compression
-        if self._compress: value = zlib.compress(value, 9)
-        return value
-    
-    def loads(self, value):
-        '''Deserializes and optionally decompresses an object.'''
-        if self._compress:
-            try:
-                value = zlib.decompress(value)
-            except zlib.error: pass
-        # Load serialized objects
-        try:
-            value = pickle.loads(value)
-        # Skip non strings
-        except (pickle.UnpicklingError, IndexError, ValueError):
-            if isinstance(value, str): pass
-        return value
-
-
-class BaseStore(Base):
-
-    '''Base Store class (based on UserDict.DictMixin).'''
-
-    def __init__(self, engine, **kw):
-        super(BaseStore, self).__init__(engine, **kw)
-        self._store = None
-
-    def __cmp__(self, other):
-        if other is None: return False
-        if isinstance(other, BaseStore):
-            return cmp(dict(self.iteritems()), dict(other.iteritems()))
-
-    def __del__(self):
-        # __init__ didn't succeed, so don't bother closing
-        if not hasattr(self, '_store'): return
-        self.close()
-
-    def __iter__(self):
-        for k in self.keys(): yield k
-
-    def __len__(self):
-        return len(self.keys())
-
-    def __repr__(self):
-        return repr(dict(self.iteritems()))
-
-    def close(self):
-        '''Closes internal store and clears object references.'''
-        try:
-            self._store.close()
-        except AttributeError: pass
-        self._store = None
-
-    def clear(self):
-        '''Removes all keys and values from a store.'''
-        for key in self.keys(): del self[key]
-
-    def items(self):
-        '''Returns a list with all key/value pairs in the store.'''
-        return list(self.iteritems())
-
-    def iteritems(self):
-        '''Lazily returns all key/value pairs in a store.'''
-        for k in self: yield (k, self[k])
-
-    def iterkeys(self):
-        '''Lazy returns all keys in a store.'''
-        return self.__iter__()
-
-    def itervalues(self):
-        '''Lazily returns all values in a store.'''
-        for _, v in self.iteritems(): yield v
-
-    def keys(self):
-        '''Returns a list with all keys in a store.'''
-        raise NotImplementedError()
-
-    def pop(self, key, *args):
-        '''Removes and returns a value from a store.
-
-        @param args Default to return if key not present.'''
-        if len(args) > 1:
-            raise TypeError('pop expected at most 2 arguments, got '\
-                + repr(1 + len(args)))
-        try:
-            value = self[key]
-        # Return default if key not in store
-        except KeyError:
-            if args: return args[0]
-        del self[key]
-        return value
-
-    def popitem(self):
-        '''Removes and returns a key, value pair from a store.'''
-        try:
-            k, v = self.iteritems().next()
-        except StopIteration:
-            raise KeyError('Store is empty.')
-        del self[k]
-        return (k, v)
-
-    def setdefault(self, key, default=None):
-        '''Returns the value corresponding to an existing key or sets the
-        to key to the default and returns the default.
-
-        @param default Default value (default: None)
-        '''
-        try:
-            return self[key]
-        except KeyError:
-            self[key] = default
-        return default
-
-    def update(self, other=None, **kw):
-        '''Adds to or overwrites the values in this store with values from
-        another store.
-
-        other Another store
-        kw Additional keys and values to store
-        '''
-        if other is None: pass
-        elif hasattr(other, 'iteritems'):
-            for k, v in other.iteritems(): self[k] = v
-        elif hasattr(other, 'keys'):
-            for k in other.keys(): self[k] = other[k]
-        else:
-            for k, v in other: self[k] = v
-        if kw: self.update(kw)
-
-    def values(self):
-        '''Returns a list with all values in a store.'''
-        return list(v for _, v in self.iteritems()) 
-    
-    
-class Shove(BaseStore):
-
-    '''Common object frontend class.'''
-    
-    def __init__(self, store='simple://', cache='simple://', **kw):
-        super(Shove, self).__init__(store, **kw)
-        # Load store
-        self._store = getbackend(store, stores, **kw)
-        # Load cache
-        self._cache = getbackend(cache, caches, **kw)
-        # Buffer for lazy writing and setting for syncing frequency
-        self._buffer, self._sync = dict(), kw.get('sync', 2)
-
+    def __init__(self, *a, **kw):
+        cache = kw.get('cache', 'simple://')
+        # Init superclass wi
+        super(Shove, self).__init__(a[0], cache, **kw)
+        # Delete single store instance
+        del self._store
+        # Load stores
+        stores = tuple(getbackend(i, stores, **kw) for i in a)
+        
     def __getitem__(self, key):
         '''Gets a item from shove.'''
         try:
         except KeyError:
             # Synchronize cache and store
             self.sync()
-            value = self._store[key]
+            value = self._stores[0][key]
             self._cache[key] = value
             return value
 
-    def __setitem__(self, key, value):
-        '''Sets an item in shove.'''
-        self._cache[key] = self._buffer[key] = value
-        # When the buffer reaches self._limit, writes the buffer to the store
-        if len(self._buffer) >= self._sync: self.sync()
-
     def __delitem__(self, key):
-        '''Deletes an item from shove.'''
+        '''Deletes an item from multiple stores.'''
         try:
             del self._cache[key]
         except KeyError: pass
         self.sync()
-        del self._store[key]
+        for store in self._stores: del self.store[key]
 
     def keys(self):
         '''Returns a list of keys in shove.'''
         self.sync()
-        return self._store.keys()
+        return self._stores[0].keys()
 
     def sync(self):
         '''Writes buffer to store.'''
-        for k, v in self._buffer.iteritems(): self._store[k] = v
+        for k, v in self._buffer.iteritems():
+            for store in self._stores: store[k] = v
         self._buffer.clear()
         
     def close(self):
-        '''Finalizes and closes shove.'''
+        '''Finalizes and closes shove stores.'''
         # If close has been called, pass
         if self._store is not None:
             self.sync()
-            self._store.close()
-            self._store = self._cache = self._buffer = None
-
-
-class FileBase(Base):
-
-    '''Base class for file based storage.'''    
-
-    def __init__(self, engine, **kw):
-        super(FileBase, self).__init__(engine, **kw)
-        if engine.startswith('file://'):
-            engine = urllib.url2pathname(engine.split('://')[1])
-        self._dir = engine
-        # Create directory
-        if not os.path.exists(self._dir): self._createdir()
-
-    def __getitem__(self, key):
-        try:
-            return self.loads(open(self._key_to_file(key), 'rb').read())
-        except:
-            raise KeyError('%s' % key)
-
-    def __setitem__(self, key, value):
-        try:
-            open(self._key_to_file(key), 'wb').write(self.dumps(value))
-        except (IOError, OSError):
-            raise KeyError('%s' % key)        
-
-    def __delitem__(self, key):
-        try:
-            os.remove(self._key_to_file(key))
-        except (IOError, OSError):
-            raise KeyError('%s' % key)
-
-    def __contains__(self, key):
-        return os.path.exists(self._key_to_file(key))
-    
-    def __len__(self):
-        return len(os.listdir(self._dir))    
-
-    def _createdir(self):
-        '''Creates the store directory.'''
-        try:
-            os.makedirs(self._dir)
-        except OSError:
-            raise EnvironmentError('Cache directory "%s" does not exist and ' \
-                'could not be created' % self._dir)    
-
-    def _key_to_file(self, key):
-        '''Gives the filesystem path for a key.'''
-        return os.path.join(self._dir, urllib.quote_plus(key))
-
-    def keys(self):
-        '''Returns a list of keys in the store.'''
-        return os.listdir(self._dir)       
-
-
-class SimpleBase(Base):
-
-    '''Single-process in-memory store base class.'''    
-    
-    def __init__(self, engine, **kw):
-        super(SimpleBase, self).__init__(engine, **kw)
-        self._store = dict()
-
-    def __getitem__(self, key):
-        try:
-            return self._store[key]
-        except:
-            raise KeyError('%s' % key)
-
-    def __setitem__(self, key, value):
-        self._store[key] = value
-
-    def __delitem__(self, key):
-        try:
-            del self._store[key]
-        except:
-            raise KeyError('%s' % key)
-
-    def __len__(self):
-        return len(self._store)        
-
-    def keys(self):
-        '''Returns a list of keys in the store.'''
-        return self._store.keys()        
-
-
-class DbBase(Base):     
-
-    '''Database common base class.'''
-
-    def __init__(self, engine, **kw):
-        super(DbBase, self).__init__(engine, **kw)
-      
-    def __delitem__(self, key):
-        self._store.delete(self._store.c.key==key).execute()
-
-    def __len__(self):
-        return self._store.count().execute().fetchone()[0]
+            for store in self._stores:
+                store.close()
+                store = None
+            self._store = self._cache = self._buffer = None

File trunk/setup.py

 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-'''setup - setuptools based setup for shove.'''
+'''setup - setuptools based setup for multishove.'''
 
 import ez_setup
 ez_setup.use_setuptools()
 except:
     from distutils.core import setup
 
-setup(name='shove',
+setup(name='multishove',
       version='0.1',
-      description='''Common object storage frontend.''',
-      long_description='''Common object storage frontend that supports
+      description='''Common frontend for multiple object stores.''',
+      long_description='''Common frontend that lazily stores objects
+in multiple objects storage backends simultaneously and supports
 dictionary-style access, object serialization and compression, and
 multiple storage and caching backends.
 
 Currently supported storage backends are:
 
 Amazon S3 Web Service
+
 Berkeley Source Database
+
 Memory
+
 Filesystem
+
 Firebird
+
 FTP
+
 DBM
+
 Durus
+
 Microsoft SQL Server
+
 MySQL
+
 Oracle
+
 PostgreSQL
+
 SQLite
+
 Subversion
+
 Zope Object Database (ZODB)
 
 Currently supported caching backends are:
 
 Memory
+
 Filesystem
+
 Firebird
+
 memcache
+
 Microsoft SQL Server
+
 MySQL
+
 Oracle
+
 PostgreSQL
+
 SQLite
 
-The simplest shove use case is:
+The use of multiple backends for storage involves
+passing multiple store URIs or instances to
+multishove following the form:
 
-from shove import Shove
+<storename> = Shove(<store_uri1>, <store_uri2> ..., cache=<cache_uri>)
 
-store = Shove()
+multishove's access API is the Python mapping API:
 
-which creates an in-memory store and cache.
+http://docs.python.org/lib/typesmapping.html
 
-The use of other backends for storage and caching involves
-passing an module URI or existing store or cache instance
-to shove following the form:
+multishove requires the shove package from:
 
-from shove import Shove
-
-<storename> = Shove(<store_uri>, <cache_uri>)
-
-The module-specific URI form is documented in its module. The
-form follows the URI form used by SQLAlchemy:
-
-http://www.sqlalchemy.org/docs/dbengine.myt#dbengine_establishing
-
-shove's access API is the Python mapping API:
-
-http://docs.python.org/lib/typesmapping.html''',
+http://cheeseshop.python.org/pypi/shove
+''',
       author='L. C. Rees',
       author_email='lcrees@gmail.com',
       license='BSD',
-      packages = ['shove', 'shove.cache', 'shove.store', 'shove.tests'],
-      test_suite='shove.tests',
+      packages = ['multishove', 'multishove.tests'],
+      test_suite='multishove.tests',
       zip_safe = True,
       keywords='object storage persistence database shelve',
       classifiers=['Development Status :: 4 - Beta',
           'Operating System :: OS Independent',
           'Programming Language :: Python',
           'Topic :: Database :: Front-Ends'],
-    install_requires = ['SQLAlchemy>=0.3', 'boto'],
-    entry_points = '''
-    [shove.stores]   
-    bsddb=shove.store.bsdb:BsdStore    
-    dbm=shove.store.dbm:DbmStore
-    durus=shove.store.durusdb:DurusStore
-    file=shove.store.file:FileStore
-    firebird=shove.store.db:DbStore
-    ftp=shove.store.ftp:FtpStore   
-    memory=shove.store.memory:MemoryStore
-    mssql=shove.store.db:DbStore
-    mysql=shove.store.db:DbStore
-    oracle=shove.store.db:DbStore
-    postgres=shove.store.db:DbStore
-    simple=shove.store.simple:SimpleStore
-    sqlite=shove.store.db:DbStore
-    s3=shove.store.s3:S3Store
-    svn=shove.store.svn:SvnStore
-    zodb=shove.store.zodb:ZodbStore
-    [shove.caches]
-    bsddb=shove.cache.bsdb:BsdCache    
-    file=shove.cache.file:FileCache
-    firebird=shove.cache.db:DbCache
-    memcache=shove.cache.memcached:MemCached
-    memory=shove.cache.memory:MemoryCache
-    mssql=shove.cache.db:DbCache
-    mysql=shove.cache.db:DbCache
-    oracle=shove.cache.db:DbCache
-    postgres=shove.cache.db:DbCache
-    simple=shove.cache.simple:SimpleCache
-    sqlite=shove.cache.db:DbCache
-    ''')
+    install_requires = ['shove'])