Lynn Rees avatar Lynn Rees committed fdf931f

[svn] made a copy

Comments (0)

Files changed (6)

branches/0.1/trunk/README

+Stores objects in multiple storage backends simultaneously with dictionary-style access, caching, and object serialization and compression.
+
+Currently supported storage backends are:
+
+Amazon S3 Web Service
+
+Berkeley Source Database
+
+Memory
+
+Filesystem
+
+Firebird
+
+FTP
+
+DBM
+
+Durus
+
+Microsoft SQL Server
+
+MySQL
+
+Oracle
+
+PostgreSQL
+
+SQLite
+
+Subversion
+
+Zope Object Database (ZODB)
+
+Currently supported caching backends are:
+
+Memory
+
+Filesystem
+
+Firebird
+
+memcache
+
+Microsoft SQL Server
+
+MySQL
+
+Oracle
+
+PostgreSQL
+
+SQLite
+
+The use of multiple backends for storage involves passing multiple store URIs or instances to multishove following the form:
+
+from multishove import MultiShove
+
+<storename> = MultiShove(<store_uri1>, <store_uri2> ..., cache=<cache_uri>)
+
+multishove's access API is the Python dictionary/mapping API:
+
+http://docs.python.org/lib/typesmapping.html
+
+multishove requires the shove package from:
+
+http://cheeseshop.python.org/pypi/shove

branches/0.1/trunk/ez_setup.py

+#!python
+"""Bootstrap setuptools installation
+
+If you want to use setuptools in your package's setup.py, just include this
+file in the same directory with it, and add this to the top of your setup.py::
+
+    from ez_setup import use_setuptools
+    use_setuptools()
+
+If you want to require a specific version of setuptools, set a download
+mirror, or use an alternate download directory, you can do so by supplying
+the appropriate options to ``use_setuptools()``.
+
+This file can also be run as a script to install or upgrade setuptools.
+"""
+import sys
+DEFAULT_VERSION = "0.6c3"
+DEFAULT_URL     = "http://cheeseshop.python.org/packages/%s/s/setuptools/" % sys.version[:3]
+
+md5_data = {
+    'setuptools-0.6b1-py2.3.egg': '8822caf901250d848b996b7f25c6e6ca',
+    'setuptools-0.6b1-py2.4.egg': 'b79a8a403e4502fbb85ee3f1941735cb',
+    'setuptools-0.6b2-py2.3.egg': '5657759d8a6d8fc44070a9d07272d99b',
+    'setuptools-0.6b2-py2.4.egg': '4996a8d169d2be661fa32a6e52e4f82a',
+    'setuptools-0.6b3-py2.3.egg': 'bb31c0fc7399a63579975cad9f5a0618',
+    'setuptools-0.6b3-py2.4.egg': '38a8c6b3d6ecd22247f179f7da669fac',
+    'setuptools-0.6b4-py2.3.egg': '62045a24ed4e1ebc77fe039aa4e6f7e5',
+    'setuptools-0.6b4-py2.4.egg': '4cb2a185d228dacffb2d17f103b3b1c4',
+    'setuptools-0.6c1-py2.3.egg': 'b3f2b5539d65cb7f74ad79127f1a908c',
+    'setuptools-0.6c1-py2.4.egg': 'b45adeda0667d2d2ffe14009364f2a4b',
+    'setuptools-0.6c2-py2.3.egg': 'f0064bf6aa2b7d0f3ba0b43f20817c27',
+    'setuptools-0.6c2-py2.4.egg': '616192eec35f47e8ea16cd6a122b7277',
+    'setuptools-0.6c3-py2.3.egg': 'f181fa125dfe85a259c9cd6f1d7b78fa',
+    'setuptools-0.6c3-py2.4.egg': 'e0ed74682c998bfb73bf803a50e7b71e',
+    'setuptools-0.6c3-py2.5.egg': 'abef16fdd61955514841c7c6bd98965e',
+}
+
+import sys, os
+
+def _validate_md5(egg_name, data):
+    if egg_name in md5_data:
+        from md5 import md5
+        digest = md5(data).hexdigest()
+        if digest != md5_data[egg_name]:
+            print >>sys.stderr, (
+                "md5 validation of %s failed!  (Possible download problem?)"
+                % egg_name
+            )
+            sys.exit(2)
+    return data
+
+
+def use_setuptools(
+    version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir,
+    download_delay=15
+):
+    """Automatically find/download setuptools and make it available on sys.path
+
+    `version` should be a valid setuptools version number that is available
+    as an egg for download under the `download_base` URL (which should end with
+    a '/').  `to_dir` is the directory where setuptools will be downloaded, if
+    it is not already available.  If `download_delay` is specified, it should
+    be the number of seconds that will be paused before initiating a download,
+    should one be required.  If an older version of setuptools is installed,
+    this routine will print a message to ``sys.stderr`` and raise SystemExit in
+    an attempt to abort the calling script.
+    """
+    try:
+        import setuptools
+        if setuptools.__version__ == '0.0.1':
+            print >>sys.stderr, (
+            "You have an obsolete version of setuptools installed.  Please\n"
+            "remove it from your system entirely before rerunning this script."
+            )
+            sys.exit(2)
+    except ImportError:
+        egg = download_setuptools(version, download_base, to_dir, download_delay)
+        sys.path.insert(0, egg)
+        import setuptools; setuptools.bootstrap_install_from = egg
+
+    import pkg_resources
+    try:
+        pkg_resources.require("setuptools>="+version)
+
+    except pkg_resources.VersionConflict, e:
+        # XXX could we install in a subprocess here?
+        print >>sys.stderr, (
+            "The required version of setuptools (>=%s) is not available, and\n"
+            "can't be installed while this script is running. Please install\n"
+            " a more recent version first.\n\n(Currently using %r)"
+        ) % (version, e.args[0])
+        sys.exit(2)
+
+def download_setuptools(
+    version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir,
+    delay = 15
+):
+    """Download setuptools from a specified location and return its filename
+
+    `version` should be a valid setuptools version number that is available
+    as an egg for download under the `download_base` URL (which should end
+    with a '/'). `to_dir` is the directory where the egg will be downloaded.
+    `delay` is the number of seconds to pause before an actual download attempt.
+    """
+    import urllib2, shutil
+    egg_name = "setuptools-%s-py%s.egg" % (version,sys.version[:3])
+    url = download_base + egg_name
+    saveto = os.path.join(to_dir, egg_name)
+    src = dst = None
+    if not os.path.exists(saveto):  # Avoid repeated downloads
+        try:
+            from distutils import log
+            if delay:
+                log.warn("""
+---------------------------------------------------------------------------
+This script requires setuptools version %s to run (even to display
+help).  I will attempt to download it for you (from
+%s), but
+you may need to enable firewall access for this script first.
+I will start the download in %d seconds.
+
+(Note: if this machine does not have network access, please obtain the file
+
+   %s
+
+and place it in this directory before rerunning this script.)
+---------------------------------------------------------------------------""",
+                    version, download_base, delay, url
+                ); from time import sleep; sleep(delay)
+            log.warn("Downloading %s", url)
+            src = urllib2.urlopen(url)
+            # Read/write all in one block, so we don't create a corrupt file
+            # if the download is interrupted.
+            data = _validate_md5(egg_name, src.read())
+            dst = open(saveto,"wb"); dst.write(data)
+        finally:
+            if src: src.close()
+            if dst: dst.close()
+    return os.path.realpath(saveto)
+
+def main(argv, version=DEFAULT_VERSION):
+    """Install or upgrade setuptools and EasyInstall"""
+
+    try:
+        import setuptools
+    except ImportError:
+        egg = None
+        try:
+            egg = download_setuptools(version, delay=0)
+            sys.path.insert(0,egg)
+            from setuptools.command.easy_install import main
+            return main(list(argv)+[egg])   # we're done here
+        finally:
+            if egg and os.path.exists(egg):
+                os.unlink(egg)
+    else:
+        if setuptools.__version__ == '0.0.1':
+            # tell the user to uninstall obsolete version
+            use_setuptools(version)
+
+    req = "setuptools>="+version
+    import pkg_resources
+    try:
+        pkg_resources.require(req)
+    except pkg_resources.VersionConflict:
+        try:
+            from setuptools.command.easy_install import main
+        except ImportError:
+            from easy_install import main
+        main(list(argv)+[download_setuptools(delay=0)])
+        sys.exit(0) # try to force an exit
+    else:
+        if argv:
+            from setuptools.command.easy_install import main
+            main(argv)
+        else:
+            print "Setuptools version",version,"or greater has been installed."
+            print '(Run "ez_setup.py -U setuptools" to reinstall or upgrade.)'
+
+
+
+def update_md5(filenames):
+    """Update our built-in md5 registry"""
+
+    import re
+    from md5 import md5
+
+    for name in filenames:
+        base = os.path.basename(name)
+        f = open(name,'rb')
+        md5_data[base] = md5(f.read()).hexdigest()
+        f.close()
+
+    data = ["    %r: %r,\n" % it for it in md5_data.items()]
+    data.sort()
+    repl = "".join(data)
+
+    import inspect
+    srcfile = inspect.getsourcefile(sys.modules[__name__])
+    f = open(srcfile, 'rb'); src = f.read(); f.close()
+
+    match = re.search("\nmd5_data = {\n([^}]+)}", src)
+    if not match:
+        print >>sys.stderr, "Internal error!"
+        sys.exit(2)
+
+    src = src[:match.start(1)] + repl + src[match.end(1):]
+    f = open(srcfile,'w')
+    f.write(src)
+    f.close()
+
+
+if __name__=='__main__':
+    if len(sys.argv)>2 and sys.argv[1]=='--md5update':
+        update_md5(sys.argv[2:])
+    else:
+        main(sys.argv[1:])
+
+
+
+
+

branches/0.1/trunk/multishove/__init__.py

+# Copyright (c) 2006 L. C. Rees
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#    1. Redistributions of source code must retain the above copyright notice, 
+#       this list of conditions and the following disclaimer.
+#    
+#    2. Redistributions in binary form must reproduce the above copyright 
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#
+#    3. Neither the name of the Portable Site Information project nor the names
+#       of its contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''Common frontend for storing objects in multiple storage backends at once.''' 
+
+from shove import BaseStore, getbackend, stores, caches 
+
+__all__ = ['MultiShove']
+    
+  
+class MultiShove(BaseStore):
+
+    '''Common frontend to multiple object stores.'''
+    
+    def __init__(self, *a, **kw):
+        # Init superclass with first store
+        super(MultiShove, self).__init__('', **kw)
+        if not a: a = ('simple://',)
+        # Load stores
+        self._stores = list(getbackend(i, stores, **kw) for i in a)
+        # Load cache
+        self._cache = getbackend(kw.get('cache', 'simple://'), caches, **kw)
+        # Buffer for lazy writing and setting for syncing frequency
+        self._buffer, self._sync = dict(), kw.get('sync', 2)        
+        
+    def __getitem__(self, key):
+        '''Gets a item from shove.'''
+        try:
+            return self._cache[key]
+        except KeyError:
+            # Synchronize cache and store
+            self.sync()
+            # Get value from first store
+            value = self._stores[0][key]
+            self._cache[key] = value
+            return value
+
+    def __setitem__(self, key, value):
+        '''Sets an item in shove.'''
+        self._cache[key] = self._buffer[key] = value
+        # When the buffer reaches self._limit, writes the buffer to the store
+        if len(self._buffer) >= self._sync: self.sync()        
+
+    def __delitem__(self, key):
+        '''Deletes an item from multiple stores.'''
+        try:
+            del self._cache[key]
+        except KeyError: pass
+        self.sync()
+        for store in self._stores: del store[key]
+
+    def keys(self):
+        '''Returns a list of keys in shove.'''
+        self.sync()
+        return self._stores[0].keys()
+
+    def sync(self):
+        '''Writes buffer to store.'''
+        for k, v in self._buffer.iteritems():
+            # Sync all stores
+            for store in self._stores: store[k] = v
+        self._buffer.clear()
+        
+    def close(self):
+        '''Finalizes and closes shove stores.'''
+        # If close has been called, pass
+        if self._stores is not None:
+            self.sync()
+            # Close stores
+            for idx, store in enumerate(self._stores):
+                store.close()
+                self._stores[idx] = None
+            self._cache = self._buffer = self._stores = None
Add a comment to this file

branches/0.1/trunk/multishove/tests/__init__.py

Empty file added.

branches/0.1/trunk/multishove/tests/test_multishove.py

+import unittest
+import os
+from multishove import MultiShove
+
+class TestMultiShove(unittest.TestCase):
+
+    stores = ('simple://', 'dbm://one.dbm', 'memory://', 'file://two', 'sqlite://', 
+        'bsddb://three.db')
+
+    def setUp(self): 
+        self.store = MultiShove(*self.stores)
+
+    def tearDown(self): 
+        self.store.close()
+        for x in os.listdir('two'): os.remove(os.path.join('two', x))
+        os.rmdir('two')
+        os.remove('one.dbm')
+        os.remove('three.db')
+
+    def test__getitem__(self):
+        self.store['max'] = 3
+        self.store.sync()
+        self.assertEqual(self.store['max'], 3)
+
+    def test__setitem__(self):
+        self.store['max'] = 3
+        self.store.sync()
+        self.assertEqual(self.store['max'], 3)
+
+    def test__delitem__(self):
+        self.store['max'] = 3
+        del self.store['max']
+        self.assertEqual('max' in self.store, False)
+
+    def test_get(self):
+        self.store['max'] = 3
+        self.store.sync()
+        self.assertEqual(self.store.get('min'), None)
+            
+    def test__cmp__(self):
+        tstore = MultiShove()
+        self.store['max'] = 3
+        tstore['max'] = 3
+        self.store.sync()
+        tstore.sync()
+        self.assertEqual(self.store, tstore)
+
+    def test__len__(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.assertEqual(len(self.store), 3)
+
+    def test_clear(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        self.store.clear()
+        self.assertEqual(len(self.store), 0)
+
+    def test_items(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.items())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iteritems(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iteritems())
+        self.assertEqual(('min', 6) in slist, True)
+
+    def test_iterkeys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.iterkeys())
+        self.assertEqual('min' in slist, True)
+
+    def test_itervalues(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = list(self.store.itervalues())
+        self.assertEqual(6 in slist, True)
+
+    def test_pop(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        item = self.store.pop('min')
+        self.assertEqual(item, 6)
+        
+    def test_popitem(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        item = self.store.popitem()
+        self.assertEqual(len(item) + len(self.store), 4)
+
+    def test_setdefault(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['powl'] = 7
+        self.store.setdefault('pow', 8)
+        self.assertEqual(self.store['pow'], 8)
+
+    def test_update(self):
+        tstore = MultiShove()
+        tstore['max'] = 3
+        tstore['min'] = 6
+        tstore['pow'] = 7
+        self.store['max'] = 2
+        self.store['min'] = 3
+        self.store['pow'] = 7
+        self.store.update(tstore)
+        self.assertEqual(self.store['min'], 6)
+
+    def test_values(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.values()
+        self.assertEqual(6 in slist, True)
+
+    def test_keys(self):
+        self.store['max'] = 3
+        self.store['min'] = 6
+        self.store['pow'] = 7
+        slist = self.store.keys()
+        self.assertEqual('min' in slist, True)
+
+if __name__ == '__main__':
+    unittest.main()

branches/0.1/trunk/setup.py

+# Copyright (c) 2006 L. C. Rees.  All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1.  Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+# 2.  Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# 3.  Neither the name of the Portable Site Information Project nor the names
+# of its contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+'''setup - setuptools based setup for multishove.'''
+
+import ez_setup
+ez_setup.use_setuptools()
+
+try:
+    from setuptools import setup
+except:
+    from distutils.core import setup
+
+setup(name='multishove',
+version='0.1',
+description='''Simultaneous storage of objects in multiple storage backends.''',
+long_description='''Stores objects in multiple storage backends simultaneously
+with dictionary-style access, caching, and object serialization and compression.
+
+Currently supported storage backends are:
+
+Amazon S3 Web Service
+
+Berkeley Source Database
+
+Memory
+
+Filesystem
+
+Firebird
+
+FTP
+
+DBM
+
+Durus
+
+Microsoft SQL Server
+
+MySQL
+
+Oracle
+
+PostgreSQL
+
+SQLite
+
+Subversion
+
+Zope Object Database (ZODB)
+
+Currently supported caching backends are:
+
+Memory
+
+Filesystem
+
+Firebird
+
+memcache
+
+Microsoft SQL Server
+
+MySQL
+
+Oracle
+
+PostgreSQL
+
+SQLite
+
+The use of multiple backends for storage involves
+passing multiple store URIs or instances to
+multishove following the form:
+
+from multishove import MultiShove
+
+<storename> = MultiShove(<store_uri1>, <store_uri2> ..., cache=<cache_uri>)
+
+multishove's access API is the Python dictionary/mapping API:
+
+http://docs.python.org/lib/typesmapping.html
+
+multishove requires the shove package from:
+
+http://cheeseshop.python.org/pypi/shove
+''',
+      author='L. C. Rees',
+      author_email='lcrees@gmail.com',
+      license='BSD',
+      packages = ['multishove', 'multishove.tests'],
+      test_suite='multishove.tests',
+      zip_safe = True,
+      keywords='object storage persistence database shelve',
+      classifiers=['Development Status :: 4 - Beta',
+          'Environment :: Web Environment',
+          'License :: OSI Approved :: BSD License',
+          'Operating System :: OS Independent',
+          'Programming Language :: Python',
+          'Topic :: Database :: Front-Ends'],
+    install_requires = ['shove'])
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.