Commits

Lynn Rees committed 947e136

[svn]

Comments (0)

Files changed (10)

branches/0.3/trunk/README

-Changes in 0.3.1
+Session (flup-compatible), caching, memoizing, and HTTP
+cache control middleware for WSGI. Supports memory,
+filesystem, database, and memcached based backends.
 
-- WsgiMemoize not triggering customized iterables (Roberto de Almeida)
+# Simple memoization example:
+
+from wsgistate.memory import memoize
+
+@memoize()
+def app(environ, start_response):
+    start_response('200 OK', [('Content-Type', 'text/plain')])
+    return ['Hello World!']
+
+if __name__ == '__main__':
+    from wsgiref.simple_server import make_server
+    http = make_server('', 8080, app)
+    http.serve_forever()
+
+# Simple session example:
+
+from wsgistate.memory import session
+
+@session()
+def app(environ, start_response):
+    session = environ['com.saddi.service.session'].session
+    count = session.get('count', 0) + 1
+    session['count'] = count
+    start_response('200 OK', [('Content-Type', 'text/plain')])
+    return ['You have been here %d times!' % count]
+
+if __name__ == '__main__':
+    from wsgiref.simple_server import make_server
+    http = make_server('', 8080, app)
+    http.serve_forever()

branches/0.3/trunk/setup.py

     from distutils.core import setup
 
 setup(name='wsgistate',
-      version='0.3.1',
+      version='0.4',
       description='''WSGI session and caching middleware.''',
       long_description='''Session (flup-compatible), caching, memoizing, and HTTP cache control
 middleware for WSGI. Supports memory, filesystem, database, and memcached based backends.

branches/0.3/trunk/wsgistate/__init__.py

 
 __all__ = ['BaseCache', 'db', 'file', 'memory', 'memcached', 'session', 'simple', 'cache']
 
+def synchronized(func):
+    '''Decorator to lock and unlock a method (Phillip J. Eby).
+
+    @param func Method to decorate
+    '''
+    def wrapper(self, *__args, **__kw):
+        self._lock.acquire()
+        try:
+            return func(self, *__args, **__kw)
+        finally:
+            self._lock.release()
+    wrapper.__name__ = func.__name__
+    wrapper.__dict__ = func.__dict__
+    wrapper.__doc__ = func.__doc__
+    return wrapper
+
 
 class BaseCache(object):
 
         raise NotImplementedError()
 
     def get_many(self, keys):
-        '''Fetch a bunch of keys from the cache.  For certain backends
-        (memcached, pgsql) this can be *much* faster when fetching multiple values.
-
-        Returns a dict mapping each key in keys to its value.  If the given
-        key is missing, it will be missing from the response dict.
+        '''Fetch a bunch of keys from the cache. Returns a dict mapping each
+        key in keys to its value.  If the given key is missing, it will be
+        missing from the response dict.
 
         @param keys Keywords of items in cache.        
         '''

branches/0.3/trunk/wsgistate/db.py

 
     def __init__(self, *a, **kw):
         super(DbCache, self).__init__(self, *a, **kw)
+        # Get table name
+        tablename = kw.get('tablename', 'cache')
         # Bind metadata
         self._metadata = BoundMetaData(a[0])
         # Make cache
-        self._cache = Table('cache', self._metadata,
+        self._cache = Table(tablename, self._metadata,
             Column('id', Integer, primary_key=True, nullable=False, unique=True),
-            Column('cache_key', String(60), nullable=False),
+            Column('key', String(60), nullable=False),
             Column('value', PickleType, nullable=False),
             Column('expires', DateTime, nullable=False))
         # Create cache if it does not exist
         if not self._cache.exists(): self._cache.create()
+        # Maximum number of entries to cull per call if cache is full
+        self._maxcull = kw.get('maxcull', 10)                
         max_entries = kw.get('max_entries', 300)
         try:
             self._max_entries = int(max_entries)
         except (ValueError, TypeError):
             self._max_entries = 300
 
-    def get(self, key, default=None):
+    def __len__(self):
+        return self._cache.count().execute().fetchone()[0]
+
+    def get(self, k, default=None):
         '''Fetch a given key from the cache.  If the key does not exist, return
         default, which itself defaults to None.
 
         @param key Keyword of item in cache.
         @param default Default value (default: None)
         '''
-        row = self._cache.select().execute(cache_key=key).fetchone()
+        row = self._cache.select().execute(key=k).fetchone()
         if row is None: return default
         if row.expires < datetime.now().replace(microsecond=0):
-            self.delete(key)
+            self.delete(k)
             return default
         return row.value
 
-    def set(self, key, val):
+    def set(self, k, v):
         '''Set a value in the cache.
 
         @param key Keyword of item in cache.
-        @param value Value to be inserted in cache.        
+        @param value Value to be inserted in cache.      
         '''
-        
-        timeout = self.timeout
-        # Get count
-        num = self._cache.count().execute().fetchone()[0]
-        if num > self._max_entries: self._cull()
+        if len(self) > self._max_entries: self._cull()
+        timeout, cache = self.timeout, self._cache
         # Get expiration time
         exp = datetime.fromtimestamp(time.time() + timeout).replace(microsecond=0)        
-        try:
-            # Update database if key already present
-            if key in self:
-                self._cache.update(self._cache.c.cache_key==key).execute(value=val, expires=exp)
-            # Insert new key if key not present
-            else:            
-                self._cache.insert().execute(cache_key=key, value=val, expires=exp)
+        #try:
+        # Update database if key already present
+        if k in self:
+            cache.update(cache.c.key==k).execute(value=v, expires=exp)
+        # Insert new key if key not present
+        else:            
+            cache.insert().execute(key=k, value=v, expires=exp)
         # To be threadsafe, updates/inserts are allowed to fail silently
-        except: pass
+        #except: pass
        
-    def delete(self, key):
+    def delete(self, k):
         '''Delete a key from the cache, failing silently.
 
         @param key Keyword of item in cache.
         '''
-        self._cache.delete().execute(cache_key=key) 
+        self._cache.delete(self._cache.c.key==k).execute()
 
     def _cull(self):
-        '''Remove items in cache that have timed out.'''
+        '''Remove items in cache to make more room.'''        
+        cache, maxcull = self._cache, self._maxcull
+        # Remove items that have timed out
         now = datetime.now().replace(microsecond=0)
-        self._cache.delete(self._cache.c.expires < now).execute()
+        cache.delete(cache.c.expires < now).execute()
+        # Remove any items over the maximum allowed number in the cache
+        if len(self) >= self._max_entries:
+            # Upper limit for key query
+            ul = maxcull * 2
+            # Get list of keys
+            keys = [i[0] for i in select([cache.c.key], limit=ul).execute().fetchall()]
+            # Get some keys at random
+            delkeys = list(random.choice(keys) for i in range(maxcull))
+            # Delete keys
+            fkeys = tuple({'key':k} for k in delkeys)
+            cache.delete(cache.c.key.in_(bindparam('key'))).execute(*fkeys)

branches/0.3/trunk/wsgistate/file.py

         if not os.path.exists(self._dir): self._createdir()
         # Remove unneeded methods and attributes
         del self._cache
-        del self._expire_info
 
     def __contains__(self, key):
         '''Tell if a given key is in the cache.'''
         @param key Keyword of item in cache.
         @param default Default value (default: None)
         '''
-        fname = self._key_to_file(key)
         try:
-            f = open(fname, 'rb')
-            exp, now = pickle.load(f), time.time()
+            exp, value = pickle.load(open(self._key_to_file(key), 'rb')) 
             # Remove item if time has expired.
-            if exp < now:
-                f.close()
-                os.remove(fname)
-            else:
-                return pickle.load(f)
+            if exp < time.time():
+                self.delete(key)
+                return default
+            return value
         except (IOError, OSError, EOFError, pickle.PickleError): pass
         return default
 
 
         @param key Keyword of item in cache.
         @param value Value to be inserted in cache.        
-        '''
-        fname = self._key_to_file(key)
+        '''        
+        if len(self.keys()) > self._max_entries: self._cull()
         try:
-            filelist = os.listdir(self._dir)
-        except (IOError, OSError):
-            self._createdir()
-            filelist = list()
-        if len(filelist) > self._max_entries: self._cull()
-        try:
-            f = open(fname, 'wb')
-            now = time.time()
-            pickle.dump(now + self.timeout, f, 2)
-            pickle.dump(value, f, 2)
+            fname = self._key_to_file(key)
+            pickle.dump((time.time() + self.timeout, value), open(fname, 'wb'), 2)
         except (IOError, OSError): pass
 
     def delete(self, key):
             os.remove(self._key_to_file(key))
         except (IOError, OSError): pass
 
-    def _cull(self):
-        '''Remove items in cache that have timed out.'''
-        try:
-            filelist = os.listdir(self._dir)
-        except (IOError, OSError):
-            self._createdir()
-            filelist = list()
-        for fname in filelist:
-            # Remove expired items from cache.
-            try:
-                f = open(fname, 'rb')
-                exp = pickle.load(f)
-                now = time.time()
-                if exp < now:
-                    f.close()
-                    try:
-                        os.remove(os.path.join(self._dir, fname))
-                    except (IOError, OSError): pass
-            except (IOError, OSError, EOFError, pickle.PickleError): pass            
-
+    def keys(self):
+        '''Returns a list of keys in the cache.'''
+        return os.listdir(self._dir)
+    
     def _createdir(self):
         '''Creates the cache directory.'''
         try:

branches/0.3/trunk/wsgistate/memcached.py

         @param default Default value (default: None)
         '''
         val = self._cache.get(key)
-        if val is None:
-            return default
-        else:
-            return val
+        if val is None: return default
+        return val
 
     def set(self, key, value):
         '''Set a value in the cache.

branches/0.3/trunk/wsgistate/memory.py

     import threading
 except ImportError:
     import dummy_threading as threading
+from wsgistate import synchronized
 from wsgistate.simple import SimpleCache
 from wsgistate.cache import WsgiMemoize
 from wsgistate.session import CookieSession, URLSession, SessionCache
         super(MemoryCache, self).__init__(*a, **kw)
         self._lock = threading.Condition()
         
+    @synchronized        
     def get(self, key, default=None):
-        '''Fetch a given key from the cache.  If the key does not exist, return
+        '''Fetch a given key from the cache. If the key does not exist, return
         default, which itself defaults to None.
 
         @param key Keyword of item in cache.
         @param default Default value (default: None)
         '''
-        self._lock.acquire()
-        try:
-            now, exp = time.time(), self._expire_info.get(key)
-            if exp is None:
-                return default
-            # Return default value if item expired
-            elif exp < now:
-                self.delete(key)
-                return default
-            else:
-                return copy.deepcopy(self._cache[key])
-        finally:
-            self._lock.release()
+        return copy.deepcopy(super(MemoryCache, self).get(key))
 
+    @synchronized
     def set(self, key, value):
         '''Set a value in the cache.  
 
         @param key Keyword of item in cache.
         @param value Value to be inserted in cache.        
         '''
-        self._lock.acquire()
-        try:
-            super(MemoryCache, self).set(key, value)
-        finally:
-            self._lock.release()
+        super(MemoryCache, self).set(key, value)
 
+    @synchronized
     def delete(self, key):
         '''Delete a key from the cache, failing silently.
 
         @param key Keyword of item in cache.
         '''
-        self._lock.acquire()
-        try:
-            super(MemoryCache, self).delete(key)
-        finally:
-            self._lock.release()
+        super(MemoryCache, self).delete(key)

branches/0.3/trunk/wsgistate/session.py

     import threading
 except ImportError:
     import dummy_threading as threading
+from wsgistate import synchronized
 
 __all__ = ['SessionCache', 'SessionManager', 'CookieSession', 'URLSession',
      'session', 'urlsession']
     length = 64
 
     def __init__(self, cache, **kw):
-        self.lock = threading.Condition()
+        self._lock = threading.Condition()
         self.checkedout, self._closed, self.cache = dict(), False, cache
         # Sets if session id is random on every access or not
         self._random = kw.get('random', False)
 
     # Public interface.
 
+    @synchronized
     def create(self):
         '''Create a new session with a unique identifier.
         
         The newly-created session should eventually be released by
         a call to checkin().            
         '''
-        self.lock.acquire()
-        try:
-            sid, sess = self.newid(), dict()
-            self.cache.set(sid, sess)            
-            self.checkedout[sid] = sess
-            return sid, sess
-        finally:
-            self.lock.release()
+        sid, sess = self.newid(), dict()
+        self.cache.set(sid, sess)            
+        self.checkedout[sid] = sess
+        return sid, sess
 
+    @synchronized
     def checkout(self, sid):
         '''Checks out a session for use. Returns the session if it exists,
         otherwise returns None. If this call succeeds, the session
 
         @param sid Session id        
         '''
-        self.lock.acquire()
-        try:
-            # If we know it's already checked out, block.
-            while sid in self.checkedout: self.lock.wait()
-            sess = self.cache.get(sid)
-            if sess is not None:
-                # Randomize session id if set and remove old session id
-                if self._random:
-                    self.cache.delete(sid)
-                    sid = self.newid()
-                # Put in checkout
-                self.checkedout[sid] = sess
-                return sid, sess
-            return None, None
-        finally:
-            self.lock.release()
+        # If we know it's already checked out, block.
+        while sid in self.checkedout: self._lock.wait()
+        sess = self.cache.get(sid)
+        if sess is not None:
+            # Randomize session id if set and remove old session id
+            if self._random:
+                self.cache.delete(sid)
+                sid = self.newid()
+            # Put in checkout
+            self.checkedout[sid] = sess
+            return sid, sess
+        return None, None
 
+    @synchronized
     def checkin(self, sid, sess):
         '''Returns the session for use by other threads/processes.
 
         @param sid Session id
         @param session Session dictionary
         '''
-        self.lock.acquire()
-        try:
-            del self.checkedout[sid]
-            self.cache.set(sid, sess)
-            self.lock.notify()
-        finally:            
-            self.lock.release()
+        del self.checkedout[sid]
+        self.cache.set(sid, sess)
+        self._lock.notify()
 
+    @synchronized
     def shutdown(self):
-        '''Clean up outstanding sessions.'''
-        self.lock.acquire()
-        try:
-            if not self._closed:
-                # Save or delete any sessions that are still out there.
-                for sid, sess in self.checkedout.iteritems():
-                    self.cache.set(sid, sess)
-                self.checkedout.clear()
-                self.cache._cull()                
-                self._closed = True
-        finally:
-            self.lock.release()
+        '''Clean up outstanding sessions.'''        
+        if not self._closed:
+            # Save or delete any sessions that are still out there.
+            for sid, sess in self.checkedout.iteritems():
+                self.cache.set(sid, sess)
+            self.checkedout.clear()
+            self.cache._cull()                
+            self._closed = True
 
     # Utilities
 

branches/0.3/trunk/wsgistate/simple.py

 '''Single-process in-memory cache backend.'''
 
 import time
+import random
 from wsgistate import BaseCache
 from wsgistate.cache import WsgiMemoize
 from wsgistate.session import CookieSession, URLSession, SessionCache
     
     def __init__(self, *a, **kw):
         super(SimpleCache, self).__init__(*a, **kw)
-        self._cache, self._expire_info = dict(), dict()
+        # Get random seed
+        random.seed()        
+        self._cache = dict()
+        # Set max entries
         max_entries = kw.get('max_entries', 300)
         try:
             self._max_entries = int(max_entries)
         except (ValueError, TypeError):
             self._max_entries = 300
+        # Set maximum number of items to cull if over max
+        self._maxcull = kw.get('maxcull', 10)
 
     def get(self, key, default=None):
         '''Fetch a given key from the cache.  If the key does not exist, return
         @param key Keyword of item in cache.
         @param default Default value (default: None)
         '''
-        now, exp = time.time(), self._expire_info.get(key)
-        if exp is None:
-            return default
+        values = self._cache.get(key)
+        if values is None: return default
         # Delete if item timed out and return default.
-        elif exp < now:
+        if values[0] < time.time():
             self.delete(key)
             return default
-        else:
-            return self._cache[key]
+        return values[1] 
 
     def set(self, key, value):
         '''Set a value in the cache.
         '''
         # Cull timed out values if over max # of entries
         if len(self._cache) >= self._max_entries: self._cull()
-        self._cache[key] = value
-        # Set timeout
-        self._expire_info[key] = time.time() + self.timeout
+        # Set value and timeout in cache
+        self._cache[key] = (time.time() + self.timeout, value)
 
     def delete(self, key):
         '''Delete a key from the cache, failing silently.
         try:
             del self._cache[key]
         except KeyError: pass
-        try:
-            del self._expire_info[key]
-        except KeyError: pass
+
+    def keys(self):
+        '''Returns a list of keys in the cache.'''
+        return self._cache.keys()
 
     def _cull(self):
-        '''Remove items in cache that have timed out.'''
-        now = time.time()
-        for key, exp in self._expire_info.iteritems():
-            if exp < now: self.delete(key)
+        '''Remove items in cache to make room.'''        
+        num, maxcull = 0, self._maxcull
+        # Cull number of items allowed (set by self._maxcull)
+        for key in self.keys():
+            # Remove only maximum # of items allowed by maxcull
+            if num <= maxcull:
+                # Remove items if expired
+                if self.get(key) is None: num += 1
+            else: break
+        # Remove any additional items up to max # of items allowed by maxcull
+        while len(self.keys()) >= self._max_entries and num <= maxcull:
+            # Cull remainder of allowed quota at random
+            self.delete(random.choice(self.keys()))
+            num += 1

branches/0.3/trunk/wsgistate/tests/test_wsgistate.py

         del cache['test']
         self.assertEqual(cache.get('test'), None)
 
-    def test_sc_set_get(self):
+    def test_sc_set_delete(self):
         '''Tests delete on SimpleCache.'''
         cache = simple.SimpleCache()
         cache.set('test', 'test')
         del cache['test']
         self.assertEqual(cache.get('test'), None)
 
-    def test_mc_set_get(self):
+    def test_mc_set_delete(self):
         '''Tests delete on MemoryCache.'''
         cache = memory.MemoryCache()
         cache.set('test', 'test')
         del cache['test']
         self.assertEqual(cache.get('test'), None)
 
-    def test_fc_set_get(self):
+    def test_fc_set_delete(self):
         '''Tests delete on FileCache.'''
         cache = file.FileCache('test_wsgistate')
         cache.set('test', 'test')
         del cache['test']
         self.assertEqual(cache.get('test'), None)
 
-    def test_db_set_get(self):
+    def test_db_set_delete(self):
         '''Tests delete on DbCache.'''
         cache = db.DbCache('sqlite:///:memory:')
         cache.set('test', 'test')
         del cache['test']
         self.assertEqual(cache.get('test'), None)
 
-    def test_mcd_set_get(self):
+    def test_mcd_set_delete(self):
         '''Tests delete on MemCache.'''
         cache = memcached.MemCached('localhost')
         cache.set('test', 'test')
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.