Commits

Eric Larson committed e9e5cfb

Summary: Removing the file cache from httpcache. The dict cache is threadsafe enough
Author: eric@ionrock.org

Comments (0)

Files changed (2)

httpcache/cache.py

 import time
 
 from contextlib import contextmanager
-from threading import Lock
 from cPickle import dump, load
 from hashlib import md5
 
     def delete(self, key):
         if key in self.data:
             self.data.pop(key)
-
-
-# Cache filename construction (original borrowed from Venus
-# http://intertwingly.net/code/venus/)
-re_url_scheme = re.compile(r'^\w+://')
-re_slash = re.compile(r'[?/:|]+')
-
-
-def safename(filename):
-    """Return a filename suitable for the cache.
-
-    Strips dangerous and common characters to create a filename we
-    can use to store the cache in.
-    """
-
-    try:
-        if re_url_scheme.match(filename):
-            if isinstance(filename, str):
-                filename = filename.decode('utf-8')
-                filename = filename.encode('idna')
-            else:
-                filename = filename.encode('idna')
-    except UnicodeError:
-        pass
-    if isinstance(filename, unicode):
-        filename = filename.encode('utf-8')
-    filemd5 = md5(filename).hexdigest()
-    filename = re_url_scheme.sub("", filename)
-    filename = re_slash.sub(",", filename)
-
-    # limit length of filename
-    if len(filename) > 200:
-        filename = filename[:200]
-    return ",".join((filename, filemd5))
-
-
-@contextmanager
-def filelock(key, mode, timeout=None, interval=.1):
-    """
-    A simple context manager that creates a temporary file for
-    locking a specific cache entry.
-
-    This was inspired pretty directly by:
-      http://amix.dk/blog/post/19531
-    """
-    lockfile = '%s.lock' % key
-    if timeout:
-        iterations = int(timeout / interval)
-    locked = os.path.exists(lockfile)
-    while locked:
-        error = FileCacheLockedException(lockfile, key)
-        if timeout:
-            # it is already locked, but we have a timeout if we
-            # want to try and block until it is available
-            time.sleep(interval)
-            iterations -= 1
-            locked = os.path.exists(lockfile)
-            if not iterations:
-                raise error
-        else:
-            # it is locked and we don't have a timeout, so raise
-            # the error
-            error
-
-    with open(lockfile, 'w+') as file_lock:
-        file_lock.write('1')
-        with open(key, mode) as opened_file:
-            yield opened_file
-    os.remove(lockfile)
-
-
-class FileCacheLockedException(Exception):
-
-    def __init__(self, lockfile, fname):
-        self.lockfile, self.fname = lockfile, fname
-        msg = '"%s" is locked. Try removing "%s"' % (lockfile, fname)
-        super(FileCacheLockedException, self).__init__(msg)
-
-
-class FileCache(BaseCache):
-    """
-    A simple threadsafe file based cache.
-
-    The file cache is a port of httplib2's directory cache. The only
-    difference is that it uses a lock when writing the file and
-    pickles the value. The pickling is used b/c we are using
-    requests.Response objects.
-    """
-
-    def __init__(self, cache, worker=None):
-        self.cache = cache
-        self.safe = safename
-        if not os.path.exists(cache):
-            os.makedirs(self.cache)
-        self.lock = Lock()
-
-    def get(self, key):
-        retval = None
-        cache_full_path = os.path.join(self.cache, safename(key))
-        try:
-            with open(cache_full_path, "rb") as f:
-                retval = load(f)
-        except (IOError, EOFError):
-            pass
-        return retval
-
-    def set(self, key, value):
-        cache_full_path = os.path.join(self.cache, safename(key))
-        with filelock(cache_full_path, "wb", timeout=1) as f:
-            dump(value, f)
-
-    def delete(self, key):
-        cache_full_path = os.path.join(self.cache, safename(key))
-        if os.path.exists(cache_full_path):
-            os.remove(cache_full_path)

httpcache/tests/test_cache.py

-"""
-Tests for our caches
-"""
-import os
-
-from threading import Thread
-# from multiprocessing import Process
-from httpcache.cache import FileCache, safename, filelock
-from httpcache.cache import FileCacheLockedException
-
-
-class RandomWriter(Thread):
-    def __init__(self, cache):
-        super(RandomWriter, self).__init__()
-        self.key = 'threading_test'
-        self.cache = cache
-
-    def run(self):
-        for x in range(0, 5):
-            value = x * 1000
-            self.cache.set(self.key, value)
-            assert self.cache.get(self.key) == value
-
-
-class TestFileCache(object):
-
-    cache_dir = 'test_file_cache'
-
-    def test_set_get_delete(self):
-        fc = FileCache(self.cache_dir)
-        fc.set('foo', 'bar')
-        assert fc.get('foo') == 'bar'
-        fc.delete('foo')
-        assert fc.get('foo') == None
-
-    def test_setting_with_multiple_threads(self):
-        fc = FileCache(self.cache_dir)
-        w1 = RandomWriter(fc)
-        w2 = RandomWriter(fc)
-        w1.start()
-        w2.start()
-        w1.join()
-        w2.join()
-        assert fc.get(w1.key) == 4000
-
-    def test_locked_raises_exception(self):
-        key, value = 'locked', {'foo': 'bar'}
-        fc = FileCache(self.cache_dir)
-        lockfile = os.path.join(self.cache_dir,
-                                '%s.lock' % safename(key))
-        with open(lockfile, 'w+') as lock:
-            lock.write('1')
-        assert os.path.exists(lockfile)
-        try:
-            fc.set(key, value)
-            assert False
-        except FileCacheLockedException:
-            assert True
-        os.remove(lockfile)
-        fc.set(key, value)
-        assert fc.get(key) == value
-        assert not os.path.exists(lockfile)
-
-    def test_filelock_timeout(self):
-        fc = FileCache(self.cache_dir)
-        fname = safename('locked')
-        lockfile = '%s.lock' % fname
-        with open(lockfile, 'w+') as lock:
-            lock.write('1')
-            try:
-                with filelock(fname, 'w+', .5):
-                    assert False
-            except FileCacheLockedException:
-                assert True
-        os.remove(lockfile)
-        assert not os.path.exists(lockfile)