Commits

Alan Justino committed 60b6f33

Added: filecache.FileSystemCachedS3BotoStorage

Caches S3Boto files on FileSystemStorage to speedup
next uses of read() and save() over it. Usefull for
thumbnail generators mainly

  • Participants
  • Parent commits 757c908

Comments (0)

Files changed (2)

File storages/backends/filecache.py

+import os
+
+from django.conf import settings
+from django.core.files.storage import Storage
+from storages.utils import import_attribute
+
+import logging
+logger = logging.getLogger(__name__)
+
+
+def load_backend(backend=None, factory=import_attribute):
+    if backend is None:  # pragma: no cover
+        raise ImproperlyConfigured("You should define a backend.")
+    if not isinstance(backend, basestring):
+        raise ImproperlyConfigured("You should specify the backend "
+                                   "as dotted import paths, "
+                                   "not instances or classes")
+    return factory(backend)
+
+
+class FileCacheMixin(object):
+    """Storage mixing to keep a copy of accessed files"""
+
+    def __init__(self, cache_storage=None, master_storage=None):
+        self.cache_storage = load_backend(cache_storage)()
+        self.master_storage = load_backend(master_storage)()
+
+    # Read operations must be done with cache_storage, after regenerating from master_storage
+    # Write operations must be done with master_storage, after cleaning cache_storage.
+    # Other operations must be done with master_storage.
+
+    def _open(self, name, mode='rb'):
+        if not self.cache_storage.exists(name):
+            self.transfer(name, origin=self.master_storage, destiny=self.cache_storage)
+        return self.cache_storage.open(name)
+
+    def _save(self, name, content):
+        master_result = self.master_storage.save(name, content)
+        self.cache_storage.delete(name)
+        return master_result
+
+    def _transfer(self, name, origin, destiny):
+        """
+        Transfers the file with the given name from the local to the remote
+        storage backend.
+
+        :param name: The name of the file to transfer
+        :param origin: Storage instance that have the file
+        :param destiny: Storage instance that will receive the file
+        :returns: `True` when the transfer succeeded, `False` if not.
+        :rtype: bool
+        """
+        try:
+            destiny.save(name, origin.open(name))
+            return True
+        except Exception, e:
+            logger.error("Unable to transfer '%s' from storage '%s' to '%s." %
+                (name, origin, destiny)
+            )
+            logger.exception(e)
+            raise e #return False
+
+    ## Storage standard interface:
+
+    def delete(self, name):
+        """
+        Deletes the specified file from the storage system.
+
+        :param name: file name
+        :type name: str
+        """
+        self.cache_storage.delete(name)
+        return self.master_storage.delete(name)
+
+    def exists(self, name):
+        """
+        Returns ``True`` if a file referened by the given name already exists
+        in the storage system, or False if the name is available for a new
+        file.
+
+        :param name: file name
+        :type name: str
+        :rtype: bool
+        """
+        return self.master_storage.exists(name)
+
+    def listdir(self, name):
+        """
+        Lists the contents of the specified path, returning a 2-tuple of lists;
+        the first item being directories, the second item being files.
+
+        :param name: file name
+        :type name: str
+        :rtype: tuple
+        """
+        return self.master_storage.listdir(name)
+
+    def size(self, name):
+        """
+        Returns the total size, in bytes, of the file specified by name.
+
+        :param name: file name
+        :type name: str
+        :rtype: int
+        """
+        return self.master_storage.size(name)
+
+    def url(self, name):
+        """
+        Returns an absolute URL where the file's contents can be accessed
+        directly by a Web browser.
+
+        :param name: file name
+        :type name: str
+        :rtype: str
+        """
+        return self.master_storage.url(name)
+
+    def accessed_time(self, name):
+        """
+        Returns the last accessed time (as datetime object) of the file
+        specified by name.
+
+        :param name: file name
+        :type name: str
+        :rtype: :class:`~python:datetime.datetime`
+        """
+        return self.master_storage.accessed_time(name)
+
+    def created_time(self, name):
+        """
+        Returns the creation time (as datetime object) of the file
+        specified by name.
+
+        :param name: file name
+        :type name: str
+        :rtype: :class:`~python:datetime.datetime`
+        """
+        return self.master_storage.created_time(name)
+
+    def modified_time(self, name):
+        """
+        Returns the last modified time (as datetime object) of the file
+        specified by name.
+
+        :param name: file name
+        :type name: str
+        :rtype: :class:`~python:datetime.datetime`
+        """
+        return self.master_storage.modified_time(name)
+
+    ### Bits derived from github:jezdez/django-queued-storage
+    def get_storage(self, name):
+        """
+        Returns the storage backend instance responsible for the file
+        with the given name (either local or remote). This method is
+        used in most of the storage API methods.
+
+        :param name: file name
+        :type name: str
+        :rtype: :class:`~django:django.core.files.storage.Storage`
+        """
+        if not name.endswith('/') and self.cache_storage.exists(name):
+            return self.cache_storage
+        else:
+            return self.master_storage
+
+    def using_cache(self, name):
+        """
+        Determines for the file with the given name whether
+        the cache storage is current used.
+
+        :param name: file name
+        :type name: str
+        :rtype: bool
+        """
+        return self.get_storage(name) is self.cache
+
+    def using_master(self, name):
+        """
+        Determines for the file with the given name whether
+        the master storage is current used.
+
+        :param name: file name
+        :type name: str
+        :rtype: bool
+        """
+        return self.get_storage(name) is self.master
+
+    ### Optimizations
+    """
+    Comes from http://www.djangosnippets.org/snippets/976/
+    (even if it already exists in S3Storage for ages)
+
+    See also Django #4339, which might add this functionality to core.
+    """
+    def get_available_name(self, name):
+        """
+        Returns a filename that's free on the target storage system, and
+        available for new content to be written to.
+        """
+        if self.exists(name):
+            self.delete(name)
+        return name
+
+
+class FileSystemCachedS3BotoStorage(FileCacheMixin, Storage):
+    """Caches S3 files on local filesystem.
+    
+    Use S3BotoStorage to serve URLs and so, and caches Django-consumed files
+    on local filesystem.
+
+    It mitigates the following problem:
+    - thumbnail creation from CDN-stored files gets faster by holding a local
+    copy of the files needed to create the thumbnails, but only needed ones
+    """
+    def __init__(self, *args, **kwargs):
+        FileCacheMixin.__init__(self,
+            cache_storage='django.core.files.storage.FileSystemStorage',
+            master_storage='storages.backends.s3boto.S3BotoStorage'
+        )

File storages/utils.py

+from django.core.exceptions import ImproperlyConfigured
+from django.utils.importlib import import_module
+
+
+def import_attribute(import_path=None):
+    if import_path is None:
+        raise ImproperlyConfigured("No import path was given.")
+    try:
+        dot = import_path.rindex('.')
+    except ValueError:
+        raise ImproperlyConfigured("%s isn't a module." % import_path)
+    module, classname = import_path[:dot], import_path[dot + 1:]
+    try:
+        mod = import_module(module)
+    except ImportError, e:
+        raise ImproperlyConfigured('Error importing module %s: "%s"' %
+                                   (module, e))
+    try:
+        return getattr(mod, classname)
+    except AttributeError:
+        raise ImproperlyConfigured(
+            'Module "%s" does not define a "%s" class.' % (module, classname))