Source

django-storages / storages / backends / filecache.py

import os

from django.conf import settings
from django.core.files.storage import Storage
from storages.utils import import_attribute

import time
import logging
logger = logging.getLogger(__name__)


def load_backend(backend=None, factory=import_attribute):
    if backend is None:  # pragma: no cover
        raise ImproperlyConfigured("You should define a backend.")
    if not isinstance(backend, basestring):
        raise ImproperlyConfigured("You should specify the backend "
                                   "as dotted import paths, "
                                   "not instances or classes")
    return factory(backend)


class FileCacheMixin(object):
    """Storage mixing to keep a copy of accessed files"""

    def __init__(self, cache_storage=None, master_storage=None):
        self.cache_storage = load_backend(cache_storage)()
        self.master_storage = load_backend(master_storage)()

    # Read operations must be done with cache_storage, after regenerating from master_storage
    # Write operations must be done with master_storage, after cleaning cache_storage.
    # Other operations must be done with master_storage.

    def _open(self, name, mode='rb'):
        if not self.cache_storage.exists(name):
            self._transfer(name, origin=self.master_storage, destiny=self.cache_storage)
        return self.cache_storage.open(name)

    def _save(self, name, content):
        master_result = self.master_storage.save(name, content)
        self.cache_storage.delete(name)
        return master_result

    def _transfer(self, name, origin, destiny):
        """
        Transfers the file with the given name from the local to the remote
        storage backend.

        :param name: The name of the file to transfer
        :param origin: Storage instance that have the file
        :param destiny: Storage instance that will receive the file
        :returns: `True` when the transfer succeeded, `False` if not.
        :rtype: bool
        """
        logger.info(u'Started to transfer "%s" from master to cache storage' % name)
        try:
            destiny.save(name, origin.open(name))
            try: # If destiny have 'path', keep mtime
                mtime = origin.modified_time(name)
                timestamp = time.mktime(mtime.timetuple())
                os.utime(destiny.path(name), (timestamp, timestamp))
            except (NotImplementedError, IOError):
                pass

            return True
        except Exception, e:
            logger.error("Unable to transfer '%s' from storage '%s' to '%s." %
                (name, origin, destiny)
            )
            logger.exception(e)
            raise e #return False

    ## Storage standard interface:

    def delete(self, name):
        """
        Deletes the specified file from the storage system.

        :param name: file name
        :type name: str
        """
        self.cache_storage.delete(name)
        return self.master_storage.delete(name)

    def exists(self, name):
        """
        Returns ``True`` if a file referened by the given name already exists
        in the storage system, or False if the name is available for a new
        file.

        :param name: file name
        :type name: str
        :rtype: bool
        """
        return self.master_storage.exists(name)

    def listdir(self, name):
        """
        Lists the contents of the specified path, returning a 2-tuple of lists;
        the first item being directories, the second item being files.

        :param name: file name
        :type name: str
        :rtype: tuple
        """
        return self.master_storage.listdir(name)

    def size(self, name):
        """
        Returns the total size, in bytes, of the file specified by name.

        :param name: file name
        :type name: str
        :rtype: int
        """
        return self.master_storage.size(name)

    def url(self, name):
        """
        Returns an absolute URL where the file's contents can be accessed
        directly by a Web browser.

        :param name: file name
        :type name: str
        :rtype: str
        """
        return self.master_storage.url(name)

    def accessed_time(self, name):
        """
        Returns the last accessed time (as datetime object) of the file
        specified by name.

        :param name: file name
        :type name: str
        :rtype: :class:`~python:datetime.datetime`
        """
        return self.master_storage.accessed_time(name)

    def created_time(self, name):
        """
        Returns the creation time (as datetime object) of the file
        specified by name.

        :param name: file name
        :type name: str
        :rtype: :class:`~python:datetime.datetime`
        """
        return self.master_storage.created_time(name)

    def modified_time(self, name):
        """
        Returns the last modified time (as datetime object) of the file
        specified by name.

        :param name: file name
        :type name: str
        :rtype: :class:`~python:datetime.datetime`
        """
        return self.master_storage.modified_time(name)

    ### Bits derived from github:jezdez/django-queued-storage
    def _get_storage(self, name):
        """
        Returns the storage backend instance responsible for the file
        with the given name (either local or remote). This method is
        used in most of the storage API methods.

        :param name: file name
        :type name: str
        :rtype: :class:`~django:django.core.files.storage.Storage`
        """
        if not name.endswith('/') and self.cache_storage.exists(name):
            return self.cache_storage
        else:
            return self.master_storage

    def using_cache(self, name):
        """
        Determines for the file with the given name whether
        the cache storage is current used.

        :param name: file name
        :type name: str
        :rtype: bool
        """
        return self._get_storage(name) is self.cache_storage

    def using_master(self, name):
        """
        Determines for the file with the given name whether
        the master storage is current used.

        :param name: file name
        :type name: str
        :rtype: bool
        """
        return self._get_storage(name) is self.master_storage

    ### Optimizations
    """
    Comes from http://www.djangosnippets.org/snippets/976/
    (even if it already exists in S3Storage for ages)

    See also Django #4339, which might add this functionality to core.
    """
    def get_available_name(self, name):
        """
        Returns a filename that's free on the target storage system, and
        available for new content to be written to.
        """
        if self.exists(name):
            self.delete(name)
        return name

    def path(self, name):
        """
        Returns a local filesystem path where the file can be retrieved.

        If cache_storage is path"able", tranfers from master_storage and serves
        from cache_storage.

        :param name: file name
        :type name: str
        :rtype: str
        """
        raise NotImplementedError()
        #try:
        #    return self._get_storage(name).path(name)
        #except NotImplementedError, e:
        #    if self.using_cache(name):
        #        raise
        #    else: # Maybe is worth to try with cache_storage if master failed
        #        self._transfer(name, origin=self.master_storage, destiny=self.cache_storage)
        #        return self.cache_storage.path(name)


class FileSystemCachedS3BotoStorage(FileCacheMixin, Storage):
    """Caches S3 files on local filesystem.
    
    Use S3BotoStorage to serve URLs and so, and caches Django-consumed files
    on local filesystem.

    It mitigates the following problem:
    - thumbnail creation from CDN-stored files gets faster by holding a local
    copy of the files needed to create the thumbnails, but only needed ones
    """
    def __init__(self, *args, **kwargs):
        FileCacheMixin.__init__(self,
            cache_storage='django.core.files.storage.FileSystemStorage',
            master_storage='storages.backends.s3boto.S3BotoStorage'
        )