Tobias McNulty avatar Tobias McNulty committed 65b3658 Merge

merge in upstream changes

Comments (0)

Files changed (16)

 *.swp
 *.tmp
 *~
+.tox/
 _build/
 build/
 dist/*
 Install
 =======
 
-See http://code.larlet.fr/doc/#django-storage for detailed explanations.
 
-TODO
-====
-
-    * Use chunks for S3Storage as in original FileSystemStorage
-    * Invite people who work on storages to add them to the repository
-    * Add more documentation
+See http://django-storages.readthedocs.org/

docs/backends/rackspace-cloudfiles.rst

     CLOUDFILES_CONTAINER = 'ContainerName'
     DEFAULT_FILE_STORAGE = 'backends.mosso.CloudFilesStorage'
 
+    # Optional - use SSL
+    CLOUDFILES_SSL = True
+
 Optionally, you can implement the following custom upload_to in your models.py file. This will upload the file using the file name only to Cloud Files (e.g. 'myfile.jpg'). If you supply a string (e.g. upload_to='some/path'), your file name will include the path (e.g. 'some/path/myfile.jpg')::
 
     from backends.mosso import cloudfiles_upload_to

examples/libcloud_project/manage.py

+#!/usr/bin/env python
+from django.core.management import execute_manager
+import imp
+try:
+    imp.find_module('settings') # Assumed to be in the same directory.
+except ImportError:
+    import sys
+    sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n" % __file__)
+    sys.exit(1)
+
+import settings
+
+if __name__ == "__main__":
+    execute_manager(settings)

examples/libcloud_project/settings.py

+# Django settings for libcloud_project project.
+
+DEBUG = True
+TEMPLATE_DEBUG = DEBUG
+
+ADMINS = (
+    # ('Your Name', 'your_email@example.com'),
+)
+
+MANAGERS = ADMINS
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
+        'NAME': 'libcloud_project.sqllite',                      # Or path to database file if using sqlite3.
+        'USER': '',                      # Not used with sqlite3.
+        'PASSWORD': '',                  # Not used with sqlite3.
+        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
+        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
+    }
+}
+
+# Local time zone for this installation. Choices can be found here:
+# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
+# although not all choices may be available on all operating systems.
+# On Unix systems, a value of None will cause Django to use the same
+# timezone as the operating system.
+# If running in a Windows environment this must be set to the same as your
+# system time zone.
+TIME_ZONE = 'America/Chicago'
+
+# Language code for this installation. All choices can be found here:
+# http://www.i18nguy.com/unicode/language-identifiers.html
+LANGUAGE_CODE = 'en-us'
+
+SITE_ID = 1
+
+# If you set this to False, Django will make some optimizations so as not
+# to load the internationalization machinery.
+USE_I18N = True
+
+# If you set this to False, Django will not format dates, numbers and
+# calendars according to the current locale
+USE_L10N = True
+
+# Absolute filesystem path to the directory that will hold user-uploaded files.
+# Example: "/home/media/media.lawrence.com/media/"
+MEDIA_ROOT = ''
+
+# URL that handles the media served from MEDIA_ROOT. Make sure to use a
+# trailing slash.
+# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
+MEDIA_URL = ''
+
+# Absolute path to the directory static files should be collected to.
+# Don't put anything in this directory yourself; store your static files
+# in apps' "static/" subdirectories and in STATICFILES_DIRS.
+# Example: "/home/media/media.lawrence.com/static/"
+STATIC_ROOT = ''
+
+# URL prefix for static files.
+# Example: "http://media.lawrence.com/static/"
+STATIC_URL = '/static/'
+
+# URL prefix for admin static files -- CSS, JavaScript and images.
+# Make sure to use a trailing slash.
+# Examples: "http://foo.com/static/admin/", "/static/admin/".
+ADMIN_MEDIA_PREFIX = '/static/admin/'
+
+# Additional locations of static files
+STATICFILES_DIRS = (
+    # Put strings here, like "/home/html/static" or "C:/www/django/static".
+    # Always use forward slashes, even on Windows.
+    # Don't forget to use absolute paths, not relative paths.
+)
+
+# List of finder classes that know how to find static files in
+# various locations.
+STATICFILES_FINDERS = (
+    'django.contrib.staticfiles.finders.FileSystemFinder',
+    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
+)
+
+# Make this unique, and don't share it with anybody.
+SECRET_KEY = 'tdzq9m9k-u*k=furpki(@wejb&^2!ea4*z1^t9waj&$)(+$4(h'
+
+# List of callables that know how to import templates from various sources.
+TEMPLATE_LOADERS = (
+    'django.template.loaders.filesystem.Loader',
+    'django.template.loaders.app_directories.Loader',
+#     'django.template.loaders.eggs.Loader',
+)
+
+MIDDLEWARE_CLASSES = (
+    'django.middleware.common.CommonMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+)
+
+ROOT_URLCONF = 'libcloud_project.urls'
+
+TEMPLATE_DIRS = (
+    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
+    # Always use forward slashes, even on Windows.
+    # Don't forget to use absolute paths, not relative paths.
+)
+
+INSTALLED_APPS = (
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.sites',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    # Uncomment the next line to enable the admin:
+    # 'django.contrib.admin',
+    # Uncomment the next line to enable admin documentation:
+    # 'django.contrib.admindocs',
+)
+
+# A sample logging configuration. The only tangible logging
+# performed by this configuration is to send an email to
+# the site admins on every HTTP 500 error.
+# See http://docs.djangoproject.com/en/dev/topics/logging for
+# more details on how to customize your logging configuration.
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'handlers': {
+        'mail_admins': {
+            'level': 'ERROR',
+            'class': 'django.utils.log.AdminEmailHandler'
+        }
+    },
+    'loggers': {
+        'django.request': {
+            'handlers': ['mail_admins'],
+            'level': 'ERROR',
+            'propagate': True,
+        },
+    }
+}
+
+#
+# Specific project configuration
+#
+from libcloud.storage.types import Provider
+LIBCLOUD_PROVIDERS = {
+    'test_google_storage': {
+        'type': Provider.GOOGLE_STORAGE,
+        'user': '<google apiv1 user (20 char)>',
+        'key': '<google apiv1 key>',
+        'bucket': '<bucket name>'
+    }
+}
+
+DEFAULT_FILE_STORAGE = 'backends.backends.LibCloudStorage'

examples/libcloud_project/test_storage.py

+import sys, os
+PROJECT_PATH = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(PROJECT_PATH)
+
+from django.core.management import setup_environ
+import settings
+setup_environ(settings)
+
+from storages.backends.apache_libcloud import LibCloudStorage
+
+# test_google_storage is a key in settings LIBCLOUD_PROVIDERS dict
+store = LibCloudStorage('test_google_storage')
+# store is your django storage object that will use google storage
+# bucket specified in configuration

examples/libcloud_project/urls.py

+from django.conf.urls.defaults import patterns, include, url
+
+# Uncomment the next two lines to enable the admin:
+# from django.contrib import admin
+# admin.autodiscover()
+
+urlpatterns = patterns('',
+    # Examples:
+    # url(r'^$', 'libcloud_project.views.home', name='home'),
+    # url(r'^libcloud_project/', include('libcloud_project.foo.urls')),
+
+    # Uncomment the admin/doc line below to enable admin documentation:
+    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
+
+    # Uncomment the next line to enable the admin:
+    # url(r'^admin/', include(admin.site.urls)),
+)
         'Programming Language :: Python',
         'Framework :: Django',
     ],
-    py_modules = ['S3'],
+    test_suite='tests.main',
     zip_safe = False,
 )

storages/backends/apache_libcloud.py

+# Django storage using libcloud providers
+# Aymeric Barantal (mric at chamal.fr) 2011
+#
+import os
+
+from django.conf import settings
+from django.core.files.storage import Storage
+from django.core.files.base import File
+from django.core.exceptions import ImproperlyConfigured
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+
+try:
+    from libcloud.storage.providers import get_driver
+    from libcloud.storage.types import ObjectDoesNotExistError
+except ImportError:
+    raise ImproperlyConfigured("Could not load libcloud")
+
+
+class LibCloudStorage(Storage):
+    """Django storage derived class using apache libcloud to operate
+    on supported providers"""
+    def __init__(self, provider_name, option=None):
+        self.provider = settings.LIBCLOUD_PROVIDERS.get(provider_name)
+        if not self.provider:
+            raise ImproperlyConfigured(
+                'LIBCLOUD_PROVIDERS %s not define or invalid' % provider_name)
+        try:
+            Driver = get_driver(self.provider['type'])
+            self.driver = Driver(
+                self.provider['user'],
+                self.provider['key'],
+                )
+        except Exception, e:
+            raise ImproperlyConfigured(
+                "Unable to create libcloud driver type %s" % \
+                (self.provider.get('type'), e))
+        self.bucket = self.provider['bucket']   # Limit to one container
+
+    def _get_bucket(self):
+        """Helper to get bucket object (libcloud container)"""
+        return self.driver.get_container(self.bucket)
+
+    def _clean_name(self, name):
+        """Clean name (windows directories)"""
+        return os.path.normpath(name).replace('\\', '/')
+
+    def _get_object(self, name):
+        """Get object by its name. Return None if object not found"""
+        clean_name = self._clean_name(name)
+        try:
+            return self.driver.get_object(self.bucket, clean_name)
+        except ObjectDoesNotExistError, e:
+            return None
+
+    def delete(self, name):
+        """Delete objet on remote"""
+        obj = self._get_object(name)
+        if obj:
+            return self.driver.delete_object(obj)
+        else:
+            raise Exception('Object to delete does not exists')
+
+    def exists(self, name):
+        obj = self._get_object(name)
+        return True if obj else False
+
+    def listdir(self, path='/'):
+        """Lists the contents of the specified path,
+        returning a 2-tuple of lists; the first item being
+        directories, the second item being files.
+        """
+        container = self._get_bucket()
+        objects = self.driver.list_container_objects(container)
+        path = self._clean_name(path)
+        if not path.endswith('/'):
+            path = "%s/" % path
+        files = []
+        dirs = []
+        # TOFIX: better algorithm to filter correctly
+        # (and not depend on google-storage empty folder naming)
+        for o in objects:
+            if path == '/':
+                if o.name.count('/') == 0:
+                    files.append(o.name)
+                elif o.name.count('/') == 1:
+                    dir_name = o.name[:o.name.index('/')]
+                    if not dir_name in dirs:
+                        dirs.append(dir_name)
+            elif o.name.startswith(path):
+                if o.name.count('/') <= path.count('/'):
+                    # TOFIX : special case for google storage with empty dir
+                    if o.name.endswith('_$folder$'):
+                        name = o.name[:-9]
+                        name = name[len(path):]
+                        dirs.append(name)
+                    else:
+                        name = o.name[len(path):]
+                        files.append(name)
+        return (dirs, files)
+
+    def size(self, name):
+        obj = self._get_object(name)
+        if obj:
+            return obj.size
+        else:
+            return -1
+
+    def url(self, name):
+        obj = self._get_object(name)
+        return self.driver.get_object_cdn_url(obj)
+
+    def _open(self, name, mode='rb'):
+        remote_file = LibCloudFile(name, self, mode=mode)
+        return remote_file
+
+    def _read(self, name, start_range=None, end_range=None):
+        obj = self._get_object(name)
+        # TOFIX : we should be able to read chunk by chunk
+        return self.driver.download_object_as_stream(obj, obj.size).next()
+
+    def _save(self, name, file):
+        self.driver.upload_object_via_stream(file, self._get_bucket(), name)
+
+
+class LibCloudFile(File):
+    """File intherited class for libcloud storage objects read and write"""
+    def __init__(self, name, storage, mode):
+        self._name = name
+        self._storage = storage
+        self._mode = mode
+        self._is_dirty = False
+        self.file = StringIO()
+        self.start_range = 0
+
+    @property
+    def size(self):
+        if not hasattr(self, '_size'):
+            self._size = self._storage.size(self._name)
+        return self._size
+
+    def read(self, num_bytes=None):
+        if num_bytes is None:
+            args = []
+            self.start_range = 0
+        else:
+            args = [self.start_range, self.start_range + num_bytes - 1]
+        data = self._storage._read(self._name, *args)
+        self.file = StringIO(data)
+        return self.file.getvalue()
+
+    def write(self, content):
+        if 'w' not in self._mode:
+            raise AttributeError("File was opened for read-only access.")
+        self.file = StringIO(content)
+        self._is_dirty = True
+
+    def close(self):
+        if self._is_dirty:
+            self._storage._save(self._name, self.file)
+        self.file.close()

storages/backends/hashpath.py

 import os, hashlib, errno
 
-from django.conf import settings
 from django.core.files.storage import FileSystemStorage
 from django.utils.encoding import force_unicode
 
         if self.exists(name):
             return name
 
-        # Try to create the directory relative to the media root
+        # Try to create the directory relative to location specified in __init__
         try:
-            os.makedirs(os.path.join(settings.MEDIA_ROOT, dir_name))
+            os.makedirs(os.path.join(self.location, dir_name))
         except OSError, e:
             if e.errno is not errno.EEXIST:
                 raise e

storages/backends/mosso.py

 # TODO: implement TTL into cloudfiles methods
 TTL = getattr(settings, 'CLOUDFILES_TTL', 600)
 CONNECTION_KWARGS = getattr(settings, 'CLOUDFILES_CONNECTION_KWARGS', {})
+SSL = getattr(settings, 'CLOUDFILES_SSL', False)
 
 
 def cloudfiles_upload_to(self, filename):
 
     def _get_container_url(self):
         if not hasattr(self, '_container_public_uri'):
-            self._container_public_uri = self.container.public_uri()
+            if SSL:
+                self._container_public_uri = self.container.public_ssl_uri()
+            else:
+                self._container_public_uri = self.container.public_uri()
         return self._container_public_uri
 
     container_url = property(_get_container_url)

storages/backends/s3boto.py

 import os
+import re
+import time
 import mimetypes
+import calendar
+from datetime import datetime
 
 try:
     from cStringIO import StringIO
 except ImportError:
-    from StringIO import StringIO
+    from StringIO import StringIO  # noqa
 
 from django.conf import settings
 from django.core.files.base import File
     raise ImproperlyConfigured("Could not load Boto's S3 bindings.\n"
                                "See http://code.google.com/p/boto/")
 
-ACCESS_KEY_NAME     = getattr(settings, 'AWS_ACCESS_KEY_ID', None)
-SECRET_KEY_NAME     = getattr(settings, 'AWS_SECRET_ACCESS_KEY', None)
-HEADERS             = getattr(settings, 'AWS_HEADERS', {})
+ACCESS_KEY_NAME = getattr(settings, 'AWS_ACCESS_KEY_ID', None)
+SECRET_KEY_NAME = getattr(settings, 'AWS_SECRET_ACCESS_KEY', None)
+HEADERS = getattr(settings, 'AWS_HEADERS', {})
 STORAGE_BUCKET_NAME = getattr(settings, 'AWS_STORAGE_BUCKET_NAME', None)
-AUTO_CREATE_BUCKET  = getattr(settings, 'AWS_AUTO_CREATE_BUCKET', False)
-DEFAULT_ACL         = getattr(settings, 'AWS_DEFAULT_ACL', 'public-read')
-BUCKET_ACL          = getattr(settings, 'AWS_BUCKET_ACL', DEFAULT_ACL)
-QUERYSTRING_AUTH    = getattr(settings, 'AWS_QUERYSTRING_AUTH', True)
-QUERYSTRING_EXPIRE  = getattr(settings, 'AWS_QUERYSTRING_EXPIRE', 3600)
-REDUCED_REDUNDANCY  = getattr(settings, 'AWS_REDUCED_REDUNDANCY', False)
-LOCATION            = getattr(settings, 'AWS_LOCATION', '')
-ENCRYPTION          = getattr(settings, 'AWS_S3_ENCRYPTION', False)
-CUSTOM_DOMAIN       = getattr(settings, 'AWS_S3_CUSTOM_DOMAIN', None)
-CALLING_FORMAT      = getattr(settings, 'AWS_S3_CALLING_FORMAT', SubdomainCallingFormat())
-SECURE_URLS         = getattr(settings, 'AWS_S3_SECURE_URLS', True)
-FILE_NAME_CHARSET   = getattr(settings, 'AWS_S3_FILE_NAME_CHARSET', 'utf-8')
-FILE_OVERWRITE      = getattr(settings, 'AWS_S3_FILE_OVERWRITE', True)
-IS_GZIPPED          = getattr(settings, 'AWS_IS_GZIPPED', False)
-PRELOAD_METADATA    = getattr(settings, 'AWS_PRELOAD_METADATA', False)
-GZIP_CONTENT_TYPES  = getattr(settings, 'GZIP_CONTENT_TYPES', (
+AUTO_CREATE_BUCKET = getattr(settings, 'AWS_AUTO_CREATE_BUCKET', False)
+DEFAULT_ACL = getattr(settings, 'AWS_DEFAULT_ACL', 'public-read')
+BUCKET_ACL = getattr(settings, 'AWS_BUCKET_ACL', DEFAULT_ACL)
+QUERYSTRING_AUTH = getattr(settings, 'AWS_QUERYSTRING_AUTH', True)
+QUERYSTRING_EXPIRE = getattr(settings, 'AWS_QUERYSTRING_EXPIRE', 3600)
+REDUCED_REDUNDANCY = getattr(settings, 'AWS_REDUCED_REDUNDANCY', False)
+LOCATION = getattr(settings, 'AWS_LOCATION', '')
+ENCRYPTION = getattr(settings, 'AWS_S3_ENCRYPTION', False)
+CUSTOM_DOMAIN = getattr(settings, 'AWS_S3_CUSTOM_DOMAIN', None)
+CALLING_FORMAT = getattr(settings, 'AWS_S3_CALLING_FORMAT',
+                         SubdomainCallingFormat())
+SECURE_URLS = getattr(settings, 'AWS_S3_SECURE_URLS', True)
+FILE_NAME_CHARSET = getattr(settings, 'AWS_S3_FILE_NAME_CHARSET', 'utf-8')
+FILE_OVERWRITE = getattr(settings, 'AWS_S3_FILE_OVERWRITE', True)
+FILE_BUFFER_SIZE = getattr(settings, 'AWS_S3_FILE_BUFFER_SIZE', 5242880)
+IS_GZIPPED = getattr(settings, 'AWS_IS_GZIPPED', False)
+PRELOAD_METADATA = getattr(settings, 'AWS_PRELOAD_METADATA', False)
+GZIP_CONTENT_TYPES = getattr(settings, 'GZIP_CONTENT_TYPES', (
     'text/css',
     'application/javascript',
-    'application/x-javascript'
+    'application/x-javascript',
 ))
 
 if IS_GZIPPED:
     from gzip import GzipFile
 
+
 def safe_join(base, *paths):
     """
     A version of django.utils._os.safe_join for S3 paths.
 
-    Joins one or more path components to the base path component intelligently.
-    Returns a normalized version of the final path.
+    Joins one or more path components to the base path component
+    intelligently. Returns a normalized version of the final path.
 
-    The final path must be located inside of the base path component (otherwise
-    a ValueError is raised).
+    The final path must be located inside of the base path component
+    (otherwise a ValueError is raised).
 
-    Paths outside the base path indicate a possible security sensitive operation.
+    Paths outside the base path indicate a possible security
+    sensitive operation.
     """
     from urlparse import urljoin
     base_path = force_unicode(base)
-    paths = map(lambda p: force_unicode(p), paths)
-    final_path = urljoin(base_path + ("/" if not base_path.endswith("/") else ""), *paths)
+    base_path = base_path.rstrip('/')
+    paths = [force_unicode(p) for p in paths]
+
+    final_path = base_path
+    for path in paths:
+        final_path = urljoin(final_path.rstrip('/') + "/", path.rstrip("/"))
+
     # Ensure final_path starts with base_path and that the next character after
     # the final path is '/' (or nothing, in which case final_path must be
     # equal to base_path).
     base_path_len = len(base_path)
     if not final_path.startswith(base_path) \
-       or final_path[base_path_len:base_path_len+1] not in ('', '/'):
+       or final_path[base_path_len:base_path_len + 1] not in ('', '/'):
         raise ValueError('the joined path is located outside of the base path'
                          ' component')
-    return final_path
+
+    return final_path.lstrip('/')
+
+# Dates returned from S3's API look something like this:
+# "Sun, 11 Mar 2012 17:01:41 GMT"
+MONTH_NAMES = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
+               'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
+DATESTR_RE = re.compile(r"^.+, (?P<day>\d{1,2}) (?P<month_name>%s) (?P<year>\d{4}) (?P<hour>\d{1,2}):(?P<minute>\d{1,2}):(?P<second>\d{1,2}) (GMT|UTC)$" % ("|".join(MONTH_NAMES)))
+def _parse_datestring(dstr):
+    """
+    Parse a simple datestring returned by the S3 API and returns
+    a datetime object in the local timezone.
+    """
+    # This regular expression and thus this function
+    # assumes the date is GMT/UTC
+    m = DATESTR_RE.match(dstr)
+    if m:
+        # This code could raise a ValueError if there is some
+        # bad data or the date is invalid.
+        datedict = m.groupdict()
+        utc_datetime = datetime(
+            int(datedict['year']),
+            int(MONTH_NAMES.index(datedict['month_name'])) + 1,
+            int(datedict['day']),
+            int(datedict['hour']),
+            int(datedict['minute']),
+            int(datedict['second']),
+        )
+
+        # Convert the UTC datetime object to local time.
+        return datetime(*time.localtime(calendar.timegm(utc_datetime.timetuple()))[:6])
+    else:
+        raise ValueError("Could not parse date string: " + dstr)
 
 class S3BotoStorage(Storage):
-    """Amazon Simple Storage Service using Boto"""
+    """
+    Amazon Simple Storage Service using Boto
+    
+    This storage backend supports opening files in read or write
+    mode and supports streaming(buffering) data in chunks to S3
+    when writing.
+    """
 
     def __init__(self, bucket=STORAGE_BUCKET_NAME, access_key=None,
-                       secret_key=None, bucket_acl=BUCKET_ACL, acl=DEFAULT_ACL, headers=HEADERS,
-                       gzip=IS_GZIPPED, gzip_content_types=GZIP_CONTENT_TYPES,
-                       querystring_auth=QUERYSTRING_AUTH, querystring_expire=QUERYSTRING_EXPIRE,
-                       reduced_redundancy=REDUCED_REDUNDANCY, encryption=ENCRYPTION,
-                       custom_domain=CUSTOM_DOMAIN, secure_urls=SECURE_URLS,
-                       location=LOCATION, file_name_charset=FILE_NAME_CHARSET,
-                       preload_metadata=PRELOAD_METADATA, calling_format=CALLING_FORMAT):
+            secret_key=None, bucket_acl=BUCKET_ACL, acl=DEFAULT_ACL,
+            headers=HEADERS, gzip=IS_GZIPPED,
+            gzip_content_types=GZIP_CONTENT_TYPES,
+            querystring_auth=QUERYSTRING_AUTH,
+            querystring_expire=QUERYSTRING_EXPIRE,
+            reduced_redundancy=REDUCED_REDUNDANCY,
+            encryption=ENCRYPTION,
+            custom_domain=CUSTOM_DOMAIN,
+            secure_urls=SECURE_URLS,
+            location=LOCATION,
+            file_name_charset=FILE_NAME_CHARSET,
+            preload_metadata=PRELOAD_METADATA,
+            calling_format=CALLING_FORMAT):
         self.bucket_acl = bucket_acl
         self.bucket_name = bucket
         self.acl = acl
         self.file_name_charset = file_name_charset
 
         if not access_key and not secret_key:
-             access_key, secret_key = self._get_access_keys()
+            access_key, secret_key = self._get_access_keys()
 
-        self.connection = S3Connection(access_key, secret_key, calling_format=calling_format)
+        self.connection = S3Connection(access_key, secret_key,
+            calling_format=calling_format)
         self._entries = {}
 
     @property
     def bucket(self):
+        """
+        Get the current bucket. If there is no current bucket object
+        create it.
+        """
         if not hasattr(self, '_bucket'):
             self._bucket = self._get_or_create_bucket(self.bucket_name)
         return self._bucket
 
     @property
     def entries(self):
+        """
+        Get the locally cached files for the bucket.
+        """
         if self.preload_metadata and not self._entries:
             self._entries = dict((self._decode_name(entry.key), entry)
                                 for entry in self.bucket.list())
         return self._entries
 
     def _get_access_keys(self):
+        """
+        Gets the access keys to use when accessing S3. If none
+        are provided to the class in the constructor or in the 
+        settings then get them from the environment variables.
+        """
         access_key = ACCESS_KEY_NAME
         secret_key = SECRET_KEY_NAME
         if (access_key or secret_key) and (not access_key or not secret_key):
+            # TODO: this seems to be broken
             access_key = os.environ.get(ACCESS_KEY_NAME)
             secret_key = os.environ.get(SECRET_KEY_NAME)
 
     def _get_or_create_bucket(self, name):
         """Retrieves a bucket if it exists, otherwise creates it."""
         try:
-            return self.connection.get_bucket(name, validate=AUTO_CREATE_BUCKET)
-        except S3ResponseError, e:
+            return self.connection.get_bucket(name,
+                validate=AUTO_CREATE_BUCKET)
+        except S3ResponseError:
             if AUTO_CREATE_BUCKET:
                 bucket = self.connection.create_bucket(name)
                 bucket.set_acl(self.bucket_acl)
                 return bucket
-            raise ImproperlyConfigured, ("Bucket specified by "
-            "AWS_STORAGE_BUCKET_NAME does not exist. Buckets can be "
-            "automatically created by setting AWS_AUTO_CREATE_BUCKET=True")
+            raise ImproperlyConfigured("Bucket specified by "
+                "AWS_STORAGE_BUCKET_NAME does not exist. "
+                "Buckets can be automatically created by setting "
+                "AWS_AUTO_CREATE_BUCKET=True")
 
     def _clean_name(self, name):
+        """
+        Cleans the name so that Windows style paths work
+        """
         # Useful for windows' paths
         return os.path.normpath(name).replace('\\', '/')
 
     def _normalize_name(self, name):
+        """
+        Normalizes the name so that paths like /path/to/ignored/../something.txt
+        work. We check to make sure that the path pointed to is not outside
+        the directory specified by the LOCATION setting.
+        """
         try:
-            return safe_join(self.location, name).lstrip('/')
+            return safe_join(self.location, name)
         except ValueError:
-            raise SuspiciousOperation("Attempted access to '%s' denied." % name)
+            raise SuspiciousOperation("Attempted access to '%s' denied." %
+                                      name)
 
     def _encode_name(self, name):
         return smart_str(name, encoding=self.file_name_charset)
         return force_unicode(name, encoding=self.file_name_charset)
 
     def _compress_content(self, content):
-        """Gzip a given string."""
+        """Gzip a given string content."""
         zbuf = StringIO()
         zfile = GzipFile(mode='wb', compresslevel=6, fileobj=zbuf)
         zfile.write(content.read())
         cleaned_name = self._clean_name(name)
         name = self._normalize_name(cleaned_name)
         headers = self.headers.copy()
-        content_type = getattr(content,'content_type', mimetypes.guess_type(name)[0] or Key.DefaultContentType)
+        content_type = getattr(content, 'content_type',
+            mimetypes.guess_type(name)[0] or Key.DefaultContentType)
 
         if self.gzip and content_type in self.gzip_content_types:
             content = self._compress_content(content)
             headers.update({'Content-Encoding': 'gzip'})
 
         content.name = cleaned_name
-        k = self.bucket.get_key(self._encode_name(name))
-        if not k:
-            k = self.bucket.new_key(self._encode_name(name))
+        encoded_name = self._encode_name(name)
+        key = self.bucket.get_key(encoded_name)
+        if not key:
+            key = self.bucket.new_key(encoded_name)
+        if self.preload_metadata:
+            self._entries[encoded_name] = key
 
-        k.set_metadata('Content-Type',content_type)
-        k.set_contents_from_file(content, headers=headers, policy=self.acl,
-                                 reduced_redundancy=self.reduced_redundancy,
-                                 encrypt_key=self.encryption)
+        key.set_metadata('Content-Type', content_type)
+        key.set_contents_from_file(content, headers=headers, policy=self.acl,
+                                   reduced_redundancy=self.reduced_redundancy,
+                                   encrypt_key=self.encryption)
         return cleaned_name
 
     def delete(self, name):
 
     def listdir(self, name):
         name = self._normalize_name(self._clean_name(name))
+        # for the bucket.list and logic below name needs to end in /
+        # But for the root path "" we leave it as an empty string
+        if name:
+            name += '/'
+
         dirlist = self.bucket.list(self._encode_name(name))
         files = []
         dirs = set()
-        base_parts = name.split("/") if name else []
+        base_parts = name.split("/")[:-1]
         for item in dirlist:
             parts = item.name.split("/")
             parts = parts[len(base_parts):]
             elif len(parts) > 1:
                 # Directory
                 dirs.add(parts[0])
-        return list(dirs),files
+        return list(dirs), files
 
     def size(self, name):
         name = self._normalize_name(self._clean_name(name))
         return self.bucket.get_key(self._encode_name(name)).size
 
     def modified_time(self, name):
-        try:
-           from dateutil import parser, tz
-        except ImportError:
-            raise NotImplementedError()
         name = self._normalize_name(self._clean_name(name))
         entry = self.entries.get(name)
-        # only call self.bucket.get_key() if the key is not found
-        # in the preloaded metadata.
         if entry is None:
             entry = self.bucket.get_key(self._encode_name(name))
-        # convert to string to date
-        last_modified_date = parser.parse(entry.last_modified)
-        # if the date has no timzone, assume UTC
-        if last_modified_date.tzinfo == None:
-            last_modified_date = last_modified_date.replace(tzinfo=tz.tzutc())
-        # convert date to local time w/o timezone
-        return last_modified_date.astimezone(tz.tzlocal()).replace(tzinfo=None)
+
+        # Parse the last_modified string to a local datetime object.
+        return _parse_datestring(entry.last_modified)
 
     def url(self, name):
         name = self._normalize_name(self._clean_name(name))
         if self.custom_domain:
-            return "%s://%s/%s" % ('https' if self.secure_urls else 'http', self.custom_domain, name)
-        else:
-            return self.connection.generate_url(self.querystring_expire, method='GET', \
-                    bucket=self.bucket.name, key=self._encode_name(name), query_auth=self.querystring_auth, \
-                    force_http=not self.secure_urls)
+            return "%s://%s/%s" % ('https' if self.secure_urls else 'http',
+                                   self.custom_domain, name)
+        return self.connection.generate_url(self.querystring_expire,
+            method='GET', bucket=self.bucket.name, key=self._encode_name(name),
+            query_auth=self.querystring_auth, force_http=not self.secure_urls)
 
     def get_available_name(self, name):
         """ Overwrite existing file with the same name. """
 
 
 class S3BotoStorageFile(File):
-    def __init__(self, name, mode, storage):
+    """
+    The default file object used by the S3BotoStorage backend.
+    
+    This file implements file streaming using boto's multipart
+    uploading functionality. The file can be opened in read or
+    write mode.
+
+    This class extends Django's File class. However, the contained
+    data is only the data contained in the current buffer. So you
+    should not access the contained file object directly. You should
+    access the data via this class.
+
+    Warning: This file *must* be closed using the close() method in
+    order to properly write the file to S3. Be sure to close the file
+    in your application.
+    """
+    # TODO: Read/Write (rw) mode may be a bit undefined at the moment. Needs testing.
+    # TODO: When Django drops support for Python 2.5, rewrite to use the 
+    #       BufferedIO streams in the Python 2.6 io module.
+
+    def __init__(self, name, mode, storage, buffer_size=FILE_BUFFER_SIZE):
         self._storage = storage
         self.name = name[len(self._storage.location):].lstrip('/')
         self._mode = mode
         self.key = storage.bucket.get_key(self._storage._encode_name(name))
+        if not self.key and 'w' in mode:
+            self.key = storage.bucket.new_key(storage._encode_name(name))
         self._is_dirty = False
         self._file = None
+        self._multipart = None
+        # 5 MB is the minimum part size (if there is more than one part).
+        # Amazon allows up to 10,000 parts.  The default supports uploads
+        # up to roughly 50 GB.  Increase the part size to accommodate 
+        # for files larger than this.
+        self._write_buffer_size = buffer_size
+        self._write_counter = 0
 
     @property
     def size(self):
 
     def write(self, *args, **kwargs):
         if 'w' not in self._mode:
-            raise AttributeError("File was opened for read-only access.")
+            raise AttributeError("File was not opened in write mode.")
         self._is_dirty = True
+        if self._multipart is None:
+            provider = self.key.bucket.connection.provider
+            upload_headers = {
+                provider.acl_header: self._storage.acl
+            }
+            upload_headers.update(self._storage.headers)
+            self._multipart = self._storage.bucket.initiate_multipart_upload(
+                self.key.name,
+                headers = upload_headers,
+                reduced_redundancy = self._storage.reduced_redundancy
+            )
+        if self._write_buffer_size <= self._buffer_file_size:
+            self._flush_write_buffer()
         return super(S3BotoStorageFile, self).write(*args, **kwargs)
 
+    @property
+    def _buffer_file_size(self):
+        pos = self.file.tell()
+        self.file.seek(0,os.SEEK_END)
+        length = self.file.tell()
+        self.file.seek(pos)
+        return length
+
+    def _flush_write_buffer(self):
+        """
+        Flushes the write buffer.
+        """
+        if self._buffer_file_size:
+            self._write_counter += 1
+            self.file.seek(0)
+            self._multipart.upload_part_from_file(
+                self.file,
+                self._write_counter,
+                headers=self._storage.headers
+            )
+            self.file.close()
+            self._file = None
+
     def close(self):
         if self._is_dirty:
-            self.key.set_contents_from_file(self._file, headers=self._storage.headers, policy=self._storage.acl)
+            self._flush_write_buffer()
+            self._multipart.complete_upload()
+        else:
+            if not self._multipart is None:
+                self._multipart.cancel_upload()
         self.key.close()

storages/tests/__init__.py

 from storages.tests.hashpath import *
+from storages.tests.s3boto import *

storages/tests/s3boto.py

+import os
+import mock
+from uuid import uuid4
+from urllib2 import urlopen
+
+from django.test import TestCase
+from django.core.files.base import ContentFile
+from django.conf import settings
+from django.core.files.storage import FileSystemStorage
+
+from boto.s3.key import Key
+
+from storages.backends import s3boto
+
+__all__ = (
+    'SafeJoinTest',
+    'S3BotoStorageTests',
+    #'S3BotoStorageFileTests',
+)
+
+class S3BotoTestCase(TestCase):
+    @mock.patch('storages.backends.s3boto.S3Connection')
+    def setUp(self, S3Connection):
+        self.storage = s3boto.S3BotoStorage()
+
+
+class SafeJoinTest(TestCase):
+    def test_normal(self):
+        path = s3boto.safe_join("", "path/to/somewhere", "other", "path/to/somewhere")
+        self.assertEquals(path, "path/to/somewhere/other/path/to/somewhere")
+
+    def test_with_dot(self):
+        path = s3boto.safe_join("", "path/./somewhere/../other", "..",
+                                ".", "to/./somewhere")
+        self.assertEquals(path, "path/to/somewhere")
+
+    def test_base_url(self):
+        path = s3boto.safe_join("base_url", "path/to/somewhere")
+        self.assertEquals(path, "base_url/path/to/somewhere")
+
+    def test_base_url_with_slash(self):
+        path = s3boto.safe_join("base_url/", "path/to/somewhere")
+        self.assertEquals(path, "base_url/path/to/somewhere")
+
+    def test_suspicious_operation(self):
+        self.assertRaises(ValueError,
+            s3boto.safe_join, "base", "../../../../../../../etc/passwd")
+    
+class S3BotoStorageTests(S3BotoTestCase):
+
+    def test_storage_save(self):
+        """
+        Test saving a file
+        """
+        name = 'test_storage_save.txt'
+        content = ContentFile('new content')
+        self.storage.save(name, content)
+        self.storage.bucket.get_key.assert_called_once_with(name)
+        
+        key = self.storage.bucket.get_key.return_value
+        key.set_metadata.assert_called_with('Content-Type', 'text/plain')
+        key.set_contents_from_file.assert_called_with(
+            content,
+            headers={},
+            policy=self.storage.acl,
+            reduced_redundancy=self.storage.reduced_redundancy,
+        )
+    
+    def test_storage_open_write(self):
+        """
+        Test opening a file in write mode
+        """
+        name = 'test_open_for_writing.txt'
+        content = 'new content'
+
+        # Set the ACL header used when creating/writing data.
+        self.storage.bucket.connection.provider.acl_header = 'x-amz-acl'
+        # Set the mocked key's bucket
+        self.storage.bucket.get_key.return_value.bucket = self.storage.bucket
+        # Set the name of the mock object
+        self.storage.bucket.get_key.return_value.name = name 
+
+        file = self.storage.open(name, 'w')
+        self.storage.bucket.get_key.assert_called_with(name)
+
+        file.write(content)
+        self.storage.bucket.initiate_multipart_upload.assert_called_with(
+            name,
+            headers={'x-amz-acl': 'public-read'},
+            reduced_redundancy=self.storage.reduced_redundancy,
+        )
+
+        # Save the internal file before closing
+        _file = file.file
+        file.close()
+        file._multipart.upload_part_from_file.assert_called_with(
+            _file, 1, headers=self.storage.headers,
+        )
+        file._multipart.complete_upload.assert_called_once()
+    
+    #def test_storage_exists_and_delete(self):
+    #    # show file does not exist
+    #    name = self.prefix_path('test_exists.txt')
+    #    self.assertFalse(self.storage.exists(name))
+    #    
+    #    # create the file
+    #    content = 'new content'
+    #    file = self.storage.open(name, 'w')
+    #    file.write(content)
+    #    file.close()
+    #    
+    #    # show file exists
+    #    self.assertTrue(self.storage.exists(name))
+    #    
+    #    # delete the file
+    #    self.storage.delete(name)
+    #    
+    #    # show file does not exist
+    #    self.assertFalse(self.storage.exists(name))
+
+    def test_storage_listdir_base(self):
+        file_names = ["some/path/1.txt", "2.txt", "other/path/3.txt", "4.txt"]
+
+        self.storage.bucket.list.return_value = []
+        for p in file_names:
+            key = mock.MagicMock(spec=Key)
+            key.name = p
+            self.storage.bucket.list.return_value.append(key)
+
+        dirs, files = self.storage.listdir("")
+
+        self.assertEqual(len(dirs), 2)
+        for directory in ["some", "other"]:
+            self.assertTrue(directory in dirs, 
+                            """ "%s" not in directory list "%s".""" % (
+                                directory, dirs))
+            
+        self.assertEqual(len(files), 2)
+        for filename in ["2.txt", "4.txt"]:
+            self.assertTrue(filename in files, 
+                            """ "%s" not in file list "%s".""" % (
+                                filename, files))
+
+    def test_storage_listdir_subdir(self):
+        file_names = ["some/path/1.txt", "some/2.txt"]
+
+        self.storage.bucket.list.return_value = []
+        for p in file_names:
+            key = mock.MagicMock(spec=Key)
+            key.name = p
+            self.storage.bucket.list.return_value.append(key)
+
+        dirs, files = self.storage.listdir("some/")
+        self.assertEqual(len(dirs), 1)
+        self.assertTrue('path' in dirs, 
+                        """ "path" not in directory list "%s".""" % (dirs,))
+            
+        self.assertEqual(len(files), 1)
+        self.assertTrue('2.txt' in files, 
+                        """ "2.txt" not in files list "%s".""" % (files,))
+
+    #def test_storage_size(self):
+    #    name = self.prefix_path('test_storage_size.txt')
+    #    content = 'new content'
+    #    f = ContentFile(content)
+    #    self.storage.save(name, f)
+    #    self.assertEqual(self.storage.size(name), f.size)
+    #    
+    #def test_storage_url(self):
+    #    name = self.prefix_path('test_storage_size.txt')
+    #    content = 'new content'
+    #    f = ContentFile(content)
+    #    self.storage.save(name, f)
+    #    self.assertEqual(content, urlopen(self.storage.url(name)).read())
+        
+#class S3BotoStorageFileTests(S3BotoTestCase):
+#    def test_multipart_upload(self):
+#        nparts = 2
+#        name = self.prefix_path("test_multipart_upload.txt")
+#        mode = 'w'
+#        f = s3boto.S3BotoStorageFile(name, mode, self.storage)
+#        content_length = 1024 * 1024# 1 MB
+#        content = 'a' * content_length
+#        
+#        bytes = 0
+#        target = f._write_buffer_size * nparts
+#        while bytes < target:
+#            f.write(content)
+#            bytes += content_length
+#            
+#        # make the buffer roll over so f._write_counter
+#        # is incremented
+#        f.write("finished")
+#        
+#        # verify upload was multipart and correctly partitioned
+#        self.assertEqual(f._write_counter, nparts)
+#        
+#        # complete the upload
+#        f.close()
+#        
+#        # verify that the remaining buffered bytes were
+#        # uploaded when the file was closed.
+#        self.assertEqual(f._write_counter, nparts+1)
+import os
+import sys
+import django
+
+BASE_PATH = os.path.dirname(__file__)
+
+def main():
+    """
+    Standalone django model test with a 'memory-only-django-installation'.
+    You can play with a django model without a complete django app installation.
+    http://www.djangosnippets.org/snippets/1044/
+    """
+    sys.exc_clear()
+
+    os.environ["DJANGO_SETTINGS_MODULE"] = "django.conf.global_settings"
+    from django.conf import global_settings
+
+    global_settings.INSTALLED_APPS = (
+        'django.contrib.auth',
+        'django.contrib.sessions',
+        'django.contrib.contenttypes',
+        'storages',
+    )
+    if django.VERSION > (1,2):
+        global_settings.DATABASES = {
+            'default': {
+                'ENGINE': 'django.db.backends.sqlite3',
+                'NAME': os.path.join(BASE_PATH, 'connpass.sqlite'),
+                'USER': '',
+                'PASSWORD': '',
+                'HOST': '',
+                'PORT': '',
+            }
+        }
+    else:
+        global_settings.DATABASE_ENGINE = "sqlite3"
+        global_settings.DATABASE_NAME = ":memory:"
+
+    global_settings.ROOT_URLCONF='beproud.django.authutils.tests.test_urls'
+    global_settings.MIDDLEWARE_CLASSES = (
+        'django.middleware.common.CommonMiddleware',
+        'django.contrib.sessions.middleware.SessionMiddleware',
+        'django.middleware.csrf.CsrfViewMiddleware',
+        'django.contrib.auth.middleware.AuthenticationMiddleware',
+        'django.contrib.messages.middleware.MessageMiddleware',
+        'beproud.django.authutils.middleware.AuthMiddleware',
+    )
+    global_settings.DEFAULT_FILE_STORAGE = 'backends.s3boto.S3BotoStorage'
+
+    from django.test.utils import get_runner
+    test_runner = get_runner(global_settings)
+
+    if django.VERSION > (1,2):
+        test_runner = test_runner()
+        failures = test_runner.run_tests(['storages'])
+    else:
+        failures = test_runner(['storages'], verbosity=1)
+    sys.exit(failures)
+
+if __name__ == '__main__':
+    main()
+# content of: tox.ini , put in same dir as setup.py
+[tox]
+envlist = django11,django12,django13
+
+[testenv]
+deps=boto
+
+[testenv:django11]
+deps=
+    mock==0.8.0
+    django==1.1.4
+    boto==2.2.2
+commands=python setup.py test
+
+[testenv:django12]
+deps=
+    mock==0.8.0
+    django==1.2.7
+    boto==2.2.2
+commands=python setup.py test
+
+[testenv:django13]
+deps=
+    mock==0.8.0
+    django==1.3.1
+    boto==2.2.2
+commands=python setup.py test
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.