Commits

Jannis Leidel committed c68ad09

Added s3.PreloadingS3Storage which preloads bucket content info to make it faster in common methods (modified_time (new), size, exists). The new modified_time method requires python-dateutil to parse the ISO 8601 date.

Comments (0)

Files changed (1)

storages/backends/s3.py

 
 class S3Storage(Storage):
     """Amazon Simple Storage Service"""
+    preload = False
 
     def __init__(self, bucket=settings.AWS_STORAGE_BUCKET_NAME,
             access_key=None, secret_key=None, acl=DEFAULT_ACL,
         self.encrypt = encrypt
         self.gzip = gzip
         self.gzip_content_types = gzip_content_types
-        
+
         if encrypt:
             try:
                 import ezPyCrypto
 
         self.connection = AWSAuthConnection(access_key, secret_key,
                             calling_format=calling_format)
-        self.generator = QueryStringAuthGenerator(access_key, secret_key, 
+        self.generator = QueryStringAuthGenerator(access_key, secret_key,
                             calling_format=calling_format,
                             is_secure=SECURE_URLS)
         self.generator.set_expires_in(QUERYSTRING_EXPIRE)
-        
+
         self.headers = HEADERS
+        self.entries = self.preload and self._preload_entries() or {}
 
     def _get_access_keys(self):
         access_key = ACCESS_KEY_NAME
 
         return None, None
 
+    def _preload_entries(self):
+        entries = self.connection.list_bucket(self.bucket).entries
+        return dict((entry.key, entry) for entry in entries)
+
     def _get_connection(self):
         return AWSAuthConnection(*self._get_access_keys())
 
         zfile.write(s)
         zfile.close()
         return zbuf.getvalue()
-        
+
     def _put_file(self, name, content):
         if self.encrypt:
-        
+
             # Create a key object
             key = self.crypto_key()
-        
+
             # Read in a public key
             fd = open(settings.CRYPTO_KEYS_PUBLIC, "rb")
             public_key = fd.read()
             fd.close()
-        
+
             # import this public key
             key.importKey(public_key)
-        
+
             # Now encrypt some text against this public key
             content = key.encString(content)
-        
+
         content_type = mimetypes.guess_type(name)[0] or "application/x-octet-stream"
-        
+
         if self.gzip and content_type in self.gzip_content_types:
             content = self._compress_string(content)
             self.headers.update({'Content-Encoding': 'gzip'})
-        
+
         self.headers.update({
-            'x-amz-acl': self.acl, 
+            'x-amz-acl': self.acl,
             'Content-Type': content_type,
             'Content-Length' : str(len(content)),
         })
         if response.http_response.status not in (200, 206):
             raise IOError("S3StorageError: %s" % response.message)
         headers = response.http_response.msg
-        
+
         if self.encrypt:
             # Read in a private key
             fd = open(settings.CRYPTO_KEYS_PRIVATE, "rb")
             private_key = fd.read()
             fd.close()
-        
+
             # Create a key object, and auto-import private key
             key = self.crypto_key(private_key)
-        
+
             # Decrypt this file
             response.object.data = key.decString(response.object.data)
-        
+
         return response.object.data, headers.get('etag', None), headers.get('content-range', None)
-        
+
     def _save(self, name, content):
         name = self._clean_name(name)
         content.open()
             content_str = content.read()
         self._put_file(name, content_str)
         return name
-    
+
     def delete(self, name):
         name = self._clean_name(name)
         response = self.connection.delete(self.bucket, name)
 
     def exists(self, name):
         name = self._clean_name(name)
+        if self.entries:
+            return name in self.entries
         response = self.connection._make_request('HEAD', self.bucket, name)
         return response.status == 200
 
     def size(self, name):
         name = self._clean_name(name)
+        if self.entries:
+            entry = self.entries.get(name)
+            if entry:
+                return entry.size
+            return 0
         response = self.connection._make_request('HEAD', self.bucket, name)
         content_length = response.getheader('Content-Length')
         return content_length and int(content_length) or 0
-    
+
     def url(self, name):
         name = self._clean_name(name)
         if QUERYSTRING_ACTIVE:
         else:
             return self.generator.make_bare_url(self.bucket, name)
 
+    def modified_time(self, name):
+        try:
+           from dateutil import parser, tz
+        except ImportError:
+            raise NotImplementedError()
+        name = self._clean_name(name)
+        if self.entries:
+            last_modified = self.entries.get(name).last_modified
+        else:
+            response = self.connection._make_request('HEAD', self.bucket, name)
+            last_modified = response.getheader('Last-Modified')
+        # convert to string to date
+        last_modified_date = parser.parse(last_modified)
+        # if the date has no timzone, assume UTC
+        if last_modified_date.tzinfo == None:
+            last_modified_date = last_modified_date.replace(tzinfo=tz.tzutc())
+        # convert date to local time w/o timezone
+        return last_modified_date.astimezone(tz.tzlocal()).replace(tzinfo=None)
+
     ## UNCOMMENT BELOW IF NECESSARY
     #def get_available_name(self, name):
     #    """ Overwrite existing file with the same name. """
     #    return name
 
 
+class PreloadingS3Storage(S3Storage):
+    """
+    Preloading Amazon Simple Storage Service
+
+    This preloads info about the bucket entries to allow faster use with
+    staticfiles's collectstatic.
+    """
+    preload = True
+
+
 class S3StorageFile(File):
     def __init__(self, name, storage, mode):
         self._name = name
         self._is_dirty = False
         self.file = StringIO()
         self.start_range = 0
-    
+
     @property
     def size(self):
         if not hasattr(self, '_size'):
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.