Rich Leland avatar Rich Leland committed 07344ab

Incorporated changes from default branch.

Comments (0)

Files changed (3)

     * Rich Leland (Mosso Cloud Files)
     * Jason Christa (patches)
     * Adam Nelson (patches)
+    * Erik CW (S3 encryption)
 
 Extra thanks to Marty for adding this in Django, 
 you can buy his very interesting book (Pro Django).

storages/backends/s3.py

 QUERYSTRING_ACTIVE= getattr(settings, 'AWS_QUERYSTRING_ACTIVE', False)
 QUERYSTRING_EXPIRE= getattr(settings, 'AWS_QUERYSTRING_EXPIRE', 60)
 SECURE_URLS= getattr(settings, 'AWS_S3_SECURE_URLS', False)
+BUCKET_PREFIX = getattr(settings, 'AWS_BUCKET_PREFIX', '')
 
+IS_GZIPPED= getattr(settings, 'AWS_IS_GZIPPED', False) 
+GZIP_CONTENT_TYPES = (
+    'text/css',
+    'application/javascript',
+    'application/x-javascript'
+)
+GZIP_CONTENT_TYPES = getattr(settings, 'GZIP_CONTENT_TYPES', GZIP_CONTENT_TYPES)
+
+if IS_GZIPPED:
+    from gzip import GzipFile
 
 class S3Storage(Storage):
     """Amazon Simple Storage Service"""
 
     def __init__(self, bucket=settings.AWS_STORAGE_BUCKET_NAME,
             access_key=None, secret_key=None, acl=DEFAULT_ACL,
-            calling_format=settings.AWS_CALLING_FORMAT):
+            calling_format=settings.AWS_CALLING_FORMAT, encrypt=False,
+            gzip=IS_GZIPPED, gzip_content_types=GZIP_CONTENT_TYPES):
         self.bucket = bucket
         self.acl = acl
+        self.encrypt = encrypt
+        self.gzip = gzip
+        self.gzip_content_types = gzip_content_types
+        
+        if encrypt:
+            try:
+                import ezPyCrypto
+            except ImportError:
+                raise ImproperlyConfigured, "Could not load ezPyCrypto.\
+                \nSee http://www.freenet.org.nz/ezPyCrypto/ to install it."
+            self.crypto_key = ezPyCrypto.key
 
         if not access_key and not secret_key:
             access_key, secret_key = self._get_access_keys()
 
     def _clean_name(self, name):
         # Useful for windows' paths
-        return os.path.normpath(name).replace('\\', '/')
+        return os.path.join(BUCKET_PREFIX, os.path.normpath(name).replace('\\', '/'))
 
+    def _compress_string(self, s):
+        """Gzip a given string."""
+        zbuf = StringIO()
+        zfile = GzipFile(mode='wb', compresslevel=6, fileobj=zbuf)
+        zfile.write(s)
+        zfile.close()
+        return zbuf.getvalue()
+        
     def _put_file(self, name, content):
+        if self.encrypt:
+        
+            # Create a key object
+            key = self.crypto_key()
+        
+            # Read in a public key
+            fd = open(settings.CRYPTO_KEYS_PUBLIC, "rb")
+            public_key = fd.read()
+            fd.close()
+        
+            # import this public key
+            key.importKey(public_key)
+        
+            # Now encrypt some text against this public key
+            content = key.encString(content)
+        
         content_type = mimetypes.guess_type(name)[0] or "application/x-octet-stream"
+        
+        if self.gzip and content_type in self.gzip_content_types:
+            content = self._compress_string(content)
+            self.headers.update({'Content-Encoding': 'gzip'})
+        
         self.headers.update({
             'x-amz-acl': self.acl, 
             'Content-Type': content_type,
         if response.http_response.status not in (200, 206):
             raise IOError("S3StorageError: %s" % response.message)
         headers = response.http_response.msg
+        
+        if self.encrypt:
+            # Read in a private key
+            fd = open(settings.CRYPTO_KEYS_PRIVATE, "rb")
+            private_key = fd.read()
+            fd.close()
+        
+            # Create a key object, and auto-import private key
+            key = self.crypto_key(private_key)
+        
+            # Decrypt this file
+            response.object.data = key.decString(response.object.data)
+        
         return response.object.data, headers.get('etag', None), headers.get('content-range', None)
         
     def _save(self, name, content):

storages/backends/s3boto.py

 import os
+import mimetypes
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
 
 from django.conf import settings
 from django.core.files.base import File
 DEFAULT_ACL         = getattr(settings, 'AWS_DEFAULT_ACL', 'public-read')
 QUERYSTRING_AUTH    = getattr(settings, 'AWS_QUERYSTRING_AUTH', True)
 QUERYSTRING_EXPIRE  = getattr(settings, 'AWS_QUERYSTRING_EXPIRE', 3600)
+LOCATION            = getattr(settings, 'AWS_LOCATION', '')
+IS_GZIPPED          = getattr(settings, 'AWS_IS_GZIPPED', False)
+GZIP_CONTENT_TYPES  = getattr(settings, 'GZIP_CONTENT_TYPES', (
+    'text/css',
+    'application/javascript',
+    'application/x-javascript'
+))
+
+if IS_GZIPPED:
+    from gzip import GzipFile
 
 class S3BotoStorage(Storage):
     """Amazon Simple Storage Service using Boto"""
     
     def __init__(self, bucket=STORAGE_BUCKET_NAME, access_key=None,
-                       secret_key=None, acl=DEFAULT_ACL, headers=HEADERS):
+                       secret_key=None, acl=DEFAULT_ACL, headers=HEADERS,
+                       gzip=IS_GZIPPED, gzip_content_types=GZIP_CONTENT_TYPES):
         self.acl = acl
         self.headers = headers
+        self.gzip = gzip
+        self.gzip_content_types = gzip_content_types
         
         if not access_key and not secret_key:
              access_key, secret_key = self._get_access_keys()
         # Useful for windows' paths
         return os.path.normpath(name).replace('\\', '/')
 
+    def _compress_content(self, content):
+        """Gzip a given string."""
+        zbuf = StringIO()
+        zfile = GzipFile(mode='wb', compresslevel=6, fileobj=zbuf)
+        zfile.write(content.read())
+        zfile.close()
+        content.file = zbuf
+        return content
+        
     def _open(self, name, mode='rb'):
         name = self._clean_name(name)
         return S3BotoStorageFile(name, mode, self)
     def _save(self, name, content):
         name = self._clean_name(name)
         headers = self.headers
+        
         if hasattr(content.file, 'content_type'):
-            headers['Content-Type'] = content.file.content_type
+            content_type = content.file.content_type
+        else:
+            content_type = mimetypes.guess_type(name)[0] or "application/x-octet-stream"
+            
+        if self.gzip and content_type in self.gzip_content_types:
+            content = self._compress_content(content)
+            headers.update({'Content-Encoding': 'gzip'})
+
+        headers.update({
+            'Content-Type': content_type,
+            'Content-Length' : len(content),
+        })
+        
         content.name = name
         k = self.bucket.get_key(name)
         if not k:
     
     def url(self, name):
         name = self._clean_name(name)
+        if self.bucket.get_key(name) is None:
+            return ''
         return self.bucket.get_key(name).generate_url(QUERYSTRING_EXPIRE, method='GET', query_auth=QUERYSTRING_AUTH)
-    
+
     def get_available_name(self, name):
         """ Overwrite existing file with the same name. """
         name = self._clean_name(name)
 class S3BotoStorageFile(File):
     def __init__(self, name, mode, storage):
         self._storage = storage
-        self._name = name
+        self.name = name
         self._mode = mode
         self.key = storage.bucket.get_key(name)
-    
+        self._is_dirty = False
+        self.file = StringIO()
+
+    @property
     def size(self):
         return self.key.size
-    
+
     def read(self, *args, **kwargs):
-        return self.key.read(*args, **kwargs)
-    
+        self.file = StringIO()
+        self._is_dirty = False
+        self.key.get_contents_to_file(self.file)
+        return self.file.getvalue()
+
     def write(self, content):
-        self.key.set_contents_from_string(content, headers=self._storage.headers, acl=self._storage.acl)
-    
+        if 'w' not in self._mode:
+            raise AttributeError("File was opened for read-only access.")
+        self.file = StringIO(content)
+        self._is_dirty = True
+
     def close(self):
+        if self._is_dirty:
+            self.key.set_contents_from_string(self.file.getvalue(), headers=self._storage.headers, acl=self._storage.acl)
         self.key.close()
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.