Commits

pendletongp committed 16dee4f

Comments (0)

Files changed (2)

storages/backends/s3boto.py

 except ImportError:
     from StringIO import StringIO  # noqa
 
+try:
+    from tempfile import SpooledTemporaryFile
+except ImportError:
+    from ..tempfile import SpooledTemporaryFile
+
 from django.core.files.base import File
 from django.core.files.storage import Storage
 from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
 
     def _get_file(self):
         if self._file is None:
-            self._file = StringIO()
+            self._file = SpooledTemporaryFile(max_size=self._storage.file_max_size)
             if 'r' in self._mode:
                 self._is_dirty = False
                 self.key.get_contents_to_file(self._file)
     ))
     url_protocol = setting('AWS_S3_URL_PROTOCOL', 'http:')
 
+    # The max amount of memory a returned file can take up before being
+    # rolled over into a temporary file on disk. Default is 0: Do not roll over.
+    file_max_size = setting('AWS_S3_MAX_MEMORY_SIZE', 0)
+
     def __init__(self, acl=None, bucket=None, **settings):
         # check if some of the settings we've provided as class attributes
         # need to be overwritten with values passed in here

storages/tempfile.py

+from tempfile import TemporaryFile, template
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+class SpooledTemporaryFile:
+    """
+    Temporary file wrapper, specialized to switch from
+    StringIO to a real file when it exceeds a certain size or
+    when a fileno is needed.
+
+    Copied from Python 2.7.1
+    """
+    _rolled = False
+
+    def __init__(self, max_size=0, mode='w+b', bufsize=-1,
+                 suffix="", prefix=template, dir=None):
+        self._file = StringIO()
+        self._max_size = max_size
+        self._rolled = False
+        self._TemporaryFileArgs = (mode, bufsize, suffix, prefix, dir)
+
+    def _check(self, file):
+        if self._rolled: return
+        max_size = self._max_size
+        if max_size and file.tell() > max_size:
+            self.rollover()
+
+    def rollover(self):
+        if self._rolled: return
+        file = self._file
+        newfile = self._file = TemporaryFile(*self._TemporaryFileArgs)
+        del self._TemporaryFileArgs
+
+        newfile.write(file.getvalue())
+        newfile.seek(file.tell(), 0)
+
+        self._rolled = True
+
+    # The method caching trick from NamedTemporaryFile
+    # won't work here, because _file may change from a
+    # _StringIO instance to a real file. So we list
+    # all the methods directly.
+
+    # Context management protocol
+    def __enter__(self):
+        if self._file.closed:
+            raise ValueError("Cannot enter context with closed file")
+        return self
+
+    def __exit__(self, exc, value, tb):
+        self._file.close()
+
+    # file protocol
+    def __iter__(self):
+        return self._file.__iter__()
+
+    def close(self):
+        self._file.close()
+
+    @property
+    def closed(self):
+        return self._file.closed
+
+    @property
+    def encoding(self):
+        return self._file.encoding
+
+    def fileno(self):
+        self.rollover()
+        return self._file.fileno()
+
+    def flush(self):
+        self._file.flush()
+
+    def isatty(self):
+        return self._file.isatty()
+
+    @property
+    def mode(self):
+        return self._file.mode
+
+    @property
+    def name(self):
+        return self._file.name
+
+    @property
+    def newlines(self):
+        return self._file.newlines
+
+    def next(self):
+        return self._file.next
+
+    def read(self, *args):
+        return self._file.read(*args)
+
+    def readline(self, *args):
+        return self._file.readline(*args)
+
+    def readlines(self, *args):
+        return self._file.readlines(*args)
+
+    def seek(self, *args):
+        self._file.seek(*args)
+
+    @property
+    def softspace(self):
+        return self._file.softspace
+
+    def tell(self):
+        return self._file.tell()
+
+    def truncate(self):
+        self._file.truncate()
+
+    def write(self, s):
+        file = self._file
+        rv = file.write(s)
+        self._check(file)
+        return rv
+
+    def writelines(self, iterable):
+        file = self._file
+        rv = file.writelines(iterable)
+        self._check(file)
+        return rv
+
+    def xreadlines(self, *args):
+        return self._file.xreadlines(*args)