Brodie Rao avatar Brodie Rao committed 1602fac

Removed X-Sendfile support and implemented per-extension file size limiting for streaming

Comments (0)

Files changed (4)

         /dl,C:\dls
 ``DL_MAX_SIZE``:
     Forbids downloads of files or directories larger than the value in bytes.
+``DL_MAX_STREAM_SIZES``:
+    Limits the maximum file size of streamed files, per file extension.
+
+    Examples::
+        .mp3,1843200
+        .mp3,1843200,.m4a,3686400
 ``DL_AUTH_HOST``:
     The hostname to use for token-based authentication.
 
     Note: Authentication is disabled when this isn't set.
 ``DL_AUTH_URI``:
     The URI to query for token-based authentication.
-    
+
     Note: Authentication is disabled when this isn't set.
 ``DL_STAT_HOST``:
     The hostname to use for the tracking of authenticated requests.
-    
+
     Note: Statistics recording is disabled when this isn't set.
 ``DL_STAT_URI``:
     The URI to query for the tracking of authenticated requests.
         * ``DEBUG``
 
     Note: ``WARNING`` is the default.
-``DL_USE_XSENDFILE``:
-    Sends X-Sendfile headers for individually served files, which can be
-    used with lighttpd or mod_xsendfile and Apache to have the web server
-    serve the files itself.
 ``DL_DAEMONIZE``:
     In standalone mode, this makes the script run as a daemon (it backgrounds
     itself).

streamtar/__init__.py

 DL_MAX_SIZE may be specified, which forbids downloads of files and
 directories larger than DL_MAX_SIZE bytes.
 
+DL_MAX_STREAM_SIZES may be specified, which maps size limits to file
+extensions (e.g. ".mp3,1843200;.m4a,3686400").
+
 If DL_AUTH_HOST and DL_AUTH_URI are specified, token-based authentication to
 a remote service is used to permit and deny access.
 
 
 WARNING is the default.
 
-If DL_USE_XSENDFILE is set, streamed files (such as MP3s) will be sent by the
-web server directly, using the X-Sendfile header.
-
 On Unix, DL_DAEMONIZE can be set to have the script fork into a daemon
 process in standalone mode.
 
     return dl_paths
 
 
+def _get_broken_uri(environ):
+    """Parses and returns DL_BROKEN_URI"""
+
+    try:
+        return bool(environ.get('DL_BROKEN_URI'))
+    except ValueError:
+        raise ValueError('DL_BROKEN_URI must be a boolean value')
+
+
 def _get_max_size(environ):
     """Parses and returns DL_MAX_SIZE"""
 
     return max_size
 
 
-def _get_broken_uri(environ):
-    """Parses and returns DL_BROKEN_URI"""
+def _get_max_stream_sizes(environ):
+    """Parses and returns DL_MAX_STREAM_SIZES"""
 
-    try:
-        return bool(environ.get('DL_BROKEN_URI'))
-    except ValueError:
-        raise ValueError('DL_BROKEN_URI must be a boolean value')
+    max_stream_sizes_str = environ.get('DL_MAX_STREAM_SIZES', '')
+    if not max_stream_sizes_str:
+        return
+
+    max_stream_sizes = {}
+    for pair in max_stream_sizes_str.split(';'):
+        pair = pair.split(',', 1)
+        if len(pair) == 2:
+            ext = pair[0].lower()
+            try:
+                max_size = int(pair[1])
+            except ValueError:
+                raise ValueError('DL_MAX_STREAM_SIZES must contain integers')
+            if max_size < 0:
+                max_size = None
+            max_stream_sizes[ext] = max_size
+    return max_stream_sizes
 
 
 def _get_standalone_addr(environ, key='DL_STANDALONE'):
     return address[0], port
 
 
-def _get_use_xsendfile(environ):
-    """Parses and returns DL_USE_XSENDFILE"""
-
-    try:
-        return bool(environ.get('DL_USE_XSENDFILE'))
-    except ValueError:
-        raise ValueError('DL_USE_XSENDFILE must be a boolean value')
-
-
 def _get_crossdomain_file(environ):
     """Parses and returns DL_CROSSDOMAIN_FILE"""
 
 
     try:
         dl_paths = _get_paths(os.environ)
+        broken_uri = _get_broken_uri(os.environ)
         max_size = _get_max_size(os.environ)
-        broken_uri = _get_broken_uri(os.environ)
-        use_xsendfile = _get_use_xsendfile(os.environ)
+        max_stream_sizes = _get_max_stream_sizes(os.environ)
         crossdomain_file = _get_crossdomain_file(os.environ)
     except ValueError, err:
         logging.error('%s', err)
         return 1
 
     from streamtar.server import make_streamtar, log_exceptions
-    app = make_streamtar(dl_paths, max_size, broken_uri, use_xsendfile)
+    app = make_streamtar(dl_paths, broken_uri, max_size, max_stream_sizes)
 
     auth_host = os.environ.get('DL_AUTH_HOST')
     auth_uri = os.environ.get('DL_AUTH_URI')
 
     if crossdomain_file:
         from streamtar.server import make_static
-        app = make_static(
-            {'/crossdomain.xml': crossdomain_file},
-            use_xsendfile
-        )(app)
+        app = make_static({'/crossdomain.xml': crossdomain_file})(app)
 
     app = log_exceptions(app)
 

streamtar/server.py

     return root
 
 
-def _handle_stream(root, base_uri, query_string, max_size=None,
-                   broken_uri=False, use_xsendfile=False):
+def _handle_stream(root, base_uri, query_string, broken_uri=False,
+                   max_size=None, max_stream_sizes=None):
     """Handles getting headers and stream object for the WSGI application
     function.
     """
 
     headers = []
-    using_xsendfile = False
     if os.path.isfile(root):
         logging.info('Request for file %r', root)
         if not _all_readable([root]):
         headers.append(('Content-Type', type_))
         if encoding:
             headers.append(('Content-Encoding', encoding))
-        if use_xsendfile:
-            logging.debug('Sending X-Sendfile with path %r', root)
-            using_xsendfile = True
-            headers.append(('X-Sendfile', root))
-            stream = ['']
-            size = os.stat(root).st_size
+        if max_stream_sizes:
+            max_size = max_stream_sizes.get(os.path.splitext(root)[1])
         else:
-            stream = FileStream(root)
-            size = stream.size()
+            max_size = None
+        stream = FileStream(root, max_size)
         ext = ''
     elif os.path.isdir(root):
         args = parse_qs(query_string)
             stream = TarStream(root, paths)
             ext = '.tar'
         logging.debug('Paths: %s', paths)
-        size = stream.size()
     else:
         logging.warn('Neither file nor directory: %r', root)
         raise ForbiddenError()
 
+    size = stream.size()
     logging.debug('Stream size: %s (maximum: %s)', size, max_size)
     if max_size and size > max_size:
         logging.warn('File too large: %r (size: %r, max: %r)', root,
     filename = os.path.basename(root.rstrip(os.sep).replace('"', ''))
     headers.append(('Content-Disposition',
                     'inline; filename="%s%s"' % (filename, ext)))
-    # mod_xsendfile sends its own Content-Length
-    if not using_xsendfile:
-        headers.append(('Content-Length', str(size)))
+    headers.append(('Content-Length', str(size)))
 
-    # mod_xsendfile sends its own ETag
-    if using_xsendfile:
-        etag = ''
-    else:
-        etag = sha.new()
-        etag.update(str(size))
-        etag.update(str(os.stat(root).st_mtime))
-        etag = etag.hexdigest()
-        headers.append(('ETag', etag))
+    etag = sha.new()
+    etag.update(str(size))
+    etag.update(str(os.stat(root).st_mtime))
+    etag = etag.hexdigest()
+    headers.append(('ETag', etag))
 
     return headers, stream, etag
 
 
-def make_streamtar(dl_paths, max_size=None, broken_uri=False,
-                   use_xsendfile=False):
+def make_streamtar(dl_paths, broken_uri=False,
+                   max_size=None, max_stream_sizes=None):
     """Creates a streamtar WSGI application function.
 
     dl_paths should be a list of 2-tuples, mapping prefixes to paths.
 
+    If broken_uri is True, an extra level of URI unescaping will be done on
+    requested URIs to compensate for weird configurations.
+
     If max_size is set, downloads greater than max_size bytes will be
     forbidden.
 
-    If broken_uri is True, an extra level of URI unescaping will be done on
-    requested URIs to compensate for weird configurations.
-
-    If use_xsendfile is True, streamed files (such as MP3s) are specified
-    in the X-Sendfile header, to be sent by the web server itself.
+    If max_stream_sizes is set, it should be a dict mapping file extensions
+    to size limits (in bytes).
 
     Sends 404 Not Found for invalid requested paths.
 
             query_string = environ.get('QUERY_STRING', '')
             base_uri = request_uri(environ, include_query=False)
             headers, stream, etag = _handle_stream(
-                root, base_uri, query_string, max_size, broken_uri,
-                use_xsendfile
+                root, base_uri, query_string, broken_uri, max_size,
+                max_stream_sizes
             )
 
             logging.debug('Sending headers: %s', headers)
     return wrapper
 
 
-def make_static(paths, use_xsendfile=False):
+def make_static(paths):
     """WSGI middleware that serves URLs mapped to paths"""
 
     def static(app):
 
                 headers = []
                 base_uri = request_uri(environ, include_query=False)
-                headers, stream, etag = _handle_stream(
-                    root, base_uri, '', None, False, use_xsendfile
-                )
+                headers, stream, etag = _handle_stream(root, base_uri, '')
 
                 logging.debug('Sending headers: %s', headers)
                 if environ.get('HTTP_IF_NONE_MATCH') == etag:

streamtar/stream.py

     file.
     """
 
-    def __init__(self, path):
+    def __init__(self, path, max_size=None):
         """Initializes a FileStream for the file at path"""
 
         self._path = path
         self._len = os.stat(path).st_size
+        if max_size is not None and max_size < self._len:
+            self._len = max_size
 
     def size(self):
         """Returns the length of the file. This can be called before
 
         file_ = open(self._path, 'rb')
         try:
+            bytes_left = self._len
             while True:
-                chunk = file_.read(BLOCKSIZE)
-                if not chunk:
+                if bytes_left < BLOCKSIZE:
+                    chunk = file_.read(bytes_left)
+                    if chunk:
+                        yield chunk
                     break
-                yield chunk
+                else:
+                    chunk = file_.read(BLOCKSIZE)
+                    if not chunk:
+                        break
+                    bytes_left -= len(chunk)
+                    yield chunk
             file_.close()
         except:
             file_.close()
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.