Commits

Kevin Chan committed 820c073

Added custom configs for remote storage; updated code to save local and
remote versions of images -- to get compatibility working for non-local
file system storage backends.

  • Participants
  • Parent commits af255e3
  • Tags 3.0.3

Comments (0)

Files changed (6)

File filebrowser/base.py

 
 # filebrowser imports
 from filebrowser.settings import *
-from filebrowser.functions import get_file_type, url_join, is_selectable, get_version_path
+from filebrowser.functions import (
+    get_file_type,
+    url_join,
+    is_selectable,
+    get_version_path,
+    get_media_url
+)
 from django.utils.encoding import force_unicode
 
 # PIL import
         import Image
 
 
+from garage.logger import logger
+
+
 class FileObject(object):
     """
     The FileObject represents a File on the Server.
-    
+
     PATH has to be relative to MEDIA_ROOT.
     """
-    
+    media_url = get_media_url()
+
     def __init__(self, path):
         self.path = path
         self.url_rel = path.replace("\\","/")
         self.filename = os.path.split(path)[1]
         self.filename_lower = self.filename.lower() # important for sorting
         self.filetype = get_file_type(self.filename)
-    
+
     def _filesize(self):
         """
         Filesize.
             return os.path.getsize(os.path.join(MEDIA_ROOT, path))
         return ""
     filesize = property(_filesize)
-    
+
     def _date(self):
         """
         Date.
             return os.path.getmtime(os.path.join(MEDIA_ROOT, self.path))
         return ""
     date = property(_date)
-    
+
     def _datetime(self):
         """
         Datetime Object.
         """
         return datetime.datetime.fromtimestamp(self.date)
     datetime = property(_datetime)
-    
+
     def _extension(self):
         """
         Extension.
         """
         return u"%s" % os.path.splitext(self.filename)[1]
     extension = property(_extension)
-    
+
     def _filetype_checked(self):
         if self.filetype == "Folder" and os.path.isdir(self.path_full):
             return self.filetype
         else:
             return ""
     filetype_checked = property(_filetype_checked)
-    
+
     def _path_full(self):
         """
         Full server PATH including MEDIA_ROOT.
         """
         return os.path.join(MEDIA_ROOT, self.path)
     path_full = property(_path_full)
-    
+
     def _path_relative(self):
         return self.path
     path_relative = property(_path_relative)
-    
+
     def _path_relative_directory(self):
         """
         Path relative to initial directory.
         value = directory_re.sub('', self.path)
         return u"%s" % value
     path_relative_directory = property(_path_relative_directory)
-    
+
     def _url_relative(self):
         return self.url_rel
     url_relative = property(_url_relative)
-    
+
     def _url_full(self):
         """
         Full URL including MEDIA_URL.
         """
-        return force_unicode(url_join(MEDIA_URL, self.url_rel))
+        return force_unicode(url_join(self.media_url, self.url_rel))
     url_full = property(_url_full)
-    
+
     def _url_save(self):
         """
         URL used for the filebrowsefield.
         else:
             return self.url_rel
     url_save = property(_url_save)
-    
+
     def _url_thumbnail(self):
         """
         Thumbnail URL.
         """
         if self.filetype == "Image":
-            return u"%s" % url_join(MEDIA_URL, get_version_path(self.path, 'fb_thumb'))
+            return u"%s" % url_join(self.media_url, get_version_path(self.path, 'fb_thumb'))
         else:
             return ""
     url_thumbnail = property(_url_thumbnail)
-    
+
     def url_admin(self):
         if self.filetype_checked == "Folder":
             directory_re = re.compile(r'^(%s)' % (DIRECTORY))
             value = directory_re.sub('', self.path)
             return u"%s" % value
         else:
-            return u"%s" % url_join(MEDIA_URL, self.path)
-    
+            return u"%s" % url_join(self.media_url, self.path)
+
     def _dimensions(self):
         """
         Image Dimensions.
         else:
             return False
     dimensions = property(_dimensions)
-    
+
     def _width(self):
         """
         Image Width.
         """
         return self.dimensions[0]
     width = property(_width)
-    
+
     def _height(self):
         """
         Image Height.
         """
         return self.dimensions[1]
     height = property(_height)
-    
+
     def _orientation(self):
         """
         Image Orientation.
         else:
             return None
     orientation = property(_orientation)
-    
+
     def _is_empty(self):
         """
         True if Folder is empty, False if not.
         else:
             return None
     is_empty = property(_is_empty)
-    
+
     def __repr__(self):
         return force_unicode(self.url_save)
-    
+
     def __str__(self):
         return force_unicode(self.url_save)
-    
+
     def __unicode__(self):
         return force_unicode(self.url_save)
-
-

File filebrowser/functions.py

 # django imports
 from django.utils.translation import ugettext as _
 from django.utils.safestring import mark_safe
-from django.core.files import File
 from django.core.files.storage import default_storage
 from django.utils.encoding import smart_str
 
         import Image
 
 
+def get_media_url():
+    """
+    Get media url.
+    * return remote media url if USE_REMOTE_STORAGE is True.
+    """
+    if USE_REMOTE_STORAGE:
+        return REMOTE_MEDIA_URL
+    else:
+        return MEDIA_URL
+
+
 def url_to_path(value):
     """
     Change URL to PATH.
     Value has to be an URL relative to MEDIA URL or a full URL (including MEDIA_URL).
-    
+
     Returns a PATH relative to MEDIA_ROOT.
     """
-    
+
     mediaurl_re = re.compile(r'^(%s)' % (MEDIA_URL))
     value = mediaurl_re.sub('', value)
     return value
     """
     Change PATH to URL.
     Value has to be a PATH relative to MEDIA_ROOT.
-    
+
     Return an URL relative to MEDIA_ROOT.
     """
-    
+
     mediaroot_re = re.compile(r'^(%s)' % (MEDIA_ROOT))
     value = mediaroot_re.sub('', value)
     return url_join(MEDIA_URL, value)
     URL has to be an absolute URL including MEDIA_URL or
     an URL relative to MEDIA_URL.
     """
-    
+
     mediaurl_re = re.compile(r'^(%s)' % (MEDIA_URL))
     value = mediaurl_re.sub('', value)
     directory_re = re.compile(r'^(%s)' % (DIRECTORY))
     """
     Construct the PATH to an Image version.
     Value has to be server-path, relative to MEDIA_ROOT.
-    
+
     version_filename = filename + version_prefix + ext
     Returns a path relative to MEDIA_ROOT.
     """
-    
+
     if os.path.isfile(smart_str(os.path.join(MEDIA_ROOT, value))):
         path, filename = os.path.split(value)
         filename, ext = os.path.splitext(filename)
-        
+
         # check if this file is a version of an other file
         # to return filename_<version>.ext instead of filename_<version>_<version>.ext
         tmp = filename.split("_")
                 # or we get a <VERSIONS_BASEDIR>/<VERSIONS_BASEDIR>/... construct
                 if VERSIONS_BASEDIR != "":
                         path = path.replace(VERSIONS_BASEDIR + "/", "")
-        
+
         version_filename = filename + "_" + version_prefix + ext
         return os.path.join(VERSIONS_BASEDIR, path, version_filename)
     else:
 def sort_by_attr(seq, attr):
     """
     Sort the sequence of objects by object's attribute
-    
+
     Arguments:
     seq  - the list or any sequence (including immutable one) of objects to sort.
     attr - the name of attribute to sort by
-    
+
     Returns:
     the sorted list of objects.
     """
     import operator
-    
+
     # Use the "Schwartzian transform"
     # Create the auxiliary list of tuples where every i-th tuple has form
     # (seq[i].attr, i, seq[i]) and sort it. The second item of tuple is needed not
     """
     URL join routine.
     """
-    
+
     if args[0].startswith("http://"):
         url = "http://"
     else:
     """
     Get Path.
     """
-    
+
     if path.startswith('.') or os.path.isabs(path) or not os.path.isdir(os.path.join(MEDIA_ROOT, DIRECTORY, path)):
         return None
     return path
     """
     Get File.
     """
-    
+
     converted_path = smart_str(os.path.join(MEDIA_ROOT, DIRECTORY, path, filename))
-    
+
     if not os.path.isfile(converted_path) and not os.path.isdir(converted_path):
         return None
     return filename
     """
     Get breadcrumbs.
     """
-    
+
     breadcrumbs = []
     dir_query = ""
     if path:
     """
     Get filterdate.
     """
-    
+
     returnvalue = ''
     dateYear = strftime("%Y", gmtime(dateTime))
     dateMonth = strftime("%m", gmtime(dateTime))
     """
     Get settings variables used for FileBrowser listing.
     """
-    
+
     settings_var = {}
     # Main
     settings_var['DEBUG'] = DEBUG
     settings_var['MAX_UPLOAD_SIZE'] = MAX_UPLOAD_SIZE
     # Convert Filenames
     settings_var['CONVERT_FILENAME'] = CONVERT_FILENAME
+    # custom configurations
+    settings_var['USE_REMOTE_STORAGE'] = USE_REMOTE_STORAGE
+    settings_var['REMOTE_UPLOAD_PATH_PREFIX'] = REMOTE_UPLOAD_PATH_PREFIX
+    settings_var['REMOTE_MEDIA_URL'] = REMOTE_MEDIA_URL
     return settings_var
 
 
-def handle_file_upload(path, file):
+def handle_file_upload_OLD(path, file):
     """
-    Handle File Upload.
+    Handle File Upload. (OLD VERSION)
     """
-    
+
     file_path = os.path.join(path, file.name)
     uploadedfile = default_storage.save(file_path, file)
     os.chmod(uploadedfile, get_settings_var().get('DEFAULT_PERMISSIONS', DEFAULT_PERMISSIONS))
     return uploadedfile
 
 
+def handle_file_upload(path, file):
+    """
+    Handle File Upload.
+    * modified to handle remote storage.
+    * saves local copy and copies to remote storage.
+    """
+    from filebrowser.remote_storage import save_local_file, copy_to_remote
+    settings = get_settings_var()
+    if USE_REMOTE_STORAGE:
+        local_path = os.path.join(path, file.name)
+        local_file = save_local_file(local_path, file)
+        os.chmod(local_file, settings.get('DEFAULT_PERMISSIONS', DEFAULT_PERMISSIONS))
+        uploadedfile = local_file
+        remote_path = copy_to_remote(local_path)
+    else:
+        file_path = os.path.join(path, file.name)
+        uploadedfile = default_storage.save(file_path, file)
+        os.chmod(uploadedfile, settings.get('DEFAULT_PERMISSIONS', DEFAULT_PERMISSIONS))
+    return uploadedfile
+
+
 def get_file_type(filename):
     """
     Get file type as defined in EXTENSIONS.
     """
-    
+
     file_extension = os.path.splitext(filename)[1].lower()
     file_type = ''
     for k,v in EXTENSIONS.iteritems():
     """
     Get select type as defined in FORMATS.
     """
-    
+
     file_extension = os.path.splitext(filename)[1].lower()
     select_types = []
     for k,v in SELECT_FORMATS.iteritems():
     Generate Version for an Image.
     value has to be a serverpath relative to MEDIA_ROOT.
     """
-    
+    from filebrowser.remote_storage import copy_to_remote
+
     # PIL's Error "Suspension not allowed here" work around:
     # s. http://mail.python.org/pipermail/image-sig/1999-August/000816.html
     if STRICT_PIL:
         except ImportError:
             import ImageFile
     ImageFile.MAXBLOCK = IMAGE_MAXBLOCK # default is 64k
-    
+
+    im = Image.open(smart_str(os.path.join(MEDIA_ROOT, value)))
+    version_path = get_version_path(value, version_prefix)
+    absolute_version_path = smart_str(os.path.join(MEDIA_ROOT, version_path))
+    version_dir = os.path.split(absolute_version_path)[0]
+    if not os.path.isdir(version_dir):
+        os.makedirs(version_dir)
+        os.chmod(version_dir, 0775)
+    version = scale_and_crop(im, VERSIONS[version_prefix]['width'], VERSIONS[version_prefix]['height'], VERSIONS[version_prefix]['opts'])
     try:
-        im = Image.open(smart_str(os.path.join(MEDIA_ROOT, value)))
-        version_path = get_version_path(value, version_prefix)
-        absolute_version_path = smart_str(os.path.join(MEDIA_ROOT, version_path))
-        version_dir = os.path.split(absolute_version_path)[0]
-        if not os.path.isdir(version_dir):
-            os.makedirs(version_dir)
-            os.chmod(version_dir, 0775)
-        version = scale_and_crop(im, VERSIONS[version_prefix]['width'], VERSIONS[version_prefix]['height'], VERSIONS[version_prefix]['opts'])
-        try:
-            version.save(absolute_version_path, quality=90, optimize=(os.path.splitext(version_path)[1].lower() != '.gif'))
-        except IOError:
-            version.save(absolute_version_path, quality=90)
-        return version_path
-    except:
-        return None
+        version.save(absolute_version_path, quality=90, optimize=(os.path.splitext(version_path)[1].lower() != '.gif'))
+    except IOError:
+        version.save(absolute_version_path, quality=90)
+
+    if USE_REMOTE_STORAGE:
+        remote_path = copy_to_remote(absolute_version_path)
+
+    return version_path
 
 
 def scale_and_crop(im, width, height, opts):
     """
     Scale and Crop.
     """
-    
+
     x, y   = [float(v) for v in im.size]
     if width:
         xr = float(width)
         yr = float(height)
     else:
         yr = float(y*width/x)
-    
+
     if 'crop' in opts:
         r = max(xr/x, yr/y)
     else:
         r = min(xr/x, yr/y)
-    
+
     if r < 1.0 or (r > 1.0 and 'upscale' in opts):
         im = im.resize((int(x*r), int(y*r)), resample=Image.ANTIALIAS)
-    
+
     if 'crop' in opts:
         x, y   = [float(v) for v in im.size]
         ex, ey = (x-min(x, xr))/2, (y-min(y, yr))/2
         if ex or ey:
             im = im.crop((int(ex), int(ey), int(x-ex), int(y-ey)))
     return im
-    
+
     # if 'crop' in opts:
     #     if 'top_left' in opts:
     #         #draw cropping box from upper left corner of image
     #             box = (int(ex), int(ey), int(x-ex), int(y-ey))
     #             im = im.resize((int(x), int(y)), resample=Image.ANTIALIAS).crop(box)
     # return im
-    
+
 scale_and_crop.valid_options = ('crop', 'upscale')
 
 
     """
     Convert Filename.
     """
-    
+
     if CONVERT_FILENAME:
         return value.replace(" ", "_").lower()
     else:
         return value
-
-

File filebrowser/remote_storage.py

+# -*- coding: utf-8 -*-
+"""
+remote_storage
+
+Helper functions to handle remote storage.
+
+* created: 2013-08-14 Kevin Chan <kefin@makedostudio.com>
+* updated: 2013-08-14 kchan
+"""
+
+import os.path
+
+from django.core.files.storage import default_storage, FileSystemStorage
+from django.core.files import File
+
+# filebrowser imports
+from filebrowser.settings import *
+
+
+# local storage
+# * this is used to save a copy of the upload file to the local file system
+#   while the a mirrot version is saved to remote storage.
+file_system_storage = FileSystemStorage()
+
+
+def local_path_to_remote_url(local_path):
+    """
+    Convert a local path to a remote media url.
+    """
+    media_url = REMOTE_MEDIA_URL
+    local_path_prefix = MEDIA_ROOT
+    remote_path_prefix = REMOTE_UPLOAD_PATH_PREFIX
+    relpath = local_path.replace(local_path_prefix, '')
+    url = '%s%s%s' % (media_url, remote_path_prefix, relpath)
+    return url
+
+
+def get_remote_path(local_path):
+    """
+    Helper function to convert local path to remote path.
+    * if USE_REMOTE_STORAGE is True.
+    """
+    local_path_prefix = MEDIA_ROOT
+    remote_path_prefix = REMOTE_UPLOAD_PATH_PREFIX
+    relpath = local_path.replace(local_path_prefix, '')
+    remote_path = '%s%s' % (remote_path_prefix, relpath)
+    return remote_path
+
+
+def save_local_file(path, file):
+    """
+    Save file to local file system.
+    """
+    local_file = file_system_storage.save(path, file)
+    return local_file
+
+
+def get_img_ext(path, default_ext='unknown'):
+    """
+    Detect image type from file path and return file extension.
+    """
+    import imghdr
+    imgtype = imghdr.what(path)
+    suffix = {
+        'rgb': 'rgb',
+        'gif': 'gif',
+        'pbm': 'pbm',
+        'pgm': 'pgm',
+        'ppm': 'ppm',
+        'tiff': 'tif',
+        'rast': 'rast',
+        'xbm': 'xbm',
+        'jpeg': 'jpg',
+        'bmp': 'bmp',
+        'png': 'png'
+    }
+    return suffix.get(imgtype, default_ext)
+
+
+def copy_to_remote(path, storage=None):
+    """
+    Push a local file to remote media storage.
+
+    :param path: local path of file
+    :param storage: storage instance to use (default: default_storage)
+    :returns: url of file on remote storage server
+    """
+    FREAD_CHUNK_SIZE = 1024
+    UNKNOWN_MIME_TYPE = 'application/x-octet-stream'
+    MIME_TYPES = {
+        'gif': 'image/gif',
+        'jpg': 'image/jpeg',
+        'png': 'image/png',
+        'bmp': 'image/bmp',
+        'tiff': 'image/tiff',
+        'unknown': UNKNOWN_MIME_TYPE,
+    }
+
+    if not path or not os.path.isfile(path):
+        remote_path = path
+    else:
+        if not storage:
+            storage = default_storage
+
+        media_url = REMOTE_MEDIA_URL
+        media_root = MEDIA_ROOT
+        subpath = path.replace(media_root, '')
+        fkey = '%s%s' % (REMOTE_UPLOAD_PATH_PREFIX, subpath)
+
+        # determine mime type
+        fext = get_img_ext(path)
+        content_type = MIME_TYPES.get(fext, UNKNOWN_MIME_TYPE)
+
+        # write file to remote server
+        file = storage.open(fkey, 'w')
+        storage.headers.update({"Content-Type": content_type})
+        f = open(path, 'rb')
+        media = File(f)
+        for chunk in media.chunks(chunk_size=FREAD_CHUNK_SIZE):
+            file.write(chunk)
+        file.close()
+        media.close()
+        f.close()
+
+        # construct remote url
+        remote_path = '%s%s' % (media_url, subpath)
+
+    return remote_path
+
+
+def delete_remote(url, storage=None):
+    """
+    Delete file on remote storage.
+    """
+    if storage is None:
+        storage = default_storage
+    media_url = REMOTE_MEDIA_URL
+    path = url.replace(media_url, '')
+    storage.delete(path)
+
+
+def rename_remote(old_local_path, new_local_path):
+    """
+    Rename remote file.
+    * delete the old one
+    * and copy the new one to remote
+    """
+    delete_remote(local_path_to_remote_url(old_local_path))
+    copy_to_remote(new_local_path)

File filebrowser/settings.py

 # default uploaded file permissions
 DEFAULT_PERMISSIONS = getattr(settings, 'FILEBROWSER_DEFAULT_PERMISSIONS', 0755)
 
+# custom settings for remote storage
+USE_REMOTE_STORAGE = getattr(settings, 'FILEBROWSER_USE_REMOTE_STORAGE', False)
+REMOTE_UPLOAD_PATH_PREFIX = getattr(settings, 'FILEBROWSER_REMOTE_UPLOAD_PATH_PREFIX', '')
+REMOTE_MEDIA_URL = getattr(settings, 'FILEBROWSER_REMOTE_MEDIA_URL', MEDIA_URL)
+
 # EXTRA TRANSLATION STRINGS
 # The following strings are not availabe within views or templates
 _('Folder')
 _('Document')
 _('Audio')
 _('Code')
-
-

File filebrowser/urls.py

-from django.conf.urls.defaults import *
+from django.conf.urls import patterns, include, url
 
 urlpatterns = patterns('',
     

File filebrowser/views.py

 from django.dispatch import Signal
 from django.core.paginator import Paginator, InvalidPage, EmptyPage
 from django.utils.encoding import smart_str
+from django.core.files.storage import default_storage
 
 try:
     # django SVN
 
 # filebrowser imports
 from filebrowser.settings import *
-from filebrowser.functions import path_to_url, sort_by_attr, get_path, get_file, get_version_path, get_breadcrumbs, get_filterdate, get_settings_var, handle_file_upload, convert_filename
+from filebrowser.functions import (
+    path_to_url,
+    sort_by_attr,
+    get_path,
+    get_file,
+    get_version_path,
+    get_breadcrumbs,
+    get_filterdate,
+    get_settings_var,
+    handle_file_upload,
+    convert_filename
+)
 from filebrowser.templatetags.fb_tags import query_helper
 from filebrowser.base import FileObject
 from filebrowser.decorators import flash_login_required
+from filebrowser.remote_storage import (
+    local_path_to_remote_url,
+    delete_remote,
+    rename_remote
+)
+
 
 # Precompile regular expressions
 filter_re = []
                 filebrowser_pre_delete.send(sender=request, path=path, filename=filename)
                 # DELETE IMAGE VERSIONS/THUMBNAILS
                 for version in VERSIONS:
+                    local_path = os.path.join(MEDIA_ROOT, get_version_path(relative_server_path, version))
                     try:
-                        os.unlink(os.path.join(MEDIA_ROOT, get_version_path(relative_server_path, version)))
+                        os.unlink(local_path)
                     except:
                         pass
+                    if USE_REMOTE_STORAGE:
+                        delete_remote(local_path_to_remote_url(local_path))
+
                 # DELETE FILE
-                os.unlink(smart_str(os.path.join(abs_path, filename)))
+                local_path = smart_str(os.path.join(abs_path, filename))
+                os.unlink(local_path)
+                if USE_REMOTE_STORAGE:
+                    delete_remote(local_path_to_remote_url(local_path))
+
                 # POST DELETE SIGNAL
                 filebrowser_post_delete.send(sender=request, path=path, filename=filename)
                 # MESSAGE & REDIRECT
             try:
                 # PRE DELETE SIGNAL
                 filebrowser_pre_delete.send(sender=request, path=path, filename=filename)
+
                 # DELETE FOLDER
-                os.rmdir(os.path.join(abs_path, filename))
+                local_path = os.path.join(abs_path, filename)
+                os.rmdir(local_path)
+                if USE_REMOTE_STORAGE:
+                    delete_remote(local_path_to_remote_url(local_path))
+
                 # POST DELETE SIGNAL
                 filebrowser_post_delete.send(sender=request, path=path, filename=filename)
                 # MESSAGE & REDIRECT
                 # DELETE IMAGE VERSIONS/THUMBNAILS
                 # regenerating versions/thumbs will be done automatically
                 for version in VERSIONS:
+                    local_path = os.path.join(MEDIA_ROOT, get_version_path(relative_server_path, version))
                     try:
-                        os.unlink(os.path.join(MEDIA_ROOT, get_version_path(relative_server_path, version)))
+                        os.unlink(local_path)
                     except:
                         pass
+                    if USE_REMOTE_STORAGE:
+                        delete_remote(local_path_to_remote_url(local_path))
+
                 # RENAME ORIGINAL
-                os.rename(os.path.join(MEDIA_ROOT, relative_server_path), os.path.join(MEDIA_ROOT, new_relative_server_path))
+                old_path = os.path.join(MEDIA_ROOT, relative_server_path)
+                new_path = os.path.join(MEDIA_ROOT, new_relative_server_path)
+                os.rename(old_path, new_path)
+                if USE_REMOTE_STORAGE:
+                    rename_remote(old_path, new_path)
+
                 # POST RENAME SIGNAL
                 filebrowser_post_rename.send(sender=request, path=path, filename=filename, new_filename=new_filename)
                 # MESSAGE & REDIRECT
         'breadcrumbs_title': _(u'Versions for "%s"') % filename
     }, context_instance=Context(request))
 versions = staff_member_required(never_cache(versions))
-
-