Aymeric Barantal avatar Aymeric Barantal committed 4634a2f

apache libcloud storage backend, work at least with google storage

Comments (0)

Files changed (5)

examples/libcloud_project/manage.py

+#!/usr/bin/env python
+from django.core.management import execute_manager
+import imp
+try:
+    imp.find_module('settings') # Assumed to be in the same directory.
+except ImportError:
+    import sys
+    sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n" % __file__)
+    sys.exit(1)
+
+import settings
+
+if __name__ == "__main__":
+    execute_manager(settings)

examples/libcloud_project/settings.py

+# Django settings for libcloud_project project.
+
+DEBUG = True
+TEMPLATE_DEBUG = DEBUG
+
+ADMINS = (
+    # ('Your Name', 'your_email@example.com'),
+)
+
+MANAGERS = ADMINS
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
+        'NAME': 'libcloud_project.sqllite',                      # Or path to database file if using sqlite3.
+        'USER': '',                      # Not used with sqlite3.
+        'PASSWORD': '',                  # Not used with sqlite3.
+        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
+        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
+    }
+}
+
+# Local time zone for this installation. Choices can be found here:
+# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
+# although not all choices may be available on all operating systems.
+# On Unix systems, a value of None will cause Django to use the same
+# timezone as the operating system.
+# If running in a Windows environment this must be set to the same as your
+# system time zone.
+TIME_ZONE = 'America/Chicago'
+
+# Language code for this installation. All choices can be found here:
+# http://www.i18nguy.com/unicode/language-identifiers.html
+LANGUAGE_CODE = 'en-us'
+
+SITE_ID = 1
+
+# If you set this to False, Django will make some optimizations so as not
+# to load the internationalization machinery.
+USE_I18N = True
+
+# If you set this to False, Django will not format dates, numbers and
+# calendars according to the current locale
+USE_L10N = True
+
+# Absolute filesystem path to the directory that will hold user-uploaded files.
+# Example: "/home/media/media.lawrence.com/media/"
+MEDIA_ROOT = ''
+
+# URL that handles the media served from MEDIA_ROOT. Make sure to use a
+# trailing slash.
+# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
+MEDIA_URL = ''
+
+# Absolute path to the directory static files should be collected to.
+# Don't put anything in this directory yourself; store your static files
+# in apps' "static/" subdirectories and in STATICFILES_DIRS.
+# Example: "/home/media/media.lawrence.com/static/"
+STATIC_ROOT = ''
+
+# URL prefix for static files.
+# Example: "http://media.lawrence.com/static/"
+STATIC_URL = '/static/'
+
+# URL prefix for admin static files -- CSS, JavaScript and images.
+# Make sure to use a trailing slash.
+# Examples: "http://foo.com/static/admin/", "/static/admin/".
+ADMIN_MEDIA_PREFIX = '/static/admin/'
+
+# Additional locations of static files
+STATICFILES_DIRS = (
+    # Put strings here, like "/home/html/static" or "C:/www/django/static".
+    # Always use forward slashes, even on Windows.
+    # Don't forget to use absolute paths, not relative paths.
+)
+
+# List of finder classes that know how to find static files in
+# various locations.
+STATICFILES_FINDERS = (
+    'django.contrib.staticfiles.finders.FileSystemFinder',
+    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
+)
+
+# Make this unique, and don't share it with anybody.
+SECRET_KEY = 'tdzq9m9k-u*k=furpki(@wejb&^2!ea4*z1^t9waj&$)(+$4(h'
+
+# List of callables that know how to import templates from various sources.
+TEMPLATE_LOADERS = (
+    'django.template.loaders.filesystem.Loader',
+    'django.template.loaders.app_directories.Loader',
+#     'django.template.loaders.eggs.Loader',
+)
+
+MIDDLEWARE_CLASSES = (
+    'django.middleware.common.CommonMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+)
+
+ROOT_URLCONF = 'libcloud_project.urls'
+
+TEMPLATE_DIRS = (
+    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
+    # Always use forward slashes, even on Windows.
+    # Don't forget to use absolute paths, not relative paths.
+)
+
+INSTALLED_APPS = (
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.sites',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    # Uncomment the next line to enable the admin:
+    # 'django.contrib.admin',
+    # Uncomment the next line to enable admin documentation:
+    # 'django.contrib.admindocs',
+)
+
+# A sample logging configuration. The only tangible logging
+# performed by this configuration is to send an email to
+# the site admins on every HTTP 500 error.
+# See http://docs.djangoproject.com/en/dev/topics/logging for
+# more details on how to customize your logging configuration.
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'handlers': {
+        'mail_admins': {
+            'level': 'ERROR',
+            'class': 'django.utils.log.AdminEmailHandler'
+        }
+    },
+    'loggers': {
+        'django.request': {
+            'handlers': ['mail_admins'],
+            'level': 'ERROR',
+            'propagate': True,
+        },
+    }
+}
+
+#
+# Specific project configuration
+#
+from libcloud.storage.types import Provider
+LIBCLOUD_PROVIDERS = {
+    'test_google_storage': {
+        'type': Provider.GOOGLE_STORAGE,
+        'user': '<google apiv1 user (20 char)>',
+        'key': '<google apiv1 key>',
+        'bucket': '<bucket name>'
+    }
+}
+
+DEFAULT_FILE_STORAGE = 'backends.backends.LibCloudStorage'

examples/libcloud_project/test_storage.py

+import sys, os
+PROJECT_PATH = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(PROJECT_PATH)
+
+from django.core.management import setup_environ
+import settings
+setup_environ(settings)
+
+from storages.backends.apache_libcloud import LibCloudStorage
+
+# test_google_storage is a key in settings LIBCLOUD_PROVIDERS dict
+store = LibCloudStorage('test_google_storage')
+# store is your django storage object that will use google storage
+# bucket specified in configuration

examples/libcloud_project/urls.py

+from django.conf.urls.defaults import patterns, include, url
+
+# Uncomment the next two lines to enable the admin:
+# from django.contrib import admin
+# admin.autodiscover()
+
+urlpatterns = patterns('',
+    # Examples:
+    # url(r'^$', 'libcloud_project.views.home', name='home'),
+    # url(r'^libcloud_project/', include('libcloud_project.foo.urls')),
+
+    # Uncomment the admin/doc line below to enable admin documentation:
+    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
+
+    # Uncomment the next line to enable the admin:
+    # url(r'^admin/', include(admin.site.urls)),
+)

storages/backends/apache_libcloud.py

+# Django storage using libcloud providers
+# Aymeric Barantal (mric at chamal.fr) 2011
+#
+import os
+
+from django.conf import settings
+from django.core.files.storage import Storage
+from django.core.files.base import File
+from django.core.exceptions import ImproperlyConfigured
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+
+try:
+    from libcloud.storage.providers import get_driver
+    from libcloud.storage.types import ObjectDoesNotExistError
+except ImportError:
+    raise ImproperlyConfigured("Could not load libcloud")
+
+
+class LibCloudStorage(Storage):
+    """Django storage derived class using apache libcloud to operate
+    on supported providers"""
+    def __init__(self, provider_name, option=None):
+        self.provider = settings.LIBCLOUD_PROVIDERS.get(provider_name)
+        if not self.provider:
+            raise ImproperlyConfigured(
+                'LIBCLOUD_PROVIDERS %s not define or invalid' % provider_name)
+        try:
+            Driver = get_driver(self.provider['type'])
+            self.driver = Driver(
+                self.provider['user'],
+                self.provider['key'],
+                )
+        except Exception, e:
+            raise ImproperlyConfigured(
+                "Unable to create libcloud driver type %s" % \
+                (self.provider.get('type'), e))
+        self.bucket = self.provider['bucket']   # Limit to one container
+
+    def _get_bucket(self):
+        """Helper to get bucket object (libcloud container)"""
+        return self.driver.get_container(self.bucket)
+
+    def _clean_name(self, name):
+        """Clean name (windows directories)"""
+        return os.path.normpath(name).replace('\\', '/')
+
+    def _get_object(self, name):
+        """Get object by its name. Return None if object not found"""
+        clean_name = self._clean_name(name)
+        try:
+            return self.driver.get_object(self.bucket, clean_name)
+        except ObjectDoesNotExistError, e:
+            return None
+
+    def delete(self, name):
+        """Delete objet on remote"""
+        obj = self._get_object(name)
+        if obj:
+            return self.driver.delete_object(obj)
+        else:
+            raise Exception('Object to delete does not exists')
+
+    def exists(self, name):
+        obj = self._get_object(name)
+        return True if obj else False
+
+    def listdir(self, path='/'):
+        """Lists the contents of the specified path,
+        returning a 2-tuple of lists; the first item being
+        directories, the second item being files.
+        """
+        container = self._get_bucket()
+        objects = self.driver.list_container_objects(container)
+        path = self._clean_name(path)
+        if not path.endswith('/'):
+            path = "%s/" % path
+        files = []
+        dirs = []
+        # TOFIX: better algorithm to filter correctly
+        # (and not depend on google-storage empty folder naming)
+        for o in objects:
+            if path == '/':
+                if o.name.count('/') == 0:
+                    files.append(o.name)
+                elif o.name.count('/') == 1:
+                    dir_name = o.name[:o.name.index('/')]
+                    if not dir_name in dirs:
+                        dirs.append(dir_name)
+            elif o.name.startswith(path):
+                if o.name.count('/') <= path.count('/'):
+                    # TOFIX : special case for google storage with empty dir
+                    if o.name.endswith('_$folder$'):
+                        name = o.name[:-9]
+                        name = name[len(path):]
+                        dirs.append(name)
+                    else:
+                        name = o.name[len(path):]
+                        files.append(name)
+        return (dirs, files)
+
+    def size(self, name):
+        obj = self._get_object(name)
+        if obj:
+            return obj.size
+        else:
+            return -1
+
+    def url(self, name):
+        obj = self._get_object(name)
+        return self.driver.get_object_cdn_url(obj)
+
+    def _open(self, name, mode='rb'):
+        remote_file = LibCloudFile(name, self, mode=mode)
+        return remote_file
+
+    def _read(self, name, start_range=None, end_range=None):
+        obj = self._get_object(name)
+        # TOFIX : we should be able to read chunk by chunk
+        return self.driver.download_object_as_stream(obj, obj.size).next()
+
+    def _save(self, name, file):
+        self.driver.upload_object_via_stream(file, self._get_bucket(), name)
+
+
+class LibCloudFile(File):
+    """File intherited class for libcloud storage objects read and write"""
+    def __init__(self, name, storage, mode):
+        self._name = name
+        self._storage = storage
+        self._mode = mode
+        self._is_dirty = False
+        self.file = StringIO()
+        self.start_range = 0
+
+    @property
+    def size(self):
+        if not hasattr(self, '_size'):
+            self._size = self._storage.size(self._name)
+        return self._size
+
+    def read(self, num_bytes=None):
+        if num_bytes is None:
+            args = []
+            self.start_range = 0
+        else:
+            args = [self.start_range, self.start_range + num_bytes - 1]
+        data = self._storage._read(self._name, *args)
+        self.file = StringIO(data)
+        return self.file.getvalue()
+
+    def write(self, content):
+        if 'w' not in self._mode:
+            raise AttributeError("File was opened for read-only access.")
+        self.file = StringIO(content)
+        self._is_dirty = True
+
+    def close(self):
+        if self._is_dirty:
+            self._storage._save(self._name, self.file)
+        self.file.close()
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.