Commits

akoha committed 8acb919

Pushing django-dbstorage to its own repo in preparation for open-sourcing it.

Comments (0)

Files changed (6)

django_dbstorage/__init__.py

Empty file added.

django_dbstorage/fields.py

+# Copied from http://www.djangosnippets.org/snippets/1669/
+#
+# We would like to use a BlobField but that doesn't exist in Django yet:
+# http://code.djangoproject.com/ticket/2417
+
+import base64
+
+from django.db import models
+
+
+class Base64Field(models.TextField):
+
+    def contribute_to_class(self, cls, name):
+        if self.db_column is None:
+            self.db_column = name
+        self.field_name = name + '_base64'
+        super(Base64Field, self).contribute_to_class(cls, self.field_name)
+        setattr(cls, name, property(self.get_data, self.set_data))
+
+    def get_data(self, obj):
+        return base64.b64decode(getattr(obj, self.field_name))
+
+    def set_data(self, obj, data):
+        setattr(obj, self.field_name, base64.b64encode(data))

django_dbstorage/models.py

+from errno import EBADF
+import os
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+from StringIO import _complain_ifclosed
+
+from django.conf import settings
+from django.db import models
+
+from django_dbstorage.fields import Base64Field
+
+
+__all__ = ['File']
+
+class FileManager(models.Manager):
+    def open(self, name, mode):
+        f = self.get(name=name)
+        f.mode = mode
+        return f
+
+
+class File(models.Model):
+    """Model for treating rows in the database as file streams."""
+
+    name = models.CharField(max_length=getattr(settings, 'PATH_MAX', 100),
+                            primary_key=True, editable=False)
+    data = Base64Field(blank=True, editable=False)
+    size = models.PositiveIntegerField(editable=False)
+
+    objects = FileManager()
+
+    def __init__(self, *args, **kwargs):
+        mode = kwargs.pop('mode', 'r+b')
+        super(File, self).__init__(*args, **kwargs)
+        self._file = StringIO()
+        self._file.write(self.data)
+        self._file.seek(0)
+        self.encoding = None
+        self.errors = None
+        self.mode = mode
+        self.newlines = None
+        self.softspace = 0
+
+    def __unicode__(self):
+        output = '%r, mode %r' % (self.name, self.mode)
+        if self.closed:
+            output = 'closed ' + output
+        return output
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        _complain_ifclosed(self.closed)
+        r = self._file.readline()
+        if not r:
+            raise StopIteration
+        return r
+
+    def close(self):
+        """close() -> None.  Flushes and releases resources held."""
+        if not self.closed:
+            if not self._isreadonly():
+                self.flush()
+            self._file = None
+
+    def _closed(self):
+        return self._file is None
+    closed = property(_closed)
+
+    def isatty(self):
+        """isatty() -> False."""
+        _complain_ifclosed(self.closed)
+        return False
+
+    def seek(self, offset, whence=0):
+        """
+        seek(offset[, whence]) -> None.  Move to new file position.
+
+        Argument offset is a byte count.  Optional argument whence defaults to
+        0 (offset from start of file, offset should be >= 0); other values are 1
+        (move relative to current position, positive or negative), and 2 (move
+        relative to end of file, usually negative, but allows seeking beyond
+        the end of a file).
+        """
+        _complain_ifclosed(self.closed)
+        return self._file.seek(offset, whence)
+
+    def tell(self):
+        """
+        tell() -> current file position, an integer (may be a long integer).
+        """
+        _complain_ifclosed(self.closed)
+        return self._file.tell()
+
+    def read(self, size=-1):
+        """read([size]) -> read at most size bytes, returned as a string."""
+        _complain_ifclosed(self.closed)
+        return self._file.read(size)
+
+    def readline(self):
+        """readline() -> next line from the file, as a string."""
+        _complain_ifclosed(self.closed)
+        return self._file.readline()
+
+    def readlines(self):
+        """readlines() -> list of strings, each a line from the file."""
+        _complain_ifclosed(self.closed)
+        return self._file.readlines()
+
+    def truncate(self, size=None):
+        """
+        truncate([size]) -> None.  Truncate the file to at most size bytes.
+
+        Size defaults to the current file position, as returned by tell().
+        """
+        _complain_ifclosed(self.closed)
+        _complain_ifreadonly(self._isreadonly())
+        position = self._file.tell()
+        try:
+            if size is not None:
+                self._file.seek(size)
+            return self._file.truncate()
+        finally:
+            self._file.seek(position)
+
+    def write(self, str):
+        """
+        write(str) -> None.  Write string str to file.
+
+        Note that due to buffering, flush() or close() are needed before
+        the database reflects the data written.
+        """
+        _complain_ifclosed(self.closed)
+        _complain_ifreadonly(self._isreadonly())
+        return self._file.write(str)
+
+    def writelines(self, iterable):
+        """
+        writelines(iterable) -> None.  Write the iterable of strings to file.
+
+        Note that newlines are not added.  This is equalivalent to calling
+        write() for each string in the iterable.
+        """
+        _complain_ifclosed(self.closed)
+        _complain_ifreadonly(self._isreadonly())
+        return self._file.writelines(iterable)
+
+    def flush(self):
+        """flush() -> None.  Flush the internal I/O buffer."""
+        _complain_ifclosed(self.closed)
+        _complain_ifreadonly(self._isreadonly())
+        self.save()
+
+    def save_base(self, *args, **kwargs):
+        self.data = self._file.getvalue()
+        self.size = self._size()
+        super(File, self).save_base(*args, **kwargs)
+
+    def _size(self):
+        """_size() -> current file size, an integer (may be a long integer)."""
+        position = self._file.tell()
+        try:
+            self._file.seek(0, 2)
+            return self._file.tell()
+        finally:
+            self._file.seek(position)
+
+    def _isreadonly(self):
+        """_isreadonly() -> true or false.  True if mode is readonly."""
+        return not ('+' in self.mode or 'w' in self.mode or 'a' in self.mode)
+
+
+def _complain_ifreadonly(readonly):
+    if readonly:
+        raise IOError(EBADF, os.strerror(EBADF))

django_dbstorage/storage.py

+from errno import EEXIST, EISDIR, ENOENT, ENOTDIR
+import os
+try:
+    set
+except NameError:
+    from sets import Set as set
+import urlparse
+
+from django.conf import settings
+from django.core.files.storage import Storage
+from django.db import IntegrityError
+
+from django_dbstorage.models import File
+
+
+class DatabaseStorage(Storage):
+    def __init__(self, location=None, base_url=None, uniquify_names=True):
+        if location is None:
+            location = settings.MEDIA_ROOT
+        self.location = location
+        if base_url is None:
+            base_url = settings.MEDIA_URL
+        self.base_url = base_url
+        self.uniquify_names = uniquify_names
+
+    def _open(self, name, mode='rb'):
+        name = self._name(name)
+        try:
+            return File.objects.open(name=name, mode=mode)
+        except File.DoesNotExist:
+            raise IOError(ENOENT, os.strerror(ENOENT))
+
+    def _save(self, name, content):
+        if name.endswith(os.path.sep):
+            raise IOError(EISDIR, os.strerror(EISDIR))
+        name = self._name(name)
+        if not name:
+            raise IOError(ENOENT, os.strerror(ENOENT))
+        # Extract the data from content
+        data = content.read()
+        # Save to the database.
+        while True:
+            try:
+                File.objects.create(name=os.path.normpath(name), data=data)
+            except IntegrityError:
+                # File exists. We need a new file name.
+                if not self.uniquify_names:
+                    raise IOError(EEXIST, os.strerror(EEXIST))
+                name = self.get_available_name(name)
+            else:
+                # OK, the file save worked. Break out of the loop.
+                break
+        return name
+
+    def get_available_name(self, name):
+        """
+        Returns a filename that's free on the target storage system, and
+        available for new content to be written to.
+        """
+        name = self._name(name)
+        dir_name, file_name = os.path.split(name)
+        file_root, file_ext = os.path.splitext(file_name)
+        # If the filename already exists, keep adding an underscore (before the
+        # file extension, if one exists) to the filename until the generated
+        # filename doesn't exist.
+        while self.exists(name):
+            file_root += '_'
+            # file_ext includes the dot.
+            name = os.path.join(dir_name, file_root + file_ext)
+        return name
+
+    def delete(self, name):
+        """
+        Deletes the specified file from the storage system.
+        """
+        name = self._name(name)
+        File.objects.filter(name=name).delete()
+
+    def exists(self, name):
+        """
+        Returns True if a file referened by the given name already exists in the
+        storage system, or False if the name is available for a new file.
+        """
+        name = self._name(name)
+        return bool(File.objects.filter(name=name).count())
+
+    def listdir(self, path):
+        """
+        Lists the contents of the specified path, returning a 2-tuple of lists;
+        the first item being directories, the second item being files.
+        """
+        path = self._name(path)
+        if path and not path.endswith(os.path.sep):
+            path += os.path.sep
+        directories, files = set(), []
+        entries = File.objects.filter(name__startswith=path)
+        entries = entries.values_list('name', flat=True)
+        if not entries and path:
+            # Pretend empty directories don't exist, except for the root.
+            raise OSError(ENOTDIR, os.strerror(ENOTDIR))
+        for entry in entries:
+            entry = entry[len(path):]
+            bits = entry.split(os.path.sep)
+            if len(bits) == 1:
+                files.append(bits[0])
+            else:
+                directories.add(bits[0])
+        # Sort the directories and files
+        directories = list(directories)
+        directories.sort()
+        files.sort()
+        return directories, files
+
+    def _name(self, name):
+        new_name = os.path.normpath(name.lstrip(os.path.sep))
+        if new_name == os.path.curdir:
+            return ''
+        if name.endswith(os.path.sep):
+            # Preserve trailing slash for directories.
+            new_name += os.path.sep
+        return new_name
+
+    def size(self, name):
+        """
+        Returns the total size, in bytes, of the file specified by name.
+        """
+        return self._open(name).size
+
+    def url(self, name):
+        """
+        Returns an absolute URL where the file's contents can be accessed
+        directly by a web browser.
+        """
+        if self.base_url is None:
+            raise ValueError("This file is not accessible via a URL.")
+        return urlparse.urljoin(self.base_url, name).replace('\\', '/')

django_dbstorage/tests.py

+from StringIO import StringIO
+
+from django.conf import settings
+from django.test import TestCase
+
+from django_dbstorage.models import File
+from django_dbstorage.storage import DatabaseStorage
+
+
+class FileTestCase(TestCase):
+    def setUp(self):
+        File.objects.all().delete()
+
+    def test_init(self):
+        f = File(name='hello.txt')
+        self.assertEqual(f.name, 'hello.txt')
+        self.assertEqual(f.data, '')
+        self.assertEqual(f.closed, False)
+        self.assertEqual(f.encoding, None)
+        self.assertEqual(f.errors, None)
+        self.assertEqual(f.mode, 'r+b')
+        self.assertEqual(f.newlines, None)
+        self.assertEqual(f.softspace, 0)
+
+    def test_create(self):
+        f = File.objects.create(name='hello.txt')
+        self.assertEqual(f.name, 'hello.txt')
+        self.assertEqual(f.data, '')
+        self.assertEqual(f.closed, False)
+        self.assertEqual(f.encoding, None)
+        self.assertEqual(f.errors, None)
+        self.assertEqual(f.mode, 'r+b')
+        self.assertEqual(f.newlines, None)
+        self.assertEqual(f.softspace, 0)
+
+    def test_get(self):
+        File.objects.create(name='hello.txt', data='Hello world!')
+        f = File.objects.get(name='hello.txt')
+        self.assertEqual(f.data, 'Hello world!')
+
+    def test_unicode(self):
+        f = File.objects.create(name='hello.txt', data='Hello world!')
+        self.assertEqual(repr(f), "<File: 'hello.txt', mode 'r+b'>")
+        f.close()
+        self.assertEqual(repr(f), "<File: closed 'hello.txt', mode 'r+b'>")
+
+    def test_iter_next(self):
+        data = ['Hello world!\n', 'Salut monde!\n']
+        f = File.objects.create(name='hello.txt', data=''.join(data))
+        for line, expected in zip(f, data):
+            self.assertEqual(line, expected)
+
+    def test_close(self):
+        f = File.objects.create(name='hello.txt')
+        self.assertFalse(f.closed)
+        f.close()
+        self.assertTrue(f.closed)
+        self.assertRaises(ValueError, f.isatty)
+        self.assertRaises(ValueError, f.seek, 0)
+        self.assertRaises(ValueError, f.tell)
+        self.assertRaises(ValueError, f.read)
+        self.assertRaises(ValueError, f.readline)
+        self.assertRaises(ValueError, f.readlines)
+        self.assertRaises(ValueError, f.truncate)
+        self.assertRaises(ValueError, f.write, '')
+        self.assertRaises(ValueError, f.writelines, [''])
+        self.assertRaises(ValueError, f.flush)
+
+    def test_isatty(self):
+        f = File.objects.create(name='hello.txt')
+        self.assertFalse(f.isatty())
+
+    def test_seek_tell(self):
+        f = File.objects.create(name='hello.txt', data='Hello world!')
+        self.assertEqual(f.tell(), 0)
+        f.seek(5)
+        self.assertEqual(f.tell(), 5)
+        f.seek(3, 1)
+        self.assertEqual(f.tell(), 8)
+        f.seek(0, 2)
+        self.assertEqual(f.tell(), 12)
+        f.seek(-2, 2)
+        self.assertEqual(f.tell(), 10)
+
+    def test_read(self):
+        f = File.objects.create(name='hello.txt', data='Hello world!')
+        self.assertEqual(f.read(1), 'H')
+        self.assertEqual(f.read(), 'ello world!')
+
+    def test_readline(self):
+        f = File.objects.create(name='hello.txt',
+                                data='Hello world!\nSalut monde!\n')
+        self.assertEqual(f.readline(), 'Hello world!\n')
+        self.assertEqual(f.readline(), 'Salut monde!\n')
+
+    def test_readlines(self):
+        f = File.objects.create(name='hello.txt',
+                                data='Hello world!\nSalut monde!\n')
+        self.assertEqual(f.readlines(), ['Hello world!\n',
+                                         'Salut monde!\n'])
+
+    def test_truncate(self):
+        f = File.objects.create(name='hello.txt', data='Hello world!')
+        self.assertEqual(f.read(5), 'Hello')
+        f.truncate()
+        self.assertEqual(f.read(), '')
+        f.seek(0)
+        self.assertEqual(f.read(), 'Hello')
+        f.seek(10)
+        f.truncate(1)
+        self.assertEqual(f.tell(), 10)
+        f.seek(0)
+        self.assertEqual(f.read(), 'H')
+
+    def test_write(self):
+        f = File.objects.create(name='hello.txt')
+        f.write('Hello ')
+        f.write('world!\n')
+        print >>f, 'Salut monde!'
+        f.flush()
+        self.assertEqual(f.data, 'Hello world!\nSalut monde!\n')
+
+    def test_flush(self):
+        f = File.objects.create(name='hello.txt',
+                                data='Hello world!\n')
+        f.seek(0, 2)
+        print >>f, 'Salut monde!'
+        f.flush()
+        f = File.objects.get(name='hello.txt')
+        self.assertEqual(f.read(), 'Hello world!\nSalut monde!\n')
+
+    def test_size(self):
+        f = File.objects.create(name='hello.txt', data='Hello world!')
+        self.assertEqual(f._size(), 12)
+        f.seek(5)
+        f.truncate()
+        self.assertEqual(f._size(), 5)
+
+    def test_readonly(self):
+        f = File.objects.create(name='hello.txt', data='Hello world!')
+        f.mode = 'rb'
+        self.assertFalse(f.isatty())
+        self.assertEqual(f.tell(), 0)
+        self.assertEqual(f.read(), 'Hello world!')
+        f.seek(0)
+        self.assertEqual(f.readline(), 'Hello world!')
+        f.seek(0)
+        self.assertEqual(f.readlines(), ['Hello world!'])
+        self.assertRaises(IOError, f.truncate)
+        self.assertRaises(IOError, f.write, '')
+        self.assertRaises(IOError, f.writelines, [''])
+        self.assertRaises(IOError, f.flush)
+
+
+class DatabaseStorageTestCase(TestCase):
+    def setUp(self):
+        File.objects.all().delete()
+
+    def test_init(self):
+        s = DatabaseStorage()
+        self.assertEqual(s.base_url, settings.MEDIA_URL)
+        s = DatabaseStorage(base_url='/media/')
+        self.assertEqual(s.base_url, '/media/')
+
+    def test_open(self):
+        content = StringIO('Hello world!')
+        s = DatabaseStorage()
+        self.assertRaises(IOError, s.open, name='hello.txt')
+        s.save(name='hello.txt', content=content)
+        f = s.open(name='hello.txt')
+        self.assertEqual(f.read(), content.getvalue())
+        self.assertRaises(IOError, f.truncate)
+        f = s.open(name='hello.txt', mode='r+b')
+        self.assertEqual(f.read(), content.getvalue())
+        f.truncate()
+
+    def test_save(self):
+        content = StringIO('Hello world!')
+        s = DatabaseStorage()
+        self.assertRaises(IOError, s.save, name='', content=content)
+        self.assertRaises(IOError, s.save, name='/', content=content)
+        self.assertRaises(IOError, s.save, name='hello/', content=content)
+        self.assertEqual(s.save(name='hello.txt', content=content),
+                         'hello.txt')
+        self.assertEqual(s.save(name='hello.txt', content=content),
+                         'hello_.txt')
+        self.assertEqual(s.save(name='/hello.txt', content=content),
+                         'hello__.txt')
+        self.assertEqual(s.save(name='/hello/goodbye.txt', content=content),
+                         'hello/goodbye.txt')
+
+    def test_get_available_name(self):
+        content = StringIO('Hello world!')
+        s = DatabaseStorage()
+        self.assertEqual(s.get_available_name('hello.txt'), 'hello.txt')
+        s.save(name='hello.txt', content=content)
+        self.assertEqual(s.get_available_name('hello.txt'), 'hello_.txt')
+
+    def test_delete(self):
+        content = StringIO('Hello world!')
+        s = DatabaseStorage()
+        s.delete('hello.txt')
+        s.save(name='hello.txt', content=content)
+        s.delete('hello.txt')
+        self.assertFalse(s.exists('hello.txt'))
+
+    def test_exists(self):
+        content = StringIO('Hello world!')
+        s = DatabaseStorage()
+        self.assertFalse(s.exists('hello.txt'))
+        s.save(name='hello.txt', content=content)
+        self.assertTrue(s.exists('hello.txt'))
+
+    def test_listdir(self):
+        content = StringIO('')
+        english = StringIO('Hello world!')
+        french = StringIO('Salut monde!')
+        s = DatabaseStorage()
+        self.assertEqual(s.listdir(''), ([], []))
+        self.assertEqual(s.listdir('/'), ([], []))
+        s.save(name='hello.txt', content=english)
+        s.save(name='salut.txt', content=french)
+        self.assertEqual(s.listdir(''), ([], ['hello.txt', 'salut.txt']))
+        self.assertEqual(s.listdir('/'), ([], ['hello.txt', 'salut.txt']))
+        s.save(name='hello/en.txt', content=english)
+        s.save(name='hello/fr.txt', content=french)
+        s.save(name='hello/docs/README', content=content)
+        self.assertEqual(s.listdir(''), (['hello'],
+                                         ['hello.txt', 'salut.txt']))
+        self.assertEqual(s.listdir('/'), (['hello'],
+                                          ['hello.txt', 'salut.txt']))
+        self.assertEqual(s.listdir('hello'), (['docs'], ['en.txt', 'fr.txt']))
+        self.assertEqual(s.listdir('/hello'), (['docs'], ['en.txt', 'fr.txt']))
+        self.assertEqual(s.listdir('hello/docs'), ([], ['README']))
+        self.assertEqual(s.listdir('/hello/docs'), ([], ['README']))
+        self.assertRaises(OSError, s.listdir, 'goodbye')
+        self.assertRaises(OSError, s.listdir, '/goodbye')
+
+    def test_name(self):
+        s = DatabaseStorage()
+        self.assertEqual(s._name(''), '')
+        self.assertEqual(s._name('/'), '')
+        self.assertEqual(s._name('//'), '')
+        self.assertEqual(s._name('hello'), 'hello')
+        self.assertEqual(s._name('/hello'), 'hello')
+        self.assertEqual(s._name('//hello'), 'hello')
+        self.assertEqual(s._name('hello/'), 'hello/')
+        self.assertEqual(s._name('hello//'), 'hello/')
+        self.assertEqual(s._name('hello/goodbye'), 'hello/goodbye')
+        self.assertEqual(s._name('hello//goodbye'), 'hello/goodbye')
+
+    def test_size(self):
+        content = StringIO('Hello world!')
+        s = DatabaseStorage()
+        self.assertRaises(IOError, s.size, name='hello.txt')
+        s.save(name='hello.txt', content=content)
+        self.assertEqual(s.size(name='hello.txt'), len(content.getvalue()))
+
+    def test_url(self):
+        s = DatabaseStorage()
+        s.base_url = None
+        self.assertRaises(ValueError, s.url, name='hello.txt')
+        s = DatabaseStorage(base_url='/media/')
+        self.assertEqual(s.url(name='hello.txt'), '/media/hello.txt')
+        self.assertEqual(s.url(name='/hello.txt'), '/hello.txt')
+#!/usr/bin/env python
+from setuptools import setup
+
+setup(
+    name='django-dbstorage',
+    version='1.0',
+    description=('A Django file storage backend for files in the database.'),
+    author='Akoha Inc.',
+    author_email='adminmail@akoha.com',
+    url='http://bitbucket.org/akoha/django-dbstorage/',
+    packages=['django_dbstorage'],
+    package_dir={'django_dbstorage': 'django_dbstorage'},
+    classifiers=[
+        'Development Status :: 5 - Production/Stable',
+        'Environment :: Web Environment',
+        'Framework :: Django',
+        'License :: OSI Approved :: MIT License',
+        'Operating System :: OS Independent',
+        'Programming Language :: Python',
+    ],
+    zip_safe=True,
+)