Commits

Thomas Waldmann committed 08add84 Merge

merged ronny's stuff

  • Participants
  • Parent commits 1efdf11, 6e6ea96

Comments (0)

Files changed (15)

 .rej$
 .~$
 .DS_Store
-
+.*\.sqlite

File backend/_util.py

     """
     def __init__(self, realfile, hash_method='sha1'):
         self._realfile = realfile
+        self._read = realfile.read
         self.hash = hashlib.new(hash_method)
         self.size = 0
 
-    def read(self, size=-1):
-        data = self._realfile.read(size)
+    def read(self, size=None):
+        #XXX: workaround for werkzeug.wsgi.LimitedStream
+        #     which expects None instead of -1 for read everything
+        if size is None:
+            data = self._read()
+        else:
+            data = self._read(size)
         self.hash.update(data)
         self.size += len(data)
         return data

File backend/storages.py

         self.meta_store[metaid] = meta
         return metaid
 
-    def _store_data(self, data):
-        dataid = make_uuid()
+    def store_revision(self, meta, data):
         # XXX Idea: we could check the type the store wants from us:
         # if it is a str/bytes (BytesStorage), just use meta "as is",
         # if it is a file (FileStorage), wrap it into StringIO and give that to the store.
-        self.data_store[dataid] = data
-        return dataid
-
-    def store_revision(self, meta, data):
-        tfw = TrackingFileWrapper(data, hash_method=HASH_ALGORITHM)
-        dataid = self._store_data(tfw)
-        meta['dataid'] = dataid
-        meta['size'] = tfw.size
-        meta[HASH_ALGORITHM] = tfw.hash.hexdigest()
+        if 'dataid' not in meta:
+            tfw = TrackingFileWrapper(data, hash_method=HASH_ALGORITHM)
+            dataid = make_uuid()
+            self.data_store[dataid] = tfw
+            meta['dataid'] = dataid
+            meta['size'] = tfw.size
+            meta[HASH_ALGORITHM] = tfw.hash.hexdigest()
+        else:
+            dataid = meta['dataid']
+            # we will just asume stuff is correct if you pass it with a data id
+            if dataid not in self.data_store:
+                self.data_store[dataid] = data
         # if something goes wrong below, the data shall be purged by a garbage collection
         metaid = self._store_meta(meta)
         return metaid

File middleware/_tests/test_serializer.py

+from storage.memory import BytesStorage, FileStorage
+
+from middleware.indexing import IndexingMiddleware, AccessDenied
+from middleware.serializer import serialize, deserialize
+from backend.storages import MutableBackend
+
+
+from StringIO import StringIO
+
+contents = [
+    (u'Foo', {'name': u'Foo'}, ''),
+    (u'Foo', {'name': u'Foo'}, '2nd'),
+    (u'Subdir', {'name': u'Subdir'}, ''),
+    (u'Subdir/Foo', {'name': u'Subdir/Foo'}, ''),
+    (u'Subdir/Bar', {'name': u'Subdir/Bar'}, ''),
+]
+
+
+
+scenarios = [
+    ('Simple', ['']),
+    ('Nested', ['', 'Subdir']),
+]
+
+
+def pytest_generate_tests(metafunc):
+    metafunc.addcall(id='Simple->Simple', param=('Simple', 'Simple'))
+
+def pytest_funcarg__source(request):
+    # scenario
+    return make_middleware(request)
+
+def pytest_funcarg__target(request):
+    # scenario
+    return make_middleware(request)
+
+def make_middleware(request):
+    tmpdir = request.getfuncargvalue('tmpdir')
+    # scenario
+
+    meta_store = BytesStorage()
+    data_store = FileStorage()
+    backend = MutableBackend(meta_store, data_store)
+    backend.create()
+    backend.open()
+    request.addfinalizer(backend.destroy)
+    request.addfinalizer(backend.close)
+    
+    mw = IndexingMiddleware(index_dir=str(tmpdir/'foo'),
+                                  backend=backend)
+    mw.create()
+    mw.open()
+    request.addfinalizer(mw.destroy)
+    request.addfinalizer(mw.close)
+    return mw
+
+
+def test_serialize_deserialize(source, target):
+
+    i = 0
+    for name, meta, data in contents:
+        item = source['name']
+        item.create_revision(dict(meta, mtime=i), StringIO(data))
+        i += 1
+
+    io = StringIO()
+    serialize(source.backend, io)
+    io.seek(0)
+    deserialize(io, target.backend)
+    target.rebuild()
+
+    print sorted(source.backend)
+    print sorted(target.backend)
+    assert sorted(source.backend)  == sorted(target.backend)

File middleware/indexing.py

         meta[COMMENT] = reason or u'destroyed'
         # TODO cleanup more metadata
         data = StringIO('') # nothing to see there
+        del meta['dataid'] # remove dataid
         revid = backend.store_revision(meta, data)
         # Note: we just stored new (empty) data for this revision, but the old
         # data file is still in storage (not referenced by THIS revision any more)

File middleware/serializer.py

+import struct
+import json
+from io import BytesIO
+from werkzeug.wsgi import LimitedStream
+
+
+def serialize(backend, targetfile):
+    targetfile.writelines(serialize_iter(backend))
+
+
+def serialize_iter(backend):
+    for revid in backend:
+        meta, data = backend.get_revision(revid)
+
+        text = json.dumps(meta, ensure_ascii=False)
+        meta_str = text.encode('utf-8')
+        yield struct.pack('!i', len(meta_str))
+        yield meta_str
+        while True:
+            block = data.read(8192)
+            if not block:
+                break
+            yield block
+
+
+def deserialize(io, backend):
+    while True:
+        meta_size_bytes = io.read(4)
+        if not meta_size_bytes:
+            return
+        meta_size = struct.unpack('!i', meta_size_bytes)[0]
+        meta_str = io.read(meta_size)
+        text = meta_str.decode('utf-8')
+        meta = json.loads(text)
+        data_size = meta[u'size']
+
+        limited = LimitedStream(io, data_size)
+        backend.store_revision(meta, limited)
+        assert limited.is_exhausted
+
+

File storage/_tests/__init__.py

 """
 
 
-from __future__ import absolute_import, division
+class FileStorageTestBase(object):
+    pass
 
-import pytest
 
-from StringIO import StringIO
+class BytesStorageTestBase(object):
+    pass
 
 
-class _StorageTestBase(object):
-    def setup_method(self, method):
-        """
-        self.st needs to be an created/opened storage
-        """
-        raise NotImplemented
 
-    def teardown_method(self, method):
-        """
-        close and destroy self.st
-        """
-        self.st.close()
-        self.st.destroy()
 
-    def test_getitem_raises(self):
-        with pytest.raises(KeyError):
-            self.st['doesnotexist']
 
 
-class FileStorageTestBase(_StorageTestBase):
-    def test_setitem_getitem_delitem(self):
-        k, v = 'key', 'value'
-        self.st[k] = StringIO(v)
-        assert v == self.st[k].read()
-        del self.st[k]
-        with pytest.raises(KeyError):
-            self.st[k]
-
-    def test_setitem_getitem_delitem_binary(self):
-        k, v = 'key', '\000\001\002'
-        self.st[k] = StringIO(v)
-        assert v == self.st[k].read()
-        assert len(v) == 3
-        del self.st[k]
-        with pytest.raises(KeyError):
-            self.st[k]
-
-    def test_iter(self):
-        kvs = set([('1', 'one'), ('2', 'two'), ('3', 'three'), ])
-        for k, v in kvs:
-            v = StringIO(v)
-            self.st[k] = v
-        result = set()
-        for k in self.st:
-            result.add((k, self.st[k].read()))
-        assert result == kvs
-
-    def test_len(self):
-        assert len(self.st) == 0
-        self.st['foo'] = StringIO('bar')
-        assert len(self.st) == 1
-        del self.st['foo']
-        assert len(self.st) == 0
-
-    def test_perf(self):
-        pytest.skip("usually we do no performance tests")
-        for i in range(1000):
-            key = value = str(i)
-            self.st[key] = StringIO(value)
-        for i in range(1000):
-            key = expected_value = str(i)
-            assert self.st[key].read() == expected_value
-        for i in range(1000):
-            key = str(i)
-            del self.st[key]
-
-
-class BytesStorageTestBase(_StorageTestBase):
-    def test_setitem_getitem_delitem(self):
-        k, v = 'key', 'value'
-        self.st[k] = v
-        assert v == self.st[k]
-        del self.st[k]
-        with pytest.raises(KeyError):
-            self.st[k]
-
-    def test_setitem_getitem_delitem_binary(self):
-        k, v = 'key', '\000\001\002'
-        self.st[k] = v
-        assert v == self.st[k]
-        assert len(v) == 3
-        del self.st[k]
-        with pytest.raises(KeyError):
-            self.st[k]
-
-    def test_iter(self):
-        kvs = set([('1', 'one'), ('2', 'two'), ('3', 'three'), ])
-        for k, v in kvs:
-            self.st[k] = v
-        result = set()
-        for k in self.st:
-            result.add((k, self.st[k]))
-        assert result == kvs
-
-    def test_len(self):
-        assert len(self.st) == 0
-        self.st['foo'] = 'bar'
-        assert len(self.st) == 1
-        del self.st['foo']
-        assert len(self.st) == 0
-
-    def test_perf(self):
-        pytest.skip("usually we do no performance tests")
-        for i in range(1000):
-            key = value = str(i)
-            self.st[key] = value
-        for i in range(1000):
-            key = expected_value = str(i)
-            assert self.st[key] == expected_value
-        for i in range(1000):
-            key = str(i)
-            del self.st[key]
-
-

File storage/_tests/conftest.py

+import pytest
+from storage.wrappers import ByteToStreamWrappingStore
+
+# memcached is not in the loop
+stores = 'fs kc memory sqlite sqlite:compressed'.split()
+
+constructors = {
+    'memory': lambda store, _: store(),
+    'fs': lambda store, tmpdir: store(str(tmpdir.join('store'))),
+    'sqlite': lambda store, tmpdir: store(str(tmpdir.join('store.sqlite')),
+                                          'test_table', compression_level=0),
+    'sqlite:compressed': lambda store, tmpdir: store(str(tmpdir.join('store.sqlite')),
+                                          'test_table', compression_level=1),
+    'kc': lambda store, tmpdir: store(str(tmpdir.join('store'))),
+}
+
+
+def pytest_generate_tests(metafunc):
+    argnames = metafunc.funcargnames
+    
+    if 'store' in argnames:
+        klasses = 'BytesStorage', 'FileStorage'
+    elif 'bst' in argnames:
+        klasses = 'BytesStorage',
+    elif 'fst' in argnames:
+        klasses = 'FileStorage',
+    else:
+        klasses = None
+
+    if klasses is not None:
+        for storename in stores:
+            for klass in klasses:
+                metafunc.addcall(
+                    id='%s/%s' % (storename, klass),
+                    param=(storename, klass))
+
+    multi_mark = getattr(metafunc.function, 'multi', None)
+    if multi_mark is not None:
+        #XXX: hack
+        storages = multi_mark.kwargs['Storage']
+        for storage in storages:
+            metafunc.addcall(id=storage.__name__, funcargs={
+                'Storage': storage,
+            })
+
+
+def make_storage(request):
+    tmpdir = request.getfuncargvalue('tmpdir')
+    storename, kind = request.param
+    storemodule = pytest.importorskip('storage.' + storename.split(':')[0])
+    klass = getattr(storemodule, kind)
+    construct = constructors.get(storename)
+    if construct is None:
+        pytest.xfail('don\'t know how to construct %s store' % (storename,))
+    store = construct(klass, tmpdir)
+    store.create()
+    store.open()
+
+    # no destroy in the normal finalizer
+    # so we can keep the data for example if its a tmpdir
+    request.addfinalizer(store.close)
+    return store
+
+
+def pytest_funcarg__bst(request):
+    return make_storage(request)
+
+
+def pytest_funcarg__fst(request):
+    return make_storage(request)
+
+
+def pytest_funcarg__store(request):
+    store = make_storage(request)
+    storename, kind = request.param
+    if kind == 'FileStorage':
+        store = ByteToStreamWrappingStore(store)
+    return store
+

File storage/_tests/test_all.py

+import pytest
+
+def test_getitem_raises(store):
+    with pytest.raises(KeyError):
+        store['doesnotexist']
+
+
+def test_setitem_getitem_delitem(store):
+    k, v = 'key', 'value'
+    store[k] = v
+    assert v == store[k]
+    del store[k]
+    with pytest.raises(KeyError):
+        store[k]
+
+def test_setitem_getitem_delitem_binary(store):
+    k, v = 'key', '\000\001\002'
+    store[k] = v
+    assert v == store[k]
+    assert len(v) == 3
+    del store[k]
+    with pytest.raises(KeyError):
+        store[k]
+
+
+def test_iter(store):
+    kvs = set([('1', 'one'), ('2', 'two'), ('3', 'three'), ])
+    for k, v in kvs:
+        store[k] = v
+    result = set()
+    for k in store:
+        result.add((k, store[k]))
+    assert result == kvs
+
+
+def test_len(store):
+    assert len(store) == 0
+    store['foo'] = 'bar'
+    assert len(store) == 1
+    del store['foo']
+    assert len(store) == 0
+
+def test_perf(store):
+    #XXX: introduce perf test option
+    pytest.skip("usually we do no performance tests")
+    for i in range(1000):
+        key = value = str(i)
+        store[key] = value
+    for i in range(1000):
+        key = expected_value = str(i)
+        assert store[key] == expected_value
+    for i in range(1000):
+        key = str(i)
+        del store[key]

File storage/_tests/test_fs.py

 
 from __future__ import absolute_import, division
 
-import os, tempfile
+import pytest
+from storage.fs import BytesStorage, FileStorage
 
-from storage.fs import BytesStorage, FileStorage
-from storage._tests import BytesStorageTestBase, FileStorageTestBase
 
-class TestBytesStorage(BytesStorageTestBase):
-    def setup_method(self, method):
-        path = tempfile.mkdtemp()
-        os.rmdir(path)
-        self.st = BytesStorage(path)
-        self.st.create()
-        self.st.open()
+@pytest.mark.multi(Storage=[BytesStorage, FileStorage])
+def test_create(tmpdir, Storage):
+    target = tmpdir.join('store')
+    assert not target.check()
 
-class TestFileStorage(FileStorageTestBase):
-    def setup_method(self, method):
-        path = tempfile.mkdtemp()
-        os.rmdir(path)
-        self.st = FileStorage(path)
-        self.st.create()
-        self.st.open()
+    store = Storage(str(target))
+    assert not target.check()
+    store.create()
+    assert target.check()
 
+    return store
 
+
+@pytest.mark.multi(Storage=[BytesStorage, FileStorage])
+def test_destroy(tmpdir, Storage):
+    store = test_create(tmpdir, Storage)
+    target = tmpdir.join('store')
+    store.destroy()
+    assert not target.check()
+

File storage/_tests/test_kc.py

 
 from __future__ import absolute_import, division
 
+import pytest
+pytest.importorskip('storage.kc')
+
 from storage.kc import BytesStorage, FileStorage
-from storage._tests import BytesStorageTestBase, FileStorageTestBase
 
 
-class TestBytesStorage(BytesStorageTestBase):
-    def setup_method(self, method):
-        self.st = BytesStorage('testdb.kch') # *.kch
-        self.st.create()
-        self.st.open()
+@pytest.mark.multi(Storage=[BytesStorage, FileStorage])
+def test_create(tmpdir, Storage):
+    target = tmpdir.join('store.kc')
+    assert not target.check()
 
-class TestFileStorage(FileStorageTestBase):
-    def setup_method(self, method):
-        self.st = FileStorage('testdb.kch') # *.kch
-        self.st.create()
-        self.st.open()
+    store = Storage(str(target))
+    assert not target.check()
+    store.create()
+    assert target.check()
 
+    return store
+
+
+@pytest.mark.multi(Storage=[BytesStorage, FileStorage])
+def test_destroy(tmpdir, Storage):
+    store = test_create(tmpdir, Storage)
+    target = tmpdir.join('store.kc')
+    store.destroy()
+    assert not target.check()
+
+

File storage/_tests/test_memory.py

 MoinMoin - memory storage tests
 """
 
+import pytest
+from storage.memory import BytesStorage, FileStorage
 
-from __future__ import absolute_import, division
+@pytest.mark.multi(Storage=[BytesStorage, FileStorage])
+def test_create( Storage):
+    store = Storage()
+    assert store._st is None
 
-from storage.memory import BytesStorage, FileStorage
-from storage._tests import BytesStorageTestBase, FileStorageTestBase
+    store.create()
+    assert store._st == {}
 
+    return store
 
-class TestBytesStorage(BytesStorageTestBase):
-    def setup_method(self, method):
-        self.st = BytesStorage()
-        self.st.create()
-        self.st.open()
+@pytest.mark.multi(Storage=[BytesStorage, FileStorage])
+def test_destroy(Storage):
+    store = test_create(Storage)
+    store.destroy()
+    assert store._st is None
 
-class TestFileStorage(FileStorageTestBase):
-    def setup_method(self, method):
-        self.st = FileStorage()
-        self.st.create()
-        self.st.open()
 
-

File storage/_tests/test_sqlite.py

 """
 
 
-from __future__ import absolute_import, division
+import pytest
+from storage.sqlite import BytesStorage, FileStorage
 
-from storage.sqlite import BytesStorage, FileStorage
-from storage._tests import BytesStorageTestBase, FileStorageTestBase
+def bytes_compressed(path):
+    return BytesStorage(path, 'test_table', compression_level=1)
+def bytes_uncompressed(path):
+    return BytesStorage(path, 'test_table', compression_level=0)
 
+def file_compressed(path):
+    return FileStorage(path, 'test_table', compression_level=1)
+def file_uncompressed(path):
+    return FileStorage(path, 'test_table', compression_level=0)
 
-class TestBytesStorage(BytesStorageTestBase):
-    def setup_method(self, method):
-        self.st = BytesStorage('testdb.sqlite', 'testbs', compression_level=0) # ':memory:' does not work, strange
-        self.st.create()
-        self.st.open()
+all_setups = pytest.mark.multi(Storage=[
+    bytes_uncompressed,
+    bytes_compressed,
+    file_uncompressed,
+    file_compressed,
+])
 
 
-class TestBytesStorageCompressed(BytesStorageTestBase):
-    def setup_method(self, method):
-        self.st = BytesStorage('testdb.sqlite', 'testbs', compression_level=1) # ':memory:' does not work, strange
-        self.st.create()
-        self.st.open()
+@all_setups
+def test_create(tmpdir, Storage):
+    dbfile = tmpdir.join('store.sqlite')
+    assert not dbfile.check()
+    store = Storage(str(dbfile))
+    assert not dbfile.check()
+    store.create()
+    assert dbfile.check()
+    return store
 
+@all_setups
+def test_destroy(tmpdir, Storage):
+    dbfile = tmpdir.join('store.sqlite')
+    store = test_create(tmpdir, Storage)
+    store.destroy()
+    #XXX: check for dropped table
 
-class TestFileStorage(FileStorageTestBase):
-    def setup_method(self, method):
-        self.st = FileStorage('testdb.sqlite', 'testfs', compression_level=0) # ':memory:' does not work, strange
-        self.st.create()
-        self.st.open()
 
-
-class TestFileStorageCompressed(FileStorageTestBase):
-    def setup_method(self, method):
-        self.st = FileStorage('testdb.sqlite', 'testfs', compression_level=1) # ':memory:' does not work, strange
-        self.st.create()
-        self.st.open()
-

File storage/memory.py

     """
     A simple dict-based in-memory storage. No persistence!
     """
+    def __init__(self):
+        self._st = None
+
     def create(self):
         self._st = {}
 

File storage/wrappers.py

+
+from io import BytesIO
+from collections import MutableMapping
+class ByteToStreamWrappingStore(MutableMapping):
+    def __init__(self, stream_store):
+        self._st = stream_store
+
+    def __iter__(self):
+        return iter(self._st)
+
+    def __setitem__(self, key, value):
+        self._st[key] = BytesIO(value)
+
+    def __getitem__(self, key):
+        return self._st[key].read()
+
+    def __delitem__(self, key):
+        del self._st[key]
+
+    def __len__(self):
+        return len(self._st)
+
+
+