Commits

Ronny Pfannschmidt committed 6287d6e

initial stake at the serializer

Comments (0)

Files changed (2)

middleware/_tests/test_serializer.py

+from storage.memory import BytesStorage, FileStorage
+
+from middleware.indexing import IndexingMiddleware, AccessDenied
+from middleware.serializer import serialize, deserialize
+from backend.storages import MutableBackend
+
+
+from StringIO import StringIO
+
+contents = [
+    (u'Foo', {'name': u'Foo'}, ''),
+    (u'Foo', {'name': u'Foo'}, '2nd'),
+    (u'Subdir', {'name': u'Subdir'}, ''),
+    (u'Subdir/Foo', {'name': u'Subdir/Foo'}, ''),
+    (u'Subdir/Bar', {'name': u'Subdir/Bar'}, ''),
+]
+
+
+
+scenarios = [
+    ('Simple', ['']),
+    ('Nested', ['', 'Subdir']),
+]
+
+
+def pytest_generate_tests(metafunc):
+    metafunc.addcall(id='Simple->Simple', param=('Simple', 'Simple'))
+
+def pytest_funcarg__source(request):
+    # scenario
+    return make_middleware(request)
+
+def pytest_funcarg__target(request):
+    # scenario
+    return make_middleware(request)
+
+def make_middleware(request):
+    tmpdir = request.getfuncargvalue('tmpdir')
+    # scenario
+
+    meta_store = BytesStorage()
+    data_store = FileStorage()
+    backend = MutableBackend(meta_store, data_store)
+    backend.create()
+    backend.open()
+    request.addfinalizer(backend.destroy)
+    request.addfinalizer(backend.close)
+    
+    mw = IndexingMiddleware(index_dir=str(tmpdir/'foo'),
+                                  backend=backend)
+    mw.create()
+    mw.open()
+    request.addfinalizer(mw.destroy)
+    request.addfinalizer(mw.close)
+    return mw
+
+
+def test_serialize_deserialize(source, target):
+
+    i = 0
+    for name, meta, data in contents:
+        item = source['name']
+        item.create_revision(dict(meta, mtime=i), StringIO(data))
+        i += 1
+
+    io = StringIO()
+    serialize(source.backend, io)
+    io.seek(0)
+    deserialize(io, target.backend)
+
+    print sorted(source.backend)
+    print sorted(target.backend)
+    assert sorted(source.backend)  == sorted(target.backend)

middleware/serializer.py

+import struct
+import json
+from werkzeug.wsgi import LimitedStream
+
+
+def serialize(backend, targetfile):
+    targetfile.writelines(serialize_iter(backend))
+
+
+def serialize_iter(backend):
+    for revid in backend:
+        meta, data = backend.get_revision(revid)
+
+        text = json.dumps(meta, ensure_ascii=False)
+        meta_str = text.encode('utf-8')
+        yield struct.pack('!i', len(meta_str))
+        yield meta_str
+        while True:
+            block = data.read(8192)
+            if not block:
+                break
+            yield block
+
+
+def deserialize(io, backend):
+    while True:
+        meta_size_bytes = io.read(4)
+        if not meta_size_bytes:
+            return
+        meta_size = struct.unpack('!i', meta_size_bytes)[0]
+        meta_str = io.read(meta_size)
+        text = meta_str.decode('utf-8')
+        meta = json.loads(text)
+        data_size = meta[u'size']
+        
+        #XXX: this shoul be, unreliable for unknown reason
+        #limited = LimitedStream(io, data_size)
+        #backend.store_revision(meta, limited)
+        #assert limited.is_exhausted
+
+        data = io.read(data_size)
+        from StringIO import StringIO
+        backend.store_revision(meta, StringIO(data))
+        
+