1. Thomas Waldmann
  2. storage-ng

Commits

Thomas Waldmann  committed 159c0a6

rename router>routing, serializer>serialization

  • Participants
  • Parent commits a86e6b7
  • Branches default

Comments (0)

Files changed (8)

File middleware/_tests/test_router.py

-# Copyright: 2011 MoinMoin:ThomasWaldmann
-# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
-
-"""
-MoinMoin - router middleware tests
-"""
-
-
-from __future__ import absolute_import, division
-
-from StringIO import StringIO
-
-import pytest
-
-from config import NAME, REVID
-
-from middleware.router import Backend as RouterBackend
-from backend.storages import MutableBackend as StorageBackend, Backend as ROBackend
-
-from storage.memory import BytesStorage as MemoryBytesStorage
-from storage.memory import FileStorage as MemoryFileStorage
-
-
-def make_ro_backend():
-    store = StorageBackend(MemoryBytesStorage(), MemoryFileStorage())
-    store.create()
-    store.store({NAME: 'test'}, StringIO(''))
-    store.store({NAME: 'test2'}, StringIO(''))
-    return ROBackend(store.meta_store, store.data_store)
-
-
-
-def pytest_funcarg__router(request):
-    root_be = StorageBackend(MemoryBytesStorage(), MemoryFileStorage())
-    sub_be = StorageBackend(MemoryBytesStorage(), MemoryFileStorage())
-    ro_be = make_ro_backend()
-    router = RouterBackend([('sub', sub_be), ('ro', ro_be), ('', root_be)])
-    router.open()
-    router.create()
-
-    @request.addfinalizer
-    def finalize():
-        router.close()
-        router.destroy()
-
-    return router
-
-def revid_split(revid):
-    # router revids are <backend_mountpoint>/<backend_revid>, split that:
-    return revid.rsplit(u'/', 1)
-
-def test_store_get_del(router):
-    root_name = u'foo'
-    root_revid = router.store(dict(name=root_name), StringIO(''))
-    sub_name = u'sub/bar'
-    sub_revid = router.store(dict(name=sub_name), StringIO(''))
-
-    assert revid_split(root_revid)[0] == ''
-    assert revid_split(sub_revid)[0] == 'sub'
-
-    # when going via the router backend, we get back fully qualified names:
-    root_meta, _ = router.retrieve(root_revid)
-    sub_meta, _ = router.retrieve(sub_revid)
-    assert root_name == root_meta[NAME]
-    assert sub_name == sub_meta[NAME]
-
-    # when looking into the storage backend, we see relative names (without mountpoint):
-    root_meta, _ = router.mapping[-1][1].retrieve(revid_split(root_revid)[1])
-    sub_meta, _ = router.mapping[0][1].retrieve(revid_split(sub_revid)[1])
-    assert root_name == root_meta[NAME]
-    assert sub_name == 'sub' + '/' + sub_meta[NAME]
-    # delete revs:
-    router.remove(root_revid)
-    router.remove(sub_revid)
-
-
-def test_store_readonly_fails(router):
-    with pytest.raises(TypeError):
-        router.store(dict(name=u'ro/testing'), StringIO(''))
-
-def test_del_readonly_fails(router):
-    ro_id = next(iter(router)) # we have only readonly items
-    print ro_id
-    with pytest.raises(TypeError):
-        router.remove(ro_id)
-
-
-def test_destroy_create_dont_touch_ro(router):
-    existing = set(router)
-    root_revid = router.store(dict(name=u'foo'), StringIO(''))
-    sub_revid = router.store(dict(name=u'sub/bar'), StringIO(''))
-
-    router.destroy()
-    router.create()
-
-    assert set(router) == existing
-
-
-def test_iter(router):
-    existing = set(router)
-    root_revid = router.store(dict(name=u'foo'), StringIO(''))
-    sub_revid = router.store(dict(name=u'sub/bar'), StringIO(''))
-    assert set(router) == (set([root_revid, sub_revid])|existing)
-

File middleware/_tests/test_routing.py

View file
+# Copyright: 2011 MoinMoin:ThomasWaldmann
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+MoinMoin - router middleware tests
+"""
+
+
+from __future__ import absolute_import, division
+
+from StringIO import StringIO
+
+import pytest
+
+from config import NAME, REVID
+
+from middleware.routing import Backend as RouterBackend
+from backend.storages import MutableBackend as StorageBackend, Backend as ROBackend
+
+from storage.memory import BytesStorage as MemoryBytesStorage
+from storage.memory import FileStorage as MemoryFileStorage
+
+
+def make_ro_backend():
+    store = StorageBackend(MemoryBytesStorage(), MemoryFileStorage())
+    store.create()
+    store.store({NAME: 'test'}, StringIO(''))
+    store.store({NAME: 'test2'}, StringIO(''))
+    return ROBackend(store.meta_store, store.data_store)
+
+
+
+def pytest_funcarg__router(request):
+    root_be = StorageBackend(MemoryBytesStorage(), MemoryFileStorage())
+    sub_be = StorageBackend(MemoryBytesStorage(), MemoryFileStorage())
+    ro_be = make_ro_backend()
+    router = RouterBackend([('sub', sub_be), ('ro', ro_be), ('', root_be)])
+    router.open()
+    router.create()
+
+    @request.addfinalizer
+    def finalize():
+        router.close()
+        router.destroy()
+
+    return router
+
+def revid_split(revid):
+    # router revids are <backend_mountpoint>/<backend_revid>, split that:
+    return revid.rsplit(u'/', 1)
+
+def test_store_get_del(router):
+    root_name = u'foo'
+    root_revid = router.store(dict(name=root_name), StringIO(''))
+    sub_name = u'sub/bar'
+    sub_revid = router.store(dict(name=sub_name), StringIO(''))
+
+    assert revid_split(root_revid)[0] == ''
+    assert revid_split(sub_revid)[0] == 'sub'
+
+    # when going via the router backend, we get back fully qualified names:
+    root_meta, _ = router.retrieve(root_revid)
+    sub_meta, _ = router.retrieve(sub_revid)
+    assert root_name == root_meta[NAME]
+    assert sub_name == sub_meta[NAME]
+
+    # when looking into the storage backend, we see relative names (without mountpoint):
+    root_meta, _ = router.mapping[-1][1].retrieve(revid_split(root_revid)[1])
+    sub_meta, _ = router.mapping[0][1].retrieve(revid_split(sub_revid)[1])
+    assert root_name == root_meta[NAME]
+    assert sub_name == 'sub' + '/' + sub_meta[NAME]
+    # delete revs:
+    router.remove(root_revid)
+    router.remove(sub_revid)
+
+
+def test_store_readonly_fails(router):
+    with pytest.raises(TypeError):
+        router.store(dict(name=u'ro/testing'), StringIO(''))
+
+def test_del_readonly_fails(router):
+    ro_id = next(iter(router)) # we have only readonly items
+    print ro_id
+    with pytest.raises(TypeError):
+        router.remove(ro_id)
+
+
+def test_destroy_create_dont_touch_ro(router):
+    existing = set(router)
+    root_revid = router.store(dict(name=u'foo'), StringIO(''))
+    sub_revid = router.store(dict(name=u'sub/bar'), StringIO(''))
+
+    router.destroy()
+    router.create()
+
+    assert set(router) == existing
+
+
+def test_iter(router):
+    existing = set(router)
+    root_revid = router.store(dict(name=u'foo'), StringIO(''))
+    sub_revid = router.store(dict(name=u'sub/bar'), StringIO(''))
+    assert set(router) == (set([root_revid, sub_revid])|existing)
+

File middleware/_tests/test_serialization.py

View file
+# Copyright: 2011 MoinMoin:RonnyPfannschmidt
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+MoinMoin - serializer / deserializer tests
+"""
+
+
+from __future__ import absolute_import, division
+
+from storage.memory import BytesStorage, FileStorage
+
+from middleware.indexing import IndexingMiddleware, AccessDenied
+from middleware.serialization import serialize, deserialize
+from backend.storages import MutableBackend
+
+
+from StringIO import StringIO
+
+contents = [
+    (u'Foo', {'name': u'Foo'}, ''),
+    (u'Foo', {'name': u'Foo'}, '2nd'),
+    (u'Subdir', {'name': u'Subdir'}, ''),
+    (u'Subdir/Foo', {'name': u'Subdir/Foo'}, ''),
+    (u'Subdir/Bar', {'name': u'Subdir/Bar'}, ''),
+]
+
+
+scenarios = [
+    ('Simple', ['']),
+    ('Nested', ['', 'Subdir']),
+]
+
+
+def pytest_generate_tests(metafunc):
+    metafunc.addcall(id='Simple->Simple', param=('Simple', 'Simple'))
+
+def pytest_funcarg__source(request):
+    # scenario
+    return make_middleware(request)
+
+def pytest_funcarg__target(request):
+    # scenario
+    return make_middleware(request)
+
+def make_middleware(request):
+    tmpdir = request.getfuncargvalue('tmpdir')
+    # scenario
+
+    meta_store = BytesStorage()
+    data_store = FileStorage()
+    backend = MutableBackend(meta_store, data_store)
+    backend.create()
+    backend.open()
+    request.addfinalizer(backend.destroy)
+    request.addfinalizer(backend.close)
+    
+    mw = IndexingMiddleware(index_dir=str(tmpdir/'foo'),
+                            backend=backend)
+    mw.create()
+    mw.open()
+    request.addfinalizer(mw.destroy)
+    request.addfinalizer(mw.close)
+    return mw
+
+
+def test_serialize_deserialize(source, target):
+    i = 0
+    for name, meta, data in contents:
+        item = source['name']
+        item.store_revision(dict(meta, mtime=i), StringIO(data))
+        i += 1
+
+    io = StringIO()
+    serialize(source.backend, io)
+    io.seek(0)
+    deserialize(io, target.backend)
+    target.rebuild()
+
+    print sorted(source.backend)
+    print sorted(target.backend)
+    assert sorted(source.backend) == sorted(target.backend)
+

File middleware/_tests/test_serializer.py

-# Copyright: 2011 MoinMoin:RonnyPfannschmidt
-# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
-
-"""
-MoinMoin - serializer / deserializer tests
-"""
-
-
-from __future__ import absolute_import, division
-
-from storage.memory import BytesStorage, FileStorage
-
-from middleware.indexing import IndexingMiddleware, AccessDenied
-from middleware.serializer import serialize, deserialize
-from backend.storages import MutableBackend
-
-
-from StringIO import StringIO
-
-contents = [
-    (u'Foo', {'name': u'Foo'}, ''),
-    (u'Foo', {'name': u'Foo'}, '2nd'),
-    (u'Subdir', {'name': u'Subdir'}, ''),
-    (u'Subdir/Foo', {'name': u'Subdir/Foo'}, ''),
-    (u'Subdir/Bar', {'name': u'Subdir/Bar'}, ''),
-]
-
-
-scenarios = [
-    ('Simple', ['']),
-    ('Nested', ['', 'Subdir']),
-]
-
-
-def pytest_generate_tests(metafunc):
-    metafunc.addcall(id='Simple->Simple', param=('Simple', 'Simple'))
-
-def pytest_funcarg__source(request):
-    # scenario
-    return make_middleware(request)
-
-def pytest_funcarg__target(request):
-    # scenario
-    return make_middleware(request)
-
-def make_middleware(request):
-    tmpdir = request.getfuncargvalue('tmpdir')
-    # scenario
-
-    meta_store = BytesStorage()
-    data_store = FileStorage()
-    backend = MutableBackend(meta_store, data_store)
-    backend.create()
-    backend.open()
-    request.addfinalizer(backend.destroy)
-    request.addfinalizer(backend.close)
-    
-    mw = IndexingMiddleware(index_dir=str(tmpdir/'foo'),
-                            backend=backend)
-    mw.create()
-    mw.open()
-    request.addfinalizer(mw.destroy)
-    request.addfinalizer(mw.close)
-    return mw
-
-
-def test_serialize_deserialize(source, target):
-    i = 0
-    for name, meta, data in contents:
-        item = source['name']
-        item.store_revision(dict(meta, mtime=i), StringIO(data))
-        i += 1
-
-    io = StringIO()
-    serialize(source.backend, io)
-    io.seek(0)
-    deserialize(io, target.backend)
-    target.rebuild()
-
-    print sorted(source.backend)
-    print sorted(target.backend)
-    assert sorted(source.backend) == sorted(target.backend)
-

File middleware/router.py

-# Copyright: 2008-2011 MoinMoin:ThomasWaldmann
-# Copyright: 2011 MoinMoin:RonnyPfannschmidt
-# Copyright: 2009 MoinMoin:ChristopherDenter
-# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
-
-"""
-MoinMoin - router middleware
-
-Routes requests to different backends depending on the item name.
-"""
-
-
-from __future__ import absolute_import, division
-
-from config import NAME
-
-from backend import BackendBase, MutableBackendBase
-
-
-class Backend(MutableBackendBase):
-    """
-    router, behaves readonly for readonly mounts
-    """
-    def __init__(self, mapping):
-        """
-        Initialize router backend.
-
-        The mapping given must satisfy the following criteria:
-            * Order matters.
-            * Mountpoints are just item names, including the special '' (empty)
-              root item name.
-            * Trailing '/' of a mountpoint will be stripped.
-            * There *must* be a backend with mountpoint '' at the very
-              end of the mapping. That backend is then used as root, which means
-              that all items that don't lie in the namespace of any other
-              backend are stored there.
-
-        :type mapping: list of tuples of mountpoint -> backend mappings
-        :param mapping: [(mountpoint, backend), ...]
-        """
-        self.mapping = [(mountpoint.rstrip('/'), backend) for mountpoint, backend in mapping]
-
-    def open(self):
-        for mountpoint, backend in self.mapping:
-            backend.open()
-
-    def close(self):
-        for mountpoint, backend in self.mapping:
-            backend.close()
-
-    def _get_backend(self, itemname):
-        """
-        For a given fully-qualified itemname (i.e. something like Company/Bosses/Mr_Joe)
-        find the backend it belongs to (given by this instance's mapping), the local
-        itemname inside that backend and the mountpoint of the backend.
-
-        :param itemname: fully-qualified itemname
-        :returns: tuple of (backend, local itemname, mountpoint)
-        """
-        for mountpoint, backend in self.mapping:
-            if itemname == mountpoint or itemname.startswith(mountpoint and mountpoint + '/' or ''):
-                lstrip = mountpoint and len(mountpoint)+1 or 0
-                return backend, itemname[lstrip:], mountpoint
-        raise AssertionError("No backend found for %r. Available backends: %r" % (itemname, self.mapping))
-
-    def __iter__(self):
-        # Note: yields <backend_mountpoint>/<backend_revid> as router revid, so that this
-        #       can be given to get_revision and be routed to the right backend.
-        for mountpoint, backend in self.mapping:
-            for revid in backend:
-                yield u'%s/%s' % (mountpoint, revid)
-
-    def retrieve(self, revid):
-        mountpoint, revid = revid.rsplit(u'/', 1)
-        backend = self._get_backend(mountpoint)[0]
-        meta, data = backend.retrieve(revid)
-        if mountpoint:
-            meta[NAME] = u'%s/%s' % (mountpoint, meta[NAME])
-        return meta, data
-
-    # writing part
-    def create(self):
-        for mountpoint, backend in self.mapping:
-            if isinstance(backend, MutableBackendBase):
-                backend.create()
-            #XXX else: log info?
-
-    def destroy(self):
-        for mountpoint, backend in self.mapping:
-            if isinstance(backend, MutableBackendBase):
-                backend.destroy()
-            #XXX else: log info?
-
-    def store(self, meta, data):
-        itemname = meta[NAME]
-        backend, itemname, mountpoint = self._get_backend(itemname)
-        if not isinstance(backend, MutableBackendBase):
-            raise TypeError('backend %r mounted at %r is readonly' % (
-                backend, mountpoint))
-        meta[NAME] = itemname
-        return u'%s/%s' % (mountpoint, backend.store(meta, data))
-
-    def remove(self, revid):
-        mountpoint, revid = revid.rsplit(u'/', 1)
-        backend = self._get_backend(mountpoint)[0]
-        if not isinstance(backend, MutableBackendBase):
-            raise TypeError('backend %r mounted at %r is readonly' % (
-                backend, mountpoint))
-        backend.remove(revid)
-

File middleware/routing.py

View file
+# Copyright: 2008-2011 MoinMoin:ThomasWaldmann
+# Copyright: 2011 MoinMoin:RonnyPfannschmidt
+# Copyright: 2009 MoinMoin:ChristopherDenter
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+MoinMoin - router middleware
+
+Routes requests to different backends depending on the item name.
+"""
+
+
+from __future__ import absolute_import, division
+
+from config import NAME
+
+from backend import BackendBase, MutableBackendBase
+
+
+class Backend(MutableBackendBase):
+    """
+    router, behaves readonly for readonly mounts
+    """
+    def __init__(self, mapping):
+        """
+        Initialize router backend.
+
+        The mapping given must satisfy the following criteria:
+            * Order matters.
+            * Mountpoints are just item names, including the special '' (empty)
+              root item name.
+            * Trailing '/' of a mountpoint will be stripped.
+            * There *must* be a backend with mountpoint '' at the very
+              end of the mapping. That backend is then used as root, which means
+              that all items that don't lie in the namespace of any other
+              backend are stored there.
+
+        :type mapping: list of tuples of mountpoint -> backend mappings
+        :param mapping: [(mountpoint, backend), ...]
+        """
+        self.mapping = [(mountpoint.rstrip('/'), backend) for mountpoint, backend in mapping]
+
+    def open(self):
+        for mountpoint, backend in self.mapping:
+            backend.open()
+
+    def close(self):
+        for mountpoint, backend in self.mapping:
+            backend.close()
+
+    def _get_backend(self, itemname):
+        """
+        For a given fully-qualified itemname (i.e. something like Company/Bosses/Mr_Joe)
+        find the backend it belongs to (given by this instance's mapping), the local
+        itemname inside that backend and the mountpoint of the backend.
+
+        :param itemname: fully-qualified itemname
+        :returns: tuple of (backend, local itemname, mountpoint)
+        """
+        for mountpoint, backend in self.mapping:
+            if itemname == mountpoint or itemname.startswith(mountpoint and mountpoint + '/' or ''):
+                lstrip = mountpoint and len(mountpoint)+1 or 0
+                return backend, itemname[lstrip:], mountpoint
+        raise AssertionError("No backend found for %r. Available backends: %r" % (itemname, self.mapping))
+
+    def __iter__(self):
+        # Note: yields <backend_mountpoint>/<backend_revid> as router revid, so that this
+        #       can be given to get_revision and be routed to the right backend.
+        for mountpoint, backend in self.mapping:
+            for revid in backend:
+                yield u'%s/%s' % (mountpoint, revid)
+
+    def retrieve(self, revid):
+        mountpoint, revid = revid.rsplit(u'/', 1)
+        backend = self._get_backend(mountpoint)[0]
+        meta, data = backend.retrieve(revid)
+        if mountpoint:
+            meta[NAME] = u'%s/%s' % (mountpoint, meta[NAME])
+        return meta, data
+
+    # writing part
+    def create(self):
+        for mountpoint, backend in self.mapping:
+            if isinstance(backend, MutableBackendBase):
+                backend.create()
+            #XXX else: log info?
+
+    def destroy(self):
+        for mountpoint, backend in self.mapping:
+            if isinstance(backend, MutableBackendBase):
+                backend.destroy()
+            #XXX else: log info?
+
+    def store(self, meta, data):
+        itemname = meta[NAME]
+        backend, itemname, mountpoint = self._get_backend(itemname)
+        if not isinstance(backend, MutableBackendBase):
+            raise TypeError('backend %r mounted at %r is readonly' % (
+                backend, mountpoint))
+        meta[NAME] = itemname
+        return u'%s/%s' % (mountpoint, backend.store(meta, data))
+
+    def remove(self, revid):
+        mountpoint, revid = revid.rsplit(u'/', 1)
+        backend = self._get_backend(mountpoint)[0]
+        if not isinstance(backend, MutableBackendBase):
+            raise TypeError('backend %r mounted at %r is readonly' % (
+                backend, mountpoint))
+        backend.remove(revid)
+

File middleware/serialization.py

View file
+# Copyright: 2011 MoinMoin:RonnyPfannschmidt
+# Copyright: 2011 MoinMoin:ThomasWaldmann
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+MoinMoin - backend serialization / deserialization
+
+We use a simple custom format here:
+
+4 bytes length of meta (m)
+m bytes metadata (json serialization, utf-8 encoded)
+        (the metadata contains the data length d in meta['size'])
+d bytes binary data
+... (repeat for all meta/data)
+4 bytes 00 (== length of next meta -> there is none, this is the end)
+"""
+
+
+from __future__ import absolute_import, division
+
+import struct
+import json
+from io import BytesIO
+
+from werkzeug.wsgi import LimitedStream
+
+
+def serialize(backend, dst):
+    dst.writelines(serialize_iter(backend))
+
+
+def serialize_iter(backend):
+    for revid in backend:
+        meta, data = backend.retrieve(revid)
+
+        text = json.dumps(meta, ensure_ascii=False)
+        meta_str = text.encode('utf-8')
+        yield struct.pack('!i', len(meta_str))
+        yield meta_str
+        while True:
+            block = data.read(8192)
+            if not block:
+                break
+            yield block
+    yield struct.pack('!i', 0)
+
+
+def deserialize(src, backend):
+    while True:
+        meta_size_bytes = src.read(4)
+        meta_size = struct.unpack('!i', meta_size_bytes)[0]
+        if not meta_size:
+            return
+        meta_str = src.read(meta_size)
+        text = meta_str.decode('utf-8')
+        meta = json.loads(text)
+        data_size = meta[u'size']
+
+        limited = LimitedStream(src, data_size)
+        backend.store(meta, limited)
+        assert limited.is_exhausted
+

File middleware/serializer.py

-# Copyright: 2011 MoinMoin:RonnyPfannschmidt
-# Copyright: 2011 MoinMoin:ThomasWaldmann
-# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
-
-"""
-MoinMoin - backend serialization / deserialization
-
-We use a simple custom format here:
-
-4 bytes length of meta (m)
-m bytes metadata (json serialization, utf-8 encoded)
-        (the metadata contains the data length d in meta['size'])
-d bytes binary data
-... (repeat for all meta/data)
-4 bytes 00 (== length of next meta -> there is none, this is the end)
-"""
-
-
-from __future__ import absolute_import, division
-
-import struct
-import json
-from io import BytesIO
-
-from werkzeug.wsgi import LimitedStream
-
-
-def serialize(backend, dst):
-    dst.writelines(serialize_iter(backend))
-
-
-def serialize_iter(backend):
-    for revid in backend:
-        meta, data = backend.retrieve(revid)
-
-        text = json.dumps(meta, ensure_ascii=False)
-        meta_str = text.encode('utf-8')
-        yield struct.pack('!i', len(meta_str))
-        yield meta_str
-        while True:
-            block = data.read(8192)
-            if not block:
-                break
-            yield block
-    yield struct.pack('!i', 0)
-
-
-def deserialize(src, backend):
-    while True:
-        meta_size_bytes = src.read(4)
-        meta_size = struct.unpack('!i', meta_size_bytes)[0]
-        if not meta_size:
-            return
-        meta_str = src.read(meta_size)
-        text = meta_str.decode('utf-8')
-        meta = json.loads(text)
-        data_size = meta[u'size']
-
-        limited = LimitedStream(src, data_size)
-        backend.store(meta, limited)
-        assert limited.is_exhausted
-