moin-2.0 / build / lib / MoinMoin / storage / middleware /

# Copyright: 2011 MoinMoin:RonnyPfannschmidt
# Copyright: 2011 MoinMoin:ThomasWaldmann
# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.

MoinMoin - backend serialization / deserialization

We use a simple custom format here:

4 bytes length of meta (m)
m bytes metadata (json serialization, utf-8 encoded)
        (the metadata contains the data length d in meta['size'])
d bytes binary data
... (repeat for all meta/data)
4 bytes 00 (== length of next meta -> there is none, this is the end)

from __future__ import absolute_import, division

import struct
import json
from io import BytesIO

from werkzeug.wsgi import LimitedStream

def serialize(backend, dst):

def serialize_rev(meta, data):
    if meta is None:
        # this is the end!
        yield struct.pack('!i', 0)
        text = json.dumps(meta, ensure_ascii=False)
        meta_str = text.encode('utf-8')
        yield struct.pack('!i', len(meta_str))
        yield meta_str
        while True:
            block =
            if not block:
            yield block

def serialize_iter(backend):
    for mountpoint, revid in backend:
        meta, data = backend.retrieve(mountpoint, revid)
        for data in serialize_rev(meta, data):
            yield data
    for data in serialize_rev(None, None):
        yield data

def deserialize(src, backend):
    while True:
        meta_size_bytes =
        meta_size = struct.unpack('!i', meta_size_bytes)[0]
        if not meta_size:
        meta_str =
        text = meta_str.decode('utf-8')
        meta = json.loads(text)
        data_size = meta[u'size']
        curr_pos = src.tell()
        limited = LimitedStream(src, data_size), limited)
        if not limited.is_exhausted:
            # if we already have the DATAID in the backend, the backend code
            # does not read from the limited stream:
            assert limited._pos == 0
            # but we must seek to get forward to the next item:
   + data_size)