Thomas Waldmann avatar Thomas Waldmann committed 8a32453

serialization: add options to serialize backends by backendname (or all of them)

fix issue with file positioning / data consumption when deserializing

Comments (0)

Files changed (3)

MoinMoin/script/maint/serialization.py

     option_list = [
         Option('--file', '-f', dest='filename', type=unicode, required=False,
                help='Filename of the output file.'),
+        Option('--backends', '-b', dest='backends', type=unicode, required=False,
+               help='Backend names to serialize (comma separated).'),
+        Option('--all-backends', '-a', dest='all_backends', action='store_true', default=False,
+               help='Serialize all configured backends.'),
     ]
 
-    def run(self, filename=None):
+    def run(self, filename=None, backends=None, all_backends=False):
         if filename is None:
             f = sys.stdout
         else:
             f = open(filename, "wb")
         with f as f:
-            serialize(app.storage.backend, f)
+            existing_backends = set(app.cfg.backend_mapping)
+            if all_backends:
+                backends = set(app.cfg.backend_mapping)
+            elif backends:
+                backends = set(backends.split(','))
+            if backends:
+                # low level - directly serialize some backend contents -
+                # this does not use the index:
+                if backends.issubset(existing_backends):
+                    for backend_name in backends:
+                        backend = app.cfg.backend_mapping.get(backend_name)
+                        serialize(backend, f)
+                else:
+                    print "Error: Wrong backend name given."
+                    print "Given Backends: %r" % backends
+                    print "Configured Backends: %r" % existing_backends
 
 
 class Deserialize(Command):

MoinMoin/storage/backends/stores.py

             dataid = meta[DATAID]
             # we will just asume stuff is correct if you pass it with a data id
             if dataid not in self.data_store:
-                # XXX issue: if we do not store if we already have the dataid in the store,
-                # XXX deserialization does not work as the fpos does not advance to the next record,
-                # XXX because we do not read from the source file. Remove the check?
                 self.data_store[dataid] = data
+            else:
+                # this is reading the data to avoid this issue:
+                # if we do not store if we already have the dataid in the store,
+                # deserialization does not work as the fpos does not advance to the next record,
+                # because we do not read from the source file. Remove the check?
+                while data.read(64*1024):
+                    pass
+
         # if something goes wrong below, the data shall be purged by a garbage collection
         metaid = self._store_meta(meta)
         return metaid

MoinMoin/storage/middleware/serialization.py

             yield block
 
 def serialize_iter(backend):
-    for mountpoint, revid in backend:
-        meta, data = backend.retrieve(mountpoint, revid)
+    for revid in backend:
+        if isinstance(revid, tuple):
+            # router middleware gives tuples and wants both values for retrieve:
+            meta, data = backend.retrieve(*revid)
+        else:
+            # lower level backends have simple revids
+            meta, data = backend.retrieve(revid)
         for data in serialize_rev(meta, data):
             yield data
     for data in serialize_rev(None, None):
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.