Commits

Thomas Waldmann  committed 8ce9e34

sqlite storage: add zlib compression (optional, off by default), tests

  • Participants
  • Parent commits 54fbe54

Comments (0)

Files changed (2)

File storage/_tests/test_sqlite.py

 
 class TestBytesStorage(BytesStorageTestBase):
     def setup_method(self, method):
-        self.st = BytesStorage('testdb.sqlite', 'testbs') # ':memory:' does not work, strange
+        self.st = BytesStorage('testdb.sqlite', 'testbs', compression_level=0) # ':memory:' does not work, strange
         self.st.create()
         self.st.open()
 
-class TestFileStorage(FileStorageTestBase):
+
+class TestBytesStorageCompressed(BytesStorageTestBase):
     def setup_method(self, method):
-        self.st = FileStorage('testdb.sqlite', 'testfs') # ':memory:' does not work, strange
+        self.st = BytesStorage('testdb.sqlite', 'testbs', compression_level=1) # ':memory:' does not work, strange
         self.st.create()
         self.st.open()
 
 
+class TestFileStorage(FileStorageTestBase):
+    def setup_method(self, method):
+        self.st = FileStorage('testdb.sqlite', 'testfs', compression_level=0) # ':memory:' does not work, strange
+        self.st.create()
+        self.st.open()
+
+
+class TestFileStorageCompressed(FileStorageTestBase):
+    def setup_method(self, method):
+        self.st = FileStorage('testdb.sqlite', 'testfs', compression_level=1) # ':memory:' does not work, strange
+        self.st.create()
+        self.st.open()
+

File storage/sqlite.py

 # License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
 
 """
-MoinMoin - sqlite3 storage
+MoinMoin - sqlite3 key/value storage (optionally with zlib/"gzip" compression)
 """
 
 
 from __future__ import absolute_import, division
 
 from StringIO import StringIO
+import zlib
 from sqlite3 import *
 
 from storage import MutableStorageBase, BytesMutableStorageBase, FileMutableStorageBase
     """
     A simple sqlite3 based storage.
     """
-    def __init__(self, db_name, table_name):
+    def __init__(self, db_name, table_name, compression_level=0):
+        """
+        Just store the params.
+
+        :param db_name: database (file)name
+        :param table_name: table to use for this storage (we only touch this table)
+        :param compression_level: zlib compression level
+                                  0 = no compr, 1 = fast/small, ..., 9 = slow/smaller
+                                  we recommend 0 for low cpu usage, 1 for low disk space usage
+                                  high compression levels don't give much better compression,
+                                  but use lots of cpu (e.g. 6 is about 2x more cpu than 1).
+        """
         self.db_name = db_name
         self.table_name = table_name
+        self.compression_level = compression_level
 
     def create(self):
         conn = connect(self.db_name)
         with self.conn:
             self.conn.execute('delete from %s where key=?' % self.table_name, (key, ))
 
+    def _compress(self, value):
+        if self.compression_level:
+            value = zlib.compress(value, self.compression_level)
+        # we store some magic start/end markers and the compression level,
+        # so we can later uncompress correctly (or rather NOT uncompress if level == 0)
+        return "{{{GZ%(level)d|%(value)s}}}" % dict(level=self.compression_level, value=value)
+
+    def _decompress(self, value):
+        if not value.startswith("{{{GZ") or not value.endswith("}}}"):
+            raise ValueError("Invalid data format in database.")
+        compression_level = int(value[5])
+        value = value[7:-3]
+        if compression_level:
+            value = zlib.decompress(value)
+        return value
 
 class BytesStorage(_Storage, BytesMutableStorageBase):
     def __getitem__(self, key):
         rows = list(self.conn.execute("select value from %s where key=?" % self.table_name, (key, )))
         if not rows:
             raise KeyError(key)
-        return str(rows[0]['value'])
+        value = str(rows[0]['value'])
+        return self._decompress(value)
 
     def __setitem__(self, key, value):
+        value = self._compress(value)
         with self.conn:
             self.conn.execute('insert into %s values (?, ?)' % self.table_name, (key, buffer(value)))
 
         rows = list(self.conn.execute("select value from %s where key=?" % self.table_name, (key, )))
         if not rows:
             raise KeyError(key)
-        return StringIO(rows[0]['value'])
+        value = str(rows[0]['value'])
+        return StringIO(self._decompress(value))
 
     def __setitem__(self, key, stream):
         value = stream.read()
+        value = self._compress(value)
         with self.conn:
             self.conn.execute('insert into %s values (?, ?)' % self.table_name, (key, buffer(value)))