Thomas Waldmann avatar Thomas Waldmann committed 4a05406

make whoosh storage type and params configurable, updated docs

Currently it only supports:

index_storage = 'FileStorage', (index_dir, ), {}

But it is now easy to add other types of whoosh storages, see
MoinMoin/storage/middleware/indexing.py (search for WHOOSH_FILESTORAGE).

Comments (0)

Files changed (6)

MoinMoin/_tests/wikiconfig.py

     _here = abspath(dirname(__file__))
     _root = abspath(join(_here, '..', '..'))
     data_dir = join(_here, 'wiki', 'data') # needed for plugins package TODO
-    index_dir = join(_here, 'wiki', 'index')
+    index_storage = 'FileStorage', (join(_here, 'wiki', 'index'), ), {}
     content_acl = None
     item_root = 'FrontPage'
     interwikiname = u'MoinTest'
     if app.cfg.create_storage:
         app.router.create()
     app.router.open()
-    app.storage = indexing.IndexingMiddleware(app.cfg.index_dir, app.router,
+    app.storage = indexing.IndexingMiddleware(app.cfg.index_storage, app.router,
                                               wiki_name=app.cfg.interwikiname,
                                               acl_rights_contents=app.cfg.acl_rights_contents)
     if app.cfg.create_index:

MoinMoin/storage/middleware/_tests/test_serialization.py

 
 from StringIO import StringIO
 
-from ..indexing import IndexingMiddleware
+from ..indexing import IndexingMiddleware, WHOOSH_FILESTORAGE
 from ..routing import Backend as RoutingBackend
 from ..serialization import serialize, deserialize
 
     request.addfinalizer(backend.destroy)
     request.addfinalizer(backend.close)
 
-    mw = IndexingMiddleware(index_dir=str(tmpdir/'foo'),
+    mw = IndexingMiddleware(index_storage=(WHOOSH_FILESTORAGE, (str(tmpdir/'foo'), ), {}),
                             backend=backend)
     mw.create()
     mw.open()

MoinMoin/storage/middleware/indexing.py

 from MoinMoin.storage.error import NoSuchItemError, ItemAlreadyExistsError
 
 
+WHOOSH_FILESTORAGE = 'FileStorage'
 INDEXES = [LATEST_REVS, ALL_REVS, ]
 
 
 
 
 class IndexingMiddleware(object):
-    def __init__(self, index_dir, backend, wiki_name=None, acl_rights_contents=[], **kw):
+    def __init__(self, index_storage, backend, wiki_name=None, acl_rights_contents=[], **kw):
         """
         Store params, create schemas.
         """
-        self.index_dir = index_dir
-        self.index_dir_tmp = index_dir + '.temp'
+        self.index_storage = index_storage
         self.backend = backend
         self.wikiname = wiki_name
         self.ix = {}  # open indexes
         # or latest revs index):
         self.common_fields = set(latest_revs_fields.keys()) & set(all_revs_fields.keys())
 
+    def get_storage_params(self, tmp=False):
+        kind, params, kw = self.index_storage
+        params, kw = list(params), dict(kw)  # better make a (mutable) copy
+        if kind == WHOOSH_FILESTORAGE:
+            # index_storage = 'FileStorage', (index_dir, ), {}
+            if tmp:
+                params[0] += '.temp'
+            from whoosh.filedb.filestore import FileStorage
+            cls = FileStorage
+        else:
+            raise ValueError("index_storage = {0!r} is not supported!".format(kind))
+        return kind, cls, params, kw
+
     def get_storage(self, tmp=False, create=False):
         """
         Get the whoosh storage (whoosh supports different kinds of storage,
         e.g. to filesystem or to GAE).
         Currently we only support the FileStorage.
         """
-        from whoosh.filedb.filestore import FileStorage
-        index_dir = self.index_dir_tmp if tmp else self.index_dir
-        if create:
-            try:
-                os.mkdir(index_dir)
-            except:
-                # ignore exception, we'll get another exception below
-                # in case there are problems with the index_dir
-                pass
-        storage = FileStorage(index_dir)
-        return storage
+        kind, cls, params, kw = self.get_storage_params(tmp)
+        if kind == WHOOSH_FILESTORAGE:
+            if create:
+                index_dir = params[0]
+                try:
+                    os.mkdir(index_dir)
+                except:
+                    # ignore exception, we'll get another exception below
+                    # in case there are problems with the index_dir
+                    pass
+        return cls(*params, **kw)
 
     def open(self):
         """
         Destroy all indexes.
         """
         # XXX this is whoosh backend specific and currently only works for FileStorage.
-        index_dir = self.index_dir_tmp if tmp else self.index_dir
-        if os.path.exists(index_dir):
-            shutil.rmtree(index_dir)
+        kind, cls, params, kw = self.get_storage_params(tmp)
+        if kind == WHOOSH_FILESTORAGE:
+            index_dir = params[0]
+            if os.path.exists(index_dir):
+                shutil.rmtree(index_dir)
 
     def move_index(self):
         """
-        Move freshly built indexes from index_dir_tmp to index_dir.
+        Move freshly built indexes from tmp storage to normal storage
         """
         # XXX this is whoosh backend specific and currently only works for FileStorage.
-        self.destroy()
-        os.rename(self.index_dir_tmp, self.index_dir)
+        kind, cls, params, kw = self.get_storage_params(False)
+        if kind == WHOOSH_FILESTORAGE:
+            _, _, params_tmp, _ = self.get_storage_params(True)
+            self.destroy()
+            index_dir, index_dir_tmp = params[0], params_tmp[0]
+            os.rename(index_dir_tmp, index_dir)
 
     def index_revision(self, meta, content, async=True):
         """

docs/admin/index.rst

 
 Configuration
 =============
-Your wiki config needs ``index_dir`` to point to a writable directory. A fast,
-local filesystem is preferred.
-Use something like::
+Your need to have a ``index_storage`` entry in your wiki config.
 
-    index_dir = "/path/to/moin-2.0/wiki/index"
+We use whoosh for indexing and as whoosh supports multiple storage backends,
+this entry is made to potentially support any storage supported by whoosh.
 
-**Note:**
-* The path MUST be absolute.
-* Moin will use `index_dir`.temp location as well, if you build an index at
+In general, this entry has the form of::
+
+    index_storage = kind, (p1, p2, ...), {kw1=..., kw2=..., ...}
+
+Currently, we only support the 'FileStorage' kind of index storage, which only
+has one parameter - the index directory::
+
+    index_storage = 'FileStorage', ("/path/to/moin-2.0/wiki/index", ), {}
+
+**Notes for FileStorage:**
+* The path MUST be absolute, writable and should be on a fast, local filesystem.
+* Moin will use `index.temp` directory as well, if you build an index at
 the `temporary location`.
 
 
 ``Sales``::
 
       interwikiname = u"Sales"
-      index_dir = "/path/to/wiki/index"
+      index_storage = 'FileStorage', ("/path/to/moin-2.0/wiki/index", ), {}
+
 
 ``Engineering``::
 
       interwikiname = u"Engineering"
-      index_dir = "/path/to/wiki/index"
+      index_storage = 'FileStorage', ("/path/to/moin-2.0/wiki/index", ), {}
+
 
 Now do the initial index building::
 
     # If that's not true, feel free to adjust the pathes.
     instance_dir = os.path.join(wikiconfig_dir, 'wiki')
     data_dir = os.path.join(instance_dir, 'data') # Note: this used to have a trailing / in the past
-    index_dir = os.path.join(instance_dir, "index")
+    index_storage = 'FileStorage', (os.path.join(instance_dir, "index"), ), {}
 
     # This provides a simple default setup for your backend configuration.
     # 'stores:fs:...' indicates that you want to use the filesystem backend.
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.