pkumar avatar pkumar committed c7e87a7 Merge

merged default from main repo

Comments (0)

Files changed (11)

Add a comment to this file

MoinMoin/_tests/wikiconfig.py

File contents unchanged.

MoinMoin/apps/frontend/views.py

     :type item_name: unicode
     :returns: the list of all items which ref item_name
     """
-    refs_here = []
-    for item in items:
-        current_item = item.name
-        try:
-            current_revision = item.get_revision(-1)
-        except NoSuchRevisionError:
-            continue
-        links = current_revision.get(ITEMLINKS, [])
-        transclusions = current_revision.get(ITEMTRANSCLUSIONS, [])
-
-        refs = set(links + transclusions)
-        if item_name in refs:
-            refs_here.append(current_item)
-    return refs_here
-
+    from MoinMoin.search.indexing import WhooshIndex
+    from whoosh.query import Term, Or
+    index_object = WhooshIndex()
+    ix = index_object.latest_revisions_index
+    with ix.searcher() as searcher:
+        q = Or([Term("itemtransclusions", item_name), Term("itemlinks", item_name)])
+        results = searcher.search(q)
+        return [result["name"] for result in results]
 
 @frontend.route('/+history/<itemname:item_name>')
 def history(item_name):

MoinMoin/script/maint/index.py

 # License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
 
 """
-MoinMoin - Manage whoosh indexes
+MoinMoin - manage whoosh indexes (building, updating, (re)moving and displaying)
 """
 
 import os, datetime

MoinMoin/search/indexing.py

                 'all_revisions_index': 'all_revisions_schema',
                }
 
-    def __init__(self, index_dir=None, cfg=None):
+    def __init__(self, index_dir=None, cfg=None, force_create=False):
         """
         Create and open indexes in index_dir
 
+        :param force_create: Create empty index in index_dir even if index exists
         :param index_dir: Directory where whoosh indexes will be created, default None
         :param cfg: Application config (app.cfg), default None
         """
             self.all_revisions_schema.add(glob, field_type, glob=True)
 
         for index_name, index_schema in self._indexes.items():
-            self.open_index(index_name, index_schema, create=True, index_dir=self._index_dir)
+            self.open_index(index_name, index_schema, create=True, force_create=force_create,
+                            index_dir=self._index_dir
+                           )
 
-    def open_index(self, indexname, schema, create=False, index_dir=None):
+    def open_index(self, indexname, schema, create=False, force_create=False, index_dir=None):
         """
         Open index <indexname> in <index_dir>. if opening fails and <create>
         is True, try creating the index and retry opening it afterwards.
         return index object.
 
-        :param index_dir: Directory where whoosh indexes will be created
+
         :param indexname: Name of created index
         :param schema: which schema applies
+        :param create: create index if index doesn't exist
+        :param force_create: force create new empty index in index_dir
+        :param index_dir: Directory where whoosh indexes will be created
         """
         index_dir = index_dir or self._cfg.index_dir
+        if force_create:
+            self.create_index(index_dir, indexname, schema)
         try:
             index = open_dir(index_dir, indexname=indexname)
             setattr(self, indexname, index)
         except (IOError, OSError) as err:
             logging.error(u"%s [while trying to create index '%s' in '%s']" % (str(err), indexname, index_dir))
 
+    def remove_index(self):
+        """
+        Create empty index in index_dir and removing old
+        """
+        for index_name, index_schema in self._indexes.items():
+            self.create_index(indexname=index_name, schema=index_schema, index_dir=self._index_dir)

MoinMoin/storage/_tests/test_indexing.py

+# Copyright: 2011 MoinMoin:MichaelMayorov
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+    MoinMoin - Test - indexing
+"""
+
+import py
+
+from MoinMoin._tests import update_item, nuke_item
+from MoinMoin._tests.wikiconfig import Config
+from MoinMoin.storage.backends.indexing import ItemIndex
+from MoinMoin.config import NAME
+
+# Revisions for tests
+document_revs = [
+                 { "wikiname": u"Test",
+                   "name": u"DocumentOne",
+                   "uuid": u"68054804bd7141609b7c441143adf83d",
+                   "rev_no": 0,
+                   "mtime":  1172969203.1,
+                   "content": u"Some not very long content line",
+                   "contenttype": u"text/plain;charset=utf-8",
+                   "tags": [u"Rest", u"in", u"peace"],
+                   "itemlinks": [u"Home", u"Find"],
+                   "itemtransclusions": [u"Another", u"Stuff"],
+                   "language": u"en",
+                   "address": u"127.0.0.1",
+                   "hostname": u"localhost",
+                 },
+                 { "wikiname": u"Test",
+                   "name": u"DocumentOne",
+                   "uuid": u"68054804bd7141609b7c441143adf83d",
+                   "rev_no": 1,
+                   "mtime":  1172969203.9,
+                   "content": u"This line should be much better, but it isn't",
+                   "contenttype": u"text/plain;charset=utf-8",
+                   "tags": [u"first_tag", u"second_tag"],
+                   "itemlinks": [u"Home", u"Find"],
+                   "itemtransclusions": [u"Another", u"Stuff"],
+                   "language": u"en",
+                   "address": u"127.0.0.1",
+                   "hostname": u"localhost",
+                 }
+                ]
+
+class TestIndexing(object):
+
+    def setup_method(self, method):
+        self.wikiconfig = Config()
+        self.item_index = ItemIndex(self.wikiconfig, force_create=True)
+        self.all_revs_ix = self.item_index.index_object.all_revisions_index
+        self.latest_revs_ix = self.item_index.index_object.latest_revisions_index
+
+    def teardown_method(self, method):
+         self.item_index.remove_index()
+
+    def test_create_item(self):
+        """ Try to search for non-existent revision, add it to backend and then search again """
+        revision = document_revs[0]
+        with self.all_revs_ix.searcher() as searcher:
+            found_document = searcher.document(name_exact=revision[NAME])
+        assert found_document is None
+        with self.latest_revs_ix.searcher() as searcher:
+            found_document = searcher.document(name_exact=revision[NAME])
+        assert found_document is None
+        backend_rev = update_item(revision[NAME], revision["rev_no"],
+                                  revision, revision["content"])
+        with self.all_revs_ix.searcher() as searcher:
+            found_document = searcher.document(name_exact=revision[NAME])
+        assert found_document is not None and found_document[NAME] == revision[NAME]
+        with self.latest_revs_ix.searcher() as searcher:
+            found_document = searcher.document(name_exact=revision[NAME])
+        assert found_document is not None and found_document[NAME] == revision[NAME]
+
+    def test_create_rev(self):
+        """ Create 2 item revisions and try to search for them in backend """
+        revision1, revision2 = document_revs
+        backend_rev = update_item(revision1[NAME], revision1["rev_no"], revision1, revision1["content"])
+        backend_rev = update_item(revision2[NAME], revision2["rev_no"], revision2, revision2["content"])
+        with self.all_revs_ix.searcher() as searcher:
+            found_documents = list(searcher.documents(name_exact=revision1[NAME]))
+        assert len(found_documents) == 2
+        with self.latest_revs_ix.searcher() as searcher:
+            found_documents = list(searcher.documents(name_exact=revision2[NAME]))
+        assert len(found_documents) == 1 and found_documents[0]["rev_no"] == 1
+
+    def test_destroy(self):
+        """ Create & Destroy test for backend item """
+        py.test.skip("Anonymous can't destroy stuff from backend, thus we leave this test for now")
+        revision = document_revs[0]
+        backend_rev = update_item(revision[NAME], revision["rev_no"], revision, revision["content"])
+        with self.all_revs_ix.searcher() as searcher:
+            found_documents = list(searcher.documents(name_exact=revision[NAME]))
+            assert len(found_documents) == 1
+            nuke_item(revision[NAME])
+            found_document = searcher.document(name_exact=revision[NAME])
+            assert found_document is None

MoinMoin/storage/backends/indexing.py

     """
     Index for Items/Revisions
     """
-    def __init__(self, cfg):
+    def __init__(self, cfg, force_create=False):
         self.wikiname = cfg.interwikiname
-        self.index_object = WhooshIndex(cfg=cfg)
+        self.index_object = WhooshIndex(force_create=force_create, cfg=cfg)
 
     def close(self):
         self.index_object.all_revisions_index.close()
         self.index_object.latest_revisions_index.close()
 
+    def remove_index(self):
+        self.index_object.remove_index()
+
     def update_item(self, metas):
         """
         update item (not revision!) metadata

docs/devel/api/MoinMoin.rst

     MoinMoin.macro
     MoinMoin.mail
     MoinMoin.script
+    MoinMoin.search
     MoinMoin.security
     MoinMoin.signalling
     MoinMoin.storage

docs/devel/api/MoinMoin.script.maint.rst

     :undoc-members:
     :show-inheritance:
 
+:mod:`index` Module
+-------------------
+.. automodule:: MoinMoin.script.maint.index
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 :mod:`reduce_revisions` Module
 ------------------------------
 

docs/devel/api/MoinMoin.search.rst

+search Package
+==============
+
+:mod:`search` Package
+---------------------
+
+.. automodule:: MoinMoin.search
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`indexing` Module
+----------------------
+
+.. automodule:: MoinMoin.search.indexing
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 .. toctree::
    :maxdepth: 2
 
+   user/searching
    user/markups
 
 Administrating MoinMoin

docs/user/searching.rst

+=========
+Searching
+=========
+Entering search queries
+=======================
+
+Usually there is a simple and rather short search query input field offered by the theme - if you submit a query from there, it will only search the current item revisions and show the related results to you.
+
+On that search results view, you will get a bigger search query input field (e.g. for refining your query) and you may also choose to additionally search in non-current revision item revisions (selecting that will search in all revisions).
+
+Simple search queries
+=====================
+Just enter one or few simple words.
+
+If you give multiple words, it will only find documents containing ALL those words (``AND`` is the default).
+
+You can use ``AND`` (default), ``OR``, ``NOT`` to refine your search.
+
+Examples
+--------
+::
+
+    # search for "wiki"
+    wiki
+
+    # search for documents containing "wiki" AND "moin"
+    wiki moin
+    # this does the same:
+    wiki AND moin
+    # search for documents containing "wiki" OR "moin"
+    wiki OR moin
+    # search for documents containing "wiki" and NOT "bad"
+    wiki NOT bad
+
+
+Wildcard queries
+----------------
+If you want to enter word fragments or if you are not sure about spelling or word form, use wildcards for the parts you do not know:
+ * ``?`` matches 1 arbitrary character
+ * ``*`` matches multiple arbitrary characters.
+
+Examples
+--------
+::
+
+    # would match on wiki, wika, wikb, ...
+    wik?
+    # would match on wiki, willi, wi, ...
+    w*i
+
+    # you can also use it for poor man's language independant stemming:
+    # matches on wash, washes, washing, washed, ...
+    wash*
+
+
+Searching in specific fields
+----------------------------
+
+As long as you do not specify otherwise, moin will search in ``name``, ``name_exact`` and ``content`` fields.
+
+To specify the field to search in, just use the `fieldname:searchterm` syntax.
+
+Examples
+--------
+::
+
+    # search in metadata fields
+    contenttype:text
+    contenttype:image/jpeg
+    tags:demo
+    mtime:201108
+    address:127.0.0.1
+    language:en 
+    hostname:localhost
+
+    # search items with item ACL that explicitly gives Joe read rights
+    acl:Joe:+read
+
+    # limiting search to a specific wiki in a wiki farm index
+    wikiname:SomeWiki
+
+
+Notes
+-----
+moin uses indexed search - keep in mind that this has some special properties:
+ * as it is using an index, it is rather fast usually
+ * because it is only using the index, it can only find what was put there
+ * if you use wildcards, it will still use the index, but in a different, slower way
+
+E.g.:
+ * ``foobar`` is put into the index somehow
+ * you search for "ooba" - you will not find it, because only ``foobar`` was put into the index
+ * solution: search for ``foobar`` - fast and will find it
+ * solution: search for ``*ooba*`` - slow, but will find it
+
+More infos
+----------
+
+`Whoosh docs about its default query language <http://packages.python.org/Whoosh/querylang.html>`_
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.