Commits

Thomas Waldmann committed 94e388c

use 'constants' for metadata keys / whoosh field names

Comments (0)

Files changed (10)

MoinMoin/apps/feed/views.py

 from MoinMoin import wikiutil
 from MoinMoin.i18n import _, L_, N_
 from MoinMoin.apps.feed import feed
-from MoinMoin.config import NAME, ACL, ACTION, ADDRESS, HOSTNAME, USERID, COMMENT, MTIME
+from MoinMoin.config import NAME, NAME_EXACT, WIKINAME, ACL, ACTION, ADDRESS, HOSTNAME, USERID, COMMENT, MTIME, REV_NO
 from MoinMoin.themes import get_editor_info
 from MoinMoin.items import Item
 from MoinMoin.util.crypto import cache_key
     if content is None:
         title = app.cfg.sitename
         feed = AtomFeed(title=title, feed_url=request.url, url=request.host_url)
-        query = Term("wikiname", app.cfg.interwikiname)
+        query = Term(WIKINAME, app.cfg.interwikiname)
         if item_name:
-            query = And([query, Term("name_exact", item_name), ])
-        history = flaskg.storage.search(query, all_revs=True, sortedby=[MTIME, "rev_no"], reverse=True, limit=100)
+            query = And([query, Term(NAME_EXACT, item_name), ])
+        history = flaskg.storage.search(query, all_revs=True, sortedby=[MTIME, REV_NO], reverse=True, limit=100)
         for doc in history:
             name = doc[NAME]
-            this_revno = doc["rev_no"]
+            this_revno = doc[REV_NO]
             item = flaskg.storage.get_item(name)
             this_rev = item.get_revision(this_revno)
             try:

MoinMoin/apps/frontend/views.py

 from MoinMoin.items import Item, NonExistent
 from MoinMoin.items import ROWS_META, COLS, ROWS_DATA
 from MoinMoin import config, user, util, wikiutil
-from MoinMoin.config import ACTION, COMMENT, CONTENTTYPE, ITEMLINKS, ITEMTRANSCLUSIONS, NAME, CONTENTTYPE_GROUPS, MTIME, TAGS
+from MoinMoin.config import ACTION, COMMENT, WIKINAME, CONTENTTYPE, ITEMLINKS, ITEMTRANSCLUSIONS, NAME, NAME_EXACT, \
+                            CONTENTTYPE_GROUPS, MTIME, TAGS, REV_NO, CONTENT
 from MoinMoin.util import crypto
 from MoinMoin.util.interwiki import url_for_item
 from MoinMoin.security.textcha import TextCha, TextChaizedForm, TextChaValid
     query = search_form['q'].value
     if valid:
         history = bool(request.values.get('history'))
-        qp = flaskg.storage.query_parser(["name_exact", "name", "content"], all_revs=history)
+        qp = flaskg.storage.query_parser([NAME_EXACT, NAME, CONTENT], all_revs=history)
         q = qp.parse(query)
         with flaskg.storage.searcher(all_revs=history) as searcher:
             results = searcher.search(q, limit=100)
             return render_template('search.html',
                                    results=results,
-                                   name_suggestions=u', '.join([word for word, score in results.key_terms('name', docs=20, numterms=10)]),
-                                   content_suggestions=u', '.join([word for word, score in results.key_terms('content', docs=20, numterms=10)]),
+                                   name_suggestions=u', '.join([word for word, score in results.key_terms(NAME, docs=20, numterms=10)]),
+                                   content_suggestions=u', '.join([word for word, score in results.key_terms(CONTENT, docs=20, numterms=10)]),
                                    query=query,
                                    medium_search_form=search_form,
                                    item_name='+search', # XXX
     :type item_name: unicode
     :returns: the list of all items which ref item_name
     """
-    q = And([Term("wikiname", app.cfg.interwikiname),
-             Or([Term("itemtransclusions", item_name), Term("itemlinks", item_name)])])
+    q = And([Term(WIKINAME, app.cfg.interwikiname),
+             Or([Term(ITEMTRANSCLUSIONS, item_name), Term(ITEMLINKS, item_name)])])
     docs = flaskg.storage.search(q, all_revs=False)
-    return [doc["name"] for doc in docs]
+    return [doc[NAME] for doc in docs]
 
 
 @frontend.route('/+history/<itemname:item_name>')
         results_per_page = flaskg.user.results_per_page
     else:
         results_per_page = app.cfg.results_per_page
-    query = And([Term("wikiname", app.cfg.interwikiname), Term("name_exact", item_name), ])
+    query = And([Term(WIKINAME, app.cfg.interwikiname), Term(NAME_EXACT, item_name), ])
     # TODO: due to how getPageContent and the template works, we need to use limit=None -
     # it would be better to use search_page (and an appropriate limit, if needed)
-    docs = flaskg.storage.search(query, all_revs=True, sortedby="rev_no", reverse=True, limit=None)
+    docs = flaskg.storage.search(query, all_revs=True, sortedby=REV_NO, reverse=True, limit=None)
     # get rid of the content value to save potentially big amounts of memory:
-    history = [dict((k, v) for k, v in doc.iteritems() if k != 'content') for doc in docs]
+    history = [dict((k, v) for k, v in doc.iteritems() if k != CONTENT) for doc in docs]
     history_page = util.getPageContent(history, offset, results_per_page)
     return render_template('history.html',
                            item_name=item_name, # XXX no item here
         results_per_page = flaskg.user.results_per_page
     else:
         results_per_page = app.cfg.results_per_page
-    query = Term("wikiname", app.cfg.interwikiname)
+    query = Term(WIKINAME, app.cfg.interwikiname)
     if bookmark_time is not None:
         query = And([query, DateRange(MTIME, start=bookmark_time, end=None)])
     # TODO: we need use limit=None to simulate previous implementation's behaviour -
     # it would be better to use search_page (and an appropriate limit, if needed)
-    history = flaskg.storage.search(query, all_revs=True, sortedby=[MTIME, "rev_no"], reverse=True, limit=None)
+    history = flaskg.storage.search(query, all_revs=True, sortedby=[MTIME, REV_NO], reverse=True, limit=None)
     item_groups = OrderedDict()
     for doc in history:
         current_item_name = doc[NAME]
 
         # Aggregating comments, authors and revno
         for doc in docs:
-            rev_no = doc["rev_no"]
+            rev_no = doc[REV_NO]
             revnos.append(rev_no)
             comment = doc.get(COMMENT)
             if comment:
     tags_counts = {}
     for doc in docs:
         tags = doc.get(TAGS, [])
-        logging.debug("name %s rev %s tags %s" % (doc[NAME], doc["rev_no"], tags))
+        logging.debug("name %s rev %s tags %s" % (doc[NAME], doc[REV_NO], tags))
         for tag in tags:
             tags_counts[tag] = tags_counts.setdefault(tag, 0) + 1
     tags_counts = sorted(tags_counts.items())
     """
     show all items' names that have tag <tag>
     """
-    query = And([Term("wikiname", app.cfg.interwikiname), Term(TAGS, tag), ])
-    docs = flaskg.storage.search(query, all_revs=False, sortedby="name_exact", limit=None)
+    query = And([Term(WIKINAME, app.cfg.interwikiname), Term(TAGS, tag), ])
+    docs = flaskg.storage.search(query, all_revs=False, sortedby=NAME_EXACT, limit=None)
     item_names = [doc[NAME] for doc in docs]
     return render_template("item_link_list.html",
                            headline=_("Items tagged with %(tag)s", tag=tag),

MoinMoin/config/__init__.py

 # some backends may use this also for other purposes.
 HASH_ALGORITHM = 'sha1'
 
+# some field names for whoosh index schema / documents in index:
+NAME_EXACT = "name_exact"
+REV_NO = "rev_no"
+WIKINAME = "wikiname"
+CONTENT = "content"
+
 # structure for contenttype groups
 CONTENTTYPE_GROUPS = [
     ('markup text items', [

MoinMoin/converter/include.py

 
 from whoosh.query import Term, And, Wildcard
 
-from MoinMoin.config import NAME
+from MoinMoin.config import NAME, NAME_EXACT, WIKINAME
 from MoinMoin import wikiutil
 from MoinMoin.items import Item
 from MoinMoin.util.mime import type_moin_document
                     if xp_include_pages.startswith('^'):
                         # get rid of the leading ^ the Include macro needed to get into "regex mode"
                         xp_include_pages = xp_include_pages[1:]
-                    query = And([Term("wikiname", app.cfg.interwikiname), Wildcard("name_exact", xp_include_pages)])
+                    query = And([Term(WIKINAME, app.cfg.interwikiname), Wildcard(NAME_EXACT, xp_include_pages)])
                     reverse = xp_include_sort == 'descending'
-                    results = flaskg.storage.search(query, all_revs=False, sortedby="name_exact", reverse=reverse, limit=None)
+                    results = flaskg.storage.search(query, all_revs=False, sortedby=NAME_EXACT, reverse=reverse, limit=None)
                     pagelist = [result[NAME] for result in results]
                     if xp_include_skipitems is not None:
                         pagelist = pagelist[xp_include_skipitems:]

MoinMoin/items/__init__.py

 from MoinMoin.util.interwiki import url_for_item
 from MoinMoin.storage.error import NoSuchItemError, NoSuchRevisionError, AccessDeniedError, \
                                    StorageError
-from MoinMoin.config import UUID, NAME, NAME_OLD, MTIME, REVERTED_TO, ACL, \
+from MoinMoin.config import UUID, NAME, NAME_OLD, NAME_EXACT, WIKINAME, MTIME, REVERTED_TO, ACL, \
                             IS_SYSITEM, SYSITEM_VERSION,  USERGROUP, SOMEDICT, \
                             CONTENTTYPE, SIZE, LANGUAGE, ITEMLINKS, ITEMTRANSCLUSIONS, \
                             TAGS, ACTION, ADDRESS, HOSTNAME, USERID, EXTRA, COMMENT, \
         """ create an index of sub items of this item """
         if self.name:
             prefix = self.name + u'/'
-            query = And([Term("wikiname", app.cfg.interwikiname), Prefix("name_exact", prefix)])
+            query = And([Term(WIKINAME, app.cfg.interwikiname), Prefix(NAME_EXACT, prefix)])
         else:
             # trick: an item of empty name can be considered as "virtual root item",
             # that has all wiki items as sub items
             prefix = u''
-            query = Term("wikiname", app.cfg.interwikiname)
-        results = flaskg.storage.search(query, all_revs=False, sortedby="name_exact", limit=None)
+            query = Term(WIKINAME, app.cfg.interwikiname)
+        results = flaskg.storage.search(query, all_revs=False, sortedby=NAME_EXACT, limit=None)
         # We only want the sub-item part of the item names, not the whole item objects.
         prefix_len = len(prefix)
         items = [(result[NAME], result[NAME][prefix_len:], result[CONTENTTYPE])
 
     def get_templates(self, contenttype=None):
         """ create a list of templates (for some specific contenttype) """
-        terms = [Term("wikiname", app.cfg.interwikiname), Term(TAGS, u'template')]
+        terms = [Term(WIKINAME, app.cfg.interwikiname), Term(TAGS, u'template')]
         if contenttype is not None:
             terms.append(Term(CONTENTTYPE, contenttype))
         query = And(terms)
-        results = flaskg.storage.search(query, all_revs=False, sortedby="name_exact", limit=None)
+        results = flaskg.storage.search(query, all_revs=False, sortedby=NAME_EXACT, limit=None)
         return [result[NAME] for result in results]
 
     def do_modify(self, contenttype, template_name):

MoinMoin/script/maint/index.py

 from whoosh.index import EmptyIndexError
 
 from MoinMoin.search.indexing import WhooshIndex
-from MoinMoin.config import MTIME, NAME, CONTENTTYPE
+from MoinMoin.config import MTIME, NAME, CONTENTTYPE, REV_NO, CONTENT
 from MoinMoin.error import FatalError
 from MoinMoin.storage.error import NoSuchItemError, NoSuchRevisionError
 from MoinMoin.util.mime import Type
                         if not found:
                             latest_rev_writer.add_document(**converted_rev)
                         # Checking that last revision is the latest
-                        elif found["rev_no"] < converted_rev["rev_no"]:
+                        elif found[REV_NO] < converted_rev[REV_NO]:
                             doc_number = latest_rev_searcher.document_number(name_exact=item.name, wikiname=interwikiname)
                             latest_rev_writer.delete_document(doc_number)
                             latest_rev_writer.add_document(**converted_rev)
                     print "*** Revisions in", indexname
                     with ix.searcher() as searcher:
                         for rev in searcher.all_stored_fields():
-                            name = rev.pop("name", u"")
-                            content = rev.pop("content", u"")
-                            for field, value in [("name", name), ] + sorted(rev.items()) + [("content", content), ]:
+                            name = rev.pop(NAME, u"")
+                            content = rev.pop(CONTENT, u"")
+                            for field, value in [(NAME, name), ] + sorted(rev.items()) + [(CONTENT, content), ]:
                                 print "%s: %s" % (field, repr(value)[:70])
                             print "\n"
                     ix.close()
             """
 
             revs_found = searcher.documents(name_exact=name, wikiname=interwikiname)
-            return [rev["rev_no"] for rev in revs_found]
+            return [rev[REV_NO] for rev in revs_found]
 
         def do_action(action, indexnames_schemas):
             if action == "build":

MoinMoin/script/maint/reduce_revisions.py

 from flask import current_app as app
 from flaskext.script import Command, Option
 
-from MoinMoin.config import NAME
+from MoinMoin.config import NAME, NAME_EXACT
 
 
 class Reduce_Revisions(Command):
     def run(self, query):
         storage = app.unprotected_storage
         if query:
-            qp = storage.query_parser(["name_exact", ], all_revs=False)
+            qp = storage.query_parser([NAME_EXACT, ], all_revs=False)
             q = qp.parse(query)
         else:
             q = Every()

MoinMoin/script/maint/set_meta.py

 from flask import g as flaskg
 from flaskext.script import Command, Option
 
-from MoinMoin.config import NAME
+from MoinMoin.config import NAME, NAME_EXACT
 from MoinMoin.script import fatal
 from MoinMoin.storage.error import NoSuchRevisionError
 
                   "only a key you want to delete (with -r set).")
 
         if query:
-            qp = storage.query_parser(["name_exact", ], all_revs=False)
+            qp = storage.query_parser([NAME_EXACT, ], all_revs=False)
             q = qp.parse(query)
         else:
             q = Every()

MoinMoin/search/indexing.py

 from whoosh.fields import Schema, TEXT, ID, IDLIST, NUMERIC, DATETIME, KEYWORD, BOOLEAN
 from whoosh.index import open_dir, create_in, EmptyIndexError
 
-from MoinMoin.config import MTIME, NAME
+from MoinMoin.config import MTIME, NAME, NAME_EXACT, REV_NO, WIKINAME, CONTENT
 from MoinMoin.search.analyzers import *
 from MoinMoin.error import FatalError
 
                 for key, value in backend_rev.items()
                 if key in schema])
     doc[MTIME] = datetime.datetime.utcfromtimestamp(backend_rev[MTIME])
-    doc["name_exact"] = backend_rev[NAME]
-    doc["rev_no"] = rev_no
-    doc["wikiname"] = wikiname
-    doc["content"] = content
+    doc[NAME_EXACT] = backend_rev[NAME]
+    doc[REV_NO] = rev_no
+    doc[WIKINAME] = wikiname
+    doc[CONTENT] = content
     return doc
 
 

MoinMoin/storage/backends/indexing.py

 from MoinMoin.storage.error import NoSuchItemError, NoSuchRevisionError, \
                                    AccessDeniedError
 from MoinMoin.config import ACL, CONTENTTYPE, UUID, NAME, NAME_OLD, MTIME, TAGS, \
-                            ADDRESS, HOSTNAME, USERID, ITEMLINKS, ITEMTRANSCLUSIONS
+                            ADDRESS, HOSTNAME, USERID, ITEMLINKS, ITEMTRANSCLUSIONS, \
+                            REV_NO
 from MoinMoin.search.indexing import backend_to_index
 from MoinMoin.converter import default_registry
 from MoinMoin.util.iri import Iri
             schema = self.index_object.all_revisions_index.schema
             with AsyncWriter(self.index_object.all_revisions_index) as async_writer:
                 converted_rev = backend_to_index(rev, revno, schema, rev_content, self.wikiname)
-                logging.debug("All revisions: adding %s %s", converted_rev[NAME], converted_rev["rev_no"])
+                logging.debug("All revisions: adding %s %s", converted_rev[NAME], converted_rev[REV_NO])
                 async_writer.add_document(**converted_rev)
-        if not latest_found_document or int(revno) > latest_found_document["rev_no"]:
+        if not latest_found_document or int(revno) > latest_found_document[REV_NO]:
             schema = self.index_object.latest_revisions_index.schema
             with AsyncWriter(self.index_object.latest_revisions_index) as async_writer:
                 converted_rev = backend_to_index(rev, revno, schema, rev_content, self.wikiname)
-                logging.debug("Latest revisions: updating %s %s", converted_rev[NAME], converted_rev["rev_no"])
+                logging.debug("Latest revisions: updating %s %s", converted_rev[NAME], converted_rev[REV_NO])
                 async_writer.update_document(**converted_rev)
 
     def remove_rev(self, uuid, revno):