1. pkumar
  2. moin-2.0

Commits

pkumar  committed 2eaf94a Merge

merged pytest2 from main repo

  • Participants
  • Parent commits c7e87a7, 328d83e
  • Branches pytest2

Comments (0)

Files changed (27)

File MoinMoin/_tests/test_test_environ.py

View file
         storage = flaskg.storage
         assert storage
         assert hasattr(storage, 'get_item')
-        assert hasattr(storage, 'history')
         assert not list(storage.iteritems())
-        assert not list(storage.history())
         itemname = u"this item shouldn't exist yet"
         assert pytest.raises(NoSuchItemError, storage.get_item, itemname)
         item = storage.create_item(itemname)

File MoinMoin/apps/feed/views.py

View file
 
 from werkzeug.contrib.atom import AtomFeed
 
+from whoosh.query import Term, And
+
 from MoinMoin import log
 logging = log.getLogger(__name__)
 
 from MoinMoin import wikiutil
 from MoinMoin.i18n import _, L_, N_
 from MoinMoin.apps.feed import feed
-from MoinMoin.config import NAME, ACL, ACTION, ADDRESS, HOSTNAME, USERID, COMMENT
+from MoinMoin.config import NAME, ACL, ACTION, ADDRESS, HOSTNAME, USERID, COMMENT, MTIME
 from MoinMoin.themes import get_editor_info
 from MoinMoin.items import Item
 from MoinMoin.util.crypto import cache_key
     if content is None:
         title = app.cfg.sitename
         feed = AtomFeed(title=title, feed_url=request.url, url=request.host_url)
-        for rev in flaskg.storage.history(item_name=item_name):
-            this_rev = rev
-            this_revno = rev.revno
-            item = rev.item
-            name = rev[NAME]
+        query = Term("wikiname", app.cfg.interwikiname)
+        if item_name:
+            query = And([query, Term("name_exact", item_name), ])
+        history = flaskg.storage.search(query, all_revs=True, sortedby=[MTIME, "rev_no"], reverse=True, limit=100)
+        for doc in history:
+            name = doc[NAME]
+            this_revno = doc["rev_no"]
+            item = flaskg.storage.get_item(name)
+            this_rev = item.get_revision(this_revno)
             try:
                 hl_item = Item.create(name, rev_no=this_revno)
                 previous_revno = this_revno - 1
                 content = _(u'MoinMoin feels unhappy.')
                 content_type = 'text'
             feed.add(title=name, title_type='text',
-                     summary=rev.get(COMMENT, ''), summary_type='text',
+                     summary=doc.get(COMMENT, ''), summary_type='text',
                      content=content, content_type=content_type,
-                     author=get_editor_info(rev, external=True),
+                     author=get_editor_info(doc, external=True),
                      url=url_for_item(name, rev=this_revno, _external=True),
-                     updated=datetime.utcfromtimestamp(rev.timestamp),
+                     updated=doc[MTIME],
                     )
         content = feed.to_string()
         app.cache.set(cid, content)

File MoinMoin/apps/frontend/_tests/test_frontend.py

View file
             assert '<html>' in rv.data
             assert '</html>' in rv.data
 
+    def test_wanteds(self):
+        with self.app.test_client() as c:
+            rv = c.get('/+wanteds')
+            assert rv.status == '200 OK'
+            assert rv.headers['Content-Type'] == 'text/html; charset=utf-8'
+            assert '<html>' in rv.data
+            assert '</html>' in rv.data
+
+    def test_orphans(self):
+        with self.app.test_client() as c:
+            rv = c.get('/+orphans')
+            assert rv.status == '200 OK'
+            assert rv.headers['Content-Type'] == 'text/html; charset=utf-8'
+            assert '<html>' in rv.data
+            assert '</html>' in rv.data
+
 class TestUsersettings(object):
     def setup_method(self, method):
         # Save original user
         if not self.user.exists():
             self.user = None
             pytest.skip("Can't create test user")
-
-
-class TestViews(object):
-    """
-    Tester class for +backrefs, +orphans and +wanted views
-    """
-    class DummyItem(object):
-        """
-        Fake storage object, simulating the page item object from the storage
-        """
-        def __init__(self, name, revision):
-            self.latest_revision = revision
-            self.name = name
-
-        def get_revision(self, *args, **kw):
-            return self.latest_revision
-
-    class DummyRevision(object):
-        """
-        Fake revision object, used for retrieving ITEMTRANSCLUSIONS and ITEMLINKS meta
-        """
-        def __init__(self, links, transclusions):
-            self.links = links
-            self.transclusions = transclusions
-
-        def get(self, meta_name, *args, **kw):
-            if meta_name == 'itemlinks':
-                return self.links
-            if meta_name == 'itemtransclusions':
-                return self.transclusions
-
-    def setup_class(self):
-        # list of tuples
-        # (page_name, links, transclusions)
-        items = [('page1', ['page2', 'page3'], ['page2']),
-                 ('page2',  ['page1', 'page3'], []),
-                 ('page3', ['page5'], ['page1']),
-                 ('page4', [], ['page5'])
-                ]
-        # we create the list of items
-        self.items = []
-        for item in items:
-            revision = self.DummyRevision(item[1], item[2])
-            page = self.DummyItem(item[0], revision)
-            self.items.append(page)
-
-    def test_orphans(self):
-        expected_orphans = sorted(['page4'])
-        result_orphans = sorted(views._orphans(self.items))
-
-        assert result_orphans == expected_orphans
-
-    def test_wanteds(self):
-        expected_wanteds = {'page5': ['page3', 'page4']}
-        result_wanteds = views._wanteds(self.items)
-
-        assert result_wanteds == expected_wanteds
-
-    def test_backrefs(self):
-        expected_backrefs = sorted(['page1', 'page2'])
-        result_backrefs = sorted(views._backrefs(self.items, 'page3'))
-
-        assert result_backrefs == expected_backrefs
-

File MoinMoin/apps/frontend/views.py

View file
 import pytz
 from babel import Locale
 
+from whoosh.query import Term, And, DateRange
+
 from MoinMoin import log
 logging = log.getLogger(__name__)
 
 from MoinMoin.items import Item, NonExistent
 from MoinMoin.items import ROWS_META, COLS, ROWS_DATA
 from MoinMoin import config, user, util, wikiutil
-from MoinMoin.config import ACTION, COMMENT, CONTENTTYPE, ITEMLINKS, ITEMTRANSCLUSIONS, NAME, CONTENTTYPE_GROUPS
+from MoinMoin.config import ACTION, COMMENT, CONTENTTYPE, ITEMLINKS, ITEMTRANSCLUSIONS, NAME, CONTENTTYPE_GROUPS, MTIME, TAGS
 from MoinMoin.util import crypto
 from MoinMoin.util.interwiki import url_for_item
 from MoinMoin.security.textcha import TextCha, TextChaizedForm, TextChaValid
 
 
 def _search(search_form, item_name):
-    from MoinMoin.search.indexing import WhooshIndex
-    from whoosh.qparser import QueryParser, MultifieldParser
-    from MoinMoin.search.analyzers import item_name_analyzer
-    from whoosh import highlight
     query = search_form['q'].value
-    pagenum = 1 # We start from first page
-    pagelen = search_form['pagelen'].value
-    index_object = WhooshIndex()
-    ix = index_object.all_revisions_index if request.values.get('search_in_all') else index_object.latest_revisions_index
-    with ix.searcher() as searcher:
-        mparser = MultifieldParser(["name_exact", "name", "content"], schema=ix.schema)
-        q = mparser.parse(query)
-        results = searcher.search_page(q, int(pagenum), pagelen=int(pagelen))
+    pagenum = 1  # We start from first page
+    pagelen = int(search_form['pagelen'].value)
+    all_revs = bool(request.values.get('search_in_all'))
+    qp = flaskg.storage.query_parser(["name_exact", "name", "content"], all_revs=all_revs)
+    q = qp.parse(query)
+    with flaskg.storage.searcher(all_revs) as searcher:
+        results = searcher.search_page(q, pagenum, pagelen)
         return render_template('search_results.html',
                                results=results,
                                query=query,
                               )
 
 
-
 @frontend.route('/<itemname:item_name>', defaults=dict(rev=-1), methods=['GET', 'POST'])
 @frontend.route('/+show/<int:rev>/<itemname:item_name>', methods=['GET', 'POST'])
 def show_item(item_name, rev):
 
 @frontend.route('/+history/<itemname:item_name>')
 def history(item_name):
-    history = flaskg.storage.history(item_name=item_name)
-
     offset = request.values.get('offset', 0)
     offset = max(int(offset), 0)
-
-    results_per_page = int(app.cfg.results_per_page)
     if flaskg.user.valid:
         results_per_page = flaskg.user.results_per_page
+    else:
+        results_per_page = app.cfg.results_per_page
+    query = And([Term("wikiname", app.cfg.interwikiname), Term("name_exact", item_name), ])
+    # TODO: due to how getPageContent and the template works, we need to use limit=None -
+    # it would be better to use search_page (and an appropriate limit, if needed)
+    docs = flaskg.storage.search(query, all_revs=True, sortedby="rev_no", reverse=True, limit=None)
+    # get rid of the content value to save potentially big amounts of memory:
+    history = [dict((k, v) for k, v in doc.iteritems() if k != 'content') for doc in docs]
     history_page = util.getPageContent(history, offset, results_per_page)
-
     return render_template('history.html',
                            item_name=item_name, # XXX no item here
                            history_page=history_page,
                           )
 
+
 @frontend.route('/+history')
 def global_history():
-    history = flaskg.storage.history(item_name='')
-    results_per_page = int(app.cfg.results_per_page)
+    bookmark_time = None
     if flaskg.user.valid:
-        bookmark_time = flaskg.user.getBookmark()
-        results_per_page = flaskg.user.results_per_page # if it is 0, means no paging
+        bm = flaskg.user.getBookmark()
+        if bm is not None:
+            bookmark_time = datetime.utcfromtimestamp(bm)
+    if flaskg.user.valid:
+        results_per_page = flaskg.user.results_per_page
     else:
-        bookmark_time = None
+        results_per_page = app.cfg.results_per_page
+    query = Term("wikiname", app.cfg.interwikiname)
+    if bookmark_time is not None:
+        query = And([query, DateRange(MTIME, start=bookmark_time, end=None)])
+    # TODO: we need use limit=None to simulate previous implementation's behaviour -
+    # it would be better to use search_page (and an appropriate limit, if needed)
+    history = flaskg.storage.search(query, all_revs=True, sortedby=[MTIME, "rev_no"], reverse=True, limit=None)
     item_groups = OrderedDict()
-    for rev in history:
-        current_item_name = rev.item.name
-        if bookmark_time and rev.timestamp <= bookmark_time:
+    for doc in history:
+        current_item_name = doc[NAME]
+        if bookmark_time and doc[MTIME] <= bookmark_time:
             break
         elif current_item_name in item_groups:
-            latest_rev = item_groups[current_item_name][0]
-            tm_latest = datetime.utcfromtimestamp(latest_rev.timestamp)
-            tm_current = datetime.utcfromtimestamp(rev.timestamp)
+            latest_doc = item_groups[current_item_name][0]
+            tm_latest = latest_doc[MTIME]
+            tm_current = doc[MTIME]
             if format_date(tm_latest) == format_date(tm_current): # this change took place on the same day
-                item_groups[current_item_name].append(rev)
+                item_groups[current_item_name].append(doc)
         else:
-            item_groups[current_item_name] = [rev]
+            item_groups[current_item_name] = [doc]
 
     # Got the item dict, now doing grouping inside them
     editor_info = namedtuple('editor_info', ['editor', 'editor_revnos'])
-    for item_name, revs in item_groups.items():
+    for item_name, docs in item_groups.items():
         item_info = {}
         editors_info = OrderedDict()
         editors = []
         revnos = []
         comments = []
-        current_rev = revs[0]
+        current_doc = docs[0]
         item_info["item_name"] = item_name
-        item_info["timestamp"] = current_rev.timestamp
-        item_info["contenttype"] = current_rev.get(CONTENTTYPE)
-        item_info["action"] = current_rev.get(ACTION)
-        item_info["name"] = current_rev.get(NAME)
+        item_info["name"] = current_doc[NAME]
+        item_info["timestamp"] = current_doc[MTIME]
+        item_info["contenttype"] = current_doc[CONTENTTYPE]
+        item_info["action"] = current_doc[ACTION]
 
         # Aggregating comments, authors and revno
-        for rev in revs:
-            revnos.append(rev.revno)
-            comment = rev.get(COMMENT)
+        for doc in docs:
+            rev_no = doc["rev_no"]
+            revnos.append(rev_no)
+            comment = doc.get(COMMENT)
             if comment:
                 comment = "#%(revno)d %(comment)s" % {
-                          'revno': rev.revno,
+                          'revno': rev_no,
                           'comment': comment
                           }
                 comments.append(comment)
-            editor = get_editor_info(rev)
+            editor = get_editor_info(doc)
             editor_name = editor["name"]
             if editor_name in editors_info:
-                editors_info[editor_name].editor_revnos.append(rev.revno)
+                editors_info[editor_name].editor_revnos.append(rev_no)
             else:
-                editors_info[editor_name] = editor_info(editor, [rev.revno])
+                editors_info[editor_name] = editor_info(editor, [rev_no])
 
         if len(revnos) == 1:
             # there is only one change for this item in the history considered
     rev_tuple = namedtuple('rev_tuple', ['rev_date', 'item_revs'])
     rev_tuples = rev_tuple(prev_date, [])
     for item_group in item_groups.values():
-        tm = datetime.utcfromtimestamp(item_group["timestamp"])
+        tm = item_group["timestamp"]
         rev_date = format_date(tm)
         if revcount < offset:
             revcount += len(item_group["revnos"])
                            previous_offset=previous_offset,
                           )
 
+def _compute_item_sets():
+    """
+    compute sets of existing, linked, transcluded and no-revision item names
+    """
+    linked = set()
+    transcluded = set()
+    existing = set()
+    docs = flaskg.storage.documents(all_revs=False, wikiname=app.cfg.interwikiname)
+    for doc in docs:
+        existing.add(doc[NAME])
+        linked.update(doc.get(ITEMLINKS, []))
+        transcluded.update(doc.get(ITEMTRANSCLUSIONS, []))
+    return existing, linked, transcluded
+
+
 @frontend.route('/+wanteds')
 def wanted_items():
-    """ Returns a page with the list of non-existing items, which are wanted items and the
-        items they are linked or transcluded to helps show what items still need
-        to be written and shows whether there are any broken links. """
-    wanteds = _wanteds(flaskg.storage.iteritems())
+    """
+    Returns a list view of non-existing items that are linked to or
+    transcluded by other items. If you want to know by which items they are
+    referred to, use the backrefs functionality of the item in question.
+    """
+    existing, linked, transcluded = _compute_item_sets()
+    referred = linked | transcluded
+    wanteds = referred - existing
     item_name = request.values.get('item_name', '') # actions menu puts it into qs
-    return render_template('wanteds.html',
+    return render_template('item_link_list.html',
                            headline=_(u'Wanted Items'),
                            item_name=item_name,
-                           wanteds=wanteds)
-
-
-def _wanteds(items):
-    """
-    Returns a dict with all the names of non-existing items which are refed by
-    other items and the items which are refed by
-
-    :param items: all the items
-    :type items: iteratable sequence
-    :returns: a dict with all the wanted items and the items which are beign refed by
-    """
-    all_items = set()
-    wanteds = {}
-    for item in items:
-        current_item = item.name
-        all_items.add(current_item)
-        try:
-            current_rev = item.get_revision(-1)
-        except NoSuchRevisionError:
-            continue
-        # converting to sets so we can get the union
-        outgoing_links = current_rev.get(ITEMLINKS, [])
-        outgoing_transclusions = current_rev.get(ITEMTRANSCLUSIONS, [])
-        outgoing_refs = set(outgoing_transclusions + outgoing_links)
-        for refed_item in outgoing_refs:
-            if refed_item not in all_items:
-                if refed_item not in wanteds:
-                    wanteds[refed_item] = []
-                wanteds[refed_item].append(current_item)
-        if current_item in wanteds:
-            # if a previously wanted item has been found in the items storage, remove it
-            del wanteds[current_item]
-
-    return wanteds
+                           item_names=wanteds)
 
 
 @frontend.route('/+orphans')
 def orphaned_items():
-    """ Return a page with the list of items not being linked or transcluded
-        by any other items, that makes
-        them sometimes not discoverable. """
-    orphan = _orphans(flaskg.storage.iteritems())
+    """
+    Return a list view of existing items not being linked or transcluded
+    by any other item (which makes them sometimes not discoverable).
+    """
+    existing, linked, transcluded = _compute_item_sets()
+    referred = linked | transcluded
+    orphans = existing - referred
     item_name = request.values.get('item_name', '') # actions menu puts it into qs
     return render_template('item_link_list.html',
                            item_name=item_name,
                            headline=_(u'Orphaned Items'),
-                           item_names=orphan)
-
-
-def _orphans(items):
-    """
-    Returns a list with the names of all existing items not being refed by any other item
-
-    :param items: the list of all items
-    :type items: iteratable sequence
-    :returns: the list of all orphaned items
-    """
-    linked_items = set()
-    transcluded_items = set()
-    all_items = set()
-    norev_items = set()
-    for item in items:
-        all_items.add(item.name)
-        try:
-            current_rev = item.get_revision(-1)
-        except NoSuchRevisionError:
-            norev_items.add(item.name)
-        else:
-            linked_items.update(current_rev.get(ITEMLINKS, []))
-            transcluded_items.update(current_rev.get(ITEMTRANSCLUSIONS, []))
-    orphans = all_items - linked_items - transcluded_items - norev_items
-    logging.info("_orphans: Ignored %d item(s) that have no revisions" % len(norev_items))
-    return list(orphans)
+                           item_names=orphans)
 
 
 @frontend.route('/+quicklink/<itemname:item_name>')
     """
     show a list or tag cloud of all tags in this wiki
     """
-    counts_tags_names = flaskg.storage.all_tags()
     item_name = request.values.get('item_name', '') # actions menu puts it into qs
-    if counts_tags_names:
-        # sort by tag name
-        counts_tags_names = sorted(counts_tags_names, key=lambda e: e[1])
+    docs = flaskg.storage.documents(all_revs=False, wikiname=app.cfg.interwikiname)
+    tags_counts = {}
+    for doc in docs:
+        tags = doc.get(TAGS, [])
+        logging.debug("name %s rev %s tags %s" % (doc[NAME], doc["rev_no"], tags))
+        for tag in tags:
+            tags_counts[tag] = tags_counts.setdefault(tag, 0) + 1
+    tags_counts = sorted(tags_counts.items())
+    if tags_counts:
         # this is a simple linear scaling
-        counts = [e[0] for e in counts_tags_names]
+        counts = [count for tags, count in tags_counts]
         count_min = min(counts)
         count_max = max(counts)
         weight_max = 9.99
             scale = weight_max / 2
         else:
             scale = weight_max / (count_max - count_min)
-        def cls(count, tag):
+        def cls(count):
             # return the css class for this tag
             weight = scale * (count - count_min)
             return "weight%d" % int(weight)  # weight0, ..., weight9
-        tags = [(cls(count, tag), tag) for count, tag, names in counts_tags_names]
+        tags = [(cls(count), tag) for tag, count in tags_counts]
     else:
         tags = []
     return render_template("global_tags.html",
     """
     show all items' names that have tag <tag>
     """
-    item_names = flaskg.storage.tagged_items(tag)
+    query = And([Term("wikiname", app.cfg.interwikiname), Term(TAGS, tag), ])
+    docs = flaskg.storage.search(query, all_revs=False, sortedby="name_exact", limit=None)
+    item_names = [doc[NAME] for doc in docs]
     return render_template("item_link_list.html",
                            headline=_("Items tagged with %(tag)s", tag=tag),
                            item_name=tag,
                            item_names=item_names)
 
-

File MoinMoin/converter/__init__.py

View file
 
 from ..util.mime import Type, type_moin_document
 
-from MoinMoin.config import CONTENTTYPE
+from MoinMoin.config import NAME, CONTENTTYPE
 
 from MoinMoin import log
 logging = log.getLogger(__name__)
         # no way
         raise TypeError("No converter for %s --> %s" % (input_contenttype, output_contenttype))
     except Exception as e: # catch all exceptions, we don't want to break an indexing run
-        logging.exception("Exception happened in conversion:")
+        logging.exception("Exception happened in conversion of item %r rev %d contenttype %s:" % (rev[NAME], rev.revno, rev[CONTENTTYPE]))
         doc = u'ERROR [%s]' % str(e)
         return doc
 

File MoinMoin/converter/include.py

View file
 # Copyright: 2008 MoinMoin:BastianBlank
-# Copyright: 2010 MoinMoin:ThomasWaldmann
+# Copyright: 2010-2011 MoinMoin:ThomasWaldmann
 # License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
 
 """
 from MoinMoin import log
 logging = log.getLogger(__name__)
 
+from flask import current_app as app
 from flask import g as flaskg
 
+from whoosh.query import Term, And, Wildcard
+
+from MoinMoin.config import NAME
 from MoinMoin import wikiutil
 from MoinMoin.items import Item
 from MoinMoin.util.mime import type_moin_document
 
                     if xp_include:
                         for entry in xp_include:
-                            name, data = entry.name, entry.data
+                            name, data = entry.name, entry.data_unescape
                             if name == 'pages':
                                 xp_include_pages = data
                             elif name == 'sort':
                     pages = ((page, link), )
 
                 elif xp_include_pages:
-                    # We have a regex of pages to include
-                    from MoinMoin.storage.terms import NameFn
-                    inc_match = re.compile(xp_include_pages)
-                    root_item = Item(name=u'')
-                    pagelist = sorted([item.name for item in root_item.list_items(NameFn(inc_match))])
-                    if xp_include_sort == 'descending':
-                        pagelist.reverse()
+                    # XXX we currently interpret xp_include_pages as wildcard, but it should be regex
+                    # for compatibility with moin 1.9. whoosh has upcoming regex support, but it is not
+                    # released yet.
+                    if xp_include_pages.startswith('^'):
+                        # get rid of the leading ^ the Include macro needed to get into "regex mode"
+                        xp_include_pages = xp_include_pages[1:]
+                    query = And([Term("wikiname", app.cfg.interwikiname), Wildcard("name_exact", xp_include_pages)])
+                    reverse = xp_include_sort == 'descending'
+                    results = flaskg.storage.search(query, all_revs=False, sortedby="name_exact", reverse=reverse, limit=None)
+                    pagelist = [result[NAME] for result in results]
                     if xp_include_skipitems is not None:
                         pagelist = pagelist[xp_include_skipitems:]
                     if xp_include_items is not None:

File MoinMoin/items/__init__.py

View file
 
 from flatland import Form, String, Integer, Boolean, Enum
 from flatland.validation import Validator, Present, IsEmail, ValueBetween, URLValidator, Converted
+
+from whoosh.query import Term, And, Prefix
+
 from MoinMoin.util.forms import FileStorage
 
 from MoinMoin.security.textcha import TextCha, TextChaizedForm, TextChaValid
         if flaskg.user.valid:
             newrev[USERID] = unicode(flaskg.user.id)
 
-    def search_items(self, term=None):
-        """ search items matching the term or,
-            if term is None, return all items
-        """
-        if term:
-            backend_items = flaskg.storage.search_items(term)
-        else:
-            # special case: we just want all items
-            backend_items = flaskg.storage.iteritems()
-        for item in backend_items:
-            yield Item.create(item=item)
-
-    list_items = search_items  # just for cosmetics
-
-    def count_items(self, term=None):
-        """
-        Return item count for matching items. See search_items() for details.
-        """
-        count = 0
-        # we intentionally use a loop to avoid creating a list with all item objects:
-        for item in self.list_items(term):
-            count += 1
-        return count
-
     def get_index(self):
         """ create an index of sub items of this item """
-        import re
-        from MoinMoin.storage.terms import NameRE
-
         if self.name:
             prefix = self.name + u'/'
+            query = And([Term("wikiname", app.cfg.interwikiname), Prefix("name_exact", prefix)])
         else:
             # trick: an item of empty name can be considered as "virtual root item",
             # that has all wiki items as sub items
             prefix = u''
-        sub_item_re = u"^%s.*" % re.escape(prefix)
-        regex = re.compile(sub_item_re, re.UNICODE)
-
-        item_iterator = self.search_items(NameRE(regex))
-
+            query = Term("wikiname", app.cfg.interwikiname)
+        results = flaskg.storage.search(query, all_revs=False, sortedby="name_exact", limit=None)
         # We only want the sub-item part of the item names, not the whole item objects.
         prefix_len = len(prefix)
-        items = [(item.name, item.name[prefix_len:], item.meta.get(CONTENTTYPE))
-                 for item in item_iterator]
-        return sorted(items)
+        items = [(result[NAME], result[NAME][prefix_len:], result[CONTENTTYPE])
+                 for result in results]
+        return items
 
     def flat_index(self, startswith=None, selected_groups=None):
         """
 
     def get_templates(self, contenttype=None):
         """ create a list of templates (for some specific contenttype) """
-        from MoinMoin.storage.terms import AND, LastRevisionMetaDataMatch
-        term = LastRevisionMetaDataMatch(TAGS, ['template']) # XXX there might be other tags
-        if contenttype:
-            term = AND(term, LastRevisionMetaDataMatch(CONTENTTYPE, contenttype))
-        item_iterator = self.search_items(term)
-        items = [item.name for item in item_iterator]
-        return sorted(items)
+        terms = [Term("wikiname", app.cfg.interwikiname), Term(TAGS, u'template')]
+        if contenttype is not None:
+            terms.append(Term(CONTENTTYPE, contenttype))
+        query = And(terms)
+        results = flaskg.storage.search(query, all_revs=False, sortedby="name_exact", limit=None)
+        return [result[NAME] for result in results]
 
     def do_modify(self, contenttype, template_name):
         # XXX think about and add item template support

File MoinMoin/items/_tests/test_Item.py

View file
         with pytest.raises(KeyError):
             item.meta['test_key']
 
-    def test_count_items(self):
-        name = u'Test_Item'
-        contenttype = u'text/plain;charset=utf-8'
-        meta = {CONTENTTYPE: contenttype}
-        item = Item.create(name)
-        item._save(meta)
-        item = Item.create(name)
-        result1 = item.count_items()
-        assert result1 == 1
-        # add another item
-        new_name = u'New_Item'
-        meta = {CONTENTTYPE: contenttype}
-        item = Item.create(new_name)
-        item._save(meta)
-        item = Item.create(new_name)
-        result2 = item.count_items()
-        assert result2 == 2
 
 class TestBinary(object):
     """ Test for arbitrary binary items """

File MoinMoin/script/maint/reduce_revisions.py

View file
 # Copyright: 2009 MoinMoin:ChristopherDenter
 # Copyright: 2011 MoinMoin:ReimarBauer
+# Copyright: 2011 MoinMoin:ThomasWaldmann
 # License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
 
 """
-    MoinMoin - Reduce Revisions of a backend
+MoinMoin - Reduce Revisions of a backend
 
-    This script removes all revisions but the last one from all selected items.
+This script removes all revisions but the last one from all selected items.
 """
 
 
-import re
 from flask import current_app as app
 from flaskext.script import Command, Option
 
-from MoinMoin.storage.terms import NameRE
+from MoinMoin.config import NAME
 
 
 class Reduce_Revisions(Command):
     description = "This command can be used to remove all revisions but the last one from all selected items."
     option_list = (
-        Option('--pattern', '-p', required=False, dest='pattern', type=unicode, default=".*",
-               help="You can limit the operation on certain items whose names match the given pattern."),
+        Option('--query', '-q', dest="query", type=unicode, default='',
+               help='Only perform the operation on items found by the given query.'),
     )
 
-    def run(self, pattern):
+    def run(self, query):
         storage = app.unprotected_storage
-        query = NameRE(re.compile(pattern))
-        # If no pattern is given, the default regex will match every item.
-        for item in storage.search_items(query):
+        if query:
+            qp = storage.query_parser(["name_exact", ], all_revs=False)
+            q = qp.parse(query)
+        else:
+            q = Every()
+        results = storage.search(q, all_revs=False, limit=None)
+        for result in results:
+            item_name = result[NAME]
+            item = storage.get_item(item_name)
             current_revno = item.next_revno - 1
             for revno in item.list_revisions():
                 if revno < current_revno:
                     rev = item.get_revision(revno)
+                    print "Destroying %r revision %d." % (item_name, revno)
                     rev.destroy()
 
         print "Finished reducing backend."

File MoinMoin/script/maint/set_meta.py

View file
 # Copyright: 2009 MoinMoin:ChristopherDenter
 # Copyright: 2011 MoinMoin:ReimarBauer
+# Copyright: 2011 MoinMoin:ThomasWaldmann
 # License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
 
 """
-    MoinMoin - Set Metadata of a revision
+MoinMoin - Set Metadata of a revision
 
-    This script duplicates the last revision of the selected item
-    and sets or removes metadata.
+This script duplicates the last revision of the selected item
+and sets or removes metadata.
 """
 
 
-import re
 from ast import literal_eval
 from shutil import copyfileobj
 
 from flask import g as flaskg
 from flaskext.script import Command, Option
 
+from MoinMoin.config import NAME
 from MoinMoin.script import fatal
-from MoinMoin.storage.terms import NameRE
 from MoinMoin.storage.error import NoSuchRevisionError
 
+
 class Set_Meta(Command):
     description = "This command can be used to set meta data of a new revision."
     option_list = (
         Option('--key', '-k', required=False, dest='key', type=unicode,
                help="The key you want to set/change in the new revision"),
-        Option('--value', '-v', dest="text", type=unicode,
+        Option('--value', '-v', dest="value", type=unicode,
                help='The value to set for the given key.'),
         Option('--remove', '-r', dest="remove", action='store_true', default=False,
                help='If you want to delete the key given, add this flag.'),
-        Option('--pattern', '-p', dest="pattern", type=unicode, default='.*',
-               help='Only perform the operation on items whose names match the pattern.')
+        Option('--query', '-q', dest="query", type=unicode, default='',
+               help='Only perform the operation on items found by the given query.')
     )
 
-    def run(self, key, text, remove, pattern):
+    def run(self, key, value, remove, query):
         storage = app.unprotected_storage
 
-        if not ((key and text) or (key and remove)) or (key and text and remove):
+        if not ((key and value) or (key and remove)) or (key and value and remove):
             fatal("You need to either specify a proper key/value pair or "
                   "only a key you want to delete (with -r set).")
 
-        query = NameRE(re.compile(pattern))
-        for item in storage.search_items(query):
+        if query:
+            qp = storage.query_parser(["name_exact", ], all_revs=False)
+            q = qp.parse(query)
+        else:
+            q = Every()
+        results = storage.search(q, all_revs=False, limit=None)
+        for result in results:
+            item_name = result[NAME]
+            item = storage.get_item(item_name)
             try:
                 last_rev = item.get_revision(-1)
             except NoSuchRevisionError:
 
             if not remove:
                 # Set or overwrite given metadata key with text
-                value = literal_eval(text)
-                next_rev[key] = value
+                next_rev[key] = literal_eval(value)
+                print "Processing %r, setting %s=%r." % (item_name, key, value)
+            else:
+                print "Processing %r, removing %s." % (item_name, key)
+
             item.commit()
 

File MoinMoin/search/_tests/test_analyzers.py

View file
 
     test_cases_query = [
                   # (query, tokens)
-                  (u'text/plain', [u'text', u'plain']),
-                  (u'text/plain;charset=utf-8', [u'text', u'plain', u'charset=utf-8']),
+                  (u'text/plain',
+                   [u'text/plain', u'text', u'plain']),
+                  (u'text/plain;charset=utf-8',
+                   [u'text/plain;charset=utf-8', u'text', u'plain', u'charset=utf-8']),
                   (u'text/html;value1=foo;value2=bar',
-                   [u'text', u'html', u'value1=foo', u'value2=bar'],
+                   [u'text/html;value1=foo;value2=bar', u'text', u'html', u'value1=foo', u'value2=bar'],
                   ),
-                  (u'text/html;value1=foo;value1=bar', [u'text', u'html', u'value1=bar'])
+                  # we normalize, sort the params:
+                  (u'text/html;value2=bar;value1=foo',
+                   [u'text/html;value1=foo;value2=bar', u'text', u'html', u'value1=foo', u'value2=bar'],
+                  ),
+                  # later values for same key overwrite earlier ones:
+                  (u'text/html;value1=foo;value1=bar',
+                   [u'text/html;value1=bar', u'text', u'html', u'value1=bar'])
                  ]
 
     def make_tokenizer(self):

File MoinMoin/search/analyzers.py

View file
         pos = start_pos
         tk = Token()
         tp = Type(value)
+        # we need to yield the complete contenttype in one piece,
+        # so we can find it with Term(CONTENTTYPE, contenttype):
+        if tp.type is not None and tp.subtype is not None:
+            # note: we do not use "value" directly, so Type.__unicode__ can normalize it:
+            tk.text = unicode(tp)
+            if positions:
+                tk.pos = pos
+                pos += 1
+            yield tk
+        # now yield the pieces:
         tk.text = tp.type
         if positions:
             tk.pos = pos

File MoinMoin/search/indexing.py

View file
     doc = dict([(str(key), value)
                 for key, value in backend_rev.items()
                 if key in schema])
-    doc[MTIME] = datetime.datetime.fromtimestamp(backend_rev[MTIME])
+    doc[MTIME] = datetime.datetime.utcfromtimestamp(backend_rev[MTIME])
     doc["name_exact"] = backend_rev[NAME]
     doc["rev_no"] = rev_no
     doc["wikiname"] = wikiname
             userid=ID(stored=True),
             address=ID(stored=True),
             hostname=ID(stored=True),
+            size=NUMERIC(stored=True),
+            action=ID(stored=True),
+            comment=TEXT(stored=True, multitoken_query="and"),
             content=TEXT(stored=True, multitoken_query="and"),
         )
 

File MoinMoin/storage/__init__.py

View file
     This class abstracts access to backends. If you want to write a specific
     backend, say a mercurial backend, you have to implement the methods below.
     A backend knows of its items and can perform several item related operations
-    such as search_items, get_item, create_item, etc.
+    such as get_item, create_item, etc.
     """
     #
     # If you need to write a backend it is sufficient
         """
         pass
 
-    def search_items(self, searchterm):
-        """
-        Takes a MoinMoin search term and returns an iterator (maybe empty) over
-        matching item objects (NOT item names!).
-
-        :type searchterm: MoinMoin search term
-        :param searchterm: The term for which to search.
-        :rtype: iterator of item objects
-        """
-        # Very simple implementation because we have no indexing
-        # or anything like that. If you want to optimize this, override it.
-        for item in self.iteritems():
-            searchterm.prepare()
-            if searchterm.evaluate(item):
-                yield item
-
     def get_item(self, itemname):
         """
         Returns item object or raises Exception if that item does not exist.
         a wiki item, as such a deletion does not really delete anything from disk but
         just hides the former existence of the item. Such a deletion is undoable, while
         having destroyed an item is not.
-        This also destroys all history related to the item. In particular, this also
-        deletes all the item's revisions and they won't turn up in history any longer.
 
         In case the item has already been destroyed by someone else (e.g. another process)
         this method should just pass silently as the job is already done.
     that defaults to None for newly created revisions in which case it will be
     assigned at commit() time. It is writable for use by converter backends, but
     care must be taken in that case to create monotone timestamps!
-    This timestamp is also retrieved via the backend's history() method.
     """
     def __init__(self, item, revno):
         """

File MoinMoin/storage/_tests/test_backends.py

View file
 from MoinMoin.storage import Item, NewRevision
 from MoinMoin.storage.backends import memory
 from MoinMoin.storage.error import NoSuchItemError, ItemAlreadyExistsError, NoSuchRevisionError, RevisionAlreadyExistsError
-from MoinMoin.storage import terms
 from MoinMoin.config import SIZE
 
 item_names = (u"quite_normal",
     def test_has_item_that_doesnt_exist(self):
         assert not self.backend.has_item(u"i_do_not_exist")
 
-    def test_search_simple(self):
-        for name in [u"songlist", u"song lyric", u"odd_SONG_item"]:
-            self.create_rev_item_helper(name)
-        self.create_meta_item_helper(u"new_song_player")
-        query_string = u"song"
-        query = terms.Name(query_string, True)
-        for num, item in enumerate(self.backend.search_items(query)):
-            assert item.name.find(query_string) != -1
-        assert num == 2
-
-    def test_search_better(self):
-        self.create_rev_item_helper(u'abcde')
-        self.create_rev_item_helper(u'abcdef')
-        self.create_rev_item_helper(u'abcdefg')
-        self.create_rev_item_helper(u'abcdefgh')
-
-        def _test_search(term, expected):
-            found = list(self.backend.search_items(term))
-            assert len(found) == expected
-
-        # must be /part/ of the name
-        yield _test_search, terms.Name(u'AbCdEf', False), 3
-        yield _test_search, terms.Name(u'AbCdEf', True), 0
-        yield _test_search, terms.Name(u'abcdef', True), 3
-        yield _test_search, terms.NameRE(re.compile(u'abcde.*')), 4
-        yield _test_search, terms.NameFn(lambda n: n == u'abcdef'), 1
-
     def test_iteritems_1(self):
         for num in range(10, 20):
             self.create_rev_item_helper(u"item_" + str(num).zfill(2))

File MoinMoin/storage/_tests/test_backends_router.py

View file
 """
 
 import os
+import time
 
 import pytest
 
 from flask import current_app as app
 
+from whoosh.query import Term, And, Every
+
+from MoinMoin.config import NAME, MTIME
 from MoinMoin.error import ConfigurationError
 from MoinMoin.storage._tests.test_backends import BackendTest
 from MoinMoin.storage.backends.memory import MemoryBackend
         assert name == ''
         assert mountpoint == 'child'
 
+    def test_search_item_history_order(self):
+        item_name = u'some item'
+        item = self.backend.create_item(item_name)
+        for rev_no in range(3):
+            rev = item.create_revision(rev_no)
+            item.commit()
+        query = Term("name_exact", item_name)
+        results = list(self.backend.search(query, all_revs=True, sortedby="rev_no"))
+        print results
+        assert results[0].get("rev_no") == 0
+        assert results[1].get("rev_no") == 1
+        assert results[2].get("rev_no") == 2
+        results = list(self.backend.search(query, all_revs=True, sortedby="rev_no", reverse=True))
+        print results
+        assert results[0].get("rev_no") == 2
+        assert results[1].get("rev_no") == 1
+        assert results[2].get("rev_no") == 0
 
-    def test_history(self):
-        order = [(u'first', 0, ), (u'second', 0, ), (u'first', 1, ), (u'a', 0), (u'child/my_subitem', 0) ]
-        for name, revno in order:
-            if revno == 0:
-                item = self.backend.create_item(name)
-            else:
-                item = self.backend.get_item(name)
-            item.create_revision(revno)
+    def test_search_global_history_order(self):
+        names = [u'foo', u'bar', u'baz', ]
+        for item_name in names:
+            item = self.backend.create_item(item_name)
+            rev = item.create_revision(0)
             item.commit()
+            time.sleep(1) # make sure we have different MTIME
+        query = Every()
+        results = list(self.backend.search(query, all_revs=True, sortedby=[MTIME, "rev_no"]))
+        print results
+        assert results[0].get(NAME) == names[0]
+        assert results[1].get(NAME) == names[1]
+        assert results[2].get(NAME) == names[2]
+        results = list(self.backend.search(query, all_revs=True, sortedby=[MTIME, "rev_no"], reverse=True))
+        print results
+        assert results[0].get(NAME) == names[2]
+        assert results[1].get(NAME) == names[1]
+        assert results[2].get(NAME) == names[0]
 
-            # Revisions are created too fast for the rev's timestamp's granularity.
-            # This only affects the RouterBackend because there several different
-            # backends are used and no means for storing simultaneously created revs
-            # in the correct order exists between backends. It affects AclWrapperBackend
-            # tests as well because those use a RouterBackend internally for real-world-likeness.
 
-            # XXX XXX
-            # You may have realized that all the items above belong to the same backend so this shouldn't actually matter.
-            # It does matter, however, once you consider that the RouterBackend uses the generic, slow history implementation.
-            # This one uses iteritems and then sorts all the revisions itself, hence discarding any information of ordering
-            # for simultaneously created revisions. If we just call history of that single backend directly, it works without
-            # time.sleep. For n backends, however, you'd have to somehow merge the revisions into one generator again, thus
-            # discarding that information again. Besides, that would be a costly operation. The ordering for simultaneosly
-            # created revisions remains the same since it's based on tuple ordering. Better proposals welcome.
-            import time
-            time.sleep(1)
-
-        for num, rev in enumerate(self.backend.history(reverse=False)):
-            name, revno = order[num]
-            assert rev.item.name == name
-            assert rev.revno == revno
-
-        order.reverse()
-        for num, rev in enumerate(self.backend.history(reverse=True)):
-            name, revno = order[num]
-            assert rev.item.name == name
-            assert rev.revno == revno
-
-    # See history function in indexing.py for comments on why this test fails.
-    @pytest.mark.xfail
-    def test_history_size_after_rename(self):
-        item = self.backend.create_item(u'first')
-        item.create_revision(0)
-        item.commit()
-        item.rename(u'second')
-        item.create_revision(1)
-        item.commit()
-        assert len([rev for rev in self.backend.history()]) == 2
-
-    def test_history_after_destroy_item(self):
-        itemname = u"I will be completely destroyed"
-        rev_data = "I will be completely destroyed, too, hopefully"
-        item = self.backend.create_item(itemname)
-        rev = item.create_revision(0)
-        rev.write(rev_data)
-        item.commit()
-
-        item.destroy()
-
-        all_rev_data = [rev.read() for rev in self.backend.history()]
-        assert not rev_data in all_rev_data
-
-        for rev in self.backend.history():
-            assert not rev.item.name == itemname
-        for rev in self.backend.history(reverse=False):
-            assert not rev.item.name == itemname
-
-    def test_history_after_destroy_revision(self):
-        itemname = u"I will see my children die"    # removed the smiley ':-(' temporarily as it slows the test in addition with a failure
-        rev_data = "I will die!"
-        persistent_rev = "I will see my sibling die :-("
-        item = self.backend.create_item(itemname)
-        rev = item.create_revision(0)
-        rev.write(rev_data)
-        item.commit()
-        rev = item.create_revision(1)
-        rev.write(persistent_rev)
-        item.commit()
-
-        rev = item.get_revision(0)
-        rev.destroy()
-
-        for rev in self.backend.history():
-            assert not (rev.item.name == itemname and rev.revno == 0)
-
-    def test_history_item_names(self):
-        item = self.backend.create_item(u'first')
-        item.create_revision(0)
-        item.commit()
-        item.rename(u'second')
-        item.create_revision(1)
-        item.commit()
-        revs_in_create_order = [rev for rev in self.backend.history(reverse=False)]
-        assert revs_in_create_order[0].revno == 0
-        assert revs_in_create_order[0].item.name == u'second'
-        assert revs_in_create_order[1].revno == 1
-        assert revs_in_create_order[1].item.name == u'second'
-

File MoinMoin/storage/_tests/test_terms.py

-# Copyright: 2008 MoinMoin:JohannesBerg
-# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
-
-"""
-    MoinMoin - Term tests.
-"""
-
-
-import re
-
-from MoinMoin.config import NAME, CONTENTTYPE
-from MoinMoin.storage import terms as term
-from MoinMoin.storage.backends.memory import MemoryBackend
-
-
-_item_contents = {
-    u'a': u'abcdefg hijklmnop',
-    u'b': u'bbbbbbb bbbbbbbbb',
-    u'c': u'Abiturienten Apfeltortor',
-    u'Lorem': u'Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Duis placerat, tortor quis sollicitudin dictum, nisi tellus aliquam quam, ac varius lacus diam eget tortor. Nulla vehicula, nisi ac hendrerit aliquam, libero erat tempor ante, lobortis placerat lacus justo vitae erat. In rutrum odio a sem. In ac risus vel diam vulputate luctus. Fusce sit amet est. Morbi consectetuer eros vel risus. In nulla lacus, ultrices id, vestibulum tempus, dictum in, mauris. Quisque rutrum faucibus nisl. Suspendisse potenti. In hac habitasse platea dictumst. Donec ac magna ac eros malesuada facilisis. Pellentesque viverra nibh nec dui. Praesent venenatis lectus vehicula eros. Phasellus pretium, ante at mollis luctus, nibh lacus ultricies eros, vitae pharetra lacus leo at neque. Nullam vel sapien. In in diam id massa nonummy suscipit. Curabitur vel dui sed tellus pellentesque pretium.',
-}
-
-_item_metadata = {
-    u'a': {'m1': 'True', 'm2': '222'},
-    u'A': {'m1': 'True', 'm2': '333'},
-    u'b': {'m1': 'False', 'm2': '222'},
-    u'c': {'m1': 'True', 'm2': '222'},
-    u'B': {'m1': 'False', 'm2': '333'},
-    u'Lorem': {'m1': '7', 'm2': '444'},
-}
-
-_lastrevision_metadata = {
-    u'a': {'a': '1'},
-    u'A': {'a': ''},
-    u'b': {'a': '0'},
-    u'c': {'a': 'False'},
-    u'B': {'a': ''},
-    u'Lorem': {'a': '42'},
-}
-
-for n in _item_contents.keys():
-    nl = n.lower()
-    nu = n.upper()
-    _item_contents[nl] = _item_contents[n].lower()
-    _item_contents[nu] = _item_contents[n].upper()
-    if not nl in _item_metadata:
-        _item_metadata[nl] = _item_metadata[n]
-    if not nu in _item_metadata:
-        _item_metadata[nu] = _item_metadata[n]
-    if not nl in _lastrevision_metadata:
-        _lastrevision_metadata[nl] = _lastrevision_metadata[n]
-    if not nu in _lastrevision_metadata:
-        _lastrevision_metadata[nu] = _lastrevision_metadata[n]
-
-memb = MemoryBackend()
-for iname, md in _item_metadata.iteritems():
-    item = memb.create_item(iname)
-    item.change_metadata()
-    item.update(md)
-    item.publish_metadata()
-
-    rev = item.create_revision(0)
-    md = _lastrevision_metadata[iname]
-    rev.update({NAME: iname, CONTENTTYPE: u"application/octet-stream"})
-    rev.update(md)
-    rev.write(_item_contents[iname])
-    item.commit()
-
-item = memb.create_item('NR')
-item.change_metadata()
-item.update({'m1': 'True'})
-item.publish_metadata()
-del item
-
-class TermTestData:
-    def __init__(self, text):
-        self.text = text
-    def read(self, size=None):
-        return self.text
-
-class CacheAssertTerm(term.Term):
-    def __init__(self):
-        term.Term.__init__(self)
-        self.evalonce = False
-
-    def _evaluate(self, item):
-        assert not self.evalonce
-        self.evalonce = True
-        return True
-
-class AssertNotCalledTerm(term.Term):
-    def _evaluate(self, item):
-        assert False
-
-class TestTerms:
-    def _evaluate(self, term, itemname, expected):
-        if itemname is not None:
-            item = memb.get_item(itemname)
-        else:
-            item = None
-        term.prepare()
-        assert expected == term.evaluate(item)
-
-    def testSimpleTextSearch(self):
-        terms = [term.Text(u'abcdefg', True), term.Text(u'ijklmn', True)]
-        for item, expected in [('a', True), ('A', False), ('b', False), ('B', False), ('lorem', False), ('NR', False)]:
-            for t in terms:
-                yield self._evaluate, t, item, expected
-
-    def testSimpleTextSearchCI(self):
-        terms = [term.Text(u'abcdefg', False), term.Text(u'ijklmn', False)]
-        for item, expected in [('a', True), ('A', True), ('b', False), ('B', False), ('lorem', False)]:
-            for t in terms:
-                yield self._evaluate, t, item, expected
-
-    def testANDOR(self):
-        tests = [
-            (True,  [1, 1, 1, 1, 1]),
-            (True,  [1, 1, 1, 1]),
-            (True,  [1, 1, 1]),
-            (True,  [1, 1]),
-            (False, [0, 1, 1]),
-            (False, [0, 1, 1, 1]),
-            (False, [1, 0, 1, 1]),
-            (False, [1, 1, 0, 1]),
-            (False, [1, 1, 1, 0]),
-            (False, [0, 1, 1, 0]),
-        ]
-        for expected, l in tests:
-            l = [term.BOOL(i) for i in l]
-            t = term.AND(*l)
-            yield self._evaluate, t, 'a', expected
-        for expected, l in tests:
-            l = [term.BOOL(1 - i) for i in l]
-            t = term.OR(*l)
-            yield self._evaluate, t, 'a', not expected
-
-    def testXOR(self):
-        tests = [
-            (False, [1, 1, 1, 1, 1]),
-            (False, [1, 1, 1, 1]),
-            (False, [1, 1, 1]),
-            (False, [1, 1]),
-            (False, [0, 1, 1]),
-            (False, [0, 1, 1, 1]),
-            (False, [1, 0, 1, 1]),
-            (False, [1, 1, 0, 1]),
-            (False, [1, 1, 1, 0]),
-            (False, [0, 1, 1, 0]),
-            (True,  [0, 0, 0, 1, 0]),
-            (True,  [0, 0, 1, 0]),
-            (True,  [1, 0, 0]),
-            (True,  [0, 1]),
-            (False, [0, 0, 0]),
-        ]
-        for expected, l in tests:
-            l = [term.BOOL(i) for i in l]
-            t = term.XOR(*l)
-            yield self._evaluate, t, 'a', expected
-
-    def testTextSearchRE(self):
-        terms = [term.TextRE(re.compile('^abc')), term.TextRE(re.compile('\shij'))]
-        for item, expected in [('a', True), ('A', False), ('b', False), ('B', False), ('lorem', False), ('NR', False)]:
-            for t in terms:
-                yield self._evaluate, t, item, expected
-
-    def testTextSearchRE2(self):
-        terms = [term.TextRE(re.compile('sollici')), term.TextRE(re.compile('susci'))]
-        for item, expected in [('a', False), ('A', False), ('b', False), ('B', False), ('lorem', True), ('NR', False)]:
-            for t in terms:
-                yield self._evaluate, t, item, expected
-
-    def testResultCaching1(self):
-        cat = CacheAssertTerm()
-        expected = True
-        t = term.AND(cat, cat, cat)
-        yield self._evaluate, t, None, expected
-
-    def testResultCaching2(self):
-        cat = CacheAssertTerm()
-        expected = True
-        t = term.OR(cat, cat, cat)
-        yield self._evaluate, t, None, expected
-
-    def testResultCaching3(self):
-        cat = CacheAssertTerm()
-        expected = False
-        t = term.AND(cat, cat, cat, term.FALSE)
-        yield self._evaluate, t, None, expected
-
-    def testResultCaching4(self):
-        cat = CacheAssertTerm()
-        expected = True
-        t = term.OR(cat, cat, cat)
-        yield self._evaluate, t, None, expected
-
-    def testShortCircuitEval1(self):
-        yield self._evaluate, term.AND(term.TRUE, term.FALSE, AssertNotCalledTerm()), None, False
-
-    def testShortCircuitEval2(self):
-        yield self._evaluate, term.OR(term.TRUE, term.FALSE, AssertNotCalledTerm()), None, True
-
-    def testSimpleTitleSearch(self):
-        for item, expected in [('a', True), ('A', False), ('b', False), ('B', False), ('lorem', False), ('NR', False)]:
-            yield self._evaluate, term.Name(u'a', True), item, expected
-
-    def testSimpleTitleSearchCI(self):
-        for item, expected in [('a', True), ('A', True), ('b', False), ('B', False), ('lorem', False), ('NR', False)]:
-            yield self._evaluate, term.Name(u'a', False), item, expected
-
-    def testTitleRESearch(self):
-        for item, expected in [('a', True), ('A', False), ('b', False), ('B', False), ('lorem', True), ('NR', False)]:
-            yield self._evaluate, term.NameRE(re.compile('(a|e)')), item, expected
-
-    def testMetaMatch1(self):
-        t = term.ItemMetaDataMatch('m1', 'True')
-        for item, expected in [('a', True), ('A', True), ('b', False), ('B', False), ('lorem', False), ('NR', True)]:
-            yield self._evaluate, t, item, expected
-
-    def testMetaMatch2(self):
-        t = term.ItemMetaDataMatch('m2', '333')
-        for item, expected in [('a', False), ('A', True), ('b', False), ('B', True), ('lorem', False), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testMetaMatch3(self):
-        t = term.ItemMetaDataMatch('m2', '444')
-        for item, expected in [('a', False), ('A', False), ('b', False), ('B', False), ('lorem', True), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testHasMeta1(self):
-        t = term.ItemHasMetaDataKey('m3')
-        for item, expected in [('a', False), ('A', False), ('b', False), ('B', False), ('lorem', False), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testHasMeta2(self):
-        t = term.ItemHasMetaDataKey('m1')
-        for item, expected in [('a', True), ('A', True), ('b', True), ('B', True), ('lorem', True), ('NR', True)]:
-            yield self._evaluate, t, item, expected
-
-    def testHasMeta3(self):
-        t = term.LastRevisionHasMetaDataKey('a')
-        for item, expected in [('a', True), ('A', True), ('b', True), ('B', True), ('lorem', True), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testHasMeta4(self):
-        t = term.LastRevisionMetaDataMatch('a', '')
-        for item, expected in [('a', False), ('A', True), ('b', False), ('B', True), ('lorem', False), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testNameFn(self):
-        t = term.NameFn(lambda x: x in ['a', 'b', 'lorem'])
-        for item, expected in [('a', True), ('A', False), ('b', True), ('B', False), ('lorem', True), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testWordCI(self):
-        t = term.Word('Curabitur', False)
-        for item, expected in [('B', False), ('Lorem', True), ('lorem', True), ('LOREM', True), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testWord(self):
-        t = term.Word('Curabitur', True)
-        for item, expected in [('B', False), ('Lorem', True), ('lorem', False), ('LOREM', False), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testWordStartCI(self):
-        t = term.WordStart('Curabi', False)
-        for item, expected in [('B', False), ('Lorem', True), ('lorem', True), ('LOREM', True), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testWordStart(self):
-        t = term.WordStart('Curabi', True)
-        for item, expected in [('c', False), ('Lorem', True), ('lorem', False), ('LOREM', False), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testWordStart2(self):
-        t = term.WordStart('abitur', True)
-        for item, expected in [('c', True), ('C', False), ('Lorem', False), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testWordStart2CI(self):
-        t = term.WordStart('abitur', False)
-        for item, expected in [('c', True), ('C', True), ('Lorem', False), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-    def testWordEndCI(self):
-        t = term.WordEnd('abitur', False)
-        for item, expected in [('c', False), ('Lorem', True), ('lorem', True), ('LOREM', True), ('NR', False)]:
-            yield self._evaluate, t, item, expected
-
-coverage_modules = ['MoinMoin.storage.terms']

File MoinMoin/storage/backends/acl.py

View file
         # up on the real backend.
         return getattr(self.backend, attr)
 
-    def search_items(self, searchterm):
-        """
-        @see: Backend.search_items.__doc__
-        """
-        for item in self.backend.search_items(searchterm):
-            if self._may(item.name, READ):
-                # The item returned needs to be wrapped because otherwise the
-                # item's methods (like create_revision) wouldn't be wrapped.
-                wrapped_item = AclWrapperItem(item, self)
-                yield wrapped_item
-
     def get_item(self, itemname):
         """
         @see: Backend.get_item.__doc__

File MoinMoin/storage/backends/indexing.py

View file
         item.publish_metadata()
         return item
 
-    def history(self, reverse=True, item_name=u'', start=None, end=None):
-        """
-        History implementation using the index.
-        """
-        for result in self._index.history(reverse=reverse, item_name=item_name, start=start, end=end):
-            # we currently create the item, the revision and yield it to stay
-            # compatible with storage api definition, but this could be changed to
-            # just return the data we get from the index (without accessing backend)
-            # TODO: A problem exists at item = self.get_item(name).
-            # In the history_size_after_rename test in test_backends.py,
-            # an item was created with the name "first" and then renamed to "second."
-            # When it runs through this history function and runs item = self.get_item("first"),
-            # it can't find it because it was already renamed to "second."
-            # Some suggested solutions are: using some neverchanging uuid to identify some specific item
-            # or continuing to use the name, but tracking name changes within the item's history.
-            rev_datetime, name, rev_no = result
-            try:
-                logging.debug("HISTORY: name %s revno %s" % (name, rev_no))
-                item = self.get_item(name)
-                yield item.get_revision(rev_no)
-            except AccessDeniedError as e:
-                # just skip items we may not access
-                pass
-            except (NoSuchItemError, NoSuchRevisionError) as e:
-                logging.exception("history processing catched exception")
+    def query_parser(self, default_fields, all_revs=False):
+        return self._index.query_parser(default_fields, all_revs=all_revs)
 
-    def all_tags(self):
-        """
-        Return a unsorted list of tuples (count, tag, tagged_itemnames) for all tags.
-        """
-        return self._index.all_tags()
+    def searcher(self, all_revs=False):
+        return self._index.searcher(all_revs=all_revs)
 
-    def tagged_items(self, tag):
-        """
-        Return a list of item names of items that are tagged with <tag>.
-        """
-        return self._index.tagged_items(tag)
+    def search(self, q, all_revs=False, **kw):
+        return self._index.search(q, all_revs=all_revs, **kw)
+
+    def search_page(self, q, all_revs=False, pagenum=1, pagelen=10, **kw):
+        return self._index.search_page(q, all_revs=all_revs, pagenum=pagenum, pagelen=pagelen, **kw)
+
+    def documents(self, all_revs=False, **kw):
+        return self._index.documents(all_revs=all_revs, **kw)
 
 
 class IndexingItemMixin(object):
     # TODO by intercepting write() to index data written to a revision
 
 from whoosh.writing import AsyncWriter
+from whoosh.qparser import QueryParser, MultifieldParser
+
 from MoinMoin.search.indexing import WhooshIndex
 
 class ItemIndex(object):
                 logging.debug("Latest revisions: removing %d", latest_doc_number)
                 async_writer.delete_document(latest_doc_number)
 
-    def history(self, mountpoint=u'', item_name=u'', reverse=True, start=None, end=None):
-        if mountpoint:
-            mountpoint += '/'
-        with self.index_object.all_revisions_index.searcher() as all_revs_searcher:
-            if item_name:
-                docs = all_revs_searcher.documents(name_exact=item_name,
-                                                   wikiname=self.wikiname
-                                                  )
-            else:
-                docs = all_revs_searcher.documents(wikiname=self.wikiname)
-            from operator import itemgetter
-            # sort by mtime and rev_no do deal better with mtime granularity for fast item rev updates
-            for doc in sorted(docs, key=itemgetter("mtime", "rev_no"), reverse=reverse)[start:end]:
-                yield (doc[MTIME], mountpoint + doc[NAME], doc["rev_no"])
+    def query_parser(self, default_fields, all_revs=False):
+        if all_revs:
+            schema = self.index_object.all_revisions_schema
+        else:
+            schema = self.index_object.latest_revisions_schema
+        if len(default_fields) > 1:
+            qp = MultifieldParser(default_fields, schema=schema)
+        elif len(default_fields) == 1:
+            qp = QueryParser(default_fields[0], schema=schema)
+        else:
+            raise ValueError("default_fields list must at least contain one field name")
+        return qp
 
-    def all_tags(self):
-        with self.index_object.latest_revisions_index.searcher() as latest_revs_searcher:
-            docs = latest_revs_searcher.documents(wikiname=self.wikiname)
-            tags_names = {}
-            for doc in docs:
-                tags = doc.get(TAGS, [])
-                logging.debug("name %s rev %s tags %s" % (doc[NAME], doc["rev_no"], tags))
-                for tag in tags:
-                    tags_names.setdefault(tag, []).append(doc[NAME])
-            counts_tags_names = [(len(names), tag, names) for tag, names in tags_names.items()]
-            return counts_tags_names
+    def searcher(self, all_revs=False):
+        """
+        Get a searcher for the right index. Always use this with "with":
 
-    def tagged_items(self, tag):
-        with self.index_object.latest_revisions_index.searcher() as latest_revs_searcher:
-            docs = latest_revs_searcher.documents(tags=tag, wikiname=self.wikiname)
-            return [doc[NAME] for doc in docs]
+        with storage.searcher(all_revs) as searcher:
+            # your code
 
+        If you do not need the searcher itself or the Result object, but rather
+        the found documents, better use search() or search_page(), see below.
+        """
+        if all_revs:
+            ix = self.index_object.all_revisions_index
+        else:
+            ix = self.index_object.latest_revisions_index
+        return ix.searcher()
+
+    def search(self, q, all_revs=False, **kw):
+        with self.searcher(all_revs) as searcher:
+            # Note: callers must consume everything we yield, so the for loop
+            # ends and the "with" is left to close the index files.
+            for hit in searcher.search(q, **kw):
+                yield hit.fields()
+
+    def search_page(self, q, all_revs=False, pagenum=1, pagelen=10, **kw):
+        with self.searcher(all_revs) as searcher:
+            # Note: callers must consume everything we yield, so the for loop
+            # ends and the "with" is left to close the index files.
+            for hit in searcher.search_page(q, pagenum, pagelen=pagelen, **kw):
+                yield hit.fields()
+
+    def documents(self, all_revs=False, **kw):
+        if all_revs:
+            ix = self.index_object.all_revisions_index
+        else:
+            ix = self.index_object.latest_revisions_index
+        with ix.searcher() as searcher:
+            # Note: callers must consume everything we yield, so the for loop
+            # ends and the "with" is left to close the index files.
+            for doc in searcher.documents(**kw):
+                yield doc
+

File MoinMoin/storage/backends/sqla.py

View file
 
         * Data.read must be changed to operate on dynamically loaded chunks. I.e., the data._chunks must
           be set to lazy='dynamic', which will then be a query instead of a collection.
-        * Find a proper solution for methods that issue many SQL queries. Especially search_items is
-          difficult, as we cannot know what data will be needed in the subsequent processing of the items
-          returned, which will result in more queries being issued. Eager loading is only a partial solution.
+        * Find a proper solution for methods that issue many SQL queries.
         * MetaData should definitely NOT simply be stored as a dict in a PickleType Column. Store that properly,
           perhaps in (a) seperate table(s).
         * Find out why RC lists an item that was just written below Trash/ as well. (Likely a UI bug.)

File MoinMoin/storage/serialization.py

View file
         return item is not None and item.name in self.item_names
 
 
-class TermMatch(XMLSelectiveGenerator):
-    def __init__(self, out, term):
-        self.term = term  # see MoinMoin.storage.terms
-        XMLSelectiveGenerator.__init__(self, out)
-
-    def shall_serialize(self, item=None, rev=None,
-                        revno=None, current_revno=None):
-        if item is not None:
-            self.term.prepare()
-            return self.term.evaluate(item)
-        return False
-
-
 def serialize(obj, xmlfile, xmlgen_cls=XMLSelectiveGenerator, *args, **kwargs):
     """
     Serialize <obj> to <xmlfile>.
 
     The default value of <xmlgen_cls> will just serialize everything. Alternatively,
     use some of XMLSelectiveGenerator child classes to do selective serialization,
-    e.g. of just a list of items or just of items that match some search term.
+    e.g. of just a list of items or just a subset of the revisions.
 
     :arg obj: object to serialize (must mix in Serializable)
     :arg xmlfile: output file (file-like or filename)

File MoinMoin/storage/terms.py

-# Copyright: 2008 MoinMoin:JohannesBerg
-# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
-
-"""
-    MoinMoin - search expression object representation
-
-    This module defines the possible search terms for a query to the
-    storage backend. This is used, for example, to implement searching,
-    page lists etc.
-
-    Note that some backends can optimise some of the search terms, for
-    example a backend that has indexed various metadata keys can optimise
-    easy expressions containing ItemMetaDataMatch terms. This is only allowed
-    for classes documented as being 'final' which hence also means that
-    their _evaluate function may not be overridden by descendent classes.
-
-    For example, that metadata backend could test if the expression is an
-    ItemMetaDataMatch expression, and if so, simply return the appropriate
-    index; or if it is an AND() expression build the page list from the
-    index, remove the ItemMetaDataMatch instance from the AND list and match
-    the resulting expression only for pages in that list. Etc.
-
-    TODO: Should we write some generic code for picking apart expressions
-          like that?
-"""
-
-
-import re
-
-from MoinMoin.storage.error import NoSuchRevisionError
-
-# Base classes
-
-class Term(object):
-    """
-    Base class for search terms.
-    """
-    # relative cost of this search term
-    _cost = 0
-
-    def __init__(self):
-        pass
-
-    def evaluate(self, item):
-        """
-        Evaluate this term and return True or False if the
-        item identified by the parameters matches.
-
-        :param item: the item
-        """
-        assert hasattr(self, '_result')
-
-        if self._result is None:
-            self._result = self._evaluate(item)
-
-        return self._result
-
-    def _evaluate(self, item):
-        """
-        Implements the actual evaluation
-        """
-        raise NotImplementedError()
-
-    def prepare(self):
-        """
-        Prepare this search term to make it ready for testing.
-        Must be called before each outermost-level evaluate.
-        """
-        self._result = None
-
-    def copy(self):
-        """
-        Make a copy of this search term.
-        """
-        return self.__class__()
-
-class UnaryTerm(Term):
-    """
-    Base class for search terms that has a single contained
-    search term, e.g. NOT.
-    """
-    def __init__(self, term):
-        Term.__init__(self)
-        assert isinstance(term, Term)
-        self.term = term
-
-    def prepare(self):
-        Term.prepare(self)
-        self.term.prepare()
-        self._cost = self.term._cost
-
-    def __repr__(self):
-        return u'<%s(%r)>' % (self.__class__.__name__, self.term)
-
-    def copy(self):
-        return self.__class__(self.term.copy())
-
-class ListTerm(Term):
-    """
-    Base class for search terms that contain multiple other
-    search terms, e.g. AND.
-    """
-    def __init__(self, *terms):
-        Term.__init__(self)
-        for e in terms:
-            assert isinstance(e, Term)
-        self.terms = list(terms)
-
-    def prepare(self):
-        Term.prepare(self)
-        # the sum of all costs is a bit of a worst-case cost...
-        self._cost = 0
-        for e in self.terms:
-            e.prepare()
-            self._cost += e._cost
-        self.terms.sort(cmp=lambda x, y: cmp(x._cost, y._cost))
-
-    def remove(self, subterm):
-        self.terms.remove(subterm)
-
-    def add(self, subterm):
-        self.terms.append(subterm)
-
-    def __repr__(self):
-        return u'<%s(%s)>' % (self.__class__.__name__,
-                              ', '.join([repr(t) for t in self.terms]))
-
-    def copy(self):
-        terms = [t.copy() for t in self.terms]
-        return self.__class__(*terms)
-
-# Logical expression classes
-
-class AND(ListTerm):
-    """
-    AND connection between multiple terms. Final.
-    """
-    def _evaluate(self, item):
-        for e in self.terms:
-            if not e.evaluate(item):
-                return False
-        return True
-
-class OR(ListTerm):
-    """
-    OR connection between multiple terms. Final.
-    """
-    def _evaluate(self, item):
-        for e in self.terms:
-            if e.evaluate(item):
-                return True
-        return False
-
-class NOT(UnaryTerm):
-    """
-    Inversion of a single term. Final.
-    """
-    def _evaluate(self, item):
-        return not self.term.evaluate(item)
-
-class XOR(ListTerm):
-    """
-    XOR connection between multiple terms, i.e. exactly
-    one must be True. Final.
-    """
-    def _evaluate(self, item):
-        count = 0
-        for e in self.terms:
-            if e.evaluate(item):
-                count += 1
-        return count == 1
-
-class _BOOL(Term):
-    _cost = 0
-    def __init__(self, val):
-        self._val = val
-
-    def prepare(self):
-        self._result = self._val
-
-    def __repr__(self):
-        return '<%s>' % str(self._val).upper()
-
-    def copy(self):
-        return self
-
-TRUE = _BOOL(True)
-FALSE = _BOOL(False)
-
-def BOOL(b):
-    if b:
-        return TRUE
-    return FALSE
-
-# Actual Moin search terms
-
-class TextRE(Term):
-    """
-    Regular expression full text match, use as last resort.
-    """
-    _cost = 1000 # almost prohibitive
-    def __init__(self, needle_re):
-        Term.__init__(self)
-        assert hasattr(needle_re, 'search')
-        self._needle_re = needle_re
-
-    def _evaluate(self, item):
-        try:
-            rev = item.get_revision(-1)
-        except NoSuchRevisionError:
-            return False
-        data = rev.read()
-        return not (not self._needle_re.search(data))
-
-    def __repr__(self):
-        return u'<term.TextRE(...)>'
-
-    def copy(self):
-        return TextRE(self._needle_re)
-
-class Text(TextRE):
-    """
-    Full text match including middle of words and over word
-    boundaries. Final.
-    """
-    def __init__(self, needle, case_sensitive):
-        flags = re.UNICODE
-        if not case_sensitive:
-            flags = flags | re.IGNORECASE
-        _needle_re = re.compile(re.escape(needle), flags)
-        TextRE.__init__(self, _needle_re)
-        self.needle = needle
-        self.case_sensitive = case_sensitive
-
-    def __repr__(self):
-        return u'<term.Text(%s, %s)>' % (self.needle, self.case_sensitive)
-
-    def copy(self):
-        return Text(self.needle, self.case_sensitive)
-
-class Word(TextRE):
-    """
-    Full text match finding exact words. Final.
-    """
-    def __init__(self, needle, case_sensitive):
-        flags = re.UNICODE
-        if not case_sensitive:
-            flags = flags | re.IGNORECASE
-        _needle_re = re.compile('\\b' + re.escape(needle) + '\\b', flags)
-        TextRE.__init__(self, _needle_re)
-        self.needle = needle
-        self.case_sensitive = case_sensitive
-
-    def __repr__(self):
-        return u'<term.Word(%s, %s)>' % (self.needle, self.case_sensitive)
-
-    def copy(self):
-        return Word(self.needle, self.case_sensitive)
-
-class WordStart(TextRE):
-    """
-    Full text match finding the start of a word. Final.
-    """
-    def __init__(self, needle, case_sensitive):
-        flags = re.UNICODE
-        if not case_sensitive:
-            flags = flags | re.IGNORECASE
-        _needle_re = re.compile('\\b' + re.escape(needle), flags)
-        TextRE.__init__(self, _needle_re)
-        self.needle = needle
-        self.case_sensitive = case_sensitive
-
-    def __repr__(self):
-        return u'<term.WordStart(%s, %s)>' % (self.needle, self.case_sensitive)
-