Commits

Thomas Waldmann committed 5ad4ae0

lots of fixes / adaptions for storage-ng code

Comments (0)

Files changed (25)

MoinMoin/_tests/__init__.py

 
 
 import os, shutil
+from StringIO import StringIO
 
 from flask import current_app as app
 from flask import g as flaskg
 
 
 # Creating and destroying test items --------------------------------
+
 def update_item(name, revno, meta, data):
     """ creates or updates an item  """
     if isinstance(data, unicode):
         data = data.encode(config.charset)
-    try:
-        item = flaskg.storage.create_item(name)
-    except ItemAlreadyExistsError:
-        item = flaskg.storage.get_item(name)
+    item = flaskg.storage[name]
 
-    rev = item.create_revision(revno)
-    for key, value in meta.items():
-        rev[key] = value
-    if not NAME in rev:
-        rev[NAME] = name
-    if not CONTENTTYPE in rev:
-        rev[CONTENTTYPE] = u'application/octet-stream'
-    rev.write(data)
-    item.commit()
-    return item
+    if NAME not in meta:
+        meta[NAME] = name
+    if CONTENTTYPE not in meta:
+        meta[CONTENTTYPE] = u'application/octet-stream'
+    rev = item.store_revision(meta, StringIO(data))
+    return rev
 
 def create_random_string_list(length=14, count=10):
     """ creates a list of random strings """

MoinMoin/_tests/test_test_environ.py

     MoinMoin - Tests for our test environment
 """
 
+from StringIO import StringIO
 
 import pytest
 
 from flask import g as flaskg
 
 from MoinMoin.conftest import init_test_app, deinit_test_app
-from MoinMoin.config import NAME, CONTENTTYPE, IS_SYSITEM, SYSITEM_VERSION
+from MoinMoin.config import NAME, CURRENT, CONTENTTYPE, IS_SYSITEM, SYSITEM_VERSION
 from MoinMoin.storage.error import NoSuchItemError
-from MoinMoin.storage.middleware.serialization import serialize, unserialize
 
 from MoinMoin._tests import wikiconfig
 
 
         storage = flaskg.storage
         assert storage
-        assert hasattr(storage, 'get_item')
-        assert not list(storage.iteritems())
+        assert hasattr(storage, '__getitem__')
         itemname = u"this item shouldn't exist yet"
-        assert pytest.raises(NoSuchItemError, storage.get_item, itemname)
-        item = storage.create_item(itemname)
-        new_rev = item.create_revision(0)
-        new_rev[NAME] = itemname
-        new_rev[CONTENTTYPE] = u'text/plain'
-        item.commit()
+        assert not storage.has_item(itemname)
+        item = storage[itemname]
+        new_rev = item.store_revision({NAME: itemname, CONTENTTYPE: u'text/plain'}, StringIO(''))
         assert storage.has_item(itemname)
-        assert not storage.has_item("FrontPage")
 
-    # Run this test twice to see if something's changed
-    test_twice = test_fresh_backends
 
+CONTENT_ACL = dict(
+        before="+All:write", # need to write to sys pages
+        default="All:read,write,admin,create,destroy",
+        after="Me:create",
+        hierarchic=False,
+)
 
 class TestStorageEnvironWithConfig(object):
+
     class Config(wikiconfig.Config):
-        content_acl = dict(
-            before="+All:write", # need to write to sys pages
-            default="All:read,write,admin,create,destroy",
-            after="Me:create",
-            hierarchic=False,
-        )
+        content_acl = CONTENT_ACL
 
-    def test_fresh_backends_with_content(self):
-        # get the items from xml file
-        backend = app.unprotected_storage
-        unserialize(backend, self.Config._test_items_xml)
+    def test_config(self):
+        assert isinstance(app.cfg, wikiconfig.Config)
+        assert app.cfg.content_acl == CONTENT_ACL
 
-        assert isinstance(app.cfg, wikiconfig.Config)
-
-        storage = flaskg.storage
-        should_be_there = (u"FrontPage", u"HelpOnLinking", u"HelpOnMoinWikiSyntax", )
-        for pagename in should_be_there:
-            assert storage.has_item(pagename)
-            item = storage.get_item(pagename)
-            rev = item.get_revision(-1)
-            assert rev.revno == 0
-            assert rev[IS_SYSITEM]
-            assert rev[SYSITEM_VERSION] == 1
-            # check whether this dirties the backend for the second iteration of the test
-            new_rev = item.create_revision(1)
-            new_rev[NAME] = pagename
-            new_rev[CONTENTTYPE] = u'text/plain'
-            item.commit()
-
-        itemname = u"OnlyForThisTest"
-        assert not storage.has_item(itemname)
-        new_item = storage.create_item(itemname)
-        new_rev = new_item.create_revision(0)
-        new_rev[NAME] = itemname
-        new_rev[CONTENTTYPE] = u'text/plain'
-        new_item.commit()
-        assert storage.has_item(itemname)
-
-        assert storage.get_backend("/").after.acl_lines[0] == "Me:create"
-
-    # Run this test twice to see if something's changed
-    test_twice = test_fresh_backends_with_content
-

MoinMoin/_tests/test_user.py

 from MoinMoin.util import crypto
 
 
+class TestSimple(object):
+    def test_create_retrieve(self):
+        name = u"foo"
+        password = u"barbaz4711"
+        email = u"foo@example.org"
+        # first create a user
+        ret = user.create_user(name, password, email, validate=False)
+        assert ret is None, "create_user returned: %s" % ret
+        # now try to use it
+        u = user.User(name=name, password=password)
+        assert u.name == name
+        assert u.email == email
+        assert u.valid
+
+
 class TestLoginWithPassword(object):
     """user: login tests"""
 
         # Create anon user for the tests
         flaskg.user = user.User()
 
-        self.user = None
-
     def teardown_method(self, method):
         """ Run after each test
 
         Remove user and reset user listing cache.
         """
-        # Remove user file and user
-        if self.user is not None:
-            del self.user
-
         # Restore original user
         flaskg.user = self.saved_user
 
 
     # Helpers ---------------------------------------------------------
 
-    def createUser(self, name, password, pwencoded=False, email=None):
-        """ helper to create test user
-        """
-        # Create user
-        self.user = user.User()
-        self.user.name = name
-        self.user.email = email
-        if not pwencoded:
-            password = crypto.crypt_password(password)
-        self.user.enc_password = password
-
-        # Validate that we are not modifying existing user data file!
-        if self.user.exists():
-            self.user = None
-            pytest.skip("Test user exists, will not override existing user data file!")
-
-        # Save test user
-        self.user.save()
-
-        # Validate user creation
-        if not self.user.exists():
-            self.user = None
-            pytest.skip("Can't create test user")
+    def createUser(self, name, password, pwencoded=False, email=None, validate=False):
+        ret = user.create_user(name, password, email, validate=validate, is_encrypted=pwencoded)
+        assert ret is None, "create_user returned: %s" % ret
 
 
 class TestGroupName(object):

MoinMoin/_tests/test_wikiutil.py

 
 from MoinMoin import config, wikiutil
 from MoinMoin._tests import wikiconfig
-from MoinMoin.storage.middleware.serialization import serialize, unserialize
 
 from werkzeug import MultiDict
 
             assert wikiutil.clean_input(instr) == outstr
 
 
-class TestSystemItem(object):
-    systemItems = (
-        'HelpOnMoinWikiSyntax',
-        'HelpOnLinking',
-        )
-    notSystemItems = (
-        'NoSuchPageYetAndWillNeverBe',
-        )
-
-    def testSystemItem(self):
-        """wikiutil: good system item names accepted, bad rejected"""
-        # get the items from xml file
-        backend = app.unprotected_storage
-        unserialize(backend, wikiconfig.Config._test_items_xml)
-
-        for name in self.systemItems:
-            assert wikiutil.isSystemItem(name)
-        for name in self.notSystemItems:
-            assert not wikiutil.isSystemItem(name)
-
-
 class TestAnchorNames(object):
     def test_anchor_name_encoding(self):
         tests = [

MoinMoin/_tests/wikiconfig.py

     _root = abspath(join(_here, '..', '..'))
     data_dir = join(_here, 'wiki', 'data') # needed for plugins package TODO
     index_dir = join(_here, 'wiki', 'index')
-    index_dir_tmp = join(_here, 'wiki', 'index_tmp')
-    _test_items_xml = join(_here, 'testitems.xml')
     content_acl = None
     item_root = 'FrontPage'
     interwikiname = u'MoinTest'
     clock.stop('create_app flask-cache')
     # init storage
     clock.start('create_app init backends')
-    app.unprotected_storage, app.storage = init_backends(app)
+    init_backends(app)
     clock.stop('create_app init backends')
     clock.start('create_app flask-babel')
     i18n_init(app)
     deinit_backends(app)
 
 
-from MoinMoin.storage.middleware import router, acl
+from MoinMoin.storage.middleware import protecting, indexing, routing
 from MoinMoin import auth, config, user
 
 
     initialize the backends
     """
     # A ns_mapping consists of several lines, where each line is made up like this:
-    # mountpoint, unprotected backend, protection to apply as a dict
-    ns_mapping = app.cfg.namespace_mapping
+    # mountpoint, unprotected backend
     # Just initialize with unprotected backends.
-    unprotected_mapping = [(ns, backend) for ns, backend, acls in ns_mapping]
-    unprotected_storage = router.RouterBackend(unprotected_mapping, cfg=app.cfg)
-    # Protect each backend with the acls provided for it in the mapping at position 2
-    amw = acl.AclWrapperBackend
-    protected_mapping = [(ns, amw(app.cfg, backend, **acls)) for ns, backend, acls in ns_mapping]
-    storage = router.RouterBackend(protected_mapping, cfg=app.cfg)
-    return unprotected_storage, storage
+    app.router = routing.Backend(app.cfg.namespace_mapping)
+    if app.cfg.create_storage:
+        app.router.create()
+    app.router.open()
+    app.storage = indexing.IndexingMiddleware(app.cfg.index_dir, app.router, wiki_name=app.cfg.interwikiname) # XXX give user name etc.
+    if app.cfg.create_storage:
+        app.storage.create()
+    app.storage.open()
 
 def deinit_backends(app):
     app.storage.close()
-    app.unprotected_storage.close()
+    app.router.close()
+    if app.cfg.destroy_storage:
+        app.storage.destroy()
+        app.router.destroy()
 
 
 def setup_user():
     flaskg.clock.start('total')
     flaskg.clock.start('init')
     try:
-        flaskg.unprotected_storage = app.unprotected_storage
+        flaskg.unprotected_storage = app.storage
 
         flaskg.user = setup_user()
+        flaskg.storage = protecting.ProtectingMiddleware(app.storage, flaskg.user, app.cfg.acl_mapping)
 
         flaskg.dicts = app.cfg.dicts()
         flaskg.groups = app.cfg.groups()
         flaskg.content_lang = app.cfg.language_default
         flaskg.current_lang = app.cfg.language_default
 
-        flaskg.storage = app.storage
-
         setup_jinja_env()
     finally:
         flaskg.clock.stop('init')

MoinMoin/apps/admin/views.py

 from MoinMoin.apps.admin import admin
 from MoinMoin import user
 from MoinMoin.storage.error import NoSuchRevisionError
-from MoinMoin.config import NAME, UUID, SIZE, EMAIL
+from MoinMoin.config import NAME, ITEMID, SIZE, EMAIL
 from MoinMoin.config import SUPERUSER
 from MoinMoin.security import require_permission
 
     """
     groups = flaskg.groups
     docs = user.search_users() # all users
-    user_accounts = [dict(uid=doc[UUID],
+    user_accounts = [dict(uid=doc[ITEMID],
                           name=doc[NAME],
                           email=doc[EMAIL],
                           disabled=False,  # TODO: add to index

MoinMoin/apps/feed/views.py

 from MoinMoin import wikiutil
 from MoinMoin.i18n import _, L_, N_
 from MoinMoin.apps.feed import feed
-from MoinMoin.config import NAME, NAME_EXACT, WIKINAME, ACL, ACTION, ADDRESS, HOSTNAME, USERID, COMMENT, MTIME, REV_NO
+from MoinMoin.config import NAME, NAME_EXACT, WIKINAME, ACL, ACTION, ADDRESS, HOSTNAME, USERID, COMMENT, MTIME, REVID
 from MoinMoin.themes import get_editor_info
 from MoinMoin.items import Item
 from MoinMoin.util.crypto import cache_key
         query = Term(WIKINAME, app.cfg.interwikiname)
         if item_name:
             query = And([query, Term(NAME_EXACT, item_name), ])
-        history = flaskg.storage.search(query, all_revs=True, sortedby=[MTIME, REV_NO], reverse=True, limit=100)
-        for doc in history:
-            name = doc[NAME]
-            this_revno = doc[REV_NO]
-            item = flaskg.storage.get_item(name)
-            this_rev = item.get_revision(this_revno)
+        history = flaskg.storage.search(query, all_revs=True, sortedby=[MTIME], reverse=True, limit=100)
+        for rev in history:
+            name = rev.meta[NAME]
+            item = rev.item
+            this_revno = rev.meta[REVID]
+            previous_revno = None # XXX TODO we need PARENT for this
+            this_rev = rev
             try:
                 hl_item = Item.create(name, rev_no=this_revno)
-                previous_revno = this_revno - 1
-                if previous_revno >= 0:
+                if previous_revno is not None:
                     # simple text diff for changes
-                    previous_rev = item.get_revision(previous_revno)
-                    content = hl_item._render_data_diff_text(previous_rev, this_rev)
+                    previous_rev = item[previous_revno]
+                    content = hl_item._render_data_diff_text(previous_rev.data, this_rev.data)
                     content = '<div><pre>%s</pre></div>' % content
                 else:
                     # full html rendering for new items
                 content = _(u'MoinMoin feels unhappy.')
                 content_type = 'text'
             feed.add(title=name, title_type='text',
-                     summary=doc.get(COMMENT, ''), summary_type='text',
+                     summary=rev.meta.get(COMMENT, ''), summary_type='text',
                      content=content, content_type=content_type,
-                     author=get_editor_info(doc, external=True),
+                     author=get_editor_info(rev.meta, external=True),
                      url=url_for_item(name, rev=this_revno, _external=True),
-                     updated=doc[MTIME],
+                     updated=rev.meta[MTIME],
                     )
         content = feed.to_string()
         app.cache.set(cid, content)

MoinMoin/apps/frontend/views.py

 from MoinMoin.items import ROWS_META, COLS, ROWS_DATA
 from MoinMoin import config, user, util, wikiutil
 from MoinMoin.config import ACTION, COMMENT, WIKINAME, CONTENTTYPE, ITEMLINKS, ITEMTRANSCLUSIONS, NAME, NAME_EXACT, \
-                            CONTENTTYPE_GROUPS, MTIME, TAGS, REV_NO, CONTENT
+                            CONTENTTYPE_GROUPS, MTIME, TAGS, ITEMID, REVID, CURRENT, CONTENT
 from MoinMoin.util import crypto
 from MoinMoin.util.interwiki import url_for_item
 from MoinMoin.security.textcha import TextCha, TextChaizedForm, TextChaValid
     return html
 
 
-@frontend.route('/<itemname:item_name>', defaults=dict(rev=-1), methods=['GET'])
-@frontend.route('/+show/<int:rev>/<itemname:item_name>', methods=['GET'])
+@frontend.route('/<itemname:item_name>', defaults=dict(rev=CURRENT), methods=['GET'])
+@frontend.route('/+show/<rev>/<itemname:item_name>', methods=['GET'])
 def show_item(item_name, rev):
     flaskg.user.addTrail(item_name)
     item_displayed.send(app._get_current_object(),
         item = Item.create(item_name, rev_no=rev)
     except AccessDeniedError:
         abort(403)
-    show_revision = show_navigation = rev >= 0
+    show_revision = show_navigation = rev != CURRENT
     # Note: rev.revno of DummyRev is None
     first_rev = None
     last_rev = None
                               data_rendered=Markup(item._render_data()),
                               show_revision=show_revision,
                               show_navigation=show_navigation,
-                              search_form=SearchForm.from_defaults(),
+                              #search_form=SearchForm.from_defaults(),
                              )
     return Response(content, status)
 
     query = And([Term(WIKINAME, app.cfg.interwikiname), Term(NAME_EXACT, item_name), ])
     # TODO: due to how getPageContent and the template works, we need to use limit=None -
     # it would be better to use search_page (and an appropriate limit, if needed)
-    docs = flaskg.storage.search(query, all_revs=True, sortedby=REV_NO, reverse=True, limit=None)
+    docs = flaskg.storage.search(query, all_revs=True, sortedby=[MTIME], reverse=True, limit=None)
     # get rid of the content value to save potentially big amounts of memory:
     history = [dict((k, v) for k, v in doc.iteritems() if k != CONTENT) for doc in docs]
     history_page = util.getPageContent(history, offset, results_per_page)
         query = And([query, DateRange(MTIME, start=bookmark_time, end=None)])
     # TODO: we need use limit=None to simulate previous implementation's behaviour -
     # it would be better to use search_page (and an appropriate limit, if needed)
-    history = flaskg.storage.search(query, all_revs=True, sortedby=[MTIME, REV_NO], reverse=True, limit=None)
+    history = flaskg.storage.search(query, all_revs=True, sortedby=[MTIME], reverse=True, limit=None) # XXX
     item_groups = OrderedDict()
     for doc in history:
         current_item_name = doc[NAME]
 
         # Aggregating comments, authors and revno
         for doc in docs:
-            rev_no = doc[REV_NO]
+            rev_no = doc[REVID]
             revnos.append(rev_no)
             comment = doc.get(COMMENT)
             if comment:
     linked = set()
     transcluded = set()
     existing = set()
-    docs = flaskg.storage.documents(all_revs=False, wikiname=app.cfg.interwikiname)
-    for doc in docs:
-        existing.add(doc[NAME])
-        linked.update(doc.get(ITEMLINKS, []))
-        transcluded.update(doc.get(ITEMTRANSCLUSIONS, []))
+    revs = flaskg.storage.documents(all_revs=False, wikiname=app.cfg.interwikiname)
+    for rev in revs:
+        existing.add(rev.meta[NAME])
+        linked.update(rev.meta.get(ITEMLINKS, []))
+        transcluded.update(rev.meta.get(ITEMTRANSCLUSIONS, []))
     return existing, linked, transcluded
 
 
             email = form['email'].value
             if form['email'].valid and email:
                 users = user.search_users(email=email)
-                u = users and user.User(users[0][UUID])
+                u = users and user.User(users[0][ITEMID])
             if u and u.valid:
                 is_ok, msg = u.mailAccountData()
                 if not is_ok:
     tags_counts = {}
     for doc in docs:
         tags = doc.get(TAGS, [])
-        logging.debug("name %s rev %s tags %s" % (doc[NAME], doc[REV_NO], tags))
+        logging.debug("name %s rev %s tags %s" % (doc[NAME], doc[REVID], tags))
         for tag in tags:
             tags_counts[tag] = tags_counts.setdefault(tag, 0) + 1
     tags_counts = sorted(tags_counts.items())

MoinMoin/apps/misc/views.py

         return dt.strftime("%Y-%m-%dT%H:%M:%S+00:00")
 
     sitemap = []
-    for doc in flaskg.storage.documents(all_revs=False, wikiname=app.cfg.interwikiname):
-        name = doc[NAME]
-        mtime = doc[MTIME]
+    for rev in flaskg.storage.documents(all_revs=False, wikiname=app.cfg.interwikiname):
+        name = rev.meta[NAME]
+        mtime = rev.meta[MTIME]
         if False: # was: wikiutil.isSystemItem(name)   XXX add back later, when we have that in the index
             if not SITEMAP_HAS_SYSTEM_ITEMS:
                 continue
         sitemap.append((name, format_timestamp(mtime), changefreq, priority))
     # add an entry for root url
     root_item = app.cfg.item_root
-    docs = list(flaskg.storage.documents(all_revs=False, wikiname=app.cfg.interwikiname, name=root_item))
-    if docs:
-        mtime = docs[0][MTIME]
+    revs = list(flaskg.storage.documents(all_revs=False, wikiname=app.cfg.interwikiname, name=root_item))
+    if revs:
+        mtime = revs[0].meta[MTIME]
         sitemap.append((u'', format_timestamp(mtime), "hourly", "1.0"))
     sitemap.sort()
     content = render_template('misc/sitemap.xml', sitemap=sitemap)
     See: http://usemod.com/cgi-bin/mb.pl?SisterSitesImplementationGuide
     """
     # XXX we currently also get trash items, fix this
-    item_names = sorted([doc[NAME] for doc in flaskg.storage.documents(all_revs=False, wikiname=app.cfg.interwikiname)])
+    item_names = sorted([rev.meta[NAME] for rev in flaskg.storage.documents(all_revs=False, wikiname=app.cfg.interwikiname)])
     content = render_template('misc/urls_names.txt', item_names=item_names)
     return Response(content, mimetype='text/plain')
 

MoinMoin/auth/openidrp.py

 from flask import current_app as app
 from MoinMoin.auth import BaseAuth, get_multistage_continuation_url
 from MoinMoin.auth import ContinueLogin, CancelLogin, MultistageFormLogin, MultistageRedirectLogin
+from MoinMoin.config import ITEMID
 from MoinMoin import user
 from MoinMoin.i18n import _, L_, N_
 
                 # we get the user with this openid associated to him
                 identity = oid_info.identity_url
                 users = user.search_users(openid=identity)
-                user_obj = users and user.User(users[0][UUID])
+                user_obj = users and user.User(users[0][ITEMID])
 
                 # if the user actually exists
                 if user_obj:

MoinMoin/config/default.py

             raise error.ConfigurationError("No storage configuration specified! You need to define a namespace_mapping. " + \
                                            "For further reference, please see HelpOnStorageConfiguration.")
 
+        if self.acl_mapping is None:
+            raise error.ConfigurationError("No acl configuration specified! You need to define a acl_mapping. " + \
+                                           "For further reference, please see HelpOnStorageConfiguration.")
+
         if self.secrets is None:  # admin did not setup a real secret
             raise error.ConfigurationError("No secret configured! You need to set secrets = 'somelongsecretstring' in your wiki config.")
 
     ('interwiki_map', {},
      "Dictionary of wiki_name -> wiki_url"),
     ('namespace_mapping', None,
-    "This needs to point to a (correctly ordered!) list of tuples, each tuple containing: Namespace identifier, backend, acl protection to be applied to that backend. " + \
-    "E.g.: [('/', FSBackend('wiki/data'), dict(default='All:read,write,create')), ]. Please see HelpOnStorageConfiguration for further reference."),
+    "This needs to point to a list of tuples, each tuple containing: Namespace identifier, backend. " + \
+    "E.g.: [('/', FSBackend('wiki/data')), ]. Please see HelpOnStorageConfiguration for further reference."),
+    ('acl_mapping', None,
+    "This needs to point to a list of tuples, each tuple containing: name prefix, acl protection to be applied to matching items. " + \
+    "E.g.: [('', dict(default='All:read,write,create')), ]. Please see HelpOnStorageConfiguration for further reference."),
+    ('create_storage', False, "Create (initialize) the storage backends before trying to use them."),
+    ('destroy_storage', False, "Destroy (empty) the storage backends after using them."),
   )),
   # ==========================================================================
   'items': ('Special item names', None, (

MoinMoin/conftest.py

 
 from MoinMoin.app import create_app_ext, destroy_app, before_wiki, teardown_wiki
 from MoinMoin._tests import maketestwiki, wikiconfig
-from MoinMoin.storage.backends import create_simple_mapping
+from MoinMoin.storage import create_simple_mapping
 from flask import g as flaskg
 
 # In the beginning following variables have no values
     return prev_app, prev_ctx, prev_cls
 
 def init_test_app(given_config):
-    namespace_mapping = create_simple_mapping("memory:", given_config.content_acl)
+    namespace_mapping, acl_mapping = create_simple_mapping("stores:memory:", given_config.content_acl)
     more_config = dict(
         namespace_mapping=namespace_mapping,
+        acl_mapping=acl_mapping,
+        create_storage = True, # create a fresh storage at each app start
+        destroy_storage = True, # kill all storage contents at app shutdown
     )
     app = create_app_ext(flask_config_dict=dict(SECRET_KEY='foobarfoobar'),
                          moin_config_class=given_config,
 
 
     def teardown(self):
-        clean_backend()
         super(MoinTestFunction, self).teardown()
 
 
 def pytest_report_header(config):
     return "The tests here are implemented only for pytest-2"
 
-def clean_backend():
-    """ method to cleanup the items created in testing process """
-    for test_item in flaskg.unprotected_storage.iteritems():
-        # some items don't have 'uuid' as key in them
-        # such items raise keyerror on test_item.destroy()
-        # add the key 'uuid' to such items
-        key_list = test_item.keys()
-        if 'uuid' not in key_list:
-            test_item.change_metadata()
-            test_item['uuid'] = 'temp_uuid'
-            test_item.publish_metadata()
-        test_item.destroy()
-
 class Module(pytest.collect.Module):
     def run(self, *args, **kwargs):
         if coverage is not None:

MoinMoin/datastruct/backends/wiki_dicts.py

 
 from flask import g as flaskg
 
-from MoinMoin.config import SOMEDICT
+from MoinMoin.config import CURRENT, SOMEDICT
 from MoinMoin.datastruct.backends import BaseDict, BaseDictsBackend, DictDoesNotExistError
 
 
 
     def _load_dict(self):
         dict_name = self.name
-        if flaskg.unprotected_storage.has_item(dict_name):
-            item = flaskg.unprotected_storage.get_item(dict_name)
-            rev = item.get_revision(-1)
-            somedict = rev.get(SOMEDICT, {})
+        item = flaskg.unprotected_storage[dict_name]
+        try:
+            rev = item[CURRENT]
+            somedict = rev.meta.get(SOMEDICT, {})
             return somedict
-        else:
+        except KeyError:
             raise DictDoesNotExistError(dict_name)
 
 
         return WikiDict(name=dict_name, backend=self)
 
     def _retrieve_items(self, dict_name):
-        item = flaskg.unprotected_storage.get_item(dict_name)
-        rev = item.get_revision(-1)
-        somedict = rev.get(SOMEDICT, {})
+        item = flaskg.unprotected_storage[dict_name]
+        rev = item.get_revision(CURRENT)
+        somedict = rev.meta.get(SOMEDICT, {})
         return somedict
 

MoinMoin/datastruct/backends/wiki_groups.py

 
 from flask import g as flaskg
 
-from MoinMoin.config import USERGROUP
+from MoinMoin.config import CURRENT, USERGROUP
 from MoinMoin.datastruct.backends import GreedyGroup, BaseGroupsBackend, GroupDoesNotExistError
 
 
         """
         To find group pages, app.cfg.cache.item_group_regexact pattern is used.
         """
-        all_items = flaskg.unprotected_storage.iteritems()
-        item_list = [item.name for item in all_items
-                     if self.item_group_regex.search(item.name)]
+        # TODO: use whoosh to search for group_regex matching items
+        item_list = [rev.name for rev in flaskg.unprotected_storage.documents(all_revs=False)
+                     if self.item_group_regex.search(rev.name)]
         return iter(item_list)
 
     def __getitem__(self, group_name):
         return WikiGroup(name=group_name, backend=self)
 
     def _retrieve_members(self, group_name):
-        item = flaskg.unprotected_storage.get_item(group_name)
-        rev = item.get_revision(-1)
-        usergroup = rev.get(USERGROUP, [])
+        item = flaskg.unprotected_storage[group_name]
+        rev = item[CURRENT]
+        usergroup = rev.meta.get(USERGROUP, [])
         return usergroup
+

MoinMoin/items/__init__.py

 from MoinMoin.util.interwiki import url_for_item
 from MoinMoin.storage.error import NoSuchItemError, NoSuchRevisionError, AccessDeniedError, \
                                    StorageError
-from MoinMoin.config import UUID, NAME, NAME_OLD, NAME_EXACT, WIKINAME, MTIME, REVERTED_TO, ACL, \
+from MoinMoin.config import NAME, NAME_OLD, NAME_EXACT, WIKINAME, MTIME, REVERTED_TO, ACL, \
                             IS_SYSITEM, SYSITEM_VERSION,  USERGROUP, SOMEDICT, \
                             CONTENTTYPE, SIZE, LANGUAGE, ITEMLINKS, ITEMTRANSCLUSIONS, \
                             TAGS, ACTION, ADDRESS, HOSTNAME, USERID, EXTRA, COMMENT, \
-                            HASH_ALGORITHM, CONTENTTYPE_GROUPS
+                            HASH_ALGORITHM, CONTENTTYPE_GROUPS, ITEMID, REVID, DATAID, \
+                            CURRENT
 
 COLS = 80
 ROWS_DATA = 20
 class DummyRev(dict):
     """ if we have no stored Revision, we use this dummy """
     def __init__(self, item, contenttype):
-        self[CONTENTTYPE] = contenttype
         self.item = item
-        self.timestamp = 0
-        self.revno = None
-    def read(self, size=-1):
-        return ''
-    def seek(self, offset, whence=0):
-        pass
-    def tell(self):
-        return 0
+        self.meta = {CONTENTTYPE: contenttype}
+        self.data = StringIO('')
 
 
 class DummyItem(object):
         return cls(name, contenttype=unicode(contenttype), **kw)
 
     @classmethod
-    def create(cls, name=u'', contenttype=None, rev_no=None, item=None):
-        if rev_no is None:
-            rev_no = -1
+    def create(cls, name=u'', contenttype=None, rev_no=CURRENT, item=None):
         if contenttype is None:
             contenttype = u'application/x-nonexistent'
 
-        try:
+        if 1: # try:
             if item is None:
-                item = flaskg.storage.get_item(name)
+                item = flaskg.storage[name]
             else:
                 name = item.name
-        except NoSuchItemError:
+        if not item: # except NoSuchItemError:
             logging.debug("No such item: %r" % name)
             item = DummyItem(name)
             rev = DummyRev(item, contenttype)
             try:
                 rev = item.get_revision(rev_no)
                 contenttype = u'application/octet-stream' # it exists
-            except NoSuchRevisionError:
+            except KeyError: # NoSuchRevisionError:
                 try:
-                    rev = item.get_revision(-1) # fall back to current revision
+                    rev = item.get_revision(CURRENT) # fall back to current revision
                     # XXX add some message about invalid revision
-                except NoSuchRevisionError:
+                except KeyError: # NoSuchRevisionError:
                     logging.debug("Item %r has no revisions." % name)
                     rev = DummyRev(item, contenttype)
                     logging.debug("Item %r, created dummy revision with contenttype %r" % (name, contenttype))
             logging.debug("Got item %r, revision: %r" % (name, rev_no))
-        contenttype = rev.get(CONTENTTYPE) or contenttype # use contenttype in case our metadata does not provide CONTENTTYPE
+        contenttype = rev.meta.get(CONTENTTYPE) or contenttype # use contenttype in case our metadata does not provide CONTENTTYPE
         logging.debug("Item %r, got contenttype %r from revision meta" % (name, contenttype))
-        logging.debug("Item %r, rev meta dict: %r" % (name, dict(rev)))
+        #logging.debug("Item %r, rev meta dict: %r" % (name, dict(rev.meta)))
 
         item = item_registry.get(name, Type(contenttype), rev=rev)
         logging.debug("ItemClass %r handles %r" % (item.__class__, contenttype))
         self.contenttype = contenttype
 
     def get_meta(self):
-        return self.rev or {}
+        return self.rev.meta
     meta = property(fget=get_meta)
 
     def _render_meta(self):
         """
         flaskg.clock.start('conv_in_dom')
         hash_name = HASH_ALGORITHM
-        hash_hexdigest = self.rev.get(hash_name)
+        hash_hexdigest = self.rev.meta.get(hash_name)
         if hash_hexdigest:
             cid = cache_key(usage="internal_representation",
                             hash_name=hash_name,
                      NAME_OLD,
                      # are automatically implanted when saving
                      NAME,
-                     UUID,
+                     ITEMID, REVID, DATAID,
                      HASH_ALGORITHM,
                      SIZE,
                      COMMENT,
                 buf = content.read(bufsize)
                 if not buf:
                     break
-                new_rev.write(buf)
+                new_rev.data.write(buf)
                 written += len(buf)
         elif isinstance(content, str):
-            new_rev.write(content)
+            new_rev.data.write(content)
             written += len(content)
         else:
             raise StorageError("unsupported content object: %r" % content)
         """
         old_item = self.rev.item
         flaskg.storage.copy_item(old_item, name=name)
-        current_rev = old_item.get_revision(-1)
+        current_rev = old_item.get_revision(CURRENT)
         # we just create a new revision with almost same meta/data to show up on RC
         self._save(current_rev, current_rev, name=name, action=u'COPY', comment=comment)
 
     def _rename(self, name, comment, action):
-        self.rev.item.rename(name)
         self._save(self.meta, self.data, name=name, action=action, comment=comment)
 
     def rename(self, name, comment=u''):
         comment = request.form.get('comment')
         return self._save(meta, data, contenttype_guessed=contenttype_guessed, comment=comment)
 
-    def _save(self, meta, data=None, name=None, action=u'SAVE', contenttype_guessed=None, comment=u''):
+    def _save(self, meta, data=None, name=None, action=u'SAVE', contenttype_guessed=None, comment=u'', overwrite=False):
         if name is None:
             name = self.name
         backend = flaskg.storage
+        storage_item = backend[name]
         try:
-            storage_item = backend.get_item(name)
-        except NoSuchItemError:
-            storage_item = backend.create_item(name)
-        try:
-            currentrev = storage_item.get_revision(-1)
-            rev_no = currentrev.revno
-            contenttype_current = currentrev.get(CONTENTTYPE)
-        except NoSuchRevisionError:
+            currentrev = storage_item.get_revision(CURRENT)
+            rev_no = currentrev.revid
+            contenttype_current = currentrev.meta.get(CONTENTTYPE)
+        except KeyError: # XXX was: NoSuchRevisionError:
             currentrev = None
-            rev_no = -1
+            rev_no = None
             contenttype_current = None
-        new_rev_no = rev_no + 1
-        newrev = storage_item.create_revision(new_rev_no)
-        for k, v in meta.iteritems():
-            # TODO Put metadata into newrev here for now. There should be a safer way
-            #      of input for this.
-            newrev[k] = v
+
+        meta = dict(meta) # we may get a read-only dict-like, copy it
 
         # we store the previous (if different) and current item name into revision metadata
         # this is useful for rename history and backends that use item uids internally
         oldname = meta.get(NAME)
         if oldname and oldname != name:
-            newrev[NAME_OLD] = oldname
-        newrev[NAME] = name
+            meta[NAME_OLD] = oldname
+        meta[NAME] = name
+
+        if comment:
+            meta[COMMENT] = unicode(comment)
+
+        if CONTENTTYPE not in meta:
+            # make sure we have CONTENTTYPE
+            meta[CONTENTTYPE] = unicode(contenttype_current or contenttype_guessed or 'application/octet-stream')
+
+        meta[ACTION] = unicode(action)
+
+        if not overwrite and REVID in meta:
+            # we usually want to create a new revision, thus we must remove the existing REVID
+            del meta[REVID]
 
         if data is None:
             if currentrev is not None:
                 # we don't have (new) data, just copy the old one.
                 # a valid usecase of this is to just edit metadata.
-                data = currentrev
+                data = currentrev.data
             else:
                 data = ''
-        size = self._write_stream(data, newrev)
 
-        # XXX if meta is from old revision, and user did not give a non-empty
-        # XXX comment, re-using the old rev's comment is wrong behaviour:
-        comment = unicode(comment or meta.get(COMMENT, ''))
-        if comment:
-            newrev[COMMENT] = comment
+        if isinstance(data, unicode):
+            data = data.encode(config.charset)
 
-        if CONTENTTYPE not in newrev:
-            # make sure we have CONTENTTYPE
-            newrev[CONTENTTYPE] = unicode(contenttype_current or contenttype_guessed or 'application/octet-stream')
+        if isinstance(data, str):
+            data = StringIO(data)
 
-        newrev[ACTION] = unicode(action)
-        storage_item.commit()
+        newrev = storage_item.store_revision(meta, data, overwrite=overwrite)
         item_modified.send(app._get_current_object(), item_name=name)
-        return new_rev_no, size
+        return None, None # XXX was: new_revno, size
 
     def get_index(self):
         """ create an index of sub items of this item """
             # that has all wiki items as sub items
             prefix = u''
             query = Term(WIKINAME, app.cfg.interwikiname)
-        results = flaskg.storage.search(query, all_revs=False, sortedby=NAME_EXACT, limit=None)
         # We only want the sub-item part of the item names, not the whole item objects.
         prefix_len = len(prefix)
-        items = [(result[NAME], result[NAME][prefix_len:], result[CONTENTTYPE])
-                 for result in results]
+        revs = flaskg.storage.search(query, all_revs=False, sortedby=NAME_EXACT, limit=None)
+        items = [(rev.meta[NAME], rev.meta[NAME][prefix_len:], rev.meta[CONTENTTYPE])
+                 for rev in revs]
         return items
 
     def flat_index(self, startswith=None, selected_groups=None):
     # XXX reads item rev data into memory!
     def get_data(self):
         if self.rev is not None:
-            return self.rev.read()
+            return self.rev.data.read()
         else:
             return ''
     data = property(fget=get_data)
         if contenttype is not None:
             terms.append(Term(CONTENTTYPE, contenttype))
         query = And(terms)
-        results = flaskg.storage.search(query, all_revs=False, sortedby=NAME_EXACT, limit=None)
-        return [result[NAME] for result in results]
+        revs = flaskg.storage.search(query, all_revs=False, sortedby=NAME_EXACT, limit=None)
+        return [rev.meta[NAME] for rev in revs]
 
     def do_modify(self, contenttype, template_name):
         # XXX think about and add item template support
             form = ModifyForm.from_defaults()
             TextCha(form).amend_form()
             form['meta_text'] = self.meta_dict_to_text(self.meta)
-            form['rev'] = self.rev.revno if self.rev.revno is not None else -1
+            form['rev'] = self.rev.revno if self.rev.revno is not None else CURRENT
         elif request.method == 'POST':
             form = ModifyForm.from_flat(request.form.items() + request.files.items())
             TextCha(form).amend_form()
                  contenttype=request.values.get('contenttype'))
 
     def do_get(self, force_attachment=False, mimetype=None):
-        hash = self.rev.get(HASH_ALGORITHM)
+        hash = self.rev.meta.get(HASH_ALGORITHM)
         if is_resource_modified(request.environ, hash): # use hash as etag
             return self._do_get_modified(hash, force_attachment=force_attachment, mimetype=mimetype)
         else:
         """
         list tar file contents (member file names)
         """
-        self.rev.seek(0)
-        tf = tarfile.open(fileobj=self.rev, mode='r')
+        self.rev.data.seek(0)
+        tf = tarfile.open(fileobj=self.rev.data, mode='r')
         return tf.getnames()
 
     def get_member(self, name):
 
         :param name: name of the data in the container file
         """
-        self.rev.seek(0)
-        tf = tarfile.open(fileobj=self.rev, mode='r')
+        self.rev.data.seek(0)
+        tf = tarfile.open(fileobj=self.rev.data, mode='r')
         return tf.extractfile(name)
 
     def put_member(self, name, content, content_length, expected_members):
         """
         list zip file contents (member file names)
         """
-        self.rev.seek(0)
-        zf = zipfile.ZipFile(self.rev, mode='r')
+        self.rev.data.seek(0)
+        zf = zipfile.ZipFile(self.rev.data, mode='r')
         return zf.namelist()
 
     def get_member(self, name):
 
         :param name: name of the data in the zip file
         """
-        self.rev.seek(0)
-        zf = zipfile.ZipFile(self.rev, mode='r')
+        self.rev.data.seek(0)
+        zf = zipfile.ZipFile(self.rev.data, mode='r')
         return zf.open(name, mode='r')
 
     def put_member(self, name, content, content_length, expected_members):
             from PIL import Image as PILImage
         except ImportError:
             # no PIL, we can't do anything, we just output the revision data as is
-            return content_type, self.rev.read()
+            return content_type, self.rev.data.read()
 
         if content_type == 'image/jpeg':
             output_type = 'JPEG'
             raise ValueError("content_type %r not supported" % content_type)
 
         # revision obj has read() seek() tell(), thus this works:
-        image = PILImage.open(self.rev)
+        image = PILImage.open(self.rev.data)
         image.load()
 
         try:
         if width or height or transpose != 1:
             # resize requested, XXX check ACL behaviour! XXX
             hash_name = HASH_ALGORITHM
-            hash_hexdigest = self.rev[hash_name]
+            hash_hexdigest = self.rev.meta[hash_name]
             cid = cache_key(usage="ImageTransform",
                             hash_name=hash_name,
                             hash_hexdigest=hash_hexdigest,
                 if mimetype:
                     content_type = mimetype
                 else:
-                    content_type = self.rev[CONTENTTYPE]
+                    content_type = self.rev.meta[CONTENTTYPE]
                 size = (width or 99999, height or 99999)
                 content_type, data = self._transform(content_type, size=size, transpose_op=transpose)
                 headers = wikiutil.file_headers(content_type=content_type, content_length=len(data))
 
     def _render_data_diff(self, oldrev, newrev):
         from MoinMoin.util.diff_html import diff
-        old_text = self.data_storage_to_internal(oldrev.read())
-        new_text = self.data_storage_to_internal(newrev.read())
-        storage_item = flaskg.storage.get_item(self.name)
+        old_text = self.data_storage_to_internal(oldrev.data.read())
+        new_text = self.data_storage_to_internal(newrev.data.read())
+        storage_item = flaskg.storage[self.name]
         revs = storage_item.list_revisions()
         diffs = [(d[0], Markup(d[1]), d[2], Markup(d[3])) for d in diff(old_text, new_text)]
         return Markup(render_template('diff_text.html',
 
     def _render_data_diff_text(self, oldrev, newrev):
         from MoinMoin.util import diff_text
-        oldlines = self.data_storage_to_internal(oldrev.read()).split('\n')
-        newlines = self.data_storage_to_internal(newrev.read()).split('\n')
+        oldlines = self.data_storage_to_internal(oldrev.data.read()).split('\n')
+        newlines = self.data_storage_to_internal(newrev.data.read()).split('\n')
         difflines = diff_text.diff(oldlines, newlines)
         return '\n'.join(difflines)
 
             else:
                 form['data_text'] = self.data_storage_to_internal(self.data)
             form['meta_text'] = self.meta_dict_to_text(self.meta)
-            form['rev'] = self.rev.revno if self.rev.revno is not None else -1
+            form['rev'] = self.rev.revno if self.rev.revno is not None else CURRENT
         elif request.method == 'POST':
             form = ModifyForm.from_flat(request.form.items() + request.files.items())
             TextCha(form).amend_form()
             TextCha(form).amend_form()
             # XXX currently this is rather pointless, as the form does not get POSTed:
             form['meta_text'] = self.meta_dict_to_text(self.meta)
-            form['rev'] = self.rev.revno if self.rev.revno is not None else -1
+            form['rev'] = self.rev.revno if self.rev.revno is not None else CURRENT
         elif request.method == 'POST':
             # this POST comes directly from TWikiDraw (not from Browser), thus no validation
             try:
             TextCha(form).amend_form()
             # XXX currently this is rather pointless, as the form does not get POSTed:
             form['meta_text'] = self.meta_dict_to_text(self.meta)
-            form['rev'] = self.rev.revno if self.rev.revno is not None else -1
+            form['rev'] = self.rev.revno if self.rev.revno is not None else CURRENT
         elif request.method == 'POST':
             # this POST comes directly from AnyWikiDraw (not from Browser), thus no validation
             try:
             TextCha(form).amend_form()
             # XXX currently this is rather pointless, as the form does not get POSTed:
             form['meta_text'] = self.meta_dict_to_text(self.meta)
-            form['rev'] = self.rev.revno if self.rev.revno is not None else -1
+            form['rev'] = self.rev.revno if self.rev.revno is not None else CURRENT
         elif request.method == 'POST':
             # this POST comes directly from SvgDraw (not from Browser), thus no validation
             try:

MoinMoin/items/_tests/test_Item.py

         item._save(meta, data, comment=comment)
         # check save result
         item = Item.create(name)
-        saved_meta, saved_data = dict(item.meta), item.data
+        saved_meta, saved_data = item.meta, item.data
         assert saved_meta[CONTENTTYPE] == contenttype
         assert saved_meta[COMMENT] == comment
         assert saved_data == data
-        assert item.rev.revno == 0
 
         data = rev1_data = data * 10000
         comment = comment + u' again'
         assert saved_meta[CONTENTTYPE] == contenttype
         assert saved_meta[COMMENT] == comment
         assert saved_data == data
-        assert item.rev.revno == 1
 
         data = ''
         comment = 'saved empty data'
         assert saved_meta[CONTENTTYPE] == contenttype
         assert saved_meta[COMMENT] == comment
         assert saved_data == data
-        assert item.rev.revno == 2
-
-        # access old revision
-        item = Item.create(name, rev_no=1)
-        assert item.data == rev1_data
 
     def testIndex(self):
         # create a toplevel and some sub-items
     def test_meta_filter(self):
         name = u'Test_item'
         contenttype = u'text/plain;charset=utf-8'
-        meta = {'test_key': 'test_val', CONTENTTYPE: contenttype, 'name': 'test_name', 'uuid': 'test_uuid'}
+        meta = {'test_key': 'test_val', CONTENTTYPE: contenttype, 'name': 'test_name'}
         item = Item.create(name)
         result = Item.meta_filter(item, meta)
-        # keys like NAME and UUID are filtered
+        # keys like NAME, ITEMID, REVID, DATAID are filtered
         expected = {'test_key': 'test_val', CONTENTTYPE: contenttype}
         assert result == expected
 
     def test_meta_dict_to_text(self):
         name = u'Test_item'
         contenttype = u'text/plain;charset=utf-8'
-        meta = {'test_key': 'test_val', CONTENTTYPE: contenttype, 'name': 'test_name', 'uuid': 'test_uuid'}
+        meta = {'test_key': 'test_val', CONTENTTYPE: contenttype, 'name': 'test_name'}
         item = Item.create(name)
         result = Item.meta_dict_to_text(item, meta)
         expected = '{\n  "contenttype": "text/plain;charset=utf-8", \n  "test_key": "test_val"\n}'
     def test_meta_text_to_dict(self):
         name = u'Test_item'
         contenttype = u'text/plain;charset=utf-8'
-        text = '{\n  "contenttype": "text/plain;charset=utf-8", \n  "test_key": "test_val", \n "name": "test_name", \n "uuid": "test_uuid"\n}'
+        text = '{\n  "contenttype": "text/plain;charset=utf-8", \n  "test_key": "test_val", \n "name": "test_name" \n}'
         item = Item.create(name)
         result = Item.meta_text_to_dict(item, text)
         expected = {'test_key': 'test_val', CONTENTTYPE: contenttype}
         # item and its contents after deletion
         item = Item.create(name)
         assert item.name == u'Test_Item'
-        assert item.meta == {'contenttype': 'application/x-nonexistent'}
+        assert item.meta[CONTENTTYPE] == 'application/x-nonexistent'
 
     def test_revert(self):
         name = u'Test_Item'
         content_length = len(filecontent)
         item.put_member('example1.txt', filecontent, content_length, expected_members=members)
 
-        item = flaskg.storage.get_item(item_name)
-        assert item.next_revno == 2
-
         item = Item.create(item_name, contenttype=u'application/x-tar')
         assert item.get_member('example1.txt').read() == filecontent
 

MoinMoin/script/__init__.py

     manager.add_option('-c', '--config', dest='config', required=False, default=wiki_config)
     manager.add_command("moin", Server(host='127.0.0.1', port=8080))
 
-    from MoinMoin.script.maint.index import IndexOperations
-    manager.add_command("index", IndexOperations())
+    from MoinMoin.script.maint import index
+    manager.add_command("index-create", index.IndexCreate())
+    manager.add_command("index-build", index.IndexBuild())
+    manager.add_command("index-update", index.IndexUpdate())
+    manager.add_command("index-destroy", index.IndexDestroy())
+    manager.add_command("index-move", index.IndexMove())
+    manager.add_command("index-optimize", index.IndexOptimize())
+    from MoinMoin.script.maint import serialization
+    manager.add_command("save", serialization.Serialize())
+    manager.add_command("load", serialization.Deserialize())
     from MoinMoin.script.account.create import Create_User
     manager.add_command("account_create", Create_User())
     from MoinMoin.script.account.disable import Disable_User
     manager.add_command("maint_create_item", Create_Item())
     from MoinMoin.script.maint.modified_systemitems import Modified_SystemItems
     manager.add_command("maint_modified_systemitems", Modified_SystemItems())
-    from MoinMoin.script.maint.xml import XML
-    manager.add_command("maint_xml", XML())
 
     return manager.run(default_command=default_command)
 

MoinMoin/script/maint/index.py

 MoinMoin - manage whoosh indexes (building, updating, (re)moving and displaying)
 """
 
-import os, datetime
 
 from flask import current_app as app
 from flask import g as flaskg
 from flaskext.script import Command, Option
-from whoosh.filedb.multiproc import MultiSegmentWriter
-from whoosh.index import open_dir, create_in, exists_in
-from whoosh.index import EmptyIndexError
-
-from MoinMoin.search.indexing import WhooshIndex
-from MoinMoin.config import MTIME, NAME, CONTENTTYPE, REV_NO, CONTENT
-from MoinMoin.error import FatalError
-from MoinMoin.storage.error import NoSuchItemError, NoSuchRevisionError
-from MoinMoin.util.mime import Type
-from MoinMoin.search.indexing import backend_to_index
-from MoinMoin.storage.middleware.indexing import convert_to_indexable
 
 from MoinMoin import log
 logging = log.getLogger(__name__)
 
-# Information about index and schema for latest and all revisions
-latest_indexname_schema = ("latest_revisions_index", "latest_revisions_schema")
-all_indexname_schema = ("all_revisions_index", "all_revisions_schema")
-both_indexnames_schemas = [latest_indexname_schema, all_indexname_schema]
 
+class IndexCreate(Command):
+    description = 'Create empty indexes.'
 
-class IndexOperations(Command):
-    description = 'Build indexes'
+    option_list = [
+        Option('--tmp', action="store_true", required=False, dest='tmp', default=False,
+            help='use the temporary location.'),
+    ]
 
-    option_list = (
-        Option('--for', required=True, dest='indexname', type=str, choices=("all-revs", "latest-revs", "both"),
-            help='For what type of indexes we will use action'),
-        Option('--action', required=True, dest='action', type=str, choices=("build", "update", "clean", "move", "show"),
-            help="""
-                  Action for given indexes:
-                  build -- Build in index_dir_tmp
-                  update -- Update in index_dir
-                  clean -- Clean index_dir
-                  move  -- Move index files from index_dir_tmp to index_dir
-                  show -- Show index contents for the given index.
-                 """
-               ),
+    def run(self, tmp):
+        unprotected_storage = flaskg.unprotected_storage = app.unprotected_storage
+        unprotected_storage.create(tmp=tmp)
+
+
+class IndexDestroy(Command):
+    description = 'Destroy the indexes.'
+
+    option_list = [
+        Option('--tmp', action="store_true", required=False, dest='tmp', default=False,
+            help='use the temporary location.'),
+    ]
+
+    def run(self, tmp):
+        unprotected_storage = flaskg.unprotected_storage = app.unprotected_storage
+        unprotected_storage.destroy(tmp=tmp)
+
+
+class IndexBuild(Command):
+    description = 'Build the indexes.'
+
+    option_list = [
+        Option('--tmp', action="store_true", required=False, dest='tmp', default=False,
+            help='use the temporary location.'),
         Option('--procs', '-p', required=False, dest='procs', type=int, default=None,
             help='Number of processors the writer will use.'),
         Option('--limitmb', '-l', required=False, dest='limitmb', type=int, default=10,
             help='Maximum memory (in megabytes) each index-writer will use for the indexing pool.'),
-                  )
+    ]
 
-    def run(self, indexname, action, procs, limitmb):
+    def run(self, tmp, procs, limitmb):
+        unprotected_storage = flaskg.unprotected_storage = app.unprotected_storage
+        unprotected_storage.rebuild(tmp=tmp, procs=procs, limitmb=limitmb)
 
-        def build_index(indexnames_schemas):
-            """
-            Building in app.cfg.index_dir_tmp
-            """
-            indexnames = [indexname for indexname, schema in indexnames_schemas]
-            if procs == 1:
-                # MultiSegmentWriter sometimes has issues and is pointless for procs == 1,
-                # so use the simple writer when --procs 1 is given:
-                _all_rev_writer = all_rev_index.writer()
-                _latest_rev_writer = latest_rev_index.writer()
-            else:
-                _all_rev_writer = MultiSegmentWriter(all_rev_index, procs, limitmb)
-                _latest_rev_writer = MultiSegmentWriter(latest_rev_index, procs, limitmb)
-            with _all_rev_writer as all_rev_writer:
-                with _latest_rev_writer as latest_rev_writer:
-                    for item in backend.iter_items_noindex():
-                        try:
-                            rev_no = None
-                            if "all_revisions_index" in indexnames:
-                                for rev_no in item.list_revisions():
-                                    revision = item.get_revision(rev_no)
-                                    rev_content = convert_to_indexable(revision)
-                                    metadata = backend_to_index(revision, rev_no, all_rev_schema, rev_content, interwikiname)
-                                    all_rev_writer.add_document(**metadata)
-                            else:
-                                revision = item.get_revision(-1)
-                                rev_no = revision.revno
-                                rev_content = convert_to_indexable(revision)
-                        except NoSuchRevisionError: # item has no such revision
-                            continue
-                        # revision is now the latest revision of this item
-                        if "latest_revisions_index" in indexnames and rev_no is not None:
-                            metadata = backend_to_index(revision, rev_no, latest_rev_schema, rev_content, interwikiname)
-                            latest_rev_writer.add_document(**metadata)
 
-        def update_index(indexnames_schemas):
-            """
-            Updating index in app.cfg.index_dir_tmp
-            """
+class IndexUpdate(Command):
+    description = 'Update the indexes.'
 
-            indexnames = [indexname for indexname, schema in indexnames_schemas]
-            create_documents = []
-            delete_documents = []
-            latest_documents = []
-            for item in backend.iter_items_noindex():
-                backend_rev_list = item.list_revisions()
-                if not backend_rev_list: # If item hasn't revisions, skipping it
-                    continue
-                name = item.get_revision(-1)[NAME]
-                index_rev_list = item_index_revs(all_rev_searcher, name)
-                add_rev_nos = set(backend_rev_list) - set(index_rev_list)
-                if add_rev_nos:
-                    if "all_revisions_index" in indexnames:
-                        create_documents.append((item, add_rev_nos))
-                    if "latest_revisions_index" in indexnames:
-                        latest_documents.append((item, max(add_rev_nos))) # Add latest revision
-                remove_rev_nos = set(index_rev_list) - set(backend_rev_list)
-                if remove_rev_nos:
-                    if "all_revisions_index" in indexnames:
-                        delete_documents.append((item, remove_rev_nos))
+    option_list = [
+        Option('--tmp', action="store_true", required=False, dest='tmp', default=False,
+            help='use the temporary location.'),
+    ]
 
-            if "latest_revisions_index" in indexnames and latest_documents:
-                with latest_rev_index.writer() as latest_rev_writer:
-                    for item, rev_no in latest_documents:
-                        revision = item.get_revision(rev_no)
-                        rev_content = convert_to_indexable(revision)
-                        converted_rev = backend_to_index(revision, rev_no, latest_rev_schema, rev_content, interwikiname)
-                        found = latest_rev_searcher.document(name_exact=item.name,
-                                                             wikiname=interwikiname
-                                                            )
-                        if not found:
-                            latest_rev_writer.add_document(**converted_rev)
-                        # Checking that last revision is the latest
-                        elif found[REV_NO] < converted_rev[REV_NO]:
-                            doc_number = latest_rev_searcher.document_number(name_exact=item.name, wikiname=interwikiname)
-                            latest_rev_writer.delete_document(doc_number)
-                            latest_rev_writer.add_document(**converted_rev)
+    def run(self, tmp):
+        unprotected_storage = flaskg.unprotected_storage = app.unprotected_storage
+        unprotected_storage.update(tmp=tmp)
 
-            if "all_revisions_index" in indexnames and delete_documents:
-                with all_rev_index.writer() as all_rev_writer:
-                    for item, rev_nos in delete_documents:
-                        for rev_no in rev_nos:
-                            doc_number = all_rev_searcher.document_number(rev_no=rev_no,
-                                                                          exact_name=item.name,
-                                                                          wikiname=interwikiname
-                                                                         )
-                            if doc_number:
-                                all_rev_writer.delete_document(doc_number)
 
-            if "all_revisions_index" in indexnames and create_documents:
-                with all_rev_index.writer() as all_rev_writer:
-                    for item, rev_nos in create_documents:
-                        for rev_no in rev_nos:
-                            revision = item.get_revision(rev_no)
-                            rev_content = convert_to_indexable(revision)
-                            converted_rev = backend_to_index(revision, rev_no, all_rev_schema, rev_content, interwikiname)
-                            all_rev_writer.add_document(**converted_rev)
+class IndexMove(Command):
+    description = 'Move the indexes from the temporary to the normal location.'
 
-        def clean_index(indexnames_schemas):
-            """
-            Clean given index in app.cfg.index_dir
-            """
-            for indexname, schema in indexnames_schemas:
-                index_object.create_index(index_dir=app.cfg.index_dir,
-                                          indexname=indexname,
-                                          schema=schema
-                                         )
+    option_list = [
+    ]
 
-        def move_index(indexnames_schemas):
-            """
-            Move given indexes from index_dir_tmp to index_dir
-            """
-            clean_index(indexnames_schemas)
-            for indexname, schema in indexnames_schemas:
-                if not exists_in(app.cfg.index_dir_tmp, indexname=indexname):
-                    raise FatalError(u"Can't find %s in %s" % (indexname, app.cfg.index_dir_tmp))
-                for filename in all_rev_index.storage.list():
-                    src_file = os.path.join(app.cfg.index_dir_tmp, filename)
-                    dst_file = os.path.join(app.cfg.index_dir, filename)
-                    if indexname in filename and os.path.exists(src_file):
-                        os.rename(src_file, dst_file)
+    def run(self):
+        unprotected_storage = flaskg.unprotected_storage = app.unprotected_storage
+        unprotected_storage.move_index()
 
-        def show_index(indexnames_schemas):
-            """
-            Print documents in given index to stdout
-            """
 
-            for indexname, schema in indexnames_schemas:
-                try:
-                    if indexname == "all_revisions_index":
-                        ix = open_dir(app.cfg.index_dir, indexname="all_revisions_index")
-                    elif indexname == "latest_revisions_index":
-                        ix = open_dir(app.cfg.index_dir, indexname="latest_revisions_index")
-                    print "*** Revisions in", indexname
-                    with ix.searcher() as searcher:
-                        for rev in searcher.all_stored_fields():
-                            name = rev.pop(NAME, u"")
-                            content = rev.pop(CONTENT, u"")
-                            for field, value in [(NAME, name), ] + sorted(rev.items()) + [(CONTENT, content), ]:
-                                print "%s: %s" % (field, repr(value)[:70])
-                            print "\n"
-                    ix.close()
-                except (IOError, OSError, EmptyIndexError) as err:
-                    raise FatalError("%s [Can not open %s index" % str(err), indexname)
+class IndexOptimize(Command):
+    description = 'Optimize the indexes.'
 
-        def item_index_revs(searcher, name):
-            """
-            Return list of found documents for given name using index searcher
-            """
+    option_list = [
+        Option('--tmp', action="store_true", required=False, dest='tmp', default=False,
+            help='use the temporary location.'),
+    ]
 
-            revs_found = searcher.documents(name_exact=name, wikiname=interwikiname)
-            return [rev[REV_NO] for rev in revs_found]
+    def run(self, tmp):
+        unprotected_storage = flaskg.unprotected_storage = app.unprotected_storage
+        unprotected_storage.optimize_index(tmp=tmp)
 
-        def do_action(action, indexnames_schemas):
-            if action == "build":
-                build_index(indexnames_schemas)
-            elif action == "update":
-                update_index(indexnames_schemas)
-            elif action == "clean":
-                clean_index(indexnames_schemas)
-            elif action == "move":
-                move_index(indexnames_schemas)
-            elif action == "show":
-                show_index(indexnames_schemas)
-
-        backend = flaskg.unprotected_storage = app.unprotected_storage
-        index_object = WhooshIndex(index_dir=app.cfg.index_dir_tmp)
-        interwikiname = app.cfg.interwikiname
-        if os.path.samefile(app.cfg.index_dir_tmp, app.cfg.index_dir):
-            raise FatalError(u"cfg.index_dir and cfg.index_dir_tmp must point to different directories.")
-
-        latest_rev_index = index_object.latest_revisions_index
-        all_rev_index = index_object.all_revisions_index
-
-        latest_rev_schema = latest_rev_index.schema
-        all_rev_schema = all_rev_index.schema
-
-        latest_rev_searcher = latest_rev_index.searcher()
-        all_rev_searcher = all_rev_index.searcher()
-
-        if indexname == "both":
-            do_action(action, both_indexnames_schemas)
-        elif indexname == "all-revs":
-            do_action(action, (all_indexname_schema, ))
-        elif indexname == "latest-revs":
-            do_action(action, (latest_indexname_schema, ))

MoinMoin/security/__init__.py

         :returns: checking function for that right
         """
         if attr in app.cfg.acl_rights_contents:
-            def may(itemname):
-                backend = flaskg.storage._get_backend(itemname)[0]
-                return backend._may(itemname, attr, username=self.name)
-            return may
+            return lambda itemname: flaskg.storage.may(itemname, attr, username=self.name)
         if attr in app.cfg.acl_rights_functions:
             may = app.cfg.cache.acl_functions.may
             return lambda: may(self.name, attr)

MoinMoin/security/_tests/test_security.py

 
     from MoinMoin._tests import wikiconfig
     class Config(wikiconfig.Config):
-        content_acl = dict(
-                hierarchic=False,
-                before=u"WikiAdmin:admin,read,write,create,destroy",
-                default=u"All:read,write",
-                after=u"All:read",
-        )
+        content_acl = dict(hierarchic=False, before=u"WikiAdmin:admin,read,write,create,destroy", default=u"All:read,write", after=u"All:read")
 
     def setup_method(self, method):
         become_trusted(username=u'WikiAdmin')
 
     from MoinMoin._tests import wikiconfig
     class Config(wikiconfig.Config):
-        content_acl = dict(
-                       hierarchic=True,
-                       before=u"WikiAdmin:admin,read,write,create,destroy",
-                       default=u"All:read,write",
-                       after=u"All:read",
-        )
+        content_acl = dict(hierarchic=True, before=u"WikiAdmin:admin,read,write,create,destroy", default=u"All:read,write", after=u"All:read")
 
     def setup_method(self, method):
         become_trusted(username=u'WikiAdmin')
             (self.mainitem_name, u'JaneDoe', ['read', 'write']), # by item acl
             (self.mainitem_name, u'JoeDoe', []), # by item acl
             (self.subitem1_name, u'WikiAdmin', ['read', 'write', 'admin', 'create', 'destroy']),
-            (self.subitem1_name, u'AnyUser', ['read']), # by after acl
+            (self.subitem1_name, u'AnyUser', ['read', 'write']), # by default acl
             (self.subitem1_name, u'JoeDoe', []), # by inherited acl from main item
             (self.subitem1_name, u'JaneDoe', ['read', 'write']), # by inherited acl from main item
             (self.subitem2_name, u'WikiAdmin', ['read', 'write', 'admin', 'create', 'destroy']),
 
 import time
 import copy
+from StringIO import StringIO
 
 from babel import parse_locale
 
 from whoosh.query import Term, And, Or
 
 from MoinMoin import config, wikiutil
-from MoinMoin.config import WIKINAME, NAME, NAME_EXACT, UUID, ACTION, CONTENTTYPE, EMAIL, OPENID
+from MoinMoin.config import WIKINAME, NAME, NAME_EXACT, ITEMID, ACTION, CONTENTTYPE, EMAIL, OPENID, CURRENT
 from MoinMoin.i18n import _, L_, N_
 from MoinMoin.util.interwiki import getInterwikiHome, getInterwikiName, is_local_wiki
 from MoinMoin.util.crypto import crypt_password, upgrade_password, valid_password, \
 from MoinMoin.storage.error import NoSuchItemError, ItemAlreadyExistsError, NoSuchRevisionError
 
 
-def create_user(username, password, email, openid=None):
+def create_user(username, password, email, openid=None, validate=True, is_encrypted=False):
     """ create a user """
     # Create user profile
     theuser = User(auth_method="new-user")
     theuser.name = unicode(username)
 
     # Don't allow creating users with invalid names
-    if not isValidName(theuser.name):
+    if validate and not isValidName(theuser.name):
         return _("""Invalid user name '%(name)s'.
 Name may contain any Unicode alpha numeric character, with optional one
 space between words. Group page name is not allowed.""", name=theuser.name)
 
     # Name required to be unique. Check if name belong to another user.
-    if search_users(name_exact=theuser.name):
+    if validate and search_users(name_exact=theuser.name):
         return _("This user name already belongs to somebody else.")
 
     pw_checker = app.cfg.password_checker
-    if pw_checker:
+    if validate and pw_checker:
         pw_error = pw_checker(theuser.name, password)
         if pw_error:
             return _("Password not acceptable: %(msg)s", msg=pw_error)
 
     # Encode password
     try:
-        theuser.enc_password = crypt_password(password)
+        if is_encrypted:
+            theuser.enc_password = password
+        else:
+            theuser.enc_password = crypt_password(password)
     except UnicodeError as err:
         # Should never happen
         return "Can't encode password: %(msg)s" % dict(msg=str(err))
 
     # try to get the email, for new users it is required
     theuser.email = email
-    if not theuser.email:
+    if validate and not theuser.email:
         return _("Please provide your email address. If you lose your"
                  " login information, you can get it by email.")
 
     # Email should be unique - see also MoinMoin/script/accounts/moin_usercheck.py
-    if theuser.email and app.cfg.user_email_unique:
+    if validate and theuser.email and app.cfg.user_email_unique:
         if search_users(email=theuser.email):
             return _("This email already belongs to somebody else.")
 
     # Openid should be unique
     theuser.openid = openid
-    if theuser.openid and search_users(openid=theuser.openid):
+    if validate and theuser.openid and search_users(openid=theuser.openid):
         return _('This OpenID already belongs to somebody else.')
 
     # save data
                                First tuple element was used for authentication.
         """
         self._user_backend = get_user_backend()
-        self._user = None
 
         self._cfg = app.cfg
         self.valid = 0
         if not self.uuid and self.auth_username:
             users = search_users(name_exact=self.auth_username)
             if users:
-                self.uuid = users[0][UUID]
+                self.uuid = users[0].meta[ITEMID]
             if not password is None:
                 check_password = password
         if self.uuid:
         elif self.name and self.name != 'anonymous':
             users = search_users(name_exact=self.name)
             if users:
-                self.uuid = users[0][UUID]
+                self.uuid = users[0].meta[ITEMID]
             if self.uuid:
                 # no password given should fail
                 self.load_from_id(password or u'')
                          password in the user account file.
         """
         try:
-            users = search_users(uuid=self.uuid) # XXX we need the name because backend API is still based on names
-            if not users:
-                raise NoSuchItemError("No user name for that uuid.")
-            name = users[0][NAME]
-            item = self._user_backend.get_item(name)
-            self._user = item.get_revision(-1)
-        except (NoSuchItemError, NoSuchRevisionError):
+            item = self._user_backend.get_item(itemid=self.uuid)
+            rev = item[CURRENT]
+        except KeyError: # was: (NoSuchItemError, NoSuchRevisionError):
             return
 
-        user_data = dict()
-        for metadata_key in self._user:
-            user_data[metadata_key] = self._user[metadata_key]
+        user_data = dict(rev.meta)
 
         # Validate data from user file. In case we need to change some
         # values, we set 'changed' flag, and later save the user data.
         Save user account data to user account file on disk.
         """
         backend_name = self.name # XXX maybe UserProfile/<name> later
-        try:
-            item = self._user_backend.get_item(backend_name)
-        except NoSuchItemError:
-            item = self._user_backend.create_item(backend_name)
-        try:
-            currentrev = item.get_revision(-1)
-            rev_no = currentrev.revno
-        except NoSuchRevisionError:
-            currentrev = None
-            rev_no = -1
-        new_rev_no = rev_no + 1
-        newrev = item.create_revision(new_rev_no)
+        item = self._user_backend[backend_name]
+        meta = {}
         for key, value in self.persistent_items():
             if isinstance(value, list):
                 value = tuple(value)
-            newrev[key] = value
-        newrev[CONTENTTYPE] = u'application/x.moin.userprofile'
-        newrev[ACTION] = u'SAVE'
-        item.commit()
+            meta[key] = value
+        meta[CONTENTTYPE] = u'application/x.moin.userprofile'
+        meta[ACTION] = u'SAVE'
+        item.store_revision(meta, StringIO(''), overwrite=True)
 
         if not self.disabled:
             self.valid = 1

MoinMoin/util/_tests/test_crypto.py

         with pytest.raises(ValueError):
             invalid_result = crypto.valid_password("MoinMoin", '{junk_value}')
 
+    def testvalidpassword2(self):
+        """ validate user password """
+        hash_val = crypto.crypt_password(u"MoinMoin")
+        result = crypto.valid_password('MoinMoin', hash_val)
+        assert result
+        result = crypto.valid_password('WrongPassword', hash_val)
+        assert not result
+        with pytest.raises(ValueError):
+            invalid_result = crypto.valid_password("MoinMoin", '{junk_value}')
+
 
 class TestToken(object):
     """ tests for the generated tokens """

MoinMoin/util/crypto.py

                 enc = crypt.crypt(pw_utf8, salt.encode('ascii'))
             else:
                 raise ValueError("missing password hash method %s handler" % method)
-
             return pw_hash == method + enc
     else:
         idx = pw_hash.index('}') + 1

MoinMoin/wikiutil.py

 from flask import request
 
 from MoinMoin import config
-from MoinMoin.config import IS_SYSITEM
+from MoinMoin.config import CURRENT, IS_SYSITEM
 
 from MoinMoin.i18n import _, L_, N_
 from MoinMoin.util import pysupport, lock
     :returns: True if page is a system item
     """
     try:
-        item = flaskg.storage.get_item(itemname)
-        return item.get_revision(-1)[IS_SYSITEM]
+        item = flaskg.storage[itemname]
+        return item[CURRENT][IS_SYSITEM]
     except (NoSuchItemError, NoSuchRevisionError, KeyError):
-        pass
-
-    return False
+        return False
 
 
 def isGroupItem(itemname):