Commits

Thomas Waldmann committed 41eb0ef Draft

make moin work for GAE (google app engine), details see below

Most of this stuff is based on work done by Guido van Rossum, thanks!

added a GAE store implementation for moin

wikiconfig: adjusted so it works with GAE by default (in this branch):
* use moin's GAE store
* use whoosh's GAE backend

logging: currently needs a hack (configured=True), otherwise it just hangs
strangely.

added a app.yaml definition file for GAE

added a appengine_main.py script for usage with GAE

Notes:
* there needs to be a "support" directory with all of moin's dependencies
(except jinja2, sphinx, py.test) on the same level as "MoinMoin" == in the top
level directory.

* we currently need the 2.5 development version of whoosh.

* I did not apply the patch for filesys.py as we currently do not import
this module at all, so there is no problem with it on GAE either.

Comments (0)

Files changed (6)

 import logging.config
 import logging.handlers  # needed for handlers defined there being configurable in logging.conf file
 
-configured = False
+# TODO: configured should be False of course, but this crashes GAE!
+# maybe this is related how we (ab)use and patch the logger as "logging".
+configured = True
+
 fallback_config = False
 
 import warnings

MoinMoin/storage/middleware/indexing.py

 
 
 WHOOSH_FILESTORAGE = 'FileStorage'
+WHOOSH_GAE = 'DatastoreStorage'
 INDEXES = [LATEST_REVS, ALL_REVS, ]
 
 
                 params[0] += '.temp'
             from whoosh.filedb.filestore import FileStorage
             cls = FileStorage
+        elif kind == WHOOSH_GAE:
+            from whoosh.filedb.gae import DatastoreStorage
+            cls = DatastoreStorage
         else:
             raise ValueError("index_storage = {0!r} is not supported!".format(kind))
         return kind, cls, params, kw

MoinMoin/storage/stores/gae.py

+# Copyright: 2011 MoinMoin:GuidoVanRossum
+# License: GNU GPL v2 (or any later version), see LICENSE.txt for details.
+
+"""
+MoinMoin - Google App Engine store
+
+Store into Google App Engine datastore (using NDB), one entity per k/v pair.
+"""
+
+from __future__ import absolute_import, division
+
+import cStringIO as StringIO
+import logging
+
+from google.appengine.ext import ndb
+
+from . import MutableStoreBase, BytesMutableStoreBase, FileMutableStoreBase
+
+
+class _MoinDirectory(ndb.Model):
+    """Used as a parent key."""
+
+
+class _MoinValue(ndb.Model):
+    """Used to store a value.
+
+    The parent is a _MoinDirectory key (but no _MoinDirectory
+    object exists).
+    """
+
+    value = ndb.BlobProperty()
+
+
+class _Store(MutableStoreBase):
+    """Keys and uris are required to be valid key names.
+
+    (This is not an onerous requirement.)
+    """
+
+    @classmethod
+    def from_uri(cls, uri):
+        return cls(uri)
+
+    def __init__(self, path):
+        logging.info('%s(%r)', self.__class__.__name__, path)
+        self._root_key = ndb.Key(_MoinDirectory, path)
+        self._query = _MoinValue.query(ancestor=self._root_key)
+
+    def create(self):
+        """Nothing to do."""
+
+    def destroy(self):
+        self._query.map(self._destroy_key, keys_only=True)
+
+    def _destroy_key(self, key):
+        return key.delete()
+
+    def __iter__(self):
+        return self._query.iter(keys_only=True)
+
+    def _getitem(self, key):
+        ent = _MoinValue.get_by_id(key, parent=self._root_key)
+        return ent and ent.value or 'null'
+
+    def _setitem(self, key, value):
+        _MoinValue(value=value, id=key, parent=self._root_key).put()
+
+    def __delitem__(self, key):
+        ndb.Key(_MoinValue, key, parent=self._root_key).delete()
+
+
+class BytesStore(_Store, BytesMutableStoreBase):
+
+    def __getitem__(self, key):
+        return self._getitem(key)
+    
+    def __setitem__(self, key, value):
+        self._setitem(key, value)
+
+
+class FileStore(_Store, FileMutableStoreBase):
+
+    def __getitem__(self, key):
+        return StringIO.StringIO(self._getitem(key))
+
+    def __setitem__(self, key, stream):
+        return self._setitem(key, stream.read())
+application: moin2-test
+version: dev
+runtime: python27
+api_version: 1
+threadsafe: true
+
+handlers:
+- url: /.*
+  script: appengine_main.application
+
+libraries:
+- name: jinja2
+  version: latest
+

appengine_main.py

+"""Main entry point for Google App Engine."""
+
+# Python imports.
+import os
+import sys
+
+# Configuration constants.
+wiki_config = 'wikiconfig.py'
+
+# Tweak sys.path.
+support_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'support'))
+if support_path not in sys.path:
+    sys.path.insert(0, support_path)
+
+# Now we can import MoinMoin.
+from MoinMoin.app import create_app
+
+# Hack: If there are no DatastoreFile instances assume we must create the index.
+from whoosh.filedb.gae import DatastoreFile
+create_index = DatastoreFile.all().get() is None
+
+# Create the WSGI application object.
+application = create_app(os.path.abspath(wiki_config), create_index)
     # If that's not true, feel free to adjust the pathes.
     instance_dir = os.path.join(wikiconfig_dir, 'wiki')
     data_dir = os.path.join(instance_dir, 'data') # Note: this used to have a trailing / in the past
-    index_storage = 'FileStorage', (os.path.join(instance_dir, "index"), ), {}
+    index_storage = 'DatastoreStorage', (), {}
 
     # This provides a simple default setup for your backend configuration.
     # 'stores:fs:...' indicates that you want to use the filesystem backend.
     namespace_mapping, acl_mapping = create_simple_mapping(
-                            uri='stores:fs:{0}/%(nsname)s/%(kind)s'.format(data_dir),
+                            uri='stores:gae:/%(nsname)s/%(kind)s',
                             # XXX we use rather relaxed ACLs for the development wiki:
                             content_acl=dict(before=u'',
                                              default=u'All:read,write,create,destroy,admin',