Commits

Anonymous committed d5fc413 Merge

[merge] from default

Comments (0)

Files changed (68)

 sstore/*
 data/*
 pylons_data/*
+ckan/i18n/*/*/*.mo
 
 # other
 pip-requirements-local.txt
+include ckan/config/deployment.ini_tmpl
 recursive-include ckan/public *
 recursive-include ckan/templates *
 include CHANGELOG.txt
 Developer Installation
 ======================
 
+These are quick instructions to get developing. For fuller instructions see :doc:`deployment`.
+
 1. Get the code and install it:
 
    We recommend installing using pip and virtualenv::
 
    NB: you'll need to setup a database -- see sqlalchemy.url config option.
    We support only PostgreSQL at this time. You'll need to install the relevant
-   python library (eg. On debiani/ubuntu: python-psycopg2)
+   python library (eg. On debian/ubuntu: python-psycopg2)
 
    NB: You may also need to create the Pylon's cache directory specified by
    cache_dir in the config file.
 
       paster serve {your-config.ini} 
 
-6. Point your browser at: localhost:5000 (if you set a different port in your
-   config file then youl will need to change 5000 to whatever port value you
-   chose).
+6. Point your browser at: http://localhost:5000/ (or a different port, depending
+   on the one given in your config file)
 
 
 Test
 
     @classmethod
     def is_authorized(cls, username, action, domain_object):
+        '''Authorize `action` by `username` on `domain_object`.
+        
+        :param username: a user identifier (may be e.g. an IP address).
+        :param action: a ckan.model.authz.Action enumeration.
+        :param domain_object: the domain object instance (or class/type in the
+            case of e.g. 'create' action).
+
+        :returns: True or False
+        '''
         if isinstance(username, str):
             username = username.decode('utf8')
         assert isinstance(username, unicode), type(username)

ckan/config/deployment.ini_tmpl

 # Guide location
 #guide_url = http://wiki.okfn.org/ckan/doc/
 
-## extra places to look for templates (comma separated list)
-## any templates found will override correspondingly named templates in in
-## ckan/templates/ (e.g. to override main layout template provide a new one
-## called layout.html)
-# extra_template_paths = %(here)s/my-extra-templates
+## extra places to look for templates and public files (comma separated lists)
+## any templates/files found will override correspondingly named ones in
+## ckan/templates/ and ckan/public
+## (e.g. to override main layout template layout.html or add in css/extra.css)
+# extra_template_paths = %(here)s/my-templates
+# extra_public_paths = %(here)s/my-public
+
+# Package form integration
+#package_edit_return_url = http://another.frontend/package/<NAME>
+#package_new_return_url = http://another.frontend/package/<NAME>
+
 
 # Logging configuration
 [loggers]

ckan/config/environment.py

 
 from sqlalchemy import engine_from_config
 from pylons import config
+from pylons.i18n.translation import ugettext
 from genshi.template import TemplateLoader
+from genshi.filters.i18n import Translator
 
 import ckan.lib.app_globals as app_globals
 import ckan.lib.helpers
 from ckan.config.routing import make_map
 from ckan import model
 
-from genshi.filters.i18n import Translator
-from pylons.i18n.translation import ugettext
 
 def load_environment(global_conf, app_conf):
     """Configure the Pylons environment via the ``pylons.config``
                  templates=[os.path.join(root, 'templates')])
 
     # Initialize config with the basic options
-    config.init_app(global_conf, app_conf, package='ckan',
-                    template_engine='genshi', paths=paths)
+    config.init_app(global_conf, app_conf, package='ckan', paths=paths)
 
     config['routes.map'] = make_map()
     config['pylons.app_globals'] = app_globals.Globals()
     # tmpl_options["genshi.loader_callback"] = template_loaded
     config['pylons.app_globals'].genshi_loader = TemplateLoader(
         template_paths, auto_reload=True, callback=template_loaded)
-    # HACK! For some reason callback=template_loaded in previous line does
-    # *not* work (this required 1h to track down!!)
-    # This does work ...
-    tmpl_options = config['buffet.template_options']
-    tmpl_options["genshi.loader_callback"] = template_loaded
 
     # CONFIGURATION OPTIONS HERE (note: all config options will override
     # any Pylons config options)

ckan/config/middleware.py

 """Pylons middleware initialization"""
+from beaker.middleware import CacheMiddleware, SessionMiddleware
 from paste.cascade import Cascade
 from paste.registry import RegistryManager
 from paste.urlparser import StaticURLParser
 from paste.deploy.converters import asbool
-
 from pylons import config
-from pylons.error import error_template
-from pylons.middleware import ErrorHandler, StaticJavascripts, StatusCodeRedirect
+from pylons.middleware import ErrorHandler, StatusCodeRedirect
 from pylons.wsgiapp import PylonsApp
+from routes.middleware import RoutesMiddleware
 
 from ckan.config.environment import load_environment
 
-def make_app(global_conf, full_stack=True, **app_conf):
+def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
     """Create a Pylons WSGI application and return it
 
     ``global_conf``
         the [DEFAULT] section of the Paste ini file.
 
     ``full_stack``
-        Whether or not this application provides a full WSGI stack (by
-        default, meaning it handles its own exceptions and errors).
-        Disable full_stack when this application is "managed" by
-        another WSGI middleware.
+        Whether this application provides a full WSGI stack (by default,
+        meaning it handles its own exceptions and errors). Disable
+        full_stack when this application is "managed" by another WSGI
+        middleware.
+
+    ``static_files``
+        Whether this application serves its own static files; disable
+        when another web server is responsible for serving them.
 
     ``app_conf``
-        The application's local configuration. Normally specified in the
-        [app:<name>] section of the Paste ini file (where <name>
+        The application's local configuration. Normally specified in
+        the [app:<name>] section of the Paste ini file (where <name>
         defaults to main).
+
     """
     # Configure the Pylons environment
     load_environment(global_conf, app_conf)
     # The Pylons WSGI app
     app = PylonsApp()
 
+    # Routing/Session/Cache Middleware
+    app = RoutesMiddleware(app, config['routes.map'])
+    app = SessionMiddleware(app, config)
+    app = CacheMiddleware(app, config)
+
     # CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares)
 
-    import pylons
-    if pylons.__version__ >= "0.9.7":
-        # Routing/Session/Cache Middleware
-        from beaker.middleware import CacheMiddleware, SessionMiddleware
-        from routes.middleware import RoutesMiddleware
-        app = RoutesMiddleware(app, config['routes.map'])
-        app = SessionMiddleware(app, config)
-        app = CacheMiddleware(app, config)
-
-    from repoze.who.config import make_middleware_with_config
-    app = make_middleware_with_config(app, global_conf,
-            app_conf['who.config_file'], app_conf['who.log_file'],
-            app_conf['who.log_level'])
-
     if asbool(full_stack):
         # Handle Python exceptions
-        app = ErrorHandler(app, global_conf,**config['pylons.errorware'])
+        app = ErrorHandler(app, global_conf, **config['pylons.errorware'])
 
         # Display error documents for 401, 403, 404 status codes (and
         # 500 when debug is disabled)
         if asbool(config['debug']):
             app = StatusCodeRedirect(app)
         else:
-            app = StatusCodeRedirect(app, [401, 403, 404, 500])
+            app = StatusCodeRedirect(app, [400, 401, 403, 404, 500])
+    
+    from repoze.who.config import make_middleware_with_config
+    app = make_middleware_with_config(app, global_conf,
+        app_conf['who.config_file'], app_conf['who.log_file'],
+        app_conf['who.log_level'])
+
 
     # Establish the Registry for this application
     app = RegistryManager(app)
 
-    # Static files
-    static_app = StaticURLParser(config['pylons.paths']['static_files'])
-    app = Cascade([static_app, app])
+    if asbool(static_files):
+        # Serve static files
+        static_app = StaticURLParser(config['pylons.paths']['static_files'])
+        static_parsers = [static_app, app]
+
+        # Configurable extra static file paths
+        extra_public_paths = app_conf.get('extra_public_paths')
+        if extra_public_paths:
+            static_parsers = [StaticURLParser(public_path) \
+                              for public_path in \
+                              extra_public_paths.split(',')] + static_parsers
+            
+        app = Cascade(static_parsers)
+
     return app

ckan/config/routing.py

     """Create, configure and return the routes Mapper"""
     map = Mapper(directory=config['pylons.paths']['controllers'],
                  always_scan=config['debug'])
+    map.minimization = False
 
     # The ErrorController route (handles 404/500 error pages); it should
     # likely stay at the top, ensuring it can always be resolved
-    map.connect('/error/:action/:id', controller='error')
+    map.connect('/error/{action}', controller='error')
+    map.connect('/error/{action}/{id}', controller='error')
 
     # CUSTOM ROUTES HERE
     map.connect('home', '/', controller='home', action='index')
     map.connect('about', '/about', controller='home', action='about')
     map.connect('stats', '/stats', controller='home', action='stats')
     maps.admin_map(map, controller='admin', url='/admin')
+    # CKAN API.
     map.connect('/api/search/:register', controller='rest', action='search')
     map.connect('/api/tag_counts', controller='rest', action='tag_counts')
     map.connect('/api', controller='rest', action='index')
     map.connect('/api/rest/:register/:id/:subregister/:id2',
         controller='rest', action='delete',
         conditions=dict(method=['DELETE']))
+    # CKAN API v1.
+    map.connect('/api/1/search/:register', controller='rest', action='search')
+    map.connect('/api/1/tag_counts', controller='rest', action='tag_counts')
+    map.connect('/api/1', controller='rest', action='index')
+    map.connect('/api/1/rest', controller='rest', action='index')
+    map.connect('/api/1/rest/:register', controller='rest', action='list',
+        conditions=dict(method=['GET']))
+    map.connect('/api/1/rest/:register', controller='rest', action='create',
+        conditions=dict(method=['POST']))
+    map.connect('/api/1/rest/:register/:id', controller='rest', action='show',
+        conditions=dict(method=['GET']))
+    map.connect('/api/1/rest/:register/:id', controller='rest', action='update',
+        conditions=dict(method=['PUT']))
+    map.connect('/api/1/rest/:register/:id', controller='rest', action='update',
+        conditions=dict(method=['POST']))
+    map.connect('/api/1/rest/:register/:id', controller='rest', action='delete',
+        conditions=dict(method=['DELETE']))
+    map.connect('/api/1/rest/:register/:id/:subregister',
+        controller='rest', action='list',
+        conditions=dict(method=['GET']))
+    map.connect('/api/1/rest/:register/:id/:subregister/:id2',
+        controller='rest', action='create',
+        conditions=dict(method=['POST']))
+    map.connect('/api/1/rest/:register/:id/:subregister/:id2',
+        controller='rest', action='show',
+        conditions=dict(method=['GET']))
+    map.connect('/api/1/rest/:register/:id/:subregister/:id2',
+        controller='rest', action='update',
+        conditions=dict(method=['PUT']))
+    map.connect('/api/1/rest/:register/:id/:subregister/:id2',
+        controller='rest', action='delete',
+        conditions=dict(method=['DELETE']))
+
+    # CKAN API v2.
+    map.connect('/api/2/search/:register', controller='rest2', action='search')
+    map.connect('/api/2/tag_counts', controller='rest2', action='tag_counts')
+    map.connect('/api/2', controller='rest2', action='index')
+    map.connect('/api/2/rest', controller='rest2', action='index')
+    map.connect('/api/2/rest/:register', controller='rest2', action='list',
+        conditions=dict(method=['GET']))
+    map.connect('/api/2/rest/:register', controller='rest2', action='create',
+        conditions=dict(method=['POST']))
+    map.connect('/api/2/rest/:register/:id', controller='rest2', action='show',
+        conditions=dict(method=['GET']))
+    map.connect('/api/2/rest/:register/:id', controller='rest2', action='update',
+        conditions=dict(method=['PUT']))
+    map.connect('/api/2/rest/:register/:id', controller='rest2', action='update',
+        conditions=dict(method=['POST']))
+    map.connect('/api/2/rest/:register/:id', controller='rest2', action='delete',
+        conditions=dict(method=['DELETE']))
+    map.connect('/api/2/rest/:register/:id/:subregister',
+        controller='rest2', action='list',
+        conditions=dict(method=['GET']))
+    map.connect('/api/2/rest/:register/:id/:subregister/:id2',
+        controller='rest2', action='create',
+        conditions=dict(method=['POST']))
+    map.connect('/api/2/rest/:register/:id/:subregister/:id2',
+        controller='rest2', action='show',
+        conditions=dict(method=['GET']))
+    map.connect('/api/2/rest/:register/:id/:subregister/:id2',
+        controller='rest2', action='update',
+        conditions=dict(method=['PUT']))
+    map.connect('/api/2/rest/:register/:id/:subregister/:id2',
+        controller='rest2', action='delete',
+        conditions=dict(method=['DELETE']))
 
     map.redirect("/packages", "/package")
     map.redirect("/packages/{url:.*}", "/package/{url}")
     map.connect('/user/logout', controller='user', action='logout')
     map.connect('/user/apikey', controller='user', action='apikey')
     map.connect('/user/:id', controller='user', action='read')
-    map.connect('/:controller/:action/:id')
-    map.connect('/:controller/', action='index')
-    map.connect('/:controller/:action/', id=None)
+    map.connect('/{controller}', action='index')
+    map.connect('/:controller/{action}')
+    map.connect('/{controller}/{action}/{id}')
+    map.redirect('/*(url)/', '/{url}',
+                 _redirect_code='301 Moved Permanently')
     map.connect('/*url', controller='template', action='view')
 
     return map

ckan/controllers/error.py

 import cgi
-import os.path
 
-import paste.fileapp
-from pylons.middleware import error_document_template, media_path
+from paste.urlparser import PkgResourcesParser
+from pylons import request
+from pylons.controllers.util import forward
+from pylons.middleware import error_document_template
+from webhelpers.html.builder import literal
 
-from ckan.lib.base import *
+from ckan.lib.base import BaseController
+from ckan.lib.base import render
 
 class ErrorController(BaseController):
+
     """Generates error documents as and when they are required.
 
     The ErrorDocuments middleware forwards to ErrorController when error
 
     This behaviour can be altered by changing the parameters to the
     ErrorDocuments middleware in your config/middleware.py file.
+
     """
 
     def document(self):
         """Render the error document"""
-        ckan_template = render("error_document_template")
-        ckan_template = ckan_template.decode('utf8')
-        response = request.environ.get('pylons.original_response')
+        ckan_template = render('error_document_template.html')
+        # ckan_template = ckan_template.decode('utf8')
+        resp = request.environ.get('pylons.original_response')
+        content = literal(resp.body) or cgi.escape(request.GET.get('message', ''))
+        # page = error_document_template % \
         page = ckan_template % \
             dict(prefix=request.environ.get('SCRIPT_NAME', ''),
-                 code=cgi.escape(request.GET.get('code', str(response.status_int))),
-                 message=h.literal(response.body) or cgi.escape(request.GET.get('message', '')))
+                 code=cgi.escape(request.GET.get('code', str(resp.status_int))),
+                 message=content)
         return page
 
     def img(self, id):
         """Serve Pylons' stock images"""
-        return self._serve_file(os.path.join(media_path, 'img', id))
+        return self._serve_file('/'.join(['media/img', id]))
 
     def style(self, id):
         """Serve Pylons' stock stylesheets"""
-        return self._serve_file(os.path.join(media_path, 'style', id))
+        return self._serve_file('/'.join(['media/style', id]))
 
     def _serve_file(self, path):
         """Call Paste's FileApp (a WSGI application) to serve the file
         at the specified path
         """
-        fapp = paste.fileapp.FileApp(path)
-        return fapp(request.environ, self.start_response)
+        request.environ['PATH_INFO'] = '/%s' % path
+        return forward(PkgResourcesParser('pylons', 'pylons'))

ckan/controllers/group.py

 import genshi
 
 from ckan.lib.base import *
-from simplejson import dumps
 import ckan.authz as authz
 import ckan.forms
 from ckan.lib.helpers import Page
             page=request.params.get('page', 1),
             items_per_page=20
         )
-        return render('group/index')
+        return render('group/index.html')
 
     def read(self, id):
         c.group = model.Group.by_name(id)
             page=request.params.get('page', 1),
             items_per_page=50
         )
-        return render('group/read')
+        return render('group/read.html')
 
     def new(self):
         record = model.Group
             except ValidationException, error:
                 fs = error.args[0]
                 c.form = self._render_edit_form(fs)
-                return render('group/edit')
+                return render('group/edit.html')
             # do not use groupname from id as may have changed
             c.groupname = c.fs.name.value
             group = model.Group.by_name(c.groupname)
             data = ckan.forms.edit_group_dict(ckan.forms.get_group_dict(), request.params)
             fs = fs.bind(data=data, session=model.Session)
         c.form = self._render_edit_form(fs)
-        return render('group/new')
+        return render('group/new.html')
 
     def edit(self, id=None): # allow id=None to allow posting
         c.error = ''
             
             fs = ckan.forms.get_group_fieldset('group_fs').bind(c.group)
             c.form = self._render_edit_form(fs)
-            return render('group/edit')
+            return render('group/edit.html')
         else:
             # id is the name (pre-edited state)
             c.groupname = id
             except ValidationException, error:
                 fs = error.args[0]
                 c.form = self._render_edit_form(fs)
-                return render('group/edit')
+                return render('group/edit.html')
             pkgs = [model.Package.by_name(name) for name in request.params.getall('Group-packages-current')]
             group.packages = pkgs
             pkgids = request.params.getall('PackageGroup--package_id')
             except ValidationException, error:
                 # TODO: sort this out 
                 # fs = error.args[0]
-                # return render('group/authz')
+                # return render('group/authz.html')
                 raise
             # now do new roles
             newrole_user_id = request.params.get('GroupRole--user_id')
         fs = ckan.forms.get_authz_fieldset('group_authz_fs').bind(group.roles)
         c.form = fs.render()
         c.new_roles_form = ckan.forms.get_authz_fieldset('new_group_roles_fs').render()
-        return render('group/authz')
+        return render('group/authz.html')
 
     def _render_edit_form(self, fs):
         # errors arrive in c.error and fs.errors
         c.fieldset = fs
         c.fieldset2 = ckan.forms.get_group_fieldset('new_package_group_fs')
-        return render('group/edit_form')
+        return render('group/edit_form.html')
 
     def _update(self, fs, group_name, group_id):
         '''

ckan/controllers/home.py

         # 3600=hourly, 86400=daily
         c.tag_counts = mycache.get_value(key='tag_counts_home_page',
                 createfunc=tag_counts, expiretime=86400)
-        return render('home/index')
+        return render('home/index.html')
 
     def license(self):
-        return render('home/license')
+        return render('home/license.html')
 
     def about(self):
-        return render('home/about')
+        return render('home/about.html')
 
     def stats(self):
         def stats_html():
             c.top_package_owners = stats.top_package_owners()
             c.new_packages_by_week = rev_stats.get_by_week('new_packages')
             c.package_revisions_by_week = rev_stats.get_by_week('package_revisions')
-            return render('home/stats')
+            return render('home/stats.html')
         if not c.user:
             mycache = cache.get_cache('stats', type='dbm')
             # 3600=hourly, 86400=daily

ckan/controllers/importer.py

     authorizer = ckan.authz.Authorizer()
 
     def index(self):
-        return render('importer/importer')
+        return render('importer/importer.html')
 
     def preview(self):
         if not c.user:
         params = dict(request.params)
         if not params.has_key('file'):
             c.error = _('Need to specify a filename.')
-            return render('importer/importer')                
+            return render('importer/importer.html')                
         if not hasattr(params['file'], 'value'):
             c.error = _('Did not receive file successfully.')
-            return render('importer/importer')
+            return render('importer/importer.html')
         file_buf = params['file'].value
         # save as temp file for when you do import
         self._save_tempfile(file_buf)
         if not file_buf:
             c.error = _('File \'%s\' not found.') % params['file'].filename
-            return render('importer/importer')
+            return render('importer/importer.html')
         try:
             importer = importer.PackageImporter(buf=file_buf)
         except importer.ImportException, e:
             c.error = _('Error importing file \'%s\' as Excel or CSV format: %s') % (params['file'].filename, e)
-            return render('importer/importer')
+            return render('importer/importer.html')
         c.import_filename = params['file'].filename.lstrip(os.sep)
         if params.has_key('log_message'):
             c.log_message = params['log_message']
             c.fs_list.append(fs)
         c.errors = len(all_errors)
         c.num_pkgs = len(c.fs_list)
-        return render('importer/preview')
+        return render('importer/preview.html')
 
     def do_import(self):
         import ckan.lib.importer as importer
             importer = importer.PackageImporter(buf=file_buf)
         except importer.ImportException, e:
             c.error = _('Error importing file \'%s\' as Excel or CSV format: %s') % (params['file'].filename, e)
-            return render('importer/importer')
+            return render('importer/importer.html')
         if 'log_message' in request.params:
             log_message = request.params.getone('log_message')
         else:
 
         model.Session.commit()
         c.message = ungettext('Imported %i package.', 'Imported %i packages.', count) % count
-        return render('importer/result')
+        return render('importer/result.html')
 
     def _get_fs(self, importer):
         for index, pkg_dict in enumerate(importer.pkg_dict()):
     def package_render(self, fs, errors, warnings):
         try:
             PackageSaver().render_preview(fs, None, None) # create a new package for now
-            preview = h.literal(render('package/read_core'))
+            preview = h.literal(render('package/read_core.html'))
         except ValidationException, error:
             c.error, fs = error.args
             preview = h.literal('<li>Errors: %s</li>\n') % c.error

ckan/controllers/package.py

 import logging
 import urlparse
 
-import simplejson
 import genshi
 from pylons import config
 
 from ckan.lib.base import *
-from ckan.lib.search import Search, SearchOptions
+from ckan.lib.search import make_search, SearchOptions
 from ckan.lib.package_saver import PackageSaver, ValidationException
 import ckan.forms
 import ckan.authz
 import ckan.rating
 import ckan.misc
-from ckan.lib.helpers import Page
 from pylons.i18n import get_lang
 
 logger = logging.getLogger('ckan.controllers')
     def index(self):
         query = ckan.authz.Authorizer().authorized_query(c.user, model.Package)
         c.package_count = query.count()
-        return render('package/index')
+        return render('package/index.html')
 
     def list(self):
         query = ckan.authz.Authorizer().authorized_query(c.user, model.Package)
-        c.page = Page(
+        c.page = h.Page(
             collection=query,
             page=request.params.get('page', 1),
             items_per_page=50
         )
-        return render('package/list')
+        return render('package/list.html')
 
     def search(self):        
         c.q = request.params.get('q') # unicode format (decoded from utf8)
                 'filter_by_downloadable': c.downloadable_only,
                 })
             # package search
-            query = Search().query(options, username=c.user)
-            c.page = Page(
+            query = make_search().query(options, username=c.user)
+            c.page = h.Page(
                 collection=query,
                 page=request.params.get('page', 1),
                 items_per_page=50
                 'return_objects': True,
                 'limit': c.tag_limit,
                 })
-            results = Search().run(options)
+            results = make_search().run(options)
             c.tags = results['results']
             c.tags_count = results['count']
 
-        return render('package/search')
+        return render('package/search.html')
 
     def read(self, id):
         pkg = model.Package.by_name(id)
         c.auth_for_change_state = self.authorizer.am_authorized(c, model.Action.CHANGE_STATE, pkg)
 
         PackageSaver().render_package(pkg)
-        return render('package/read') 
+        return render('package/read.html') 
 
     def history(self, id):
         if 'diff' in request.params or 'selected1' in request.params:
             feed.content_type = 'application/atom+xml'
             return feed.writeString('utf-8')
         c.pkg_revisions = c.pkg.all_related_revisions
-        return render('package/history')
+        return render('package/history.html')
 
     def new(self):
         c.error = ''
                 model.setup_default_user_roles(pkg, admins)
                 model.repo.commit_and_remove()
 
-                h.redirect_to(action='read', id=pkgname)
+                self._form_commit_redirect(pkgname, 'new')
             except ValidationException, error:
                 fs = error.args[0]
                 c.form = self._render_edit_form(fs, request.params,
                         clear_session=True)
-                return render('package/new')
+                return render('package/new.html')
             except KeyError, error:
                 abort(400, ('Missing parameter: %s' % error.args).encode('utf8'))
 
                 PackageSaver().render_preview(fs, id, record.id,
                                               log_message=log_message,
                                               author=c.author)
-                c.preview = h.literal(render('package/read_core'))
+                c.preview = h.literal(render('package/read_core.html'))
             except ValidationException, error:
                 fs = error.args[0]
                 c.form = self._render_edit_form(fs, request.params,
                         clear_session=True)
-                return render('package/new')
-        return render('package/new')
+                return render('package/new.html')
+        return render('package/new.html')
 
     def edit(self, id=None): # allow id=None to allow posting
         # TODO: refactor to avoid duplication between here and new
                 self._adjust_license_id_options(pkg, fs)
             fs = fs.bind(pkg)
             c.form = self._render_edit_form(fs, request.params)
-            return render('package/edit')
+            return render('package/edit.html')
         elif request.params.has_key('commit'):
             # id is the name (pre-edited state)
             pkgname = id
                 PackageSaver().commit_pkg(fs, id, pkg.id, log_message, c.author)
                 # do not use pkgname from id as may have changed
                 pkgname = fs.name.value
-                h.redirect_to(action='read', id=pkgname)
+                self._form_commit_redirect(pkgname, 'edit')
             except ValidationException, error:
                 fs = error.args[0]
                 c.form = self._render_edit_form(fs, request.params,
                         clear_session=True)
-                return render('package/edit')
+                return render('package/edit.html')
             except KeyError, error:
                 abort(400, 'Missing parameter: %s' % error.args)
         else: # Must be preview
                 PackageSaver().render_preview(fs, id, pkg.id,
                                               log_message=log_message,
                                               author=c.author)
-                read_core_html = render('package/read_core') #utf8 format
+                read_core_html = render('package/read_core.html') #utf8 format
                 c.preview = h.literal(read_core_html)
                 c.form = self._render_edit_form(fs, request.params)
             except ValidationException, error:
                 fs = error.args[0]
                 c.form = self._render_edit_form(fs, request.params,
                         clear_session=True)
-                return render('package/edit')
-            return render('package/edit') # uses c.form and c.preview
+                return render('package/edit.html')
+            return render('package/edit.html') # uses c.form and c.preview
 
+    def _form_commit_redirect(self, pkgname, action):
+        '''This redirects the user to the CKAN package/read page,
+        unless there is request parameter giving an alternate location,
+        perhaps an external website.
+        @param pkgname - Name of the package just edited
+        @param action - What the action of the edit was
+        '''
+        assert action in ('new', 'edit')
+        url = request.params.get('return_to') or \
+              config.get('package_%s_return_url' % action)
+        if url:
+            url = url.replace('<NAME>', pkgname)
+        else:
+            url = h.url_for(action='read', id=pkgname)
+        redirect(url)        
+        
     def _adjust_license_id_options(self, pkg, fs):
         options = fs.license_id.render_opts['options']
         is_included = False
             except ValidationException, error:
                 # TODO: sort this out 
                 # fs = error.args
-                # return render('package/authz')
+                # return render('package/authz.html')
                 raise
             # now do new roles
             newrole_user_id = request.params.get('PackageRole--user_id')
         fs = ckan.forms.get_authz_fieldset('package_authz_fs').bind(c.pkg.roles)
         c.form = fs.render()
         c.new_roles_form = ckan.forms.get_authz_fieldset('new_package_roles_fs').render()
-        return render('package/authz')
+        return render('package/authz.html')
 
     def rate(self, id):
         package_name = id
             model.Session.clear()
         edit_form_html = fs.render()
         c.form = h.literal(edit_form_html)
-        return h.literal(render('package/edit_form'))
+        return h.literal(render('package/edit_form.html'))
 
     def _update_authz(self, fs):
         validation = fs.validate()

ckan/controllers/rest.py

 import sqlalchemy.orm
-import simplejson
 
 from ckan.lib.base import *
+from ckan.lib.helpers import json
 import ckan.model as model
 import ckan.forms
-from ckan.lib.search import Search, SearchOptions
+from ckan.lib.search import make_search, SearchOptions
 import ckan.authz
 import ckan.rating
 
-class RestController(BaseController):
+class BaseRestController(BaseController):
+
+    ref_package_with_attr = 'id'
+
+    def _list_package_refs(self, packages):
+        return [getattr(p, self.ref_package_with_attr) for p in packages]
+
+    def _get_pkg(self, id):
+        pkg = model.Session.query(model.Package).get(id)
+        if pkg == None:
+            pkg = model.Package.by_name(id)
+            # Todo: Make sure package names can't be changed to look like package IDs?
+        return pkg
 
     def index(self):
-        return render('rest/index')
+        return render('rest/index.html')
 
     def list(self, register, subregister=None, id=None):
         if register == 'revision':
             return self._finish_ok([rev.id for rev in revs])
         elif register == u'package' and not subregister:
             query = ckan.authz.Authorizer().authorized_query(self._get_username(), model.Package)
-            packages = query.all() 
-            results = [package.name for package in packages]
+            packages = query.all()
+            results = self._list_package_refs(packages)
             return self._finish_ok(results)
         elif register == u'package' and subregister == 'relationships':
             #TODO authz stuff for this and related packages
-            pkg = model.Package.by_name(id)
+            pkg = self._get_pkg(id)
             if not pkg:
                 response.status_int = 404
                 return 'First package named in request was not found.'
                 'timestamp': model.strftimestamp(rev.timestamp),
                 'author': rev.author,
                 'message': rev.message,
-                'packages': [p.name for p in rev.packages],
+                'packages': self._list_package_refs(rev.packages),
             }
             return self._finish_ok(response_data)
         elif register == u'changeset':
             _dict = changeset.as_dict()
             return self._finish_ok(_dict)
         elif register == u'package' and not subregister:
-            pkg = model.Package.by_name(id)
-            if pkg is None:
+            pkg = self._get_pkg(id)
+            if pkg == None:
                 response.status_int = 404
                 return ''
-
             if not self._check_access(pkg, model.Action.READ):
                 return ''
             _dict = pkg.as_dict()
             return self._finish_ok(_dict)
         elif register == u'package' and (subregister == 'relationships' or subregister in model.PackageRelationship.get_all_types()):
-            pkg1 = model.Package.by_name(id)
-            pkg2 = model.Package.by_name(id2)
+            pkg1 = self._get_pkg(id)
+            pkg2 = self._get_pkg(id2)
             if not pkg1:
                 response.status_int = 404
                 return 'First package named in address was not found.'
     def create(self, register, id=None, subregister=None, id2=None):
         # Check an API key given
         if not self._check_access(None, None):
-            return simplejson.dumps(_('Access denied'))
+            return json.dumps(_('Access denied'))
         try:
             request_data = self._get_request_data()
         except ValueError, inst:
                 request_fa_dict = ckan.forms.edit_package_dict(ckan.forms.get_package_dict(fs=fs), request_data)
                 fs = fs.bind(model.Package, data=request_fa_dict, session=model.Session)
             elif register == 'package' and subregister in model.PackageRelationship.get_all_types():
-                pkg1 = model.Package.by_name(id)
-                pkg2 = model.Package.by_name(id2)
+                pkg1 = self._get_pkg(id)
+                pkg2 = self._get_pkg(id2)
                 if not pkg1:
                     response.status_int = 404
                     return 'First package named in address was not found.'
             validation = fs.validate()
             if not validation:
                 response.status_int = 409
-                return simplejson.dumps(repr(fs.errors))
+                return json.dumps(repr(fs.errors))
             rev = model.repo.new_revision()
             rev.author = self.rest_api_user
             rev.message = _(u'REST API: Create object %s') % str(fs.name.value)
             model.Session.rollback()
             raise
         obj = fs.model
+        location = "%s/%s" % (request.path, obj.id)
+        response.headers['Location'] = location
         return self._finish_ok(obj.as_dict())
             
     def update(self, register, id, subregister=None, id2=None):
+        # must be logged in to start with
+        if not self._check_access(None, None):
+            return json.dumps(_('Access denied'))
+
         if register == 'package' and not subregister:
-            entity = model.Package.by_name(id)
+            entity = self._get_pkg(id)
+            if entity == None:
+                response.status_int = 404
+                return 'Package was not found.'
         elif register == 'package' and subregister in model.PackageRelationship.get_all_types():
-            pkg1 = model.Package.by_name(id)
-            pkg2 = model.Package.by_name(id2)
+            pkg1 = self._get_pkg(id)
+            pkg2 = self._get_pkg(id2)
             if not pkg1:
                 response.status_int = 404
                 return 'First package named in address was not found.'
         if (not subregister and \
             not self._check_access(entity, model.Action.EDIT)) \
             or not self._check_access(None, None):
-            return simplejson.dumps(_('Access denied'))
+            return json.dumps(_('Access denied'))
 
         try:
             request_data = self._get_request_data()
             validation = fs.validate()
             if not validation:
                 response.status_int = 409
-                return simplejson.dumps(repr(fs.errors))
+                return json.dumps(repr(fs.errors))
             try:
                 rev = model.repo.new_revision()
                 rev.author = self.rest_api_user
                 return self._update_package_relationship(entity, comment)
 
     def delete(self, register, id, subregister=None, id2=None):
+        # must be logged in to start with
+        if not self._check_access(None, None):
+            return json.dumps(_('Access denied'))
+
         if register == 'package' and not subregister:
-            entity = model.Package.by_name(id)
+            entity = self._get_pkg(id)
             if not entity:
                 response.status_int = 404
                 return 'Package was not found.'
             revisioned_details = 'Package: %s' % entity.name
         elif register == 'package' and subregister in model.PackageRelationship.get_all_types():
-            pkg1 = model.Package.by_name(id)
-            pkg2 = model.Package.by_name(id2)
+            pkg1 = self._get_pkg(id)
+            pkg2 = self._get_pkg(id2)
             if not pkg1:
                 response.status_int = 404
                 return 'First package named in address was not found.'
             return ''
 
         if not self._check_access(entity, model.Action.PURGE):
-            return simplejson.dumps(_('Access denied'))
+            return json.dumps(_('Access denied'))
 
         if revisioned_details:
             rev = model.repo.new_revision()
                 if not request.params['qjson']:
                     response.status_int = 400
                     return gettext('Blank qjson parameter')
-                params = simplejson.loads(request.params['qjson'])
+                params = json.loads(request.params['qjson'])
             elif request.params.values() and request.params.values() != [u''] and request.params.values() != [u'1']:
                 params = request.params
             else:
             options = SearchOptions(params)
             options.search_tags = False
             options.return_objects = False
+            options.ref_entity_with_attr = self.ref_package_with_attr
             username = self._get_username()
-            results = Search().run(options, username)
+            results = make_search().run(options, username)
             return self._finish_ok(results)
 
     def tag_counts(self):
                               'rating':5}
         """
         # check options
-        package_name = params.get('package')
+        package_ref = params.get('package')
         rating = params.get('rating')
         user = self.rest_api_user
         opts_err = None
-        if not package_name:
-            opts_err = gettext('You must supply a package name (parameter "package").')
+        if not package_ref:
+            opts_err = gettext('You must supply a package id or name (parameter "package").')
         elif not rating:
             opts_err = gettext('You must supply a rating (parameter "rating").')
         else:
             except ValueError:
                 opts_err = gettext('Rating must be an integer value.')
             else:
-                package = model.Package.by_name(package_name)
+                package = self._get_pkg(package_ref)
                 if rating < ckan.rating.MIN_RATING or rating > ckan.rating.MAX_RATING:
                     opts_err = gettext('Rating must be between %i and %i.') % (ckan.rating.MIN_RATING, ckan.rating.MAX_RATING)
                 elif not package:
-                    opts_err = gettext('Package with name %r does not exist.') % package_name
+                    opts_err = gettext('Package with name %r does not exist.') % package_ref
         if opts_err:
             self.log.debug(opts_err)
             response.status_int = 400
         user = model.User.by_name(self.rest_api_user)
         ckan.rating.set_rating(user, package, rating_int)
 
-        response.headers['Content-Type'] = 'application/json'
-        package = model.Package.by_name(package_name)
+        package = self._get_pkg(package_ref)
         ret_dict = {'rating average':package.get_average_rating(),
                     'rating count': len(package.ratings)}
         return self._finish_ok(ret_dict)
                 request.params.items(), str(inst)
             )
             raise ValueError, msg
-        request_data = simplejson.loads(request_data, encoding='utf8')
+        request_data = json.loads(request_data, encoding='utf8')
         if not isinstance(request_data, dict):
             raise ValueError, _("Request params must be in form of a json encoded dictionary.")
         # ensure unicode values
         for key, val in request_data.items():
-            # if val is str then assume it is ascii, since simplejson converts
+            # if val is str then assume it is ascii, since json converts
             # utf8 encoded JSON to unicode
             request_data[key] = self._make_unicode(val)
         return request_data
 
     def _finish_ok(self, response_data=None):
         response.status_int = 200
-        response.headers['Content-Type'] = 'application/json'
+        response.headers['Content-Type'] = 'application/json;charset=utf-8'
         if response_data is not None:
-            return simplejson.dumps(response_data)
+            return json.dumps(response_data)
         else:
             return ''
 
+class RestController(BaseRestController):
+    # Implements CKAN API Version 1.
+
+    ref_package_with_attr = 'name'
+
+
+

ckan/controllers/rest2.py

+import sqlalchemy.orm
+
+from ckan.lib.base import *
+from ckan.controllers.rest import BaseRestController
+from ckan.lib.helpers import json
+import ckan.model as model
+import ckan.forms
+from ckan.lib.search import make_search, SearchOptions
+import ckan.authz
+import ckan.rating
+
+class Rest2Controller(BaseRestController):
+
+    ref_package_with_attr = 'id'
+

ckan/controllers/revision.py

                 items_per_page=50
             )
             
-            return render('revision/list')
+            return render('revision/list.html')
 
     def read(self, id=None):
         if id is None:
         c.packages = [ pkg.continuity for pkg in pkgs ]
         pkgtags = model.Session.query(model.PackageTagRevision).filter_by(revision=c.revision)
         c.pkgtags = [ pkgtag.continuity for pkgtag in pkgtags ]
-        return render('revision/read')
+        return render('revision/read.html')
 
     def diff(self, id=None):
         if 'diff' not in request.params or 'oldid' not in request.params:
         c.diff = diff.items()
         c.diff.sort()
         c.pkg = pkg
-        return render('revision/diff')
+        return render('revision/diff.html')
 
     def _has_purge_permissions(self):
         authorizer = ckan.authz.Authorizer()
     def purge(self, id=None):
         if id is None:
             c.error = _('No revision id specified')
-            return render('revision/purge')
+            return render('revision/purge.html')
         if not self._has_purge_permissions():
             c.error = _('You are not authorized to perform this action')
-            return render('revision/purge')
+            return render('revision/purge.html')
         else:
             revision = model.Session.query(model.Revision).get(id)
             try:
                 # is this a security risk?
                 # probably not because only admins get to here
                 c.error = _('Purge of revision failed: %s') % inst
-            return render('revision/purge')
+            return render('revision/purge.html')
 

ckan/controllers/tag.py

 from ckan.lib.base import *
-from simplejson import dumps
+from ckan.lib.helpers import json
 
 class TagController(BaseController):
 
             items_per_page=100
         )
            
-        return render('tag/index')
+        return render('tag/index.html')
 
     def read(self, id):
         c.tag = model.Tag.by_name(id)
         if c.tag is None:
             abort(404)
-        return render('tag/read')
+        return render('tag/read.html')
 
     def autocomplete(self):
         incomplete = request.params.get('incomplete', '')
                 "Name": tagName
             }
             resultSet["ResultSet"]["Result"].append(result)
-        return dumps(resultSet)
+        return json.dumps(resultSet)
 

ckan/controllers/user.py

 from ckan.lib.base import *
 
 def login_form():
-    return render('user/login_form').replace('FORM_ACTION', '%s')
+    return render('user/login_form.html').replace('FORM_ACTION', '%s')
 
 class UserController(BaseController):
 
-    def index(self, id):
-        if not c.user or c.user != id:
+    def index(self, id=None):
+        if not c.user:
             h.redirect_to(controller='user', action='login', id=None)
         return self.read()
 
-    def read(self, id):
+    def read(self, id=None):
         if id:
             user = model.Session.query(model.User).get(id)
         else:
         c.num_edits = revisions_q.count()
         c.num_pkg_admin = model.Session.query(model.PackageRole).filter_by(user=user, role=model.Role.ADMIN).count()
         c.activity = revisions_q.limit(20).all()
-        return render('user/read')
+        return render('user/read.html')
 
     def login(self):
         if c.user:
                 model.Session.commit()
             h.redirect_to(controller='user', action=None, id=None)
         else:
-            form = render('user/openid_form')
+            form = render('user/openid_form.html')
             # /login_openid page need not exist -- request gets intercepted by openid plugin
             form = form.replace('FORM_ACTION', '/login_openid')
             return form
 
     def logout(self):
         c.user = None
-        return render('user/logout')
+        return render('user/logout.html')
 
     def apikey(self):
         # logged in
         else:
             user = model.User.by_name(c.user)
             c.api_key = user.apikey
-        return render('user/apikey')
+        return render('user/apikey.html')
 
     def edit(self):
         # logged in
                 model.Session.commit()
             h.redirect_to(controller='user', action='read', id=user.id)
             
-        return render('user/edit')
+        return render('user/edit.html')
         
     def _format_about(self, about):
         about_formatted = ckan.misc.MarkdownFormat().to_html(about)

ckan/forms/authz.py

 
 class RolesRenderer(formalchemy.fields.FieldRenderer):
     def render(self, **kwargs):
-        selected = kwargs.get('selected', None) or unicode(self._value)
+        selected = kwargs.get('selected', None) or unicode(self.value)
         options = [(role, role) for role in model.Role.get_all()]
         select = fa_h.select(self.name, selected, options, **kwargs)
         return select

ckan/forms/builder.py

 import formalchemy
-from pylons.templating import render
+from pylons.templating import render_genshi as render
 from pylons import c
 
 from ckan import model
         self.added_fields = []
         self.options = self.fs._fields # {field_name:fs.field}
         self.includes = None
-        self.set_form_template('package/form')
+        self.set_form_template('package/form.html')
 
     def add_field(self, field):
         if isinstance(field, common.ConfiguredField):

ckan/forms/common.py

 from formalchemy import helpers as fa_h
 import formalchemy
 import genshi
-from pylons.templating import render
+from pylons.templating import render_genshi as render
 from pylons import c
 from pylons.i18n import _, ungettext, N_, gettext
 
 class TextExtraRenderer(formalchemy.fields.TextFieldRenderer):
     def _get_value(self):
         extras = self.field.parent.model.extras # db
-        return self._value or extras.get(self.field.name, u'') or u''
+        return self.value or extras.get(self.field.name, u'') or u''
 
     def render(self, **kwargs):
         value = self._get_value()
     class DateRangeRenderer(formalchemy.fields.FieldRenderer):
         def _get_value(self):
             extras = self.field.parent.model.extras
-            if self._value:
-                from_form, to_form = self._value
+            if self.value:
+                from_form, to_form = self.value
             else:
                 from_ = extras.get(self.field.name + '-from') or u''
                 to = extras.get(self.field.name + '-to') or u''
     class TextRangeRenderer(formalchemy.fields.FieldRenderer):
         def _get_value(self):
             extras = self.field.parent.model.extras
-            if self._value:
-                from_form, to_form = self._value
+            if self.value:
+                from_form, to_form = self.value
             else:
                 from_ = extras.get(self.field.name + '-from') or u''
                 to = extras.get(self.field.name + '-to') or u''
 
     class ResourcesRenderer(formalchemy.fields.FieldRenderer):
         def render(self, **kwargs):
-            c.resources = self._value or []
+            c.resources = self.value or []
             # [:] does a copy, so we don't change original
             c.resources = c.resources[:]
             c.resources.extend([None])
             c.id = self.name
-            return render('package/form_resources')            
+            return render('package/form_resources.html')            
 
         def stringify_value(self, v):
             # actually returns dict here for _value
                 field_values.append({
                     'name':'%s-newfield%s' % (self.name, i)})
             c.fields = field_values
-            html = render('package/form_extra_fields')
+            html = render('package/form_extra_fields.html')
             return h.literal(html)
 
         def render_readonly(self, **kwargs):
     class SelectRenderer(formalchemy.fields.FieldRenderer):
         def _get_value(self, **kwargs):
             extras = self.field.parent.model.extras
-            return unicode(kwargs.get('selected', '') or self._value or extras.get(self.field.name, ''))
+            return unicode(kwargs.get('selected', '') or self.value or extras.get(self.field.name, ''))
 
         def render(self, options, **kwargs):
             selected = self._get_value()
     class CheckboxExtraRenderer(formalchemy.fields.CheckBoxFieldRenderer):
         def _get_value(self):
             extras = self.field.parent.model.extras
-            return bool(self._value or extras.get(self.field.name) == u'yes')
+            return bool(self.value or extras.get(self.field.name) == u'yes')
 
         def render(self, **kwargs):
             value = self._get_value()

ckan/forms/group.py

 
     for field in fs._fields.values():
         if not filter(lambda x: field.renderer.name.endswith(x), exclude):
-            if field.renderer._value:
-                indict[field.renderer.name] = field.renderer._value
+            if field.renderer.value:
+                indict[field.renderer.name] = field.renderer.value
             else:
                 indict[field.renderer.name] = u''
 

ckan/forms/package_ca.py

 #       though it is technically an agency. This is because it is SC is
 #       particularly important for this database.
 
-ministries = ['Agriculture and Agri-Food', 'Canadian Heritage', 'Citizenship and Immigration', 'Environment', 'Finance', 'Fisheries and Oceans', 'Foreign Affairs and International Trade', 'Health', 'Human Resources and Social Development', 'Indian and Northern Affairs', 'Industry', 'Intergovernmental Affairs', 'Justice', 'National Defence', 'Natural Resources', 'Public Safety', 'Public Works and Government Services', 'Statistics Canada', 'Transport', 'Veterans Affairs', 'Western Economic Diversification']
+ministries = ['Agriculture and Agri-Food', 'Canadian Heritage', 'Citizenship and Immigration', 'Environment Canada', 'Finance', 'Fisheries and Oceans', 'Foreign Affairs and International Trade', 'Health Canada', 'Human Resources and Social Development', 'Indian and Northern Affairs', 'Industry Canada', 'Intergovernmental Affairs', 'Justice (Department of)', 'National Defence', 'Natural Resources', 'Privy Council Office', 'Public Safety', 'Public Works', 'Statistics Canada', 'Transport Canada', 'Treasury Board Secretariat', 'Veterans Affairs', 'Western Economic Diversification']
 
 agencies = ['Air Transport Security Authority', 'Artists and Producers Professional Relations Tribunal', 'Atlantic Canada Opportunities Agency', 'Atlantic Pilotage Authority', 'Atomic Energy of Canada Limited', 'Auditor General', 'Bank of Canada', 'Border Services Agency', 'Business Development Bank of Canada', 'Business Service Centres', 'Canada Information Office', 'Canada Lands Company', 'Canada-Newfoundland Offshore Petroleum Board', 'Canada-Nova Scotia Offshore Petroleum Board', 'Canada Post Corporation', 'Canadian Broadcasting Corporation', 'Cape Breton Development Office', 'Centre for Occupational Health and Safety', 'Centre on Substance Abuse', 'Climate Change Secretariat', 'Commercial Corporation', 'Competition Bureau', 'Competition Tribunal', 'Copyright Board', 'Correctional Service', 'Council for the Arts', 'Courts Administration Service', 'Cultural Property Export Review Board', 'Dairy Commission', 'Defence Construction Canada', 'Defence Research and Development Canada', 'Deposit Insurance Corporation', 'Economic Development Agency for Quebec Regions', 'Elections Canada', 'Enterprise Cape Breton Corporation', 'Environment and Sustainable Development Commissioner', 'Environmental Assessment Agency', 'Environmental Protection Review Canada', 'Ethics Commissioner', 'Export Development Canada', 'Federal Bridge Corporation Limited', 'Federal Judicial Affairs Commissioner', 'Federal Science for Sustainable Development', 'Financial Consumer Agency', 'Firearms Centre', 'Food Inspection Agency', 'Freshwater Fish Marketing Corporation', 'Grain Commission', 'Human Rights Commission', 'Human Rights Tribunal', 'Industrial Relations Board', 'Institutes of Health Research', 'Intergovernmental Conference Secretariat', 'International Development Agency', 'International Trade Tribunal', 'Library and Archives Canada', 'Mortgage and Housing Corporation', 'National Capital Commission', 'National Film Board', 'National Parole Board', 'National Research Council', 'National Round Table on the Environment and the Economy', 'National Search and Rescue Secretariat', 'Natural Sciences and Engineering Research Council of Canada', 'Northern Pipeline Agency', 'Nuclear Safety Commission', 'Official Languages Commissioner', 'Pari-Mutuel Agency', 'Parks Canada', 'Pension Plan Investment Board', 'Polar Commission', 'Police College', 'Privacy Commissioner of Canada', 'Public Health Agency of Canada', 'Public Service Commission of Canada', 'Radio-Television and Telecommunications Commission', 'Revenue Agency', 'Review Tribunals Commissioner', 'Royal Canadian Mint', 'School of Public Service', 'Social Sciences and Humanities Research Council', 'Space Agency', 'Standards Council of Canada', 'Tourism Commission', 'Transportation Agency', 'Transportation Safety Board', 'Treasury Board Secretariat', 'VIA Rail Canada', 'Wheat Board']
 

ckan/forms/package_dict.py

             if blank:
                 indict[field.renderer.name] = u''
             else:
-                if field.renderer._value:
-                    indict[field.renderer.name] = field.renderer._value
+                if field.renderer.value:
+                    indict[field.renderer.name] = field.renderer.value
                 else:
                     indict[field.renderer.name] = u''
 

ckan/forms/package_gov.py

 
     class GeoCoverageRenderer(formalchemy.fields.FieldRenderer):
         def _get_value(self):
-            form_regions = self._value # params
+            form_regions = self.value # params
             if not form_regions:
                 extras = self.field.parent.model.extras # db
                 db_regions = extras.get(self.field.name, []) or []
         fs = self.field.parent
         pkg_dict = {}
         for field_name, field in fs.render_fields.items():
-            pkg_dict[field_name] = field.renderer._value
+            pkg_dict[field_name] = field.renderer.value
         tag_suggestions = schema_gov.suggest_tags(pkg_dict)
         html = literal("<div>Suggestions (preview refreshes): %s</div>") % ' '.join(tag_suggestions)
         html += common.TagField.TagEditRenderer.render(self, **kwargs)

ckan/lib/app_globals.py

 """The application's Globals object"""
-from pylons import config
 
 class Globals(object):
+
     """Globals acts as a container for objects available throughout the
     life of the application
+
     """
 
     def __init__(self):
         """One instance of Globals is created during application
-        initialization and is available during requests via the 'g'
-        variable
+        initialization and is available during requests via the
+        'app_globals' variable
+
         """
-        pass
 """The base Controller API
 
-Provides the BaseController class for subclassing, and other objects
-utilized by Controllers.
+Provides the BaseController class for subclassing.
 """
 import logging
 
 from pylons.controllers.util import abort, etag_cache, redirect_to, redirect
 from pylons.decorators import jsonify, validate
 from pylons.i18n import _, ungettext, N_, gettext
-from pylons.templating import render
+from pylons.templating import render_genshi as render
 
 import ckan
 import ckan.lib.helpers as h

ckan/lib/create_test_data.py

     max_args = 1
     min_args = 0
     author = u'tester'
+
+    pkg_names = []
+    tag_names = []
+    group_names = set()
+    user_names = []
     
     def command(self):
         self._load_config()
 
     @classmethod
     def create_arbitrary(self, package_dicts,
-                         relationships=[], extra_user_names=[]):
+                         relationships=[], extra_user_names=[],
+                         commit_changesets=False):
+        assert isinstance(relationships, (list, tuple))
+        assert isinstance(extra_user_names, (list, tuple))
         import ckan.model as model
         model.Session.remove()
-        self.pkg_names = []
-        self.tag_names = []
-        self.group_names = set()
-        self.user_names = extra_user_names
+        new_user_names = extra_user_names
+        new_group_names = set()
+        
         admins_list = [] # list of (package_name, admin_names)
         if package_dicts:
             rev = model.repo.new_revision() 
                     elif attr == 'download_url':
                         pkg.add_resource(unicode(val))
                     elif attr == 'resources':
+                        assert isinstance(val, (list, tuple))
                         for res_dict in val:
                             pkg.add_resource(
                                 url=unicode(res_dict['url']),
                             group = model.Group.by_name(group_name)
                             if not group:
                                 group = model.Group(name=group_name)
-                                self.group_names.add(group_name)
                                 model.Session.add(group)
+                                new_group_names.add(group_name)
                             pkg.groups.append(group)
                     elif attr == 'license':
                         pkg.license_id = val
                     elif attr == 'admins':
                         assert isinstance(val, list)
                         admins_list.append((item['name'], val))
-                        for user in val:
-                            if user not in self.user_names:
-                                self.user_names.append(user)
+                        for user_name in val:
+                            if user_name not in new_user_names:
+                                new_user_names.append(user_name)
                     else:
                         raise NotImplementedError(attr)
                 self.pkg_names.append(item['name'])
                 model.setup_default_user_roles(pkg)
             model.repo.commit_and_remove()
 
-        for user_name in self.user_names:
-            user = model.User(name=unicode(user_name))
-            model.Session.add(user)
+        needs_commit = False
+        for user_name in new_user_names:
+            if not model.User.by_name(unicode(user_name)):
+                user = model.User(name=unicode(user_name))
+                model.Session.add(user)
+                self.user_names.append(user_name)
+                needs_commit = True
 
         for pkg_name, admins in admins_list:
             pkg = model.Package.by_name(unicode(pkg_name))
             admins = [model.User.by_name(unicode(user_name)) for user_name in self.user_names]
             model.setup_default_user_roles(pkg, admins)
+            needs_commit = True
 
-        for group_name in self.group_names:
+        for group_name in new_group_names:
+            group = model.Group.by_name(unicode(group_name))
             model.setup_default_user_roles(group)
+            self.group_names.add(group_name)
+            needs_commit = True
+
+        if needs_commit:
+            model.repo.commit_and_remove()
+            needs_commit = False
 
         if relationships:
             rev = model.repo.new_revision() 
             for subject_name, relationship, object_name in relationships:
                 pkg(subject_name).add_relationship(
                     unicode(relationship), pkg(object_name))
+                needs_commit = True
 
             model.repo.commit_and_remove()
-    
+        
+        if commit_changesets:
+            from ckan.model.changeset import ChangesetRegister
+            changeset_ids = ChangesetRegister().commit()
+
     @classmethod
-    def create(self):
+    def create(self, commit_changesets=False):
         import ckan.model as model
         model.Session.remove()
         rev = model.repo.new_revision()
                              description=u'Roger likes these books.')
         for obj in [david, roger]:
             model.Session.add(obj)
-        self.group_names = (u'david', u'roger')
+        self.group_names.add(u'david')
+        self.group_names.add(u'roger')
         david.packages = [pkg1, pkg2]
         roger.packages = [pkg1]
         # authz
 
         model.repo.commit_and_remove()
 
+        if commit_changesets:
+            from ckan.model.changeset import ChangesetRegister
+            changeset_ids = ChangesetRegister().commit()
 
     @classmethod
     def delete(self):
+        '''Purges packages etc. that were created by this class.'''
         import ckan.model as model
         for pkg_name in self.pkg_names:
             pkg = model.Package.by_name(unicode(pkg_name))
                 tag.purge()
         revs = model.Session.query(model.Revision).filter_by(author=self.author)
         for rev in revs:
+            for pkg in rev.packages:
+                pkg.purge()
             model.Session.delete(rev)
         for group_name in self.group_names:
             group = model.Group.by_name(unicode(group_name))

ckan/lib/dumper.py

 import csv
-import simplejson
 import datetime
 from sqlalchemy import orm
 
 import ckan.model as model
 import ckan.model
+from helpers import json
 
 class SimpleDumper(object):
     '''Dumps just package data but including tags, groups, license text etc'''
         for pkg in query:
             pkg_dict = pkg.as_dict()
             pkgs.append(pkg_dict)
-        simplejson.dump(pkgs, dump_file_obj, indent=4)
+        json.dump(pkgs, dump_file_obj, indent=4)
 
 class Dumper(object):
     '''Dumps the database in same structure as it appears in the database'''
         if verbose:
             print '---------------------------------'
             print 'Dumping to %s' % dump_path
-        simplejson.dump(dump_struct, file(dump_path, 'w'), indent=4, sort_keys=True)
+        json.dump(dump_struct, file(dump_path, 'w'), indent=4, sort_keys=True)
 
     def cvt_record_to_dict(self, record, table):
         out = {}
         return out
 
     def load_json(self, dump_path, verbose=False):
-        dump_struct = simplejson.load(open(dump_path))
+        dump_struct = json.load(open(dump_path))
 
         if verbose:
             print 'Building table...'

ckan/lib/search.py

 import sqlalchemy
-import simplejson
 
 from pylons import config
 
     order_by = 'rank'
     all_fields = False
     return_objects = False
+    ref_entity_with_attr = 'name'
 
     def __init__(self, kw_dict):
         if not kw_dict.keys():
     def __str__(self):
         return repr(self.__dict__)
 
-class Search:
+class SQLSearch:
     _tokens = [ 'name', 'title', 'notes', 'tags', 'groups', 'author', 'maintainer', 'update_frequency', 'geographic_granularity', 'geographic_coverage', 'temporal_granularity', 'temporal_coverage', 'national_statistic', 'categories', 'precision', 'department', 'agency', 'external_reference']
     # Note: all tokens must be in the search vector (see model/full_search.py)
     _open_licenses = None
                     results.append(result)
                 self._results['results'] = results
             else:
-                self._results['results'] = [entity.name for entity in self._results['results']]
- 
+                attr_name = self._options.ref_entity_with_attr
+                self._results['results'] = [getattr(entity, attr_name) for entity in self._results['results']]
+    
+    def index_package(self, package):
+        pass
+        
+    def index_group(self, group):
+        pass
+        
+    def index_tag(self, tag):
+        pass
+
+
+class SolrSearch(SQLSearch):
+    _solr_fields = ["entity_type", "tags", "groups", "res_description", "res_format", 
+                    "res_url", "text", "urls", "indexed_ts"]
+
+    def __init__(self, solr_url=None):
+        if solr_url is None:
+            solr_url = config.get('solr_url', 'http://localhost:8983/solr')
+        # import inline to avoid external dependency 
+        from solr import SolrConnection # == solrpy 
+        self._conn = SolrConnection(solr_url)
+
+    def _open_license_query_part():
+        if self._open_licenses is None:
+            self._update_open_licenses()
+        licenses = ["+%d" % id for id in self.open_licenses]
+        licenses = " OR ".join(licenses)
+        return "license_id:(%s) " % licenses
+
+    def _build_package_query(self, authorized_package_query,
+                             general_terms, field_specific_terms):
+        orm_query = authorized_package_query
+        orm_query = orm_query.filter(model.package_search_table.c.package_id==model.Package.id)
+
+        # Full search by general_terms (and field specific terms but not by field)
+        query = u""
+        for field, term in field_specific_terms.items():
+            query += field + u":" + term + u" "
+        for term in general_terms:
+            query += term + u" "
+
+        # Filter for options
+        if self._options.filter_by_downloadable:
+            query += u"res_url:[* TO *] " # not null resource URL 
+        if self._options.filter_by_openness:
+            query += self._open_license_query_part()
+        
+        self._solr_results = self._conn.query(query, #sort=sorting, 
+                                        rows=self._options.limit,
+                                        start=self._options.offset)
+        entity_ids = [r.get('id') for r in self._solr_results.results]
+        orm_query = orm_query.filter(model.Package.id.in_(entity_ids))
+        orm_query = orm_query.add_column(sqlalchemy.func.now())
+        
+        if self._options.order_by and self._options.order_by != 'rank':
+            if hasattr(model.Package, self._options.order_by):
+                model_attr = getattr(model.Package, self._options.order_by)
+                orm_query = orm_query.order_by(model_attr)
+        
+        return orm_query
+        
+    def _run_query(self, query):
+        if self._options.entity == 'package':
+            self._results['count'] = query.count()
+            results = [(r, self._solr_results.get('score', 0)) for r in query]
+            self._results['results'] = results
+        else:
+            SQLSearch._run_query(self, query)
+    
+    def index_package(self, package):
+        return self.index_package_dict(package.as_dict())
+    
+    def index_package_dict(self, package):
+        index_fields = self._solr_fields + package.keys()
+            
+        # include the extras in the main namespace
+        extras = package.get('extras', {})
+        if 'extras' in package:
+            del package['extras']
+        for (key, value) in extras.items():
+            if key not in index_fields:
+                package[key] = value
+
+        # flatten the structure for indexing: 
+        for resource in package.get('resources', []):
+            for (okey, nkey) in [('description', 'res_description'),
+                                 ('format', 'res_format'),
+                                 ('url', 'res_url')]:
+                package[nkey] = package.get(nkey, []) + [resource.get(okey, u'')]
+        if 'resources' in package:
+            del package['resources']
+
+        package['entity_type'] = u"package"
+        package = dict([(str(k), v) for (k, v) in package.items()])
+
+        # send to solr:    
+        self._conn.add(**package)
+
+
+ENGINES = {
+    'sql': SQLSearch, 
+    'solr': SolrSearch
+    }
+
+
+def make_search(engine=None, **kwargs):
+    if engine is None:
+        engine = config.get('search_engine', 'sql')
+    klass = ENGINES.get(engine.strip().lower())
+    return klass(**kwargs)

ckan/migration/tests/test_v0-11.py

 class TestMigrate11B(object):
     # NB Run this part with model code at v11
     def test_0_basic(self):
-        from ckan.lib.search import Search, SearchOptions
-        result = Search().search(u'annakarenina')
+        from ckan.lib.search import make_search, SearchOptions
+        result = make_search().search(u'annakarenina')
         assert result['count'] == 1, result
         assert 'annakarenina' in result['results'], result['results']
 
     def test_1_notes(self):
-        from ckan.lib.search import Search, SearchOptions
-        result = Search().search(u'italicized')
+        from ckan.lib.search import make_search, SearchOptions
+        result = make_search().search(u'italicized')
         assert result['count'] == 1, result
         assert 'annakarenina' in result['results'], result['results']
         

ckan/model/authz.py

+'''For an overview of CKAN authorization system and model see
+doc/authorization.rst.
+
+'''
 from meta import *
 from core import DomainObject, Package, System
 from group import Group
 PSEUDO_USER__LOGGED_IN = u'logged_in'
 PSEUDO_USER__VISITOR = u'visitor'
 
+class NotRealUserException(Exception):
+    pass
+
+## ======================================
+## Action and Role Enums
+
 class Enum(object):
     @classmethod
     def is_valid(self, val):
     EDITOR = u'editor'
     READER = u'reader'
 
+default_role_actions = [
+    (Role.EDITOR, Action.EDIT),
+    (Role.EDITOR, Action.CREATE),
+    (Role.EDITOR, Action.READ),        
+    (Role.READER, Action.CREATE),
+    (Role.READER, Action.READ),
+    ]
+
+
+## ======================================
+## Table Definitions
+
 role_action_table = Table('role_action', metadata,
            Column('id', UnicodeText, primary_key=True, default=make_uuid),
            Column('role', UnicodeText),
            Column('user_object_role_id', UnicodeText, ForeignKey('user_object_role.id'), primary_key=True),
            )
 
+
 class RoleAction(DomainObject):
     pass
 
+# dictionary mapping protected objects (e.g. Package) to related ObjectRole
+protected_objects = {}
+
 class UserObjectRole(DomainObject):
-    pass
+    name = None
+    protected_object = None
+
+    @classmethod
+    def get_object_role_class(self, domain_obj):
+        protected_object = protected_objects.get(domain_obj.__class__, None)
+        if protected_object is None:
+            # TODO: make into an authz exception
+            msg = '%s is not a protected object, i.e. a subject of authorization' % domain_obj
+            raise Exception(msg)
+        else:
+            return protected_object
+
+    @classmethod
+    def user_has_role(cls, user, role, domain_obj):
+        assert isinstance(user, User), user
+        assert Role.is_valid(role), role
+        q = cls._query(user, role, domain_obj)
+        return q.count() == 1
+
+    @classmethod
+    def _query(cls, user, role, domain_obj):
+        q = Session.query(cls).filter_by(role=role)
+        # some protected objects are not "contextual"
+        if cls.name is not None:
+            # e.g. filter_by(package=domain_obj)
+            q = q.filter_by(**dict({cls.name: domain_obj}))
+        q = q.filter_by(user=user)
+        return q
+
+    @classmethod
+    def add_user_to_role(cls, user, role, domain_obj):
+        # role assignment already exists
+        if cls.user_has_role(user, role, domain_obj):
+            return
+        objectrole = cls(role=role, user=user)
+        if cls.name is not None:
+            setattr(objectrole, cls.name, domain_obj)
+        Session.add(objectrole)
+
+    @classmethod
+    def remove_user_from_role(cls, user, role, domain_obj):
+        q = self._query(user, role, domain_obj)
+        uo_role = q.one()
+        Session.delete(ou_role)
+        Session.commit()
+        Session.remove()
 
 class PackageRole(UserObjectRole):
-    pass
+    protected_object = Package
+    name = 'package'
+protected_objects[PackageRole.protected_object] = PackageRole
 
 class GroupRole(UserObjectRole):
-    pass
+    protected_object = Group
+    name = 'group'
+protected_objects[GroupRole.protected_object] = GroupRole
 
 class SystemRole(UserObjectRole):
-    pass
+    protected_object = System
+    name = None
+protected_objects[SystemRole.protected_object] = SystemRole
 
-mapper(RoleAction, role_action_table)
-       
-mapper(UserObjectRole, user_object_role_table,
-    polymorphic_on=user_object_role_table.c.context,
-    polymorphic_identity=u'user_object',
-    properties={
-        'user': orm.relation(User,
-            backref=orm.backref('roles',
-                cascade='all, delete, delete-orphan'
-            )
-        )
-    },
-    order_by=[user_object_role_table.c.id],
-)