Commits

Benoit Boissinot committed 8ad2e7a Merge

[merge] merge with i18n changes

Comments (0)

Files changed (63)

+include ckan/config/deployment.ini_tmpl
 recursive-include ckan/public *
 recursive-include ckan/templates *
 include CHANGELOG.txt
 Developer Installation
 ======================
 
+These are quick instructions to get developing. For fuller instructions see :doc:`deployment`.
+
 1. Get the code and install it:
 
    We recommend installing using pip and virtualenv::
 
    NB: you'll need to setup a database -- see sqlalchemy.url config option.
    We support only PostgreSQL at this time. You'll need to install the relevant
-   python library (eg. On debiani/ubuntu: python-psycopg2)
+   python library (eg. On debian/ubuntu: python-psycopg2)
 
    NB: You may also need to create the Pylon's cache directory specified by
    cache_dir in the config file.
 
       paster serve {your-config.ini} 
 
-6. Point your browser at: localhost:5000 (if you set a different port in your
-   config file then youl will need to change 5000 to whatever port value you
-   chose).
+6. Point your browser at: http://localhost:5000/ (or a different port, depending
+   on the one given in your config file)
 
 
 Test
 import os
 import logging
 import sys
-import gzip
+import zipfile
 import traceback
 import datetime
 
 LOG_FILENAME = os.path.expanduser('~/gov-daily.log')
 ONS_CACHE_DIR = os.path.expanduser('~/ons_data')
-DUMP_FILE_BASE = os.path.expanduser('~/data.gov.uk-daily')
+DUMP_DIR = os.path.expanduser('~/dump/')
+# e.g. hmg.ckan.net-20091125.json.gz	
+DUMP_FILE_BASE = os.path.expanduser('hmg.ckan.net-%Y-%m-%d')
+TMP_FILEPATH = '/tmp/dump'
 USAGE = '''Daily script for government
 Usage: python %s [config.ini]
 ''' % sys.argv[0]
 
 # Dump
 logging.info('Creating database dump')
-dump_filepath_base = DUMP_FILE_BASE
-csv_filepath = dump_filepath_base + '.csv.gz'
-json_filepath = dump_filepath_base + '.json.gz'
-csv_file = gzip.open(csv_filepath, 'wb')
-json_file = gzip.open(json_filepath, 'wb')
+if not os.path.exists(DUMP_DIR):
+    logging.info('Creating dump dir: %s' % DUMP_DIR)
+    os.makedirs(DUMP_DIR)
+dump_file_base = start_time.strftime(DUMP_FILE_BASE)
+csv_dump_filename_base = dump_file_base + '.csv'
+json_dump_filename_base = dump_file_base + '.json'
+csv_filepath = DUMP_DIR + csv_dump_filename_base + '.zip'
+json_filepath = DUMP_DIR + json_dump_filename_base + '.zip'
 query = model.Session.query(model.Package)
+tmp_file = open(TMP_FILEPATH, 'w')
 logging.info('Creating CSV file: %s' % csv_filepath)
-dumper.SimpleDumper().dump_csv(csv_file, query)
+dumper.SimpleDumper().dump_csv(tmp_file, query)
+tmp_file.close()
+csv_file = zipfile.ZipFile(csv_filepath, 'w', zipfile.ZIP_DEFLATED)
+csv_file.write(TMP_FILEPATH, dump_file_base)
 csv_file.close()
+tmp_file = open(TMP_FILEPATH, 'w')
 logging.info('Creating JSON file: %s' % json_filepath)
-dumper.SimpleDumper().dump_json(json_file, query)
+dumper.SimpleDumper().dump_json(tmp_file, query)
+tmp_file.close()
+json_file = zipfile.ZipFile(json_filepath, 'w', zipfile.ZIP_DEFLATED)
+json_file.write(TMP_FILEPATH, dump_file_base)
 json_file.close()
 #logging.info('Transferring dumps to ckan.net')
 #TODO

ckan/config/deployment.ini_tmpl

 # Guide location
 #guide_url = http://wiki.okfn.org/ckan/doc/
 
-## extra places to look for templates (comma separated list)
-## any templates found will override correspondingly named templates in in
-## ckan/templates/ (e.g. to override main layout template provide a new one
-## called layout.html)
-# extra_template_paths = %(here)s/my-extra-templates
+## extra places to look for templates and public files (comma separated lists)
+## any templates/files found will override correspondingly named ones in
+## ckan/templates/ and ckan/public
+## (e.g. to override main layout template layout.html or add in css/extra.css)
+# extra_template_paths = %(here)s/my-templates
+# extra_public_paths = %(here)s/my-public
 
 # Logging configuration
 [loggers]

ckan/config/environment.py

 import pylons
 from sqlalchemy import engine_from_config
 from pylons import config
+from pylons.i18n.translation import ugettext
 from genshi.template import TemplateLoader
+from genshi.filters.i18n import Translator
 
 import ckan.lib.app_globals as app_globals
 import ckan.lib.helpers
 from ckan.config.routing import make_map
 from ckan import model
 
-from genshi.filters.i18n import Translator
 
 def load_environment(global_conf, app_conf):
     """Configure the Pylons environment via the ``pylons.config``
                  templates=[os.path.join(root, 'templates')])
 
     # Initialize config with the basic options
-    config.init_app(global_conf, app_conf, package='ckan',
-                    template_engine='genshi', paths=paths)
+    config.init_app(global_conf, app_conf, package='ckan', paths=paths)
 
     config['routes.map'] = make_map()
     config['pylons.app_globals'] = app_globals.Globals()
     # tmpl_options["genshi.loader_callback"] = template_loaded
     config['pylons.app_globals'].genshi_loader = TemplateLoader(
         template_paths, auto_reload=True, callback=template_loaded)
-    # HACK! For some reason callback=template_loaded in previous line does
-    # *not* work (this required 1h to track down!!)
-    # This does work ...
-    tmpl_options = config['buffet.template_options']
-    tmpl_options["genshi.loader_callback"] = template_loaded
 
     # CONFIGURATION OPTIONS HERE (note: all config options will override
     # any Pylons config options)

ckan/config/middleware.py

 """Pylons middleware initialization"""
+from beaker.middleware import CacheMiddleware, SessionMiddleware
 from paste.cascade import Cascade
 from paste.registry import RegistryManager
 from paste.urlparser import StaticURLParser
 from paste.deploy.converters import asbool
-
 from pylons import config
-from pylons.error import error_template
-from pylons.middleware import ErrorHandler, StaticJavascripts, StatusCodeRedirect
+from pylons.middleware import ErrorHandler, StatusCodeRedirect
 from pylons.wsgiapp import PylonsApp
+from routes.middleware import RoutesMiddleware
 
 from ckan.config.environment import load_environment
 
-def make_app(global_conf, full_stack=True, **app_conf):
+def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
     """Create a Pylons WSGI application and return it
 
     ``global_conf``
         the [DEFAULT] section of the Paste ini file.
 
     ``full_stack``
-        Whether or not this application provides a full WSGI stack (by
-        default, meaning it handles its own exceptions and errors).
-        Disable full_stack when this application is "managed" by
-        another WSGI middleware.
+        Whether this application provides a full WSGI stack (by default,
+        meaning it handles its own exceptions and errors). Disable
+        full_stack when this application is "managed" by another WSGI
+        middleware.
+
+    ``static_files``
+        Whether this application serves its own static files; disable
+        when another web server is responsible for serving them.
 
     ``app_conf``
-        The application's local configuration. Normally specified in the
-        [app:<name>] section of the Paste ini file (where <name>
+        The application's local configuration. Normally specified in
+        the [app:<name>] section of the Paste ini file (where <name>
         defaults to main).
+
     """
     # Configure the Pylons environment
     load_environment(global_conf, app_conf)
     # The Pylons WSGI app
     app = PylonsApp()
 
+    # Routing/Session/Cache Middleware
+    app = RoutesMiddleware(app, config['routes.map'])
+    app = SessionMiddleware(app, config)
+    app = CacheMiddleware(app, config)
+
     # CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares)
 
-    import pylons
-    if pylons.__version__ >= "0.9.7":
-        # Routing/Session/Cache Middleware
-        from beaker.middleware import CacheMiddleware, SessionMiddleware
-        from routes.middleware import RoutesMiddleware
-        app = RoutesMiddleware(app, config['routes.map'])
-        app = SessionMiddleware(app, config)
-        app = CacheMiddleware(app, config)
-
-    from repoze.who.config import make_middleware_with_config
-    app = make_middleware_with_config(app, global_conf,
-            app_conf['who.config_file'], app_conf['who.log_file'],
-            app_conf['who.log_level'])
-
     if asbool(full_stack):
         # Handle Python exceptions
-        app = ErrorHandler(app, global_conf,**config['pylons.errorware'])
+        app = ErrorHandler(app, global_conf, **config['pylons.errorware'])
 
         # Display error documents for 401, 403, 404 status codes (and
         # 500 when debug is disabled)
         if asbool(config['debug']):
             app = StatusCodeRedirect(app)
         else:
-            app = StatusCodeRedirect(app, [401, 403, 404, 500])
+            app = StatusCodeRedirect(app, [400, 401, 403, 404, 500])
+    
+    from repoze.who.config import make_middleware_with_config
+    app = make_middleware_with_config(app, global_conf,
+        app_conf['who.config_file'], app_conf['who.log_file'],
+        app_conf['who.log_level'])
+
 
     # Establish the Registry for this application
     app = RegistryManager(app)
 
-    # Static files
-    static_app = StaticURLParser(config['pylons.paths']['static_files'])
-    app = Cascade([static_app, app])
+    if asbool(static_files):
+        # Serve static files
+        static_app = StaticURLParser(config['pylons.paths']['static_files'])
+        static_parsers = [static_app, app]
+
+        # Configurable extra static file paths
+        extra_public_paths = app_conf.get('extra_public_paths')
+        if extra_public_paths:
+            static_parsers = [StaticURLParser(public_path) \
+                              for public_path in \
+                              extra_public_paths.split(',')] + static_parsers
+            
+        app = Cascade(static_parsers)
+
     return app

ckan/config/routing.py

     """Create, configure and return the routes Mapper"""
     map = Mapper(directory=config['pylons.paths']['controllers'],
                  always_scan=config['debug'])
+    map.minimization = False
 
     # The ErrorController route (handles 404/500 error pages); it should
     # likely stay at the top, ensuring it can always be resolved
-    map.connect('/error/:action/:id', controller='error')
+    map.connect('/error/{action}', controller='error')
+    map.connect('/error/{action}/{id}', controller='error')
 
     # CUSTOM ROUTES HERE
     map.connect('home', '/', controller='home', action='index')
     map.connect('about', '/about', controller='home', action='about')
     map.connect('stats', '/stats', controller='home', action='stats')
     maps.admin_map(map, controller='admin', url='/admin')
+    # CKAN API.
     map.connect('/api/search/:register', controller='rest', action='search')
     map.connect('/api/tag_counts', controller='rest', action='tag_counts')
     map.connect('/api', controller='rest', action='index')
     map.connect('/api/rest/:register/:id/:subregister/:id2',
         controller='rest', action='delete',
         conditions=dict(method=['DELETE']))
+    # CKAN API v1.
+    map.connect('/api/1/search/:register', controller='rest', action='search')
+    map.connect('/api/1/tag_counts', controller='rest', action='tag_counts')
+    map.connect('/api/1', controller='rest', action='index')
+    map.connect('/api/1/rest', controller='rest', action='index')
+    map.connect('/api/1/rest/:register', controller='rest', action='list',
+        conditions=dict(method=['GET']))
+    map.connect('/api/1/rest/:register', controller='rest', action='create',
+        conditions=dict(method=['POST']))
+    map.connect('/api/1/rest/:register/:id', controller='rest', action='show',
+        conditions=dict(method=['GET']))
+    map.connect('/api/1/rest/:register/:id', controller='rest', action='update',
+        conditions=dict(method=['PUT']))
+    map.connect('/api/1/rest/:register/:id', controller='rest', action='update',
+        conditions=dict(method=['POST']))
+    map.connect('/api/1/rest/:register/:id', controller='rest', action='delete',
+        conditions=dict(method=['DELETE']))
+    map.connect('/api/1/rest/:register/:id/:subregister',
+        controller='rest', action='list',
+        conditions=dict(method=['GET']))
+    map.connect('/api/1/rest/:register/:id/:subregister/:id2',
+        controller='rest', action='create',
+        conditions=dict(method=['POST']))
+    map.connect('/api/1/rest/:register/:id/:subregister/:id2',
+        controller='rest', action='show',
+        conditions=dict(method=['GET']))
+    map.connect('/api/1/rest/:register/:id/:subregister/:id2',
+        controller='rest', action='update',
+        conditions=dict(method=['PUT']))
+    map.connect('/api/1/rest/:register/:id/:subregister/:id2',
+        controller='rest', action='delete',
+        conditions=dict(method=['DELETE']))
+
+    # CKAN API v2.
+    map.connect('/api/2/search/:register', controller='rest2', action='search')
+    map.connect('/api/2/tag_counts', controller='rest2', action='tag_counts')
+    map.connect('/api/2', controller='rest2', action='index')
+    map.connect('/api/2/rest', controller='rest2', action='index')
+    map.connect('/api/2/rest/:register', controller='rest2', action='list',
+        conditions=dict(method=['GET']))
+    map.connect('/api/2/rest/:register', controller='rest2', action='create',
+        conditions=dict(method=['POST']))
+    map.connect('/api/2/rest/:register/:id', controller='rest2', action='show',
+        conditions=dict(method=['GET']))
+    map.connect('/api/2/rest/:register/:id', controller='rest2', action='update',
+        conditions=dict(method=['PUT']))
+    map.connect('/api/2/rest/:register/:id', controller='rest2', action='update',
+        conditions=dict(method=['POST']))
+    map.connect('/api/2/rest/:register/:id', controller='rest2', action='delete',
+        conditions=dict(method=['DELETE']))
+    map.connect('/api/2/rest/:register/:id/:subregister',
+        controller='rest2', action='list',
+        conditions=dict(method=['GET']))
+    map.connect('/api/2/rest/:register/:id/:subregister/:id2',
+        controller='rest2', action='create',
+        conditions=dict(method=['POST']))
+    map.connect('/api/2/rest/:register/:id/:subregister/:id2',
+        controller='rest2', action='show',
+        conditions=dict(method=['GET']))
+    map.connect('/api/2/rest/:register/:id/:subregister/:id2',
+        controller='rest2', action='update',
+        conditions=dict(method=['PUT']))
+    map.connect('/api/2/rest/:register/:id/:subregister/:id2',
+        controller='rest2', action='delete',
+        conditions=dict(method=['DELETE']))
 
     map.redirect("/packages", "/package")
     map.redirect("/packages/{url:.*}", "/package/{url}")
     map.connect('/user/logout', controller='user', action='logout')
     map.connect('/user/apikey', controller='user', action='apikey')
     map.connect('/user/:id', controller='user', action='read')
-    map.connect('/:controller/:action/:id')
-    map.connect('/:controller/', action='index')
-    map.connect('/:controller/:action/', id=None)
+    map.connect('/{controller}', action='index')
+    map.connect('/:controller/{action}')
+    map.connect('/{controller}/{action}/{id}')
+    map.redirect('/*(url)/', '/{url}',
+                 _redirect_code='301 Moved Permanently')
     map.connect('/*url', controller='template', action='view')
 
     return map

ckan/controllers/error.py

 import cgi
-import os.path
 
-import paste.fileapp
-from pylons.middleware import error_document_template, media_path
+from paste.urlparser import PkgResourcesParser
+from pylons import request
+from pylons.controllers.util import forward
+from pylons.middleware import error_document_template
+from webhelpers.html.builder import literal
 
-from ckan.lib.base import *
+from ckan.lib.base import BaseController
+from ckan.lib.base import render
 
 class ErrorController(BaseController):
+
     """Generates error documents as and when they are required.
 
     The ErrorDocuments middleware forwards to ErrorController when error
 
     This behaviour can be altered by changing the parameters to the
     ErrorDocuments middleware in your config/middleware.py file.
+
     """
 
     def document(self):
         """Render the error document"""
-        ckan_template = render("error_document_template")
-        ckan_template = ckan_template.decode('utf8')
-        response = request.environ.get('pylons.original_response')
+        ckan_template = render('error_document_template.html')
+        # ckan_template = ckan_template.decode('utf8')
+        resp = request.environ.get('pylons.original_response')
+        content = literal(resp.body) or cgi.escape(request.GET.get('message', ''))
+        # page = error_document_template % \
         page = ckan_template % \
             dict(prefix=request.environ.get('SCRIPT_NAME', ''),
-                 code=cgi.escape(request.GET.get('code', str(response.status_int))),
-                 message=h.literal(response.body) or cgi.escape(request.GET.get('message', '')))
+                 code=cgi.escape(request.GET.get('code', str(resp.status_int))),
+                 message=content)
         return page
 
     def img(self, id):
         """Serve Pylons' stock images"""
-        return self._serve_file(os.path.join(media_path, 'img', id))
+        return self._serve_file('/'.join(['media/img', id]))
 
     def style(self, id):
         """Serve Pylons' stock stylesheets"""
-        return self._serve_file(os.path.join(media_path, 'style', id))
+        return self._serve_file('/'.join(['media/style', id]))
 
     def _serve_file(self, path):
         """Call Paste's FileApp (a WSGI application) to serve the file
         at the specified path
         """
-        fapp = paste.fileapp.FileApp(path)
-        return fapp(request.environ, self.start_response)
+        request.environ['PATH_INFO'] = '/%s' % path
+        return forward(PkgResourcesParser('pylons', 'pylons'))

ckan/controllers/group.py

             page=request.params.get('page', 1),
             items_per_page=20
         )
-        return render('group/index')
+        return render('group/index.html')
 
     def read(self, id):
         c.group = model.Group.by_name(id)
             page=request.params.get('page', 1),
             items_per_page=50
         )
-        return render('group/read')
+        return render('group/read.html')
 
     def new(self):
         record = model.Group
             except ValidationException, error:
                 fs = error.args[0]
                 c.form = self._render_edit_form(fs)
-                return render('group/edit')
+                return render('group/edit.html')
             # do not use groupname from id as may have changed
             c.groupname = c.fs.name.value
             group = model.Group.by_name(c.groupname)
             data = ckan.forms.edit_group_dict(ckan.forms.get_group_dict(), request.params)
             fs = fs.bind(data=data, session=model.Session)
         c.form = self._render_edit_form(fs)
-        return render('group/new')
+        return render('group/new.html')
 
     def edit(self, id=None): # allow id=None to allow posting
         c.error = ''
             
             fs = ckan.forms.get_group_fieldset('group_fs').bind(c.group)
             c.form = self._render_edit_form(fs)
-            return render('group/edit')
+            return render('group/edit.html')
         else:
             # id is the name (pre-edited state)
             c.groupname = id
             except ValidationException, error:
                 fs = error.args[0]
                 c.form = self._render_edit_form(fs)
-                return render('group/edit')
+                return render('group/edit.html')
             pkgs = [model.Package.by_name(name) for name in request.params.getall('Group-packages-current')]
             group.packages = pkgs
             pkgids = request.params.getall('PackageGroup--package_id')
             except ValidationException, error:
                 # TODO: sort this out 
                 # fs = error.args[0]
-                # return render('group/authz')
+                # return render('group/authz.html')
                 raise
             # now do new roles
             newrole_user_id = request.params.get('GroupRole--user_id')
         fs = ckan.forms.get_authz_fieldset('group_authz_fs').bind(group.roles)
         c.form = fs.render()
         c.new_roles_form = ckan.forms.get_authz_fieldset('new_group_roles_fs').render()
-        return render('group/authz')
+        return render('group/authz.html')
 
     def _render_edit_form(self, fs):
         # errors arrive in c.error and fs.errors
         c.fieldset = fs
         c.fieldset2 = ckan.forms.get_group_fieldset('new_package_group_fs')
-        return render('group/edit_form')
+        return render('group/edit_form.html')
 
     def _update(self, fs, group_name, group_id):
         '''

ckan/controllers/home.py

         # 3600=hourly, 86400=daily
         c.tag_counts = mycache.get_value(key='tag_counts_home_page',
                 createfunc=tag_counts, expiretime=86400)
-        return render('home/index')
+        return render('home/index.html')
 
     def license(self):
-        return render('home/license')
+        return render('home/license.html')
 
     def about(self):
-        return render('home/about')
+        return render('home/about.html')
 
     def stats(self):
         def stats_html():
             c.top_package_owners = stats.top_package_owners()
             c.new_packages_by_week = rev_stats.get_by_week('new_packages')
             c.package_revisions_by_week = rev_stats.get_by_week('package_revisions')
-            return render('home/stats')
+            return render('home/stats.html')
         if not c.user:
             mycache = cache.get_cache('stats', type='dbm')
             # 3600=hourly, 86400=daily

ckan/controllers/importer.py

     authorizer = ckan.authz.Authorizer()
 
     def index(self):
-        return render('importer/importer')
+        return render('importer/importer.html')
 
     def preview(self):
         if not c.user:
         params = dict(request.params)
         if not params.has_key('file'):
             c.error = _('Need to specify a filename.')
-            return render('importer/importer')                
+            return render('importer/importer.html')                
         if not hasattr(params['file'], 'value'):
             c.error = _('Did not receive file successfully.')
-            return render('importer/importer')
+            return render('importer/importer.html')
         file_buf = params['file'].value
         # save as temp file for when you do import
         self._save_tempfile(file_buf)
         if not file_buf:
             c.error = _('File \'%s\' not found.') % params['file'].filename
-            return render('importer/importer')
+            return render('importer/importer.html')
         try:
             importer = importer.PackageImporter(buf=file_buf)
         except importer.ImportException, e:
             c.error = _('Error importing file \'%s\' as Excel or CSV format: %s') % (params['file'].filename, e)
-            return render('importer/importer')
+            return render('importer/importer.html')
         c.import_filename = params['file'].filename.lstrip(os.sep)
         if params.has_key('log_message'):
             c.log_message = params['log_message']
             c.fs_list.append(fs)
         c.errors = len(all_errors)
         c.num_pkgs = len(c.fs_list)
-        return render('importer/preview')
+        return render('importer/preview.html')
 
     def do_import(self):
         import ckan.lib.importer as importer
             importer = importer.PackageImporter(buf=file_buf)
         except importer.ImportException, e:
             c.error = _('Error importing file \'%s\' as Excel or CSV format: %s') % (params['file'].filename, e)
-            return render('importer/importer')
+            return render('importer/importer.html')
         if 'log_message' in request.params:
             log_message = request.params.getone('log_message')
         else:
 
         model.Session.commit()
         c.message = ungettext('Imported %i package.', 'Imported %i packages.', count) % count
-        return render('importer/result')
+        return render('importer/result.html')
 
     def _get_fs(self, importer):
         for index, pkg_dict in enumerate(importer.pkg_dict()):
     def package_render(self, fs, errors, warnings):
         try:
             PackageSaver().render_preview(fs, None, None) # create a new package for now
-            preview = h.literal(render('package/read_core'))
+            preview = h.literal(render('package/read_core.html'))
         except ValidationException, error:
             c.error, fs = error.args
             preview = h.literal('<li>Errors: %s</li>\n') % c.error

ckan/controllers/package.py

 import ckan.authz
 import ckan.rating
 import ckan.misc
-from ckan.lib.helpers import Page
 from pylons.i18n import get_lang
 
 logger = logging.getLogger('ckan.controllers')
     def index(self):
         query = ckan.authz.Authorizer().authorized_query(c.user, model.Package)
         c.package_count = query.count()
-        return render('package/index')
+        return render('package/index.html')
 
     def list(self):
         query = ckan.authz.Authorizer().authorized_query(c.user, model.Package)
-        c.page = Page(
+        c.page = h.Page(
             collection=query,
             page=request.params.get('page', 1),
             items_per_page=50
         )
-        return render('package/list')
+        return render('package/list.html')
 
     def search(self):        
         c.q = request.params.get('q') # unicode format (decoded from utf8)
                 })
             # package search
             query = make_search().query(options, username=c.user)
-            c.page = Page(
+            c.page = h.Page(
                 collection=query,
                 page=request.params.get('page', 1),
                 items_per_page=50
             c.tags = results['results']
             c.tags_count = results['count']
 
-        return render('package/search')
+        return render('package/search.html')
 
     def read(self, id):
         pkg = model.Package.by_name(id)
         c.auth_for_change_state = self.authorizer.am_authorized(c, model.Action.CHANGE_STATE, pkg)
 
         PackageSaver().render_package(pkg)
-        return render('package/read') 
+        return render('package/read.html') 
 
     def history(self, id):
         if 'diff' in request.params or 'selected1' in request.params:
             feed.content_type = 'application/atom+xml'
             return feed.writeString('utf-8')
         c.pkg_revisions = c.pkg.all_related_revisions
-        return render('package/history')
+        return render('package/history.html')
 
     def new(self):
         c.error = ''
                 model.setup_default_user_roles(pkg, admins)
                 model.repo.commit_and_remove()
 
-                h.redirect_to(action='read', id=pkgname)
+                self._post_edit_redirect(pkgname)
             except ValidationException, error:
                 fs = error.args[0]
                 c.form = self._render_edit_form(fs, request.params,
                         clear_session=True)
-                return render('package/new')
+                return render('package/new.html')
             except KeyError, error:
                 abort(400, ('Missing parameter: %s' % error.args).encode('utf8'))
 
                 PackageSaver().render_preview(fs, id, record.id,
                                               log_message=log_message,
                                               author=c.author)
-                c.preview = h.literal(render('package/read_core'))
+                c.preview = h.literal(render('package/read_core.html'))
             except ValidationException, error:
                 fs = error.args[0]
                 c.form = self._render_edit_form(fs, request.params,
                         clear_session=True)
-                return render('package/new')
-        return render('package/new')
+                return render('package/new.html')
+        return render('package/new.html')
 
     def edit(self, id=None): # allow id=None to allow posting
         # TODO: refactor to avoid duplication between here and new
                 self._adjust_license_id_options(pkg, fs)
             fs = fs.bind(pkg)
             c.form = self._render_edit_form(fs, request.params)
-            return render('package/edit')
+            return render('package/edit.html')
         elif request.params.has_key('commit'):
             # id is the name (pre-edited state)
             pkgname = id
                 PackageSaver().commit_pkg(fs, id, pkg.id, log_message, c.author)
                 # do not use pkgname from id as may have changed
                 pkgname = fs.name.value
-                h.redirect_to(action='read', id=pkgname)
+                self._post_edit_redirect(pkgname)
             except ValidationException, error:
                 fs = error.args[0]
                 c.form = self._render_edit_form(fs, request.params,
                         clear_session=True)
-                return render('package/edit')
+                return render('package/edit.html')
             except KeyError, error:
                 abort(400, 'Missing parameter: %s' % error.args)
         else: # Must be preview
                 PackageSaver().render_preview(fs, id, pkg.id,
                                               log_message=log_message,
                                               author=c.author)
-                read_core_html = render('package/read_core') #utf8 format
+                read_core_html = render('package/read_core.html') #utf8 format
                 c.preview = h.literal(read_core_html)
                 c.form = self._render_edit_form(fs, request.params)
             except ValidationException, error:
                 fs = error.args[0]
                 c.form = self._render_edit_form(fs, request.params,
                         clear_session=True)
-                return render('package/edit')
-            return render('package/edit') # uses c.form and c.preview
+                return render('package/edit.html')
+            return render('package/edit.html') # uses c.form and c.preview
 
+    def _post_edit_redirect(self, pkgname):
+        '''This redirects the user to the package/read page,
+        unless there is request parameter giving an alternate location.
+        @param pkgname - Name of the package just edited
+        '''
+        url = request.params.get('return_to')
+        if url:
+            url = url.replace('<NAME>', pkgname)
+        else:
+            url = h.url_for(action='read', id=pkgname)
+        redirect(url)        
+        
     def _adjust_license_id_options(self, pkg, fs):
         options = fs.license_id.render_opts['options']
         is_included = False
             except ValidationException, error:
                 # TODO: sort this out 
                 # fs = error.args
-                # return render('package/authz')
+                # return render('package/authz.html')
                 raise
             # now do new roles
             newrole_user_id = request.params.get('PackageRole--user_id')
         fs = ckan.forms.get_authz_fieldset('package_authz_fs').bind(c.pkg.roles)
         c.form = fs.render()
         c.new_roles_form = ckan.forms.get_authz_fieldset('new_package_roles_fs').render()
-        return render('package/authz')
+        return render('package/authz.html')
 
     def rate(self, id):
         package_name = id
             model.Session.clear()
         edit_form_html = fs.render()
         c.form = h.literal(edit_form_html)
-        return h.literal(render('package/edit_form'))
+        return h.literal(render('package/edit_form.html'))
 
     def _update_authz(self, fs):
         validation = fs.validate()

ckan/controllers/rest.py

 import ckan.authz
 import ckan.rating
 
-class RestController(BaseController):
+class BaseRestController(BaseController):
+
+    ref_package_with_attr = 'id'
+
+    def _list_package_refs(self, packages):
+        return [getattr(p, self.ref_package_with_attr) for p in packages]
+
+    def _get_pkg(self, id):
+        pkg = model.Session.query(model.Package).get(id)
+        if pkg == None:
+            pkg = model.Package.by_name(id)
+            # Todo: Make sure package names can't be changed to look like package IDs?
+        return pkg
 
     def index(self):
-        return render('rest/index')
+        return render('rest/index.html')
 
     def list(self, register, subregister=None, id=None):
         if register == 'revision':
             return self._finish_ok([rev.id for rev in revs])
         elif register == u'package' and not subregister:
             query = ckan.authz.Authorizer().authorized_query(self._get_username(), model.Package)
-            packages = query.all() 
-            results = [package.name for package in packages]
+            packages = query.all()
+            results = self._list_package_refs(packages)
             return self._finish_ok(results)
         elif register == u'package' and subregister == 'relationships':
             #TODO authz stuff for this and related packages
-            pkg = model.Package.by_name(id)
+            pkg = self._get_pkg(id)
             if not pkg:
                 response.status_int = 404
                 return 'First package named in request was not found.'
                 'timestamp': model.strftimestamp(rev.timestamp),
                 'author': rev.author,
                 'message': rev.message,
-                'packages': [p.name for p in rev.packages],
+                'packages': self._list_package_refs(rev.packages),
             }
             return self._finish_ok(response_data)
         elif register == u'changeset':
             _dict = changeset.as_dict()
             return self._finish_ok(_dict)
         elif register == u'package' and not subregister:
-            pkg = model.Package.by_name(id)
-            if pkg is None:
+            pkg = self._get_pkg(id)
+            if pkg == None:
                 response.status_int = 404
                 return ''
-
             if not self._check_access(pkg, model.Action.READ):
                 return ''
             _dict = pkg.as_dict()
-            # Set 'license' in _dict to cater for old clients.
-            # Todo: Remove this ASAP.
-            pkg_license = pkg.license
-            if pkg_license:
-                _dict['license'] = pkg_license.title
-            else:
-                _dict['license'] = _dict.get('license_id', '')
             return self._finish_ok(_dict)
         elif register == u'package' and (subregister == 'relationships' or subregister in model.PackageRelationship.get_all_types()):
-            pkg1 = model.Package.by_name(id)
-            pkg2 = model.Package.by_name(id2)
+            pkg1 = self._get_pkg(id)
+            pkg2 = self._get_pkg(id2)
             if not pkg1:
                 response.status_int = 404
                 return 'First package named in address was not found.'
                 request_fa_dict = ckan.forms.edit_package_dict(ckan.forms.get_package_dict(fs=fs), request_data)
                 fs = fs.bind(model.Package, data=request_fa_dict, session=model.Session)
             elif register == 'package' and subregister in model.PackageRelationship.get_all_types():
-                pkg1 = model.Package.by_name(id)
-                pkg2 = model.Package.by_name(id2)
+                pkg1 = self._get_pkg(id)
+                pkg2 = self._get_pkg(id2)
                 if not pkg1:
                     response.status_int = 404
                     return 'First package named in address was not found.'
             model.Session.rollback()
             raise
         obj = fs.model
+        location = "%s/%s" % (request.path, obj.id)
+        response.headers['Location'] = location
         return self._finish_ok(obj.as_dict())
             
     def update(self, register, id, subregister=None, id2=None):
+        # must be logged in to start with
+        if not self._check_access(None, None):
+            return json.dumps(_('Access denied'))
+
         if register == 'package' and not subregister:
-            entity = model.Package.by_name(id)
+            entity = self._get_pkg(id)
+            if entity == None:
+                response.status_int = 404
+                return 'Package was not found.'
         elif register == 'package' and subregister in model.PackageRelationship.get_all_types():
-            pkg1 = model.Package.by_name(id)
-            pkg2 = model.Package.by_name(id2)
+            pkg1 = self._get_pkg(id)
+            pkg2 = self._get_pkg(id2)
             if not pkg1:
                 response.status_int = 404
                 return 'First package named in address was not found.'
                 return self._update_package_relationship(entity, comment)
 
     def delete(self, register, id, subregister=None, id2=None):
+        # must be logged in to start with
+        if not self._check_access(None, None):
+            return json.dumps(_('Access denied'))
+
         if register == 'package' and not subregister:
-            entity = model.Package.by_name(id)
+            entity = self._get_pkg(id)
             if not entity:
                 response.status_int = 404
                 return 'Package was not found.'
             revisioned_details = 'Package: %s' % entity.name
         elif register == 'package' and subregister in model.PackageRelationship.get_all_types():
-            pkg1 = model.Package.by_name(id)
-            pkg2 = model.Package.by_name(id2)
+            pkg1 = self._get_pkg(id)
+            pkg2 = self._get_pkg(id2)
             if not pkg1:
                 response.status_int = 404
                 return 'First package named in address was not found.'
             options = SearchOptions(params)
             options.search_tags = False
             options.return_objects = False
+            options.ref_entity_with_attr = self.ref_package_with_attr
             username = self._get_username()
             results = make_search().run(options, username)
             return self._finish_ok(results)
                               'rating':5}
         """
         # check options
-        package_name = params.get('package')
+        package_ref = params.get('package')
         rating = params.get('rating')
         user = self.rest_api_user
         opts_err = None
-        if not package_name:
-            opts_err = gettext('You must supply a package name (parameter "package").')
+        if not package_ref:
+            opts_err = gettext('You must supply a package id or name (parameter "package").')
         elif not rating:
             opts_err = gettext('You must supply a rating (parameter "rating").')
         else:
             except ValueError:
                 opts_err = gettext('Rating must be an integer value.')
             else:
-                package = model.Package.by_name(package_name)
+                package = self._get_pkg(package_ref)
                 if rating < ckan.rating.MIN_RATING or rating > ckan.rating.MAX_RATING:
                     opts_err = gettext('Rating must be between %i and %i.') % (ckan.rating.MIN_RATING, ckan.rating.MAX_RATING)
                 elif not package:
-                    opts_err = gettext('Package with name %r does not exist.') % package_name
+                    opts_err = gettext('Package with name %r does not exist.') % package_ref
         if opts_err:
             self.log.debug(opts_err)
             response.status_int = 400
         ckan.rating.set_rating(user, package, rating_int)
 
         response.headers['Content-Type'] = 'application/json'
-        package = model.Package.by_name(package_name)
+        package = self._get_pkg(package_ref)
         ret_dict = {'rating average':package.get_average_rating(),
                     'rating count': len(package.ratings)}
         return self._finish_ok(ret_dict)
         else:
             return ''
 
+class RestController(BaseRestController):
+    # Implements CKAN API Version 1.
+
+    ref_package_with_attr = 'name'
+
+
+

ckan/controllers/rest2.py

+import sqlalchemy.orm
+
+from ckan.lib.base import *
+from ckan.controllers.rest import BaseRestController
+from ckan.lib.helpers import json
+import ckan.model as model
+import ckan.forms
+from ckan.lib.search import make_search, SearchOptions
+import ckan.authz
+import ckan.rating
+
+class Rest2Controller(BaseRestController):
+
+    ref_package_with_attr = 'id'
+

ckan/controllers/revision.py

                 items_per_page=50
             )
             
-            return render('revision/list')
+            return render('revision/list.html')
 
     def read(self, id=None):
         if id is None:
         c.packages = [ pkg.continuity for pkg in pkgs ]
         pkgtags = model.Session.query(model.PackageTagRevision).filter_by(revision=c.revision)
         c.pkgtags = [ pkgtag.continuity for pkgtag in pkgtags ]
-        return render('revision/read')
+        return render('revision/read.html')
 
     def diff(self, id=None):
         if 'diff' not in request.params or 'oldid' not in request.params:
         c.diff = diff.items()
         c.diff.sort()
         c.pkg = pkg
-        return render('revision/diff')
+        return render('revision/diff.html')
 
     def _has_purge_permissions(self):
         authorizer = ckan.authz.Authorizer()
     def purge(self, id=None):
         if id is None:
             c.error = _('No revision id specified')
-            return render('revision/purge')
+            return render('revision/purge.html')
         if not self._has_purge_permissions():
             c.error = _('You are not authorized to perform this action')
-            return render('revision/purge')
+            return render('revision/purge.html')
         else:
             revision = model.Session.query(model.Revision).get(id)
             try:
                 # is this a security risk?
                 # probably not because only admins get to here
                 c.error = _('Purge of revision failed: %s') % inst
-            return render('revision/purge')
+            return render('revision/purge.html')
 

ckan/controllers/tag.py

             items_per_page=100
         )
            
-        return render('tag/index')
+        return render('tag/index.html')
 
     def read(self, id):
         c.tag = model.Tag.by_name(id)
         if c.tag is None:
             abort(404)
-        return render('tag/read')
+        return render('tag/read.html')
 
     def autocomplete(self):
         incomplete = request.params.get('incomplete', '')

ckan/controllers/user.py

 from ckan.lib.base import *
 
 def login_form():
-    return render('user/login_form').replace('FORM_ACTION', '%s')
+    return render('user/login_form.html').replace('FORM_ACTION', '%s')
 
 class UserController(BaseController):
 
-    def index(self, id):
-        if not c.user or c.user != id:
+    def index(self, id=None):
+        if not c.user:
             h.redirect_to(controller='user', action='login', id=None)
         return self.read()
 
-    def read(self, id):
+    def read(self, id=None):
         if id:
             user = model.Session.query(model.User).get(id)
         else:
         c.num_edits = revisions_q.count()
         c.num_pkg_admin = model.Session.query(model.PackageRole).filter_by(user=user, role=model.Role.ADMIN).count()
         c.activity = revisions_q.limit(20).all()
-        return render('user/read')
+        return render('user/read.html')
 
     def login(self):
         if c.user:
                 model.Session.commit()
             h.redirect_to(controller='user', action=None, id=None)
         else:
-            form = render('user/openid_form')
+            form = render('user/openid_form.html')
             # /login_openid page need not exist -- request gets intercepted by openid plugin
             form = form.replace('FORM_ACTION', '/login_openid')
             return form
 
     def logout(self):
         c.user = None
-        return render('user/logout')
+        return render('user/logout.html')
 
     def apikey(self):
         # logged in
         else:
             user = model.User.by_name(c.user)
             c.api_key = user.apikey
-        return render('user/apikey')
+        return render('user/apikey.html')
 
     def edit(self):
         # logged in
                 model.Session.commit()
             h.redirect_to(controller='user', action='read', id=user.id)
             
-        return render('user/edit')
+        return render('user/edit.html')
         
     def _format_about(self, about):
         about_formatted = ckan.misc.MarkdownFormat().to_html(about)

ckan/forms/authz.py

 
 class RolesRenderer(formalchemy.fields.FieldRenderer):
     def render(self, **kwargs):
-        selected = kwargs.get('selected', None) or unicode(self._value)
+        selected = kwargs.get('selected', None) or unicode(self.value)
         options = [(role, role) for role in model.Role.get_all()]
         select = fa_h.select(self.name, selected, options, **kwargs)
         return select

ckan/forms/builder.py

 import formalchemy
-from pylons.templating import render
+from pylons.templating import render_genshi as render
 from pylons import c
 
 from ckan import model
         self.added_fields = []
         self.options = self.fs._fields # {field_name:fs.field}
         self.includes = None
-        self.set_form_template('package/form')
+        self.set_form_template('package/form.html')
 
     def add_field(self, field):
         if isinstance(field, common.ConfiguredField):

ckan/forms/common.py

 from formalchemy import helpers as fa_h
 import formalchemy
 import genshi
-from pylons.templating import render
+from pylons.templating import render_genshi as render
 from pylons import c
 from pylons.i18n import _, ungettext, N_, gettext
 
 class TextExtraRenderer(formalchemy.fields.TextFieldRenderer):
     def _get_value(self):
         extras = self.field.parent.model.extras # db
-        return self._value or extras.get(self.field.name, u'') or u''
+        return self.value or extras.get(self.field.name, u'') or u''
 
     def render(self, **kwargs):
         value = self._get_value()
     class DateRangeRenderer(formalchemy.fields.FieldRenderer):
         def _get_value(self):
             extras = self.field.parent.model.extras
-            if self._value:
-                from_form, to_form = self._value
+            if self.value:
+                from_form, to_form = self.value
             else:
                 from_ = extras.get(self.field.name + '-from') or u''
                 to = extras.get(self.field.name + '-to') or u''
     class TextRangeRenderer(formalchemy.fields.FieldRenderer):
         def _get_value(self):
             extras = self.field.parent.model.extras
-            if self._value:
-                from_form, to_form = self._value
+            if self.value:
+                from_form, to_form = self.value
             else:
                 from_ = extras.get(self.field.name + '-from') or u''
                 to = extras.get(self.field.name + '-to') or u''
 
     class ResourcesRenderer(formalchemy.fields.FieldRenderer):
         def render(self, **kwargs):
-            c.resources = self._value or []
+            c.resources = self.value or []
             # [:] does a copy, so we don't change original
             c.resources = c.resources[:]
             c.resources.extend([None])
             c.id = self.name
-            return render('package/form_resources')            
+            return render('package/form_resources.html')            
 
         def stringify_value(self, v):
             # actually returns dict here for _value
                 field_values.append({
                     'name':'%s-newfield%s' % (self.name, i)})
             c.fields = field_values
-            html = render('package/form_extra_fields')
+            html = render('package/form_extra_fields.html')
             return h.literal(html)
 
         def render_readonly(self, **kwargs):
     class SelectRenderer(formalchemy.fields.FieldRenderer):
         def _get_value(self, **kwargs):
             extras = self.field.parent.model.extras
-            return unicode(kwargs.get('selected', '') or self._value or extras.get(self.field.name, ''))
+            return unicode(kwargs.get('selected', '') or self.value or extras.get(self.field.name, ''))
 
         def render(self, options, **kwargs):
             selected = self._get_value()
     class CheckboxExtraRenderer(formalchemy.fields.CheckBoxFieldRenderer):
         def _get_value(self):
             extras = self.field.parent.model.extras
-            return bool(self._value or extras.get(self.field.name) == u'yes')
+            return bool(self.value or extras.get(self.field.name) == u'yes')
 
         def render(self, **kwargs):
             value = self._get_value()

ckan/forms/group.py

 
     for field in fs._fields.values():
         if not filter(lambda x: field.renderer.name.endswith(x), exclude):
-            if field.renderer._value:
-                indict[field.renderer.name] = field.renderer._value
+            if field.renderer.value:
+                indict[field.renderer.name] = field.renderer.value
             else:
                 indict[field.renderer.name] = u''
 

ckan/forms/package_ca.py

 #       though it is technically an agency. This is because it is SC is
 #       particularly important for this database.
 
-ministries = ['Agriculture and Agri-Food', 'Canadian Heritage', 'Citizenship and Immigration', 'Environment', 'Finance', 'Fisheries and Oceans', 'Foreign Affairs and International Trade', 'Health', 'Human Resources and Social Development', 'Indian and Northern Affairs', 'Industry', 'Intergovernmental Affairs', 'Justice', 'National Defence', 'Natural Resources', 'Public Safety', 'Public Works and Government Services', 'Statistics Canada', 'Transport', 'Veterans Affairs', 'Western Economic Diversification']
+ministries = ['Agriculture and Agri-Food', 'Canadian Heritage', 'Citizenship and Immigration', 'Environment Canada', 'Finance', 'Fisheries and Oceans', 'Foreign Affairs and International Trade', 'Health Canada', 'Human Resources and Social Development', 'Indian and Northern Affairs', 'Industry Canada', 'Intergovernmental Affairs', 'Justice (Department of)', 'National Defence', 'Natural Resources', 'Privy Council Office', 'Public Safety', 'Public Works', 'Statistics Canada', 'Transport Canada', 'Treasury Board Secretariat', 'Veterans Affairs', 'Western Economic Diversification']
 
 agencies = ['Air Transport Security Authority', 'Artists and Producers Professional Relations Tribunal', 'Atlantic Canada Opportunities Agency', 'Atlantic Pilotage Authority', 'Atomic Energy of Canada Limited', 'Auditor General', 'Bank of Canada', 'Border Services Agency', 'Business Development Bank of Canada', 'Business Service Centres', 'Canada Information Office', 'Canada Lands Company', 'Canada-Newfoundland Offshore Petroleum Board', 'Canada-Nova Scotia Offshore Petroleum Board', 'Canada Post Corporation', 'Canadian Broadcasting Corporation', 'Cape Breton Development Office', 'Centre for Occupational Health and Safety', 'Centre on Substance Abuse', 'Climate Change Secretariat', 'Commercial Corporation', 'Competition Bureau', 'Competition Tribunal', 'Copyright Board', 'Correctional Service', 'Council for the Arts', 'Courts Administration Service', 'Cultural Property Export Review Board', 'Dairy Commission', 'Defence Construction Canada', 'Defence Research and Development Canada', 'Deposit Insurance Corporation', 'Economic Development Agency for Quebec Regions', 'Elections Canada', 'Enterprise Cape Breton Corporation', 'Environment and Sustainable Development Commissioner', 'Environmental Assessment Agency', 'Environmental Protection Review Canada', 'Ethics Commissioner', 'Export Development Canada', 'Federal Bridge Corporation Limited', 'Federal Judicial Affairs Commissioner', 'Federal Science for Sustainable Development', 'Financial Consumer Agency', 'Firearms Centre', 'Food Inspection Agency', 'Freshwater Fish Marketing Corporation', 'Grain Commission', 'Human Rights Commission', 'Human Rights Tribunal', 'Industrial Relations Board', 'Institutes of Health Research', 'Intergovernmental Conference Secretariat', 'International Development Agency', 'International Trade Tribunal', 'Library and Archives Canada', 'Mortgage and Housing Corporation', 'National Capital Commission', 'National Film Board', 'National Parole Board', 'National Research Council', 'National Round Table on the Environment and the Economy', 'National Search and Rescue Secretariat', 'Natural Sciences and Engineering Research Council of Canada', 'Northern Pipeline Agency', 'Nuclear Safety Commission', 'Official Languages Commissioner', 'Pari-Mutuel Agency', 'Parks Canada', 'Pension Plan Investment Board', 'Polar Commission', 'Police College', 'Privacy Commissioner of Canada', 'Public Health Agency of Canada', 'Public Service Commission of Canada', 'Radio-Television and Telecommunications Commission', 'Revenue Agency', 'Review Tribunals Commissioner', 'Royal Canadian Mint', 'School of Public Service', 'Social Sciences and Humanities Research Council', 'Space Agency', 'Standards Council of Canada', 'Tourism Commission', 'Transportation Agency', 'Transportation Safety Board', 'Treasury Board Secretariat', 'VIA Rail Canada', 'Wheat Board']
 

ckan/forms/package_dict.py

             if blank:
                 indict[field.renderer.name] = u''
             else:
-                if field.renderer._value:
-                    indict[field.renderer.name] = field.renderer._value
+                if field.renderer.value:
+                    indict[field.renderer.name] = field.renderer.value
                 else:
                     indict[field.renderer.name] = u''
 

ckan/forms/package_gov.py

 
     class GeoCoverageRenderer(formalchemy.fields.FieldRenderer):
         def _get_value(self):
-            form_regions = self._value # params
+            form_regions = self.value # params
             if not form_regions:
                 extras = self.field.parent.model.extras # db
                 db_regions = extras.get(self.field.name, []) or []
         fs = self.field.parent
         pkg_dict = {}
         for field_name, field in fs.render_fields.items():
-            pkg_dict[field_name] = field.renderer._value
+            pkg_dict[field_name] = field.renderer.value
         tag_suggestions = schema_gov.suggest_tags(pkg_dict)
         html = literal("<div>Suggestions (preview refreshes): %s</div>") % ' '.join(tag_suggestions)
         html += common.TagField.TagEditRenderer.render(self, **kwargs)

ckan/getdata/data4nr.py

         rev = self._new_revision()
         assert os.path.exists(csv_filepath)
         f_obj = open(csv_filepath, "r")
+        dialect = csv.excel
+        dialect.escapechar = '\\' # for openoffice
+        reader = csv.reader(f_obj, dialect)
         self._current_filename = os.path.basename(csv_filepath)
-        reader = csv.reader(f_obj)
         index = 0
         reader.next()
         for row_list in reader:

ckan/getdata/ons_import.py

         date_released = u''
         if item['pubDate']:
             try:
-                date_released = field_types.DateType.iso_to_db(item['pubDate'], '%a, %d %b %Y %H:%M:%S %Z')
+                iso_date = field_types.DateType.strip_iso_timezone(item['pubDate'])
+                date_released = field_types.DateType.iso_to_db(iso_date, '%a, %d %b %Y %H:%M:%S')
             except TypeError, e:
                 self._log(logging.warning, 'Warning: Could not read format of publication (release) date: %r' % e.args)
         extras['date_released'] = date_released

ckan/lib/app_globals.py

 """The application's Globals object"""
-from pylons import config
 
 class Globals(object):
+
     """Globals acts as a container for objects available throughout the
     life of the application
+
     """
 
     def __init__(self):
         """One instance of Globals is created during application
-        initialization and is available during requests via the 'g'
-        variable
+        initialization and is available during requests via the
+        'app_globals' variable
+
         """
-        pass
 """The base Controller API
 
-Provides the BaseController class for subclassing, and other objects
-utilized by Controllers.
+Provides the BaseController class for subclassing.
 """
 import logging
 
 from pylons.controllers.util import abort, etag_cache, redirect_to, redirect
 from pylons.decorators import jsonify, validate
 from pylons.i18n import _, ungettext, N_, gettext
-from pylons.templating import render
+from pylons.templating import render_genshi as render
 
 import ckan
 import ckan.lib.helpers as h

ckan/lib/create_test_data.py

     max_args = 1
     min_args = 0
     author = u'tester'
+
+    pkg_names = []
+    tag_names = []
+    group_names = set()
+    user_names = []
     
     def command(self):
         self._load_config()
 
     @classmethod
     def create_arbitrary(self, package_dicts,
-                         relationships=[], extra_user_names=[]):
+                         relationships=[], extra_user_names=[],
+                         commit_changesets=False):
+        assert isinstance(relationships, (list, tuple))
+        assert isinstance(extra_user_names, (list, tuple))
         import ckan.model as model
         model.Session.remove()
-        self.pkg_names = []
-        self.tag_names = []
-        self.group_names = set()
-        self.user_names = extra_user_names
+        new_user_names = extra_user_names
+        new_group_names = set()
+        
         admins_list = [] # list of (package_name, admin_names)
         if package_dicts:
             rev = model.repo.new_revision() 
                     elif attr == 'download_url':
                         pkg.add_resource(unicode(val))
                     elif attr == 'resources':
+                        assert isinstance(val, (list, tuple))
                         for res_dict in val:
                             pkg.add_resource(
                                 url=unicode(res_dict['url']),
                             group = model.Group.by_name(group_name)
                             if not group:
                                 group = model.Group(name=group_name)
-                                self.group_names.add(group_name)
                                 model.Session.add(group)
+                                new_group_names.add(group_name)
                             pkg.groups.append(group)
                     elif attr == 'license':
                         pkg.license_id = val
                     elif attr == 'admins':
                         assert isinstance(val, list)
                         admins_list.append((item['name'], val))
-                        for user in val:
-                            if user not in self.user_names:
-                                self.user_names.append(user)
+                        for user_name in val:
+                            if user_name not in new_user_names:
+                                new_user_names.append(user_name)
                     else:
                         raise NotImplementedError(attr)
                 self.pkg_names.append(item['name'])
                 model.setup_default_user_roles(pkg)
             model.repo.commit_and_remove()
 
-        for user_name in self.user_names:
-            user = model.User(name=unicode(user_name))
-            model.Session.add(user)
+        needs_commit = False
+        for user_name in new_user_names:
+            if not model.User.by_name(unicode(user_name)):
+                user = model.User(name=unicode(user_name))
+                model.Session.add(user)
+                self.user_names.append(user_name)
+                needs_commit = True
 
         for pkg_name, admins in admins_list:
             pkg = model.Package.by_name(unicode(pkg_name))
             admins = [model.User.by_name(unicode(user_name)) for user_name in self.user_names]
             model.setup_default_user_roles(pkg, admins)
+            needs_commit = True
 
-        for group_name in self.group_names:
+        for group_name in new_group_names:
+            group = model.Group.by_name(unicode(group_name))
             model.setup_default_user_roles(group)
+            self.group_names.add(group_name)
+            needs_commit = True
+
+        if needs_commit:
+            model.repo.commit_and_remove()
+            needs_commit = False
 
         if relationships:
             rev = model.repo.new_revision() 
             for subject_name, relationship, object_name in relationships:
                 pkg(subject_name).add_relationship(
                     unicode(relationship), pkg(object_name))
+                needs_commit = True
 
             model.repo.commit_and_remove()
-    
+        
+        if commit_changesets:
+            from ckan.model.changeset import ChangesetRegister
+            changeset_ids = ChangesetRegister().commit()
+
     @classmethod
-    def create(self):
+    def create(self, commit_changesets=False):
         import ckan.model as model
         model.Session.remove()
         rev = model.repo.new_revision()
                              description=u'Roger likes these books.')
         for obj in [david, roger]:
             model.Session.add(obj)
-        self.group_names = (u'david', u'roger')
+        self.group_names.add(u'david')
+        self.group_names.add(u'roger')
         david.packages = [pkg1, pkg2]
         roger.packages = [pkg1]
         # authz
 
         model.repo.commit_and_remove()
 
+        if commit_changesets:
+            from ckan.model.changeset import ChangesetRegister
+            changeset_ids = ChangesetRegister().commit()
 
     @classmethod
     def delete(self):
+        '''Purges packages etc. that were created by this class.'''
         import ckan.model as model
         for pkg_name in self.pkg_names:
             pkg = model.Package.by_name(unicode(pkg_name))
                 tag.purge()
         revs = model.Session.query(model.Revision).filter_by(author=self.author)
         for rev in revs:
+            for pkg in rev.packages:
+                pkg.purge()
             model.Session.delete(rev)
         for group_name in self.group_names:
             group = model.Group.by_name(unicode(group_name))

ckan/lib/dumper.py

         for pkg in query:
             pkg_dict = pkg.as_dict()
             pkgs.append(pkg_dict)
-        json.dump(pkgs, dump_file_obj, indent=4, sort_keys=True)
+        json.dump(pkgs, dump_file_obj, indent=4)
 
 class Dumper(object):
     '''Dumps the database in same structure as it appears in the database'''
                 update.execute()
 
 class PackagesCsvWriter:
-    title_order = ('name', 'title', 'version')
-    
     def __init__(self, package_dict_list=None):
         self._rows = []
         self._col_titles = []
         titles_set = set()
         for row_dict in package_dict_list:
             for key in row_dict.keys():
-                titles_set.add(key)
-        self._col_titles = []
-        for ordered_title in self.title_order:
-            if ordered_title in titles_set:
-                titles_set.remove(ordered_title)
-                self._col_titles.append(ordered_title)
-        self._col_titles.extend(list(titles_set))
+                if key not in self._col_titles:
+                    self._col_titles.append(key)
         for row_dict in package_dict_list:
             self._add_row_dict(row_dict)
         

ckan/lib/field_types.py

     default_db_separator = '-'
     default_form_separator = '/'
     word_match = re.compile('[A-Za-z]+')
+    timezone_match = re.compile('(\s[A-Z]{3})|(\s[+-]\d\d:?\d\d)')
     months_chopped = [month[:3] for month in months]
 
     @classmethod
     def iso_to_db(self, iso_date, format):
-        # e.g. 'Wed, 06 Jan 2010 09:30:00 GMT'
-        #      '%a, %d %b %Y %H:%M:%S %Z'
-        if iso_date.endswith('+0100'):
-            iso_date = iso_date.replace('+0100', 'BST')
+        # e.g. 'Wed, 06 Jan 2010 09:30:00'
+        #      '%a, %d %b %Y %H:%M:%S'
         assert isinstance(iso_date, (unicode, str))
         try:
             date_tuple = time.strptime(iso_date, format)
         return date_str
 
     @classmethod
+    def strip_iso_timezone(self, iso_date):
+        return self.timezone_match.sub('', iso_date)
+
+    @classmethod
     def form_to_db(self, form_str, may_except=True):
         '''
         27/2/2005 -> 2005-02-27

ckan/lib/search.py

     order_by = 'rank'
     all_fields = False
     return_objects = False
+    ref_entity_with_attr = 'name'
 
     def __init__(self, kw_dict):
         if not kw_dict.keys():
                     results.append(result)
                 self._results['results'] = results
             else:
-                self._results['results'] = [entity.name for entity in self._results['results']]
+                attr_name = self._options.ref_entity_with_attr
+                self._results['results'] = [getattr(entity, attr_name) for entity in self._results['results']]
     
     def index_package(self, package):
         pass
                     "res_url", "text", "urls", "indexed_ts"]
 
     def __init__(self, solr_url=None):
-        if solr_url is None: 
+        if solr_url is None:
             solr_url = config.get('solr_url', 'http://localhost:8983/solr')
         # import inline to avoid external dependency 
         from solr import SolrConnection # == solrpy 

ckan/model/core.py

 from pylons import config
 from meta import *
 import vdm.sqlalchemy
+from sqlalchemy.util import OrderedDict
 
 from types import make_uuid
 import full_search
         sess.delete(self)
 
     def as_dict(self):
-        _dict = {}
+        _dict = OrderedDict()
         table = orm.class_mapper(self.__class__).mapped_table
         for col in table.c:
             val = getattr(self, col.name)
 
     def as_dict(self):
         _dict = DomainObject.as_dict(self)
+        # Set 'license' in _dict to cater for old clients.
+        # Todo: Remove this ASAP.
+        _dict['license'] = self.license.title if self.license else _dict.get('license_id', '')
         _dict['tags'] = [tag.name for tag in self.tags]
         _dict['groups'] = [group.name for group in self.groups]
         _dict['extras'] = dict([(extra.key, extra.value) for key, extra in self._extras.items()])

ckan/public/css/ckan.master.css

 @import url(buttons.css);
 @import url(autocomplete.css);
 @import url(stars.css);
+@import url(extra.css);
 
 /* ============= */
 /* = Utilities = */
 .package .details td.package-label {
   width: 150px;
 }
-

ckan/templates/group/authz.html

     <form action="" method="post">
       <h3>Update Existing Roles</h3>
       <table>
-      ${HTML(c.form)}
+      ${h.literal(c.form)}
       </table>
 
       <h3>Create New User Roles</h3>
-      ${HTML(c.new_roles_form)}
+      ${h.literal(c.new_roles_form)}
         
       <br/>
 

ckan/templates/layout.html

-<html
-  xmlns="http://www.w3.org/1999/xhtml"
+<html xmlns="http://www.w3.org/1999/xhtml"
   xmlns:i18n="http://genshi.edgewall.org/i18n"
-  xmlns:py="http://genshi.edgewall.org/"
+  xmlns:py="http://genshi.edgewall.org/" 
   xmlns:xi="http://www.w3.org/2001/XInclude"
+  xmlns:doap="http://usefulinc.com/ns/doap"
+  xmlns:foaf="http://xmlns.com/foaf/0.1/"
   py:strip=""
   >
-
-<xi:include href="_util.html" />
-
-<head>
-  <title>CKAN - Comprehensive Knowledge Archive Network - ${page_title()}</title>
-  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
-  <link rel="shortcut icon" href="http://m.okfn.org/gfx/logo/favicon.ico" type="image/x-icon" />
-  <py:choose>
-  <py:when test="defined('optional_feed')">
-    ${optional_feed()}
-  </py:when>
-  <py:otherwise>
-  <link rel="alternate" type="application/atom+xml" title="CKAN Recent Changes" href="/revision/list?format=atom&amp;days=1" />
-  </py:otherwise>
-  </py:choose>
-
-  <style type="text/css"> 
-    @import url(http://m.okfn.org/kforge/css/structure.css);
-    @import url(http://m.okfn.org/kforge/css/display.css);
-    @import url(http://m.okfn.org/kforge/css/navigation.css);
-    @import url(http://m.okfn.org/kforge/css/print.css);
-  </style> 
-
-  <link rel="stylesheet" href="/css/ckan.master.css" type="text/css" media="screen, print" />
-
-  <!--[if IE]>
-  <link rel="stylesheet" href="/css/ie.css" type="text/css" media="screen, print" />
-  <![endif]-->
-
-  <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.4/jquery.min.js"></script>
-  <script type="text/javascript" src="/scripts/application.js"></script>
-
-  <py:if test="defined('optional_head')">
-    ${optional_head()}
-  </py:if>
-</head>
-
-<body>
-<div id="airlock">
-  <!--[if IE]>
-    <hr class="holder" />
-  <![endif]-->
-  <div id="top">
-    <div id="top-inner">
-      <div id="top-bar">
-        <topbar py:choose="">
-	<p py:when="c.user" i18n:msg="openid_icon, user">
-            Logged in as <strong>${h.icon('openid')} ${c.user}</strong>
-            &middot; <a href="${h.url_for(controller='user',action=None)}">My account</a>
-	    &middot; <a href="${h.url_for('/logout_openid')}">Logout</a>
-          </p>
-          <p py:otherwise="" i18n:msg="openid_icon">
-            <a href="${h.url_for(controller='user',action='login', id=None)}">Login with ${h.icon('openid')} OpenID</a>
-          </p>
-        </topbar>
-      </div><!-- /top-bar -->
-
-      <h1>
-        <a href="${h.url_for('home')}"><img style="padding: 0; margin-top: -10px; margin-bottom: -10px;" src="/images/ckan_logo_fullname_long.png" alt="CKAN Logo" /></a>
-        <!-- <a href="/" title="CKAN Home">Comprehensive Knowledge Archive Network</a> -->
-      </h1>
-
-    </div><!-- /top-inner -->
-
-    <h3 class="hidden">Sections:</h3>
-    <ul id="navigation">
-      <li>${h.nav_link(c, _('Home'), controller='home', action='index', id=None)}</li>
-      <li>${h.nav_link(c, _('Packages'), controller='package', action='index', id=None)}</li>
-      <li>${h.nav_link(c, _('Tags'), controller='tag', action='index', id=None)}</li>
-      <li>${h.nav_link(c, _('Groups'), controller='group', action='index', id=None)}</li>
-      <li>${h.nav_link(c, _('Recent changes'), controller='revision', action='index', id=None)}</li>
-			<li>${h.nav_link(c, _('API'), controller='rest', action='index', id=None)}</li>
-    </ul>
-    <h3 class="hidden">In this section:</h3>
-    <!--! This is a default (blank) subnav. -->
-    <py:match path="minornavigation"></py:match>
-    <ul id="subnav"><minornavigation></minornavigation></ul>
-
-    <!--[if IE]>
-    <hr class="holder" />
-    <![endif]-->
-
-  </div><!-- /top -->
-
-  <p class="hidden"><a href="#main" title="Skip to page content">[ Skip to main content ]</a></p>
-
-  <div id="primary" class="sidebar">
-    <primarysidebar>
-    <!-- Primary Side Bar Goes Here -->
-    </primarysidebar>
-
-    <!--[if IE]>
-    <hr class="primary" />
-    <![endif]-->
-  </div><!-- /primary -->
-
-  <div id="main">
-    <content>
-      <p>Master content template placeholder &hellip; please replace me.</p>
-    </content>
-
-    <div id="footer">
-      <p>
-        <a href="http://validator.w3.org/check/referer" title="Valid XHTML 1.1">XHTML</a>
-        | <a href="http://jigsaw.w3.org/css-validator/check/referer">CSS</a>
-        | <a href="http://www.okfn.org/ckan/">Project Home Page</a>
-        | <a href="http://www.okfn.org/contact/">Contact Us</a>
-        | <a href="http://www.okfn.org/privacy-policy/">Privacy Policy</a>
-      </p>
-      <p i18n:msg="">
-        <img style="margin-bottom: -5px;" src="http://m.okfn.org/images/logo/okf_logo_white_and_green_tiny.png" /> An <a href="http://www.okfn.org/">Open Knowledge Foundation</a> Project
-      </p>
-      <p i18n:msg="version">
-        v${c.__version__}
-        | (c) Open Knowledge Foundation
-	| All material available under an <a href="${h.url_for('license')}">open license</a>
-        | <a href="http://www.opendefinition.org/1.0/"><img
-            style="border: none; margin-bottom: -4px;"
-            src="http://m.okfn.org/images/ok_buttons/ok_90x15_blue.png"
-            alt="This Content and Data is Open" /></a>
-        | <a href="http://www.opendefinition.org/1.0/"><img
-            style="border: none; margin-bottom: -4px;"
-            src="http://m.okfn.org/images/ok_buttons/od_80x15_blue.png"
-            alt="This Content and Data is Open" /></a>
-      </p>
-    </div><!-- /footer -->
-
-    <!--[if IE]>
-    <hr class="main" />
-    <![endif]-->
-
-  </div><!-- /main -->
-
-</div><!-- /airlock -->
-
-<!--! No need to load GA in development -->
-<py:if test="not config['debug']">
-  <!-- Google Analytics -->
-  <script src='http://www.google-analytics.com/ga.js' type='text/javascript'></script>
-  <script type="text/javascript">
-  try {
-  var pageTracker = _gat._getTracker("UA-8271754-2");
-  pageTracker._setDomainName(".ckan.net");
-  pageTracker._trackPageview();
-  } catch(err) {}
-  </script>
-  <!-- /Google Analytics -->
-</py:if>
-
-</body>
+  <xi:include href="layout_base.html" />
 </html>

ckan/templates/layout_base.html

+<html
+  xmlns="http://www.w3.org/1999/xhtml"
+  xmlns:i18n="http://genshi.edgewall.org/i18n"
+  xmlns:py="http://genshi.edgewall.org/"
+  xmlns:xi="http://www.w3.org/2001/XInclude"
+  py:strip=""
+  >
+
+<xi:include href="_util.html" />
+
+<head>
+  <title>CKAN - Comprehensive Knowledge Archive Network - ${page_title()}</title>
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+  <link rel="shortcut icon" href="http://m.okfn.org/gfx/logo/favicon.ico" type="image/x-icon" />
+  <py:choose>
+  <py:when test="defined('optional_feed')">
+    ${optional_feed()}
+  </py:when>
+  <py:otherwise>
+  <link rel="alternate" type="application/atom+xml" title="CKAN Recent Changes" href="/revision/list?format=atom&amp;days=1" />
+  </py:otherwise>
+  </py:choose>
+
+  <style type="text/css"> 
+    @import url(http://m.okfn.org/kforge/css/structure.css);
+    @import url(http://m.okfn.org/kforge/css/display.css);
+    @import url(http://m.okfn.org/kforge/css/navigation.css);
+    @import url(http://m.okfn.org/kforge/css/print.css);
+  </style> 
+
+  <link rel="stylesheet" href="/css/ckan.master.css" type="text/css" media="screen, print" />
+
+  <!--[if IE]>
+  <link rel="stylesheet" href="/css/ie.css" type="text/css" media="screen, print" />
+  <![endif]-->
+
+  <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.4/jquery.min.js"></script>
+  <script type="text/javascript" src="/scripts/application.js"></script>
+
+  <py:if test="defined('optional_head')">
+    ${optional_head()}
+  </py:if>
+</head>
+
+<body>
+<div id="airlock">
+  <!--[if IE]>
+    <hr class="holder" />
+  <![endif]-->
+  <div id="top">
+    <div id="top-inner">
+      <div id="top-bar">
+        <topbar py:choose="">
+	<p py:when="c.user" i18n:msg="openid_icon, user">
+            Logged in as <strong>${h.icon('openid')} ${c.user}</strong>
+            &middot; <a href="${h.url_for(controller='user',action=None)}">My account</a>
+	    &middot; <a href="${h.url_for('/logout_openid')}">Logout</a>
+          </p>
+          <p py:otherwise="" i18n:msg="openid_icon">
+            <a href="${h.url_for(controller='user',action='login', id=None)}">Login with ${h.icon('openid')} OpenID</a>
+          </p>
+        </topbar>
+      </div><!-- /top-bar -->
+
+      <h1>
+        <a href="${h.url_for('home')}"><img style="padding: 0; margin-top: -10px; margin-bottom: -10px;" src="/images/ckan_logo_fullname_long.png" alt="CKAN Logo" /></a>
+        <!-- <a href="/" title="CKAN Home">Comprehensive Knowledge Archive Network</a> -->
+      </h1>
+
+    </div><!-- /top-inner -->
+
+    <h3 class="hidden">Sections:</h3>