Marcin Kuzminski avatar Marcin Kuzminski committed ca41d54 Merge

Merge with 6aa7db1c083a1384ebff5c2bb3c943a035bb310d - celery branch

Comments (0)

Files changed (64)

 - full permissions per project read/write/admin access even on mercurial request
 - mako templates let's you cusmotize look and feel of application.
 - diffs annotations and source code all colored by pygments.
-- mercurial branch graph and yui-flot powered graphs
+- mercurial branch graph and yui-flot powered graphs with zooming
 - admin interface for performing user/permission managments as well as repository
   managment. 
+- full text search of source codes with indexing daemons using whoosh
+  (no external search servers required all in one application)
+- async tasks for speed and performance using celery (works without them too)  
 - Additional settings for mercurial web, (hooks editable from admin
   panel !) also manage paths, archive, remote messages  
 - backup scripts can do backup of whole app and send it over scp to desired location
 **Incoming**
 
 - code review based on hg-review (when it's stable)
-- git support (when vcs can handle it)
-- full text search of source codes with indexing daemons using whoosh
-  (no external search servers required all in one application)
-- manage hg ui() per repo, add hooks settings, per repo, and not globally
-- other cools stuff that i can figure out
+- git support (when vcs can handle it - almost there !)
+- commit based wikis
+- in server forks
+- clonning from remote repositories into hg-app 
+- other cools stuff that i can figure out (or You can help me figure out)
 
 .. note::
    This software is still in beta mode. 
    
 - create new virtualenv and activate it - highly recommend that you use separate
   virtual-env for whole application
-- download hg app from default (not demo) branch from bitbucket and run 
+- download hg app from default branch from bitbucket and run 
   'python setup.py install' this will install all required dependencies needed
 - run paster setup-app production.ini it should create all needed tables 
-  and an admin account. 
+  and an admin account make sure You specify correct path to repositories. 
 - remember that the given path for mercurial repositories must be write 
   accessible for the application
 - run paster serve development.ini - or you can use manage-hg_app script.
 - use admin account you created to login.
 - default permissions on each repository is read, and owner is admin. So remember
   to update these.
+- in order to use full power of async tasks, You must install message broker
+  preferrably rabbitmq and start celeryd daemon. The app should gain some speed 
+  than. For installation instructions 
+  You can visit: http://ask.github.com/celery/getting-started/index.html. All
+  needed configs are inside hg-app ie. celeryconfig.py
      
+# List of modules to import when celery starts.
+import sys
+import os
+import ConfigParser
+root = os.getcwd()
+
+PYLONS_CONFIG_NAME = 'development.ini'
+
+sys.path.append(root)
+config = ConfigParser.ConfigParser({'here':root})
+config.read('%s/%s' % (root, PYLONS_CONFIG_NAME))
+PYLONS_CONFIG = config
+
+CELERY_IMPORTS = ("pylons_app.lib.celerylib.tasks",)
+
+## Result store settings.
+CELERY_RESULT_BACKEND = "database"
+CELERY_RESULT_DBURI = dict(config.items('app:main'))['sqlalchemy.db1.url']
+CELERY_RESULT_SERIALIZER = 'json'
+
+
+BROKER_CONNECTION_MAX_RETRIES = 30
+
+## Broker settings.
+BROKER_HOST = "localhost"
+BROKER_PORT = 5672
+BROKER_VHOST = "rabbitmqhost"
+BROKER_USER = "rabbitmq"
+BROKER_PASSWORD = "qweqwe"
+
+## Worker settings
+## If you're doing mostly I/O you can have more processes,
+## but if mostly spending CPU, try to keep it close to the
+## number of CPUs on your machine. If not set, the number of CPUs/cores
+## available will be used.
+CELERYD_CONCURRENCY = 2
+# CELERYD_LOG_FILE = "celeryd.log"
+CELERYD_LOG_LEVEL = "DEBUG"
+CELERYD_MAX_TASKS_PER_CHILD = 1
+
+#Tasks will never be sent to the queue, but executed locally instead.
+CELERY_ALWAYS_EAGER = False
+
+#===============================================================================
+# EMAIL SETTINGS
+#===============================================================================
+pylons_email_config = dict(config.items('DEFAULT'))
+
+CELERY_SEND_TASK_ERROR_EMAILS = True
+
+#List of (name, email_address) tuples for the admins that should receive error e-mails.
+ADMINS = [('Administrator', pylons_email_config.get('email_to'))]
+
+#The e-mail address this worker sends e-mails from. Default is "celery@localhost".
+SERVER_EMAIL = pylons_email_config.get('error_email_from')
+
+#The mail server to use. Default is "localhost".
+MAIL_HOST = pylons_email_config.get('smtp_server')
+
+#Username (if required) to log on to the mail server with.
+MAIL_HOST_USER = pylons_email_config.get('smtp_username')
+
+#Password (if required) to log on to the mail server with.
+MAIL_HOST_PASSWORD = pylons_email_config.get('smtp_password')
+
+MAIL_PORT = pylons_email_config.get('smtp_port')
+
+
+#===============================================================================
+# INSTRUCTIONS FOR RABBITMQ
+#===============================================================================
+# rabbitmqctl add_user rabbitmq qweqwe
+# rabbitmqctl add_vhost rabbitmqhost
+# rabbitmqctl set_permissions -p rabbitmqhost rabbitmq ".*" ".*" ".*"
 ################################################################################
 ################################################################################
-# pylons_app - Pylons environment configuration                                #
+# hg-app - Pylons environment configuration                                    #
 #                                                                              # 
 # The %(here)s variable will be replaced with the parent directory of this file#
 ################################################################################
 
 [DEFAULT]
 debug = true
-############################################
-## Uncomment and replace with the address ##
-## which should receive any error reports ##
-############################################
+################################################################################
+## Uncomment and replace with the address which should receive                ## 
+## any error reports after application crash								  ##
+## Additionally those settings will be used by hg-app mailing system          ##
+################################################################################
 #email_to = admin@localhost
+#error_email_from = paste_error@localhost
+#app_email_from = hg-app-noreply@localhost
+#error_message =
+
 #smtp_server = mail.server.com
-#error_email_from = paste_error@localhost
 #smtp_username = 
-#smtp_password = 
-#error_message = 'mercurial crash !'
+#smtp_password =
+#smtp_port = 
+#smtp_use_tls = 
 
 [server:main]
 ##nr of threads to spawn
 threadpool_workers = 5
 
 ##max request before
-threadpool_max_requests = 2
+threadpool_max_requests = 6
 
 ##option to use threads of process
-use_threadpool = true
+use_threadpool = false
 
 use = egg:Paste#http
 host = 127.0.0.1
 ###       BEAKER SESSION        ####
 ####################################
 ## Type of storage used for the session, current types are 
-## dbm, file, memcached, database, and memory. 
+## "dbm", "file", "memcached", "database", and "memory". 
 ## The storage uses the Container API 
 ##that is also used by the cache system.
 beaker.session.type = file
 ################################################################################
 ################################################################################
-# pylons_app - Pylons environment configuration                                #
+# hg-app - Pylons environment configuration                                    #
 #                                                                              # 
 # The %(here)s variable will be replaced with the parent directory of this file#
 ################################################################################
 
 [DEFAULT]
 debug = true
-############################################
-## Uncomment and replace with the address ##
-## which should receive any error reports ##
-############################################
+################################################################################
+## Uncomment and replace with the address which should receive                ## 
+## any error reports after application crash								  ##
+## Additionally those settings will be used by hg-app mailing system          ##
+################################################################################
 #email_to = admin@localhost
+#error_email_from = paste_error@localhost
+#app_email_from = hg-app-noreply@localhost
+#error_message =
+
 #smtp_server = mail.server.com
-#error_email_from = paste_error@localhost
 #smtp_username = 
 #smtp_password = 
-#error_message = 'mercurial crash !'
+#smtp_port = 
+#smtp_use_tls = false
 
 [server:main]
 ##nr of threads to spawn
 threadpool_workers = 5
 
-##max request before
+##max request before thread respawn
 threadpool_max_requests = 2
 
 ##option to use threads of process

pylons_app/__init__.py

 """
 Created on April 9, 2010
 Hg app, a web based mercurial repository managment based on pylons
+versioning implementation: http://semver.org/
 @author: marcink
 """
 
-VERSION = (0, 8, 2, 'beta')
+VERSION = (0, 8, 3, 'beta')
 
 __version__ = '.'.join((str(each) for each in VERSION[:4]))
 

pylons_app/config/deployment.ini_tmpl

 
 [DEFAULT]
 debug = true
-############################################
-## Uncomment and replace with the address ##
-## which should receive any error reports ##
-############################################
+################################################################################
+## Uncomment and replace with the address which should receive                ## 
+## any error reports after application crash								  ##
+## Additionally those settings will be used by hg-app mailing system          ##
+################################################################################
 #email_to = admin@localhost
+#error_email_from = paste_error@localhost
+#app_email_from = hg-app-noreply@localhost
+#error_message =
+
 #smtp_server = mail.server.com
-#error_email_from = paste_error@localhost
 #smtp_username = 
 #smtp_password = 
-#error_message = 'hp-app crash !'
+#smtp_port = 
+#smtp_use_tls = false
 
 [server:main]
 ##nr of threads to spawn

pylons_app/config/environment.py

 
     #sets the c attribute access when don't existing attribute are accessed
     config['pylons.strict_tmpl_context'] = True
-    test = os.path.split(config['__file__'])[-1] == 'tests.ini'
+    test = os.path.split(config['__file__'])[-1] == 'test.ini'
+    if test:
+        from pylons_app.lib.utils import create_test_env, create_test_index
+        create_test_env('/tmp', config)
+        create_test_index('/tmp/*', True)
+        
     #MULTIPLE DB configs
     # Setup the SQLAlchemy database engine
     if config['debug'] and not test:

pylons_app/config/routing.py

     #SEARCH
     map.connect('search', '/_admin/search', controller='search')
     
-    #LOGIN/LOGOUT
+    #LOGIN/LOGOUT/REGISTER/SIGN IN
     map.connect('login_home', '/_admin/login', controller='login')
     map.connect('logout_home', '/_admin/logout', controller='login', action='logout')
     map.connect('register', '/_admin/register', controller='login', action='register')
+    map.connect('reset_password', '/_admin/password_reset', controller='login', action='password_reset')
         
     #FEEDS
     map.connect('rss_feed_home', '/{repo_name:.*}/feed/rss',
                 controller='changeset', revision='tip',
                 conditions=dict(function=check_repo))
     map.connect('raw_changeset_home', '/{repo_name:.*}/raw-changeset/{revision}',
-                controller='changeset',action='raw_changeset', revision='tip',
+                controller='changeset', action='raw_changeset', revision='tip',
                 conditions=dict(function=check_repo))
     map.connect('summary_home', '/{repo_name:.*}/summary',
                 controller='summary', conditions=dict(function=check_repo))
     map.connect('files_diff_home', '/{repo_name:.*}/diff/{f_path:.*}',
                 controller='files', action='diff', revision='tip', f_path='',
                 conditions=dict(function=check_repo))
-    map.connect('files_raw_home', '/{repo_name:.*}/rawfile/{revision}/{f_path:.*}',
+    map.connect('files_rawfile_home', '/{repo_name:.*}/rawfile/{revision}/{f_path:.*}',
                 controller='files', action='rawfile', revision='tip', f_path='',
                 conditions=dict(function=check_repo))
+    map.connect('files_raw_home', '/{repo_name:.*}/raw/{revision}/{f_path:.*}',
+                controller='files', action='raw', revision='tip', f_path='',
+                conditions=dict(function=check_repo))
     map.connect('files_annotate_home', '/{repo_name:.*}/annotate/{revision}/{f_path:.*}',
                 controller='files', action='annotate', revision='tip', f_path='',
                 conditions=dict(function=check_repo))    

pylons_app/controllers/admin/settings.py

     ApplicationUiSettingsForm
 from pylons_app.model.hg_model import HgModel
 from pylons_app.model.user_model import UserModel
+from pylons_app.lib.celerylib import tasks, run_task
 import formencode
 import logging
 import traceback
             invalidate_cache('cached_repo_list')
             h.flash(_('Repositories sucessfully rescanned'), category='success')            
         
+        if setting_id == 'whoosh':
+            repo_location = get_hg_ui_settings()['paths_root_path']
+            full_index = request.POST.get('full_index', False)
+            task = run_task(tasks.whoosh_index, repo_location, full_index)
+            
+            h.flash(_('Whoosh reindex task scheduled'), category='success')
         if setting_id == 'global':
             
             application_form = ApplicationSettingsForm()()
         # url('admin_settings_my_account_update', id=ID)
         user_model = UserModel()
         uid = c.hg_app_user.user_id
-        _form = UserForm(edit=True, old_data={'user_id':uid})()
+        _form = UserForm(edit=True, old_data={'user_id':uid,
+                                              'email':c.hg_app_user.email})()
         form_result = {}
         try:
             form_result = _form.to_python(dict(request.POST))
                     category='success')
                            
         except formencode.Invalid as errors:
-            #c.user = self.sa.query(User).get(c.hg_app_user.user_id)
+            c.user = self.sa.query(User).get(c.hg_app_user.user_id)
+            c.user_repos = []
+            for repo in c.cached_repo_list.values():
+                if repo.dbrepo.user.username == c.user.username:
+                    c.user_repos.append(repo)            
             return htmlfill.render(
                 render('admin/users/user_edit_my_account.html'),
                 defaults=errors.value,

pylons_app/controllers/admin/users.py

         #           method='put')
         # url('user', id=ID)
         user_model = UserModel()
-        _form = UserForm(edit=True, old_data={'user_id':id})()
+        c.user = user_model.get_user(id)
+        
+        _form = UserForm(edit=True, old_data={'user_id':id,
+                                              'email':c.user.email})()
         form_result = {}
         try:
             form_result = _form.to_python(dict(request.POST))
             h.flash(_('User updated succesfully'), category='success')
                            
         except formencode.Invalid as errors:
-            c.user = user_model.get_user(id)
             return htmlfill.render(
                 render('admin/users/user_edit.html'),
                 defaults=errors.value,
         """GET /users/id/edit: Form to edit an existing item"""
         # url('edit_user', id=ID)
         c.user = self.sa.query(User).get(id)
+        if not c.user:
+            return redirect(url('users'))
         if c.user.username == 'default':
             h.flash(_("You can't edit this user since it's" 
               " crucial for entire application"), category='warning')

pylons_app/controllers/files.py

                                    'repository.admin')       
     def __before__(self):
         super(FilesController, self).__before__()
+        c.file_size_limit = 250 * 1024 #limit of file size to display
 
     def index(self, repo_name, revision, f_path):
         hg_model = HgModel()
                              revision=next_rev, f_path=f_path)   
                     
             c.changeset = repo.get_changeset(revision)
-
                         
             c.cur_rev = c.changeset.raw_id
             c.rev_nr = c.changeset.revision
         response.content_disposition = 'attachment; filename=%s' \
                                                     % f_path.split('/')[-1] 
         return file_node.content
+
+    def raw(self, repo_name, revision, f_path):
+        hg_model = HgModel()
+        c.repo = hg_model.get_repo(c.repo_name)
+        file_node = c.repo.get_changeset(revision).get_node(f_path)
+        response.content_type = 'text/plain'
+        
+        return file_node.content
     
     def annotate(self, repo_name, revision, f_path):
         hg_model = HgModel()

pylons_app/controllers/login.py

 from pylons.controllers.util import abort, redirect
 from pylons_app.lib.auth import AuthUser, HasPermissionAnyDecorator
 from pylons_app.lib.base import BaseController, render
-from pylons_app.model.forms import LoginForm, RegisterForm
+import pylons_app.lib.helpers as h 
+from pylons.i18n.translation import _
+from pylons_app.model.forms import LoginForm, RegisterForm, PasswordResetForm
 from pylons_app.model.user_model import UserModel
 import formencode
 import logging
 
     def index(self):
         #redirect if already logged in
-        c.came_from = request.GET.get('came_from',None)
+        c.came_from = request.GET.get('came_from', None)
         
         if c.hg_app_user.is_authenticated:
             return redirect(url('hg_home'))
                         
         return render('/login.html')
     
-    @HasPermissionAnyDecorator('hg.admin', 'hg.register.auto_activate', 
+    @HasPermissionAnyDecorator('hg.admin', 'hg.register.auto_activate',
                                'hg.register.manual_activate')
     def register(self):
         user_model = UserModel()
                 form_result = register_form.to_python(dict(request.POST))
                 form_result['active'] = c.auto_active
                 user_model.create_registration(form_result)
+                h.flash(_('You have successfully registered into hg-app'),
+                            category='success')                
                 return redirect(url('login_home'))
                                
             except formencode.Invalid as errors:
                     encoding="UTF-8")
         
         return render('/register.html')
-    
+
+    def password_reset(self):
+        user_model = UserModel()
+        if request.POST:
+                
+            password_reset_form = PasswordResetForm()()
+            try:
+                form_result = password_reset_form.to_python(dict(request.POST))
+                user_model.reset_password(form_result)
+                h.flash(_('Your new password was sent'),
+                            category='success')                 
+                return redirect(url('login_home'))
+                               
+            except formencode.Invalid as errors:
+                return htmlfill.render(
+                    render('/password_reset.html'),
+                    defaults=errors.value,
+                    errors=errors.error_dict or {},
+                    prefix_error=False,
+                    encoding="UTF-8")
+        
+        return render('/password_reset.html')
+        
     def logout(self):
         session['hg_app_user'] = AuthUser()
         session.save()

pylons_app/controllers/search.py

 from pylons.controllers.util import abort, redirect
 from pylons_app.lib.auth import LoginRequired
 from pylons_app.lib.base import BaseController, render
-from pylons_app.lib.indexers import ANALYZER, IDX_LOCATION, SCHEMA, IDX_NAME
-from webhelpers.html.builder import escape
-from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter, \
-    ContextFragmenter
+from pylons_app.lib.indexers import IDX_LOCATION, SCHEMA, IDX_NAME, ResultWrapper
+from webhelpers.paginate import Page
+from webhelpers.util import update_params
 from pylons.i18n.translation import _
 from whoosh.index import open_dir, EmptyIndexError
 from whoosh.qparser import QueryParser, QueryParserError
     def __before__(self):
         super(SearchController, self).__before__()    
 
-
     def index(self):
         c.formated_results = []
         c.runtime = ''
-        search_items = set()
         c.cur_query = request.GET.get('q', None)
         if c.cur_query:
             cur_query = c.cur_query.lower()
         
-        
         if c.cur_query:
+            p = int(request.params.get('page', 1))
+            highlight_items = set()
             try:
                 idx = open_dir(IDX_LOCATION, indexname=IDX_NAME)
                 searcher = idx.searcher()
-            
+
                 qp = QueryParser("content", schema=SCHEMA)
                 try:
                     query = qp.parse(unicode(cur_query))
                     
                     if isinstance(query, Phrase):
-                        search_items.update(query.words)
+                        highlight_items.update(query.words)
                     else:
                         for i in query.all_terms():
-                            search_items.add(i[1])
-                        
+                            if i[0] == 'content':
+                                highlight_items.add(i[1])
+
+                    matcher = query.matcher(searcher)
+                    
                     log.debug(query)
-                    log.debug(search_items)
+                    log.debug(highlight_items)
                     results = searcher.search(query)
+                    res_ln = len(results)
                     c.runtime = '%s results (%.3f seconds)' \
-                    % (len(results), results.runtime)
+                    % (res_ln, results.runtime)
+                    
+                    def url_generator(**kw):
+                        return update_params("?q=%s" % c.cur_query, **kw)
 
-                    analyzer = ANALYZER
-                    formatter = HtmlFormatter('span',
-                        between='\n<span class="break">...</span>\n') 
-                    
-                    #how the parts are splitted within the same text part
-                    fragmenter = SimpleFragmenter(200)
-                    #fragmenter = ContextFragmenter(search_items)
-                    
-                    for res in results:
-                        d = {}
-                        d.update(res)
-                        hl = highlight(escape(res['content']), search_items,
-                                                         analyzer=analyzer,
-                                                         fragmenter=fragmenter,
-                                                         formatter=formatter,
-                                                         top=5)
-                        f_path = res['path'][res['path'].find(res['repository']) \
-                                             + len(res['repository']):].lstrip('/')
-                        d.update({'content_short':hl,
-                                  'f_path':f_path})
-                        #del d['content']
-                        c.formated_results.append(d)
-                                                    
+                    c.formated_results = Page(
+                                ResultWrapper(searcher, matcher, highlight_items),
+                                page=p, item_count=res_ln,
+                                items_per_page=10, url=url_generator)
+                           
                 except QueryParserError:
                     c.runtime = _('Invalid search query. Try quoting it.')
-
+                searcher.close()
             except (EmptyIndexError, IOError):
                 log.error(traceback.format_exc())
                 log.error('Empty Index data')
                 c.runtime = _('There is no index to search in. Please run whoosh indexer')
-            
-
-                
+                        
         # Return a rendered template
         return render('/search/search.html')

pylons_app/controllers/summary.py

 summary controller for pylons
 @author: marcink
 """
-from datetime import datetime, timedelta
-from pylons import tmpl_context as c, request
+from pylons import tmpl_context as c, request, url
 from pylons_app.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from pylons_app.lib.base import BaseController, render
-from pylons_app.lib.helpers import person
 from pylons_app.lib.utils import OrderedDict
 from pylons_app.model.hg_model import HgModel
+from pylons_app.model.db import Statistics
+from webhelpers.paginate import Page
+from pylons_app.lib.celerylib import run_task
+from pylons_app.lib.celerylib.tasks import get_commits_stats
+from datetime import datetime, timedelta
 from time import mktime
-from webhelpers.paginate import Page
 import calendar
 import logging
 
         c.repo_branches = OrderedDict()
         for name, hash in c.repo_info.branches.items()[:10]:
             c.repo_branches[name] = c.repo_info.get_changeset(hash)
+        
+        td = datetime.today() + timedelta(days=1) 
+        y, m, d = td.year, td.month, td.day
+        
+        ts_min_y = mktime((y - 1, (td - timedelta(days=calendar.mdays[m])).month,
+                            d, 0, 0, 0, 0, 0, 0,))
+        ts_min_m = mktime((y, (td - timedelta(days=calendar.mdays[m])).month,
+                            d, 0, 0, 0, 0, 0, 0,))
+        
+        ts_max_y = mktime((y, m, d, 0, 0, 0, 0, 0, 0,))
+            
+        run_task(get_commits_stats, c.repo_info.name, ts_min_y, ts_max_y)
+        c.ts_min = ts_min_m
+        c.ts_max = ts_max_y
+        
+        
+        stats = self.sa.query(Statistics)\
+            .filter(Statistics.repository == c.repo_info.dbrepo)\
+            .scalar()
 
-        c.commit_data = self.__get_commit_stats(c.repo_info)
+        if stats:
+            c.commit_data = stats.commit_activity
+            c.overview_data = stats.commit_activity_combined
+        else:
+            import json
+            c.commit_data = json.dumps({})
+            c.overview_data = json.dumps([[ts_min_y, 0], [ts_max_y, 0] ])
         
         return render('summary/summary.html')
 
-
-
-    def __get_commit_stats(self, repo):
-        aggregate = OrderedDict()
-        
-        #graph range
-        td = datetime.today() + timedelta(days=1) 
-        y, m, d = td.year, td.month, td.day
-        c.ts_min = mktime((y, (td - timedelta(days=calendar.mdays[m])).month,
-                            d, 0, 0, 0, 0, 0, 0,))
-        c.ts_max = mktime((y, m, d, 0, 0, 0, 0, 0, 0,))
-        
-        def author_key_cleaner(k):
-            k = person(k)
-            k = k.replace('"', "'") #for js data compatibilty
-            return k
-                
-        for cs in repo[:200]:#added limit 200 until fix #29 is made
-            k = '%s-%s-%s' % (cs.date.timetuple()[0], cs.date.timetuple()[1],
-                              cs.date.timetuple()[2])
-            timetupple = [int(x) for x in k.split('-')]
-            timetupple.extend([0 for _ in xrange(6)])
-            k = mktime(timetupple)
-            if aggregate.has_key(author_key_cleaner(cs.author)):
-                if aggregate[author_key_cleaner(cs.author)].has_key(k):
-                    aggregate[author_key_cleaner(cs.author)][k]["commits"] += 1
-                    aggregate[author_key_cleaner(cs.author)][k]["added"] += len(cs.added)
-                    aggregate[author_key_cleaner(cs.author)][k]["changed"] += len(cs.changed)
-                    aggregate[author_key_cleaner(cs.author)][k]["removed"] += len(cs.removed)
-                    
-                else:
-                    #aggregate[author_key_cleaner(cs.author)].update(dates_range)
-                    if k >= c.ts_min and k <= c.ts_max:
-                        aggregate[author_key_cleaner(cs.author)][k] = {}
-                        aggregate[author_key_cleaner(cs.author)][k]["commits"] = 1
-                        aggregate[author_key_cleaner(cs.author)][k]["added"] = len(cs.added)
-                        aggregate[author_key_cleaner(cs.author)][k]["changed"] = len(cs.changed)
-                        aggregate[author_key_cleaner(cs.author)][k]["removed"] = len(cs.removed) 
-                                            
-            else:
-                if k >= c.ts_min and k <= c.ts_max:
-                    aggregate[author_key_cleaner(cs.author)] = OrderedDict()
-                    #aggregate[author_key_cleaner(cs.author)].update(dates_range)
-                    aggregate[author_key_cleaner(cs.author)][k] = {}
-                    aggregate[author_key_cleaner(cs.author)][k]["commits"] = 1
-                    aggregate[author_key_cleaner(cs.author)][k]["added"] = len(cs.added)
-                    aggregate[author_key_cleaner(cs.author)][k]["changed"] = len(cs.changed)
-                    aggregate[author_key_cleaner(cs.author)][k]["removed"] = len(cs.removed)                 
-        
-        d = ''
-        tmpl0 = u""""%s":%s"""
-        tmpl1 = u"""{label:"%s",data:%s,schema:["commits"]},"""
-        for author in aggregate:
-            
-            d += tmpl0 % (author,
-                          tmpl1 \
-                          % (author,
-                        [{"time":x,
-                          "commits":aggregate[author][x]['commits'],
-                          "added":aggregate[author][x]['added'],
-                          "changed":aggregate[author][x]['changed'],
-                          "removed":aggregate[author][x]['removed'],
-                          } for x in aggregate[author]]))
-        if d == '':
-            d = '"%s":{label:"%s",data:[[0,1],]}' \
-                % (author_key_cleaner(repo.contact),
-                   author_key_cleaner(repo.contact))
-        return d
-
-

pylons_app/lib/auth.py

 import bcrypt
 from decorator import decorator
 import logging
+import random
 
 log = logging.getLogger(__name__) 
 
+class PasswordGenerator(object):
+    """This is a simple class for generating password from
+        different sets of characters
+        usage:
+        passwd_gen = PasswordGenerator()
+        #print 8-letter password containing only big and small letters of alphabet
+        print passwd_gen.gen_password(8, passwd_gen.ALPHABETS_BIG_SMALL)        
+    """
+    ALPHABETS_NUM = r'''1234567890'''#[0]
+    ALPHABETS_SMALL = r'''qwertyuiopasdfghjklzxcvbnm'''#[1]
+    ALPHABETS_BIG = r'''QWERTYUIOPASDFGHJKLZXCVBNM'''#[2]
+    ALPHABETS_SPECIAL = r'''`-=[]\;',./~!@#$%^&*()_+{}|:"<>?'''    #[3]
+    ALPHABETS_FULL = ALPHABETS_BIG + ALPHABETS_SMALL + ALPHABETS_NUM + ALPHABETS_SPECIAL#[4]
+    ALPHABETS_ALPHANUM = ALPHABETS_BIG + ALPHABETS_SMALL + ALPHABETS_NUM#[5]
+    ALPHABETS_BIG_SMALL = ALPHABETS_BIG + ALPHABETS_SMALL
+    ALPHABETS_ALPHANUM_BIG = ALPHABETS_BIG + ALPHABETS_NUM#[6]
+    ALPHABETS_ALPHANUM_SMALL = ALPHABETS_SMALL + ALPHABETS_NUM#[7]
+            
+    def __init__(self, passwd=''):
+        self.passwd = passwd
+
+    def gen_password(self, len, type):
+        self.passwd = ''.join([random.choice(type) for _ in xrange(len)])
+        return self.passwd
+
+    
 def get_crypt_password(password):
     """Cryptographic function used for password hashing based on sha1
     @param password: password to hash
 
             p = request.environ.get('PATH_INFO')
             if request.environ.get('QUERY_STRING'):
-                p+='?'+request.environ.get('QUERY_STRING')
-            log.debug('redirecting to login page with %s',p)                
-            return redirect(url('login_home',came_from=p))
+                p += '?' + request.environ.get('QUERY_STRING')
+            log.debug('redirecting to login page with %s', p)                
+            return redirect(url('login_home', came_from=p))
 
 class PermsDecorator(object):
     """Base class for decorators"""

pylons_app/lib/celerylib/__init__.py

+from pylons_app.lib.pidlock import DaemonLock, LockHeld
+from vcs.utils.lazy import LazyProperty
+from decorator import decorator
+import logging
+import os
+import sys
+import traceback
+from hashlib import md5
+log = logging.getLogger(__name__)
+
+class ResultWrapper(object):
+    def __init__(self, task):
+        self.task = task
+        
+    @LazyProperty
+    def result(self):
+        return self.task
+
+def run_task(task, *args, **kwargs):
+    try:
+        t = task.delay(*args, **kwargs)
+        log.info('running task %s', t.task_id)
+        return t
+    except Exception, e:
+        print e
+        if e.errno == 111:
+            log.debug('Unnable to connect. Sync execution')
+        else:
+            log.error(traceback.format_exc())
+        #pure sync version
+        return ResultWrapper(task(*args, **kwargs))
+
+
+class LockTask(object):
+    """LockTask decorator"""
+    
+    def __init__(self, func):
+        self.func = func
+        
+    def __call__(self, func):
+        return decorator(self.__wrapper, func)
+    
+    def __wrapper(self, func, *fargs, **fkwargs):
+        params = []
+        params.extend(fargs)
+        params.extend(fkwargs.values())
+        lockkey = 'task_%s' % \
+           md5(str(self.func) + '-' + '-'.join(map(str, params))).hexdigest()
+        log.info('running task with lockkey %s', lockkey)
+        try:
+            l = DaemonLock(lockkey)
+            return func(*fargs, **fkwargs)
+            l.release()
+        except LockHeld:
+            log.info('LockHeld')
+            return 'Task with key %s already running' % lockkey   
+
+            
+            
+
+        
+        
+    
+    
+    
+  

pylons_app/lib/celerylib/tasks.py

+from celery.decorators import task
+from celery.task.sets import subtask
+from celeryconfig import PYLONS_CONFIG as config
+from pylons.i18n.translation import _
+from pylons_app.lib.celerylib import run_task, LockTask
+from pylons_app.lib.helpers import person
+from pylons_app.lib.smtp_mailer import SmtpMailer
+from pylons_app.lib.utils import OrderedDict
+from operator import itemgetter
+from vcs.backends.hg import MercurialRepository
+from time import mktime
+import traceback
+import json
+
+__all__ = ['whoosh_index', 'get_commits_stats',
+           'reset_user_password', 'send_email']
+
+def get_session():
+    from sqlalchemy import engine_from_config
+    from sqlalchemy.orm import sessionmaker, scoped_session
+    engine = engine_from_config(dict(config.items('app:main')), 'sqlalchemy.db1.')
+    sa = scoped_session(sessionmaker(bind=engine))
+    return sa
+
+def get_hg_settings():
+    from pylons_app.model.db import HgAppSettings
+    try:
+        sa = get_session()
+        ret = sa.query(HgAppSettings).all()
+    finally:
+        sa.remove()
+        
+    if not ret:
+        raise Exception('Could not get application settings !')
+    settings = {}
+    for each in ret:
+        settings['hg_app_' + each.app_settings_name] = each.app_settings_value    
+    
+    return settings
+
+def get_hg_ui_settings():
+    from pylons_app.model.db import HgAppUi
+    try:
+        sa = get_session()
+        ret = sa.query(HgAppUi).all()
+    finally:
+        sa.remove()
+        
+    if not ret:
+        raise Exception('Could not get application ui settings !')
+    settings = {}
+    for each in ret:
+        k = each.ui_key
+        v = each.ui_value
+        if k == '/':
+            k = 'root_path'
+        
+        if k.find('.') != -1:
+            k = k.replace('.', '_')
+        
+        if each.ui_section == 'hooks':
+            v = each.ui_active
+        
+        settings[each.ui_section + '_' + k] = v  
+    
+    return settings   
+
+@task
+def whoosh_index(repo_location, full_index):
+    log = whoosh_index.get_logger()
+    from pylons_app.lib.pidlock import DaemonLock
+    from pylons_app.lib.indexers.daemon import WhooshIndexingDaemon, LockHeld
+    try:
+        l = DaemonLock()
+        WhooshIndexingDaemon(repo_location=repo_location)\
+            .run(full_index=full_index)
+        l.release()
+        return 'Done'
+    except LockHeld:
+        log.info('LockHeld')
+        return 'LockHeld'    
+
+
+@task
+@LockTask('get_commits_stats')
+def get_commits_stats(repo_name, ts_min_y, ts_max_y):
+    author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty
+        
+    from pylons_app.model.db import Statistics, Repository
+    log = get_commits_stats.get_logger()
+    commits_by_day_author_aggregate = {}
+    commits_by_day_aggregate = {}
+    repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '')
+    repo = MercurialRepository(repos_path + repo_name)
+
+    skip_date_limit = True
+    parse_limit = 350 #limit for single task changeset parsing
+    last_rev = 0
+    last_cs = None
+    timegetter = itemgetter('time')
+    
+    sa = get_session()
+    
+    dbrepo = sa.query(Repository)\
+        .filter(Repository.repo_name == repo_name).scalar()
+    cur_stats = sa.query(Statistics)\
+        .filter(Statistics.repository == dbrepo).scalar()
+    if cur_stats:
+        last_rev = cur_stats.stat_on_revision
+    
+    if last_rev == repo.revisions[-1]:
+        #pass silently without any work
+        return True
+    
+    if cur_stats:
+        commits_by_day_aggregate = OrderedDict(
+                                       json.loads(
+                                        cur_stats.commit_activity_combined))
+        commits_by_day_author_aggregate = json.loads(cur_stats.commit_activity)
+    
+    for cnt, rev in enumerate(repo.revisions[last_rev:]):
+        last_cs = cs = repo.get_changeset(rev)
+        k = '%s-%s-%s' % (cs.date.timetuple()[0], cs.date.timetuple()[1],
+                          cs.date.timetuple()[2])
+        timetupple = [int(x) for x in k.split('-')]
+        timetupple.extend([0 for _ in xrange(6)])
+        k = mktime(timetupple)
+        if commits_by_day_author_aggregate.has_key(author_key_cleaner(cs.author)):
+            try:
+                l = [timegetter(x) for x in commits_by_day_author_aggregate\
+                        [author_key_cleaner(cs.author)]['data']]
+                time_pos = l.index(k)
+            except ValueError:
+                time_pos = False
+                
+            if time_pos >= 0 and time_pos is not False:
+                
+                datadict = commits_by_day_author_aggregate\
+                    [author_key_cleaner(cs.author)]['data'][time_pos]
+                
+                datadict["commits"] += 1
+                datadict["added"] += len(cs.added)
+                datadict["changed"] += len(cs.changed)
+                datadict["removed"] += len(cs.removed)
+                #print datadict
+                
+            else:
+                #print 'ELSE !!!!'
+                if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
+                    
+                    datadict = {"time":k,
+                                "commits":1,
+                                "added":len(cs.added),
+                                "changed":len(cs.changed),
+                                "removed":len(cs.removed),
+                               }
+                    commits_by_day_author_aggregate\
+                        [author_key_cleaner(cs.author)]['data'].append(datadict)
+                                        
+        else:
+            #print k, 'nokey ADDING'
+            if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
+                commits_by_day_author_aggregate[author_key_cleaner(cs.author)] = {
+                                    "label":author_key_cleaner(cs.author),
+                                    "data":[{"time":k,
+                                             "commits":1,
+                                             "added":len(cs.added),
+                                             "changed":len(cs.changed),
+                                             "removed":len(cs.removed),
+                                             }],
+                                    "schema":["commits"],
+                                    }               
+    
+#        #gather all data by day
+        if commits_by_day_aggregate.has_key(k):
+            commits_by_day_aggregate[k] += 1
+        else:
+            commits_by_day_aggregate[k] = 1
+        
+        if cnt >= parse_limit:
+            #don't fetch to much data since we can freeze application
+            break
+
+    overview_data = []
+    for k, v in commits_by_day_aggregate.items():
+        overview_data.append([k, v])
+    overview_data = sorted(overview_data, key=itemgetter(0))
+        
+    if not commits_by_day_author_aggregate:
+        commits_by_day_author_aggregate[author_key_cleaner(repo.contact)] = {
+            "label":author_key_cleaner(repo.contact),
+            "data":[0, 1],
+            "schema":["commits"],
+        }
+
+    stats = cur_stats if cur_stats else Statistics()
+    stats.commit_activity = json.dumps(commits_by_day_author_aggregate)
+    stats.commit_activity_combined = json.dumps(overview_data)
+    stats.repository = dbrepo
+    stats.stat_on_revision = last_cs.revision
+    stats.languages = json.dumps({'_TOTAL_':0, '':0})
+    
+    try:
+        sa.add(stats)
+        sa.commit()    
+    except:
+        log.error(traceback.format_exc())
+        sa.rollback()
+        return False
+    
+    run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y)
+                            
+    return True
+
+@task
+def reset_user_password(user_email):
+    log = reset_user_password.get_logger()
+    from pylons_app.lib import auth
+    from pylons_app.model.db import User
+    
+    try:
+        try:
+            sa = get_session()
+            user = sa.query(User).filter(User.email == user_email).scalar()
+            new_passwd = auth.PasswordGenerator().gen_password(8,
+                             auth.PasswordGenerator.ALPHABETS_BIG_SMALL)
+            if user:
+                user.password = auth.get_crypt_password(new_passwd)
+                sa.add(user)
+                sa.commit()
+                log.info('change password for %s', user_email)
+            if new_passwd is None:
+                raise Exception('unable to generate new password')
+            
+        except:
+            log.error(traceback.format_exc())
+            sa.rollback()
+        
+        run_task(send_email, user_email,
+                 "Your new hg-app password",
+                 'Your new hg-app password:%s' % (new_passwd))
+        log.info('send new password mail to %s', user_email)
+        
+        
+    except:
+        log.error('Failed to update user password')
+        log.error(traceback.format_exc())
+    return True
+
+@task    
+def send_email(recipients, subject, body):
+    log = send_email.get_logger()
+    email_config = dict(config.items('DEFAULT')) 
+    mail_from = email_config.get('app_email_from')
+    user = email_config.get('smtp_username')
+    passwd = email_config.get('smtp_password')
+    mail_server = email_config.get('smtp_server')
+    mail_port = email_config.get('smtp_port')
+    tls = email_config.get('smtp_use_tls')
+    ssl = False
+    
+    try:
+        m = SmtpMailer(mail_from, user, passwd, mail_server,
+                       mail_port, ssl, tls)
+        m.send(recipients, subject, body)  
+    except:
+        log.error('Mail sending failed')
+        log.error(traceback.format_exc())
+        return False
+    return True

pylons_app/lib/db_manage.py

 log = logging.getLogger(__name__)
 
 class DbManage(object):
-    def __init__(self, log_sql, dbname,tests=False):
+    def __init__(self, log_sql, dbname, tests=False):
         self.dbname = dbname
         self.tests = tests
         dburi = 'sqlite:////%s' % jn(ROOT, self.dbname)
         if override:
             log.info("database exisist and it's going to be destroyed")
             if self.tests:
-                destroy=True
+                destroy = True
             else:
                 destroy = ask_ok('Are you sure to destroy old database ? [y/n]')
             if not destroy:
             import getpass
             username = raw_input('Specify admin username:')
             password = getpass.getpass('Specify admin password:')
-            self.create_user(username, password, True)
+            email = raw_input('Specify admin email:')
+            self.create_user(username, password, email, True)
         else:
             log.info('creating admin and regular test users')
-            self.create_user('test_admin', 'test', True)
-            self.create_user('test_regular', 'test', False)
+            self.create_user('test_admin', 'test', 'test_admin@mail.com', True)
+            self.create_user('test_regular', 'test', 'test_regular@mail.com', False)
+            self.create_user('test_regular2', 'test', 'test_regular2@mail.com', False)
             
         
     
-    def config_prompt(self,test_repo_path=''):
+    def config_prompt(self, test_repo_path=''):
         log.info('Setting up repositories config')
         
         if not self.tests and not test_repo_path:
             path = test_repo_path
             
         if not os.path.isdir(path):
-            log.error('You entered wrong path: %s',path)
+            log.error('You entered wrong path: %s', path)
             sys.exit()
         
         hooks1 = HgAppUi()
             raise        
         log.info('created ui config')
                     
-    def create_user(self, username, password, admin=False):
+    def create_user(self, username, password, email='', admin=False):
         log.info('creating administrator user %s', username)
         new_user = User()
         new_user.username = username
         new_user.password = get_crypt_password(password)
         new_user.name = 'Hg'
         new_user.lastname = 'Admin'
-        new_user.email = 'admin@localhost'
+        new_user.email = email
         new_user.admin = admin
         new_user.active = True
         

pylons_app/lib/helpers.py

     return literal(annotate_highlight(filenode, url_func, **kwargs))
       
 def repo_name_slug(value):
+    """Return slug of name of repository
+    This function is called on each creation/modification
+    of repository to prevent bad names in repo
     """
-    Return slug of name of repository
-    """
-    slug = urlify(value)
-    for c in """=[]\;'"<>,/~!@#$%^&*()+{}|:""":
+    slug = remove_formatting(value)
+    slug = strip_tags(slug)
+    
+    for c in """=[]\;'"<>,/~!@#$%^&*()+{}|: """:
         slug = slug.replace(c, '-')
     slug = recursive_replace(slug, '-')
+    slug = collapse(slug, '-')
     return slug
 
 def get_changeset_safe(repo, rev):
 isodatesec = lambda  x: util.datestr(x, '%Y-%m-%d %H:%M:%S %1%2')
 localdate = lambda  x: (x[0], util.makedate()[1])
 rfc822date = lambda  x: util.datestr(x, "%a, %d %b %Y %H:%M:%S %1%2")
+rfc822date_notz = lambda  x: util.datestr(x, "%a, %d %b %Y %H:%M:%S")
 rfc3339date = lambda  x: util.datestr(x, "%Y-%m-%dT%H:%M:%S%1:%2")
 time_ago = lambda x: util.datestr(_age(x), "%a, %d %b %Y %H:%M:%S %1%2")
 

pylons_app/lib/indexers/__init__.py

+from os.path import dirname as dn, join as jn
+from pylons_app.config.environment import load_environment
+from pylons_app.model.hg_model import HgModel
+from shutil import rmtree
+from webhelpers.html.builder import escape
+from vcs.utils.lazy import LazyProperty
+
+from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter
+from whoosh.fields import TEXT, ID, STORED, Schema, FieldType
+from whoosh.index import create_in, open_dir
+from whoosh.formats import Characters
+from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter   
+
+import os
 import sys
-import os
-from pidlock import LockHeld, DaemonLock
 import traceback
 
-from os.path import dirname as dn
-from os.path import join as jn
-
 #to get the pylons_app import
 sys.path.append(dn(dn(dn(os.path.realpath(__file__)))))
 
-from pylons_app.config.environment import load_environment
-from pylons_app.model.hg_model import HgModel
-from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter
-from whoosh.fields import TEXT, ID, STORED, Schema
-from whoosh.index import create_in, open_dir
-from shutil import rmtree
 
 #LOCATION WE KEEP THE INDEX
 IDX_LOCATION = jn(dn(dn(dn(dn(os.path.abspath(__file__))))), 'data', 'index')
 
 #EXTENSIONS WE WANT TO INDEX CONTENT OFF
-INDEX_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c', 
-                    'cfm', 'cpp', 'cs', 'css', 'diff', 'do', 'el', 'erl', 'h', 
-                    'htm', 'html', 'ini', 'java', 'js', 'jsp', 'jspx', 'lisp', 
-                    'lua', 'm', 'mako', 'ml', 'pas', 'patch', 'php', 'php3', 
-                    'php4', 'phtml', 'pm', 'py', 'rb', 'rst', 's', 'sh', 'sql', 
-                    'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml','xsl','xslt', 
+INDEX_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c',
+                    'cfg', 'cfm', 'cpp', 'cs', 'css', 'diff', 'do', 'el', 'erl',
+                    'h', 'htm', 'html', 'ini', 'java', 'js', 'jsp', 'jspx', 'lisp',
+                    'lua', 'm', 'mako', 'ml', 'pas', 'patch', 'php', 'php3',
+                    'php4', 'phtml', 'pm', 'py', 'rb', 'rst', 's', 'sh', 'sql',
+                    'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt',
                     'yaws']
 
 #CUSTOM ANALYZER wordsplit + lowercase filter
 ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()
 
+
 #INDEX SCHEMA DEFINITION
 SCHEMA = Schema(owner=TEXT(),
                 repository=TEXT(stored=True),
                 path=ID(stored=True, unique=True),
-                content=TEXT(stored=True, analyzer=ANALYZER),
-                modtime=STORED(),extension=TEXT(stored=True))
+                content=FieldType(format=Characters(ANALYZER),
+                             scorable=True, stored=True),
+                modtime=STORED(), extension=TEXT(stored=True))
 
-IDX_NAME = 'HG_INDEX'
+
+IDX_NAME = 'HG_INDEX'
+FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n') 
+FRAGMENTER = SimpleFragmenter(200)
+                            
+class ResultWrapper(object):
+    def __init__(self, searcher, matcher, highlight_items):
+        self.searcher = searcher
+        self.matcher = matcher
+        self.highlight_items = highlight_items
+        self.fragment_size = 200 / 2
+    
+    @LazyProperty
+    def doc_ids(self):
+        docs_id = []
+        while self.matcher.is_active():
+            docnum = self.matcher.id()
+            chunks = [offsets for offsets in self.get_chunks()]
+            docs_id.append([docnum, chunks])
+            self.matcher.next()
+        return docs_id   
+        
+    def __str__(self):
+        return '<%s at %s>' % (self.__class__.__name__, len(self.doc_ids))
+
+    def __repr__(self):
+        return self.__str__()
+
+    def __len__(self):
+        return len(self.doc_ids)
+
+    def __iter__(self):
+        """
+        Allows Iteration over results,and lazy generate content
+
+        *Requires* implementation of ``__getitem__`` method.
+        """
+        for docid in self.doc_ids:
+            yield self.get_full_content(docid)
+
+    def __getslice__(self, i, j):
+        """
+        Slicing of resultWrapper
+        """
+        slice = []
+        for docid in self.doc_ids[i:j]:
+            slice.append(self.get_full_content(docid))
+        return slice   
+                            
+
+    def get_full_content(self, docid):
+        res = self.searcher.stored_fields(docid[0])
+        f_path = res['path'][res['path'].find(res['repository']) \
+                             + len(res['repository']):].lstrip('/')
+        
+        content_short = self.get_short_content(res, docid[1])
+        res.update({'content_short':content_short,
+                    'content_short_hl':self.highlight(content_short),
+                    'f_path':f_path})
+        
+        return res        
+    
+    def get_short_content(self, res, chunks):
+        
+        return ''.join([res['content'][chunk[0]:chunk[1]] for chunk in chunks])
+    
+    def get_chunks(self):
+        """
+        Smart function that implements chunking the content
+        but not overlap chunks so it doesn't highlight the same
+        close occurences twice.
+        @param matcher:
+        @param size:
+        """
+        memory = [(0, 0)]
+        for span in self.matcher.spans():
+            start = span.startchar or 0
+            end = span.endchar or 0
+            start_offseted = max(0, start - self.fragment_size)
+            end_offseted = end + self.fragment_size
+            
+            if start_offseted < memory[-1][1]:
+                start_offseted = memory[-1][1]
+            memory.append((start_offseted, end_offseted,))    
+            yield (start_offseted, end_offseted,)  
+        
+    def highlight(self, content, top=5):
+        hl = highlight(escape(content),
+                 self.highlight_items,
+                 analyzer=ANALYZER,
+                 fragmenter=FRAGMENTER,
+                 formatter=FORMATTER,
+                 top=top)
+        return hl 

pylons_app/lib/indexers/daemon.py

 project_path = dn(dn(dn(dn(os.path.realpath(__file__)))))
 sys.path.append(project_path)
 
-from pidlock import LockHeld, DaemonLock
-import traceback
-from pylons_app.config.environment import load_environment
+from pylons_app.lib.pidlock import LockHeld, DaemonLock
 from pylons_app.model.hg_model import HgModel
 from pylons_app.lib.helpers import safe_unicode
 from whoosh.index import create_in, open_dir
 from shutil import rmtree
-from pylons_app.lib.indexers import ANALYZER, INDEX_EXTENSIONS, IDX_LOCATION, \
-SCHEMA, IDX_NAME
+from pylons_app.lib.indexers import INDEX_EXTENSIONS, IDX_LOCATION, SCHEMA, IDX_NAME
 
 import logging
-import logging.config
-logging.config.fileConfig(jn(project_path, 'development.ini'))
+
 log = logging.getLogger('whooshIndexer')
+# create logger
+log.setLevel(logging.DEBUG)
+log.propagate = False
+# create console handler and set level to debug
+ch = logging.StreamHandler()
+ch.setLevel(logging.DEBUG)
+
+# create formatter
+formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+
+# add formatter to ch
+ch.setFormatter(formatter)
+
+# add ch to logger
+log.addHandler(ch)
 
 def scan_paths(root_location):
     return HgModel.repo_scan('/', root_location, None, True)
         WhooshIndexingDaemon(repo_location=repo_location)\
             .run(full_index=full_index)
         l.release()
+        reload(logging)
     except LockHeld:
         sys.exit(1)
 

pylons_app/lib/indexers/pidlock.py

-import os, time
-import sys
-from warnings import warn
-
-class LockHeld(Exception):pass
-
-
-class DaemonLock(object):
-    '''daemon locking
-    USAGE:
-    try:
-        l = lock()
-        main()
-        l.release()
-    except LockHeld:
-        sys.exit(1)
-    '''
-
-    def __init__(self, file=None, callbackfn=None,
-                 desc='daemon lock', debug=False):
-
-        self.pidfile = file if file else os.path.join(os.path.dirname(__file__),
-                                                      'running.lock')
-        self.callbackfn = callbackfn
-        self.desc = desc
-        self.debug = debug
-        self.held = False
-        #run the lock automatically !
-        self.lock()
-
-    def __del__(self):
-        if self.held:
-
-#            warn("use lock.release instead of del lock",
-#                    category = DeprecationWarning,
-#                    stacklevel = 2)
-
-            # ensure the lock will be removed
-            self.release()
-
-
-    def lock(self):
-        '''
-        locking function, if lock is present it will raise LockHeld exception
-        '''
-        lockname = '%s' % (os.getpid())
-
-        self.trylock()
-        self.makelock(lockname, self.pidfile)
-        return True
-
-    def trylock(self):
-        running_pid = False
-        try:
-            pidfile = open(self.pidfile, "r")
-            pidfile.seek(0)
-            running_pid = pidfile.readline()
-            if self.debug:
-                print 'lock file present running_pid: %s, checking for execution'\
-                % running_pid
-            # Now we check the PID from lock file matches to the current
-            # process PID
-            if running_pid:
-                if os.path.exists("/proc/%s" % running_pid):
-                        print "You already have an instance of the program running"
-                        print "It is running as process %s" % running_pid
-                        raise LockHeld
-                else:
-                        print "Lock File is there but the program is not running"
-                        print "Removing lock file for the: %s" % running_pid
-                        self.release()
-        except IOError, e:
-            if e.errno != 2:
-                raise
-
-
-    def release(self):
-        '''
-        releases the pid by removing the pidfile
-        '''
-        if self.callbackfn:
-            #execute callback function on release
-            if self.debug:
-                print 'executing callback function %s' % self.callbackfn
-            self.callbackfn()
-        try:
-            if self.debug:
-                print 'removing pidfile %s' % self.pidfile
-            os.remove(self.pidfile)
-            self.held = False
-        except OSError, e:
-            if self.debug:
-                print 'removing pidfile failed %s' % e
-            pass
-
-    def makelock(self, lockname, pidfile):
-        '''
-        this function will make an actual lock
-        @param lockname: acctual pid of file
-        @param pidfile: the file to write the pid in
-        '''
-        if self.debug:
-            print 'creating a file %s and pid: %s' % (pidfile, lockname)
-        pidfile = open(self.pidfile, "wb")
-        pidfile.write(lockname)
-        pidfile.close
-        self.held = True
-
-
-def main():
-    print 'func is running'
-    cnt = 20
-    while 1:
-        print cnt
-        if cnt == 0:
-            break
-        time.sleep(1)
-        cnt -= 1
-
-
-if __name__ == "__main__":
-    try:
-        l = DaemonLock(desc='test lock')
-        main()
-        l.release()
-    except LockHeld:
-        sys.exit(1)

pylons_app/lib/pidlock.py

+import os, time
+import sys
+from warnings import warn
+
+class LockHeld(Exception):pass
+
+
+class DaemonLock(object):
+    """daemon locking
+    USAGE:
+    try:
+        l = lock()
+        main()
+        l.release()
+    except LockHeld:
+        sys.exit(1)
+    """
+
+    def __init__(self, file=None, callbackfn=None,
+                 desc='daemon lock', debug=False):
+
+        self.pidfile = file if file else os.path.join(os.path.dirname(__file__),
+                                                      'running.lock')
+        self.callbackfn = callbackfn
+        self.desc = desc
+        self.debug = debug
+        self.held = False
+        #run the lock automatically !
+        self.lock()
+
+    def __del__(self):
+        if self.held:
+
+#            warn("use lock.release instead of del lock",
+#                    category = DeprecationWarning,
+#                    stacklevel = 2)
+
+            # ensure the lock will be removed
+            self.release()
+
+
+    def lock(self):
+        """
+        locking function, if lock is present it will raise LockHeld exception
+        """
+        lockname = '%s' % (os.getpid())
+
+        self.trylock()
+        self.makelock(lockname, self.pidfile)
+        return True
+
+    def trylock(self):
+        running_pid = False
+        try:
+            pidfile = open(self.pidfile, "r")
+            pidfile.seek(0)
+            running_pid = pidfile.readline()
+            if self.debug:
+                print 'lock file present running_pid: %s, checking for execution'\
+                % running_pid
+            # Now we check the PID from lock file matches to the current
+            # process PID
+            if running_pid:
+                if os.path.exists("/proc/%s" % running_pid):
+                        print "You already have an instance of the program running"
+                        print "It is running as process %s" % running_pid
+                        raise LockHeld
+                else:
+                        print "Lock File is there but the program is not running"
+                        print "Removing lock file for the: %s" % running_pid
+                        self.release()
+        except IOError, e:
+            if e.errno != 2:
+                raise
+
+
+    def release(self):
+        """
+        releases the pid by removing the pidfile
+        """
+        if self.callbackfn:
+            #execute callback function on release
+            if self.debug:
+                print 'executing callback function %s' % self.callbackfn
+            self.callbackfn()
+        try:
+            if self.debug:
+                print 'removing pidfile %s' % self.pidfile
+            os.remove(self.pidfile)
+            self.held = False
+        except OSError, e:
+            if self.debug:
+                print 'removing pidfile failed %s' % e
+            pass
+
+    def makelock(self, lockname, pidfile):
+        """
+        this function will make an actual lock
+        @param lockname: acctual pid of file
+        @param pidfile: the file to write the pid in
+        """
+        if self.debug:
+            print 'creating a file %s and pid: %s' % (pidfile, lockname)
+        pidfile = open(self.pidfile, "wb")
+        pidfile.write(lockname)
+        pidfile.close
+        self.held = True
+
+
+def main():
+    print 'func is running'
+    cnt = 20
+    while 1:
+        print cnt
+        if cnt == 0:
+            break
+        time.sleep(1)
+        cnt -= 1
+
+
+if __name__ == "__main__":
+    try:
+        l = DaemonLock(desc='test lock')
+        main()
+        l.release()
+    except LockHeld:
+        sys.exit(1)

pylons_app/lib/smtp_mailer.py

+import logging
+import smtplib
+import mimetypes
+from email.mime.multipart import MIMEMultipart
+from email.mime.image import MIMEImage
+from email.mime.audio import MIMEAudio
+from email.mime.base import MIMEBase
+from email.mime.text import MIMEText
+from email.utils import formatdate
+from email import encoders
+
+class SmtpMailer(object):
+    """simple smtp mailer class
+    
+    mailer = SmtpMailer(mail_from, user, passwd, mail_server, mail_port, ssl, tls)
+    mailer.send(recipients, subject, body, attachment_files)    
+    
+    :param recipients might be a list of string or single string
+    :param attachment_files is a dict of {filename:location} 
+    it tries to guess the mimetype and attach the file
+    """
+
+    def __init__(self, mail_from, user, passwd, mail_server,
+                    mail_port=None, ssl=False, tls=False):
+        
+        self.mail_from = mail_from
+        self.mail_server = mail_server
+        self.mail_port = mail_port
+        self.user = user
+        self.passwd = passwd
+        self.ssl = ssl
+        self.tls = tls
+        self.debug = False
+        
+    def send(self, recipients=[], subject='', body='', attachment_files={}):
+
+        if isinstance(recipients, basestring):
+            recipients = [recipients]
+        if self.ssl:
+            smtp_serv = smtplib.SMTP_SSL(self.mail_server, self.mail_port)
+        else:
+            smtp_serv = smtplib.SMTP(self.mail_server, self.mail_port)
+
+        if self.tls:
+            smtp_serv.starttls()
+         
+        if self.debug:    
+            smtp_serv.set_debuglevel(1)
+
+        smtp_serv.ehlo("mailer")
+
+        #if server requires authorization you must provide login and password
+        smtp_serv.login(self.user, self.passwd)
+
+        date_ = formatdate(localtime=True)
+        msg = MIMEMultipart()
+        msg['From'] = self.mail_from
+        msg['To'] = ','.join(recipients)
+        msg['Date'] = date_
+        msg['Subject'] = subject
+        msg.preamble = 'You will not see this in a MIME-aware mail reader.\n'
+
+        msg.attach(MIMEText(body))
+
+        if attachment_files:
+            self.__atach_files(msg, attachment_files)
+
+        smtp_serv.sendmail(self.mail_from, recipients, msg.as_string())
+        logging.info('MAIL SEND TO: %s' % recipients)
+        smtp_serv.quit()
+
+
+    def __atach_files(self, msg, attachment_files):
+        if isinstance(attachment_files, dict):
+            for f_name, msg_file in attachment_files.items():
+                ctype, encoding = mimetypes.guess_type(f_name)
+                logging.info("guessing file %s type based on %s" , ctype, f_name)
+                if ctype is None or encoding is not None:
+                    # No guess could be made, or the file is encoded (compressed), so
+                    # use a generic bag-of-bits type.
+                    ctype = 'application/octet-stream'
+                maintype, subtype = ctype.split('/', 1)
+                if maintype == 'text':
+                    # Note: we should handle calculating the charset
+                    file_part = MIMEText(self.get_content(msg_file), 
+                                         _subtype=subtype)
+                elif maintype == 'image':
+                    file_part = MIMEImage(self.get_content(msg_file), 
+                                          _subtype=subtype)
+                elif maintype == 'audio':
+                    file_part = MIMEAudio(self.get_content(msg_file), 
+                                          _subtype=subtype)
+                else:
+                    file_part = MIMEBase(maintype, subtype)
+                    file_part.set_payload(self.get_content(msg_file))
+                    # Encode the payload using Base64
+                    encoders.encode_base64(msg)
+                # Set the filename parameter
+                file_part.add_header('Content-Disposition', 'attachment', 
+                                     filename=f_name)
+                file_part.add_header('Content-Type', ctype, name=f_name)
+                msg.attach(file_part)
+        else:
+            raise Exception('Attachment files should be' 
+                            'a dict in format {"filename":"filepath"}')    
+
+    def get_content(self, msg_file):
+        '''
+        Get content based on type, if content is a string do open first
+        else just read because it's a probably open file object
+        @param msg_file:
+        '''
+        if isinstance(msg_file, str):
+            return open(msg_file, "rb").read()
+        else:
+            #just for safe seek to 0
+            msg_file.seek(0)
+            return msg_file.read()

pylons_app/lib/timerproxy.py

 from sqlalchemy.interfaces import ConnectionProxy
 import time
-import logging
-log = logging.getLogger('timerproxy')
+from sqlalchemy import log
 BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = xrange(30, 38)
 
 def color_sql(sql):
     sql = sql.replace('\n', '')
     sql = one_space_trim(sql)
     sql = sql\
-        .replace(',',',\n\t')\
+        .replace(',', ',\n\t')\
         .replace('SELECT', '\n\tSELECT \n\t')\
         .replace('UPDATE', '\n\tUPDATE \n\t')\
         .replace('DELETE', '\n\tDELETE \n\t')\
 
 
 class TimerProxy(ConnectionProxy):
+    
+    def __init__(self):
+        super(TimerProxy, self).__init__()
+        self.logging_name = 'timerProxy'
+        self.log = log.instance_logger(self, True)
+        
     def cursor_execute(self, execute, cursor, statement, parameters, context, executemany):
+        
         now = time.time()
         try:
-            log.info(">>>>> STARTING QUERY >>>>>")
+            self.log.info(">>>>> STARTING QUERY >>>>>")
             return execute(cursor, statement, parameters, context)
         finally:
             total = time.time() - now
             try:
-                log.info(format_sql("Query: %s" % statement % parameters))
+                self.log.info(format_sql("Query: %s" % statement % parameters))
             except TypeError:
-                log.info(format_sql("Query: %s %s" % (statement, parameters)))
-            log.info("<<<<< TOTAL TIME: %f <<<<<" % total)
-
-
-
-
+                self.log.info(format_sql("Query: %s %s" % (statement, parameters)))
+            self.log.info("<<<<< TOTAL TIME: %f <<<<<" % total)

pylons_app/lib/utils.py

 from vcs.utils.lazy import LazyProperty
 import logging
 import os
+
 log = logging.getLogger(__name__)
 
 
     
     revision = -1
     message = ''
+    author = ''
     
     @LazyProperty
     def raw_id(self):
 
     def __ne__(self, other):
         return not self == other
+
+
+#===============================================================================
+# TEST FUNCTIONS
+#===============================================================================
+def create_test_index(repo_location, full_index):
+    """Makes default test index
+    @param repo_location:
+    @param full_index:
+    """
+    from pylons_app.lib.indexers.daemon import WhooshIndexingDaemon
+    from pylons_app.lib.pidlock import DaemonLock, LockHeld
+    from pylons_app.lib.indexers import IDX_LOCATION
+    import shutil
+    
+    if os.path.exists(IDX_LOCATION):
+        shutil.rmtree(IDX_LOCATION)
+         
+    try:
+        l = DaemonLock()
+        WhooshIndexingDaemon(repo_location=repo_location)\
+            .run(full_index=full_index)
+        l.release()
+    except LockHeld:
+        pass    
+    
+def create_test_env(repos_test_path, config):
+    """Makes a fresh database and 
+    install test repository into tmp dir
+    """
+    from pylons_app.lib.db_manage import DbManage
+    import tarfile
+    import shutil
+    from os.path import dirname as dn, join as jn, abspath
+    
+    log = logging.getLogger('TestEnvCreator')
+    # create logger
+    log.setLevel(logging.DEBUG)
+    log.propagate = True
+    # create console handler and set level to debug
+    ch = logging.StreamHandler()
+    ch.setLevel(logging.DEBUG)
+    
+    # create formatter
+    formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+    
+    # add formatter to ch
+    ch.setFormatter(formatter)
+    
+    # add ch to logger
+    log.addHandler(ch)
+    
+    #PART ONE create db
+    log.debug('making test db')
+    dbname = config['sqlalchemy.db1.url'].split('/')[-1]
+    dbmanage = DbManage(log_sql=True, dbname=dbname, tests=True)
+    dbmanage.create_tables(override=True)
+    dbmanage.config_prompt(repos_test_path)
+    dbmanage.create_default_user()
+    dbmanage.admin_prompt()
+    dbmanage.create_permissions()
+    dbmanage.populate_default_permissions()
+    
+    #PART TWO make test repo
+    log.debug('making test vcs repo')
+    if os.path.isdir('/tmp/vcs_test'):
+        shutil.rmtree('/tmp/vcs_test')
+        
+    cur_dir = dn(dn(abspath(__file__)))
+    tar = tarfile.open(jn(cur_dir, 'tests', "vcs_test.tar.gz"))
+    tar.extractall('/tmp')
+    tar.close()

pylons_app/model/__init__.py

 """The application's model objects"""
 import logging
-import sqlalchemy as sa
-from sqlalchemy import orm
 from pylons_app.model import meta
-from pylons_app.model.meta import Session
 log = logging.getLogger(__name__)
 
-# Add these two imports:
-import datetime
-from sqlalchemy import schema, types
-
 def init_model(engine):
     """Call me before using any of the tables or classes in the model"""
     log.info("INITIALIZING DB MODELS")

pylons_app/model/caching_query.py

+"""caching_query.py
+
+Represent persistence structures which allow the usage of
+Beaker caching with SQLAlchemy.
+
+The three new concepts introduced here are:
+
+ * CachingQuery - a Query subclass that caches and
+   retrieves results in/from Beaker.
+ * FromCache - a query option that establishes caching
+   parameters on a Query
+ * RelationshipCache - a variant of FromCache which is specific
+   to a query invoked during a lazy load.
+ * _params_from_query - extracts value parameters from 
+   a Query.
+
+The rest of what's here are standard SQLAlchemy and
+Beaker constructs.
+   
+"""
+from sqlalchemy.orm.interfaces import MapperOption
+from sqlalchemy.orm.query import Query
+from sqlalchemy.sql import visitors
+
+class CachingQuery(Query):
+    """A Query subclass which optionally loads full results from a Beaker 
+    cache region.
+    
+    The CachingQuery stores additional state that allows it to consult
+    a Beaker cache before accessing the database:
+    
+    * A "region", which is a cache region argument passed to a 
+      Beaker CacheManager, specifies a particular cache configuration
+      (including backend implementation, expiration times, etc.)
+    * A "namespace", which is a qualifying name that identifies a
+      group of keys within the cache.  A query that filters on a name 
+      might use the name "by_name", a query that filters on a date range 
+      to a joined table might use the name "related_date_range".
+      
+    When the above state is present, a Beaker cache is retrieved.
+    
+    The "namespace" name is first concatenated with 
+    a string composed of the individual entities and columns the Query 
+    requests, i.e. such as ``Query(User.id, User.name)``.