Marcin Kuzminski avatar Marcin Kuzminski committed 1f47ade

Bumped sqlalchemy version to 0.7, replaced timerproxy with new event system for sqlalchemy.
TimerProxy is now longer valid profilling method for sqlalchemy

Comments (0)

Files changed (9)

 #########################################################
 ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 #########################################################
-sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db
-#sqlalchemy.db1.url = postgresql://postgres:qwe@localhost/rhodecode
-sqlalchemy.db1.echo = True
+#sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db
+sqlalchemy.db1.url = postgresql://postgres:qwe@localhost/rhodecode
+sqlalchemy.db1.echo = false
 sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.convert_unicode = true
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
-level = NOTSET
+level = DEBUG
 formatter = color_formatter
 
 [handler_console_sql]
 class = StreamHandler
 args = (sys.stderr,)
-level = NOTSET
+level = DEBUG
 formatter = color_formatter_sql
 
 ################
 #########################################################
 #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db
 sqlalchemy.db1.url = postgresql://postgres:qwe@localhost/rhodecode
-sqlalchemy.db1.echo = True
+sqlalchemy.db1.echo = false
 sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.convert_unicode = true
 

rhodecode/config/deployment.ini_tmpl

 ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 #########################################################
 sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db
-sqlalchemy.db1.echo = True
+sqlalchemy.db1.echo = false
 sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.convert_unicode = true
 

rhodecode/config/environment.py

 
     #MULTIPLE DB configs
     # Setup the SQLAlchemy database engine
-    if config['debug'] and not test:
-        #use query time debugging.
-        sa_engine_db1 = engine_from_config(config, 'sqlalchemy.db1.',
-                                           proxy=TimerProxy())
-    else:
-        sa_engine_db1 = engine_from_config(config, 'sqlalchemy.db1.')
+    sa_engine_db1 = engine_from_config(config, 'sqlalchemy.db1.')
 
     init_model(sa_engine_db1)
 

rhodecode/lib/__init__.py

 def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs):
     """
     Custom engine_from_config functions that makes sure we use NullPool for
-    file based sqlite databases. This prevents errors on sqlite.
+    file based sqlite databases. This prevents errors on sqlite. This only 
+    applies to sqlalchemy versions < 0.7.0
 
     """
+    import sqlalchemy
     from sqlalchemy import engine_from_config as efc
-    from sqlalchemy.pool import NullPool
+    import logging
 
-    url = configuration[prefix + 'url']
+    if int(sqlalchemy.__version__.split('.')[1]) < 7:
 
-    if url.startswith('sqlite'):
-        kwargs.update({'poolclass': NullPool})
+        # This solution should work for sqlalchemy < 0.7.0, and should use
+        # proxy=TimerProxy() for execution time profiling
 
-    return efc(configuration, prefix, **kwargs)
+        from sqlalchemy.pool import NullPool
+        url = configuration[prefix + 'url']
+
+        if url.startswith('sqlite'):
+            kwargs.update({'poolclass': NullPool})
+        return efc(configuration, prefix, **kwargs)
+    else:
+        import time
+        from sqlalchemy import event
+        from sqlalchemy.engine import Engine
+
+        log = logging.getLogger('timerproxy')
+        BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = xrange(30, 38)
+        engine = efc(configuration, prefix, **kwargs)
+
+        def color_sql(sql):
+            COLOR_SEQ = "\033[1;%dm"
+            COLOR_SQL = YELLOW
+            normal = '\x1b[0m'
+            return ''.join([COLOR_SEQ % COLOR_SQL, sql, normal])
+
+        if configuration['debug']:
+            #attach events only for debug configuration
+
+            def before_cursor_execute(conn, cursor, statement,
+                                    parameters, context, executemany):
+                context._query_start_time = time.time()
+                log.info(color_sql(">>>>> STARTING QUERY >>>>>"))
+
+
+            def after_cursor_execute(conn, cursor, statement,
+                                    parameters, context, executemany):
+                total = time.time() - context._query_start_time
+                log.info(color_sql("<<<<< TOTAL TIME: %f <<<<<" % total))
+
+            event.listen(engine, "before_cursor_execute",
+                         before_cursor_execute)
+            event.listen(engine, "after_cursor_execute",
+                         after_cursor_execute)
+
+    return engine
 
 
 def age(curdate):

rhodecode/lib/colored_formatter.py

         def_record = logging.Formatter.format(self, record)
         end = RESET_SEQ
 
-        colored_record = start + def_record + end
+        colored_record = ''.join([start, def_record, end])
         return colored_record
 
 
         def_record = format_sql(logging.Formatter.format(self, record))
         end = RESET_SEQ
 
-        colored_record = start + def_record + end
+        colored_record = ''.join([start, def_record, end])
         return colored_record

rhodecode/lib/timerproxy.py

     COLOR_SEQ = "\033[1;%dm"
     COLOR_SQL = YELLOW
     normal = '\x1b[0m'
-    return COLOR_SEQ % COLOR_SQL + sql + normal
+    return ''.join([COLOR_SEQ % COLOR_SQL, sql, normal])
 
 
 class TimerProxy(ConnectionProxy):

rhodecode/model/db.py

 
 class RhodeCodeSettings(Base):
     __tablename__ = 'rhodecode_settings'
-    __table_args__ = (UniqueConstraint('app_settings_name'), {'useexisting':True})
+    __table_args__ = (UniqueConstraint('app_settings_name'), {'extend_existing':True})
     app_settings_id = Column("app_settings_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     app_settings_name = Column("app_settings_name", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     app_settings_value = Column("app_settings_value", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
 class RhodeCodeUi(Base):
     __tablename__ = 'rhodecode_ui'
-    __table_args__ = {'useexisting':True}
+    __table_args__ = {'extend_existing':True}
     ui_id = Column("ui_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     ui_section = Column("ui_section", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_key = Column("ui_key", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
 class User(Base):
     __tablename__ = 'users'
-    __table_args__ = (UniqueConstraint('username'), UniqueConstraint('email'), {'useexisting':True})
+    __table_args__ = (UniqueConstraint('username'), UniqueConstraint('email'), {'extend_existing':True})
     user_id = Column("user_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     username = Column("username", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     password = Column("password", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
 class UserLog(Base):
     __tablename__ = 'user_logs'
-    __table_args__ = {'useexisting':True}
+    __table_args__ = {'extend_existing':True}
     user_log_id = Column("user_log_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
-    repository_id = Column("repository_id", Integer(length=None, convert_unicode=False, assert_unicode=None), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
+    repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
     repository_name = Column("repository_name", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     user_ip = Column("user_ip", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     action = Column("action", UnicodeText(length=1200000, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
 class UsersGroup(Base):
     __tablename__ = 'users_groups'
-    __table_args__ = {'useexisting':True}
+    __table_args__ = {'extend_existing':True}
 
     users_group_id = Column("users_group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_name = Column("users_group_name", String(length=255, convert_unicode=False, assert_unicode=None), nullable=False, unique=True, default=None)
 
 class UsersGroupMember(Base):
     __tablename__ = 'users_groups_members'
-    __table_args__ = {'useexisting':True}
+    __table_args__ = {'extend_existing':True}
 
     users_group_member_id = Column("users_group_member_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
 
 class Repository(Base):
     __tablename__ = 'repositories'
-    __table_args__ = (UniqueConstraint('repo_name'), {'useexisting':True},)
+    __table_args__ = (UniqueConstraint('repo_name'), {'extend_existing':True},)
     __mapper_args__ = {'extension':RepositoryMapper()}
 
     repo_id = Column("repo_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
 class Group(Base):
     __tablename__ = 'groups'
     __table_args__ = (UniqueConstraint('group_name', 'group_parent_id'),
-                      CheckConstraint('group_id != group_parent_id'), {'useexisting':True},)
+                      CheckConstraint('group_id != group_parent_id'), {'extend_existing':True},)
     __mapper_args__ = {'order_by':'group_name'}
 
     group_id = Column("group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
 
 class Permission(Base):
     __tablename__ = 'permissions'
-    __table_args__ = {'useexisting':True}
+    __table_args__ = {'extend_existing':True}
     permission_id = Column("permission_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     permission_name = Column("permission_name", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     permission_longname = Column("permission_longname", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
 class RepoToPerm(Base):
     __tablename__ = 'repo_to_perm'
-    __table_args__ = (UniqueConstraint('user_id', 'repository_id'), {'useexisting':True})
+    __table_args__ = (UniqueConstraint('user_id', 'repository_id'), {'extend_existing':True})
     repo_to_perm_id = Column("repo_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
 
 class UserToPerm(Base):
     __tablename__ = 'user_to_perm'
-    __table_args__ = (UniqueConstraint('user_id', 'permission_id'), {'useexisting':True})
+    __table_args__ = (UniqueConstraint('user_id', 'permission_id'), {'extend_existing':True})
     user_to_perm_id = Column("user_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
 
 class UsersGroupRepoToPerm(Base):
     __tablename__ = 'users_group_repo_to_perm'
-    __table_args__ = (UniqueConstraint('users_group_id', 'permission_id'), {'useexisting':True})
+    __table_args__ = (UniqueConstraint('users_group_id', 'permission_id'), {'extend_existing':True})
     users_group_to_perm_id = Column("users_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
 
 class GroupToPerm(Base):
     __tablename__ = 'group_to_perm'
-    __table_args__ = (UniqueConstraint('group_id', 'permission_id'), {'useexisting':True})
+    __table_args__ = (UniqueConstraint('group_id', 'permission_id'), {'extend_existing':True})
 
     group_to_perm_id = Column("group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
 
 class Statistics(Base):
     __tablename__ = 'statistics'
-    __table_args__ = (UniqueConstraint('repository_id'), {'useexisting':True})
+    __table_args__ = (UniqueConstraint('repository_id'), {'extend_existing':True})
     stat_id = Column("stat_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=True, default=None)
     stat_on_revision = Column("stat_on_revision", Integer(), nullable=False)
     __tablename__ = 'user_followings'
     __table_args__ = (UniqueConstraint('user_id', 'follows_repository_id'),
                       UniqueConstraint('user_id', 'follows_user_id')
-                      , {'useexisting':True})
+                      , {'extend_existing':True})
 
     user_following_id = Column("user_following_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
 
 class CacheInvalidation(Base):
     __tablename__ = 'cache_invalidation'
-    __table_args__ = (UniqueConstraint('cache_key'), {'useexisting':True})
+    __table_args__ = (UniqueConstraint('cache_key'), {'extend_existing':True})
     cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     cache_key = Column("cache_key", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     cache_args = Column("cache_args", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
 class DbMigrateVersion(Base):
     __tablename__ = 'db_migrate_version'
-    __table_args__ = {'useexisting':True}
+    __table_args__ = {'extend_existing':True}
     repository_id = Column('repository_id', String(250), primary_key=True)
     repository_path = Column('repository_path', Text)
     version = Column('version', Integer)
 requirements = [
         "Pylons==1.0.0",
         "WebHelpers>=1.2",
-        "SQLAlchemy>=0.6.6",
+        "SQLAlchemy>=0.7.0",
         "Mako>=0.4.0",
         "vcs>=0.2.1",
         "pygments>=1.4",
         "mercurial>=1.8.1",
-        "whoosh>=1.8.0",
+        "whoosh>=1.8.4",
         "celery>=2.2.5",
         "babel",
         "python-dateutil>=1.5.0,<2.0.0",
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.