Commits

Anonymous committed 0862575

BEP-0010 ticket numbering branch merged to trunk

Comments (0)

Files changed (7)

bloodhound_multiproduct/multiproduct/api.py

 
 from multiproduct.dbcursor import GLOBAL_PRODUCT
 from multiproduct.model import Product, ProductResourceMap, ProductSetting
-from multiproduct.util import EmbeddedLinkFormatter, IDENTIFIER
+from multiproduct.util import EmbeddedLinkFormatter, IDENTIFIER, \
+    using_sqlite_backend
 
 __all__ = ['MultiProductSystem', 'PRODUCT_SYNTAX_DELIMITER']
 
-DB_VERSION = 4
+DB_VERSION = 5
 DB_SYSTEM_KEY = 'bloodhound_multi_product_version'
 PLUGIN_NAME = 'Bloodhound multi product'
 
                 self._create_product_tables_for_plugins(db)
                 db_installed_version = self._update_db_version(db, 4)
 
+            if db_installed_version < 5:
+                table_defs = self._add_product_column_to_tables(
+                    ['ticket'], db_installed_version)
+                self._modify_ticket_pk(db, table_defs)
+                db_installed_version = self._update_db_version(db, 5)
+
             self.env.enable_multiproduct_schema(True)
 
     def _add_column_product_to_ticket(self, db):
                     (t.name, t.key, PLUGIN_NAME, current_version, 3))
         return table_defs
 
-    def _get_table_columns(self, table_defs):
+    def _get_table_columns(self, table_defs, all_columns=False):
         table_columns = dict()
         for table in table_defs:
             table_definition = \
             column_names = \
                 [column.name for column in table_definition.columns]
             table_columns[table.name] = \
-                [c for c in column_names if c != 'product']
+                [c for c in column_names if all_columns or c != 'product']
         return table_columns
 
     def _insert_default_product(self, db):
         for statement in db_connector.to_sql(ProductSetting._get_schema()):
             db(statement)
 
+    def _modify_ticket_pk(self, db, table_defs):
+        self.log.debug("Modifying ticket primary key: id -> uid")
+        table_columns = self._get_table_columns(table_defs, True)
+        db_connector, _ = DatabaseManager(self.env)._get_connector()
+
+        def rename_id_to_uid(table):
+            for c in table.columns:
+                if c.name == 'id':
+                    c.name = 'uid'
+                    break
+            table.key = ['uid']
+
+        def add_new_id_column(table):
+            id_column = Column('id', type='int', auto_increment=True)
+            if using_sqlite_backend(self.env):
+                # sqlite does not support multiple auto increment columns
+                id_column.auto_increment = False
+            table.columns.append(id_column)
+            table.indices.append(Index(['product', 'id'], unique=True))
+
+
+        for t in table_defs:
+            rename_id_to_uid(t)
+            add_new_id_column(t)
+
+            temp_table_name, cols = self._create_temp_table(
+                db, t.name, table_columns, table_defs)
+            db("""INSERT INTO ticket (%s, uid)
+                       SELECT %s, id FROM ticket_temp""" %
+                (cols, cols))
+            self._drop_temp_table(db, temp_table_name)
+            db.update_sequence(db.cursor(), 'ticket', 'id')
+            db.update_sequence(db.cursor(), 'ticket', 'uid')
+
     # IResourceChangeListener methods
     def match_resource(self, resource):
         return isinstance(resource, Product)

bloodhound_multiproduct/multiproduct/dbcursor.py

 import sqlparse.sql as Types
 
 from multiproduct.cache import lru_cache
+from multiproduct.util import using_sqlite_backend
 
 __all__ = ['BloodhoundIterableCursor', 'BloodhoundConnectionWrapper', 'ProductEnvContextManager']
 
                     'permission',
                     'wiki',
                     'report',
-                    ]
+                   ]
 PRODUCT_COLUMN = 'product'
 GLOBAL_PRODUCT = ''
 
 def translate_sql(env, sql):
     translator = None
     log = None
+    product_prefix = None
     if env is not None:
-        # FIXME: This is the right way to do it but breaks translation
-        # if trac.db.api.DatabaseManager(self.env).debug_sql:
-        if (env.parent or env).config['trac'].get('debug_sql', False):
+        if trac.db.api.DatabaseManager(env).debug_sql:
             log = env.log
         product_prefix = env.product.prefix if env.product else GLOBAL_PRODUCT
         translator = BloodhoundProductSQLTranslate(SKIP_TABLES,
                                                    TRANSLATE_TABLES,
                                                    PRODUCT_COLUMN,
-                                                   product_prefix)
+                                                   product_prefix,
+                                                   env)
     if log:
         log.debug('Original SQl: %s', sql)
     realsql = translator.translate(sql) if (translator is not None) else sql
     if log:
         log.debug('SQL: %s', realsql)
+
     return realsql
 
 class BloodhoundIterableCursor(trac.db.util.IterableCursor):
                         'JOIN', 'INNER JOIN']
     _from_end_words = ['WHERE', 'GROUP', 'HAVING', 'ORDER', 'UNION', 'LIMIT']
 
-    def __init__(self, skip_tables, translate_tables, product_column, product_prefix):
+    def __init__(self, skip_tables, translate_tables, product_column, product_prefix, env=None):
         self._skip_tables = skip_tables
         self._translate_tables = translate_tables
         self._product_column = product_column
         self._product_prefix = product_prefix
+        self._using_sqlite = env is None or using_sqlite_backend(env)
 
     def _sqlparse_underline_hack(self, token):
         underline_token = lambda token: token.ttype == Tokens.Token.Error and token.value == '_'
         token = self._token_next(parent, start_token)
         if not token.match(Tokens.Keyword, 'INTO'):
             raise Exception("Invalid INSERT statement")
-        def insert_extra_column(tablename, columns_token):
+        def insert_extra_columns(tablename, columns_token):
+            columns_present = []
             if tablename in self._translate_tables and \
                isinstance(columns_token, Types.Parenthesis):
                 ptoken = self._token_first(columns_token)
                 last_token = ptoken
                 while ptoken:
                     if isinstance(ptoken, Types.IdentifierList):
-                        if any(i.get_name() == 'product'
-                               for i in ptoken.get_identifiers()
-                               if isinstance(i, Types.Identifier)):
-                            return True
+                        if not 'product' in columns_present \
+                           and any(i.get_name() == 'product'
+                                for i in ptoken.get_identifiers()
+                                if isinstance(i, Types.Identifier)):
+                            columns_present.append('product')
+                        elif not 'id' in columns_present \
+                             and tablename == 'ticket' \
+                             and isinstance(ptoken, Types.IdentifierList) \
+                             and any((t.ttype is None or t.is_keyword)
+                                    and t.value == 'id'
+                                    for t in ptoken.get_identifiers()):
+                            columns_present.append('id')
                     last_token = ptoken
                     ptoken = self._token_next(columns_token, ptoken)
                 if not last_token or \
                    not last_token.match(Tokens.Punctuation, ')'):
                     raise Exception("Invalid INSERT statement, unable to find column parenthesis end")
-                for keyword in [',', ' ', self._product_column]:
+
+                columns_to_insert = []
+                if not 'product' in columns_present:
+                    columns_to_insert += [',', ' ', self._product_column]
+                if self._using_sqlite \
+                   and tablename == 'ticket'\
+                   and not 'id' in columns_present:
+                    columns_to_insert += [',', ' ', 'id']
+                for keyword in columns_to_insert:
                     self._token_insert_before(columns_token, last_token, Types.Token(Tokens.Keyword, keyword))
-            return False
-        def insert_extra_column_value(tablename, ptoken, before_token):
+            return columns_present
+
+        def insert_extra_column_values(tablename, ptoken, before_token,
+                columns_present):
             if tablename in self._translate_tables:
-                for keyword in [',', "'", self._product_prefix, "'"]:
+                values_to_insert = []
+                if not 'product' in columns_present:
+                    values_to_insert += [',', "'", self._product_prefix, "'"]
+                if self._using_sqlite \
+                   and tablename == 'ticket' \
+                   and not 'id' in columns_present:
+                    values_to_insert += [
+                        ',', """COALESCE((SELECT MAX(id) FROM ticket
+                                           WHERE product='%s'), 0)+1""" %
+                             (self._product_prefix,)
+                    ]
+                for keyword in values_to_insert:
                     self._token_insert_before(ptoken, before_token, Types.Token(Tokens.Keyword, keyword))
             return
+
         tablename = None
         table_name_token = self._token_next(parent, token)
-        has_product_column = False
+        columns_present = []
         if isinstance(table_name_token, Types.Function):
             token = self._token_first(table_name_token)
             if isinstance(token, Types.Identifier):
                 if columns_token.match(Tokens.Keyword, 'VALUES'):
                     token = columns_token
                 else:
-                    has_product_column = insert_extra_column(tablename, columns_token)
+                    columns_present = insert_extra_columns(tablename, columns_token)
                     token = self._token_next(parent, table_name_token)
         else:
             tablename = table_name_token.value
             if columns_token.match(Tokens.Keyword, 'VALUES'):
                 token = columns_token
             else:
-                has_product_column = insert_extra_column(tablename, columns_token)
+                columns_present = insert_extra_columns(tablename, columns_token)
                 token = self._token_next(parent, columns_token)
-        if has_product_column:
-            pass  # INSERT already has product, no translation needed
-        elif token.match(Tokens.Keyword, 'VALUES'):
+
+        if token.match(Tokens.Keyword, 'VALUES'):
             separators = [',', '(', ')']
             token = self._token_next(parent, token)
             while token:
                     if not last_token or \
                        not last_token.match(Tokens.Punctuation, ')'):
                         raise Exception("Invalid INSERT statement, unable to find column value parenthesis end")
-                    insert_extra_column_value(tablename, token, last_token)
+                    insert_extra_column_values(tablename, token, last_token, columns_present)
                 elif not token.match(Tokens.Punctuation, separators) and\
                      not token.match(Tokens.Keyword, separators) and\
                      not token.is_whitespace():

bloodhound_multiproduct/multiproduct/util.py

 
 from genshi.builder import tag
 
+import trac.db.api
 from trac.util.text import unquote_label
 from trac.wiki.formatter import LinkFormatter
 from trac.core import Component, ComponentMeta
                  else fullmatch.group('label'))
         return self._make_lhref_link(match, fullmatch, rel, ns, target, label)
 
+
+def using_sqlite_backend(env):
+    connector, arg = trac.db.api._parse_db_str(
+        trac.db.api.DatabaseManager(env).connection_uri
+    )
+    return connector == 'sqlite'
+
 #----------------------
 # Useful regex
 #----------------------

bloodhound_multiproduct/tests/db/cursor.py

     'insert_with_product': [
         (
 """INSERT INTO ticket (summary, product) VALUES ('S', 'swlcu')""",
-"""INSERT INTO ticket (summary, product) VALUES ('S', 'swlcu')"""
+"""INSERT INTO ticket (summary, product, id) VALUES ('S', 'swlcu',COALESCE((SELECT MAX(id) FROM ticket\nWHERE product='PRODUCT'), 0)+1)"""
         ),
     ],
 

bloodhound_multiproduct/tests/ticket/model.py

 import shutil
 import unittest
 
-from trac.ticket.model import Milestone
+from trac.ticket.model import Milestone, Ticket
 from trac.ticket.tests.model import TicketTestCase, TicketCommentTestCase, \
         TicketCommentEditTestCase, TicketCommentDeleteTestCase, EnumTestCase, \
         MilestoneTestCase, ComponentTestCase, VersionTestCase
 from trac.util.datefmt import to_utimestamp, utc
 
+from multiproduct.model import Product
 from multiproduct.env import ProductEnvironment
 from tests.env import MultiproductTestCase
 
         self.global_env.reset_db()
         self.env = self.global_env = None
 
+    def _get_ticket_uid(self, tid):
+        with self.env.db_query as db:
+            rows = db("""SELECT uid FROM ticket WHERE id=%s""", (tid, ))
+            return rows[0][0] if rows else -1
+
+    def test_insert_into_multiple_products(self):
+        # UIDs are global, autoincremented
+        # IDs are product-scoped, incremented in the SQL translator
+        self.env = ProductEnvironment(self.global_env, self.default_product)
+
+        tid = self._insert_ticket('hello kitty', reporter='admin')
+        ticket = Ticket(self.env, tid)
+        self.assertEqual(tid, 1)
+        self.assertEqual(self._get_ticket_uid(tid), 1)
+        self.assertEqual(ticket.id, tid)
+        tid = self._insert_ticket('hello kitteh', reporter='admin')
+        ticket = Ticket(self.env, tid)
+        self.assertEqual(tid, 2)
+        self.assertEqual(self._get_ticket_uid(tid), 2)
+        self.assertEqual(ticket.id, tid)
+
+        p2 = Product(self.global_env)
+        p2.prefix = 'p2'
+        p2.name = 'product, too'
+        p2.owner = 'admin'
+        p2.insert()
+        self.env = ProductEnvironment(self.global_env, p2)
+
+        tid = self._insert_ticket('hello catty', reporter='admin')
+        ticket = Ticket(self.env, tid)
+        self.assertEqual(tid, 1)
+        self.assertEqual(self._get_ticket_uid(tid), 3)
+        self.assertEqual(ticket.id, tid)
+        tid = self._insert_ticket('hello ocelot', reporter='admin')
+        ticket = Ticket(self.env, tid)
+        self.assertEqual(tid, 2)
+        self.assertEqual(self._get_ticket_uid(tid), 4)
+        self.assertEqual(ticket.id, tid)
+
+
 class ProductTicketCommentTestCase(MultiproductTestCase):
 
     @property

bloodhound_multiproduct/tests/upgrade.py

         self.assertEqual(len(products), 1)
         self.assertEqual(products[0].prefix, 'xxx')
 
+    def test_migration_to_multiproduct_preserves_ticket_ids(self):
+        for ticket_id in (1, 3, 5, 7):
+            with self.env.db_transaction as db:
+                cursor = db.cursor()
+                cursor.execute("INSERT INTO ticket (id) VALUES (%i)" % ticket_id)
+                db.update_sequence(cursor, 'ticket')
+
+        self._enable_multiproduct()
+        self.env.upgrade()
+
+        for ticket_id in (1, 3, 5, 7):
+            with self.product('@'):
+                ticket = Ticket(self.env, ticket_id)
+            self.assertEqual(ticket.id, ticket_id)
+
+    def test_can_insert_tickets_after_upgrade(self):
+        t1 = Ticket(self.env)
+        t1.summary = "test"
+        t1.insert()
+        self.assertEqual(t1.id, 1)
+
+        self._enable_multiproduct()
+        self.env.upgrade()
+
+        with self.product('@'):
+            ticket = Ticket(self.env)
+            ticket.summary = 'test'
+            ticket.insert()
+            self.assertEqual(ticket.id, 2)
+
+    def test_can_insert_tickets_with_same_id_to_different_products(self):
+        self._enable_multiproduct()
+        self.env.upgrade()
+
+        self.env.db_transaction("INSERT INTO ticket (id, summary)"
+                                "            VALUES (1, 'first product')")
+        t1 = Ticket(self.env, 1)
+
+        with self.product('@'):
+            self.env.db_transaction("INSERT INTO ticket (id, summary)"
+                                    "            VALUES (1, 'second product')")
+            t2 = Ticket(self.env, 1)
+
+        self.assertEqual(t1.id, t2.id)
+        self.assertNotEqual(t1['summary'], t2['summary'])
+
+    def test_batch_ticket_insert_after_upgrade(self):
+        self._enable_multiproduct()
+        self.env.upgrade()
+        with self.env.db_direct_transaction as db:
+            db("""CREATE TABLE "@_tmp" (summary text, product text)""")
+            for summary in "abcdef":
+                db("""INSERT INTO "@_tmp" VALUES ('%s', '@')""" % (summary,))
+
+        with self.product('@'):
+            with self.env.db_transaction as db:
+                db("""INSERT INTO ticket (summary) SELECT summary FROM tmp""")
+
+
+
     def _enable_multiproduct(self):
         self._update_config('components', 'multiproduct.*', 'enabled')
 

trac/trac/ticket/model.py

                            % (','.join(std_fields),
                               ','.join(['%s'] * len(std_fields))),
                            [values[name] for name in std_fields])
-            tkt_id = db.get_last_id(cursor, 'ticket')
+            if getattr(self.env, '_multiproduct_schema_enabled', False):
+                tkt_id = db.get_last_id(cursor, 'ticket', 'uid')
+                rows = db("""SELECT id FROM ticket WHERE uid=%s""", (tkt_id,))
+                tkt_id = rows[0][0] if rows else -1
+            else:
+                tkt_id = db.get_last_id(cursor, 'ticket')
+
 
             # Insert custom fields
             if custom_fields:
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.