Commits

cmlenz  committed 5284d71

Improvements to the Bugzilla importer contributed by asmodai. Thanks!

  • Participants
  • Parent commits 7b1844f
  • Branches trunk

Comments (0)

Files changed (2)

- * Brad Anderson           brad@dsource.org
- * Christopher Armstrong   radix
- * Jani Averbach           jaa@jaa.iki.fi
- * Juanma Barranquero      lektu@terra.es
- * Christian Boos          cboos@bct-technology.com
- * Rocky Burt              rocky.burt@myrealbox.com
- * Toni Brkic              toni.brkic@switchcore.com
- * Wesley Crucius          wcrucius@sandc.com
- * Daragh Fitzpatrick      Daragh@i2i-Tech.com
- * dju'
- * Felix Colins            felix@keyghost.com
- * Eric Gillespie          epg@netbsd.org
- * Matthew Good            trac@matt-good.net
- * Shun-ichi Goto          gotoh@taiyo.co.jp
- * Chris Green             cmgreen@uab.edu
- * Mikael Hallendal        micke@imendio.com
- * Stephen Hansen          shansen@advpubtech.com
- * Laurie Harper           zodiac@holoweb.net
- * Francois Harvey         fharvey@securiweb.net
- * Michael Hope            michael.hope@hamjet.co.nz
- * Richard Hult            richard@imendio.com
- * Nuutti Kotivuori        naked@iki.fi
- * Ian Leader              ian.leader@line.co.uk
- * Christopher Lenz        cmlenz@gmx.de
- * Ivo Looser              ivo.looser@gmail.com
- * Angel Marin             anmar@gmx.net
- * Keir Mierle             keir@cs.utoronto.ca
- * James Moger             jamesm@transonic.com
- * Tim Moloney             moloney@mrsl.com
- * Jennifer Murtell        jen@jmurtell.com
- * Jacob Norda             jacobnorda@gmail.com
- * Juracy Filho            juracy@gmail.com
- * Cap Petschulat          cap@cdres.com
- * Nicholas Riley          sabi
- * Manuzhai                manuzhai@gmail.com
- * Mark Rowe               mark.rowe@bdash.net.nz
- * Olliver Rutherfurd      ollie
- * pkou                    pkou@ua.fm
- * Michael Scherer         misc@mandrake.org
- * Andreas Schrattenecker  vittorio
- * Emmeran Seehuber        rototor@rototor.de
- * Noah Slater             nslater@gmail.com
- * Bill Soudan             bill@soudan.net
- * Ludvig Strigeus
- * Alec Thomas             alec@swapoff.org
- * Jani Tiainen            redetin@luukku.com
- * Zilvinas Valinskas      zilvinas@gemtek.lt
- * Jason Vasquez           jason@mugfu.com
- * Dmitry Yusupov          dmitry_yus@yahoo.com
+ * Brad Anderson                  brad@dsource.org
+ * Christopher Armstrong          radix
+ * Jani Averbach                  jaa@jaa.iki.fi
+ * Juanma Barranquero             lektu@terra.es
+ * Christian Boos                 cboos@bct-technology.com
+ * Rocky Burt                     rocky.burt@myrealbox.com
+ * Toni Brkic                     toni.brkic@switchcore.com
+ * Wesley Crucius                 wcrucius@sandc.com
+ * Daragh Fitzpatrick             Daragh@i2i-Tech.com
+ * dju'                           
+ * Felix Colins                   felix@keyghost.com
+ * Eric Gillespie                 epg@netbsd.org
+ * Matthew Good                   trac@matt-good.net
+ * Shun-ichi Goto                 gotoh@taiyo.co.jp
+ * Chris Green                    cmgreen@uab.edu
+ * Mikael Hallendal               micke@imendio.com
+ * Stephen Hansen                 shansen@advpubtech.com
+ * Laurie Harper                  zodiac@holoweb.net
+ * Francois Harvey                fharvey@securiweb.net
+ * Michael Hope                   michael.hope@hamjet.co.nz
+ * Richard Hult                   richard@imendio.com
+ * Nuutti Kotivuori               naked@iki.fi
+ * Ian Leader                     ian.leader@line.co.uk
+ * Christopher Lenz               cmlenz@gmx.de
+ * Ivo Looser                     ivo.looser@gmail.com
+ * Angel Marin                    anmar@gmx.net
+ * Keir Mierle                    keir@cs.utoronto.ca
+ * James Moger                    jamesm@transonic.com
+ * Tim Moloney                    moloney@mrsl.com
+ * Jennifer Murtell               jen@jmurtell.com
+ * Jacob Norda                    jacobnorda@gmail.com
+ * Jeroen Ruigrok van der Werven  asmodai@in-nomine.org
+ * Juracy Filho                   juracy@gmail.com
+ * Cap Petschulat                 cap@cdres.com
+ * Nicholas Riley                 sabi
+ * Manuzhai                       manuzhai@gmail.com
+ * Mark Rowe                      mark.rowe@bdash.net.nz
+ * Olliver Rutherfurd             ollie
+ * pkou                           pkou@ua.fm
+ * Michael Scherer                misc@mandrake.org
+ * Andreas Schrattenecker         vittorio
+ * Emmeran Seehuber               rototor@rototor.de
+ * Noah Slater                    nslater@gmail.com
+ * Bill Soudan                    bill@soudan.net
+ * Ludvig Strigeus                
+ * Alec Thomas                    alec@swapoff.org
+ * Jani Tiainen                   redetin@luukku.com
+ * Zilvinas Valinskas             zilvinas@gemtek.lt
+ * Jason Vasquez                  jason@mugfu.com
+ * Dmitry Yusupov                 dmitry_yus@yahoo.com
 
 The ever so elusive Anonymous.
 

File contrib/bugzilla2trac.py

 """
 Import a Bugzilla items into a Trac database.
 
-Requires:  Trac 0.7.1 from http://trac.edgewall.com/
+Requires:  Trac 0.9b1 from http://trac.edgewall.com/
            Python 2.3 from http://www.python.org/
            MySQL >= 3.23 from http://www.mysql.org/
 
 Copyright 2004, Dmitry Yusupov <dmitry_yus@yahoo.com>
 
 Many enhancements, Bill Soudan <bill@soudan.net>
+Other enhancements, Florent Guillaume <fg@nuxeo.com>
+Reworked, Jeroen Ruigrok van der Werven <asmodai@tendra.org>
+
+$Id$
 """
 
+import re
+
 ###
 ### Conversion Settings -- edit these before running if desired
 ###
 # Bugzilla version.  You can find this in Bugzilla's globals.pl file.
 #
 # Currently, the following bugzilla versions are known to work:
-#   2.11
+#   2.11 (2110), 2.16.5 (2165), 2.18.3 (2183), 2.19.1 (2191)
 #
 # If you run this script on a version not listed here and it is successful,
-# please report it to the Trac mailing list so we can update the list.
-BZ_VERSION = '2.16.5'
+# please report it to the Trac mailing list and drop a note to
+# asmodai@tendra.org so we can update the list.
+BZ_VERSION = 2180
 
 # MySQL connection parameters for the Bugzilla database.  These can also 
 # be specified on the command line.
-BZ_DB = ''
-BZ_HOST = 'localhost'
-BZ_USER = ''
-BZ_PASSWORD = ''
+BZ_DB = ""
+BZ_HOST = ""
+BZ_USER = ""
+BZ_PASSWORD = ""
 
 # Path to the Trac environment.
-TRAC_ENV = ''
+TRAC_ENV = "/usr/local/trac"
 
 # If true, all existing Trac tickets and attachments will be removed 
 # prior to import.
-TRAC_CLEAN = False
+TRAC_CLEAN = True
 
 # Enclose imported ticket description and comments in a {{{ }}} 
 # preformat block?  This formats the text in a fixed-point font.
 PREFORMAT_COMMENTS = False
 
+# Replace bug numbers in comments with #xyz
+REPLACE_BUG_NO = False
+
+# Severities
+SEVERITIES = [
+    ("blocker",  "1"),
+    ("critical", "2"),
+    ("major",    "3"),
+    ("normal",   "4"),
+    ("minor",    "5"),
+    ("trivial",  "6")
+]
+
+# Priorities
+# If using the default Bugzilla priorities of P1 - P5, do not change anything
+# here.
+# If you have other priorities defined please change the P1 - P5 mapping to
+# the order you want.  You can also collapse multiple priorities on bugzilla's
+# side into the same priority on Trac's side, simply adjust PRIORITIES_MAP.
+PRIORITIES = [
+    ("highest", "1"),
+    ("high",    "2"),
+    ("normal",  "3"),
+    ("low",     "4"),
+    ("lowest",  "5")
+]
+
+# Bugzilla: Trac
+# NOTE: Use lowercase.
+PRIORITIES_MAP = {
+    "p1": "highest",
+    "p2": "high",
+    "p3": "normal",
+    "p4": "low",
+    "p5": "lowest"
+}
+
 # By default, all bugs are imported from Bugzilla.  If you add a list
 # of products here, only bugs from those products will be imported.
 PRODUCTS = []
+# These Bugzilla products will be ignored during import.
+IGNORE_PRODUCTS = []
 
-# Trac doesn't have the concept of a product.  Instead, this script can
-# assign keywords in the ticket entry to represent products.
-#
-# ex. PRODUCT_KEYWORDS = { 'product1' : 'PRODUCT1_KEYWORD' }
-PRODUCT_KEYWORDS = {}
+# These milestones are ignored
+IGNORE_MILESTONES = ["---"]
+
+# These logins are converted to these user ids
+LOGIN_MAP = {
+    #'some.user@example.com': 'someuser',
+}
+
+# These emails are removed from CC list
+IGNORE_CC = [
+    #'loser@example.com',
+]
+
+# The 'component' field in Trac can come either from the Product or
+# or from the Component field of Bugzilla. COMPONENTS_FROM_PRODUCTS
+# switches the behavior.
+# If COMPONENTS_FROM_PRODUCTS is True:
+# - Bugzilla Product -> Trac Component
+# - Bugzilla Component -> Trac Keyword
+# IF COMPONENTS_FROM_PRODUCTS is False:
+# - Bugzilla Product -> Trac Keyword
+# - Bugzilla Component -> Trac Component
+COMPONENTS_FROM_PRODUCTS = False
+
+# If COMPONENTS_FROM_PRODUCTS is True, the default owner for each
+# Trac component is inferred from a default Bugzilla component.
+DEFAULT_COMPONENTS = ["default", "misc", "main"]
+
+# This mapping can assign keywords in the ticket entry to represent
+# products or components (depending on COMPONENTS_FROM_PRODUCTS).
+# The keyword will be ignored if empty.
+KEYWORDS_MAPPING = {
+    #'Bugzilla_product_or_component': 'Keyword',
+    "default": "",
+    "misc": "",
+    }
+
+# If this is True, products or components are all set as keywords
+# even if not mentionned in KEYWORDS_MAPPING.
+MAP_ALL_KEYWORDS = True
+
 
 # Bug comments that should not be imported.  Each entry in list should
 # be a regular expression.
 IGNORE_COMMENTS = [
-   '^Created an attachment \(id='
+   "^Created an attachment \(id="
 ]
 
 ###########################################################################
 # There is some special magic for open in the code:  if there is no
 # Bugzilla owner, open is mapped to 'new' instead.
 STATUS_TRANSLATE = {
-  'unconfirmed' : 'new',
-  'open' : 'assigned',
-  'resolved' : 'closed',
-  'verified' : 'closed',
-  'released' : 'closed'
+  "unconfirmed": "new",
+  "open":        "assigned",
+  "resolved":    "closed",
+  "verified":    "closed",
+  "released":    "closed"
 }
 
 # Translate Bugzilla statuses into Trac keywords.  This provides a way 
 # to retain the Bugzilla statuses in Trac.  e.g. when a bug is marked 
 # 'verified' in Bugzilla it will be assigned a VERIFIED keyword.
 STATUS_KEYWORDS = {
-  'verified' : 'VERIFIED',
-  'released' : 'RELEASED'
+  "verified": "VERIFIED",
+  "released": "RELEASED"
 }
 
 # Some fields in Bugzilla do not have equivalents in Trac.  Changes in
 # fields listed here will not be imported into the ticket change history,
 # otherwise you'd see changes for fields that don't exist in Trac.
-IGNORED_ACTIVITY_FIELDS = ['everconfirmed']
+IGNORED_ACTIVITY_FIELDS = ["everconfirmed"]
+
+# Regular expression and its replacement
+BUG_NO_RE = re.compile(r"\b(bug #?)([0-9])")
+BUG_NO_REPL = r"#\2"
 
 ###
 ### Script begins here
 ###
 
 import os
-import re
 import sys
 import string
 import StringIO
 
 import MySQLdb
 import MySQLdb.cursors
-import trac.env
+try:
+    from trac.env import Environment
+except:
+    from trac.Environment import Environment
+from trac.attachment import Attachment
 
 if not hasattr(sys, 'setdefaultencoding'):
     reload(sys)
 sys.setdefaultencoding('latin1')
 
 # simulated Attachment class for trac.add
-class Attachment:
-    def __init__(self, name, data):
-        self.filename = name
-        self.file = StringIO.StringIO(data.tostring())
+#class Attachment:
+#    def __init__(self, name, data):
+#        self.filename = name
+#        self.file = StringIO.StringIO(data.tostring())
   
 # simple field translation mapping.  if string not in
 # mapping, just return string, otherwise return value
 
 class TracDatabase(object):
     def __init__(self, path):
-        self.env = trac.env.Environment(path)
+        self.env = Environment(path)
         self._db = self.env.get_db_cnx()
         self._db.autocommit = False
         self.loginNameCache = {}
     
     def hasTickets(self):
         c = self.db().cursor()
-        c.execute('''SELECT count(*) FROM Ticket''')
+        c.execute("SELECT count(*) FROM Ticket")
         return int(c.fetchall()[0][0]) > 0
 
     def assertNoTickets(self):
         self.assertNoTickets()
         
         c = self.db().cursor()
-        c.execute("""DELETE FROM enum WHERE type='severity'""")
+        c.execute("DELETE FROM enum WHERE type='severity'")
         for value, i in s:
-            print "inserting severity ", value, " ", i
-            c.execute("""INSERT INTO enum (type, name, value) VALUES (%s, %s, %s)""",
-                      "severity", value.encode('utf-8'), i)
+            print "  inserting severity '%s' - '%s'" % (value, i)
+            c.execute("""INSERT INTO enum (type, name, value)
+                                   VALUES (%s, %s, %s)""",
+                      ("severity", value.encode('utf-8'), i))
         self.db().commit()
     
     def setPriorityList(self, s):
         self.assertNoTickets()
         
         c = self.db().cursor()
-        c.execute("""DELETE FROM enum WHERE type='priority'""")
+        c.execute("DELETE FROM enum WHERE type='priority'")
         for value, i in s:
-            print "inserting priority ", value, " ", i
-            c.execute("""INSERT INTO enum (type, name, value) VALUES (%s, %s, %s)""",
-                      "priority",
-                      value.encode('utf-8'),
-                      i)
+            print "  inserting priority '%s' - '%s'" % (value, i)
+            c.execute("""INSERT INTO enum (type, name, value)
+                                   VALUES (%s, %s, %s)""",
+                      ("priority", value.encode('utf-8'), i))
         self.db().commit()
 
     
         self.assertNoTickets()
         
         c = self.db().cursor()
-        c.execute("""DELETE FROM component""")
+        c.execute("DELETE FROM component")
         for comp in l:
-            print "inserting component '",comp[key],"', owner",  comp['owner']
-            c.execute("""INSERT INTO component (name, owner) VALUES (%s, %s)""",
-                      comp[key].encode('utf-8'), comp['owner'].encode('utf-8'))
+            print "  inserting component '%s', owner '%s'" % \
+                            (comp[key], comp['owner'])
+            c.execute("INSERT INTO component (name, owner) VALUES (%s, %s)",
+                      (comp[key].encode('utf-8'),
+                       comp['owner'].encode('utf-8')))
         self.db().commit()
     
     def setVersionList(self, v, key):
         self.assertNoTickets()
         
         c = self.db().cursor()
-        c.execute("""DELETE FROM version""")
+        c.execute("DELETE FROM version")
         for vers in v:
-            print "inserting version ", vers[key]
-            c.execute("""INSERT INTO version (name) VALUES (%s)""",
-                      vers[key].encode('utf-8'))
+            print "  inserting version '%s'" % (vers[key])
+            c.execute("INSERT INTO version (name) VALUES (%s)", (vers[key],))
         self.db().commit()
         
     def setMilestoneList(self, m, key):
         self.assertNoTickets()
         
         c = self.db().cursor()
-        c.execute("""DELETE FROM milestone""")
+        c.execute("DELETE FROM milestone")
         for ms in m:
-            print "inserting milestone ", ms[key]
-            c.execute("""INSERT INTO milestone (name) VALUES (%s)""",
-                      ms[key].encode('utf-8'))
+            milestone = ms[key]
+            print "  inserting milestone '%s'" % (milestone)
+            c.execute("INSERT INTO milestone (name) VALUES (%s)",
+                      (milestone.encode('utf-8')))
         self.db().commit()
     
-    def addTicket(self, id, time, changetime, component,
-                  severity, priority, owner, reporter, cc,
-                  version, milestone, status, resolution,
+    def addTicket(self, id, time, changetime, component, severity, priority,
+                  owner, reporter, cc, version, milestone, status, resolution,
                   summary, description, keywords):
         c = self.db().cursor()
         
         desc = description.encode('utf-8')
+        type = "defect"
+        
+        if severity.lower() == "enhancement":
+                severity = "minor"
+                type = "enhancement"
         
         if PREFORMAT_COMMENTS:
           desc = '{{{\n%s\n}}}' % desc
 
-        print "inserting ticket %s -- %s" % (id, summary)
-        c.execute("""INSERT INTO ticket (id, time, changetime, component,
-                                         severity, priority, owner, reporter, cc,
-                                         version, milestone, status, resolution,
-                                         summary, description, keywords)
-                                 VALUES (%s, %s, %s, %s,
-                                         %s, %s, %s, %s, %s,
-                                         %s, %s, %s, %s,
-                                         %s, %s, %s)""",
-                  id, time.strftime('%s'), changetime.strftime('%s'), component.encode('utf-8'),
-                  severity.encode('utf-8'), priority.encode('utf-8'), owner, reporter, cc,
-                  version, milestone.encode('utf-8'), status.lower(), resolution,
-                  summary.encode('utf-8'), desc, keywords)
+        if REPLACE_BUG_NO:
+            if BUG_NO_RE.search(desc):
+                desc = re.sub(BUG_NO_RE, BUG_NO_REPL, desc)
+
+        if PRIORITIES_MAP.has_key(priority):
+            priority = PRIORITIES_MAP[priority]
+
+        print "  inserting ticket %s -- %s" % (id, summary)
+
+        c.execute("""INSERT INTO ticket (id, type, time, changetime, component,
+                                         severity, priority, owner, reporter,
+                                         cc, version, milestone, status,
+                                         resolution, summary, description,
+                                         keywords)
+                                 VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s,
+                                         %s, %s, %s, %s, %s, %s, %s, %s)""",
+                  (id, type.encode('utf-8'), time.strftime('%s'),
+                   changetime.strftime('%s'), component.encode('utf-8'),
+                   severity.encode('utf-8'), priority.encode('utf-8'), owner,
+                   reporter, cc, version, milestone.encode('utf-8'),
+                   status.lower(), resolution, summary.encode('utf-8'), desc,
+                   keywords))
         
         self.db().commit()
-        return self.db().db.sqlite_last_insert_rowid()
+        return self.db().get_last_id(c, 'ticket')
     
     def addTicketComment(self, ticket, time, author, value):
         comment = value.encode('utf-8')
         if PREFORMAT_COMMENTS:
           comment = '{{{\n%s\n}}}' % comment
 
+        if REPLACE_BUG_NO:
+            if BUG_NO_RE.search(comment):
+                comment = re.sub(BUG_NO_RE, BUG_NO_REPL, comment)
+
         c = self.db().cursor()
-        c.execute("""INSERT INTO ticket_change (ticket, time, author, field, oldvalue, newvalue)
-                                 VALUES        (%s, %s, %s, %s, %s, %s)""",
-                  ticket, time.strftime('%s'), author, 'comment', '', comment)
+        c.execute("""INSERT INTO ticket_change (ticket, time, author, field,
+                                                oldvalue, newvalue)
+                                        VALUES (%s, %s, %s, %s, %s, %s)""",
+                  (ticket, time.strftime('%s'), author, 'comment', '', comment))
         self.db().commit()
 
     def addTicketChange(self, ticket, time, author, field, oldvalue, newvalue):
         c = self.db().cursor()
-        c.execute("""INSERT INTO ticket_change (ticket, time, author, field, oldvalue, newvalue)
-                                 VALUES        (%s, %s, %s, %s, %s, %s)""",
-                  ticket, time.strftime('%s'), author, field, oldvalue.encode('utf-8'), newvalue.encode('utf-8'))
+
+        if field == "priority":
+            if PRIORITIES_MAP.has_key(oldvalue.lower()):
+                oldvalue = PRIORITIES_MAP[oldvalue.lower()]
+            if PRIORITIES_MAP.has_key(newvalue.lower()):
+                newvalue = PRIORITIES_MAP[newvalue.lower()]
+
+        # Doesn't make sense if we go from highest -> highest, for example.
+        if oldvalue == newvalue:
+            return
+        
+        c.execute("""INSERT INTO ticket_change (ticket, time, author, field,
+                                                oldvalue, newvalue)
+                                        VALUES (%s, %s, %s, %s, %s, %s)""",
+                  (ticket, time.strftime('%s'), author, field,
+                   oldvalue.encode('utf-8'), newvalue.encode('utf-8')))
         self.db().commit()
         
-    def addAttachment(self, id, attachment, description, author):
-        print 'inserting attachment for ticket %s -- %s' % (id, description)
-        attachment.filename = attachment.filename.encode('utf-8')
-        self.env.create_attachment(self.db(), 'ticket', str(id), attachment, description.encode('utf-8'),
-            author, 'unknown')
+    def addAttachment(self, author, a):
+        description = a['description'].encode('utf-8')
+        id = a['bug_id']
+        filename = a['filename'].encode('utf-8')
+        filedata = StringIO.StringIO(a['thedata'].tostring())
+        filesize = len(filedata.getvalue())
+        time = a['creation_ts']
+        print "    ->inserting attachment '%s' for ticket %s -- %s" % \
+                (filename, id, description)
+
+        attachment = Attachment(self.env, 'ticket', id)
+        attachment.author = author
+        attachment.description = description
+        attachment.insert(filename, filedata, filesize, time.strftime('%s'))
+        del attachment
         
     def getLoginName(self, cursor, userid):
         if userid not in self.loginNameCache:
-            cursor.execute("SELECT * FROM profiles WHERE userid = %s" % userid)
+            cursor.execute("SELECT * FROM profiles WHERE userid = %s", (userid))
             loginName = cursor.fetchall()
 
             if loginName:
                 loginName = loginName[0]['login_name']
             else:
-                print 'warning: unknown bugzilla userid %d, recording as anonymous' % userid
-                loginName = 'anonymous'
+                print """WARNING: unknown bugzilla userid %d, recording as
+                         anonymous""" % (userid)
+                loginName = "anonymous"
+
+            loginName = LOGIN_MAP.get(loginName, loginName)
 
             self.loginNameCache[userid] = loginName
 
 
     def getFieldName(self, cursor, fieldid):
         if fieldid not in self.fieldNameCache:
-            cursor.execute("SELECT * FROM fielddefs WHERE fieldid = %s" % fieldid)
+            cursor.execute("SELECT * FROM fielddefs WHERE fieldid = %s",
+                           (fieldid))
             fieldName = cursor.fetchall()
 
             if fieldName:
                 fieldName = fieldName[0]['name'].lower()
             else:
-                print 'warning: unknown bugzilla fieldid %d, recording as unknown' % userid
-                fieldName = 'unknown'
+                print "WARNING: unknown bugzilla fieldid %d, \
+                                recording as unknown" % (userid)
+                fieldName = "unknown"
 
             self.fieldNameCache[fieldid] = fieldName
 
         return self.fieldNameCache[fieldid]
 
-def productFilter(fieldName, products):
-    first = True
-    result = ''
-    for product in products:
-        if not first: 
-            result += " or "
-        first = False
-        result += "%s = '%s'" % (fieldName, product)
-    return result
+def makeWhereClause(fieldName, values, negative=False):
+    if not values:
+        return ''
+    if negative:
+        connector, op = ' AND ', '!='
+    else:
+        connector, op = ' OR ', '='
+    clause = connector.join(["%s %s '%s'" % (fieldName, op, value) for value in values])
+    return ' ' + clause
 
 def convert(_db, _host, _user, _password, _env, _force):
     activityFields = FieldTranslator()
 
     # account for older versions of bugzilla
-    if BZ_VERSION == '2.11':
-        print 'Using Buzvilla v%s schema.' % BZ_VERSION
-        activityFields['removed'] = 'oldvalue'
-        activityFields['added'] = 'newvalue'
+    print "Using Bugzilla v%s schema." % BZ_VERSION
+    if BZ_VERSION == 2110:
+        activityFields['removed'] = "oldvalue"
+        activityFields['added'] = "newvalue"
 
     # init Bugzilla environment
-    print "Bugzilla MySQL('%s':'%s':'%s':'%s'): connecting..." % (_db, _host, _user, _password)
+    print "Bugzilla MySQL('%s':'%s':'%s':'%s'): connecting..." % \
+            (_db, _host, _user, ("*" * len(_password)))
     mysql_con = MySQLdb.connect(host=_host, 
                 user=_user, passwd=_password, db=_db, compress=1, 
                 cursorclass=MySQLdb.cursors.DictCursor)
 
     # force mode...
     if _force == 1:
-        print "cleaning all tickets..."
+        print "\nCleaning all tickets..."
         c = trac.db().cursor()
-        c.execute("""DELETE FROM ticket_change""")
+        c.execute("DELETE FROM ticket_change")
         trac.db().commit()
-        c.execute("""DELETE FROM ticket""")
+        
+        c.execute("DELETE FROM ticket")
         trac.db().commit()
-        c.execute("""DELETE FROM attachment""")
-        os.system('rm -rf %s' % trac.env.get_attachments_dir())
-        os.mkdir(trac.env.get_attachments_dir())
+        
+        c.execute("DELETE FROM attachment")
+	attachments_dir = os.path.join(os.path.normpath(trac.env.path),
+                                "attachments")
+        # Straight from the Python documentation.
+        for root, dirs, files in os.walk(attachments_dir, topdown=False):
+            for name in files:
+                os.remove(os.path.join(root, name))
+            for name in dirs:
+                os.rmdir(os.path.join(root, name))
+        if not os.stat(attachments_dir):
+            os.mkdir(attachments_dir)
         trac.db().commit()
+        print "All tickets cleaned..."
 
 
-    print
-    print "1. import severities..."
-    severities = (('blocker', '1'), ('critical', '2'), ('major', '3'), ('normal', '4'),
-        ('minor', '5'), ('trivial', '6'), ('enhancement', '7'))
-    trac.setSeverityList(severities)
+    print "\n0. Filtering products..."
+    mysql_cur.execute("SELECT name FROM products")
+    products = []
+    for line in mysql_cur.fetchall():
+        product = line['name']
+        if PRODUCTS and product not in PRODUCTS:
+            continue
+        if product in IGNORE_PRODUCTS:
+            continue
+        products.append(product)
+    PRODUCTS[:] = products
+    print "  Using products", " ".join(PRODUCTS)
 
-    print
-    print "2. import components..."
-    sql = "SELECT value, initialowner AS owner FROM components"
-    if PRODUCTS:
-       sql += " WHERE %s" % productFilter('program', PRODUCTS)
-    mysql_cur.execute(sql)
-    components = mysql_cur.fetchall()
-    for component in components:
-    		component['owner'] = trac.getLoginName(mysql_cur, component['owner'])
-    trac.setComponentList(components, 'value')
+    print "\n1. Import severities..."
+    trac.setSeverityList(SEVERITIES)
 
-    print
-    print "3. import priorities..."
-    priorities = (('P1', '1'), ('P2', '2'), ('P3', '3'), ('P4', '4'), ('P5', '5'))
-    trac.setPriorityList(priorities)
+    print "\n2. Import components..."
+    if not COMPONENTS_FROM_PRODUCTS:
+    	if BZ_VERSION >= 2180:
+	    sql = """SELECT DISTINCT c.name AS name, c.initialowner AS owner
+                                FROM components AS c, products AS p
+	                       WHERE c.product_id = p.id AND"""
+	    sql += makeWhereClause('p.name', PRODUCTS)
+	else:
+	    sql = "SELECT value AS name, initialowner AS owner FROM components"
+            sql += " WHERE" + makeWhereClause('program', PRODUCTS)
+        mysql_cur.execute(sql)
+        components = mysql_cur.fetchall()
+        for component in components:
+            component['owner'] = trac.getLoginName(mysql_cur,
+                                                   component['owner'])
+        trac.setComponentList(components, 'name')
+    else:
+        sql = """SELECT program AS product, value AS comp, initialowner AS owner
+                   FROM components"""
+        sql += " WHERE" + makeWhereClause('program', PRODUCTS)
+        mysql_cur.execute(sql)
+        lines = mysql_cur.fetchall()
+        all_components = {} # product -> components
+        all_owners = {} # product, component -> owner
+        for line in lines:
+            product = line['product']
+            comp = line['comp']
+            owner = line['owner']
+            all_components.setdefault(product, []).append(comp)
+            all_owners[(product, comp)] = owner
+        component_list = []
+        for product, components in all_components.items():
+            # find best default owner
+            default = None
+            for comp in DEFAULT_COMPONENTS:
+                if comp in components:
+                    default = comp
+                    break
+            if default is None:
+                default = components[0]
+            owner = all_owners[(product, default)]
+            owner_name = trac.getLoginName(mysql_cur, owner)
+            component_list.append({'product': product, 'owner': owner_name})
+        trac.setComponentList(component_list, 'product')
 
-    print
-    print "4. import versions..."
-    sql = "SELECT DISTINCTROW value FROM versions"
-    if PRODUCTS:
-       sql += " WHERE %s" % productFilter('program', PRODUCTS)
+    print "\n3. Import priorities..."
+    trac.setPriorityList(PRIORITIES)
+
+    print "\n4. Import versions..."
+    if BZ_VERSION >= 2180:
+        sql = """SELECT DISTINCTROW versions.value AS value
+                               FROM products, versions"""
+	sql += " WHERE" + makeWhereClause('products.name', PRODUCTS)
+    else:
+    	sql = "SELECT DISTINCTROW value FROM versions"
+    	sql += " WHERE" + makeWhereClause('program', PRODUCTS)
     mysql_cur.execute(sql)
     versions = mysql_cur.fetchall()
     trac.setVersionList(versions, 'value')
 
-    print
-    print "5. import milestones..."
-    mysql_cur.execute("SELECT value FROM milestones")
+    print "\n5. Import milestones..."
+    sql = "SELECT DISTINCT value FROM milestones"
+    sql += " WHERE" + makeWhereClause('value', IGNORE_MILESTONES, negative=True)
+    mysql_cur.execute(sql)
     milestones = mysql_cur.fetchall()
-    if milestones[0] == '---':
-        trac.setMilestoneList(milestones, 'value')
-    else:
-        trac.setMilestoneList([], '')
+    trac.setMilestoneList(milestones, 'value')
 
-    print
-    print '6. retrieving bugs...'
-    sql = "SELECT * FROM bugs "
-    if PRODUCTS:
-       sql += " WHERE %s" % productFilter('product', PRODUCTS)
-    sql += " ORDER BY bug_id"
+    print "\n6. Retrieving bugs..."
+    sql = """SELECT DISTINCT b.*, c.name AS component, p.name AS product
+                        FROM bugs AS b, components AS c, products AS p """
+    sql += " WHERE (" + makeWhereClause('p.name', PRODUCTS)
+    sql += ") AND b.product_id = p.id"
+    sql += " AND b.component_id = c.id"
+    sql += " ORDER BY b.bug_id"
     mysql_cur.execute(sql)
     bugs = mysql_cur.fetchall()
+
     
-    print
-    print "7. import bugs and bug activity..."
+    print "\n7. Import bugs and bug activity..."
     for bug in bugs:
         bugid = bug['bug_id']
         
         ticket['id'] = bugid
         ticket['time'] = bug['creation_ts']
         ticket['changetime'] = bug['delta_ts']
-        ticket['component'] = bug['component']
+        if COMPONENTS_FROM_PRODUCTS:
+            ticket['component'] = bug['product']
+        else:
+            ticket['component'] = bug['component']
         ticket['severity'] = bug['bug_severity']
-        ticket['priority'] = bug['priority']
+        ticket['priority'] = bug['priority'].lower()
 
         ticket['owner'] = trac.getLoginName(mysql_cur, bug['assigned_to'])
         ticket['reporter'] = trac.getLoginName(mysql_cur, bug['reporter'])
 
-        mysql_cur.execute("SELECT * FROM cc WHERE bug_id = %s" % bugid)
+        mysql_cur.execute("SELECT * FROM cc WHERE bug_id = %s", bugid)
         cc_records = mysql_cur.fetchall()
         cc_list = []
         for cc in cc_records:
             cc_list.append(trac.getLoginName(mysql_cur, cc['who']))
+        cc_list = [cc for cc in cc_list if '@' in cc and cc not in IGNORE_CC]
         ticket['cc'] = string.join(cc_list, ', ')
 
         ticket['version'] = bug['version']
 
-        if bug['target_milestone'] == '---':
-            ticket['milestone'] = ''
-        else:
-            ticket['milestone'] = bug['target_milestone']
+        target_milestone = bug['target_milestone']
+        if target_milestone in IGNORE_MILESTONES:
+            target_milestone = ''
+        ticket['milestone'] = target_milestone
 
         bug_status = bug['bug_status'].lower()
         ticket['status'] = statusXlator[bug_status]
 
         ticket['summary'] = bug['short_desc']
 
-        keywords = string.split(bug['keywords'], ' ')
-
         mysql_cur.execute("SELECT * FROM longdescs WHERE bug_id = %s" % bugid) 
         longdescs = list(mysql_cur.fetchall())
 
                 if re.match(comment, desc['thetext']):
                     ignore = True
                     
-            if ignore: continue
+            if ignore:
+                    continue
             
             trac.addTicketComment(ticket=bugid,
-                time=desc['bug_when'],
+                time = desc['bug_when'],
                 author=trac.getLoginName(mysql_cur, desc['who']),
-                value=desc['thetext'])
+                value = desc['thetext'])
 
-        mysql_cur.execute("SELECT * FROM bugs_activity WHERE bug_id = %s ORDER BY bug_when" % bugid)
+        mysql_cur.execute("""SELECT * FROM bugs_activity WHERE bug_id = %s
+                           ORDER BY bug_when""" % bugid)
         bugs_activity = mysql_cur.fetchall()
         resolution = ''
         ticketChanges = []
+        keywords = []
         for activity in bugs_activity:
             field_name = trac.getFieldName(mysql_cur, activity['fieldid']).lower()
             
             added = activity[activityFields['added']]
 
             # statuses and resolutions are in lowercase in trac
-            if field_name == 'resolution' or field_name == 'bug_status':
+            if field_name == "resolution" or field_name == "bug_status":
                 removed = removed.lower()
                 added = added.lower()
 
             # remember most recent resolution, we need this later
-            if field_name == 'resolution':
+            if field_name == "resolution":
                 resolution = added.lower()
 
-            keywordChange = False
-            oldKeywords = string.join(keywords, " ")
+            add_keywords = []
+            remove_keywords = []
 
             # convert bugzilla field names...
-            if field_name == 'bug_severity':
-                field_name = 'severity'
-            elif field_name == 'assigned_to':
-                field_name = 'owner'
-            elif field_name == 'bug_status':
-                field_name = 'status'
+            if field_name == "bug_severity":
+                field_name = "severity"
+            elif field_name == "assigned_to":
+                field_name = "owner"
+            elif field_name == "bug_status":
+                field_name = "status"
                 if removed in STATUS_KEYWORDS:
-                    kw = STATUS_KEYWORDS[removed]
-                    if kw in keywords:
-                        keywords.remove(kw)
-                    else:
-                        oldKeywords = string.join(keywords + [ kw ], " ")
-                    keywordChange = True
+                    remove_keywords.append(STATUS_KEYWORDS[removed])
                 if added in STATUS_KEYWORDS:
-                    kw = STATUS_KEYWORDS[added]
-                    keywords.append(kw)
-                    keywordChange = True
+                    add_keywords.append(STATUS_KEYWORDS[added])
                 added = statusXlator[added]
                 removed = statusXlator[removed]
-            elif field_name == 'short_desc':
-                field_name = 'summary'
-            elif field_name == 'product':
-                if removed in PRODUCT_KEYWORDS:
-                    kw = PRODUCT_KEYWORDS[removed]
-                    if kw in keywords:
-                        keywords.remove(kw)
-                    else:
-                        oldKeywords = string.join(keywords + [ kw ], " ")
-                    keywordChange = True
-                if added in PRODUCT_KEYWORDS:
-                    kw = PRODUCT_KEYWORDS[added]
-                    keywords.append(kw)
-                    keywordChange = True
+            elif field_name == "short_desc":
+                field_name = "summary"
+            elif field_name == "product" and COMPONENTS_FROM_PRODUCTS:
+                field_name = "component"
+            elif ((field_name == "product" and not COMPONENTS_FROM_PRODUCTS) or
+                  (field_name == "component" and COMPONENTS_FROM_PRODUCTS)):
+                if MAP_ALL_KEYWORDS or removed in KEYWORDS_MAPPING:
+                    kw = KEYWORDS_MAPPING.get(removed, removed)
+                    if kw:
+                        remove_keywords.append(kw)
+                if MAP_ALL_KEYWORDS or added in KEYWORDS_MAPPING:
+                    kw = KEYWORDS_MAPPING.get(added, added)
+                    if kw:
+                        add_keywords.append(kw)
+                if field_name == "component":
+                    # just keep the keyword change
+                    added = removed = ""
+            elif field_name == "target_milestone":
+                field_name = "milestone"
+                if added in IGNORE_MILESTONES:
+                    added = ""
+                if removed in IGNORE_MILESTONES:
+                    removed = ""
 
             ticketChange = {}
             ticketChange['ticket'] = bugid
             ticketChange['time'] = activity['bug_when']
-            ticketChange['author'] = trac.getLoginName(mysql_cur, activity['who'])
+            ticketChange['author'] = trac.getLoginName(mysql_cur,
+                                                       activity['who'])
             ticketChange['field'] = field_name
             ticketChange['oldvalue'] = removed
             ticketChange['newvalue'] = added
 
-            if keywordChange:
-                newKeywords = string.join(keywords, " ")
-                ticketChangeKw = ticketChange
-                ticketChangeKw['field'] = 'keywords'
-                ticketChangeKw['oldvalue'] = oldKeywords
-                ticketChangeKw['newvalue'] = newKeywords
-                #trac.addTicketChange(ticket=bugid, time=activity['bug_when'],
-                #    author=trac.getLoginName(mysql_cur, activity['who']),
-                #    field='keywords', oldvalue=oldKeywords, newvalue=newKeywords)
-                ticketChanges.append(ticketChangeKw)
+            if add_keywords or remove_keywords:
+                # ensure removed ones are in old
+                old_keywords = keywords + [kw for kw in remove_keywords if kw
+                                           not in keywords]
+                # remove from new
+                keywords = [kw for kw in keywords if kw not in remove_keywords]
+                # add to new
+                keywords += [kw for kw in add_keywords if kw not in keywords]
+                if old_keywords != keywords:
+                    ticketChangeKw = ticketChange.copy()
+                    ticketChangeKw['field'] = "keywords"
+                    ticketChangeKw['oldvalue'] = ' '.join(old_keywords)
+                    ticketChangeKw['newvalue'] = ' '.join(keywords)
+                    ticketChanges.append(ticketChangeKw)
 
             if field_name in IGNORED_ACTIVITY_FIELDS:
                 continue
 
-            # skip changes that have no effect (think translation!)
+            # Skip changes that have no effect (think translation!).
             if added == removed:
                 continue
                 
-            # bugzilla splits large summary changes into two records
+            # Bugzilla splits large summary changes into two records.
             for oldChange in ticketChanges:
-              if (field_name == 'summary'
+              if (field_name == "summary"
                   and oldChange['field'] == ticketChange['field'] 
                   and oldChange['time'] == ticketChange['time'] 
                   and oldChange['author'] == ticketChange['author']):
                   oldChange['oldvalue'] += " " + ticketChange['oldvalue'] 
                   oldChange['newvalue'] += " " + ticketChange['newvalue']
                   break
+              # cc sometime appear in different activities with same time
+              if (field_name == "cc" \
+                  and oldChange['time'] == ticketChange['time']):
+                  oldChange['newvalue'] += ", " + ticketChange['newvalue']
+                  break
             else:
-                #trac.addTicketChange(ticket=bugid, time=activity['bug_when'],
-                #    author=trac.getLoginName(mysql_cur, activity['who']),
-                #    field=field_name, oldvalue=removed, newvalue=added)
                 ticketChanges.append (ticketChange)
 
         for ticketChange in ticketChanges:
             trac.addTicketChange (**ticketChange)
 
-
-        # for some reason, bugzilla v2.11 seems to clear the resolution
-        # when you mark a bug as closed.  let's remember it and restore
+        # For some reason, bugzilla v2.11 seems to clear the resolution
+        # when you mark a bug as closed.  Let's remember it and restore
         # it if the ticket is closed but there's no resolution.
-        if not ticket['resolution'] and ticket['status'] == 'closed':
+        if not ticket['resolution'] and ticket['status'] == "closed":
             ticket['resolution'] = resolution
 
-        if bug['bug_status'] in STATUS_KEYWORDS:
-            kw = STATUS_KEYWORDS[bug['bug_status']]
-            # may have already been added during activity import
+        bug_status = bug['bug_status']
+        if bug_status in STATUS_KEYWORDS:
+            kw = STATUS_KEYWORDS[bug_status]
             if kw not in keywords:
                 keywords.append(kw)
 
-        if bug['product'] in PRODUCT_KEYWORDS:
-            kw = PRODUCT_KEYWORDS[bug['product']]
-            # may have already been added during activity import
-            if kw not in keywords:
+        product = bug['product']
+        if product in KEYWORDS_MAPPING and not COMPONENTS_FROM_PRODUCTS:
+            kw = KEYWORDS_MAPPING.get(product, product)
+            if kw and kw not in keywords:
                 keywords.append(kw)
 
+        component = bug['component']
+        if (COMPONENTS_FROM_PRODUCTS and \
+            (MAP_ALL_KEYWORDS or component in KEYWORDS_MAPPING)):
+            kw = KEYWORDS_MAPPING.get(component, component)
+            if kw and kw not in keywords:
+                keywords.append(kw)
+
+        ticket['keywords'] = string.join(keywords)                
+        ticketid = trac.addTicket(**ticket)
+
         mysql_cur.execute("SELECT * FROM attachments WHERE bug_id = %s" % bugid)
         attachments = mysql_cur.fetchall()
         for a in attachments:
             author = trac.getLoginName(mysql_cur, a['submitter_id'])
+            trac.addAttachment(author, a)
             
-            tracAttachment = Attachment(a['filename'], a['thedata'])
-            trac.addAttachment(bugid, tracAttachment, a['description'], author)
-            
-        ticket['keywords'] = string.join(keywords)                
-        ticketid = trac.addTicket(**ticket)
+    print "\n8. Importing users and passwords..."
+    if BZ_VERSION >= 2180:
+        mysql_cur.execute("SELECT login_name, cryptpassword FROM profiles")
+        users = mysql_cur.fetchall()
+    htpasswd = file("htpasswd", 'w')
+    for user in users:
+        if LOGIN_MAP.has_key(user['login_name']):
+            login = LOGIN_MAP[user['login_name']]
+        else:
+            login = user['login_name']
+        htpasswd.write(login + ":" + user['cryptpassword'] + "\n")
 
-    print "Success!"
+    htpasswd.close()
+    print "  Bugzilla users converted to htpasswd format, see 'htpasswd'."
+
+    print "\nAll tickets converted."
+
+def log(msg):
+    print "DEBUG: %s" % (msg)
 
 def usage():
-    print "bugzilla2trac - Imports a bug database from Bugzilla into Trac."
-    print
-    print "Usage: bugzilla2trac.py [options]"
-    print
-    print "Available Options:"
-    print "  --db <MySQL dbname>              - Bugzilla's database"
-    print "  --tracenv /path/to/trac/env      - full path to Trac db environment"
-    print "  -h | --host <MySQL hostname>     - Bugzilla's DNS host name"
-    print "  -u | --user <MySQL username>     - effective Bugzilla's database user"
-    print "  -p | --passwd <MySQL password>   - Bugzilla's user password"
-    print "  -c | --clean                     - remove current Trac tickets before importing"
-    print "  --help | help                    - this help info"
-    print
-    print "Additional configuration options can be defined directly in the script."
-    print
+    print """bugzilla2trac - Imports a bug database from Bugzilla into Trac.
+
+Usage: bugzilla2trac.py [options]
+
+Available Options:
+  --db <MySQL dbname>              - Bugzilla's database name
+  --tracenv /path/to/trac/env      - Full path to Trac db environment
+  -h | --host <MySQL hostname>     - Bugzilla's DNS host name
+  -u | --user <MySQL username>     - Effective Bugzilla's database user
+  -p | --passwd <MySQL password>   - Bugzilla's user password
+  -c | --clean                     - Remove current Trac tickets before
+                                     importing
+  --help | help                    - This help info
+
+Additional configuration options can be defined directly in the script.
+"""
     sys.exit(0)
 
 def main():
     global BZ_DB, BZ_HOST, BZ_USER, BZ_PASSWORD, TRAC_ENV, TRAC_CLEAN
     if len (sys.argv) > 1:
-        if sys.argv[1] in ['--help','help'] or len(sys.argv) < 4:
-            usage()
-        iter = 1
-        while iter < len(sys.argv):
-            if sys.argv[iter] in ['--db'] and iter+1 < len(sys.argv):
-                BZ_DB = sys.argv[iter+1]
-                iter = iter + 1
-            elif sys.argv[iter] in ['-h', '--host'] and iter+1 < len(sys.argv):
-                BZ_HOST = sys.argv[iter+1]
-                iter = iter + 1
-            elif sys.argv[iter] in ['-u', '--user'] and iter+1 < len(sys.argv):
-                BZ_USER = sys.argv[iter+1]
-                iter = iter + 1
-            elif sys.argv[iter] in ['-p', '--passwd'] and iter+1 < len(sys.argv):
-                BZ_PASSWORD = sys.argv[iter+1]
-                iter = iter + 1
-            elif sys.argv[iter] in ['--tracenv'] and iter+1 < len(sys.argv):
-                TRAC_ENV = sys.argv[iter+1]
-                iter = iter + 1
-            elif sys.argv[iter] in ['-c', '--clean']:
-                TRAC_CLEAN = 1
-            else:
-                print "Error: unknown parameter: " + sys.argv[iter]
-                sys.exit(0)
-            iter = iter + 1
-    else:
-        usage()
+    	if sys.argv[1] in ['--help','help'] or len(sys.argv) < 4:
+    	    usage()
+    	iter = 1
+    	while iter < len(sys.argv):
+    	    if sys.argv[iter] in ['--db'] and iter+1 < len(sys.argv):
+    	        BZ_DB = sys.argv[iter+1]
+    	        iter = iter + 1
+    	    elif sys.argv[iter] in ['-h', '--host'] and iter+1 < len(sys.argv):
+    	        BZ_HOST = sys.argv[iter+1]
+    	        iter = iter + 1
+    	    elif sys.argv[iter] in ['-u', '--user'] and iter+1 < len(sys.argv):
+    	        BZ_USER = sys.argv[iter+1]
+    	        iter = iter + 1
+    	    elif sys.argv[iter] in ['-p', '--passwd'] and iter+1 < len(sys.argv):
+    	        BZ_PASSWORD = sys.argv[iter+1]
+    	        iter = iter + 1
+    	    elif sys.argv[iter] in ['--tracenv'] and iter+1 < len(sys.argv):
+    	        TRAC_ENV = sys.argv[iter+1]
+    	        iter = iter + 1
+    	    elif sys.argv[iter] in ['-c', '--clean']:
+    	        TRAC_CLEAN = 1
+    	    else:
+    	        print "Error: unknown parameter: " + sys.argv[iter]
+    	        sys.exit(0)
+    	    iter = iter + 1
         
     convert(BZ_DB, BZ_HOST, BZ_USER, BZ_PASSWORD, TRAC_ENV, TRAC_CLEAN)