Commits

dirkbaechle committed 3db61f7

- first version
- can read SCons issues from the Tigris bug tracker in XML format to a set of files
- can merge the XML files to a single file in reduced format
- probably needs some further tweaking, like adding/removing data fields
- there is no support for interfacing the google.code tracker (uploading the ripped
issues again), yet

Comments (0)

Files changed (10)

issues/scons_tigris/data/TigrisActivity.py

+'''
+Created on 08.04.2010
+
+@author: dirk
+'''
+
+import io.xml
+
+class TigrisActivity:
+    '''
+    A single activity in a Tigris issue.
+    '''
+
+
+    def __init__(self, who='', when='', field='', old='', new=''):
+        '''
+        Constructor
+        '''
+        self.who = who
+        self.when = when
+        self.field = field
+        self.old = old
+        self.new = new
+        
+    def appendNode(self, doc, root):
+        e = doc.createElement("activity")
+        io.xml.addChildText(doc, e, "user", self.who)
+        io.xml.addChildText(doc, e, "when", self.when)
+        io.xml.addChildText(doc, e, "field_name", self.field)
+        io.xml.addChildText(doc, e, "oldvalue", self.old)
+        io.xml.addChildText(doc, e, "newvalue", self.new)
+        root.appendChild(e)
+    
+    def parseNode(self, node):
+        self.who = io.xml.getChildText(node, "user")
+        self.when = io.xml.getChildText(node, "when")
+        self.field = io.xml.getChildText(node, "field_name")
+        self.old = io.xml.getChildText(node, "oldvalue")
+        self.new = io.xml.getChildText(node, "newvalue")
+        

issues/scons_tigris/data/TigrisAttachment.py

+'''
+Created on 08.04.2010
+
+@author: dirk
+'''
+
+import io.xml
+
+class TigrisAttachment:
+    '''
+    A single attachment in a Tigris issue.
+    '''
+
+    def __init__(self, who='', when='', file=''):
+        '''
+        Constructor
+        '''
+        self.who = who
+        self.when = when
+        self.file = file
+        
+    def appendNode(self, doc, root):
+        e = doc.createElement("attachment")
+        io.xml.addChildText(doc, e, "submitting_username", self.who)
+        io.xml.addChildText(doc, e, "date", self.when)
+        io.xml.addChildText(doc, e, "filename", self.file)
+        root.appendChild(e)
+    
+    def parseNode(self, node):
+        self.who = io.xml.getChildText(node, "submitting_username")
+        self.when = io.xml.getChildText(node, "date")
+        self.file = io.xml.getChildText(node, "filename")
+        

issues/scons_tigris/data/TigrisIssue.py

+# coding: latin-1
+'''
+Created on 08.04.2010
+
+@author: dirk
+'''
+
+import TigrisActivity
+import TigrisAttachment
+import TigrisIssueMessage
+
+import io.xml
+import xml.dom.minidom
+
+
+class TigrisIssue:
+    '''
+    A complete Tigris issue with its messages and activities.
+    '''
+    def __init__(self):
+       
+        self.messages = []
+        self.activities = []
+        self.attachments = []
+        self.id = ''
+        self.desc = ''
+        self.status = ''
+        self.assigned = ''
+        self.type = ''
+        self.delta_ts = ''
+        self.creation_ts = ''
+
+    def readXML(self, xpath):
+        # Parse XML
+        dom = xml.dom.minidom.parse(xpath)
+        for i in dom.getElementsByTagName('issue'):
+            self.parseNode(i)
+
+    def appendNode(self, doc, root):
+        e = doc.createElement("issue")
+        io.xml.addChildText(doc, e, "issue_id", self.id)
+        io.xml.addChildText(doc, e, "short_desc", self.desc)
+        io.xml.addChildText(doc, e, "issue_status", self.status)
+        io.xml.addChildText(doc, e, "assigned_to", self.assigned)
+        io.xml.addChildText(doc, e, "issue_type", self.type)
+        io.xml.addChildText(doc, e, "delta_ts", self.delta_ts)
+        io.xml.addChildText(doc, e, "creation_ts", self.creation_ts)
+        
+        for m in self.messages:
+            m.appendNode(doc, e)
+        for att in self.attachments:
+            att.appendNode(doc, e)
+        for act in self.activities:
+            act.appendNode(doc, e)
+        
+        root.appendChild(e)
+    
+    def parseNode(self, i):
+        self.id = io.xml.getChildText(i, 'issue_id')
+        self.desc = io.xml.getChildText(i, 'short_desc')
+        self.status = io.xml.getChildText(i, 'issue_status')
+        self.assigned = io.xml.getChildText(i, 'assigned_to')
+        self.type = io.xml.getChildText(i, 'issue_type')
+        self.delta_ts = io.xml.getChildText(i, 'delta_ts')
+        self.creation_ts = io.xml.getChildText(i, 'creation_ts')
+
+        for m in i.getElementsByTagName('long_desc'):            
+            tm = TigrisIssueMessage.TigrisIssueMessage()
+            tm.parseNode(m)
+            self.messages.append(tm)
+
+        for att in i.getElementsByTagName('attachment'):            
+            ta = TigrisAttachment.TigrisAttachment()
+            ta.parseNode(att)
+            self.attachments.append(ta)
+
+        for act in i.getElementsByTagName('activity'):            
+            tc = TigrisActivity.TigrisActivity()
+            tc.parseNode(act)
+            self.activities.append(tc)
+
+    def getLastMessageDate(self):
+        d = ''
+        for m in self.messages:
+            if m.when > d:
+                d = m.when
+        
+        return d
+    
+    def getLastAttachmentDate(self):
+        d = ''
+        for a in self.attachments:
+            if a.when > d:
+                d = a.when
+        
+        return d
+
+    def getLastActivityDate(self, field=''):
+        d = ''
+        if field != '':
+            for a in self.activities:
+                if (a.field == field) and (a.when > d):
+                    d = a.when
+        else:
+            for a in self.activities:
+                if a.when > d:
+                    d = a.when
+
+        return d
+
+    def getDeltaDate(self):
+        d = self.getLastMessageDate()
+        n = self.getLastAttachmentDate()
+        if n > d:
+            d = n
+        n = self.getLastActivityDate()
+        if n > d:
+            d = n
+        
+        return d

issues/scons_tigris/data/TigrisIssueList.py

+'''
+Created on 12.04.2010
+
+@author: dirk
+'''
+
+import os
+import xml.dom.minidom
+import TigrisIssue
+
+class TigrisIssueList:
+    '''
+    List of Tigris issues, where each thread may consist of
+    several messages/activities/attachments
+    '''
+
+    def __init__(self):
+        '''
+        Constructor
+        '''
+        self.issues = []
+        
+    def scanXML(self, spath):
+        for path, dirs, files in os.walk(spath):
+            for f in files:
+                if f.endswith('.xml') and not f.find('_') >= 0:
+                    i = TigrisIssue.TigrisIssue()
+                    i.readXML(os.path.join(path,f))
+                    self.issues.append(i)
+    
+    def writeFile(self, fpath):
+        doc = xml.dom.minidom.Document()
+        root = doc.createElement(u'issues')
+        doc.appendChild(root)
+        for i in self.issues:
+            i.appendNode(doc, root)
+        
+        o = open(fpath, 'w')
+        o.write(doc.toxml(encoding="utf-8"))
+        o.close()
+        
+    def readFile(self, fpath):
+        dom = xml.dom.minidom.parse(fpath)
+        for node in dom.getElementsByTagName('issue'):
+            i = TigrisIssue.TigrisIssue()
+            i.parseNode(node)
+            self.issues.append(i)
+        
+        

issues/scons_tigris/data/TigrisIssueMessage.py

+'''
+Created on 08.04.2010
+
+@author: dirk
+'''
+
+import io.xml
+
+class TigrisIssueMessage:
+    '''
+    A single message in a Tigris issue.
+    '''
+
+    def __init__(self, who='', when='', text=''):
+        '''
+        Constructor
+        '''
+        self.who = who
+        self.when = when
+        self.text = text
+        
+    def appendNode(self, doc, root):
+        e = doc.createElement("long_desc")
+        io.xml.addChildText(doc, e, "who", self.who)
+        io.xml.addChildText(doc, e, "issue_when", self.when)
+        io.xml.addChildText(doc, e, "thetext", self.text)
+        root.appendChild(e)
+    
+    def parseNode(self, node):
+        self.who = io.xml.getChildText(node, "who")
+        self.when = io.xml.getChildText(node, "issue_when")
+        self.text = io.xml.getChildText(node, "thetext")
+        

issues/scons_tigris/data/__init__.py

Empty file added.

issues/scons_tigris/io/__init__.py

Empty file added.

issues/scons_tigris/io/xml.py

+'''
+Some helper functions for working with xml.dom.minidom classes/definitions
+Created on 24.02.2010
+
+@author: dirk
+'''
+
+def addChildText(doc, root, cname, text):
+    key = doc.createElement(cname)
+    t = doc.createTextNode(text)
+    key.appendChild(t)
+    root.appendChild(key)
+
+def addChildInt(doc, root, cname, i):
+    txt = ""
+    try:
+        txt = str(i)
+    except:
+        pass
+    addChildText(doc, root, cname, txt)
+    
+def addChildBool(doc, root, cname, b):
+    if b:
+        addChildText(doc, root, cname, "1")
+    else:
+        addChildText(doc, root, cname, "0")
+
+def getText(root):
+    txt = ""
+    for e in root.childNodes:
+        if (e.nodeType == e.TEXT_NODE):
+            txt += e.data
+        if (e.nodeType == e.ENTITY_REFERENCE_NODE):
+            print e.data
+    return txt
+
+def getChildText(root, cname):
+    txt = ""
+    for e in root.childNodes:
+        if e.nodeType == e.ELEMENT_NODE and e.localName == cname:
+            txt += getText(e)
+            #txt = str(e.normalize())
+            break
+    return txt
+
+def getChildInt(root, cname, default=0):
+    txt = getChildText(root, cname)
+    i = default
+    try:
+        i = int(txt)
+    except:
+        pass
+    
+    return i
+
+bool_texts = ['true', 'True', 'TRUE', '1']
+
+def getChildBool(root, cname, default=False):
+    txt = getChildText(root, cname)
+    if txt == "":
+        return default
+    else:
+        if txt in bool_texts:
+            return True
+    
+    return False

issues/scons_tigris/merge_issues.py

+import sys
+from data.TigrisIssueList import TigrisIssueList
+
+def usage():
+    print "merge_issues.py xmlfile"
+
+def main():
+    if len(sys.argv) < 2:
+        usage()
+        sys.exit(0)
+
+    tl = TigrisIssueList()
+    tl.scanXML('.')
+
+    print "%d issues read" % len(tl.issues)
+
+    tl.writeFile(sys.argv[1])
+    
+if __name__ == "__main__":
+    main()

issues/scons_tigris/pull_issues.py

+import os
+import sys
+import urllib
+
+def getIssue(fid):
+    query = urllib.urlencode({'id':fid})
+    urllib.urlretrieve("http://scons.tigris.org/issues/xml.cgi?%s" % query, "%d.xml" % fid)
+
+def main():
+    max = 0
+    cur = 0
+    if len(sys.argv) > 1:
+        sl = sys.argv[1].split('-')
+        if len(sl) > 1:
+            cur = int(sl[0])-1
+            max = int(sl[1])
+        else:
+            max = int(sys.argv[1])
+            
+    while 1:
+        if max > 0:
+            if cur == max:
+                break
+        cur += 1
+        print "%d" % cur
+        getIssue(cur)
+        # Check for contents
+        f = open('%d.xml' % cur)
+        content = f.read()
+        f.close()
+        if content.find('issue status_code="404" status_message="NotFound"') >= 0:
+            # Delete the last file
+            os.remove('%d.xml' % cur)
+            break
+
+if __name__ == "__main__":
+    main()
+