Commits

Ezio Melotti committed 90207bf

#544: add an issues stats page, a script to generate initial data, and modify the roundup-summary to update them.

Comments (0)

Files changed (3)

html/issue.stats.html

+<tal:block metal:use-macro="templates/page/macros/icing">
+<title metal:fill-slot="head_title" >
+  <span tal:omit-tag="true" i18n:translate="" >Issues stats</span>
+  <span tal:condition="request/dispname"
+   tal:replace="python:' - %s '%request.dispname"
+  /> - <span tal:replace="config/TRACKER_NAME" />
+</title>
+
+<metal:slot fill-slot="more-javascript">
+<script type="text/javascript" src="http://cdn.jsdelivr.net/jquery/2.1.1/jquery.min.js"></script>
+<script type="text/javascript" src="http://cdn.jsdelivr.net/jqplot/1.0.8/jquery.jqplot.js"></script>
+<link rel="stylesheet" type="text/css" href="http://cdn.jsdelivr.net/jqplot/1.0.8/jquery.jqplot.css">
+<script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/jqPlot/1.0.8/plugins/jqplot.dateAxisRenderer.min.js"></script>
+<script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/jqPlot/1.0.8/plugins/jqplot.barRenderer.min.js"></script>
+<script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/jqPlot/1.0.8/plugins/jqplot.canvasTextRenderer.min.js"></script>
+<script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/jqPlot/1.0.8/plugins/jqplot.canvasAxisTickRenderer.min.js"></script>
+<script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/jqPlot/1.0.8/plugins/jqplot.categoryAxisRenderer.min.js"></script>
+<script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/jqPlot/1.0.8/plugins/jqplot.pointLabels.min.js"></script>
+
+<script type="text/javascript">
+function make_chart(id, title, series, type, labels, ticks) {
+    if (labels == null)
+        var legend = {show: false};
+    else
+        var legend = {show: true, location: 'nw', labels: labels};
+    if (type == 'line') {
+        var series_renderer = $.jqplot.LineRenderer;
+        var axis_renderer = $.jqplot.DateAxisRenderer;
+        var show_label = false;
+    }
+    else {
+        var series_renderer = $.jqplot.BarRenderer;
+        var axis_renderer = $.jqplot.CategoryAxisRenderer;
+        var show_label = true;
+    }
+    var plot = $.jqplot(id, series, {
+        title: title,
+        seriesColors: ["#3771a0", "#fcd449"],
+        seriesDefaults: {renderer:series_renderer, showMarker:false,
+                         pointLabels: { show: show_label },
+                         rendererOptions: {fillToZero: true}},
+        axes: {xaxis:{renderer:axis_renderer, ticks:ticks,
+                      tickRenderer: $.jqplot.CanvasAxisTickRenderer,
+                      tickOptions: {angle: -30}}},
+        legend: legend,
+    });
+}
+
+function zip(first, second) {
+    var res = [];
+    // assume same length
+    for (var k = 0; k < first.length; k++) {
+        res.push([first[k], second[k]]);
+    }
+    return res;
+}
+
+$(document).ready(function(){
+    $.getJSON("@@file/issue.stats.json", function(j) {
+        var dates = [];
+        for (var k = 0; k < j.timespan.length; k++) {
+            dates.push(j.timespan[k][1]);
+        }
+
+        var num = -25;  // show only last 25 weeks
+
+        make_chart('open_patches', 'Open issues (total)',
+                   [zip(dates, j.open), zip(dates, j.patches)],
+                   'line', ['Open issues', 'Open issues with patches']);
+        make_chart('open_deltas', 'Open issues deltas (weekly)',
+                   [j.open_delta.slice(num)], 'bar', null, dates.slice(num));
+        make_chart('open_closed_week', 'Opened and Closed (weekly)',
+                   [j.total_delta.slice(num), j.closed_delta.slice(num)],
+                   'bar', ['Opened', 'Closed'], dates.slice(num));
+        make_chart('closed_total', 'Closed and Total issues (total)',
+                   [zip(dates, j.closed), zip(dates, j.total)],
+                   'line', ['Closed', 'Total']);
+    });
+});
+</script>
+</metal:slot>
+
+<span metal:fill-slot="body_title" tal:omit-tag="true">
+  <span tal:omit-tag="true" i18n:translate="" >Issues stats</span>
+  <span tal:condition="request/dispname"
+   tal:replace="python:' - %s' % request.dispname" />
+</span>
+<tal:block metal:fill-slot="content">
+
+<p>These charts are updated weekly.  JavaScript must be enabled to see the charts.</p>
+
+<p>Total number of open issues and open issues with patches:</p>
+<div id="open_patches"></div>
+
+<p>Delta of open issues compared with the previous week.  If the delta is positive,
+the total number of open issues increased; if negative, the total number decreased.</p>
+<div id="open_deltas"></div>
+
+<p>Number of issues that have been opened and closed during each week.
+The difference between these two values is shown in the previous graph.</p>
+<div id="open_closed_week"></div>
+
+<p>The number of closed issues, and the number of issues regardless of their status:</p>
+<div id="closed_total"></div>
+
+</tal:block>
+</tal:block>

scripts/issuestats.py

+# Search for the weekly summary reports from bugs.python.org in the
+# python-dev archives and plot the result.
+#
+# $ issuestats.py collect
+#
+# Collects statistics from the mailing list and saves to
+# issue.stats.json
+#
+# $ issuestats.py plot
+#
+# Written by Ezio Melotti.
+# Based on the work of Petri Lehtinen (https://gist.github.com/akheron/2723809).
+#
+# Requires Python 3.
+#
+# This script is used to generate a JSON file with historical data that can
+# be copied in the html/ dir of the bugs.python.org Roundup instance and used
+# by the html/issue.stats.html page.  The roundup-summary script can update the
+# JSON file weekly.
+#
+
+
+import os
+import re
+import sys
+import json
+import gzip
+import errno
+import argparse
+import datetime
+import tempfile
+import webbrowser
+import urllib.parse
+import urllib.request
+
+from collections import defaultdict
+
+
+MONTH_NAMES = [datetime.date(2012, n, 1).strftime('%B') for n in range(1, 13)]
+ARCHIVE_URL = 'http://mail.python.org/pipermail/python-dev/%s'
+
+STARTYEAR = 2011
+STARTMONTH = 1  # February
+
+NOW = datetime.date.today()
+ENDYEAR = NOW.year
+ENDMONTH = NOW.month
+
+STATISTICS_FILENAME = 'issue.stats.json'
+
+activity_re = re.compile('ACTIVITY SUMMARY \((\d{4}-\d\d-\d\d) - '
+                         '(\d{4}-\d\d-\d\d)\)')
+count_re = re.compile('\s+(open|closed|total)\s+(\d+)\s+\(([^)]+)\)')
+patches_re = re.compile('Open issues with patches: (\d+)')
+
+
+def find_statistics(source):
+    print(source)
+    monthly_data = {}
+    with gzip.open(source) as file:
+        parsing = False
+        for line in file:
+            line = line.decode('utf-8')
+            if not parsing:
+                m = activity_re.match(line)
+                if not m:
+                    continue
+                start_end = m.groups()
+                if start_end in monthly_data:
+                    continue
+                monthly_data[start_end] = weekly_data = {}
+                parsing = True
+                continue
+            m = count_re.match(line)
+            if parsing and m:
+                type, count, delta = m.groups()
+                weekly_data[type] = int(count)
+                weekly_data[type + '_delta'] = int(delta)
+            m = patches_re.match(line)
+            if parsing and m:
+                weekly_data['patches'] = int(m.group(1))
+                parsing = False
+    print('  ', len(monthly_data), 'reports found')
+    return monthly_data
+
+
+def collect_data():
+    try:
+        os.mkdir('cache')
+    except OSError as exc:
+        if exc.errno != errno.EEXIST:
+            raise
+
+    statistics = {}
+
+    for year in range(STARTYEAR, ENDYEAR + 1):
+        # Assume STARTYEAR != ENDYEAR
+        if year == STARTYEAR:
+            month_range = range(STARTMONTH, 12)
+        elif year == ENDYEAR:
+            month_range = range(0, ENDMONTH)
+        else:
+            month_range = range(12)
+
+        for month in month_range:
+            prefix = '%04d-%s' % (year, MONTH_NAMES[month])
+
+            archive = prefix + '.txt.gz'
+            archive_path = os.path.join('cache', archive)
+
+            if not os.path.exists(archive_path):
+                print('Downloading %s' % archive)
+                url = ARCHIVE_URL % urllib.parse.quote(archive)
+                urllib.request.urlretrieve(url, archive_path)
+
+
+            print('Processing %s' % prefix)
+            statistics.update(find_statistics(archive_path))
+
+
+    statistics2 = defaultdict(list)
+    for key, val in sorted(statistics.items()):
+        statistics2['timespan'].append(key)
+        for k2, v2 in val.items():
+            statistics2[k2].append(v2)
+
+    with open(STATISTICS_FILENAME, 'w') as fobj:
+        json.dump(statistics2, fobj)
+
+    print('Now run "plot".')
+
+
+HTML = """<!DOCTYPE html>
+<html>
+<head>
+<script type="text/javascript" src="http://cdn.jsdelivr.net/jquery/2.1.1/jquery.min.js"></script>
+<script type="text/javascript" src="http://cdn.jsdelivr.net/jqplot/1.0.8/jquery.jqplot.js"></script>
+<script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/jqPlot/1.0.8/plugins/jqplot.dateAxisRenderer.min.js"></script>
+<script type="text/javascript">
+function make_chart(id, title, dates, values) {
+    var data = [];
+    for (var k = 0; k < dates.length; k++) {
+        data.push([dates[k], values[k]]);
+    }
+    var plot = $.jqplot(id, [data], {
+        title: title,
+        series: [{showMarker:false}],
+        axes: {xaxis:{renderer:$.jqplot.DateAxisRenderer}},
+    });
+}
+$(document).ready(function(){
+    var j = %s;
+    var dates = [];
+    for (var k = 0; k < j.timespan.length; k++) {
+        console.log(j.timespan[k][1]);
+        dates.push(j.timespan[k][1]);
+    }
+    console.log(dates);
+    var open = [];
+    for (var k = 0; k < dates.length; k++) {
+        open.push([dates[k], j.open[k]]);
+    }
+    console.log(open);
+    make_chart('open', 'Open issues', dates, j.open);
+    make_chart('open_delta', 'Open issues (deltas)', dates, j.open_delta);
+    make_chart('open_week', 'Opened per week', dates, j.wopened);
+    make_chart('closed', 'Closed issues', dates, j.closed);
+    make_chart('closed_delta', 'Closed issues (deltas)', dates, j.closed_delta);
+    make_chart('closed_week', 'Closed per week', dates, j.wclosed);
+    make_chart('total', 'Total issues', dates, j.total);
+    make_chart('total_delta', 'Total issues (deltas)', dates, j.total_delta);
+    make_chart('patches', 'Open issues with patches', dates, j.patches);
+});
+</script>
+<link rel="stylesheet" type="text/css" href="http://cdn.jsdelivr.net/jqplot/1.0.8/jquery.jqplot.css">
+<style type="text/css">div { margin-bottom: 1em; }</style>
+</head>
+<body>
+<div id="open"></div>
+<div id="open_delta"></div>
+<div id="open_week"></div>
+<div id="closed"></div>
+<div id="closed_delta"></div>
+<div id="closed_week"></div>
+<div id="total"></div>
+<div id="total_delta"></div>
+<div id="patches"></div>
+</body>
+</html>
+"""
+
+def plot_statistics():
+    try:
+        with open(STATISTICS_FILENAME) as j:
+            json = j.read()
+    except FileNotFoundError:
+        sys.exit('You need to run "collect" first.')
+    with tempfile.NamedTemporaryFile('w', delete=False) as tf:
+        tf.write(HTML % json)
+    webbrowser.open(tf.name)
+
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('command', choices=['collect', 'plot'])
+
+    args = parser.parse_args()
+
+    if args.command == 'collect':
+        collect_data()
+    elif args.command == 'plot':
+        plot_statistics()
+
+
+if __name__ == '__main__':
+    main()

scripts/roundup-summary

                         Print journal for all the transactions in the given
                         date range.
     -D, --DEBUG         Print email content without sending it if -m is used.
+    --update-stats-file=FILENAME
+                        Append tracker stats to JSON file FILENAME.
 """
 
 # This script has a class (Report) that filters the issues and generates txt
 import sys
 # hardcode the path to roundup
 sys.path.insert(1, '/home/roundup/lib/python2.6/site-packages')
-#sys.path.insert(1, '/opt/tracker-roundup/lib/python2.6/site-packages/')
 
 import cgi
+import json
+import os.path
 import optparse
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
     advanced.add_option(
         '-D', '--DEBUG', dest='debug', action='store_true', default=False,
         help='Print email content without sending it if -m is used.')
+    advanced.add_option(
+        '--update-stats-file', dest='update_stats', metavar='FILENAME',
+        help='Append tracker stats to JSON file FILENAME.')
     parser.add_option_group(advanced)
 
     # Get the command line args:
         smtp.sendmail(config.ADMIN_EMAIL, recipient, msg.as_string())
 
 
+def update_stats_file(stats, filename):
+    # This function will look for a JSON file in the html/ dir of the instance
+    # and append the data for the current week.  If the file doesn't exist it
+    # will create a new one.  This function won't happend the same data twice
+    # in a single day, but if this script is run on different dates it will.
+    # In order to generate an initial JSON file with historical data, use
+    # the scripts/issuestats.py script.  These data are used by the
+    # issue.stats.html page.
+    stats = dict(stats)  # make a copy
+    stats_file = os.path.join(instance_home, 'html', OPTIONS.update_stats)
+    try:
+        with open(stats_file) as fi:
+            j = json.load(fi)
+    except IOError:
+        j = dict(open=[], closed=[], total=[], timespan=[], patches=[],
+                 open_delta=[], closed_delta=[], total_delta=[])
+    timespan = stats.pop('timespan').split(' - ')
+    if timespan in j['timespan']:
+        return  # we already updated the file today
+    j['timespan'].append(timespan)
+    for k, v in stats.items():
+        if k in j:
+            j[k].append(v)
+    with open(stats_file, 'w') as fo:
+        json.dump(j, fo)
+
+
 def main():
     """Create the report and print or send it."""
     issue_attrs = issues_map()
         for recipient in OPTIONS.mailTo.split(','):
             send_report(recipient, txt_report, html_report)
 
+    if OPTIONS.update_stats:
+        update_stats_file(report.header_content, OPTIONS.update_stats)
 
 
 if __name__ == '__main__':