Commits

Andy Mikhailenko committed c6d6c6d

Added proper time series (with custom interval length and quantity) to the financial charts. Of course this will be useful for any kind of chart, e.g. to track activities, etc.

Comments (0)

Files changed (6)

orgtool/__init__.py

 # -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2009—2010 Andrey Mikhailenko and contributors
+#
+#  This file is part of OrgTool.
+#
+#  OrgTool is free software under terms of the GNU Lesser
+#  General Public License version 3 (LGPLv3) as published by the Free
+#  Software Foundation. See the file README for copying conditions.
+#
 """
 Orgtool
 =======

orgtool/ext/finances/__init__.py

 # -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2009—2010 Andrey Mikhailenko and contributors
+#
+#  This file is part of OrgTool.
+#
+#  OrgTool is free software under terms of the GNU Lesser
+#  General Public License version 3 (LGPLv3) as published by the Free
+#  Software Foundation. See the file README for copying conditions.
+#
 """
 Finances
 ========

orgtool/ext/finances/templates/dashboard.html

 <p>See also: <a href="{{ url_for('orgtool.ext.finances.plan_index') }}">all contracts</a>, 
              <a href="{{ url_for('orgtool.ext.finances.event_index') }}">all payments</a>.</p>
 
-<img src="{{ chart_url_for_payments(payments, only_smooth=False, 
+<img src="{{ chart_url_for_payments(payments, legend=True, title='The Big Picture (last two years)',
                                     width=900, height=250,
                                     currency=default_currency,
-                                    max_days=300) }}" alt="All payments"/>
+                                    scale='months', max_intervals=24) }}" alt="All payments"/>
 
 <ul style="overflow: hidden; list-style: none;">
     {% for column in contracts.where_not(next_date_time=None)|slice(3) %}
         <li style="width: 33%; float: left; margin: 0;">
             {% for plan in column %}
-                {% if plan.next_date_time %}
+                {#% if plan.next_date_time %#}
                     <h2><a href="{{ url_for('orgtool.ext.finances.plan', pk=plan.pk) }}">{{ plan }}</a></h2>
                         {% if plan.next_date_time < today %}
                             <strong>???</strong>
 
                         <span title="{{ plan.next_date_time.strftime("%d %B %Y") }}">{{ render_rel_delta(plan.next_date_time) }}</span>
                         {% set chart_url = chart_url_for_payments(plan.events,
-                                    max_days=365, 
-                                    only_smooth=False,
+                                    legend=False,
+                                    scale='months', max_intervals=6, 
                                     currency=default_currency,
                         ) %}
                         {% if chart_url %}
                         {% else %}
                             <p>No payments within last year.</p>
                         {% endif %}
-                {% endif %}
+                {#% endif %#}
             {% endfor %}
         </li>
     {% endfor %}

orgtool/ext/finances/templates/plan_detail.html

 
 <p><a href="{{ admin_url_for(object, 'finances') }}">View in admin</a></p>
 
-<img src="{{ chart_url_for_payments(object.events, width=800, height=300, only_smooth=False) }}" alt="payments"/>
+<p><img src="{{ chart_url_for_payments(object.events, width=800, height=300,
+                                       scale='days', max_intervals=30,
+                                       title='Payments: last month') }}" 
+        alt="Payments (last month)"/></p>
+<p><img src="{{ chart_url_for_payments(object.events, width=800, height=300,
+                                       scale='months', max_intervals=12,
+                                       title='Payments: last year') }}" 
+        alt="Payments (last year)"/></p>
+<p><img src="{{ chart_url_for_payments(object.events, width=800, height=300,
+                                       scale='years', max_intervals=5,
+                                       title='Payments: last 5 years') }}" 
+        alt="Payments (last 5 years)"/></p>
 
-<div style="float: right; width: 48%;"
+<div style="float: right; width: 48%;">
     {% if object.stakeholder %}
         <p>Заинтересованное лицо: {{ object.stakeholder|urlize(40, true) }}</p>
     {% endif %}

orgtool/ext/finances/utils.py

 # -*- coding: utf-8 -*-
-
+#
+#  Copyright (c) 2009—2010 Andrey Mikhailenko and contributors
+#
+#  This file is part of OrgTool.
+#
+#  OrgTool is free software under terms of the GNU Lesser
+#  General Public License version 3 (LGPLv3) as published by the Free
+#  Software Foundation. See the file README for copying conditions.
+#
 import datetime
-import decimal
+from dateutil.relativedelta import relativedelta
+from decimal import Decimal as D
 import logging
+import math
 import urllib2
 import re
 import yaml
 
 from tool import app
 
+from orgtool.utils.timeseries import group_by_date
+
 
 ROOT_MODULE = __name__.rpartition('.')[0]
 
     rate = re.search(r'^(\d+\.\d+)', data['rhs'])
     if not rate:
         raise CalculationError('Unexpected rate format: {rhs}'.format(**data))
-    return decimal.Decimal(rate.group(1).strip())
+    return D(rate.group(1).strip())
 
 
 def get_default_currency():
 
     #get n sma first and calculate the next n period ema
     sma = sum(s[:n]) / n
-    multiplier = 2 / float(1 + n)
+    multiplier = 2 / D(1 + n)
     ema.append(sma)
 
     #EMA(current) = ( (Price(current) - EMA(prev) ) x Multiplier) + EMA(prev)
 
     return ema
 
-# TODO: extract this to a template filter
+# TODO: extract this to a Tool template filter
 def _get_rel_delta(dt, precision=2):
-    from dateutil.relativedelta import relativedelta
-    import datetime
     now = datetime.datetime.utcnow()
     if not isinstance(dt, datetime.datetime):
         now = now.date()
     parts, is_past = _get_rel_delta(dt)
     return not bool(parts)
 
-def chart_url_for_payments(payments, width=300, height=100, only_smooth=True,
-                           max_days=None, currency=None):
+def chart_url_for_payments(payments, width=300, height=100, scale='months',
+                           max_intervals=6, currency=None, legend=True,
+                           title=None):
+    """ TODO:
+
+    dark.aggregates: нужно получать разные агрегаты (сумма, кол-во) по группам
+    и притом иметь возможность доступиться к исходным элементам через агрегат.
+
+    Например, хочется поставить маркеры с текстом и датой для выдающихся
+    платежей. Поскольку минимальное значение за период у нас зачастую получается
+    суммированием отдельных значений (т.е. неск-ко платежей в течение месяца),
+    мы не можем просто найти платеж с известной нам минимальной суммой: такого
+    отдельного может просто не существовать. Поэтому маркером надо указывать
+    вообще не на платеж, а на агрегированное значение. Возможно, при этом
+    перечисляя описания соотв. платежей (как в ohloh). (Гуглодиаграммы
+    поддерживают многострочные маркеры.)
+
+    + надо корректно обрабатывать отсутствие данных на каких-то отрезках.
+    Похоже, сейчас это не так просто. ГуглоAPI это умеет, но у нас тут всё
+    завязано на нули, так что лучше пока не трогать.
+    """
+    assert max_intervals, 'unlimited depth is not (yet) supported'
+
     payments = payments.where_not(amount=None).order_by('date_time')  # NOT reversed
-    if max_days:
-        min_date = datetime.datetime.utcnow() - datetime.timedelta(days=max_days)
-        payments = payments.where(date_time__gte=min_date)
-    if not payments:
-        return None
-
-    # http://chart.apis.google.com/chart?cht=lc&chs=400x200&chd=t:10,-20,60,40&chco=318CE7&chds=-20,60&chxt=r,r&chxr=0,-20,60|1,-20,60&chxtc=1,-400&chxp=1,0&chxs=1,0000dd,13,-1,t,FF0000&chxl=1:|zero
+    until = datetime.datetime.utcnow()
+    since = until - relativedelta(**{scale: max_intervals})
+    payments = payments.where(date_time__gte=since)
 
     main_color, smooth_color = 'FFBF00', '8DB600', #'318CE7'   #, #'FE6F5E'# 'FAE7B5'
-    """
-    url_template = (
-        'http://chart.apis.google.com/chart?'
-        #'cht=lc&'                                 # line chart
-        'cht=lc:nda&'                             # hide axis
-        + (
-            'chd=t:{smooth_series}&'  # data series
-            if only_smooth else
-            'chd=t:{smooth_series}|{amount_series}&'  # data series
-        ) +
-        'chds={min_amount},{max_amount}&'         # fix axis range
-        'chs={width}x{height}&'                             # image size
-        #'chm=H,FF0000,0,0,1&'              # mark zero amount
-        'chxt=r&'                                 # show ticks on the right side
-        # axis labels:
-        'chxl=0:|{min_amount}|zero|{max_amount}&chxp=0,0|-100,100&chxr=0,{min_amount},{max_amount},0&chxtc=0,-{width}&'
-        'chco={main_color},{smooth_color}&'
-        'chm={annotations}&'
-    )
-    """
-    #print payments
-    amounts = [float(p.get_amount_as(currency) if currency else p.amount)
-               for p in payments if p.amount is not None]
-    #print 'amounts', amounts
-    _flat = lambda xs: ','.join(['{0:.2f}'.format(x) for x in xs])
-    #amount_series = _flat(amounts)
+    grouped = group_by_date(payments, 'date_time', scale,
+                            since=since, until=until)
+    group_labels = []
+    amounts = []
+    scale_to_date_fmt = {
+        'hours': '%H',
+        'days': '%d',
+        'months': '%b',
+        'years': '%Y',
+    }
+    for date, group in grouped:
+        print date, group
+        fmt = scale_to_date_fmt.get(scale, '%d %b')
+        label = date.strftime(fmt)
+        group_labels.append(label)
+        #group_labels.append(date.strftime('%b'))  # TODO: mind scale
+        amounts.append(
+            D(sum(p.get_amount_as(currency) if currency else p.amount
+                             for p in group))
+        )
     smoothing_period = len(amounts) / 3 or len(amounts) # 120->30, 14->6..10, 80->30
-
-#    for i in reversed(range(1,10)):
-#        if i < len(amounts) / 2:
-#            smoothing_period = len(amounts) / i
-#            break
-    # (len(amounts) / 10 if 10 < len(amounts) else 5) or 3
     smooth_data = calculate_ema(amounts, smoothing_period)
     if not smooth_data:
         return ''
 
+    #_flat = lambda xs: ','.join(['{0:.2f}'.format(x) for x in xs])
     #smooth_series = _flat(smooth_data)
-    min_amount = min(smooth_data if only_smooth else amounts) - 1
-    max_amount = max(smooth_data if only_smooth else amounts) + 1
-    if max_amount < 0:
-        max_amount = 0
-    if 0 < min_amount:
-        min_amount = 0
+    min_amount = min(amounts)    #smooth_data if only_smooth else amounts)
+    max_amount = max(amounts)    #smooth_data if only_smooth else amounts)
+    safe_min_amount = min_amount - 1
+    safe_max_amount = max_amount + 1
+    if safe_max_amount < 0:
+        safe_max_amount = 0
+    if 0 < safe_min_amount:
+        safe_min_amount = 0
 #        if min_amount == max_amount:
 #            min_amount = 0 if 0 < min_amount else -max_amount
 
-    #'''
-    # FIXME HACK -- this is broken by design:
-    year_marks = {}
-    for i, payment in enumerate(payments):
-        year_marks.setdefault(payment.date_time.year, (i, payment))
-    annotations = '|'.join([ 'A{text},,1,{point},{size}'.format(
-        text = year_marks[year][1].date_time.strftime('%b %Y'),
-        # HACK: fixing "ensure_series"
-        point = year_marks[year][0] if 1 < len(amounts) else year_marks[year][0]+1,  # payment index in query AND amount index
-        size = 8,
-    ) for year in sorted(year_marks)])
-
-
-    #return url_template.format(**locals())
-    #'''
-
-
-    from pygooglechart import Chart
-    from pygooglechart import SimpleLineChart
-    from pygooglechart import Axis
-    import math
-
     def _prep_boundary(x):
         value = math.ceil(abs(x))
         return -value if x < 0 else value
-    min_y, max_y = [_prep_boundary(x) for x in (min_amount, max_amount)]
+    min_y, max_y = [_prep_boundary(x) for x in (safe_min_amount,
+                                                safe_max_amount)]
 
-    chart = SimpleLineChart(width, height, y_range=[min_y, max_y])
+    def prep(val):
+        #return '%.01f'%val if isinstance(val, float) else str(val)
+        return str(val.quantize(D('.1'))) if isinstance(val, D) else str(val)
+    def join(*vals):
+        return ','.join(prep(v) for v in vals)
 
-    chart.add_data(smooth_data)
-    if not only_smooth:
-        chart.add_data(amounts)
-    chart.set_colours([smooth_color, main_color])
+    span = (max_amount + abs(min_amount)) if 0 < max_amount else max_amount + min_amount
+    if max_amount <= 0:
+        zero_border = 1
+    elif max_amount == span:
+        zero_border = 0
+    else:
+        zero_border = max_amount * D('0.001')
+    zero_border = D(zero_border).quantize(D('.1'))
 
-    chart.set_axis_labels(Axis.RIGHT, [min_y, max_y])
-    chart_url = chart.get_url()
+    # positive balance (green)
+    marker_positive = ''
+    if 0 < max_amount:
+        print 'MAX Y', max_amount
+        marker_positive = 'r,EEFFEE,0,{start},{end},{z_index}'.format(
+            start = 1,
+            end = zero_border,
+            z_index = -2,
+        )
 
-    chxr = 'chxr=0,{min_y},{max_y}'.format(**locals())
+    # negative balance (red)
+    marker_negative = ''
+    if min_amount < 0:
+        print 'MIN Y', min_amount
+        marker_negative = 'r,FFEEEE,0,{start},{end},{z_index}'.format(
+            start = zero_border,
+            end = 0,
+            z_index = -2,
+        )
 
-    # NOTE: pygooglechart does not support some stuff we need here
-    return '&'.join([
-        chart_url,
+    # grid made out of markers. Simple grid would get hidden behind
+    # positive/negative area merkers. It also didn't follow data ticks.
+    marker_grid_v = 'V,cccccc,0,::2,0.5,-1'
+    marker_grid_h = 'h,cccccc,0,0:2:.2,0.5,-1'
+
+    markers = [
+        marker_positive,
+        marker_negative,
+        marker_grid_v,
+        marker_grid_h
+    ]
+
+    dynamic_markers = []
+    """ TODO (no earlier than Dark gets smarter aggregates)
+    def make_dynamic_marker(points, text):
+        tmpl = 'y;s={style};d=bb,{text},{fg},{bg};ds={series};dp={points}'
+        return tmpl.format(
+            series = 0,   # axis A
+            points = points,
+            text = text,
+            style = 'bubble_text_small', # icon_string_constant
+            fg = 'ff0000',
+            bg = 'ffffff',
+        )
+    # minimum value
+    for aggregate in groups:
+        if aggregate == min_amount:
+            dm = make_dynamic_marker(
+                points = ???,
+                text = '\n'.join(p.summary for p in aggregate.items)
+            )
+            dynamic_markers.append(dm)
+            print 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', dm
+    # maximum value
+    #...
+    """
+
+    chart = Chart(title, width, height, with_legend=legend)
+    facts = Axis(
+        range = (min_y, max_y),
+        scale = (0, 100, min_y, max_y),
+        align = 'y',
+        color = main_color,
+        style = '1',
+        values = amounts,
+        legend = 'Facts',
+        labels = group_labels,
+    )
+    ema = Axis(
+        range = (0, len(amounts)),
+        scale = (0, 100, min_y, max_y),
+        align = 'x',
+        color = smooth_color,
+        style = '2,3,2', # width=1, dash segment length=4, blank segment length=1
+        values = smooth_data,
+        legend = 'Average',
+    )
+    chart.axes.append(facts)
+    chart.axes.append(ema)
+    chart.markers = [m for m in markers if m]
+    chart.dynamic_markers = dynamic_markers
+
+    return chart.render()
+
+    '''
+    url = (
+        'http://chart.apis.google.com/chart?'
+        # plot title
+        '&chtt={plot_title}'
+        # axis labels
+        '&chxl={axis_labels_a}'
+        # axis label positions
+        '&chxp={axis_label_pos_a}'
+        # axis ranges
+        '&chxr={axis_range_a}|{axis_range_b}'
+        '&chxs=0,676767,11.5,0,lt,676767|1,676767,11.5,0,lt,676767'
         # visible axes
-        'chxt=r,r',
+        '&chxt={axis_align_a},{axis_align_b}'
+        '&chs={plot_width}x{plot_height}'
+        '&cht=lxy'
+        '&chco={axis_color_a},{axis_color_b}'
+        # scale with custom range
+        '&chds={axis_scale_a},{axis_scale_b}'
+        '&chd=t:-1|{axis_values_a}|-1|{axis_values_b}'
+        '&chdl={axis_legend_a}|{axis_legend_b}'
+        '&chdlp=b'
+#        '&chg={plot_grid_steps},{plot_grid_dash}'
+        '&chls={axis_style_a}|{axis_style_b}'
+        '&chma=5,5,5,25'
+        # markers
+        '&chm={markers}'
+        '&chem={enhanced_markers}'
+    ).format(
+        plot_title = title,  # {since}—{until}'.format(**locals()),
+        plot_width = width,
+        plot_height = height,
+        axis_range_a = join(0, min_y, max_y),
+        axis_range_b = join(1, 0, len(amounts)),
+        axis_align_a = 'y',
+        axis_align_b = 'x',
+        axis_color_a = main_color,
+        axis_color_b = smooth_color,
+        axis_style_a = '1',     # width=1
+        axis_style_b = '2,3,2', # width=1, dash segment length=4, blank segment length=1
+        # min scale, max scale, min value, max value
+        axis_scale_a = join(0, 100, min_y, max_y),  # main data
+        axis_scale_b = join(0, 100, min_y, max_y),  # smooth data
+        # comma-separated series of values
+        axis_values_a = join(*amounts),
+        axis_values_b = join(*smooth_data),
+        axis_legend_a = 'Facts',
+        axis_legend_b = 'Average',
+        # labels
+        axis_labels_a = '1:|' + '|'.join(group_labels),
+        axis_label_pos_a = '',  # ?
+        # markers
+        markers = '|'.join([m for m in markers if m]),
+        enhanced_markers = '|'.join(dynamic_markers),
+    )
+#    return url
+    '''
 
-        # scale with custom range
-        'chds={min_y},{max_y}'.format(**locals()),
+
+#--- TODO: extract code below to Dark(?)
+
+class Chart(object):
+    def __init__(self, title, width, height, with_legend=True):
+        self.title = title
+        self.width = width
+        self.height = height
+        self.axes = []
+        self.markers = []
+        self.dynamic_markers = []
+        self.with_legend = with_legend
+
+    def __str__(self):
+        return self.render()
+
+    def render(self):
+        bits = '&'.join('='.join([k,v]) for k,v in self._url_bits() if v)
+        return 'http://chart.apis.google.com/chart?' + bits
+
+    def dump(self):
+        for param, value in self._url_bits():
+            print '{param}\t{value}'.format(**locals())
+
+    def _collect_axis_labels(self):
+        for i, axis in enumerate(self.axes, 1):
+            if axis.labels:
+                labels = '|'.join(str(x) for x in axis.labels if x)
+                yield '{i}:|{labels}'.format(**locals())
+
+    def _collect_axis_label_positions(self):
+        for i, axis in enumerate(self.axes, 1):
+            if axis.label_positions:
+                labels = ','.join(str(x) for x in axis.labels if x)
+                yield '{i},{labels}'.format(**locals())
+
+    def _url_bits(self):
+        # plot title
+        if self.with_legend:
+            yield 'chtt', self.title
+            # axis labels
+            yield 'chxl', '|'.join(self._collect_axis_labels())
+            # axis label positions
+            yield 'chxp', '|'.join(self._collect_axis_label_positions())
+            yield 'chdl', '|'.join(a.legend for a in self.axes if a.legend)
 
         # axis ranges
-        'chxr=0,{min_y},{max_y}|1,{min_y},{max_y}'.format(**locals()),
+        yield 'chxr', '|'.join('{0},{1},{2}'.format(i, *a.range)
+                                    for i,a in enumerate(self.axes))
+        yield 'chxs', '0,676767,11.5,0,lt,676767|1,676767,11.5,0,lt,676767'
+        # visible axes
+        yield 'chxt', ','.join(a.align for a in self.axes)
+        yield 'chs', 'x'.join(str(x) for x in [self.width, self.height])
+        yield 'cht', 'lxy'
+        yield 'chco', ','.join(a.color for a in self.axes)
+        # scale with custom range
+        scales = (','.join(str(x) for x in a.scale) for a in self.axes)
+        yield 'chds', ','.join(scales)
+        values = (a.get_values() for a in self.axes)
+        yield 'chd', 't:' + '|'.join(values)
+        yield 'chdlp', 'b'
+        yield 'chls', '|'.join(a.style for a in self.axes if a.style)
+        yield 'chma', '5,5,5,25'
+        # markers
+        yield 'chm', '|'.join(m for m in self.markers)
+        yield 'chem', '|'.join(m for m in self.dynamic_markers)
 
-        # axis tick styles
-        'chxtc=1,-{width}'.format(**locals()),
 
-        # axis labels
-        'chxl=1:|sea level',
+class Axis(object):
+    def __init__(self, values, scale, range, labels=None,
+                 label_positions=None, align='x', color='000000',
+                 style='1', legend=None):
+        self.values = values
+        self.scale = scale
+        self.range = range
+        self.labels = labels
+        self.label_positions = label_positions
+        self.align = align
+        self.color = color
+        self.style = style
+        self.legend = legend or None
 
-        # axis label positions
-        'chxp=1,0',
-
-        # axis label styles
-        'chxs=1,318CE7,13,-1,t,318CE7', #550000',
-
-#        chxr,
-#        'chxl=0:{min_y},zero,{max_y}'.format(**locals()),
-#        'chxp=0,{min_y},0,{max_y}'.format(**locals()),
-##        'chxs=|0N*sz2*,0000FF'
-##        zeromark
-
-        # annotations
-        'chm={annotations}&'.format(**locals()),
-    ])
-
+    def get_values(self):
+        return '-1|' + ','.join(prep(x) for x in self.values)

orgtool/utils/dates.py

 # -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2009—2010 Andrey Mikhailenko and contributors
+#
+#  This file is part of OrgTool.
+#
+#  OrgTool is free software under terms of the GNU Lesser
+#  General Public License version 3 (LGPLv3) as published by the Free
+#  Software Foundation. See the file README for copying conditions.
+#
 """
 Functions for parsing "informal" date definitions.
 """
-
 import datetime
 from dateutil.rrule import *
 from lepl import *