Commits

Robert Brewer committed 281f5a4

Much faster balance charts due to new defaultlist class.

  • Participants
  • Parent commits 1509709

Comments (0)

Files changed (4)

File flowrate/__init__.py

                      for tx in ledger.transactions.itervalues()]
             if not years:
                 years = [datetime.date.today().year]
+        years.sort()
         if not months:
             months = range(1, 13)
+        months.sort()
 
         if dategroup == 'year':
             coerce_date = cmpdates.strkeys['years']
-            dategroups = [str(y) for y in years]
+            dategroups = [(str(y),
+                           datetime.date(y, 1, 1).toordinal(),
+                           datetime.date(y + 1, 1, 1).toordinal())
+                          for y in years]
         elif dategroup == 'day':
             coerce_date = cmpdates.strkeys['days']
             dategroups = []
             for y in years:
                 for m in months:
-                    for d in days or xrange(1, calendar.monthrange(y, m)[1] + 1):
-                        dategroups.append('%04d-%02d-%02d' % (y, m, d))
+                    for d in sorted(days) or xrange(1, calendar.monthrange(y, m)[1] + 1):
+                        lo = datetime.date(y, m, d)
+                        hi = lo + datetime.timedelta(days=1)
+                        dategroups.append(('%04d-%02d-%02d' % (y, m, d),
+                                           lo.toordinal(), hi.toordinal()))
         else:
             # dategroup == 'month' or other
             coerce_date = cmpdates.strkeys['months']
-            dategroups = ['%04d-%02d' % (y, m) for y in years for m in months]
+            dategroups = [('%04d-%02d' % (y, m),
+                           datetime.date(y, m, 1).toordinal(),
+                           (datetime.date(y, m, calendar.monthrange(y, m)[1]) +
+                            datetime.timedelta(days=1)).toordinal())
+                          for y in years for m in months]
 
         # Calculate the budget of each requested account for each
         # dategroup in the output. Send [budget, actual] pairs.
         for a in accounts:
             acct = ledger.accounts[a]
             budgets[str(a)] = bucket = {}
-            for dg in dategroups:
+            for dg, lo, hi in dategroups:
                 # Grab transactions matching the balance "date"
-                actual = sum(v for d, v in acct.activity.iteritems()
-                             if coerce_date(d) == dg)
+                actual = sum(acct.activity[lo:hi])
 
                 # Grab obligations matching the balance "date"
-                budget = sum(v for d, v in acct.budgets.iteritems()
-                             if coerce_date(d) == dg)
+                budget = sum(acct.budgets[lo:hi])
 
                 bucket[dg] = [budget, actual]
 
                      for tx in ledger.transactions.itervalues()]
             if not years:
                 years = [datetime.date.today().year]
+        years.sort()
         if not months:
             months = range(1, 13)
+        months.sort()
 
         if dategroup == 'year':
             balance_dates = [datetime.date(y, 12, 31) for y in years]
             dategroups = [str(y) for y in years]
         elif dategroup == 'day':
             balance_dates = []
+            dategroups = []
             for y in years:
                 for m in months:
-                    for d in days or xrange(1, calendar.monthrange(y, m)[1] + 1):
+                    for d in sorted(days) or xrange(1, calendar.monthrange(y, m)[1] + 1):
                         balance_dates.append(datetime.date(y, m, d))
-            dategroups = [d.isoformat() for d in balance_dates]
+                        dategroups.append('%04d-%02d-%02d' % (y, m, d))
         else:
             # dategroup == 'month' or other
             balance_dates = [datetime.date(y, m, calendar.monthrange(y, m)[1])
         # Calculate the balance of each requested account for each
         # dategroup in the output.
         balances = {}
-        age = datetime.date.today() - datetime.timedelta(days=30)
+        age = (datetime.date.today() - datetime.timedelta(days=30)).toordinal()
+        #print "Age", age
         dates_and_groups = zip(balance_dates, dategroups)
         for a in accounts:
             acct = ledger.accounts[a]
+            #print "Account", a, acct.activity._initial
+            #print acct.activity._data
             balances[str(a)] = bucket = {}
+
+            lastbd = None
+            curbal = 0
             for bd, dg in dates_and_groups:
-                # Grab all transactions older than the balance date
-                actual = sum(v for d, v in acct.activity.iteritems()
-                             if d <= bd)
+                hi = bd.toordinal() + 1
 
-                if age <= bd:
+                # Grab all transactions older than or equal to the balance date
+                actual = sum(acct.activity[lastbd:hi])
+
+                if lastbd is None or lastbd < age:
+                    lo = age
+                else:
+                    lo = lastbd
+
+                if lo < hi:
                     # Grab obligations between 30 days ago and the balance date...
-                    budget = sum(v for d, v in acct.budgets.iteritems()
-                                 if age <= d <= bd)
+                    budget = sum(acct.budgets[lo:hi])
 
                     # ...but reduce by their fulfillments
-                    fulfillment = sum(v for d, v in acct.fulfillments.iteritems()
-                                      if age <= d <= bd)
+                    fulfillment = sum(acct.fulfillments[lo:hi])
                 else:
-                    budget = fulfillment = 0
+                    budget, fulfillment = 0, 0
 
-                bucket[dg] = actual + budget - fulfillment
+                subtotal = actual + (budget - fulfillment)
+                curbal += subtotal
+                #print "%s: [%s(%s):%s] = %s + %s - %s = %s -> %s" % (dg, lastbd, lo, hi, actual, budget, fulfillment, subtotal, curbal)
+                bucket[dg] = curbal
+                lastbd = hi
 
         b['data'] = balances
 

File flowrate/flows.py

         da = ledger.accounts[self['debit']]
 
         for ob in obs:
+            obord = ob['postdate'].toordinal()
             sys.stdout.write(".")
             sys.stdout.flush()
             obrem = ob['remaining']
             obdate = coerce_date(ob['postdate'])
 
             # Start by adding all of the obligation to budgets
-            ca.budgets[ob['postdate']] += ob['amount'] * ob['credit_mult']
-            da.budgets[ob['postdate']] += ob['amount'] * ob['debit_mult']
+            ca.budgets[obord] += ob['amount'] * ob['credit_mult']
+            da.budgets[obord] += ob['amount'] * ob['debit_mult']
 
             for tx in txtable.get(obdate, []):
                 used = sum(f['amount'] for f in tx.fulfillments)
                     f_amt = min(obrem, txrem)
                     tx.fulfillments.append({'obligation': ob, 'amount': f_amt})
 
-                    ca.fulfillments[ob['postdate']] += f_amt * ob['credit_mult']
-                    da.fulfillments[ob['postdate']] += f_amt * ob['debit_mult']
+                    ca.fulfillments[obord] += f_amt * ob['credit_mult']
+                    da.fulfillments[obord] += f_amt * ob['debit_mult']
 
                     # Likewise, reduce the remaining obligation amount.
                     obrem -= f_amt
         da = ledger.accounts[self['debit']]
         while self.obligations:
             ob = self.obligations.pop()
-            ca.budgets[ob['postdate']] -= ob['amount'] * ob['credit_mult']
-            da.budgets[ob['postdate']] -= ob['amount'] * ob['debit_mult']
+            obord = ob['postdate'].toordinal()
+            ca.budgets[obord] -= ob['amount'] * ob['credit_mult']
+            da.budgets[obord] -= ob['amount'] * ob['debit_mult']
             spent = (ob['amount'] - ob['remaining'])
-            ca.fulfillments[ob['postdate']] -= spent * ob['credit_mult']
-            da.fulfillments[ob['postdate']] -= spent * ob['debit_mult']
+            ca.fulfillments[obord] -= spent * ob['credit_mult']
+            da.fulfillments[obord] -= spent * ob['debit_mult']
 
 
 def fulfill(tx):
             tx.fulfillments.append({'obligation': ob, 'amount': f_amt})
 
             # Add the amount to the account's fulfillments.
+            obord = ob['postdate'].toordinal()
             ca = ledger.accounts[ob['credit']].fulfillments
-            ca[ob['postdate']] += f_amt * ob['credit_mult']
+            ca[obord] += f_amt * ob['credit_mult']
             da = ledger.accounts[ob['debit']].fulfillments
-            da[ob['postdate']] += f_amt * ob['debit_mult']
+            da[obord] += f_amt * ob['debit_mult']
 
             ob['remaining'] -= f_amt
             txrem -= f_amt
         ob['remaining'] += f['amount']
 
         # Subtract the amount from the accounts' fulfillments.
+        obord = ob['postdate'].toordinal()
         ca = ledger.accounts[ob['credit']].fulfillments
-        ca[ob['postdate']] -= f['amount'] * ob['credit_mult']
+        ca[obord] -= f['amount'] * ob['credit_mult']
         da = ledger.accounts[ob['debit']].fulfillments
-        da[ob['postdate']] -= f['amount'] * ob['debit_mult']
+        da[obord] -= f['amount'] * ob['debit_mult']
 
 
 def find_transactions(filters):

File flowrate/ledger.py

 import flowrate
 
 
+class defaultlist(object):
+    """defaultlist(default_factory) --> boundless list with default factory
+
+    Note that negative indices do *not* count backward from the end. They are
+    valid indices just like any positive integer.
+    """
+
+    def __init__(self, default_factory=int):
+        self._data = []
+        self._initial = 0
+        self.default_factory = default_factory
+
+    def append(self, item):
+        self._data.append(item)
+
+    def extend(self, items):
+        self._data.extend(items)
+
+    def __len__(self):
+        return len(self._data)
+
+    def __getitem__(self, key):
+        key = self._grow(key)
+        #print "Fetching", key, self._data[key]
+        return self._data[key]
+
+    def __setitem__(self, key, value):
+        key = self._grow(key)
+        self._data[key] = value
+
+    def _grow(self, key):
+        """Grow the list to encompass the key. Return the adjusted key."""
+        #print "before", key, self._initial, self._data
+
+        # Calculate boundaries
+        final = self._initial + len(self._data)
+        if isinstance(key, slice):
+            lo, hi = key.start, key.stop
+        else:
+            lo, hi = key, key + 1
+
+        # Find the new low
+        minima = []
+        if lo is not None:
+            minima.append(lo)
+        if self._data:
+            minima.append(self._initial)
+        if not minima and hi is not None:
+            minima.append(hi - 1)
+        newlow = min(minima)
+
+        # Find the new high
+        maxima = []
+        if hi is not None:
+            maxima.append(hi)
+        if self._data:
+            maxima.append(final)
+        if not maxima and lo is not None:
+            maxima.append(lo)
+        newhigh = max(maxima)
+
+        # Grow boundaries if needed
+        #print "Resizing from %s:%s to %s:%s" % (self._initial, final, newlow, newhigh)
+        if self._data:
+            if newlow < self._initial:
+                self._data = [self.default_factory()
+                              for i in xrange(self._initial - newlow)
+                              ] + self._data
+            if newhigh > final:
+                self._data.extend(self.default_factory()
+                                  for i in xrange(newhigh - final))
+        else:
+            self._data = [self.default_factory()
+                          for i in xrange(newhigh - newlow)]
+        self._initial = newlow
+
+        # Modify the key to match our new boundaries.
+        if isinstance(key, slice):
+            key = slice(lo if lo is None else max(lo - self._initial, 0),
+                        hi if hi is None else max(hi - self._initial, 0),
+                        key.step)
+        else:
+            key = lo if lo is None else max(lo - self._initial, 0)
+
+        #print "after", key, self._initial, self._data
+        return key
+
+
 accounts = {}
 
 class Account(object):
         self.id = id
         self.row = dict((k, kwargs[k]) for k in self.fields)
 
-        # A dict of transaction activity on this account,
+        # A defaultlist of transaction activity on this account,
         # one entry per datetime.date.
         # Each value is a balance *difference* for the day.
         # So to read the balance at the end of 2012-04-13:
-        #   sum(v for d, v in activity.iteritems() if d <= date(2012, 4, 13))
-        self.activity = defaultdict(int)
+        #   sum(activity[:date(2012, 4, 14)])
+        self.activity = defaultlist()
 
-        # A dict of obligation budgets on this account,
+        # A defaultlist of obligation budgets on this account,
         # one entry per datetime.date.
         # Each value is a budget for the day.
         # So to read the total budget for 2012-04:
-        #   sum(v for d, v in budgets.iteritems()
-        #       if d.year == 2012 and d.month = 4)
-        self.budgets = defaultdict(int)
+        #   sum(budgets[date(2012, 4, 1):date(2012, 5, 1)])
+        self.budgets = defaultlist()
 
-        # A dict of sum(obligation fulfillments) on this account,
+        # A defaultlist of sum(obligation fulfillments) on this account,
         # one entry per datetime.date.
         # Each value is a balance *difference* for the day.
         # So to read the balance fulfilled at the end of 2012-04-13:
-        #   sum(v for d, v in fulfillments.iteritems()
-        #       if d <= date(2012, 4, 13))
-        self.fulfillments = defaultdict(int)
+        #   sum(fulfillments[:date(2012, 4, 14)])
+        self.fulfillments = defaultlist()
 
     @classmethod
     def load_all(cls):
     def activate(self):
         """Apply this transaction's amount to daily balances."""
         ca = accounts[self['credit']].activity
-        ca[self['postdate']] += self['amount'] * self['credit_mult']
+        ca[self['postdate'].toordinal()] += self['amount'] * self['credit_mult']
         da = accounts[self['debit']].activity
-        da[self['postdate']] += self['amount'] * self['debit_mult']
+        da[self['postdate'].toordinal()] += self['amount'] * self['debit_mult']
 
     def deactivate(self):
         """Apply this transaction's amount to daily balances."""
         ca = accounts[self['credit']].activity
-        ca[self['postdate']] -= self['amount'] * self['credit_mult']
+        ca[self['postdate'].toordinal()] -= self['amount'] * self['credit_mult']
         da = accounts[self['debit']].activity
-        da[self['postdate']] -= self['amount'] * self['debit_mult']
+        da[self['postdate'].toordinal()] -= self['amount'] * self['debit_mult']
 
     def __getitem__(self, key):
         return self.row[key]

File flowrate/testing/test_defaultlist.py

+import datetime
+import decimal
+
+from flowrate.testing import FlowrateTest
+from flowrate.ledger import defaultlist
+
+
+class TestDefaultList(FlowrateTest):
+
+    def test_initial_zero(self):
+        d = defaultlist()
+        d[0:10] = range(10)
+        self.assertEqual(d._data, range(10))
+        self.assertEqual(d._initial, 0)
+
+        self.assertEqual(d[3:7], [3, 4, 5, 6])
+
+    def test_initial_positive(self):
+        d = defaultlist()
+        d[100:110] = range(10)
+        self.assertEqual(d._data, range(10))
+        self.assertEqual(d._initial, 100)
+
+        self.assertEqual(d[103:107], [3, 4, 5, 6])
+
+    def test_initial_negative(self):
+        d = defaultlist()
+        d[-110:-100] = range(10)
+        self.assertEqual(d._data, range(10))
+        self.assertEqual(d._initial, -110)
+
+        self.assertEqual(d[-107:-103], [3, 4, 5, 6])
+
+    def test_grow_high(self):
+        d = defaultlist()
+        d[100:110] = range(10)
+        d[112] = 12
+        self.assertEqual(d._data, range(10) + [0, 0, 12])
+        self.assertEqual(d._initial, 100)
+
+        self.assertEqual(d[109:114], [9, 0, 0, 12, 0])
+
+        d[111:115] = [11, 72, 13, 14]
+        self.assertEqual(d._data, range(10) + [0, 11, 72, 13, 14])
+        self.assertEqual(d._initial, 100)
+
+        self.assertEqual(d[109:117], [9, 0, 11, 72, 13, 14, 0, 0])
+
+    def test_grow_low(self):
+        d = defaultlist()
+        d[100:110] = range(10)
+        d[98] = 98
+        self.assertEqual(d._data, [98, 0] + range(10))
+        self.assertEqual(d._initial, 98)
+
+        self.assertEqual(d[94:102], [0, 0, 0, 0, 98, 0, 0, 1])
+
+        d[95:99] = [95, 96, 97, 888]
+        self.assertEqual(d._data, [0, 95, 96, 97, 888, 0] + range(10))
+        self.assertEqual(d._initial, 94)
+
+        self.assertEqual(d[94:102], [0, 95, 96, 97, 888, 0, 0, 1])
+
+