Commits

Robert Brewer committed c557000

Stage 3 of variables in flows: get/set as JSON and auto-recalc all on change. Next: an HTML UI.

  • Participants
  • Parent commits f63ee0e

Comments (0)

Files changed (7)

File flowrate/__init__.py

 
 import calendar
 import datetime
+import decimal
 import heapq
 import os
 os.umask(000)
 from cherrypy.lib import cpstats, static
 
 import simplejson
+def json_processor(entity):
+    """Read application/json data into request.json."""
+    if not entity.headers.get("Content-Length", ""):
+        raise cherrypy.HTTPError(411)
+    
+    body = entity.fp.read()
+    try:
+        cherrypy.serving.request.json = simplejson.loads(
+            body.decode('utf-8'), parse_float=decimal.Decimal)
+    except ValueError:
+        raise cherrypy.HTTPError(400, 'Invalid JSON document')
+def json_handler(*args, **kwargs):
+    value = cherrypy.serving.request._json_inner_handler(*args, **kwargs)
+    return simplejson.dumps(value, use_decimal=True)
+
 from sqlalchemy import create_engine
 
 db = None
 
 from flowrate import csvutil, flows
-from flowrate.variables import ReferenceFinder
+from flowrate.variables import ReferenceFinder, environment as env
 
 
 def set_db(connstring, **kwargs):
 (but note that the server will ignore any members other than those
 in the 'body' object). GET to return it. DELETE to destroy it.""")
     
-    @cherrypy.tools.json_out()
+    @cherrypy.tools.json_out(handler=json_handler)
     def GET(self):
         """Return the transaction."""
         req = cherrypy.serving.request
                 },
             }
 
-    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_in(processor=json_processor)
     @cherrypy.tools.last_content_type()
     def PUT(self):
         """Update the given transaction entity."""
         # This triggers tools.trailing_slash to force a redirect if missing
         self.index = self
 
-    @cherrypy.tools.json_out()
+    @cherrypy.tools.json_out(handler=json_handler)
     def GET(self, accounts='', credits='', debits='',
             years='', months='', days='', description='',
             # This querystring arg doesn't actually do anything; it's here
 
         return t
 
-    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_in(processor=json_processor)
     @cherrypy.tools.last_content_type()
     def POST(self):
         """Insert the given transaction entity."""
     exposed = True
     description = "A table of account budgets."
 
-    @cherrypy.tools.json_out()
+    @cherrypy.tools.json_out(handler=json_handler)
     def GET(self, accounts='', years='', months='', days='', dategroup='month',
             **kwargs):
         """Return a table of budgets."""
 (but note that the server will ignore any members other than those
 in the 'body' object). GET to return it. DELETE to destroy it.""")
     
-    @cherrypy.tools.json_out()
+    @cherrypy.tools.json_out(handler=json_handler)
     def GET(self):
         """Return the flow."""
         req = cherrypy.serving.request
                     },
                 }
 
-    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_in(processor=json_processor)
     @cherrypy.tools.last_content_type()
     def PUT(self):
         """Update the given flow entity."""
         # This triggers tools.trailing_slash to force a redirect if missing
         self.index = self
 
-    @cherrypy.tools.json_out()
+    @cherrypy.tools.json_out(handler=json_handler)
     def GET(self, accounts='', credits='', debits='',
             years='', months='', days='', description='',
             # This querystring arg doesn't actually do anything; it's here
 
         return t
 
-    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_in(processor=json_processor)
     @cherrypy.tools.last_content_type()
     def POST(self):
         """Insert the given flow entity."""
         return Flow()
 
 
+class Variable(object):
+    
+    exposed = True
+    description = (
+"""A Flowrate Variable. PUT a Variable representation to replace it
+(but note that the server will ignore any members other than those
+in the 'body' object). GET to return it. DELETE to destroy it.""")
+
+    @cherrypy.tools.json_out(handler=json_handler)
+    def GET(self):
+        """Return the variable."""
+        req = cherrypy.serving.request
+
+        try:
+            var = env.variables[req.varname]
+        except KeyError:
+            raise cherrypy.NotFound()
+
+        return {'self': cherrypy.url(),
+                'description': self.description % vars(),
+                'body': {
+                    'source': var.source,
+                    'references': list(var.references),
+                    },
+                }
+
+    @cherrypy.tools.json_in(processor=json_processor)
+    @cherrypy.tools.last_content_type()
+    def PUT(self):
+        """Update the given variable entity."""
+        req = cherrypy.serving.request
+        name = req.varname
+
+        if "body" not in req.json:
+            raise cherrypy.HTTPError("400 No Body",
+                "Variables MUST include a 'body' member.")
+
+        # Accept the new variable definition.
+        vals = req.json["body"]
+        exists = name in env.variables
+        env.bind(name, vals['source'])
+
+        # Calculate this variable and all its dependents.
+        dependents = list(env.calc(name))
+        dependents.append(name)
+
+        # Recalc any flows which depend on these variables.
+        for row in db.execute(
+                "SELECT * FROM flows WHERE variables && %(names)s;",
+                {"names": dependents or '{}'}).fetchall():
+            flow = flows.Flow(row.id, row)
+            flow.clear_obligations()
+            flow.obligate()
+
+        if exists:
+            db.execute("UPDATE variables SET source = %s WHERE name = %s;",
+                       (vals['source'], name))
+            cherrypy.response.status = 204
+        else:
+            db.execute("INSERT INTO variables (name, source) VALUES (%s, %s);",
+                       (name, vals['source']))
+            cherrypy.response.headers['Location'] = cherrypy.url()
+            cherrypy.response.status = 201
+
+
+class Variables(object):
+    
+    exposed = True
+    description = """A catalog of available variables."""
+
+    def __init__(self):
+        # This triggers tools.trailing_slash to force a redirect if missing
+        self.index = self
+
+    @cherrypy.tools.json_out(handler=json_handler)
+    def GET(self):
+        """Return a catalog of variables."""
+        t = {
+            "self": cherrypy.url(qs=cherrypy.request.query_string),
+            "description": self.description % vars(),
+            }
+        t['data'] = [
+            {"link": cherrypy.url("/variables/%s" % k),
+             "name": k,
+             "source": v.source,
+             } for k, v in env.variables.iteritems()]
+
+        return t
+
+    def _cp_dispatch(self, vpath):
+        cherrypy.serving.request.varname = vpath.pop(0)
+        return Variable()
+
+
 class Balances(object):
 
     exposed = True
     description = "A table of account balances."
 
-    @cherrypy.tools.json_out()
+    @cherrypy.tools.json_out(handler=json_handler)
     def GET(self, accounts='', years='', months='', days='', dategroup='month'):
         """Return a table of balances."""
         b = {
 in the 'body' object). Performing a DELETE on this resource will
 delete all its transactions! Use with care!""")
 
-    @cherrypy.tools.json_out()
+    @cherrypy.tools.json_out(handler=json_handler)
     def GET(self):
         """Return information about the account."""
         req = cherrypy.serving.request
                 },
             }
     
-    @cherrypy.tools.json_in()
+    @cherrypy.tools.json_in(processor=json_processor)
     @cherrypy.tools.last_content_type()
     def PUT(self):
         """Accept (upsert) the given account entity."""
         # This triggers tools.trailing_slash to force a redirect if missing
         self.index = self
 
-    @cherrypy.tools.json_out()
+    @cherrypy.tools.json_out(handler=json_handler)
     def GET(self):
         rows = db.execute(
             "SELECT id, name, type FROM accounts ORDER BY id;").fetchall()
     balances = Balances()
     budgets = Budgets()
     flows = Flows()
+    variables = Variables()
     transactions = Transactions()
     json2_js = json2_js
     common_js = common_js
         # This triggers tools.trailing_slash to force a redirect if missing
         self.index = self
 
-    @cherrypy.tools.json_out()
+    @cherrypy.tools.json_out(handler=json_handler)
     def GET(self):
         return {
             "self": cherrypy.url(),

File flowrate/flowrate.sql

 
 CREATE TABLE variables (
     name text NOT NULL,
-    expr text,
-    variables text[] NOT NULL DEFAULT '{}'
+    source text
 );
 
 

File flowrate/flows.py

 
 class Flow(object):
 
-    def __init__(self, id):
+    def __init__(self, id, row=None):
         self.id = id
-        self.row = None
+        self.row = row
 
     def find(self):
         """Set self.row to a DB row matching the given ID, or None."""
         refs = set()
         finder = ReferenceFinder()
         for key in ('start', 'end', 'amount'):
-            val = vals[key].strip()
+            val = vals[key]
+            if not isinstance(val, basestring):
+                # Allow 'amount' entries to be numbers
+                val = str(val)
+            val = val.strip()
             if val.startswith("="):
                 newrefs = finder.find(val[1:])
                 refs.update(newrefs)
         elif self.unit == 'years':
             unit, day = lambda d: d.year, lambda d: d.timetuple().tm_yday
 
-        escrow = datetime.date(2014, 2, 1)
-
         start = self.range_start.strip()
         if start.startswith("="):
             # It's a Python expression
-            start = eval(start[1:], env, locals())
+            start = env.eval(start[1:], locals())
         else:
             # Assume it's an ISO date YYYY-MM-DD
             start = datetime.date(*map(int, start.split("-")))
         end = self.range_end.strip()
         if end.startswith("="):
             # It's a Python expression
-            end = eval(end[1:], env, locals())
+            end = env.eval(end[1:], locals())
         else:
             # Assume it's an ISO date YYYY-MM-DD
             end = datetime.date(*map(int, end.split("-")))
                 amount = self.amount.strip()
                 if amount.startswith("="):
                     # It's a Python expression
-                    amount = eval(amount[1:], env, locals())
+                    amount = env.eval(amount[1:], locals())
                 else:
                     # Assume it's a number
                     amount = decimal.Decimal(amount)
 
         obs = {}
         for ob in self.obligations():
-            print ob
             # TODO: this is slow. Can we change it to an "INSERT INTO ... FROM"?
             row = flowrate.db.execute(
                 "INSERT INTO obligations "

File flowrate/run.py

 
 import flowrate
 from flowrate import dbutil
-from flowrate import variables
+from flowrate.variables import environment as env
 
 
 class Postgres(plugins.SimplePlugin):
     def start(self):
         self.bus.log("Loading Flowrate variables...")
 
-        allvars = dict(
-            (row['name'], [row['expr'], row['variables']])
-            for row in flowrate.db.execute("SELECT * FROM variables;"
-                                           ).fetchall())
+        for row in flowrate.db.execute("SELECT * FROM variables;"
+                                       ).fetchall():
+            env.bind(row['name'], row['source'])
 
-        # Evaluate in topological order.
-        # Start with all the ones which reference no others.
-        Q = [(name, expr)
-             for name, (expr, refs) in allvars.iteritems() if not refs]
-        while Q:
-            name, expr = Q.pop()
+        env.calc_all()
 
-            # Eval and store result.
-            variables.bind(name, expr)
-
-            # Mark all vars which reference "name".
-            for k, (e, r) in allvars.iteritems():
-                if name in r:
-                    r.remove(name)
-                    # Add if all references have been evaluated.
-                    if not r:
-                        Q.append((k, e))
-
-        # Error if there are any circular definitions.
-        remaining = [(name, expr, refs)
-                     for name, (expr, refs) in allvars.iteritems() if refs]
-        if remaining:
-            raise ValueError("Circular dependencies found.", remaining)
-
-        print "Environment:", variables.environment
+        #print "Environment:"
+        #for name, source in env.variables.iteritems():
+        #    print name, source, '=>', env.locals[name]
 
     start.priority = Postgres.start.priority + 5
 

File flowrate/testing/__init__.py

+import decimal
 import difflib
 import os
 thisdir = os.path.abspath(os.path.dirname(__file__))
         return
 
     fname = os.path.join(thisdir, 'test.cfg')
-    config = simplejson.loads(open(fname, 'rb').read())
+    config = simplejson.loads(open(fname, 'rb').read(),
+                              parse_float=decimal.Decimal)
 
     webtest.WebCase.PORT = config['port']
 
             compare(expected, actual)
         except AssertionError:
             A = simplejson.dumps(
-                    expected, sort_keys=True, indent='    ').splitlines()
+                    expected, sort_keys=True, indent='    ', use_decimal=True
+                    ).splitlines()
             B = simplejson.dumps(
-                    actual, sort_keys=True, indent='    ').splitlines()
+                    actual, sort_keys=True, indent='    ', use_decimal=True
+                    ).splitlines()
             msg = "\n".join(difflib.unified_diff(
                 A, B, fromfile="expected", tofile="actual"))
             raise AssertionError("JSON documents do not match.\n" + msg)
 
     @property
     def json(self):
-        return simplejson.loads(self.body)
+        return simplejson.loads(self.body, parse_float=decimal.Decimal)
 
     def define_account(self, id, **kwargs):
         """Add the given account to the system."""
         loc = '/accounts/%s' % id
-        b = simplejson.dumps({'body': kwargs})
+        b = simplejson.dumps({'body': kwargs}, use_decimal=True)
         h = [('Content-Type', 'application/json'),
              ('Content-Length', str(len(b)))]
         self.getPage(loc, method="PUT", headers=h, body=b)
 
     def define_flow(self, **kwargs):
         """Add the given flow to the system."""
-        b = simplejson.dumps({'body': kwargs})
+        b = simplejson.dumps({'body': kwargs}, use_decimal=True)
         h = [('Content-Type', 'application/json'),
              ('Content-Length', str(len(b)))]
         self.getPage('/flows/', method="POST", headers=h, body=b)
         self.assertEqual(self.json['body'], kwargs)
         return loc
 
+    def define_variable(self, name, **kwargs):
+        """Add the given variable to the system."""
+        loc = '/variables/%s' % name
+        b = simplejson.dumps({'body': kwargs}, use_decimal=True)
+        h = [('Content-Type', 'application/json'),
+             ('Content-Length', str(len(b)))]
+        self.getPage(loc, method="PUT", headers=h, body=b)
+        self.assertStatus((201, 204))
+        self.getPage(loc)
+        self.assertStatus(200)
+        self.assertJSONEqual(kwargs, self.json['body'])
+        return loc
+
     def add_transaction(self, **kwargs):
         """Add the given transaction to the system."""
-        b = simplejson.dumps({'body': kwargs})
+        b = simplejson.dumps({'body': kwargs}, use_decimal=True)
         h = [('Content-Type', 'application/json'),
              ('Content-Length', str(len(b)))]
         self.getPage('/transactions/', method="POST", headers=h, body=b)
         return loc
 
     def tx_from_flow(self, flow, **kwargs):
+        amt = flow['amount']
+        if isinstance(amt, basestring):
+            amt = decimal.Decimal(amt)
         tx = {'credit': flow['credit'], 'debit': flow['debit'],
-              'description': flow['description'], 'amount': flow['amount']}
+              'description': flow['description'], 'amount': amt}
         tx.update(kwargs)
         return tx
 

File flowrate/testing/test_flowrate.py

+import datetime
+import decimal
+
 from flowrate.testing import start, FlowrateTest
 start()
 
 class TestWorkflow(FlowrateTest):
 
     def test_basic_workflow(self):
+        today = datetime.date.today()
+        start = datetime.date(today.year, 1, 1)
+        end = datetime.date(today.year, 12, 31)
+
         self.define_account(id=1012, type="asset", name="Checking")
 
         # Add a flow
         self.define_account(id=3011, type="income", name="Paycheck")
-        flow = dict(start='2012-01-01', end='2012-12-31',
+        flow = dict(start=start.isoformat(), end=end.isoformat(),
                     credit=self.base() + '/accounts/3011',
                     debit=self.base() + '/accounts/1012',
                     period=1, unit='months', days=[15, 28],
                     description='External Deposit MyCompany',
-                    amount=1500.30)
+                    amount='1500.30')
         self.define_flow(**flow)
 
         # Assert unfulfilled obligations for the new flow
-        expected = [self.tx_from_flow(flow, postdate='2012-%02d-%02d' % (m, d))
+        expected = [self.tx_from_flow(flow,
+                        postdate='%d-%02d-%02d' % (today.year, m, d))
                     for m in range(1, 13) for d in (15, 28)]
-        self.getPage('/transactions/?accounts=3011&years=2012')
+        self.getPage('/transactions/?accounts=3011&years=%s' % today.year)
         self.assertStatus(200)
-        self.assertJSONEqual(self.json['data'], list(reversed(expected)))
+        self.assertJSONEqual(list(reversed(expected)), self.json['data'])
 
-        tx = dict(postdate='2012-01-14',
+        # Add a transaction which mostly fulfills
+        # one of the two obligations for this month
+        tx = dict(postdate=datetime.date(today.year, today.month, 14).isoformat(),
                   credit=self.base() + '/accounts/3011',
                   debit=self.base() + '/accounts/1012',
                   description='External Deposit MyCompany',
-                  amount=1500.0)
+                  amount=decimal.Decimal('1500.00'))
         txloc = self.add_transaction(**tx)
 
-        flowtx2 = self.tx_from_flow(flow, postdate='2012-01-28')
-        flowtx1 = self.tx_from_flow(flow, amount=0.30, postdate='2012-01-15')
+        flowtx2 = self.tx_from_flow(flow,
+            postdate=datetime.date(today.year, today.month, 28).isoformat())
+        flowtx1 = self.tx_from_flow(flow,
+            amount=decimal.Decimal('0.30'),
+            postdate=datetime.date(today.year, today.month, 15).isoformat())
         tx['id'] = txloc
 
-        self.getPage('/transactions/?accounts=3011&years=2012&months=1')
+        self.getPage('/transactions/?accounts=3011&years=%s&months=%s' %
+                     (today.year, today.month))
         self.assertStatus(200)
-        self.assertJSONEqual(self.json['data'], [flowtx2, flowtx1, tx])
+        self.assertJSONEqual([flowtx2, flowtx1, tx], self.json['data'])
 
+
+class TestVariables(FlowrateTest):
+
+    def test_variable_in_flow(self):
+        today = datetime.date.today()
+
+        self.define_account(id=1012, type="asset", name="Checking")
+
+        # Define variables
+        self.define_variable(name="escrow",
+                             source="datetime.date(%s, 1, 1)" %
+                                    (today.year + 1))
+        self.define_variable(name="down", source="11000")
+
+        # Add a flow
+        self.define_account(id=4413, type="expense", name="Mortgage & Rent")
+        flow = dict(start="=escrow + months(1)", end="=eom(start)",
+                    credit=self.base() + '/accounts/4413',
+                    debit=self.base() + '/accounts/1012',
+                    period=1, unit='months', days=[6],
+                    description='Down Payment (less deposit)',
+                    amount='=down - 5000')
+        self.define_flow(**flow)
+
+        # Assert unfulfilled obligations for the new flow
+        tx = {'credit': flow['credit'], 'debit': flow['debit'],
+              'description': flow['description'],
+              'amount': decimal.Decimal('6000.00'),
+              'postdate': datetime.date(today.year + 1, 2, 6).isoformat(),
+              }
+        self.getPage('/transactions/?accounts=4413')
+        self.assertStatus(200)
+        self.assertJSONEqual([tx], self.json['data'])
+

File flowrate/variables.py

 def eom(d):
     return d.replace(day=calendar.monthrange(d.year, d.month)[1])
 
-environment = {
-    'datetime': datetime,
-    'decimal': decimal,
-    'days': days,
-    'months': months,
-    'eom': eom,
-    }
 
-def bind(name, source):
-    """Evaluate the given source and add its result to the environment."""
-    environment[name] = eval(source, {}, environment)
+missing = object()
 
+class Expression(object):
+
+    def __init__(self, source):
+        self.source = source
+        self.references = ReferenceFinder().find(source)
+
+
+class Environment(object):
+
+    globals = {
+        'datetime': datetime,
+        'decimal': decimal,
+        'days': days,
+        'months': months,
+        'eom': eom,
+        }
+
+    def __init__(self):
+        # Maintain one dict with references to Expressions...
+        self.variables = {}
+        # ...and another with references to their values, for use with eval().
+        self.locals = {}
+
+    def eval(self, source, extra_locals=None):
+        if extra_locals:
+            l = self.locals.copy()
+            l.update(extra_locals)
+        else:
+            l = self.locals
+
+        return eval(source, self.globals, l)
+
+    def bind(self, name, source):
+        """Register the given source."""
+        self.variables[name] = Expression(source)
+
+    def calc(self, name):
+        """Recalc the variable. Recalc dependents and return their names."""
+        source = self.variables[name].source
+        self.locals[name] = self.eval(source)
+
+        # Re-calc any other variables which depend on this one (topologically).
+        names = self.get_referrers(name, recursive=True)
+        self.calc_all(names)
+
+        return names
+
+    def get_referrers(self, name, recursive=False):
+        """Yield names of variables which refer to the given name.
+
+        If 'recursive' is True, then recurse and yield all names which refer
+        to the original referrers, and so on.
+        """
+        for k, expr in self.variables.iteritems():
+            if name in expr.references:
+                yield k
+
+                if recursive:
+                    for n in self.get_referrers(k, recursive):
+                        yield n
+
+    def calc_all(self, names=None):
+        """Calculate the given expressions in topological order.
+
+        If 'names' is given, it MUST be a list of names (from self.variables)
+        to calculate. If None, all names in self.variables are calculated.
+        Use this to pass in a portion of the graph of expressions.
+        """
+        if names is None:
+            names = self.variables.keys()
+
+        # Make a copy of the references so we can mutate them as we traverse.
+        # However, omit any name which isn't in our 'names' argument;
+        # we rely on emptying the 'refs' list to know when a node is ready
+        # to calculate, and we assume that any name not passed in is
+        # already calculated.
+        allrefs = dict(
+            (name, [r for r in self.variables[name].references if r in names])
+            for name in names)
+
+        # Start with all the ones which reference no others.
+        Q = [name for name, refs in allrefs.iteritems() if not refs]
+
+        while Q:
+            name = Q.pop()
+
+            # Eval and store result.
+            self.locals[name] = self.eval(self.variables[name].source)
+
+            # Mark all vars which reference "name".
+            for k, refs in allrefs.iteritems():
+                if name in refs:
+                    refs.remove(name)
+                    # Add (to the end!) if all references have been evaluated.
+                    if not refs:
+                        Q.append(k)
+
+        # Error if there are any circular definitions.
+        remaining = [(k, refs) for k, refs in allrefs.iteritems() if refs]
+        if remaining:
+            raise ValueError("Circular dependencies found.", remaining)
+
+environment = Environment()
+