Olemis Lang avatar Olemis Lang committed 910c62a

GViz QL : Parse GViz QL using operator precedence parser. Tests for FROM clause ... [ok]

gvizql.py => Total tests : 61 , Failures : 7 , Errors : 41

Comments (0)

Files changed (7)

trac-dev/gviz/tracgviz/aggregate.py

+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+
+# Copyright 2009-2011 Olemis Lang <olemis at gmail.com>
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+r"""Aggregation functions defined by Google Visualization API (version 7.0)
+
+https://developers.google.com/chart/interactive/docs/querylanguage#aggregation_functions
+
+Copyright 2009-2011 Olemis Lang <olemis at gmail.com>
+Licensed under the Apache License, Version 2.0 
+"""
+__author__ = 'Olemis Lang'
+__all__ = 'avg count max min sum'.split()
+
+def notimpl(*args, **kwds):
+  raise NotImplementedError('Aggregation not supported ... yet')
+
+avg = lambda *args, **kwds: notimpl(*args, **kwds)
+avg.return_type = 'number'
+
+count = lambda *args, **kwds: notimpl(*args, **kwds)
+count.return_type = 'number'
+
+max = lambda *args, **kwds: notimpl(*args, **kwds)
+
+min = lambda *args, **kwds: notimpl(*args, **kwds)
+
+sum = lambda *args, **kwds: notimpl(*args, **kwds)
+sum.return_type = 'number'
+

trac-dev/gviz/tracgviz/api.py

 from xmlrpclib import DateTime
 from datetime import datetime
 
-try:
-  from gvizql import prepare_ql_data
-except ImportError :
-  def prepare_ql_data(provider, tq, req, **params):
-    r"""Fall back to this implementation if `Pygments` import fails.
-    Please read `gvizql.prepare_ql_data` docstrings for further details.
-    """
-    if not tq:
-      sch = provider.get_data_schema.im_func.func_code
-      sch_args = (sch.co_argcount > 1) and (req,) or ()
-      data = provider.get_data(req, None, **params)
-      return (provider.get_data_schema(*sch_args), data)
-    else:
-      raise GVizUnsupportedQueryOp("Unable to process query : " \
-                                    "Pygments is not installed ?")
-
 class GVizException(Exception):
   r"""Base class for all exception types defined in this package"""
 
     pattern has been specified by the end-user.
     """
 
+class GVizRuntimeError(GVizException):
+    r"""An error detected at run-time when processing GViz request.
+    """
+
 def gviz_col(col_id, col_doc):
     r"""This function can be used to document the meaning of the 
     different columns in the table returned by GViz data sources as 
 # TODO : Cache the requests and optimize the call/response life cycle
 #            by using the `reqId` parameter.
 
+# Imports introducing circular reference with tracgviz.gvizql
+
+try:
+  from gvizql import prepare_ql_data
+except ImportError :
+  def prepare_ql_data(provider, tq, req, **params):
+    r"""Fall back to this implementation if `Pygments` import fails.
+    Please read `gvizql.prepare_ql_data` docstrings for further details.
+    """
+    if not tq:
+      sch = provider.get_data_schema.im_func.func_code
+      sch_args = (sch.co_argcount > 1) and (req,) or ()
+      data = provider.get_data(req, None, **params)
+      return (provider.get_data_schema(*sch_args), data)
+    else:
+      raise GVizUnsupportedQueryOp("Unable to process query : " \
+                                    "Pygments is not installed ?")
+

trac-dev/gviz/tracgviz/grammar.py

 from tracgviz.util.parsing import Any, EndMarker, NonTerminal, \
     OperatorPrecedenceParser as Parser
 
+LessPrecedence = Parser.LessPrecedence
+SamePrecedence = Parser.SamePrecedence
+MorePrecedence = Parser.MorePrecedence
+
 # Grammar
 # =======
 
-GVIZ_QL_START_STATE = 'GVIZ_SQL'
+GVIZ_QL_START_STATE = 'GVIZQL'
+
+GVIZ_QL_TERMINALS = [ 
+    (Operator.Word.Boolean, 'and'), 
+    (Operator.Word.Boolean, 'or'), 
+    (Operator.Word.Boolean, 'not'), 
+    (Number, Any), 
+    (Name.Variable, Any), 
+    (String, Any), 
+    (Literal.Date, Any), 
+    (Name.Constant, Any), 
+    (Name.Builtin, Any), 
+    (Name.Function, Any), 
+    (Punctuation, ','), 
+    (Punctuation, '('), 
+    (Punctuation, ')'), 
+    (Operator.Arithmetic, '+'), 
+    (Operator.Arithmetic, '*'), 
+    (Operator.Arithmetic, '-'), 
+    (Operator.Arithmetic, '/'), 
+    (Operator.Comparison, Any), 
+    (Operator.Word.Comparison, Any), 
+    (Keyword.Reserved, 'select'),
+    (Keyword.Reserved, 'where'),
+    (Keyword.Reserved, 'group by'),
+    (Keyword.Reserved, 'pivot'),
+    (Keyword.Reserved, 'order by'),
+    (Keyword.Reserved, 'limit'),
+    (Keyword.Reserved, 'offset'),
+    (Keyword.Reserved, 'label'),
+    (Keyword.Reserved, 'format'),
+    (Keyword.Reserved, 'options'),
+    (Name.Other, '*'),
+    (Keyword.Reserved, 'from'),
+    (Keyword, 'asc'),
+    (Keyword, 'desc'),
+  ]
+
+_And, _Or, _Not, _Number, _Var, _Str, _Date, _Const, _Builtin, _Function, \
+_Comma, _OpenP, _CloseP, _Plus, _Mult, _Minus, _Div, _BoolOp, \
+_BoolWordOp, _Select, _Where, _GroupBy, _Pivot, _OrderBy, _Limit, _Offset, \
+_Label, _Format, _Options, _Wildchar, _From, _Asc, _Desc = GVIZ_QL_TERMINALS
+
+# GViz QL non-terminals
+GVIZQL, CLAUSE, SEQ, COL_SEQ, COLUMN, BOOL_EXPR, LABEL_SEQ, \
+LABEL_EXPR, VALUE, ARITHMETIC_EXPR, TERM, FACTOR, SIMPLE_EXPR, \
+BOOL_VALUE, BOOL_EXPR, OR_EXPR, BODY, ORDER_SEQ, \
+ORDER_EXPR = [(NonTerminal, x) for x in (
+  'GVIZQL', 'CLAUSE', 'SEQ', 'COLSEQ', 'COLUMN', 'BOOL_EXPR', 'LABELSEQ', \
+  'LABEL_EXPR', 'VALUE', 'ARITHMETIC_EXPR', 'TERM', 'FACTOR', 'SIMPLE_EXPR', \
+  'BOOL_VALUE', 'BOOL_EXPR', 'OR_EXPR', 'BODY', 'ORDER_SEQ', 'ORDER_EXPR' )]
 
 def gvizql_grammar():
-  GVIZ_QL_TERMINALS = [ 
-      (Operator.Word.Boolean, 'and'), 
-      (Operator.Word.Boolean, 'or'), 
-      (Operator.Word.Boolean, 'not'), 
-      (Number, Any), 
-      (Name.Variable, Any), 
-      (String, Any), 
-      (Literal.Date, Any), 
-      (Name.Constant, Any), 
-      (Name.Builtin, Any), 
-      (Name.Function, Any), 
-      (Punctuation, ','), 
-      (Punctuation, '('), 
-      (Punctuation, ')'), 
-      (Operator.Arithmetic, '+'), 
-      (Operator.Arithmetic, '*'), 
-      (Operator.Arithmetic, '-'), 
-      (Operator.Arithmetic, '/'), 
-      (Operator.Comparison, Any), 
-      (Operator.Word.Comparison, Any), 
-      (Keyword.Reserved, 'select'),
-      (Keyword.Reserved, 'where'),
-      (Keyword.Reserved, 'group by'),
-      (Keyword.Reserved, 'pivot'),
-      (Keyword.Reserved, 'order by'),
-      (Keyword.Reserved, 'limit'),
-      (Keyword.Reserved, 'offset'),
-      (Keyword.Reserved, 'label'),
-      (Keyword.Reserved, 'format'),
-      (Keyword.Reserved, 'options'),
-    ]
-
-  _And, _Or, _Not, _Number, _Var, _Str, _Date, _Const, _Builtin, _Function, \
-  _Comma, _OpenP, _CloseP, _Plus, _Mult, _Minus, _Div, _BoolOp, \
-  _BoolWordOp, _Select, _Where, _GroupBy, _Pivot, _OrderBy, _Limit, _Offset, \
-  _Label, _Format, _Options = GVIZ_QL_TERMINALS
-
-  # GViz QL non-terminals
-  GVIZQL, CLAUSE, SEQ, COL_SEQ, COLUMN, BOOL_EXPR, LABEL_SEQ, \
-  LABEL_EXPR, VALUE, ARITHMETIC_EXPR, TERM, FACTOR, SIMPLE_EXPR, \
-  BOOL_VALUE, BOOL_EXPR, OR_EXPR = [(NonTerminal, x) for x in (
-    'GVIZQL', 'CLAUSE', 'SEQ', 'COLSEQ', 'COLUMN', 'BOOL_EXPR', 'LABELSEQ', \
-    'LABEL_EXPR', 'VALUE', 'ARITHMETIC_EXPR', 'TERM', 'FACTOR', 'SIMPLE_EXPR', \
-    'BOOL_VALUE', 'BOOL_EXPR', 'OR_EXPR' )]
-
   GVIZ_QL_PRODUCTIONS = [
       # SQL clauses
-      ('gvizql',  GVIZQL,     [ GVIZQL, (Whitespace, Any), CLAUSE ] ),
-      ('',        GVIZQL,     [ CLAUSE ] ),
+      ('gvizql',  GVIZQL,     [ BODY, (Whitespace, '\n') ] ),
+      ('body',    BODY,       [ BODY, (Whitespace, ' '), CLAUSE ] ),
+      ('',        BODY,       [ CLAUSE ] ),
       ('select',  CLAUSE,     [ _Select, SEQ ] ),
+      ('from',    CLAUSE,     [ _From, COLUMN ] ),
+      ('selall',  CLAUSE,     [ _Select, _Wildchar ] ),
       ('where',   CLAUSE,     [ _Where, BOOL_EXPR ] ),
-      ('groupby', CLAUSE,     [ _GroupBy, COL_SEQ ] ),
-      ('pivot',   CLAUSE,     [ _Pivot, COL_SEQ ] ),
-      ('orderby', CLAUSE,     [ _OrderBy, SEQ ] ),
+      ('groupby', CLAUSE,     [ _GroupBy, SEQ ] ),
+      ('pivot',   CLAUSE,     [ _Pivot, SEQ ] ),
+      ('orderby', CLAUSE,     [ _OrderBy, ORDER_SEQ ] ),
       ('limit',   CLAUSE,     [ _Limit, _Number ] ),
       ('offset',  CLAUSE,     [ _Offset, _Number ] ),
       ('label',   CLAUSE,     [ _Label, LABEL_SEQ ] ),
       ('format',  CLAUSE,     [ _Format, LABEL_SEQ ] ),
       ('options', CLAUSE,     [ _Options, COL_SEQ ] ),
+
+      # Special sequences
       ('colseq',  COL_SEQ,    [ COL_SEQ , _Comma , COLUMN ] ),
       ('',        COL_SEQ,    [ COLUMN ] ),
       ('column',  COLUMN,     [ _Var ] ),
       ('lblseq',  LABEL_SEQ,  [ LABEL_SEQ, _Comma, LABEL_EXPR ] ),
       ('',        LABEL_SEQ,  [ LABEL_EXPR ] ),
       ('lblexpr', LABEL_EXPR, [ _Var, _Str ] ),
+      ('orderseq',  ORDER_SEQ,  [ ORDER_SEQ, _Comma, ORDER_EXPR ] ),
+      ('',          ORDER_SEQ,  [ ORDER_EXPR ] ),
+      ('',          ORDER_EXPR, [ ARITHMETIC_EXPR ] ),
+      ('orderasc',  ORDER_EXPR, [ ARITHMETIC_EXPR , _Asc ] ),
+      ('orderdsc',  ORDER_EXPR, [ ARITHMETIC_EXPR , _Desc ] ),
 
       # Boolean expressions
       ('boolexp', BOOL_EXPR,  [ BOOL_EXPR, _And, OR_EXPR ] ),
       ('',        SIMPLE_EXPR,      [ VALUE ] ),
 
       # Function calls
-      ('bfunc',     VALUE, [ _Builtin, _OpenP, _CloseP ] ),
+      ('bfunc',     VALUE, [ _Builtin, _OpenP, COLUMN, _CloseP ] ),
       ('func',      VALUE, [ _Function, _OpenP, _CloseP ] ),
-      ('bfuncargs', VALUE, [ _Builtin, _OpenP, SEQ, _CloseP ] ),
       ('funcargs',  VALUE, [ _Function, _OpenP, SEQ, _CloseP ] ),
       ('seq',       SEQ, [ SEQ, _Comma, ARITHMETIC_EXPR ] ),
       ('',          SEQ, [ ARITHMETIC_EXPR ] ),
     ]
   return GVIZ_QL_PRODUCTIONS
 
-# Generated precedence and productions lookup tree
+def pretty_gviz_grammar():
+  r"""Generate he readable representation of GViz QL grammar.
+
+  PS: shown below
+  """
+  # Generate grammar precedence and productions lookup table
+  from tracgviz.util.parsing import OperatorPrecedenceParser as Parser
+
+  productions = [(prod_id, nt[1], ps) for prod_id, nt, ps in gvizql_grammar()]
+  prec, prod = Parser.process_productions(*productions)
+
+  # Readable representation for mappings and dicts
+
+  from pprint import pprint
+  from StringIO import StringIO
+
+  sio = StringIO()
+  pprint(prec, stream=sio)
+  prec_text = sio.getvalue()
+
+  sio = StringIO()
+  pprint(prod, stream=sio)
+  prod_text = sio.getvalue()
+
+  sio = None
+
+  # Perform substitutions
+
+  def find_token(tkn):
+    for attr, value in globals().iteritems():
+      if value == tkn:
+        return attr
+
+  token_subs = dict([tkn, find_token(tkn)] for tkn in GVIZ_QL_TERMINALS)
+
+  for tkn, attrnm in token_subs.iteritems():
+    prec_text = prec_text.replace(str(tkn), attrnm)
+    prod_text = prod_text.replace(str(tkn), attrnm)
+
+  for tkn in ('Token.Grammar.Relationship.', 'Token.Grammar.'):
+    prec_text = prec_text.replace(tkn, '')
+    prod_text = prod_text.replace(tkn, '')
+
+  return prec_text, prod_text
+
+# Generated precedence and productions lookup tree (gen_readable_gviz_grammar)
 # These are made available to improve performance because of the fact that
 # no production processing is needed , they are ready to use 
 
-GVIZ_GRAMMAR_PRECEDENCE = {
- ((EndMarker, 'ARITHMETIC_EXPR'), _Date): LessPrecedence,
+GVIZ_GRAMMAR_PRODUCTIONS = {(NonTerminal, Any): {_Format: {EndMarker: 'format'},
+        _From: {EndMarker: 'from'},
+        _GroupBy: {EndMarker: 'groupby'},
+        _Label: {EndMarker: 'label'},
+        _Options: {EndMarker: 'options'},
+        _OrderBy: {EndMarker: 'orderby'},
+        _Pivot: {EndMarker: 'pivot'},
+        _Select: {EndMarker: 'select'},
+        _Where: {EndMarker: 'where'},
+        _Mult: {(NonTerminal, Any): {EndMarker: 'mult'}},
+        _Plus: {(NonTerminal, Any): {EndMarker: 'sum'}},
+        _Minus: {(NonTerminal, Any): {EndMarker: 'sub'}},
+        _Div: {(NonTerminal, Any): {EndMarker: 'div'}},
+        _BoolOp: {(NonTerminal, Any): {EndMarker: 'cmp'}},
+        _And: {(NonTerminal, Any): {EndMarker: 'boolexp'}},
+        _Not: {EndMarker: 'not'},
+        _Or: {(NonTerminal, Any): {EndMarker: 'orexp'}},
+        _BoolWordOp: {(NonTerminal, Any): {EndMarker: 'strcmp'}},
+        _Comma: {(NonTerminal, Any): {EndMarker: 'seq'}},
+        (Token.Text.Whitespace, ' '): {(NonTerminal, Any): {EndMarker: 'body'}}},
+ _Asc: {(NonTerminal, Any): {EndMarker: 'orderasc'}},
+ _Desc: {(NonTerminal, Any): {EndMarker: 'orderdsc'}},
+ _Date: {EndMarker: 'date'},
+ _Number: {_Limit: {EndMarker: 'limit'},
+        _Offset: {EndMarker: 'offset'},
+        EndMarker: 'number'},
+ _Str: {_Var: {EndMarker: 'lblexpr'},
+        EndMarker: 'str'},
+ _Const: {EndMarker: 'const'},
+ _Wildchar: {_Select: {EndMarker: 'selall'}},
+ _Var: {EndMarker: 'var'},
+ _CloseP: {(NonTerminal, Any): {_OpenP: {_Builtin: {EndMarker: 'bfunc'},
+                _Function: {EndMarker: 'funcargs'},
+                EndMarker: 'par'}},
+        _OpenP: {_Function: {EndMarker: 'func'}}},
+ (Token.Text.Whitespace, '\n'): {(NonTerminal, Any): {EndMarker: 'gvizql'}}}
+
+
+GVIZ_GRAMMAR_PRECEDENCE = {((EndMarker, 'ARITHMETIC_EXPR'), _Date): LessPrecedence,
  ((EndMarker, 'ARITHMETIC_EXPR'), _Number): LessPrecedence,
  ((EndMarker, 'ARITHMETIC_EXPR'), _Str): LessPrecedence,
  ((EndMarker, 'ARITHMETIC_EXPR'), _Builtin): LessPrecedence,
  ((EndMarker, 'ARITHMETIC_EXPR'), _Minus): LessPrecedence,
  ((EndMarker, 'ARITHMETIC_EXPR'), _Div): LessPrecedence,
  ((EndMarker, 'ARITHMETIC_EXPR'), _OpenP): LessPrecedence,
+ ((EndMarker, 'BODY'), _Format): LessPrecedence,
+ ((EndMarker, 'BODY'), _From): LessPrecedence,
+ ((EndMarker, 'BODY'), _GroupBy): LessPrecedence,
+ ((EndMarker, 'BODY'), _Label): LessPrecedence,
+ ((EndMarker, 'BODY'), _Limit): LessPrecedence,
+ ((EndMarker, 'BODY'), _Offset): LessPrecedence,
+ ((EndMarker, 'BODY'), _Options): LessPrecedence,
+ ((EndMarker, 'BODY'), _OrderBy): LessPrecedence,
+ ((EndMarker, 'BODY'), _Pivot): LessPrecedence,
+ ((EndMarker, 'BODY'), _Select): LessPrecedence,
+ ((EndMarker, 'BODY'), _Where): LessPrecedence,
+ ((EndMarker, 'BODY'), (Token.Text.Whitespace, ' ')): LessPrecedence,
  ((EndMarker, 'BOOL_EXPR'), _Date): LessPrecedence,
  ((EndMarker, 'BOOL_EXPR'), _Number): LessPrecedence,
  ((EndMarker, 'BOOL_EXPR'), _Str): LessPrecedence,
  ((EndMarker, 'BOOL_VALUE'), _BoolWordOp): LessPrecedence,
  ((EndMarker, 'BOOL_VALUE'), _OpenP): LessPrecedence,
  ((EndMarker, 'CLAUSE'), _Format): LessPrecedence,
+ ((EndMarker, 'CLAUSE'), _From): LessPrecedence,
  ((EndMarker, 'CLAUSE'), _GroupBy): LessPrecedence,
  ((EndMarker, 'CLAUSE'), _Label): LessPrecedence,
  ((EndMarker, 'CLAUSE'), _Limit): LessPrecedence,
  ((EndMarker, 'FACTOR'), _Div): LessPrecedence,
  ((EndMarker, 'FACTOR'), _OpenP): LessPrecedence,
  ((EndMarker, 'GVIZQL'), _Format): LessPrecedence,
+ ((EndMarker, 'GVIZQL'), _From): LessPrecedence,
  ((EndMarker, 'GVIZQL'), _GroupBy): LessPrecedence,
  ((EndMarker, 'GVIZQL'), _Label): LessPrecedence,
  ((EndMarker, 'GVIZQL'), _Limit): LessPrecedence,
  ((EndMarker, 'GVIZQL'), _Pivot): LessPrecedence,
  ((EndMarker, 'GVIZQL'), _Select): LessPrecedence,
  ((EndMarker, 'GVIZQL'), _Where): LessPrecedence,
- ((EndMarker, 'GVIZQL'), (Token.Text.Whitespace, Any)): LessPrecedence,
+ ((EndMarker, 'GVIZQL'), (Token.Text.Whitespace, '\n')): LessPrecedence,
+ ((EndMarker, 'GVIZQL'), (Token.Text.Whitespace, ' ')): LessPrecedence,
  ((EndMarker, 'LABELSEQ'), _Var): LessPrecedence,
  ((EndMarker, 'LABELSEQ'), _Comma): LessPrecedence,
  ((EndMarker, 'LABEL_EXPR'), _Var): LessPrecedence,
+ ((EndMarker, 'ORDER_EXPR'), _Asc): LessPrecedence,
+ ((EndMarker, 'ORDER_EXPR'), _Desc): LessPrecedence,
+ ((EndMarker, 'ORDER_EXPR'), _Date): LessPrecedence,
+ ((EndMarker, 'ORDER_EXPR'), _Number): LessPrecedence,
+ ((EndMarker, 'ORDER_EXPR'), _Str): LessPrecedence,
+ ((EndMarker, 'ORDER_EXPR'), _Builtin): LessPrecedence,
+ ((EndMarker, 'ORDER_EXPR'), _Const): LessPrecedence,
+ ((EndMarker, 'ORDER_EXPR'), _Function): LessPrecedence,
+ ((EndMarker, 'ORDER_EXPR'), _Var): LessPrecedence,
+ ((EndMarker, 'ORDER_EXPR'), _Mult): LessPrecedence,
+ ((EndMarker, 'ORDER_EXPR'), _Plus): LessPrecedence,
+ ((EndMarker, 'ORDER_EXPR'), _Minus): LessPrecedence,
+ ((EndMarker, 'ORDER_EXPR'), _Div): LessPrecedence,
+ ((EndMarker, 'ORDER_EXPR'), _OpenP): LessPrecedence,
+ ((EndMarker, 'ORDER_SEQ'), _Asc): LessPrecedence,
+ ((EndMarker, 'ORDER_SEQ'), _Desc): LessPrecedence,
+ ((EndMarker, 'ORDER_SEQ'), _Date): LessPrecedence,
+ ((EndMarker, 'ORDER_SEQ'), _Number): LessPrecedence,
+ ((EndMarker, 'ORDER_SEQ'), _Str): LessPrecedence,
+ ((EndMarker, 'ORDER_SEQ'), _Builtin): LessPrecedence,
+ ((EndMarker, 'ORDER_SEQ'), _Const): LessPrecedence,
+ ((EndMarker, 'ORDER_SEQ'), _Function): LessPrecedence,
+ ((EndMarker, 'ORDER_SEQ'), _Var): LessPrecedence,
+ ((EndMarker, 'ORDER_SEQ'), _Mult): LessPrecedence,
+ ((EndMarker, 'ORDER_SEQ'), _Plus): LessPrecedence,
+ ((EndMarker, 'ORDER_SEQ'), _Minus): LessPrecedence,
+ ((EndMarker, 'ORDER_SEQ'), _Div): LessPrecedence,
+ ((EndMarker, 'ORDER_SEQ'), _OpenP): LessPrecedence,
+ ((EndMarker, 'ORDER_SEQ'), _Comma): LessPrecedence,
  ((EndMarker, 'OR_EXPR'), _Date): LessPrecedence,
  ((EndMarker, 'OR_EXPR'), _Number): LessPrecedence,
  ((EndMarker, 'OR_EXPR'), _Str): LessPrecedence,
  ((EndMarker, 'VALUE'), _Const): LessPrecedence,
  ((EndMarker, 'VALUE'), _Function): LessPrecedence,
  ((EndMarker, 'VALUE'), _Var): LessPrecedence,
+ (_Asc, (EndMarker, 'BODY')): MorePrecedence,
+ (_Asc, (EndMarker, 'CLAUSE')): MorePrecedence,
+ (_Asc, (EndMarker, 'ORDER_EXPR')): MorePrecedence,
+ (_Asc, (EndMarker, 'ORDER_SEQ')): MorePrecedence,
+ (_Asc, _Comma): MorePrecedence,
+ (_Asc, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_Asc, (Token.Text.Whitespace, ' ')): MorePrecedence,
+ (_Desc, (EndMarker, 'BODY')): MorePrecedence,
+ (_Desc, (EndMarker, 'CLAUSE')): MorePrecedence,
+ (_Desc, (EndMarker, 'ORDER_EXPR')): MorePrecedence,
+ (_Desc, (EndMarker, 'ORDER_SEQ')): MorePrecedence,
+ (_Desc, _Comma): MorePrecedence,
+ (_Desc, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_Desc, (Token.Text.Whitespace, ' ')): MorePrecedence,
+ (_Format, (EndMarker, 'BODY')): MorePrecedence,
  (_Format, (EndMarker, 'CLAUSE')): MorePrecedence,
- (_Format, (EndMarker, 'GVIZQL')): MorePrecedence,
  (_Format, _Var): LessPrecedence,
  (_Format, _Comma): LessPrecedence,
- (_Format, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_Format, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_Format, (Token.Text.Whitespace, ' ')): MorePrecedence,
+ (_From, (EndMarker, 'BODY')): MorePrecedence,
+ (_From, (EndMarker, 'CLAUSE')): MorePrecedence,
+ (_From, _Var): LessPrecedence,
+ (_From, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_From, (Token.Text.Whitespace, ' ')): MorePrecedence,
+ (_GroupBy, (EndMarker, 'BODY')): MorePrecedence,
  (_GroupBy, (EndMarker, 'CLAUSE')): MorePrecedence,
- (_GroupBy, (EndMarker, 'GVIZQL')): MorePrecedence,
+ (_GroupBy, _Date): LessPrecedence,
+ (_GroupBy, _Number): LessPrecedence,
+ (_GroupBy, _Str): LessPrecedence,
+ (_GroupBy, _Builtin): LessPrecedence,
+ (_GroupBy, _Const): LessPrecedence,
+ (_GroupBy, _Function): LessPrecedence,
  (_GroupBy, _Var): LessPrecedence,
+ (_GroupBy, _Mult): LessPrecedence,
+ (_GroupBy, _Plus): LessPrecedence,
+ (_GroupBy, _Minus): LessPrecedence,
+ (_GroupBy, _Div): LessPrecedence,
+ (_GroupBy, _OpenP): LessPrecedence,
  (_GroupBy, _Comma): LessPrecedence,
- (_GroupBy, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_GroupBy, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_GroupBy, (Token.Text.Whitespace, ' ')): MorePrecedence,
+ (_Label, (EndMarker, 'BODY')): MorePrecedence,
  (_Label, (EndMarker, 'CLAUSE')): MorePrecedence,
- (_Label, (EndMarker, 'GVIZQL')): MorePrecedence,
  (_Label, _Var): LessPrecedence,
  (_Label, _Comma): LessPrecedence,
- (_Label, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_Label, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_Label, (Token.Text.Whitespace, ' ')): MorePrecedence,
  (_Limit, _Number): SamePrecedence,
  (_Offset, _Number): SamePrecedence,
+ (_Options, (EndMarker, 'BODY')): MorePrecedence,
  (_Options, (EndMarker, 'CLAUSE')): MorePrecedence,
- (_Options, (EndMarker, 'GVIZQL')): MorePrecedence,
  (_Options, _Var): LessPrecedence,
  (_Options, _Comma): LessPrecedence,
- (_Options, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_Options, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_Options, (Token.Text.Whitespace, ' ')): MorePrecedence,
+ (_OrderBy, (EndMarker, 'BODY')): MorePrecedence,
  (_OrderBy, (EndMarker, 'CLAUSE')): MorePrecedence,
- (_OrderBy, (EndMarker, 'GVIZQL')): MorePrecedence,
+ (_OrderBy, _Asc): LessPrecedence,
+ (_OrderBy, _Desc): LessPrecedence,
  (_OrderBy, _Date): LessPrecedence,
  (_OrderBy, _Number): LessPrecedence,
  (_OrderBy, _Str): LessPrecedence,
  (_OrderBy, _Div): LessPrecedence,
  (_OrderBy, _OpenP): LessPrecedence,
  (_OrderBy, _Comma): LessPrecedence,
- (_OrderBy, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_OrderBy, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_OrderBy, (Token.Text.Whitespace, ' ')): MorePrecedence,
+ (_Pivot, (EndMarker, 'BODY')): MorePrecedence,
  (_Pivot, (EndMarker, 'CLAUSE')): MorePrecedence,
- (_Pivot, (EndMarker, 'GVIZQL')): MorePrecedence,
+ (_Pivot, _Date): LessPrecedence,
+ (_Pivot, _Number): LessPrecedence,
+ (_Pivot, _Str): LessPrecedence,
+ (_Pivot, _Builtin): LessPrecedence,
+ (_Pivot, _Const): LessPrecedence,
+ (_Pivot, _Function): LessPrecedence,
  (_Pivot, _Var): LessPrecedence,
+ (_Pivot, _Mult): LessPrecedence,
+ (_Pivot, _Plus): LessPrecedence,
+ (_Pivot, _Minus): LessPrecedence,
+ (_Pivot, _Div): LessPrecedence,
+ (_Pivot, _OpenP): LessPrecedence,
  (_Pivot, _Comma): LessPrecedence,
- (_Pivot, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_Pivot, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_Pivot, (Token.Text.Whitespace, ' ')): MorePrecedence,
+ (_Select, (EndMarker, 'BODY')): MorePrecedence,
  (_Select, (EndMarker, 'CLAUSE')): MorePrecedence,
- (_Select, (EndMarker, 'GVIZQL')): MorePrecedence,
  (_Select, _Date): LessPrecedence,
  (_Select, _Number): LessPrecedence,
  (_Select, _Str): LessPrecedence,
  (_Select, _Builtin): LessPrecedence,
  (_Select, _Const): LessPrecedence,
  (_Select, _Function): LessPrecedence,
+ (_Select, _Wildchar): SamePrecedence,
  (_Select, _Var): LessPrecedence,
  (_Select, _Mult): LessPrecedence,
  (_Select, _Plus): LessPrecedence,
  (_Select, _Div): LessPrecedence,
  (_Select, _OpenP): LessPrecedence,
  (_Select, _Comma): LessPrecedence,
- (_Select, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_Select, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_Select, (Token.Text.Whitespace, ' ')): MorePrecedence,
+ (_Where, (EndMarker, 'BODY')): MorePrecedence,
  (_Where, (EndMarker, 'CLAUSE')): MorePrecedence,
- (_Where, (EndMarker, 'GVIZQL')): MorePrecedence,
  (_Where, _Date): LessPrecedence,
  (_Where, _Number): LessPrecedence,
  (_Where, _Str): LessPrecedence,
  (_Where, _Or): LessPrecedence,
  (_Where, _BoolWordOp): LessPrecedence,
  (_Where, _OpenP): LessPrecedence,
- (_Where, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_Where, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_Where, (Token.Text.Whitespace, ' ')): MorePrecedence,
  (_Date, (EndMarker, 'ARITHMETIC_EXPR')): MorePrecedence,
+ (_Date, (EndMarker, 'BODY')): MorePrecedence,
  (_Date, (EndMarker, 'BOOL_EXPR')): MorePrecedence,
  (_Date, (EndMarker, 'BOOL_VALUE')): MorePrecedence,
  (_Date, (EndMarker, 'CLAUSE')): MorePrecedence,
  (_Date, (EndMarker, 'FACTOR')): MorePrecedence,
- (_Date, (EndMarker, 'GVIZQL')): MorePrecedence,
+ (_Date, (EndMarker, 'ORDER_EXPR')): MorePrecedence,
+ (_Date, (EndMarker, 'ORDER_SEQ')): MorePrecedence,
  (_Date, (EndMarker, 'OR_EXPR')): MorePrecedence,
  (_Date, (EndMarker, 'SEQ')): MorePrecedence,
  (_Date, (EndMarker, 'SIMPLE_EXPR')): MorePrecedence,
  (_Date, (EndMarker, 'TERM')): MorePrecedence,
  (_Date, (EndMarker, 'VALUE')): MorePrecedence,
+ (_Date, _Asc): MorePrecedence,
+ (_Date, _Desc): MorePrecedence,
  (_Date, _Mult): MorePrecedence,
  (_Date, _Plus): MorePrecedence,
  (_Date, _Minus): MorePrecedence,
  (_Date, _BoolWordOp): MorePrecedence,
  (_Date, _CloseP): MorePrecedence,
  (_Date, _Comma): MorePrecedence,
- (_Date, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_Date, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_Date, (Token.Text.Whitespace, ' ')): MorePrecedence,
  (_Number, (EndMarker, 'ARITHMETIC_EXPR')): MorePrecedence,
+ (_Number, (EndMarker, 'BODY')): MorePrecedence,
  (_Number, (EndMarker, 'BOOL_EXPR')): MorePrecedence,
  (_Number, (EndMarker, 'BOOL_VALUE')): MorePrecedence,
  (_Number, (EndMarker, 'CLAUSE')): MorePrecedence,
  (_Number, (EndMarker, 'FACTOR')): MorePrecedence,
- (_Number, (EndMarker, 'GVIZQL')): MorePrecedence,
+ (_Number, (EndMarker, 'ORDER_EXPR')): MorePrecedence,
+ (_Number, (EndMarker, 'ORDER_SEQ')): MorePrecedence,
  (_Number, (EndMarker, 'OR_EXPR')): MorePrecedence,
  (_Number, (EndMarker, 'SEQ')): MorePrecedence,
  (_Number, (EndMarker, 'SIMPLE_EXPR')): MorePrecedence,
  (_Number, (EndMarker, 'TERM')): MorePrecedence,
  (_Number, (EndMarker, 'VALUE')): MorePrecedence,
+ (_Number, _Asc): MorePrecedence,
+ (_Number, _Desc): MorePrecedence,
  (_Number, _Mult): MorePrecedence,
  (_Number, _Plus): MorePrecedence,
  (_Number, _Minus): MorePrecedence,
  (_Number, _BoolWordOp): MorePrecedence,
  (_Number, _CloseP): MorePrecedence,
  (_Number, _Comma): MorePrecedence,
- (_Number, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_Number, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_Number, (Token.Text.Whitespace, ' ')): MorePrecedence,
  (_Str, (EndMarker, 'ARITHMETIC_EXPR')): MorePrecedence,
+ (_Str, (EndMarker, 'BODY')): MorePrecedence,
  (_Str, (EndMarker, 'BOOL_EXPR')): MorePrecedence,
  (_Str, (EndMarker, 'BOOL_VALUE')): MorePrecedence,
  (_Str, (EndMarker, 'CLAUSE')): MorePrecedence,
  (_Str, (EndMarker, 'FACTOR')): MorePrecedence,
- (_Str, (EndMarker, 'GVIZQL')): MorePrecedence,
  (_Str, (EndMarker, 'LABELSEQ')): MorePrecedence,
  (_Str, (EndMarker, 'LABEL_EXPR')): MorePrecedence,
+ (_Str, (EndMarker, 'ORDER_EXPR')): MorePrecedence,
+ (_Str, (EndMarker, 'ORDER_SEQ')): MorePrecedence,
  (_Str, (EndMarker, 'OR_EXPR')): MorePrecedence,
  (_Str, (EndMarker, 'SEQ')): MorePrecedence,
  (_Str, (EndMarker, 'SIMPLE_EXPR')): MorePrecedence,
  (_Str, (EndMarker, 'TERM')): MorePrecedence,
  (_Str, (EndMarker, 'VALUE')): MorePrecedence,
+ (_Str, _Asc): MorePrecedence,
+ (_Str, _Desc): MorePrecedence,
  (_Str, _Mult): MorePrecedence,
  (_Str, _Plus): MorePrecedence,
  (_Str, _Minus): MorePrecedence,
  (_Str, _BoolWordOp): MorePrecedence,
  (_Str, _CloseP): MorePrecedence,
  (_Str, _Comma): MorePrecedence,
- (_Str, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_Str, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_Str, (Token.Text.Whitespace, ' ')): MorePrecedence,
  (_Builtin, _OpenP): SamePrecedence,
  (_Const, (EndMarker, 'ARITHMETIC_EXPR')): MorePrecedence,
+ (_Const, (EndMarker, 'BODY')): MorePrecedence,
  (_Const, (EndMarker, 'BOOL_EXPR')): MorePrecedence,
  (_Const, (EndMarker, 'BOOL_VALUE')): MorePrecedence,
  (_Const, (EndMarker, 'CLAUSE')): MorePrecedence,
  (_Const, (EndMarker, 'FACTOR')): MorePrecedence,
- (_Const, (EndMarker, 'GVIZQL')): MorePrecedence,
+ (_Const, (EndMarker, 'ORDER_EXPR')): MorePrecedence,
+ (_Const, (EndMarker, 'ORDER_SEQ')): MorePrecedence,
  (_Const, (EndMarker, 'OR_EXPR')): MorePrecedence,
  (_Const, (EndMarker, 'SEQ')): MorePrecedence,
  (_Const, (EndMarker, 'SIMPLE_EXPR')): MorePrecedence,
  (_Const, (EndMarker, 'TERM')): MorePrecedence,
  (_Const, (EndMarker, 'VALUE')): MorePrecedence,
+ (_Const, _Asc): MorePrecedence,
+ (_Const, _Desc): MorePrecedence,
  (_Const, _Mult): MorePrecedence,
  (_Const, _Plus): MorePrecedence,
  (_Const, _Minus): MorePrecedence,
  (_Const, _BoolWordOp): MorePrecedence,
  (_Const, _CloseP): MorePrecedence,
  (_Const, _Comma): MorePrecedence,
- (_Const, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_Const, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_Const, (Token.Text.Whitespace, ' ')): MorePrecedence,
  (_Function, _OpenP): SamePrecedence,
+ (_Wildchar, (EndMarker, 'BODY')): MorePrecedence,
+ (_Wildchar, (EndMarker, 'CLAUSE')): MorePrecedence,
+ (_Wildchar, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_Wildchar, (Token.Text.Whitespace, ' ')): MorePrecedence,
  (_Var, (EndMarker, 'ARITHMETIC_EXPR')): MorePrecedence,
+ (_Var, (EndMarker, 'BODY')): MorePrecedence,
  (_Var, (EndMarker, 'BOOL_EXPR')): MorePrecedence,
  (_Var, (EndMarker, 'BOOL_VALUE')): MorePrecedence,
  (_Var, (EndMarker, 'CLAUSE')): MorePrecedence,
  (_Var, (EndMarker, 'COLSEQ')): MorePrecedence,
  (_Var, (EndMarker, 'COLUMN')): MorePrecedence,
  (_Var, (EndMarker, 'FACTOR')): MorePrecedence,
- (_Var, (EndMarker, 'GVIZQL')): MorePrecedence,
+ (_Var, (EndMarker, 'ORDER_EXPR')): MorePrecedence,
+ (_Var, (EndMarker, 'ORDER_SEQ')): MorePrecedence,
  (_Var, (EndMarker, 'OR_EXPR')): MorePrecedence,
  (_Var, (EndMarker, 'SEQ')): MorePrecedence,
  (_Var, (EndMarker, 'SIMPLE_EXPR')): MorePrecedence,
  (_Var, (EndMarker, 'TERM')): MorePrecedence,
  (_Var, (EndMarker, 'VALUE')): MorePrecedence,
+ (_Var, _Asc): MorePrecedence,
+ (_Var, _Desc): MorePrecedence,
  (_Var, _Str): SamePrecedence,
  (_Var, _Mult): MorePrecedence,
  (_Var, _Plus): MorePrecedence,
  (_Var, _BoolWordOp): MorePrecedence,
  (_Var, _CloseP): MorePrecedence,
  (_Var, _Comma): MorePrecedence,
- (_Var, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_Var, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_Var, (Token.Text.Whitespace, ' ')): MorePrecedence,
  (_Mult, (EndMarker, 'ARITHMETIC_EXPR')): MorePrecedence,
+ (_Mult, (EndMarker, 'BODY')): MorePrecedence,
  (_Mult, (EndMarker, 'BOOL_EXPR')): MorePrecedence,
  (_Mult, (EndMarker, 'BOOL_VALUE')): MorePrecedence,
  (_Mult, (EndMarker, 'CLAUSE')): MorePrecedence,
- (_Mult, (EndMarker, 'GVIZQL')): MorePrecedence,
+ (_Mult, (EndMarker, 'ORDER_EXPR')): MorePrecedence,
+ (_Mult, (EndMarker, 'ORDER_SEQ')): MorePrecedence,
  (_Mult, (EndMarker, 'OR_EXPR')): MorePrecedence,
  (_Mult, (EndMarker, 'SEQ')): MorePrecedence,
  (_Mult, (EndMarker, 'TERM')): MorePrecedence,
+ (_Mult, _Asc): MorePrecedence,
+ (_Mult, _Desc): MorePrecedence,
  (_Mult, _Date): LessPrecedence,
  (_Mult, _Number): LessPrecedence,
  (_Mult, _Str): LessPrecedence,
  (_Mult, _OpenP): LessPrecedence,
  (_Mult, _CloseP): MorePrecedence,
  (_Mult, _Comma): MorePrecedence,
- (_Mult, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_Mult, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_Mult, (Token.Text.Whitespace, ' ')): MorePrecedence,
  (_Plus, (EndMarker, 'ARITHMETIC_EXPR')): MorePrecedence,
+ (_Plus, (EndMarker, 'BODY')): MorePrecedence,
  (_Plus, (EndMarker, 'BOOL_EXPR')): MorePrecedence,
  (_Plus, (EndMarker, 'BOOL_VALUE')): MorePrecedence,
  (_Plus, (EndMarker, 'CLAUSE')): MorePrecedence,
- (_Plus, (EndMarker, 'GVIZQL')): MorePrecedence,
+ (_Plus, (EndMarker, 'ORDER_EXPR')): MorePrecedence,
+ (_Plus, (EndMarker, 'ORDER_SEQ')): MorePrecedence,
  (_Plus, (EndMarker, 'OR_EXPR')): MorePrecedence,
  (_Plus, (EndMarker, 'SEQ')): MorePrecedence,
+ (_Plus, _Asc): MorePrecedence,
+ (_Plus, _Desc): MorePrecedence,
  (_Plus, _Date): LessPrecedence,
  (_Plus, _Number): LessPrecedence,
  (_Plus, _Str): LessPrecedence,
  (_Plus, _OpenP): LessPrecedence,
  (_Plus, _CloseP): MorePrecedence,
  (_Plus, _Comma): MorePrecedence,
- (_Plus, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_Plus, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_Plus, (Token.Text.Whitespace, ' ')): MorePrecedence,
  (_Minus, (EndMarker, 'ARITHMETIC_EXPR')): MorePrecedence,
+ (_Minus, (EndMarker, 'BODY')): MorePrecedence,
  (_Minus, (EndMarker, 'BOOL_EXPR')): MorePrecedence,
  (_Minus, (EndMarker, 'BOOL_VALUE')): MorePrecedence,
  (_Minus, (EndMarker, 'CLAUSE')): MorePrecedence,
- (_Minus, (EndMarker, 'GVIZQL')): MorePrecedence,
+ (_Minus, (EndMarker, 'ORDER_EXPR')): MorePrecedence,
+ (_Minus, (EndMarker, 'ORDER_SEQ')): MorePrecedence,
  (_Minus, (EndMarker, 'OR_EXPR')): MorePrecedence,
  (_Minus, (EndMarker, 'SEQ')): MorePrecedence,
+ (_Minus, _Asc): MorePrecedence,
+ (_Minus, _Desc): MorePrecedence,
  (_Minus, _Date): LessPrecedence,
  (_Minus, _Number): LessPrecedence,
  (_Minus, _Str): LessPrecedence,
  (_Minus, _OpenP): LessPrecedence,
  (_Minus, _CloseP): MorePrecedence,
  (_Minus, _Comma): MorePrecedence,
- (_Minus, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_Minus, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_Minus, (Token.Text.Whitespace, ' ')): MorePrecedence,
  (_Div, (EndMarker, 'ARITHMETIC_EXPR')): MorePrecedence,
+ (_Div, (EndMarker, 'BODY')): MorePrecedence,
  (_Div, (EndMarker, 'BOOL_EXPR')): MorePrecedence,
  (_Div, (EndMarker, 'BOOL_VALUE')): MorePrecedence,
  (_Div, (EndMarker, 'CLAUSE')): MorePrecedence,
  (_Div, (EndMarker, 'FACTOR')): MorePrecedence,
- (_Div, (EndMarker, 'GVIZQL')): MorePrecedence,
+ (_Div, (EndMarker, 'ORDER_EXPR')): MorePrecedence,
+ (_Div, (EndMarker, 'ORDER_SEQ')): MorePrecedence,
  (_Div, (EndMarker, 'OR_EXPR')): MorePrecedence,
  (_Div, (EndMarker, 'SEQ')): MorePrecedence,
  (_Div, (EndMarker, 'TERM')): MorePrecedence,
+ (_Div, _Asc): MorePrecedence,
+ (_Div, _Desc): MorePrecedence,
  (_Div, _Date): LessPrecedence,
  (_Div, _Number): LessPrecedence,
  (_Div, _Str): LessPrecedence,
  (_Div, _OpenP): LessPrecedence,
  (_Div, _CloseP): MorePrecedence,
  (_Div, _Comma): MorePrecedence,
- (_Div, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_Div, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_Div, (Token.Text.Whitespace, ' ')): MorePrecedence,
+ (_BoolOp, (EndMarker, 'BODY')): MorePrecedence,
  (_BoolOp, (EndMarker, 'BOOL_EXPR')): MorePrecedence,
  (_BoolOp, (EndMarker, 'BOOL_VALUE')): MorePrecedence,
  (_BoolOp, (EndMarker, 'CLAUSE')): MorePrecedence,
- (_BoolOp, (EndMarker, 'GVIZQL')): MorePrecedence,
  (_BoolOp, (EndMarker, 'OR_EXPR')): MorePrecedence,
  (_BoolOp, _Date): LessPrecedence,
  (_BoolOp, _Number): LessPrecedence,
  (_BoolOp, _And): MorePrecedence,
  (_BoolOp, _Or): MorePrecedence,
  (_BoolOp, _OpenP): LessPrecedence,
- (_BoolOp, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_BoolOp, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_BoolOp, (Token.Text.Whitespace, ' ')): MorePrecedence,
+ (_And, (EndMarker, 'BODY')): MorePrecedence,
  (_And, (EndMarker, 'BOOL_EXPR')): MorePrecedence,
  (_And, (EndMarker, 'CLAUSE')): MorePrecedence,
- (_And, (EndMarker, 'GVIZQL')): MorePrecedence,
  (_And, _Date): LessPrecedence,
  (_And, _Number): LessPrecedence,
  (_And, _Str): LessPrecedence,
  (_And, _Or): LessPrecedence,
  (_And, _BoolWordOp): LessPrecedence,
  (_And, _OpenP): LessPrecedence,
- (_And, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_And, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_And, (Token.Text.Whitespace, ' ')): MorePrecedence,
+ (_Not, (EndMarker, 'BODY')): MorePrecedence,
  (_Not, (EndMarker, 'BOOL_EXPR')): MorePrecedence,
  (_Not, (EndMarker, 'BOOL_VALUE')): MorePrecedence,
  (_Not, (EndMarker, 'CLAUSE')): MorePrecedence,
- (_Not, (EndMarker, 'GVIZQL')): MorePrecedence,
  (_Not, (EndMarker, 'OR_EXPR')): MorePrecedence,
  (_Not, _Date): LessPrecedence,
  (_Not, _Number): LessPrecedence,
  (_Not, _Or): MorePrecedence,
  (_Not, _BoolWordOp): LessPrecedence,
  (_Not, _OpenP): LessPrecedence,
- (_Not, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_Not, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_Not, (Token.Text.Whitespace, ' ')): MorePrecedence,
+ (_Or, (EndMarker, 'BODY')): MorePrecedence,
  (_Or, (EndMarker, 'BOOL_EXPR')): MorePrecedence,
  (_Or, (EndMarker, 'CLAUSE')): MorePrecedence,
- (_Or, (EndMarker, 'GVIZQL')): MorePrecedence,
  (_Or, (EndMarker, 'OR_EXPR')): MorePrecedence,
  (_Or, _Date): LessPrecedence,
  (_Or, _Number): LessPrecedence,
  (_Or, _Or): MorePrecedence,
  (_Or, _BoolWordOp): LessPrecedence,
  (_Or, _OpenP): LessPrecedence,
- (_Or, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_Or, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_Or, (Token.Text.Whitespace, ' ')): MorePrecedence,
+ (_BoolWordOp, (EndMarker, 'BODY')): MorePrecedence,
  (_BoolWordOp, (EndMarker, 'BOOL_EXPR')): MorePrecedence,
  (_BoolWordOp, (EndMarker, 'BOOL_VALUE')): MorePrecedence,
  (_BoolWordOp, (EndMarker, 'CLAUSE')): MorePrecedence,
- (_BoolWordOp, (EndMarker, 'GVIZQL')): MorePrecedence,
  (_BoolWordOp, (EndMarker, 'OR_EXPR')): MorePrecedence,
  (_BoolWordOp, _Date): LessPrecedence,
  (_BoolWordOp, _Number): LessPrecedence,
  (_BoolWordOp, _And): MorePrecedence,
  (_BoolWordOp, _Or): MorePrecedence,
  (_BoolWordOp, _OpenP): LessPrecedence,
- (_BoolWordOp, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_BoolWordOp, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_BoolWordOp, (Token.Text.Whitespace, ' ')): MorePrecedence,
  (_OpenP, _Date): LessPrecedence,
  (_OpenP, _Number): LessPrecedence,
  (_OpenP, _Str): LessPrecedence,
  (_OpenP, _CloseP): SamePrecedence,
  (_OpenP, _Comma): LessPrecedence,
  (_CloseP, (EndMarker, 'ARITHMETIC_EXPR')): MorePrecedence,
+ (_CloseP, (EndMarker, 'BODY')): MorePrecedence,
  (_CloseP, (EndMarker, 'BOOL_EXPR')): MorePrecedence,
  (_CloseP, (EndMarker, 'BOOL_VALUE')): MorePrecedence,
  (_CloseP, (EndMarker, 'CLAUSE')): MorePrecedence,
  (_CloseP, (EndMarker, 'FACTOR')): MorePrecedence,
- (_CloseP, (EndMarker, 'GVIZQL')): MorePrecedence,
+ (_CloseP, (EndMarker, 'ORDER_EXPR')): MorePrecedence,
+ (_CloseP, (EndMarker, 'ORDER_SEQ')): MorePrecedence,
  (_CloseP, (EndMarker, 'OR_EXPR')): MorePrecedence,
  (_CloseP, (EndMarker, 'SEQ')): MorePrecedence,
  (_CloseP, (EndMarker, 'SIMPLE_EXPR')): MorePrecedence,
  (_CloseP, (EndMarker, 'TERM')): MorePrecedence,
  (_CloseP, (EndMarker, 'VALUE')): MorePrecedence,
+ (_CloseP, _Asc): MorePrecedence,
+ (_CloseP, _Desc): MorePrecedence,
  (_CloseP, _Mult): MorePrecedence,
  (_CloseP, _Plus): MorePrecedence,
  (_CloseP, _Minus): MorePrecedence,
  (_CloseP, _BoolWordOp): MorePrecedence,
  (_CloseP, _CloseP): MorePrecedence,
  (_CloseP, _Comma): MorePrecedence,
- (_CloseP, (Token.Text.Whitespace, Any)): MorePrecedence,
+ (_CloseP, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_CloseP, (Token.Text.Whitespace, ' ')): MorePrecedence,
+ (_Comma, (EndMarker, 'BODY')): MorePrecedence,
  (_Comma, (EndMarker, 'CLAUSE')): MorePrecedence,
  (_Comma, (EndMarker, 'COLSEQ')): MorePrecedence,
- (_Comma, (EndMarker, 'GVIZQL')): MorePrecedence,
  (_Comma, (EndMarker, 'LABELSEQ')): MorePrecedence,
+ (_Comma, (EndMarker, 'ORDER_SEQ')): MorePrecedence,
  (_Comma, (EndMarker, 'SEQ')): MorePrecedence,
+ (_Comma, _Asc): LessPrecedence,
+ (_Comma, _Desc): LessPrecedence,
  (_Comma, _Date): LessPrecedence,
  (_Comma, _Number): LessPrecedence,
  (_Comma, _Str): LessPrecedence,
  (_Comma, _OpenP): LessPrecedence,
  (_Comma, _CloseP): MorePrecedence,
  (_Comma, _Comma): MorePrecedence,
- (_Comma, (Token.Text.Whitespace, Any)): MorePrecedence,
- ((Token.Text.Whitespace, Any), (EndMarker, 'GVIZQL')): MorePrecedence,
- ((Token.Text.Whitespace, Any), _Format): LessPrecedence,
- ((Token.Text.Whitespace, Any), _GroupBy): LessPrecedence,
- ((Token.Text.Whitespace, Any), _Label): LessPrecedence,
- ((Token.Text.Whitespace, Any), _Limit): LessPrecedence,
- ((Token.Text.Whitespace, Any), _Offset): LessPrecedence,
- ((Token.Text.Whitespace, Any), _Options): LessPrecedence,
- ((Token.Text.Whitespace, Any), _OrderBy): LessPrecedence,
- ((Token.Text.Whitespace, Any), _Pivot): LessPrecedence,
- ((Token.Text.Whitespace, Any), _Select): LessPrecedence,
- ((Token.Text.Whitespace, Any), _Where): LessPrecedence,
- ((Token.Text.Whitespace, Any), (Token.Text.Whitespace, Any)): MorePrecedence}
+ (_Comma, (Token.Text.Whitespace, '\n')): MorePrecedence,
+ (_Comma, (Token.Text.Whitespace, ' ')): MorePrecedence,
+ ((Token.Text.Whitespace, '\n'), (EndMarker, 'GVIZQL')): MorePrecedence,
+ ((Token.Text.Whitespace, ' '), (EndMarker, 'BODY')): MorePrecedence,
+ ((Token.Text.Whitespace, ' '), _Format): LessPrecedence,
+ ((Token.Text.Whitespace, ' '), _From): LessPrecedence,
+ ((Token.Text.Whitespace, ' '), _GroupBy): LessPrecedence,
+ ((Token.Text.Whitespace, ' '), _Label): LessPrecedence,
+ ((Token.Text.Whitespace, ' '), _Limit): LessPrecedence,
+ ((Token.Text.Whitespace, ' '), _Offset): LessPrecedence,
+ ((Token.Text.Whitespace, ' '), _Options): LessPrecedence,
+ ((Token.Text.Whitespace, ' '), _OrderBy): LessPrecedence,
+ ((Token.Text.Whitespace, ' '), _Pivot): LessPrecedence,
+ ((Token.Text.Whitespace, ' '), _Select): LessPrecedence,
+ ((Token.Text.Whitespace, ' '), _Where): LessPrecedence,
+ ((Token.Text.Whitespace, ' '), (Token.Text.Whitespace, '\n')): MorePrecedence,
+ ((Token.Text.Whitespace, ' '), (Token.Text.Whitespace, ' ')): MorePrecedence}
 
-GVIZ_GRAMMAR_PRODUCTIONS = {
- (NonTerminal, Any): {
-        _Format: {EndMarker: 'format'},
-        _GroupBy: {EndMarker: 'groupby'},
-        _Label: {EndMarker: 'label'},
-        _Options: {EndMarker: 'options'},
-        _OrderBy: {EndMarker: 'orderby'},
-        _Pivot: {EndMarker: 'pivot'},
-        _Select: {EndMarker: 'select'},
-        _Where: {EndMarker: 'where'},
-        _Mult: {(NonTerminal, Any): {EndMarker: 'mult'}},
-        _Plus: {(NonTerminal, Any): {EndMarker: 'sum'}},
-        _Minus: {(NonTerminal, Any): {EndMarker: 'sub'}},
-        _Div: {(NonTerminal, Any): {EndMarker: 'div'}},
-        _BoolOp: {(NonTerminal, Any): {EndMarker: 'cmp'}},
-        _And: {(NonTerminal, Any): {EndMarker: 'boolexp'}},
-        _Not: {EndMarker: 'not'},
-        _Or: {(NonTerminal, Any): {EndMarker: 'orexp'}},
-        _BoolWordOp: {(NonTerminal, Any): {EndMarker: 'strcmp'}},
-        _Comma: {(NonTerminal, Any): {EndMarker: 'seq'}},
-        (Token.Text.Whitespace, Any): {(NonTerminal, Any): {EndMarker: 'gvizql'}}},
- _Date: {EndMarker: 'date'},
- _Number: {_Limit: {EndMarker: 'limit'},
-        _Offset: {EndMarker: 'offset'},
-        EndMarker: 'number'},
- _Str: {_Var: {EndMarker: 'lblexpr'},
-        EndMarker: 'str'},
- _Const: {EndMarker: 'const'},
- _Var: {EndMarker: 'var'},
- _CloseP: {
-        (NonTerminal, Any): {
-            _OpenP: {
-                _Builtin: {EndMarker: 'bfuncargs'},
-                _Function: {EndMarker: 'funcargs'},
-                EndMarker: 'par'}},
-        _OpenP: {
-            _Builtin: {EndMarker: 'bfunc'},
-            _Function: {EndMarker: 'func'}}}}
-
-

trac-dev/gviz/tracgviz/gvizql.py

           'defaultParser'
 __metaclass__ = type
 
+from ast import literal_eval
+from datetime import datetime
 from itertools import islice, izip
+import operator
 from pygments.lexer import RegexLexer
 from pygments.filter import Filter
 from pygments.token import *
 from re import compile
 from weakref import proxy
 
+from tracgviz.util.parsing import OperatorPrecedenceParser,  \
+    InvalidParserConfiguration
+
 #------------------------------------------------------
 #   GVizQL clause handlers 
 #------------------------------------------------------
 
   Instances of this class have to override the following fields:
     - `_PROPS` :      a static dictionary containing the following keys
+      * `idx_eval`      a number indicating the relative order of 
+                        this clause at evaluation time.
       * `idx_syntax`    a number indicating the relative syntantic 
                         order of this clause.
-      * `idx_eval`      a number indicating the relative order of 
-                        this clause at evaluation time.
       * `keyw`          the keyword introducing (i.e. identifying) 
                         this clause.
     - `transform` :   override this method in order to apply the 
   __metaclass__ = GVizQLClauseType
   __abstract__ = True
 
-  ERR_MSGS = {
-      'PARSING' : "%(clause)s clause is not supported yet.",
-      'EVAL' :    "Unable to evaluate %(clause)s clause. " \
-                  "Either the whole clause or an specific feature " \
-                  "is not supported yet."
-    }
+  ERR_MSG = "Unable to evaluate %(clause)s clause. " \
+      "Either the whole clause or an specific feature " \
+      "is not supported yet."
 
-  def unsupported(self, at='PARSING', **vals):
+  def unsupported(self, **vals):
     r"""Indicate that this clause is not supported yet.
     """
     vals['clause'] = self.get_props('keyw').upper()
-    msg = self.ERR_MSGS.get(at, '') % vals
+    msg = self.ERR_MSG % vals
     raise GVizUnsupportedQueryOp(msg)
 
   def transform(self, schema, data):
     Note: Default behavior is to indicate that the clause is not 
     supported.
     """
-    self.unsupported(at='EVAL')
+    self.unsupported()
+
+  # Internal methods
+
+  @staticmethod
+  def _column_accessor(schema):
+    r"""Functor returning a callable providing access to the values in 
+    a given row of the result set by column name.
+    """
+    schcols = dict([col[0], i] for i, col in enumerate(schema))
+
+    def get_col_value(row, colnm):
+      try:
+        return row[schcols[colnm]]
+      except KeyError:
+        raise GVizRuntimeError("Unknown column " + colnm)
+
+    return get_col_value
+
+  @staticmethod
+  def _schema_accessor(schema):
+    r"""Functor returning a callable providing access to columns schema 
+    by column name.
+    """
+    schcols = dict([col[0], i] for i, col in enumerate(schema))
+
+    def get_col_schema(colnm):
+      try:
+        return schema[schcols[colnm]]
+      except KeyError:
+        raise GVizRuntimeError("Unknown column " + colnm)
+
+    return get_col_schema
+
+  @staticmethod
+  def _resolve_schema(expr, get_col_schema):
+    r"""Determine the new schema obtained after transforming base result set
+    by appying an arithmetic or boolean expression.
+    """
+    return expr['schema'](get_col_schema) if expr['is_schema_callable'] \
+        else expr['schema']
+
+  @staticmethod
+  def _eval_expr(row, expr, get_col_value):
+    r"""Evaluate an arithmetic or boolean expression against a row
+    in the data set. Accessor function might be needed to access row values
+    by column name.
+    """
+    return expr['eval'](row, get_col_value) if expr['is_eval_callable'] \
+        else expr['eval']
 
 class GVizSelectClause(GVizQLClauseHandler):
   _PROPS = {'idx_syntax' : 0, 'idx_eval': 7, 'keyw' : 'select'}
   KEYWORDS = ('select',)
 
-  def __init__(self, ctx):
-    r"""Initialize the expression using the enclosing parsing context
-    (see docs for `GVizQLParsingContext`). Try to parse the items 
-    in SELECT clause.
+  def __init__(self, seq):
+    r"""Initialize SELECT clause handler with the sequence of expressions
+    specified in input GViz QL statement.
     """
-    tkn, val = ctx.stream.next()
-    if tkn is Name.Other and val == '*':
-      self.cols = None
-      tkn, val = ctx.stream.next()
-    elif tkn is Name.Variable:
-      if val.startswith('`'):
-        val = val[1:-1]
-      self.cols = cols = [val]
-      for tkn, val in ctx.stream :
-        if tkn is Punctuation and val == ',' :
-          tkn, val = ctx.stream.next()
-          if tkn is Name.Variable :     # FIXME: Just column names so far
-            if val.startswith('`'):
-              val = val[1:-1]
-            cols.append(val)
-          elif tkn in (Name.Function, Name.Builtin):
-            raise GVizUnsupportedQueryOp("Scalar and aggregate functions " \
-                                          "not supported yet : token %s" % \
-                                          (val,))
-          else:
-            raise GVizInvalidQuery("Syntax error or unsupported " \
-                                    "feature. Unexpected token %s." \
-                                                            % (val,))
-        else:
-          break
-    elif tkn in (Name.Function, Name.Builtin):
-      raise GVizUnsupportedQueryOp("Scalar and aggregate functions " \
-                                    "not supported yet : token %s" % \
-                                    (val,))
-    else:
-      if tkn is Whitespace and val == '\n' :
-        val = '<EOL>'
-      raise GVizInvalidQuery("Syntax error or unsupported " \
-                              "feature. Unexpected token %s." % (val,))
+    self.cols_mapping = None if seq == '*' else seq
+    self.cols = [self._resolve_schema(c, lambda colnm: (colnm, None))[0] 
+        for c in self.cols_mapping]
 
   def transform(self, schema, data):
-    r"""Arrange values considering the given column order.
+    r"""Arrange and transform values considering the given expressions list.
     """
-    cols = self.cols
-    if cols is None:
+    cols = self.cols_mapping
+    if not cols:
       return schema, data
     else:
-      idxmap = [None] * len(cols)
-      schcols = dict([col[0], i] for i, col in enumerate(schema))
-      for i, colnm in enumerate(cols):
-        try:
-          idxmap[i] = schcols[colnm]
-        except KeyError, exc:
-          raise GVizInvalidQuery("Unable to evaluate SELECT clause. " \
-                                  "Unknown column %s." % (exc.message,))
-      new_schema = [schema[i] for i in idxmap]
-      new_data = ([row[i] for i in idxmap] for row in data)
+      get_col_value = self._column_accessor(schema)
+      get_col_schema = self._schema_accessor(schema)
+      new_schema = [self._resolve_schema(c, get_col_schema) \
+          for c in self.cols_mapping]
+      new_data = ( [self._eval_expr(row, c, get_col_value) \
+              for c in self.cols_mapping] for row in data )
       return new_schema, new_data
 
 class GVizFromClause(GVizQLClauseHandler):
   _PROPS = {'idx_syntax' : 1, 'idx_eval': 0, 'keyw' : 'from'}
   KEYWORDS = ('from',)
 
-  def __init__(self, ctx):
-    r"""Initialize the expression using the enclosing parsing context
-    (see docs for `GVizQLParsingContext`). Try to parse the items 
-    in FROM clause.
+  def __init__(self, name):
+    r"""Always fail with deprecation message.
+
+    FROM clause is deprecated and has been removed from the language.
     """
-    tkn, val = ctx.stream.next()
-    if tkn is Name.Variable :
-      if val.startswith('`'):
-        val = val[1:-1]
-      self.basetable = val
-      ctx.stream.next()
-    else :
-      raise GVizInvalidQuery("Syntax error: Table name expected but" \
-                              " %s found." % (val,))
+    self.basetable = name['schema'](lambda colnm: (colnm, None))[0]
+
   def transform(self, schema, data):
     r"""Notify that base table has not been processed by data source. 
     This is made this way since it may hide a bug in the underlying 
     data source implementation.
 
-    FROM clause is irrevelevant for transformations. This clause is 
-    available so that the underlying data sources be able to handle 
-    it. This feature will be supported in upcoming versions.
+    FROM clause is deprecated and has been removed from the language.
     """
-    raise GVizInvalidQuery("Unable to evaluate FROM clause. " \
-                              "Data source doesn't contain base " \
-                              "table %s." % (self.basetable,))
+    raise GVizInvalidQuery("The from clause has been eliminated" \
+        " from the language.")
 
 class GVizWhereClause(GVizQLClauseHandler):
   _PROPS = {'idx_syntax' : 2, 'idx_eval': 3, 'keyw' : 'where'}
   KEYWORDS = ('where',)
 
-  def __init__(self, ctx):
-    r"""Initialize the expression using the enclosing parsing context
-    (see docs for `GVizQLParsingContext`). Try to parse the items 
-    in WHERE clause.
+  def __init__(self, expr):
+    r"""Initialize clause instance with a boolean expression evaluated
+    in order to filter rows in the result set.
     """
-    self.unsupported()
+    if expr['is_schema_callable'] or expr['schema'][1] != 'boolean':
+      raise GVizInvalidQuery("Syntax error: Boolean expression expected " \
+          "in where clause." + 
+          (" Got %s" % (expr['schema'][1])) if not expr['is_schema_callable']
+              else '')
+    self.expr = expr
+    self.filter = self._resolve_schema(expr, lambda colnm: (colnm, None))[0]
+
+  def transform(self, schema, data):
+    r"""Filter rows that do not satisfy target predicate.
+    """
+    get_col_value = self._column_accessor(schema)
+    new_data = ( row for row in data \
+        if bool(self._eval_expr(row, self.expr, get_col_value)) )
+    return (schema, new_data)
 
 class GVizGroupByClause(GVizQLClauseHandler):
   _PROPS = {'idx_syntax' : 3, 'idx_eval': 1, 'keyw' : 'group by'}
   KEYWORDS = ('group', 'by')
 
-  def __init__(self, ctx):
-    r"""Initialize the expression using the enclosing parsing context
-    (see docs for `GVizQLParsingContext`). Try to parse the items 
-    in GROUP BY clause.
+  def __init__(self, seq):
+    r"""Initialize this clause with a sequence of expressions specifying
+    how to group multiple rows in the base data set.
+    """
+    self.groups_def = seq
+
+  def transform(self, schema, data):
+    r"""Create groups.
     """
     self.unsupported()
 
   _PROPS = {'idx_syntax' : 4, 'idx_eval': 2, 'keyw' : 'pivot'}
   KEYWORDS = ('pivot',)
 
-  def __init__(self, ctx):
-    r"""Initialize the expression using the enclosing parsing context
-    (see docs for `GVizQLParsingContext`). Try to parse the items 
-    in the SELECT clause.
+  def __init__(self, seq):
+    r"""Initialize this clause with the expressions transforming values into
+    new columns.
+    """
+    self.pivot_def = seq
+
+  def transform(self, schema, data):
+    r"""Create new columns from data.
     """
     self.unsupported()
 
   _PROPS = {'idx_syntax' : 5, 'idx_eval': 4, 'keyw' : 'order by'}
   KEYWORDS = ['asc', 'desc', 'order', 'by']
 
-  def __init__(self, ctx):
-    r"""Initialize the expression using the enclosing parsing context
-    (see docs for `GVizQLParsingContext`). Try to parse the items 
-    in the SELECT clause.
+  def __init__(self, seq):
+    r"""Initialize this clause with expressions used to sort base result set.
+    """
+    if isinstance(seq, tuple):
+      seq = Sequence([seq])
+    elif isinstance(seq, dict):
+      seq = Sequence([(seq, True)])
+    self.order_def = seq
+
+  def transform(self, schema, data):
+    r"""Sort rows in base result set by evaluating target expressions.
     """
     self.unsupported()
 
   _PROPS = {'idx_syntax' : 6, 'idx_eval': 6, 'keyw' : 'limit'}
   KEYWORDS = ('limit',)
 
-  def __init__(self, ctx):
-    r"""Initialize the expression using the enclosing parsing context
-    (see docs for `GVizQLParsingContext`). Try to parse the items 
-    in the SELECT clause.
+  def __init__(self, number):
+    r"""Initialize this clause with an integet value
     """
-    tkn, val = ctx.stream.next()
-    if tkn is Number.Integer :
-      self.cnt = int(val)
-      ctx.stream.next()
-    else :
-      raise GVizInvalidQuery("Syntax error: Integer number expected " \
-                              "but %s found." % (val,))
+    self.cnt = number
 
   def transform(self, schema, data):
     r"""Retrieve no more than `cnt` items.
   _PROPS = {'idx_syntax' : 7, 'idx_eval': 5, 'keyw' : 'offset'}
   KEYWORDS = ('offset',)
 
-  def __init__(self, ctx):
-    r"""Initialize the expression using the enclosing parsing context
-    (see docs for `GVizQLParsingContext`). Try to parse the items 
-    in the SELECT clause.
+  def __init__(self, number):
+    r"""Initialize this clause with an integet value
     """
-    tkn, val = ctx.stream.next()
-    if tkn is Number.Integer :
-      self.skip = int(val)
-      ctx.stream.next()
-    else :
-      raise GVizInvalidQuery("Syntax error: Integer number expected " \
-                              "but %s found." % (val,))
+    self.skip = number
 
   def transform(self, schema, data):
     r"""Skip the number of items determined by `skip` attribute.
   _PROPS = {'idx_syntax' : 8, 'idx_eval': 10, 'keyw' : 'label'}
   KEYWORDS = ('label',)
 
-  def __init__(self, ctx):
-    r"""Initialize the expression using the enclosing parsing context
-    (see docs for `GVizQLParsingContext`). Try to parse the items 
-    in the SELECT clause.
+  def __init__(self, seq):
+    r"""Initialize this clause with a sequence of formatting expressions
     """
-    self.labels = labels = []
-    tkn, val = Punctuation, ','               # Enter the cycle initially
-    while tkn is Punctuation and val == ',':
-      tkn, val = ctx.stream.next()
-      if tkn is Name.Variable :
-        if val.startswith('`'):
-          val = val[1:-1]
-        colnm = val
-        tkn, val = ctx.stream.next()
-        if tkn in [String.Single, String.Double]:
-          val = val[1:-1]                         # Remove quotes
-          labels.append((colnm, val.encode('utf-8', 'ignore')))
-        else :
-          raise GVizInvalidQuery("Syntax error: String constant " \
-                                "expected but %s found." % (val,))
-        tkn, val = ctx.stream.next()
-      else :
-        raise GVizInvalidQuery("Syntax error: Column name expected " \
-                                "but %s found." % (val,))
+    self.labels = list(seq)
 
   def transform(self, schema, data):
     r"""(Add | modify) the schema in order to (include | update) 
   _PROPS = {'idx_syntax' : 9, 'idx_eval': 8, 'keyw' : 'format'}
   KEYWORDS = ('format',)
 
-  def __init__(self, ctx):
-    r"""Initialize the expression using the enclosing parsing context
-    (see docs for `GVizQLParsingContext`). Try to parse the items 
-    in the SELECT clause.
+  def __init__(self, seq):
+    r"""Initialize this clause with a sequence of formatting expressions
+    """
+    self.patterns = dict(seq)
+
+  def transform(self, schema, data):
+    r"""Annotate result set with formatted values.
     """
     self.unsupported()
 
   _PROPS = {'idx_syntax' : 10, 'idx_eval': 9, 'keyw' : 'options'}
   KEYWORDS = ('options',)
 
-  def __init__(self, ctx):
-    r"""Initialize the expression using the enclosing parsing context
-    (see docs for `GVizQLParsingContext`). Try to parse the items 
-    in the SELECT clause.
+  def __init__(self, seq):
+    r"""Initialize this clause with a list of option names.
+    """
+    self.options = seq
+
+  def transform(self, schema, data):
+    r"""Condition the result set according to options.
     """
     self.unsupported()
 
           ],
       }
 
+  from tracgviz.grammar import GVIZ_GRAMMAR_PRECEDENCE, GVIZ_QL_START_STATE, \
+      GVIZ_GRAMMAR_PRODUCTIONS
+
   def __init__(self, *args, **kwds):
     super(GVizQLParser, self).__init__(*args, **kwds)
     self.noisy = True
+    self.parser = OperatorPrecedenceParser()
+    self.parser.start_state = self.GVIZ_QL_START_STATE
+    self.parser.precedence = self.GVIZ_GRAMMAR_PRECEDENCE
+    self.parser.productions_tree = self.GVIZ_GRAMMAR_PRODUCTIONS
 
   def parse(self, tq):
     r"""Parse a GVizQL expression.
 
     @param tq                       the GViz QL expression.
-    @return                         an instance 
+    @return                         an instance of GVizQLExpression 
     @throw GVizInvalidQuery         if a syntax error or other error 
                                     is detected in the input string.
     @throw GVizUnsupportedQueryOp   if the GViz QL query expression 
                                     contains a clause or feature 
                                     that's not supported yet.
     """
-    global GVizUnsupportedQueryOp, GVizInvalidQuery
-    from tracgviz.api import GVizUnsupportedQueryOp as GU, GVizInvalidQuery as GI
-    GVizUnsupportedQueryOp, GVizInvalidQuery = GU, GI
+    tkns = iter(self.get_tokens(tq))
 
-    tkns = iter(self.get_tokens(tq))
+    builder = GVizQLExpressionBuilder(self.parser)
     try:
-      tkn, val = tkns.next()
-    except StopIteration:
-      raise GVizInvalidQuery("No token found: empty string ?")
-    else:
-      order = None
-      expr = GVizQLExpression()
-      ch = None
-      try:
-        while True:         # Try to parse the next clause
-          if tkn is Whitespace and val == '\n':
-            break
-          elif tkn is not Keyword.Reserved:   # Check for a valid clause 
-            last = ch.get_props('keyw')
-            raise GVizInvalidQuery("Invalid token %s found %s: " \
-                                    "Clause name expected." % \
-                                        (val, ch and 'after ' + last or ''))
-          clause_type = GVizQLClauseType.CLAUSE_CACHE[val]
-          clause_order = clause_type.get_props('idx_syntax')
-          if not clause_order <= order:
-            order = clause_order
-          else:
-            raise GVizInvalidQuery("Invalid token %s found: " \
-                                    "Unexpected clause." % (val,))
-          ch = clause_type(self._ctx)     # Consume tokens until the 
-                                          # clause is over
-          expr._add_handler(ch)
-          tkn, val = self._ctx._last
-      except StopIteration:             # TODO: Trailing whitespace? (bug ?)
-        pass
-      if not expr._handlers:            # Something was actually parsed ?
-        raise GVizInvalidQuery("No token found: empty string ?")
-      return expr
+      result = self.parser.parse(tkns, on_reduce=builder.dispatch, \
+          on_accept=builder.handle_gvizql)
+    except SyntaxError, exc:
+      raise GVizInvalidQuery(exc)
+    return result
 
   def get_tokens(self, tq):
     try:
         if p.noisy :
           raise GVizInvalidQuery("Invalid token %s " % t[1:])
       elif t[0] is not Whitespace:
+
+        # Artificial whitespace between SQL clauses
+        if t[0] in Keyword.Reserved and self._last != (None, None):
+          yield (Whitespace, ' ')
+
         if self._last == (Keyword.Reserved, 'select') and \
             t == (Operator, '*'):
           t = Name.Other, '*'
     """
     self._handlers = dict()
     self._attrmap = dict()
+
   def _add_handler(self, ch):
     r"""Add a clause handler to this expression object. Bind the 
     handler's attributes to lazy arguments.
     sch = provider.get_data_schema(*sch_args)
     return tq.transform(sch, data)
 
+class Sequence(list):
+  pass
+
+class GVizQLExpressionBuilder:
+  r"""Stub class responsible for handling parser callback invocations 
+  in order to build instances of `GVizQLExpression`.
+  """
+  def __init__(self, parser, scalar_funcs=None, agg_funcs=None):
+    self.parser = parser
+    if scalar_funcs is None:
+      scalar_funcs = self._default_scalar_funcs()
+    if agg_funcs is None:
+      agg_funcs = self._default_aggregate_funcs()
+    self.scalar_funcs = scalar_funcs
+    self.agg_funcs = agg_funcs
+    self.now = None
+
+  @classmethod
+  def _default_scalar_funcs(cls):
+    from tracgviz import scalar
+    return dict([nm, getattr(scalar, nm)] for nm in scalar.__all__)
+
+  @classmethod
+  def _default_aggregate_funcs(cls):
+    from tracgviz import aggregate
+    return dict([nm, getattr(aggregate, nm)] for nm in aggregate.__all__)
+
+  def dispatch(self, prod_id, *args):
+    try:
+      handler = getattr(self, 'handle_' + prod_id)
+    except AttributeError:
+      raise InvalidParserConfiguration(self.parser, 
+          'Unknown production ' + prod_id)
+    else:
+      result = handler(*args)
+      if isinstance(result, dict):
+        result['production'] = prod_id
+      return result
+
+  # Special markers 
+
+  Sequence = Sequence
+
+  @property
+  def SyntaxError(self):
+    return GVizInvalidQuery
+
+  # Grammar production handlers
+
+  def handle_number(self, value):
+    if value[0] in Number.Integer:
+      _value = int(value[1])
+    else:
+      _value = float(value[1])
+    return dict(
+        eval=_value, 
+        schema=(value[1], 'number'), 
+        is_eval_callable=False,
+        is_schema_callable=False
+      )
+
+  def handle_var(self, name):
+    colnm = name[1]
+    if colnm.startswith('`'):
+      colnm = colnm[1:-1]
+    return dict(
+        eval=lambda row, getvalue: getvalue(row, colnm) , 
+        schema=lambda get_col_schema: get_col_schema(colnm),
+        is_eval_callable=True,
+        is_schema_callable=True)
+
+  def handle_str(self, value):
+    return dict(
+        eval=literal_eval(value[1]), 
+        schema=(value[1], 'string'),
+        is_eval_callable=False,
+        is_schema_callable=False)
+
+  def handle_date(self, value):
+    try:
+      kind, _value = value[1].split()
+      _value = literal_eval(_value)
+      f, fmtstr = {
+          'date' : (datetime.date, '%Y-%m-%d'),
+          'timeofday' : (datetime.time, '%H:%M:%S'),
+          'datetime' : (lambda x: x, '%Y-%m-%d %H:%M:%S'),
+        }.get(kind)
+      if not kind:
+        raise self.SyntaxError('Invalid date literal %s' % value[1])
+      _value = datetime.strptime(_value, fmtstr)
+    except ValueError:
+      raise self.SyntaxError('Invalid date literal %s' % value[1])
+    return dict(
+        eval=f(_value), 
+        schema=(value[1], kind), 
+        is_eval_callable=False,
+        is_schema_callable=False)
+
+  def handle_const(self, value):
+    try:
+      _value = {
+          'true' : True,
+          'false' : False,
+          'null' : None
+        }.get(value[1])
+    except KeyError:
+      raise self.SyntaxError('Unexpected constant %s' % (value[1],))
+    return dict(
+        eval=_value, 
+        schema=(value[1], 'boolean' if value[1] != 'null' else None), 
+        is_eval_callable=False,
+        is_schema_callable=False
+      )
+
+  def handle_seq(self, seq, _, expr):
+    _seq = seq[1]
+    _expr = expr[1]
+    if isinstance(_seq, self.Sequence):
+      _seq.append(_expr)
+    else:
+      _seq = self.Sequence([_seq, _expr])
+    return _seq
+
+  def _unsupported(self, *args):
+    raise NotImplementedError('Feature not supported ... yet')
+
+  def handle_bfunc(self, funcnm, _, colnm, __):
+    funcnm = funcnm[1]
+    colnm = colnm[1]
+    try:
+      f = self.agg_funcs[funcnm]
+    except KeyError:
+      raise LookupError('Unknown aggregation function %s' % (funcnm,))
+    return_type = getattr(f, 'return_type', None)
+    func_label = '%s()' % (funcnm,)
+    return dict(
+        eval=self._unsupported, 
+        schema=(func_label, return_type) if return_type else 
+            lambda get_col_schema: (func_label, get_col_schema(colnm)),
+        is_eval_callable=True,
+        is_schema_callable=bool(return_type)
+      )
+
+  def handle_func(self, funcnm, _, __):
+    funcnm = funcnm[1]
+    try:
+      f = self.scalar_funcs[funcnm]
+      return_type = r.return_type
+    except KeyError:
+      raise LookupError('Unknown scalar function %s' % (funcnm,))
+    except AttributeError:
+      raise GVizRuntimeError('Unknown return tyoe for scalar function %s' % \
+          funcnm)
+    if funcnm == 'now':
+      # Unique time measurement
+      if self.now is None:
+        self.now = f()
+      return dict(
+          eval=self.now, 
+          schema=('now()', return_type), 
+          is_eval_callable=False,
+          is_schema_callable=False
+        )
+    else:
+      return dict(
+          eval=lambda *args: f(), 
+          schema=('%s()' % (funcnm,), return_type),
+          is_eval_callable=True,
+          is_schema_callable=False
+        )
+
+  def handle_funcargs(self, funcnm, _, seq, __):
+    funcnm = funcnm[1]
+    seq = seq[1]
+    try:
+      f = self.scalar_funcs[funcnm]
+      return_type = r.return_type
+    except KeyError:
+      raise LookupError('Unknown scalar function %s' % (funcnm,))
+    except AttributeError:
+      raise GVizRuntimeError('Unknown return tyoe for scalar function %s' % \
+          funcnm)
+
+    # GViz QL scalar functions (except `now`) are deterministic however ...
+    is_eval_callable, eval_func = self._eval_expr(f, seq, 
+        getattr(f, 'deterministic', True))
+    return dict(
+        eval=eval_func, 
+        schema=(self._function_label(funcnm, seq), return_type),
+        is_eval_callable=is_eval_callable,
+        is_schema_callable=False
+      )
+
+  def handle_par(self, _, expr, __):
+    return expr[1]
+
+  def _handle_binary_op(self, lexpr, op, rexpr):
+    op = op[1]
+    lexpr = lexpr[1]
+    rexpr = rexpr[1]
+
+    if op == 'like':
+      raise NotImplementedError('Try `matches` instead of `like`')
+
+    f = {
+        '+' : 'add', '-' : 'sub', '*' : 'mul', # '/' : 'div',
+        'and' : 'and_', 'or' : 'or_',
+        '<=' : 'le', '<' : 'lt', '>' : 'gt', '>=' : 'ge', 
+        '=' : 'eq', '!=' : 'ne', '<>' : 'ne', 
+        'is' : 'is_', 'is not' : 'is_not', 
+        'contains' : 'contains',
+      }.get(op)
+    if f is not None:
+      f = getattr(operator, f)
+    else:
+      from re import match
+
+      def div(a, b):
+        try:
+          return operator.div(a, b)
+        except ZeroDivisionError:
+          return None
+      f = {
+          '/' : div, 
+          'starts with' : lambda a, b : a.startswith(b),
+          'ends with' : lambda a, b : a.endswith(b),
+          'matches' : lambda a, b: bool(match('^%s$' % (b,), a)),
+        }.get(op)
+
+      if f is None:
+        raise self.SyntaxError('Unknown operator ' + op)
+
+    is_eval_callable, eval_func = self._eval_expr(f, [lexpr, rexpr])
+    return_type = 'number' if op in '-+*/' else 'boolean'
+    if lexpr['is_schema_callable']:
+      lschema = lexpr['schema'](lambda colnm : (colnm, None))
+    else:
+      lschema = lexpr['schema']
+    if rexpr['is_schema_callable']:
+      rschema = rexpr['schema'](lambda colnm : (colnm, None))
+    else:
+      rschema = rexpr['schema']
+    return dict(
+        eval=eval_func,
+        schema=('%s %s %s' % (lschema[0], op, rschema[0]), return_type),
+        is_eval_callable=is_eval_callable,
+        is_schema_callable=False
+      )
+
+  handle_sum = handle_sub = handle_mul = handle_div = handle_strcmp = \
+      handle_cmp = handle_orexp = handle_boolexp = _handle_binary_op
+
+  def handle_not(self, _, expr):
+    expr = expr[1]
+    not_label='not ' + expr['label']
+    if expr.get('is_eval_callable'):
+      f = expr['eval']
+      return dict(
+          eval=lambda *args: not bool(f(*args)), 
+          label=(not_label, 'boolean'), 
+          is_eval_callable=True,
+          is_schema_callable=False
+        )
+    else:
+      return dict(
+          eval=not bool(f), 
+          schema=(not_label, 'boolean'), 
+          is_eval_callable=False,
+          is_schema_callable=False)
+
+  def _handle_explicit_order(self, expr, direction):
+    return (expr[1], direction[1] == 'asc')
+
+  handle_orderasc = handle_order_desc = _handle_explicit_order
+
+  def handle_orderseq(self, orderdef, _, neworder):
+    orderdef = orderdef[1]
+    neworder = neworder[1]
+    if not isinstance(neworder, tuple):
+      # Default order = asc
+      neworder = (neworder, True)
+    if isinstance(orderdef, self.Sequence):
+      orderdef.append(neworder)
+    else:
+      if not isinstance(orderdef, tuple):
+        # Default order = asc
+        orderdef = (orderdef, True)
+      orderdef = self.Sequence([orderdef, neworder])
+    return orderdef
+
+  def handle_lblexpr(self, colnm, label):
+    return (colnm[1], literal_eval(label[1]))
+
+  def handle_lblseq(self, labels, _, label_expr):
+    labels = labels[1]
+    label_expr = label_expr[1]
+    if isinstance(labels, self.Sequence):
+      labels.append(label_expr)
+    else:
+      labels = self.Sequence([labels, label_expr])
+    return labels
+
+  def handle_column(self, colnm):
+    return colnm[1]
+
+  def handle_colseq(self, colseq, _, colnm):
+    colseq = colseq[1]
+    colnm = colnm[1]
+    if isinstance(colseq, self.Sequence):
+      colseq.append(colnm)
+    else:
+      colseq = self.Sequence([colseq, colnm])
+    return colseq
+
+  def _handle_number_clause(self, keyword, number):
+    keyword = keyword[1]
+    try:
+      if number[0] not in Number.Integer:
+        raise ValueError()
+      number = int(number[1])
+    except ValueError:
+      raise self.SyntaxError('Integer number expected in clause %s . Got %s' %
+          (keyword, number[1]) )
+    clause_type = GVizQLClauseType.CLAUSE_CACHE[keyword]
+    ch = clause_type(number)
+    return ch
+
+  handle_limit = handle_offset = _handle_number_clause
+
+  def _handle_complex_clause(self, keyword, body):
+    keyword = keyword[1]
+    clause_type = GVizQLClauseType.CLAUSE_CACHE[keyword]
+    ch = clause_type(body[1])
+    return ch
+
+  handle_select = handle_selall = handle_where = handle_groupby = \
+      handle_pivot = handle_orderby = handle_label = handle_format = \
+      handle_options = handle_from = _handle_complex_clause
+
+  def handle_body(self, expr_or_clause, _=None, newclause=None):
+    if isinstance(expr_or_clause, tuple):
+      expr_or_clause = expr_or_clause[1]
+    if isinstance(expr_or_clause, GVizQLExpression) :
+      gviz_expr = expr_or_clause 
+    else:
+      gviz_expr = GVizQLExpression()
+      gviz_expr._add_handler(expr_or_clause)
+    if newclause is not None:
+      gviz_expr._add_handler(newclause[1])
+    return gviz_expr
+
+  handle_gvizql = handle_body
+
+  # Internal methods
+
+  def _function_label(self, funcnm, seq):
+    return '%s(%s)' % (funcnm, ', '.join(x['label'] for x in seq))
+
+  def _eval_expr(self, f, seq, deterministic=True):
+    r"""Functor computing evaluation function for a compound statement.
+
+    : param f :       function or callable transforming values in
+                      sub-statements values according to the particular
+                      semantics of the target non-terminal state
+                      e.g. `sum` for production E -> E + T
+    : param seq :     a sequence of dictionaries containing the definition of
+                      the values and transformations to evaluate 
+                      sub-expressions.
+    : param deterministic : a boolean flag indicating whether f is 
+                      deterministic (i.e. will always output the same result
+                      for the same inputs) e.g. `sum` is deterministic
+                      whereas `datetime.datetime.now` and `random.random` 
+                      are not.
+    : return :        a binary tuple of the form `(is_eval_callable, value)`
+                      where `is_eval_callable` indicates whether `value`
+                      is a constant result (e.g. like in `upper('foo')` )
+                      or a callable object.
+    """
+    expand = set(i for i, x in enumerate(seq) if x.get('is_eval_callable'))
+    values = [x['eval'] for x in seq]
+    if not expand:
+      if deterministic:
+        # A constant or a function applied to a constant e.g. upper("foo")
+        # ... so it becomes a constant as well ;)
+        return False, f(*values)
+      else:
+        return True, lambda *args: f(*values)
+    else:
+      return True, lambda *args: f(*(v(*args) if i in expand else v \
+              for i,v in enumerate(vals))),
+
 #------------------------------------------------------
 #   Global Testing
 #------------------------------------------------------
   from string import whitespace
   import sys
 
-  from api import GVizInvalidQuery
-  from testing.dutest import MultiTestLoader, DocTestLoader
+  from tracgviz.api import GVizInvalidQuery
+  from tracgviz.testing.dutest import MultiTestLoader, DocTestLoader
 
   def parse(expr, *attrs, **kwds):
     # Test lexical analysis
         print '\nNotSupported  :(',
       except GVizInvalidQuery, exc :
         print 
-        print exc.message
+        print unicode(exc)
       else:
         for attrnm in attrs :
           print
                                         TEST_DATA_GVIZDATA)
         except Exception, exc:
           print
-          print exc.__class__.__name__, ' : ', exc.message,
+          print exc.__class__.__name__, ' : ', unicode(exc),
         else:
           print
           print "= Columns =",
                           )])
   return l.loadTestsFromModule(sys.modules[__name__])
 
+# Imports introducing circular reference with tracgviz.api
+
+from tracgviz.api import GVizUnsupportedQueryOp, GVizInvalidQuery, \
+    GVizRuntimeError
+

trac-dev/gviz/tracgviz/scalar.py

+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+
+# Copyright 2009-2011 Olemis Lang <olemis at gmail.com>
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+r"""Scalar functions defined by Google Visualization API (version 7.0)
+
+https://developers.google.com/chart/interactive/docs/querylanguage#scalar_functions
+
+Copyright 2009-2011 Olemis Lang <olemis at gmail.com>
+Licensed under the Apache License, Version 2.0 
+"""
+__author__ = 'Olemis Lang'
+__all__ = 'year month day hour minute second millisecond quarter dayOfWeek ' \
+    'now dateDiff toDate upper lower'.split()
+
+from datetime import datetime, date
+
+def check_datatype(value, types, expected_msg=None):
+  if not isinstance(value, types):
+    msg = '%s expected but received %s' % (expected_msg, value) \
+        if expected_msg else 'Unexpected value %s' % (value,)
+    raise ValueError(msg)
+
+def year(value):
+  check_datatype(value, (datetime, date), 'Date value')
+  return value.date
+
+year.return_type = 'number'
+
+def month(value):
+  check_datatype(value, (datetime, date), 'Date value')
+  return value.month - 1
+
+month.return_type = 'number'
+
+def day(value):
+  check_datatype(value, (datetime, date), 'Date value')
+  return value.day
+
+day.return_type = 'number'
+
+def hour(value):
+  check_datatype(value, (time, date), 'Time value')
+  return value.hour
+
+hour.return_type = 'number'
+
+def minute(value):
+  check_datatype(value, (time, date), 'Time value')
+  return value.minute
+
+minute.return_type = 'number'
+
+def second(value):
+  check_datatype(value, (time, date), 'Time value')
+  return value.second
+
+second.return_type = 'number'
+
+def millisecond(value):
+  check_datatype(value, (time, date), 'Time value')
+  return value.microsecond / 1000
+
+millisecond.return_type = 'number'
+
+def quarter(value):
+  return 1 + month(value) / 3
+
+quarter.return_type = 'number'
+
+def dayOfWeek(value):
+  check_datatype(value, (datetime, date), 'Date value')
+  return value.isoweekday() % 7 + 1
+
+dayOfWeek.return_type = 'number'
+
+def dateDiff(until, since):
+  check_datatype(since, (datetime, date), 'Date value')
+  check_datatype(until, (datetime, date), 'Date value')
+  return (until - since).days
+
+dateDiff.return_type = 'number'
+
+def now(value):
+  return datetime.now()
+
+now.return_type = 'datetime'
+now.deterministic = False
+
+def toDate(value):
+  check_datatype(value, (date, datetime, int, long), 'Date value')
+  if isinstance(value, date):
+    return value
+  elif isinstance(value, datetime):
+    return value.date()
+  elif isinstance(value, (int, long)):
+    return date.fromtimestamp(value / 1000)
+  raise ValueError('Date value expected but received %s' % (value,))
+
+toDate.return_type = 'date'
+
+def upper(value):
+  check_datatype(value, basestring, 'String value')
+  return value.upper()
+
+upper.return_type = 'string'
+
+def lower(value):
+  check_datatype(value, basestring, 'String value')
+  return value.lower()
+
+lower.return_type = 'string'
+

trac-dev/gviz/tracgviz/testing/test_gvizql.py

       *****
       * Result
       *****
-      GVizInvalidQuery  :  Unable to evaluate FROM clause. 
-                    Data source doesn't contain base table employees.
+      GVizInvalidQuery  :  The from clause has been eliminated from the language.
 
       >>> parse("from `employees`", 'basetable')
       *****
       *****
       * Result
       *****
-      GVizInvalidQuery  :  Unable to evaluate FROM clause. 
-                    Data source doesn't contain base table employees.
+      GVizInvalidQuery  :  The from clause has been eliminated from the language.
 
       >>> parse("select dept , salary from emp_data", 'basetable', 'cols')
       *****
       *****
       * Result
       *****
-      GVizInvalidQuery  :  Unable to evaluate FROM clause. 
-                      Data source doesn't contain base table emp_data.
+      GVizInvalidQuery  :  The from clause has been eliminated from the language.