Commits

Olemis Lang committed 2f29abc

GViz QL: Fine-grained operator tokens

  • Participants
  • Parent commits 98c842c
  • Branches gviz_ql

Comments (0)

Files changed (5)

trac-dev/gviz/TODO

 
 - GViz API QL: Parse and evaluate expressions in `select` clause.
 
+- GViz API QL: Incremental production definition for operator precedence 
+  grammar parser
+
+- GViz API QL: Augment GViz QL grammar to include SQL clauses and refactor 
+  current parsing solution.
+

trac-dev/gviz/tracgviz/grammar.py

+#!/usr/bin/env python
+
+# Copyright 2009-2011 Olemis Lang <olemis at gmail.com>
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+r"""Internal representation of operator precedence grammar used to parse
+Google Visualization API Query Language expressions.
+
+Copyright 2009-2011 Olemis Lang <olemis at gmail.com>
+Licensed under the Apache License, Version 2.0 
+"""
+__author__ = 'Olemis Lang'
+
+__all__ = '', ''
+
+from pygments.token import *
+
+from tracgviz.util.parsing import Any, EndMarker, gen_precedence, NonTerminal, \
+    OperatorPrecedenceParser as Parser
+
+# Grammar
+# =======
+#
+#  Default start state BOOL_EXPR
+#
+#  1.   VALUE -> Number
+#  2.   VALUE -> Variable
+#  3.   VALUE -> String
+#  4.   VALUE -> Date
+#  5.   VALUE -> Constant
+#  6.   VALUE -> Builtin ( )
+#  7.   VALUE -> Builtin ( SUITE )
+#  8.   VALUE -> Function ( )
+#  9.   VALUE -> Function ( SUITE )
+#  10.  SUITE -> SUITE , ARITMETIC_EXPR
+#  10a. SUITE -> ARITMETIC_EXPR
+#  13.  ARITMETIC_EXPR -> ARITMETIC_EXPR + TERM
+#  14.  ARITMETIC_EXPR -> ARITMETIC_EXPR - TERM
+#  14a. ARITMETIC_EXPR -> TERM
+#  15.  TERM -> TERM * FACTOR
+#  15.a TERM -> FACTOR
+#  16.  FACTOR -> FACTOR / SIMPLE_EXPR
+#  16a. FACTOR -> SIMPLE_EXPR
+#  17.  SIMPLE_EXPR -> ( ARITHMETIC_EXPR )
+#  17a. SIMPLE_EXPR -> VALUE
+#  18.  BOOL_VALUE -> ARITHMETIC_EXPR Operator.Comparison ARITHMETIC_EXPR
+#  19.  BOOL_VALUE -> ARITHMETIC_EXPR Operator.Word.Comparison ARITHMETIC_EXPR
+#  20.  BOOL_VALUE -> not BOOL_VALUE
+#  21.  BOOL_EXPR -> BOOL_EXPR and OR_EXPR
+#  21a. BOOL_EXPR -> OR_EXPR
+#  22.  OR_EXPR -> OR_EXPR or BOOL_VALUE
+#  22a. OR_EXPR -> BOOL_VALUE
+
+_And        = (Operator.Word.Boolean, 'and')
+_Or         = (Operator.Word.Boolean, 'or')
+_Not        = (Operator.Word.Boolean, 'not')
+_Number     = (Number, Any)
+_Var        = (Name.Variable, Any)
+_Str        = (String, Any)
+_Date       = (Literal.Date, Any)
+_Const      = (Name.Constant, Any)
+_Builtin    = (Name.Builtin, Any)
+_Function   = (Name.Function, Any)
+_Comma      = (Punctuation, ',')
+_OpenP      = (Punctuation, '(')
+_CloseP     = (Punctuation, ')')
+_Sum        = (Operator.Arithmetic, '+')
+_Multiply   = (Operator.Arithmetic, '*')
+_Subtract   = (Operator.Arithmetic, '-')
+_Divide     = (Operator.Arithmetic, '/')
+_BoolOp     = (Operator.Comparison, Any)
+_BoolWordOp = (Operator.Word.Comparison, Any)
+_EndAE      = (EndMarker, 'ARITHMETIC_EXPR')
+_EndBE      = (EndMarker, 'BOOL_EXPR')
+_EndSuite   = (EndMarker, 'SUITE')
+
+GVIZ_GRAMMAR_PRECEDENCE = {
+    CloseP : {
+        Parser.MorePrecedence : [Multiply, Add, CloseP, EndE],
+      },
+    Var : {
+        Parser.MorePrecedence : [Multiply, Add, CloseP, EndE],
+      },
+    Multiply : {
+        Parser.MorePrecedence : [Multiply, Add, CloseP, EndE],
+        Parser.LessPrecedence : [OpenP, Var],
+      },
+    Add : {
+        Parser.MorePrecedence : [Add, CloseP, EndE],
+        Parser.LessPrecedence : [Multiply, OpenP, Var],
+      },
+    OpenP : {
+        Parser.LessPrecedence : [OpenP, Var, Multiply, Add,],
+        Parser.SamePrecedence : [CloseP, ],
+      },
+    EndE : {
+        Parser.LessPrecedence : [OpenP, Var, Multiply, Add,],
+      },
+  }
+
+GVIZ_GRAMMAR_PRECEDENCE = gen_precedence(GVIZ_GRAMMAR_PRECEDENCE) 
+
+GVIZ_GRAMMAR_PRODUCTIONS = {
+    (Name, Any) : {
+        EndMarker : '6',
+      },
+    (Punctuation, ')') : {
+        (NonTerminal, Any) : {
+            (Punctuation, '(') : {
+                EndMarker: '5',
+              }
+          }
+      },
+    (NonTerminal, Any) : {
+        (Operator, '*') :{
+            (NonTerminal, Any) : {
+                EndMarker: '3',
+              }
+          },
+        (Operator, '+') :{
+            (NonTerminal, Any) : {
+                EndMarker: '1',
+              }
+          },
+      },
+  }
+

trac-dev/gviz/tracgviz/gvizql.py

       setattr(self, attr, None)
     self._last = (None, None)
 
+  OPERATOR_OVERRIDES = {
+      '*' : Operator.Arithmetic,
+      '+' : Operator.Arithmetic,
+      '-' : Operator.Arithmetic,
+      '/' : Operator.Arithmetic,
+      '<=' : Operator.Comparison,
+      '<' : Operator.Comparison,
+      '>' : Operator.Comparison,
+      '>=' : Operator.Comparison,
+      '=' : Operator.Comparison,
+      '!=' : Operator.Comparison,
+      '<>' : Operator.Comparison,
+      'is' : Operator.Word.Comparison,
+      'is not' : Operator.Word.Comparison,
+      'and' : Operator.Word.Boolean,
+      'or' : Operator.Word.Boolean,
+      'not' : Operator.Word.Boolean,
+      'contains' : Operator.Word.Comparison,
+      'starts with' : Operator.Word.Comparison,
+      'ends with' : Operator.Word.Comparison,
+      'matches' : Operator.Word.Comparison,
+      'like' : Operator.Word.Comparison,
+    }
+
   def filter(self, p, stream):
     r""" Store lookahead token.
     """
         if self._last == (Keyword.Reserved, 'select') and \
             t == (Operator, '*'):
           t = Name.Other, '*'
+        elif t[0] in Operator and t[1] in self.OPERATOR_OVERRIDES:
+          t = (self.OPERATOR_OVERRIDES[t[1]], t[1])
       elif t[1].endswith('\n'):
         t = (Whitespace, '\n')
       else:

trac-dev/gviz/tracgviz/testing/test_gvizql.py

       *****
       Token.Keyword.Reserved where
       Token.Name.Variable salary
-      Token.Operator >=
+      Token.Operator.Comparison >=
       Token.Literal.Number.Integer 600
       *****
       * Parsing
       *****
       Token.Keyword.Reserved where
       Token.Name.Variable dept
-      Token.Operator !=
+      Token.Operator.Comparison !=
       Token.Literal.String.Single 'Eng'
-      Token.Operator.Word and
+      Token.Operator.Word.Boolean and
       Token.Literal.Date date '2005-01-21'
-      Token.Operator <
+      Token.Operator.Comparison <
       Token.Name.Variable hireDate
       *****
       * Parsing
       *****
       Token.Keyword.Reserved where
       Token.Name.Variable salary
-      Token.Operator >
+      Token.Operator.Comparison >
       Token.Literal.Number.Integer 700
       *****
       * Parsing
       *****
       Token.Keyword.Reserved where
       Token.Name.Variable salary
-      Token.Operator >
+      Token.Operator.Comparison >
       Token.Literal.Number 700.35
       *****
       * Parsing
       Token.Name.Variable name
       Token.Keyword.Reserved where
       Token.Name.Variable salary
-      Token.Operator >
+      Token.Operator.Comparison >
       Token.Literal.Number.Integer 700
       *****
       * Parsing
       Token.Keyword.Reserved where
       Token.Punctuation (
       Token.Name.Variable dept
-      Token.Operator <>
+      Token.Operator.Comparison <>
       Token.Literal.String.Single 'Eng'
-      Token.Operator.Word and
+      Token.Operator.Word.Boolean and
       Token.Name.Variable isSenior
-      Token.Operator =
+      Token.Operator.Comparison =
       Token.Name.Variable true
       Token.Punctuation )
-      Token.Operator.Word or
+      Token.Operator.Word.Boolean or
       Token.Punctuation (
       Token.Name.Variable dept
-      Token.Operator =
+      Token.Operator.Comparison =
       Token.Literal.String.Single 'Sales'
       Token.Punctuation )
-      Token.Operator.Word or
+      Token.Operator.Word.Boolean or
       Token.Name.Variable seniorityStartTime
-      Token.Operator.Word is
+      Token.Operator.Word.Comparison is
       Token.Keyword.Constant null
       *****
       * Parsing
       *****
       Token.Keyword.Reserved where
       Token.Name.Variable name
-      Token.Operator.Word contains
+      Token.Operator.Word.Comparison contains
       Token.Literal.String.Single 'John'
       *****
       * Parsing
       *****
       Token.Keyword.Reserved where
       Token.Name.Variable dept
-      Token.Operator.Word starts with
+      Token.Operator.Word.Comparison starts with
       Token.Literal.String.Single 'engineering'
       *****
       * Parsing
       *****
       Token.Keyword.Reserved where
       Token.Literal.String.Single 'engineering'
-      Token.Operator.Word starts with
+      Token.Operator.Word.Comparison starts with
       Token.Name.Variable dept
       *****
       * Parsing
       *****
       Token.Keyword.Reserved where
       Token.Literal.String.Single 'cowboy'
-      Token.Operator.Word ends with
+      Token.Operator.Word.Comparison ends with
       Token.Name.Variable suffix
       *****
       * Parsing
       *****
       Token.Keyword.Reserved where
       Token.Name.Variable country
-      Token.Operator.Word matches
+      Token.Operator.Word.Comparison matches
       Token.Literal.String.Single '.*ia'
       *****
       * Parsing
       *****
       Token.Keyword.Reserved where
       Token.Name.Variable name
-      Token.Operator.Word like
+      Token.Operator.Word.Comparison like
       Token.Literal.String.Single 'fre%'
       *****
       * Parsing

trac-dev/gviz/tracgviz/util/parsing.py

 """
 __author__ = 'Olemis Lang'
 
-__all__ = 'OperatorPrecedenceParser'
+__all__ = 'OperatorPrecedenceParser', 'Any', 'EndMarker', 'gen_precedence', \
+    'NonTerminal'
 __metaclass__ = type
 
 from itertools import ifilter
         for tkn2 in v2
   )
 
-
 #------------------------------------------------------
 #   Exception classes
 #------------------------------------------------------