Commits

Anonymous committed f389258

Leysin sprint work:

more tests and bug fixes on the parser package

Comments (0)

Files changed (6)

pypy/module/recparser/__init__.py

          'decode_string_literal': 'pyparser.decode_string_literal',
          'install_compiler_hook' : 'pypy.interpreter.pycompiler.install_compiler_hook',
          'insert_grammar_rule' : 'pypy.interpreter.pycompiler.insert_grammar_rule',
-         'rules' : 'pypy.interpreter.pyparser.pythonparse.grammar_rules',
-         'parse_grammar' : 'pypy.interpreter.pyparser.pythonparse.parse_grammar',
+         #'rules' : 'pypy.interpreter.pyparser.pythonparse.grammar_rules',
+         #'parse_grammar' : 'pypy.interpreter.pyparser.pythonparse.parse_grammar',
          }
 
 # Automatically exports each AST class

pypy/module/recparser/codegen.py

 
     def get_size(self):
         s = 0
-        for i in insns:
+        for i in self.insns:
             s += i.size()
         return s
 

pypy/module/recparser/compat.py

 """Compatibility layer for CPython's parser module"""
 
-from pythonparse import parse_python_source
-from pythonutil import PYTHON_PARSER
+from pypy.interpreter.pyparser.tuplebuilder import TupleBuilder
+from pythonparse import make_pyparser
+from pythonutil import pypy_parse
+import symbol # XXX use PYTHON_PARSER.symbols ?
 from compiler import transformer, compile as pycompile
 
+PYTHON_PARSER = make_pyparser()
+
 def suite( source ):
     strings = [line+'\n' for line in source.split('\n')]
-    builder = parse_python_source( strings, PYTHON_PARSER, "file_input" )
+    builder = TupleBuilder(PYTHON_PARSER)
+    PYTHON_PARSER.parse_source(source, 'exec', builder)
     nested_tuples = builder.stack[-1].as_tuple()
     if builder.source_encoding is not None:
         return (symbol.encoding_decl, nested_tuples, builder.source_encoding)
 
 def expr( source ):
     strings = [line+'\n' for line in source.split('\n')]
-    builder = parse_python_source( strings, PYTHON_PARSER, "eval_input" )
+    builder = TupleBuilder(PYTHON_PARSER)
+    PYTHON_PARSER.parse_source(source, 'eval', builder)
     nested_tuples = builder.stack[-1].as_tuple()
     if builder.source_encoding is not None:
         return (symbol.encoding_decl, nested_tuples, builder.source_encoding)

pypy/module/recparser/pyparser.py

 
     def visit_tokennode( self, node ):
         space = self.space
-        tokens = space.parser.tokens
+        tokens = space.default_compiler.parser.tokens
         num = node.name
         lineno = node.lineno
         if node.value is not None:
         else:
             if num not in ( tokens['NEWLINE'], tokens['INDENT'],
                             tokens['DEDENT'], tokens['ENDMARKER'] ):
-                val = space.parser.tok_values[num]
+                val = space.default_compiler.parser.tok_rvalues[num]
             else:
                 val = node.value or ''
         if self.line_info:
     totuple = interp2app(STType.descr_totuple),
 )
 
-def parse_python_source(space, source, goal):
-    builder = grammar.BaseGrammarBuilder(debug=False, rules=PYTHON_PARSER.rules)
+def parse_python_source(space, source, mode):
+    builder = grammar.BaseGrammarBuilder(debug=False, parser=PYTHON_PARSER)
     builder.space = space
     try:
-        PYTHON_PARSER.parse_source(source, goal, builder )
+        PYTHON_PARSER.parse_source(source, mode, builder )
         return builder.stack[-1]
     except SyntaxError, e:
         raise OperationError(space.w_SyntaxError,
 
 def suite( space, source ):
     # make the annotator life easier (don't use str.splitlines())
-    syntaxtree = parse_python_source( space, source, "file_input" )
+    syntaxtree = parse_python_source( space, source, "exec" )
     return space.wrap( STType(space, syntaxtree) )
 
 suite.unwrap_spec = [ObjSpace, str]
 
 def expr( space, source ):
     # make the annotator life easier (don't use str.splitlines())
-    syntaxtree = parse_python_source( space, source, "eval_input" )
+    syntaxtree = parse_python_source( space, source, "eval" )
     return space.wrap( STType(space, syntaxtree) )
 
 expr.unwrap_spec = [ObjSpace, str]
 
 
 def source2ast(space, source):
-    from pypy.interpreter.pyparser.pythonutil import AstBuilder, PYTHON_PARSER
-    builder = AstBuilder(parser=PYTHON_PARSER, space=space)
-    PYTHON_PARSER.parse_source(source, 'file_input', builder)
-    ast_tree = builder.rule_stack[-1]
-    return space.wrap(ast_tree)
+    from pypy.interpreter.pyparser.pythonutil import source2ast
+    return space.wrap(source2ast(source, 'exec', space=space))    
 source2ast.unwrap_spec = [ObjSpace, str]
 
 

pypy/module/recparser/test/test_dyn_grammarrules.py

+class AppTest_InsertGrammarRules:
+
+    def test_do_while(self):
+        import parser
+
+        newrules = """
+        compound_stmt: if_stmt | on_stmt | unless_stmt | dountil_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef
+        dountil_stmt: 'do' ':' suite 'until' test
+        unless_stmt: 'unless' test ':' suite
+        on_stmt: 'on' NAME '=' test ':' suite ['else' ':' suite]
+        """
+
+        def build_dountil_stmt(items):
+            """ 'do' ':' suite 'until' ':' test """
+            lineno = items[0].lineno
+            suite = items[2]
+            test = items[-1]
+            while_stmt = parser.ASTWhile(parser.ASTNot(test), suite, None, lineno)
+            return parser.ASTStmt([suite, while_stmt], lineno)
+
+        def build_unless_stmt(its):
+            """ 'unless' test ':' suite  """
+            lineno = its[0].lineno
+            return parser.ASTIf([(parser.ASTNot(its[1]), its[3])], None, lineno)
+
+        def make_assignment(var, node):
+            # XXX: consts.OP_APPLY
+            return parser.ASTAssign([parser.ASTAssName('x', 0)], node)
+
+        def build_on_stmt(items):
+            """ 'on' NAME = test ':' suite  'else' ':' suite"""
+            varname = items[1].value
+            test = items[3]
+            suite = items[5]
+            assign = make_assignment(varname, test)
+            if len(items) == 9:
+                else_ = items[-1]
+            else:
+                else_ = None
+            test = parser.ASTIf([(parser.ASTName(varname), suite)], else_, items[0].lineno)
+            return parser.ASTStmt([assign, test], items[0].lineno)
+
+        parser.insert_grammar_rule(newrules, {'dountil_stmt' : build_dountil_stmt,
+                                              'unless_stmt': build_unless_stmt,
+                                              'on_stmt' : build_on_stmt,
+                                              })
+
+        # now we should be able to use do...until and unless statements
+        d = {}
+        exec '''
+a = 0
+do:
+    a += 1
+until True
+
+b = 0
+unless a == 2: b = 3
+        ''' in d
+        assert d['a'] == 1
+        assert d['b'] == 3
+

pypy/module/recparser/test/test_parser.py

     def test_enc_minimal(self):
         import parser
         parser.suite("# -*- coding: koi8-u -*-*\ngreat()")
+        
+    def test_simple_ass_totuple(self):
+        import parser
+        parser.suite("a = 3").totuple()
+