Commits

Anonymous committed e3597a4 Merge

merged latest default in await-keyword

Comments (0)

Files changed (12)

+
 =====================================
-PyPy: Python in Python Implementation 
+Pypy fork which implements the 'await' keyword for Python for asyncronous programming.
+=====================================
+
+Extension Author: Jonathan Slenders, Mobile Vikings
+
+This extension aims to provide a more user-friendly syntax for asyncronous
+programming. It was inspired by c#, which also supports the async/await
+keywords.
+
+The idea is that 'await', just like 'yield' creates a generator, only is this
+generator flagged as a special one, which creates an async scope, and that it
+allows a return statement inside.
+
+Two methods were added to the interpreter: `sys.setawaithandler` and
+`sys.setawaitresultwrapper`, they both accept one paramter which should be a
+callable.
+The first should be used to register an I/O schedular. This function
+will receive the generator object when such an async method has been called;
+the result of this method will be returned instead.
+The second wrapper function is used to wrap the return value of the generator.
+It's impossible to return values from a generator, but the return value will
+be turned into an additional yield statement, first wrapped through this
+function, so that the I/O schedular can recognize this final yield as the
+result value.
+
+Example:
+
+::
+
+    >>>> import sys
+    >>>> def wrapper(generator):
+    ....    print 'before'
+    ....    for i in generator:
+    ....        print i
+    ....    print 'after'
+    ....    # return AsyncTask()
+    ....
+    >>>> def result_wrapper(result):
+    ....     return '< %s >' % result
+    ....
+    >>>> def g():
+    ....  a = await 1
+    ....  b = await 2
+    ....  return 'result'
+    ....
+    >>>> g()
+    <generator object g at 0x0ae9272c>
+    >>>> sys.setawaithandler(wrapper)
+    >>>> sys.setawaitresultwrapper(result_wrapper)
+    >>>> g()
+    before
+    1
+    2
+    < result >
+    after
+
+This can be used for all kinds of asyncronous frameworks where you prefer to
+have at least some kind of syntax to indicate delegation to the asyncronous I/O
+loop.
+
+For Twisted Matrix for instance, the following code:
+
+::
+
+    @defer.inlineCallbacks
+    def async_function(deferred_param):
+        a = yield deferred_param
+        b = yield some_call(a)
+        yield defer.returnValue(b)
+
+can now be written as:
+
+::
+
+    def async_function(deferred_param):
+        a = await deferred_param
+        b = await some_call(a)
+        return b
+
+
+
+
+
+=====================================
+PyPy: Python in Python Implementation
 =====================================
 
 Welcome to PyPy!

pypy/interpreter/astcompiler/ast.py

 
 
 class Yield(expr):
+    await_expr = False
 
     def __init__(self, value, lineno, col_offset):
         self.value = value
             self.value.sync_app_attrs(space)
 
 
+class Await(Yield):
+    await_expr = True
+
+
 class Compare(expr):
 
     def __init__(self, left, ops, comparators, lineno, col_offset):

pypy/interpreter/astcompiler/astbuilder.py

             return ast.Break(flow_node.lineno, flow_node.column)
         elif first_child_type == syms.continue_stmt:
             return ast.Continue(flow_node.lineno, flow_node.column)
-        elif first_child_type == syms.yield_stmt:
+        elif first_child_type in (syms.yield_stmt, syms.await_expr):
             yield_expr = self.handle_expr(first_child.children[0])
             return ast.Expr(yield_expr, flow_node.lineno, flow_node.column)
         elif first_child_type == syms.return_stmt:
             targets = []
             for i in range(0, len(stmt.children) - 2, 2):
                 target_node = stmt.children[i]
-                if target_node.type == syms.yield_expr:
+                if target_node.type in (syms.yield_expr, syms.await_expr):
                     self.error("can't assign to yield expression", target_node)
                 target_expr = self.handle_testlist(target_node)
                 self.set_context(target_expr, ast.Store)
                 else:
                     exp = None
                 return ast.Yield(exp, expr_node.lineno, expr_node.column)
+            elif expr_node_type == syms.await_expr:
+                if len(expr_node.children) == 2:
+                    exp = self.handle_testlist(expr_node.children[1])
+                else:
+                    exp = None
+                return ast.Await(exp, expr_node.lineno, expr_node.column)
             elif expr_node_type == syms.factor:
                 if len(expr_node.children) == 1:
                     expr_node = expr_node.children[0]
             if second_child.type == tokens.RPAR:
                 return ast.Tuple(None, ast.Load, atom_node.lineno,
                                  atom_node.column)
-            elif second_child.type == syms.yield_expr:
+            elif second_child.type in (syms.yield_expr, syms.await_expr):
                 return self.handle_expr(second_child)
             return self.handle_testlist_gexp(second_child)
         elif first_child_type == tokens.LSQB:

pypy/interpreter/astcompiler/codegen.py

         elif call_type == 3:
             op = ops.CALL_FUNCTION_VAR_KW
         self.emit_op_arg(op, arg)
-    
+
     def _call_has_no_star_args(self, call):
         return not call.starargs and not call.kwargs
 
             flags |= consts.CO_NESTED
         if scope.is_generator:
             flags |= consts.CO_GENERATOR
+
+            # *** Python-extension for the 'await' keyword
+            # If this is a special generator, created with the
+            # await keyword, set this bitfield.
+            if scope.is_await_generator:
+                flags |= consts.CO_AWAIT_GENERATOR
+
         if scope.has_variable_arg:
             flags |= consts.CO_VARARGS
         if scope.has_keywords_arg:

pypy/interpreter/astcompiler/consts.py

 CO_NESTED = 0x0010
 CO_GENERATOR = 0x0020
 CO_NOFREE = 0x0040
+CO_AWAIT_GENERATOR = 0x0080 # *** Python-extension for the 'await' keyword
 CO_GENERATOR_ALLOWED = 0x1000
 CO_FUTURE_DIVISION = 0x2000
 CO_FUTURE_ABSOLUTE_IMPORT = 0x4000

pypy/interpreter/astcompiler/symtable.py

         self.has_variable_arg = False
         self.has_keywords_arg = False
         self.is_generator = False
+        self.is_await_generator = False
         self.optimized = True
         self.return_with_value = False
         self.import_star = None
 
     def note_yield(self, yield_node):
         if self.return_with_value:
+            # TODO: make sure that users are not mixing 'yield' and 'await in
+            # the same function.
             raise SyntaxError("'return' with argument inside generator",
                               self.ret.lineno, self.ret.col_offset)
         self.is_generator = True
+        self.is_await_generator = yield_node.await_expr
 
     def note_return(self, ret):
         if ret.value:
-            if self.is_generator:
+            # TODO: make sure that users are not mixing 'yield' and 'await in
+            # the same function.
+            if self.is_generator and not self.is_await_generator:
                 raise SyntaxError("'return' with argument inside generator",
                                   ret.lineno, ret.col_offset)
             self.return_with_value = True

pypy/interpreter/executioncontext.py

         # that tracing occurs whenever self.w_tracefunc or self.is_tracing
         # is modified.
         self.w_tracefunc = None        # if not None, no JIT
+        self.w_await_schedular_func = None
+        self.w_await_result_wrapper_func = None
         self.is_tracing = 0
         self.compiler = space.createcompiler()
         self.profilefunc = None        # if not None, no JIT
     def gettrace(self):
         return self.w_tracefunc
 
+    # << Python extension: 'await' keyword
+    def setawaithandler(self, w_func):
+        """Set await I/O schedular func."""
+        if self.space.is_w(w_func, self.space.w_None):
+            self.w_await_schedular_func = None
+        else:
+            self.w_await_schedular_func = w_func
+
+    def setawaitresultwrapper(self, w_func):
+        """Set await I/O schedular result wrapper."""
+        if self.space.is_w(w_func, self.space.w_None):
+            self.w_await_result_wrapper_func = None
+        else:
+            self.w_await_result_wrapper_func = w_func
+
+    def getawaithandler(self):
+        return self.w_await_schedular_func
+
+    def getawaitresultwrapper(self):
+        return self.w_await_result_wrapper_func
+    # >>
+
     def setprofile(self, w_func):
         """Set the global trace function."""
         if self.space.is_w(w_func, self.space.w_None):

pypy/interpreter/function.py

             w_arg = frame.peekvalue(nargs-1-i)
             new_frame.locals_stack_w[i] = w_arg
 
-        return new_frame.run()
+        result = new_frame.run()
+        result = self._wrap_await_result(result)
+        return result
+
+    def _wrap_await_result(self, result):
+        """
+        Python-extension for the 'await' keyword. Wrap the last iteration of an
+        await-generator through executioncontext.w_await_schedular_func.
+        """
+        # Does the code object of this function contain the CO_AWAIT_GENERATOR
+        # flag? (Set by the ast.)
+        from pypy.interpreter.pycode import PyCode
+        from pypy.interpreter.astcompiler.consts import CO_GENERATOR, CO_AWAIT_GENERATOR
+        is_await_generator = (self.getcode().co_flags & CO_AWAIT_GENERATOR)
+
+        # If so, and if an I/O schedular is set, get the result through the
+        # I/O schedular function.
+        if is_await_generator:
+            context = self.space.getexecutioncontext()
+            if context.w_await_schedular_func:
+                schedular = context.w_await_schedular_func
+                assert isinstance(schedular, Function)
+                result = schedular.funccall(result)
+            else:
+                raise OperationError(self.space.w_TypeError,
+                    self.space.wrap("Calling an asyncronous function (with an 'await'-keyword) while no I/O schedular has been set")
+                )
+        return result
 
     @jit.unroll_safe
     def _flat_pycall_defaults(self, code, nargs, frame, defs_to_load):

pypy/interpreter/generator.py

+from pypy.interpreter.astcompiler.consts import CO_AWAIT_GENERATOR
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.error import OperationError
+from pypy.interpreter.function import Function
 from pypy.interpreter.pyopcode import LoopBlock
 from rpython.rlib import jit
 
             # if the frame is now marked as finished, it was RETURNed from
             if frame.frame_finished_execution:
                 self.frame = None
-                raise OperationError(space.w_StopIteration, space.w_None)
+
+                # When an w_await_result_wrapper_func has been
+                # registered. We can accept a return parameter, and
+                # append use it to wrap the last item before adding it to the list.
+                context = space.getexecutioncontext()
+                is_await_generator = (self.pycode.co_flags & CO_AWAIT_GENERATOR)
+                if is_await_generator and context.w_await_result_wrapper_func:
+                    wrapper = context.w_await_result_wrapper_func
+                    assert isinstance(wrapper, Function)
+                    return wrapper.funccall(w_result)
+                else:
+                    raise OperationError(space.w_StopIteration, space.w_None)
             else:
                 return w_result     # YIELDed
         finally:
                         break
                     # if the frame is now marked as finished, it was RETURNed from
                     if frame.frame_finished_execution:
+                        # When an w_await_result_wrapper_func has been
+                        # registered. We can accept a return parameter, and
+                        # append use it to wrap the last item before adding it to the list.
+                        context = space.getexecutioncontext()
+                        is_await_generator = (pycode.co_flags & CO_AWAIT_GENERATOR)
+                        if is_await_generator and context.w_await_result_wrapper_func:
+                            wrapper = context.w_await_result_wrapper_func
+                            assert isinstance(wrapper, Function)
+                            results.append(wrapper.funccall(w_result))
+
                         break
                     results.append(w_result)     # YIELDed
             finally:

pypy/interpreter/pyparser/data/Grammar2.7

 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | print_stmt  | del_stmt | pass_stmt | flow_stmt |
              import_stmt | global_stmt | exec_stmt | assert_stmt)
-expr_stmt: testlist (augassign (yield_expr|testlist) |
-                     ('=' (yield_expr|testlist))*)
+expr_stmt: testlist (augassign (yield_expr|await_expr|testlist) |
+                     ('=' (yield_expr|await_expr|testlist))*)
 augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
             '<<=' | '>>=' | '**=' | '//=')
 # For normal assignments, additional restrictions enforced by the interpreter
 break_stmt: 'break'
 continue_stmt: 'continue'
 return_stmt: 'return' [testlist]
-yield_stmt: yield_expr
+yield_stmt: yield_expr | await_expr
 raise_stmt: 'raise' [test [',' test [',' test]]]
 import_stmt: import_name | import_from
 import_name: 'import' dotted_as_names
 term: factor (('*'|'/'|'%'|'//') factor)*
 factor: ('+'|'-'|'~') factor | power
 power: atom trailer* ['**' factor]
-atom: ('(' [yield_expr|testlist_comp] ')' |
+atom: ('(' [yield_expr|await_expr|testlist_comp] ')' |
        '[' [listmaker] ']' |
        '{' [dictorsetmaker] '}' |
        '`' testlist1 '`' |
 classdef: 'class' NAME ['(' [testlist] ')'] ':' suite
 
 arglist: (argument ',')* (argument [',']
-                         |'*' test (',' argument)* [',' '**' test] 
+                         |'*' test (',' argument)* [',' '**' test]
                          |'**' test)
 # The reason that keywords are test nodes instead of NAME is that using NAME
 # results in an ambiguity. ast.c makes sure it's a NAME.
 encoding_decl: NAME
 
 yield_expr: 'yield' [testlist]
+
+# *** Python-extension: the 'await' keyword.
+await_expr: 'await' [testlist]

pypy/module/sys/__init__.py

         """NOT_RPYTHON""" # because parent __init__ isn't
         if space.config.translating:
             del self.__class__.interpleveldefs['pypy_getudir']
-        super(Module, self).__init__(space, w_name) 
+        super(Module, self).__init__(space, w_name)
         self.recursionlimit = 100
         self.w_default_encoder = None
         self.defaultencoding = "ascii"
         self.filesystemencoding = None
 
     interpleveldefs = {
-        '__name__'              : '(space.wrap("sys"))', 
-        '__doc__'               : '(space.wrap("PyPy sys module"))', 
+        '__name__'              : '(space.wrap("sys"))',
+        '__doc__'               : '(space.wrap("PyPy sys module"))',
 
-        'platform'              : 'space.wrap(sys.platform)', 
+        'platform'              : 'space.wrap(sys.platform)',
         'maxint'                : 'space.wrap(sys.maxint)',
         'maxsize'               : 'space.wrap(sys.maxint)',
-        'byteorder'             : 'space.wrap(sys.byteorder)', 
+        'byteorder'             : 'space.wrap(sys.byteorder)',
         'maxunicode'            : 'space.wrap(vm.MAXUNICODE)',
         'stdin'                 : 'state.getio(space).w_stdin',
         '__stdin__'             : 'state.getio(space).w_stdin',
         'stderr'                : 'state.getio(space).w_stderr',
         '__stderr__'            : 'state.getio(space).w_stderr',
         'pypy_objspaceclass'    : 'space.wrap(repr(space))',
-        #'prefix'               : # added by pypy_initial_path() when it 
+        #'prefix'               : # added by pypy_initial_path() when it
         #'exec_prefix'          : # succeeds, pointing to trunk or /usr
         'path'                  : 'state.get(space).w_path',
-        'modules'               : 'state.get(space).w_modules', 
+        'modules'               : 'state.get(space).w_modules',
         'argv'                  : 'state.get(space).w_argv',
         'py3kwarning'           : 'space.w_False',
-        'warnoptions'           : 'state.get(space).w_warnoptions', 
+        'warnoptions'           : 'state.get(space).w_warnoptions',
         'builtin_module_names'  : 'space.w_None',
         'pypy_getudir'          : 'state.pypy_getudir',    # not translated
         'pypy_find_stdlib'      : 'initpath.pypy_find_stdlib',
         'pypy_find_executable'  : 'initpath.pypy_find_executable',
         'pypy_resolvedirof'     : 'initpath.pypy_resolvedirof',
 
-        '_getframe'             : 'vm._getframe', 
-        '_current_frames'       : 'currentframes._current_frames', 
-        'setrecursionlimit'     : 'vm.setrecursionlimit', 
-        'getrecursionlimit'     : 'vm.getrecursionlimit', 
-        'setcheckinterval'      : 'vm.setcheckinterval', 
-        'getcheckinterval'      : 'vm.getcheckinterval', 
-        'exc_info'              : 'vm.exc_info', 
-        'exc_clear'             : 'vm.exc_clear', 
+        '_getframe'             : 'vm._getframe',
+        '_current_frames'       : 'currentframes._current_frames',
+        'setrecursionlimit'     : 'vm.setrecursionlimit',
+        'getrecursionlimit'     : 'vm.getrecursionlimit',
+        'setcheckinterval'      : 'vm.setcheckinterval',
+        'getcheckinterval'      : 'vm.getcheckinterval',
+        'exc_info'              : 'vm.exc_info',
+        'exc_clear'             : 'vm.exc_clear',
         'settrace'              : 'vm.settrace',
         'gettrace'              : 'vm.gettrace',
+
+        'setawaithandler'      : 'vm.setawaithandler', # Python-extension: handlers for the 'await' keywords
+        'getawaithandler'      : 'vm.getawaithandler', # Python-extension: handlers for the 'await' keywords
+        'setawaitresultwrapper' : 'vm.setawaitresultwrapper', # Python-extension: handlers for the 'await' keywords
+        'getawaitresultwrapper' : 'vm.getawaitresultwrapper', # Python-extension: handlers for the 'await' keywords
+
         'setprofile'            : 'vm.setprofile',
         'getprofile'            : 'vm.getprofile',
         'call_tracing'          : 'vm.call_tracing',
         'getsizeof'             : 'vm.getsizeof',
-        
-        'executable'            : 'space.wrap("py.py")', 
+
+        'executable'            : 'space.wrap("py.py")',
         'api_version'           : 'version.get_api_version(space)',
         'version_info'          : 'version.get_version_info(space)',
         'version'               : 'version.get_version(space)',
         '_mercurial'            : 'version.get_repo_info(space)',
         'hexversion'            : 'version.get_hexversion(space)',
 
-        'displayhook'           : 'hook.displayhook', 
-        '__displayhook__'       : 'hook.__displayhook__', 
+        'displayhook'           : 'hook.displayhook',
+        '__displayhook__'       : 'hook.__displayhook__',
         'meta_path'             : 'space.wrap([])',
         'path_hooks'            : 'space.wrap([])',
         'path_importer_cache'   : 'space.wrap({})',
         'dont_write_bytecode'   : 'space.w_False',
-        
-        'getdefaultencoding'    : 'interp_encoding.getdefaultencoding', 
+
+        'getdefaultencoding'    : 'interp_encoding.getdefaultencoding',
         'setdefaultencoding'    : 'interp_encoding.setdefaultencoding',
         'getfilesystemencoding' : 'interp_encoding.getfilesystemencoding',
 
     if sys.platform == 'win32':
         interpleveldefs['winver'] = 'version.get_winver(space)'
         interpleveldefs['getwindowsversion'] = 'vm.getwindowsversion'
-    
+
     appleveldefs = {
-        'excepthook'            : 'app.excepthook', 
-        '__excepthook__'        : 'app.excepthook', 
-        'exit'                  : 'app.exit', 
+        'excepthook'            : 'app.excepthook',
+        '__excepthook__'        : 'app.excepthook',
+        'exit'                  : 'app.exit',
         'exitfunc'              : 'app.exitfunc',
         'callstats'             : 'app.callstats',
         'copyright'             : 'app.copyright_str',
         'flags'                 : 'app.null_sysflags',
     }
 
-    def setbuiltinmodule(self, w_module, name): 
+    def setbuiltinmodule(self, w_module, name):
         w_name = self.space.wrap(name)
         w_modules = self.get('modules')
         self.space.setitem(w_modules, w_name, w_module)
 
     def getmodule(self, name):
         space = self.space
-        w_modules = self.get('modules') 
-        try: 
+        w_modules = self.get('modules')
+        try:
             return space.getitem(w_modules, space.wrap(name))
-        except OperationError, e: 
-            if not e.match(space, space.w_KeyError): 
-                raise 
-            return None 
+        except OperationError, e:
+            if not e.match(space, space.w_KeyError):
+                raise
+            return None
 
-    def setmodule(self, w_module): 
+    def setmodule(self, w_module):
         space = self.space
         w_name = self.space.getattr(w_module, space.wrap('__name__'))
         w_modules = self.get('modules')
         self.space.setitem(w_modules, w_name, w_module)
 
     def getdictvalue(self, space, attr):
-        """ specialize access to dynamic exc_* attributes. """ 
-        value = MixedModule.getdictvalue(self, space, attr) 
-        if value is not None: 
+        """ specialize access to dynamic exc_* attributes. """
+        value = MixedModule.getdictvalue(self, space, attr)
+        if value is not None:
             return value
         if attr == 'exc_type':
             operror = space.getexecutioncontext().sys_exc_info()
                 return space.w_None
             else:
                 return space.wrap(operror.get_traceback())
-        return None 
+        return None
 
     def get_w_default_encoder(self):
         if self.w_default_encoder is not None:

pypy/module/sys/vm.py

 See the debugger chapter in the library manual."""
     return space.getexecutioncontext().gettrace()
 
+# << Python extension: the 'await' keyword.
+def setawaithandler(space, w_func):
+    """ Set handler functions for generators which contain an 'await' statement """
+    space.getexecutioncontext().setawaithandler(w_func)
+
+def getawaithandler(space):
+    """ Return the 'await' handlers """
+    return space.getexecutioncontext().getawaithandler()
+
+def setawaitresultwrapper(space, w_func):
+    """ Set wrapper for the return result in 'await' generators. """
+    space.getexecutioncontext().setawaitresultwrapper(w_func)
+
+def getawaitresultwrapper(space):
+    """ Return the result wrapper """
+    return space.getexecutioncontext().getawaitresultwrapper()
+# >>
+
 def setprofile(space, w_func):
     """Set the profiling function.  It will be called on each function call
 and return.  See the profiler chapter in the library manual."""