Commits

Philip Jenvey committed 7536507 Merge

merge py3k

Comments (0)

Files changed (9)

lib-python/2.7/xml/sax/saxutils.py

         except AttributeError:
             pass
     # wrap a binary writer with TextIOWrapper
-    class UnbufferedTextIOWrapper(io.TextIOWrapper):
-        def write(self, s):
-            super(UnbufferedTextIOWrapper, self).write(s)
-            self.flush()
-    return UnbufferedTextIOWrapper(buffer, encoding=encoding,
+    return _UnbufferedTextIOWrapper(buffer, encoding=encoding,
                                    errors='xmlcharrefreplace',
                                    newline='\n')
+# PyPy: moved this class outside the function above
+class _UnbufferedTextIOWrapper(io.TextIOWrapper):
+    def write(self, s):
+        super(_UnbufferedTextIOWrapper, self).write(s)
+        self.flush()
 
 class XMLGenerator(handler.ContentHandler):
 

pypy/interpreter/baseobjspace.py

             return buf.as_str()
 
     def str_or_None_w(self, w_obj):
-        if self.is_w(w_obj, self.w_None):
-            return None
-        return self.str_w(w_obj)
+        return None if self.is_none(w_obj) else self.str_w(w_obj)
 
     def str_w(self, w_obj):
         """

pypy/module/_ssl/interp_ssl.py

-from __future__ import with_statement
-from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
-from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.typedef import TypeDef, GetSetProperty
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+import weakref
 
+from rpython.rlib import rpoll, rsocket
 from rpython.rlib.rarithmetic import intmask
-from rpython.rlib import rpoll, rsocket
 from rpython.rlib.ropenssl import *
 from rpython.rlib.rposix import get_errno, set_errno
+from rpython.rtyper.lltypesystem import lltype, rffi
 
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
+from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.typedef import GetSetProperty, TypeDef
 from pypy.module._socket import interp_socket
 from pypy.module.exceptions import interp_exceptions
-import weakref
 
 
 ## user defined constants
 
         Mix string into the OpenSSL PRNG state.  entropy (a float) is a lower
         bound on the entropy contained in string."""
-
-        buf = rffi.str2charp(string)
-        try:
+        with rffi.scoped_str2charp(string) as buf:
             libssl_RAND_add(buf, len(string), entropy)
-        finally:
-            rffi.free_charp(buf)
 
     def RAND_status(space):
         """RAND_status() -> 0 or 1
 
-        Returns 1 if the OpenSSL PRNG has been seeded with enough data and 0 if not.
-        It is necessary to seed the PRNG with RAND_add() on some platforms before
-        using the ssl() function."""
+        Returns 1 if the OpenSSL PRNG has been seeded with enough data
+        and 0 if not.  It is necessary to seed the PRNG with RAND_add()
+        on some platforms before using the ssl() function."""
 
         res = libssl_RAND_status()
         return space.wrap(res)
         Queries the entropy gather daemon (EGD) on socket path.  Returns number
         of bytes read.  Raises socket.sslerror if connection to EGD fails or
         if it does provide enough data to seed PRNG."""
-
-        socket_path = rffi.str2charp(path)
-        try:
+        with rffi.scoped_str2charp(path) as socket_path:
             bytes = libssl_RAND_egd(socket_path)
-        finally:
-            rffi.free_charp(socket_path)
         if bytes == -1:
-            msg = "EGD connection failed or EGD did not return"
-            msg += " enough data to seed the PRNG"
-            raise ssl_error(space, msg)
+            raise ssl_error(space,
+                            "EGD connection failed or EGD did not return "
+                            "enough data to seed the PRNG")
         return space.wrap(bytes)
 
 
         of bytes written."""
         w_socket = self._get_socket(space)
 
-        sockstate = check_socket_and_wait_for_timeout(space, w_socket, True)
+        sockstate = checkwait(space, w_socket, True)
         if sockstate == SOCKET_HAS_TIMED_OUT:
             raise ssl_error(space, "The write operation timed out")
         elif sockstate == SOCKET_HAS_BEEN_CLOSED:
             err = libssl_SSL_get_error(self.ssl, num_bytes)
 
             if err == SSL_ERROR_WANT_READ:
-                sockstate = check_socket_and_wait_for_timeout(
-                    space, w_socket, False)
+                sockstate = checkwait(space, w_socket, False)
             elif err == SSL_ERROR_WANT_WRITE:
-                sockstate = check_socket_and_wait_for_timeout(
-                    space, w_socket, True)
+                sockstate = checkwait(space, w_socket, True)
             else:
                 sockstate = SOCKET_OPERATION_OK
 
 
         count = libssl_SSL_pending(self.ssl)
         if not count:
-            sockstate = check_socket_and_wait_for_timeout(
-                space, w_socket, False)
+            sockstate = checkwait(space, w_socket, False)
             if sockstate == SOCKET_HAS_TIMED_OUT:
                 raise ssl_error(space, "The read operation timed out")
             elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
-                raise ssl_error(space, "Underlying socket too large for select().")
+                raise ssl_error(space,
+                                "Underlying socket too large for select().")
             elif sockstate == SOCKET_HAS_BEEN_CLOSED:
                 if libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN:
                     if space.is_none(w_buf):
                         return space.wrapbytes('')
                     else:
                         return space.wrap(0)
-                raise ssl_error(space, "Socket closed without SSL shutdown handshake")
+                raise ssl_error(space,
+                                "Socket closed without SSL shutdown handshake")
 
         rwbuffer = None
         if not space.is_none(w_buf):
                 err = libssl_SSL_get_error(self.ssl, count)
 
                 if err == SSL_ERROR_WANT_READ:
-                    sockstate = check_socket_and_wait_for_timeout(self.space,
-                        self.w_socket, False)
+                    sockstate = checkwait(space, w_socket, False)
                 elif err == SSL_ERROR_WANT_WRITE:
-                    sockstate = check_socket_and_wait_for_timeout(self.space,
-                        self.w_socket, True)
+                    sockstate = checkwait(space, w_socket, True)
                 elif (err == SSL_ERROR_ZERO_RETURN and
                    libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN):
                     if space.is_none(w_buf):
                     sockstate = SOCKET_OPERATION_OK
 
                 if sockstate == SOCKET_HAS_TIMED_OUT:
-                    raise ssl_error(self.space, "The read operation timed out")
+                    raise ssl_error(space, "The read operation timed out")
                 elif sockstate == SOCKET_IS_NONBLOCKING:
                     break
 
                     break
 
             if count <= 0:
-                raise _ssl_seterror(self.space, self, count)
+                raise _ssl_seterror(space, self, count)
 
             result = buf.str(count)
 
         if rwbuffer is not None:
             rwbuffer.setslice(0, result)
-            return self.space.wrap(count)
+            return space.wrap(count)
         else:
             return space.wrapbytes(result)
 
             err = libssl_SSL_get_error(self.ssl, ret)
             # XXX PyErr_CheckSignals()
             if err == SSL_ERROR_WANT_READ:
-                sockstate = check_socket_and_wait_for_timeout(
-                    space, w_socket, False)
+                sockstate = checkwait(space, w_socket, False)
             elif err == SSL_ERROR_WANT_WRITE:
-                sockstate = check_socket_and_wait_for_timeout(
-                    space, w_socket, True)
+                sockstate = checkwait(space, w_socket, True)
             else:
                 sockstate = SOCKET_OPERATION_OK
             if sockstate == SOCKET_HAS_TIMED_OUT:
             elif sockstate == SOCKET_HAS_BEEN_CLOSED:
                 raise ssl_error(space, "Underlying socket has been closed.")
             elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
-                raise ssl_error(space, "Underlying socket too large for select().")
+                raise ssl_error(space,
+                                "Underlying socket too large for select().")
             elif sockstate == SOCKET_IS_NONBLOCKING:
                 break
 
             raise ssl_error(space, "Underlying socket has been closed")
 
         zeros = 0
-
         while True:
             # Disable read-ahead so that unwrap can work correctly.
             # Otherwise OpenSSL might read in too much data,
             # Possibly retry shutdown until timeout or failure
             ssl_err = libssl_SSL_get_error(self.ssl, ret)
             if ssl_err == SSL_ERROR_WANT_READ:
-                sockstate = check_socket_and_wait_for_timeout(
-                    space, w_socket, False)
+                sockstate = checkwait(space, w_socket, False)
             elif ssl_err == SSL_ERROR_WANT_WRITE:
-                sockstate = check_socket_and_wait_for_timeout(
-                    space, w_socket, True)
+                sockstate = checkwait(space, w_socket, True)
             else:
                 break
 
                 else:
                     raise ssl_error(space, "The write operation timed out")
             elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT:
-                raise ssl_error(space, "Underlying socket too large for select().")
+                raise ssl_error(space,
+                                "Underlying socket too large for select().")
             elif sockstate != SOCKET_OPERATION_OK:
                 # Retain the SSL error code
                 break
             return space.w_None
 
         name = libssl_SSL_CIPHER_get_name(current)
-        if name:
-            w_name = space.wrap(rffi.charp2str(name))
-        else:
-            w_name = space.w_None
+        w_name = space.wrap(rffi.charp2str(name)) if name else space.w_None
 
         proto = libssl_SSL_CIPHER_get_version(current)
-        if proto:
-            w_proto = space.wrap(rffi.charp2str(proto))
-        else:
-            w_proto = space.w_None
+        w_proto = space.wrap(rffi.charp2str(proto)) if proto else space.w_None
 
         bits = libssl_SSL_CIPHER_get_bits(current,
                                           lltype.nullptr(rffi.INTP.TO))
         w_bits = space.newint(bits)
-
         return space.newtuple([w_name, w_proto, w_bits])
 
     @unwrap_spec(der=bool)
     def peer_certificate(self, space, der=False):
         """peer_certificate([der=False]) -> certificate
 
-        Returns the certificate for the peer.  If no certificate was provided,
-        returns None.  If a certificate was provided, but not validated, returns
-        an empty dictionary.  Otherwise returns a dict containing information
-        about the peer certificate.
+        Returns the certificate for the peer.  If no certificate was
+        provided, returns None.  If a certificate was provided, but not
+        validated, returns an empty dictionary.  Otherwise returns a
+        dict containing information about the peer certificate.
 
-        If the optional argument is True, returns a DER-encoded copy of the
-        peer certificate, or None if no certificate was provided.  This will
-        return the certificate even if it wasn't validated."""
+        If the optional argument is True, returns a DER-encoded copy of
+        the peer certificate, or None if no certificate was provided.
+        This will return the certificate even if it wasn't validated.
+        """
         if not self.peer_cert:
             return space.w_None
 
                     raise _ssl_seterror(space, self, length)
                 try:
                     # this is actually an immutable bytes sequence
-                    return space.wrap(rffi.charpsize2str(buf_ptr[0],
-                                                         length))
+                    return space.wrap(rffi.charpsize2str(buf_ptr[0], length))
                 finally:
                     libssl_OPENSSL_free(buf_ptr[0])
         else:
                 name = libssl_sk_GENERAL_NAME_value(names, j)
                 gntype = intmask(name[0].c_type)
                 if gntype == GEN_DIRNAME:
-                    # we special-case DirName as a tuple of tuples of attributes
+                    # we special-case DirName as a tuple of tuples of
+                    # attributes
                     dirname = libssl_pypy_GENERAL_NAME_dirn(name)
                     w_t = space.newtuple([
                             space.wrap("DirName"),
                             _create_tuple_for_X509_NAME(space, dirname)
                             ])
                 elif gntype in (GEN_EMAIL, GEN_DNS, GEN_URI):
-                    # GENERAL_NAME_print() doesn't handle NULL bytes in ASN1_string
-                    # correctly, CVE-2013-4238
+                    # GENERAL_NAME_print() doesn't handle NULL bytes in
+                    # ASN1_string correctly, CVE-2013-4238
                     if gntype == GEN_EMAIL:
                         v = space.wrap("email")
                     elif gntype == GEN_DNS:
 
     sock_fd = space.int_w(space.call_method(w_sock, "fileno"))
     w_timeout = space.call_method(w_sock, "gettimeout")
-    if space.is_none(w_timeout):
-        has_timeout = False
-    else:
-        has_timeout = True
+    has_timeout = not space.is_none(w_timeout)
 
     ss.ssl = libssl_SSL_new(ctx) # new ssl struct
     libssl_SSL_set_fd(ss.ssl, sock_fd) # set the socket for SSL
     # The ACCEPT_MOVING_WRITE_BUFFER flag is necessary because the address
     # of a str object may be changed by the garbage collector.
-    libssl_SSL_set_mode(ss.ssl,
-                        SSL_MODE_AUTO_RETRY | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER)
+    libssl_SSL_set_mode(
+        ss.ssl, SSL_MODE_AUTO_RETRY | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER)
 
     if server_hostname:
         libssl_SSL_set_tlsext_host_name(ss.ssl, server_hostname);
     ss.w_socket = weakref.ref(w_sock)
     return ss
 
-def check_socket_and_wait_for_timeout(space, w_sock, writing):
+def checkwait(space, w_sock, writing):
     """If the socket has a timeout, do a select()/poll() on the socket.
     The argument writing indicates the direction.
     Returns one of the possibilities in the timeout_state enum (above)."""

pypy/module/_winreg/interp_winreg.py

                                        space.wrap(nValues[0]),
                                        space.wrap(l)])
 
-def str_or_None_w(space, w_obj):
-    if space.is_w(w_obj, space.w_None):
-        return None
-    return space.str_w(w_obj)
-
 def ConnectRegistry(space, w_machine, w_hkey):
     """key = ConnectRegistry(computer_name, key)
 
 
 The return value is the handle of the opened key.
 If the function fails, an EnvironmentError exception is raised."""
-    machine = str_or_None_w(space, w_machine)
+    machine = space.str_or_None_w(w_machine)
     hkey = hkey_w(w_hkey, space)
     with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey:
         ret = rwinreg.RegConnectRegistry(machine, hkey, rethkey)

rpython/jit/metainterp/pyjitpl.py

     def do_residual_call(self, funcbox, argboxes, descr, pc,
                          assembler_call=False,
                          assembler_call_jd=None):
-        # First build allboxes: it may need some reordering from the
-        # list provided in argboxes, depending on the order in which
-        # the arguments are expected by the function
-        #
-        allboxes = self._build_allboxes(funcbox, argboxes, descr)
-        effectinfo = descr.get_extra_info()
-        if (assembler_call or
-                effectinfo.check_forces_virtual_or_virtualizable()):
-            # residual calls require attention to keep virtualizables in-sync
-            self.metainterp.clear_exception()
-            if effectinfo.oopspecindex == EffectInfo.OS_JIT_FORCE_VIRTUAL:
-                resbox = self._do_jit_force_virtual(allboxes, descr, pc)
+        debug_start("jit-residual-call")
+        try:
+            # First build allboxes: it may need some reordering from the
+            # list provided in argboxes, depending on the order in which
+            # the arguments are expected by the function
+            #
+            allboxes = self._build_allboxes(funcbox, argboxes, descr)
+            effectinfo = descr.get_extra_info()
+            if (assembler_call or
+                    effectinfo.check_forces_virtual_or_virtualizable()):
+                # residual calls require attention to keep virtualizables in-sync
+                self.metainterp.clear_exception()
+                if effectinfo.oopspecindex == EffectInfo.OS_JIT_FORCE_VIRTUAL:
+                    resbox = self._do_jit_force_virtual(allboxes, descr, pc)
+                    if resbox is not None:
+                        return resbox
+                self.metainterp.vable_and_vrefs_before_residual_call()
+                resbox = self.metainterp.execute_and_record_varargs(
+                    rop.CALL_MAY_FORCE, allboxes, descr=descr)
+                if effectinfo.is_call_release_gil():
+                    self.metainterp.direct_call_release_gil()
+                self.metainterp.vrefs_after_residual_call()
+                vablebox = None
+                if assembler_call:
+                    vablebox = self.metainterp.direct_assembler_call(
+                        assembler_call_jd)
                 if resbox is not None:
-                    return resbox
-            self.metainterp.vable_and_vrefs_before_residual_call()
-            resbox = self.metainterp.execute_and_record_varargs(
-                rop.CALL_MAY_FORCE, allboxes, descr=descr)
-            if effectinfo.is_call_release_gil():
-                self.metainterp.direct_call_release_gil()
-            self.metainterp.vrefs_after_residual_call()
-            vablebox = None
-            if assembler_call:
-                vablebox = self.metainterp.direct_assembler_call(
-                    assembler_call_jd)
-            if resbox is not None:
-                self.make_result_of_lastop(resbox)
-            self.metainterp.vable_after_residual_call(funcbox)
-            self.metainterp.generate_guard(rop.GUARD_NOT_FORCED, None)
-            if vablebox is not None:
-                self.metainterp.history.record(rop.KEEPALIVE, [vablebox], None)
-            self.metainterp.handle_possible_exception()
-            # XXX refactor: direct_libffi_call() is a hack
-            if effectinfo.oopspecindex == effectinfo.OS_LIBFFI_CALL:
-                self.metainterp.direct_libffi_call()
-            return resbox
-        else:
-            effect = effectinfo.extraeffect
-            if effect == effectinfo.EF_LOOPINVARIANT:
-                return self.execute_varargs(rop.CALL_LOOPINVARIANT, allboxes,
-                                            descr, False, False)
-            exc = effectinfo.check_can_raise()
-            pure = effectinfo.check_is_elidable()
-            return self.execute_varargs(rop.CALL, allboxes, descr, exc, pure)
+                    self.make_result_of_lastop(resbox)
+                self.metainterp.vable_after_residual_call(funcbox)
+                self.metainterp.generate_guard(rop.GUARD_NOT_FORCED, None)
+                if vablebox is not None:
+                    self.metainterp.history.record(rop.KEEPALIVE, [vablebox], None)
+                self.metainterp.handle_possible_exception()
+                # XXX refactor: direct_libffi_call() is a hack
+                if effectinfo.oopspecindex == effectinfo.OS_LIBFFI_CALL:
+                    self.metainterp.direct_libffi_call()
+                return resbox
+            else:
+                effect = effectinfo.extraeffect
+                if effect == effectinfo.EF_LOOPINVARIANT:
+                    return self.execute_varargs(rop.CALL_LOOPINVARIANT, allboxes,
+                                                descr, False, False)
+                exc = effectinfo.check_can_raise()
+                pure = effectinfo.check_is_elidable()
+                return self.execute_varargs(rop.CALL, allboxes, descr, exc, pure)
+        finally:
+            debug_stop("jit-residual-call")
 
     def do_conditional_call(self, condbox, funcbox, argboxes, descr, pc):
         if isinstance(condbox, ConstInt) and condbox.value == 0:

rpython/jit/tool/oparser_model.py

 
     class ExtendedTreeLoop(model.TreeLoop):
 
+        def as_json(self):
+            return {
+                'comment': self.comment,
+                'name': self.name,
+                'operations': [op.as_json() for op in self.operations],
+                'inputargs': self.inputargs,
+                'last_offset': self.last_offset
+            }
+
         def getboxes(self):
             def opboxes(operations):
                 for op in operations:

rpython/tool/jitlogparser/logparser2json.py

+#!/usr/bin/env python
+""" Convert logfile (from jit-log-opt and jit-backend) to json format.
+Usage:
+
+logparser2json.py <logfile.log> <outfile.json>
+"""
+
+import os
+import sys
+import json
+from rpython.tool.jitlogparser.parser import import_log, parse_log_counts
+from rpython.tool.logparser import extract_category
+from rpython.tool.jitlogparser.storage import LoopStorage
+
+def mangle_descr(descr):
+    if descr.startswith('TargetToken('):
+        return descr[len('TargetToken('):-1]
+    if descr.startswith('<Guard'):
+        return 'bridge-' + str(int(descr[len('<Guard0x'):-1], 16))
+    if descr.startswith('<Loop'):
+        return 'entry-' + descr[len('<Loop'):-1]
+    return descr.replace(" ", '-')
+
+def create_loop_dict(loops):
+    d = {}
+    for loop in loops:
+        d[mangle_descr(loop.descr)] = loop
+    return d
+
+def main(progname, logfilename, outfilename):
+    storage = LoopStorage(extrapath=os.path.dirname(progname))
+    log, loops = import_log(logfilename)
+    parse_log_counts(extract_category(log, 'jit-backend-count'), loops)
+    storage.loops = [loop for loop in loops
+                     if not loop.descr.startswith('bridge')]
+    storage.loop_dict = create_loop_dict(loops)
+    json.dump([loop.force_asm().as_json() for loop in storage.loops],
+              open(outfilename, "w"), indent=4)
+
+if __name__ == '__main__':
+    if len(sys.argv) != 3:
+        print __doc__
+        sys.exit(1)
+    main(sys.argv[0], sys.argv[1], sys.argv[2])

rpython/tool/jitlogparser/parser.py

         if self._is_guard:
             self.guard_no = int(self.descr[len('<Guard0x'):-1], 16)
 
+    def as_json(self):
+        d = {
+            'name': self.name,
+            'args': self.args,
+            'res': self.res,
+        }
+        if self.descr is not None:
+            d['descr'] = self.descr
+        if self.bridge is not None:
+            d['bridge'] = self.bridge.as_json()
+        if self.asm is not None:
+            d['asm'] = self.asm
+        return d
+
     def setfailargs(self, failargs):
         self.failargs = failargs
 

rpython/tool/jitlogparser/test/test_storage.py

     tmppath.join("x.py").write("def f(): pass") # one code
     s = LoopStorage(str(tmppath))
     assert s.load_code(str(tmppath.join('x.py'))) == s.load_code('x.py')
+