Commits

wlav committed 72e0d45 Merge

merge default into branch

Comments (0)

Files changed (14)

pypy/doc/release-2.3.0.rst

+=======================================
+PyPy 2.3 - XXXX TODO
+=======================================
+
+We're pleased to announce PyPy 2.3, which targets version 2.7.6 of the Python
+language. This release updates the stdlib from 2.7.3, jumping directly to 2.7.6.
+
+This release also contains several bugfixes and performance improvements. 
+
+You can download the PyPy 2.3 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project. We showed quite a bit of progress on all three projects (see below)
+and we're slowly running out of funds.
+Please consider donating more so we can finish those projects!  The three
+projects are:
+
+* Py3k (supporting Python 3.x): the release PyPy3 2.2 is imminent.
+
+* STM (software transactional memory): a preview will be released very soon,
+  as soon as we fix a few bugs
+
+* NumPy: the work done is included in the PyPy 2.2 release. More details below.
+
+.. _`Raspberry Pi Foundation`: http://www.raspberrypi.org
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy 2.2 and cpython 2.7.2`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+This release supports x86 machines running Linux 32/64, Mac OS X 64, Windows
+32, or ARM (ARMv6 or ARMv7, with VFPv3).
+
+Work on the native Windows 64 is still stalling, we would welcome a volunteer
+to handle that.
+
+.. _`pypy 2.2 and cpython 2.7.2`: http://speed.pypy.org
+
+Highlights
+==========
+
+* Our Garbage Collector is now "incremental".  It should avoid almost
+  all pauses due to a major collection taking place.  Previously, it
+  would pause the program (rarely) to walk all live objects, which
+  could take arbitrarily long if your process is using a whole lot of
+  RAM.  Now the same work is done in steps.  This should make PyPy
+  more responsive, e.g. in games.  There are still other pauses, from
+  the GC and the JIT, but they should be on the order of 5
+  milliseconds each.
+
+* The JIT counters for hot code were never reset, which meant that a
+  process running for long enough would eventually JIT-compile more
+  and more rarely executed code.  Not only is it useless to compile
+  such code, but as more compiled code means more memory used, this
+  gives the impression of a memory leak.  This has been tentatively
+  fixed by decreasing the counters from time to time.
+
+* NumPy has been split: now PyPy only contains the core module, called
+  ``_numpypy``.  The ``numpy`` module itself has been moved to
+  ``https://bitbucket.org/pypy/numpy`` and ``numpypy`` disappeared.
+  You need to install NumPy separately with a virtualenv:
+  ``pip install git+https://bitbucket.org/pypy/numpy.git``;
+  or directly:
+  ``git clone https://bitbucket.org/pypy/numpy.git``;
+  ``cd numpy``; ``pypy setup.py install``.
+
+* non-inlined calls have less overhead
+
+* Things that use ``sys.set_trace`` are now JITted (like coverage)
+
+* JSON decoding is now very fast (JSON encoding was already very fast)
+
+* various buffer copying methods experience speedups (like list-of-ints to
+  ``int[]`` buffer from cffi)
+
+* We finally wrote (hopefully) all the missing ``os.xxx()`` functions,
+  including ``os.startfile()`` on Windows and a handful of rare ones
+  on Posix.
+
+* numpy has a rudimentary C API that cooperates with ``cpyext``
+
+Cheers,
+Armin Rigo and Maciej Fijalkowski

pypy/module/cppyy/capi/loadable_capi.py

             'stdstring2stdstring'      : ([c_object],                 c_object),
         }
 
+        # size/offset are backend-specific but fixed after load
+        self.c_sizeof_farg = 0
+        self.c_offset_farg = 0
+
+
 def load_reflection_library(space):
     state = space.fromcache(State)
     if state.library is None:
         from pypy.module._cffi_backend.libraryobj import W_Library
         state.library = W_Library(space, reflection_library, rdynload.RTLD_LOCAL | rdynload.RTLD_LAZY)
+        if state.library:
+            # fix constants
+            state.c_sizeof_farg = _cdata_to_size_t(space, call_capi(space, 'function_arg_sizeof', []))
+            state.c_offset_farg = _cdata_to_size_t(space, call_capi(space, 'function_arg_typeoffset', []))
     return state.library
 
 def verify_backend(space):
     return _cdata_to_ptr(space, call_capi(space, 'allocate_function_args', [_Arg(l=size)]))
 def c_deallocate_function_args(space, cargs):
     call_capi(space, 'deallocate_function_args', [_Arg(vp=cargs)])
-@jit.elidable
 def c_function_arg_sizeof(space):
-    return _cdata_to_size_t(space, call_capi(space, 'function_arg_sizeof', []))
-@jit.elidable
+    state = space.fromcache(State)
+    return state.c_sizeof_farg
 def c_function_arg_typeoffset(space):
-    return _cdata_to_size_t(space, call_capi(space, 'function_arg_typeoffset', []))
+    state = space.fromcache(State)
+    return state.c_offset_farg
 
 # scope reflection information -----------------------------------------------
 def c_is_namespace(space, scope):
 def c_base_name(space, cppclass, base_index):
     args = [_Arg(l=cppclass.handle), _Arg(l=base_index)]
     return charp2str_free(space, call_capi(space, 'base_name', args))
-@jit.elidable_promote('2')
 def c_is_subtype(space, derived, base):
+    jit.promote(base)
     if derived == base:
         return bool(1)
     return space.bool_w(call_capi(space, 'is_subtype', [_Arg(l=derived.handle), _Arg(l=base.handle)]))
 
-@jit.elidable_promote('1,2,4')
 def _c_base_offset(space, derived_h, base_h, address, direction):
     args = [_Arg(l=derived_h), _Arg(l=base_h), _Arg(l=address), _Arg(l=direction)]
     return _cdata_to_size_t(space, call_capi(space, 'base_offset', args))

pypy/module/cppyy/converter.py

 from rpython.rlib.rarithmetic import r_singlefloat
 from rpython.rlib import jit_libffi, rfloat
 
-from pypy.module._rawffi.interp_rawffi import unpack_simple_shape
+from pypy.module._rawffi.interp_rawffi import letter2tp
 from pypy.module._rawffi.array import W_Array, W_ArrayInstance
 
 from pypy.module.cppyy import helper, capi, ffitypes
     def __getattr__(self, name):
         if name.startswith('array_'):
             typecode = name[len('array_'):]
-            arr = self.space.interp_w(W_Array, unpack_simple_shape(self.space, self.space.wrap(typecode)))
+            arr = self.space.interp_w(W_Array, letter2tp(self.space, typecode))
             setattr(self, name, arr)
             return arr
         raise AttributeError(name)
         if ptrval == 0:
             from pypy.module.cppyy import interp_cppyy
             return interp_cppyy.get_nullptr(space)
-        arr = space.interp_w(W_Array, unpack_simple_shape(space, space.wrap('P')))
+        arr = space.interp_w(W_Array, letter2tp(space, 'P'))
         return arr.fromaddress(space, ptrval, sys.maxint)
 
     def to_memory(self, space, w_obj, w_value, offset):

pypy/module/cppyy/interp_cppyy.py

     def get_returntype(self):
         return self.space.wrap(self.converter.name)
 
-    @jit.elidable_promote()
     def _get_offset(self, cppinstance):
         if cppinstance:
             assert lltype.typeOf(cppinstance.cppclass.handle) == lltype.typeOf(self.scope.handle)
-            offset = self.offset + capi.c_base_offset(self.space,
-                cppinstance.cppclass, self.scope, cppinstance.get_rawobject(), 1)
+            offset = self.offset + cppinstance.cppclass.get_base_offset(cppinstance, self.scope)
         else:
             offset = self.offset
         return offset
         self.datamembers[name] = new_dm
         return new_dm
 
-    @jit.elidable_promote()
     def dispatch(self, name, signature):
         overload = self.get_overload(name)
         sig = '(%s)' % signature
     def find_datamember(self, name):
         raise self.missing_attribute_error(name)
 
+    def get_base_offset(self, cppinstance, calling_scope):
+        assert self == cppinstance.cppclass
+        return 0
+
     def get_cppthis(self, cppinstance, calling_scope):
         assert self == cppinstance.cppclass
         return cppinstance.get_rawobject()
 
 class W_ComplexCPPClass(W_CPPClass):
 
-    def get_cppthis(self, cppinstance, calling_scope):
+    def get_base_offset(self, cppinstance, calling_scope):
         assert self == cppinstance.cppclass
         offset = capi.c_base_offset(self.space,
                                     self, calling_scope, cppinstance.get_rawobject(), 1)
+        return offset
+
+    def get_cppthis(self, cppinstance, calling_scope):
+        assert self == cppinstance.cppclass
+        offset = self.get_base_offset(cppinstance, calling_scope)
         return capi.direct_ptradd(cppinstance.get_rawobject(), offset)
 
 W_ComplexCPPClass.typedef = TypeDef(

pypy/module/micronumpy/compile.py

     def newcomplex(self, r, i):
         return ComplexObject(r, i)
 
+    def getitem(self, obj, index):
+        assert isinstance(obj, ListObject)
+        assert isinstance(index, IntObject)
+        return obj.items[index.intval]
+
     def listview(self, obj, number=-1):
         assert isinstance(obj, ListObject)
         if number != -1:

pypy/module/micronumpy/test/test_ndarray.py

         from numpypy import array, zeros
         a = array([-1.2, 3.4, 5.7, -3.0, 2.7])
         assert a.max() == 5.7
+        assert a.max().shape == ()
+        assert a.max(axis=(0,)) == 5.7
+        assert a.max(axis=(0,)).shape == ()
         assert a.max(keepdims=True) == 5.7
         assert a.max(keepdims=True).shape == (1,)
         b = array([])
         from numpypy import array, zeros
         a = array([-1.2, 3.4, 5.7, -3.0, 2.7])
         assert a.min() == -3.0
+        assert a.min().shape == ()
+        assert a.min(axis=(0,)) == -3.0
+        assert a.min(axis=(0,)).shape == ()
         assert a.min(keepdims=True) == -3.0
         assert a.min(keepdims=True).shape == (1,)
         b = array([])

pypy/module/micronumpy/test/test_ufuncs.py

 
         a = zeros((2, 2)) + 1
         assert (add.reduce(a, axis=1) == [2, 2]).all()
+        assert (add.reduce(a, axis=(1,)) == [2, 2]).all()
         exc = raises(ValueError, add.reduce, a, axis=2)
         assert exc.value[0] == "'axis' entry is out of bounds"
 

pypy/module/micronumpy/ufuncs.py

         if space.is_none(w_axis):
             axis = maxint
         else:
+            if space.isinstance_w(w_axis, space.w_tuple) and space.len_w(w_axis) == 1:
+                w_axis = space.getitem(w_axis, space.wrap(0))
             axis = space.int_w(w_axis)
             if axis < -shapelen or axis >= shapelen:
                 raise oefmt(space.w_ValueError, "'axis' entry is out of bounds")

rpython/jit/metainterp/resume.py

         self._add_pending_fields(pending_setfields)
 
         storage.rd_consts = self.memo.consts
-        dump_storage(storage, liveboxes)
         return liveboxes[:]
 
     def _number_virtuals(self, liveboxes, optimizer, num_env_virtuals):
 
     def int_add_const(self, base, offset):
         return base + offset
-
-# ____________________________________________________________
-
-def dump_storage(storage, liveboxes):
-    "For profiling only."
-    debug_start("jit-resume")
-    if have_debug_prints():
-        debug_print('Log storage', compute_unique_id(storage))
-        frameinfo = storage.rd_frame_info_list
-        while frameinfo is not None:
-            try:
-                jitcodename = frameinfo.jitcode.name
-            except AttributeError:
-                jitcodename = str(compute_unique_id(frameinfo.jitcode))
-            debug_print('\tjitcode/pc', jitcodename,
-                        frameinfo.pc,
-                        'at', compute_unique_id(frameinfo))
-            frameinfo = frameinfo.prev
-        numb = storage.rd_numb
-        while numb:
-            debug_print('\tnumb', str([untag(numb.nums[i])
-                                       for i in range(len(numb.nums))]),
-                        'at', compute_unique_id(numb))
-            numb = numb.prev
-        for const in storage.rd_consts:
-            debug_print('\tconst', const.repr_rpython())
-        for box in liveboxes:
-            if box is None:
-                debug_print('\tbox', 'None')
-            else:
-                debug_print('\tbox', box.repr_rpython())
-        if storage.rd_virtuals is not None:
-            for virtual in storage.rd_virtuals:
-                if virtual is None:
-                    debug_print('\t\t', 'None')
-                else:
-                    virtual.debug_prints()
-        if storage.rd_pendingfields:
-            debug_print('\tpending setfields')
-            for i in range(len(storage.rd_pendingfields)):
-                lldescr = storage.rd_pendingfields[i].lldescr
-                num = storage.rd_pendingfields[i].num
-                fieldnum = storage.rd_pendingfields[i].fieldnum
-                itemindex = storage.rd_pendingfields[i].itemindex
-                debug_print("\t\t", str(lldescr), str(untag(num)), str(untag(fieldnum)), itemindex)
-
-    debug_stop("jit-resume")

rpython/jit/metainterp/test/test_resume.py

         except KeyError:
             value = self.values[box] = OptValue(box)
         return value
-        
+
+
+# ____________________________________________________________
+
+def dump_storage(storage, liveboxes):
+    "For profiling only."
+    debug_start("jit-resume")
+    if have_debug_prints():
+        debug_print('Log storage', compute_unique_id(storage))
+        frameinfo = storage.rd_frame_info_list
+        while frameinfo is not None:
+            try:
+                jitcodename = frameinfo.jitcode.name
+            except AttributeError:
+                jitcodename = str(compute_unique_id(frameinfo.jitcode))
+            debug_print('\tjitcode/pc', jitcodename,
+                        frameinfo.pc,
+                        'at', compute_unique_id(frameinfo))
+            frameinfo = frameinfo.prev
+        numb = storage.rd_numb
+        while numb:
+            debug_print('\tnumb', str([untag(numb.nums[i])
+                                       for i in range(len(numb.nums))]),
+                        'at', compute_unique_id(numb))
+            numb = numb.prev
+        for const in storage.rd_consts:
+            debug_print('\tconst', const.repr_rpython())
+        for box in liveboxes:
+            if box is None:
+                debug_print('\tbox', 'None')
+            else:
+                debug_print('\tbox', box.repr_rpython())
+        if storage.rd_virtuals is not None:
+            for virtual in storage.rd_virtuals:
+                if virtual is None:
+                    debug_print('\t\t', 'None')
+                else:
+                    virtual.debug_prints()
+        if storage.rd_pendingfields:
+            debug_print('\tpending setfields')
+            for i in range(len(storage.rd_pendingfields)):
+                lldescr = storage.rd_pendingfields[i].lldescr
+                num = storage.rd_pendingfields[i].num
+                fieldnum = storage.rd_pendingfields[i].fieldnum
+                itemindex = storage.rd_pendingfields[i].itemindex
+                debug_print("\t\t", str(lldescr), str(untag(num)), str(untag(fieldnum)), itemindex)
+
+    debug_stop("jit-resume")
+
 
 def test_tag():
     assert tag(3, 1) == rffi.r_short(3<<2|1)

rpython/rlib/bitmanipulation.py

-from rpython.rlib import unroll
-
-
-class BitSplitter(dict):
-    def __getitem__(self, lengths):
-        if isinstance(lengths, int):
-            lengths = (lengths, )
-        if lengths in self:
-            return dict.__getitem__(self, lengths)
-        unrolling_lenghts = unroll.unrolling_iterable(lengths)
-        def splitbits(integer):
-            result = ()
-            sum = 0
-            for length in unrolling_lenghts:
-                sum += length
-                n = integer & ((1<<length) - 1)
-                assert n >= 0
-                result += (n, )
-                integer = integer >> length
-            assert sum <= 32
-            return result
-        splitbits.func_name += "_" + "_".join([str(i) for i in lengths])
-        self[lengths] = splitbits
-        return splitbits
-
-    def _freeze_(self):
-        # as this class is not in __builtin__, we need to explicitly tell
-        # the flow space that the object is frozen and the accesses can
-        # be constant-folded.
-        return True
-
-splitter = BitSplitter()

rpython/rlib/rzlib.py

 
 from rpython.rlib import rgc
 from rpython.rlib.rstring import StringBuilder
+from rpython.rtyper.annlowlevel import llstr
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
 from rpython.rtyper.tool import rffi_platform
 from rpython.translator.platform import platform as compiler, CompilationError
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
     """
     # Prepare the input buffer for the stream
     with lltype.scoped_alloc(rffi.CCHARP.TO, len(data)) as inbuf:
-        for i in xrange(len(data)):
-            inbuf[i] = data[i]
+        copy_string_to_raw(llstr(data), inbuf, 0, len(data))
         stream.c_next_in = rffi.cast(Bytefp, inbuf)
         rffi.setintfield(stream, 'c_avail_in', len(data))
 

rpython/rlib/test/test_bitmanipulation.py

-from rpython.rlib.bitmanipulation import splitter
-
-
-def test_simple_splitbits():
-    assert ((1, ) * 4) == splitter[8,8,8,8](0x01010101)
-    assert ((255, ) * 4) == splitter[8,8,8,8](0xFfFfFfFf)
-
-def test_fancy_splitbits():
-    assert (4,3,2,1) == splitter[8,8,8,8](0x01020304)
-    assert (1,3,7,15) == splitter[1,2,3,4](0xFfFfFfFf)
-    
-def test_format_splitbits():
-    x = 0xAA
-    assert (x & 3, ) == splitter[2](x)
- 

rpython/rlib/test/test_rsocket.py

 def test_connect_with_timeout_fail():
     s = RSocket()
     s.settimeout(0.1)
-    if sys.platform == 'win32':
-        addr = '169.254.169.254'
-    else:
-        addr = '240.240.240.240'
     with py.test.raises(SocketTimeout):
-        s.connect(INETAddress(addr, 12345))
+        s.connect(INETAddress('10.255.255.10', 12345))
     s.close()
 
 def test_connect_with_timeout_succeed():