Commits

Amaury Forgeot d'Arc committed fe5a0f5

Merge from trunk
svn merge -r79456:79701 ../trunk

Comments (0)

Files changed (163)

ctypes_configure/configure.py

 C_HEADER = """
 #include <stdio.h>
 #include <stddef.h>   /* for offsetof() */
+#include <stdint.h>   /* FreeBSD: for uint64_t */
 
 void dump(char* key, int value) {
     printf("%s: %d\\n", key, value);

lib-python/conftest.py

     RegrTest('test_augassign.py', core=True),
     RegrTest('test_base64.py'),
     RegrTest('test_bastion.py'),
-    RegrTest('test_binascii.py'),
+    RegrTest('test_binascii.py', usemodules='binascii'),
 
     RegrTest('test_binhex.py'),
 

lib-python/modified-2.5.2/distutils/msvccompiler.py

 from distutils import log
 from distutils.util import get_platform
 
-import _winreg
+try:
+    import _winreg
+except ImportError:
+    pass
+else:
+    RegOpenKeyEx = _winreg.OpenKeyEx
+    RegEnumKey = _winreg.EnumKey
+    RegEnumValue = _winreg.EnumValue
+    RegError = _winreg.error
 
-RegOpenKeyEx = _winreg.OpenKeyEx
-RegEnumKey = _winreg.EnumKey
-RegEnumValue = _winreg.EnumValue
-RegError = _winreg.error
-
-HKEYS = (_winreg.HKEY_USERS,
-         _winreg.HKEY_CURRENT_USER,
-         _winreg.HKEY_LOCAL_MACHINE,
-         _winreg.HKEY_CLASSES_ROOT)
+    HKEYS = (_winreg.HKEY_USERS,
+             _winreg.HKEY_CURRENT_USER,
+             _winreg.HKEY_LOCAL_MACHINE,
+             _winreg.HKEY_CLASSES_ROOT)
 
 VS_BASE = r"Software\Microsoft\VisualStudio\%0.1f"
 VSEXPRESS_BASE = r"Software\Microsoft\VCExpress\%0.1f"

lib-python/modified-2.5.2/distutils/tests/test_msvccompiler.py

         else:
             self.fail("could not find a suitable manifest")
 
+class MsvcCompilerSimplerTestCase(unittest.TestCase):
+    def test_import_module(self):
+        from distutils.msvccompiler import MSVCCompiler
+
 def test_suite():
     if sys.platform == 'win32':
         return unittest.makeSuite(MsvcCompilerTestCase)
     else:
-        return unittest.TestSuite([])
+        return unittest.makeSuite(MsvcCompilerSimplerTestCase)
 
 if __name__ == "__main__":
     unittest.main(defaultTest="test_suite")

lib_pypy/_locale.py

 # load the platform-specific cache made by running locale.ctc.py
 from ctypes_config_cache._locale_cache import *
 
+import __pypy__
+
 
 # Ubuntu Gusty i386 structure
 class lconv(Structure):
     ul = ''.join(ul)
     string.letters = ul
 
+@__pypy__.builtinify
 def setlocale(category, locale=None):
     "(integer,string=None) -> string. Activates/queries locale processing."
     if locale:
         groups.append(0)
     return groups
 
+@__pypy__.builtinify
 def localeconv():
     "() -> dict. Returns numeric and monetary locale-specific parameters."
 
     }
     return result
 
+@__pypy__.builtinify
 def strcoll(s1, s2):
     "string,string -> int. Compares two strings according to the locale."
 
     # Collate the strings.
     return _wcscoll(s1, s2)
 
+@__pypy__.builtinify
 def strxfrm(s):
     "string -> string. Returns a string that behaves for cmp locale-aware."
 
         _strxfrm(buf, s, n2)
     return buf.value
 
+@__pypy__.builtinify
 def getdefaultlocale():
     # TODO: Port code from CPython for Windows and Mac OS
     raise NotImplementedError()
         raise ValueError("unsupported langinfo constant")
 
 if HAS_LIBINTL:
+    @__pypy__.builtinify
     def gettext(msg):
         """gettext(msg) -> string
         Return translation of msg."""
         return _gettext(msg)
 
+    @__pypy__.builtinify
     def dgettext(domain, msg):
         """dgettext(domain, msg) -> string
         Return translation of msg in domain."""
         return _dgettext(domain, msg)
 
+    @__pypy__.builtinify
     def dcgettext(domain, msg, category):
         """dcgettext(domain, msg, category) -> string
         Return translation of msg in domain and category."""
         return _dcgettext(domain, msg, category)
 
+    @__pypy__.builtinify
     def textdomain(domain):
         """textdomain(domain) -> string
         Set the C library's textdomain to domain, returning the new domain."""
         return _textdomain(domain)
 
+    @__pypy__.builtinify
     def bindtextdomain(domain, dir):
         """bindtextdomain(domain, dir) -> string
         Bind the C library's domain to dir."""
         return dirname
 
     if HAS_BIND_TEXTDOMAIN_CODESET:
+        @__pypy__.builtinify
         def bind_textdomain_codeset(domain, codeset):
             """bind_textdomain_codeset(domain, codeset) -> string
             Bind the C library's domain to codeset."""

lib_pypy/_marshal.py

 This module contains functions that can read and write Python values in a binary format. The format is specific to Python, but independent of machine architecture issues (e.g., you can write a Python value to a file on a PC, transport the file to a Sun, and read it back there). Details of the format may change between Python versions.
 """
 
-import types
+import types, __pypy__
 from _codecs import utf_8_decode, utf_8_encode
 
 TYPE_NULL     = '0'
 
 version = 1
 
+@__pypy__.builtinify
 def dump(x, f, version=version):
     # XXX 'version' is ignored, we always dump in a version-0-compatible format
     m = _Marshaller(f.write)
     m.dump(x)
 
+@__pypy__.builtinify
 def load(f):
     um = _Unmarshaller(f.read)
     return um.load()
 
+@__pypy__.builtinify
 def dumps(x, version=version):
     # XXX 'version' is ignored, we always dump in a version-0-compatible format
     buffer = []
     m.dump(x)
     return ''.join(buffer)
 
+@__pypy__.builtinify
 def loads(s):
     um = _FastUnmarshaller(s)
     return um.load()

lib_pypy/_minimal_curses.py

 
 # ____________________________________________________________
 
+import __pypy__
+
+@__pypy__.builtinify
 def setupterm(termstr, fd):
     err = ctypes.c_int(0)
     result = clib.setupterm(termstr, fd, ctypes.byref(err))
     if result == ERR:
         raise error("setupterm() failed (err=%d)" % err.value)
 
+@__pypy__.builtinify
 def tigetstr(cap):
     result = clib.tigetstr(cap)
     if ctypes.cast(result, ctypes.c_void_p).value == ERR:
         return None
     return ctypes.cast(result, ctypes.c_char_p).value
 
+@__pypy__.builtinify
 def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0):
     result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9)
     if result is None:

lib_pypy/_pypy_interact.py

 
 
 def interactive_console(mainmodule=None):
+    # set sys.{ps1,ps2} just before invoking the interactive interpreter. This
+    # mimics what CPython does in pythonrun.c
+    if not hasattr(sys, 'ps1'):
+        sys.ps1 = '>>>> '
+    if not hasattr(sys, 'ps2'):
+        sys.ps2 = '.... '
+    #
     try:
         from _pypy_irc_topic import some_topic
         text = "And now for something completely different: ``%s''" % (
         print text
     except ImportError:
         pass
+    #
     try:
         from pyrepl.simple_interact import check
         if not check():

lib_pypy/binascii.py

+"""A pure Python implementation of binascii.
+
+Rather slow and buggy in corner cases.
+PyPy provides an RPython version too.
+"""
+
 class Error(Exception):
     pass
 
         if (c > '~' or
             c == '=' or
             (header and c == '_') or
-            (c == '.' and linelen == 0 and (inp == len(data) or
+            (c == '.' and linelen == 0 and (inp+1 == len(data) or
                                             data[inp+1] == '\n' or
                                             data[inp+1] == '\r')) or
             (not istext and (c == '\r' or c == '\n')) or

lib_pypy/cPickle.py

 
 from pickle import *
 from pickle import __doc__, __version__, format_version, compatible_formats
+import __pypy__
 
 BadPickleGet = KeyError
 UnpickleableError = PicklingError
     def getvalue(self):
         return self.__f and self.__f.getvalue()
 
+@__pypy__.builtinify
 def dump(obj, file, protocol=None):
     Pickler(file, protocol).dump(obj)
 
+@__pypy__.builtinify
 def dumps(obj, protocol=None):
     file = StringIO()
     Pickler(file, protocol).dump(obj)

lib_pypy/cmath.py

 
 # much code borrowed from mathmodule.c
 
-import math
+import math, __pypy__
 from math import e, pi
-        
+
 
 # constants
 _one = complex(1., 0.)
 
 
 
+@__pypy__.builtinify
 def phase(x):
     x = _to_complex(x)
     return math.atan2(x.imag, x.real)
 
 
+@__pypy__.builtinify
 def polar(x):
     x = _to_complex(x)
     phi = math.atan2(x.imag, x.real)
     return r, phi
 
 
+@__pypy__.builtinify
 def rect(r, phi):
     return complex(r * math.cos(phi), r * math.sin(phi))
 
 
+@__pypy__.builtinify
 def acos(x):
     """acos(x)
 
     return -(_prodi(log((x+(_i*sqrt((_one-(x*x))))))))
 
 
+@__pypy__.builtinify
 def acosh(x):
     """acosh(x)
 
     return z+z
 
 
+@__pypy__.builtinify
 def asin(x):
     """asin(x)
 
     return -(_prodi(log((sqrt_1_minus_x_sq+_prodi(x)))))
 
 
+@__pypy__.builtinify
 def asinh(x):
     """asinh(x)
 
     return z+z
 
 
+@__pypy__.builtinify
 def atan(x):
     """atan(x)
     
     return _halfi*log(((_i+x)/(_i-x)))
 
 
+@__pypy__.builtinify
 def atanh(x):
     """atanh(x)
 
     return _half*log((_one+x)/(_one-x))
 
 
+@__pypy__.builtinify
 def cos(x):
     """cos(x)
 
     return complex(real, imag)
 
 
+@__pypy__.builtinify
 def cosh(x):
     """cosh(x)
     
     return complex(real, imag)
 
 
+@__pypy__.builtinify
 def exp(x):
     """exp(x)
     
     return complex(real, imag)
 
 
+@__pypy__.builtinify
 def log(x, base=None):
     """log(x)
 
     return complex(real, imag)
 
 
+@__pypy__.builtinify
 def log10(x):
     """log10(x)
 
     return complex(real, imag)
 
 
+@__pypy__.builtinify
 def sin(x):
     """sin(x)
 
     return complex(real, imag)
 
 
+@__pypy__.builtinify
 def sinh(x):
     """sinh(x)
 
     return complex(real, imag)
 
 
+@__pypy__.builtinify
 def sqrt(x):
     """sqrt(x)
 
 _sqrt_half = sqrt(_half)
 
 
+@__pypy__.builtinify
 def tan(x):
     """tan(x)
 
     return complex(real, imag)
 
 
+@__pypy__.builtinify
 def tanh(x):
     """tanh(x)
 

lib_pypy/ctypes_support.py

     def _where_is_errno():
         return standard_c_lib.__errno_location()
 
-elif sys.platform == 'darwin':
+elif sys.platform in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9'):
     standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int)
     def _where_is_errno():
         return standard_c_lib.__error()
 """ This module provides ctypes version of cpython's grp module
 """
 
-import sys
+import sys, __pypy__
 if sys.platform == 'win32':
     raise ImportError("No grp module on Windows")
 
     return Group(res.contents.gr_name, res.contents.gr_passwd,
                  res.contents.gr_gid, mem)
 
+@__pypy__.builtinify
 def getgrgid(gid):
     res = libc.getgrgid(gid)
     if not res:
         raise KeyError(gid)
     return _group_from_gstruct(res)
 
+@__pypy__.builtinify
 def getgrnam(name):
     if not isinstance(name, str):
         raise TypeError("expected string")
         raise KeyError(name)
     return _group_from_gstruct(res)
 
+@__pypy__.builtinify
 def getgrall():
     libc.setgrent()
     lst = []

lib_pypy/itertools.py

            'ifilterfalse', 'imap', 'islice', 'izip', 'repeat', 'starmap',
            'takewhile', 'tee']
 
+import __pypy__
+
 
 class chain(object):
     """Make an iterator that returns elements from the first iterable
     def __iter__(self):
         return self
 
-    
+
+@__pypy__.builtinify
 def tee(iterable, n=2):
     """Return n independent iterators from a single iterable.
     Note : once tee() has made a split, the original iterable

lib_pypy/msvcrt.py

 from ctypes_support import standard_c_lib as _c
 from ctypes_support import get_errno
 import errno
+import __pypy__
 
 try:
     open_osfhandle = _c._open_osfhandle
 _locking.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int]
 _locking.restype = ctypes.c_int
 
+@__pypy__.builtinify
 def locking(fd, mode, nbytes):
     '''lock or unlock a number of bytes in a file.'''
     rv = _locking(fd, mode, nbytes)
 exception is raised if the entry asked for cannot be found.
 """
 
-import sys
+import sys, __pypy__
 if sys.platform == 'win32':
     raise ImportError("No pwd module on Windows")
 
 _endpwent.argtypes = None
 _endpwent.restype = None
 
+@__pypy__.builtinify
 def mkpwent(pw):
     pw = pw.contents
     return struct_passwd(pw)
 
+@__pypy__.builtinify
 def getpwuid(uid):
     """
     getpwuid(uid) -> (pw_name,pw_passwd,pw_uid,
         raise KeyError("getpwuid(): uid not found: %s" % uid)
     return mkpwent(pw)
 
+@__pypy__.builtinify
 def getpwnam(name):
     """
     getpwnam(name) -> (pw_name,pw_passwd,pw_uid,
         raise KeyError("getpwname(): name not found: %s" % name)
     return mkpwent(pw)
 
+@__pypy__.builtinify
 def getpwall():
     """
-    "getpwall() -> list_of_entries
+    getpwall() -> list_of_entries
     Return a list of all available password database entries, in arbitrary order.
     See pwd.__doc__ for more on password database entries.
     """

lib_pypy/pyexpat.py

 import ctypes
 import ctypes.util
 from ctypes import c_char_p, c_int, c_void_p, POINTER, c_char, c_wchar_p
-import sys
+import sys, __pypy__
 
 # load the platform-specific cache made by running pyexpat.ctc.py
 from ctypes_config_cache._pyexpat_cache import *
         new_parser._set_unknown_encoding_handler()
         return new_parser
 
+@__pypy__.builtinify
 def ErrorString(errno):
     return XML_ErrorString(errno)[:200]
 
+@__pypy__.builtinify
 def ParserCreate(encoding=None, namespace_separator=None, intern=None):
     if (not isinstance(encoding, str) and
         not encoding is None):

lib_pypy/pypy_test/hack___pypy__.py

+# here only to make test runs work even if not running on top of PyPy
+import sys, new
+
+def builtinify(f):
+    return f
+
+pypy = new.module('__pypy__')
+pypy.builtinify = builtinify
+sys.modules.setdefault('__pypy__', pypy)

lib_pypy/pypy_test/test_structseq.py

 from __future__ import absolute_import
 import py
-from .._structseq import *
+from .._structseq import structseqfield, structseqtype
 
 
 class mydata:

lib_pypy/readline.py

 are only stubs at the moment.
 """
 
-# Note that PyPy contains also a built-in module 'readline' which will hide
-# this one if compiled in.  However the built-in module is incomplete;
-# don't use it.
-
 from pyrepl.readline import *

lib_pypy/resource.py

-import sys
+import sys, __pypy__
 if sys.platform == 'win32':
     raise ImportError('resource module not available for win32')
 
     ru_nvcsw = _structseq.structseqfield(14)
     ru_nivcsw = _structseq.structseqfield(15)
 
+@__pypy__.builtinify
 def rlimit_check_bounds(rlim_cur, rlim_max):
     if rlim_cur > rlim_t_max:
         raise ValueError("%d does not fit into rlim_t" % rlim_cur)
         ("rlim_max", rlim_t),
     )
 
+@__pypy__.builtinify
 def getrusage(who):
     ru = _struct_rusage()
     ret = _getrusage(who, byref(ru))
         ru.ru_nivcsw,
         ))
 
+@__pypy__.builtinify
 def getrlimit(resource):
     if not(0 <= resource < RLIM_NLIMITS):
         return ValueError("invalid resource specified")
         raise error(errno)
     return (rlim.rlim_cur, rlim.rlim_max)
 
+@__pypy__.builtinify
 def setrlimit(resource, rlim):
     if not(0 <= resource < RLIM_NLIMITS):
         return ValueError("invalid resource specified")
         else:
             raise error(errno)
 
+@__pypy__.builtinify
 def getpagesize():
     pagesize = 0
     if _getpagesize:

lib_pypy/syslog.py

 syslog facility.
 """
 
-import sys
+import sys, __pypy__
 if sys.platform == 'win32':
     raise ImportError("No syslog on Windows")
 
 _setlogmask.argtypes = (c_int,)
 _setlogmask.restype = c_int
 
+@__pypy__.builtinify
 def openlog(ident, option, facility):
     _openlog(ident, option, facility)
 
+@__pypy__.builtinify
 def syslog(arg1, arg2=None):
     if arg2 is not None:
         priority, message = arg1, arg2
         priority, message = LOG_INFO, arg1
     _syslog(priority, "%s", message)
 
+@__pypy__.builtinify
 def closelog():
     _closelog()
 
+@__pypy__.builtinify
 def setlogmask(mask):
     return _setlogmask(mask)
 
+@__pypy__.builtinify
 def LOG_MASK(pri):
     return (1 << pri)
 
+@__pypy__.builtinify
 def LOG_UPTO(pri):
     return (1 << (pri + 1)) - 1
 

pypy/annotation/description.py

                     return self
         return None
 
+    def maybe_return_immutable_list(self, attr, s_result):
+        # hack: 'x.lst' where lst is listed in _immutable_fields_ as 'lst[*]'
+        # should really return an immutable list as a result.  Implemented
+        # by changing the result's annotation (but not, of course, doing an
+        # actual copy in the rtyper).  Tested in pypy.rpython.test.test_rlist,
+        # test_immutable_list_out_of_instance.
+        search = '%s[*]' % (attr,)
+        cdesc = self
+        while cdesc is not None:
+            if '_immutable_fields_' in cdesc.classdict:
+                if search in cdesc.classdict['_immutable_fields_'].value:
+                    s_result.listdef.never_resize()
+                    s_copy = s_result.listdef.offspring()
+                    s_copy.listdef.mark_as_immutable()
+                    return s_copy
+            cdesc = cdesc.basedesc
+        return s_result     # common case
+
     def consider_call_site(bookkeeper, family, descs, args, s_result):
         from pypy.annotation.model import SomeInstance, SomePBC, s_None
         if len(descs) == 1:

pypy/annotation/listdef.py

     resized = False    # True for lists resized after creation
     range_step = None  # the step -- only for lists only created by a range()
     dont_change_any_more = False   # set to True when too late for changes
-    must_not_mutate = False   # list_not_modified_any_more()
+    immutable = False  # for getattr out of _immutable_fields_ = ['attr[*]']
     must_not_resize = False   # make_sure_not_resized()
 
     # what to do if range_step is different in merge.
         if not self.mutated:
             if self.dont_change_any_more:
                 raise TooLateForChange
-            if self.must_not_mutate:
-                raise ListChangeUnallowed("mutating list")
+            self.immutable = False
             self.mutated = True
 
     def resize(self):
                     # things more general
                     self, other = other, self
 
-            if other.must_not_mutate:
-                if self.mutated:
-                    raise ListChangeUnallowed("list merge with a mutated")
-                self.must_not_mutate = True
+            self.immutable &= other.immutable
             if other.must_not_resize:
                 if self.resized:
                     raise ListChangeUnallowed("list merge with a resized")
         self.listitem.generalize(s_value)
 
     def __repr__(self):
-        return '<[%r]%s%s>' % (self.listitem.s_value,
+        return '<[%r]%s%s%s%s>' % (self.listitem.s_value,
                                self.listitem.mutated and 'm' or '',
-                               self.listitem.resized and 'r' or '')
+                               self.listitem.resized and 'r' or '',
+                               self.listitem.immutable and 'I' or '',
+                               self.listitem.must_not_resize and '!R' or '')
 
     def mutate(self):
         self.listitem.mutate()
             raise ListChangeUnallowed("list already resized")
         self.listitem.must_not_resize = True
 
-    def never_mutate(self):
+    def mark_as_immutable(self):
+        # Sets the 'immutable' flag.  Note that unlike "never resized",
+        # the immutable flag is only a hint.  It is cleared again e.g.
+        # when we merge with a "normal" list that doesn't have it.  It
+        # is thus expected to live only shortly, mostly for the case
+        # of writing 'x.list[n]'.
         self.never_resize()
-        if self.listitem.mutated:
-            raise ListChangeUnallowed("list already mutated")
-        self.listitem.must_not_mutate = True
+        if not self.listitem.mutated:
+            self.listitem.immutable = True
+        #else: it's fine, don't set immutable=True at all (see
+        #      test_can_merge_immutable_list_with_regular_list)
 
 MOST_GENERAL_LISTDEF = ListDef(None, SomeObject())
 

pypy/annotation/test/test_annrpython.py

         a.build_types(fn, [])
         # assert did not raise ListChangeUnallowed
 
-    def test_list_not_modified_any_more(self):
-        from pypy.rlib.debug import list_not_modified_any_more
-
-        def pycode(consts):
-            return list_not_modified_any_more(consts)
-        def build1():
-            return pycode(consts=[1])
-        def build2():
-            return pycode(consts=[0])
-        def fn():
-            build1()
-            build2()
+    def test_return_immutable_list(self):
+        class A:
+            _immutable_fields_ = 'lst[*]'
+        def f(n):
+            a = A()
+            l1 = [n, 0]
+            l1[1] = n+1
+            a.lst = l1
+            return a.lst
 
         a = self.RPythonAnnotator()
-        a.translator.config.translation.list_comprehension_operations = True
-        a.build_types(fn, [])
-        # assert did not raise ListChangeUnallowed
+        s = a.build_types(f, [int])
+        assert s.listdef.listitem.immutable
+
+    def test_immutable_list_is_actually_resized(self):
+        class A:
+            _immutable_fields_ = 'lst[*]'
+        def f(n):
+            a = A()
+            l1 = [n]
+            l1.append(n+1)
+            a.lst = l1
+            return a.lst
+
+        a = self.RPythonAnnotator()
+        py.test.raises(ListChangeUnallowed, a.build_types, f, [int])
+
+    def test_can_merge_immutable_list_with_regular_list(self):
+        class A:
+            _immutable_fields_ = 'lst[*]'
+        def foo(lst):
+            pass
+
+        def f(n):
+            a = A()
+            l1 = [n, 0]
+            l1[1] = n+1
+            a.lst = l1
+            if n > 0:
+                foo(a.lst)
+            else:
+                lst = [0]
+                lst[0] = n
+                foo(lst)
+
+        a = self.RPythonAnnotator()
+        a.build_types(f, [int])
+
+        def f(n):
+            a = A()
+            l1 = [n, 0]
+            l1[1] = n+1
+            a.lst = l1
+            if n > 0:
+                lst = [0]
+                lst[0] = n
+                foo(lst)
+            else:
+                foo(a.lst)
+
+        a = self.RPythonAnnotator()
+        a.build_types(f, [int])
+
 
 def g(n):
     return [0,1,2,n]

pypy/annotation/unaryop.py

                     if basedef.classdesc.all_enforced_attrs is not None:
                         if attr in basedef.classdesc.all_enforced_attrs:
                             raise HarmlesslyBlocked("get enforced attr")
+            elif isinstance(s_result, SomeList):
+                s_result = ins.classdef.classdesc.maybe_return_immutable_list(
+                    attr, s_result)
             return s_result
         return SomeObject()
     getattr.can_only_throw = []

pypy/config/pypyoption.py

      "crypt", "signal", "_rawffi", "termios", "zlib", "bz2",
      "struct", "_hashlib", "_md5", "_sha", "_minimal_curses", "cStringIO",
      "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
-     "_bisect", "_multiprocessing", '_warnings']
+     "_bisect", "binascii", "_multiprocessing", '_warnings']
 ))
 
 translation_modules = default_modules.copy()
                suggests=[("objspace.allworkingmodules", False)]),
 
     BoolOption("geninterp", "specify whether geninterp should be used",
-               cmdline=None,
                default=True),
 
     BoolOption("logbytecodes",

pypy/config/test/test_support.py

 import os, sys, py
 
 cpuinfo = """
-processor	: 0
+processor\t: 0
 
-processor	: 1
-vendor_id	: GenuineIntel
-cpu family	: 6
-model		: 37
-model name	: Intel(R) Core(TM) i7 CPU       L 620  @ 2.00GHz
-stepping	: 2
+processor\t: 1
+vendor_id\t: GenuineIntel
+cpu family\t: 6
+model\t\t: 37
+model name\t: Intel(R) Core(TM) i7 CPU       L 620  @ 2.00GHz
+stepping\t: 2
 
-processor	: 2
-vendor_id	: GenuineIntel
-cpu family	: 6
-model		: 37
-model name	: Intel(R) Core(TM) i7 CPU       L 620  @ 2.00GHz
-stepping	: 2
+processor\t: 2
+vendor_id\t: GenuineIntel
+cpu family\t: 6
+model\t\t: 37
+model name\t: Intel(R) Core(TM) i7 CPU       L 620  @ 2.00GHz
+stepping\t: 2
 
-processor	: 3
-vendor_id	: GenuineIntel
-cpu family	: 6
-model		: 37
-model name	: Intel(R) Core(TM) i7 CPU       L 620  @ 2.00GHz
-stepping	: 2
-cpu MHz		: 1199.000
-cache size	: 4096 KB
-physical id	: 0
-siblings	: 4
+processor\t: 3
+vendor_id\t: GenuineIntel
+cpu family\t: 6
+model\t\t: 37
+model name\t: Intel(R) Core(TM) i7 CPU       L 620  @ 2.00GHz
+stepping\t: 2
+cpu MHz\t\t: 1199.000
+cache size\t: 4096 KB
+physical id\t: 0
+siblings\t: 4
 """
 
 class FakeEnviron:
         self.runtest_finish()
 
     def runtest_open(self):
-        leakfinder.start_tracking_allocations()
+        if not getattr(self.obj, 'dont_track_allocations', False):
+            leakfinder.start_tracking_allocations()
 
     def runtest_perform(self):
         super(PyPyTestFunction, self).runtest()
 
     def runtest_close(self):
-        if leakfinder.TRACK_ALLOCATIONS:
+        if (not getattr(self.obj, 'dont_track_allocations', False)
+            and leakfinder.TRACK_ALLOCATIONS):
             self._pypytest_leaks = leakfinder.stop_tracking_allocations(False)
         else:            # stop_tracking_allocations() already called
             self._pypytest_leaks = None

pypy/doc/config/objspace.usemodules.binascii.txt

+Use the RPython 'binascii' module.

pypy/doc/config/translation.jit_debug.txt

-Choose the level of debugging output in the JIT.
-'off' means none, other values give increasingly more.

pypy/doc/cpython_differences.txt

 
     __builtin__
     `__pypy__`_
+    _ast
+    _bisect
     _codecs
     _lsprof
     `_minimal_curses`_
     _random
     `_rawffi`_
+    _ssl
     _socket
     _sre
     _weakref
+    array
     bz2
     cStringIO
+    `cpyext`_
     crypt
     errno
     exceptions
 
 * Supported by being rewritten in pure Python (possibly using ``ctypes``):
   see the `lib_pypy/`_ directory.  Examples of modules that we
-  support this way: ``ctypes``, ``array``, ``cPickle``,
+  support this way: ``ctypes``, ``cPickle``,
   ``cStringIO``, ``cmath``, ``dbm`` (?), ``datetime``, ``binascii``...  
   Note that some modules are both in there and in the list above;
   by default, the built-in module is used (but can be disabled
 
 The extension modules (i.e. modules written in C, in the standard CPython)
 that are neither mentioned above nor in `lib_pypy/`_ are not available in PyPy.
+(You may have a chance to use them anyway with `cpyext`_.)
 
 .. the nonstandard modules are listed below...
 .. _`__pypy__`: __pypy__-module.html
 .. _`_rawffi`: ctypes-implementation.html
 .. _`_minimal_curses`: config/objspace.usemodules._minimal_curses.html
+.. _`cpyext`: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html
 .. _Stackless: stackless.html
 
 
 .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-1.html
 .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-2.html
 
-The built-in function ``id()`` returns numbers that are not addresses
-for most of PyPy's garbage collectors.
-This is most visible in the default repr: a typical PyPy object can
-pretend to be located ``at 0x00000009``.  This is just its ``id()``, not
-its real address (because an object can move around in some GCs). Calling
-``id`` a lot can lead to performance problem.
+Using the default GC called ``minimark``, the built-in function ``id()``
+works like it does in CPython.  With other GCs it returns numbers that
+are not real addresses (because an object can move around several times)
+and calling it a lot can lead to performance problem.
 
 Note that if you have a long chain of objects, each with a reference to
 the next one, and each with a __del__, PyPy's GC will perform badly.  On
 
 There is also an experimental support for CPython extension modules, so
 they'll run without change (from current observation, rather with little
-change) on trunk. It has been a part of 1.3 release, but support is still
+change) on trunk. It has been a part of 1.4 release, but support is still
 in alpha phase.
 
 .. _`extension modules`: cpython_differences.html#extension-modules

pypy/doc/index.txt

 Getting into PyPy ... 
 =============================================
 
-* `Release 1.3`_: the latest official release
+* `Release 1.4`_: the latest official release
 
 * `PyPy Blog`_: news and status info about PyPy 
 
 .. _`Documentation`: docindex.html 
 .. _`Getting Started`: getting-started.html
 .. _papers: extradoc.html
-.. _`Release 1.3`: http://pypy.org/download.html
+.. _`Release 1.4`: http://pypy.org/download.html

pypy/doc/release-1.4.0.txt

 PyPy 1.4: Ouroboros in practice
 ===============================
 
-Hello.
-
 We're pleased to announce the 1.4 release of PyPy. This is a major breakthrough
 in our long journey, as PyPy 1.4 is the first PyPy release that can translate
-itself faster than CPython. Starting today, we plan to start using PyPy for our
-own development.
+itself faster than CPython.  Starting today, we are using PyPy more for
+our every-day development.  So may you :) You can download it here:
 
-Among other features, this release includes numerous performance improvements
+    http://pypy.org/download.html
+
+What is PyPy
+============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement
+for CPython. It's fast (`pypy 1.4 and cpython 2.6`_ comparison)
+
+Among its new features, this release includes numerous performance improvements
 (which made fast self-hosting possible), a 64-bit JIT backend, as well
 as serious stabilization. As of now, we can consider the 32-bit and 64-bit
-linux versions of PyPy stable enoughto run in production.
+linux versions of PyPy stable enough to run `in production`_.
+
+Numerous speed achievements are described on `our blog`_. Normalized speed
+charts comparing `pypy 1.4 and pypy 1.3`_ as well as `pypy 1.4 and cpython 2.6`_
+are available on benchmark website. For the impatient: yes, we got a lot faster!
 
 More highlights
 ===============
 
-* Virtualenv support: now PyPy is fully compatible with virtualenv_: note that
+* PyPy's built-in Just-in-Time compiler is fully transparent and
+  automatically generated; it now also has very reasonable memory
+  requirements.  The total memory used by a very complex and
+  long-running process (translating PyPy itself) is within 1.5x to
+  at most 2x the memory needed by CPython, for a speed-up of 2x.
+
+* More compact instances.  All instances are as compact as if
+  they had ``__slots__``.  This can give programs a big gain in
+  memory.  (In the example of translation above, we already have
+  carefully placed ``__slots__``, so there is no extra win.)
+
+* `Virtualenv support`_: now PyPy is fully compatible with virtualenv_: note that
   to use it, you need a recent version of virtualenv (>= 1.5).
 
 * Faster (and JITted) regular expressions - huge boost in speeding up
-  sre module.
+  the `re` module.
 
-* Faster (and JITted) calls to functions like map().
+* Other speed improvements, like JITted calls to functions like map().
 
 .. _virtualenv: http://pypi.python.org/pypi/virtualenv
+.. _`Virtualenv support`: http://morepypy.blogspot.com/2010/08/using-virtualenv-with-pypy.html
+.. _`in production`: http://morepypy.blogspot.com/2010/11/running-large-radio-telescope-software.html
+.. _`our blog`: http://morepypy.blogspot.com
+.. _`pypy 1.4 and pypy 1.3`: http://speed.pypy.org/comparison/?exe=1%2B41,1%2B172&ben=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20&env=1&hor=false&bas=1%2B41&chart=normal+bars
+.. _`pypy 1.4 and cpython 2.6`: http://speed.pypy.org/comparison/?exe=2%2B35,1%2B172&ben=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20&env=1&hor=false&bas=2%2B35&chart=normal+bars
+
+Cheers,
+
+Carl Friedrich Bolz, Antonio Cuni, Maciej Fijalkowski,
+Amaury Forgeot d'Arc, Armin Rigo and the PyPy team

pypy/doc/sprint-reports.txt

   * `D�sseldorf (October 2006)`_
   * `Leysin (January 2007)`_
   * `Hildesheim (Feb 2007)`_ (also `EU report writing sprint`_)
+  * `G�teborg (November 2007)`_
+  * `Leysin (January 2008)`_
+  * `Berlin (May 2008)`_
+  * `Vilnius after EuroPython (July 2008)`_
+  * `D�sseldorf (August 2008)`_
+  * `Wroclaw (February 2009)`_
+  * `Leysin (April 2009)`_
+  * `G�teborg (August 2009)`_
+  * `D�sseldorf (November 2009)`_
+  * `CERN (July 2010)`_
+  * `D�sseldorf (October 2010)`_
 
     .. _Hildesheim (Feb 2003): http://codespeak.net/pypy/extradoc/sprintinfo/HildesheimReport.html
     .. _Gothenburg (May 2003): http://codespeak.net/pypy/extradoc/sprintinfo/gothenburg-2003-sprintreport.txt
     .. _Hildesheim (Feb 2007): http://codespeak.net/pypy/extradoc/sprintinfo/trillke-2007/sprint-report.txt
     .. _`EU report writing sprint`: http://codespeak.net/pypy/extradoc/sprintinfo/trillke-2007/eu-report-sprint-report.txt
     .. _`PyCon/Dallas (Feb 2006)`: http://codespeak.net/pypy/extradoc/sprintinfo/pycon06/sprint-report.txt
+    .. _`G�teborg (November 2007)`: http://morepypy.blogspot.com/2007_11_01_archive.html
+    .. _`Leysin (January 2008)`: http://morepypy.blogspot.com/2008/01/leysin-winter-sport-sprint-started.html
+    .. _`Berlin (May 2008)`: http://morepypy.blogspot.com/2008_05_01_archive.html
+    .. _`Vilnius after EuroPython (July 2008)`: http://morepypy.blogspot.com/2008/07/europython-2008-pypy-talks-and-sprint.html
+    .. _`D�sseldorf (August 2008)`: http://morepypy.blogspot.com/2008_10_01_archive.html
+    .. _`Wroclaw (February 2009)`: http://morepypy.blogspot.com/2009/02/wroclaw-2009-sprint-progress-report.html
+    .. _`Leysin (April 2009)`: http://morepypy.blogspot.com/2009/04/leysin-sprint-report.html
+    .. _`G�teborg (August 2009)`: http://morepypy.blogspot.com/2009/08/gothenburg-jit-sprint-report.html
+    .. _`D�sseldorf (November 2009)`: http://morepypy.blogspot.com/2009/11/dusseldorf-sprint-report.html
+    .. _`CERN (July 2010)`: http://morepypy.blogspot.com/2010/07/cern-sprint-report-wrapping-c-libraries.html
+    .. _`D�sseldorf (October 2010)`: http://morepypy.blogspot.com/2010/10/dusseldorf-sprint-report-2010.html
+

pypy/doc/statistic/release_dates.dat

 2006-06-25,"PyPy 0.9"
 2007-02-17,"PyPy 0.99"
 2007-03-27,"PyPy 1.0"
+2009-04-28,"PyPy 1.1"
+2010-03-12,"PyPy 1.2"
+2010-06-26,"PyPy 1.3"
+2010-11-26,"PyPy 1.4"

pypy/doc/statistic/sprint_dates.dat

 PyPy sprints
 location, begin, end
 "Hildesheim",2003-02-17,2003-02-23
-"Gothenburg",2003-05-24,2003-05-31
-"LovainLaNeuve",2003-06-21,2003-06-24
+"Göteborg",2003-05-24,2003-05-31
+"Louvain-la-Neuve",2003-06-21,2003-06-24
 "Berlin",2003-09-29,2003-10-04
 "Amsterdam",2003-12-14,2003-12-21
-"Europython/Gothenburg",2004-06-01,2004-06-07
+"Europython/Göteborg",2004-06-01,2004-06-07
 "Vilnius",2004-11-15,2004-11-23
 "Leysin",2005-01-22,2005-01-29
 "PyCon/Washington",2005-03-19,2005-03-22
-"Europython/Gothenburg",2005-07-01,2005-07-07
+"Europython/Göteborg",2005-07-01,2005-07-07
 "Hildesheim",2005-07-25,2005-07-31
 "Heidelberg",2005-08-22,2005-08-29
 "Paris",2005-10-10,2005-10-16
-"Gothenburg",2005-12-05,2005-12-11
+"Göteborg",2005-12-05,2005-12-11
 "Mallorca",2006-01-23,2006-01-29
 "Pycon/Dallas",2006-02-27,2006-03-02
 "Louvain-la-Neuve",2006-03-06,2006-03-10
 "Japan",2006-04-23,2006-04-29
-"Duesseldorf",2006-06-02,2006-06-09
+"Düsseldorf",2006-06-02,2006-06-09
 "Europython/Genf",2006-07-06,2006-07-09
 "Limerick",2006-08-21,2006-08-27
-"Duesseldorf",2006-10-30,2006-11-05
+"Düsseldorf",2006-10-30,2006-11-05
 "Leysin",2007-01-08,2007-01-14
 "Hildesheim",2007-03-01,2007-03-05
+"Hildesheim",2007-03-18,2007-03-23
+"Bern",2007-10-22,2007-10-26
+"Göteborg",2007-11-19,2007-11-25
+"Leysin",2008-01-12,2008-01-19
+"Berlin",2008-05-17,2008-05-22
+"EuroPython/Vilnius",2008-07-10,2008-07-12
+"Düsseldorf",2008-08-05,2008-08-13
+"Wroclaw",2009-02-07,2009-02-14
+"Leysin",2009-04-14,2009-04-21
+"Göteborg",2009-08-18,2009-08-25
+"Düsseldorf",2009-11-06,2009-11-13
+"CERN",2010-07-05,2010-07-09
+"Düsseldorf",2010-10-25,2010-10-31

pypy/interpreter/function.py

         self.w_func_dict = func.w_func_dict
         self.w_module = func.w_module
 
-    def descr_builtinfunction__new__(space, w_subtype, w_func):
-        func = space.interp_w(Function, w_func)
-        bltin = space.allocate_instance(BuiltinFunction, w_subtype)
-        BuiltinFunction.__init__(bltin, func)
-        return space.wrap(bltin)
+    def descr_builtinfunction__new__(space, w_subtype):
+        raise OperationError(space.w_TypeError,
+                     space.wrap("cannot create 'builtin_function' instances"))
 
     def descr_function_repr(self):
         return self.space.wrap('<built-in function %s>' % (self.name,))

pypy/interpreter/gateway.py

             # these decorators are known to return the same function
             # object, we may ignore them
             assert '\n' in source
-            source = source[source.find('\n') + 1:]
+            source = source[source.find('\n') + 1:].lstrip()
         assert source.startswith("def "), "can only transform functions" 
         source = source[4:]
     p = source.find('(')

pypy/interpreter/pycode.py

     CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED,
     CO_GENERATOR, CO_CONTAINSGLOBALS)
 from pypy.rlib.rarithmetic import intmask
-from pypy.rlib.debug import make_sure_not_resized, list_not_modified_any_more
+from pypy.rlib.debug import make_sure_not_resized
 from pypy.rlib import jit
 from pypy.rlib.objectmodel import compute_hash
 from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT
         self.co_stacksize = stacksize
         self.co_flags = flags
         self.co_code = code
-        self.co_consts_w = list_not_modified_any_more(consts)
+        self.co_consts_w = consts
         self.co_names_w = [space.new_interned_str(aname) for aname in names]
         self.co_varnames = varnames
         self.co_freevars = freevars
         dis.dis(co)
 
     def fget_co_consts(space, self):
-        return space.newtuple(self.co_consts_w[:])
+        return space.newtuple(self.co_consts_w)
     
     def fget_co_names(space, self):
         return space.newtuple(self.co_names_w)
             w(self.co_stacksize), 
             w(self.co_flags),
             w(self.co_code), 
-            space.newtuple(self.co_consts_w[:]), 
+            space.newtuple(self.co_consts_w), 
             space.newtuple(self.co_names_w), 
             space.newtuple([w(v) for v in self.co_varnames]), 
             w(self.co_filename),

pypy/interpreter/test/test_gateway.py

         w_res = space.call_args(w_g, args)
         assert space.eq_w(w_res, space.wrap((-1, 0)))
 
+class AppTestPyTestMark:
+    @py.test.mark.unlikely_to_exist
+    def test_anything(self):
+        pass
+
 
 class TestPassThroughArguments:
     

pypy/jit/backend/conftest.py

 """
 import py, random
 
+option = py.test.config.option
+
 def pytest_addoption(parser):
     group = parser.getgroup('random test options')
     group.addoption('--random-seed', action="store", type="int",

pypy/jit/backend/detect_cpu.py

                 'i86pc': 'x86',    # Solaris/Intel
                 'x86':   'x86',    # Apple
                 'Power Macintosh': 'ppc',
-                'x86_64': 'x86', 
+                'x86_64': 'x86',
+                'amd64': 'x86'     # freebsd
                 }[mach]
     except KeyError:
         return mach

pypy/jit/backend/llsupport/asmmemmgr.py

+import sys
+from pypy.rlib.rarithmetic import intmask, r_uint, LONG_BIT
+from pypy.rlib.objectmodel import we_are_translated
+from pypy.rlib import rmmap
+from pypy.rpython.lltypesystem import lltype, llmemory, rffi
+
+
+class AsmMemoryManager(object):
+    LARGE_ALLOC_SIZE = 1024 * 1024   # 1MB
+    MIN_FRAGMENT = 64
+    NUM_INDICES = 32     # good for all sizes between 64 bytes and ~490 KB
+    _allocated = None
+
+    def __init__(self, large_alloc_size = LARGE_ALLOC_SIZE,
+                       min_fragment     = MIN_FRAGMENT,
+                       num_indices      = NUM_INDICES):
+        self.total_memory_allocated = r_uint(0)
+        self.total_mallocs = r_uint(0)
+        self.large_alloc_size = large_alloc_size
+        self.min_fragment = min_fragment
+        self.num_indices = num_indices
+        self.free_blocks = {}      # map {start: stop}
+        self.free_blocks_end = {}  # map {stop: start}
+        self.blocks_by_size = [[] for i in range(self.num_indices)]
+
+    def malloc(self, minsize, maxsize):
+        """Allocate executable memory, between minsize and maxsize bytes,
+        and return a pair (start, stop).  Does not perform any rounding
+        of minsize and maxsize.
+        """
+        result = self._allocate_block(minsize)
+        (start, stop) = result
+        smaller_stop = start + maxsize
+        if smaller_stop + self.min_fragment <= stop:
+            self._add_free_block(smaller_stop, stop)
+            stop = smaller_stop
+            result = (start, stop)
+        self.total_mallocs += stop - start
+        return result   # pair (start, stop)
+
+    def free(self, start, stop):
+        """Free a block (start, stop) returned by a previous malloc()."""
+        self.total_mallocs -= (stop - start)
+        self._add_free_block(start, stop)
+
+    def _allocate_large_block(self, minsize):
+        # Compute 'size' from 'minsize': it must be rounded up to
+        # 'large_alloc_size'.  Additionally, we use the following line
+        # to limit how many mmap() requests the OS will see in total:
+        minsize = max(minsize, intmask(self.total_memory_allocated >> 4))
+        size = minsize + self.large_alloc_size - 1
+        size = (size // self.large_alloc_size) * self.large_alloc_size
+        data = rmmap.alloc(size)
+        if not we_are_translated():
+            if self._allocated is None:
+                self._allocated = []
+            self._allocated.append((data, size))
+            if sys.maxint > 2147483647:
+                # Hack to make sure that mcs are not within 32-bits of one
+                # another for testing purposes
+                rmmap.hint.pos += 0x80000000 - size
+        self.total_memory_allocated += size
+        data = rffi.cast(lltype.Signed, data)
+        return self._add_free_block(data, data + size)
+
+    def _get_index(self, length):
+        i = 0
+        while length > self.min_fragment:
+            length = (length * 3) >> 2
+            i += 1
+            if i == self.num_indices - 1:
+                break
+        return i
+
+    def _add_free_block(self, start, stop):
+        # Merge with the block on the left
+        if start in self.free_blocks_end:
+            left_start = self.free_blocks_end[start]
+            self._del_free_block(left_start, start)
+            start = left_start
+        # Merge with the block on the right
+        if stop in self.free_blocks:
+            right_stop = self.free_blocks[stop]
+            self._del_free_block(stop, right_stop)
+            stop = right_stop
+        # Add it to the dicts
+        self.free_blocks[start] = stop
+        self.free_blocks_end[stop] = start
+        i = self._get_index(stop - start)
+        self.blocks_by_size[i].append(start)
+        return start
+
+    def _del_free_block(self, start, stop):
+        del self.free_blocks[start]
+        del self.free_blocks_end[stop]
+        i = self._get_index(stop - start)
+        self.blocks_by_size[i].remove(start)
+
+    def _allocate_block(self, length):
+        # First look in the group of index i0 if there is a block that is
+        # big enough.  Following an idea found in the Linux malloc.c, we
+        # prefer the oldest entries rather than the newest one, to let
+        # them have enough time to coalesce into bigger blocks.  It makes
+        # a big difference on the purely random test (30% of total usage).
+        i0 = self._get_index(length)
+        bbs = self.blocks_by_size[i0]
+        for j in range(len(bbs)):
+            start = bbs[j]
+            stop = self.free_blocks[start]
+            if start + length <= stop:
+                del bbs[j]
+                break   # found a block big enough
+        else:
+            # Then look in the larger groups
+            i = i0 + 1
+            while i < self.num_indices:
+                if len(self.blocks_by_size[i]) > 0:
+                    # any block found in a larger group is big enough
+                    start = self.blocks_by_size[i].pop(0)
+                    stop = self.free_blocks[start]
+                    break
+                i += 1
+            else:
+                # Exhausted the memory.  Allocate the resulting block.
+                start = self._allocate_large_block(length)
+                stop = self.free_blocks[start]
+                i = self._get_index(stop - start)
+                assert self.blocks_by_size[i][-1] == start
+                self.blocks_by_size[i].pop()
+        #
+        del self.free_blocks[start]
+        del self.free_blocks_end[stop]
+        return (start, stop)
+
+    def _delete(self):
+        "NOT_RPYTHON"
+        if self._allocated:
+            for data, size in self._allocated:
+                rmmap.free(data, size)
+        self._allocated = None
+
+
+class BlockBuilderMixin(object):
+    _mixin_ = True
+    # A base class to generate assembler.  It is equivalent to just a list
+    # of chars, but it is potentially more efficient for that usage.
+    # It works by allocating the assembler SUBBLOCK_SIZE bytes at a time.
+    # Ideally, this number should be a power of two that fits the GC's most
+    # compact allocation scheme (which is so far 35 * WORD for minimark.py).
+    WORD = LONG_BIT // 8
+    SUBBLOCK_SIZE = 32 * WORD
+    SUBBLOCK_PTR = lltype.Ptr(lltype.GcForwardReference())
+    SUBBLOCK = lltype.GcStruct('SUBBLOCK',
+                   ('prev', SUBBLOCK_PTR),
+                   ('data', lltype.FixedSizeArray(lltype.Char, SUBBLOCK_SIZE)))
+    SUBBLOCK_PTR.TO.become(SUBBLOCK)
+
+    gcroot_markers = None
+    gcroot_markers_total_size = 0
+
+    def __init__(self, translated=None):
+        if translated is None:
+            translated = we_are_translated()
+        if translated:
+            self.init_block_builder()
+        else:
+            self._become_a_plain_block_builder()
+
+    def init_block_builder(self):
+        self._cursubblock = lltype.nullptr(self.SUBBLOCK)
+        self._baserelpos = -self.SUBBLOCK_SIZE
+        self._make_new_subblock()
+
+    def _make_new_subblock(self):
+        nextsubblock = lltype.malloc(self.SUBBLOCK)
+        nextsubblock.prev = self._cursubblock
+        self._cursubblock = nextsubblock
+        self._cursubindex = 0
+        self._baserelpos += self.SUBBLOCK_SIZE
+    _make_new_subblock._dont_inline_ = True
+
+    def writechar(self, char):
+        index = self._cursubindex
+        if index == self.SUBBLOCK_SIZE:
+            self._make_new_subblock()
+            index = 0
+        self._cursubblock.data[index] = char
+        self._cursubindex = index + 1
+
+    def overwrite(self, index, char):
+        assert 0 <= index < self.get_relative_pos()
+        block = self._cursubblock
+        index -= self._baserelpos
+        while index < 0:
+            block = block.prev
+            index += self.SUBBLOCK_SIZE
+        block.data[index] = char
+
+    def get_relative_pos(self):
+        return self._baserelpos + self._cursubindex
+
+    def copy_to_raw_memory(self, addr):
+        # indirection for _become_a_plain_block_builder() and for subclasses
+        self._copy_to_raw_memory(addr)
+
+    def _copy_to_raw_memory(self, addr):
+        block = self._cursubblock
+        blocksize = self._cursubindex
+        targetindex = self._baserelpos
+        while targetindex >= 0:
+            dst = rffi.cast(rffi.CCHARP, addr + targetindex)
+            for j in range(blocksize):
+                dst[j] = block.data[j]
+            block = block.prev
+            blocksize = self.SUBBLOCK_SIZE
+            targetindex -= self.SUBBLOCK_SIZE
+        assert not block
+
+    def materialize(self, asmmemmgr, allblocks, gcrootmap=None):
+        size = self.get_relative_pos()
+        malloced = asmmemmgr.malloc(size, size)
+        allblocks.append(malloced)
+        rawstart = malloced[0]
+        self.copy_to_raw_memory(rawstart)
+        if self.gcroot_markers is not None:
+            assert gcrootmap is not None
+            gcrootmap.add_raw_gcroot_markers(asmmemmgr,
+                                             allblocks,
+                                             self.gcroot_markers,
+                                             self.gcroot_markers_total_size,
+                                             rawstart)
+        return rawstart
+
+    def _become_a_plain_block_builder(self):
+        # hack purely for speed of tests
+        self._data = []
+        self.writechar = self._data.append
+        self.overwrite = self._data.__setitem__
+        self.get_relative_pos = self._data.__len__
+        def plain_copy_to_raw_memory(addr):
+            dst = rffi.cast(rffi.CCHARP, addr)
+            for i, c in enumerate(self._data):
+                dst[i] = c
+        self._copy_to_raw_memory = plain_copy_to_raw_memory
+
+    def insert_gcroot_marker(self, mark):
+        if self.gcroot_markers is None:
+            self.gcroot_markers = []
+        self.gcroot_markers.append((self.get_relative_pos(), mark))
+        self.gcroot_markers_total_size += len(mark)

pypy/jit/backend/llsupport/gc.py

 from pypy.jit.backend.llsupport.descr import GcCache, get_field_descr
 from pypy.jit.backend.llsupport.descr import GcPtrFieldDescr
 from pypy.jit.backend.llsupport.descr import get_call_descr
+from pypy.rpython.memory.gctransform import asmgcroot
 
 # ____________________________________________________________
 
         return False
     def has_write_barrier_class(self):
         return None
+    def freeing_block(self, start, stop):
+        pass
 
 # ____________________________________________________________
 
     LOC_EBP_PLUS  = 2
     LOC_EBP_MINUS = 3
 
-    GCMAP_ARRAY = rffi.CArray(llmemory.Address)
+    GCMAP_ARRAY = rffi.CArray(lltype.Signed)
     CALLSHAPE_ARRAY = rffi.CArray(rffi.UCHAR)
 
     def __init__(self):
+        # '_gcmap' is an array of length '_gcmap_maxlength' of addresses.
+        # '_gcmap_curlength' tells how full the array really is.
+        # The addresses are actually grouped in pairs:
+        #     (addr-after-the-CALL-in-assembler, addr-of-the-call-shape).
+        # '_gcmap_deadentries' counts pairs marked dead (2nd item is NULL).
+        # '_gcmap_sorted' is True only if we know the array is sorted.
         self._gcmap = lltype.nullptr(self.GCMAP_ARRAY)
         self._gcmap_curlength = 0
         self._gcmap_maxlength = 0
+        self._gcmap_deadentries = 0
+        self._gcmap_sorted = True
 
     def initialize(self):
         # hack hack hack.  Remove these lines and see MissingRTypeAttribute
         # when the rtyper tries to annotate these methods only when GC-ing...
         self.gcmapstart()
         self.gcmapend()
+        self.gcmarksorted()
 
     def gcmapstart(self):
-        return llmemory.cast_ptr_to_adr(self._gcmap)
+        return rffi.cast(llmemory.Address, self._gcmap)
 
     def gcmapend(self):
         addr = self.gcmapstart()
         if self._gcmap_curlength:
-            addr += llmemory.sizeof(llmemory.Address)*self._gcmap_curlength
+            addr += rffi.sizeof(lltype.Signed) * self._gcmap_curlength
+            if not we_are_translated() and type(addr) is long:
+                from pypy.rpython.lltypesystem import ll2ctypes
+                addr = ll2ctypes._lladdress(addr)       # XXX workaround
         return addr
 
-    def put(self, retaddr, callshapeaddr):
+    def gcmarksorted(self):
+        # Called by the GC when it is about to sort [gcmapstart():gcmapend()].
+        # Returns the previous sortedness flag -- i.e. returns True if it
+        # is already sorted, False if sorting is needed.
+        sorted = self._gcmap_sorted
+        self._gcmap_sorted = True
+        return sorted
+
+    @rgc.no_collect
+    def _put(self, retaddr, callshapeaddr):
         """'retaddr' is the address just after the CALL.
-        'callshapeaddr' is the address returned by encode_callshape()."""
+        'callshapeaddr' is the address of the raw 'shape' marker.
+        Both addresses are actually integers here."""
         index = self._gcmap_curlength
         if index + 2 > self._gcmap_maxlength:
-            self._enlarge_gcmap()
+            index = self._enlarge_gcmap()
         self._gcmap[index] = retaddr
         self._gcmap[index+1] = callshapeaddr
         self._gcmap_curlength = index + 2
+        self._gcmap_sorted = False
 
+    @rgc.no_collect
     def _enlarge_gcmap(self):
-        newlength = 250 + self._gcmap_maxlength * 2
-        newgcmap = lltype.malloc(self.GCMAP_ARRAY, newlength, flavor='raw',
-                                 track_allocation=False)   # YYY leak
         oldgcmap = self._gcmap
-        for i in range(self._gcmap_curlength):
-            newgcmap[i] = oldgcmap[i]
-        self._gcmap = newgcmap
-        self._gcmap_maxlength = newlength
-        if oldgcmap:
-            lltype.free(oldgcmap, flavor='raw', track_allocation=False)
+        if self._gcmap_deadentries * 3 * 2 > self._gcmap_maxlength:
+            # More than 1/3rd of the entries are dead.  Don't actually
+            # enlarge the gcmap table, but just clean up the dead entries.
+            newgcmap = oldgcmap
+        else:
+            # Normal path: enlarge the array.
+            newlength = 250 + (self._gcmap_maxlength // 3) * 4
+            newgcmap = lltype.malloc(self.GCMAP_ARRAY, newlength, flavor='raw',
+                                     track_allocation=False)
+            self._gcmap_maxlength = newlength
+        #
+        j = 0
+        i = 0
+        end = self._gcmap_curlength
+        while i < end:
+            if oldgcmap[i + 1]:
+                newgcmap[j] = oldgcmap[i]
+                newgcmap[j + 1] = oldgcmap[i + 1]
+                j += 2
+            i += 2
+        self._gcmap_curlength = j
+        self._gcmap_deadentries = 0
+        if oldgcmap != newgcmap:
+            self._gcmap = newgcmap
+            if oldgcmap:
+                lltype.free(oldgcmap, flavor='raw', track_allocation=False)
+        return j
+
+    def add_raw_gcroot_markers(self, asmmemmgr, allblocks,
+                               markers, total_size, rawstart):
+        """The interface is a bit custom, but this routine writes the
+        shapes of gcroots (for the GC to use) into raw memory."""
+        # xxx so far, we never try to share them.  But right now
+        # the amount of potential sharing would not be too large.
+        dst = 1
+        stop = 0
+        for relpos, shape in markers:
+            #
+            if dst + len(shape) > stop:
+                # No more space in the previous raw block,
+                # allocate a raw block of memory big enough to fit
+                # as many of the remaining 'shapes' as possible
+                start, stop = asmmemmgr.malloc(len(shape), total_size)
+                # add the raw block to 'compiled_loop_token.asmmemmgr_blocks'
+                allblocks.append((start, stop))
+                dst = start
+            #
+            # add the entry 'pos_after_call -> dst' to the table
+            self._put(rawstart + relpos, dst)
+            # Copy 'shape' into the raw memory, reversing the order
+            # of the bytes.  Similar to compress_callshape() in
+            # trackgcroot.py.
+            total_size -= len(shape)
+            src = len(shape) - 1
+            while src >= 0:
+                rffi.cast(rffi.CCHARP, dst)[0] = shape[src]
+                dst += 1
+                src -= 1
+
+    @rgc.no_collect
+    def freeing_block(self, start, stop):
+        # if [start:stop] is a raw block of assembler, then look up the
+        # corresponding gcroot markers, and mark them as freed now in
+        # self._gcmap by setting the 2nd address of every entry to NULL.
+        gcmapstart = self.gcmapstart()
+        gcmapend   = self.gcmapend()
+        if gcmapstart == gcmapend:
+            return
+        if not self.gcmarksorted():
+            asmgcroot.sort_gcmap(gcmapstart, gcmapend)
+        # A note about gcmarksorted(): the deletion we do here keeps the
+        # array sorted.  This avoids needing too many sort_gcmap()s.
+        # Indeed, freeing_block() is typically called many times in a row,
+        # so it will call sort_gcmap() at most the first time.
+        startaddr = rffi.cast(llmemory.Address, start)
+        stopaddr  = rffi.cast(llmemory.Address, stop)
+        item = asmgcroot.binary_search(gcmapstart, gcmapend, startaddr)
+        # 'item' points to one of the entries.  Because the whole array
+        # is sorted, we know that it points either to the first entry we
+        # want to kill, or to the previous entry.
+        if item.address[0] < startaddr:
+            item += asmgcroot.arrayitemsize    # go forward one entry
+            assert item == gcmapend or item.address[0] >= startaddr
+        while item != gcmapend and item.address[0] < stopaddr:
+            item.address[1] = llmemory.NULL
+            self._gcmap_deadentries += 1
+            item += asmgcroot.arrayitemsize
 
     def get_basic_shape(self, is_64_bit=False):
         # XXX: Should this code even really know about stack frame layout of
         assert reg_index > 0
         shape.append(chr(self.LOC_REG | (reg_index << 2)))
 
-    def compress_callshape(self, shape):