Commits

Laurence Tratt committed 21ecaa0 Merge

Merge default.

  • Participants
  • Parent commits 3a4f3f6, 4cd1bc8
  • Branches more_strategies

Comments (0)

Files changed (76)

lib-python/2.7/json/encoder.py

 
 from __pypy__.builders import StringBuilder, UnicodeBuilder
 
+class StringOrUnicodeBuilder(object):
+    def __init__(self):
+        self._builder = StringBuilder()
+    def append(self, string):
+        try:
+            self._builder.append(string)
+        except UnicodeEncodeError:
+            ub = UnicodeBuilder()
+            ub.append(self._builder.build())
+            self._builder = ub
+            ub.append(string)
+    def build(self):
+        return self._builder.build()
+
+
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
 ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
 HAS_UTF8 = re.compile(r'[\x80-\xff]')
         if self.ensure_ascii:
             builder = StringBuilder()
         else:
-            builder = UnicodeBuilder()
+            builder = StringOrUnicodeBuilder()
         self.__encode(o, markers, builder, 0)
         return builder.build()
 

lib-python/2.7/weakref.py

         def remove(wr, selfref=ref(self)):
             self = selfref()
             if self is not None:
-                del self.data[wr.key]
+                # Changed this for PyPy: made more resistent.  The
+                # issue is that in some corner cases, self.data
+                # might already be changed or removed by the time
+                # this weakref's callback is called.  If that is
+                # the case, we don't want to randomly kill an
+                # unrelated entry.
+                if self.data.get(wr.key) is wr:
+                    del self.data[wr.key]
         self._remove = remove
         UserDict.UserDict.__init__(self, *args, **kw)
 
         try:
             o = self.data.pop(key)()
         except KeyError:
+            o = None
+        if o is None:
             if args:
                 return args[0]
-            raise
-        if o is None:
             raise KeyError, key
         else:
             return o
+        # The logic above was fixed in PyPy
 
     def setdefault(self, key, default=None):
         try:
-            wr = self.data[key]
+            o = self.data[key]()
         except KeyError:
+            o = None
+        if o is None:
             self.data[key] = KeyedRef(default, self._remove, key)
             return default
         else:
-            return wr()
+            return o
+        # The logic above was fixed in PyPy
 
     def update(self, dict=None, **kwargs):
         d = self.data

lib_pypy/cffi.egg-info

 Metadata-Version: 1.0
 Name: cffi
-Version: 0.7
+Version: 0.8
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski

lib_pypy/cffi/__init__.py

 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "0.7.2"
-__version_info__ = (0, 7, 2)
+__version__ = "0.8"
+__version_info__ = (0, 8)

lib_pypy/cffi/api.py

 import types
+from .lock import allocate_lock
 
 try:
     callable
             # rely on it!  It's probably not going to work well.)
 
         self._backend = backend
+        self._lock = allocate_lock()
         self._parser = cparser.Parser()
         self._cached_btypes = {}
         self._parsed_types = types.ModuleType('parsed_types').__dict__
             if name.startswith('RTLD_'):
                 setattr(self, name, getattr(backend, name))
         #
-        self.BVoidP = self._get_cached_btype(model.voidp_type)
+        with self._lock:
+            self.BVoidP = self._get_cached_btype(model.voidp_type)
         if isinstance(backend, types.ModuleType):
             # _cffi_backend: attach these constants to the class
             if not hasattr(FFI, 'NULL'):
             if not isinstance(csource, basestring):
                 raise TypeError("cdef() argument must be a string")
             csource = csource.encode('ascii')
-        self._parser.parse(csource, override=override)
-        self._cdefsources.append(csource)
-        if override:
-            for cache in self._function_caches:
-                cache.clear()
+        with self._lock:
+            self._parser.parse(csource, override=override)
+            self._cdefsources.append(csource)
+            if override:
+                for cache in self._function_caches:
+                    cache.clear()
 
     def dlopen(self, name, flags=0):
         """Load and return a dynamic library identified by 'name'.
         library we only look for the actual (untyped) symbols.
         """
         assert isinstance(name, basestring) or name is None
-        lib, function_cache = _make_ffi_library(self, name, flags)
-        self._function_caches.append(function_cache)
-        self._libraries.append(lib)
+        with self._lock:
+            lib, function_cache = _make_ffi_library(self, name, flags)
+            self._function_caches.append(function_cache)
+            self._libraries.append(lib)
         return lib
 
+    def _typeof_locked(self, cdecl):
+        # call me with the lock!
+        key = cdecl
+        if key in self._parsed_types:
+            return self._parsed_types[key]
+        #
+        if not isinstance(cdecl, str):    # unicode, on Python 2
+            cdecl = cdecl.encode('ascii')
+        #
+        type = self._parser.parse_type(cdecl)
+        really_a_function_type = type.is_raw_function
+        if really_a_function_type:
+            type = type.as_function_pointer()
+        btype = self._get_cached_btype(type)
+        result = btype, really_a_function_type
+        self._parsed_types[key] = result
+        return result
+
     def _typeof(self, cdecl, consider_function_as_funcptr=False):
         # string -> ctype object
         try:
-            btype, cfaf = self._parsed_types[cdecl]
-            if consider_function_as_funcptr and not cfaf:
-                raise KeyError
+            result = self._parsed_types[cdecl]
         except KeyError:
-            key = cdecl
-            if not isinstance(cdecl, str):    # unicode, on Python 2
-                cdecl = cdecl.encode('ascii')
-            cfaf = consider_function_as_funcptr
-            type = self._parser.parse_type(cdecl,
-                       consider_function_as_funcptr=cfaf)
-            btype = self._get_cached_btype(type)
-            self._parsed_types[key] = btype, cfaf
+            with self._lock:
+                result = self._typeof_locked(cdecl)
+        #
+        btype, really_a_function_type = result
+        if really_a_function_type and not consider_function_as_funcptr:
+            raise CDefError("the type %r is a function type, not a "
+                            "pointer-to-function type" % (cdecl,))
         return btype
 
     def typeof(self, cdecl):
         """Parse the C type given as a string and return the
-        corresponding Python type: <class 'ffi.CData<...>'>.
+        corresponding <ctype> object.
         It can also be used on 'cdata' instance to get its C type.
         """
         if isinstance(cdecl, basestring):
             res = _builtin_function_type(cdecl)
             if res is not None:
                 return res
+        if (isinstance(cdecl, types.FunctionType)
+                and hasattr(cdecl, '_cffi_base_type')):
+            with self._lock:
+                return self._get_cached_btype(cdecl._cffi_base_type)
         raise TypeError(type(cdecl))
 
     def sizeof(self, cdecl):
         data.  Later, when this new cdata object is garbage-collected,
         'destructor(old_cdata_object)' will be called.
         """
-        try:
-            gc_weakrefs = self.gc_weakrefs
-        except AttributeError:
-            from .gc_weakref import GcWeakrefs
-            gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self)
-        return gc_weakrefs.build(cdata, destructor)
+        with self._lock:
+            try:
+                gc_weakrefs = self.gc_weakrefs
+            except AttributeError:
+                from .gc_weakref import GcWeakrefs
+                gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self)
+            return gc_weakrefs.build(cdata, destructor)
 
     def _get_cached_btype(self, type):
+        assert self._lock.acquire(False) is False
+        # call me with the lock!
         try:
             BType = self._cached_btypes[type]
         except KeyError:
 
     def _pointer_to(self, ctype):
         from . import model
-        return model.pointer_cache(self, ctype)
+        with self._lock:
+            return model.pointer_cache(self, ctype)
 
     def addressof(self, cdata, field=None):
         """Return the address of a <cdata 'struct-or-union'>.
         variables, which must anyway be accessed directly from the
         lib object returned by the original FFI instance.
         """
-        self._parser.include(ffi_to_include._parser)
-        self._cdefsources.append('[')
-        self._cdefsources.extend(ffi_to_include._cdefsources)
-        self._cdefsources.append(']')
+        with ffi_to_include._lock:
+            with self._lock:
+                self._parser.include(ffi_to_include._parser)
+                self._cdefsources.append('[')
+                self._cdefsources.extend(ffi_to_include._cdefsources)
+                self._cdefsources.append(']')
 
     def new_handle(self, x):
         return self._backend.newp_handle(self.BVoidP, x)
         backendlib = backend.load_library(path, flags)
     copied_enums = []
     #
-    def make_accessor(name):
+    def make_accessor_locked(name):
         key = 'function ' + name
         if key in ffi._parser._declarations:
             tp = ffi._parser._declarations[key]
                     if enumname not in library.__dict__:
                         library.__dict__[enumname] = enumval
             copied_enums.append(True)
+            if name in library.__dict__:
+                return
         #
-        if name in library.__dict__:   # copied from an enum value just above,
-            return                     # or multithread's race condition
         raise AttributeError(name)
     #
+    def make_accessor(name):
+        with ffi._lock:
+            if name in library.__dict__ or name in FFILibrary.__dict__:
+                return    # added by another thread while waiting for the lock
+            make_accessor_locked(name)
+    #
     class FFILibrary(object):
         def __getattr__(self, name):
             make_accessor(name)
     except (KeyError, AttributeError, TypeError):
         return None
     else:
-        return ffi._get_cached_btype(tp)
+        with ffi._lock:
+            return ffi._get_cached_btype(tp)

lib_pypy/cffi/cparser.py

                 if 1 <= linenum <= len(csourcelines):
                     line = csourcelines[linenum-1]
         if line:
-            msg = 'cannot parse "%s"\n%s' % (line, msg)
+            msg = 'cannot parse "%s"\n%s' % (line.strip(), msg)
         else:
             msg = 'parse error\n%s' % (msg,)
         raise api.CDefError(msg)
             #
             if decl.name:
                 tp = self._get_type(node, partial_length_ok=True)
-                if self._is_constant_declaration(node):
+                if self._is_constant_globalvar(node):
                     self._declare('constant ' + decl.name, tp)
                 else:
                     self._declare('variable ' + decl.name, tp)
 
-    def parse_type(self, cdecl, consider_function_as_funcptr=False):
+    def parse_type(self, cdecl):
         ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl)
         assert not macros
         exprnode = ast.ext[-1].type.args.params[0]
         if isinstance(exprnode, pycparser.c_ast.ID):
             raise api.CDefError("unknown identifier '%s'" % (exprnode.name,))
-        return self._get_type(exprnode.type,
-                     consider_function_as_funcptr=consider_function_as_funcptr)
+        return self._get_type(exprnode.type)
 
     def _declare(self, name, obj):
         if name in self._declarations:
             return model.ConstPointerType(type)
         return model.PointerType(type)
 
-    def _get_type(self, typenode, convert_array_to_pointer=False,
-                  name=None, partial_length_ok=False,
-                  consider_function_as_funcptr=False):
+    def _get_type(self, typenode, name=None, partial_length_ok=False):
         # first, dereference typedefs, if we have it already parsed, we're good
         if (isinstance(typenode, pycparser.c_ast.TypeDecl) and
             isinstance(typenode.type, pycparser.c_ast.IdentifierType) and
             len(typenode.type.names) == 1 and
             ('typedef ' + typenode.type.names[0]) in self._declarations):
             type = self._declarations['typedef ' + typenode.type.names[0]]
-            if isinstance(type, model.ArrayType):
-                if convert_array_to_pointer:
-                    return type.item
-            else:
-                if (consider_function_as_funcptr and
-                        isinstance(type, model.RawFunctionType)):
-                    return type.as_function_pointer()
             return type
         #
         if isinstance(typenode, pycparser.c_ast.ArrayDecl):
             # array type
-            if convert_array_to_pointer:
-                return self._get_type_pointer(self._get_type(typenode.type))
             if typenode.dim is None:
                 length = None
             else:
         #
         if isinstance(typenode, pycparser.c_ast.FuncDecl):
             # a function type
-            result = self._parse_function_type(typenode, name)
-            if consider_function_as_funcptr:
-                result = result.as_function_pointer()
-            return result
+            return self._parse_function_type(typenode, name)
         #
         # nested anonymous structs or unions end up here
         if isinstance(typenode, pycparser.c_ast.Struct):
             isinstance(params[0].type.type, pycparser.c_ast.IdentifierType)
                 and list(params[0].type.type.names) == ['void']):
             del params[0]
-        args = [self._get_type(argdeclnode.type,
-                               convert_array_to_pointer=True,
-                               consider_function_as_funcptr=True)
+        args = [self._as_func_arg(self._get_type(argdeclnode.type))
                 for argdeclnode in params]
         result = self._get_type(typenode.type)
         return model.RawFunctionType(tuple(args), result, ellipsis)
 
-    def _is_constant_declaration(self, typenode, const=False):
-        if isinstance(typenode, pycparser.c_ast.ArrayDecl):
-            return self._is_constant_declaration(typenode.type)
+    def _as_func_arg(self, type):
+        if isinstance(type, model.ArrayType):
+            return model.PointerType(type.item)
+        elif isinstance(type, model.RawFunctionType):
+            return type.as_function_pointer()
+        else:
+            return type
+
+    def _is_constant_globalvar(self, typenode):
         if isinstance(typenode, pycparser.c_ast.PtrDecl):
-            const = 'const' in typenode.quals
-            return self._is_constant_declaration(typenode.type, const)
+            return 'const' in typenode.quals
         if isinstance(typenode, pycparser.c_ast.TypeDecl):
-            return const or 'const' in typenode.quals
+            return 'const' in typenode.quals
         return False
 
     def _get_struct_union_enum_type(self, kind, type, name=None, nested=False):
         return tp
 
     def _make_partial(self, tp, nested):
-        if not isinstance(tp, model.StructType):
+        if not isinstance(tp, model.StructOrUnion):
             raise api.CDefError("%s cannot be partial" % (tp,))
         if not tp.has_c_name() and not nested:
             raise NotImplementedError("%s is partial but has no C name" %(tp,))
             if (isinstance(exprnode, pycparser.c_ast.ID) and
                     exprnode.name == '__dotdotdotarray__'):
                 self._partial_length = True
-                return None
+                return '...'
         #
         raise api.FFIError("unsupported expression: expected a "
                            "simple numeric constant")

lib_pypy/cffi/gc_weakref.py

 
     def build(self, cdata, destructor):
         # make a new cdata of the same type as the original one
-        new_cdata = self.ffi.cast(self.ffi.typeof(cdata), cdata)
+        new_cdata = self.ffi.cast(self.ffi._backend.typeof(cdata), cdata)
         self.data[ref(new_cdata, self.remove)] = destructor, cdata
         return new_cdata

lib_pypy/cffi/lock.py

+import sys
+
+if sys.version_info < (3,):
+    try:
+        from thread import allocate_lock
+    except ImportError:
+        from dummy_thread import allocate_lock
+else:
+    try:
+        from _thread import allocate_lock
+    except ImportError:
+        from _dummy_thread import allocate_lock
+
+
+##import sys
+##l1 = allocate_lock
+
+##class allocate_lock(object):
+##    def __init__(self):
+##        self._real = l1()
+##    def __enter__(self):
+##        for i in range(4, 0, -1):
+##            print sys._getframe(i).f_code
+##        print
+##        return self._real.__enter__()
+##    def __exit__(self, *args):
+##        return self._real.__exit__(*args)
+##    def acquire(self, f):
+##        assert f is False
+##        return self._real.acquire(f)

lib_pypy/cffi/model.py

 import weakref
+from .lock import allocate_lock
+
 
 class BaseTypeByIdentity(object):
     is_array_type = False
+    is_raw_function = False
 
     def get_c_name(self, replace_with='', context='a C file'):
         result = self.c_name_with_marker
     # a function, but not a pointer-to-function.  The backend has no
     # notion of such a type; it's used temporarily by parsing.
     _base_pattern = '(&)(%s)'
+    is_raw_function = True
 
     def build_backend_type(self, ffi, finishlist):
         from . import api
     _base_pattern       = " const *&"
     _base_pattern_array = "(const *&)"
 
-    def build_backend_type(self, ffi, finishlist):
-        BPtr = PointerType(self.totype).get_cached_btype(ffi, finishlist)
-        return BPtr
-
 const_voidp_type = ConstPointerType(void_type)
 
 
         self.item = item
         self.length = length
         #
-        if self.length is None:
+        if length is None:
             brackets = '&[]'
+        elif length == '...':
+            brackets = '&[/*...*/]'
         else:
-            brackets = '&[%d]' % self.length
+            brackets = '&[%d]' % length
         self.c_name_with_marker = (
             self.item.c_name_with_marker.replace('&', brackets))
 
         return ArrayType(self.item, newlength)
 
     def build_backend_type(self, ffi, finishlist):
+        if self.length == '...':
+            from . import api
+            raise api.CDefError("cannot render the type %r: unknown length" %
+                                (self,))
         self.item.get_cached_btype(ffi, finishlist)   # force the item BType
         BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist)
         return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length)
 class StructOrUnion(StructOrUnionOrEnum):
     fixedlayout = None
     completed = False
+    partial = False
 
     def __init__(self, name, fldnames, fldtypes, fldbitsize):
         self.name = name
             return    # not completing it: it's an opaque struct
         #
         self.completed = 1
-        fldtypes = tuple(tp.get_cached_btype(ffi, finishlist)
-                         for tp in self.fldtypes)
         #
         if self.fixedlayout is None:
+            fldtypes = [tp.get_cached_btype(ffi, finishlist)
+                        for tp in self.fldtypes]
             lst = list(zip(self.fldnames, fldtypes, self.fldbitsize))
             ffi._backend.complete_struct_or_union(BType, lst, self)
             #
         else:
+            fldtypes = []
             fieldofs, fieldsize, totalsize, totalalignment = self.fixedlayout
             for i in range(len(self.fldnames)):
                 fsize = fieldsize[i]
                 ftype = self.fldtypes[i]
                 #
-                if isinstance(ftype, ArrayType) and ftype.length is None:
+                if isinstance(ftype, ArrayType) and ftype.length == '...':
                     # fix the length to match the total size
                     BItemType = ftype.item.get_cached_btype(ffi, finishlist)
                     nlen, nrest = divmod(fsize, ffi.sizeof(BItemType))
                     ftype = ftype.resolve_length(nlen)
                     self.fldtypes = (self.fldtypes[:i] + (ftype,) +
                                      self.fldtypes[i+1:])
-                    BArrayType = ftype.get_cached_btype(ffi, finishlist)
-                    fldtypes = (fldtypes[:i] + (BArrayType,) +
-                                fldtypes[i+1:])
-                    continue
                 #
-                bitemsize = ffi.sizeof(fldtypes[i])
-                if bitemsize != fsize:
-                    self._verification_error(
-                        "field '%s.%s' is declared as %d bytes, but is "
-                        "really %d bytes" % (self.name,
-                                             self.fldnames[i] or '{}',
-                                             bitemsize, fsize))
+                BFieldType = ftype.get_cached_btype(ffi, finishlist)
+                if isinstance(ftype, ArrayType) and ftype.length is None:
+                    assert fsize == 0
+                else:
+                    bitemsize = ffi.sizeof(BFieldType)
+                    if bitemsize != fsize:
+                        self._verification_error(
+                            "field '%s.%s' is declared as %d bytes, but is "
+                            "really %d bytes" % (self.name,
+                                                 self.fldnames[i] or '{}',
+                                                 bitemsize, fsize))
+                fldtypes.append(BFieldType)
+            #
             lst = list(zip(self.fldnames, fldtypes, self.fldbitsize, fieldofs))
             ffi._backend.complete_struct_or_union(BType, lst, self,
                                                   totalsize, totalalignment)
         from .ffiplatform import VerificationError
         raise VerificationError(msg)
 
-
-class StructType(StructOrUnion):
-    kind = 'struct'
-    partial = False
-
     def check_not_partial(self):
         if self.partial and self.fixedlayout is None:
             from . import ffiplatform
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
-        
-        return global_cache(self, ffi, 'new_struct_type',
+        #
+        return global_cache(self, ffi, 'new_%s_type' % self.kind,
                             self.get_official_name(), key=self)
 
 
+class StructType(StructOrUnion):
+    kind = 'struct'
+
+
 class UnionType(StructOrUnion):
     kind = 'union'
 
-    def build_backend_type(self, ffi, finishlist):
-        finishlist.append(self)
-        return global_cache(self, ffi, 'new_union_type',
-                            self.get_official_name(), key=self)
-
 
 class EnumType(StructOrUnionOrEnum):
     kind = 'enum'
         self.baseinttype = baseinttype
         self.build_c_name_with_marker()
 
+    def force_the_name(self, forcename):
+        StructOrUnionOrEnum.force_the_name(self, forcename)
+        if self.forcename is None:
+            name = self.get_official_name()
+            self.forcename = '$' + name.replace(' ', '_')
+
     def check_not_partial(self):
         if self.partial and not self.partial_resolved:
             from . import ffiplatform
     tp = StructType(structname, None, None, None)
     return NamedPointerType(tp, name)
 
+
+global_lock = allocate_lock()
+
 def global_cache(srctype, ffi, funcname, *args, **kwds):
     key = kwds.pop('key', (funcname, args))
     assert not kwds
         res = getattr(ffi._backend, funcname)(*args)
     except NotImplementedError as e:
         raise NotImplementedError("%r: %s" % (srctype, e))
-    ffi._backend.__typecache[key] = res
-    return res
+    # note that setdefault() on WeakValueDictionary is not atomic
+    # and contains a rare bug (http://bugs.python.org/issue19542);
+    # we have to use a lock and do it ourselves
+    cache = ffi._backend.__typecache
+    with global_lock:
+        res1 = cache.get(key)
+        if res1 is None:
+            cache[key] = res
+            return res
+        else:
+            return res1
 
 def pointer_cache(ffi, BType):
     return global_cache('?', ffi, 'new_pointer_type', BType)

lib_pypy/cffi/vengine_cpy.py

     def patch_extension_kwds(self, kwds):
         pass
 
-    def find_module(self, module_name, path, so_suffix):
+    def find_module(self, module_name, path, so_suffixes):
         try:
             f, filename, descr = imp.find_module(module_name, path)
         except ImportError:
         # Note that after a setuptools installation, there are both .py
         # and .so files with the same basename.  The code here relies on
         # imp.find_module() locating the .so in priority.
-        if descr[0] != so_suffix:
+        if descr[0] not in so_suffixes:
             return None
         return filename
 
             def __dir__(self):
                 return FFILibrary._cffi_dir + list(self.__dict__)
         library = FFILibrary()
-        module._cffi_setup(lst, ffiplatform.VerificationError, library)
+        if module._cffi_setup(lst, ffiplatform.VerificationError, library):
+            import warnings
+            warnings.warn("reimporting %r might overwrite older definitions"
+                          % (self.verifier.get_module_name()))
         #
         # finally, call the loaded_cpy_xxx() functions.  This will perform
         # the final adjustments, like copying the Python->C wrapper
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(tp))
         elif isinstance(tp, model.ArrayType):
-            return '_cffi_from_c_deref((char *)%s, _cffi_type(%d))' % (
-                var, self._gettypenum(tp))
+            return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
+                var, self._gettypenum(model.PointerType(tp.item)))
         elif isinstance(tp, model.StructType):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
         prnt('  static Py_ssize_t nums[] = {')
         prnt('    sizeof(%s),' % cname)
         prnt('    offsetof(struct _cffi_aligncheck, y),')
-        for fname, _, fbitsize in tp.enumfields():
+        for fname, ftype, fbitsize in tp.enumfields():
             if fbitsize >= 0:
                 continue      # xxx ignore fbitsize for now
             prnt('    offsetof(%s, %s),' % (cname, fname))
-            prnt('    sizeof(((%s *)0)->%s),' % (cname, fname))
+            if isinstance(ftype, model.ArrayType) and ftype.length is None:
+                prnt('    0,  /* %s */' % ftype._get_c_name())
+            else:
+                prnt('    sizeof(((%s *)0)->%s),' % (cname, fname))
         prnt('    -1')
         prnt('  };')
         prnt('  return _cffi_get_struct_layout(nums);')
         #
         function = getattr(module, layoutfuncname)
         layout = function()
-        if isinstance(tp, model.StructType) and tp.partial:
+        if isinstance(tp, model.StructOrUnion) and tp.partial:
             # use the function()'s sizes and offsets to guide the
             # layout of the struct
             totalsize = layout[0]
                     continue        # xxx ignore fbitsize for now
                 check(layout[i], ffi.offsetof(BStruct, fname),
                       "wrong offset for field %r" % (fname,))
-                BField = ffi._get_cached_btype(ftype)
-                check(layout[i+1], ffi.sizeof(BField),
-                      "wrong size for field %r" % (fname,))
+                if layout[i+1] != 0:
+                    BField = ffi._get_cached_btype(ftype)
+                    check(layout[i+1], ffi.sizeof(BField),
+                          "wrong size for field %r" % (fname,))
                 i += 2
             assert i == len(layout)
 
     # constants, likely declared with '#define'
 
     def _generate_cpy_const(self, is_int, name, tp=None, category='const',
-                            vartp=None, delayed=True):
+                            vartp=None, delayed=True, size_too=False):
         prnt = self._prnt
         funcname = '_cffi_%s_%s' % (category, name)
         prnt('static int %s(PyObject *lib)' % funcname)
                  '(unsigned long long)(%s));' % (name,))
         prnt('  if (o == NULL)')
         prnt('    return -1;')
+        if size_too:
+            prnt('  {')
+            prnt('    PyObject *o1 = o;')
+            prnt('    o = Py_BuildValue("On", o1, (Py_ssize_t)sizeof(%s));'
+                 % (name,))
+            prnt('    Py_DECREF(o1);')
+            prnt('    if (o == NULL)')
+            prnt('      return -1;')
+            prnt('  }')
         prnt('  res = PyObject_SetAttrString(lib, "%s", o);' % name)
         prnt('  Py_DECREF(o);')
         prnt('  if (res < 0)')
         prnt('static int %s(PyObject *lib)' % funcname)
         prnt('{')
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
-            prnt('  if (%s != %d) {' % (enumerator, enumvalue))
+            if enumvalue < 0:
+                prnt('  if ((%s) >= 0 || (long)(%s) != %dL) {' % (
+                    enumerator, enumerator, enumvalue))
+            else:
+                prnt('  if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % (
+                    enumerator, enumerator, enumvalue))
+            prnt('    char buf[64];')
+            prnt('    if ((%s) < 0)' % enumerator)
+            prnt('        snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator)
+            prnt('    else')
+            prnt('        snprintf(buf, 63, "%%lu", (unsigned long)(%s));' %
+                 enumerator)
             prnt('    PyErr_Format(_cffi_VerificationError,')
-            prnt('                 "enum %s: %s has the real value %d, '
-                 'not %d",')
-            prnt('                 "%s", "%s", (int)%s, %d);' % (
-                name, enumerator, enumerator, enumvalue))
+            prnt('                 "enum %s: %s has the real value %s, '
+                 'not %s",')
+            prnt('                 "%s", "%s", buf, "%d");' % (
+                name, enumerator, enumvalue))
             prnt('    return -1;')
             prnt('  }')
         prnt('  return %s;' % self._chained_list_constants[True])
 
     def _generate_cpy_variable_collecttype(self, tp, name):
         if isinstance(tp, model.ArrayType):
-            self._do_collect_type(tp)
+            tp_ptr = model.PointerType(tp.item)
         else:
             tp_ptr = model.PointerType(tp)
-            self._do_collect_type(tp_ptr)
+        self._do_collect_type(tp_ptr)
 
     def _generate_cpy_variable_decl(self, tp, name):
         if isinstance(tp, model.ArrayType):
             tp_ptr = model.PointerType(tp.item)
-            self._generate_cpy_const(False, name, tp, vartp=tp_ptr)
+            self._generate_cpy_const(False, name, tp, vartp=tp_ptr,
+                                     size_too = (tp.length == '...'))
         else:
             tp_ptr = model.PointerType(tp)
             self._generate_cpy_const(False, name, tp_ptr, category='var')
     _loading_cpy_variable = _loaded_noop
 
     def _loaded_cpy_variable(self, tp, name, module, library):
+        value = getattr(library, name)
         if isinstance(tp, model.ArrayType):   # int a[5] is "constant" in the
-            return                            # sense that "a=..." is forbidden
+                                              # sense that "a=..." is forbidden
+            if tp.length == '...':
+                assert isinstance(value, tuple)
+                (value, size) = value
+                BItemType = self.ffi._get_cached_btype(tp.item)
+                length, rest = divmod(size, self.ffi.sizeof(BItemType))
+                if rest != 0:
+                    raise ffiplatform.VerificationError(
+                        "bad size: %r does not seem to be an array of %s" %
+                        (name, tp.item))
+                tp = tp.resolve_length(length)
+            # 'value' is a <cdata 'type *'> which we have to replace with
+            # a <cdata 'type[N]'> if the N is actually known
+            if tp.length is not None:
+                BArray = self.ffi._get_cached_btype(tp)
+                value = self.ffi.cast(BArray, value)
+                setattr(library, name, value)
+            return
         # remove ptr=<cdata 'int *'> from the library instance, and replace
         # it by a property on the class, which reads/writes into ptr[0].
-        ptr = getattr(library, name)
+        ptr = value
         delattr(library, name)
         def getter(library):
             return ptr[0]
 
     def _generate_setup_custom(self):
         prnt = self._prnt
-        prnt('static PyObject *_cffi_setup_custom(PyObject *lib)')
+        prnt('static int _cffi_setup_custom(PyObject *lib)')
         prnt('{')
-        prnt('  if (%s < 0)' % self._chained_list_constants[True])
-        prnt('    return NULL;')
-        prnt('  Py_INCREF(Py_None);')
-        prnt('  return Py_None;')
+        prnt('  return %s;' % self._chained_list_constants[True])
         prnt('}')
 
 cffimod_header = r'''
 static void *_cffi_exports[_CFFI_NUM_EXPORTS];
 static PyObject *_cffi_types, *_cffi_VerificationError;
 
-static PyObject *_cffi_setup_custom(PyObject *lib);   /* forward */
+static int _cffi_setup_custom(PyObject *lib);   /* forward */
 
 static PyObject *_cffi_setup(PyObject *self, PyObject *args)
 {
     PyObject *library;
+    int was_alive = (_cffi_types != NULL);
     if (!PyArg_ParseTuple(args, "OOO", &_cffi_types, &_cffi_VerificationError,
                                        &library))
         return NULL;
     Py_INCREF(_cffi_types);
     Py_INCREF(_cffi_VerificationError);
-    return _cffi_setup_custom(library);
+    if (_cffi_setup_custom(library) < 0)
+        return NULL;
+    return PyBool_FromLong(was_alive);
 }
 
 static void _cffi_init(void)

lib_pypy/cffi/vengine_gen.py

         # up in kwds['export_symbols'].
         kwds.setdefault('export_symbols', self.export_symbols)
 
-    def find_module(self, module_name, path, so_suffix):
-        basename = module_name + so_suffix
-        if path is None:
-            path = sys.path
-        for dirname in path:
-            filename = os.path.join(dirname, basename)
-            if os.path.isfile(filename):
-                return filename
-        return None
+    def find_module(self, module_name, path, so_suffixes):
+        for so_suffix in so_suffixes:
+            basename = module_name + so_suffix
+            if path is None:
+                path = sys.path
+            for dirname in path:
+                filename = os.path.join(dirname, basename)
+                if os.path.isfile(filename):
+                    return filename
 
     def collect_types(self):
         pass      # not needed in the generic engine
             newfunction = self._load_constant(False, tp, name, module)
         else:
             indirections = []
+            base_tp = tp
             if any(isinstance(typ, model.StructOrUnion) for typ in tp.args):
                 indirect_args = []
                 for i, typ in enumerate(tp.args):
             wrappername = '_cffi_f_%s' % name
             newfunction = module.load_function(BFunc, wrappername)
             for i, typ in indirections:
-                newfunction = self._make_struct_wrapper(newfunction, i, typ)
+                newfunction = self._make_struct_wrapper(newfunction, i, typ,
+                                                        base_tp)
         setattr(library, name, newfunction)
         type(library)._cffi_dir.append(name)
 
-    def _make_struct_wrapper(self, oldfunc, i, tp):
+    def _make_struct_wrapper(self, oldfunc, i, tp, base_tp):
         backend = self.ffi._backend
         BType = self.ffi._get_cached_btype(tp)
         def newfunc(*args):
             args = args[:i] + (backend.newp(BType, args[i]),) + args[i+1:]
             return oldfunc(*args)
+        newfunc._cffi_base_type = base_tp
         return newfunc
 
     # ----------
         prnt('  static ssize_t nums[] = {')
         prnt('    sizeof(%s),' % cname)
         prnt('    offsetof(struct _cffi_aligncheck, y),')
-        for fname, _, fbitsize in tp.enumfields():
+        for fname, ftype, fbitsize in tp.enumfields():
             if fbitsize >= 0:
                 continue      # xxx ignore fbitsize for now
             prnt('    offsetof(%s, %s),' % (cname, fname))
-            prnt('    sizeof(((%s *)0)->%s),' % (cname, fname))
+            if isinstance(ftype, model.ArrayType) and ftype.length is None:
+                prnt('    0,  /* %s */' % ftype._get_c_name())
+            else:
+                prnt('    sizeof(((%s *)0)->%s),' % (cname, fname))
         prnt('    -1')
         prnt('  };')
         prnt('  return nums[i];')
             return     # nothing to do with opaque structs
         layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name)
         #
-        BFunc = self.ffi.typeof("ssize_t(*)(ssize_t)")
+        BFunc = self.ffi._typeof_locked("ssize_t(*)(ssize_t)")[0]
         function = module.load_function(BFunc, layoutfuncname)
         layout = []
         num = 0
             if x < 0: break
             layout.append(x)
             num += 1
-        if isinstance(tp, model.StructType) and tp.partial:
+        if isinstance(tp, model.StructOrUnion) and tp.partial:
             # use the function()'s sizes and offsets to guide the
             # layout of the struct
             totalsize = layout[0]
                     continue        # xxx ignore fbitsize for now
                 check(layout[i], ffi.offsetof(BStruct, fname),
                       "wrong offset for field %r" % (fname,))
-                BField = ffi._get_cached_btype(ftype)
-                check(layout[i+1], ffi.sizeof(BField),
-                      "wrong size for field %r" % (fname,))
+                if layout[i+1] != 0:
+                    BField = ffi._get_cached_btype(ftype)
+                    check(layout[i+1], ffi.sizeof(BField),
+                          "wrong size for field %r" % (fname,))
                 i += 2
             assert i == len(layout)
 
     def _load_constant(self, is_int, tp, name, module):
         funcname = '_cffi_const_%s' % name
         if is_int:
-            BFunc = self.ffi.typeof("int(*)(long long*)")
+            BType = self.ffi._typeof_locked("long long*")[0]
+            BFunc = self.ffi._typeof_locked("int(*)(long long*)")[0]
             function = module.load_function(BFunc, funcname)
-            p = self.ffi.new("long long*")
+            p = self.ffi.new(BType)
             negative = function(p)
             value = int(p[0])
             if value < 0 and not negative:
-                value += (1 << (8*self.ffi.sizeof("long long")))
+                BLongLong = self.ffi._typeof_locked("long long")[0]
+                value += (1 << (8*self.ffi.sizeof(BLongLong)))
         else:
-            BFunc = self.ffi.typeof(tp.get_c_name('(*)(void)', name))
+            BFunc = self.ffi._typeof_locked(tp.get_c_name('(*)(void)', name))[0]
             function = module.load_function(BFunc, funcname)
             value = function()
         return value
         prnt('int %s(char *out_error)' % funcname)
         prnt('{')
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
-            prnt('  if (%s != %d) {' % (enumerator, enumvalue))
+            if enumvalue < 0:
+                prnt('  if ((%s) >= 0 || (long)(%s) != %dL) {' % (
+                    enumerator, enumerator, enumvalue))
+            else:
+                prnt('  if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % (
+                    enumerator, enumerator, enumvalue))
+            prnt('    char buf[64];')
+            prnt('    if ((%s) < 0)' % enumerator)
+            prnt('        snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator)
+            prnt('    else')
+            prnt('        snprintf(buf, 63, "%%lu", (unsigned long)(%s));' %
+                 enumerator)
             prnt('    snprintf(out_error, 255,'
-                             '"%s has the real value %d, not %d",')
-            prnt('            "%s", (int)%s, %d);' % (
-                enumerator, enumerator, enumvalue))
+                             ' "%s has the real value %s, not %s",')
+            prnt('            "%s", buf, "%d");' % (
+                enumerator, enumvalue))
             prnt('    return -1;')
             prnt('  }')
         prnt('  return 0;')
             tp.enumvalues = tuple(enumvalues)
             tp.partial_resolved = True
         else:
-            BFunc = self.ffi.typeof("int(*)(char*)")
+            BType = self.ffi._typeof_locked("char[]")[0]
+            BFunc = self.ffi._typeof_locked("int(*)(char*)")[0]
             funcname = '_cffi_e_%s_%s' % (prefix, name)
             function = module.load_function(BFunc, funcname)
-            p = self.ffi.new("char[]", 256)
+            p = self.ffi.new(BType, 256)
             if function(p) < 0:
                 error = self.ffi.string(p)
                 if sys.version_info >= (3,):
 
     def _generate_gen_variable_decl(self, tp, name):
         if isinstance(tp, model.ArrayType):
+            if tp.length == '...':
+                prnt = self._prnt
+                funcname = '_cffi_sizeof_%s' % (name,)
+                self.export_symbols.append(funcname)
+                prnt("size_t %s(void)" % funcname)
+                prnt("{")
+                prnt("  return sizeof(%s);" % (name,))
+                prnt("}")
             tp_ptr = model.PointerType(tp.item)
             self._generate_gen_const(False, name, tp_ptr)
         else:
     def _loaded_gen_variable(self, tp, name, module, library):
         if isinstance(tp, model.ArrayType):   # int a[5] is "constant" in the
                                               # sense that "a=..." is forbidden
+            if tp.length == '...':
+                funcname = '_cffi_sizeof_%s' % (name,)
+                BFunc = self.ffi._typeof_locked('size_t(*)(void)')[0]
+                function = module.load_function(BFunc, funcname)
+                size = function()
+                BItemType = self.ffi._get_cached_btype(tp.item)
+                length, rest = divmod(size, self.ffi.sizeof(BItemType))
+                if rest != 0:
+                    raise ffiplatform.VerificationError(
+                        "bad size: %r does not seem to be an array of %s" %
+                        (name, tp.item))
+                tp = tp.resolve_length(length)
             tp_ptr = model.PointerType(tp.item)
             value = self._load_constant(False, tp_ptr, name, module)
             # 'value' is a <cdata 'type *'> which we have to replace with
         # remove ptr=<cdata 'int *'> from the library instance, and replace
         # it by a property on the class, which reads/writes into ptr[0].
         funcname = '_cffi_var_%s' % name
-        BFunc = self.ffi.typeof(tp.get_c_name('*(*)(void)', name))
+        BFunc = self.ffi._typeof_locked(tp.get_c_name('*(*)(void)', name))[0]
         function = module.load_function(BFunc, funcname)
         ptr = function()
         def getter(library):

lib_pypy/cffi/verifier.py

             k2 = k2.lstrip('0').rstrip('L')
             modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key,
                                               k1, k2)
-        suffix = _get_so_suffix()
+        suffix = _get_so_suffixes()[0]
         self.tmpdir = tmpdir or _caller_dir_pycache()
         self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c')
         self.modulefilename = os.path.join(self.tmpdir, modulename + suffix)
     def write_source(self, file=None):
         """Write the C source code.  It is produced in 'self.sourcefilename',
         which can be tweaked beforehand."""
-        if self._has_source and file is None:
-            raise ffiplatform.VerificationError("source code already written")
-        self._write_source(file)
+        with self.ffi._lock:
+            if self._has_source and file is None:
+                raise ffiplatform.VerificationError(
+                    "source code already written")
+            self._write_source(file)
 
     def compile_module(self):
         """Write the C source code (if not done already) and compile it.
         This produces a dynamic link library in 'self.modulefilename'."""
-        if self._has_module:
-            raise ffiplatform.VerificationError("module already compiled")
-        if not self._has_source:
-            self._write_source()
-        self._compile_module()
+        with self.ffi._lock:
+            if self._has_module:
+                raise ffiplatform.VerificationError("module already compiled")
+            if not self._has_source:
+                self._write_source()
+            self._compile_module()
 
     def load_library(self):
         """Get a C module from this Verifier instance.
         operations to the C module.  If necessary, the C code is written
         and compiled first.
         """
-        if not self._has_module:
-            self._locate_module()
+        with self.ffi._lock:
             if not self._has_module:
-                self.compile_module()
-        return self._load_library()
+                self._locate_module()
+                if not self._has_module:
+                    if not self._has_source:
+                        self._write_source()
+                    self._compile_module()
+            return self._load_library()
 
     def get_module_name(self):
         basename = os.path.basename(self.modulefilename)
 
     def get_extension(self):
         if not self._has_source:
-            self._write_source()
+            with self.ffi._lock:
+                if not self._has_source:
+                    self._write_source()
         sourcename = ffiplatform.maybe_relative_path(self.sourcefilename)
         modname = self.get_module_name()
         return ffiplatform.get_extension(sourcename, modname, **self.kwds)
             else:
                 path = None
             filename = self._vengine.find_module(self.get_module_name(), path,
-                                                 _get_so_suffix())
+                                                 _get_so_suffixes())
             if filename is None:
                 return
             self.modulefilename = filename
     if keep_so:
         suffix = '.c'   # only remove .c files
     else:
-        suffix = _get_so_suffix().lower()
+        suffix = _get_so_suffixes()[0].lower()
     for fn in filelist:
         if fn.lower().startswith('_cffi_') and (
                 fn.lower().endswith(suffix) or fn.lower().endswith('.c')):
         except OSError:
             pass
 
-def _get_so_suffix():
+def _get_so_suffixes():
+    suffixes = []
     for suffix, mode, type in imp.get_suffixes():
         if type == imp.C_EXTENSION:
-            return suffix
-    # bah, no C_EXTENSION available.  Occurs on pypy without cpyext
-    if sys.platform == 'win32':
-        return ".pyd"
-    else:
-        return ".so"
+            suffixes.append(suffix)
+
+    if not suffixes:
+        # bah, no C_EXTENSION available.  Occurs on pypy without cpyext
+        if sys.platform == 'win32':
+            suffixes = [".pyd"]
+        else:
+            suffixes = [".so"]
+
+    return suffixes
 
 def _ensure_dir(filename):
     try:

lib_pypy/pyrepl/reader.py

 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-import types
+import types, re
 from pyrepl import unicodedata_
 from pyrepl import commands
 from pyrepl import input
 
+_r_csi_seq = re.compile(r"\033\[[ -@]*[A-~]")
+
 def _make_unctrl_map():
     uc_map = {}
     for c in map(unichr, range(256)):
         excluded from the length calculation.  So also a copy of the prompt
         is returned with these control characters removed.  """
 
+        # The logic below also ignores the length of common escape
+        # sequences if they were not explicitly within \x01...\x02.
+        # They are CSI (or ANSI) sequences  ( ESC [ ... LETTER )
+
         out_prompt = ''
         l = len(prompt)
         pos = 0
                 break
             # Found start and end brackets, subtract from string length
             l = l - (e-s+1)
-            out_prompt += prompt[pos:s] + prompt[s+1:e]
+            keep = prompt[pos:s]
+            l -= sum(map(len, _r_csi_seq.findall(keep)))
+            out_prompt += keep + prompt[s+1:e]
             pos = e+1
-        out_prompt += prompt[pos:]
+        keep = prompt[pos:]
+        l -= sum(map(len, _r_csi_seq.findall(keep)))
+        out_prompt += keep
         return out_prompt, l
 
     def bow(self, p=None):

pypy/doc/contributor.rst

   David Schneider
   Holger Krekel
   Christian Tismer
+  Matti Picus
   Hakan Ardo
   Benjamin Peterson
-  Matti Picus
   Philip Jenvey
   Anders Chrigstrom
   Brian Kearns
+  Manuel Jacob
   Eric van Riet Paap
+  Wim Lavrijsen
   Richard Emslie
   Alexander Schremmer
-  Wim Lavrijsen
   Dan Villiom Podlaski Christiansen
-  Manuel Jacob
+  Ronan Lamy
   Lukas Diekmann
   Sven Hager
   Anders Lehmann
   Aurelien Campeas
   Niklaus Haldimann
-  Ronan Lamy
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
+  Romain Guillebert
   Guido Wesdorp
   Lawrence Oluyede
+  Remi Meier
   Bartosz Skowron
   Daniel Roberts
   Niko Matsakis
   Ludovic Aubry
   Alexander Hesse
   Jacob Hallen
-  Romain Guillebert
   Jason Creighton
   Alex Martelli
   Michal Bendowski
   Jan de Mooij
+  stian
   Michael Foord
   Stephan Diehl
   Stefan Schwarzer
   Valentino Volonghi
   Tomek Meka
   Patrick Maupin
-  stian
   Bob Ippolito
   Bruno Gola
   Jean-Paul Calderone
   Simon Burton
   Marius Gedminas
   John Witulski
+  Konstantin Lopuhin
   Greg Price
   Dario Bertini
   Mark Pearse
   Simon Cross
-  Konstantin Lopuhin
   Andreas Stührk
   Jean-Philippe St. Pierre
   Guido van Rossum
   Pavel Vinogradov
+  Paweł Piotr Przeradowski
   Paul deGrandis
   Ilya Osadchiy
   Adrian Kuhn
   Boris Feigin
+  Stefano Rivera
   tav
+  Taavi Burns
   Georg Brandl
   Bert Freudenberg
   Stian Andreassen
-  Stefano Rivera
   Wanja Saatkamp
   Gerald Klix
   Mike Blume
-  Taavi Burns
   Oscar Nierstrasz
+  Stefan H. Muller
+  Laurence Tratt
+  Rami Chowdhury
   David Malcolm
   Eugene Oden
   Henry Mason
   David Ripton
   Dusty Phillips
   Lukas Renggli
+  Edd Barrett
   Guenter Jantzen
   Tobias Oberstein
-  Remi Meier
   Ned Batchelder
   Amit Regmi
   Ben Young
   Nicolas Chauvat
   Andrew Durdin
+  Andrew Chambers
   Michael Schneider
   Nicholas Riley
   Jason Chu
   Olivier Dormond
   Jared Grubb
   Karl Bartel
+  Tobias Pape
   Brian Dorsey
   Victor Stinner
+  Andrews Medina
   Stuart Williams
   Jasper Schulz
+  Christian Hudon
   Toby Watson
   Antoine Pitrou
   Aaron Iles
   Neil Shepperd
   Mikael Schönenberg
   Elmo Mäntynen
-  Tobias Pape
   Jonathan David Riehl
   Stanislaw Halik
   Anders Qvist
   Alexander Sedov
   Corbin Simpson
   Christopher Pope
-  Laurence Tratt
-  Guillebert Romain
   Christian Tismer 
   Dan Stromberg
   Stefano Parmesan
-  Christian Hudon
   Alexis Daboville
   Jens-Uwe Mager
   Carl Meyer
   Karl Ramm
   Pieter Zieschang
   Gabriel
-  Paweł Piotr Przeradowski
   Andrew Dalke
   Sylvain Thenault
   Nathan Taylor
   Martin Blais
   Lene Wagner
   Tomo Cocoa
-  Andrews Medina
   roberto@goyle
+  Yury V. Zaytsev
+  Anna Katrina Dominguez
   William Leslie
   Bobby Impollonia
   timo@eistee.fritz.box
   Andrew Thompson
   Yusei Tahara
+  Ben Darnell
   Roberto De Ioris
   Juan Francisco Cantero Hurtado
   Godefroid Chappelle
   Akira Li
   Gustavo Niemeyer
   Stephan Busemann
-  Anna Katrina Dominguez
+  Rafał Gałczyński
   Christian Muirhead
   James Lan
   shoma hosaka
   Dinu Gherman
   Chris Lambacher
   coolbutuseless@gmail.com
+  w31rd0
   Jim Baker
   Rodrigo Araújo
   Armin Ronacher
   Even Wiik Thomassen
   jbs
   soareschen
+  Mike Bayer
   Flavio Percoco
   Kristoffer Kleine
   yasirs
   Michael Chermside
   Anna Ravencroft
-  Andrew Chambers
   Julien Phalip
   Dan Loewenherz
 

pypy/doc/interpreter.rst

   control flow of a function (such as ``while`` and ``try`` constructs)
 
 - a value stack where bytecode interpretation pulls object
-  from and puts results on.
+  from and puts results on.  (``locals_stack_w`` is actually a single
+  list containing both the local scope and the value stack.)
 
 - a reference to the *globals* dictionary, containing
   module-level name-value bindings
 
 - the class ``PyFrame`` is defined in `pypy/interpreter/pyframe.py`_.
 
-- the file `pypy/interpreter/pyopcode.py`_ add support for all Python opcode.
-
-- nested scope support is added to the ``PyFrame`` class in
-  `pypy/interpreter/nestedscope.py`_.
+- the file `pypy/interpreter/pyopcode.py`_ add support for all Python opcodes.
 
 .. _Code:
 
 * ``co_name`` name of the code object (often the function name)
 * ``co_lnotab`` a helper table to compute the line-numbers corresponding to bytecodes
 
-In PyPy, code objects also have the responsibility of creating their Frame_ objects
-via the `'create_frame()`` method.  With proper parser and compiler support this would
-allow to create custom Frame objects extending the execution of functions
-in various ways.  The several Frame_ classes already utilize this flexibility
-in order to implement Generators and Nested Scopes.
-
 .. _Function:
 
 Function and Method classes

pypy/doc/tool/makecontributor.py

+# NOTE: run this script with LANG=en_US.UTF-8
+
 import py
 import sys
 from collections import defaultdict
         if show_numbers:
             print '%5d %s' % (n, name)
         else:
-            print name
+            print '  ' + name
 
 if __name__ == '__main__':
     show_numbers = '-n' in sys.argv

pypy/doc/whatsnew-head.rst

 .. branch: safe-win-mmap
 .. branch: boolean-indexing-cleanup
 .. branch: cpyext-best_base
+.. branch: cpyext-int
 .. branch: fileops2
 
 .. branch: nobold-backtrace

pypy/module/__pypy__/test/test_signal.py

                 interrupted = []
                 print('--- start ---')
                 thread.start_new_thread(subthread, ())
-                for j in range(10):
+                for j in range(30):
                     if len(done): break
                     print('.')
                     time.sleep(0.25)

pypy/module/_cffi_backend/__init__.py

     appleveldefs = {
         }
     interpleveldefs = {
-        '__version__': 'space.wrap("0.7")',
+        '__version__': 'space.wrap("0.8")',
 
         'load_library': 'libraryobj.load_library',
 

pypy/module/_cffi_backend/cdataobj.py

     _cdata = lltype.nullptr(rffi.CCHARP.TO)
 
     def __init__(self, space, cdata, ctype):
-        from pypy.module._cffi_backend import ctypeprim
+        from pypy.module._cffi_backend import ctypeobj
         assert lltype.typeOf(cdata) == rffi.CCHARP
-        assert isinstance(ctype, ctypeprim.W_CType)
+        assert isinstance(ctype, ctypeobj.W_CType)
         self.space = space
         self._cdata = cdata    # don't forget keepalive_until_here!
         self.ctype = ctype
                 keepalive_until_here(w_value)
                 return
         #
+        # A fast path for <char[]>[0:N] = "somestring".
+        from pypy.module._cffi_backend import ctypeprim
         space = self.space
+        if (space.isinstance_w(w_value, space.w_str) and
+                isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar)):
+            from rpython.rtyper.annlowlevel import llstr
+            from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
+            value = space.str_w(w_value)
+            if len(value) != length:
+                raise operationerrfmt(space.w_ValueError,
+                                      "need a string of length %d, got %d",
+                                      length, len(value))
+            copy_string_to_raw(llstr(value), cdata, 0, length)
+            return
+        #
         w_iter = space.iter(w_value)
         for i in range(length):
             try:
         space = self.space
         if isinstance(w_other, W_CData):
             from pypy.module._cffi_backend import ctypeptr, ctypearray
+            from pypy.module._cffi_backend import ctypevoid
             ct = w_other.ctype
             if isinstance(ct, ctypearray.W_CTypeArray):
                 ct = ct.ctptr
             #
             if (ct is not self.ctype or
                    not isinstance(ct, ctypeptr.W_CTypePointer) or
-                   ct.ctitem.size <= 0):
+                   (ct.ctitem.size <= 0 and not ct.is_void_ptr)):
                 raise operationerrfmt(space.w_TypeError,
                     "cannot subtract cdata '%s' and cdata '%s'",
                     self.ctype.name, ct.name)
             #
+            itemsize = ct.ctitem.size
+            if itemsize <= 0: itemsize = 1
             diff = (rffi.cast(lltype.Signed, self._cdata) -
-                    rffi.cast(lltype.Signed, w_other._cdata)) // ct.ctitem.size
+                    rffi.cast(lltype.Signed, w_other._cdata)) // itemsize
             return space.wrap(diff)
         #
         return self._add_or_sub(w_other, -1)
     __getitem__ = interp2app(W_CData.getitem),
     __setitem__ = interp2app(W_CData.setitem),
     __add__ = interp2app(W_CData.add),
+    __radd__ = interp2app(W_CData.add),
     __sub__ = interp2app(W_CData.sub),
     __getattr__ = interp2app(W_CData.getattr),
     __setattr__ = interp2app(W_CData.setattr),

pypy/module/_cffi_backend/ctypearray.py

         datasize = self.size
         #
         if datasize < 0:
-            if (space.isinstance_w(w_init, space.w_list) or
-                space.isinstance_w(w_init, space.w_tuple)):
-                length = space.int_w(space.len(w_init))
-            elif space.isinstance_w(w_init, space.w_basestring):
-                # from a string, we add the null terminator
-                length = space.int_w(space.len(w_init)) + 1
-            else:
-                length = space.getindex_w(w_init, space.w_OverflowError)
-                if length < 0:
-                    raise OperationError(space.w_ValueError,
-                                         space.wrap("negative array length"))
-                w_init = space.w_None
-            #
+            from pypy.module._cffi_backend import misc
+            w_init, length = misc.get_new_array_length(space, w_init)
             try:
                 datasize = ovfcheck(length * self.ctitem.size)
             except OverflowError:

pypy/module/_cffi_backend/ctypeptr.py

 Pointers.
 """
 
-from pypy.interpreter.error import OperationError, operationerrfmt, wrap_oserror
-
 from rpython.rlib import rposix
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rarithmetic import ovfcheck
+from rpython.rtyper.annlowlevel import llstr, llunicode
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw, copy_unicode_to_raw
 
+from pypy.interpreter.error import OperationError, operationerrfmt, wrap_oserror
 from pypy.module._cffi_backend import cdataobj, misc, ctypeprim, ctypevoid
 from pypy.module._cffi_backend.ctypeobj import W_CType
 
 
 class W_CTypePtrOrArray(W_CType):
-    _attrs_            = ['ctitem', 'can_cast_anything', 'is_struct_ptr',
-                          'length']
-    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'is_struct_ptr',
-                          'length']
+    _attrs_            = ['ctitem', 'can_cast_anything', 'length']
+    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'length']
     length = -1
 
     def __init__(self, space, size, extra, extra_position, ctitem,
                  could_cast_anything=True):
-        from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
         name, name_position = ctitem.insert_name(extra, extra_position)
         W_CType.__init__(self, space, size, name, name_position)
         # this is the "underlying type":
         #  - for functions, it is the return type
         self.ctitem = ctitem
         self.can_cast_anything = could_cast_anything and ctitem.cast_anything
-        self.is_struct_ptr = isinstance(ctitem, W_CTypeStructOrUnion)
 
     def is_char_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveChar)
                                       "initializer string is too long for '%s'"
                                       " (got %d characters)",
                                       self.name, n)
-            for i in range(n):
-                cdata[i] = s[i]
+            copy_string_to_raw(llstr(s), cdata, 0, n)
             if n != self.length:
                 cdata[n] = '\x00'
         elif isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar):
                                       " (got %d characters)",
                                       self.name, n)
             unichardata = rffi.cast(rffi.CWCHARP, cdata)
-            for i in range(n):
-                unichardata[i] = s[i]
+            copy_unicode_to_raw(llunicode(s), unichardata, 0, n)
             if n != self.length:
                 unichardata[n] = u'\x00'
         else:
         return cdataobj.W_CData(self.space, ptrdata, self)
 
     def convert_from_object(self, cdata, w_ob):
-        space = self.space
         if not isinstance(w_ob, cdataobj.W_CData):
             raise self._convert_error("cdata pointer", w_ob)
         other = w_ob.ctype
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
     def newp(self, w_init):
+        from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
         space = self.space
         ctitem = self.ctitem
         datasize = ctitem.size
             raise operationerrfmt(space.w_TypeError,
                 "cannot instantiate ctype '%s' of unknown size",
                                   self.name)
-        if self.is_struct_ptr:
+        if isinstance(ctitem, W_CTypeStructOrUnion):
             # 'newp' on a struct-or-union pointer: in this case, we return
             # a W_CDataPtrToStruct object which has a strong reference
             # to a W_CDataNewOwning that really contains the structure.
+            #
+            if ctitem.with_var_array and not space.is_w(w_init, space.w_None):
+                datasize = ctitem.convert_struct_from_object(
+                    lltype.nullptr(rffi.CCHARP.TO), w_init, datasize)
+            #
             cdatastruct = cdataobj.W_CDataNewOwning(space, datasize, ctitem)
             cdata = cdataobj.W_CDataPtrToStructOrUnion(space,
                                                        cdatastruct._cdata,
     def add(self, cdata, i):
         space = self.space
         ctitem = self.ctitem
+        itemsize = ctitem.size
         if ctitem.size < 0:
-            raise operationerrfmt(space.w_TypeError,
+            if self.is_void_ptr:
+                itemsize = 1
+            else:
+                raise operationerrfmt(space.w_TypeError,
                                   "ctype '%s' points to items of unknown size",
                                   self.name)
-        p = rffi.ptradd(cdata, i * self.ctitem.size)
+        p = rffi.ptradd(cdata, i * itemsize)
         return cdataobj.W_CData(space, p, self)
 
     def cast(self, w_ob):
 
     def convert_argument_from_object(self, cdata, w_ob):
         from pypy.module._cffi_backend.ctypefunc import set_mustfree_flag
-        space = self.space
         result = (not isinstance(w_ob, cdataobj.W_CData) and
                   self._prepare_pointer_call_argument(w_ob, cdata))
         if result == 0:
         space = self.space
         ctype2 = cdata.ctype
         if (isinstance(ctype2, W_CTypeStructOrUnion) or
-            (isinstance(ctype2, W_CTypePtrOrArray) and ctype2.is_struct_ptr)):
+               (isinstance(ctype2, W_CTypePtrOrArray) and
+                isinstance(ctype2.ctitem, W_CTypeStructOrUnion))):
             ptrdata = rffi.ptradd(cdata._cdata, offset)
             return cdataobj.W_CData(space, ptrdata, self)
         else:

pypy/module/_cffi_backend/ctypestruct.py

 from rpython.rlib import jit
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong, intmask
-from rpython.rtyper.lltypesystem import rffi
+from rpython.rlib.rarithmetic import ovfcheck
+from rpython.rtyper.lltypesystem import lltype, rffi
 
 from pypy.module._cffi_backend import cdataobj, ctypeprim, misc
 from pypy.module._cffi_backend.ctypeobj import W_CType
 
 class W_CTypeStructOrUnion(W_CType):
     _immutable_fields_ = ['alignment?', 'fields_list?', 'fields_dict?',
-                          'custom_field_pos?']
+                          'custom_field_pos?', 'with_var_array?']
     # fields added by complete_struct_or_union():
     alignment = -1
     fields_list = None
     fields_dict = None
     custom_field_pos = False
+    with_var_array = False
 
     def __init__(self, space, name):
         W_CType.__init__(self, space, -1, name, len(name))
         pass
 
     def convert_from_object(self, cdata, w_ob):
-        space = self.space
-        if self._copy_from_same(cdata, w_ob):
-            return
+        if not self._copy_from_same(cdata, w_ob):
+            self.convert_struct_from_object(cdata, w_ob, optvarsize=-1)
 
+    @jit.look_inside_iff(
+        lambda self, cdata, w_ob, optvarsize: jit.isvirtual(w_ob)
+    )
+    def convert_struct_from_object(self, cdata, w_ob, optvarsize):
         self._check_only_one_argument_for_union(w_ob)
 
+        space = self.space
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
             lst_w = space.listview(w_ob)
                         "too many initializers for '%s' (got %d)",
                                       self.name, len(lst_w))
             for i in range(len(lst_w)):
-                self.fields_list[i].write(cdata, lst_w[i])
+                optvarsize = self.fields_list[i].write_v(cdata, lst_w[i],
+                                                         optvarsize)
+            return optvarsize
 
         elif space.isinstance_w(w_ob, space.w_dict):
             lst_w = space.fixedview(w_ob)
                 except KeyError:
                     space.raise_key_error(w_key)
                     assert 0
-                cf.write(cdata, space.getitem(w_ob, w_key))
+                optvarsize = cf.write_v(cdata, space.getitem(w_ob, w_key),
+                                        optvarsize)
+            return optvarsize
 
         else:
-            raise self._convert_error("list or tuple or dict or struct-cdata",
-                                      w_ob)
+            if optvarsize == -1:
+                msg = "list or tuple or dict or struct-cdata"
+            else:
+                msg = "list or tuple or dict"
+            raise self._convert_error(msg, w_ob)
 
     @jit.elidable
     def _getcfield_const(self, attr):
         else:
             self.ctype.convert_from_object(cdata, w_ob)
 
+    def write_v(self, cdata, w_ob, optvarsize):
+        # a special case for var-sized C99 arrays
+        from pypy.module._cffi_backend import ctypearray
+        ct = self.ctype
+        if isinstance(ct, ctypearray.W_CTypeArray) and ct.length < 0:
+            space = ct.space
+            w_ob, varsizelength = misc.get_new_array_length(space, w_ob)
+            if optvarsize != -1:
+                # in this mode, the only purpose of this function is to compute
+                # the real size of the structure from a var-sized C99 array
+                assert cdata == lltype.nullptr(rffi.CCHARP.TO)
+                itemsize = ct.ctitem.size
+                try:
+                    varsize = ovfcheck(itemsize * varsizelength)
+                    size = ovfcheck(self.offset + varsize)
+                except OverflowError:
+                    raise OperationError(space.w_OverflowError,
+                        space.wrap("array size would overflow a ssize_t"))
+                assert size >= 0
+                return max(size, optvarsize)
+            # if 'value' was only an integer, get_new_array_length() returns
+            # w_ob = space.w_None.  Detect if this was the case,
+            # and if so, stop here, leaving the content uninitialized
+            # (it should be zero-initialized from somewhere else).
+            if space.is_w(w_ob, space.w_None):
+                return optvarsize
+        #
+        if optvarsize == -1:
+            self.write(cdata, w_ob)
+        return optvarsize
+
     def convert_bitfield_to_object(self, cdata):
         ctype = self.ctype
         space = ctype.space

pypy/module/_cffi_backend/misc.py

 
 # ____________________________________________________________
 
+def get_new_array_length(space, w_value):
+    if (space.isinstance_w(w_value, space.w_list) or
+        space.isinstance_w(w_value, space.w_tuple)):
+        return (w_value, space.int_w(space.len(w_value)))
+    elif space.isinstance_w(w_value, space.w_basestring):
+        # from a string, we add the null terminator
+        return (w_value, space.int_w(space.len(w_value)) + 1)
+    else:
+        explicitlength = space.getindex_w(w_value, space.w_OverflowError)
+        if explicitlength < 0:
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("negative array length"))
+        return (space.w_None, explicitlength)
+
+# ____________________________________________________________
+
 @specialize.arg(0)
 def _raw_memcopy_tp(TPP, source, dest):
     # in its own function: LONGLONG may make the whole function jit-opaque