Commits

mattip  committed 7dd1946 Merge

merge default into branch

  • Participants
  • Parent commits d2bfba5, ca509b8
  • Branches stdlib-2.7.5

Comments (0)

Files changed (50)

File lib_pypy/cffi.egg-info

 Metadata-Version: 1.0
 Name: cffi
-Version: 0.7
+Version: 0.8
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski

File lib_pypy/cffi/__init__.py

 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "0.7.2"
-__version_info__ = (0, 7, 2)
+__version__ = "0.8"
+__version_info__ = (0, 8)

File lib_pypy/cffi/api.py

 import types
+from .lock import allocate_lock
 
 try:
     callable
             # rely on it!  It's probably not going to work well.)
 
         self._backend = backend
+        self._lock = allocate_lock()
         self._parser = cparser.Parser()
         self._cached_btypes = {}
         self._parsed_types = types.ModuleType('parsed_types').__dict__
             if name.startswith('RTLD_'):
                 setattr(self, name, getattr(backend, name))
         #
-        self.BVoidP = self._get_cached_btype(model.voidp_type)
+        with self._lock:
+            self.BVoidP = self._get_cached_btype(model.voidp_type)
         if isinstance(backend, types.ModuleType):
             # _cffi_backend: attach these constants to the class
             if not hasattr(FFI, 'NULL'):
             if not isinstance(csource, basestring):
                 raise TypeError("cdef() argument must be a string")
             csource = csource.encode('ascii')
-        self._parser.parse(csource, override=override)
-        self._cdefsources.append(csource)
-        if override:
-            for cache in self._function_caches:
-                cache.clear()
+        with self._lock:
+            self._parser.parse(csource, override=override)
+            self._cdefsources.append(csource)
+            if override:
+                for cache in self._function_caches:
+                    cache.clear()
 
     def dlopen(self, name, flags=0):
         """Load and return a dynamic library identified by 'name'.
         library we only look for the actual (untyped) symbols.
         """
         assert isinstance(name, basestring) or name is None
-        lib, function_cache = _make_ffi_library(self, name, flags)
-        self._function_caches.append(function_cache)
-        self._libraries.append(lib)
+        with self._lock:
+            lib, function_cache = _make_ffi_library(self, name, flags)
+            self._function_caches.append(function_cache)
+            self._libraries.append(lib)
         return lib
 
+    def _typeof_locked(self, cdecl):
+        # call me with the lock!
+        key = cdecl
+        if key in self._parsed_types:
+            return self._parsed_types[key]
+        #
+        if not isinstance(cdecl, str):    # unicode, on Python 2
+            cdecl = cdecl.encode('ascii')
+        #
+        type = self._parser.parse_type(cdecl)
+        if hasattr(type, 'as_function_pointer'):
+            really_a_function_type = True
+            type = type.as_function_pointer()
+        else:
+            really_a_function_type = False
+        btype = self._get_cached_btype(type)
+        result = btype, really_a_function_type
+        self._parsed_types[key] = result
+        return result
+
     def _typeof(self, cdecl, consider_function_as_funcptr=False):
         # string -> ctype object
         try:
-            btype, cfaf = self._parsed_types[cdecl]
-            if consider_function_as_funcptr and not cfaf:
-                raise KeyError
+            result = self._parsed_types[cdecl]
         except KeyError:
-            key = cdecl
-            if not isinstance(cdecl, str):    # unicode, on Python 2
-                cdecl = cdecl.encode('ascii')
-            cfaf = consider_function_as_funcptr
-            type = self._parser.parse_type(cdecl,
-                       consider_function_as_funcptr=cfaf)
-            btype = self._get_cached_btype(type)
-            self._parsed_types[key] = btype, cfaf
+            with self._lock:
+                result = self._typeof_locked(cdecl)
+        #
+        btype, really_a_function_type = result
+        if really_a_function_type and not consider_function_as_funcptr:
+            raise CDefError("the type %r is a function type, not a "
+                            "pointer-to-function type" % (cdecl,))
         return btype
 
     def typeof(self, cdecl):
         """Parse the C type given as a string and return the
-        corresponding Python type: <class 'ffi.CData<...>'>.
+        corresponding <ctype> object.
         It can also be used on 'cdata' instance to get its C type.
         """
         if isinstance(cdecl, basestring):
             res = _builtin_function_type(cdecl)
             if res is not None:
                 return res
+        if (isinstance(cdecl, types.FunctionType)
+                and hasattr(cdecl, '_cffi_base_type')):
+            with self._lock:
+                return self._get_cached_btype(cdecl._cffi_base_type)
         raise TypeError(type(cdecl))
 
     def sizeof(self, cdecl):
         data.  Later, when this new cdata object is garbage-collected,
         'destructor(old_cdata_object)' will be called.
         """
-        try:
-            gc_weakrefs = self.gc_weakrefs
-        except AttributeError:
-            from .gc_weakref import GcWeakrefs
-            gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self)
-        return gc_weakrefs.build(cdata, destructor)
+        with self._lock:
+            try:
+                gc_weakrefs = self.gc_weakrefs
+            except AttributeError:
+                from .gc_weakref import GcWeakrefs
+                gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self)
+            return gc_weakrefs.build(cdata, destructor)
 
     def _get_cached_btype(self, type):
+        assert self._lock.acquire(False) is False
+        # call me with the lock!
         try:
             BType = self._cached_btypes[type]
         except KeyError:
 
     def _pointer_to(self, ctype):
         from . import model
-        return model.pointer_cache(self, ctype)
+        with self._lock:
+            return model.pointer_cache(self, ctype)
 
     def addressof(self, cdata, field=None):
         """Return the address of a <cdata 'struct-or-union'>.
         variables, which must anyway be accessed directly from the
         lib object returned by the original FFI instance.
         """
-        self._parser.include(ffi_to_include._parser)
-        self._cdefsources.append('[')
-        self._cdefsources.extend(ffi_to_include._cdefsources)
-        self._cdefsources.append(']')
+        with ffi_to_include._lock:
+            with self._lock:
+                self._parser.include(ffi_to_include._parser)
+                self._cdefsources.append('[')
+                self._cdefsources.extend(ffi_to_include._cdefsources)
+                self._cdefsources.append(']')
 
     def new_handle(self, x):
         return self._backend.newp_handle(self.BVoidP, x)
         backendlib = backend.load_library(path, flags)
     copied_enums = []
     #
-    def make_accessor(name):
+    def make_accessor_locked(name):
         key = 'function ' + name
         if key in ffi._parser._declarations:
             tp = ffi._parser._declarations[key]
                     if enumname not in library.__dict__:
                         library.__dict__[enumname] = enumval
             copied_enums.append(True)
+            if name in library.__dict__:
+                return
         #
-        if name in library.__dict__:   # copied from an enum value just above,
-            return                     # or multithread's race condition
         raise AttributeError(name)
     #
+    def make_accessor(name):
+        with ffi._lock:
+            if name in library.__dict__ or name in FFILibrary.__dict__:
+                return    # added by another thread while waiting for the lock
+            make_accessor_locked(name)
+    #
     class FFILibrary(object):
         def __getattr__(self, name):
             make_accessor(name)
     except (KeyError, AttributeError, TypeError):
         return None
     else:
-        return ffi._get_cached_btype(tp)
+        with ffi._lock:
+            return ffi._get_cached_btype(tp)

File lib_pypy/cffi/cparser.py

                 if 1 <= linenum <= len(csourcelines):
                     line = csourcelines[linenum-1]
         if line:
-            msg = 'cannot parse "%s"\n%s' % (line, msg)
+            msg = 'cannot parse "%s"\n%s' % (line.strip(), msg)
         else:
             msg = 'parse error\n%s' % (msg,)
         raise api.CDefError(msg)
             #
             if decl.name:
                 tp = self._get_type(node, partial_length_ok=True)
-                if self._is_constant_declaration(node):
+                if self._is_constant_globalvar(node):
                     self._declare('constant ' + decl.name, tp)
                 else:
                     self._declare('variable ' + decl.name, tp)
 
-    def parse_type(self, cdecl, consider_function_as_funcptr=False):
+    def parse_type(self, cdecl):
         ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl)
         assert not macros
         exprnode = ast.ext[-1].type.args.params[0]
         if isinstance(exprnode, pycparser.c_ast.ID):
             raise api.CDefError("unknown identifier '%s'" % (exprnode.name,))
-        return self._get_type(exprnode.type,
-                     consider_function_as_funcptr=consider_function_as_funcptr)
+        return self._get_type(exprnode.type)
 
     def _declare(self, name, obj):
         if name in self._declarations:
             return model.ConstPointerType(type)
         return model.PointerType(type)
 
-    def _get_type(self, typenode, convert_array_to_pointer=False,
-                  name=None, partial_length_ok=False,
-                  consider_function_as_funcptr=False):
+    def _get_type(self, typenode, name=None, partial_length_ok=False):
         # first, dereference typedefs, if we have it already parsed, we're good
         if (isinstance(typenode, pycparser.c_ast.TypeDecl) and
             isinstance(typenode.type, pycparser.c_ast.IdentifierType) and
             len(typenode.type.names) == 1 and
             ('typedef ' + typenode.type.names[0]) in self._declarations):
             type = self._declarations['typedef ' + typenode.type.names[0]]
-            if isinstance(type, model.ArrayType):
-                if convert_array_to_pointer:
-                    return type.item
-            else:
-                if (consider_function_as_funcptr and
-                        isinstance(type, model.RawFunctionType)):
-                    return type.as_function_pointer()
             return type
         #
         if isinstance(typenode, pycparser.c_ast.ArrayDecl):
             # array type
-            if convert_array_to_pointer:
-                return self._get_type_pointer(self._get_type(typenode.type))
             if typenode.dim is None:
                 length = None
             else:
         #
         if isinstance(typenode, pycparser.c_ast.FuncDecl):
             # a function type
-            result = self._parse_function_type(typenode, name)
-            if consider_function_as_funcptr:
-                result = result.as_function_pointer()
-            return result
+            return self._parse_function_type(typenode, name)
         #
         # nested anonymous structs or unions end up here
         if isinstance(typenode, pycparser.c_ast.Struct):
             isinstance(params[0].type.type, pycparser.c_ast.IdentifierType)
                 and list(params[0].type.type.names) == ['void']):
             del params[0]
-        args = [self._get_type(argdeclnode.type,
-                               convert_array_to_pointer=True,
-                               consider_function_as_funcptr=True)
+        args = [self._as_func_arg(self._get_type(argdeclnode.type))
                 for argdeclnode in params]
         result = self._get_type(typenode.type)
         return model.RawFunctionType(tuple(args), result, ellipsis)
 
-    def _is_constant_declaration(self, typenode, const=False):
-        if isinstance(typenode, pycparser.c_ast.ArrayDecl):
-            return self._is_constant_declaration(typenode.type)
+    def _as_func_arg(self, type):
+        if isinstance(type, model.ArrayType):
+            return model.PointerType(type.item)
+        elif isinstance(type, model.RawFunctionType):
+            return type.as_function_pointer()
+        else:
+            return type
+
+    def _is_constant_globalvar(self, typenode):
         if isinstance(typenode, pycparser.c_ast.PtrDecl):
-            const = 'const' in typenode.quals
-            return self._is_constant_declaration(typenode.type, const)
+            return 'const' in typenode.quals
         if isinstance(typenode, pycparser.c_ast.TypeDecl):
-            return const or 'const' in typenode.quals
+            return 'const' in typenode.quals
         return False
 
     def _get_struct_union_enum_type(self, kind, type, name=None, nested=False):
         return tp
 
     def _make_partial(self, tp, nested):
-        if not isinstance(tp, model.StructType):
+        if not isinstance(tp, model.StructOrUnion):
             raise api.CDefError("%s cannot be partial" % (tp,))
         if not tp.has_c_name() and not nested:
             raise NotImplementedError("%s is partial but has no C name" %(tp,))
             if (isinstance(exprnode, pycparser.c_ast.ID) and
                     exprnode.name == '__dotdotdotarray__'):
                 self._partial_length = True
-                return None
+                return '...'
         #
         raise api.FFIError("unsupported expression: expected a "
                            "simple numeric constant")

File lib_pypy/cffi/gc_weakref.py

 
     def build(self, cdata, destructor):
         # make a new cdata of the same type as the original one
-        new_cdata = self.ffi.cast(self.ffi.typeof(cdata), cdata)
+        new_cdata = self.ffi.cast(self.ffi._backend.typeof(cdata), cdata)
         self.data[ref(new_cdata, self.remove)] = destructor, cdata
         return new_cdata

File lib_pypy/cffi/lock.py

+import sys
+
+if sys.version_info < (3,):
+    try:
+        from thread import allocate_lock
+    except ImportError:
+        from dummy_thread import allocate_lock
+else:
+    try:
+        from _thread import allocate_lock
+    except ImportError:
+        from _dummy_thread import allocate_lock
+
+
+##import sys
+##l1 = allocate_lock
+
+##class allocate_lock(object):
+##    def __init__(self):
+##        self._real = l1()
+##    def __enter__(self):
+##        for i in range(4, 0, -1):
+##            print sys._getframe(i).f_code
+##        print
+##        return self._real.__enter__()
+##    def __exit__(self, *args):
+##        return self._real.__exit__(*args)
+##    def acquire(self, f):
+##        assert f is False
+##        return self._real.acquire(f)

File lib_pypy/cffi/model.py

     _base_pattern       = " const *&"
     _base_pattern_array = "(const *&)"
 
-    def build_backend_type(self, ffi, finishlist):
-        BPtr = PointerType(self.totype).get_cached_btype(ffi, finishlist)
-        return BPtr
-
 const_voidp_type = ConstPointerType(void_type)
 
 
         self.item = item
         self.length = length
         #
-        if self.length is None:
+        if length is None or length == '...':
             brackets = '&[]'
         else:
-            brackets = '&[%d]' % self.length
+            brackets = '&[%d]' % length
         self.c_name_with_marker = (
             self.item.c_name_with_marker.replace('&', brackets))
 
         return ArrayType(self.item, newlength)
 
     def build_backend_type(self, ffi, finishlist):
+        if self.length == '...':
+            from . import api
+            raise api.CDefError("cannot render the type %r: unknown length" %
+                                (self,))
         self.item.get_cached_btype(ffi, finishlist)   # force the item BType
         BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist)
         return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length)
 class StructOrUnion(StructOrUnionOrEnum):
     fixedlayout = None
     completed = False
+    partial = False
 
     def __init__(self, name, fldnames, fldtypes, fldbitsize):
         self.name = name
             return    # not completing it: it's an opaque struct
         #
         self.completed = 1
-        fldtypes = tuple(tp.get_cached_btype(ffi, finishlist)
-                         for tp in self.fldtypes)
         #
         if self.fixedlayout is None:
+            fldtypes = [tp.get_cached_btype(ffi, finishlist)
+                        for tp in self.fldtypes]
             lst = list(zip(self.fldnames, fldtypes, self.fldbitsize))
             ffi._backend.complete_struct_or_union(BType, lst, self)
             #
         else:
+            fldtypes = []
             fieldofs, fieldsize, totalsize, totalalignment = self.fixedlayout
             for i in range(len(self.fldnames)):
                 fsize = fieldsize[i]
                 ftype = self.fldtypes[i]
                 #
-                if isinstance(ftype, ArrayType) and ftype.length is None:
+                if isinstance(ftype, ArrayType) and ftype.length == '...':
                     # fix the length to match the total size
                     BItemType = ftype.item.get_cached_btype(ffi, finishlist)
                     nlen, nrest = divmod(fsize, ffi.sizeof(BItemType))
                     ftype = ftype.resolve_length(nlen)
                     self.fldtypes = (self.fldtypes[:i] + (ftype,) +
                                      self.fldtypes[i+1:])
-                    BArrayType = ftype.get_cached_btype(ffi, finishlist)
-                    fldtypes = (fldtypes[:i] + (BArrayType,) +
-                                fldtypes[i+1:])
-                    continue
                 #
-                bitemsize = ffi.sizeof(fldtypes[i])
-                if bitemsize != fsize:
-                    self._verification_error(
-                        "field '%s.%s' is declared as %d bytes, but is "
-                        "really %d bytes" % (self.name,
-                                             self.fldnames[i] or '{}',
-                                             bitemsize, fsize))
+                BFieldType = ftype.get_cached_btype(ffi, finishlist)
+                if isinstance(ftype, ArrayType) and ftype.length is None:
+                    assert fsize == 0
+                else:
+                    bitemsize = ffi.sizeof(BFieldType)
+                    if bitemsize != fsize:
+                        self._verification_error(
+                            "field '%s.%s' is declared as %d bytes, but is "
+                            "really %d bytes" % (self.name,
+                                                 self.fldnames[i] or '{}',
+                                                 bitemsize, fsize))
+                fldtypes.append(BFieldType)
+            #
             lst = list(zip(self.fldnames, fldtypes, self.fldbitsize, fieldofs))
             ffi._backend.complete_struct_or_union(BType, lst, self,
                                                   totalsize, totalalignment)
         from .ffiplatform import VerificationError
         raise VerificationError(msg)
 
-
-class StructType(StructOrUnion):
-    kind = 'struct'
-    partial = False
-
     def check_not_partial(self):
         if self.partial and self.fixedlayout is None:
             from . import ffiplatform
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
-        
-        return global_cache(self, ffi, 'new_struct_type',
+        #
+        return global_cache(self, ffi, 'new_%s_type' % self.kind,
                             self.get_official_name(), key=self)
 
 
+class StructType(StructOrUnion):
+    kind = 'struct'
+
+
 class UnionType(StructOrUnion):
     kind = 'union'
 
-    def build_backend_type(self, ffi, finishlist):
-        finishlist.append(self)
-        return global_cache(self, ffi, 'new_union_type',
-                            self.get_official_name(), key=self)
-
 
 class EnumType(StructOrUnionOrEnum):
     kind = 'enum'
         self.baseinttype = baseinttype
         self.build_c_name_with_marker()
 
+    def force_the_name(self, forcename):
+        StructOrUnionOrEnum.force_the_name(self, forcename)
+        if self.forcename is None:
+            name = self.get_official_name()
+            self.forcename = '$' + name.replace(' ', '_')
+
     def check_not_partial(self):
         if self.partial and not self.partial_resolved:
             from . import ffiplatform

File lib_pypy/cffi/vengine_cpy.py

     def patch_extension_kwds(self, kwds):
         pass
 
-    def find_module(self, module_name, path, so_suffix):
+    def find_module(self, module_name, path, so_suffixes):
         try:
             f, filename, descr = imp.find_module(module_name, path)
         except ImportError:
         # Note that after a setuptools installation, there are both .py
         # and .so files with the same basename.  The code here relies on
         # imp.find_module() locating the .so in priority.
-        if descr[0] != so_suffix:
+        if descr[0] not in so_suffixes:
             return None
         return filename
 
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(tp))
         elif isinstance(tp, model.ArrayType):
-            return '_cffi_from_c_deref((char *)%s, _cffi_type(%d))' % (
-                var, self._gettypenum(tp))
+            return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
+                var, self._gettypenum(model.PointerType(tp.item)))
         elif isinstance(tp, model.StructType):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
         prnt('  static Py_ssize_t nums[] = {')
         prnt('    sizeof(%s),' % cname)
         prnt('    offsetof(struct _cffi_aligncheck, y),')
-        for fname, _, fbitsize in tp.enumfields():
+        for fname, ftype, fbitsize in tp.enumfields():
             if fbitsize >= 0:
                 continue      # xxx ignore fbitsize for now
             prnt('    offsetof(%s, %s),' % (cname, fname))
-            prnt('    sizeof(((%s *)0)->%s),' % (cname, fname))
+            if isinstance(ftype, model.ArrayType) and ftype.length is None:
+                prnt('    0,  /* %s */' % ftype._get_c_name())
+            else:
+                prnt('    sizeof(((%s *)0)->%s),' % (cname, fname))
         prnt('    -1')
         prnt('  };')
         prnt('  return _cffi_get_struct_layout(nums);')
         #
         function = getattr(module, layoutfuncname)
         layout = function()
-        if isinstance(tp, model.StructType) and tp.partial:
+        if isinstance(tp, model.StructOrUnion) and tp.partial:
             # use the function()'s sizes and offsets to guide the
             # layout of the struct
             totalsize = layout[0]
                     continue        # xxx ignore fbitsize for now
                 check(layout[i], ffi.offsetof(BStruct, fname),
                       "wrong offset for field %r" % (fname,))
-                BField = ffi._get_cached_btype(ftype)
-                check(layout[i+1], ffi.sizeof(BField),
-                      "wrong size for field %r" % (fname,))
+                if layout[i+1] != 0:
+                    BField = ffi._get_cached_btype(ftype)
+                    check(layout[i+1], ffi.sizeof(BField),
+                          "wrong size for field %r" % (fname,))
                 i += 2
             assert i == len(layout)
 
     # constants, likely declared with '#define'
 
     def _generate_cpy_const(self, is_int, name, tp=None, category='const',
-                            vartp=None, delayed=True):
+                            vartp=None, delayed=True, size_too=False):
         prnt = self._prnt
         funcname = '_cffi_%s_%s' % (category, name)
         prnt('static int %s(PyObject *lib)' % funcname)
                  '(unsigned long long)(%s));' % (name,))
         prnt('  if (o == NULL)')
         prnt('    return -1;')
+        if size_too:
+            prnt('  {')
+            prnt('    PyObject *o1 = o;')
+            prnt('    o = Py_BuildValue("On", o1, (Py_ssize_t)sizeof(%s));'
+                 % (name,))
+            prnt('    Py_DECREF(o1);')
+            prnt('    if (o == NULL)')
+            prnt('      return -1;')
+            prnt('  }')
         prnt('  res = PyObject_SetAttrString(lib, "%s", o);' % name)
         prnt('  Py_DECREF(o);')
         prnt('  if (res < 0)')
 
     def _generate_cpy_variable_collecttype(self, tp, name):
         if isinstance(tp, model.ArrayType):
-            self._do_collect_type(tp)
+            tp_ptr = model.PointerType(tp.item)
         else:
             tp_ptr = model.PointerType(tp)
-            self._do_collect_type(tp_ptr)
+        self._do_collect_type(tp_ptr)
 
     def _generate_cpy_variable_decl(self, tp, name):
         if isinstance(tp, model.ArrayType):
             tp_ptr = model.PointerType(tp.item)
-            self._generate_cpy_const(False, name, tp, vartp=tp_ptr)
+            self._generate_cpy_const(False, name, tp, vartp=tp_ptr,
+                                     size_too = (tp.length == '...'))
         else:
             tp_ptr = model.PointerType(tp)
             self._generate_cpy_const(False, name, tp_ptr, category='var')
     _loading_cpy_variable = _loaded_noop
 
     def _loaded_cpy_variable(self, tp, name, module, library):
+        value = getattr(library, name)
         if isinstance(tp, model.ArrayType):   # int a[5] is "constant" in the
-            return                            # sense that "a=..." is forbidden
+                                              # sense that "a=..." is forbidden
+            if tp.length == '...':
+                assert isinstance(value, tuple)
+                (value, size) = value
+                BItemType = self.ffi._get_cached_btype(tp.item)
+                length, rest = divmod(size, self.ffi.sizeof(BItemType))
+                if rest != 0:
+                    raise ffiplatform.VerificationError(
+                        "bad size: %r does not seem to be an array of %s" %
+                        (name, tp.item))
+                tp = tp.resolve_length(length)
+            # 'value' is a <cdata 'type *'> which we have to replace with
+            # a <cdata 'type[N]'> if the N is actually known
+            if tp.length is not None:
+                BArray = self.ffi._get_cached_btype(tp)
+                value = self.ffi.cast(BArray, value)
+                setattr(library, name, value)
+            return
         # remove ptr=<cdata 'int *'> from the library instance, and replace
         # it by a property on the class, which reads/writes into ptr[0].
-        ptr = getattr(library, name)
+        ptr = value
         delattr(library, name)
         def getter(library):
             return ptr[0]

File lib_pypy/cffi/vengine_gen.py

         # up in kwds['export_symbols'].
         kwds.setdefault('export_symbols', self.export_symbols)
 
-    def find_module(self, module_name, path, so_suffix):
-        basename = module_name + so_suffix
-        if path is None:
-            path = sys.path
-        for dirname in path:
-            filename = os.path.join(dirname, basename)
-            if os.path.isfile(filename):
-                return filename
-        return None
+    def find_module(self, module_name, path, so_suffixes):
+        for so_suffix in so_suffixes:
+            basename = module_name + so_suffix
+            if path is None:
+                path = sys.path
+            for dirname in path:
+                filename = os.path.join(dirname, basename)
+                if os.path.isfile(filename):
+                    return filename
 
     def collect_types(self):
         pass      # not needed in the generic engine
             newfunction = self._load_constant(False, tp, name, module)
         else:
             indirections = []
+            base_tp = tp
             if any(isinstance(typ, model.StructOrUnion) for typ in tp.args):
                 indirect_args = []
                 for i, typ in enumerate(tp.args):
             wrappername = '_cffi_f_%s' % name
             newfunction = module.load_function(BFunc, wrappername)
             for i, typ in indirections:
-                newfunction = self._make_struct_wrapper(newfunction, i, typ)
+                newfunction = self._make_struct_wrapper(newfunction, i, typ,
+                                                        base_tp)
         setattr(library, name, newfunction)
         type(library)._cffi_dir.append(name)
 
-    def _make_struct_wrapper(self, oldfunc, i, tp):
+    def _make_struct_wrapper(self, oldfunc, i, tp, base_tp):
         backend = self.ffi._backend
         BType = self.ffi._get_cached_btype(tp)
         def newfunc(*args):
             args = args[:i] + (backend.newp(BType, args[i]),) + args[i+1:]
             return oldfunc(*args)
+        newfunc._cffi_base_type = base_tp
         return newfunc
 
     # ----------
         prnt('  static ssize_t nums[] = {')
         prnt('    sizeof(%s),' % cname)
         prnt('    offsetof(struct _cffi_aligncheck, y),')
-        for fname, _, fbitsize in tp.enumfields():
+        for fname, ftype, fbitsize in tp.enumfields():
             if fbitsize >= 0:
                 continue      # xxx ignore fbitsize for now
             prnt('    offsetof(%s, %s),' % (cname, fname))
-            prnt('    sizeof(((%s *)0)->%s),' % (cname, fname))
+            if isinstance(ftype, model.ArrayType) and ftype.length is None:
+                prnt('    0,  /* %s */' % ftype._get_c_name())
+            else:
+                prnt('    sizeof(((%s *)0)->%s),' % (cname, fname))
         prnt('    -1')
         prnt('  };')
         prnt('  return nums[i];')
             return     # nothing to do with opaque structs
         layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name)
         #
-        BFunc = self.ffi.typeof("ssize_t(*)(ssize_t)")
+        BFunc = self.ffi._typeof_locked("ssize_t(*)(ssize_t)")[0]
         function = module.load_function(BFunc, layoutfuncname)
         layout = []
         num = 0
             if x < 0: break
             layout.append(x)
             num += 1
-        if isinstance(tp, model.StructType) and tp.partial:
+        if isinstance(tp, model.StructOrUnion) and tp.partial:
             # use the function()'s sizes and offsets to guide the
             # layout of the struct
             totalsize = layout[0]
                     continue        # xxx ignore fbitsize for now
                 check(layout[i], ffi.offsetof(BStruct, fname),
                       "wrong offset for field %r" % (fname,))
-                BField = ffi._get_cached_btype(ftype)
-                check(layout[i+1], ffi.sizeof(BField),
-                      "wrong size for field %r" % (fname,))
+                if layout[i+1] != 0:
+                    BField = ffi._get_cached_btype(ftype)
+                    check(layout[i+1], ffi.sizeof(BField),
+                          "wrong size for field %r" % (fname,))
                 i += 2
             assert i == len(layout)
 
     def _load_constant(self, is_int, tp, name, module):
         funcname = '_cffi_const_%s' % name
         if is_int:
-            BFunc = self.ffi.typeof("int(*)(long long*)")
+            BType = self.ffi._typeof_locked("long long*")[0]
+            BFunc = self.ffi._typeof_locked("int(*)(long long*)")[0]
             function = module.load_function(BFunc, funcname)
-            p = self.ffi.new("long long*")
+            p = self.ffi.new(BType)
             negative = function(p)
             value = int(p[0])
             if value < 0 and not negative:
-                value += (1 << (8*self.ffi.sizeof("long long")))
+                BLongLong = self.ffi._typeof_locked("long long")[0]
+                value += (1 << (8*self.ffi.sizeof(BLongLong)))
         else:
-            BFunc = self.ffi.typeof(tp.get_c_name('(*)(void)', name))
+            BFunc = self.ffi._typeof_locked(tp.get_c_name('(*)(void)', name))[0]
             function = module.load_function(BFunc, funcname)
             value = function()
         return value
             tp.enumvalues = tuple(enumvalues)
             tp.partial_resolved = True
         else:
-            BFunc = self.ffi.typeof("int(*)(char*)")
+            BType = self.ffi._typeof_locked("char[]")[0]
+            BFunc = self.ffi._typeof_locked("int(*)(char*)")[0]
             funcname = '_cffi_e_%s_%s' % (prefix, name)
             function = module.load_function(BFunc, funcname)
-            p = self.ffi.new("char[]", 256)
+            p = self.ffi.new(BType, 256)
             if function(p) < 0:
                 error = self.ffi.string(p)
                 if sys.version_info >= (3,):
 
     def _generate_gen_variable_decl(self, tp, name):
         if isinstance(tp, model.ArrayType):
+            if tp.length == '...':
+                prnt = self._prnt
+                funcname = '_cffi_sizeof_%s' % (name,)
+                self.export_symbols.append(funcname)
+                prnt("size_t %s(void)" % funcname)
+                prnt("{")
+                prnt("  return sizeof(%s);" % (name,))
+                prnt("}")
             tp_ptr = model.PointerType(tp.item)
             self._generate_gen_const(False, name, tp_ptr)
         else:
     def _loaded_gen_variable(self, tp, name, module, library):
         if isinstance(tp, model.ArrayType):   # int a[5] is "constant" in the
                                               # sense that "a=..." is forbidden
+            if tp.length == '...':
+                funcname = '_cffi_sizeof_%s' % (name,)
+                BFunc = self.ffi._typeof_locked('size_t(*)(void)')[0]
+                function = module.load_function(BFunc, funcname)
+                size = function()
+                BItemType = self.ffi._get_cached_btype(tp.item)
+                length, rest = divmod(size, self.ffi.sizeof(BItemType))
+                if rest != 0:
+                    raise ffiplatform.VerificationError(
+                        "bad size: %r does not seem to be an array of %s" %
+                        (name, tp.item))
+                tp = tp.resolve_length(length)
             tp_ptr = model.PointerType(tp.item)
             value = self._load_constant(False, tp_ptr, name, module)
             # 'value' is a <cdata 'type *'> which we have to replace with
         # remove ptr=<cdata 'int *'> from the library instance, and replace
         # it by a property on the class, which reads/writes into ptr[0].
         funcname = '_cffi_var_%s' % name
-        BFunc = self.ffi.typeof(tp.get_c_name('*(*)(void)', name))
+        BFunc = self.ffi._typeof_locked(tp.get_c_name('*(*)(void)', name))[0]
         function = module.load_function(BFunc, funcname)
         ptr = function()
         def getter(library):

File lib_pypy/cffi/verifier.py

             k2 = k2.lstrip('0').rstrip('L')
             modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key,
                                               k1, k2)
-        suffix = _get_so_suffix()
+        suffix = _get_so_suffixes()[0]
         self.tmpdir = tmpdir or _caller_dir_pycache()
         self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c')
         self.modulefilename = os.path.join(self.tmpdir, modulename + suffix)
     def write_source(self, file=None):
         """Write the C source code.  It is produced in 'self.sourcefilename',
         which can be tweaked beforehand."""
-        if self._has_source and file is None:
-            raise ffiplatform.VerificationError("source code already written")
-        self._write_source(file)
+        with self.ffi._lock:
+            if self._has_source and file is None:
+                raise ffiplatform.VerificationError(
+                    "source code already written")
+            self._write_source(file)
 
     def compile_module(self):
         """Write the C source code (if not done already) and compile it.
         This produces a dynamic link library in 'self.modulefilename'."""
-        if self._has_module:
-            raise ffiplatform.VerificationError("module already compiled")
-        if not self._has_source:
-            self._write_source()
-        self._compile_module()
+        with self.ffi._lock:
+            if self._has_module:
+                raise ffiplatform.VerificationError("module already compiled")
+            if not self._has_source:
+                self._write_source()
+            self._compile_module()
 
     def load_library(self):
         """Get a C module from this Verifier instance.
         operations to the C module.  If necessary, the C code is written
         and compiled first.
         """
-        if not self._has_module:
-            self._locate_module()
+        with self.ffi._lock:
             if not self._has_module:
-                self.compile_module()
-        return self._load_library()
+                self._locate_module()
+                if not self._has_module:
+                    if not self._has_source:
+                        self._write_source()
+                    self._compile_module()
+            return self._load_library()
 
     def get_module_name(self):
         basename = os.path.basename(self.modulefilename)
 
     def get_extension(self):
         if not self._has_source:
-            self._write_source()
+            with self.ffi._lock:
+                if not self._has_source:
+                    self._write_source()
         sourcename = ffiplatform.maybe_relative_path(self.sourcefilename)
         modname = self.get_module_name()
         return ffiplatform.get_extension(sourcename, modname, **self.kwds)
             else:
                 path = None
             filename = self._vengine.find_module(self.get_module_name(), path,
-                                                 _get_so_suffix())
+                                                 _get_so_suffixes())
             if filename is None:
                 return
             self.modulefilename = filename
     if keep_so:
         suffix = '.c'   # only remove .c files
     else:
-        suffix = _get_so_suffix().lower()
+        suffix = _get_so_suffixes()[0].lower()
     for fn in filelist:
         if fn.lower().startswith('_cffi_') and (
                 fn.lower().endswith(suffix) or fn.lower().endswith('.c')):
         except OSError:
             pass
 
-def _get_so_suffix():
+def _get_so_suffixes():
+    suffixes = []
     for suffix, mode, type in imp.get_suffixes():
         if type == imp.C_EXTENSION:
-            return suffix
-    # bah, no C_EXTENSION available.  Occurs on pypy without cpyext
-    if sys.platform == 'win32':
-        return ".pyd"
-    else:
-        return ".so"
+            suffixes.append(suffix)
+
+    if not suffixes:
+        # bah, no C_EXTENSION available.  Occurs on pypy without cpyext
+        if sys.platform == 'win32':
+            suffixes = [".pyd"]
+        else:
+            suffixes = [".so"]
+
+    return suffixes
 
 def _ensure_dir(filename):
     try:

File pypy/doc/how-to-release.rst

 * post announcement on morepypy.blogspot.com
 * send announcements to pypy-dev, python-list,
   python-announce, python-dev ...
+
+* add a tag on jitviewer that corresponds to pypy release
+* add a tag on codespeed that corresponds to pypy release
+

File pypy/doc/whatsnew-head.rst

 .. branch: safe-win-mmap
 .. branch: boolean-indexing-cleanup
 .. branch: cpyext-best_base
+.. branch: cpyext-int
 .. branch: fileops2
 
 .. branch: nobold-backtrace

File pypy/module/_cffi_backend/__init__.py

     appleveldefs = {
         }
     interpleveldefs = {
-        '__version__': 'space.wrap("0.7")',
+        '__version__': 'space.wrap("0.8")',
 
         'load_library': 'libraryobj.load_library',
 

File pypy/module/_cffi_backend/cdataobj.py

     _cdata = lltype.nullptr(rffi.CCHARP.TO)
 
     def __init__(self, space, cdata, ctype):
-        from pypy.module._cffi_backend import ctypeprim
+        from pypy.module._cffi_backend import ctypeobj
         assert lltype.typeOf(cdata) == rffi.CCHARP
-        assert isinstance(ctype, ctypeprim.W_CType)
+        assert isinstance(ctype, ctypeobj.W_CType)
         self.space = space
         self._cdata = cdata    # don't forget keepalive_until_here!
         self.ctype = ctype
                 keepalive_until_here(w_value)
                 return
         #
+        # A fast path for <char[]>[0:N] = "somestring".
+        from pypy.module._cffi_backend import ctypeprim
         space = self.space
+        if (space.isinstance_w(w_value, space.w_str) and
+                isinstance(ctitem, ctypeprim.W_CTypePrimitiveChar)):
+            from rpython.rtyper.annlowlevel import llstr
+            from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
+            value = space.str_w(w_value)
+            if len(value) != length:
+                raise operationerrfmt(space.w_ValueError,
+                                      "need a string of length %d, got %d",
+                                      length, len(value))
+            copy_string_to_raw(llstr(value), cdata, 0, length)
+            return
+        #
         w_iter = space.iter(w_value)
         for i in range(length):
             try:
         space = self.space
         if isinstance(w_other, W_CData):
             from pypy.module._cffi_backend import ctypeptr, ctypearray
+            from pypy.module._cffi_backend import ctypevoid
             ct = w_other.ctype
             if isinstance(ct, ctypearray.W_CTypeArray):
                 ct = ct.ctptr
             #
             if (ct is not self.ctype or
                    not isinstance(ct, ctypeptr.W_CTypePointer) or
-                   ct.ctitem.size <= 0):
+                   (ct.ctitem.size <= 0 and not ct.is_void_ptr)):
                 raise operationerrfmt(space.w_TypeError,
                     "cannot subtract cdata '%s' and cdata '%s'",
                     self.ctype.name, ct.name)
             #
+            itemsize = ct.ctitem.size
+            if itemsize <= 0: itemsize = 1
             diff = (rffi.cast(lltype.Signed, self._cdata) -
-                    rffi.cast(lltype.Signed, w_other._cdata)) // ct.ctitem.size
+                    rffi.cast(lltype.Signed, w_other._cdata)) // itemsize
             return space.wrap(diff)
         #
         return self._add_or_sub(w_other, -1)
     __getitem__ = interp2app(W_CData.getitem),
     __setitem__ = interp2app(W_CData.setitem),
     __add__ = interp2app(W_CData.add),
+    __radd__ = interp2app(W_CData.add),
     __sub__ = interp2app(W_CData.sub),
     __getattr__ = interp2app(W_CData.getattr),
     __setattr__ = interp2app(W_CData.setattr),

File pypy/module/_cffi_backend/ctypearray.py

         datasize = self.size
         #
         if datasize < 0:
-            if (space.isinstance_w(w_init, space.w_list) or
-                space.isinstance_w(w_init, space.w_tuple)):
-                length = space.int_w(space.len(w_init))
-            elif space.isinstance_w(w_init, space.w_basestring):
-                # from a string, we add the null terminator
-                length = space.int_w(space.len(w_init)) + 1
-            else:
-                length = space.getindex_w(w_init, space.w_OverflowError)
-                if length < 0:
-                    raise OperationError(space.w_ValueError,
-                                         space.wrap("negative array length"))
-                w_init = space.w_None
-            #
+            from pypy.module._cffi_backend import misc
+            w_init, length = misc.get_new_array_length(space, w_init)
             try:
                 datasize = ovfcheck(length * self.ctitem.size)
             except OverflowError:

File pypy/module/_cffi_backend/ctypeptr.py

 Pointers.
 """
 
-from pypy.interpreter.error import OperationError, operationerrfmt, wrap_oserror
-
 from rpython.rlib import rposix
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rarithmetic import ovfcheck
+from rpython.rtyper.annlowlevel import llstr, llunicode
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw, copy_unicode_to_raw
 
+from pypy.interpreter.error import OperationError, operationerrfmt, wrap_oserror
 from pypy.module._cffi_backend import cdataobj, misc, ctypeprim, ctypevoid
 from pypy.module._cffi_backend.ctypeobj import W_CType
 
 
 class W_CTypePtrOrArray(W_CType):
-    _attrs_            = ['ctitem', 'can_cast_anything', 'is_struct_ptr',
-                          'length']
-    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'is_struct_ptr',
-                          'length']
+    _attrs_            = ['ctitem', 'can_cast_anything', 'length']
+    _immutable_fields_ = ['ctitem', 'can_cast_anything', 'length']
     length = -1
 
     def __init__(self, space, size, extra, extra_position, ctitem,
                  could_cast_anything=True):
-        from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
         name, name_position = ctitem.insert_name(extra, extra_position)
         W_CType.__init__(self, space, size, name, name_position)
         # this is the "underlying type":
         #  - for functions, it is the return type
         self.ctitem = ctitem
         self.can_cast_anything = could_cast_anything and ctitem.cast_anything
-        self.is_struct_ptr = isinstance(ctitem, W_CTypeStructOrUnion)
 
     def is_char_ptr_or_array(self):
         return isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveChar)
                                       "initializer string is too long for '%s'"
                                       " (got %d characters)",
                                       self.name, n)
-            for i in range(n):
-                cdata[i] = s[i]
+            copy_string_to_raw(llstr(s), cdata, 0, n)
             if n != self.length:
                 cdata[n] = '\x00'
         elif isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar):
                                       " (got %d characters)",
                                       self.name, n)
             unichardata = rffi.cast(rffi.CWCHARP, cdata)
-            for i in range(n):
-                unichardata[i] = s[i]
+            copy_unicode_to_raw(llunicode(s), unichardata, 0, n)
             if n != self.length:
                 unichardata[n] = u'\x00'
         else:
         return cdataobj.W_CData(self.space, ptrdata, self)
 
     def convert_from_object(self, cdata, w_ob):
-        space = self.space
         if not isinstance(w_ob, cdataobj.W_CData):
             raise self._convert_error("cdata pointer", w_ob)
         other = w_ob.ctype
         W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem)
 
     def newp(self, w_init):
+        from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
         space = self.space
         ctitem = self.ctitem
         datasize = ctitem.size
             raise operationerrfmt(space.w_TypeError,
                 "cannot instantiate ctype '%s' of unknown size",
                                   self.name)
-        if self.is_struct_ptr:
+        if isinstance(ctitem, W_CTypeStructOrUnion):
             # 'newp' on a struct-or-union pointer: in this case, we return
             # a W_CDataPtrToStruct object which has a strong reference
             # to a W_CDataNewOwning that really contains the structure.
+            #
+            if ctitem.with_var_array and not space.is_w(w_init, space.w_None):
+                datasize = ctitem.convert_struct_from_object(
+                    lltype.nullptr(rffi.CCHARP.TO), w_init, datasize)
+            #
             cdatastruct = cdataobj.W_CDataNewOwning(space, datasize, ctitem)
             cdata = cdataobj.W_CDataPtrToStructOrUnion(space,
                                                        cdatastruct._cdata,
     def add(self, cdata, i):
         space = self.space
         ctitem = self.ctitem
+        itemsize = ctitem.size
         if ctitem.size < 0:
-            raise operationerrfmt(space.w_TypeError,
+            if self.is_void_ptr:
+                itemsize = 1
+            else:
+                raise operationerrfmt(space.w_TypeError,
                                   "ctype '%s' points to items of unknown size",
                                   self.name)
-        p = rffi.ptradd(cdata, i * self.ctitem.size)
+        p = rffi.ptradd(cdata, i * itemsize)
         return cdataobj.W_CData(space, p, self)
 
     def cast(self, w_ob):
 
     def convert_argument_from_object(self, cdata, w_ob):
         from pypy.module._cffi_backend.ctypefunc import set_mustfree_flag
-        space = self.space
         result = (not isinstance(w_ob, cdataobj.W_CData) and
                   self._prepare_pointer_call_argument(w_ob, cdata))
         if result == 0:
         space = self.space
         ctype2 = cdata.ctype
         if (isinstance(ctype2, W_CTypeStructOrUnion) or
-            (isinstance(ctype2, W_CTypePtrOrArray) and ctype2.is_struct_ptr)):
+               (isinstance(ctype2, W_CTypePtrOrArray) and
+                isinstance(ctype2.ctitem, W_CTypeStructOrUnion))):
             ptrdata = rffi.ptradd(cdata._cdata, offset)
             return cdataobj.W_CData(space, ptrdata, self)
         else:

File pypy/module/_cffi_backend/ctypestruct.py

 from rpython.rlib import jit
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong, intmask
-from rpython.rtyper.lltypesystem import rffi
+from rpython.rlib.rarithmetic import ovfcheck
+from rpython.rtyper.lltypesystem import lltype, rffi
 
 from pypy.module._cffi_backend import cdataobj, ctypeprim, misc
 from pypy.module._cffi_backend.ctypeobj import W_CType
 
 class W_CTypeStructOrUnion(W_CType):
     _immutable_fields_ = ['alignment?', 'fields_list?', 'fields_dict?',
-                          'custom_field_pos?']
+                          'custom_field_pos?', 'with_var_array?']
     # fields added by complete_struct_or_union():
     alignment = -1
     fields_list = None
     fields_dict = None
     custom_field_pos = False
+    with_var_array = False
 
     def __init__(self, space, name):
         W_CType.__init__(self, space, -1, name, len(name))
         pass
 
     def convert_from_object(self, cdata, w_ob):
-        space = self.space
-        if self._copy_from_same(cdata, w_ob):
-            return
+        if not self._copy_from_same(cdata, w_ob):
+            self.convert_struct_from_object(cdata, w_ob, optvarsize=-1)
 
+    def convert_struct_from_object(self, cdata, w_ob, optvarsize):
         self._check_only_one_argument_for_union(w_ob)
 
+        space = self.space
         if (space.isinstance_w(w_ob, space.w_list) or
             space.isinstance_w(w_ob, space.w_tuple)):
             lst_w = space.listview(w_ob)
                         "too many initializers for '%s' (got %d)",
                                       self.name, len(lst_w))
             for i in range(len(lst_w)):
-                self.fields_list[i].write(cdata, lst_w[i])
+                optvarsize = self.fields_list[i].write_v(cdata, lst_w[i],
+                                                         optvarsize)
+            return optvarsize
 
         elif space.isinstance_w(w_ob, space.w_dict):
             lst_w = space.fixedview(w_ob)
                 except KeyError:
                     space.raise_key_error(w_key)
                     assert 0
-                cf.write(cdata, space.getitem(w_ob, w_key))
+                optvarsize = cf.write_v(cdata, space.getitem(w_ob, w_key),
+                                        optvarsize)
+            return optvarsize
 
         else:
-            raise self._convert_error("list or tuple or dict or struct-cdata",
-                                      w_ob)
+            if optvarsize == -1:
+                msg = "list or tuple or dict or struct-cdata"
+            else:
+                msg = "list or tuple or dict"
+            raise self._convert_error(msg, w_ob)
 
     @jit.elidable
     def _getcfield_const(self, attr):
         else:
             self.ctype.convert_from_object(cdata, w_ob)
 
+    def write_v(self, cdata, w_ob, optvarsize):
+        # a special case for var-sized C99 arrays
+        from pypy.module._cffi_backend import ctypearray
+        ct = self.ctype
+        if isinstance(ct, ctypearray.W_CTypeArray) and ct.length < 0:
+            space = ct.space
+            w_ob, varsizelength = misc.get_new_array_length(space, w_ob)
+            if optvarsize != -1:
+                # in this mode, the only purpose of this function is to compute
+                # the real size of the structure from a var-sized C99 array
+                assert cdata == lltype.nullptr(rffi.CCHARP.TO)
+                itemsize = ct.ctitem.size
+                try:
+                    varsize = ovfcheck(itemsize * varsizelength)
+                    size = ovfcheck(self.offset + varsize)
+                except OverflowError:
+                    raise OperationError(space.w_OverflowError,
+                        space.wrap("array size would overflow a ssize_t"))
+                assert size >= 0
+                return max(size, optvarsize)
+            # if 'value' was only an integer, get_new_array_length() returns
+            # w_ob = space.w_None.  Detect if this was the case,
+            # and if so, stop here, leaving the content uninitialized
+            # (it should be zero-initialized from somewhere else).
+            if space.is_w(w_ob, space.w_None):
+                return optvarsize
+        #
+        if optvarsize == -1:
+            self.write(cdata, w_ob)
+        return optvarsize
+
     def convert_bitfield_to_object(self, cdata):
         ctype = self.ctype
         space = ctype.space

File pypy/module/_cffi_backend/misc.py

 
 # ____________________________________________________________
 
+def get_new_array_length(space, w_value):
+    if (space.isinstance_w(w_value, space.w_list) or
+        space.isinstance_w(w_value, space.w_tuple)):
+        return (w_value, space.int_w(space.len(w_value)))
+    elif space.isinstance_w(w_value, space.w_basestring):
+        # from a string, we add the null terminator
+        return (w_value, space.int_w(space.len(w_value)) + 1)
+    else:
+        explicitlength = space.getindex_w(w_value, space.w_OverflowError)
+        if explicitlength < 0:
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("negative array length"))
+        return (space.w_None, explicitlength)
+
+# ____________________________________________________________
+
 @specialize.arg(0)
 def _raw_memcopy_tp(TPP, source, dest):
     # in its own function: LONGLONG may make the whole function jit-opaque

File pypy/module/_cffi_backend/newtype.py

     fields_list = []
     fields_dict = {}
     custom_field_pos = False
+    with_var_array = False
 
-    for w_field in fields_w:
+    for i in range(len(fields_w)):
+        w_field = fields_w[i]
         field_w = space.fixedview(w_field)
         if not (2 <= len(field_w) <= 4):
             raise OperationError(space.w_TypeError,
                                   "duplicate field name '%s'", fname)
         #
         if ftype.size < 0:
-            raise operationerrfmt(space.w_TypeError,
+            if (isinstance(ftype, ctypearray.W_CTypeArray) and fbitsize < 0
+                    and (i == len(fields_w) - 1 or foffset != -1)):
+                with_var_array = True
+            else:
+                raise operationerrfmt(space.w_TypeError,
                     "field '%s.%s' has ctype '%s' of unknown size",
                                   w_ctype.name, fname, ftype.name)
         #
                 fields_list.append(fld)
                 fields_dict[fname] = fld
 
-            boffset += ftype.size * 8
+            if ftype.size >= 0:
+                boffset += ftype.size * 8
             prev_bitfield_size = 0
 
         else:
     w_ctype.fields_list = fields_list
     w_ctype.fields_dict = fields_dict
     w_ctype.custom_field_pos = custom_field_pos
+    w_ctype.with_var_array = with_var_array
 
 # ____________________________________________________________
 

File pypy/module/_cffi_backend/test/_backend_test_c.py

     assert repr(a) == "<cdata 'int[3][5]' owning %d bytes>" % (
         3*5*size_of_int(),)
     assert repr(a + 0).startswith("<cdata 'int(*)[5]' 0x")
+    assert 0 + a == a + 0 != 1 + a == a + 1
     assert repr(a[0]).startswith("<cdata 'int[5]' 0x")
     assert repr((a + 0)[0]).startswith("<cdata 'int[5]' 0x")
     assert repr(a[0] + 0).startswith("<cdata 'int *' 0x")
 def test_void_errors():
     py.test.raises(ValueError, alignof, new_void_type())
     py.test.raises(TypeError, newp, new_pointer_type(new_void_type()), None)
-    x = cast(new_pointer_type(new_void_type()), 42)
-    py.test.raises(TypeError, "x + 1")
-    py.test.raises(TypeError, "x - 1")
 
 def test_too_many_items():
     BChar = new_primitive_type("char")
     _test_bitfield_details(flag=4)
 
 
+def test_struct_array_no_length():
+    BInt = new_primitive_type("int")
+    BIntP = new_pointer_type(BInt)
+    BArray = new_array_type(BIntP, None)
+    BStruct = new_struct_type("foo")
+    py.test.raises(TypeError, complete_struct_or_union,
+                   BStruct, [('x', BArray),
+                             ('y', BInt)])
+    #
+    BStruct = new_struct_type("foo")
+    complete_struct_or_union(BStruct, [('x', BInt),
+                                       ('y', BArray)])
+    assert sizeof(BStruct) == size_of_int()
+    d = BStruct.fields
+    assert len(d) == 2
+    assert d[0][0] == 'x'
+    assert d[0][1].type is BInt
+    assert d[0][1].offset == 0
+    assert d[0][1].bitshift == -1
+    assert d[0][1].bitsize == -1
+    assert d[1][0] == 'y'
+    assert d[1][1].type is BArray
+    assert d[1][1].offset == size_of_int()
+    assert d[1][1].bitshift == -1
+    assert d[1][1].bitsize == -1
+    #
+    p = newp(new_pointer_type(BStruct))
+    p.x = 42
+    assert p.x == 42
+    assert typeof(p.y) is BIntP
+    assert p.y == cast(BIntP, p) + 1
+    #
+    p = newp(new_pointer_type(BStruct), [100])
+    assert p.x == 100
+    #
+    # Tests for
+    #    ffi.new("struct_with_var_array *", [field.., [the_array_items..]])
+    #    ffi.new("struct_with_var_array *", [field.., array_size])
+    plist = []
+    for i in range(20):
+        if i % 2 == 0:
+            p = newp(new_pointer_type(BStruct), [100, [200, i, 400]])
+        else:
+            p = newp(new_pointer_type(BStruct), [100, 3])
+            p.y[1] = i
+            p.y[0] = 200
+            assert p.y[2] == 0
+            p.y[2] = 400
+        plist.append(p)
+    for i in range(20):
+        p = plist[i]
+        assert p.x == 100
+        assert p.y[0] == 200
+        assert p.y[1] == i
+        assert p.y[2] == 400
+        assert list(p.y[0:3]) == [200, i, 400]
+    #
+    # the following assignment works, as it normally would, for any array field
+    p.y = [500, 600]
+    assert list(p.y[0:3]) == [500, 600, 400]
+    #
+    # error cases
+    py.test.raises(TypeError, "p.y = cast(BIntP, 0)")
+    py.test.raises(TypeError, "p.y = 15")
+    py.test.raises(TypeError, "p.y = None")
+    #
+    # accepting this may be specified by the C99 standard,
+    # or a GCC strangeness...
+    BStruct2 = new_struct_type("bar")
+    complete_struct_or_union(BStruct2, [('f', BStruct),
+                                        ('n', BInt)])
+    p = newp(new_pointer_type(BStruct2), {'n': 42})
+    assert p.n == 42
+    #
+    # more error cases
+    py.test.raises(TypeError, newp, new_pointer_type(BStruct), [100, None])
+    BArray4 = new_array_type(BIntP, 4)
+    BStruct4 = new_struct_type("test4")
+    complete_struct_or_union(BStruct4, [('a', BArray4)])   # not varsized
+    py.test.raises(TypeError, newp, new_pointer_type(BStruct4), [None])
+    py.test.raises(TypeError, newp, new_pointer_type(BStruct4), [4])
+    p = newp(new_pointer_type(BStruct4), [[10, 20, 30]])
+    assert p.a[0] == 10
+    assert p.a[1] == 20
+    assert p.a[2] == 30
+    assert p.a[3] == 0
+
+def test_struct_array_no_length_explicit_position():
+    BInt = new_primitive_type("int")
+    BIntP = new_pointer_type(BInt)
+    BArray = new_array_type(BIntP, None)
+    BStruct = new_struct_type("foo")
+    complete_struct_or_union(BStruct, [('x', BArray, -1, 0), # actually 3 items
+                                       ('y', BInt, -1, 12)])
+    p = newp(new_pointer_type(BStruct), [[10, 20], 30])
+    assert p.x[0] == 10
+    assert p.x[1] == 20
+    assert p.x[2] == 0
+    assert p.y == 30
+    p = newp(new_pointer_type(BStruct), {'x': [40], 'y': 50})
+    assert p.x[0] == 40
+    assert p.x[1] == 0
+    assert p.x[2] == 0
+    assert p.y == 50
+    p = newp(new_pointer_type(BStruct), {'y': 60})
+    assert p.x[0] == 0
+    assert p.x[1] == 0
+    assert p.x[2] == 0
+    assert p.y == 60
+    #
+    # This "should" work too, allocating a larger structure
+    # (a bit strange in this case, but useful in general)
+    plist = []
+    for i in range(20):
+        p = newp(new_pointer_type(BStruct), [[10, 20, 30, 40, 50, 60, 70]])
+        plist.append(p)
+    for i in range(20):
+        p = plist[i]
+        assert p.x[0] == 10
+        assert p.x[1] == 20
+        assert p.x[2] == 30
+        assert p.x[3] == 40 == p.y
+        assert p.x[4] == 50
+        assert p.x[5] == 60
+        assert p.x[6] == 70
+
+def test_ass_slice():
+    BChar = new_primitive_type("char")
+    BArray = new_array_type(new_pointer_type(BChar), None)
+    p = newp(BArray, b"foobar")
+    p[2:5] = [b"*", b"Z", b"T"]
+    p[1:3] = b"XY"
+    assert list(p) == [b"f", b"X", b"Y", b"Z", b"T", b"r", b"\x00"]
+    py.test.raises(TypeError, "p[1:5] = u+'XYZT'")
+    py.test.raises(TypeError, "p[1:5] = [1, 2, 3, 4]")
+    #
+    BUniChar = new_primitive_type("wchar_t")
+    BArray = new_array_type(new_pointer_type(BUniChar), None)
+    p = newp(BArray, u+"foobar")
+    p[2:5] = [u+"*", u+"Z", u+"T"]
+    p[1:3] = u+"XY"
+    assert list(p) == [u+"f", u+"X", u+"Y", u+"Z", u+"T", u+"r", u+"\x00"]
+    py.test.raises(TypeError, "p[1:5] = b'XYZT'")
+    py.test.raises(TypeError, "p[1:5] = [1, 2, 3, 4]")
+
+def test_void_p_arithmetic():
+    BVoid = new_void_type()
+    BInt = new_primitive_type("intptr_t")
+    p = cast(new_pointer_type(BVoid), 100000)
+    assert int(cast(BInt, p)) == 100000
+    assert int(cast(BInt, p + 42)) == 100042
+    assert int(cast(BInt, p - (-42))) == 100042
+    assert (p + 42) - p == 42
+    q = cast(new_pointer_type(new_primitive_type("char")), 100000)
+    py.test.raises(TypeError, "p - q")
+    py.test.raises(TypeError, "q - p")
+    py.test.raises(TypeError, "p + cast(new_primitive_type('int'), 42)")
+    py.test.raises(TypeError, "p - cast(new_primitive_type('int'), 42)")
+
+
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "0.7"
+    assert __version__ == "0.8"

File pypy/module/array/interp_array.py

     def descr_getslice(self, space, w_i, w_j):
         return space.getitem(self, space.newslice(w_i, w_j, space.w_None))
 
-
     def descr_setitem(self, space, w_idx, w_item):
         "x.__setitem__(i, y) <==> x[i]=y"
         if space.isinstance_w(w_idx, space.w_slice):
                     self.buffer[i] = w_item.buffer[j]
                     j += 1
 
-        # We can't look into this function until ptradd works with things (in the
-        # JIT) other than rffi.CCHARP
-        @jit.dont_look_inside
         def delitem(self, space, i, j):
             if i < 0:
                 i += self.len
                 lltype.free(oldbuffer, flavor='raw')
 
         # Add and mul methods
-
         def descr_add(self, space, w_other):
             if not isinstance(w_other, W_Array):
                 return space.w_NotImplemented
             a = mytype.w_class(space)
             a.setlen(self.len + w_other.len, overallocate=False)
-            for i in range(self.len):
-                a.buffer[i] = self.buffer[i]
-            for i in range(w_other.len):
-                a.buffer[i + self.len] = w_other.buffer[i]
+            if self.len:
+                rffi.c_memcpy(
+                    rffi.cast(rffi.VOIDP, a.buffer),
+                    rffi.cast(rffi.VOIDP, self.buffer),
+                    self.len * mytype.bytes
+                )
+            if w_other.len:
+                rffi.c_memcpy(
+                    rffi.cast(rffi.VOIDP, rffi.ptradd(a.buffer, self.len)),
+                    rffi.cast(rffi.VOIDP, w_other.buffer),
+                    w_other.len * mytype.bytes
+                )
             return a
 
         def descr_inplace_add(self, space, w_other):
             oldlen = self.len
             otherlen = w_other.len
             self.setlen(oldlen + otherlen)
-            for i in range(otherlen):
-                self.buffer[oldlen + i] = w_other.buffer[i]
+            if otherlen:
+                rffi.c_memcpy(
+                    rffi.cast(rffi.VOIDP, rffi.ptradd(self.buffer, oldlen)),
+                    rffi.cast(rffi.VOIDP, w_other.buffer),
+                    otherlen * mytype.bytes
+                )
             return self
 
         def descr_mul(self, space, w_repeat):

File pypy/module/cpyext/intobject.py

     "Type description of PyIntObject"
     make_typedescr(space.w_int.instancetypedef,
                    basestruct=PyIntObject.TO,
+                   attach=int_attach,
                    realize=int_realize)
 
+def int_attach(space, py_obj, w_obj):
+    """
+    Fills a newly allocated PyIntObject with the given int object. The
+    value must not be modified.
+    """
+    py_int = rffi.cast(PyIntObject, py_obj)
+    py_int.c_ob_ival = space.int_w(w_obj)
+
 def int_realize(space, obj):
     intval = rffi.cast(lltype.Signed, rffi.cast(PyIntObject, obj).c_ob_ival)
     w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type))

File pypy/module/cpyext/test/test_intobject.py

 
                 return (PyObject *)enumObj;
              """),
-            ], 
+            ],
             prologue="""
             typedef struct
             {
         assert isinstance(a, int)
         assert a == int(a) == 42
         assert a.name == "ULTIMATE_ANSWER"
+
+    def test_int_cast(self):
+        mod = self.import_extension('foo', [
+                #prove it works for ints
+                ("test_int", "METH_NOARGS",
+                """
+                PyObject * obj = PyInt_FromLong(42);
+                if (!PyInt_Check(obj)) {
+                    Py_DECREF(obj);
+                    PyErr_SetNone(PyExc_ValueError);
+                    return NULL;
+                }
+                PyObject * val = PyInt_FromLong(((PyIntObject *)obj)->ob_ival);
+                Py_DECREF(obj);
+                return val;
+                """
+                ),
+                ])
+        i = mod.test_int()
+        assert isinstance(i, int)
+        assert i == 42

File pypy/module/cpyext/test/test_ndarrayobject.py

-import py
-
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rtyper.lltypesystem import rffi, lltype
         arr = mod.test_FromObject()
         dt = mod.test_DescrFromType(11)
         assert dt.num == 11
+
+

File pypy/module/micronumpy/interp_numarray.py

         # by converting nonnative byte order.
         if self.is_scalar():
             return space.wrap(0)
-        s = self.get_dtype().name
-        if not self.get_dtype().is_native():
-            s = s[1:]
-        dtype = interp_dtype.get_dtype_cache(space).dtypes_by_name[s]
+        if not self.get_dtype().is_flexible_type():
+            s = self.get_dtype().name
+            if not self.get_dtype().is_native():
+                s = s[1:]
+            dtype = interp_dtype.get_dtype_cache(space).dtypes_by_name[s]
+        else:
+            dtype = self.get_dtype()
         contig = self.implementation.astype(space, dtype)
         return contig.argsort(space, w_axis)
 

File pypy/module/micronumpy/test/test_sorting.py

             exp = sorted(range(len(exp)), key=exp.__getitem__)
             c = a.copy()
             res = a.argsort()
-            assert (res == exp).all(), \
-                'a,res,dtype %r,%r,%r' % (a,res,dtype)
+            assert (res == exp).all(), '%r\n%r\n%r' % (a,res,exp)
             assert (a == c).all() # not modified
 
             a = arange(100, dtype=dtype)
         for dtype in ['int', 'float', 'int16', 'float32', 'uint64',
                       'i2', complex]:
             a = array([6, 4, -1, 3, 8, 3, 256+20, 100, 101], dtype=dtype)
-            b = sorted(list(a))
-            c = a.copy()
-            a.sort()
-            assert (a == b).all(), \
-                'a,orig,dtype %r,%r,%r' % (a,c,dtype)
+            exp = sorted(list(a))
+            res = a.copy()
+            res.sort()
+            assert (res == exp).all(), '%r\n%r\n%r' % (a,res,exp)
 
             a = arange(100, dtype=dtype)
             c = a.copy()
             #assert (a == b).all(), \
             #    'a,orig,dtype %r,%r,%r' % (a,c,dtype)
 
-
 # tests from numpy/tests/test_multiarray.py
     def test_sort_corner_cases(self):
         # test ordering for floats and complex containing nans. It is only
         assert (r == array([('a', 1), ('c', 3), ('b', 255), ('d', 258)],
                                  dtype=mydtype)).all()
 
-
 # tests from numpy/tests/test_regression.py
     def test_sort_bigendian(self):
         skip('not implemented yet')
         y = fromstring("\x00\x01\x00\x02", dtype="S2")
         x.sort(kind='q')
         assert (x == y).all()
+
+    def test_string_mergesort(self):
+        import numpypy as np
+        import sys
+        x = np.array(['a'] * 32)
+        if '__pypy__' in sys.builtin_module_names:
+            exc = raises(NotImplementedError, "x.argsort(kind='m')")
+            assert 'non-numeric types' in exc.value.message
+        else:
+            assert (x.argsort(kind='m') == np.arange(32)).all()

File pypy/module/mmap/interp_mmap.py

         if step == 0:  # index only
             return space.wrap(self.mmap.getitem(start))
         elif step == 1:
+            if stop - start < 0:
+                return space.wrap("")
             return space.wrap(self.mmap.getslice(start, stop - start))
         else:
             res = "".join([self.mmap.getitem(i)

File pypy/module/mmap/test/test_mmap.py

         m = mmap(f.fileno(), 6)
         assert m[-3:7] == "bar"
 
+        assert m[1:0:1] == ""
+
         f.close()
 
     def test_sequence_type(self):

File pypy/module/posix/__init__.py

     if hasattr(os, 'fpathconf'):
         interpleveldefs['fpathconf'] = 'interp_posix.fpathconf'
         interpleveldefs['pathconf_names'] = 'space.wrap(os.pathconf_names)'
+    if hasattr(os, 'pathconf'):
+        interpleveldefs['pathconf'] = 'interp_posix.pathconf'
+    if hasattr(os, 'confstr'):
+        interpleveldefs['confstr'] = 'interp_posix.confstr'
+        interpleveldefs['confstr_names'] = 'space.wrap(os.confstr_names)'
     if hasattr(os, 'ttyname'):
         interpleveldefs['ttyname'] = 'interp_posix.ttyname'
     if hasattr(os, 'getloadavg'):
     for name in ['setsid', 'getuid', 'geteuid', 'getgid', 'getegid', 'setuid',
                  'seteuid', 'setgid', 'setegid', 'getgroups', 'getpgrp',
                  'setpgrp', 'getppid', 'getpgid', 'setpgid', 'setreuid',
-                 'setregid', 'getsid', 'setsid', 'fstatvfs', 'statvfs']:
+                 'setregid', 'getsid', 'setsid', 'fstatvfs', 'statvfs',
+                 'setgroups', 'initgroups', 'tcgetpgrp', 'tcsetpgrp',
+                 'getresuid', 'getresgid', 'setresuid', 'setresgid']:
         if hasattr(os, name):
             interpleveldefs[name] = 'interp_posix.%s' % (name,)
     # not visible via os, inconsistency in nt:

File pypy/module/posix/interp_posix.py

 
     Return list of supplemental group IDs for the process.
     """
-    return space.newlist([space.wrap(e) for e in os.getgroups()])
+    try:
+        list = os.getgroups()
+    except OSError, e:
+        raise wrap_oserror(space, e)
+    return space.newlist([space.wrap(e) for e in list])
+
+def setgroups(space, w_list):
+    """ setgroups(list)
+
+    Set the groups of the current process to list.
+    """
+    list = []
+    for w_gid in space.unpackiterable(w_list):
+        gid = space.int_w(w_gid)
+        check_uid_range(space, gid)
+        list.append(gid)
+    try:
+        os.setgroups(list[:])
+    except OSError, e:
+        raise wrap_oserror(space, e)
+
+@unwrap_spec(username=str, gid=c_gid_t)