Romain Guillebert avatar Romain Guillebert committed 4d92d4a Merge

Merge default

Comments (0)

Files changed (277)

 
 all: pypy-c
 
+PYPY_EXECUTABLE := $(shell which pypy)
+URAM := $(shell python -c "import sys; print 4.5 if sys.maxint>1<<32 else 2.5")
+
+ifeq ($(PYPY_EXECUTABLE),)
+RUNINTERP = python
+else
+RUNINTERP = $(PYPY_EXECUTABLE)
+endif
+
 pypy-c:
-	@echo "Building PyPy with JIT, it'll take about 40 minutes and 4G of RAM"
-	@sleep 3
-	rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
+	@echo
+	@echo "===================================================================="
+ifeq ($(PYPY_EXECUTABLE),)
+	@echo "Building a regular (jitting) version of PyPy, using CPython."
+	@echo "This takes around 2 hours and $(URAM) GB of RAM."
+	@echo "Note that pre-installing a PyPy binary would reduce this time"
+	@echo "and produce basically the same result."
+else
+	@echo "Building a regular (jitting) version of PyPy, using"
+	@echo "$(PYPY_EXECUTABLE) to run the translation itself."
+	@echo "This takes up to 1 hour and $(URAM) GB of RAM."
+endif
+	@echo
+	@echo "For more control (e.g. to use multiple CPU cores during part of"
+	@echo "the process) you need to run \`\`rpython/bin/rpython'' directly."
+	@echo "For more information see \`\`http://pypy.org/download.html''."
+	@echo "===================================================================="
+	@echo
+	@sleep 5
+	$(RUNINTERP) rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
+
+# Note: the -jN option, or MAKEFLAGS=-jN, are not usable.  They are
+# replaced with an opaque --jobserver option by the time this Makefile
+# runs.  We cannot get their original value either:
+# http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html
 =====================================
-PyPy: Python in Python Implementation 
+PyPy: Python in Python Implementation
 =====================================
 
 Welcome to PyPy!
 Building
 ========
 
-build with::
+build with:
 
-  rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
+.. code-block:: console
+
+    $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
 
 This ends up with ``pypy-c`` binary in the main pypy directory. We suggest
-to use virtualenv with the resulting pypy-c as the interpreter, you can
+to use virtualenv with the resulting pypy-c as the interpreter; you can
 find more details about various installation schemes here:
 
 http://doc.pypy.org/en/latest/getting-started.html#installing-pypy

lib-python/2.7/json/encoder.py

 
 from __pypy__.builders import StringBuilder, UnicodeBuilder
 
+class StringOrUnicodeBuilder(object):
+    def __init__(self):
+        self._builder = StringBuilder()
+    def append(self, string):
+        try:
+            self._builder.append(string)
+        except UnicodeEncodeError:
+            ub = UnicodeBuilder()
+            ub.append(self._builder.build())
+            self._builder = ub
+            ub.append(string)
+    def build(self):
+        return self._builder.build()
+
+
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
 ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
 HAS_UTF8 = re.compile(r'[\x80-\xff]')
         if self.ensure_ascii:
             builder = StringBuilder()
         else:
-            builder = UnicodeBuilder()
+            builder = StringOrUnicodeBuilder()
         self.__encode(o, markers, builder, 0)
         return builder.build()
 

lib-python/2.7/string.py

     must be of the same length.
 
     """
-    if len(fromstr) != len(tostr):
+    n = len(fromstr)
+    if n != len(tostr):
         raise ValueError, "maketrans arguments must have same length"
-    global _idmapL
-    if not _idmapL:
-        _idmapL = list(_idmap)
-    L = _idmapL[:]
-    fromstr = map(ord, fromstr)
-    for i in range(len(fromstr)):
-        L[fromstr[i]] = tostr[i]
-    return ''.join(L)
+    # this function has been rewritten to suit PyPy better; it is
+    # almost 10x faster than the original.
+    buf = bytearray(256)
+    for i in range(256):
+        buf[i] = i
+    for i in range(n):
+        buf[ord(fromstr[i])] = tostr[i]
+    return str(buf)
 
 
 

lib-python/2.7/test/test_mailbox.py

     def _delete_recursively(self, target):
         # Delete a file or delete a directory recursively
         if os.path.isdir(target):
-            for path, dirs, files in os.walk(target, topdown=False):
-                for name in files:
-                    os.remove(os.path.join(path, name))
-                for name in dirs:
-                    os.rmdir(os.path.join(path, name))
-            os.rmdir(target)
+            test_support.rmtree(target)
         elif os.path.exists(target):
-            os.remove(target)
+            test_support.unlink(target)
 
 
 class TestMailbox(TestBase):
         msg = self._box.get(key1)
         self.assertEqual(msg['from'], 'foo')
         self.assertEqual(msg.fp.read(), '1')
+        msg.fp.close()
 
     def test_getitem(self):
         # Retrieve message using __getitem__()
         # Get file representations of messages
         key0 = self._box.add(self._template % 0)
         key1 = self._box.add(_sample_message)
-        self.assertEqual(self._box.get_file(key0).read().replace(os.linesep, '\n'),
+        msg0 = self._box.get_file(key0)
+        self.assertEqual(msg0.read().replace(os.linesep, '\n'),
                          self._template % 0)
-        self.assertEqual(self._box.get_file(key1).read().replace(os.linesep, '\n'),
+        msg1 = self._box.get_file(key1)
+        self.assertEqual(msg1.read().replace(os.linesep, '\n'),
                          _sample_message)
+        msg0.close()
+        msg1.close()
 
     def test_get_file_can_be_closed_twice(self):
         # Issue 11700
         self._box.add(contents[0])
         self._box.add(contents[1])
         self._box.add(contents[2])
+        oldbox = self._box
         method()
         if should_call_close:
             self._box.close()
         self.assertEqual(len(keys), 3)
         for key in keys:
             self.assertIn(self._box.get_string(key), contents)
+        oldbox.close()
 
     def test_dump_message(self):
         # Write message representations to disk
     def setUp(self):
         # create a new maildir mailbox to work with:
         self._dir = test_support.TESTFN
+        if os.path.isdir(self._dir):
+            test_support.rmtree(self._dir)
+        if os.path.isfile(self._dir):
+            test_support.unlink(self._dir)
         os.mkdir(self._dir)
         os.mkdir(os.path.join(self._dir, "cur"))
         os.mkdir(os.path.join(self._dir, "tmp"))
 
     def tearDown(self):
         map(os.unlink, self._msgfiles)
-        os.rmdir(os.path.join(self._dir, "cur"))
-        os.rmdir(os.path.join(self._dir, "tmp"))
-        os.rmdir(os.path.join(self._dir, "new"))
-        os.rmdir(self._dir)
+        test_support.rmdir(os.path.join(self._dir, "cur"))
+        test_support.rmdir(os.path.join(self._dir, "tmp"))
+        test_support.rmdir(os.path.join(self._dir, "new"))
+        test_support.rmdir(self._dir)
 
     def createMessage(self, dir, mbox=False):
         t = int(time.time() % 1000000)
         self.createMessage("cur")
         self.mbox = mailbox.Maildir(test_support.TESTFN)
         #self.assertTrue(len(self.mbox.boxes) == 1)
-        self.assertIsNot(self.mbox.next(), None)
+        msg = self.mbox.next()
+        self.assertIsNot(msg, None)
+        msg.fp.close()
         self.assertIs(self.mbox.next(), None)
         self.assertIs(self.mbox.next(), None)
 
         self.createMessage("new")
         self.mbox = mailbox.Maildir(test_support.TESTFN)
         #self.assertTrue(len(self.mbox.boxes) == 1)
-        self.assertIsNot(self.mbox.next(), None)
+        msg = self.mbox.next()
+        self.assertIsNot(msg, None)
+        msg.fp.close()
         self.assertIs(self.mbox.next(), None)
         self.assertIs(self.mbox.next(), None)
 
         self.createMessage("new")
         self.mbox = mailbox.Maildir(test_support.TESTFN)
         #self.assertTrue(len(self.mbox.boxes) == 2)
-        self.assertIsNot(self.mbox.next(), None)
-        self.assertIsNot(self.mbox.next(), None)
+        msg = self.mbox.next()
+        self.assertIsNot(msg, None)
+        msg.fp.close()
+        msg = self.mbox.next()
+        self.assertIsNot(msg, None)
+        msg.fp.close()
         self.assertIs(self.mbox.next(), None)
         self.assertIs(self.mbox.next(), None)
 
         import email.parser
         fname = self.createMessage("cur", True)
         n = 0
-        for msg in mailbox.PortableUnixMailbox(open(fname),
+        fid = open(fname)
+        for msg in mailbox.PortableUnixMailbox(fid,
                                                email.parser.Parser().parse):
             n += 1
             self.assertEqual(msg["subject"], "Simple Test")
             self.assertEqual(len(str(msg)), len(FROM_)+len(DUMMY_MESSAGE))
+        fid.close()
         self.assertEqual(n, 1)
 
 ## End: classes from the original module (for backward compatibility).

lib-python/2.7/test/test_mmap.py

 
     def setUp(self):
         if os.path.exists(TESTFN):
-            os.unlink(TESTFN)
+            unlink(TESTFN)
 
     def tearDown(self):
         try:

lib-python/2.7/test/test_multiprocessing.py

 #!/usr/bin/env python
 
+## FIXME: remove when https://bugs.pypy.org/issue1644 is resolved
+import sys
+if sys.platform.startswith('freebsd'):
+    raise Exception("This test hangs on FreeBSD. Test deactivated for now until https://bugs.pypy.org/issue1644 get resolved")
+
 #
 # Unit tests for the multiprocessing package
 #

lib-python/2.7/test/test_old_mailbox.py

         self.createMessage("cur")
         self.mbox = mailbox.Maildir(test_support.TESTFN)
         self.assertTrue(len(self.mbox) == 1)
-        self.assertTrue(self.mbox.next() is not None)
+        msg = self.mbox.next()
+        self.assertTrue(msg is not None)
+        msg.fp.close()
         self.assertTrue(self.mbox.next() is None)
         self.assertTrue(self.mbox.next() is None)
 
         self.createMessage("new")
         self.mbox = mailbox.Maildir(test_support.TESTFN)
         self.assertTrue(len(self.mbox) == 1)
-        self.assertTrue(self.mbox.next() is not None)
+        msg = self.mbox.next()
+        self.assertTrue(msg is not None)
+        msg.fp.close()
         self.assertTrue(self.mbox.next() is None)
         self.assertTrue(self.mbox.next() is None)
 
         self.createMessage("new")
         self.mbox = mailbox.Maildir(test_support.TESTFN)
         self.assertTrue(len(self.mbox) == 2)
-        self.assertTrue(self.mbox.next() is not None)
-        self.assertTrue(self.mbox.next() is not None)
+        msg = self.mbox.next()
+        self.assertTrue(msg is not None)
+        msg.fp.close()
+        msg = self.mbox.next()
+        self.assertTrue(msg is not None)
+        msg.fp.close()
         self.assertTrue(self.mbox.next() is None)
         self.assertTrue(self.mbox.next() is None)
 

lib-python/2.7/test/test_os.py

         self.assertFalse(os.path.exists(name),
                     "file already exists for temporary file")
         # make sure we can create the file
-        open(name, "w")
+        open(name, "w").close()
         self.files.append(name)
 
     def test_tempnam(self):

lib-python/2.7/test/test_support.py

     except KeyError:
         pass
 
+if sys.platform.startswith("win"):
+    def _waitfor(func, pathname, waitall=False):
+        # Peform the operation
+        func(pathname)
+        # Now setup the wait loop
+        if waitall:
+            dirname = pathname
+        else:
+            dirname, name = os.path.split(pathname)
+            dirname = dirname or '.'
+        # Check for `pathname` to be removed from the filesystem.
+        # The exponential backoff of the timeout amounts to a total
+        # of ~1 second after which the deletion is probably an error
+        # anyway.
+        # Testing on a i7@4.3GHz shows that usually only 1 iteration is
+        # required when contention occurs.
+        timeout = 0.001
+        while timeout < 1.0:
+            # Note we are only testing for the existance of the file(s) in
+            # the contents of the directory regardless of any security or
+            # access rights.  If we have made it this far, we have sufficient
+            # permissions to do that much using Python's equivalent of the
+            # Windows API FindFirstFile.
+            # Other Windows APIs can fail or give incorrect results when
+            # dealing with files that are pending deletion.
+            L = os.listdir(dirname)
+            if not (L if waitall else name in L):
+                return
+            # Increase the timeout and try again
+            time.sleep(timeout)
+            timeout *= 2
+        warnings.warn('tests may fail, delete still pending for ' + pathname,
+                      RuntimeWarning, stacklevel=4)
+
+    def _unlink(filename):
+        _waitfor(os.unlink, filename)
+
+    def _rmdir(dirname):
+        _waitfor(os.rmdir, dirname)
+
+    def _rmtree(path):
+        def _rmtree_inner(path):
+            for name in os.listdir(path):
+                fullname = os.path.join(path, name)
+                if os.path.isdir(fullname):
+                    _waitfor(_rmtree_inner, fullname, waitall=True)
+                    os.rmdir(fullname)
+                else:
+                    os.unlink(fullname)
+        _waitfor(_rmtree_inner, path, waitall=True)
+        _waitfor(os.rmdir, path)
+else:
+    _unlink = os.unlink
+    _rmdir = os.rmdir
+    _rmtree = shutil.rmtree
+
 def unlink(filename):
     try:
-        os.unlink(filename)
+        _unlink(filename)
     except OSError:
         pass
 
+def rmdir(dirname):
+    try:
+        _rmdir(dirname)
+    except OSError as error:
+        # The directory need not exist.
+        if error.errno != errno.ENOENT:
+            raise
+
 def rmtree(path):
     try:
-        shutil.rmtree(path)
+        _rmtree(path)
     except OSError, e:
         # Unix returns ENOENT, Windows returns ESRCH.
         if e.errno not in (errno.ENOENT, errno.ESRCH):

lib-python/2.7/test/test_tarfile.py

 
     def test_extract_hardlink(self):
         # Test hardlink extraction (e.g. bug #857297).
-        tar = tarfile.open(tarname, errorlevel=1, encoding="iso8859-1")
+        with tarfile.open(tarname, errorlevel=1, encoding="iso8859-1") as tar:
+            tar.extract("ustar/regtype", TEMPDIR)
+            self.addCleanup(os.remove, os.path.join(TEMPDIR, "ustar/regtype"))
 
-        tar.extract("ustar/regtype", TEMPDIR)
-        try:
             tar.extract("ustar/lnktype", TEMPDIR)
-        except EnvironmentError, e:
-            if e.errno == errno.ENOENT:
-                self.fail("hardlink not extracted properly")
+            self.addCleanup(os.remove, os.path.join(TEMPDIR, "ustar/lnktype"))
+            with open(os.path.join(TEMPDIR, "ustar/lnktype"), "rb") as f:
+                data = f.read()
+            self.assertEqual(md5sum(data), md5_regtype)
 
-        data = open(os.path.join(TEMPDIR, "ustar/lnktype"), "rb").read()
-        self.assertEqual(md5sum(data), md5_regtype)
-
-        try:
             tar.extract("ustar/symtype", TEMPDIR)
-        except EnvironmentError, e:
-            if e.errno == errno.ENOENT:
-                self.fail("symlink not extracted properly")
-
-        data = open(os.path.join(TEMPDIR, "ustar/symtype"), "rb").read()
-        self.assertEqual(md5sum(data), md5_regtype)
+            self.addCleanup(os.remove, os.path.join(TEMPDIR, "ustar/symtype"))
+            with open(os.path.join(TEMPDIR, "ustar/symtype"), "rb") as f:
+                data = f.read()
+            self.assertEqual(md5sum(data), md5_regtype)
 
     def test_extractall(self):
         # Test if extractall() correctly restores directory permissions
         # constructor in case of an error. For the test we rely on
         # the fact that opening an empty file raises a ReadError.
         empty = os.path.join(TEMPDIR, "empty")
-        open(empty, "wb").write("")
+        open(empty, "wb").close()
 
         try:
             tar = object.__new__(tarfile.TarFile)
             else:
                 self.fail("ReadError not raised")
         finally:
-            os.remove(empty)
+            test_support.unlink(empty)
 
 
 class StreamReadTest(CommonReadTest):
     def setUp(self):
         self.tarname = tmpname
         if os.path.exists(self.tarname):
-            os.remove(self.tarname)
+            test_support.unlink(self.tarname)
 
     def _add_testfile(self, fileobj=None):
         tar = tarfile.open(self.tarname, "a", fileobj=fileobj)

lib-python/2.7/traceback.py

     return list
 
 
-def print_exception(etype, value, tb, limit=None, file=None):
+def print_exception(etype, value, tb, limit=None, file=None, _encoding=None):
     """Print exception up to 'limit' stack trace entries from 'tb' to 'file'.
 
     This differs from print_tb() in the following ways: (1) if
     if tb:
         _print(file, 'Traceback (most recent call last):')
         print_tb(tb, limit, file)
-    lines = format_exception_only(etype, value)
+    lines = format_exception_only(etype, value, _encoding)
     for line in lines:
         _print(file, line, '')
 
     list = list + format_exception_only(etype, value)
     return list
 
-def format_exception_only(etype, value):
+def format_exception_only(etype, value, _encoding=None):
     """Format the exception part of a traceback.
 
     The arguments are the exception type and value such as given by
     if (isinstance(etype, BaseException) or
         isinstance(etype, types.InstanceType) or
         etype is None or type(etype) is str):
-        return [_format_final_exc_line(etype, value)]
+        return [_format_final_exc_line(etype, value, _encoding)]
 
     stype = etype.__name__
 
     if not issubclass(etype, SyntaxError):
-        return [_format_final_exc_line(stype, value)]
+        return [_format_final_exc_line(stype, value, _encoding)]
 
     # It was a syntax error; show exactly where the problem was found.
     lines = []
                 lines.append('   %s^\n' % ''.join(caretspace))
         value = msg
 
-    lines.append(_format_final_exc_line(stype, value))
+    lines.append(_format_final_exc_line(stype, value, _encoding))
     return lines
 
-def _format_final_exc_line(etype, value):
+def _format_final_exc_line(etype, value, _encoding=None):
     """Return a list of a single line -- normal case for format_exception_only"""
-    valuestr = _some_str(value)
+    valuestr = _some_str(value, _encoding)
     if value is None or not valuestr:
         line = "%s\n" % etype
     else:
         line = "%s: %s\n" % (etype, valuestr)
     return line
 
-def _some_str(value):
+def _some_str(value, _encoding=None):
     try:
         return str(value)
     except Exception:
         pass
     try:
         value = unicode(value)
-        return value.encode("ascii", "backslashreplace")
+        return value.encode(_encoding or "ascii", "backslashreplace")
     except Exception:
         pass
     return '<unprintable %s object>' % type(value).__name__

lib-python/2.7/weakref.py

         def remove(wr, selfref=ref(self)):
             self = selfref()
             if self is not None:
-                del self.data[wr.key]
+                # Changed this for PyPy: made more resistent.  The
+                # issue is that in some corner cases, self.data
+                # might already be changed or removed by the time
+                # this weakref's callback is called.  If that is
+                # the case, we don't want to randomly kill an
+                # unrelated entry.
+                if self.data.get(wr.key) is wr:
+                    del self.data[wr.key]
         self._remove = remove
         UserDict.UserDict.__init__(self, *args, **kw)
 
         try:
             o = self.data.pop(key)()
         except KeyError:
+            o = None
+        if o is None:
             if args:
                 return args[0]
-            raise
-        if o is None:
             raise KeyError, key
         else:
             return o
+        # The logic above was fixed in PyPy
 
     def setdefault(self, key, default=None):
         try:
-            wr = self.data[key]
+            o = self.data[key]()
         except KeyError:
+            o = None
+        if o is None:
             self.data[key] = KeyedRef(default, self._remove, key)
             return default
         else:
-            return wr()
+            return o
+        # The logic above was fixed in PyPy
 
     def update(self, dict=None, **kwargs):
         d = self.data

lib_pypy/_sqlite3.py

 if _has_load_extension():
     _ffi.cdef("int sqlite3_enable_load_extension(sqlite3 *db, int onoff);")
 
-_lib = _ffi.verify("""
-#include <sqlite3.h>
-""", libraries=['sqlite3']
-)
+if sys.platform.startswith('freebsd'):
+    _lib = _ffi.verify("""
+    #include <sqlite3.h>
+    """, libraries=['sqlite3'],
+         include_dirs=['/usr/local/include'],
+         library_dirs=['/usr/local/lib']
+    )
+else:
+    _lib = _ffi.verify("""
+    #include <sqlite3.h>
+    """, libraries=['sqlite3']
+    )
 
 exported_sqlite_symbols = [
     'SQLITE_ALTER_TABLE',

lib_pypy/_tkinter/tklib.py

     incdirs = ['/usr/local/include/tcl8.5', '/usr/local/include/tk8.5', '/usr/X11R6/include']
     linklibs = ['tk85', 'tcl85']
     libdirs = ['/usr/local/lib', '/usr/X11R6/lib']
+elif sys.platform.startswith("freebsd"):
+    incdirs = ['/usr/local/include/tcl8.6', '/usr/local/include/tk8.6', '/usr/local/include/X11', '/usr/local/include']
+    linklibs = ['tk86', 'tcl86']
+    libdirs = ['/usr/local/lib']
+elif sys.platform == 'win32':
+    incdirs = []
+    linklibs = ['tcl85', 'tk85']
+    libdirs = []
 else:
     incdirs=['/usr/include/tcl']
     linklibs=['tcl', 'tk']

lib_pypy/cffi.egg-info

 Metadata-Version: 1.0
 Name: cffi
-Version: 0.7
+Version: 0.8
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski

lib_pypy/cffi/__init__.py

 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "0.7.2"
-__version_info__ = (0, 7, 2)
+__version__ = "0.8"
+__version_info__ = (0, 8)

lib_pypy/cffi/api.py

 import types
+from .lock import allocate_lock
 
 try:
     callable
             # rely on it!  It's probably not going to work well.)
 
         self._backend = backend
+        self._lock = allocate_lock()
         self._parser = cparser.Parser()
         self._cached_btypes = {}
         self._parsed_types = types.ModuleType('parsed_types').__dict__
             if name.startswith('RTLD_'):
                 setattr(self, name, getattr(backend, name))
         #
-        self.BVoidP = self._get_cached_btype(model.voidp_type)
+        with self._lock:
+            self.BVoidP = self._get_cached_btype(model.voidp_type)
         if isinstance(backend, types.ModuleType):
             # _cffi_backend: attach these constants to the class
             if not hasattr(FFI, 'NULL'):
             if not isinstance(csource, basestring):
                 raise TypeError("cdef() argument must be a string")
             csource = csource.encode('ascii')
-        self._parser.parse(csource, override=override)
-        self._cdefsources.append(csource)
-        if override:
-            for cache in self._function_caches:
-                cache.clear()
+        with self._lock:
+            self._parser.parse(csource, override=override)
+            self._cdefsources.append(csource)
+            if override:
+                for cache in self._function_caches:
+                    cache.clear()
 
     def dlopen(self, name, flags=0):
         """Load and return a dynamic library identified by 'name'.
         library we only look for the actual (untyped) symbols.
         """
         assert isinstance(name, basestring) or name is None
-        lib, function_cache = _make_ffi_library(self, name, flags)
-        self._function_caches.append(function_cache)
-        self._libraries.append(lib)
+        with self._lock:
+            lib, function_cache = _make_ffi_library(self, name, flags)
+            self._function_caches.append(function_cache)
+            self._libraries.append(lib)
         return lib
 
+    def _typeof_locked(self, cdecl):
+        # call me with the lock!
+        key = cdecl
+        if key in self._parsed_types:
+            return self._parsed_types[key]
+        #
+        if not isinstance(cdecl, str):    # unicode, on Python 2
+            cdecl = cdecl.encode('ascii')
+        #
+        type = self._parser.parse_type(cdecl)
+        really_a_function_type = type.is_raw_function
+        if really_a_function_type:
+            type = type.as_function_pointer()
+        btype = self._get_cached_btype(type)
+        result = btype, really_a_function_type
+        self._parsed_types[key] = result
+        return result
+
     def _typeof(self, cdecl, consider_function_as_funcptr=False):
         # string -> ctype object
         try:
-            btype, cfaf = self._parsed_types[cdecl]
-            if consider_function_as_funcptr and not cfaf:
-                raise KeyError
+            result = self._parsed_types[cdecl]
         except KeyError:
-            key = cdecl
-            if not isinstance(cdecl, str):    # unicode, on Python 2
-                cdecl = cdecl.encode('ascii')
-            cfaf = consider_function_as_funcptr
-            type = self._parser.parse_type(cdecl,
-                       consider_function_as_funcptr=cfaf)
-            btype = self._get_cached_btype(type)
-            self._parsed_types[key] = btype, cfaf
+            with self._lock:
+                result = self._typeof_locked(cdecl)
+        #
+        btype, really_a_function_type = result
+        if really_a_function_type and not consider_function_as_funcptr:
+            raise CDefError("the type %r is a function type, not a "
+                            "pointer-to-function type" % (cdecl,))
         return btype
 
     def typeof(self, cdecl):
         """Parse the C type given as a string and return the
-        corresponding Python type: <class 'ffi.CData<...>'>.
+        corresponding <ctype> object.
         It can also be used on 'cdata' instance to get its C type.
         """
         if isinstance(cdecl, basestring):
             res = _builtin_function_type(cdecl)
             if res is not None:
                 return res
+        if (isinstance(cdecl, types.FunctionType)
+                and hasattr(cdecl, '_cffi_base_type')):
+            with self._lock:
+                return self._get_cached_btype(cdecl._cffi_base_type)
         raise TypeError(type(cdecl))
 
     def sizeof(self, cdecl):
         data.  Later, when this new cdata object is garbage-collected,
         'destructor(old_cdata_object)' will be called.
         """
-        try:
-            gc_weakrefs = self.gc_weakrefs
-        except AttributeError:
-            from .gc_weakref import GcWeakrefs
-            gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self)
-        return gc_weakrefs.build(cdata, destructor)
+        with self._lock:
+            try:
+                gc_weakrefs = self.gc_weakrefs
+            except AttributeError:
+                from .gc_weakref import GcWeakrefs
+                gc_weakrefs = self.gc_weakrefs = GcWeakrefs(self)
+            return gc_weakrefs.build(cdata, destructor)
 
     def _get_cached_btype(self, type):
+        assert self._lock.acquire(False) is False
+        # call me with the lock!
         try:
             BType = self._cached_btypes[type]
         except KeyError:
     errno = property(_get_errno, _set_errno, None,
                      "the value of 'errno' from/to the C calls")
 
+    def getwinerror(self, code=-1):
+        return self._backend.getwinerror(code)
+
     def _pointer_to(self, ctype):
         from . import model
-        return model.pointer_cache(self, ctype)
+        with self._lock:
+            return model.pointer_cache(self, ctype)
 
     def addressof(self, cdata, field=None):
         """Return the address of a <cdata 'struct-or-union'>.
         variables, which must anyway be accessed directly from the
         lib object returned by the original FFI instance.
         """
-        self._parser.include(ffi_to_include._parser)
-        self._cdefsources.append('[')
-        self._cdefsources.extend(ffi_to_include._cdefsources)
-        self._cdefsources.append(']')
+        with ffi_to_include._lock:
+            with self._lock:
+                self._parser.include(ffi_to_include._parser)
+                self._cdefsources.append('[')
+                self._cdefsources.extend(ffi_to_include._cdefsources)
+                self._cdefsources.append(']')
 
     def new_handle(self, x):
         return self._backend.newp_handle(self.BVoidP, x)
         backendlib = backend.load_library(path, flags)
     copied_enums = []
     #
-    def make_accessor(name):
+    def make_accessor_locked(name):
         key = 'function ' + name
         if key in ffi._parser._declarations:
             tp = ffi._parser._declarations[key]
                     if enumname not in library.__dict__:
                         library.__dict__[enumname] = enumval
             copied_enums.append(True)
+            if name in library.__dict__:
+                return
         #
-        if name in library.__dict__:   # copied from an enum value just above,
-            return                     # or multithread's race condition
         raise AttributeError(name)
     #
+    def make_accessor(name):
+        with ffi._lock:
+            if name in library.__dict__ or name in FFILibrary.__dict__:
+                return    # added by another thread while waiting for the lock
+            make_accessor_locked(name)
+    #
     class FFILibrary(object):
         def __getattr__(self, name):
             make_accessor(name)
     except (KeyError, AttributeError, TypeError):
         return None
     else:
-        return ffi._get_cached_btype(tp)
+        with ffi._lock:
+            return ffi._get_cached_btype(tp)

lib_pypy/cffi/cparser.py

                 if 1 <= linenum <= len(csourcelines):
                     line = csourcelines[linenum-1]
         if line:
-            msg = 'cannot parse "%s"\n%s' % (line, msg)
+            msg = 'cannot parse "%s"\n%s' % (line.strip(), msg)
         else:
             msg = 'parse error\n%s' % (msg,)
         raise api.CDefError(msg)
             #
             if decl.name:
                 tp = self._get_type(node, partial_length_ok=True)
-                if self._is_constant_declaration(node):
+                if self._is_constant_globalvar(node):
                     self._declare('constant ' + decl.name, tp)
                 else:
                     self._declare('variable ' + decl.name, tp)
 
-    def parse_type(self, cdecl, consider_function_as_funcptr=False):
+    def parse_type(self, cdecl):
         ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl)
         assert not macros
         exprnode = ast.ext[-1].type.args.params[0]
         if isinstance(exprnode, pycparser.c_ast.ID):
             raise api.CDefError("unknown identifier '%s'" % (exprnode.name,))
-        return self._get_type(exprnode.type,
-                     consider_function_as_funcptr=consider_function_as_funcptr)
+        return self._get_type(exprnode.type)
 
     def _declare(self, name, obj):
         if name in self._declarations:
             return model.ConstPointerType(type)
         return model.PointerType(type)
 
-    def _get_type(self, typenode, convert_array_to_pointer=False,
-                  name=None, partial_length_ok=False,
-                  consider_function_as_funcptr=False):
+    def _get_type(self, typenode, name=None, partial_length_ok=False):
         # first, dereference typedefs, if we have it already parsed, we're good
         if (isinstance(typenode, pycparser.c_ast.TypeDecl) and
             isinstance(typenode.type, pycparser.c_ast.IdentifierType) and
             len(typenode.type.names) == 1 and
             ('typedef ' + typenode.type.names[0]) in self._declarations):
             type = self._declarations['typedef ' + typenode.type.names[0]]
-            if isinstance(type, model.ArrayType):
-                if convert_array_to_pointer:
-                    return type.item
-            else:
-                if (consider_function_as_funcptr and
-                        isinstance(type, model.RawFunctionType)):
-                    return type.as_function_pointer()
             return type
         #
         if isinstance(typenode, pycparser.c_ast.ArrayDecl):
             # array type
-            if convert_array_to_pointer:
-                return self._get_type_pointer(self._get_type(typenode.type))
             if typenode.dim is None:
                 length = None
             else:
         #
         if isinstance(typenode, pycparser.c_ast.FuncDecl):
             # a function type
-            result = self._parse_function_type(typenode, name)
-            if consider_function_as_funcptr:
-                result = result.as_function_pointer()
-            return result
+            return self._parse_function_type(typenode, name)
         #
         # nested anonymous structs or unions end up here
         if isinstance(typenode, pycparser.c_ast.Struct):
             isinstance(params[0].type.type, pycparser.c_ast.IdentifierType)
                 and list(params[0].type.type.names) == ['void']):
             del params[0]
-        args = [self._get_type(argdeclnode.type,
-                               convert_array_to_pointer=True,
-                               consider_function_as_funcptr=True)
+        args = [self._as_func_arg(self._get_type(argdeclnode.type))
                 for argdeclnode in params]
         result = self._get_type(typenode.type)
         return model.RawFunctionType(tuple(args), result, ellipsis)
 
-    def _is_constant_declaration(self, typenode, const=False):
-        if isinstance(typenode, pycparser.c_ast.ArrayDecl):
-            return self._is_constant_declaration(typenode.type)
+    def _as_func_arg(self, type):
+        if isinstance(type, model.ArrayType):
+            return model.PointerType(type.item)
+        elif isinstance(type, model.RawFunctionType):
+            return type.as_function_pointer()
+        else:
+            return type
+
+    def _is_constant_globalvar(self, typenode):
         if isinstance(typenode, pycparser.c_ast.PtrDecl):
-            const = 'const' in typenode.quals
-            return self._is_constant_declaration(typenode.type, const)
+            return 'const' in typenode.quals
         if isinstance(typenode, pycparser.c_ast.TypeDecl):
-            return const or 'const' in typenode.quals
+            return 'const' in typenode.quals
         return False
 
     def _get_struct_union_enum_type(self, kind, type, name=None, nested=False):
         return tp
 
     def _make_partial(self, tp, nested):
-        if not isinstance(tp, model.StructType):
+        if not isinstance(tp, model.StructOrUnion):
             raise api.CDefError("%s cannot be partial" % (tp,))
         if not tp.has_c_name() and not nested:
             raise NotImplementedError("%s is partial but has no C name" %(tp,))
             if (isinstance(exprnode, pycparser.c_ast.ID) and
                     exprnode.name == '__dotdotdotarray__'):
                 self._partial_length = True
-                return None
+                return '...'
         #
         raise api.FFIError("unsupported expression: expected a "
                            "simple numeric constant")

lib_pypy/cffi/gc_weakref.py

 
     def build(self, cdata, destructor):
         # make a new cdata of the same type as the original one
-        new_cdata = self.ffi.cast(self.ffi.typeof(cdata), cdata)
+        new_cdata = self.ffi.cast(self.ffi._backend.typeof(cdata), cdata)
         self.data[ref(new_cdata, self.remove)] = destructor, cdata
         return new_cdata

lib_pypy/cffi/lock.py

+import sys
+
+if sys.version_info < (3,):
+    try:
+        from thread import allocate_lock
+    except ImportError:
+        from dummy_thread import allocate_lock
+else:
+    try:
+        from _thread import allocate_lock
+    except ImportError:
+        from _dummy_thread import allocate_lock
+
+
+##import sys
+##l1 = allocate_lock
+
+##class allocate_lock(object):
+##    def __init__(self):
+##        self._real = l1()
+##    def __enter__(self):
+##        for i in range(4, 0, -1):
+##            print sys._getframe(i).f_code
+##        print
+##        return self._real.__enter__()
+##    def __exit__(self, *args):
+##        return self._real.__exit__(*args)
+##    def acquire(self, f):
+##        assert f is False
+##        return self._real.acquire(f)

lib_pypy/cffi/model.py

 import weakref
+from .lock import allocate_lock
+
 
 class BaseTypeByIdentity(object):
     is_array_type = False
+    is_raw_function = False
 
     def get_c_name(self, replace_with='', context='a C file'):
         result = self.c_name_with_marker
     # a function, but not a pointer-to-function.  The backend has no
     # notion of such a type; it's used temporarily by parsing.
     _base_pattern = '(&)(%s)'
+    is_raw_function = True
 
     def build_backend_type(self, ffi, finishlist):
         from . import api
     _base_pattern       = " const *&"
     _base_pattern_array = "(const *&)"
 
-    def build_backend_type(self, ffi, finishlist):
-        BPtr = PointerType(self.totype).get_cached_btype(ffi, finishlist)
-        return BPtr
-
 const_voidp_type = ConstPointerType(void_type)
 
 
         self.item = item
         self.length = length
         #
-        if self.length is None:
+        if length is None:
             brackets = '&[]'
+        elif length == '...':
+            brackets = '&[/*...*/]'
         else:
-            brackets = '&[%d]' % self.length
+            brackets = '&[%d]' % length
         self.c_name_with_marker = (
             self.item.c_name_with_marker.replace('&', brackets))
 
         return ArrayType(self.item, newlength)
 
     def build_backend_type(self, ffi, finishlist):
+        if self.length == '...':
+            from . import api
+            raise api.CDefError("cannot render the type %r: unknown length" %
+                                (self,))
         self.item.get_cached_btype(ffi, finishlist)   # force the item BType
         BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist)
         return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length)
 class StructOrUnion(StructOrUnionOrEnum):
     fixedlayout = None
     completed = False
+    partial = False
 
     def __init__(self, name, fldnames, fldtypes, fldbitsize):
         self.name = name
             return    # not completing it: it's an opaque struct
         #
         self.completed = 1
-        fldtypes = tuple(tp.get_cached_btype(ffi, finishlist)
-                         for tp in self.fldtypes)
         #
         if self.fixedlayout is None:
+            fldtypes = [tp.get_cached_btype(ffi, finishlist)
+                        for tp in self.fldtypes]
             lst = list(zip(self.fldnames, fldtypes, self.fldbitsize))
             ffi._backend.complete_struct_or_union(BType, lst, self)
             #
         else:
+            fldtypes = []
             fieldofs, fieldsize, totalsize, totalalignment = self.fixedlayout
             for i in range(len(self.fldnames)):
                 fsize = fieldsize[i]
                 ftype = self.fldtypes[i]
                 #
-                if isinstance(ftype, ArrayType) and ftype.length is None:
+                if isinstance(ftype, ArrayType) and ftype.length == '...':
                     # fix the length to match the total size
                     BItemType = ftype.item.get_cached_btype(ffi, finishlist)
                     nlen, nrest = divmod(fsize, ffi.sizeof(BItemType))
                     ftype = ftype.resolve_length(nlen)
                     self.fldtypes = (self.fldtypes[:i] + (ftype,) +
                                      self.fldtypes[i+1:])
-                    BArrayType = ftype.get_cached_btype(ffi, finishlist)
-                    fldtypes = (fldtypes[:i] + (BArrayType,) +
-                                fldtypes[i+1:])
-                    continue
                 #
-                bitemsize = ffi.sizeof(fldtypes[i])
-                if bitemsize != fsize:
-                    self._verification_error(
-                        "field '%s.%s' is declared as %d bytes, but is "
-                        "really %d bytes" % (self.name,
-                                             self.fldnames[i] or '{}',
-                                             bitemsize, fsize))
+                BFieldType = ftype.get_cached_btype(ffi, finishlist)
+                if isinstance(ftype, ArrayType) and ftype.length is None:
+                    assert fsize == 0
+                else:
+                    bitemsize = ffi.sizeof(BFieldType)
+                    if bitemsize != fsize:
+                        self._verification_error(
+                            "field '%s.%s' is declared as %d bytes, but is "
+                            "really %d bytes" % (self.name,
+                                                 self.fldnames[i] or '{}',
+                                                 bitemsize, fsize))
+                fldtypes.append(BFieldType)
+            #
             lst = list(zip(self.fldnames, fldtypes, self.fldbitsize, fieldofs))
             ffi._backend.complete_struct_or_union(BType, lst, self,
                                                   totalsize, totalalignment)
         from .ffiplatform import VerificationError
         raise VerificationError(msg)
 
-
-class StructType(StructOrUnion):
-    kind = 'struct'
-    partial = False
-
     def check_not_partial(self):
         if self.partial and self.fixedlayout is None:
             from . import ffiplatform
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
-        
-        return global_cache(self, ffi, 'new_struct_type',
+        #
+        return global_cache(self, ffi, 'new_%s_type' % self.kind,
                             self.get_official_name(), key=self)
 
 
+class StructType(StructOrUnion):
+    kind = 'struct'
+
+
 class UnionType(StructOrUnion):
     kind = 'union'
 
-    def build_backend_type(self, ffi, finishlist):
-        finishlist.append(self)
-        return global_cache(self, ffi, 'new_union_type',
-                            self.get_official_name(), key=self)
-
 
 class EnumType(StructOrUnionOrEnum):
     kind = 'enum'
         self.baseinttype = baseinttype
         self.build_c_name_with_marker()
 
+    def force_the_name(self, forcename):
+        StructOrUnionOrEnum.force_the_name(self, forcename)
+        if self.forcename is None:
+            name = self.get_official_name()
+            self.forcename = '$' + name.replace(' ', '_')
+
     def check_not_partial(self):
         if self.partial and not self.partial_resolved:
             from . import ffiplatform
     tp = StructType(structname, None, None, None)
     return NamedPointerType(tp, name)
 
+
+global_lock = allocate_lock()
+
 def global_cache(srctype, ffi, funcname, *args, **kwds):
     key = kwds.pop('key', (funcname, args))
     assert not kwds
         res = getattr(ffi._backend, funcname)(*args)
     except NotImplementedError as e:
         raise NotImplementedError("%r: %s" % (srctype, e))
-    ffi._backend.__typecache[key] = res
-    return res
+    # note that setdefault() on WeakValueDictionary is not atomic
+    # and contains a rare bug (http://bugs.python.org/issue19542);
+    # we have to use a lock and do it ourselves
+    cache = ffi._backend.__typecache
+    with global_lock:
+        res1 = cache.get(key)
+        if res1 is None:
+            cache[key] = res
+            return res
+        else:
+            return res1
 
 def pointer_cache(ffi, BType):
     return global_cache('?', ffi, 'new_pointer_type', BType)

lib_pypy/cffi/vengine_cpy.py

     def patch_extension_kwds(self, kwds):
         pass
 
-    def find_module(self, module_name, path, so_suffix):
+    def find_module(self, module_name, path, so_suffixes):
         try:
             f, filename, descr = imp.find_module(module_name, path)
         except ImportError:
         # Note that after a setuptools installation, there are both .py
         # and .so files with the same basename.  The code here relies on
         # imp.find_module() locating the .so in priority.
-        if descr[0] != so_suffix:
+        if descr[0] not in so_suffixes:
             return None
         return filename
 
             def __dir__(self):
                 return FFILibrary._cffi_dir + list(self.__dict__)
         library = FFILibrary()
-        module._cffi_setup(lst, ffiplatform.VerificationError, library)
+        if module._cffi_setup(lst, ffiplatform.VerificationError, library):
+            import warnings
+            warnings.warn("reimporting %r might overwrite older definitions"
+                          % (self.verifier.get_module_name()))
         #
         # finally, call the loaded_cpy_xxx() functions.  This will perform
         # the final adjustments, like copying the Python->C wrapper
             return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
                 var, self._gettypenum(tp))
         elif isinstance(tp, model.ArrayType):
-            return '_cffi_from_c_deref((char *)%s, _cffi_type(%d))' % (
-                var, self._gettypenum(tp))
+            return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % (
+                var, self._gettypenum(model.PointerType(tp.item)))
         elif isinstance(tp, model.StructType):
             if tp.fldnames is None:
                 raise TypeError("'%s' is used as %s, but is opaque" % (
         prnt('  static Py_ssize_t nums[] = {')
         prnt('    sizeof(%s),' % cname)
         prnt('    offsetof(struct _cffi_aligncheck, y),')
-        for fname, _, fbitsize in tp.enumfields():
+        for fname, ftype, fbitsize in tp.enumfields():
             if fbitsize >= 0:
                 continue      # xxx ignore fbitsize for now
             prnt('    offsetof(%s, %s),' % (cname, fname))
-            prnt('    sizeof(((%s *)0)->%s),' % (cname, fname))
+            if isinstance(ftype, model.ArrayType) and ftype.length is None:
+                prnt('    0,  /* %s */' % ftype._get_c_name())
+            else:
+                prnt('    sizeof(((%s *)0)->%s),' % (cname, fname))
         prnt('    -1')
         prnt('  };')
         prnt('  return _cffi_get_struct_layout(nums);')
         #
         function = getattr(module, layoutfuncname)
         layout = function()
-        if isinstance(tp, model.StructType) and tp.partial:
+        if isinstance(tp, model.StructOrUnion) and tp.partial:
             # use the function()'s sizes and offsets to guide the
             # layout of the struct
             totalsize = layout[0]
                     continue        # xxx ignore fbitsize for now
                 check(layout[i], ffi.offsetof(BStruct, fname),
                       "wrong offset for field %r" % (fname,))
-                BField = ffi._get_cached_btype(ftype)
-                check(layout[i+1], ffi.sizeof(BField),
-                      "wrong size for field %r" % (fname,))
+                if layout[i+1] != 0:
+                    BField = ffi._get_cached_btype(ftype)
+                    check(layout[i+1], ffi.sizeof(BField),
+                          "wrong size for field %r" % (fname,))
                 i += 2
             assert i == len(layout)
 
     # constants, likely declared with '#define'
 
     def _generate_cpy_const(self, is_int, name, tp=None, category='const',
-                            vartp=None, delayed=True):
+                            vartp=None, delayed=True, size_too=False):
         prnt = self._prnt
         funcname = '_cffi_%s_%s' % (category, name)
         prnt('static int %s(PyObject *lib)' % funcname)
                  '(unsigned long long)(%s));' % (name,))
         prnt('  if (o == NULL)')
         prnt('    return -1;')
+        if size_too:
+            prnt('  {')
+            prnt('    PyObject *o1 = o;')
+            prnt('    o = Py_BuildValue("On", o1, (Py_ssize_t)sizeof(%s));'
+                 % (name,))
+            prnt('    Py_DECREF(o1);')
+            prnt('    if (o == NULL)')
+            prnt('      return -1;')
+            prnt('  }')
         prnt('  res = PyObject_SetAttrString(lib, "%s", o);' % name)
         prnt('  Py_DECREF(o);')
         prnt('  if (res < 0)')
         prnt('static int %s(PyObject *lib)' % funcname)
         prnt('{')
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
-            prnt('  if (%s != %d) {' % (enumerator, enumvalue))
+            if enumvalue < 0:
+                prnt('  if ((%s) >= 0 || (long)(%s) != %dL) {' % (
+                    enumerator, enumerator, enumvalue))
+            else:
+                prnt('  if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % (
+                    enumerator, enumerator, enumvalue))
+            prnt('    char buf[64];')
+            prnt('    if ((%s) < 0)' % enumerator)
+            prnt('        snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator)
+            prnt('    else')
+            prnt('        snprintf(buf, 63, "%%lu", (unsigned long)(%s));' %
+                 enumerator)
             prnt('    PyErr_Format(_cffi_VerificationError,')
-            prnt('                 "enum %s: %s has the real value %d, '
-                 'not %d",')
-            prnt('                 "%s", "%s", (int)%s, %d);' % (
-                name, enumerator, enumerator, enumvalue))
+            prnt('                 "enum %s: %s has the real value %s, '
+                 'not %s",')
+            prnt('                 "%s", "%s", buf, "%d");' % (
+                name, enumerator, enumvalue))
             prnt('    return -1;')
             prnt('  }')
         prnt('  return %s;' % self._chained_list_constants[True])
 
     def _generate_cpy_variable_collecttype(self, tp, name):
         if isinstance(tp, model.ArrayType):
-            self._do_collect_type(tp)
+            tp_ptr = model.PointerType(tp.item)
         else:
             tp_ptr = model.PointerType(tp)
-            self._do_collect_type(tp_ptr)
+        self._do_collect_type(tp_ptr)
 
     def _generate_cpy_variable_decl(self, tp, name):
         if isinstance(tp, model.ArrayType):
             tp_ptr = model.PointerType(tp.item)
-            self._generate_cpy_const(False, name, tp, vartp=tp_ptr)
+            self._generate_cpy_const(False, name, tp, vartp=tp_ptr,
+                                     size_too = (tp.length == '...'))
         else:
             tp_ptr = model.PointerType(tp)
             self._generate_cpy_const(False, name, tp_ptr, category='var')
     _loading_cpy_variable = _loaded_noop
 
     def _loaded_cpy_variable(self, tp, name, module, library):
+        value = getattr(library, name)
         if isinstance(tp, model.ArrayType):   # int a[5] is "constant" in the
-            return                            # sense that "a=..." is forbidden
+                                              # sense that "a=..." is forbidden
+            if tp.length == '...':
+                assert isinstance(value, tuple)
+                (value, size) = value
+                BItemType = self.ffi._get_cached_btype(tp.item)
+                length, rest = divmod(size, self.ffi.sizeof(BItemType))
+                if rest != 0:
+                    raise ffiplatform.VerificationError(
+                        "bad size: %r does not seem to be an array of %s" %
+                        (name, tp.item))
+                tp = tp.resolve_length(length)
+            # 'value' is a <cdata 'type *'> which we have to replace with
+            # a <cdata 'type[N]'> if the N is actually known
+            if tp.length is not None:
+                BArray = self.ffi._get_cached_btype(tp)
+                value = self.ffi.cast(BArray, value)
+                setattr(library, name, value)
+            return
         # remove ptr=<cdata 'int *'> from the library instance, and replace
         # it by a property on the class, which reads/writes into ptr[0].
-        ptr = getattr(library, name)
+        ptr = value
         delattr(library, name)
         def getter(library):
             return ptr[0]
 
     def _generate_setup_custom(self):
         prnt = self._prnt
-        prnt('static PyObject *_cffi_setup_custom(PyObject *lib)')
+        prnt('static int _cffi_setup_custom(PyObject *lib)')
         prnt('{')
-        prnt('  if (%s < 0)' % self._chained_list_constants[True])
-        prnt('    return NULL;')
-        prnt('  Py_INCREF(Py_None);')
-        prnt('  return Py_None;')
+        prnt('  return %s;' % self._chained_list_constants[True])
         prnt('}')
 
 cffimod_header = r'''
 static void *_cffi_exports[_CFFI_NUM_EXPORTS];
 static PyObject *_cffi_types, *_cffi_VerificationError;
 
-static PyObject *_cffi_setup_custom(PyObject *lib);   /* forward */
+static int _cffi_setup_custom(PyObject *lib);   /* forward */
 
 static PyObject *_cffi_setup(PyObject *self, PyObject *args)
 {
     PyObject *library;
+    int was_alive = (_cffi_types != NULL);
     if (!PyArg_ParseTuple(args, "OOO", &_cffi_types, &_cffi_VerificationError,
                                        &library))
         return NULL;
     Py_INCREF(_cffi_types);
     Py_INCREF(_cffi_VerificationError);
-    return _cffi_setup_custom(library);
+    if (_cffi_setup_custom(library) < 0)
+        return NULL;
+    return PyBool_FromLong(was_alive);
 }
 
 static void _cffi_init(void)

lib_pypy/cffi/vengine_gen.py

         # up in kwds['export_symbols'].
         kwds.setdefault('export_symbols', self.export_symbols)
 
-    def find_module(self, module_name, path, so_suffix):
-        basename = module_name + so_suffix
-        if path is None:
-            path = sys.path
-        for dirname in path:
-            filename = os.path.join(dirname, basename)
-            if os.path.isfile(filename):
-                return filename
-        return None
+    def find_module(self, module_name, path, so_suffixes):
+        for so_suffix in so_suffixes:
+            basename = module_name + so_suffix
+            if path is None:
+                path = sys.path
+            for dirname in path:
+                filename = os.path.join(dirname, basename)
+                if os.path.isfile(filename):
+                    return filename
 
     def collect_types(self):
         pass      # not needed in the generic engine
             newfunction = self._load_constant(False, tp, name, module)
         else:
             indirections = []
+            base_tp = tp
             if any(isinstance(typ, model.StructOrUnion) for typ in tp.args):
                 indirect_args = []
                 for i, typ in enumerate(tp.args):
             wrappername = '_cffi_f_%s' % name
             newfunction = module.load_function(BFunc, wrappername)
             for i, typ in indirections:
-                newfunction = self._make_struct_wrapper(newfunction, i, typ)
+                newfunction = self._make_struct_wrapper(newfunction, i, typ,
+                                                        base_tp)
         setattr(library, name, newfunction)
         type(library)._cffi_dir.append(name)
 
-    def _make_struct_wrapper(self, oldfunc, i, tp):
+    def _make_struct_wrapper(self, oldfunc, i, tp, base_tp):
         backend = self.ffi._backend
         BType = self.ffi._get_cached_btype(tp)
         def newfunc(*args):
             args = args[:i] + (backend.newp(BType, args[i]),) + args[i+1:]
             return oldfunc(*args)
+        newfunc._cffi_base_type = base_tp
         return newfunc
 
     # ----------
         prnt('  static ssize_t nums[] = {')
         prnt('    sizeof(%s),' % cname)
         prnt('    offsetof(struct _cffi_aligncheck, y),')
-        for fname, _, fbitsize in tp.enumfields():
+        for fname, ftype, fbitsize in tp.enumfields():
             if fbitsize >= 0:
                 continue      # xxx ignore fbitsize for now
             prnt('    offsetof(%s, %s),' % (cname, fname))
-            prnt('    sizeof(((%s *)0)->%s),' % (cname, fname))
+            if isinstance(ftype, model.ArrayType) and ftype.length is None:
+                prnt('    0,  /* %s */' % ftype._get_c_name())
+            else:
+                prnt('    sizeof(((%s *)0)->%s),' % (cname, fname))
         prnt('    -1')
         prnt('  };')
         prnt('  return nums[i];')
             return     # nothing to do with opaque structs
         layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name)
         #
-        BFunc = self.ffi.typeof("ssize_t(*)(ssize_t)")
+        BFunc = self.ffi._typeof_locked("ssize_t(*)(ssize_t)")[0]
         function = module.load_function(BFunc, layoutfuncname)
         layout = []
         num = 0
             if x < 0: break
             layout.append(x)
             num += 1
-        if isinstance(tp, model.StructType) and tp.partial:
+        if isinstance(tp, model.StructOrUnion) and tp.partial:
             # use the function()'s sizes and offsets to guide the
             # layout of the struct
             totalsize = layout[0]
                     continue        # xxx ignore fbitsize for now
                 check(layout[i], ffi.offsetof(BStruct, fname),
                       "wrong offset for field %r" % (fname,))
-                BField = ffi._get_cached_btype(ftype)
-                check(layout[i+1], ffi.sizeof(BField),
-                      "wrong size for field %r" % (fname,))
+                if layout[i+1] != 0:
+                    BField = ffi._get_cached_btype(ftype)
+                    check(layout[i+1], ffi.sizeof(BField),
+                          "wrong size for field %r" % (fname,))
                 i += 2
             assert i == len(layout)
 
     def _load_constant(self, is_int, tp, name, module):
         funcname = '_cffi_const_%s' % name
         if is_int:
-            BFunc = self.ffi.typeof("int(*)(long long*)")
+            BType = self.ffi._typeof_locked("long long*")[0]
+            BFunc = self.ffi._typeof_locked("int(*)(long long*)")[0]
             function = module.load_function(BFunc, funcname)
-            p = self.ffi.new("long long*")
+            p = self.ffi.new(BType)
             negative = function(p)
             value = int(p[0])
             if value < 0 and not negative:
-                value += (1 << (8*self.ffi.sizeof("long long")))
+                BLongLong = self.ffi._typeof_locked("long long")[0]
+                value += (1 << (8*self.ffi.sizeof(BLongLong)))
         else:
-            BFunc = self.ffi.typeof(tp.get_c_name('(*)(void)', name))
+            BFunc = self.ffi._typeof_locked(tp.get_c_name('(*)(void)', name))[0]
             function = module.load_function(BFunc, funcname)
             value = function()
         return value
         prnt('int %s(char *out_error)' % funcname)
         prnt('{')
         for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues):
-            prnt('  if (%s != %d) {' % (enumerator, enumvalue))
+            if enumvalue < 0:
+                prnt('  if ((%s) >= 0 || (long)(%s) != %dL) {' % (
+                    enumerator, enumerator, enumvalue))
+            else:
+                prnt('  if ((%s) < 0 || (unsigned long)(%s) != %dUL) {' % (
+                    enumerator, enumerator, enumvalue))
+            prnt('    char buf[64];')
+            prnt('    if ((%s) < 0)' % enumerator)
+            prnt('        snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator)
+            prnt('    else')
+            prnt('        snprintf(buf, 63, "%%lu", (unsigned long)(%s));' %
+                 enumerator)
             prnt('    snprintf(out_error, 255,'
-                             '"%s has the real value %d, not %d",')
-            prnt('            "%s", (int)%s, %d);' % (
-                enumerator, enumerator, enumvalue))
+                             ' "%s has the real value %s, not %s",')
+            prnt('            "%s", buf, "%d");' % (
+                enumerator, enumvalue))
             prnt('    return -1;')
             prnt('  }')
         prnt('  return 0;')
             tp.enumvalues = tuple(enumvalues)
             tp.partial_resolved = True
         else:
-            BFunc = self.ffi.typeof("int(*)(char*)")
+            BType = self.ffi._typeof_locked("char[]")[0]
+            BFunc = self.ffi._typeof_locked("int(*)(char*)")[0]
             funcname = '_cffi_e_%s_%s' % (prefix, name)
             function = module.load_function(BFunc, funcname)
-            p = self.ffi.new("char[]", 256)
+            p = self.ffi.new(BType, 256)
             if function(p) < 0:
                 error = self.ffi.string(p)
                 if sys.version_info >= (3,):
 
     def _generate_gen_variable_decl(self, tp, name):
         if isinstance(tp, model.ArrayType):
+            if tp.length == '...':
+                prnt = self._prnt
+                funcname = '_cffi_sizeof_%s' % (name,)
+                self.export_symbols.append(funcname)
+                prnt("size_t %s(void)" % funcname)
+                prnt("{")
+                prnt("  return sizeof(%s);" % (name,))
+                prnt("}")
             tp_ptr = model.PointerType(tp.item)
             self._generate_gen_const(False, name, tp_ptr)
         else:
     def _loaded_gen_variable(self, tp, name, module, library):
         if isinstance(tp, model.ArrayType):   # int a[5] is "constant" in the
                                               # sense that "a=..." is forbidden
+            if tp.length == '...':
+                funcname = '_cffi_sizeof_%s' % (name,)
+                BFunc = self.ffi._typeof_locked('size_t(*)(void)')[0]
+                function = module.load_function(BFunc, funcname)
+                size = function()
+                BItemType = self.ffi._get_cached_btype(tp.item)
+                length, rest = divmod(size, self.ffi.sizeof(BItemType))
+                if rest != 0:
+                    raise ffiplatform.VerificationError(
+                        "bad size: %r does not seem to be an array of %s" %
+                        (name, tp.item))
+                tp = tp.resolve_length(length)
             tp_ptr = model.PointerType(tp.item)
             value = self._load_constant(False, tp_ptr, name, module)
             # 'value' is a <cdata 'type *'> which we have to replace with
         # remove ptr=<cdata 'int *'> from the library instance, and replace
         # it by a property on the class, which reads/writes into ptr[0].
         funcname = '_cffi_var_%s' % name
-        BFunc = self.ffi.typeof(tp.get_c_name('*(*)(void)', name))
+        BFunc = self.ffi._typeof_locked(tp.get_c_name('*(*)(void)', name))[0]
         function = module.load_function(BFunc, funcname)
         ptr = function()
         def getter(library):

lib_pypy/cffi/verifier.py

             k2 = k2.lstrip('0').rstrip('L')
             modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key,
                                               k1, k2)
-        suffix = _get_so_suffix()
+        suffix = _get_so_suffixes()[0]
         self.tmpdir = tmpdir or _caller_dir_pycache()
         self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c')
         self.modulefilename = os.path.join(self.tmpdir, modulename + suffix)
     def write_source(self, file=None):
         """Write the C source code.  It is produced in 'self.sourcefilename',
         which can be tweaked beforehand."""
-        if self._has_source and file is None:
-            raise ffiplatform.VerificationError("source code already written")
-        self._write_source(file)
+        with self.ffi._lock:
+            if self._has_source and file is None:
+                raise ffiplatform.VerificationError(
+                    "source code already written")
+            self._write_source(file)
 
     def compile_module(self):
         """Write the C source code (if not done already) and compile it.
         This produces a dynamic link library in 'self.modulefilename'."""
-        if self._has_module:
-            raise ffiplatform.VerificationError("module already compiled")
-        if not self._has_source:
-            self._write_source()