Commits

mattip committed 7292e1a Merge

merge default into branch

  • Participants
  • Parent commits 2514183, 207a4cb
  • Branches ufuncapi

Comments (0)

Files changed (72)

 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
 0000000000000000000000000000000000000000 release-2.3.0
 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3
+32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
+32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1
+32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
+0000000000000000000000000000000000000000 release-2.2=3.1
     Stian Andreassen
     Laurence Tratt
     Wanja Saatkamp
+    Ivan Sichmann Freitas
     Gerald Klix
     Mike Blume
     Oscar Nierstrasz
     Alejandro J. Cura
     Jacob Oscarson
     Travis Francis Athougies
+    Ryan Gonzalez
     Kristjan Valur Jonsson
+    Sebastian Pawluś
     Neil Blakey-Milner
     anatoly techtonik
     Lutz Paelike
     Michael Hudson-Doyle
     Anders Sigfridsson
     Yasir Suhail
+    rafalgalczynski@gmail.com
     Floris Bruynooghe
     Laurens Van Houtven
     Akira Li
     Zooko Wilcox-O Hearn
     Tomer Chachamu
     Christopher Groskopf
+    Asmo Soinio
+    Stefan Marr
     jiaaro
     opassembler.py
     Antony Lee

lib-python/2.7/imputil.py

     saved back to the filesystem for future imports. The source file's
     modification timestamp must be provided as a Long value.
     """
-    codestring = open(pathname, 'rU').read()
+    with open(pathname, 'rU') as fp:
+        codestring = fp.read()
     if codestring and codestring[-1] != '\n':
         codestring = codestring + '\n'
     code = __builtin__.compile(codestring, pathname, 'exec')
         self.desc = desc
 
     def import_file(self, filename, finfo, fqname):
-        fp = open(filename, self.desc[1])
-        module = imp.load_module(fqname, fp, filename, self.desc)
+        with open(filename, self.desc[1]) as fp:
+            module = imp.load_module(fqname, fp, filename, self.desc)
         module.__file__ = filename
         return 0, module, { }
 

lib-python/2.7/modulefinder.py

 
     def run_script(self, pathname):
         self.msg(2, "run_script", pathname)
-        fp = open(pathname, READ_MODE)
-        stuff = ("", "r", imp.PY_SOURCE)
-        self.load_module('__main__', fp, pathname, stuff)
+        with open(pathname, READ_MODE) as fp:
+            stuff = ("", "r", imp.PY_SOURCE)
+            self.load_module('__main__', fp, pathname, stuff)
 
     def load_file(self, pathname):
         dir, name = os.path.split(pathname)
         name, ext = os.path.splitext(name)
-        fp = open(pathname, READ_MODE)
-        stuff = (ext, "r", imp.PY_SOURCE)
-        self.load_module(name, fp, pathname, stuff)
+        with open(pathname, READ_MODE) as fp:
+            stuff = (ext, "r", imp.PY_SOURCE)
+            self.load_module(name, fp, pathname, stuff)
 
     def import_hook(self, name, caller=None, fromlist=None, level=-1):
         self.msg(3, "import_hook", name, caller, fromlist, level)
         fp, buf, stuff = self.find_module("__init__", m.__path__)
         self.load_module(fqname, fp, buf, stuff)
         self.msgout(2, "load_package ->", m)
+        if fp:
+            fp.close()
         return m
 
     def add_module(self, fqname):

lib-python/2.7/test/test_argparse.py

 
     def tearDown(self):
         os.chdir(self.old_dir)
+        import gc
+        # Force a collection which should close FileType() options
+        gc.collect()
         for root, dirs, files in os.walk(self.temp_dir, topdown=False):
             for name in files:
                 os.chmod(os.path.join(self.temp_dir, name), stat.S_IWRITE)

lib_pypy/_tkinter/license.terms

+This software is copyrighted by the Regents of the University of
+California, Sun Microsystems, Inc., and other parties.  The following
+terms apply to all files associated with the software unless explicitly
+disclaimed in individual files.
+
+The authors hereby grant permission to use, copy, modify, distribute,
+and license this software and its documentation for any purpose, provided
+that existing copyright notices are retained in all copies and that this
+notice is included verbatim in any distributions. No written agreement,
+license, or royalty fee is required for any of the authorized uses.
+Modifications to this software may be copyrighted by their authors
+and need not follow the licensing terms described here, provided that
+the new terms are clearly indicated on the first page of each file where
+they apply.
+
+IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY
+FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
+ARISING OUT OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY
+DERIVATIVES THEREOF, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT.  THIS SOFTWARE
+IS PROVIDED ON AN "AS IS" BASIS, AND THE AUTHORS AND DISTRIBUTORS HAVE
+NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR
+MODIFICATIONS.
+
+GOVERNMENT USE: If you are acquiring this software on behalf of the
+U.S. government, the Government shall have only "Restricted Rights"
+in the software and related documentation as defined in the Federal 
+Acquisition Regulations (FARs) in Clause 52.227.19 (c) (2).  If you
+are acquiring the software on behalf of the Department of Defense, the
+software shall be classified as "Commercial Computer Software" and the
+Government shall have only "Restricted Rights" as defined in Clause
+252.227-7013 (c) (1) of DFARs.  Notwithstanding the foregoing, the
+authors grant the U.S. Government and others acting in its behalf
+permission to use and distribute the software in accordance with the
+terms specified in this license.

pypy/doc/contributor.rst

   Stian Andreassen
   Laurence Tratt
   Wanja Saatkamp
+  Ivan Sichmann Freitas
   Gerald Klix
   Mike Blume
   Oscar Nierstrasz
   Alejandro J. Cura
   Jacob Oscarson
   Travis Francis Athougies
+  Ryan Gonzalez
   Kristjan Valur Jonsson
+  Sebastian Pawluś
   Neil Blakey-Milner
   anatoly techtonik
   Lutz Paelike
   Michael Hudson-Doyle
   Anders Sigfridsson
   Yasir Suhail
+  rafalgalczynski@gmail.com
   Floris Bruynooghe
   Laurens Van Houtven
   Akira Li
   Zooko Wilcox-O Hearn
   Tomer Chachamu
   Christopher Groskopf
+  Asmo Soinio
+  Stefan Marr
   jiaaro
   opassembler.py
   Antony Lee

pypy/doc/index-of-release-notes.rst

 
 .. toctree::
 
+   release-2.3.1.rst
    release-2.3.0.rst
    release-2.2.1.rst
    release-2.2.0.rst

pypy/doc/index.rst

 
 * `FAQ`_: some frequently asked questions.
 
-* `Release 2.3.0`_: the latest official release
+* `Release 2.3.1`_: the latest official release
 
 * `PyPy Blog`_: news and status info about PyPy 
 
 .. _`Getting Started`: getting-started.html
 .. _`Papers`: extradoc.html
 .. _`Videos`: video-index.html
-.. _`Release 2.3.0`: http://pypy.org/download.html
+.. _`Release 2.3.1`: http://pypy.org/download.html
 .. _`speed.pypy.org`: http://speed.pypy.org
 .. _`RPython toolchain`: translation.html
 .. _`potential project ideas`: project-ideas.html

pypy/doc/release-2.3.1.rst

+=================================================
+PyPy 2.3.1 - Terrestrial Arthropod Trap Revisited
+=================================================
+
+We're pleased to announce PyPy 2.3.1, a feature-and-bugfix improvement over our
+recent release last month.
+
+This release contains several bugfixes and enhancements.
+
+You can download the PyPy 2.3.1 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project, and for those who donate to our three sub-projects.
+We've shown quite a bit of progress 
+but we're slowly running out of funds.
+Please consider donating more, or even better convince your employer to donate,
+so we can finish those projects!  The three sub-projects are:
+
+* `Py3k`_ (supporting Python 3.x): the release PyPy3 2.3 is imminent.
+
+* `STM`_ (software transactional memory): a preview will be released very soon,
+  once we fix a few bugs
+
+* `NumPy`_ which requires installation of our fork of upstream numpy, available `on bitbucket`_
+
+.. _`Py3k`: http://pypy.org/py3donate.html
+.. _`STM`: http://pypy.org/tmdonate2.html
+.. _`NumPy`: http://pypy.org/numpydonate.html
+.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy   
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy 2.3 and cpython 2.7.x`_ performance comparison;
+note that cpython's speed has not changed since 2.7.2)
+due to its integrated tracing JIT compiler.
+
+This release supports x86 machines running Linux 32/64, Mac OS X 64, Windows,
+and OpenBSD,
+as well as newer ARM hardware (ARMv6 or ARMv7, with VFPv3) running Linux. 
+
+While we support 32 bit python on Windows, work on the native Windows 64
+bit python is still stalling, we would welcome a volunteer
+to `handle that`_.
+
+.. _`pypy 2.3 and cpython 2.7.x`: http://speed.pypy.org
+.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation
+
+Highlights
+==========
+
+Issues with the 2.3 release were resolved after being reported by users to
+our new issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at
+#pypy. Here is a summary of the user-facing changes;
+for more information see `whats-new`_:
+
+* The built-in ``struct`` module was renamed to ``_struct``, solving issues
+  with IDLE and other modules.
+
+* Support for compilation with gcc-4.9
+
+* A rewrite of packaging.py which produces our downloadable packages to
+  modernize command line argument handling and to document third-party
+  contributions in our LICENSE file
+
+* A CFFI-based version of the gdbm module is now included in our downloads
+
+* Many issues were resolved_ since the 2.3 release on May 8
+
+.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html
+.. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved
+Please try it out and let us know what you think. We especially welcome
+success stories, we know you are using PyPy, please tell us about it!
+
+Cheers
+
+The PyPy Team
+

pypy/doc/whatsnew-2.3.1.rst

 
 Support compilation with gcc-4.9
 
-Fixes for issues #1769, #1764, #1762, #1752
+Added support for the stdlib gdbm module via cffi
 
+Annotator cleanups
+
+.. branch: release-2.3.x
+
+.. branch: unify-call-ops
+
+.. branch packaging
+Use argparse for packaging.py, and add third-party components to LICENSE file.
+Also mention that gdbm is GPL.
+Do not crash the packaging process on failure in CFFI or license-building,
+rather complete the build step and return -1.

pypy/doc/whatsnew-head.rst

 =======================
 
 .. this is a revision shortly after release-2.3.x
-.. startrev: b2cc67adbaad
+.. startrev: 87fdc76bccb4
 
-Added support for the stdlib gdbm module via cffi
 
-Annotator cleanups
-
-.. branch: release-2.3.x
-

pypy/doc/windows.rst

 64bit Windows.  See at the end of this page for what is missing
 for a full 64bit translation.
 
-To build pypy-c you need a C compiler.  Microsoft Visual Studio is
-preferred, but can also use the mingw32 port of gcc.
+To build pypy-c you need a working python environment, and a C compiler.
+It is possible to translate with a CPython 2.6 or later, but this is not
+the preferred way, because it will take a lot longer to run – depending
+on your architecture, between two and three times as long. So head to 
+`our downloads`_ and get the latest stable version.
+
+Microsoft Visual Studio is preferred as a compiler, but there are reports 
+of success with the mingw32 port of gcc.
 
 
 Translating PyPy with Visual Studio
 **Note:** PyPy is currently not supported for 64 bit Windows, and translation
 will fail in this case.
 
-The compiler is all you need to build pypy-c, but it will miss some
+Python and a C compiler are all you need to build pypy, but it will miss some
 modules that relies on third-party libraries.  See below how to get
 and build them.
 
+Please see the `non-windows instructions`_ for more information, especially note
+that translation is RAM-hungry. A standard translation requires around 4GB, so
+special preparations are necessary, or you may want to use the method in the
+notes of the `build instructions`_ to reduce memory usage at the price of a
+slower translation::
+
+    set PYPY_GC_MAX_DELTA=200MB
+    pypy --jit loop_longevity=300 ../../rpython/bin/rpython -Ojit targetpypystandalone
+    set PYPY_GC_MAX_DELTA=
+
 Preping Windows for the Large Build
 -----------------------------------
 
 
 Then you need to execute::
 
-    editbin /largeaddressaware pypy.exe
+    editbin /largeaddressaware translator.exe
 
-on the pypy.exe file you compiled.
+where ``translator.exe`` is the pypy.exe or cpython.exe you will use to 
+translate with. 
 
 Installing external packages
 ----------------------------
 .. _`msys for mingw`: http://sourceforge.net/projects/mingw-w64/files/External%20binary%20packages%20%28Win64%20hosted%29/MSYS%20%2832-bit%29   
 .. _`libffi source files`: http://sourceware.org/libffi/
 .. _`RPython translation toolchain`: translation.html
-
+.. _`our downloads`: http://pypy.org/download.html   
+.. _`non-windows instructions`: getting-started-python.html#translating-the-pypy-python-interpreter
+.. _`build instructions`: http://pypy.org/download.html#building-from-source
 
 What is missing for a full 64-bit translation
 ---------------------------------------------

pypy/module/__builtin__/app_io.py

 """
 
 import sys
+from _ast import PyCF_ACCEPT_NULL_BYTES
 
 def execfile(filename, glob=None, loc=None):
     """execfile(filename[, globals[, locals]])
     finally:
         f.close()
     #Don't exec the source directly, as this loses the filename info
-    co = compile(source.rstrip()+"\n", filename, 'exec')
+    co = compile(source.rstrip()+"\n", filename, 'exec',
+                 PyCF_ACCEPT_NULL_BYTES)
     exec co in glob, loc
 
 def _write_prompt(stdout, prompt):

pypy/module/__builtin__/test/test_builtin.py

 import sys
 
+from rpython.tool.udir import udir
+
 class AppTestBuiltinApp:
     def setup_class(cls):
+        space = cls.space
         class X(object):
             def __eq__(self, other):
                 raise OverflowError
         try:
             d[X()]
         except OverflowError:
-            cls.w_sane_lookup = cls.space.wrap(True)
+            cls.w_sane_lookup = space.wrap(True)
         except KeyError:
-            cls.w_sane_lookup = cls.space.wrap(False)
+            cls.w_sane_lookup = space.wrap(False)
         # starting with CPython 2.6, when the stack is almost out, we
         # can get a random error, instead of just a RuntimeError.
         # For example if an object x has a __getattr__, we can get
         # AttributeError if attempting to call x.__getattr__ runs out
         # of stack.  That's annoying, so we just work around it.
         if cls.runappdirect:
-            cls.w_safe_runtimerror = cls.space.wrap(True)
+            cls.w_safe_runtimerror = space.wrap(True)
         else:
-            cls.w_safe_runtimerror = cls.space.wrap(sys.version_info < (2, 6))
+            cls.w_safe_runtimerror = space.wrap(sys.version_info < (2, 6))
+
+        emptyfile = udir.join('emptyfile.py')
+        emptyfile.write('')
+        nullbytes = udir.join('nullbytes.py')
+        nullbytes.write('#abc\x00def\n')
+        cls.w_emptyfile = space.wrap(str(emptyfile))
+        cls.w_nullbytes = space.wrap(str(nullbytes))
 
     def test_builtin_names(self):
         import __builtin__
         assert setattr(x, 'x', 11) == None
         assert delattr(x, 'x') == None
         # To make this test, we need autopath to work in application space.
-        #self.assertEquals(execfile('emptyfile.py'), None)
+        assert execfile(self.emptyfile) == None
 
     def test_divmod(self):
         assert divmod(15,10) ==(1,5)
         assert firstlineno == 2
 
     def test_compile_null_bytes(self):
-        import _ast
         raises(TypeError, compile, '\x00', 'mymod', 'exec', 0)
-        raises(SyntaxError, compile, '\x00', 'mymod', 'exec',
-               _ast.PyCF_ACCEPT_NULL_BYTES)
         src = "#abc\x00def\n"
         raises(TypeError, compile, src, 'mymod', 'exec')
         raises(TypeError, compile, src, 'mymod', 'exec', 0)
-        compile(src, 'mymod', 'exec', _ast.PyCF_ACCEPT_NULL_BYTES)  # works
+        execfile(self.nullbytes) # works
+
+    def test_compile_null_bytes_flag(self):
+        try:
+            from _ast import PyCF_ACCEPT_NULL_BYTES
+        except ImportError:
+            skip('PyPy only (requires _ast.PyCF_ACCEPT_NULL_BYTES)')
+        raises(SyntaxError, compile, '\x00', 'mymod', 'exec',
+               PyCF_ACCEPT_NULL_BYTES)
+        src = "#abc\x00def\n"
+        compile(src, 'mymod', 'exec', PyCF_ACCEPT_NULL_BYTES)  # works
 
     def test_print_function(self):
         import __builtin__
 
 class TestInternal:
     def test_execfile(self, space):
-        from rpython.tool.udir import udir
         fn = str(udir.join('test_execfile'))
         f = open(fn, 'w')
         print >>f, "i=42"

pypy/module/_ast/test/test_ast.py

File contents unchanged.

pypy/module/cpyext/pystate.py

 
 @cpython_api([], PyGILState_STATE, error=CANNOT_FAIL)
 def PyGILState_Ensure(space):
+    # XXX XXX XXX THIS IS A VERY MINIMAL IMPLEMENTATION THAT WILL HAPPILY
+    # DEADLOCK IF CALLED TWICE ON THE SAME THREAD, OR CRASH IF CALLED IN A
+    # NEW THREAD.  We should very carefully follow what CPython does instead.
     if rffi.aroundstate.after:
         # After external call is before entering Python
         rffi.aroundstate.after()
 
 @cpython_api([PyGILState_STATE], lltype.Void)
 def PyGILState_Release(space, state):
+    # XXX XXX XXX We should very carefully follow what CPython does instead.
     if rffi.aroundstate.before:
         # Before external call is after running Python
         rffi.aroundstate.before()

pypy/module/imp/importing.py

         self.lockcounter = 0
 
     def lock_held_by_someone_else(self):
-        return self.lockowner is not None and not self.lock_held()
+        me = self.space.getexecutioncontext()   # used as thread ident
+        return self.lockowner is not None and self.lockowner is not me
 
-    def lock_held(self):
-        me = self.space.getexecutioncontext()   # used as thread ident
-        return self.lockowner is me
+    def lock_held_by_anyone(self):
+        return self.lockowner is not None
 
     def acquire_lock(self):
         # this function runs with the GIL acquired so there is no race

pypy/module/imp/interp_imp.py

 
 def lock_held(space):
     if space.config.objspace.usemodules.thread:
-        return space.wrap(importing.getimportlock(space).lock_held())
+        return space.wrap(importing.getimportlock(space).lock_held_by_anyone())
     else:
         return space.w_False
 

pypy/module/micronumpy/__init__.py

         'empty': 'ctors.zeros',
         'empty_like': 'ctors.empty_like',
         'fromstring': 'ctors.fromstring',
+        'frombuffer': 'ctors.frombuffer',
 
         'concatenate': 'arrayops.concatenate',
         'count_nonzero': 'arrayops.count_nonzero',

pypy/module/micronumpy/ctors.py

 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
+from rpython.rlib.buffer import SubBuffer
 from rpython.rlib.rstring import strip_spaces
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.module.micronumpy import descriptor, loop
         return _fromstring_bin(space, s, count, length, dtype)
     else:
         return _fromstring_text(space, s, count, sep, length, dtype)
+
+
+def _getbuffer(space, w_buffer):
+    try:
+        return space.writebuf_w(w_buffer)
+    except OperationError as e:
+        if not e.match(space, space.w_TypeError):
+            raise
+        return space.readbuf_w(w_buffer)
+
+
+@unwrap_spec(count=int, offset=int)
+def frombuffer(space, w_buffer, w_dtype=None, count=-1, offset=0):
+    dtype = space.interp_w(descriptor.W_Dtype,
+        space.call_function(space.gettypefor(descriptor.W_Dtype), w_dtype))
+    if dtype.elsize == 0:
+        raise oefmt(space.w_ValueError, "itemsize cannot be zero in type")
+
+    try:
+        buf = _getbuffer(space, w_buffer)
+    except OperationError as e:
+        if not e.match(space, space.w_TypeError):
+            raise
+        w_buffer = space.getattr(w_buffer, space.wrap('__buffer__'))
+        buf = _getbuffer(space, w_buffer)
+
+    ts = buf.getlength()
+    if offset < 0 or offset > ts:
+        raise oefmt(space.w_ValueError,
+                    "offset must be non-negative and no greater than "
+                    "buffer length (%d)", ts)
+
+    s = ts - offset
+    if offset:
+        buf = SubBuffer(buf, offset, s)
+
+    n = count
+    itemsize = dtype.elsize
+    assert itemsize > 0
+    if n < 0:
+        if s % itemsize != 0:
+            raise oefmt(space.w_ValueError,
+                        "buffer size must be a multiple of element size")
+        n = s / itemsize
+    else:
+        if s < n * itemsize:
+            raise oefmt(space.w_ValueError,
+                        "buffer is smaller than requested size")
+
+    try:
+        storage = buf.get_raw_address()
+    except ValueError:
+        a = W_NDimArray.from_shape(space, [n], dtype=dtype)
+        loop.fromstring_loop(space, a, dtype, itemsize, buf.as_str())
+        return a
+    else:
+        writable = not buf.readonly
+    return W_NDimArray.from_shape_and_storage(space, [n], storage, dtype=dtype,
+                                              w_base=w_buffer, writable=writable)

pypy/module/micronumpy/test/test_ndarray.py

 
 
 class AppTestSupport(BaseNumpyAppTest):
+    spaceconfig = {'usemodules': ['micronumpy', 'array']}
+
     def setup_class(cls):
         import struct
         BaseNumpyAppTest.setup_class.im_func(cls)
         cls.w_float64val = cls.space.wrap(struct.pack('d', 300.4))
         cls.w_ulongval = cls.space.wrap(struct.pack('L', 12))
 
+    def test_frombuffer(self):
+        import numpy as np
+        exc = raises(AttributeError, np.frombuffer, None)
+        assert str(exc.value) == "'NoneType' object has no attribute '__buffer__'"
+        exc = raises(AttributeError, np.frombuffer, memoryview(self.data))
+        assert str(exc.value) == "'memoryview' object has no attribute '__buffer__'"
+        exc = raises(ValueError, np.frombuffer, self.data, 'S0')
+        assert str(exc.value) == "itemsize cannot be zero in type"
+        exc = raises(ValueError, np.frombuffer, self.data, offset=-1)
+        assert str(exc.value) == "offset must be non-negative and no greater than buffer length (32)"
+        exc = raises(ValueError, np.frombuffer, self.data, count=100)
+        assert str(exc.value) == "buffer is smaller than requested size"
+        for data in [self.data, buffer(self.data)]:
+            a = np.frombuffer(data)
+            for i in range(4):
+                assert a[i] == i + 1
+
+        import array
+        data = array.array('c', 'testing')
+        a = np.frombuffer(data, 'c')
+        assert a.base is data
+        a[2] = 'Z'
+        assert data.tostring() == 'teZting'
+
+        data = buffer(data)
+        a = np.frombuffer(data, 'c')
+        assert a.base is data
+        exc = raises(ValueError, "a[2] = 'Z'")
+        assert str(exc.value) == "assignment destination is read-only"
+
+        class A(object):
+            __buffer__ = 'abc'
+
+        data = A()
+        a = np.frombuffer(data, 'c')
+        #assert a.base is data.__buffer__
+        assert a.tostring() == 'abc'
+
     def test_fromstring(self):
         import sys
         from numpypy import fromstring, dtype

pypy/module/sys/__init__.py

         """NOT_RPYTHON""" # because parent __init__ isn't
         if space.config.translating:
             del self.__class__.interpleveldefs['pypy_getudir']
-        super(Module, self).__init__(space, w_name) 
+        super(Module, self).__init__(space, w_name)
         self.recursionlimit = 100
         self.w_default_encoder = None
         self.defaultencoding = "ascii"
         self.debug = True
 
     interpleveldefs = {
-        '__name__'              : '(space.wrap("sys"))', 
-        '__doc__'               : '(space.wrap("PyPy sys module"))', 
+        '__name__'              : '(space.wrap("sys"))',
+        '__doc__'               : '(space.wrap("PyPy sys module"))',
 
-        'platform'              : 'space.wrap(sys.platform)', 
+        'platform'              : 'space.wrap(sys.platform)',
         'maxint'                : 'space.wrap(sys.maxint)',
         'maxsize'               : 'space.wrap(sys.maxint)',
-        'byteorder'             : 'space.wrap(sys.byteorder)', 
+        'byteorder'             : 'space.wrap(sys.byteorder)',
         'maxunicode'            : 'space.wrap(vm.MAXUNICODE)',
         'stdin'                 : 'state.getio(space).w_stdin',
         '__stdin__'             : 'state.getio(space).w_stdin',
         'stderr'                : 'state.getio(space).w_stderr',
         '__stderr__'            : 'state.getio(space).w_stderr',
         'pypy_objspaceclass'    : 'space.wrap(repr(space))',
-        #'prefix'               : # added by pypy_initial_path() when it 
+        #'prefix'               : # added by pypy_initial_path() when it
         #'exec_prefix'          : # succeeds, pointing to trunk or /usr
         'path'                  : 'state.get(space).w_path',
-        'modules'               : 'state.get(space).w_modules', 
+        'modules'               : 'state.get(space).w_modules',
         'argv'                  : 'state.get(space).w_argv',
         'py3kwarning'           : 'space.w_False',
-        'warnoptions'           : 'state.get(space).w_warnoptions', 
+        'warnoptions'           : 'state.get(space).w_warnoptions',
         'builtin_module_names'  : 'space.w_None',
         'pypy_getudir'          : 'state.pypy_getudir',    # not translated
         'pypy_find_stdlib'      : 'initpath.pypy_find_stdlib',
         'pypy_find_executable'  : 'initpath.pypy_find_executable',
         'pypy_resolvedirof'     : 'initpath.pypy_resolvedirof',
 
-        '_getframe'             : 'vm._getframe', 
-        '_current_frames'       : 'currentframes._current_frames', 
-        'setrecursionlimit'     : 'vm.setrecursionlimit', 
-        'getrecursionlimit'     : 'vm.getrecursionlimit', 
-        'setcheckinterval'      : 'vm.setcheckinterval', 
-        'getcheckinterval'      : 'vm.getcheckinterval', 
-        'exc_info'              : 'vm.exc_info', 
-        'exc_clear'             : 'vm.exc_clear', 
+        '_getframe'             : 'vm._getframe',
+        '_current_frames'       : 'currentframes._current_frames',
+        'setrecursionlimit'     : 'vm.setrecursionlimit',
+        'getrecursionlimit'     : 'vm.getrecursionlimit',
+        'setcheckinterval'      : 'vm.setcheckinterval',
+        'getcheckinterval'      : 'vm.getcheckinterval',
+        'exc_info'              : 'vm.exc_info',
+        'exc_clear'             : 'vm.exc_clear',
         'settrace'              : 'vm.settrace',
         'gettrace'              : 'vm.gettrace',
         'setprofile'            : 'vm.setprofile',
         'getprofile'            : 'vm.getprofile',
         'call_tracing'          : 'vm.call_tracing',
         'getsizeof'             : 'vm.getsizeof',
-        
-        'executable'            : 'space.wrap("py.py")', 
+
+        'executable'            : 'space.wrap("py.py")',
         'api_version'           : 'version.get_api_version(space)',
         'version_info'          : 'version.get_version_info(space)',
         'version'               : 'version.get_version(space)',
         '_mercurial'            : 'version.get_repo_info(space)',
         'hexversion'            : 'version.get_hexversion(space)',
 
-        'displayhook'           : 'hook.displayhook', 
-        '__displayhook__'       : 'hook.__displayhook__', 
+        'displayhook'           : 'hook.displayhook',
+        '__displayhook__'       : 'hook.__displayhook__',
         'meta_path'             : 'space.wrap([])',
         'path_hooks'            : 'space.wrap([])',
         'path_importer_cache'   : 'space.wrap({})',
         'dont_write_bytecode'   : 'space.w_False',
-        
-        'getdefaultencoding'    : 'interp_encoding.getdefaultencoding', 
+
+        'getdefaultencoding'    : 'interp_encoding.getdefaultencoding',
         'setdefaultencoding'    : 'interp_encoding.setdefaultencoding',
         'getfilesystemencoding' : 'interp_encoding.getfilesystemencoding',
 
         w_modules = self.get('modules')
         try:
             return space.getitem(w_modules, space.wrap(name))
-        except OperationError, e: 
-            if not e.match(space, space.w_KeyError): 
-                raise 
-            return None 
+        except OperationError, e:
+            if not e.match(space, space.w_KeyError):
+                raise
+            return None
 
-    def setmodule(self, w_module): 
+    def setmodule(self, w_module):
         space = self.space
         w_name = self.space.getattr(w_module, space.wrap('__name__'))
         w_modules = self.get('modules')
         self.space.setitem(w_modules, w_name, w_module)
 
     def getdictvalue(self, space, attr):
-        """ specialize access to dynamic exc_* attributes. """ 
-        value = MixedModule.getdictvalue(self, space, attr) 
-        if value is not None: 
+        """ specialize access to dynamic exc_* attributes. """
+        value = MixedModule.getdictvalue(self, space, attr)
+        if value is not None:
             return value
         if attr == 'exc_type':
             operror = space.getexecutioncontext().sys_exc_info()
                 return space.w_None
             else:
                 return space.wrap(operror.get_traceback())
-        return None 
+        return None
 
     def get_w_default_encoder(self):
         if self.w_default_encoder is not None:

pypy/module/thread/test/test_import_lock.py

         self.waitfor(lambda: done)
         assert done
 
+    def test_lock_held_by_another_thread(self):
+        import thread, imp
+        lock_held = thread.allocate_lock()
+        test_complete = thread.allocate_lock()
+        lock_released = thread.allocate_lock()
+        def other_thread():
+            imp.acquire_lock()        # 3
+            assert imp.lock_held()
+            lock_held.release()       # 4
+            test_complete.acquire()   # 7
+            imp.release_lock()        # 8
+            lock_released.release()   # 9
+        lock_held.acquire()
+        test_complete.acquire()
+        lock_released.acquire()
+        #
+        thread.start_new_thread(other_thread, ())  # 1
+        lock_held.acquire()                        # 2
+        assert imp.lock_held()                     # 5
+        test_complete.release()                    # 6
+        lock_released.acquire()                    # 10
+
 class TestImportLock:
     def test_lock(self, space, monkeypatch):
         from pypy.module.imp.importing import getimportlock, importhook

pypy/tool/release/package.py

 #!/usr/bin/env python
-""" A sample script that packages PyPy, provided that it's already built.
+""" packages PyPy, provided that it's already built.
 It uses 'pypy/goal/pypy-c' and parts of the rest of the working
 copy.  Usage:
 
-    package.py [--nostrip] [--without-tk] root-pypy-dir [name-of-archive] [name-of-pypy-c] [destination-for-tarball] [pypy-c-path]
+    package.py [--options]
 
-Usually you would do:   package.py ../../.. pypy-VER-PLATFORM
-The output is found in the directory /tmp/usession-YOURNAME/build/.
+Usually you would do:   package.py --version-name pypy-VER-PLATFORM
+The output is found in the directory from --builddir,
+by default /tmp/usession-YOURNAME/build/.
 """
 
 import shutil
 import sys
 import os
 #Add toplevel repository dir to sys.path
-sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
+basedir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+sys.path.insert(0,basedir)
 import py
 import fnmatch
-from rpython.tool.udir import udir
 import subprocess
+import glob
 
 if sys.version_info < (2,6): py.test.skip("requires 2.6 so far")
 
 class PyPyCNotFound(Exception):
     pass
 
-def fix_permissions(basedir):
+class MissingDependenciesError(Exception):
+    pass
+
+def fix_permissions(dirname):
     if sys.platform != 'win32':
-        os.system("chmod -R a+rX %s" % basedir)
-        os.system("chmod -R g-w %s" % basedir)
+        os.system("chmod -R a+rX %s" % dirname)
+        os.system("chmod -R g-w %s" % dirname)
 
-def package(basedir, name='pypy-nightly', rename_pypy_c='pypy',
-            copy_to_dir=None, override_pypy_c=None, nostrip=False,
-            withouttk=False):
-    assert '/' not in rename_pypy_c
+sep_template = "\nThis copy of PyPy includes a copy of %s, which is licensed under the following terms:\n\n"
+
+def generate_license_linux(basedir, options):
+    base_file = str(basedir.join('LICENSE'))
+    with open(base_file) as fid:
+        txt = fid.read()
+    searches = [("bzip2","libbz2-*", "copyright", '---------'),
+                ("openssl", "openssl*", "copyright", 'LICENSE ISSUES'),
+               ]
+    if not options.no_tk:
+        name = 'Tcl/Tk'
+        txt += "License for '%s'" %name
+        txt += '\n' + "="*(14 + len(name)) + '\n'
+        txt += sep_template % name
+        base_file = str(basedir.join('lib_pypy/_tkinter/license.terms'))
+        with open(base_file, 'r') as fid:
+            txt += fid.read()
+    for name, pat, fname, first_line in searches:
+        txt += "License for '" + name + "'"
+        txt += '\n' + "="*(14 + len(name)) + '\n'
+        txt += sep_template % name
+        dirs = glob.glob(options.license_base + "/" +pat)
+        if not dirs:
+            raise ValueError, "Could not find "+ options.license_base + "/" + pat
+        if len(dirs) > 2:
+            raise ValueError, "Multiple copies of "+pat
+        dir = dirs[0]
+        with open(os.path.join(dir, fname)) as fid:
+            # Read up to the line dividing the packaging header from the actual copyright
+            for line in fid:
+                if first_line in line:
+                    break
+            txt += line
+            for line in fid:
+                txt += line
+            if len(line.strip())<1:
+                txt += '\n'
+    txt += third_party_header
+    # Do something for gdbm, which is GPL
+    txt += gdbm_bit
+    return txt
+
+def generate_license_windows(basedir, options):
+    base_file = str(basedir.join('LICENSE'))
+    with open(base_file) as fid:
+        txt = fid.read()
+    # shutil.copyfileobj(open("crtlicense.txt"), out) # We do not ship msvc runtime files
+    if not options.no_tk:
+        name = 'Tcl/Tk'
+        txt += "License for '%s'" %name
+        txt += '\n' + "="*(14 + len(name)) + '\n'
+        txt += sep_template % name
+        base_file = str(basedir.join('lib_pypy/_tkinter/license.terms'))
+        with open(base_file, 'r') as fid:
+            txt += fid.read()
+    for name, pat, file in (("bzip2","bzip2-*", "LICENSE"),
+                      ("openssl", "openssl-*", "LICENSE")):
+        txt += sep_template % name
+        dirs = glob.glob(options.license_base + "/" +pat)
+        if not dirs:
+            raise ValueError, "Could not find "+ options.license_base + "/" + pat
+        if len(dirs) > 2:
+            raise ValueError, "Multiple copies of "+pat
+        dir = dirs[0]
+        with open(os.path.join(dir, file)) as fid:
+            txt += fid.read()
+    return txt
+
+def generate_license_darwin(basedir, options):
+    # where are copyright files on macos?
+    return generate_license_linux(basedir, options)
+
+if sys.platform == 'win32':
+    generate_license = generate_license_windows
+elif sys.platform == 'darwin':
+    generate_license = generate_license_darwin
+else:
+    generate_license = generate_license_linux
+
+def create_cffi_import_libraries(pypy_c, options):
+    modules = ['_sqlite3']
+    subprocess.check_call([str(pypy_c), '-c', 'import _sqlite3'])
+    if not sys.platform == 'win32':
+        modules += ['_curses', 'syslog', 'gdbm', '_sqlite3']
+    if not options.no_tk:
+        modules.append(('_tkinter'))
+    for module in modules:
+        try:
+            subprocess.check_call([str(pypy_c), '-c', 'import ' + module])
+        except subprocess.CalledProcessError:
+            print >>sys.stderr, """Building {0} bindings failed.
+You can either install development headers package or
+add --without-{0} option to skip packaging binary CFFI extension.""".format(module)
+            raise MissingDependenciesError(module)
+
+def create_package(basedir, options):
+    retval = 0
+    name = options.name
+    if not name:
+        name = 'pypy-nightly'
+    rename_pypy_c = options.pypy_c
+    override_pypy_c = options.override_pypy_c
+
     basedir = py.path.local(basedir)
-    if override_pypy_c is None:
+    if not override_pypy_c:
         basename = 'pypy-c'
         if sys.platform == 'win32':
             basename += '.exe'
             raise PyPyCNotFound(
                 'Bogus path: %r does not exist (see docstring for more info)'
                 % (os.path.dirname(str(pypy_c)),))
-    win_extras = ['libpypy-c.dll', 'libexpat.dll', 'sqlite3.dll',
-                      'libeay32.dll', 'ssleay32.dll']
-    subprocess.check_call([str(pypy_c), '-c', 'import _sqlite3'])
-    if not sys.platform == 'win32':
-        subprocess.check_call([str(pypy_c), '-c', 'import _curses'])
-        subprocess.check_call([str(pypy_c), '-c', 'import syslog'])
-        subprocess.check_call([str(pypy_c), '-c', 'import gdbm'])
-    if not withouttk:
+    if not options.no_cffi:
         try:
-            subprocess.check_call([str(pypy_c), '-c', 'import _tkinter'])
-        except subprocess.CalledProcessError:
-            print >>sys.stderr, """Building Tk bindings failed.
-You can either install Tk development headers package or
-add --without-tk option to skip packaging binary CFFI extension."""
-            sys.exit(1)
-        #Can the dependencies be found from cffi somehow?    
-        win_extras += ['tcl85.dll', 'tk85.dll']    
+            create_cffi_import_libraries(pypy_c, options)
+        except MissingDependenciesError:
+            # This is a non-fatal error
+            retval = -1
+
     if sys.platform == 'win32' and not rename_pypy_c.lower().endswith('.exe'):
         rename_pypy_c += '.exe'
     binaries = [(pypy_c, rename_pypy_c)]
     #
-    builddir = udir.ensure("build", dir=True)
+    builddir = options.builddir
     pypydir = builddir.ensure(name, dir=True)
     includedir = basedir.join('include')
     # Recursively copy all headers, shutil has only ignore
     pypydir.ensure('include', dir=True)
 
     if sys.platform == 'win32':
-        #Don't include a mscvrXX.dll, users should get their own.
-        #Instructions are provided on the website.
-
         # Can't rename a DLL: it is always called 'libpypy-c.dll'
+        win_extras = ['libpypy-c.dll', 'libexpat.dll', 'sqlite3.dll',
+                          'libeay32.dll', 'ssleay32.dll']
+        if not options.no_tk:
+            win_extras += ['tcl85.dll', 'tk85.dll']
 
         for extra in win_extras:
             p = pypy_c.dirpath().join(extra)
                     continue
             print "Picking %s" % p
             binaries.append((p, p.basename))
-        importlib_name = 'python27.lib'    
+        importlib_name = 'python27.lib'
         if pypy_c.dirpath().join(importlib_name).check():
             shutil.copyfile(str(pypy_c.dirpath().join(importlib_name)),
                         str(pypydir.join('include/python27.lib')))
             # XXX users will complain that they cannot compile cpyext
             # modules for windows, has the lib moved or are there no
             # exported functions in the dll so no import library is created?
-        if not withouttk:
+        if not options.no_tk:
             try:
                 p = pypy_c.dirpath().join('tcl85.dll')
                 if not p.check():
 tk85.dll and tcl85.dll found, expecting to find runtime in ..\\lib
 directory next to the dlls, as per build instructions."""
                 import traceback;traceback.print_exc()
-                sys.exit(1)
+                raise MissingDependenciesError('Tk runtime')
 
     # Careful: to copy lib_pypy, copying just the hg-tracked files
     # would not be enough: there are also ctypes_config_cache/_*_cache.py.
                     str(pypydir.join('lib_pypy')),
                     ignore=ignore_patterns('.svn', 'py', '*.pyc', '*~',
                                            '*.c', '*.o'))
-    for file in ['LICENSE', 'README.rst']:
+    for file in ['README.rst',]:
         shutil.copy(str(basedir.join(file)), str(pypydir))
     for file in ['_testcapimodule.c', '_ctypes_test.c']:
-        shutil.copyfile(str(basedir.join('lib_pypy', file)), 
+        shutil.copyfile(str(basedir.join('lib_pypy', file)),
                         str(pypydir.join('lib_pypy', file)))
+    try:
+        license = generate_license(basedir, options)
+        with open(str(pypydir.join('LICENSE')), 'w') as LICENSE:
+            LICENSE.write(license)
+    except:
+        # Non-fatal error, use original LICENCE file
+        import traceback;traceback.print_exc()
+        base_file = str(basedir.join('LICENSE'))
+        with open(base_file) as fid:
+            license = fid.read()
+        with open(str(pypydir.join('LICENSE')), 'w') as LICENSE:
+            LICENSE.write(license)
+        retval = -1
     #
     spdir = pypydir.ensure('site-packages', dir=True)
     shutil.copy(str(basedir.join('site-packages', 'README')), str(spdir))
     for source, target in binaries:
         archive = bindir.join(target)
         shutil.copy(str(source), str(archive))
+    fix_permissions(builddir)
+
     old_dir = os.getcwd()
-    fix_permissions(builddir)
     try:
         os.chdir(str(builddir))
-        #
-        # 'strip' fun: see issue #587
-        if not nostrip:
+        if not options.nostrip:
             for source, target in binaries:
                 if sys.platform == 'win32':
                     pass
                 elif sys.platform == 'darwin':
+                    # 'strip' fun: see issue #587 for why -x
                     os.system("strip -x " + str(bindir.join(target)))    # ignore errors
                 else:
                     os.system("strip " + str(bindir.join(target)))    # ignore errors
                 raise OSError('"tar" returned exit status %r' % e)
     finally:
         os.chdir(old_dir)
-    if copy_to_dir is not None:
-        print "Copying %s to %s" % (archive, copy_to_dir)
-        shutil.copy(archive, str(copy_to_dir))
+    if options.targetdir:
+        print "Copying %s to %s" % (archive, options.targetdir)
+        shutil.copy(archive, options.targetdir)
     else:
         print "Ready in %s" % (builddir,)
-    return builddir # for tests
+    return retval, builddir # for tests
 
+def package(*args):
+    try:
+        import argparse
+    except ImportError:
+        import imp
+        argparse = imp.load_source('argparse', 'lib-python/2.7/argparse.py')
+    if sys.platform == 'win32':
+        pypy_exe = 'pypy.exe'
+        license_base = os.path.join(basedir, r'..\..\..\local') # as on buildbot YMMV
+    else:
+        pypy_exe = 'pypy'
+        license_base = '/usr/share/doc'
+    parser = argparse.ArgumentParser()
+    args = list(args)
+    args[0] = str(args[0])
+    parser.add_argument('--without-tk', dest='no_tk', action='store_true',
+        help='build and package the cffi tkinter module')
+    parser.add_argument('--without-cffi', dest='no_cffi', action='store_true',
+        help='do not pre-import any cffi modules')
+    parser.add_argument('--nostrip', dest='nostrip', action='store_true',
+        help='do not strip the exe, making it ~10MB larger')
+    parser.add_argument('--rename_pypy_c', dest='pypy_c', type=str, default=pypy_exe,
+        help='target executable name, defaults to "pypy"')
+    parser.add_argument('--archive-name', dest='name', type=str, default='',
+        help='pypy-VER-PLATFORM')
+    parser.add_argument('--license_base', type=str, default=license_base,
+        help='where to start looking for third party upstream licensing info')
+    parser.add_argument('--builddir', type=str, default='',
+        help='tmp dir for packaging')
+    parser.add_argument('--targetdir', type=str, default='',
+        help='destination dir for archive')
+    parser.add_argument('--override_pypy_c', type=str, default='',
+        help='use as pypy exe instead of pypy/goal/pypy-c')
+    # Positional arguments, for backward compatability with buldbots
+    parser.add_argument('extra_args', help='optional interface to positional arguments', nargs=argparse.REMAINDER,
+        metavar='[root-pypy-dir] [name-of-archive] [name-of-pypy-c] [destination-for-tarball] [pypy-c-path]',
+        )
+    options = parser.parse_args(args)
 
-def print_usage():
-    print >>sys.stderr, __doc__
-    sys.exit(1)
+    # Handle positional arguments, choke if both methods are used
+    for i,target, default in ([1, 'name', ''], [2, 'pypy_c', pypy_exe],
+                              [3, 'targetdir', ''], [4,'override_pypy_c', '']):
+        if len(options.extra_args)>i:
+            if getattr(options, target) != default:
+                print 'positional argument',i,target,'already has value',getattr(options, target)
+                parser.print_help()
+                return
+            setattr(options, target, options.extra_args[i])
+    if os.environ.has_key("PYPY_PACKAGE_NOSTRIP"):
+        options.nostrip = True
+
+    if os.environ.has_key("PYPY_PACKAGE_WITHOUTTK"):
+        options.tk = True
+    if not options.builddir:
+        # The import actually creates the udir directory
+        from rpython.tool.udir import udir
+        options.builddir = udir.ensure("build", dir=True)
+    assert '/' not in options.pypy_c
+    return create_package(basedir, options)
+
+
+third_party_header = '''\n\nLicenses and Acknowledgements for Incorporated Software
+=======================================================
+
+This section is an incomplete, but growing list of licenses and acknowledgements
+for third-party software incorporated in the PyPy distribution.
+
+'''
+
+gdbm_bit = '''gdbm
+----
+
+The gdbm module includes code from gdbm.h, which is distributed under the terms
+of the GPL license version 2 or any later version.
+'''
 
 
 if __name__ == '__main__':
-    if len(sys.argv) == 1:
-        print_usage()
-
-    args = sys.argv[1:]
-    kw = {}
-
-    for i, arg in enumerate(args):
-        if arg == '--nostrip':
-            kw['nostrip'] = True
-        elif arg == '--without-tk':
-            kw['withouttk'] = True
-        elif not arg.startswith('--'):
-            break
-        else:
-            print_usage()
-
-    if os.environ.has_key("PYPY_PACKAGE_NOSTRIP"):
-        kw['nostrip'] = True
-
-    if os.environ.has_key("PYPY_PACKAGE_WITHOUTTK"):
-        kw['withouttk'] = True
-
-    args = args[i:]
-    package(*args, **kw)
+    import sys
+    retval, _ = package(*sys.argv[1:])
+    sys.exit(retval)

pypy/tool/release/test/test_package.py

 
 import py
 from pypy.conftest import pypydir
-from pypy.tool.release import package
+from pypy.tool.release import package, package
 from pypy.module.sys.version import  CPYTHON_VERSION
 import tarfile, zipfile, sys
 
     else:
         fake_pypy_c = False
     try:
-        builddir = package.package(py.path.local(pypydir).dirpath(), test,
+        retval, builddir = package.package(py.path.local(pypydir).dirpath(), test,
                                    rename_pypy_c)
+        assert retval == 0
         prefix = builddir.join(test)
         cpyver = '%d.%d' % CPYTHON_VERSION[:2]
         assert prefix.join('lib-python', cpyver, 'test').check()
             pypy_c.remove()
 
 def test_with_zipfile_module():
-    from pypy.tool.release import package
     prev = package.USE_ZIPFILE_MODULE
     try:
         package.USE_ZIPFILE_MODULE = True
     check(file1, 0644)
     check(file2, 0644)
     check(pypy,  0755)
+
+def test_generate_license():
+    from os.path import dirname, abspath
+    class Options(object):
+        pass
+    options = Options()
+    basedir = dirname(dirname(dirname(dirname(dirname(abspath(__file__))))))
+    options.no_tk = False
+    if sys.platform == 'win32':
+        # Following recommended build setup at
+        # http://doc.pypy.org/en/latest/windows.html#abridged-method-for-ojit-builds-using-visual-studio-2008
+        options.license_base = dirname(basedir) + '/local'
+    else:
+        options.license_base = '/usr/share/doc'
+    license = package.generate_license(py.path.local(basedir), options)
+    assert 'bzip2' in license
+    assert 'openssl' in license
+    assert 'Tcl' in license
+

rpython/annotator/binaryop.py

 Binary operations between SomeValues.
 """
 
-import py
 import operator
 from rpython.tool.pairtype import pair, pairtype
 from rpython.annotator.model import (
     SomeObject, SomeInteger, SomeBool, s_Bool, SomeString, SomeChar, SomeList,
-    SomeDict, SomeOrderedDict, SomeUnicodeCodePoint, SomeUnicodeString,
+    SomeDict, SomeUnicodeCodePoint, SomeUnicodeString,
     SomeTuple, SomeImpossibleValue, s_ImpossibleValue, SomeInstance,
-    SomeBuiltinMethod, SomeIterator, SomePBC, SomeFloat, s_None, SomeByteArray,
-    SomeWeakRef, SomeSingleFloat,
+    SomeBuiltinMethod, SomeIterator, SomePBC, SomeNone, SomeFloat, s_None,
+    SomeByteArray, SomeWeakRef, SomeSingleFloat,
     SomeLongFloat, SomeType, SomeConstantType, unionof, UnionError,
     read_can_only_throw, add_knowntypedata,
     merge_knowntypedata,)
 
 # mixing Nones with other objects
 
-def _make_none_union(classname, constructor_args='', glob=None):
-    if glob is None:
-        glob = globals()
-    loc = locals()
-    source = py.code.Source("""
-        class __extend__(pairtype(%(classname)s, SomePBC)):
-            def union((obj, pbc)):
-                if pbc.isNone():
-                    return %(classname)s(%(constructor_args)s)
-                else:
-                    raise UnionError(pbc, obj)
+class __extend__(pairtype(SomeObject, SomeNone)):
+    def union((obj, none)):
+        return obj.noneify()
 
-        class __extend__(pairtype(SomePBC, %(classname)s)):
-            def union((pbc, obj)):
-                if pbc.isNone():
-                    return %(classname)s(%(constructor_args)s)
-                else:
-                    raise UnionError(pbc, obj)
-    """ % loc)
-    exec source.compile() in glob
+class __extend__(pairtype(SomeNone, SomeObject)):
+    def union((none, obj)):
+        return obj.noneify()
 
-_make_none_union('SomeInstance',   'classdef=obj.classdef, can_be_None=True')
-_make_none_union('SomeString',      'no_nul=obj.no_nul, can_be_None=True')
-_make_none_union('SomeUnicodeString', 'can_be_None=True')
-_make_none_union('SomeList',         'obj.listdef')
-_make_none_union('SomeOrderedDict',          'obj.dictdef')
-_make_none_union('SomeDict',          'obj.dictdef')
-_make_none_union('SomeWeakRef',         'obj.classdef')
+class __extend__(pairtype(SomeImpossibleValue, SomeNone)):
+    def union((imp1, none)):
+        return s_None
 
-# getitem on SomePBCs, in particular None fails
+class __extend__(pairtype(SomeNone, SomeImpossibleValue)):
+    def union((none, imp2)):
+        return s_None
+
 
 class __extend__(pairtype(SomePBC, SomeObject)):
     def getitem((pbc, o)):
-        if not pbc.isNone():
-            raise AnnotatorError("getitem on %r" % pbc)
+        raise AnnotatorError("getitem on %r" % pbc)
+
+    def setitem((pbc, o), s_value):
+        raise AnnotatorError("setitem on %r" % pbc)
+
+class __extend__(pairtype(SomeNone, SomeObject)):
+    def getitem((none, o)):
         return s_ImpossibleValue
 
-    def setitem((pbc, o), s_value):
-        if not pbc.isNone():
-            raise AnnotatorError("setitem on %r" % pbc)
+    def setitem((none, o), s_value):
+        return None
 
 class __extend__(pairtype(SomePBC, SomeString)):
     def add((pbc, o)):
-        if not pbc.isNone():
-            raise AnnotatorError('add on %r' % pbc)
+        raise AnnotatorError('add on %r' % pbc)
+
+class __extend__(pairtype(SomeNone, SomeString)):
+    def add((none, o)):
         return s_ImpossibleValue
 
 class __extend__(pairtype(SomeString, SomePBC)):
     def add((o, pbc)):
-        if not pbc.isNone():
-            raise AnnotatorError('add on %r' % pbc)
+        raise AnnotatorError('add on %r' % pbc)
+
+class __extend__(pairtype(SomeString, SomeNone)):
+    def add((o, none)):
         return s_ImpossibleValue
 
 #_________________________________________

rpython/annotator/bookkeeper.py

 from rpython.annotator.dictdef import DictDef
 from rpython.annotator import description
 from rpython.annotator.signature import annotationoftype
-from rpython.annotator.argument import simple_args, complex_args
+from rpython.annotator.argument import simple_args
 from rpython.rlib.objectmodel import r_dict, Symbolic
 from rpython.tool.algo.unionfind import UnionFind
 from rpython.rtyper import extregistry
                 self.consider_call_site(call_op)
 
             for pbc, args_s in self.emulated_pbc_calls.itervalues():
-                self.consider_call_site_for_pbc(pbc, 'simple_call',
-                                                args_s, s_ImpossibleValue, None)
+                args = simple_args(args_s)
+                self.consider_call_site_for_pbc(pbc, args,
+                                                s_ImpossibleValue, None)
             self.emulated_pbc_calls = {}
         finally:
             self.leave()
             args_s = [lltype_to_annotation(adtmeth.ll_ptrtype)] + args_s
         if isinstance(s_callable, SomePBC):
             s_result = binding(call_op.result, s_ImpossibleValue)
-            self.consider_call_site_for_pbc(s_callable, call_op.opname, args_s,
+            args = call_op.build_args(args_s)
+            self.consider_call_site_for_pbc(s_callable, args,
                                             s_result, call_op)
 
-    def consider_call_site_for_pbc(self, s_callable, opname, args_s, s_result,
+    def consider_call_site_for_pbc(self, s_callable, args, s_result,
                                    call_op):
         descs = list(s_callable.descriptions)
-        if not descs:
-            return
         family = descs[0].getcallfamily()
-        args = self.build_args(opname, args_s)
         s_callable.getKind().consider_call_site(self, family, descs, args,
                                                 s_result, call_op)
 
         attr = s_attr.const
 
         descs = list(pbc.descriptions)
-        if not descs:
-            return s_ImpossibleValue
-
         first = descs[0]
         if len(descs) == 1:
             return first.s_read_attribute(attr)
         annotations).
         """
         descs = list(pbc.descriptions)
-        if not descs:
-            return s_ImpossibleValue
         first = descs[0]
         first.mergecallfamilies(*descs[1:])
 
             assert self.annotator.binding(op.args[pos]) == s_type
         return op
 
-    def build_args(self, op, args_s):
-        if op == "simple_call":
-            return simple_args(args_s)
-        elif op == "call_args":
-            return complex_args(args_s)
-
     def ondegenerated(self, what, s_value, where=None, called_from_graph=None):
         self.annotator.ondegenerated(what, s_value, where=where,
                                      called_from_graph=called_from_graph)

rpython/annotator/classdef.py

 """
 Type inference for user-defined classes.
 """
-from rpython.annotator.model import SomePBC, s_ImpossibleValue, unionof
-from rpython.annotator.model import SomeInteger, SomeTuple, SomeString, AnnotatorError
+from rpython.annotator.model import (
+    SomePBC, s_ImpossibleValue, unionof, s_None, SomeInteger,
+    SomeTuple, SomeString, AnnotatorError)
 from rpython.annotator import description
 
 
         # check for method demotion and after-the-fact method additions
         if isinstance(s_newvalue, SomePBC):
             attr = self.name
-            if (not s_newvalue.isNone() and
-                s_newvalue.getKind() == description.MethodDesc):
+            if s_newvalue.getKind() == description.MethodDesc:
                 # is method
                 if homedef.classdesc.read_attribute(attr, None) is None:
                     if not homedef.check_missing_attribute_update(attr):
         if uplookup is not None:
             d.append(updesc.bind_self(self, flags))
 
-        if d or pbc.can_be_None:
+        if d:
             return SomePBC(d, can_be_None=pbc.can_be_None)
+        elif pbc.can_be_None:
+            return s_None
         else:
             return s_ImpossibleValue
 

rpython/annotator/model.py

     def can_be_none(self):
         return True
 
+    def noneify(self):
+        raise UnionError(self, s_None)
+
     def nonnoneify(self):
         return self
 
     "Stands for an object which is known to be a string."
     knowntype = str
 
+    def noneify(self):
+        return SomeString(can_be_None=True, no_nul=self.no_nul)
+
 
 class SomeUnicodeString(SomeStringOrUnicode):
     "Stands for an object which is known to be an unicode string"
     knowntype = unicode
 
+    def noneify(self):
+        return SomeUnicodeString(can_be_None=True, no_nul=self.no_nul)
+
 
 class SomeByteArray(SomeStringOrUnicode):
     immutable = False
     def can_be_none(self):
         return True
 
+    def noneify(self):
+        return SomeList(self.listdef)
+
 
 class SomeTuple(SomeObject):
     "Stands for a tuple of known length."
         else:
             return '{...%s...}' % (len(const),)
 
+    def noneify(self):
+        return type(self)(self.dictdef)
+
 class SomeOrderedDict(SomeDict):
     try:
         from collections import OrderedDict as knowntype
     def nonnoneify(self):
         return SomeInstance(self.classdef, can_be_None=False)
 
+    def noneify(self):
+        return SomeInstance(self.classdef, can_be_None=True)
+
 
 class SomePBC(SomeObject):
     """Stands for a global user instance, built prior to the analysis,
     immutable = True
 
     def __init__(self, descriptions, can_be_None=False, subset_of=None):
+        assert descriptions
         # descriptions is a set of Desc instances
         descriptions = set(descriptions)
         self.descriptions = descriptions
         self.can_be_None = can_be_None
         self.subset_of = subset_of
         self.simplify()
-        if self.isNone():
-            self.knowntype = type(None)
-            self.const = None
-        else:
-            knowntype = reduce(commonbase,
-                               [x.knowntype for x in descriptions])
-            if knowntype == type(Exception):
-                knowntype = type
-            if knowntype != object:
-                self.knowntype = knowntype
-            if len(descriptions) == 1 and not can_be_None:
-                # hack for the convenience of direct callers to SomePBC():
-                # only if there is a single object in descriptions
-                desc, = descriptions
-                if desc.pyobj is not None:
-                    self.const = desc.pyobj
-            elif len(descriptions) > 1:
-                from rpython.annotator.description import ClassDesc
-                if self.getKind() is ClassDesc:
-                    # a PBC of several classes: enforce them all to be
-                    # built, without support for specialization.  See
-                    # rpython/test/test_rpbc.test_pbc_of_classes_not_all_used
-                    for desc in descriptions:
-                        desc.getuniqueclassdef()
+        knowntype = reduce(commonbase, [x.knowntype for x in descriptions])
+        if knowntype == type(Exception):
+            knowntype = type
+        if knowntype != object:
+            self.knowntype = knowntype
+        if len(descriptions) == 1 and not can_be_None:
+            # hack for the convenience of direct callers to SomePBC():
+            # only if there is a single object in descriptions
+            desc, = descriptions
+            if desc.pyobj is not None:
+                self.const = desc.pyobj
+        elif len(descriptions) > 1:
+            from rpython.annotator.description import ClassDesc
+            if self.getKind() is ClassDesc:
+                # a PBC of several classes: enforce them all to be
+                # built, without support for specialization.  See
+                # rpython/test/test_rpbc.test_pbc_of_classes_not_all_used
+                for desc in descriptions:
+                    desc.getuniqueclassdef()
 
     def any_description(self):
         return iter(self.descriptions).next()
             kinds.add(x.__class__)
         if len(kinds) > 1:
             raise AnnotatorError("mixing several kinds of PBCs: %r" % kinds)
-        if not kinds:
-            raise ValueError("no 'kind' on the 'None' PBC")
         return kinds.pop()
 
     def simplify(self):
-        if self.descriptions:
-            # We check that the set only contains a single kind of Desc instance
-            kind = self.getKind()
-            # then we remove unnecessary entries in self.descriptions:
-            # some MethodDescs can be 'shadowed' by others
-            if len(self.descriptions) > 1:
-                kind.simplify_desc_set(self.descriptions)
-        else:
-            assert self.can_be_None, "use s_ImpossibleValue"
-
-    def isNone(self):
-        return len(self.descriptions) == 0
+        # We check that the set only contains a single kind of Desc instance
+        kind = self.getKind()
+        # then we remove unnecessary entries in self.descriptions:
+        # some MethodDescs can be 'shadowed' by others
+        if len(self.descriptions) > 1:
+            kind.simplify_desc_set(self.descriptions)
 
     def can_be_none(self):
         return self.can_be_None
 
     def nonnoneify(self):
-        if self.isNone():
-            return s_ImpossibleValue
-        else:
-            return SomePBC(self.descriptions, can_be_None=False)
+        return SomePBC(self.descriptions, can_be_None=False,
+                subset_of=self.subset_of)
+
+    def noneify(self):
+        return SomePBC(self.descriptions, can_be_None=True,
+                subset_of=self.subset_of)
 
     def fmt_descriptions(self, pbis):
         if hasattr(self, 'const'):
         else:
             return kt.__name__
 
+class SomeNone(SomeObject):
+    knowntype = type(None)
+    const = None
+
+    def __init__(self):
+        pass
+
+    def is_constant(self):
+        return True
+
+    def is_immutable_constant(self):
+        return True
+
+    def nonnoneify(self):
+        return s_ImpossibleValue
+
+
 class SomeConstantType(SomePBC):
     can_be_None = False
     subset_of = None
         return False
 
 
-s_None = SomePBC([], can_be_None=True)
+s_None = SomeNone()
 s_Bool = SomeBool()
 s_Int = SomeInteger()
 s_ImpossibleValue = SomeImpossibleValue()
         # 'classdef' is None for known-to-be-dead weakrefs.
         self.classdef = classdef
 
+    def noneify(self):
+        return SomeWeakRef(self.classdef)
+
 # ____________________________________________________________
 
 
 
 
 def not_const(s_obj):
-    if s_obj.is_constant() and not isinstance(s_obj, SomePBC):
+    if s_obj.is_constant() and not isinstance(s_obj, (SomePBC, SomeNone)):
         new_s_obj = SomeObject.__new__(s_obj.__class__)
         dic = new_s_obj.__dict__ = s_obj.__dict__.copy()
         if 'const' in dic:

rpython/annotator/signature.py

         for i, argtype in enumerate(self.argtypes):
             if isinstance(argtype, (types.FunctionType, types.MethodType)):
                 argtype = argtype(*inputcells)
-            if isinstance(argtype, lltype.LowLevelType) and\
-                argtype is lltype.Void:
+            if argtype is lltype.Void:
                 # XXX the mapping between Void and annotation
                 # is not quite well defined
                 s_input = inputcells[i]
-                assert isinstance(s_input, annmodel.SomePBC)
+                assert isinstance(s_input, (annmodel.SomePBC, annmodel.SomeNone))
                 assert s_input.is_constant()
                 args_s.append(s_input)
             elif argtype is None:

rpython/annotator/test/test_model.py

     py.test.raises(AnnotatorError, compile_function, blocked_inference)
 
 
-if __name__ == '__main__':
-    for name, value in globals().items():
-        if name.startswith('test_'):
-            value()
-
+def test_not_const():
+    s_int = SomeInteger()
+    s_int.const = 2
+    assert s_int != SomeInteger()
+    assert not_const(s_int) == SomeInteger()
+    assert not_const(s_None) == s_None

rpython/annotator/unaryop.py

 from rpython.annotator.model import (SomeObject, SomeInteger, SomeBool,
     SomeString, SomeChar, SomeList, SomeDict, SomeTuple, SomeImpossibleValue,
     SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeBuiltinMethod,
-    SomeFloat, SomeIterator, SomePBC, SomeType, s_ImpossibleValue,
+    SomeFloat, SomeIterator, SomePBC, SomeNone, SomeType, s_ImpossibleValue,
     s_Bool, s_None, unionof, add_knowntypedata,
     HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString, SomeByteArray)
 from rpython.annotator.bookkeeper import getbookkeeper, immutablevalue
     getattr.can_only_throw = []
 
     def setattr(self, s_attr, s_value):
-        if not self.isNone():
-            raise AnnotatorError("Cannot modify attribute of a pre-built constant")
+        raise AnnotatorError("Cannot modify attribute of a pre-built constant")
 
     def call(self, args):
         bookkeeper = getbookkeeper()
         return SomePBC(d, can_be_None=self.can_be_None)
 
     def bool_behavior(self, s):
-        if self.isNone():
-            s.const = False
-        elif not self.can_be_None:
+        if not self.can_be_None:
             s.const = True
 
     def len(self):
-        if self.isNone():
-            # this None could later be generalized into an empty list,
-            # whose length is the constant 0; so let's tentatively answer 0.
-            return immutablevalue(0)
-        else:
-            # This should probably never happen
-            raise AnnotatorError("Cannot call len on a pbc")
+        raise AnnotatorError("Cannot call len on a pbc")
+
+class __extend__(SomeNone):
+    def bind_callables_under(self, classdef, name):
+        return self
+
+    def getattr(self, s_attr):
+        return s_ImpossibleValue
+    getattr.can_only_throw = []
+
+    def setattr(self, s_attr, s_value):
+        return None
+
+    def call(self, args):
+        return s_ImpossibleValue
+
+    def bool_behavior(self, s):
+        s.const = False
+
+    def len(self):
+        # XXX: this None could later be generalized into an empty list,
+        # whose length is the constant 0; so let's tentatively answer 0.
+        return immutablevalue(0)
 
 #_________________________________________
 # weakrefs

rpython/flowspace/operation.py

                                      SpaceOperation)
 from rpython.flowspace.specialcase import register_flow_sc
 from rpython.annotator.model import SomeTuple
+from rpython.annotator.argument import ArgumentsForTranslation
 from rpython.flowspace.specialcase import SPECIAL_CASES
 
 
                 return sc(ctx, *args_w)
         return ctx.do_op(self)
 
+    def build_args(self, args_s):
+        return ArgumentsForTranslation(list(args_s))
+
 
 class CallArgs(SingleDispatchMixin, CallOp):
     opname = 'call_args'
                     "should not call %r with keyword arguments" % (fn,))
         return ctx.do_op(self)
 
+    def build_args(self, args_s):
+        return ArgumentsForTranslation.fromshape(args_s[0].const,
+                                                list(args_s[1:]))
+
 
 # Other functions that get directly translated to SpaceOperators
 func2op[type] = op.type

rpython/jit/backend/detect_cpu.py

             'AMD64': MODEL_X86,    # win64
             'armv7l': MODEL_ARM,
             'armv6l': MODEL_ARM,
-            }[mach]
+            }.get(mach)
+
+    if result is None:
+        raise ProcessorAutodetectError, "unknown machine name %s" % mach
     #