Philip Jenvey avatar Philip Jenvey committed 7aecd86 Merge

merge default

Comments (0)

Files changed (60)

     $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
 
 This ends up with ``pypy-c`` binary in the main pypy directory. We suggest
-to use virtualenv with the resulting pypy-c as the interpreter, you can
+to use virtualenv with the resulting pypy-c as the interpreter; you can
 find more details about various installation schemes here:
 
 http://doc.pypy.org/en/latest/getting-started.html#installing-pypy

lib-python/2.7/test/test_mailbox.py

     def _delete_recursively(self, target):
         # Delete a file or delete a directory recursively
         if os.path.isdir(target):
-            for path, dirs, files in os.walk(target, topdown=False):
-                for name in files:
-                    os.remove(os.path.join(path, name))
-                for name in dirs:
-                    os.rmdir(os.path.join(path, name))
-            os.rmdir(target)
+            test_support.rmtree(target)
         elif os.path.exists(target):
-            os.remove(target)
+            test_support.unlink(target)
 
 
 class TestMailbox(TestBase):
         msg = self._box.get(key1)
         self.assertEqual(msg['from'], 'foo')
         self.assertEqual(msg.fp.read(), '1')
+        msg.fp.close()
 
     def test_getitem(self):
         # Retrieve message using __getitem__()
         # Get file representations of messages
         key0 = self._box.add(self._template % 0)
         key1 = self._box.add(_sample_message)
-        self.assertEqual(self._box.get_file(key0).read().replace(os.linesep, '\n'),
+        msg0 = self._box.get_file(key0)
+        self.assertEqual(msg0.read().replace(os.linesep, '\n'),
                          self._template % 0)
-        self.assertEqual(self._box.get_file(key1).read().replace(os.linesep, '\n'),
+        msg1 = self._box.get_file(key1)
+        self.assertEqual(msg1.read().replace(os.linesep, '\n'),
                          _sample_message)
+        msg0.close()
+        msg1.close()
 
     def test_get_file_can_be_closed_twice(self):
         # Issue 11700
         self._box.add(contents[0])
         self._box.add(contents[1])
         self._box.add(contents[2])
+        oldbox = self._box
         method()
         if should_call_close:
             self._box.close()
         self.assertEqual(len(keys), 3)
         for key in keys:
             self.assertIn(self._box.get_string(key), contents)
+        oldbox.close()
 
     def test_dump_message(self):
         # Write message representations to disk
     def setUp(self):
         # create a new maildir mailbox to work with:
         self._dir = test_support.TESTFN
+        if os.path.isdir(self._dir):
+            test_support.rmtree(self._dir)
+        if os.path.isfile(self._dir):
+            test_support.unlink(self._dir)
         os.mkdir(self._dir)
         os.mkdir(os.path.join(self._dir, "cur"))
         os.mkdir(os.path.join(self._dir, "tmp"))
 
     def tearDown(self):
         map(os.unlink, self._msgfiles)
-        os.rmdir(os.path.join(self._dir, "cur"))
-        os.rmdir(os.path.join(self._dir, "tmp"))
-        os.rmdir(os.path.join(self._dir, "new"))
-        os.rmdir(self._dir)
+        test_support.rmdir(os.path.join(self._dir, "cur"))
+        test_support.rmdir(os.path.join(self._dir, "tmp"))
+        test_support.rmdir(os.path.join(self._dir, "new"))
+        test_support.rmdir(self._dir)
 
     def createMessage(self, dir, mbox=False):
         t = int(time.time() % 1000000)
         self.createMessage("cur")
         self.mbox = mailbox.Maildir(test_support.TESTFN)
         #self.assertTrue(len(self.mbox.boxes) == 1)
-        self.assertIsNot(self.mbox.next(), None)
+        msg = self.mbox.next()
+        self.assertIsNot(msg, None)
+        msg.fp.close()
         self.assertIs(self.mbox.next(), None)
         self.assertIs(self.mbox.next(), None)
 
         self.createMessage("new")
         self.mbox = mailbox.Maildir(test_support.TESTFN)
         #self.assertTrue(len(self.mbox.boxes) == 1)
-        self.assertIsNot(self.mbox.next(), None)
+        msg = self.mbox.next()
+        self.assertIsNot(msg, None)
+        msg.fp.close()
         self.assertIs(self.mbox.next(), None)
         self.assertIs(self.mbox.next(), None)
 
         self.createMessage("new")
         self.mbox = mailbox.Maildir(test_support.TESTFN)
         #self.assertTrue(len(self.mbox.boxes) == 2)
-        self.assertIsNot(self.mbox.next(), None)
-        self.assertIsNot(self.mbox.next(), None)
+        msg = self.mbox.next()
+        self.assertIsNot(msg, None)
+        msg.fp.close()
+        msg = self.mbox.next()
+        self.assertIsNot(msg, None)
+        msg.fp.close()
         self.assertIs(self.mbox.next(), None)
         self.assertIs(self.mbox.next(), None)
 
         import email.parser
         fname = self.createMessage("cur", True)
         n = 0
-        for msg in mailbox.PortableUnixMailbox(open(fname),
+        fid = open(fname)
+        for msg in mailbox.PortableUnixMailbox(fid,
                                                email.parser.Parser().parse):
             n += 1
             self.assertEqual(msg["subject"], "Simple Test")
             self.assertEqual(len(str(msg)), len(FROM_)+len(DUMMY_MESSAGE))
+        fid.close()
         self.assertEqual(n, 1)
 
 ## End: classes from the original module (for backward compatibility).

lib-python/2.7/test/test_mmap.py

 
     def setUp(self):
         if os.path.exists(TESTFN):
-            os.unlink(TESTFN)
+            unlink(TESTFN)
 
     def tearDown(self):
         try:

lib-python/2.7/test/test_os.py

         self.assertFalse(os.path.exists(name),
                     "file already exists for temporary file")
         # make sure we can create the file
-        open(name, "w")
+        open(name, "w").close()
         self.files.append(name)
 
     def test_tempnam(self):

lib-python/2.7/test/test_support.py

     except KeyError:
         pass
 
+if sys.platform.startswith("win"):
+    def _waitfor(func, pathname, waitall=False):
+        # Peform the operation
+        func(pathname)
+        # Now setup the wait loop
+        if waitall:
+            dirname = pathname
+        else:
+            dirname, name = os.path.split(pathname)
+            dirname = dirname or '.'
+        # Check for `pathname` to be removed from the filesystem.
+        # The exponential backoff of the timeout amounts to a total
+        # of ~1 second after which the deletion is probably an error
+        # anyway.
+        # Testing on a i7@4.3GHz shows that usually only 1 iteration is
+        # required when contention occurs.
+        timeout = 0.001
+        while timeout < 1.0:
+            # Note we are only testing for the existance of the file(s) in
+            # the contents of the directory regardless of any security or
+            # access rights.  If we have made it this far, we have sufficient
+            # permissions to do that much using Python's equivalent of the
+            # Windows API FindFirstFile.
+            # Other Windows APIs can fail or give incorrect results when
+            # dealing with files that are pending deletion.
+            L = os.listdir(dirname)
+            if not (L if waitall else name in L):
+                return
+            # Increase the timeout and try again
+            time.sleep(timeout)
+            timeout *= 2
+        warnings.warn('tests may fail, delete still pending for ' + pathname,
+                      RuntimeWarning, stacklevel=4)
+
+    def _unlink(filename):
+        _waitfor(os.unlink, filename)
+
+    def _rmdir(dirname):
+        _waitfor(os.rmdir, dirname)
+
+    def _rmtree(path):
+        def _rmtree_inner(path):
+            for name in os.listdir(path):
+                fullname = os.path.join(path, name)
+                if os.path.isdir(fullname):
+                    _waitfor(_rmtree_inner, fullname, waitall=True)
+                    os.rmdir(fullname)
+                else:
+                    os.unlink(fullname)
+        _waitfor(_rmtree_inner, path, waitall=True)
+        _waitfor(os.rmdir, path)
+else:
+    _unlink = os.unlink
+    _rmdir = os.rmdir
+    _rmtree = shutil.rmtree
+
 def unlink(filename):
     try:
-        os.unlink(filename)
+        _unlink(filename)
     except OSError:
         pass
 
+def rmdir(dirname):
+    try:
+        _rmdir(dirname)
+    except OSError as error:
+        # The directory need not exist.
+        if error.errno != errno.ENOENT:
+            raise
+
 def rmtree(path):
     try:
-        shutil.rmtree(path)
+        _rmtree(path)
     except OSError, e:
         # Unix returns ENOENT, Windows returns ESRCH.
         if e.errno not in (errno.ENOENT, errno.ESRCH):

lib-python/2.7/test/test_tarfile.py

 
     def test_extract_hardlink(self):
         # Test hardlink extraction (e.g. bug #857297).
-        tar = tarfile.open(tarname, errorlevel=1, encoding="iso8859-1")
+        with tarfile.open(tarname, errorlevel=1, encoding="iso8859-1") as tar:
+            tar.extract("ustar/regtype", TEMPDIR)
+            self.addCleanup(os.remove, os.path.join(TEMPDIR, "ustar/regtype"))
 
-        tar.extract("ustar/regtype", TEMPDIR)
-        try:
             tar.extract("ustar/lnktype", TEMPDIR)
-        except EnvironmentError, e:
-            if e.errno == errno.ENOENT:
-                self.fail("hardlink not extracted properly")
+            self.addCleanup(os.remove, os.path.join(TEMPDIR, "ustar/lnktype"))
+            with open(os.path.join(TEMPDIR, "ustar/lnktype"), "rb") as f:
+                data = f.read()
+            self.assertEqual(md5sum(data), md5_regtype)
 
-        data = open(os.path.join(TEMPDIR, "ustar/lnktype"), "rb").read()
-        self.assertEqual(md5sum(data), md5_regtype)
-
-        try:
             tar.extract("ustar/symtype", TEMPDIR)
-        except EnvironmentError, e:
-            if e.errno == errno.ENOENT:
-                self.fail("symlink not extracted properly")
-
-        data = open(os.path.join(TEMPDIR, "ustar/symtype"), "rb").read()
-        self.assertEqual(md5sum(data), md5_regtype)
+            self.addCleanup(os.remove, os.path.join(TEMPDIR, "ustar/symtype"))
+            with open(os.path.join(TEMPDIR, "ustar/symtype"), "rb") as f:
+                data = f.read()
+            self.assertEqual(md5sum(data), md5_regtype)
 
     def test_extractall(self):
         # Test if extractall() correctly restores directory permissions
         # constructor in case of an error. For the test we rely on
         # the fact that opening an empty file raises a ReadError.
         empty = os.path.join(TEMPDIR, "empty")
-        open(empty, "wb").write("")
+        open(empty, "wb").close()
 
         try:
             tar = object.__new__(tarfile.TarFile)
             else:
                 self.fail("ReadError not raised")
         finally:
-            os.remove(empty)
+            test_support.unlink(empty)
 
 
 class StreamReadTest(CommonReadTest):
     def setUp(self):
         self.tarname = tmpname
         if os.path.exists(self.tarname):
-            os.remove(self.tarname)
+            test_support.unlink(self.tarname)
 
     def _add_testfile(self, fileobj=None):
         tar = tarfile.open(self.tarname, "a", fileobj=fileobj)

lib_pypy/cffi/api.py

     errno = property(_get_errno, _set_errno, None,
                      "the value of 'errno' from/to the C calls")
 
+    def getwinerror(self, code=-1):
+        return self._backend.getwinerror(code)
+
     def _pointer_to(self, ctype):
         from . import model
         with self._lock:
 # built documents.
 #
 # The short X.Y version.
-version = '2.1'
+version = '2.2'
 # The full version, including alpha/beta/rc tags.
-release = '2.1.0'
+release = '2.2.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

pypy/doc/how-to-release.rst

 * send announcements to pypy-dev, python-list,
   python-announce, python-dev ...
 
-* add a tag on jitviewer that corresponds to pypy release
-* add a tag on codespeed that corresponds to pypy release
+* add a tag on the pypy/jitviewer repo that corresponds to pypy release
+* add a tag on the codespeed web site that corresponds to pypy release
 

pypy/doc/index.rst

 
 * `FAQ`_: some frequently asked questions.
 
-* `Release 2.1.0`_: the latest official release
+* `Release 2.2.0`_: the latest official release
 
 * `PyPy Blog`_: news and status info about PyPy 
 
 .. _`Getting Started`: getting-started.html
 .. _`Papers`: extradoc.html
 .. _`Videos`: video-index.html
-.. _`Release 2.1.0`: http://pypy.org/download.html
+.. _`Release 2.2.0`: http://pypy.org/download.html
 .. _`speed.pypy.org`: http://speed.pypy.org
 .. _`RPython toolchain`: translation.html
 .. _`potential project ideas`: project-ideas.html

pypy/doc/release-2.2.0.rst

+=======================================
+PyPy 2.2 - Incrementalism
+=======================================
+
+We're pleased to announce PyPy 2.2, which targets version 2.7.3 of the Python
+language. This release main highlight is the introduction of the incremental
+garbage collector, sponsored by the `Raspberry Pi Foundation`_.
+
+This release also contains several bugfixes and performance improvements. 
+
+You can download the PyPy 2.2 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project. We showed quite a bit of progress on all three projects (see below)
+and we're slowly running out of funds.
+Please consider donating more so we can finish those projects!  The three
+projects are:
+
+* Py3k (supporting Python 3.x): the release PyPy3 2.2 is imminent.
+
+* STM (software transactional memory): a preview will be released very soon,
+  as soon as we fix a few bugs
+
+* NumPy: the work done is included in the PyPy 2.2 release. More details below.
+
+.. _`Raspberry Pi Foundation`: http://www.raspberrypi.org
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy 2.2 and cpython 2.7.2`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+This release supports x86 machines running Linux 32/64, Mac OS X 64, Windows
+32, or ARM (ARMv6 or ARMv7, with VFPv3).
+
+Work on the native Windows 64 is still stalling, we would welcome a volunteer
+to handle that.
+
+.. _`pypy 2.2 and cpython 2.7.2`: http://speed.pypy.org
+
+Highlights
+==========
+
+* Our Garbage Collector is now "incremental".  It should avoid almost
+  all pauses due to a major collection taking place.  Previously, it
+  would pause the program (rarely) to walk all live objects, which
+  could take arbitrarily long if your process is using a whole lot of
+  RAM.  Now the same work is done in steps.  This should make PyPy
+  more responsive, e.g. in games.  There are still other pauses, from
+  the GC and the JIT, but they should be on the order of 5
+  milliseconds each.
+
+* The JIT counters for hot code were never reset, which meant that a
+  process running for long enough would eventually JIT-compile more
+  and more rarely executed code.  Not only is it useless to compile
+  such code, but as more compiled code means more memory used, this
+  gives the impression of a memory leak.  This has been tentatively
+  fixed by decreasing the counters from time to time.
+
+* NumPy has been split: now PyPy only contains the core module, called
+  ``_numpypy``.  The ``numpy`` module itself has been moved to
+  ``https://bitbucket.org/pypy/numpy`` and ``numpypy`` disappeared.
+  You need to install NumPy separately with a virtualenv:
+  ``pip install git+https://bitbucket.org/pypy/numpy.git``;
+  or by directly doing
+  ``git clone https://bitbucket.org/pypy/numpy.git``,
+  ``cd numpy``, ``python setup.py install``.
+
+* non-inlined calls have less overhead
+
+* Things that use ``sys.set_trace`` are now JITted (like coverage)
+
+* JSON decoding is now very fast (JSON encoding was already very fast)
+
+* various buffer copying methods experience speedups (like list-of-ints to
+  ``int[]`` buffer from cffi)
+
+* We finally wrote (hopefully) all the missing ``os.xxx()`` functions,
+  including ``os.startfile()`` on Windows and a handful of rare ones
+  on Posix.
+
+* numpy has a rudimentary C API that cooperates with ``cpyext``
+
+Cheers,
+Armin Rigo and Maciej Fijalkowski

pypy/doc/sandbox.rst

 Introduction
 ------------
 
-It is possible to compile a version of pypy-c that runs
-fully "virtualized", i.e. where an external process controls all
-input/output.  Such a pypy-c is a secure sandbox: it is safe to run
-any untrusted Python code with it.  The Python code cannot see or
-modify any local file except via interaction with the external
-process.  It is also impossible to do any other I/O or consume more
-than some amount of RAM or CPU time or real time.  This works with no
-OS support at all - just ANSI C code generated in a careful way.  It's
-the kind of thing you could embed in a browser plug-in, for example
-(it would be safe even if it wasn't run as a separate process,
-actually).
+PyPy offers sandboxing at a level similar to OS-level sandboxing (e.g.
+SECCOMP_ on Linux), but implemented in a fully portable way.  To use it,
+a (regular, trusted) program launches a subprocess that is a special
+sandboxed version of PyPy.  This subprocess can run arbitrary untrusted
+Python code, but all its input/output is serialized to a stdin/stdout
+pipe instead of being directly performed.  The outer process reads the
+pipe and decides which commands are allowed or not (sandboxing), or even
+reinterprets them differently (virtualization).  A potential attacker
+can have arbitrary code run in the subprocess, but cannot actually do
+any input/output not controlled by the outer process.  Additional
+barriers are put to limit the amount of RAM and CPU time used.
 
-For comparison, trying to plug CPython into a special virtualizing C
-library is not only OS-specific, but unsafe, because one of the known
-ways to segfault CPython could be used by an attacker to trick CPython
-into issuing malicious system calls directly.  The C code generated by
+Note that this is very different from sandboxing at the Python language
+level, i.e. placing restrictions on what kind of Python code the
+attacker is allowed to run (why? read about pysandbox_).
+
+.. _SECCOMP: http://code.google.com/p/seccompsandbox/wiki/overview
+.. _pysandbox: https://mail.python.org/pipermail/python-dev/2013-November/130132.html
+
+Another point of comparison: if we were instead to try to plug CPython
+into a special virtualizing C library, we would get a result
+that is not only OS-specific, but unsafe, because CPython can be
+segfaulted (in many ways, all of them really, really obscure).
+Given enough efforts, an attacker can turn almost any
+segfault into a vulnerability.  The C code generated by
 PyPy is not segfaultable, as long as our code generators are correct -
-that's a lower number of lines of code to trust.  For the paranoid, in
-this case we also generate systematic run-time checks against buffer
-overflows.
+that's a lower number of lines of code to trust.  For the paranoid,
+PyPy translated with sandboxing also contains systematic run-time
+checks (against buffer overflows for example)
+that are normally only present in debugging versions.
 
 .. warning::
   

pypy/doc/whatsnew-2.2.rst

+======================
+What's new in PyPy 2.2
+======================
+
+.. this is a revision shortly after release-2.1-beta
+.. startrev: 4eb52818e7c0
+
+.. branch: sanitise_bytecode_dispatch
+Make PyPy's bytecode dispatcher easy to read, and less reliant on RPython
+magic. There is no functional change, though the removal of dead code leads
+to many fewer tests to execute.
+
+.. branch: fastjson
+Fast json decoder written in RPython, about 3-4x faster than the pure Python
+decoder which comes with the stdlib
+
+.. branch: improve-str2charp
+Improve the performance of I/O writing up to 15% by using memcpy instead of
+copying char-by-char in str2charp and get_nonmovingbuffer
+
+.. branch: flowoperators
+Simplify rpython/flowspace/ code by using more metaprogramming.  Create
+SpaceOperator class to gather static information about flow graph operations.
+
+.. branch: package-tk
+Adapt package.py script to compile CFFI tk extension. Add a --without-tk switch
+to optionally skip it.
+
+.. branch: distutils-cppldflags
+Copy CPython's implementation of customize_compiler, dont call split on
+environment variables, honour CFLAGS, CPPFLAGS, LDSHARED and LDFLAGS on Unices.
+
+.. branch: precise-instantiate
+When an RPython class is instantiated via an indirect call (that is, which
+class is being instantiated isn't known precisely) allow the optimizer to have
+more precise information about which functions can be called. Needed for Topaz.
+
+.. branch: ssl_moving_write_buffer
+
+.. branch: pythoninspect-fix
+Make PyPy respect PYTHONINSPECT variable set via os.putenv in the same process
+to start interactive prompt when the script execution finishes. This adds
+new __pypy__.os.real_getenv call that bypasses Python cache and looksup env
+in the underlying OS. Translatorshell now works on PyPy.
+
+.. branch: add-statvfs
+Added os.statvfs and os.fstatvfs
+
+.. branch: statvfs_tests
+Added some addition tests for statvfs.
+
+.. branch: ndarray-subtype
+Allow subclassing ndarray, i.e. matrix
+
+.. branch: ndarray-sort
+Implement ndarray in-place sorting (for numeric types, no non-native byte order)
+
+.. branch: pypy-pyarray
+Implement much of numpy's c api in cpyext, allows (slow) access to ndarray
+from c
+
+.. branch: kill-ootype
+
+.. branch: fast-slowpath
+Added an abstraction for functions with a fast and slow path in the JIT. This
+speeds up list.append() and list.pop().
+
+.. branch: curses_fixes
+
+.. branch: foldable-getarrayitem-indexerror
+Constant-fold reading out of constant tuples in PyPy.
+
+.. branch: mro-reorder-numpypy-str
+No longer delegate numpy string_ methods to space.StringObject, in numpy
+this works by kind of by accident. Support for merging the refactor-str-types
+branch
+
+.. branch: kill-typesystem
+Remove the "type system" abstraction, now that there is only ever one kind of
+type system used.
+
+.. branch: kill-gen-store-back-in
+Kills gen_store_back_in_virtualizable - should improve non-inlined calls by
+a bit
+
+.. branch: dotviewer-linewidth
+.. branch: reflex-support
+.. branch: numpypy-inplace-op
+.. branch: rewritten-loop-logging
+.. branch: no-release-gil
+.. branch: safe-win-mmap
+.. branch: boolean-indexing-cleanup
+.. branch: cpyext-best_base
+.. branch: cpyext-int
+.. branch: fileops2
+
+.. branch: nobold-backtrace
+Work on improving UnionError messages and stack trace displays.
+
+.. branch: improve-errors-again
+More improvements and refactorings of error messages.
+
+.. branch: improve-errors-again2
+Unbreak tests in rlib.
+
+.. branch: less-stringly-ops
+Use subclasses of SpaceOperation instead of SpaceOperator objects.
+Random cleanups in flowspace.
+
+.. branch: file-support-in-rpython
+make open() and friends rpython
+
+.. branch: incremental-gc
+Added the new incminimark GC which performs GC in incremental steps
+
+.. branch: fast_cffi_list_init
+fastpath for cffi.new("long[]")
+
+.. branch: remove-eval-frame
+remove a pointless abstraction
+
+.. branch: jit-settrace
+Allow the jit to continue running when sys.settrace() is active, necessary to
+make coverage.py fast
+
+.. branch: remove-numpypy
+Remove lib_pypy/numpypy in favor of external numpy fork
+
+.. branch: jit-counter
+Tweak the jit counters: decay them at minor collection (actually
+only every 32 minor collection is enough). Should avoid the "memory
+leaks" observed in long-running processes, actually created by the
+jit compiling more and more rarely executed paths.
+
+.. branch: fix-trace-jit
+Fixed the usage of sys.settrace() with the JIT. Also made it so using
+sys.settrace() doesn't cause the GIL to be released on every single iteration.
+
+.. branch: rordereddict
+Implement OrderedDict in RPython

pypy/doc/whatsnew-head.rst

-======================
-What's new in PyPy 2.1
-======================
+=======================
+What's new in PyPy 2.2+
+=======================
 
-.. this is a revision shortly after release-2.1-beta
-.. startrev: 4eb52818e7c0
+.. this is a revision shortly after release-2.2.x
+.. startrev: 4cd1bc8b3111
 
-.. branch: sanitise_bytecode_dispatch
-Make PyPy's bytecode dispatcher easy to read, and less reliant on RPython
-magic. There is no functional change, though the removal of dead code leads
-to many fewer tests to execute.
+.. branch: release-2.2.x
 
-.. branch: fastjson
-Fast json decoder written in RPython, about 3-4x faster than the pure Python
-decoder which comes with the stdlib
-
-.. branch: improve-str2charp
-Improve the performance of I/O writing up to 15% by using memcpy instead of
-copying char-by-char in str2charp and get_nonmovingbuffer
-
-.. branch: flowoperators
-Simplify rpython/flowspace/ code by using more metaprogramming.  Create
-SpaceOperator class to gather static information about flow graph operations.
-
-.. branch: package-tk
-Adapt package.py script to compile CFFI tk extension. Add a --without-tk switch
-to optionally skip it.
-
-.. branch: distutils-cppldflags
-Copy CPython's implementation of customize_compiler, dont call split on
-environment variables, honour CFLAGS, CPPFLAGS, LDSHARED and LDFLAGS on Unices.
-
-.. branch: precise-instantiate
-When an RPython class is instantiated via an indirect call (that is, which
-class is being instantiated isn't known precisely) allow the optimizer to have
-more precise information about which functions can be called. Needed for Topaz.
-
-.. branch: ssl_moving_write_buffer
-
-.. branch: pythoninspect-fix
-Make PyPy respect PYTHONINSPECT variable set via os.putenv in the same process
-to start interactive prompt when the script execution finishes. This adds
-new __pypy__.os.real_getenv call that bypasses Python cache and looksup env
-in the underlying OS. Translatorshell now works on PyPy.
-
-.. branch: add-statvfs
-Added os.statvfs and os.fstatvfs
-
-.. branch: statvfs_tests
-Added some addition tests for statvfs.
-
-.. branch: ndarray-subtype
-Allow subclassing ndarray, i.e. matrix
-
-.. branch: ndarray-sort
-Implement ndarray in-place sorting (for numeric types, no non-native byte order)
-
-.. branch: pypy-pyarray
-Implement much of numpy's c api in cpyext, allows (slow) access to ndarray
-from c
-
-.. branch: kill-ootype
-
-.. branch: fast-slowpath
-Added an abstraction for functions with a fast and slow path in the JIT. This
-speeds up list.append() and list.pop().
-
-.. branch: curses_fixes
-
-.. branch: foldable-getarrayitem-indexerror
-Constant-fold reading out of constant tuples in PyPy.
-
-.. branch: mro-reorder-numpypy-str
-No longer delegate numpy string_ methods to space.StringObject, in numpy
-this works by kind of by accident. Support for merging the refactor-str-types
-branch
-
-.. branch: kill-typesystem
-Remove the "type system" abstraction, now that there is only ever one kind of
-type system used.
-
-.. branch: kill-gen-store-back-in
-Kills gen_store_back_in_virtualizable - should improve non-inlined calls by
-a bit
-
-.. branch: dotviewer-linewidth
-.. branch: reflex-support
-.. branch: numpypy-inplace-op
-.. branch: rewritten-loop-logging
-.. branch: no-release-gil
-.. branch: safe-win-mmap
-.. branch: boolean-indexing-cleanup
-.. branch: cpyext-best_base
-.. branch: cpyext-int
-.. branch: fileops2
-
-.. branch: nobold-backtrace
-Work on improving UnionError messages and stack trace displays.
-
-.. branch: improve-errors-again
-More improvements and refactorings of error messages.
-
-.. branch: improve-errors-again2
-Unbreak tests in rlib.
-
-.. branch: less-stringly-ops
-Use subclasses of SpaceOperation instead of SpaceOperator objects.
-Random cleanups in flowspace.
-
-.. branch: file-support-in-rpython
-make open() and friends rpython
-
-.. branch: incremental-gc
-Added the new incminimark GC which performs GC in incremental steps
-
-.. branch: fast_cffi_list_init
-fastpath for cffi.new("long[]")
-
-.. branch: remove-eval-frame
-remove a pointless abstraction
-
-.. branch: jit-settrace
-Allow the jit to continue running when sys.settrace() is active, necessary to
-make coverage.py fast
-
-.. branch: remove-numpypy
-Remove lib_pypy/numpypy in favor of external numpy fork
-
-.. branch: jit-counter
-Tweak the jit counters: decay them at minor collection (actually
-only every 32 minor collection is enough). Should avoid the "memory
-leaks" observed in long-running processes, actually created by the
-jit compiling more and more rarely executed paths.
-
-.. branch: fix-trace-jit
-Fixed the usage of sys.settrace() with the JIT. Also made it so using
-sys.settrace() doesn't cause the GIL to be released on every single iteration.
-
-.. branch: rordereddict
-Implement OrderedDict in RPython
+.. branch: numpy-newbyteorder
+Clean up numpy types, add newbyteorder functionality

pypy/module/_cffi_backend/__init__.py

+import sys
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload
 
         'FFI_DEFAULT_ABI': 'ctypefunc._get_abi(space, "FFI_DEFAULT_ABI")',
         'FFI_CDECL': 'ctypefunc._get_abi(space,"FFI_DEFAULT_ABI")',#win32 name
         }
+    if sys.platform == 'win32':
+        interpleveldefs['getwinerror'] = 'cerrno.getwinerror'
 
 for _name in ["RTLD_LAZY", "RTLD_NOW", "RTLD_GLOBAL", "RTLD_LOCAL",
               "RTLD_NODELETE", "RTLD_NOLOAD", "RTLD_DEEPBIND"]:

pypy/module/_cffi_backend/cerrno.py

 def set_errno(space, errno):
     ec = get_errno_container(space)
     ec._cffi_saved_errno = errno
+
+# ____________________________________________________________
+
+@unwrap_spec(code=int)
+def getwinerror(space, code=-1):
+    from rpython.rlib.rwin32 import FormatError
+    if code == -1:
+        ec = get_errno_container(space)
+        code = ec._cffi_saved_LastError
+    message = FormatError(code)
+    return space.newtuple([space.wrap(code), space.wrap(message)])

pypy/module/_cffi_backend/test/_backend_test_c.py

     #
     res = GetLastError()
     assert res == 42
+    #
+    SetLastError(2)
+    code, message = getwinerror()
+    assert code == 2
+    assert message == "The system cannot find the file specified"
+    #
+    code, message = getwinerror(1155)
+    assert code == 1155
+    assert message == ("No application is associated with the "
+                       "specified file for this operation")
 
 def test_nonstandard_integer_types():
     for typename in ['int8_t', 'uint8_t', 'int16_t', 'uint16_t', 'int32_t',

pypy/module/_pypyjson/interp_decoder.py

 import math
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib import rfloat
+from rpython.rlib import rfloat, runicode
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import unwrap_spec
             return # help the annotator to know that we'll never go beyond
                    # this point
         #
-        uchr = unichr(val)
+        uchr = runicode.code_to_unichr(val)     # may be a surrogate pair again
         utf8_ch = unicodehelper.encode_utf8(self.space, uchr)
         builder.append(utf8_ch)
         return i

pypy/module/_pypyjson/test/test__pypyjson.py

 # -*- encoding: utf-8 -*-
-import py, sys
 from pypy.module._pypyjson.interp_decoder import JSONDecoder
 
 def test_skip_whitespace():
 class AppTest(object):
     spaceconfig = {"objspace.usemodules._pypyjson": True}
 
-    def setup_class(cls):
-        cls.w_run_on_16bit = cls.space.wrap(sys.maxunicode == 65535)
-
     def test_raise_on_bytes(self):
         import _pypyjson
         raises(TypeError, _pypyjson.loads, b"42")
         raises(ValueError, """_pypyjson.loads('["extra comma",]')""")
 
     def test_unicode_surrogate_pair(self):
-        if self.run_on_16bit:
-            skip("XXX fix me or mark definitely skipped")
         import _pypyjson
         expected = 'z\U0001d120x'
         res = _pypyjson.loads('"z\\ud834\\udd20x"')

pypy/module/array/interp_array.py

 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.buffer import RWBuffer
-from pypy.interpreter.error import OperationError
-from pypy.interpreter.gateway import interp2app, unwrap_spec, interpindirect2app
-from pypy.interpreter.typedef import GetSetProperty, make_weakref_descr, TypeDef
+from pypy.interpreter.error import OperationError, operationerrfmt
+from pypy.interpreter.gateway import (
+    interp2app, interpindirect2app, unwrap_spec)
+from pypy.interpreter.typedef import (
+    GetSetProperty, TypeDef, make_weakref_descr)
 from pypy.objspace.std.floatobject import W_FloatObject
 
 
 def descr_typecode(space, self):
     return space.wrap(self.typecode)
 
-arr_eq_driver = jit.JitDriver(name='array_eq_driver', greens = ['comp_func'], reds = 'auto')
+arr_eq_driver = jit.JitDriver(name='array_eq_driver', greens=['comp_func'],
+                              reds='auto')
 EQ, NE, LT, LE, GT, GE = range(6)
 
 def compare_arrays(space, arr1, arr2, comp_op):
-    if (not isinstance(arr1, W_ArrayBase) or
-        not isinstance(arr2, W_ArrayBase)):
+    if not (isinstance(arr1, W_ArrayBase) and isinstance(arr2, W_ArrayBase)):
         return space.w_NotImplemented
     if comp_op == EQ and arr1.len != arr2.len:
         return space.w_False
             raise OperationError(self.space.w_ValueError, self.space.wrap(msg))
         oldlen = self.len
         new = len(s) / self.itemsize
+        if not new:
+            return
         self.setlen(oldlen + new)
         cbuf = self._charbuf_start()
-        copy_string_to_raw(llstr(s), rffi.ptradd(cbuf, oldlen * self.itemsize), 0, len(s))
+        copy_string_to_raw(llstr(s), rffi.ptradd(cbuf, oldlen * self.itemsize),
+                           0, len(s))
         self._charbuf_stop()
 
     @unwrap_spec(n=int)
     def descr_tofile(self, space, w_f):
         """ tofile(f)
 
-        Write all items (as machine values) to the file object f.  Also called as
-        write.
+        Write all items (as machine values) to the file object f.  Also
+        called as write.
         """
         w_s = self.descr_tobytes(space)
         space.call_method(w_f, 'write', w_s)
     def descr_byteswap(self, space):
         """ byteswap()
 
-        Byteswap all items of the array.  If the items in the array are not 1, 2,
-        4, or 8 bytes in size, RuntimeError is raised.
+        Byteswap all items of the array.  If the items in the array are
+        not 1, 2, 4, or 8 bytes in size, RuntimeError is raised.
         """
         if self.itemsize not in [1, 2, 4, 8]:
             msg = "byteswap not supported for this array"
 W_ArrayBase.typedef = TypeDef(
     'array',
     __new__ = interp2app(w_array),
-    __module__   = 'array',
+    __module__ = 'array',
 
     __len__ = interp2app(W_ArrayBase.descr_len),
     __eq__ = interp2app(W_ArrayBase.descr_eq),
 
 
 class TypeCode(object):
-    def __init__(self, itemtype, unwrap, canoverflow=False, signed=False, method='__int__'):
+    def __init__(self, itemtype, unwrap, canoverflow=False, signed=False,
+                 method='__int__'):
         self.itemtype = itemtype
         self.bytes = rffi.sizeof(itemtype)
         self.arraytype = lltype.Array(itemtype, hints={'nolength': True})
         if self.canoverflow:
             assert self.bytes <= rffi.sizeof(rffi.ULONG)
             if self.bytes == rffi.sizeof(rffi.ULONG) and not signed and \
-                   self.unwrap == 'int_w':
+                    self.unwrap == 'int_w':
                 # Treat this type as a ULONG
                 self.unwrap = 'bigint_w'
                 self.canoverflow = False
             try:
                 item = unwrap(w_item)
             except OperationError, e:
-                if isinstance(w_item, W_FloatObject): # Odd special case from cpython
+                if isinstance(w_item, W_FloatObject):
+                    # Odd special case from cpython
                     raise
                 if mytype.method != '' and e.match(space, space.w_TypeError):
                     try:
                         item = unwrap(space.call_method(w_item, mytype.method))
                     except OperationError:
                         msg = 'array item must be ' + mytype.unwrap[:-2]
-                        raise OperationError(space.w_TypeError, space.wrap(msg))
+                        raise operationerrfmt(space.w_TypeError, msg)
                 else:
                     raise
             if mytype.unwrap == 'bigint_w':
                         some = 0
                     self.allocated = size + some
                     if zero:
-                        new_buffer = lltype.malloc(mytype.arraytype,
-                                                   self.allocated, flavor='raw',
-                                                   add_memory_pressure=True,
-                                                   zero=True)
+                        new_buffer = lltype.malloc(
+                            mytype.arraytype, self.allocated, flavor='raw',
+                            add_memory_pressure=True, zero=True)
                     else:
-                        new_buffer = lltype.malloc(mytype.arraytype,
-                                                   self.allocated, flavor='raw',
-                                                   add_memory_pressure=True)
+                        new_buffer = lltype.malloc(
+                            mytype.arraytype, self.allocated, flavor='raw',
+                            add_memory_pressure=True)
                         for i in range(min(size, self.len)):
                             new_buffer[i] = self.buffer[i]
                 else:
             if i >= j:
                 return None
             oldbuffer = self.buffer
-            self.buffer = lltype.malloc(mytype.arraytype,
-                          max(self.len - (j - i), 0), flavor='raw',
-                          add_memory_pressure=True)
+            self.buffer = lltype.malloc(
+                mytype.arraytype, max(self.len - (j - i), 0), flavor='raw',
+                add_memory_pressure=True)
             if i:
                 rffi.c_memcpy(
                     rffi.cast(rffi.VOIDP, self.buffer),

pypy/module/array/test/test_array.py

             raises(OverflowError, a.append, 2 ** (8 * b))
 
     def test_fromstring(self):
+        a = self.array('l')
+        a.fromstring('')
+        assert not len(a)
+
         for t in 'bBhHiIlLfd':
             a = self.array(t)
             a.fromstring('\x00' * a.itemsize * 2)

pypy/module/cpyext/include/patchlevel.h

 #define PY_VERSION		"3.2.3"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "2.2.1-alpha0"
+#define PYPY_VERSION "2.3.0-alpha0"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */

pypy/module/micronumpy/arrayimpl/concrete.py

         return backstrides
 
     def getitem(self, index):
-        return self.dtype.getitem(self, index)
+        return self.dtype.itemtype.read(self, index, 0)
+
+    def getitem_bool(self, index):
+        return self.dtype.itemtype.read_bool(self, index, 0)
 
     def setitem(self, index, value):
-        self.dtype.setitem(self, index, value)
+        self.dtype.itemtype.store(self, index, 0, value)
 
     def setslice(self, space, arr):
         impl = arr.implementation
         loop.setslice(space, shape, self, impl)
 
     def get_size(self):
-        return self.size // self.dtype.itemtype.get_element_size()
+        return self.size // self.dtype.get_size()
 
     def get_storage_size(self):
         return self.size
     def create_dot_iter(self, shape, skip):
         r = calculate_dot_strides(self.get_strides(), self.get_backstrides(),
                                   shape, skip)
-        return iter.MultiDimViewIterator(self, self.dtype, self.start, r[0], r[1], shape)
+        return iter.MultiDimViewIterator(self, self.start, r[0], r[1], shape)
 
     def swapaxes(self, space, orig_arr, axis1, axis2):
         shape = self.get_shape()[:]
                 support.product(shape) > support.product(self.get_shape()):
             r = calculate_broadcast_strides(self.get_strides(),
                                             self.get_backstrides(),
-                                            self.get_shape(), shape, backward_broadcast)
-            return iter.MultiDimViewIterator(self, self.dtype, self.start, r[0], r[1], shape)
-
+                                            self.get_shape(), shape,
+                                            backward_broadcast)
+            return iter.MultiDimViewIterator(self, self.start,
+                                             r[0], r[1], shape)
         if not require_index:
             return iter.ConcreteArrayIterator(self)
-        else:
-            if len(self.get_shape()) == 1:
-                return iter.OneDimViewIterator(self, self.dtype, self.start,
-                        self.get_strides(), self.get_shape())
-            else:
-                return iter.MultiDimViewIterator(self, self.dtype, self.start,
-                        self.get_strides(), self.get_backstrides(), self.get_shape())
+        if len(self.get_shape()) == 1:
+            return iter.OneDimViewIterator(self, self.start,
+                                           self.get_strides(),
+                                           self.get_shape())
+        return iter.MultiDimViewIterator(self, self.start,
+                                         self.get_strides(),
+                                         self.get_backstrides(),
+                                         self.get_shape())
 
     def fill(self, box):
-        self.dtype.fill(self.storage, box, 0, self.size)
+        self.dtype.itemtype.fill(self.storage, self.dtype.get_size(),
+                                 box, 0, self.size, 0)
 
     def set_shape(self, space, orig_array, new_shape):
         strides, backstrides = support.calc_strides(new_shape, self.dtype,
         self.storage = parent.storage
         self.order = parent.order
         self.dtype = dtype
-        self.size = support.product(shape) * self.dtype.itemtype.get_element_size()
+        self.size = support.product(shape) * self.dtype.get_size()
         self.start = start
         self.orig_arr = orig_arr
 
                                             self.get_backstrides(),
                                             self.get_shape(), shape,
                                             backward_broadcast)
-            return iter.MultiDimViewIterator(self.parent, self.dtype,
-                                             self.start, r[0], r[1], shape)
+            return iter.MultiDimViewIterator(self, self.start,
+                                             r[0], r[1], shape)
         if len(self.get_shape()) == 1:
-            return iter.OneDimViewIterator(self.parent, self.dtype, self.start,
-                    self.get_strides(), self.get_shape())
-        return iter.MultiDimViewIterator(self.parent, self.dtype, self.start,
-                                    self.get_strides(),
-                                    self.get_backstrides(), self.get_shape())
+            return iter.OneDimViewIterator(self, self.start,
+                                           self.get_strides(),
+                                           self.get_shape())
+        return iter.MultiDimViewIterator(self, self.start,
+                                         self.get_strides(),
+                                         self.get_backstrides(),
+                                         self.get_shape())
 
     def set_shape(self, space, orig_array, new_shape):
         if len(self.get_shape()) < 2 or self.size == 0:

pypy/module/micronumpy/arrayimpl/scalar.py

                             )
 
     def descr_getitem(self, space, _, w_idx):
+        if space.isinstance_w(w_idx, space.w_tuple):
+            if space.len_w(w_idx) == 0:
+                return self.get_scalar_value()
         raise OperationError(space.w_IndexError,
                              space.wrap("scalars cannot be indexed"))
 

pypy/module/micronumpy/conversion_utils.py

 from pypy.interpreter.error import OperationError
 from pypy.module.micronumpy.constants import *
 
+
+def byteorder_converter(space, new_order):
+    endian = new_order[0]
+    if endian not in (NPY_BIG, NPY_LITTLE, NPY_NATIVE, NPY_IGNORE, NPY_SWAP):
+        ch = endian
+        if ch in ('b', 'B'):
+            endian = NPY_BIG
+        elif ch in ('l', 'L'):
+            endian = NPY_LITTLE
+        elif ch in ('n', 'N'):
+            endian = NPY_NATIVE
+        elif ch in ('i', 'I'):
+            endian = NPY_IGNORE
+        elif ch in ('s', 'S'):
+            endian = NPY_SWAP
+        else:
+            raise OperationError(space.w_ValueError, space.wrap(
+                "%s is an unrecognized byteorder" % new_order))
+    return endian
+
+
 def clipmode_converter(space, w_mode):
     if space.is_none(w_mode):
         return NPY_RAISE
     raise OperationError(space.w_TypeError,
                          space.wrap("clipmode not understood"))
 
+
 def order_converter(space, w_order, default):
     if space.is_none(w_order):
         return default
         else:
             raise OperationError(space.w_TypeError, space.wrap(
                 "order not understood"))
+
+
+def multi_axis_converter(space, w_axis, ndim):
+    if space.is_none(w_axis):
+        return [True] * ndim
+    out = [False] * ndim
+    if not space.isinstance_w(w_axis, space.w_tuple):
+        w_axis = space.newtuple([w_axis])
+    for w_item in space.fixedview(w_axis):
+        item = space.int_w(w_item)
+        axis = item
+        if axis < 0:
+            axis += ndim
+        if axis < 0 or axis >= ndim:
+            raise OperationError(space.w_ValueError, space.wrap(
+                "'axis' entry %d is out of bounds [-%d, %d)" %
+                (item, ndim, ndim)))
+        if out[axis]:
+            raise OperationError(space.w_ValueError, space.wrap(
+                "duplicate value in 'axis'"))
+        out[axis] = True
+    return out

pypy/module/micronumpy/interp_boxes.py

         return convert_to_array(space, w_values)
 
     @unwrap_spec(decimals=int)
-    def descr_round(self, space, decimals=0):
+    def descr_round(self, space, decimals=0, w_out=None):
+        if not space.is_none(w_out):
+            raise OperationError(space.w_NotImplementedError, space.wrap(
+                "out not supported"))
         v = self.convert_to(self.get_dtype(space))
         return self.get_dtype(space).itemtype.round(v, decimals)
 
     def descr_get_dtype(self, space):
         return self.get_dtype(space)
 
+    def descr_get_size(self, space):
+        return space.wrap(1)
+
     def descr_get_itemsize(self, space):
         return self.get_dtype(space).descr_get_itemsize(space)
 
     copy = interp2app(W_GenericBox.descr_copy),
 
     dtype = GetSetProperty(W_GenericBox.descr_get_dtype),
+    size = GetSetProperty(W_GenericBox.descr_get_size),
     itemsize = GetSetProperty(W_GenericBox.descr_get_itemsize),
     nbytes = GetSetProperty(W_GenericBox.descr_get_itemsize),
     shape = GetSetProperty(W_GenericBox.descr_get_shape),

pypy/module/micronumpy/interp_dtype.py

 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import (TypeDef, GetSetProperty,
-    interp_attrproperty, interp_attrproperty_w)
+                                      interp_attrproperty, interp_attrproperty_w)
 from pypy.module.micronumpy import types, interp_boxes, base
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import LONG_BIT, r_longlong, r_ulonglong
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib import jit
+from pypy.module.micronumpy.conversion_utils import byteorder_converter
 from pypy.module.micronumpy.constants import *
 
 
 def decode_w_dtype(space, w_dtype):
     if space.is_none(w_dtype):
         return None
-    return space.interp_w(W_Dtype,
-          space.call_function(space.gettypefor(W_Dtype), w_dtype))
+    return space.interp_w(
+        W_Dtype, space.call_function(space.gettypefor(W_Dtype), w_dtype))
+
 
 @jit.unroll_safe
 def dtype_agreement(space, w_arr_list, shape, out=None):
     out = base.W_NDimArray.from_shape(space, shape, dtype)
     return out
 
+
 class W_Dtype(W_Root):
-    _immutable_fields_ = ["itemtype?", "num", "kind", "name?", "char", "w_box_type", "byteorder", "float_type"]
+    _immutable_fields_ = ["itemtype?", "num", "kind", "name?", "char",
+                          "w_box_type", "byteorder", "size?", "float_type",
+                          "fields?", "fieldnames?", "shape", "subdtype", "base"]
 
     def __init__(self, itemtype, num, kind, name, char, w_box_type, byteorder=NPY_NATIVE,
-                 alternate_constructors=[], aliases=[], float_type=None,
+                 size=1, alternate_constructors=[], aliases=[], float_type=None,
                  fields=None, fieldnames=None, shape=[], subdtype=None):
         self.itemtype = itemtype
         self.num = num
         self.char = char
         self.w_box_type = w_box_type
         self.byteorder = byteorder
+        self.size = size
         self.alternate_constructors = alternate_constructors
         self.aliases = aliases
         self.float_type = float_type
     def coerce(self, space, w_item):
         return self.itemtype.coerce(space, self, w_item)
 
-    def getitem(self, arr, i):
-        item = self.itemtype.read(arr, i, 0)
-        return item
-
-    def getitem_bool(self, arr, i):
-        return self.itemtype.read_bool(arr, i, 0)
-
-    def setitem(self, arr, i, box):
-        self.itemtype.store(arr, i, 0, box)
-
-    def fill(self, storage, box, start, stop):
-        self.itemtype.fill(storage, self.get_size(), box, start, stop, 0)
-
     def is_int_type(self):
         return (self.kind == NPY_SIGNEDLTR or self.kind == NPY_UNSIGNEDLTR or
                 self.kind == NPY_GENBOOLLTR)
         return self.kind == NPY_COMPLEXLTR
 
     def is_float_type(self):
-        return (self.kind == NPY_FLOATINGLTR or self.float_type is not None)
+        return self.kind == NPY_FLOATINGLTR or self.kind == NPY_COMPLEXLTR
 
     def is_bool_type(self):
         return self.kind == NPY_GENBOOLLTR
         return self.byteorder in (NPY_NATIVE, NPY_NATBYTE)
 
     def get_size(self):
-        return self.itemtype.get_element_size()
+        return self.size * self.itemtype.get_element_size()
 
     def get_name(self):
         if self.char == 'S':
         return space.wrap("dtype('%s')" % self.get_name())
 
     def descr_get_itemsize(self, space):
-        return space.wrap(self.itemtype.get_element_size())
+        return space.wrap(self.get_size())
 
     def descr_get_alignment(self, space):
         return space.wrap(self.itemtype.alignment)
         if basic == NPY_UNICODELTR:
             size >>= 2
             endian = NPY_NATBYTE
-        elif size <= 1:
+        elif size // (self.size or 1) <= 1:
             endian = NPY_IGNORE
         else:
             endian = self.byteorder
             self.fields = None
         else:
             self.fields = {}
-            ofs_and_items = []
             size = 0
             for key in space.listview(w_fields):
                 value = space.getitem(w_fields, key)
                 offset = space.int_w(space.getitem(value, space.wrap(1)))
                 self.fields[space.str_w(key)] = offset, dtype
 
-                ofs_and_items.append((offset, dtype.itemtype))
-                size += dtype.itemtype.get_element_size()
+                size += dtype.get_size()
 
-            self.itemtype = types.RecordType(ofs_and_items, size)
-            self.name = "void" + str(8 * self.itemtype.get_element_size())
+            self.itemtype = types.RecordType()
+            self.size = size
+            self.name = "void" + str(8 * self.get_size())
 
     def descr_get_names(self, space):
         if self.fieldnames is None:
                         raise
                     break
 
+    def descr_get_hasobject(self, space):
+        return space.w_False
+
     def descr_getitem(self, space, w_item):
         if self.fields is None:
             raise OperationError(space.w_KeyError, space.wrap(
         w_class = space.type(self)
 
         kind = self.kind
-        elemsize = self.itemtype.get_element_size()
+        elemsize = self.get_size()
         builder_args = space.newtuple([space.wrap("%s%d" % (kind, elemsize)), space.wrap(0), space.wrap(1)])
 
         version = space.wrap(3)
         fields = space.getitem(w_data, space.wrap(4))
         self.set_fields(space, fields)
 
+    @unwrap_spec(new_order=str)
+    def descr_newbyteorder(self, space, new_order=NPY_SWAP):
+        newendian = byteorder_converter(space, new_order)
+        endian = self.byteorder
+        if endian != NPY_IGNORE:
+            if newendian == NPY_SWAP:
+                endian = NPY_OPPBYTE if self.is_native() else NPY_NATBYTE
+            elif newendian != NPY_IGNORE:
+                endian = newendian
+        itemtype = self.itemtype.__class__(endian in (NPY_NATIVE, NPY_NATBYTE))
+        return W_Dtype(itemtype, self.num, self.kind, self.name, self.char,
+                       self.w_box_type, endian, size=self.size)
+
+
 def dtype_from_list(space, w_lst):
     lst_w = space.listview(w_lst)
     fields = {}
     offset = 0
-    ofs_and_items = []
     fieldnames = []
     for w_elem in lst_w:
         size = 1
         if space.len_w(w_elem) == 3:
             w_fldname, w_flddesc, w_shape = space.fixedview(w_elem)
             if not base.issequence_w(space, w_shape):
-                w_shape = space.newtuple([w_shape,])
+                w_shape = space.newtuple([w_shape])
         else:
             w_fldname, w_flddesc = space.fixedview(w_elem, 2)
         subdtype = descr__new__(space, space.gettypefor(W_Dtype), w_flddesc, w_shape=w_shape)
             raise OperationError(space.w_ValueError, space.wrap("two fields with the same name"))
         assert isinstance(subdtype, W_Dtype)
         fields[fldname] = (offset, subdtype)
-        ofs_and_items.append((offset, subdtype.itemtype))
-        offset += subdtype.itemtype.get_element_size() * size
+        offset += subdtype.get_size() * size
         fieldnames.append(fldname)
-    itemtype = types.RecordType(ofs_and_items, offset)
-    return W_Dtype(itemtype, NPY_VOID, NPY_VOIDLTR, "void" + str(8 * itemtype.get_element_size()),
-                   NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), fields=fields,
-                   fieldnames=fieldnames)
+    itemtype = types.RecordType()
+    return W_Dtype(itemtype, NPY_VOID, NPY_VOIDLTR,
+                   "void" + str(8 * offset * itemtype.get_element_size()),
+                   NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox),
+                   fields=fields, fieldnames=fieldnames, size=offset)
+
 
 def dtype_from_dict(space, w_dict):
     raise OperationError(space.w_NotImplementedError, space.wrap(
         "dtype from dict"))
 
+
 def dtype_from_spec(space, name):
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "dtype from spec"))
 
+
 def descr__new__(space, w_subtype, w_dtype, w_align=None, w_copy=None, w_shape=None):
     # w_align and w_copy are necessary for pickling
     cache = get_dtype_cache(space)
 
-    if w_shape is not None and (space.isinstance_w(w_shape, space.w_int) or space.len_w(w_shape) > 0):
+    if w_shape is not None and (space.isinstance_w(w_shape, space.w_int) or
+                                space.len_w(w_shape) > 0):
         subdtype = descr__new__(space, w_subtype, w_dtype, w_align, w_copy)
         assert isinstance(subdtype, W_Dtype)
         size = 1
             dim = space.int_w(w_dim)
             shape.append(dim)
             size *= dim
-        return W_Dtype(types.VoidType(subdtype.itemtype.get_element_size() * size), NPY_VOID, NPY_VOIDLTR, "void" + str(8 * subdtype.itemtype.get_element_size() * size),
-                    NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox), shape=shape, subdtype=subdtype)
+        return W_Dtype(types.VoidType(), NPY_VOID, NPY_VOIDLTR,
+                       "void" + str(8 * subdtype.get_size() * size),
+                       NPY_VOIDLTR, space.gettypefor(interp_boxes.W_VoidBox),
+                       shape=shape, subdtype=subdtype,
+                       size=subdtype.get_size() * size)
 
     if space.is_none(w_dtype):
         return cache.w_float64dtype
             return cache.dtypes_by_name[name]
         except KeyError:
             pass
-        if name[0] in 'VSUc' or name[0] in '<>=' and name[1] in 'VSUc':
+        if name[0] in 'VSUc' or name[0] in '<>=|' and name[1] in 'VSUc':
             return variable_dtype(space, name)
         raise OperationError(space.w_TypeError, space.wrap(
-                       "data type %s not understood" % name))
+            "data type %s not understood" % name))
     elif space.isinstance_w(w_dtype, space.w_list):
         return dtype_from_list(space, w_dtype)
     elif space.isinstance_w(w_dtype, space.w_tuple):
 
     __reduce__ = interp2app(W_Dtype.descr_reduce),
     __setstate__ = interp2app(W_Dtype.descr_setstate),
+    newbyteorder = interp2app(W_Dtype.descr_newbyteorder),
 
     type = interp_attrproperty_w("w_box_type", cls=W_Dtype),
     kind = interp_attrproperty("kind", cls=W_Dtype),
     isnative = GetSetProperty(W_Dtype.descr_get_isnative),
     fields = GetSetProperty(W_Dtype.descr_get_fields),
     names = GetSetProperty(W_Dtype.descr_get_names),
+    hasobject = GetSetProperty(W_Dtype.descr_get_hasobject),
 )
 W_Dtype.typedef.acceptable_as_base_class = False
 
 
 def variable_dtype(space, name):
-    if name[0] in '<>=':
+    if name[0] in '<>=|':
         name = name[1:]
     char = name[0]
     if len(name) == 1:
         size = 1
 
     if char == NPY_STRINGLTR:
-        itemtype = types.StringType(size)
+        itemtype = types.StringType()
         basename = 'string'
         num = NPY_STRING
         w_box_type = space.gettypefor(interp_boxes.W_StringBox)
     elif char == NPY_VOIDLTR:
-        itemtype = types.VoidType(size)
+        itemtype = types.VoidType()
         basename = 'void'
         num = NPY_VOID
         w_box_type = space.gettypefor(interp_boxes.W_VoidBox)
     elif char == NPY_UNICODELTR:
-        itemtype = types.UnicodeType(size)
+        itemtype = types.UnicodeType()
         basename = 'unicode'
         num = NPY_UNICODE
         w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox)
         assert False
 
     return W_Dtype(itemtype, num, char,
-                   basename + str(8 * itemtype.get_element_size()),
-                   char, w_box_type)
+                   basename + str(8 * size * itemtype.get_element_size()),
+                   char, w_box_type, size=size)
+
 
 def new_string_dtype(space, size):
-    itemtype = types.StringType(size)
+    itemtype = types.StringType()
     return W_Dtype(
         itemtype,
+        size=size,
         num=NPY_STRING,
         kind=NPY_STRINGLTR,
-        name='string' + str(8 * itemtype.get_element_size()),
+        name='string' + str(8 * size * itemtype.get_element_size()),
         char=NPY_STRINGLTR,
         w_box_type = space.gettypefor(interp_boxes.W_StringBox),
     )
 
+
 def new_unicode_dtype(space, size):
-    itemtype = types.UnicodeType(size)
+    itemtype = types.UnicodeType()
     return W_Dtype(
         itemtype,
+        size=size,
         num=NPY_UNICODE,
         kind=NPY_UNICODELTR,
-        name='unicode' + str(8 * itemtype.get_element_size()),
+        name='unicode' + str(8 * size * itemtype.get_element_size()),
         char=NPY_UNICODELTR,
         w_box_type = space.gettypefor(interp_boxes.W_UnicodeBox),
     )
             name="int32",
             char=NPY_INTLTR,
             w_box_type=space.gettypefor(interp_boxes.W_Int32Box),
-       )
+        )
         self.w_uint32dtype = W_Dtype(
             types.UInt32(),
             num=NPY_UINT,
             alternate_constructors=[space.w_int,
                                     space.gettypefor(interp_boxes.W_IntegerBox),
                                     space.gettypefor(interp_boxes.W_SignedIntegerBox),
-                                   ],
+                                    ],
             aliases=['int'],
         )
         self.w_ulongdtype = W_Dtype(
             name="uint%d" % LONG_BIT,
             char=NPY_ULONGLTR,
             w_box_type=space.gettypefor(interp_boxes.W_ULongBox),
-            alternate_constructors=[ space.gettypefor(interp_boxes.W_UnsignedIntegerBox),
-                                   ],
+            alternate_constructors=[space.gettypefor(interp_boxes.W_UnsignedIntegerBox),
+                                    ],
             aliases=['uint'],
         )
         self.w_int64dtype = W_Dtype(
             w_box_type = space.gettypefor(interp_boxes.W_Float64Box),
             alternate_constructors=[space.w_float,
                                     space.gettypefor(interp_boxes.W_NumberBox),
-                                   ],
+                                    ],
             aliases=["float", "double"],
         )
         self.w_floatlongdtype = W_Dtype(
             float_type = self.w_floatlongdtype,
         )
         self.w_stringdtype = W_Dtype(
-            types.StringType(0),
+            types.StringType(),
+            size=0,
             num=NPY_STRING,
             kind=NPY_STRINGLTR,
             name='string',
             aliases=["str"],
         )
         self.w_unicodedtype = W_Dtype(
-            types.UnicodeType(0),
+            types.UnicodeType(),
+            size=0,
             num=NPY_UNICODE,
             kind=NPY_UNICODELTR,
             name='unicode',
             alternate_constructors=[space.w_unicode],
         )
         self.w_voiddtype = W_Dtype(
-            types.VoidType(0),
+            types.VoidType(),
+            size=0,
             num=NPY_VOID,
             kind=NPY_VOIDLTR,
             name='void',
             self.w_intpdtype, self.w_uintpdtype,
         ]
         self.float_dtypes_by_num_bytes = sorted(
-            (dtype.itemtype.get_element_size(), dtype)
+            (dtype.get_size(), dtype)
             for dtype in float_dtypes
         )
         self.dtypes_by_num = {}
         for dtype in reversed(self.builtin_dtypes):
             self.dtypes_by_num[dtype.num] = dtype
             self.dtypes_by_name[dtype.name] = dtype
-            can_name = dtype.kind + str(dtype.itemtype.get_element_size())
+            can_name = dtype.kind + str(dtype.get_size())
             self.dtypes_by_name[can_name] = dtype
             self.dtypes_by_name[NPY_NATBYTE + can_name] = dtype
             self.dtypes_by_name[NPY_NATIVE + can_name] = dtype
             new_name = NPY_OPPBYTE + can_name
             itemtype = type(dtype.itemtype)(False)
             self.dtypes_by_name[new_name] = W_Dtype(
-                itemtype,
-                dtype.num, dtype.kind, new_name, dtype.char, dtype.w_box_type,
-                byteorder=NPY_OPPBYTE, float_type=dtype.float_type)
+                itemtype, dtype.num, dtype.kind, new_name, dtype.char,
+                dtype.w_box_type, byteorder=NPY_OPPBYTE,
+                float_type=dtype.float_type)
             if dtype.kind != dtype.char:
                 can_name = dtype.char
                 self.dtypes_by_name[NPY_NATBYTE + can_name] = dtype
                 self.dtypes_by_name[NPY_NATIVE + can_name] = dtype
                 new_name = NPY_OPPBYTE + can_name
                 self.dtypes_by_name[new_name] = W_Dtype(
-                    itemtype,
-                    dtype.num, dtype.kind, new_name, dtype.char, dtype.w_box_type,
-                    byteorder=NPY_OPPBYTE, float_type=dtype.float_type)
+                    itemtype, dtype.num, dtype.kind, new_name, dtype.char,
+                    dtype.w_box_type, byteorder=NPY_OPPBYTE,
+                    float_type=dtype.float_type)
 
             for alias in dtype.aliases:
                 self.dtypes_by_name[alias] = dtype
         for k, v in typeinfo_partial.iteritems():
             space.setitem(w_typeinfo, space.wrap(k), space.gettypefor(v))
         for k, dtype in typeinfo_full.iteritems():
-            itemsize = dtype.itemtype.get_element_size()
+            itemsize = dtype.get_size()
             items_w = [space.wrap(dtype.char),
                        space.wrap(dtype.num),
-                       space.wrap(itemsize * 8), # in case of changing
+                       space.wrap(itemsize * 8),  # in case of changing
                        # number of bits per byte in the future
-                       space.wrap(itemsize / (2 if dtype.kind == NPY_COMPLEXLTR else 1) or 1)]
+                       space.wrap(itemsize /
+                                  (2 if dtype.kind == NPY_COMPLEXLTR else 1)
+                                  or 1)]
             if dtype.is_int_type():
                 if dtype.kind == NPY_GENBOOLLTR:
                     w_maxobj = space.wrap(1)
             space.setitem(w_typeinfo, space.wrap(k), space.newtuple(items_w))
         self.w_typeinfo = w_typeinfo
 
+
 def get_dtype_cache(space):
     return space.fromcache(DtypeCache)

pypy/module/micronumpy/interp_flagsobj.py

 class W_FlagsObject(W_Root):
     def __init__(self, arr):
         self.arr = arr
+        self.flags = 0
 
     def descr_get_contiguous(self, space):
         return space.w_True
         raise OperationError(space.w_KeyError, space.wrap(
             "Unknown flag"))
 
+    def eq(self, space, w_other):
+        if not isinstance(w_other, W_FlagsObject):
+            return False
+        return self.flags == w_other.flags
+
+    def descr_eq(self, space, w_other):
+        return space.wrap(self.eq(space, w_other))
+
+    def descr_ne(self, space, w_other):
+        return space.wrap(not self.eq(space, w_other))
+
 W_FlagsObject.typedef = TypeDef("flagsobj",
     __module__ = "numpy",
     __getitem__ = interp2app(W_FlagsObject.descr_getitem),
     __setitem__ = interp2app(W_FlagsObject.descr_setitem),
+    __eq__ = interp2app(W_FlagsObject.descr_eq),
+    __ne__ = interp2app(W_FlagsObject.descr_ne),
 
     contiguous = GetSetProperty(W_FlagsObject.descr_get_contiguous),
     c_contiguous = GetSetProperty(W_FlagsObject.descr_get_contiguous),

pypy/module/micronumpy/interp_numarray.py

 from rpython.rlib import jit
 from rpython.rlib.rstring import StringBuilder
 from pypy.module.micronumpy.arrayimpl.base import BaseArrayImplementation
-from pypy.module.micronumpy.conversion_utils import order_converter
+from pypy.module.micronumpy.conversion_utils import order_converter, multi_axis_converter
 from pypy.module.micronumpy.constants import *
 
 def _find_shape(space, w_size, dtype):
         return space.wrap(len(self.get_shape()))
 
     def descr_get_itemsize(self, space):
-        return space.wrap(self.get_dtype().itemtype.get_element_size())
+        return space.wrap(self.get_dtype().get_size())
 
     def descr_get_nbytes(self, space):
-        return space.wrap(self.get_size() * self.get_dtype().itemtype.get_element_size())
+        return space.wrap(self.get_size() * self.get_dtype().get_size())
 
     def descr_fill(self, space, w_value):
         self.fill(self.get_dtype().coerce(space, w_value))
 
     def descr_setitem(self, space, w_idx, w_value):
         if isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool_type():
-            try:
-                self.setitem_filter(space, w_idx, convert_to_array(space, w_value))
-            except ValueError, e:
-                raise OperationError(space.w_ValueError, space.wrap(str(e)))
+            self.setitem_filter(space, w_idx, convert_to_array(space, w_value))
             return
         try:
             self.implementation.descr_setitem(space, self, w_idx, w_value)
         # by converting nonnative byte order.
         if self.is_scalar():
             return space.wrap(0)
-        if not self.get_dtype().is_flexible_type():
-            s = self.get_dtype().name
-            if not self.get_dtype().is_native():
-                s = s[1:]
-            dtype = interp_dtype.get_dtype_cache(space).dtypes_by_name[s]
-        else:
-            dtype = self.get_dtype()
+        dtype = self.get_dtype().descr_newbyteorder(space, NPY_NATIVE)
         contig = self.implementation.astype(space, dtype)
         return contig.argsort(space, w_axis)
 
         raise OperationError(space.w_NotImplementedError, space.wrap(
             "itemset not implemented yet"))
 
-    @unwrap_spec(neworder=str)
-    def descr_newbyteorder(self, space, neworder):
-        raise OperationError(space.w_NotImplementedError, space.wrap(
-            "newbyteorder not implemented yet"))
+    @unwrap_spec(new_order=str)
+    def descr_newbyteorder(self, space, new_order=NPY_SWAP):
+        return self.descr_view(space,
+            self.get_dtype().descr_newbyteorder(space, new_order))
 
     @unwrap_spec(w_axis=WrappedDefault(None),
                  w_out=WrappedDefault(None))
         return self.implementation.sort(space, w_axis, w_order)
 
     def descr_squeeze(self, space, w_axis=None):
+        cur_shape = self.get_shape()
         if not space.is_none(w_axis):
-            raise OperationError(space.w_NotImplementedError, space.wrap(
-                "axis unsupported for squeeze"))
-        cur_shape = self.get_shape()
-        new_shape = [s for s in cur_shape if s != 1]
+            axes = multi_axis_converter(space, w_axis, len(cur_shape))
+            new_shape = []
+            for i in range(len(cur_shape)):
+                if axes[i]:
+                    if cur_shape[i] != 1:
+                        raise OperationError(space.w_ValueError, space.wrap(
+                            "cannot select an axis to squeeze out " \
+                            "which has size greater than one"))
+                else:
+                    new_shape.append(cur_shape[i])
+        else:
+            new_shape = [s for s in cur_shape if s != 1]
         if len(cur_shape) == len(new_shape):
             return self
         return wrap_impl(space, space.type(self), self,
     diagonal = interp2app(W_NDimArray.descr_diagonal),
     trace = interp2app(W_NDimArray.descr_trace),
     view = interp2app(W_NDimArray.descr_view),
+    newbyteorder = interp2app(W_NDimArray.descr_newbyteorder),
 
     ctypes = GetSetProperty(W_NDimArray.descr_get_ctypes), # XXX unimplemented
     __array_interface__ = GetSetProperty(W_NDimArray.descr_array_iface),
     # not an array or incorrect dtype
     shape, elems_w = find_shape_and_elems(space, w_object, dtype)
     if dtype is None or (
-                 dtype.is_str_or_unicode() and dtype.itemtype.get_size() < 1):
+                 dtype.is_str_or_unicode() and dtype.get_size() < 1):
         for w_elem in elems_w:
             dtype = interp_ufuncs.find_dtype_for_scalar(space, w_elem,
                                                         dtype)
 
         if dtype is None:
             dtype = interp_dtype.get_dtype_cache(space).w_float64dtype
-    if dtype.is_str_or_unicode() and dtype.itemtype.get_size() < 1:
+    if dtype.is_str_or_unicode() and dtype.get_size() < 1:
         # promote S0 -> S1, U0 -> U1
         dtype = interp_dtype.variable_dtype(space, dtype.char + '1')
     if ndmin > len(shape):

pypy/module/micronumpy/interp_support.py

     return space.wrap(a)
 
 def _fromstring_bin(space, s, count, length, dtype):
-    itemsize = dtype.itemtype.get_element_size()
+    itemsize = dtype.get_size()
     assert itemsize >= 0
     if count == -1:
         count = length / itemsize
             "string is smaller than requested size"))
 
     a = W_NDimArray.from_shape(space, [count], dtype=dtype)
-    loop.fromstring_loop(a, dtype, itemsize, s)
+    loop.fromstring_loop(space, a, dtype, itemsize, s)
     return space.wrap(a)
 
 @unwrap_spec(s=str, count=int, sep=str, w_dtype=WrappedDefault(None))

pypy/module/micronumpy/interp_ufuncs.py

         if dt2.is_record_type():
             return dt2
         if dt1.is_str_or_unicode():
-            if dt2.itemtype.get_element_size() >= \
-                    dt1.itemtype.get_element_size():
+            if dt2.get_size() >= dt1.get_size():
                 return dt2
             return dt1
         return dt2
             return interp_dtype.variable_dtype(space,
                                                'S%d' % space.len_w(w_obj))
         elif current_guess.num == NPY_STRING:
-            if current_guess.itemtype.get_size() < space.len_w(w_obj):
+            if current_guess.get_size() < space.len_w(w_obj):
                 return interp_dtype.variable_dtype(space,
                                                    'S%d' % space.len_w(w_obj))
         return current_guess

pypy/module/micronumpy/iter.py

         return [space.wrap(self.indexes[i]) for i in range(shapelen)]
 
 class ConcreteArrayIterator(base.BaseArrayIterator):
-    _immutable_fields_ = ['dtype', 'skip', 'size']
+    _immutable_fields_ = ['array', 'skip', 'size']
     def __init__(self, array):
         self.array = array
         self.offset = 0
-        self.dtype = array.dtype
-        self.skip = self.dtype.itemtype.get_element_size()
+        self.skip = array.dtype.get_size()