Commits

Ronan Lamy committed f91e272 Merge

hg merge default

Comments (0)

Files changed (92)

 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0
 0000000000000000000000000000000000000000 release-2.3.0
 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3
+32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
+32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1
+32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1
+0000000000000000000000000000000000000000 release-2.2=3.1
     Stian Andreassen
     Laurence Tratt
     Wanja Saatkamp
+    Ivan Sichmann Freitas
     Gerald Klix
     Mike Blume
     Oscar Nierstrasz
     Alejandro J. Cura
     Jacob Oscarson
     Travis Francis Athougies
+    Ryan Gonzalez
     Kristjan Valur Jonsson
+    Sebastian Pawluś
     Neil Blakey-Milner
     anatoly techtonik
     Lutz Paelike
     Michael Hudson-Doyle
     Anders Sigfridsson
     Yasir Suhail
+    rafalgalczynski@gmail.com
     Floris Bruynooghe
     Laurens Van Houtven
     Akira Li
     Zooko Wilcox-O Hearn
     Tomer Chachamu
     Christopher Groskopf
+    Asmo Soinio
+    Stefan Marr
     jiaaro
     opassembler.py
     Antony Lee

lib_pypy/_tkinter/license.terms

+This software is copyrighted by the Regents of the University of
+California, Sun Microsystems, Inc., and other parties.  The following
+terms apply to all files associated with the software unless explicitly
+disclaimed in individual files.
+
+The authors hereby grant permission to use, copy, modify, distribute,
+and license this software and its documentation for any purpose, provided
+that existing copyright notices are retained in all copies and that this
+notice is included verbatim in any distributions. No written agreement,
+license, or royalty fee is required for any of the authorized uses.
+Modifications to this software may be copyrighted by their authors
+and need not follow the licensing terms described here, provided that
+the new terms are clearly indicated on the first page of each file where
+they apply.
+
+IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY
+FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
+ARISING OUT OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY
+DERIVATIVES THEREOF, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT.  THIS SOFTWARE
+IS PROVIDED ON AN "AS IS" BASIS, AND THE AUTHORS AND DISTRIBUTORS HAVE
+NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR
+MODIFICATIONS.
+
+GOVERNMENT USE: If you are acquiring this software on behalf of the
+U.S. government, the Government shall have only "Restricted Rights"
+in the software and related documentation as defined in the Federal 
+Acquisition Regulations (FARs) in Clause 52.227.19 (c) (2).  If you
+are acquiring the software on behalf of the Department of Defense, the
+software shall be classified as "Commercial Computer Software" and the
+Government shall have only "Restricted Rights" as defined in Clause
+252.227-7013 (c) (1) of DFARs.  Notwithstanding the foregoing, the
+authors grant the U.S. Government and others acting in its behalf
+permission to use and distribute the software in accordance with the
+terms specified in this license.

pypy/config/pypyoption.py

             try:
                 for name in modlist:
                     __import__(name)
-            except (ImportError, CompilationError, py.test.skip.Exception), e:
+            except (ImportError, CompilationError, py.test.skip.Exception) as e:
                 errcls = e.__class__.__name__
                 raise Exception(
                     "The module %r is disabled\n" % (modname,) +

pypy/doc/coding-guide.rst

         while True:
             try:
                 w_key = space.next(w_iter)
-            except OperationError, e:
+            except OperationError as e:
                 if not e.match(space, space.w_StopIteration):
                     raise       # re-raise other app-level exceptions
                 break
 
 **objects**
 
-  Normal rules apply. Special methods are not honoured, except ``__init__``,
-  ``__del__`` and ``__iter__``.
+  Normal rules apply. The only special methods that are honoured are
+  ``__init__``, ``__del__``, ``__len__``, ``__getitem__``, ``__setitem__``,
+  ``__getslice__``, ``__setslice__``, and ``__iter__``. To handle slicing,
+  ``__getslice__`` and ``__setslice__`` must be used; using ``__getitem__`` and
+   ``__setitem__`` for slicing isn't supported. Additionally, using negative
+   indices for slicing is still not support, even when using ``__getslice__``.
 
 This layout makes the number of types to take care about quite limited.
 
 
     try:
         ...
-    except OperationError, e:
+    except OperationError as e:
         if not e.match(space, space.w_XxxError):
             raise
         ...

pypy/doc/config/translation.log.txt

 
 These must be enabled by setting the PYPYLOG environment variable.
 The exact set of features supported by PYPYLOG is described in
-pypy/translation/c/src/debug_print.h.
+rpython/translator/c/src/debug_print.h.

pypy/doc/contributor.rst

   Stian Andreassen
   Laurence Tratt
   Wanja Saatkamp
+  Ivan Sichmann Freitas
   Gerald Klix
   Mike Blume
   Oscar Nierstrasz
   Alejandro J. Cura
   Jacob Oscarson
   Travis Francis Athougies
+  Ryan Gonzalez
   Kristjan Valur Jonsson
+  Sebastian Pawluś
   Neil Blakey-Milner
   anatoly techtonik
   Lutz Paelike
   Michael Hudson-Doyle
   Anders Sigfridsson
   Yasir Suhail
+  rafalgalczynski@gmail.com
   Floris Bruynooghe
   Laurens Van Houtven
   Akira Li
   Zooko Wilcox-O Hearn
   Tomer Chachamu
   Christopher Groskopf
+  Asmo Soinio
+  Stefan Marr
   jiaaro
   opassembler.py
   Antony Lee

pypy/doc/index-of-release-notes.rst

 
 .. toctree::
 
+   release-2.3.1.rst
    release-2.3.0.rst
    release-2.2.1.rst
    release-2.2.0.rst

pypy/doc/index.rst

 
 * `FAQ`_: some frequently asked questions.
 
-* `Release 2.3.0`_: the latest official release
+* `Release 2.3.1`_: the latest official release
 
 * `PyPy Blog`_: news and status info about PyPy 
 
 .. _`Getting Started`: getting-started.html
 .. _`Papers`: extradoc.html
 .. _`Videos`: video-index.html
-.. _`Release 2.3.0`: http://pypy.org/download.html
+.. _`Release 2.3.1`: http://pypy.org/download.html
 .. _`speed.pypy.org`: http://speed.pypy.org
 .. _`RPython toolchain`: translation.html
 .. _`potential project ideas`: project-ideas.html

pypy/doc/man/pypy.1.rst

 ``PYPYLOG``
     If set to a non-empty value, enable logging, the format is:
 
-    *fname*
+    *fname* or *+fname*
         logging for profiling: includes all
         ``debug_start``/``debug_stop`` but not any nested
         ``debug_print``.
         *fname* can be ``-`` to log to *stderr*.
-        Note that using a : in fname is a bad idea, Windows
-        users, beware.
+        The *+fname* form can be used if there is a *:* in fname
 
     ``:``\ *fname*
         Full logging, including ``debug_print``.

pypy/doc/release-2.3.1.rst

+=================================================
+PyPy 2.3.1 - Terrestrial Arthropod Trap Revisited
+=================================================
+
+We're pleased to announce PyPy 2.3.1, a feature-and-bugfix improvement over our
+recent release last month.
+
+This release contains several bugfixes and enhancements.
+
+You can download the PyPy 2.3.1 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project, and for those who donate to our three sub-projects.
+We've shown quite a bit of progress 
+but we're slowly running out of funds.
+Please consider donating more, or even better convince your employer to donate,
+so we can finish those projects!  The three sub-projects are:
+
+* `Py3k`_ (supporting Python 3.x): the release PyPy3 2.3 is imminent.
+
+* `STM`_ (software transactional memory): a preview will be released very soon,
+  once we fix a few bugs
+
+* `NumPy`_ which requires installation of our fork of upstream numpy, available `on bitbucket`_
+
+.. _`Py3k`: http://pypy.org/py3donate.html
+.. _`STM`: http://pypy.org/tmdonate2.html
+.. _`NumPy`: http://pypy.org/numpydonate.html
+.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy   
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy 2.3 and cpython 2.7.x`_ performance comparison;
+note that cpython's speed has not changed since 2.7.2)
+due to its integrated tracing JIT compiler.
+
+This release supports x86 machines running Linux 32/64, Mac OS X 64, Windows,
+and OpenBSD,
+as well as newer ARM hardware (ARMv6 or ARMv7, with VFPv3) running Linux. 
+
+While we support 32 bit python on Windows, work on the native Windows 64
+bit python is still stalling, we would welcome a volunteer
+to `handle that`_.
+
+.. _`pypy 2.3 and cpython 2.7.x`: http://speed.pypy.org
+.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation
+
+Highlights
+==========
+
+Issues with the 2.3 release were resolved after being reported by users to
+our new issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at
+#pypy. Here is a summary of the user-facing changes;
+for more information see `whats-new`_:
+
+* The built-in ``struct`` module was renamed to ``_struct``, solving issues
+  with IDLE and other modules.
+
+* Support for compilation with gcc-4.9
+
+* A rewrite of packaging.py which produces our downloadable packages to
+  modernize command line argument handling and to document third-party
+  contributions in our LICENSE file
+
+* A CFFI-based version of the gdbm module is now included in our downloads
+
+* Many issues were resolved_ since the 2.3 release on May 8
+
+.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html
+.. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved
+Please try it out and let us know what you think. We especially welcome
+success stories, we know you are using PyPy, please tell us about it!
+
+Cheers
+
+The PyPy Team
+

pypy/doc/whatsnew-2.3.1.rst

 
 Support compilation with gcc-4.9
 
-Fixes for issues #1769, #1764, #1762, #1752
+Added support for the stdlib gdbm module via cffi
 
+Annotator cleanups
+
+.. branch: release-2.3.x
+
+.. branch: unify-call-ops
+
+.. branch packaging
+Use argparse for packaging.py, and add third-party components to LICENSE file.
+Also mention that gdbm is GPL.
+Do not crash the packaging process on failure in CFFI or license-building,
+rather complete the build step and return -1.

pypy/doc/whatsnew-head.rst

 =======================
 
 .. this is a revision shortly after release-2.3.x
-.. startrev: b2cc67adbaad
+.. startrev: ca9b7cf02cf4
 
-Added support for the stdlib gdbm module via cffi
+.. branch: fix-bytearray-complexity
+Bytearray operations no longer copy the bytearray unnecessarily
 
-Annotator cleanups
-
-.. branch: release-2.3.x
-
-.. branch: unify-call-ops
+Added support for ``__getitem__``, ``__setitem__``, ``__getslice__``,
+``__setslice__``,  and ``__len__`` to RPython

pypy/doc/windows.rst

 64bit Windows.  See at the end of this page for what is missing
 for a full 64bit translation.
 
-To build pypy-c you need a C compiler.  Microsoft Visual Studio is
-preferred, but can also use the mingw32 port of gcc.
+To build pypy-c you need a working python environment, and a C compiler.
+It is possible to translate with a CPython 2.6 or later, but this is not
+the preferred way, because it will take a lot longer to run – depending
+on your architecture, between two and three times as long. So head to 
+`our downloads`_ and get the latest stable version.
+
+Microsoft Visual Studio is preferred as a compiler, but there are reports 
+of success with the mingw32 port of gcc.
 
 
 Translating PyPy with Visual Studio
 **Note:** PyPy is currently not supported for 64 bit Windows, and translation
 will fail in this case.
 
-The compiler is all you need to build pypy-c, but it will miss some
+Python and a C compiler are all you need to build pypy, but it will miss some
 modules that relies on third-party libraries.  See below how to get
 and build them.
 
+Please see the `non-windows instructions`_ for more information, especially note
+that translation is RAM-hungry. A standard translation requires around 4GB, so
+special preparations are necessary, or you may want to use the method in the
+notes of the `build instructions`_ to reduce memory usage at the price of a
+slower translation::
+
+    set PYPY_GC_MAX_DELTA=200MB
+    pypy --jit loop_longevity=300 ../../rpython/bin/rpython -Ojit targetpypystandalone
+    set PYPY_GC_MAX_DELTA=
+
 Preping Windows for the Large Build
 -----------------------------------
 
 
 Then you need to execute::
 
-    editbin /largeaddressaware pypy.exe
+    editbin /largeaddressaware translator.exe
 
-on the pypy.exe file you compiled.
+where ``translator.exe`` is the pypy.exe or cpython.exe you will use to 
+translate with. 
 
 Installing external packages
 ----------------------------
 .. _`msys for mingw`: http://sourceforge.net/projects/mingw-w64/files/External%20binary%20packages%20%28Win64%20hosted%29/MSYS%20%2832-bit%29   
 .. _`libffi source files`: http://sourceware.org/libffi/
 .. _`RPython translation toolchain`: translation.html
-
+.. _`our downloads`: http://pypy.org/download.html   
+.. _`non-windows instructions`: getting-started-python.html#translating-the-pypy-python-interpreter
+.. _`build instructions`: http://pypy.org/download.html#building-from-source
 
 What is missing for a full 64-bit translation
 ---------------------------------------------

pypy/module/__builtin__/app_io.py

 """
 
 import sys
+from _ast import PyCF_ACCEPT_NULL_BYTES
 
 def execfile(filename, glob=None, loc=None):
     """execfile(filename[, globals[, locals]])
     finally:
         f.close()
     #Don't exec the source directly, as this loses the filename info
-    co = compile(source.rstrip()+"\n", filename, 'exec')
+    co = compile(source.rstrip()+"\n", filename, 'exec',
+                 PyCF_ACCEPT_NULL_BYTES)
     exec co in glob, loc
 
 def _write_prompt(stdout, prompt):

pypy/module/__builtin__/test/test_builtin.py

 import sys
 
+from rpython.tool.udir import udir
+
 class AppTestBuiltinApp:
     def setup_class(cls):
+        space = cls.space
         class X(object):
             def __eq__(self, other):
                 raise OverflowError
         try:
             d[X()]
         except OverflowError:
-            cls.w_sane_lookup = cls.space.wrap(True)
+            cls.w_sane_lookup = space.wrap(True)
         except KeyError:
-            cls.w_sane_lookup = cls.space.wrap(False)
+            cls.w_sane_lookup = space.wrap(False)
         # starting with CPython 2.6, when the stack is almost out, we
         # can get a random error, instead of just a RuntimeError.
         # For example if an object x has a __getattr__, we can get
         # AttributeError if attempting to call x.__getattr__ runs out
         # of stack.  That's annoying, so we just work around it.
         if cls.runappdirect:
-            cls.w_safe_runtimerror = cls.space.wrap(True)
+            cls.w_safe_runtimerror = space.wrap(True)
         else:
-            cls.w_safe_runtimerror = cls.space.wrap(sys.version_info < (2, 6))
+            cls.w_safe_runtimerror = space.wrap(sys.version_info < (2, 6))
+
+        emptyfile = udir.join('emptyfile.py')
+        emptyfile.write('')
+        nullbytes = udir.join('nullbytes.py')
+        nullbytes.write('#abc\x00def\n')
+        cls.w_emptyfile = space.wrap(str(emptyfile))
+        cls.w_nullbytes = space.wrap(str(nullbytes))
 
     def test_builtin_names(self):
         import __builtin__
         assert setattr(x, 'x', 11) == None
         assert delattr(x, 'x') == None
         # To make this test, we need autopath to work in application space.
-        #self.assertEquals(execfile('emptyfile.py'), None)
+        assert execfile(self.emptyfile) == None
 
     def test_divmod(self):
         assert divmod(15,10) ==(1,5)
         assert firstlineno == 2
 
     def test_compile_null_bytes(self):
-        import _ast
         raises(TypeError, compile, '\x00', 'mymod', 'exec', 0)
-        raises(SyntaxError, compile, '\x00', 'mymod', 'exec',
-               _ast.PyCF_ACCEPT_NULL_BYTES)
         src = "#abc\x00def\n"
         raises(TypeError, compile, src, 'mymod', 'exec')
         raises(TypeError, compile, src, 'mymod', 'exec', 0)
-        compile(src, 'mymod', 'exec', _ast.PyCF_ACCEPT_NULL_BYTES)  # works
+        execfile(self.nullbytes) # works
+
+    def test_compile_null_bytes_flag(self):
+        try:
+            from _ast import PyCF_ACCEPT_NULL_BYTES
+        except ImportError:
+            skip('PyPy only (requires _ast.PyCF_ACCEPT_NULL_BYTES)')
+        raises(SyntaxError, compile, '\x00', 'mymod', 'exec',
+               PyCF_ACCEPT_NULL_BYTES)
+        src = "#abc\x00def\n"
+        compile(src, 'mymod', 'exec', PyCF_ACCEPT_NULL_BYTES)  # works
 
     def test_print_function(self):
         import __builtin__
 
 class TestInternal:
     def test_execfile(self, space):
-        from rpython.tool.udir import udir
         fn = str(udir.join('test_execfile'))
         f = open(fn, 'w')
         print >>f, "i=42"

pypy/module/micronumpy/tool/numready/page.html

         <h3>numpy compatability test results, generated automatically by running</br>
         <code>pypy/module/micronumpy/tool/numready/main.py &lt;path-to-latest-pypy&gt;</code></h3>
         <h3>Overall: {{ msg }}</h3>
+        <h3><i><b>Warning:</b> a positive result does not mean the function is actually working!  It only means that the function/module/constant is present.  It may be missing other things.</h3>
         <table>
             <thead>
                 <tr>

pypy/module/pypyjit/test_pypy_c/test_string.py

         log = self.run(main, [1000])
         assert log.result == main(1000)
         loop, = log.loops_by_filename(self.filepath)
+        # NB: since the stringbuilder2-perf branch we get more operations than
+        # before, but a lot less branches that might fail randomly.
         assert loop.match("""
-            i7 = int_gt(i4, 0)
-            guard_true(i7, descr=...)
+            i100 = int_gt(i95, 0)
+            guard_true(i100, descr=...)
             guard_not_invalidated(descr=...)
-            p9 = call(ConstClass(ll_int2dec__Signed), i4, descr=<Callr . i EF=3>)
+            p101 = call(ConstClass(ll_int2dec__Signed), i95, descr=<Callr . i EF=3>)
             guard_no_exception(descr=...)
-            i10 = strlen(p9)
-            i11 = int_is_true(i10)
-            guard_true(i11, descr=...)
-            i13 = strgetitem(p9, 0)
-            i15 = int_eq(i13, 45)
-            guard_false(i15, descr=...)
-            i17 = int_neg(i10)
-            i19 = int_gt(i10, 23)
-            guard_false(i19, descr=...)
-            p21 = newstr(23)
-            copystrcontent(p9, p21, 0, 0, i10)
-            i25 = int_add(1, i10)
-            i26 = int_gt(i25, 23)
-            guard_false(i26, descr=...)
-            strsetitem(p21, i10, 32)
-            i30 = int_add(i10, i25)
-            i31 = int_gt(i30, 23)
-            guard_false(i31, descr=...)
-            copystrcontent(p9, p21, 0, i25, i10)
-            i33 = int_lt(i30, 23)
-            guard_true(i33, descr=...)
-            p35 = call(ConstClass(ll_shrink_array__rpy_stringPtr_Signed), p21, i30, descr=<Callr . ri EF=4 OS=3>)
+            i102 = strlen(p101)
+            i103 = int_is_true(i102)
+            guard_true(i103, descr=...)
+            i104 = strgetitem(p101, 0)
+            i105 = int_eq(i104, 45)
+            guard_false(i105, descr=...)
+            i106 = int_neg(i102)
+            i107 = int_gt(i102, 23)
+            p108 = new(descr=<SizeDescr .+>)
+            p110 = newstr(23)
+            setfield_gc(..., descr=<Field. stringbuilder.+>)
+            setfield_gc(..., descr=<Field. stringbuilder.+>)
+            setfield_gc(..., descr=<Field. stringbuilder.+>)
+            cond_call(i107, ConstClass(stringbuilder_append_overflow__stringbuilderPtr_rpy_stringPtr_Signed), p108, p101, i102, descr=<Callv 0 rri EF=4>)
             guard_no_exception(descr=...)
-            i37 = strlen(p35)
-            i38 = int_add_ovf(i5, i37)
+            i111 = getfield_gc(p108, descr=<FieldS stringbuilder.skip .+>)
+            i112 = int_sub(i102, i111)
+            i113 = getfield_gc(p108, descr=<FieldS stringbuilder.current_pos .+>)
+            p114 = getfield_gc(p108, descr=<FieldP stringbuilder.current_buf .+>)
+            copystrcontent(p101, p114, i111, i113, i112)
+            i115 = int_add(i113, i112)
+            i116 = getfield_gc(p108, descr=<FieldS stringbuilder.current_end .+>)
+            setfield_gc(p108, i115, descr=<FieldS stringbuilder.current_pos .+>)
+            i117 = int_eq(i115, i116)
+            cond_call(i117, ConstClass(stringbuilder_grow__stringbuilderPtr_Signed), p108, 1, descr=<Callv 0 ri EF=4>)
+            guard_no_exception(descr=...)
+            i118 = getfield_gc(p108, descr=<FieldS stringbuilder.current_pos .+>)
+            i119 = int_add(i118, 1)
+            p120 = getfield_gc(p108, descr=<FieldP stringbuilder.current_buf .+>)
+            strsetitem(p120, i118, 32)
+            i121 = getfield_gc(p108, descr=<FieldS stringbuilder.current_end .+>)
+            i122 = int_sub(i121, i119)
+            setfield_gc(..., descr=<FieldS stringbuilder.+>)
+            setfield_gc(..., descr=<FieldS stringbuilder.+>)
+            i123 = int_gt(i102, i122)
+            cond_call(i123, ConstClass(stringbuilder_append_overflow__stringbuilderPtr_rpy_stringPtr_Signed), p108, p101, i102, descr=<Callv 0 rri EF=4>)
+            guard_no_exception(descr=...)
+            i124 = getfield_gc(p108, descr=<FieldS stringbuilder.skip .+>)
+            i125 = int_sub(i102, i124)
+            i126 = getfield_gc(p108, descr=<FieldS stringbuilder.current_pos .+>)
+            p127 = getfield_gc(p108, descr=<FieldP stringbuilder.current_buf .+>)
+            copystrcontent(p101, p127, i124, i126, i125)
+            i128 = int_add(i126, i125)
+            setfield_gc(p108, i128, descr=<FieldS stringbuilder.current_pos .+>)
+            p135 = call(..., descr=<Callr . r EF=4)     # ll_build
+            guard_no_exception(descr=...)
+            i136 = strlen(p135)
+            i137 = int_add_ovf(i92, i136)
             guard_no_overflow(descr=...)
-            i40 = int_sub(i4, 1)
+            i138 = int_sub(i95, 1)
             --TICK--
             jump(..., descr=...)
         """)

pypy/objspace/std/bytearrayobject.py

 """The builtin bytearray implementation"""
 
 from rpython.rlib.objectmodel import (
-    import_from_mixin, newlist_hint, resizelist_hint)
+    import_from_mixin, newlist_hint, resizelist_hint, specialize)
 from rpython.rlib.buffer import Buffer
-from rpython.rlib.rstring import StringBuilder
+from rpython.rlib.rstring import StringBuilder, ByteListBuilder
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.signature import Signature
 from pypy.objspace.std.sliceobject import W_SliceObject
 from pypy.objspace.std.stdtypedef import StdTypeDef
-from pypy.objspace.std.stringmethods import StringMethods
+from pypy.objspace.std.stringmethods import StringMethods, _get_buffer
+from pypy.objspace.std.bytesobject import W_BytesObject
 from pypy.objspace.std.util import get_positive_index
 
 NON_HEX_MSG = "non-hexadecimal number found in fromhex() arg at position %d"
 class W_BytearrayObject(W_Root):
     import_from_mixin(StringMethods)
 
-    def __init__(w_self, data):
-        w_self.data = data
+    def __init__(self, data):
+        self.data = data
 
-    def __repr__(w_self):
+    def __repr__(self):
         """representation for debugging purposes"""
-        return "%s(%s)" % (w_self.__class__.__name__, ''.join(w_self.data))
+        return "%s(%s)" % (self.__class__.__name__, ''.join(self.data))
 
     def buffer_w(self, space, flags):
         return BytearrayBuffer(self.data, False)
         return ''.join(self.data)
 
     def _new(self, value):
-        return W_BytearrayObject(_make_data(value))
+        return W_BytearrayObject(value)
+
+    def _new_from_buffer(self, buffer):
+        return W_BytearrayObject([buffer[i] for i in range(len(buffer))])
 
     def _new_from_list(self, value):
         return W_BytearrayObject(value)
             raise oefmt(space.w_IndexError, "bytearray index out of range")
         return space.wrap(ord(character))
 
-    _val = charbuf_w
+    def _val(self, space):
+        return self.data
+
+    @staticmethod
+    def _use_rstr_ops(space, w_other):
+        return False
 
     @staticmethod
     def _op_val(space, w_other):
         assert len(char) == 1
         return str(char)[0]
 
-    _builder = StringBuilder
+    def _multi_chr(self, char):
+        return [char]
+
+    @staticmethod
+    def _builder(size=100):
+        return ByteListBuilder(size)
 
     def _newlist_unwrapped(self, space, res):
-        return space.newlist([W_BytearrayObject(_make_data(i)) for i in res])
+        return space.newlist([W_BytearrayObject(i) for i in res])
 
     def _isupper(self, ch):
         return ch.isupper()
         return space.wrap(''.join(self.data))
 
     def descr_eq(self, space, w_other):
+        if isinstance(w_other, W_BytearrayObject):
+            return space.newbool(self.data == w_other.data)
+
         try:
-            res = self._val(space) == self._op_val(space, w_other)
+            buffer = _get_buffer(space, w_other)
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 return space.w_NotImplemented
             raise
-        return space.newbool(res)
+
+        value = self._val(space)
+        buffer_len = buffer.getlength()
+
+        if len(value) != buffer_len:
+            return space.newbool(False)
+
+        min_length = min(len(value), buffer_len)
+        return space.newbool(_memcmp(value, buffer, min_length) == 0)
 
     def descr_ne(self, space, w_other):
+        if isinstance(w_other, W_BytearrayObject):
+            return space.newbool(self.data != w_other.data)
+
         try:
-            res = self._val(space) != self._op_val(space, w_other)
+            buffer = _get_buffer(space, w_other)
         except OperationError as e:
             if e.match(space, space.w_TypeError):
                 return space.w_NotImplemented
             raise
-        return space.newbool(res)
+
+        value = self._val(space)
+        buffer_len = buffer.getlength()
+
+        if len(value) != buffer_len:
+            return space.newbool(True)
+
+        min_length = min(len(value), buffer_len)
+        return space.newbool(_memcmp(value, buffer, min_length) != 0)
+
+    def _comparison_helper(self, space, w_other):
+        value = self._val(space)
+
+        if isinstance(w_other, W_BytearrayObject):
+            other = w_other.data
+            other_len = len(other)
+            cmp = _memcmp(value, other, min(len(value), len(other)))
+        elif isinstance(w_other, W_BytesObject):
+            other = self._op_val(space, w_other)
+            other_len = len(other)
+            cmp = _memcmp(value, other, min(len(value), len(other)))
+        else:
+            try:
+                buffer = _get_buffer(space, w_other)
+            except OperationError as e:
+                if e.match(space, space.w_TypeError):
+                    return False, 0, 0
+                raise
+            other_len = len(buffer)
+            cmp = _memcmp(value, buffer, min(len(value), len(buffer)))
+
+        return True, cmp, other_len
 
     def descr_lt(self, space, w_other):
-        try:
-            res = self._val(space) < self._op_val(space, w_other)
-        except OperationError as e:
-            if e.match(space, space.w_TypeError):
-                return space.w_NotImplemented
-            raise
-        return space.newbool(res)
+        success, cmp, other_len = self._comparison_helper(space, w_other)
+        if not success:
+            return space.w_NotImplemented
+        return space.newbool(cmp < 0 or (cmp == 0 and self._len() < other_len))
 
     def descr_le(self, space, w_other):
-        try:
-            res = self._val(space) <= self._op_val(space, w_other)
-        except OperationError as e:
-            if e.match(space, space.w_TypeError):
-                return space.w_NotImplemented
-            raise
-        return space.newbool(res)
+        success, cmp, other_len = self._comparison_helper(space, w_other)
+        if not success:
+            return space.w_NotImplemented
+        return space.newbool(cmp < 0 or (cmp == 0 and self._len() <= other_len))
 
     def descr_gt(self, space, w_other):
-        try:
-            res = self._val(space) > self._op_val(space, w_other)
-        except OperationError as e:
-            if e.match(space, space.w_TypeError):
-                return space.w_NotImplemented
-            raise
-        return space.newbool(res)
+        success, cmp, other_len = self._comparison_helper(space, w_other)
+        if not success:
+            return space.w_NotImplemented
+        return space.newbool(cmp > 0 or (cmp == 0 and self._len() > other_len))
 
     def descr_ge(self, space, w_other):
-        try:
-            res = self._val(space) >= self._op_val(space, w_other)
-        except OperationError as e:
-            if e.match(space, space.w_TypeError):
-                return space.w_NotImplemented
-            raise
-        return space.newbool(res)
+        success, cmp, other_len = self._comparison_helper(space, w_other)
+        if not success:
+            return space.w_NotImplemented
+        return space.newbool(cmp > 0 or (cmp == 0 and self._len() >= other_len))
 
     def descr_iter(self, space):
         return space.newseqiter(self)
     def descr_inplace_add(self, space, w_other):
         if isinstance(w_other, W_BytearrayObject):
             self.data += w_other.data
+            return self
+
+        if isinstance(w_other, W_BytesObject):
+            self._inplace_add(self._op_val(space, w_other))
         else:
-            self.data += self._op_val(space, w_other)
+            self._inplace_add(_get_buffer(space, w_other))
         return self
 
+    @specialize.argtype(1)
+    def _inplace_add(self, other):
+        for i in range(len(other)):
+            self.data.append(other[i])
+
     def descr_inplace_mul(self, space, w_times):
         try:
             times = space.getindex_w(w_times, space.w_OverflowError)
         if space.isinstance_w(w_sub, space.w_int):
             char = space.int_w(w_sub)
             return _descr_contains_bytearray(self.data, space, char)
+
         return self._StringMethods_descr_contains(space, w_sub)
 
+    def descr_add(self, space, w_other):
+        if isinstance(w_other, W_BytearrayObject):
+            return self._new(self.data + w_other.data)
+
+        if isinstance(w_other, W_BytesObject):
+            return self._add(self._op_val(space, w_other))
+
+        try:
+            buffer = _get_buffer(space, w_other)
+        except OperationError as e:
+            if e.match(space, space.w_TypeError):
+                return space.w_NotImplemented
+            raise
+        return self._add(buffer)
+
+    @specialize.argtype(1)
+    def _add(self, other):
+        return self._new(self.data + [other[i] for i in range(len(other))])
+
     def descr_reverse(self, space):
         self.data.reverse()
 
 
+
 # ____________________________________________________________
 # helpers for slow paths, moved out because they contain loops
 
 
     def setitem(self, index, char):
         self.data[index] = char
+
+
+@specialize.argtype(1)
+def _memcmp(selfvalue, buffer, length):
+    for i in range(length):
+        if selfvalue[i] < buffer[i]:
+            return -1
+        if selfvalue[i] > buffer[i]:
+            return 1
+    return 0

pypy/objspace/std/bytesobject.py

     _immutable_fields_ = ['_value']
 
     def __init__(self, str):
+        assert str is not None
         self._value = str
 
     def __repr__(self):
     _val = str_w
 
     @staticmethod
+    def _use_rstr_ops(space, w_other):
+        from pypy.objspace.std.unicodeobject import W_UnicodeObject
+        return (isinstance(w_other, W_BytesObject) or
+                isinstance(w_other, W_UnicodeObject))
+
+    @staticmethod
     def _op_val(space, w_other):
         try:
             return space.str_w(w_other)

pypy/objspace/std/stringmethods.py

 """Functionality shared between bytes/bytearray/unicode"""
 
 from rpython.rlib import jit
-from rpython.rlib.objectmodel import specialize
+from rpython.rlib.objectmodel import specialize, newlist_hint
 from rpython.rlib.rarithmetic import ovfcheck
-from rpython.rlib.rstring import endswith, replace, rsplit, split, startswith
+from rpython.rlib.rstring import (
+    find, rfind, count, endswith, replace, rsplit, split, startswith)
+from rpython.rlib.buffer import Buffer
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import WrappedDefault, unwrap_spec
             space, lenself, w_start, w_end, upper_bound=upper_bound)
         return (value, start, end)
 
+    def _multi_chr(self, c):
+        return c
+
     def descr_len(self, space):
         return space.wrap(self._len())
 
 
     def descr_contains(self, space, w_sub):
         value = self._val(space)
-        other = self._op_val(space, w_sub)
-        return space.newbool(value.find(other) >= 0)
+        if self._use_rstr_ops(space, w_sub):
+            other = self._op_val(space, w_sub)
+            return space.newbool(value.find(other) >= 0)
+
+        from pypy.objspace.std.bytesobject import W_BytesObject
+        if isinstance(w_sub, W_BytesObject):
+            other = self._op_val(space, w_sub)
+            res = find(value, other, 0, len(value))
+        else:
+            buffer = _get_buffer(space, w_sub)
+            res = find(value, buffer, 0, len(value))
+
+        return space.newbool(res >= 0)
 
     def descr_add(self, space, w_other):
-        try:
-            other = self._op_val(space, w_other)
-        except OperationError as e:
-            if e.match(space, space.w_TypeError):
-                return space.w_NotImplemented
-            raise
-        return self._new(self._val(space) + other)
+        if self._use_rstr_ops(space, w_other):
+            try:
+                other = self._op_val(space, w_other)
+            except OperationError as e:
+                if e.match(space, space.w_TypeError):
+                    return space.w_NotImplemented
+                raise
+            return self._new(self._val(space) + other)
+
+        # Bytearray overrides this method, CPython doesn't support contacting
+        # buffers and strs, and unicodes are always handled above
+        return space.w_NotImplemented
 
     def descr_mul(self, space, w_times):
         try:
         if times <= 0:
             return self._empty()
         if self._len() == 1:
-            return self._new(self._val(space)[0] * times)
+            return self._new(self._multi_chr(self._val(space)[0]) * times)
         return self._new(self._val(space) * times)
 
     descr_rmul = descr_mul
         d = width - len(value)
         if d > 0:
             offset = d//2 + (d & width & 1)
-            fillchar = fillchar[0]    # annotator hint: it's a single character
+            fillchar = self._multi_chr(fillchar[0])
             centered = offset * fillchar + value + (d - offset) * fillchar
         else:
             centered = value
 
     def descr_count(self, space, w_sub, w_start=None, w_end=None):
         value, start, end = self._convert_idx_params(space, w_start, w_end)
-        return space.newint(value.count(self._op_val(space, w_sub), start,
-                                        end))
+
+        if self._use_rstr_ops(space, w_sub):
+            return space.newint(value.count(self._op_val(space, w_sub), start,
+                                            end))
+
+        from pypy.objspace.std.bytearrayobject import W_BytearrayObject
+        from pypy.objspace.std.bytesobject import W_BytesObject
+        if isinstance(w_sub, W_BytearrayObject):
+            res = count(value, w_sub.data, start, end)
+        elif isinstance(w_sub, W_BytesObject):
+            res = count(value, w_sub._value, start, end)
+        else:
+            buffer = _get_buffer(space, w_sub)
+            res = count(value, buffer, start, end)
+
+        return space.wrap(max(res, 0))
 
     def descr_decode(self, space, w_encoding=None, w_errors=None):
         from pypy.objspace.std.unicodeobject import (
             _get_encoding_and_errors, decode_object, unicode_from_string)
         encoding, errors = _get_encoding_and_errors(space, w_encoding,
                                                     w_errors)
-        if encoding is None and errors is None:
+
+        from pypy.objspace.std.bytearrayobject import W_BytearrayObject
+        if (encoding is None and errors is None and
+            not isinstance(self, W_BytearrayObject)):
             return unicode_from_string(space, self)
         return decode_object(space, self, encoding, errors)
 
         if not value:
             return self._empty()
 
-        splitted = value.split(self._chr('\t'))
+        if self._use_rstr_ops(space, self):
+            splitted = value.split(self._chr('\t'))
+        else:
+            splitted = split(value, self._chr('\t'))
+
         try:
             ovfcheck(len(splitted) * tabsize)
         except OverflowError:
         expanded = oldtoken = splitted.pop(0)
 
         for token in splitted:
-            expanded += self._chr(' ') * self._tabindent(oldtoken,
+            expanded += self._multi_chr(self._chr(' ')) * self._tabindent(oldtoken,
                                                          tabsize) + token
             oldtoken = token
 
 
     def descr_find(self, space, w_sub, w_start=None, w_end=None):
         (value, start, end) = self._convert_idx_params(space, w_start, w_end)
-        res = value.find(self._op_val(space, w_sub), start, end)
+
+        if self._use_rstr_ops(space, w_sub):
+            res = value.find(self._op_val(space, w_sub), start, end)
+            return space.wrap(res)
+
+        from pypy.objspace.std.bytearrayobject import W_BytearrayObject
+        from pypy.objspace.std.bytesobject import W_BytesObject
+        if isinstance(w_sub, W_BytearrayObject):
+            res = find(value, w_sub.data, start, end)
+        elif isinstance(w_sub, W_BytesObject):
+            res = find(value, w_sub._value, start, end)
+        else:
+            buffer = _get_buffer(space, w_sub)
+            res = find(value, buffer, start, end)
+
         return space.wrap(res)
 
     def descr_rfind(self, space, w_sub, w_start=None, w_end=None):
         (value, start, end) = self._convert_idx_params(space, w_start, w_end)
-        res = value.rfind(self._op_val(space, w_sub), start, end)
+
+        if self._use_rstr_ops(space, w_sub):
+            res = value.rfind(self._op_val(space, w_sub), start, end)
+            return space.wrap(res)
+
+        from pypy.objspace.std.bytearrayobject import W_BytearrayObject
+        from pypy.objspace.std.bytesobject import W_BytesObject
+        if isinstance(w_sub, W_BytearrayObject):
+            res = rfind(value, w_sub.data, start, end)
+        elif isinstance(w_sub, W_BytesObject):
+            res = rfind(value, w_sub._value, start, end)
+        else:
+            buffer = _get_buffer(space, w_sub)
+            res = rfind(value, buffer, start, end)
+
         return space.wrap(res)
 
     def descr_index(self, space, w_sub, w_start=None, w_end=None):
         (value, start, end) = self._convert_idx_params(space, w_start, w_end)
-        res = value.find(self._op_val(space, w_sub), start, end)
+
+        from pypy.objspace.std.bytearrayobject import W_BytearrayObject
+        from pypy.objspace.std.bytesobject import W_BytesObject
+        if self._use_rstr_ops(space, w_sub):
+            res = value.find(self._op_val(space, w_sub), start, end)
+        elif isinstance(w_sub, W_BytearrayObject):
+            res = find(value, w_sub.data, start, end)
+        elif isinstance(w_sub, W_BytesObject):
+            res = find(value, w_sub._value, start, end)
+        else:
+            buffer = _get_buffer(space, w_sub)
+            res = find(value, buffer, start, end)
+
         if res < 0:
             raise oefmt(space.w_ValueError,
                         "substring not found in string.index")
-
         return space.wrap(res)
 
     def descr_rindex(self, space, w_sub, w_start=None, w_end=None):
         (value, start, end) = self._convert_idx_params(space, w_start, w_end)
-        res = value.rfind(self._op_val(space, w_sub), start, end)
+
+        from pypy.objspace.std.bytearrayobject import W_BytearrayObject
+        from pypy.objspace.std.bytesobject import W_BytesObject
+        if self._use_rstr_ops(space, w_sub):
+            res = value.rfind(self._op_val(space, w_sub), start, end)
+        elif isinstance(w_sub, W_BytearrayObject):
+            res = rfind(value, w_sub.data, start, end)
+        elif isinstance(w_sub, W_BytesObject):
+            res = rfind(value, w_sub._value, start, end)
+        else:
+            buffer = _get_buffer(space, w_sub)
+            res = rfind(value, buffer, start, end)
+
         if res < 0:
             raise oefmt(space.w_ValueError,
                         "substring not found in string.rindex")
-
         return space.wrap(res)
 
     @specialize.arg(2)
         value = self._val(space)
 
         prealloc_size = len(value) * (size - 1)
+        unwrapped = newlist_hint(size)
         for i in range(size):
             w_s = list_w[i]
             check_item = self._join_check_item(space, w_s)
                             i, w_s)
             elif check_item == 2:
                 return self._join_autoconvert(space, list_w)
-            prealloc_size += len(self._op_val(space, w_s))
+            # XXX Maybe the extra copy here is okay? It was basically going to
+            #     happen anyway, what with being placed into the builder
+            unwrapped.append(self._op_val(space, w_s))
+            prealloc_size += len(unwrapped[i])
 
         sb = self._builder(prealloc_size)
         for i in range(size):
             if value and i != 0:
                 sb.append(value)
-            sb.append(self._op_val(space, list_w[i]))
+            sb.append(unwrapped[i])
         return self._new(sb.build())
 
     def _join_autoconvert(self, space, list_w):
                         "ljust() argument 2 must be a single character")
         d = width - len(value)
         if d > 0:
-            fillchar = fillchar[0]    # annotator hint: it's a single character
+            fillchar = self._multi_chr(fillchar[0])
             value += d * fillchar
 
         return self._new(value)
                         "rjust() argument 2 must be a single character")
         d = width - len(value)
         if d > 0:
-            fillchar = fillchar[0]    # annotator hint: it's a single character
+            fillchar = self._multi_chr(fillchar[0])
             value = d * fillchar + value
 
         return self._new(value)
         return self._new(builder.build())
 
     def descr_partition(self, space, w_sub):
+        from pypy.objspace.std.bytearrayobject import W_BytearrayObject
         value = self._val(space)
-        sub = self._op_val(space, w_sub)
-        if not sub:
-            raise oefmt(space.w_ValueError, "empty separator")
-        pos = value.find(sub)
+
+        if self._use_rstr_ops(space, w_sub):
+            sub = self._op_val(space, w_sub)
+            sublen = len(sub)
+            if sublen == 0:
+                raise oefmt(space.w_ValueError, "empty separator")
+
+            pos = value.find(sub)
+        else:
+            sub = _get_buffer(space, w_sub)
+            sublen = sub.getlength()
+            if sublen == 0:
+                raise oefmt(space.w_ValueError, "empty separator")
+
+            pos = find(value, sub, 0, len(value))
+            if pos != -1 and isinstance(self, W_BytearrayObject):
+                w_sub = self._new_from_buffer(sub)
+
         if pos == -1:
-            from pypy.objspace.std.bytearrayobject import W_BytearrayObject
             if isinstance(self, W_BytearrayObject):
                 self = self._new(value)
             return space.newtuple([self, self._empty(), self._empty()])
         else:
-            from pypy.objspace.std.bytearrayobject import W_BytearrayObject
-            if isinstance(self, W_BytearrayObject):
-                w_sub = self._new(sub)
             return space.newtuple(
                 [self._sliced(space, value, 0, pos, self), w_sub,
-                 self._sliced(space, value, pos+len(sub), len(value), self)])
+                 self._sliced(space, value, pos + sublen, len(value), self)])
 
     def descr_rpartition(self, space, w_sub):
+        from pypy.objspace.std.bytearrayobject import W_BytearrayObject
         value = self._val(space)
-        sub = self._op_val(space, w_sub)
-        if not sub:
-            raise oefmt(space.w_ValueError, "empty separator")
-        pos = value.rfind(sub)
+
+        if self._use_rstr_ops(space, w_sub):
+            sub = self._op_val(space, w_sub)
+            sublen = len(sub)
+            if sublen == 0:
+                raise oefmt(space.w_ValueError, "empty separator")
+
+            pos = value.rfind(sub)
+        else:
+            sub = _get_buffer(space, w_sub)
+            sublen = sub.getlength()
+            if sublen == 0:
+                raise oefmt(space.w_ValueError, "empty separator")
+
+            pos = rfind(value, sub, 0, len(value))
+            if pos != -1 and isinstance(self, W_BytearrayObject):
+                w_sub = self._new_from_buffer(sub)
+
         if pos == -1:
-            from pypy.objspace.std.bytearrayobject import W_BytearrayObject
             if isinstance(self, W_BytearrayObject):
                 self = self._new(value)
             return space.newtuple([self._empty(), self._empty(), self])
         else:
-            from pypy.objspace.std.bytearrayobject import W_BytearrayObject
-            if isinstance(self, W_BytearrayObject):
-                w_sub = self._new(sub)
             return space.newtuple(
                 [self._sliced(space, value, 0, pos, self), w_sub,
-                 self._sliced(space, value, pos+len(sub), len(value), self)])
+                 self._sliced(space, value, pos + sublen, len(value), self)])
 
     @unwrap_spec(count=int)
     def descr_replace(self, space, w_old, w_new, count=-1):
         input = self._val(space)
+
         sub = self._op_val(space, w_old)
         by = self._op_val(space, w_new)
         try:
             res = replace(input, sub, by, count)
         except OverflowError:
             raise oefmt(space.w_OverflowError, "replace string is too long")
+
         return self._new(res)
 
     @unwrap_spec(maxsplit=int)
             return self._newlist_unwrapped(space, res)
 
         by = self._op_val(space, w_sep)
-        bylen = len(by)
-        if bylen == 0:
+        if len(by) == 0:
             raise oefmt(space.w_ValueError, "empty separator")
         res = split(value, by, maxsplit)
+
         return self._newlist_unwrapped(space, res)
 
     @unwrap_spec(maxsplit=int)
             return self._newlist_unwrapped(space, res)
 
         by = self._op_val(space, w_sep)
-        bylen = len(by)
-        if bylen == 0:
+        if len(by) == 0:
             raise oefmt(space.w_ValueError, "empty separator")
         res = rsplit(value, by, maxsplit)
+
         return self._newlist_unwrapped(space, res)
 
     @unwrap_spec(keepends=bool)
             for char in string:
                 buf.append(table[ord(char)])
         else:
+            # XXX Why not preallocate here too?
             buf = self._builder()
             deletion_table = [False] * 256
-            for c in deletechars:
-                deletion_table[ord(c)] = True
+            for i in range(len(deletechars)):
+                deletion_table[ord(deletechars[i])] = True
             for char in string:
                 if not deletion_table[ord(char)]:
                     buf.append(table[ord(char)])
     def descr_zfill(self, space, width):
         selfval = self._val(space)
         if len(selfval) == 0:
-            return self._new(self._chr('0') * width)
+            return self._new(self._multi_chr(self._chr('0')) * width)
         num_zeros = width - len(selfval)
         if num_zeros <= 0:
             # cannot return self, in case it is a subclass of str
 @specialize.argtype(0)
 def _descr_getslice_slowpath(selfvalue, start, step, sl):
     return [selfvalue[start + i*step] for i in range(sl)]
+
+def _get_buffer(space, w_obj):
+    return space.buffer_w(w_obj, space.BUF_SIMPLE)
+
+

pypy/objspace/std/test/test_bytearrayobject.py

         assert bytearray('hello').rindex('l') == 3
         assert bytearray('hello').index(bytearray('e')) == 1
         assert bytearray('hello').find('l') == 2
+        assert bytearray('hello').find('l', -2) == 3
         assert bytearray('hello').rfind('l') == 3
 
+
         # these checks used to not raise in pypy but they should
         raises(TypeError, bytearray('hello').index, ord('e'))
         raises(TypeError, bytearray('hello').rindex, ord('e'))
         u = b.decode('utf-8')
         assert isinstance(u, unicode)
         assert u == u'abcdefghi'
+        assert b.decode().encode() == b
 
     def test_int(self):
         assert int(bytearray('-1234')) == -1234

pypy/objspace/std/unicodeobject.py

     _val = unicode_w
 
     @staticmethod
+    def _use_rstr_ops(space, w_other):
+        # Always return true because we always need to copy the other
+        # operand(s) before we can do comparisons
+        return True
+
+    @staticmethod
     def _op_val(space, w_other):
         if isinstance(w_other, W_UnicodeObject):
             return w_other._value

pypy/tool/release/package.py

 #!/usr/bin/env python
-""" A sample script that packages PyPy, provided that it's already built.
+""" packages PyPy, provided that it's already built.
 It uses 'pypy/goal/pypy-c' and parts of the rest of the working
 copy.  Usage:
 
-    package.py [--nostrip] [--without-tk] root-pypy-dir [name-of-archive] [name-of-pypy-c] [destination-for-tarball] [pypy-c-path]
+    package.py [--options]
 
-Usually you would do:   package.py ../../.. pypy-VER-PLATFORM
-The output is found in the directory /tmp/usession-YOURNAME/build/.
+Usually you would do:   package.py --version-name pypy-VER-PLATFORM
+The output is found in the directory from --builddir,
+by default /tmp/usession-YOURNAME/build/.
 """
 
 import shutil
 import sys
 import os
 #Add toplevel repository dir to sys.path
-sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
+basedir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+sys.path.insert(0,basedir)
 import py
 import fnmatch
-from rpython.tool.udir import udir
 import subprocess
+import glob
 
 if sys.version_info < (2,6): py.test.skip("requires 2.6 so far")
 
 class PyPyCNotFound(Exception):
     pass
 
-def fix_permissions(basedir):
+class MissingDependenciesError(Exception):
+    pass
+
+def fix_permissions(dirname):
     if sys.platform != 'win32':
-        os.system("chmod -R a+rX %s" % basedir)
-        os.system("chmod -R g-w %s" % basedir)
+        os.system("chmod -R a+rX %s" % dirname)
+        os.system("chmod -R g-w %s" % dirname)
 
-def package(basedir, name='pypy-nightly', rename_pypy_c='pypy',
-            copy_to_dir=None, override_pypy_c=None, nostrip=False,
-            withouttk=False):
-    assert '/' not in rename_pypy_c
+sep_template = "\nThis copy of PyPy includes a copy of %s, which is licensed under the following terms:\n\n"
+
+def generate_license_linux(basedir, options):
+    base_file = str(basedir.join('LICENSE'))
+    with open(base_file) as fid:
+        txt = fid.read()
+    searches = [("bzip2","libbz2-*", "copyright", '---------'),
+                ("openssl", "openssl*", "copyright", 'LICENSE ISSUES'),
+               ]
+    if not options.no_tk:
+        name = 'Tcl/Tk'
+        txt += "License for '%s'" %name
+        txt += '\n' + "="*(14 + len(name)) + '\n'
+        txt += sep_template % name
+        base_file = str(basedir.join('lib_pypy/_tkinter/license.terms'))
+        with open(base_file, 'r') as fid:
+            txt += fid.read()
+    for name, pat, fname, first_line in searches:
+        txt += "License for '" + name + "'"
+        txt += '\n' + "="*(14 + len(name)) + '\n'
+        txt += sep_template % name
+        dirs = glob.glob(options.license_base + "/" +pat)
+        if not dirs:
+            raise ValueError, "Could not find "+ options.license_base + "/" + pat
+        if len(dirs) > 2:
+            raise ValueError, "Multiple copies of "+pat
+        dir = dirs[0]
+        with open(os.path.join(dir, fname)) as fid:
+            # Read up to the line dividing the packaging header from the actual copyright
+            for line in fid:
+                if first_line in line:
+                    break
+            txt += line
+            for line in fid:
+                txt += line
+            if len(line.strip())<1:
+                txt += '\n'
+    txt += third_party_header
+    # Do something for gdbm, which is GPL
+    txt += gdbm_bit
+    return txt
+
+def generate_license_windows(basedir, options):
+    base_file = str(basedir.join('LICENSE'))
+    with open(base_file) as fid:
+        txt = fid.read()
+    # shutil.copyfileobj(open("crtlicense.txt"), out) # We do not ship msvc runtime files
+    if not options.no_tk:
+        name = 'Tcl/Tk'
+        txt += "License for '%s'" %name
+        txt += '\n' + "="*(14 + len(name)) + '\n'
+        txt += sep_template % name
+        base_file = str(basedir.join('lib_pypy/_tkinter/license.terms'))
+        with open(base_file, 'r') as fid:
+            txt += fid.read()
+    for name, pat, file in (("bzip2","bzip2-*", "LICENSE"),
+                      ("openssl", "openssl-*", "LICENSE")):
+        txt += sep_template % name
+        dirs = glob.glob(options.license_base + "/" +pat)
+        if not dirs:
+            raise ValueError, "Could not find "+ options.license_base + "/" + pat
+        if len(dirs) > 2:
+            raise ValueError, "Multiple copies of "+pat
+        dir = dirs[0]
+        with open(os.path.join(dir, file)) as fid:
+            txt += fid.read()
+    return txt
+
+def generate_license_darwin(basedir, options):
+    # where are copyright files on macos?
+    return generate_license_linux(basedir, options)
+
+if sys.platform == 'win32':
+    generate_license = generate_license_windows
+elif sys.platform == 'darwin':
+    generate_license = generate_license_darwin
+else:
+    generate_license = generate_license_linux
+
+def create_cffi_import_libraries(pypy_c, options):
+    modules = ['_sqlite3']
+    subprocess.check_call([str(pypy_c), '-c', 'import _sqlite3'])
+    if not sys.platform == 'win32':
+        modules += ['_curses', 'syslog', 'gdbm', '_sqlite3']
+    if not options.no_tk:
+        modules.append(('_tkinter'))
+    for module in modules:
+        try:
+            subprocess.check_call([str(pypy_c), '-c', 'import ' + module])
+        except subprocess.CalledProcessError:
+            print >>sys.stderr, """Building {0} bindings failed.
+You can either install development headers package or
+add --without-{0} option to skip packaging binary CFFI extension.""".format(module)
+            raise MissingDependenciesError(module)
+
+def create_package(basedir, options):
+    retval = 0
+    name = options.name
+    if not name:
+        name = 'pypy-nightly'
+    rename_pypy_c = options.pypy_c
+    override_pypy_c = options.override_pypy_c
+
     basedir = py.path.local(basedir)
-    if override_pypy_c is None:
+    if not override_pypy_c:
         basename = 'pypy-c'
         if sys.platform == 'win32':
             basename += '.exe'
             raise PyPyCNotFound(
                 'Bogus path: %r does not exist (see docstring for more info)'
                 % (os.path.dirname(str(pypy_c)),))
-    win_extras = ['libpypy-c.dll', 'libexpat.dll', 'sqlite3.dll',
-                      'libeay32.dll', 'ssleay32.dll']
-    subprocess.check_call([str(pypy_c), '-c', 'import _sqlite3'])
-    if not sys.platform == 'win32':
-        subprocess.check_call([str(pypy_c), '-c', 'import _curses'])
-        subprocess.check_call([str(pypy_c), '-c', 'import syslog'])
-        subprocess.check_call([str(pypy_c), '-c', 'import gdbm'])
-    if not withouttk:
+    if not options.no_cffi:
         try:
-            subprocess.check_call([str(pypy_c), '-c', 'import _tkinter'])
-        except subprocess.CalledProcessError:
-            print >>sys.stderr, """Building Tk bindings failed.
-You can either install Tk development headers package or
-add --without-tk option to skip packaging binary CFFI extension."""
-            sys.exit(1)
-        #Can the dependencies be found from cffi somehow?    
-        win_extras += ['tcl85.dll', 'tk85.dll']    
+            create_cffi_import_libraries(pypy_c, options)
+        except MissingDependenciesError:
+            # This is a non-fatal error
+            retval = -1
+
     if sys.platform == 'win32' and not rename_pypy_c.lower().endswith('.exe'):
         rename_pypy_c += '.exe'
     binaries = [(pypy_c, rename_pypy_c)]
     #
-    builddir = udir.ensure("build", dir=True)
+    builddir = options.builddir
     pypydir = builddir.ensure(name, dir=True)
     includedir = basedir.join('include')
     # Recursively copy all headers, shutil has only ignore
     pypydir.ensure('include', dir=True)
 
     if sys.platform == 'win32':
-        #Don't include a mscvrXX.dll, users should get their own.
-        #Instructions are provided on the website.
-
         # Can't rename a DLL: it is always called 'libpypy-c.dll'
+        win_extras = ['libpypy-c.dll', 'libexpat.dll', 'sqlite3.dll',
+                          'libeay32.dll', 'ssleay32.dll']
+        if not options.no_tk:
+            win_extras += ['tcl85.dll', 'tk85.dll']
 
         for extra in win_extras:
             p = pypy_c.dirpath().join(extra)
                     continue
             print "Picking %s" % p
             binaries.append((p, p.basename))
-        importlib_name = 'python27.lib'    
+        importlib_name = 'python27.lib'
         if pypy_c.dirpath().join(importlib_name).check():
             shutil.copyfile(str(pypy_c.dirpath().join(importlib_name)),
                         str(pypydir.join('include/python27.lib')))
             # XXX users will complain that they cannot compile cpyext
             # modules for windows, has the lib moved or are there no
             # exported functions in the dll so no import library is created?
-        if not withouttk:
+        if not options.no_tk:
             try:
                 p = pypy_c.dirpath().join('tcl85.dll')
                 if not p.check():
 tk85.dll and tcl85.dll found, expecting to find runtime in ..\\lib
 directory next to the dlls, as per build instructions."""
                 import traceback;traceback.print_exc()
-                sys.exit(1)
+                raise MissingDependenciesError('Tk runtime')
 
     # Careful: to copy lib_pypy, copying just the hg-tracked files
     # would not be enough: there are also ctypes_config_cache/_*_cache.py.
                     str(pypydir.join('lib_pypy')),
                     ignore=ignore_patterns('.svn', 'py', '*.pyc', '*~',
                                            '*.c', '*.o'))
-    for file in ['LICENSE', 'README.rst']:
+    for file in ['README.rst',]:
         shutil.copy(str(basedir.join(file)), str(pypydir))
     for file in ['_testcapimodule.c', '_ctypes_test.c']:
-        shutil.copyfile(str(basedir.join('lib_pypy', file)), 
+        shutil.copyfile(str(basedir.join('lib_pypy', file)),
                         str(pypydir.join('lib_pypy', file)))
+    try:
+        license = generate_license(basedir, options)
+        with open(str(pypydir.join('LICENSE')), 'w') as LICENSE:
+            LICENSE.write(license)
+    except:
+        # Non-fatal error, use original LICENCE file
+        import traceback;traceback.print_exc()
+        base_file = str(basedir.join('LICENSE'))
+        with open(base_file) as fid:
+            license = fid.read()
+        with open(str(pypydir.join('LICENSE')), 'w') as LICENSE:
+            LICENSE.write(license)
+        retval = -1
     #
     spdir = pypydir.ensure('site-packages', dir=True)
     shutil.copy(str(basedir.join('site-packages', 'README')), str(spdir))
     for source, target in binaries:
         archive = bindir.join(target)
         shutil.copy(str(source), str(archive))
+    fix_permissions(builddir)
+
     old_dir = os.getcwd()
-    fix_permissions(builddir)
     try:
         os.chdir(str(builddir))
-        #
-        # 'strip' fun: see issue #587
-        if not nostrip:
+        if not options.nostrip:
             for source, target in binaries:
                 if sys.platform == 'win32':
                     pass
                 elif sys.platform == 'darwin':
+                    # 'strip' fun: see issue #587 for why -x
                     os.system("strip -x " + str(bindir.join(target)))    # ignore errors
                 else:
                     os.system("strip " + str(bindir.join(target)))    # ignore errors
                 raise OSError('"tar" returned exit status %r' % e)
     finally:
         os.chdir(old_dir)
-    if copy_to_dir is not None:
-        print "Copying %s to %s" % (archive, copy_to_dir)
-        shutil.copy(archive, str(copy_to_dir))
+    if options.targetdir:
+        print "Copying %s to %s" % (archive, options.targetdir)
+        shutil.copy(archive, options.targetdir)
     else:
         print "Ready in %s" % (builddir,)
-    return builddir # for tests
+    return retval, builddir # for tests
 
+def package(*args):
+    try:
+        import argparse
+    except ImportError:
+        import imp
+        argparse = imp.load_source('argparse', 'lib-python/2.7/argparse.py')
+    if sys.platform == 'win32':
+        pypy_exe = 'pypy.exe'
+        license_base = os.path.join(basedir, r'..\..\..\local') # as on buildbot YMMV
+    else:
+        pypy_exe = 'pypy'
+        license_base = '/usr/share/doc'
+    parser = argparse.ArgumentParser()
+    args = list(args)
+    args[0] = str(args[0])
+    parser.add_argument('--without-tk', dest='no_tk', action='store_true',
+        help='build and package the cffi tkinter module')
+    parser.add_argument('--without-cffi', dest='no_cffi', action='store_true',
+        help='do not pre-import any cffi modules')
+    parser.add_argument('--nostrip', dest='nostrip', action='store_true',
+        help='do not strip the exe, making it ~10MB larger')
+    parser.add_argument('--rename_pypy_c', dest='pypy_c', type=str, default=pypy_exe,
+        help='target executable name, defaults to "pypy"')
+    parser.add_argument('--archive-name', dest='name', type=str, default='',
+        help='pypy-VER-PLATFORM')
+    parser.add_argument('--license_base', type=str, default=license_base,
+        help='where to start looking for third party upstream licensing info')
+    parser.add_argument('--builddir', type=str, default='',
+        help='tmp dir for packaging')
+    parser.add_argument('--targetdir', type=str, default='',
+        help='destination dir for archive')
+    parser.add_argument('--override_pypy_c', type=str, default='',
+        help='use as pypy exe instead of pypy/goal/pypy-c')
+    # Positional arguments, for backward compatability with buldbots
+    parser.add_argument('extra_args', help='optional interface to positional arguments', nargs=argparse.REMAINDER,
+        metavar='[root-pypy-dir] [name-of-archive] [name-of-pypy-c] [destination-for-tarball] [pypy-c-path]',
+        )
+    options = parser.parse_args(args)
 
-def print_usage():
-    print >>sys.stderr, __doc__
-    sys.exit(1)
+    # Handle positional arguments, choke if both methods are used
+    for i,target, default in ([1, 'name', ''], [2, 'pypy_c', pypy_exe],
+                              [3, 'targetdir', ''], [4,'override_pypy_c', '']):
+        if len(options.extra_args)>i:
+            if getattr(options, target) != default:
+                print 'positional argument',i,target,'already has value',getattr(options, target)
+                parser.print_help()
+                return
+            setattr(options, target, options.extra_args[i])
+    if os.environ.has_key("PYPY_PACKAGE_NOSTRIP"):
+        options.nostrip = True
+
+    if os.environ.has_key("PYPY_PACKAGE_WITHOUTTK"):
+        options.tk = True
+    if not options.builddir:
+        # The import actually creates the udir directory
+        from rpython.tool.udir import udir
+        options.builddir = udir.ensure("build", dir=True)
+    assert '/' not in options.pypy_c
+    return create_package(basedir, options)
+
+
+third_party_header = '''\n\nLicenses and Acknowledgements for Incorporated Software
+=======================================================
+
+This section is an incomplete, but growing list of licenses and acknowledgements
+for third-party software incorporated in the PyPy distribution.
+
+'''
+
+gdbm_bit = '''gdbm
+----
+
+The gdbm module includes code from gdbm.h, which is distributed under the terms
+of the GPL license version 2 or any later version.
+'''
 
 
 if __name__ == '__main__':
-    if len(sys.argv) == 1:
-        print_usage()
-
-    args = sys.argv[1:]
-    kw = {}
-
-    for i, arg in enumerate(args):
-        if arg == '--nostrip':
-            kw['nostrip'] = True
-        elif arg == '--without-tk':
-            kw['withouttk'] = True
-        elif not arg.startswith('--'):
-            break
-        else:
-            print_usage()
-
-    if os.environ.has_key("PYPY_PACKAGE_NOSTRIP"):
-        kw['nostrip'] = True
-
-    if os.environ.has_key("PYPY_PACKAGE_WITHOUTTK"):
-        kw['withouttk'] = True
-
-    args = args[i:]
-    package(*args, **kw)
+    import sys
+    retval, _ = package(*sys.argv[1:])
+    sys.exit(retval)

pypy/tool/release/test/test_package.py

 
 import py
 from pypy.conftest import pypydir
-from pypy.tool.release import package
+from pypy.tool.release import package, package
 from pypy.module.sys.version import  CPYTHON_VERSION
 import tarfile, zipfile, sys
 
     pypy_c = py.path.local(pypydir).join('goal', basename)
     if not pypy_c.check():
         if sys.platform == 'win32':
-            assert False, "test on win32 requires exe"
-        pypy_c.write("#!/bin/sh")
-        pypy_c.chmod(0755)
+            import os, shutil
+            for d in os.environ['PATH'].split(';'):
+                if os.path.exists(os.path.join(d, 'cmd.exe')):
+                    shutil.copy(os.path.join(d, 'cmd.exe'), str(pypy_c))
+                    break
+            else:
+                assert False, 'could not find cmd.exe'
+        else:    
+            pypy_c.write("#!/bin/sh")
+            pypy_c.chmod(0755)
         fake_pypy_c = True
     else:
         fake_pypy_c = False
     try:
-        builddir = package.package(py.path.local(pypydir).dirpath(), test,
+        retval, builddir = package.package(py.path.local(pypydir).dirpath(), test,
                                    rename_pypy_c)
+        assert retval == 0
         prefix = builddir.join(test)
         cpyver = '%d.%d' % CPYTHON_VERSION[:2]
         assert prefix.join('lib-python', cpyver, 'test').check()
             pypy_c.remove()
 
 def test_with_zipfile_module():
-    from pypy.tool.release import package
     prev = package.USE_ZIPFILE_MODULE
     try:
         package.USE_ZIPFILE_MODULE = True
     check(file1, 0644)
     check(file2, 0644)
     check(pypy,  0755)
+
+def test_generate_license():
+    from os.path import dirname, abspath, join
+    class Options(object):
+        pass
+    options = Options()
+    basedir = dirname(dirname(dirname(dirname(dirname(abspath(__file__))))))
+    options.no_tk = False
+    if sys.platform == 'win32':
+         # as on buildbot YMMV
+        options.license_base = join(basedir, r'..\..\..\local')
+    else:
+        options.license_base = '/usr/share/doc'
+    license = package.generate_license(py.path.local(basedir), options)
+    assert 'bzip2' in license
+    assert 'openssl' in license
+    assert 'Tcl' in license
+

rpython/annotator/binaryop.py

             return super(thistype, pair(ins1, ins2)).improve()
 
 
+class __extend__(pairtype(SomeInstance, SomeObject)):
+    def getitem((s_ins, s_idx)):
+        return s_ins._emulate_call("__getitem__", s_idx)
+
+    def setitem((s_ins, s_idx), s_value):
+        return s_ins._emulate_call("__setitem__", s_idx, s_value)
+
+
 class __extend__(pairtype(SomeIterator, SomeIterator)):
 
     def union((iter1, iter2)):

rpython/annotator/test/test_annrpython.py