1. Pypy
  2. Untitled project
  3. pypy

Commits

Amaury Forgeot d'Arc  committed a0e3eb8 Merge

Merge default

  • Participants
  • Parent commits d1414c1, 1d42dc6
  • Branches py3k

Comments (0)

Files changed (81)

File lib-python/modified-2.7/test/test_array.py

View file
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a + b")
-
-        self.assertRaises(TypeError, "a + 'bad'")
+        with self.assertRaises(TypeError):
+            a + b
+        with self.assertRaises(TypeError):
+            a + 'bad'
 
     def test_iadd(self):
         a = array.array(self.typecode, self.example[::-1])
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a += b")
-
-        self.assertRaises(TypeError, "a += 'bad'")
+        with self.assertRaises(TypeError):
+            a += b
+        with self.assertRaises(TypeError):
+            a += 'bad'
 
     def test_mul(self):
         a = 5*array.array(self.typecode, self.example)
             array.array(self.typecode)
         )
 
-        self.assertRaises(TypeError, "a * 'bad'")
+        with self.assertRaises(TypeError):
+            a * 'bad'
 
     def test_imul(self):
         a = array.array(self.typecode, self.example)
         a *= -1
         self.assertEqual(a, array.array(self.typecode))
 
-        self.assertRaises(TypeError, "a *= 'bad'")
+        with self.assertRaises(TypeError):
+            a *= 'bad'
 
     def test_getitem(self):
         a = array.array(self.typecode, self.example)

File lib_pypy/resource.py

View file
 
 from ctypes_support import standard_c_lib as libc
 from ctypes_support import get_errno
-from ctypes import Structure, c_int, c_long, byref, sizeof, POINTER
+from ctypes import Structure, c_int, c_long, byref, POINTER
 from errno import EINVAL, EPERM
 import _structseq
 
 
 @builtinify
 def getpagesize():
-    pagesize = 0
     if _getpagesize:
         return _getpagesize()
     else:

File pypy/config/pypyoption.py

View file
 
 pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [
     ChoiceOption("name", "Object Space name",
-                 ["std", "flow", "thunk", "dump", "taint"],
+                 ["std", "flow", "thunk", "dump"],
                  "std",
                  cmdline='--objspace -o'),
 

File pypy/doc/__pypy__-module.rst

View file
 .. _`thunk object space docs`: objspace-proxies.html#thunk
 .. _`interface section of the thunk object space docs`: objspace-proxies.html#thunk-interface
 
-.. broken:
-
-    Taint Object Space Functionality
-    ================================
-
-    When the taint object space is used (choose with :config:`objspace.name`),
-    the following names are put into ``__pypy__``:
-
-     - ``taint``
-     - ``is_tainted``
-     - ``untaint``
-     - ``taint_atomic``
-     - ``_taint_debug``
-     - ``_taint_look``
-     - ``TaintError``
-
-    Those are all described in the `interface section of the taint object space
-    docs`_.
-
-    For more detailed explanations and examples see the `taint object space docs`_.
-
-    .. _`taint object space docs`: objspace-proxies.html#taint
-    .. _`interface section of the taint object space docs`: objspace-proxies.html#taint-interface
 
 Transparent Proxy Functionality
 ===============================

File pypy/doc/config/objspace.name.txt

View file
 for normal usage):
 
   * thunk_: The thunk object space adds lazy evaluation to PyPy.
-  * taint_: The taint object space adds soft security features.
   * dump_:  Using this object spaces results in the dumpimp of all operations
     to a log.
 
 .. _`Object Space Proxies`: ../objspace-proxies.html
 .. _`Standard Object Space`: ../objspace.html#standard-object-space
 .. _thunk: ../objspace-proxies.html#thunk
-.. _taint: ../objspace-proxies.html#taint
 .. _dump: ../objspace-proxies.html#dump

File pypy/doc/index.rst

View file
 .. _`object space`: objspace.html
 .. _FlowObjSpace: objspace.html#the-flow-object-space 
 .. _`trace object space`: objspace.html#the-trace-object-space 
-.. _`taint object space`: objspace-proxies.html#taint
 .. _`thunk object space`: objspace-proxies.html#thunk
 .. _`transparent proxies`: objspace-proxies.html#tproxy
 .. _`Differences between PyPy and CPython`: cpython_differences.html

File pypy/doc/objspace-proxies.rst

View file
    function behaves lazily: all calls to it return a thunk object.
 
 
-.. broken right now:
-
-    .. _taint:
-
-    The Taint Object Space
-    ======================
-
-    Motivation
-    ----------
-
-    The Taint Object Space provides a form of security: "tainted objects",
-    inspired by various sources, see [D12.1]_ for a more detailed discussion. 
-
-    The basic idea of this kind of security is not to protect against
-    malicious code but to help with handling and boxing sensitive data. 
-    It covers two kinds of sensitive data: secret data which should not leak, 
-    and untrusted data coming from an external source and that must be 
-    validated before it is used.
-
-    The idea is that, considering a large application that handles these
-    kinds of sensitive data, there are typically only a small number of
-    places that need to explicitly manipulate that sensitive data; all the
-    other places merely pass it around, or do entirely unrelated things.
-
-    Nevertheless, if a large application needs to be reviewed for security,
-    it must be entirely carefully checked, because it is possible that a
-    bug at some apparently unrelated place could lead to a leak of sensitive
-    information in a way that an external attacker could exploit.  For
-    example, if any part of the application provides web services, an
-    attacker might be able to issue unexpected requests with a regular web
-    browser and deduce secret information from the details of the answers he
-    gets.  Another example is the common CGI attack where an attacker sends
-    malformed inputs and causes the CGI script to do unintended things.
-
-    An approach like that of the Taint Object Space allows the small parts
-    of the program that manipulate sensitive data to be explicitly marked.
-    The effect of this is that although these small parts still need a
-    careful security review, the rest of the application no longer does,
-    because even a bug would be unable to leak the information.
-
-    We have implemented a simple two-level model: objects are either
-    regular (untainted), or sensitive (tainted).  Objects are marked as
-    sensitive if they are secret or untrusted, and only declassified at
-    carefully-checked positions (e.g. where the secret data is needed, or
-    after the untrusted data has been fully validated).
-
-    It would be simple to extend the code for more fine-grained scales of
-    secrecy.  For example it is typical in the literature to consider
-    user-specified lattices of secrecy levels, corresponding to multiple
-    "owners" that cannot access data belonging to another "owner" unless
-    explicitly authorized to do so.
-
-    Tainting and untainting
-    -----------------------
-
-    Start a py.py with the Taint Object Space and try the following example::
-
-        $ py.py -o taint
-        >>>> from __pypy__ import taint
-        >>>> x = taint(6)
-
-        # x is hidden from now on.  We can pass it around and
-        # even operate on it, but not inspect it.  Taintness
-        # is propagated to operation results.
-
-        >>>> x
-        TaintError
-
-        >>>> if x > 5: y = 2   # see below
-        TaintError
-
-        >>>> y = x + 5         # ok
-        >>>> lst = [x, y]
-        >>>> z = lst.pop()
-        >>>> t = type(z)       # type() works too, tainted answer
-        >>>> t
-        TaintError
-        >>>> u = t is int      # even 'is' works
-        >>>> u
-        TaintError
-
-    Notice that using a tainted boolean like ``x > 5`` in an ``if``
-    statement is forbidden.  This is because knowing which path is followed
-    would give away a hint about ``x``; in the example above, if the
-    statement ``if x > 5: y = 2`` was allowed to run, we would know
-    something about the value of ``x`` by looking at the (untainted) value
-    in the variable ``y``.
-
-    Of course, there is a way to inspect tainted objects.  The basic way is
-    to explicitly "declassify" it with the ``untaint()`` function.  In an
-    application, the places that use ``untaint()`` are the places that need
-    careful security review.  To avoid unexpected objects showing up, the
-    ``untaint()`` function must be called with the exact type of the object
-    to declassify.  It will raise ``TaintError`` if the type doesn't match::
-
-        >>>> from __pypy__ import taint
-        >>>> untaint(int, x)
-        6
-        >>>> untaint(int, z)
-        11
-        >>>> untaint(bool, x > 5)
-        True
-        >>>> untaint(int, x > 5)
-        TaintError
-
-
-    Taint Bombs
-    -----------
-
-    In this area, a common problem is what to do about failing operations.
-    If an operation raises an exception when manipulating a tainted object,
-    then the very presence of the exception can leak information about the
-    tainted object itself.  Consider::
-
-        >>>> 5 / (x-6)
-
-    By checking if this raises ``ZeroDivisionError`` or not, we would know
-    if ``x`` was equal to 6 or not.  The solution to this problem in the
-    Taint Object Space is to introduce *Taint Bombs*.  They are a kind of
-    tainted object that doesn't contain a real object, but a pending
-    exception.  Taint Bombs are indistinguishable from normal tainted
-    objects to unprivileged code. See::
-
-        >>>> x = taint(6)
-        >>>> i = 5 / (x-6)     # no exception here
-        >>>> j = i + 1         # nor here
-        >>>> k = j + 5         # nor here
-        >>>> untaint(int, k)
-        TaintError
-
-    In the above example, all of ``i``, ``j`` and ``k`` contain a Taint
-    Bomb.  Trying to untaint it raises an exception - a generic
-    ``TaintError``.  What we win is that the exception gives little away,
-    and most importantly it occurs at the point where ``untaint()`` is
-    called, not where the operation failed.  This means that all calls to
-    ``untaint()`` - but not the rest of the code - must be carefully
-    reviewed for what occurs if they receive a Taint Bomb; they might catch
-    the ``TaintError`` and give the user a generic message that something
-    went wrong, if we are reasonably careful that the message or even its
-    presence doesn't give information away.  This might be a
-    problem by itself, but there is no satisfying general solution here:
-    it must be considered on a case-by-case basis.  Again, what the
-    Taint Object Space approach achieves is not solving these problems, but
-    localizing them to well-defined small parts of the application - namely,
-    around calls to ``untaint()``.
-
-    The ``TaintError`` exception deliberately does not include any
-    useful error messages, because they might give information away.
-    Of course, this makes debugging quite a bit harder; a difficult
-    problem to solve properly.  So far we have implemented a way to peek in a Taint
-    Box or Bomb, ``__pypy__._taint_look(x)``, and a "debug mode" that
-    prints the exception as soon as a Bomb is created - both write
-    information to the low-level stderr of the application, where we hope
-    that it is unlikely to be seen by anyone but the application
-    developer.
-
-
-    Taint Atomic functions
-    ----------------------
-
-    Occasionally, a more complicated computation must be performed on a
-    tainted object.  This requires first untainting the object, performing the
-    computations, and then carefully tainting the result again (including
-    hiding all exceptions into Bombs).
-
-    There is a built-in decorator that does this for you::
-
-        >>>> @__pypy__.taint_atomic
-        >>>> def myop(x, y):
-        ....     while x > 0:
-        ....         x -= y
-        ....     return x
-        ....
-        >>>> myop(42, 10)
-        -8
-        >>>> z = myop(taint(42), 10)
-        >>>> z
-        TaintError
-        >>>> untaint(int, z)
-        -8
-
-    The decorator makes a whole function behave like a built-in operation.
-    If no tainted argument is passed in, the function behaves normally.  But
-    if any of the arguments is tainted, it is automatically untainted - so
-    the function body always sees untainted arguments - and the eventual
-    result is tainted again (possibly in a Taint Bomb).
-
-    It is important for the function marked as ``taint_atomic`` to have no
-    visible side effects, as these could cause information leakage.
-    This is currently not enforced, which means that all ``taint_atomic``
-    functions have to be carefully reviewed for security (but not the
-    callers of ``taint_atomic`` functions).
-
-    A possible future extension would be to forbid side-effects on
-    non-tainted objects from all ``taint_atomic`` functions.
-
-    An example of usage: given a tainted object ``passwords_db`` that
-    references a database of passwords, we can write a function
-    that checks if a password is valid as follows::
-
-        @taint_atomic
-        def validate(passwords_db, username, password):
-            assert type(passwords_db) is PasswordDatabase
-            assert type(username) is str
-            assert type(password) is str
-            ...load username entry from passwords_db...
-            return expected_password == password
-
-    It returns a tainted boolean answer, or a Taint Bomb if something
-    went wrong.  A caller can do::
-
-        ok = validate(passwords_db, 'john', '1234')
-        ok = untaint(bool, ok)
-
-    This can give three outcomes: ``True``, ``False``, or a ``TaintError``
-    exception (with no information on it) if anything went wrong.  If even
-    this is considered giving too much information away, the ``False`` case
-    can be made indistinguishable from the ``TaintError`` case (simply by
-    raising an exception in ``validate()`` if the password is wrong).
-
-    In the above example, the security results achieved are the following:
-    as long as ``validate()`` does not leak information, no other part of
-    the code can obtain more information about a passwords database than a
-    Yes/No answer to a precise query.
-
-    A possible extension of the ``taint_atomic`` decorator would be to check
-    the argument types, as ``untaint()`` does, for the same reason: to
-    prevent bugs where a function like ``validate()`` above is accidentally
-    called with the wrong kind of tainted object, which would make it
-    misbehave.  For now, all ``taint_atomic`` functions should be
-    conservative and carefully check all assumptions on their input
-    arguments.
-
-
-    .. _`taint-interface`:
-
-    Interface
-    ---------
-
-    .. _`like a built-in operation`:
-
-    The basic rule of the Tainted Object Space is that it introduces two new
-    kinds of objects, Tainted Boxes and Tainted Bombs (which are not types
-    in the Python sense).  Each box internally contains a regular object;
-    each bomb internally contains an exception object.  An operation
-    involving Tainted Boxes is performed on the objects contained in the
-    boxes, and gives a Tainted Box or a Tainted Bomb as a result (such an
-    operation does not let an exception be raised).  An operation called
-    with a Tainted Bomb argument immediately returns the same Tainted Bomb.
-
-    In a PyPy running with (or translated with) the Taint Object Space,
-    the ``__pypy__`` module exposes the following interface:
-
-    * ``taint(obj)``
-
-        Return a new Tainted Box wrapping ``obj``.  Return ``obj`` itself
-        if it is already tainted (a Box or a Bomb).
-
-    * ``is_tainted(obj)``
-
-        Check if ``obj`` is tainted (a Box or a Bomb).
-
-    * ``untaint(type, obj)``
-
-        Untaints ``obj`` if it is tainted.  Raise ``TaintError`` if the type
-        of the untainted object is not exactly ``type``, or if ``obj`` is a
-        Bomb.
-
-    * ``taint_atomic(func)``
-
-        Return a wrapper function around the callable ``func``.  The wrapper
-        behaves `like a built-in operation`_ with respect to untainting the
-        arguments, tainting the result, and returning a Bomb.
-
-    * ``TaintError``
-
-        Exception.  On purpose, it provides no attribute or error message.
-
-    * ``_taint_debug(level)``
-
-        Set the debugging level to ``level`` (0=off).  At level 1 or above,
-        all Taint Bombs print a diagnostic message to stderr when they are
-        created.
-
-    * ``_taint_look(obj)``
-
-        For debugging purposes: prints (to stderr) the type and address of
-        the object in a Tainted Box, or prints the exception if ``obj`` is
-        a Taint Bomb.
-
-
 .. _dump:
 
 The Dump Object Space

File pypy/interpreter/executioncontext.py

View file
     def decrement_ticker(self, by):
         value = self._ticker
         if self.has_bytecode_counter:    # this 'if' is constant-folded
-            value -= by
-            self._ticker = value
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                self._ticker = value
         return value
 
 

File pypy/interpreter/pyparser/pytokenizer.py

View file
                         parenlev = parenlev - 1
                         if parenlev < 0:
                             raise TokenError("unmatched '%s'" % initial, line,
-                                             lnum-1, 0, token_list)
+                                             lnum, start + 1, token_list)
                     if token in python_opmap:
                         punct = python_opmap[token]
                     else:

File pypy/interpreter/pyparser/test/test_pyparse.py

View file
         assert exc.lineno == 1
         assert exc.offset == 5
         assert exc.lastlineno == 5
+        exc = py.test.raises(SyntaxError, parse, "abc)").value
+        assert exc.msg == "unmatched ')'"
+        assert exc.lineno == 1
+        assert exc.offset == 4
 
     def test_is(self):
         self.parse("x is y")

File pypy/jit/backend/llgraph/llimpl.py

View file
     'unicodegetitem'  : (('ref', 'int'), 'int'),
     'unicodesetitem'  : (('ref', 'int', 'int'), 'int'),
     'cast_ptr_to_int' : (('ref',), 'int'),
+    'cast_int_to_ptr' : (('int',), 'ref'),
     'debug_merge_point': (('ref', 'int'), None),
     'force_token'     : ((), 'int'),
     'call_may_force'  : (('int', 'varargs'), 'intorptr'),
     def op_new_array(self, arraydescr, count):
         return do_new_array(arraydescr.ofs, count)
 
-    def op_cast_ptr_to_int(self, descr, ptr):
-        return cast_to_int(ptr)
-
     def op_force_token(self, descr):
         opaque_frame = _to_opaque(self)
         return llmemory.cast_ptr_to_adr(opaque_frame)

File pypy/jit/backend/llsupport/gc.py

View file
     def freeing_block(self, start, stop):
         pass
 
+    def record_constptrs(self, op, gcrefs_output_list):
+        for i in range(op.numargs()):
+            v = op.getarg(i)
+            if isinstance(v, ConstPtr) and bool(v.value):
+                p = v.value
+                rgc._make_sure_does_not_move(p)
+                gcrefs_output_list.append(p)
+
 # ____________________________________________________________
 
 class GcLLDescr_boehm(GcLLDescription):
     get_funcptr_for_newstr = None
     get_funcptr_for_newunicode = None
 
+    def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
+        # record all GCREFs too, because Boehm cannot see them and keep them
+        # alive if they end up as constants in the assembler
+        for op in operations:
+            self.record_constptrs(op, gcrefs_output_list)
+        return GcLLDescription.rewrite_assembler(self, cpu, operations,
+                                                 gcrefs_output_list)
+
 
 # ____________________________________________________________
 # All code below is for the hybrid or minimark GC
             funcptr(llmemory.cast_ptr_to_adr(gcref_struct),
                     llmemory.cast_ptr_to_adr(gcref_newptr))
 
-    def record_constptrs(self, op, gcrefs_output_list):
-        for i in range(op.numargs()):
-            v = op.getarg(i)
-            if isinstance(v, ConstPtr) and bool(v.value):
-                p = v.value
-                rgc._make_sure_does_not_move(p)
-                gcrefs_output_list.append(p)
-
     def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
         # Perform two kinds of rewrites in parallel:
         #

File pypy/jit/backend/test/runner_test.py

View file
         return u''.join(u.chars)
 
 
-    def test_casts(self):
-        py.test.skip("xxx fix or kill")
-        from pypy.rpython.lltypesystem import lltype, llmemory
-        TP = lltype.GcStruct('x')
-        x = lltype.malloc(TP)        
-        x = lltype.cast_opaque_ptr(llmemory.GCREF, x)
+    def test_cast_int_to_ptr(self):
+        res = self.execute_operation(rop.CAST_INT_TO_PTR,
+                                     [BoxInt(-17)],  'ref').value
+        assert lltype.cast_ptr_to_int(res) == -17
+
+    def test_cast_ptr_to_int(self):
+        x = lltype.cast_int_to_ptr(llmemory.GCREF, -19)
         res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [BoxPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
-        res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [ConstPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
+                                     [BoxPtr(x)], 'int').value
+        assert res == -19
 
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
         #
         cpu.bh_strsetitem(x, 4, ord('/'))
         assert str.chars[4] == '/'
-        #
-##        x = cpu.bh_newstr(5)
-##        y = cpu.bh_cast_ptr_to_int(x)
-##        z = cpu.bh_cast_ptr_to_int(x)
-##        y = rffi.get_real_int(y)
-##        z = rffi.get_real_int(z)
-##        assert type(y) == type(z) == int and y == z
 
     def test_sorting_of_fields(self):
         S = self.S

File pypy/jit/backend/x86/assembler.py

View file
 
     def genop_same_as(self, op, arglocs, resloc):
         self.mov(arglocs[0], resloc)
-    #genop_cast_ptr_to_int = genop_same_as
+    genop_cast_ptr_to_int = genop_same_as
+    genop_cast_int_to_ptr = genop_same_as
 
     def genop_int_mod(self, op, arglocs, resloc):
         if IS_X86_32:

File pypy/jit/backend/x86/regalloc.py

View file
         self.possibly_free_var(op.getarg(0))
         resloc = self.force_allocate_reg(op.result)
         self.Perform(op, [argloc], resloc)
-    #consider_cast_ptr_to_int = consider_same_as
+    consider_cast_ptr_to_int = consider_same_as
+    consider_cast_int_to_ptr = consider_same_as
 
     def consider_strlen(self, op):
         args = op.getarglist()

File pypy/jit/backend/x86/tool/viewcode.py

View file
 """
 
 import autopath
-import operator, sys, os, re, py, new
+import new
+import operator
+import py
+import re
+import sys
+import subprocess
 from bisect import bisect_left
 
 # don't use pypy.tool.udir here to avoid removing old usessions which
     f = open(tmpfile, 'wb')
     f.write(data)
     f.close()
-    g = os.popen(objdump % {
+    p = subprocess.Popen(objdump % {
         'file': tmpfile,
         'origin': originaddr,
         'backend': objdump_backend_option[backend_name],
-    }, 'r')
-    result = g.readlines()
-    g.close()
-    lines = result[6:]   # drop some objdump cruft
+    }, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running objdump: %s' %
+                              stderr)
+    # drop some objdump cruft
+    lines = stdout.splitlines()[6:]
     return format_code_dump_with_labels(originaddr, lines, label_list)
 
 def format_code_dump_with_labels(originaddr, lines, label_list):
     #
     print 'loading symbols from %s...' % (filename,)
     symbols = {}
-    g = os.popen(symbollister % filename, "r")
-    for line in g:
+    p = subprocess.Popen(symbollister % filename, shell=True,
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running nm: %s' %
+                              stderr)
+    for line in stdout.splitlines():
         match = re_symbolentry.match(line)
         if match:
             addr = long(match.group(1), 16)
             if name.startswith('pypy_g_'):
                 name = '\xb7' + name[7:]
             symbols[addr] = name
-    g.close()
     print '%d symbols found' % (len(symbols),)
     return symbols
 

File pypy/jit/codewriter/jtransform.py

View file
             # the special return value None forces op.result to be considered
             # equal to op.args[0]
             return [op0, op1, None]
+        if (hints.get('promote_string') and
+            op.args[0].concretetype is not lltype.Void):
+            S = lltype.Ptr(rstr.STR)
+            assert op.args[0].concretetype == S
+            self._register_extra_helper(EffectInfo.OS_STREQ_NONNULL,
+                                        "str.eq_nonnull",
+                                        [S, S],
+                                        lltype.Signed,
+                                        EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+            descr, p = self.callcontrol.callinfocollection.callinfo_for_oopspec(
+                EffectInfo.OS_STREQ_NONNULL)
+            # XXX this is fairly ugly way of creating a constant,
+            #     however, callinfocollection has no better interface
+            c = Constant(p.adr.ptr, lltype.typeOf(p.adr.ptr))
+            op1 = SpaceOperation('str_guard_value', [op.args[0], c, descr],
+                                 op.result)
+            return [SpaceOperation('-live-', [], None), op1, None]
         else:
             log.WARNING('ignoring hint %r at %r' % (hints, self.graph))
 
 
     def rewrite_op_cast_ptr_to_int(self, op):
         if self._is_gc(op.args[0]):
-            #return op
-            raise NotImplementedError("cast_ptr_to_int")
+            return op
 
     def rewrite_op_force_cast(self, op):
         v_arg = op.args[0]
     def rewrite_op_jit_force_virtual(self, op):
         return self._do_builtin_call(op)
 
+    def rewrite_op_jit_is_virtual(self, op):
+        raise Exception, (
+            "'vref.virtual' should not be used from jit-visible code")
+
     def rewrite_op_jit_force_virtualizable(self, op):
         # this one is for virtualizables
         vinfo = self.get_vinfo(op.args[0])

File pypy/jit/codewriter/test/test_jtransform.py

View file
             if i == oopspecindex:
                 return True
         return False
+    def callinfo_for_oopspec(self, oopspecindex):
+        assert oopspecindex == effectinfo.EffectInfo.OS_STREQ_NONNULL
+        class c:
+            class adr:
+                ptr = 1
+        return ('calldescr', c)
 
 class FakeBuiltinCallControl:
     def __init__(self):
              EI.OS_STR2UNICODE:([PSTR], PUNICODE),
              EI.OS_STR_CONCAT: ([PSTR, PSTR], PSTR),
              EI.OS_STR_SLICE:  ([PSTR, INT, INT], PSTR),
+             EI.OS_STREQ_NONNULL:  ([PSTR, PSTR], INT),
              EI.OS_UNI_CONCAT: ([PUNICODE, PUNICODE], PUNICODE),
              EI.OS_UNI_SLICE:  ([PUNICODE, INT, INT], PUNICODE),
              EI.OS_UNI_EQUAL:  ([PUNICODE, PUNICODE], lltype.Bool),
     assert op1.args[2] == ListOfKind('ref', [v1, v2])
     assert op1.result == v3
 
+def test_str_promote():
+    PSTR = lltype.Ptr(rstr.STR)
+    v1 = varoftype(PSTR)
+    v2 = varoftype(PSTR)
+    op = SpaceOperation('hint',
+                        [v1, Constant({'promote_string': True}, lltype.Void)],
+                        v2)
+    tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
+    op0, op1, _ = tr.rewrite_operation(op)
+    assert op1.opname == 'str_guard_value'
+    assert op1.args[0] == v1
+    assert op1.args[2] == 'calldescr'
+    assert op1.result == v2
+    assert op0.opname == '-live-'
+
 def test_unicode_concat():
     # test that the oopspec is present and correctly transformed
     PSTR = lltype.Ptr(rstr.UNICODE)

File pypy/jit/metainterp/blackhole.py

View file
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import debug_start, debug_stop
+from pypy.rlib.debug import debug_start, debug_stop, ll_assert
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
     @arguments("r", returns="r")
     def bhimpl_cast_opaque_ptr(a):
         return a
+    @arguments("r", returns="i")
+    def bhimpl_cast_ptr_to_int(a):
+        i = lltype.cast_ptr_to_int(a)
+        ll_assert((i & 1) == 1, "bhimpl_cast_ptr_to_int: not an odd int")
+        return i
+    @arguments("i", returns="r")
+    def bhimpl_cast_int_to_ptr(i):
+        ll_assert((i & 1) == 1, "bhimpl_cast_int_to_ptr: not an odd int")
+        return lltype.cast_int_to_ptr(llmemory.GCREF, i)
 
     @arguments("i", returns="i")
     def bhimpl_int_copy(a):
     @arguments("f")
     def bhimpl_float_guard_value(a):
         pass
+    @arguments("r", "i", "d", returns="r")
+    def bhimpl_str_guard_value(a, i, d):
+        return a
 
     @arguments("self", "i")
     def bhimpl_int_push(self, a):

File pypy/jit/metainterp/graphpage.py

View file
     def get_display_text(self):
         return None
 
-def display_loops(loops, errmsg=None, highlight_loops=()):
-    graphs = [(loop, loop in highlight_loops) for loop in loops]    
+def display_loops(loops, errmsg=None, highlight_loops={}):
+    graphs = [(loop, highlight_loops.get(loop, 0)) for loop in loops]    
     for graph, highlight in graphs:
         for op in graph.get_operations():
             if is_interesting_guard(op):
     def add_graph(self, graph, highlight=False):
         graphindex = len(self.graphs)
         self.graphs.append(graph)
-        if highlight:
-            self.highlight_graphs[graph] = True
+        self.highlight_graphs[graph] = highlight
         for i, op in enumerate(graph.get_operations()):
             self.all_operations[op] = graphindex, i
 
             self.dotgen.emit('subgraph cluster%d {' % graphindex)
         label = graph.get_display_text()
         if label is not None:
-            if self.highlight_graphs.get(graph):
-                fillcolor = '#f084c2'
+            colorindex = self.highlight_graphs.get(graph, 0)
+            if colorindex == 1:
+                fillcolor = '#f084c2'    # highlighted graph
+            elif colorindex == 2:
+                fillcolor = '#808080'    # invalidated graph
             else:
-                fillcolor = '#84f0c2'
+                fillcolor = '#84f0c2'    # normal color
             self.dotgen.emit_node(graphname, shape="octagon",
                                   label=label, fillcolor=fillcolor)
             self.pendingedges.append((graphname,

File pypy/jit/metainterp/history.py

View file
 
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
+from pypy.rlib.objectmodel import compute_identity_hash
 
 # ____________________________________________________________
 
     getref._annspecialcase_ = 'specialize:arg(1)'
 
     def _get_hash_(self):
-        raise NotImplementedError
+        return compute_identity_hash(self)
 
     def clonebox(self):
         raise NotImplementedError
     def _get_str(self):
         raise NotImplementedError
 
+    def same_box(self, other):
+        return self is other
+
 class AbstractDescr(AbstractValue):
     __slots__ = ()
 
     def constbox(self):
         return self
 
+    def same_box(self, other):
+        return self.same_constant(other)
+
     def same_constant(self, other):
         raise NotImplementedError
 
     def __repr__(self):
         return 'Const(%s)' % self._getrepr_()
 
-    def __eq__(self, other):
-        "NOT_RPYTHON"
-        # Remember that you should not compare Consts with '==' in RPython.
-        # Consts have no special __hash__, in order to force different Consts
-        # from being considered as different keys when stored in dicts
-        # (as they always are after translation).  Use a dict_equal_consts()
-        # to get the other behavior (i.e. using this __eq__).
-        if self.__class__ is not other.__class__:
-            return False
-        try:
-            return self.value == other.value
-        except TypeError:
-            if (isinstance(self.value, Symbolic) and
-                isinstance(other.value, Symbolic)):
-                return self.value is other.value
-            raise
-
-    def __ne__(self, other):
-        return not (self == other)
-
 
 class ConstInt(Const):
     type = INT
 
 # ____________________________________________________________
 
-def dict_equal_consts():
-    "NOT_RPYTHON"
-    # Returns a dict in which Consts that compare as equal
-    # are identified when used as keys.
-    return r_dict(dc_eq, dc_hash)
-
-def dc_eq(c1, c2):
-    return c1 == c2
-
-def dc_hash(c):
-    "NOT_RPYTHON"
-    # This is called during translation only.  Avoid using identityhash(),
-    # to avoid forcing a hash, at least on lltype objects.
-    if not isinstance(c, Const):
-        return hash(c)
-    if isinstance(c.value, Symbolic):
-        return id(c.value)
-    try:
-        if isinstance(c, ConstPtr):
-            p = lltype.normalizeptr(c.value)
-            if p is not None:
-                return hash(p._obj)
-            else:
-                return 0
-        return c._get_hash_()
-    except lltype.DelayedPointer:
-        return -2      # xxx risk of changing hash...
 
 def make_hashable_int(i):
     from pypy.rpython.lltypesystem.ll2ctypes import NotCtypesAllocatedStructure
     failed_states = None
     retraced_count = 0
     terminating = False # see TerminatingLoopToken in compile.py
+    invalidated = False
     outermost_jitdriver_sd = None
     # and more data specified by the backend when the loop is compiled
     number = -1
         self.loops = []
         self.locations = []
         self.aborted_keys = []
+        self.invalidated_token_numbers = set()
 
     def set_history(self, history):
         self.operations = history.operations
             if loop in loops:
                 loops.remove(loop)
             loops.append(loop)
-        display_loops(loops, errmsg, extraloops)
+        highlight_loops = dict.fromkeys(extraloops, 1)
+        for loop in loops:
+            if hasattr(loop, '_looptoken_number') and (
+                    loop._looptoken_number in self.invalidated_token_numbers):
+                highlight_loops.setdefault(loop, 2)
+        display_loops(loops, errmsg, highlight_loops)
 
 # ----------------------------------------------------------------
 

File pypy/jit/metainterp/memmgr.py

View file
         debug_print("Loop tokens before:", oldtotal)
         max_generation = self.current_generation - (self.max_age-1)
         for looptoken in self.alive_loops.keys():
-            if 0 <= looptoken.generation < max_generation:
+            if (0 <= looptoken.generation < max_generation or
+                looptoken.invalidated):
                 del self.alive_loops[looptoken]
         newtotal = len(self.alive_loops)
         debug_print("Loop tokens freed: ", oldtotal - newtotal)

File pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py

View file
         def _variables_equal(box, varname, strict):
             if varname not in virtuals:
                 if strict:
-                    assert box == oparse.getvar(varname)
+                    assert box.same_box(oparse.getvar(varname))
                 else:
                     assert box.value == oparse.getvar(varname).value
             else:

File pypy/jit/metainterp/optimizeopt/util.py

View file
     for i in range(len(args1)):
         arg1 = args1[i]
         arg2 = args2[i]
-        if isinstance(arg1, history.Const):
-            if arg1.__class__ is not arg2.__class__:
+        if arg1 is None:
+            if arg2 is not None:
                 return False
-            if not arg1.same_constant(arg2):
-                return False
-        else:
-            if not arg1 is arg2:
-                return False
+        elif not arg1.same_box(arg2):
+            return False
     return True
 
 def args_hash(args):
     for arg in args:
         if arg is None:
             y = 17
-        elif isinstance(arg, history.Const):
+        else:
             y = arg._get_hash_()
-        else:
-            y = compute_identity_hash(arg)
         res = intmask((1000003 * res) ^ y)
     return res
 
         for i in range(op1.numargs()):
             x = op1.getarg(i)
             y = op2.getarg(i)
-            assert x == remap.get(y, y)
+            assert x.same_box(remap.get(y, y))
         if op2.result in remap:
-            assert op1.result == remap[op2.result]
+            if op2.result is None:
+                assert op1.result == remap[op2.result]
+            else:
+                assert op1.result.same_box(remap[op2.result])
         else:
             remap[op2.result] = op1.result
         if op1.getopnum() != rop.JUMP:      # xxx obscure
             assert len(op1.getfailargs()) == len(op2.getfailargs())
             if strict_fail_args:
                 for x, y in zip(op1.getfailargs(), op2.getfailargs()):
-                    assert x == remap.get(y, y)
+                    if x is None:
+                        assert remap.get(y, y) is None
+                    else:
+                        assert x.same_box(remap.get(y, y))
             else:
                 fail_args1 = set(op1.getfailargs())
                 fail_args2 = set([remap.get(y, y) for y in op2.getfailargs()])
-                assert fail_args1 == fail_args2
+                for x in fail_args1:
+                    for y in fail_args2:
+                        if x.same_box(y):
+                            fail_args2.remove(y)
+                            break
+                    else:
+                        assert False
     assert len(oplist1) == len(oplist2)
     print '-'*totwidth
     return True

File pypy/jit/metainterp/optimizeopt/vstring.py

View file
             return True
         #
         if v1.is_nonnull() and v2.is_nonnull():
-            if l1box is not None and l2box is not None and (
-                l1box == l2box or (isinstance(l1box, ConstInt) and
-                                   isinstance(l2box, ConstInt) and
-                                   l1box.value == l2box.value)):
+            if l1box is not None and l2box is not None and l1box.same_box(l2box):
                 do = EffectInfo.OS_STREQ_LENGTHOK
             else:
                 do = EffectInfo.OS_STREQ_NONNULL

File pypy/jit/metainterp/pyjitpl.py

View file
             if registers[i] is oldbox:
                 registers[i] = newbox
         if not we_are_translated():
-            assert oldbox not in registers[count:]
+            for b in registers[count:]:
+                assert not oldbox.same_box(b)
+                
 
     def make_result_of_lastop(self, resultbox):
         got_type = resultbox.type
-        if not we_are_translated():
-            typeof = {'i': history.INT,
-                      'r': history.REF,
-                      'f': history.FLOAT}
-            assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
+        # XXX disabled for now, conflicts with str_guard_value
+        #if not we_are_translated():
+        #    typeof = {'i': history.INT,
+        #              'r': history.REF,
+        #              'f': history.FLOAT}
+        #    assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
         target_index = ord(self.bytecode[self.pc-1])
         if got_type == history.INT:
             self.registers_i[target_index] = resultbox
                     'cast_float_to_int', 'cast_int_to_float',
                     'cast_float_to_singlefloat', 'cast_singlefloat_to_float',
                     'float_neg', 'float_abs',
+                    'cast_ptr_to_int', 'cast_int_to_ptr',
                     ]:
         exec py.code.Source('''
             @arguments("box")
     def _opimpl_guard_value(self, orgpc, box):
         self.implement_guard_value(orgpc, box)
 
+    @arguments("orgpc", "box", "box", "descr")
+    def opimpl_str_guard_value(self, orgpc, box, funcbox, descr):
+        if isinstance(box, Const):
+            return box     # no promotion needed, already a Const
+        else:
+            constbox = box.constbox()
+            resbox = self.do_residual_call(funcbox, descr, [box, constbox])
+            promoted_box = resbox.constbox()
+            # This is GUARD_VALUE because GUARD_TRUE assumes the existance
+            # of a label when computing resumepc
+            self.generate_guard(rop.GUARD_VALUE, resbox, [promoted_box],
+                                resumepc=orgpc)
+            self.metainterp.replace_box(box, constbox)
+            return constbox
+
     opimpl_int_guard_value = _opimpl_guard_value
     opimpl_ref_guard_value = _opimpl_guard_value
     opimpl_float_guard_value = _opimpl_guard_value

File pypy/jit/metainterp/quasiimmut.py

View file
 from pypy.rpython.lltypesystem import lltype, rclass
 from pypy.rpython.annlowlevel import cast_base_ptr_to_instance
 from pypy.jit.metainterp.history import AbstractDescr
+from pypy.rlib.objectmodel import we_are_translated
 
 
 def get_mutate_field_name(fieldname):
 
 class QuasiImmut(object):
     llopaque = True
+    compress_limit = 30
     
     def __init__(self, cpu):
         self.cpu = cpu
         # list of weakrefs to the LoopTokens that must be invalidated if
         # this value ever changes
         self.looptokens_wrefs = []
-        self.compress_limit = 30
 
     def hide(self):
         qmut_ptr = self.cpu.ts.cast_instance_to_base_ref(self)
         self.looptokens_wrefs.append(wref_looptoken)
 
     def compress_looptokens_list(self):
-        self.looptokens_wrefs = [wref for wref in self.looptokens_wrefs
-                                      if wref() is not None]
+        newlist = []
+        for wref in self.looptokens_wrefs:
+            looptoken = wref()
+            if looptoken is not None and not looptoken.invalidated:
+                newlist.append(wref)
+        self.looptokens_wrefs = newlist
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
     def invalidate(self):
         self.looptokens_wrefs = []
         for wref in wrefs:
             looptoken = wref()
-            if looptoken is not None:
+            if looptoken is not None and not looptoken.invalidated:
+                looptoken.invalidated = True
                 self.cpu.invalidate_loop(looptoken)
+                if not we_are_translated():
+                    self.cpu.stats.invalidated_token_numbers.add(
+                        looptoken.number)
 
 
 class QuasiImmutDescr(AbstractDescr):

File pypy/jit/metainterp/resoperation.py

View file
     'INT_INVERT/1',
     #
     'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
+    'CAST_PTR_TO_INT/1',
+    'CAST_INT_TO_PTR/1',
     #
     'PTR_EQ/2b',
     'PTR_NE/2b',

File pypy/jit/metainterp/test/test_ajit.py

View file
 from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
-    isconstant, isvirtual)
+    isconstant, isvirtual, promote_string)
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.rpython.ootypesystem import ootype
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
     def test_tagged(self):
-        py.test.skip("implement me")
         from pypy.rlib.objectmodel import UnboxedValue
         class Base(object):
             __slots__ = ()
                 pc += 1
             return pc
         res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
+
+    def test_rerased(self):
+        from pypy.rlib.rerased import erase_int, unerase_int, new_erasing_pair
+        eraseX, uneraseX = new_erasing_pair("X")
+        #
+        class X:
+            def __init__(self, a, b):
+                self.a = a
+                self.b = b
+        #
+        def f(i, j):
+            # 'j' should be 0 or 1, not other values
+            if j > 0:
+                e = eraseX(X(i, j))
+            else:
+                try:
+                    e = erase_int(i)
+                except OverflowError:
+                    return -42
+            if j & 1:
+                x = uneraseX(e)
+                return x.a - x.b
+            else:
+                return unerase_int(e)
+        #
+        x = self.interp_operations(f, [-128, 0], taggedpointers=True)
+        assert x == -128
+        bigint = sys.maxint//2 + 1
+        x = self.interp_operations(f, [bigint, 0], taggedpointers=True)
+        assert x == -42
+        x = self.interp_operations(f, [1000, 1], taggedpointers=True)
+        assert x == 999

File pypy/jit/metainterp/test/test_memmgr.py

View file
 
 class FakeLoopToken:
     generation = 0
+    invalidated = False
 
 
 class _TestMemoryManager:

File pypy/jit/metainterp/test/test_quasiimmut.py

View file
 
 class QuasiImmutTests(object):
 
+    def setup_method(self, meth):
+        self.prev_compress_limit = QuasiImmut.compress_limit
+        QuasiImmut.compress_limit = 1
+
+    def teardown_method(self, meth):
+        QuasiImmut.compress_limit = self.prev_compress_limit
+
     def test_simple_1(self):
         myjitdriver = JitDriver(greens=['foo'], reds=['x', 'total'])
         class Foo:
             return total
 
         res = self.meta_interp(main, [])
-        self.check_loop_count(9)
+        self.check_tree_loop_count(6)
         assert res == main()
 
     def test_change_during_running(self):
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          call_may_force=0, guard_not_forced=0)
 
     def test_list_simple_1(self):
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          getarrayitem_gc=0, getarrayitem_gc_pure=0,
                          call_may_force=0, guard_not_forced=0)
 
+    def test_invalidated_loop_is_not_used_any_more_as_target(self):
+        myjitdriver = JitDriver(greens=['foo'], reds=['x'])
+        class Foo:
+            _immutable_fields_ = ['step?']
+        @dont_look_inside
+        def residual(x, foo):
+            if x == 20:
+                foo.step = 1
+        def f(x):
+            foo = Foo()
+            foo.step = 2
+            while x > 0:
+                myjitdriver.jit_merge_point(foo=foo, x=x)
+                residual(x, foo)
+                x -= foo.step
+            return foo.step
+        res = self.meta_interp(f, [60])
+        assert res == 1
+        self.check_tree_loop_count(4)   # at least not 2 like before
+
 
 class TestLLtypeGreenFieldsTests(QuasiImmutTests, LLJitMixin):
     pass

File pypy/jit/metainterp/test/test_resume.py

View file
+from __future__ import with_statement
 import py
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.jit.metainterp.optimizeopt.optimizer import OptValue
     reader.consume_boxes(info, bi, br, bf)
     b1s = reader.liveboxes[0]
     b2s = reader.liveboxes[1]
-    assert bi == [b1s, ConstInt(111), b1s]
-    assert br == [ConstPtr(gcrefnull), b2s]
+    assert_same(bi, [b1s, ConstInt(111), b1s])
+    assert_same(br, [ConstPtr(gcrefnull), b2s])
     bi, br, bf = [None]*2, [None]*0, [None]*0
     info = MyBlackholeInterp([lltype.Signed,
                               lltype.Signed]).get_current_position_info()
     reader.consume_boxes(info, bi, br, bf)
-    assert bi == [ConstInt(222), ConstInt(333)]
+    assert_same(bi, [ConstInt(222), ConstInt(333)])
     bi, br, bf = [None]*2, [None]*1, [None]*0
     info = MyBlackholeInterp([lltype.Signed, llmemory.GCREF,
                               lltype.Signed]).get_current_position_info()
     reader.consume_boxes(info, bi, br, bf)
     b3s = reader.liveboxes[2]
-    assert bi == [b1s, b3s]
-    assert br == [b2s]
+    assert_same(bi, [b1s, b3s])
+    assert_same(br, [b2s])
     #
 
+def assert_same(list1, list2):
+    assert len(list1) == len(list2)
+    for b1, b2 in zip(list1, list2):
+        assert b1.same_box(b2)
+
 def test_simple_read_tagged_ints():
     storage = Storage()
     storage.rd_consts = []
     metainterp = MyMetaInterp()
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
     lst = reader.consume_boxes()
-    assert lst == [b1t, b1t, b3t]
+    assert_same(lst, [b1t, b1t, b3t])
     lst = reader.consume_boxes()
-    assert lst == [ConstInt(2), ConstInt(3)]
+    assert_same(lst, [ConstInt(2), ConstInt(3)])
     lst = reader.consume_boxes()
-    assert lst == [b1t, ConstInt(1), b1t, b1t]
+    assert_same(lst, [b1t, ConstInt(1), b1t, b1t])
     assert metainterp.trace == []    
 
 
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
     lst = reader.consume_boxes()
     c1t = ConstInt(111)
-    assert lst == [c1t, b2t, b3t]
+    assert_same(lst, [c1t, b2t, b3t])
     lst = reader.consume_boxes()
-    assert lst == [ConstInt(2), ConstInt(3)]
+    assert_same(lst, [ConstInt(2), ConstInt(3)])
     lst = reader.consume_boxes()
-    assert lst == [c1t, ConstInt(1), c1t, b2t]
+    assert_same(lst, [c1t, ConstInt(1), c1t, b2t])
     assert metainterp.trace == []
 
 
     # check that we get the operations in 'expected', in a possibly different
     # order.
     assert len(trace) == len(expected)
-    for x in trace:
-        assert x in expected
-        expected.remove(x)
+    with CompareableConsts():
+        for x in trace:
+            assert x in expected
+            expected.remove(x)
+
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.NODE)
     assert ptr.value == 111
     assert ptr2.parent.value == 33
     assert ptr2.parent.next == ptr
 
+class CompareableConsts(object):
+    def __init__(self):
+        self.oldeq = None
+        
+    def __enter__(self):
+        assert self.oldeq is None
+        self.oldeq = Const.__eq__
+        Const.__eq__ = Const.same_box
+        
+    def __exit__(self, type, value, traceback):
+        Const.__eq__ = self.oldeq
+
 def test_virtual_adder_make_varray():
     b2s, b4s = [BoxPtr(), BoxInt(4)]
     c1s = ConstInt(111)
         (rop.SETARRAYITEM_GC, [b2t,ConstInt(1), c1s], None,
                               LLtypeMixin.arraydescr),
         ]
-    for x, y in zip(expected, trace):
-        assert x == y
+    with CompareableConsts():
+        for x, y in zip(expected, trace):
+            assert x == y
     #
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(lltype.GcArray(lltype.Signed))
         (rop.SETFIELD_GC, [b2t, c1s],  None, LLtypeMixin.adescr),
         (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.bdescr),
         ]
-    for x, y in zip(expected, trace):
-        assert x == y
+    with CompareableConsts():
+        for x, y in zip(expected, trace):
+            assert x == y
     #
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.S)

File pypy/jit/metainterp/test/test_string.py

View file
 from pypy.jit.codewriter.policy import StopAtXPolicy
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
 from pypy.rlib.debug import debug_print
-from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted
+from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted,\
+     promote_string
 from pypy.rlib.rstring import StringBuilder
 from pypy.rpython.ootypesystem import ootype
 
                           'jump': 1, 'int_is_true': 1,
                           'guard_not_invalidated': 1})
 
+    def test_promote_string(self):
+        driver = JitDriver(greens = [], reds = ['n'])
+        
+        def f(n):
+            while n < 21:
+                driver.jit_merge_point(n=n)
+                promote_string(str(n % 3))
+                n += 1
+            return 0
+
+        self.meta_interp(f, [0])
+        self.check_loops(call=3 + 1) # one for int2str
 
 #class TestOOtype(StringTests, OOJitMixin):
 #    CALL = "oosend"

File pypy/jit/metainterp/test/test_virtualref.py

View file
 from pypy.rpython.llinterp import LLException
 from pypy.rlib.jit import JitDriver, dont_look_inside, vref_None
 from pypy.rlib.jit import virtual_ref, virtual_ref_finish, InvalidVirtualRef
+from pypy.rlib.jit import non_virtual_ref
 from pypy.rlib.objectmodel import compute_unique_id
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, _get_jitcodes
 from pypy.jit.metainterp.resoperation import rop
         res = self.meta_interp(fn, [10])
         assert res == 6
 
+    def test_is_virtual(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                vref = virtual_ref(x)
+                res1 = residual(vref)
+                virtual_ref_finish(vref, x)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 1
+
+    def test_is_not_virtual_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                res1 = residual(vref_None)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
+    def test_is_not_virtual_non_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                res1 = residual(non_virtual_ref(x))
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
 
 class TestLLtype(VRefTests, LLJitMixin):
     pass

File pypy/jit/metainterp/virtualref.py

View file
     def replace_force_virtual_with_call(self, graphs):
         # similar to rvirtualizable2.replace_force_virtualizable_with_call().
         c_force_virtual_ptr = None
+        c_is_virtual_ptr = None
         force_virtual_count = 0
         for graph in graphs:
             for block in graph.iterblocks():
                         op.opname = 'direct_call'
                         op.args = [c_force_virtual_ptr, op.args[0]]
                         force_virtual_count += 1
+                    #
+                    if op.opname == 'jit_is_virtual':
+                        if c_is_virtual_ptr is None:
+                            c_is_virtual_ptr = self.get_is_virtual_fnptr()
+                        #
+                        op.opname = 'direct_call'
+                        op.args = [c_is_virtual_ptr, op.args[0]]
         #
         if c_force_virtual_ptr is not None:
             log("replaced %d 'jit_force_virtual' with %r" % (force_virtual_count,
             force_virtual_if_necessary)
         return inputconst(lltype.typeOf(funcptr), funcptr)
 
+    def get_is_virtual_fnptr(self):
+        #
+        def is_virtual(inst):
+            if not inst:
+                return False
+            return inst.typeptr == self.jit_virtual_ref_vtable
+        #
+        FUNC = lltype.FuncType([rclass.OBJECTPTR], lltype.Bool)
+        funcptr = self.warmrunnerdesc.helper_func(lltype.Ptr(FUNC), is_virtual)
+        return inputconst(lltype.typeOf(funcptr), funcptr)
+
     def force_virtual(self, inst):
         vref = lltype.cast_pointer(lltype.Ptr(self.JIT_VIRTUAL_REF), inst)
         token = vref.virtual_token

File pypy/jit/metainterp/warmstate.py

View file
         if self.compiled_merge_points_wref is not None:
             for wref in self.compiled_merge_points_wref:
                 looptoken = wref()
-                if looptoken is not None:
+                if looptoken is not None and not looptoken.invalidated:
                     result.append(looptoken)
         return result
 

File pypy/module/__builtin__/__init__.py

View file
         'any'           : 'app_functional.any',
         'all'           : 'app_functional.all',
         'sum'           : 'app_functional.sum',
+        'map'           : 'app_functional.map',
+        'reduce'        : 'app_functional.reduce',
+        'filter'        : 'app_functional.filter',
         'vars'          : 'app_inspect.vars',
         'dir'           : 'app_inspect.dir',
 
         'enumerate'     : 'functional.W_Enumerate',
         'min'           : 'functional.min',
         'max'           : 'functional.max',
-        'map'           : 'functional.map',
         'zip'           : 'functional.zip',
-        'reduce'        : 'functional.reduce',
         'reversed'      : 'functional.reversed',
-        'filter'        : 'functional.filter',
         'super'         : 'descriptor.W_Super',
         'staticmethod'  : 'descriptor.StaticMethod',
         'classmethod'   : 'descriptor.ClassMethod',
            builtin = space.interpclass_w(w_builtin)
            if isinstance(builtin, module.Module):
                return builtin
-       # no builtin! make a default one.  Given them None, at least.
+       # no builtin! make a default one.  Give them None, at least.
        builtin = module.Module(space, None)
        space.setitem(builtin.w_dict, space.wrap('None'), space.w_None)
        return builtin

File pypy/module/__builtin__/app_functional.py

View file
         # Very intentionally *not* +=, that would have different semantics if
         # start was a mutable type, such as a list
         last = last + x
-    return last
+    return last
+
+def map(func, *collections):
+    """map(function, sequence[, sequence, ...]) -> list
+
+Return a list of the results of applying the function to the items of
+the argument sequence(s).  If more than one sequence is given, the
+function is called with an argument list consisting of the corresponding
+item of each sequence, substituting None for missing values when not all
+sequences have the same length.  If the function is None, return a list of
+the items of the sequence (or a list of tuples if more than one sequence)."""
+    if not collections:
+        raise TypeError("map() requires at least two arguments")
+    num_collections = len(collections)
+    none_func = func is None
+    if num_collections == 1:
+        if none_func:
+            return list(collections[0])
+        else:
+            # Special case for the really common case of a single collection,
+            # this can be eliminated if we could unroll that loop that creates
+            # `args` based on whether or not len(collections) was constant
+            result = []
+            for item in collections[0]:
+                result.append(func(item))
+            return result
+    result = []
+    # Pair of (iterator, has_finished)
+    iterators = [(iter(seq), False) for seq in collections]
+    while True:
+        cont = False
+        args = []
+        for idx, (iterator, has_finished) in enumerate(iterators):
+            val = None
+            if not has_finished:
+                try:
+                    val = next(iterator)
+                except StopIteration:
+                    iterators[idx] = (None, True)
+                else:
+                    cont = True
+            args.append(val)
+        args = tuple(args)
+        if cont:
+            if none_func:
+                result.append(args)
+            else:
+                result.append(func(*args))
+        else:
+            return result
+
+sentinel = object()
+
+def reduce(func, sequence, initial=sentinel):
+    """reduce(function, sequence[, initial]) -> value
+
+Apply a function of two arguments cumulatively to the items of a sequence,
+from left to right, so as to reduce the sequence to a single value.
+For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates
+((((1+2)+3)+4)+5).  If initial is present, it is placed before the items
+of the sequence in the calculation, and serves as a default when the
+sequence is empty."""
+    iterator = iter(sequence)
+    if initial is sentinel:
+        try:
+            initial = next(iterator)
+        except StopIteration:
+            raise TypeError("reduce() of empty sequence with no initial value")
+    result = initial
+    for item in iterator:
+        result = func(result, item)
+    return result
+
+def filter(func, seq):
+    """filter(function or None, sequence) -> list, tuple, or string
+
+Return those items of sequence for which function(item) is true.  If
+function is None, return the items that are true.  If sequence is a tuple
+or string, return the same type, else return a list."""
+    if func is None:
+        func = bool
+    if isinstance(seq, str):
+        return _filter_string(func, seq, str)
+    elif isinstance(seq, unicode):
+        return _filter_string(func, seq, unicode)
+    elif isinstance(seq, tuple):
+        return _filter_tuple(func, seq)
+    result = []
+    for item in seq:
+        if func(item):
+            result.append(item)
+    return result
+
+def _filter_string(func, string, str_type):
+    if func is bool and type(string) is str_type:
+        return string
+    result = []
+    for i in range(len(string)):
+        # You must call __getitem__ on the strings, simply iterating doesn't
+        # work :/
+        item = string[i]
+        if func(item):
+            if not isinstance(item, str_type):
+                raise TypeError("__getitem__ returned a non-string type")
+            result.append(item)
+    return str_type().join(result)
+
+def _filter_tuple(func, seq):
+    result = []
+    for i in range(len(seq)):
+        # Again, must call __getitem__, at least there are tests.
+        item = seq[i]
+        if func(item):
+            result.append(item)
+    return tuple(result)

File pypy/module/__builtin__/compiling.py

View file
         raise OperationError(space.w_TypeError,
               w('eval() arg 1 must be a string or code object'))
 
-    caller = space.getexecutioncontext().gettopframe_nohidden()
     if space.is_w(w_globals, space.w_None):
+        caller = space.getexecutioncontext().gettopframe_nohidden()
         if caller is None:
             w_globals = space.newdict()
             if space.is_w(w_locals, space.w_None):
     elif space.is_w(w_locals, space.w_None):
         w_locals = w_globals
 
-    try:
-        space.getitem(w_globals, space.wrap('__builtins__'))
-    except OperationError, e:
-        if not e.match(space, space.w_KeyError):
-            raise
-        if caller is not None:
-            w_builtin = space.builtin.pick_builtin(caller.w_globals)
-            space.setitem(w_globals, space.wrap('__builtins__'), w_builtin)
+    # xxx removed: adding '__builtins__' to the w_globals dict, if there
+    # is none.  This logic was removed as costly (it requires to get at
+    # the gettopframe_nohidden()).  I bet no test fails, and it's a really
+    # obscure case.
 
     return codeobj.exec_code(space, w_globals, w_locals)
 

File pypy/module/__builtin__/functional.py

View file
     """
     return min_max(space, __args__, "min")
 
-@unwrap_spec(collections_w="args_w")
-def map(space, w_func, collections_w):
-    """does 3 separate things, hence this enormous docstring.
-       1.  if function is None, return a list of tuples, each with one
-           item from each collection.  If the collections have different
-           lengths,  shorter ones are padded with None.
-
-       2.  if function is not None, and there is only one collection,
-           apply function to every item in the collection and return a
-           list of the results.
-
-       3.  if function is not None, and there are several collections,
-           repeatedly call the function with one argument from each
-           collection.  If the collections have different lengths,
-           shorter ones are padded with None
-    """
-    if not collections_w:
-        msg = "map() requires at least two arguments"
-        raise OperationError(space.w_TypeError, space.wrap(msg))
-    none_func = space.is_w(w_func, space.w_None)
-    if len(collections_w) == 1:
-        w_collection = collections_w[0]
-        if none_func:
-            result_w = space.unpackiterable(w_collection)
-        else:
-            result_w = map_single_collection(space, w_func, w_collection)
-    else:
-        result_w = map_multiple_collections(space, w_func, collections_w,
-                                            none_func)
-    return space.newlist(result_w)
-
-def map_single_collection(space, w_func, w_collection):
-    """Special case for 'map(func, coll)', where 'func' is not None and there