Commits

Armin Rigo committed d3aa922 Merge

Merge jit-ordereddict: add minimal support in the JIT for OrderedDict,
for bytearray, and for a few extra details, motivated by Hippy.

Comments (0)

Files changed (17)

rpython/annotator/binaryop.py

         return SomeByteArray(can_be_None=can_be_None)
 
     def add((b1, b2)):
-        result = SomeByteArray()
-        if b1.is_immutable_constant() and b2.is_immutable_constant():
-            result.const = b1.const + b2.const
-        return result
+        return SomeByteArray()
 
 class __extend__(pairtype(SomeByteArray, SomeInteger)):
     def getitem((s_b, s_i)):
                  pairtype(SomeChar, SomeByteArray),
                  pairtype(SomeByteArray, SomeChar)):
     def add((b1, b2)):
-        result = SomeByteArray()
-        if b1.is_immutable_constant() and b2.is_immutable_constant():
-            result.const = b1.const + b2.const
-        return result
+        return SomeByteArray()
 
 class __extend__(pairtype(SomeChar, SomeChar)):
 

rpython/annotator/builtin.py

     return constpropagate(unicode, [s_unicode], SomeUnicodeString())
 
 def builtin_bytearray(s_str):
-    return constpropagate(bytearray, [s_str], SomeByteArray())
+    return SomeByteArray()
 
 def our_issubclass(cls1, cls2):
     """ we're going to try to be less silly in the face of old-style classes"""

rpython/annotator/model.py

 
 
 class SomeStringOrUnicode(SomeObject):
-    """Base class for shared implementation of SomeString and SomeUnicodeString.
+    """Base class for shared implementation of SomeString,
+    SomeUnicodeString and SomeByteArray.
 
     Cannot be an annotation."""
 
         if can_be_None:
             self.can_be_None = True
         if no_nul:
+            assert self.immutable   #'no_nul' cannot be used with SomeByteArray
             self.no_nul = True
 
     def can_be_none(self):
 
 
 class SomeByteArray(SomeStringOrUnicode):
+    immutable = False
     knowntype = bytearray
 
 

rpython/annotator/test/test_annrpython.py

             return bytearray("xyz")
 
         a = self.RPythonAnnotator()
-        assert isinstance(a.build_types(f, []), annmodel.SomeByteArray)
+        s = a.build_types(f, [])
+        assert isinstance(s, annmodel.SomeByteArray)
+        assert not s.is_constant()   # never a constant!
 
     def test_bytearray_add(self):
         def f(a):

rpython/jit/backend/llgraph/runner.py

 
 class ArrayDescr(AbstractDescr):
     def __init__(self, A):
-        self.A = A
+        self.A = self.OUTERA = A
+        if isinstance(A, lltype.Struct):
+            self.A = A._flds[A._arrayfld]
 
     def __repr__(self):
-        return 'ArrayDescr(%r)' % (self.A,)
+        return 'ArrayDescr(%r)' % (self.OUTERA,)
 
     def is_array_of_pointers(self):
         return getkind(self.A.OF) == 'ref'
 
     def bh_arraylen_gc(self, a, descr):
         array = a._obj.container
+        if descr.A is not descr.OUTERA:
+            array = getattr(array, descr.OUTERA._arrayfld)
         return array.getlength()
 
     def bh_getarrayitem_gc(self, a, index, descr):

rpython/jit/backend/llsupport/test/test_descr.py

                          " <Array of Char > > >")
     # caching:
     assert fielddescr is get_field_arraylen_descr(c0, rstr.STR)
+
+def test_bytearray_descr():
+    c0 = GcCache(False)
+    descr = get_array_descr(c0, rstr.STR)   # for bytearray
+    assert descr.flag == FLAG_UNSIGNED
+    assert descr.basesize == struct.calcsize("PP")         # hash, length
+    assert descr.lendescr.offset == struct.calcsize("P")   # hash
+    assert not descr.is_array_of_pointers()

rpython/jit/codewriter/jtransform.py

 from rpython.rlib.jit import _we_are_jitted
 from rpython.rlib.rgc import lltype_is_gc
 from rpython.rtyper.lltypesystem import lltype, llmemory, rstr, rclass, rffi
+from rpython.rtyper.lltypesystem import rbytearray
 from rpython.rtyper.rclass import IR_QUASIIMMUTABLE, IR_QUASIIMMUTABLE_ARRAY
 from rpython.translator.unsimplify import varoftype
 
         return SpaceOperation('arraylen_gc', [op.args[0], arraydescr],
                               op.result)
 
+    def rewrite_op_getarraysubstruct(self, op):
+        ARRAY = op.args[0].concretetype.TO
+        assert ARRAY._gckind == 'raw'
+        assert ARRAY._hints.get('nolength') is True
+        return self.rewrite_op_direct_ptradd(op)
+
     def _array_of_voids(self, ARRAY):
         return ARRAY.OF == lltype.Void
 
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strlen"
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodelen"
+        elif optype == lltype.Ptr(rbytearray.BYTEARRAY):
+            bytearraydescr = self.cpu.arraydescrof(rbytearray.BYTEARRAY)
+            return SpaceOperation('arraylen_gc', [op.args[0], bytearraydescr],
+                                  op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodelen"
+            assert 0, "supported type %r" % (optype,)
         return SpaceOperation(opname, [op.args[0]], op.result)
 
     def rewrite_op_getinteriorfield(self, op):
         elif optype == lltype.Ptr(rstr.UNICODE):
             opname = "unicodegetitem"
             return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+        elif optype == lltype.Ptr(rbytearray.BYTEARRAY):
+            bytearraydescr = self.cpu.arraydescrof(rbytearray.BYTEARRAY)
+            v_index = op.args[2]
+            return SpaceOperation('getarrayitem_gc_i',
+                                  [op.args[0], v_index, bytearraydescr],
+                                  op.result)
         else:
             v_inst, v_index, c_field = op.args
             if op.result.concretetype is lltype.Void:
             opname = "unicodesetitem"
             return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
                                   op.result)
+        elif optype == lltype.Ptr(rbytearray.BYTEARRAY):
+            bytearraydescr = self.cpu.arraydescrof(rbytearray.BYTEARRAY)
+            opname = "setarrayitem_gc_i"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3],
+                                           bytearraydescr], op.result)
         else:
             v_inst, v_index, c_field, v_value = op.args
             if v_value.concretetype is lltype.Void:
                     "stroruni.copy_string_to_raw": EffectInfo.OS_UNI_COPY_TO_RAW
                     }
             CHR = lltype.UniChar
+        elif SoU.TO == rbytearray.BYTEARRAY:
+            raise NotSupported("bytearray operation")
         else:
             assert 0, "args[0].concretetype must be STR or UNICODE"
         #

rpython/jit/codewriter/support.py

 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.rtyper.llinterp import LLInterpreter
 from rpython.rtyper.lltypesystem import lltype, rclass, rffi, llmemory, rstr as ll_rstr, rdict as ll_rdict
+from rpython.rtyper.lltypesystem import rordereddict
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.lltypesystem.module import ll_math
 from rpython.translator.translator import TranslationContext
 
     # ---------- dict ----------
 
-    def _ll_0_newdict(DICT):
-        return ll_rdict.ll_newdict(DICT)
-    _ll_0_newdict.need_result_type = True
-
-    _ll_2_dict_delitem = ll_rdict.ll_dict_delitem
     _ll_1_dict_copy = ll_rdict.ll_copy
     _ll_1_dict_clear = ll_rdict.ll_clear
     _ll_2_dict_update = ll_rdict.ll_update
 
     _ll_1_dict_resize = ll_rdict.ll_dict_resize
 
+    # ---------- ordered dict ----------
+
+    _ll_1_odict_copy = rordereddict.ll_dict_copy
+    _ll_1_odict_clear = rordereddict.ll_dict_clear
+    _ll_2_odict_update = rordereddict.ll_dict_update
+
+    _ll_1_odict_keys   = rordereddict.ll_dict_keys
+    _ll_1_odict_values = rordereddict.ll_dict_values
+    _ll_1_odict_items  = rordereddict.ll_dict_items
+    _ll_1_odict_keys  .need_result_type = True
+    _ll_1_odict_values.need_result_type = True
+    _ll_1_odict_items .need_result_type = True
+
+    _odictnext_keys   = staticmethod(rordereddict.ll_dictnext_group['keys'])
+    _odictnext_values = staticmethod(rordereddict.ll_dictnext_group['values'])
+    _odictnext_items  = staticmethod(rordereddict.ll_dictnext_group['items'])
+
+    def _ll_1_odictiter_nextkeys(iter):
+        return LLtypeHelpers._odictnext_keys(None, iter)
+    def _ll_1_odictiter_nextvalues(iter):
+        return LLtypeHelpers._odictnext_values(None, iter)
+    def _ll_1_odictiter_nextitems(RES, iter):
+        return LLtypeHelpers._odictnext_items(lltype.Ptr(RES), iter)
+    _ll_1_odictiter_nextitems.need_result_type = True
+
+    _ll_1_odict_resize = rordereddict.ll_dict_resize
+
     # ---------- strings and unicode ----------
 
     _ll_1_str_str2unicode = ll_rstr.LLHelpers.ll_str2unicode

rpython/jit/metainterp/test/test_bytearray.py

+import py
+from rpython.jit.metainterp.test.support import LLJitMixin
+from rpython.rlib.jit import JitDriver, dont_look_inside
+
+class TestByteArray(LLJitMixin):
+
+    def test_getitem(self):
+        x = bytearray("foobar")
+        def fn(n):
+            assert n >= 0
+            return x[n]
+        res = self.interp_operations(fn, [3])
+        assert res == ord('b')
+
+    def test_getitem_negative(self):
+        x = bytearray("foobar")
+        def fn(n):
+            return x[n]
+        res = self.interp_operations(fn, [-2])
+        assert res == ord('a')
+
+    def test_len(self):
+        x = bytearray("foobar")
+        def fn(n):
+            return len(x)
+        res = self.interp_operations(fn, [3])
+        assert res == 6
+
+    def test_setitem(self):
+        @dont_look_inside
+        def make_me():
+            return bytearray("foobar")
+        def fn(n):
+            assert n >= 0
+            x = make_me()
+            x[n] = 3
+            return x[3] + 1000 * x[4]
+
+        res = self.interp_operations(fn, [3])
+        assert res == 3 + 1000 * ord('a')
+
+    def test_setitem_negative(self):
+        @dont_look_inside
+        def make_me():
+            return bytearray("foobar")
+        def fn(n):
+            x = make_me()
+            x[n] = 3
+            return x[3] + 1000 * x[4]
+
+        res = self.interp_operations(fn, [-2])
+        assert res == ord('b') + 1000 * 3
+
+    def test_new_bytearray(self):
+        def fn(n, m):
+            x = bytearray(str(n))
+            x[m] = 0x34
+            return int(str(x))
+
+        assert fn(610978, 3) == 610478
+        res = self.interp_operations(fn, [610978, 3])
+        assert res == 610478
+
+    def test_slice(self):
+        py.test.skip("XXX later")
+        def fn(n, m):
+            x = bytearray(str(n))
+            x = x[1:5]
+            x[m] = 0x35
+            return int(str(x))
+        res = self.interp_operations(fn, [610978, 1])
+        assert res == 1597
+
+    def test_bytearray_from_bytearray(self):
+        def fn(n, m):
+            x = bytearray(str(n))
+            y = bytearray(x)
+            x[m] = 0x34
+            return int(str(x)) + int(str(y))
+
+        res = self.interp_operations(fn, [610978, 3])
+        assert res == 610478 + 610978

rpython/jit/metainterp/test/test_dict.py

 from rpython.jit.metainterp.test.support import LLJitMixin
 from rpython.rlib.jit import JitDriver
 from rpython.rlib import objectmodel
+from collections import OrderedDict
 
 class DictTests:
+    def _freeze_(self):
+        return True
 
     def test_dict_set_none(self):
         def fn(n):
-            d = {}
+            d = self.newdict()
             d[0] = None
             return bool(d[n])
         res = self.interp_operations(fn, [0])
                                         ]:
             myjitdriver = JitDriver(greens = [], reds = ['n', 'dct'])
             def f(n):
-                dct = {}
+                dct = self.newdict()
                 while n > 0:
                     myjitdriver.can_enter_jit(n=n, dct=dct)
                     myjitdriver.jit_merge_point(n=n, dct=dct)
                                         ]:
             myjitdriver = JitDriver(greens = [], reds = ['total', 'it'])
             def f(n):
-                dct = {n: 100, 50: n+1}
+                dct = self.newdict()
+                dct[n] = 100
+                dct[50] = n + 1
                 it = getattr(dct, name)()
                 total = 0
                 while True:
             assert res == expected
 
     def test_dict_trace_hash(self):
+        if type(self.newdict()) is not dict:
+            py.test.skip("this is an r_dict test")
         myjitdriver = JitDriver(greens = [], reds = ['total', 'dct'])
         def key(x):
             return x % 2
     def test_dict_setdefault(self):
         myjitdriver = JitDriver(greens = [], reds = ['total', 'dct'])
         def f(n):
-            dct = {}
+            dct = self.newdict()
             total = n
             while total:
                 myjitdriver.jit_merge_point(total=total, dct=dct)
         self.check_resops(new=0, new_with_vtable=0)
 
     def test_dict_as_counter(self):
+        if type(self.newdict()) is not dict:
+            py.test.skip("this is an r_dict test")
         myjitdriver = JitDriver(greens = [], reds = ['total', 'dct'])
         def key(x):
             return x % 2
         self.check_resops(int_mod=2) # key + eq, but cached
 
     def test_repeated_lookup(self):
+        if type(self.newdict()) is not dict:
+            py.test.skip("this is an r_dict test")
         myjitdriver = JitDriver(greens = [], reds = ['n', 'd'])
         class Wrapper(object):
             _immutable_fields_ = ["value"]
         def f(n):
             while n > 0:
                 driver.jit_merge_point(n=n)
-                d = {1: 1}
+                d = self.newdict()
+                d[1] = 1
                 for elem in d:
                     n -= elem
             return n
 
 
 class TestLLtype(DictTests, LLJitMixin):
-    pass
+    @staticmethod
+    def newdict():
+        return {}
+
+class TestLLOrderedDict(DictTests, LLJitMixin):
+    @staticmethod
+    def newdict():
+        return OrderedDict()
+
+    def test_dict_is_ordered(self):
+        def fn(n):
+            d = OrderedDict()
+            d[3] = 5
+            d[n] = 9
+            d[2] = 6
+            d[1] = 4
+            lst = d.items()
+            assert len(lst) == 4
+            return (    lst[0][0] +       10*lst[0][1] +
+                    100*lst[1][0] +     1000*lst[1][1] +
+                  10000*lst[3][0] +   100000*lst[2][1] +
+                1000000*lst[2][0] + 10000000*lst[3][1])
+
+        res = self.interp_operations(fn, [0])
+        assert res == fn(0)
+
+    def test_unrolling_of_dict_iter(self):
+        py.test.skip("XXX fix me: ordereddict generates a mess for now")

rpython/jit/metainterp/test/test_rawmem.py

                                        'raw_store': 1, 'raw_load': 1,
                                        'finish': 1})
 
+    def test_getarraysubstruct(self):
+        A2 = lltype.Array(('a', lltype.Signed), ('b', lltype.Signed),
+                          hints={'nolength': True})
+        p = lltype.malloc(A2, 10, flavor='raw', immortal=True, zero=True)
+        p[2].b = 689
+        def f(n, m):
+            p[n].a = 55
+            p[n].b = 44
+            p[4].b = 66
+            return p[m].b
+
+        # run with 'disable_optimizations' to prevent an error
+        # 'Symbolics cannot be compared!' in the optimizer for int_mul
+        res = self.interp_operations(f, [7, 2], disable_optimizations=True)
+        assert res == 689
+        res = self.interp_operations(f, [7, 4], disable_optimizations=True)
+        assert res == 66
+        res = self.interp_operations(f, [2, 2], disable_optimizations=True)
+        assert res == 44
+
 class TestRawMem(RawMemTests, LLJitMixin):
     pass

rpython/rtyper/lltypesystem/ll2ctypes.py

         # bigger structure at once
         parent, parentindex = lltype.parentlink(container)
         if parent is not None:
-            convert_struct(parent)
+            if isinstance(parent, lltype._struct):
+                convert_struct(parent)
+            elif isinstance(parent, lltype._array):
+                convert_array(parent)
+            else:
+                raise AssertionError(type(parent))
             return
         # regular case: allocate a new ctypes Structure of the proper type
         cls = get_ctypes_type(STRUCT)

rpython/rtyper/lltypesystem/llmemory.py

         if isinstance(ofs, FieldOffset) and ofs.TYPE is self.adr.ptr._TYPE.TO:
             fieldadr = getattr(self.adr.ptr, ofs.fldname)
             return AddressAsInt(cast_ptr_to_adr(fieldadr))
+        if (isinstance(ofs, ItemOffset) and
+            isinstance(self.adr.ptr._TYPE.TO, lltype.Array) and
+            self.adr.ptr._TYPE.TO._hints.get('nolength') is True and
+            ofs.TYPE is self.adr.ptr._TYPE.TO.OF):
+            itemadr = self.adr.ptr[ofs.repeat]
+            return AddressAsInt(cast_ptr_to_adr(itemadr))
         return NotImplemented
     def __repr__(self):
         try:

rpython/rtyper/lltypesystem/rordereddict.py

         ll_dict_remove_deleted_items(d)
     else:
         ll_dict_reindex(d, new_size)
-ll_dict_resize.oopspec = 'dict.resize(d)'
+ll_dict_resize.oopspec = 'odict.resize(d)'
 
 def ll_dict_reindex(d, new_size):
     ll_malloc_indexes_and_choose_lookup(d, new_size)
     @jit.look_inside_iff(lambda RETURNTYPE, iter: jit.isvirtual(iter)
                          and (iter.dict is None or
                               jit.isvirtual(iter.dict)))
-    @jit.oopspec("dictiter.next%s(iter)" % kind)
+    @jit.oopspec("odictiter.next%s(iter)" % kind)
     def ll_dictnext(RETURNTYPE, iter):
         # note that RETURNTYPE is None for keys and values
         dict = iter.dict
 
     ll_dict_reindex(newdict, _ll_len_of_d_indexes(dict))
     return newdict
-ll_dict_copy.oopspec = 'dict.copy(dict)'
+ll_dict_copy.oopspec = 'odict.copy(dict)'
 
 def ll_dict_clear(d):
     if d.num_used_items == 0:
     d.num_used_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     # old_entries.delete() XXX
-ll_dict_clear.oopspec = 'dict.clear(d)'
+ll_dict_clear.oopspec = 'odict.clear(d)'
 
 def ll_dict_update(dic1, dic2):
     i = 0
             index = dic1.lookup_function(dic1, key, hash, FLAG_STORE)
             _ll_dict_setitem_lookup_done(dic1, key, value, hash, index)
         i += 1
-ll_dict_update.oopspec = 'dict.update(dic1, dic2)'
+ll_dict_update.oopspec = 'odict.update(dic1, dic2)'
 
 # this is an implementation of keys(), values() and items()
 # in a single function.
             i += 1
         assert p == res.ll_length()
         return res
-    ll_kvi.oopspec = 'dict.%s(dic)' % kind
+    ll_kvi.oopspec = 'odict.%s(dic)' % kind
     return ll_kvi
 
 ll_dict_keys   = _make_ll_keys_values_items('keys')

rpython/rtyper/lltypesystem/test/test_ll2ctypes.py

     def test_llgcopaque_eq(self):
         assert _llgcopaque(1) != None
         assert _llgcopaque(0) == None
+
+    def test_array_of_struct(self):
+        A2 = lltype.Array(('a', lltype.Signed), ('b', lltype.Signed))
+        a = lltype.malloc(A2, 10, flavor='raw')
+        a[3].b = 42
+        ac = lltype2ctypes(a[3])
+        assert ac.contents.b == 42
+        ac.contents.a = 17
+        assert a[3].a == 17
+        #lltype.free(a, flavor='raw')
+        py.test.skip("free() not working correctly here...")

rpython/rtyper/rstr.py

         return hop.gendirectcall(self.ll.ll_str2unicode, v_str)
 
     def rtype_bytearray(self, hop):
-        if hop.args_s[0].is_constant():
-            # convertion errors occur during annotation, so cannot any more:
-            hop.exception_cannot_occur()
-            return hop.inputconst(hop.r_result, hop.s_result.const)
         hop.exception_is_here()
         return hop.gendirectcall(self.ll.ll_str2bytearray,
                                  hop.inputarg(hop.args_r[0].repr, 0))

rpython/rtyper/test/test_rbytearray.py

 
     def test_getslice(self):
         def f(x):
-            return str(bytearray(str(x))[1:2])
+            b = bytearray(str(x))
+            b = b[1:3]
+            b[0] += 5
+            return str(b)
 
-        ll_res = self.interpret(f, [123])
-        assert hlstr(ll_res) == "2"
+        ll_res = self.interpret(f, [12345])
+        assert hlstr(ll_res) == f(12345) == "73"
+
+    def test_bytearray_not_constant(self):
+        for constant in ['f', 'foo']:
+            def f(x):
+                i = 0
+                total = 0
+                while i < x:
+                    b = bytearray(constant)
+                    b[0] = b[0] + 1
+                    total += b[0]
+                    i += 1
+                return total
+            ll_res = self.interpret(f, [5])
+            assert ll_res == f(5)