Commits

Remi Meier committed 2a83d4e

implement the additional barriers with fastpaths in assembler.py and gc.py

  • Participants
  • Parent commits afce459
  • Branches stmgc-c4

Comments (0)

Files changed (6)

File rpython/jit/backend/llsupport/assembler.py

         self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn)
         if gc_ll_descr.stm:
             descrs = [gc_ll_descr.A2Rdescr, gc_ll_descr.Q2Rdescr,
+                      gc_ll_descr.A2Idescr, gc_ll_descr.A2Vdescr,
                       gc_ll_descr.A2Wdescr, gc_ll_descr.V2Wdescr]
         else:
             descrs = [gc_ll_descr.write_barrier_descr]

File rpython/jit/backend/llsupport/descr.py

     field_size = 0
     flag = '\x00'
     stm_dont_track_raw_accesses = False
-    immutable = False
+    _immutable = False
 
     def __init__(self, name, offset, field_size, flag,
                  stm_dont_track_raw_accesses=False,
         self.field_size = field_size
         self.flag = flag
         self.stm_dont_track_raw_accesses = stm_dont_track_raw_accesses
-        self.immutable = immutable
+        self._immutable = immutable
 
     def is_immutable(self):
-        return self.immutable
+        return self._immutable
     
     def is_pointer_field(self):
         return self.flag == FLAG_POINTER
         name = '%s.%s' % (STRUCT._name, fieldname)
         stm_dont_track_raw_accesses = STRUCT._hints.get(
             'stm_dont_track_raw_accesses', False)
-        immutable = STRUCT._immutable_field(fieldname)
+        immutable = bool(STRUCT._immutable_field(fieldname))
         fielddescr = FieldDescr(name, offset, size, flag,
                                 stm_dont_track_raw_accesses,
                                 immutable)
     lendescr = None
     flag = '\x00'
     vinfo = None
-    immutable = False
+    _immutable = False
 
     def __init__(self, basesize, itemsize, lendescr, flag,
                  immutable=False):
         self.itemsize = itemsize
         self.lendescr = lendescr    # or None, if no length
         self.flag = flag
-        self.immutable = immutable
+        self._immutable = immutable
 
     def is_immutable(self):
-        return self.immutable
+        return self._immutable
     
     def is_array_of_pointers(self):
         return self.flag == FLAG_POINTER
         else:
             lendescr = get_field_arraylen_descr(gccache, ARRAY_OR_STRUCT)
         flag = get_type_flag(ARRAY_INSIDE.OF)
-        immutable = ARRAY_INSIDE._immutable_field()
+        immutable = bool(ARRAY_INSIDE._immutable_field())
         arraydescr = ArrayDescr(basesize, itemsize, lendescr, flag,
                                 immutable)
         if ARRAY_OR_STRUCT._gckind == 'gc':
 class InteriorFieldDescr(AbstractDescr):
     arraydescr = ArrayDescr(0, 0, None, '\x00')  # workaround for the annotator
     fielddescr = FieldDescr('', 0, 0, '\x00')
-    immutable = False
+    _immutable = False
 
     def __init__(self, arraydescr, fielddescr, immutable=False):
         assert arraydescr.flag == FLAG_STRUCT
         self.arraydescr = arraydescr
         self.fielddescr = fielddescr
-        self.immutable = immutable
+        self._immutable = immutable
 
     def is_immutable(self):
-        return self.immutable
+        return self._immutable
     
     def sort_key(self):
         return self.fielddescr.sort_key()
         else:
             REALARRAY = getattr(ARRAY, arrayfieldname)
         fielddescr = get_field_descr(gc_ll_descr, REALARRAY.OF, name)
-        immutable = arraydescr.is_immutable() or fielddescr.is_immutable()
+        immutable = bool(arraydescr.is_immutable() or fielddescr.is_immutable())
         descr = InteriorFieldDescr(arraydescr, fielddescr, immutable)
         cache[(ARRAY, name, arrayfieldname)] = descr
         return descr

File rpython/jit/backend/llsupport/gc.py

 
 class STMReadBarrierDescr(STMBarrierDescr):
     def __init__(self, gc_ll_descr, stmcat):
-        assert stmcat in ['A2R', 'Q2R']
-        if stmcat == 'A2R':
-            STMBarrierDescr.__init__(self, gc_ll_descr, stmcat,
-                                     'stm_DirectReadBarrier')
-        else:
-            STMBarrierDescr.__init__(self, gc_ll_descr, stmcat,
-                                     'stm_RepeatReadBarrier')
+        assert stmcat in ['A2R', 'Q2R', 'A2I']
+        func = {'A2R': 'stm_DirectReadBarrier',
+                'Q2R': 'stm_RepeatReadBarrier',
+                'A2I': 'stm_ImmutReadBarrier',
+                }
+        
+        STMBarrierDescr.__init__(self, gc_ll_descr, stmcat,
+                                 func[stmcat])
 
     @specialize.arg(2)
     def _do_barrier(self, gcref_struct, returns_modified_object):
             rcp = rffi.cast(CP, read_cache[0])
             if rcp[index] == objint:
                 return gcref_struct
-        else: # 'Q2R'
+        elif self.stmcat == 'Q2R':
             # is GCFLAG_PUBLIC_TO_PRIVATE or GCFLAG_MOVED set?
             if not (objhdr.h_tid &
                     (StmGC.GCFLAG_PUBLIC_TO_PRIVATE | StmGC.GCFLAG_MOVED)):
                 # no.
                 return gcref_struct
+        else: # A2I
+            # GCFLAG_STUB set?
+            if not (objhdr.h_tid & StmGC.GCFLAG_STUB):
+                return gcref_struct
         
         funcptr = self.get_barrier_funcptr(returns_modified_object)
         res = funcptr(objadr)
         
 class STMWriteBarrierDescr(STMBarrierDescr):
     def __init__(self, gc_ll_descr, stmcat):
-        assert stmcat in ['A2W', 'V2W']
-        if stmcat == 'A2W':
-            STMBarrierDescr.__init__(self, gc_ll_descr, stmcat,
-                                     'stm_WriteBarrier')
-        else:
-            STMBarrierDescr.__init__(self, gc_ll_descr, stmcat,
-                                     'stm_RepeatWriteBarrier')
+        assert stmcat in ['A2W', 'V2W', 'A2V']
+        func = {'A2W':'stm_WriteBarrier',
+                'V2W':'stm_RepeatWriteBarrier',
+                'A2V':'stm_WriteBarrier',
+                }
+
+        STMBarrierDescr.__init__(self, gc_ll_descr, stmcat,
+                                 func[stmcat])
 
 
     @specialize.arg(2)
         from rpython.memory.gc.stmgc import StmGC
         objadr = llmemory.cast_ptr_to_adr(gcref_struct)
         objhdr = rffi.cast(StmGC.GCHDRP, gcref_struct)
+
+        # for A2W, we check h_revision and WRITE_BARRIER
+        # for V2W, we only check WRITE_BARRIER
+        # for A2V, we only check h_revision
         
         # if it is a repeated WB or h_revision == privat_rev of transaction
         priv_rev = self.llop1.stm_get_adr_of_private_rev_num(rffi.SIGNEDP)
         if self.stmcat == 'V2W' or objhdr.h_revision == priv_rev[0]:
             # also WRITE_BARRIER not set?
-            if not (objhdr.h_tid & StmGC.GCFLAG_WRITE_BARRIER):
+            if (self.stmcat == 'A2V'
+                or not (objhdr.h_tid & StmGC.GCFLAG_WRITE_BARRIER)):
                 return gcref_struct
         
         funcptr = self.get_barrier_funcptr(returns_modified_object)
     def _setup_write_barrier(self):
         if self.stm:
             self.A2Rdescr = STMReadBarrierDescr(self, 'A2R')
-            self.A2Idescr = STMReadBarrierDescr(self, 'A2R') # XXX
+            self.A2Idescr = STMReadBarrierDescr(self, 'A2I')
             self.Q2Rdescr = STMReadBarrierDescr(self, 'Q2R')
             self.A2Wdescr = STMWriteBarrierDescr(self, 'A2W')
-            self.A2Vdescr = STMWriteBarrierDescr(self, 'A2W') # XXX
+            self.A2Vdescr = STMWriteBarrierDescr(self, 'A2V')
             self.V2Wdescr = STMWriteBarrierDescr(self, 'V2W')
             self.write_barrier_descr = "wbdescr: do not use"
         else:

File rpython/jit/backend/llsupport/test/test_descr.py

             descr_s = get_size_descr(c0, STRUCT)
             assert descr_s.count_fields_if_immutable() == expected
 
+def test_is_immutable():
+    U = lltype.Struct('U', ('x', lltype.Char),
+                      hints={'immutable':True})
+    V = lltype.Struct('V', ('x', lltype.Char))
+    gc = GcCache(False)
+    assert get_field_descr(gc, U, 'x').is_immutable()
+    assert not get_field_descr(gc, V, 'x').is_immutable()
+
+    A1 = lltype.GcArray(lltype.Char, hints={'immutable':True})
+    A2 = lltype.GcArray(lltype.Char)
+    assert get_array_descr(gc, A1).is_immutable()
+    assert not get_array_descr(gc, A2).is_immutable()
+
+    I1 = lltype.GcArray(('z', lltype.Char), hints={'immutable':True})
+    I2 = lltype.GcArray(('z', lltype.Char))
+    assert get_interiorfield_descr(gc, I1, 'z').is_immutable()
+    assert not get_interiorfield_descr(gc, I2, 'z').is_immutable()
+    
+    
 def test_get_field_descr():
     U = lltype.Struct('U')
     T = lltype.GcStruct('T')

File rpython/jit/backend/x86/assembler.py

             helper_num += 2
         #
         # FASTPATH:
-        #
+        # do slowpath IF:
         # A2W:
         # (obj->h_revision != stm_private_rev_num)
         #     || (obj->h_tid & GCFLAG_WRITE_BARRIER) != 0)
         # V2W:
         # (obj->h_tid & GCFLAG_WRITE_BARRIER) != 0)
+        # A2V:
+        # (obj->h_revision != stm_private_rev_num)
         # A2R:
         # (obj->h_revision != stm_private_rev_num)
         #     && (FXCACHE_AT(obj) != obj)))
         # Q2R:
         # (obj->h_tid & (GCFLAG_PUBLIC_TO_PRIVATE | GCFLAG_MOVED) != 0)
+        # A2I:
+        # (obj->h_tid & GCFLAG_STUB)
         if IS_X86_32:   # XXX: todo
             todo()
         jz_location = 0
         jz_location2 = 0
         jnz_location = 0
         # compare h_revision with stm_private_rev_num
-        if descr.stmcat in ['A2W', 'A2R']:
+        if descr.stmcat in ['A2W', 'A2R', 'A2V']:
             rn = self._get_stm_private_rev_num_addr()
             if we_are_translated():
                 # during tests, _get_stm_private_rev_num_addr returns
             else:
                 mc.CMP(X86_64_SCRATCH_REG, mem(loc_base, StmGC.H_REVISION))
             #
-            if descr.stmcat == 'A2R':
+            if descr.stmcat in ('A2R', 'A2V'):
                 # jump to end if h_rev==priv_rev
                 mc.J_il8(rx86.Conditions['Z'], 0) # patched below
                 jz_location = mc.get_relative_pos()
-            else: # write_barrier
+            else: # A2W
                 # jump to slowpath if h_rev!=priv_rev
                 mc.J_il8(rx86.Conditions['NZ'], 0) # patched below
                 jnz_location = mc.get_relative_pos()
             jz_location2 = mc.get_relative_pos()
         #
         # check flags:
-        if descr.stmcat in ['A2W', 'V2W', 'Q2R']:
+        if descr.stmcat in ['A2W', 'V2W', 'Q2R', 'A2I']:
             flags = 0
+            off = 0
             if descr.stmcat in ['A2W', 'V2W']:
                 # obj->h_tid & GCFLAG_WRITE_BARRIER) != 0
-                assert IS_X86_64 and (StmGC.GCFLAG_WRITE_BARRIER >> 32) > 0
-                assert (StmGC.GCFLAG_WRITE_BARRIER >> 40) == 0
-                flags = StmGC.GCFLAG_WRITE_BARRIER >> 32
+                flags = StmGC.GCFLAG_WRITE_BARRIER
             elif descr.stmcat == 'Q2R':
                 # obj->h_tid & PUBLIC_TO_PRIVATE|MOVED
                 flags = StmGC.GCFLAG_PUBLIC_TO_PRIVATE | StmGC.GCFLAG_MOVED
-                assert IS_X86_64 and (flags >> 32) > 0
-                assert (flags >> 40) == 0
+            elif descr.stmcat == 'A2I':
+                # obj->h_tid & STUB
+                flags = StmGC.GCFLAG_STUB
+
+            assert IS_X86_64
+            if (flags >> 32) > 0 and (flags >> 40) == 0:
                 flags = flags >> 32
-
-            off = 4
+                off = 4
+            elif (flags >> 40) > 0 and (flags >> 48) == 0:
+                flags = flags >> 40
+                off = 5
+            #
             if loc_base == ebp:
                 mc.TEST8_bi(StmGC.H_TID + off, flags)
             else:

File rpython/jit/backend/x86/test/test_stm_integration.py

             return obj
 
         self.A2Rdescr = FakeSTMBarrier(self, 'A2R', read_barrier)
+        self.A2Idescr = FakeSTMBarrier(self, 'A2I', read_barrier)
         self.Q2Rdescr = FakeSTMBarrier(self, 'Q2R', read_barrier)
         self.A2Wdescr = FakeSTMBarrier(self, 'A2W', write_barrier)
+        self.A2Vdescr = FakeSTMBarrier(self, 'A2V', write_barrier)
         self.V2Wdescr = FakeSTMBarrier(self, 'V2W', write_barrier)
         
         self.do_write_barrier = None
         
 
         self.a2wd = cpu.gc_ll_descr.A2Wdescr
+        self.a2vd = cpu.gc_ll_descr.A2Vdescr
         self.v2wd = cpu.gc_ll_descr.V2Wdescr
         self.a2rd = cpu.gc_ll_descr.A2Rdescr
+        self.a2id = cpu.gc_ll_descr.A2Idescr
         self.q2rd = cpu.gc_ll_descr.Q2Rdescr
 
         TP = rffi.CArray(lltype.Signed)
             else:
                 self.assert_in(called, [sgcref])
 
+    def test_gc_immutable_read_barrier_fastpath(self):
+        from rpython.jit.backend.llsupport.gc import STMReadBarrierDescr
+        descr = STMReadBarrierDescr(self.cpu.gc_ll_descr, 'A2I')
+
+        called = []
+        def read(obj):
+            called.append(obj)
+            return obj
+
+        functype = lltype.Ptr(lltype.FuncType(
+            [llmemory.Address], llmemory.Address))
+        funcptr = llhelper(functype, read)
+        descr.b_failing_case_ptr = funcptr
+        descr.llop1 = fakellop()
+
+        # -------- TEST --------
+        for flags in [StmGC.GCFLAG_STUB, 0]:
+            called[:] = []
+            
+            s = self.allocate_prebuilt_s()
+            sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s)
+            s.h_tid |= flags
+
+            descr._do_barrier(sgcref, returns_modified_object=True)
+                        
+            # check if rev-fastpath worked
+            if not flags:
+                # fastpath
+                self.assert_not_in(called, [sgcref])
+            else:
+                self.assert_in(called, [sgcref])
+
+
 
     def test_gc_write_barrier_fastpath(self):
         from rpython.jit.backend.llsupport.gc import STMWriteBarrierDescr
         descr._do_barrier(sgcref, 
             returns_modified_object=True)
         self.assert_in(called, [sgcref])
-        
-        
-        
+
+    def test_gc_noptr_write_barrier_fastpath(self):
+        from rpython.jit.backend.llsupport.gc import STMWriteBarrierDescr
+        descr = STMWriteBarrierDescr(self.cpu.gc_ll_descr, 'A2V')
+
+        called = []
+        def write(obj):
+            called.append(obj)
+            return obj
+
+        functype = lltype.Ptr(lltype.FuncType(
+            [llmemory.Address], llmemory.Address))
+        funcptr = llhelper(functype, write)
+        descr.b_failing_case_ptr = funcptr
+        descr.llop1 = fakellop()
+
+        # -------- TEST --------
+        for rev in [fakellop.PRIV_REV+4, fakellop.PRIV_REV]:
+            called[:] = []
+            
+            s = self.allocate_prebuilt_s()
+            sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s)
+            s.h_revision = rev
+
+            descr._do_barrier(sgcref, returns_modified_object=True)
+                        
+            # check if fastpath worked
+            if rev == fakellop.PRIV_REV:
+                # fastpath
+                self.assert_not_in(called, [sgcref])
+            else:
+                self.assert_in(called, [sgcref])
+                
+            # now set WRITE_BARRIER -> no effect
+            called[:] = []
+            s.h_tid |= StmGC.GCFLAG_WRITE_BARRIER
+            descr._do_barrier(sgcref, returns_modified_object=True)
+            if rev == fakellop.PRIV_REV:
+                # fastpath
+                self.assert_not_in(called, [sgcref])
+            else:
+                self.assert_in(called, [sgcref])
+
+                        
         
     def test_read_barrier_fastpath(self):
         cpu = self.cpu
             else:
                 self.assert_in(called_on, [sgcref])
 
+    def test_immutable_read_barrier_fastpath(self):
+        cpu = self.cpu
+        cpu.gc_ll_descr.init_nursery(100)
+        cpu.setup_once()
+
+        called_on = cpu.gc_ll_descr.rb_called_on
+        for flags in [StmGC.GCFLAG_STUB, 0]:
+            cpu.gc_ll_descr.clear_lists()
+            self.clear_read_cache()
+            
+            s = self.allocate_prebuilt_s()
+            sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s)
+            s.h_tid |= flags
+            
+            p0 = BoxPtr()
+            operations = [
+                ResOperation(rop.COND_CALL_STM_B, [p0], None,
+                             descr=self.a2id),
+                ResOperation(rop.FINISH, [p0], None, 
+                             descr=BasicFinalDescr(0)),
+                ]
+            inputargs = [p0]
+            looptoken = JitCellToken()
+            cpu.compile_loop(None, inputargs, operations, looptoken)
+            self.cpu.execute_token(looptoken, sgcref)
+            
+            # check if rev-fastpath worked
+            if not flags:
+                # fastpath
+                self.assert_not_in(called_on, [sgcref])
+            else:
+                self.assert_in(called_on, [sgcref])
+
+
 
     def test_write_barrier_fastpath(self):
         cpu = self.cpu
         self.cpu.execute_token(looptoken, sgcref)
         self.assert_in(called_on, [sgcref])
 
+    def test_noptr_write_barrier_fastpath(self):
+        cpu = self.cpu
+        cpu.gc_ll_descr.init_nursery(100)
+        cpu.setup_once()
+        PRIV_REV = rffi.cast(lltype.Signed, StmGC.PREBUILT_REVISION)
+        self.priv_rev_num[0] = PRIV_REV
+        called_on = cpu.gc_ll_descr.wb_called_on
+        
+        for rev in [PRIV_REV+4, PRIV_REV]:
+            cpu.gc_ll_descr.clear_lists()
+            
+            s = self.allocate_prebuilt_s()
+            sgcref = lltype.cast_opaque_ptr(llmemory.GCREF, s)
+            s.h_revision = rev
+            
+            p0 = BoxPtr()
+            operations = [
+                ResOperation(rop.COND_CALL_STM_B, [p0], None,
+                             descr=self.a2vd),
+                ResOperation(rop.FINISH, [p0], None, 
+                             descr=BasicFinalDescr(0)),
+                ]
+            inputargs = [p0]
+            looptoken = JitCellToken()
+            cpu.compile_loop(None, inputargs, operations, looptoken)
+            self.cpu.execute_token(looptoken, sgcref)
+            
+            # check if rev-fastpath worked
+            if rev == PRIV_REV:
+                # fastpath and WRITE_BARRIER not set
+                self.assert_not_in(called_on, [sgcref])
+            else:
+                self.assert_in(called_on, [sgcref])
+
+            # now set WRITE_BARRIER -> no effect
+            cpu.gc_ll_descr.clear_lists()
+            s.h_tid |= StmGC.GCFLAG_WRITE_BARRIER
+            self.cpu.execute_token(looptoken, sgcref)
+            if rev == PRIV_REV:
+                # fastpath and WRITE_BARRIER not set
+                self.assert_not_in(called_on, [sgcref])
+            else:
+                self.assert_in(called_on, [sgcref])
+
             
     def test_ptr_eq_fastpath(self):
         cpu = self.cpu