Commits

Maciej Fijalkowski  committed f09f03b

a bit of refactoring but now one can get to details about counters from the interpreter if he really feels like it

  • Participants
  • Parent commits ed92f26
  • Branches even-more-jit-hooks

Comments (0)

Files changed (10)

File pypy/jit/metainterp/jitprof.py

 from pypy.rlib.debug import debug_print, debug_start, debug_stop
 from pypy.rlib.debug import have_debug_prints
 from pypy.jit.metainterp.jitexc import JitException
+from pypy.rlib.jit import Counters
 
-counters="""
-TRACING
-BACKEND
-OPS
-RECORDED_OPS
-GUARDS
-OPT_OPS
-OPT_GUARDS
-OPT_FORCINGS
-ABORT_TOO_LONG
-ABORT_BRIDGE
-ABORT_BAD_LOOP
-ABORT_ESCAPE
-ABORT_FORCE_QUASIIMMUT
-NVIRTUALS
-NVHOLES
-NVREUSED
-TOTAL_COMPILED_LOOPS
-TOTAL_COMPILED_BRIDGES
-TOTAL_FREED_LOOPS
-TOTAL_FREED_BRIDGES
-"""
 
-counter_names = []
-
-def _setup():
-    names = counters.split()
-    for i, name in enumerate(names):
-        globals()[name] = i
-        counter_names.append(name)
-    global ncounters
-    ncounters = len(names)
-_setup()
-
-JITPROF_LINES = ncounters + 1 + 1 # one for TOTAL, 1 for calls, update if needed
+JITPROF_LINES = Counters.ncounters + 1 + 1
+# one for TOTAL, 1 for calls, update if needed
 _CPU_LINES = 4       # the last 4 lines are stored on the cpu
 
 class BaseProfiler(object):
     def count(self, kind, inc=1):
         pass
 
-    def count_ops(self, opnum, kind=OPS):
+    def count_ops(self, opnum, kind=Counters.OPS):
         pass
 
+    def get_counter(self, num):
+        return -1.0
+
 class Profiler(BaseProfiler):
     initialized = False
     timer = time.time
         self.starttime = self.timer()
         self.t1 = self.starttime
         self.times = [0, 0]
-        self.counters = [0] * (ncounters - _CPU_LINES)
+        self.counters = [0] * (Counters.ncounters - _CPU_LINES)
         self.calls = 0
         self.current = []
 
             return
         self.times[ev1] += self.t1 - t0
 
-    def start_tracing(self):   self._start(TRACING)
-    def end_tracing(self):     self._end  (TRACING)
+    def start_tracing(self):   self._start(Counters.TRACING)
+    def end_tracing(self):     self._end  (Counters.TRACING)
 
-    def start_backend(self):   self._start(BACKEND)
-    def end_backend(self):     self._end  (BACKEND)
+    def start_backend(self):   self._start(Counters.BACKEND)
+    def end_backend(self):     self._end  (Counters.BACKEND)
 
     def count(self, kind, inc=1):
         self.counters[kind] += inc        
-    
-    def count_ops(self, opnum, kind=OPS):
+
+    def get_counter(self, num):
+        if num == Counters.TOTAL_COMPILED_LOOPS:
+            return float(self.cpu.total_compiled_loops)
+        elif num == Counters.TOTAL_COMPILED_BRIDGES:
+            return float(self.cpu.total_compiled_bridges)
+        elif num == Counters.TOTAL_FREED_LOOPS:
+            return float(self.cpu.total_freed_loops)
+        elif num == Counters.TOTAL_FREED_BRIDGES:
+            return float(self.cpu.total_freed_bridges)
+        return float(self.counters[num])
+
+    def count_ops(self, opnum, kind=Counters.OPS):
         from pypy.jit.metainterp.resoperation import rop
         self.counters[kind] += 1
-        if opnum == rop.CALL and kind == RECORDED_OPS:# or opnum == rop.OOSEND:
+        if opnum == rop.CALL and kind == Counters.RECORDED_OPS:# or opnum == rop.OOSEND:
             self.calls += 1
 
     def print_stats(self):
         cnt = self.counters
         tim = self.times
         calls = self.calls
-        self._print_line_time("Tracing", cnt[TRACING],   tim[TRACING])
-        self._print_line_time("Backend", cnt[BACKEND],   tim[BACKEND])
+        self._print_line_time("Tracing", cnt[Counters.TRACING],
+                              tim[Counters.TRACING])
+        self._print_line_time("Backend", cnt[Counters.BACKEND],
+                              tim[Counters.BACKEND])
         line = "TOTAL:      \t\t%f" % (self.tk - self.starttime, )
         debug_print(line)
-        self._print_intline("ops", cnt[OPS])
-        self._print_intline("recorded ops", cnt[RECORDED_OPS])
+        self._print_intline("ops", cnt[Counters.OPS])
+        self._print_intline("recorded ops", cnt[Counters.RECORDED_OPS])
         self._print_intline("  calls", calls)
-        self._print_intline("guards", cnt[GUARDS])
-        self._print_intline("opt ops", cnt[OPT_OPS])
-        self._print_intline("opt guards", cnt[OPT_GUARDS])
-        self._print_intline("forcings", cnt[OPT_FORCINGS])
-        self._print_intline("abort: trace too long", cnt[ABORT_TOO_LONG])
-        self._print_intline("abort: compiling", cnt[ABORT_BRIDGE])
-        self._print_intline("abort: vable escape", cnt[ABORT_ESCAPE])
-        self._print_intline("abort: bad loop", cnt[ABORT_BAD_LOOP])
+        self._print_intline("guards", cnt[Counters.GUARDS])
+        self._print_intline("opt ops", cnt[Counters.OPT_OPS])
+        self._print_intline("opt guards", cnt[Counters.OPT_GUARDS])
+        self._print_intline("forcings", cnt[Counters.OPT_FORCINGS])
+        self._print_intline("abort: trace too long",
+                            cnt[Counters.ABORT_TOO_LONG])
+        self._print_intline("abort: compiling", cnt[Counters.ABORT_BRIDGE])
+        self._print_intline("abort: vable escape", cnt[Counters.ABORT_ESCAPE])
+        self._print_intline("abort: bad loop", cnt[Counters.ABORT_BAD_LOOP])
         self._print_intline("abort: force quasi-immut",
-                                               cnt[ABORT_FORCE_QUASIIMMUT])
-        self._print_intline("nvirtuals", cnt[NVIRTUALS])
-        self._print_intline("nvholes", cnt[NVHOLES])
-        self._print_intline("nvreused", cnt[NVREUSED])
+                            cnt[Counters.ABORT_FORCE_QUASIIMMUT])
+        self._print_intline("nvirtuals", cnt[Counters.NVIRTUALS])
+        self._print_intline("nvholes", cnt[Counters.NVHOLES])
+        self._print_intline("nvreused", cnt[Counters.NVREUSED])
         cpu = self.cpu
         if cpu is not None:   # for some tests
             self._print_intline("Total # of loops",

File pypy/jit/metainterp/optimizeopt/optimizer.py

             o.turned_constant(value)
 
     def forget_numberings(self, virtualbox):
-        self.metainterp_sd.profiler.count(jitprof.OPT_FORCINGS)
+        self.metainterp_sd.profiler.count(jitprof.Counters.OPT_FORCINGS)
         self.resumedata_memo.forget_numberings(virtualbox)
 
     def getinterned(self, box):
             else:
                 self.ensure_imported(value)
                 op.setarg(i, value.force_box(self))
-        self.metainterp_sd.profiler.count(jitprof.OPT_OPS)
+        self.metainterp_sd.profiler.count(jitprof.Counters.OPT_OPS)
         if op.is_guard():
-            self.metainterp_sd.profiler.count(jitprof.OPT_GUARDS)
+            self.metainterp_sd.profiler.count(jitprof.Counters.OPT_GUARDS)
             if self.replaces_guard and op in self.replaces_guard:
                 self.replace_op(self.replaces_guard[op], op)
                 del self.replaces_guard[op]

File pypy/jit/metainterp/pyjitpl.py

 from pypy.jit.metainterp import executor
 from pypy.jit.metainterp.logger import Logger
 from pypy.jit.metainterp.jitprof import EmptyProfiler
-from pypy.jit.metainterp.jitprof import GUARDS, RECORDED_OPS, ABORT_ESCAPE
-from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG, ABORT_BRIDGE, \
-                                        ABORT_FORCE_QUASIIMMUT, ABORT_BAD_LOOP
+from pypy.rlib.jit import Counters
 from pypy.jit.metainterp.jitexc import JitException, get_llexception
 from pypy.jit.metainterp.heapcache import HeapCache
 from pypy.rlib.objectmodel import specialize
             from pypy.jit.metainterp.quasiimmut import do_force_quasi_immutable
             do_force_quasi_immutable(self.metainterp.cpu, box.getref_base(),
                                      mutatefielddescr)
-            raise SwitchToBlackhole(ABORT_FORCE_QUASIIMMUT)
+            raise SwitchToBlackhole(Counters.ABORT_FORCE_QUASIIMMUT)
         self.generate_guard(rop.GUARD_ISNULL, mutatebox, resumepc=orgpc)
 
     def _nonstandard_virtualizable(self, pc, box):
         guard_op = metainterp.history.record(opnum, moreargs, None,
                                              descr=resumedescr)
         self.capture_resumedata(resumedescr, resumepc)
-        self.metainterp.staticdata.profiler.count_ops(opnum, GUARDS)
+        self.metainterp.staticdata.profiler.count_ops(opnum, Counters.GUARDS)
         # count
         metainterp.attach_debug_info(guard_op)
         return guard_op
             return resbox.constbox()
         # record the operation
         profiler = self.staticdata.profiler
-        profiler.count_ops(opnum, RECORDED_OPS)
+        profiler.count_ops(opnum, Counters.RECORDED_OPS)
         self.heapcache.invalidate_caches(opnum, descr, argboxes)
         op = self.history.record(opnum, argboxes, resbox, descr)
         self.attach_debug_info(op)
             if greenkey_of_huge_function is not None:
                 warmrunnerstate.disable_noninlinable_function(
                     greenkey_of_huge_function)
-            raise SwitchToBlackhole(ABORT_TOO_LONG)
+            raise SwitchToBlackhole(Counters.ABORT_TOO_LONG)
 
     def _interpret(self):
         # Execute the frames forward until we raise a DoneWithThisFrame,
         try:
             self.prepare_resume_from_failure(key.guard_opnum, dont_change_position)
             if self.resumekey_original_loop_token is None:   # very rare case
-                raise SwitchToBlackhole(ABORT_BRIDGE)
+                raise SwitchToBlackhole(Counters.ABORT_BRIDGE)
             self.interpret()
         except SwitchToBlackhole, stb:
             self.run_blackhole_interp_to_cancel_tracing(stb)
                 # raises in case it works -- which is the common case
                 if self.partial_trace:
                     if  start != self.retracing_from:
-                        raise SwitchToBlackhole(ABORT_BAD_LOOP) # For now
+                        raise SwitchToBlackhole(Counters.ABORT_BAD_LOOP) # For now
                 self.compile_loop(original_boxes, live_arg_boxes, start, resumedescr)
                 # creation of the loop was cancelled!
                 self.cancel_count += 1
                     if memmgr:
                         if self.cancel_count > memmgr.max_unroll_loops:
                             self.staticdata.log('cancelled too many times!')
-                            raise SwitchToBlackhole(ABORT_BAD_LOOP)
+                            raise SwitchToBlackhole(Counters.ABORT_BAD_LOOP)
                 self.staticdata.log('cancelled, tracing more...')
 
         # Otherwise, no loop found so far, so continue tracing.
             if vinfo.tracing_after_residual_call(virtualizable):
                 # the virtualizable escaped during CALL_MAY_FORCE.
                 self.load_fields_from_virtualizable()
-                raise SwitchToBlackhole(ABORT_ESCAPE, raising_exception=True)
+                raise SwitchToBlackhole(Counters.ABORT_ESCAPE,
+                                        raising_exception=True)
                 # ^^^ we set 'raising_exception' to True because we must still
                 # have the eventual exception raised (this is normally done
                 # after the call to vable_after_residual_call()).

File pypy/jit/metainterp/resume.py

         self.cached_virtuals.clear()
 
     def update_counters(self, profiler):
-        profiler.count(jitprof.NVIRTUALS, self.nvirtuals)
-        profiler.count(jitprof.NVHOLES, self.nvholes)
-        profiler.count(jitprof.NVREUSED, self.nvreused)
+        profiler.count(jitprof.Counters.NVIRTUALS, self.nvirtuals)
+        profiler.count(jitprof.Counters.NVHOLES, self.nvholes)
+        profiler.count(jitprof.Counters.NVREUSED, self.nvreused)
 
 _frame_info_placeholder = (None, 0, 0)
 

File pypy/jit/metainterp/test/test_jitiface.py

 
-from pypy.rlib.jit import JitDriver, JitHookInterface
+from pypy.rlib.jit import JitDriver, JitHookInterface, Counters
 from pypy.rlib import jit_hooks
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.jit.codewriter.policy import JitPolicy
-from pypy.jit.metainterp.jitprof import ABORT_FORCE_QUASIIMMUT
 from pypy.jit.metainterp.resoperation import rop
 from pypy.rpython.annlowlevel import hlstr
+from pypy.jit.metainterp.jitprof import Profiler
 
 class TestJitHookInterface(LLJitMixin):
     def test_abort_quasi_immut(self):
         assert f(100, 7) == 721
         res = self.meta_interp(f, [100, 7], policy=JitPolicy(iface))
         assert res == 721
-        assert reasons == [ABORT_FORCE_QUASIIMMUT] * 2
+        assert reasons == [Counters.ABORT_FORCE_QUASIIMMUT] * 2
 
     def test_on_compile(self):
         called = []
             assert jit_hooks.resop_getresult(op) == box5
 
         self.meta_interp(main, [])
+
+    def test_get_stats(self):
+        driver = JitDriver(greens = [], reds = ['i', 's'])
+
+        def loop(i):
+            s = 0
+            while i > 0:
+                driver.jit_merge_point(i=i, s=s)
+                if i % 2:
+                    s += 1
+                i -= 1
+                s+= 2
+            return s
+
+        def main():
+            loop(30)
+            stats = jit_hooks.get_stats()
+            assert jit_hooks.stats_get_counter_value(stats,
+                           Counters.TOTAL_COMPILED_LOOPS) == 1
+            assert jit_hooks.stats_get_counter_value(stats,
+                           Counters.TOTAL_COMPILED_BRIDGES) == 1
+            assert jit_hooks.stats_get_counter_value(stats,
+                           Counters.TRACING) >= 0
+            
+
+        self.meta_interp(main, [], ProfilerClass=Profiler)

File pypy/jit/metainterp/warmspot.py

 from pypy.annotation import model as annmodel
 from pypy.rpython.llinterp import LLException
 from pypy.rpython.test.test_llinterp import get_interpreter, clear_tcache
+from pypy.rpython.annlowlevel import cast_instance_to_base_ptr
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
 from pypy.objspace.flow.model import checkgraph, Link, copygraph
 from pypy.rlib.objectmodel import we_are_translated
 def find_set_param(graphs):
     return _find_jit_marker(graphs, 'set_param')
 
+def find_get_stats(graphs):
+    return _find_jit_marker(graphs, 'get_stats', False)
+
 def find_force_quasi_immutable(graphs):
     results = []
     for graph in graphs:
         self.rewrite_access_helpers()
         self.codewriter.make_jitcodes(verbose=verbose)
         self.rewrite_can_enter_jits()
-        self.rewrite_set_param()
+        self.rewrite_set_param_and_get_stats()
         self.rewrite_force_virtual(vrefinfo)
         self.rewrite_force_quasi_immutable()
         self.add_finish()
         # make sure we make a copy of function so it no longer belongs
         # to extregistry
         func = op.args[1].value
-        func = func_with_new_name(func, func.func_name + '_compiled')
+        if func.func_name == 'get_stats':
+            # get special treatment since we rewrite it to a call that accepts
+            # jit driver
+            def func():
+                return lltype.cast_opaque_ptr(llmemory.GCREF,
+                                              cast_instance_to_base_ptr(self))
+        else:
+            func = func_with_new_name(func, func.func_name + '_compiled')
         ptr = self.helper_func(FUNCPTR, func)
         op.opname = 'direct_call'
         op.args = [Constant(ptr, FUNCPTR)] + op.args[2:]
             call_final_function(self.translator, finish,
                                 annhelper = self.annhelper)
 
-    def rewrite_set_param(self):
+    def rewrite_set_param_and_get_stats(self):
         from pypy.rpython.lltypesystem.rstr import STR
 
         closures = {}
             op.opname = 'direct_call'
             op.args[:3] = [closures[key]]
 
+        for graph, block, i in find_get_stats(graphs):
+            xxx
+
     def rewrite_force_virtual(self, vrefinfo):
         if self.cpu.ts.name != 'lltype':
             py.test.skip("rewrite_force_virtual: port it to ootype")

File pypy/module/pypyjit/interp_resop.py

     Set a compiling hook that will be called each time a loop is compiled.
     The hook will be called with the following signature:
     hook(jitdriver_name, loop_type, greenkey or guard_number, operations,
-         assembler_addr, assembler_length)
+         assembler_addr, assembler_length, loopno)
 
     jitdriver_name is the name of this particular jitdriver, 'pypyjit' is
     the main interpreter loop
     in case loop is not `bridge`, greenkey will be a tuple of constants
     or a string describing it.
 
-    for the interpreter loop` it'll be a tuple
+    for the main interpreter loop` it'll be a tuple
     (code, offset, is_being_profiled)
 
     assembler_addr is an integer describing where assembler starts,
     can be accessed via ctypes, assembler_lenght is the lenght of compiled
     asm
 
+    loopno is the unique loop identifier (int)
+
     Note that jit hook is not reentrant. It means that if the code
     inside the jit hook is itself jitted, it will get compiled, but the
     jit hook won't be called for that.
     optimizations on Python level.
 
     The hook will be called with the following signature:
-    hook(jitdriver_name, loop_type, greenkey or guard_number, operations)
+    hook(jitdriver_name, loop_type, greenkey or guard_number, operations,
+         loopno)
 
     jitdriver_name is the name of this particular jitdriver, 'pypyjit' is
     the main interpreter loop
     inside the jit hook is itself jitted, it will get compiled, but the
     jit hook won't be called for that.
 
+    loopno is the unique loop identifier (int)
+
     Result value will be the resulting list of operations, or None
     """
     cache = space.fromcache(Cache)

File pypy/rlib/jit.py

                 raise ValueError
 set_user_param._annspecialcase_ = 'specialize:arg(0)'
 
-
 # ____________________________________________________________
 #
 # Annotation and rtyping of some of the JitDriver methods
         instance, overwrite for custom behavior
         """
 
-    def get_stats(self):
-        """ Returns various statistics
-        """
-        raise NotImplementedError
-
 def record_known_class(value, cls):
     """
     Assure the JIT that value is an instance of cls. This is not a precise
         v_cls = hop.inputarg(classrepr, arg=1)
         return hop.genop('jit_record_known_class', [v_inst, v_cls],
                          resulttype=lltype.Void)
+
+class Counters(object):
+    counters="""
+    TRACING
+    BACKEND
+    OPS
+    RECORDED_OPS
+    GUARDS
+    OPT_OPS
+    OPT_GUARDS
+    OPT_FORCINGS
+    ABORT_TOO_LONG
+    ABORT_BRIDGE
+    ABORT_BAD_LOOP
+    ABORT_ESCAPE
+    ABORT_FORCE_QUASIIMMUT
+    NVIRTUALS
+    NVHOLES
+    NVREUSED
+    TOTAL_COMPILED_LOOPS
+    TOTAL_COMPILED_BRIDGES
+    TOTAL_FREED_LOOPS
+    TOTAL_FREED_BRIDGES
+    """
+
+    counter_names = []
+
+    @staticmethod
+    def _setup():
+        names = Counters.counters.split()
+        for i, name in enumerate(names):
+            setattr(Counters, name, i)
+            Counters.counter_names.append(name)
+        Counters.ncounters = len(names)
+
+Counters._setup()

File pypy/rlib/jit_hooks.py

 def box_isconst(llbox):
     from pypy.jit.metainterp.history import Const
     return isinstance(_cast_to_box(llbox), Const)
+
+@register_helper(annmodel.SomePtr(llmemory.GCREF))
+def get_stats():
+    """ Returns various statistics, including how many times each loop
+    was run. Note that on this level the resulting instance is completely
+    opaque, you need to use the interface specified below.
+
+    Note that since we pass warmspot by closure, the actual implementation
+    is in warmspot.py, this is just a placeholder
+    """
+    from pypy.rpython.lltypesystem import lltype, llmemory
+    return lltype.nullptr(llmemory.GCREF.TO) # unusable without the actual JIT
+
+# ------------------------- stats interface ---------------------------
+
+def _cast_to_warmrunnerdesc(llref):
+    from pypy.jit.metainterp.warmspot import WarmRunnerDesc
+
+    ptr = lltype.cast_opaque_ptr(rclass.OBJECTPTR, llref)
+    return cast_base_ptr_to_instance(WarmRunnerDesc, ptr)
+
+@register_helper(annmodel.SomeFloat())
+def stats_get_counter_value(llref, no):
+    return _cast_to_warmrunnerdesc(llref).metainterp_sd.profiler.get_counter(no)

File pypy/translator/goal/richards.py

 
 import time
 
-
-
 def schedule():
     t = taskWorkArea.taskList
     while t is not None: