Commits

Alexander Hesse committed 4f3d2f5

Moved pypy.tool.error, pypy.tool.leakfinder, pypy.tool.logparser, pypy.tool.progressbar and pypy.tool.stdlib_opcode to rpython

Comments (0)

Files changed (48)

pypy/interpreter/astcompiler/assemble.py

 
 from pypy.interpreter.astcompiler import ast, symtable
 from pypy.interpreter import pycode
-from pypy.tool import stdlib_opcode as ops
+from rpython.tool import stdlib_opcode as ops
 
 from pypy.interpreter.error import OperationError
 from rpython.rlib.objectmodel import we_are_translated

pypy/interpreter/astcompiler/codegen.py

 from pypy.interpreter.astcompiler import ast, assemble, symtable, consts, misc
 from pypy.interpreter.astcompiler import optimize # For side effects
 from pypy.interpreter.pyparser.error import SyntaxError
-from pypy.tool import stdlib_opcode as ops
+from rpython.tool import stdlib_opcode as ops
 from pypy.interpreter.error import OperationError
 
 

pypy/interpreter/astcompiler/optimize.py

 import sys
 
 from pypy.interpreter.astcompiler import ast, consts, misc
-from pypy.tool import stdlib_opcode as ops
+from rpython.tool import stdlib_opcode as ops
 from pypy.interpreter.error import OperationError
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.runicode import MAXUNICODE

pypy/interpreter/astcompiler/test/test_compiler.py

 from pypy.interpreter.pyparser.test import expressions
 from pypy.interpreter.pycode import PyCode
 from pypy.interpreter.pyparser.error import SyntaxError, IndentationError
-from pypy.tool import stdlib_opcode as ops
+from rpython.tool import stdlib_opcode as ops
 
 def compile_with_astcompiler(expr, mode, space):
     p = pyparse.PythonParser(space)

pypy/interpreter/pycode.py

 from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import compute_hash
-from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT
+from rpython.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT
 
 
 class BytecodeCorruption(Exception):

pypy/interpreter/pyframe.py

 from rpython.rlib.debug import make_sure_not_resized, check_nonneg
 from rpython.rlib.rarithmetic import intmask, r_uint
 from rpython.rlib import jit
-from pypy.tool import stdlib_opcode
-from pypy.tool.stdlib_opcode import host_bytecode_spec
+from rpython.tool import stdlib_opcode
+from rpython.tool.stdlib_opcode import host_bytecode_spec
 
 # Define some opcodes used
 g = globals()

pypy/interpreter/pyopcode.py

 from rpython.rlib.rarithmetic import r_uint, intmask
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.debug import check_nonneg
-from pypy.tool.stdlib_opcode import (bytecode_spec,
+from rpython.tool.stdlib_opcode import (bytecode_spec,
                                      unrolling_all_opcode_descs)
 
 def unaryoperation(operationname):

pypy/module/_continuation/interp_pickle.py

-from pypy.tool import stdlib_opcode as pythonopcode
+from rpython.tool import stdlib_opcode as pythonopcode
 from rpython.rlib import jit
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.pyframe import PyFrame

pypy/module/cpyext/test/test_cpyext.py

 from pypy.module.cpyext.pyobject import Py_DecRef, InvalidPointerException
 from rpython.translator.goal import autopath
 from rpython.tool.identity_dict import identity_dict
-from pypy.tool import leakfinder
+from rpython.tool import leakfinder
 
 @api.cpython_api([], api.PyObject)
 def PyPy_Crash1(space):

pypy/module/cpyext/test/test_pystate.py

 from rpython.rtyper.lltypesystem import lltype
 from pypy.module.cpyext.test.test_cpyext import LeakCheckingTest, freeze_refcnts
 from pypy.module.cpyext.pystate import PyThreadState_Get, PyInterpreterState_Head
-from pypy.tool import leakfinder
+from rpython.tool import leakfinder
 
 class AppTestThreads(AppTestCpythonExtensionBase):
     def test_allow_threads(self):

pypy/module/pypyjit/interp_jit.py

 JUMP_ABSOLUTE = opmap['JUMP_ABSOLUTE']
 
 def get_printable_location(next_instr, is_being_profiled, bytecode):
-    from pypy.tool.stdlib_opcode import opcode_method_names
+    from rpython.tool.stdlib_opcode import opcode_method_names
     name = opcode_method_names[ord(bytecode.co_code[next_instr])]
     return '%s #%d %s' % (bytecode.get_repr(), next_instr, name)
 

pypy/module/pypyjit/test_pypy_c/test_00_model.py

 import py
 from lib_pypy import disassembler
 from rpython.tool.udir import udir
-from pypy.tool import logparser
+from rpython.tool import logparser
 from rpython.jit.tool.jitoutput import parse_prof
 from pypy.module.pypyjit.test_pypy_c.model import (Log, find_ids_range,
                                                    find_ids,

pypy/module/sys/vm.py

                            space.w_None])
 
 def exc_info_direct(space, frame):
-    from pypy.tool import stdlib_opcode
+    from rpython.tool import stdlib_opcode
     # In order to make the JIT happy, we try to return (exc, val, None)
     # instead of (exc, val, tb).  We can do that only if we recognize
     # the following pattern in the bytecode:

pypy/tool/error.py

-
-""" error handling features, just a way of displaying errors
-"""
-
-from rpython.tool.ansi_print import ansi_log
-from rpython.flowspace.model import Variable
-import sys
-
-import py
-log = py.log.Producer("error")
-py.log.setconsumer("error", ansi_log)
-
-SHOW_TRACEBACK = False
-SHOW_ANNOTATIONS = True
-SHOW_DEFAULT_LINES_OF_CODE = 0
-
-from pypy.interpreter.pytraceback import offset2lineno
-
-def source_lines1(graph, block, operindex=None, offset=None, long=False, \
-    show_lines_of_code=SHOW_DEFAULT_LINES_OF_CODE):
-    if block is not None:
-        if block is graph.returnblock:
-            return ['<return block>']
-    try:
-        source = graph.source
-    except AttributeError:
-        return ['no source!']
-    else:
-        graph_lines = source.split("\n")
-        if offset is not None:
-            linestart = offset2lineno(graph.func.func_code, offset)
-            linerange = (linestart, linestart)
-            here = None
-        else:
-            if block is None or not block.operations:
-                return []
-            def toline(operindex):
-                return offset2lineno(graph.func.func_code, block.operations[operindex].offset)
-            if operindex is None:
-                linerange =  (toline(0), toline(-1))
-                if not long:
-                    return ['?']
-                here = None
-            else:
-                operline = toline(operindex)
-                if long:
-                    linerange =  (toline(0), toline(-1))
-                    here = operline
-                else:
-                    linerange = (operline, operline)
-                    here = None
-        lines = ["Happened at file %s line %d" % (graph.filename, here or linerange[0]), ""]
-        for n in range(max(0, linerange[0]-show_lines_of_code), \
-            min(linerange[1]+1+show_lines_of_code, len(graph_lines)+graph.startline)):
-            if n == here:
-                prefix = '==> '
-            else:
-                prefix = '    '
-            lines.append(prefix + graph_lines[n-graph.startline])
-        lines.append("")
-        return lines
-
-def source_lines(graph, *args, **kwds):
-    lines = source_lines1(graph, *args, **kwds)
-    return ['In %r:' % (graph,)] + lines
-
-class AnnotatorError(Exception):
-    pass
-
-class NoSuchAttrError(Exception):
-    pass
-
-class ErrorWrapper(object):
-    def __init__(self, msg):
-        self.msg = msg
-
-    def __repr__(self):
-        return '<%s>' % (self.msg,)
-
-def gather_error(annotator, graph, block, operindex):
-    msg = [""]
-
-    if operindex is not None:
-        oper = block.operations[operindex]
-        if oper.opname == 'simple_call':
-            format_simple_call(annotator, oper, msg)
-    else:
-        oper = None
-    msg.append(" " + str(oper))
-    msg += source_lines(graph, block, operindex, long=True)
-    if oper is not None:
-        if SHOW_ANNOTATIONS:
-            msg.append("Known variable annotations:")
-            for arg in oper.args + [oper.result]:
-                if isinstance(arg, Variable):
-                    try:
-                        msg.append(" " + str(arg) + " = " + str(annotator.binding(arg)))
-                    except KeyError:
-                        pass
-    return "\n".join(msg)
-
-def format_blocked_annotation_error(annotator, blocked_blocks):
-    text = []
-    for block, (graph, index) in blocked_blocks.items():
-        text.append('-+' * 30)
-        text.append("Blocked block -- operation cannot succeed")
-        text.append(gather_error(annotator, graph, block, index))
-    return '\n'.join(text)
-
-def format_simple_call(annotator, oper, msg):
-    msg.append("Occurred processing the following simple_call:")
-    try:
-        descs = annotator.bindings[oper.args[0]].descriptions
-    except (KeyError, AttributeError), e:
-        msg.append("      (%s getting at the binding!)" % (
-            e.__class__.__name__,))
-        return
-    for desc in list(descs):
-        func = desc.pyobj
-        if func is None:
-            r = repr(desc)
-        else:
-            try:
-                if isinstance(func, type):
-                    func_name = "%s.__init__" % func.__name__
-                    func = func.__init__.im_func
-                else:
-                    func_name = func.func_name
-                r = "function %s <%s, line %s>" % (func_name,
-                       func.func_code.co_filename, func.func_code.co_firstlineno)
-            except (AttributeError, TypeError):
-                r = repr(desc)
-        msg.append("  %s returning" % (r,))
-        msg.append("")
-
-def debug(drv, use_pdb=True):
-    # XXX unify some code with rpython.translator.goal.translate
-    from rpython.translator.tool.pdbplus import PdbPlusShow
-    from rpython.translator.driver import log
-    t = drv.translator
-    class options:
-        huge = 100
-
-    tb = None
-    import traceback
-    errmsg = ["Error:\n"]
-    exc, val, tb = sys.exc_info()
-
-    errmsg.extend([" %s" % line for line in traceback.format_exception(exc, val, [])])
-    block = getattr(val, '__annotator_block', None)
-    if block:
-        class FileLike:
-            def write(self, s):
-                errmsg.append(" %s" % s)
-        errmsg.append("Processing block:\n")
-        t.about(block, FileLike())
-    log.ERROR(''.join(errmsg))
-
-    log.event("start debugger...")
-
-    if use_pdb:
-        pdb_plus_show = PdbPlusShow(t)
-        pdb_plus_show.start(tb)

pypy/tool/jitlogparser/parser.py

 
 from rpython.jit.metainterp.resoperation import opname
 from rpython.jit.tool.oparser import OpParser
-from pypy.tool.logparser import parse_log_file, extract_category
+from rpython.tool.logparser import parse_log_file, extract_category
 from copy import copy
 
 def parse_code_data(arg):

pypy/tool/leakfinder.py

-import sys, gc
-import cStringIO
-import traceback
-
-# Track allocations to detect memory leaks.
-# So far, this is used for lltype.malloc(flavor='raw').
-TRACK_ALLOCATIONS = False
-ALLOCATED = {}
-
-class MallocMismatch(Exception):
-    def __str__(self):
-        dict = self.args[0]
-        dict2 = {}
-        for obj, traceback in dict.items():
-            traceback = traceback.splitlines()
-            if len(traceback) > 8:
-                traceback = ['    ...'] + traceback[-6:]
-            traceback = '\n'.join(traceback)
-            dict2.setdefault(traceback, [])
-            dict2[traceback].append(obj)
-        lines = ['{']
-        for traceback, objs in dict2.items():
-            lines.append('')
-            for obj in objs:
-                lines.append('%s:' % (obj,))
-            lines.append(traceback)
-        lines.append('}')
-        return '\n'.join(lines)
-
-def start_tracking_allocations():
-    global TRACK_ALLOCATIONS
-    if TRACK_ALLOCATIONS:
-        result = ALLOCATED.copy()   # nested start
-    else:
-        result = None
-    TRACK_ALLOCATIONS = True
-    ALLOCATED.clear()
-    return result
-
-def stop_tracking_allocations(check, prev=None):
-    global TRACK_ALLOCATIONS
-    assert TRACK_ALLOCATIONS
-    for i in range(5):
-        if not ALLOCATED:
-            break
-        gc.collect()
-    result = ALLOCATED.copy()
-    ALLOCATED.clear()
-    if prev is None:
-        TRACK_ALLOCATIONS = False
-    else:
-        ALLOCATED.update(prev)
-    if check and result:
-        raise MallocMismatch(result)
-    return result
-
-def remember_malloc(obj, framedepth=1):
-    if TRACK_ALLOCATIONS:
-        frame = sys._getframe(framedepth)
-        sio = cStringIO.StringIO()
-        traceback.print_stack(frame, limit=10, file=sio)
-        tb = sio.getvalue()
-        ALLOCATED[obj] = tb
-
-def remember_free(obj):
-    if TRACK_ALLOCATIONS:
-        if obj not in ALLOCATED:
-            # rehashing is needed because some objects' hash may change
-            # e.g. when lltype objects are turned into <C object>
-            items = ALLOCATED.items()
-            ALLOCATED.clear()
-            ALLOCATED.update(items)
-        del ALLOCATED[obj]

pypy/tool/logparser.py

-#! /usr/bin/env python
-"""
-Syntax:
-    python logparser.py <action> <logfilename> <output> <options...>
-
-Actions:
-    draw-time      draw a timeline image of the log (format PNG by default)
-    print-summary  print a summary of the log
-"""
-import autopath
-import sys, re
-from rpython.rlib.debug import DebugLog
-from pypy.tool import progressbar
-
-def parse_log_file(filename, verbose=True):
-    f = open(filename, 'r')
-    if f.read(2) == 'BZ':
-        f.close()
-        import bz2
-        f = bz2.BZ2File(filename, 'r')
-    else:
-        f.seek(0)
-    lines = f.readlines()
-    f.close()
-    #
-    return parse_log(lines, verbose=verbose)
-
-def parse_log(lines, verbose=False):
-    color = "(?:\x1b.*?m)?"
-    r_start = re.compile(color + r"\[([0-9a-fA-F]+)\] \{([\w-]+)" + color + "$")
-    r_stop  = re.compile(color + r"\[([0-9a-fA-F]+)\] ([\w-]+)\}" + color + "$")
-    lasttime = 0
-    log = DebugLog()
-    time_decrase = False
-    performance_log = True
-    nested = 0
-    #
-    if verbose and sys.stdout.isatty():
-        progress = progressbar.ProgressBar(color='green')
-        counter = 0
-    else:
-        progress = None
-    single_percent = len(lines) / 100
-    if verbose:
-        vnext = 0
-    else:
-        vnext = -1
-    for i, line in enumerate(lines):
-        if i == vnext:
-            if progress is not None:
-                progress.render(counter)
-                counter += 1
-                vnext += single_percent
-            else:
-                sys.stderr.write('%d%%..' % int(100.0*i/len(lines)))
-                vnext += 500000
-        line = line.rstrip()
-        match = r_start.match(line)
-        if match:
-            record = log.debug_start
-            nested += 1
-        else:
-            match = r_stop.match(line)
-            if match:
-                record = log.debug_stop
-                nested -= 1
-            else:
-                log.debug_print(line)
-                performance_log = performance_log and nested == 0
-                continue
-        time = int(int(match.group(1), 16))
-        time_decrase = time_decrase or time < lasttime
-        lasttime = time
-        record(match.group(2), time=int(match.group(1), 16))
-    if verbose:
-        sys.stderr.write('loaded\n')
-    if performance_log and time_decrase:
-        print ("The time decreases!  The log file may have been"
-               " produced on a multi-CPU machine and the process"
-               " moved between CPUs.")
-    return log
-
-def extract_category(log, catprefix='', toplevel=False):
-    got = []
-    resulttext = []
-    for entry in log:
-        if entry[0] == 'debug_print':
-            resulttext.append(entry[1])
-        elif len(entry) == 4:
-            got.extend(extract_category(
-                entry[3], catprefix, toplevel=entry[0].startswith(catprefix)))
-        else:
-            resulttext.append('... LOG TRUCATED ...')
-    if toplevel:
-        resulttext.append('')
-        got.insert(0, '\n'.join(resulttext))
-    return got
-
-def print_log(log):
-    for entry in log:
-        if entry[0] == 'debug_print':
-            print entry[1]
-        else:
-            print "{%s" % entry[0]
-            if len(entry)>3:
-                print_log(entry[3])
-            print "%s}" % entry[0]
-
-def kill_category(log, catprefix=''):
-    newlog = []
-    for entry in log:
-        if not entry[0].startswith(catprefix):
-            if len(entry) > 3:
-                newlog.append(entry[:3] + 
-                              (kill_category(entry[3], catprefix),))
-            else:
-                newlog.append(entry)
-    return newlog
-
-def getsubcategories(log):
-    return [entry for entry in log if entry[0] != 'debug_print']
-
-def gettimebounds(log):
-    # returns (mintime, maxtime)
-    maincats = getsubcategories(log)
-    return (maincats[0][1], maincats[-1][2])
-
-def gettotaltimes(log):
-    # returns a dict {'label' or None: totaltime}
-    def rectime(category1, timestart1, timestop1, subcats):
-        substartstop = []
-        for entry in getsubcategories(subcats):
-            rectime(*entry)
-            substartstop.append(entry[1:3])   # (start, stop)
-        # compute the total time for category1 as the part of the
-        # interval [timestart1, timestop1] that is not covered by
-        # any interval from one of the subcats.
-        mytime = 0
-        substartstop.sort()
-        for substart, substop in substartstop:
-            if substart >= timestop1:
-                break
-            if substart > timestart1:
-                mytime += substart - timestart1
-            if timestart1 < substop:
-                timestart1 = substop
-        if timestart1 < timestop1:
-            mytime += timestop1 - timestart1
-        #
-        try:
-            result[category1] += mytime
-        except KeyError:
-            result[category1] = mytime
-    #
-    result = {}
-    timestart0, timestop0 = gettimebounds(log)
-    rectime(None, timestart0, timestop0, log)
-    return result
-
-# ____________________________________________________________
-
-
-COLORS = {
-    None: (248, 248, 248),
-    '': (160, 160, 160),
-    'gc-': (224, 0, 0),
-    'gc-minor': (192, 0, 16),
-    'gc-collect': (255, 0, 0),
-    'jit-': (0, 224, 0),
-    'jit-running': (192, 255, 160),
-    'jit-tracing': (0, 255, 0),
-    'jit-optimize': (160, 255, 0),
-    'jit-backend': (0, 255, 144),
-    'jit-blackhole': (0, 160, 0),
-    }
-SUMMARY = {
-    None: 'normal execution',
-    '': 'other',
-    'gc-': 'gc',
-    'jit-': 'jit',
-    'jit-running': 'jit-running',
-    }
-
-def getcolor(category):
-    while category not in COLORS:
-        category = category[:-1]
-    return COLORS[category]
-
-def getlightercolor((r, g, b)):
-    return ((r*2+255)//3, (g*2+255)//3, (b*2+255)//3)
-
-def getdarkercolor((r, g, b)):
-    return (r*2//3, g*2//3, b*2//3)
-
-def getlabel(text, _cache={}):
-    try:
-        return _cache[text]
-    except KeyError:
-        pass
-    from PIL import Image, ImageDraw
-    if None not in _cache:
-        image = Image.new("RGBA", (1, 1), (0, 0, 0, 0))
-        draw = ImageDraw.Draw(image)
-        _cache[None] = draw
-    else:
-        draw = _cache[None]
-    sx, sy = draw.textsize(text)
-    texthoriz = Image.new("RGBA", (sx, sy), (0, 0, 0, 0))
-    ImageDraw.Draw(texthoriz).text((0, 0), text, fill=(0, 0, 0))
-    textvert = texthoriz.rotate(90)
-    _cache[text] = sx, sy, texthoriz, textvert
-    return _cache[text]
-
-def bevelrect(draw, (x1, y1, x2, y2), color):
-    if x2 <= x1:
-        x2 = x1 + 1   # minimal width
-    elif x2 >= x1 + 4:
-        draw.line((x1, y1+1, x1, y2-1), fill=getlightercolor(color))
-        x1 += 1
-        x2 -= 1
-        draw.line((x2, y1+1, x2, y2-1), fill=getdarkercolor(color))
-    draw.line((x1, y1, x2-1, y1), fill=getlightercolor(color))
-    y1 += 1
-    y2 -= 1
-    draw.line((x1, y2, x2-1, y2), fill=getdarkercolor(color))
-    draw.rectangle((x1, y1, x2-1, y2-1), fill=color)
-
-# ----------
-
-def get_timeline_image(log, width, height):
-    from PIL import Image, ImageDraw
-    timestart0, timestop0 = gettimebounds(log)
-    assert timestop0 > timestart0
-    timefactor = float(width) / (timestop0 - timestart0)
-    #
-    def recdraw(sublist, subheight):
-        firstx1 = None
-        for category1, timestart1, timestop1, subcats in sublist:
-            x1 = int((timestart1 - timestart0) * timefactor)
-            x2 = int((timestop1 - timestart0) * timefactor)
-            y1 = (height - subheight) / 2
-            y2 = y1 + subheight
-            y1 = int(y1)
-            y2 = int(y2)
-            color = getcolor(category1)
-            if firstx1 is None:
-                firstx1 = x1
-            bevelrect(draw, (x1, y1, x2, y2), color)
-            subcats = getsubcategories(subcats)
-            if subcats:
-                x2 = recdraw(subcats, subheight * 0.94) - 1
-            sx, sy, texthoriz, textvert = getlabel(category1)
-            if sx <= x2-x1-8:
-                image.paste(texthoriz, (x1+5, y1+5), texthoriz)
-            elif sy <= x2-x1-2:
-                image.paste(textvert, (x1+1, y1+5), textvert)
-        return firstx1
-    #
-    image = Image.new("RGBA", (width, height), (255, 255, 255, 0))
-    draw = ImageDraw.Draw(image)
-    recdraw(getsubcategories(log), height)
-    return image
-
-# ----------
-
-def render_histogram(times, time0, labels, width, barheight):
-    # Render a histogram showing horizontal time bars are given by the
-    # 'times' dictionary.  Each entry has the label specified by 'labels',
-    # or by default the key used in 'times'.
-    from PIL import Image, ImageDraw
-    times = [(time, key) for (key, time) in times.items()]
-    times.sort()
-    times.reverse()
-    images = []
-    for time, key in times:
-        fraction = float(time) / time0
-        if fraction < 0.01:
-            break
-        color = getcolor(key)
-        image = Image.new("RGBA", (width, barheight), (255, 255, 255, 0))
-        draw = ImageDraw.Draw(image)
-        x2 = int(fraction * width)
-        bevelrect(draw, (0, 0, x2, barheight), color)
-        # draw the labels "x%" and "key"
-        percent = "%.1f%%" % (100.0 * fraction,)
-        s1x, s1y, textpercent, vtextpercent = getlabel(percent)
-        s2x, _, textlabel, _ = getlabel(labels.get(key, key))
-        t1x = 5
-        if t1x + s1x >= x2 - 3:
-            if t1x + s1y < x2 - 3:
-                textpercent = vtextpercent
-                s1x = s1y
-            else:
-                t1x = x2 + 6
-        t2x = t1x + s1x + 12
-        if t2x + s2x >= x2 - 3:
-            t2x = max(t2x, x2 + 8)
-        image.paste(textpercent, (t1x, 5), textpercent)
-        image.paste(textlabel,   (t2x, 5), textlabel)
-        images.append(image)
-    if not images:
-        return None
-    return combine(images, spacing=0, border=1, horizontal=False)
-
-def get_timesummary_single_image(totaltimes, totaltime0, componentdict,
-                                 width, barheight):
-    # Compress the totaltimes dict so that its only entries left are
-    # from componentdict.  We do that by taking the times assigned to
-    # subkeys in totaltimes and adding them to the superkeys specified
-    # in componentdict.
-    totaltimes = totaltimes.copy()
-    for key, value in totaltimes.items():
-        if key in componentdict:
-            continue
-        del totaltimes[key]
-        if key is not None:
-            while key not in componentdict:
-                key = key[:-1]
-            try:
-                totaltimes[key] += value
-            except KeyError:
-                totaltimes[key] = value
-    return render_histogram(totaltimes, totaltime0, componentdict,
-                            width, barheight)
-
-def get_timesummary_image(log, summarywidth, summarybarheight):
-    timestart0, timestop0 = gettimebounds(log)
-    totaltime0 = timestop0 - timestart0
-    totaltimes = gettotaltimes(log)
-    spacing = 50
-    width = (summarywidth - spacing) // 2
-    img1 = get_timesummary_single_image(totaltimes, totaltime0, SUMMARY,
-                                        width, summarybarheight)
-    if None in totaltimes:
-        del totaltimes[None]
-    img2 = render_histogram(totaltimes, totaltime0, {},
-                            width, summarybarheight)
-    if img2 is None:
-        return img1
-    return combine([img1, img2], spacing=spacing, horizontal=True)
-
-# ----------
-
-def combine(imagelist, spacing=50, border=0, horizontal=False):
-    if len(imagelist) <= 1 and not border:
-        return imagelist[0]
-    from PIL import Image, ImageDraw
-    wlist = [image.size[0] for image in imagelist]
-    hlist = [image.size[1] for image in imagelist]
-    if horizontal:
-        w = sum(wlist) + spacing*(len(imagelist)-1)
-        h = max(hlist)
-    else:
-        w = max(wlist)
-        h = sum(hlist) + spacing*(len(imagelist)-1)
-    w += 2*border
-    h += 2*border
-    bigimage = Image.new("RGBA", (w, h), (255, 255, 255, 0))
-    if border:
-        draw = ImageDraw.Draw(bigimage)
-        draw.rectangle((0, 0, w-1, border-1), fill=(0, 0, 0))
-        draw.rectangle((0, h-border, w-1, h-1), fill=(0, 0, 0))
-        draw.rectangle((0, 0, border-1, h-1), fill=(0, 0, 0))
-        draw.rectangle((w-1, 0, w-border, h-1), fill=(0, 0, 0))
-    x = border
-    y = border
-    for image in imagelist:
-        bigimage.paste(image, (x, y))
-        if horizontal:
-            x += image.size[0] + spacing
-        else:
-            y += image.size[1] + spacing
-    return bigimage
-
-def draw_timeline_image(log, output=None, mainwidth=3000, mainheight=150,
-                        summarywidth=850, summarybarheight=40):
-    mainwidth = int(mainwidth)
-    mainheight = int(mainheight)
-    summarywidth = int(summarywidth)
-    summarybarheight = int(summarybarheight)
-    images = []
-    if mainwidth > 0 and mainheight > 0:
-        images.append(get_timeline_image(log, mainwidth, mainheight))
-    if summarywidth > 0 and summarybarheight > 0:
-        images.append(get_timesummary_image(log, summarywidth,
-                                                 summarybarheight))
-    image = combine(images, horizontal=False)
-    if output is None:
-        image.save(sys.stdout, format='png')
-    else:
-        image.save(output)
-
-def print_summary(log, out):
-    totaltimes = gettotaltimes(log)
-    if out == '-':
-        outfile = sys.stdout
-    else:
-        outfile = open(out, "w")
-    l = totaltimes.items()
-    l.sort(cmp=lambda a, b: cmp(b[1], a[1]))
-    total = sum([b for a, b in l])
-    for a, b in l:
-        if a is None:
-            a = 'interpret'
-        s = " " * (50 - len(a))
-        print >>outfile, a, s, str(b*100/total) + "%"
-    if out != '-':
-        outfile.close()
-
-# ____________________________________________________________
-
-
-ACTIONS = {
-    'draw-time': (draw_timeline_image, ['output=',
-                                        'mainwidth=', 'mainheight=',
-                                        'summarywidth=', 'summarybarheight=',
-                                        ]),
-    'print-summary': (print_summary, []),
-    }
-
-if __name__ == '__main__':
-    import getopt
-    if len(sys.argv) < 3:
-        print __doc__
-        sys.exit(2)
-    action = sys.argv[1]
-    func, longopts = ACTIONS[action]
-    options, args = getopt.gnu_getopt(sys.argv[2:], '', longopts)
-    if len(args) != 2:
-        print __doc__
-        sys.exit(2)
-
-    kwds = {}
-    for name, value in options:
-        assert name.startswith('--')
-        kwds[name[2:]] = value
-    log = parse_log_file(args[0])
-    func(log, args[1], **kwds)

pypy/tool/progressbar.py

-# -*- coding: utf-8 -*-
-# Copyright: 2009 Nadia Alramli
-# License: BSD
-"""Draws an animated terminal progress bar
-Usage:
-    p = ProgressBar("blue")
-    p.render(percentage, message)
-"""
- 
-from pypy.tool import terminal
-import sys
- 
-class ProgressBar(object):
-    """Terminal progress bar class"""
-    TEMPLATE = (
-     '%(percent)-2s%% %(color)s%(progress)s%(normal)s%(empty)s %(message)s\n'
-    )
-    PADDING = 7
- 
-    def __init__(self, color=None, width=None, block='.', empty=' '):
-        """
-        color -- color name (BLUE GREEN CYAN RED MAGENTA YELLOW WHITE BLACK)
-        width -- bar width (optinal)
-        block -- progress display character (default '.')
-        empty -- bar display character (default ' ')
-        """
-        if color:
-            self.color = getattr(terminal, color.upper())
-        else:
-            self.color = ''
-        if width and width < terminal.COLUMNS - self.PADDING:
-            self.width = width
-        else:
-            # Adjust to the width of the terminal
-            self.width = terminal.COLUMNS - self.PADDING
-        self.block = block
-        self.empty = empty
-        self.progress = None
-        self.lines = 0
- 
-    def render(self, percent, message = ''):
-        """Print the progress bar
-        percent -- the progress percentage %
-        message -- message string (optional)
-        """
-        inline_msg_len = 0
-        if message:
-            # The length of the first line in the message
-            inline_msg_len = len(message.splitlines()[0])
-        if inline_msg_len + self.width + self.PADDING > terminal.COLUMNS:
-            # The message is too long to fit in one line.
-            # Adjust the bar width to fit.
-            bar_width = terminal.COLUMNS - inline_msg_len -self.PADDING
-        else:
-            bar_width = self.width
- 
-        # Check if render is called for the first time
-        if self.progress != None:
-            self.clear()
-        self.progress = (bar_width * percent) / 100
-        data = self.TEMPLATE % {
-            'percent': percent,
-            'color': self.color,
-            'progress': self.block * self.progress,
-            'normal': terminal.NORMAL,
-            'empty': self.empty * (bar_width - self.progress),
-            'message': message
-        }
-        sys.stdout.write(data)
-        sys.stdout.flush()
-        # The number of lines printed
-        self.lines = len(data.splitlines())
- 
-    def clear(self):
-        """Clear all printed lines"""
-        sys.stdout.write(
-            self.lines * (terminal.UP + terminal.BOL + terminal.CLEAR_EOL)
-        )

pypy/tool/pydis.py

 import autopath
 import sys
 
-from pypy.tool import stdlib_opcode
-from pypy.tool.stdlib_opcode import *
+from rpython.tool import stdlib_opcode
+from rpython.tool.stdlib_opcode import *
 
 __all__ = ["dis","pydisassemble","distb","disco"] + stdlib_opcode.__all__
 

pypy/tool/pytest/plugins.py

 # pytest hooks, installed by pypy.conftest.
 
 import py
-from pypy.tool import leakfinder
+from rpython.tool import leakfinder
 
 class LeakFinder:
     """Track memory allocations during test execution.

pypy/tool/stdlib_opcode.py

-"""
-Opcodes PyPy compiles Python source to.
-Also gives access to opcodes of the host Python PyPy was bootstrapped with
-(module attributes with the `host_` prefix).
-"""
-
-# load opcode.py as pythonopcode from our own lib
-
-__all__ = ['opmap', 'opname', 'HAVE_ARGUMENT',
-           'hasconst', 'hasname', 'hasjrel', 'hasjabs',
-           'haslocal', 'hascompare', 'hasfree', 'cmp_op']
-
-# ____________________________________________________________
-# RPython-friendly helpers and structures
-
-class _BaseOpcodeDesc(object):
-    def __init__(self, bytecode_spec, name, index, methodname):
-        self.bytecode_spec = bytecode_spec
-        self.name = name
-        self.methodname = methodname
-        self.index = index
-        self.hasarg = index >= self.HAVE_ARGUMENT
-
-    def _freeze_(self):
-        return True
-
-    def is_enabled(self, space):
-        """Check if the opcode should be enabled in the space's configuration.
-        (Returns True for all standard opcodes.)"""
-        opt = space.config.objspace.opcodes
-        return getattr(opt, self.name, True)
-    is_enabled._annspecialcase_ = 'specialize:memo'
-
-    # for predictable results, we try to order opcodes most-used-first
-    opcodeorder = [124, 125, 100, 105, 1, 131, 116, 111, 106, 83, 23, 93, 113, 25, 95, 64, 112, 66, 102, 110, 60, 92, 62, 120, 68, 87, 32, 136, 4, 103, 24, 63, 18, 65, 15, 55, 121, 3, 101, 22, 12, 80, 86, 135, 126, 90, 140, 104, 2, 33, 20, 108, 107, 31, 134, 132, 88, 30, 133, 130, 137, 141, 61, 122, 11, 40, 74, 73, 51, 96, 21, 42, 56, 85, 82, 89, 142, 77, 78, 79, 91, 76, 97, 57, 19, 43, 84, 50, 41, 99, 53, 26]
-
-    def sortkey(self):
-        try:
-            i = self.opcodeorder.index(self.index)
-        except ValueError:
-            i = 1000000
-        return i, self.index
-
-    def __cmp__(self, other):
-        return (cmp(self.__class__, other.__class__) or
-                cmp(self.sortkey(), other.sortkey()))
-
-    def __str__(self):
-        return "<OpcodeDesc code=%d name=%s at %x>" % (self.index, self.name, id(self))
-    
-    __repr__ = __str__
-
-class _baseopcodedesc:
-    """A namespace mapping OPCODE_NAME to _BaseOpcodeDescs."""
-    pass
-
-
-class BytecodeSpec(object):
-    """A bunch of mappings describing a bytecode instruction set."""
-
-    def __init__(self, name, opmap, HAVE_ARGUMENT):
-        """NOT_RPYTHON."""
-        class OpcodeDesc(_BaseOpcodeDesc):
-            HAVE_ARGUMENT = HAVE_ARGUMENT
-        class opcodedesc(_baseopcodedesc):
-            """A namespace mapping OPCODE_NAME to OpcodeDescs."""
-        
-        self.name = name
-        self.OpcodeDesc = OpcodeDesc
-        self.opcodedesc = opcodedesc
-        self.HAVE_ARGUMENT = HAVE_ARGUMENT
-        # opname -> opcode
-        self.opmap = opmap
-        # opcode -> method name
-        self.method_names = tbl = ['MISSING_OPCODE'] * 256
-        # opcode -> opdesc
-        self.opdescmap = {}
-        for name, index in opmap.items():
-            tbl[index] = methodname = name.replace('+', '_')
-            desc = OpcodeDesc(self, name, index, methodname)
-            setattr(self.opcodedesc, name, desc)
-            self.opdescmap[index] = desc
-        # fill the ordered opdesc list
-        self.ordered_opdescs = lst = self.opdescmap.values() 
-        lst.sort()
-    
-    def to_globals(self):
-        """NOT_RPYTHON. Add individual opcodes to the module constants."""
-        g = globals()
-        g.update(self.opmap)
-        g['SLICE'] = self.opmap["SLICE+0"]
-        g['STORE_SLICE'] = self.opmap["STORE_SLICE+0"]
-        g['DELETE_SLICE'] = self.opmap["DELETE_SLICE+0"]
-
-    def __str__(self):
-        return "<%s bytecode>" % (self.name,)
-    
-    __repr__ = __str__
-
-
-# Initialization
-
-from rpython.rlib.unroll import unrolling_iterable
-
-from opcode import (
-    opmap as host_opmap, HAVE_ARGUMENT as host_HAVE_ARGUMENT)
-
-def load_pypy_opcode():
-    from pypy.tool.lib_pypy import LIB_PYTHON
-    opcode_path = LIB_PYTHON.join('opcode.py')
-    d = {}
-    execfile(str(opcode_path), d)
-    for name in __all__:
-        if name in d:
-            globals()[name] = d[name]
-    return d
-
-load_pypy_opcode()
-del load_pypy_opcode
-
-bytecode_spec = BytecodeSpec('pypy', opmap, HAVE_ARGUMENT)
-host_bytecode_spec = BytecodeSpec('host', host_opmap, host_HAVE_ARGUMENT)
-bytecode_spec.to_globals()
-
-opcode_method_names = bytecode_spec.method_names
-opcodedesc = bytecode_spec.opcodedesc
-
-unrolling_all_opcode_descs = unrolling_iterable(
-    bytecode_spec.ordered_opdescs + host_bytecode_spec.ordered_opdescs)
-unrolling_opcode_descs = unrolling_iterable(
-    bytecode_spec.ordered_opdescs)

pypy/tool/test/test_error.py

 """
 
 from rpython.translator.translator import TranslationContext
-from pypy.tool.error import AnnotatorError
+from rpython.tool.error import AnnotatorError
 from rpython.annotator.model import UnionError
 
 import py

pypy/tool/test/test_leakfinder.py

 import py
-from pypy.tool import leakfinder
+from rpython.tool import leakfinder
 
 def test_start_stop():
     leakfinder.start_tracking_allocations()

pypy/tool/test/test_logparser.py

 from rpython.tool.udir import udir
-from pypy.tool.logparser import *
+from rpython.tool.logparser import *
 
 
 globalpath = udir.join('test_logparser.log')

rpython/annotator/annrpython.py

 import types
 from rpython.tool.ansi_print import ansi_log
 from rpython.tool.pairtype import pair
-from pypy.tool.error import (format_blocked_annotation_error,
+from rpython.tool.error import (format_blocked_annotation_error,
                              AnnotatorError, gather_error, ErrorWrapper)
 from rpython.flowspace.model import (Variable, Constant, FunctionGraph,
                                       c_last_exception, checkgraph)

rpython/annotator/binaryop.py

 from rpython.annotator.bookkeeper import getbookkeeper
 from rpython.flowspace.model import Variable, Constant
 from rpython.rlib import rarithmetic
-from pypy.tool.error import AnnotatorError
+from rpython.tool.error import AnnotatorError
 
 # convenience only!
 def immutablevalue(x):

rpython/annotator/builtin.py

 from rpython.annotator.bookkeeper import getbookkeeper
 from rpython.annotator import description
 from rpython.flowspace.model import Constant
-from pypy.tool.error import AnnotatorError
+from rpython.tool.error import AnnotatorError
 import rpython.rlib.rarithmetic
 import rpython.rlib.objectmodel
 

rpython/annotator/unaryop.py

 from rpython.annotator import builtin
 from rpython.annotator.binaryop import _clone ## XXX where to put this?
 from rpython.rtyper import extregistry
-from pypy.tool.error import AnnotatorError
+from rpython.tool.error import AnnotatorError
 
 # convenience only!
 def immutablevalue(x):

rpython/flowspace/bytecode.py

 """
 Bytecode handling classes and functions for use by the flow space.
 """
-from pypy.tool.stdlib_opcode import (host_bytecode_spec, EXTENDED_ARG,
+from rpython.tool.stdlib_opcode import (host_bytecode_spec, EXTENDED_ARG,
         HAVE_ARGUMENT)
 from rpython.flowspace.argument import Signature
 from rpython.flowspace.flowcontext import BytecodeCorruption

rpython/flowspace/flowcontext.py

 import sys
 import collections
 
-from pypy.tool.error import source_lines
-from pypy.tool.stdlib_opcode import host_bytecode_spec
+from rpython.tool.error import source_lines
+from rpython.tool.stdlib_opcode import host_bytecode_spec
 from rpython.flowspace.argument import ArgumentsForTranslation
 from rpython.flowspace.model import (Constant, Variable, Block, Link,
     UnwrapException, c_last_exception)

rpython/flowspace/test/test_objspace.py

 from rpython.flowspace.objspace import FlowObjSpace
 from rpython.flowspace.flowcontext import FlowingError, FlowSpaceFrame
 from pypy import conftest
-from pypy.tool.stdlib_opcode import bytecode_spec, host_bytecode_spec
+from rpython.tool.stdlib_opcode import bytecode_spec, host_bytecode_spec
 
 import os
 import operator

rpython/jit/backend/x86/test/test_runner.py

         self.cpu.setup_once()
 
     def test_debugger_on(self):
-        from pypy.tool.logparser import parse_log_file, extract_category
+        from rpython.tool.logparser import parse_log_file, extract_category
         from rpython.rlib import debug
 
         targettoken, preambletoken = TargetToken(), TargetToken()

rpython/jit/backend/x86/tool/viewcode.py

         sys.exit(2)
     #
     import cStringIO
-    from pypy.tool import logparser
+    from rpython.tool import logparser
     log1 = logparser.parse_log_file(sys.argv[1])
     text1 = logparser.extract_category(log1, catprefix='jit-backend-dump')
     f = cStringIO.StringIO()

rpython/jit/tl/spli/interpreter.py

 import os
-from pypy.tool import stdlib_opcode
+from rpython.tool import stdlib_opcode
 from rpython.jit.tl.spli import objects, pycode
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.jit import JitDriver, promote, dont_look_inside

rpython/jit/tool/findadrinlog.py

 import autopath
 import sys, re
-from pypy.tool import logparser
+from rpython.tool import logparser
 
 # fflush(pypy_debug_file)
 

rpython/jit/tool/loopviewer.py

 import sys
 import optparse
 from pprint import pprint
-from pypy.tool import logparser
+from rpython.tool import logparser
 from rpython.jit.tool.oparser import parse
 from rpython.jit.metainterp.history import ConstInt
 from rpython.rtyper.lltypesystem import llmemory, lltype

rpython/jit/tool/showstats.py

 
 import autopath
 import sys, py
-from pypy.tool import logparser
+from rpython.tool import logparser
 from rpython.jit.tool.oparser import parse
 from rpython.jit.metainterp.resoperation import rop
 from rpython.rtyper.lltypesystem import lltype, llmemory

rpython/jit/tool/test/test_jitoutput.py

 from rpython.jit.backend.llgraph import runner
 from rpython.jit.metainterp.jitprof import Profiler, JITPROF_LINES
 from rpython.jit.tool.jitoutput import parse_prof
-from pypy.tool.logparser import parse_log, extract_category
+from rpython.tool.logparser import parse_log, extract_category
 
 def test_really_run():
     """ This test checks whether output of jitprof did not change.

rpython/jit/tool/traceviewer.py

 import autopath
 from rpython.translator.tool.graphpage import GraphPage
 from rpython.translator.tool.make_dot import DotGen
-from pypy.tool import logparser
-from pypy.tool import progressbar
+from rpython.tool import logparser
+from rpython.tool import progressbar
 
 class SubPage(GraphPage):
     def compute(self, graph):

rpython/rtyper/lltypesystem/lltype.py

                                    base_int, normalizedinttype, longlongmask, longlonglongmask)
 from rpython.rlib.objectmodel import Symbolic
 from rpython.tool.identity_dict import identity_dict
-from pypy.tool import leakfinder
+from rpython.tool import leakfinder
 from types import NoneType
 from rpython.rlib.rarithmetic import maxint, is_valid_int, is_emulated_long
 import weakref

rpython/rtyper/lltypesystem/test/test_lltype.py

 from rpython.rtyper.lltypesystem.lltype import *
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.tool.identity_dict import identity_dict
-from pypy.tool import leakfinder
+from rpython.tool import leakfinder
 
 def isweak(p, T):
     try:

rpython/rtyper/ootypesystem/test/test_oortype.py

 from rpython.translator.translator import TranslationContext, graphof
 from rpython.rtyper.test.test_llinterp import interpret
 from rpython.rlib.objectmodel import r_dict
-from pypy.tool.error import AnnotatorError
+from rpython.tool.error import AnnotatorError
 from rpython.rtyper.ootypesystem import ooregistry # side effects
 
 def gengraph(f, args=[], viewBefore=False, viewAfter=False, mangle=True):

rpython/rtyper/test/test_llinterp.py

 from rpython.annotator import model as annmodel
 from rpython.annotator.model import lltype_to_annotation
 from rpython.rlib.rarithmetic import r_uint, ovfcheck
-from pypy.tool import leakfinder
+from rpython.tool import leakfinder
 from pypy import conftest
 
 

rpython/tool/error.py

+
+""" error handling features, just a way of displaying errors
+"""
+
+from rpython.tool.ansi_print import ansi_log
+from rpython.flowspace.model import Variable
+import sys
+
+import py
+log = py.log.Producer("error")
+py.log.setconsumer("error", ansi_log)
+
+SHOW_TRACEBACK = False
+SHOW_ANNOTATIONS = True
+SHOW_DEFAULT_LINES_OF_CODE = 0
+
+from pypy.interpreter.pytraceback import offset2lineno
+
+def source_lines1(graph, block, operindex=None, offset=None, long=False, \
+    show_lines_of_code=SHOW_DEFAULT_LINES_OF_CODE):
+    if block is not None:
+        if block is graph.returnblock:
+            return ['<return block>']
+    try:
+        source = graph.source
+    except AttributeError:
+        return ['no source!']
+    else:
+        graph_lines = source.split("\n")
+        if offset is not None:
+            linestart = offset2lineno(graph.func.func_code, offset)
+            linerange = (linestart, linestart)
+            here = None
+        else:
+            if block is None or not block.operations:
+                return []
+            def toline(operindex):
+                return offset2lineno(graph.func.func_code, block.operations[operindex].offset)
+            if operindex is None:
+                linerange =  (toline(0), toline(-1))
+                if not long:
+                    return ['?']
+                here = None
+            else:
+                operline = toline(operindex)
+                if long:
+                    linerange =  (toline(0), toline(-1))
+                    here = operline
+                else:
+                    linerange = (operline, operline)
+                    here = None
+        lines = ["Happened at file %s line %d" % (graph.filename, here or linerange[0]), ""]
+        for n in range(max(0, linerange[0]-show_lines_of_code), \
+            min(linerange[1]+1+show_lines_of_code, len(graph_lines)+graph.startline)):
+            if n == here:
+                prefix = '==> '
+            else:
+                prefix = '    '
+            lines.append(prefix + graph_lines[n-graph.startline])
+        lines.append("")
+        return lines
+
+def source_lines(graph, *args, **kwds):
+    lines = source_lines1(graph, *args, **kwds)
+    return ['In %r:' % (graph,)] + lines
+
+class AnnotatorError(Exception):
+    pass
+
+class NoSuchAttrError(Exception):
+    pass
+
+class ErrorWrapper(object):
+    def __init__(self, msg):
+        self.msg = msg
+
+    def __repr__(self):
+        return '<%s>' % (self.msg,)
+
+def gather_error(annotator, graph, block, operindex):
+    msg = [""]
+
+    if operindex is not None:
+        oper = block.operations[operindex]
+        if oper.opname == 'simple_call':
+            format_simple_call(annotator, oper, msg)
+    else:
+        oper = None
+    msg.append(" " + str(oper))
+    msg += source_lines(graph, block, operindex, long=True)
+    if oper is not None:
+        if SHOW_ANNOTATIONS:
+            msg.append("Known variable annotations:")
+            for arg in oper.args + [oper.result]:
+                if isinstance(arg, Variable):
+                    try:
+                        msg.append(" " + str(arg) + " = " + str(annotator.binding(arg)))
+                    except KeyError:
+                        pass
+    return "\n".join(msg)
+
+def format_blocked_annotation_error(annotator, blocked_blocks):
+    text = []
+    for block, (graph, index) in blocked_blocks.items():
+        text.append('-+' * 30)
+        text.append("Blocked block -- operation cannot succeed")
+        text.append(gather_error(annotator, graph, block, index))
+    return '\n'.join(text)
+
+def format_simple_call(annotator, oper, msg):
+    msg.append("Occurred processing the following simple_call:")
+    try:
+        descs = annotator.bindings[oper.args[0]].descriptions
+    except (KeyError, AttributeError), e:
+        msg.append("      (%s getting at the binding!)" % (
+            e.__class__.__name__,))
+        return
+    for desc in list(descs):
+        func = desc.pyobj
+        if func is None:
+            r = repr(desc)
+        else:
+            try:
+                if isinstance(func, type):
+                    func_name = "%s.__init__" % func.__name__
+                    func = func.__init__.im_func
+                else:
+                    func_name = func.func_name
+                r = "function %s <%s, line %s>" % (func_name,
+                       func.func_code.co_filename, func.func_code.co_firstlineno)
+            except (AttributeError, TypeError):
+                r = repr(desc)
+        msg.append("  %s returning" % (r,))
+        msg.append("")
+
+def debug(drv, use_pdb=True):
+    # XXX unify some code with rpython.translator.goal.translate
+    from rpython.translator.tool.pdbplus import PdbPlusShow
+    from rpython.translator.driver import log
+    t = drv.translator
+    class options:
+        huge = 100
+
+    tb = None
+    import traceback
+    errmsg = ["Error:\n"]
+    exc, val, tb = sys.exc_info()
+
+    errmsg.extend([" %s" % line for line in traceback.format_exception(exc, val, [])])
+    block = getattr(val, '__annotator_block', None)
+    if block:
+        class FileLike:
+            def write(self, s):
+                errmsg.append(" %s" % s)
+        errmsg.append("Processing block:\n")
+        t.about(block, FileLike())
+    log.ERROR(''.join(errmsg))
+
+    log.event("start debugger...")
+
+    if use_pdb:
+        pdb_plus_show = PdbPlusShow(t)
+        pdb_plus_show.start(tb)

rpython/tool/leakfinder.py

+import sys, gc
+import cStringIO
+import traceback
+
+# Track allocations to detect memory leaks.
+# So far, this is used for lltype.malloc(flavor='raw').
+TRACK_ALLOCATIONS = False
+ALLOCATED = {}
+
+class MallocMismatch(Exception):
+    def __str__(self):
+        dict = self.args[0]
+        dict2 = {}
+        for obj, traceback in dict.items():
+            traceback = traceback.splitlines()
+            if len(traceback) > 8:
+                traceback = ['    ...'] + traceback[-6:]
+            traceback = '\n'.join(traceback)
+            dict2.setdefault(traceback, [])
+            dict2[traceback].append(obj)
+        lines = ['{']
+        for traceback, objs in dict2.items():
+            lines.append('')
+            for obj in objs:
+                lines.append('%s:' % (obj,))
+            lines.append(traceback)
+        lines.append('}')
+        return '\n'.join(lines)
+
+def start_tracking_allocations():
+    global TRACK_ALLOCATIONS
+    if TRACK_ALLOCATIONS:
+        result = ALLOCATED.copy()   # nested start
+    else:
+        result = None
+    TRACK_ALLOCATIONS = True
+    ALLOCATED.clear()
+    return result
+
+def stop_tracking_allocations(check, prev=None):
+    global TRACK_ALLOCATIONS
+    assert TRACK_ALLOCATIONS
+    for i in range(5):
+        if not ALLOCATED:
+            break
+        gc.collect()
+    result = ALLOCATED.copy()
+    ALLOCATED.clear()
+    if prev is None:
+        TRACK_ALLOCATIONS = False
+    else:
+        ALLOCATED.update(prev)
+    if check and result:
+        raise MallocMismatch(result)
+    return result
+
+def remember_malloc(obj, framedepth=1):
+    if TRACK_ALLOCATIONS:
+        frame = sys._getframe(framedepth)
+        sio = cStringIO.StringIO()
+        traceback.print_stack(frame, limit=10, file=sio)
+        tb = sio.getvalue()
+        ALLOCATED[obj] = tb
+
+def remember_free(obj):
+    if TRACK_ALLOCATIONS:
+        if obj not in ALLOCATED:
+            # rehashing is needed because some objects' hash may change
+            # e.g. when lltype objects are turned into <C object>
+            items = ALLOCATED.items()
+            ALLOCATED.clear()
+            ALLOCATED.update(items)
+        del ALLOCATED[obj]

rpython/tool/logparser.py

+#! /usr/bin/env python
+"""
+Syntax:
+    python logparser.py <action> <logfilename> <output> <options...>
+
+Actions:
+    draw-time      draw a timeline image of the log (format PNG by default)
+    print-summary  print a summary of the log
+"""
+import autopath
+import sys, re
+from rpython.rlib.debug import DebugLog
+from rpython.tool import progressbar
+
+def parse_log_file(filename, verbose=True):
+    f = open(filename, 'r')
+    if f.read(2) == 'BZ':
+        f.close()
+        import bz2
+        f = bz2.BZ2File(filename, 'r')
+    else:
+        f.seek(0)
+    lines = f.readlines()
+    f.close()
+    #
+    return parse_log(lines, verbose=verbose)
+
+def parse_log(lines, verbose=False):
+    color = "(?:\x1b.*?m)?"
+    r_start = re.compile(color + r"\[([0-9a-fA-F]+)\] \{([\w-]+)" + color + "$")
+    r_stop  = re.compile(color + r"\[([0-9a-fA-F]+)\] ([\w-]+)\}" + color + "$")
+    lasttime = 0
+    log = DebugLog()
+    time_decrase = False
+    performance_log = True
+    nested = 0
+    #
+    if verbose and sys.stdout.isatty():
+        progress = progressbar.ProgressBar(color='green')
+        counter = 0
+    else:
+        progress = None
+    single_percent = len(lines) / 100
+    if verbose:
+        vnext = 0
+    else:
+        vnext = -1
+    for i, line in enumerate(lines):
+        if i == vnext:
+            if progress is not None:
+                progress.render(counter)
+                counter += 1
+                vnext += single_percent
+            else:
+                sys.stderr.write('%d%%..' % int(100.0*i/len(lines)))
+                vnext += 500000
+        line = line.rstrip()
+        match = r_start.match(line)
+        if match:
+            record = log.debug_start
+            nested += 1
+        else:
+            match = r_stop.match(line)
+            if match:
+                record = log.debug_stop
+                nested -= 1
+            else:
+                log.debug_print(line)
+                performance_log = performance_log and nested == 0
+                continue
+        time = int(int(match.group(1), 16))
+        time_decrase = time_decrase or time < lasttime
+        lasttime = time
+        record(match.group(2), time=int(match.group(1), 16))
+    if verbose:
+        sys.stderr.write('loaded\n')
+    if performance_log and time_decrase:
+        print ("The time decreases!  The log file may have been"
+               " produced on a multi-CPU machine and the process"
+               " moved between CPUs.")
+    return log
+
+def extract_category(log, catprefix='', toplevel=False):
+    got = []
+    resulttext = []
+    for entry in log:
+        if entry[0] == 'debug_print':
+            resulttext.append(entry[1])
+        elif len(entry) == 4:
+            got.extend(extract_category(
+                entry[3], catprefix, toplevel=entry[0].startswith(catprefix)))
+        else:
+            resulttext.append('... LOG TRUCATED ...')
+    if toplevel:
+        resulttext.append('')
+        got.insert(0, '\n'.join(resulttext))
+    return got
+
+def print_log(log):
+    for entry in log:
+        if entry[0] == 'debug_print':
+            print entry[1]
+        else:
+            print "{%s" % entry[0]
+            if len(entry)>3:
+                print_log(entry[3])
+            print "%s}" % entry[0]
+
+def kill_category(log, catprefix=''):
+    newlog = []
+    for entry in log:
+        if not entry[0].startswith(catprefix):
+            if len(entry) > 3:
+                newlog.append(entry[:3] + 
+                              (kill_category(entry[3], catprefix),))
+            else:
+                newlog.append(entry)
+    return newlog
+
+def getsubcategories(log):
+    return [entry for entry in log if entry[0] != 'debug_print']
+
+def gettimebounds(log):
+    # returns (mintime, maxtime)
+    maincats = getsubcategories(log)
+    return (maincats[0][1], maincats[-1][2])
+
+def gettotaltimes(log):
+    # returns a dict {'label' or None: totaltime}
+    def rectime(category1, timestart1, timestop1, subcats):
+        substartstop = []
+        for entry in getsubcategories(subcats):
+            rectime(*entry)
+            substartstop.append(entry[1:3])   # (start, stop)
+        # compute the total time for category1 as the part of the
+        # interval [timestart1, timestop1] that is not covered by
+        # any interval from one of the subcats.
+        mytime = 0
+        substartstop.sort()
+        for substart, substop in substartstop:
+            if substart >= timestop1:
+                break
+            if substart > timestart1:
+                mytime += substart - timestart1
+            if timestart1 < substop:
+                timestart1 = substop
+        if timestart1 < timestop1:
+            mytime += timestop1 - timestart1
+        #
+        try:
+            result[category1] += mytime
+        except KeyError:
+            result[category1] = mytime
+    #
+    result = {}
+    timestart0, timestop0 = gettimebounds(log)
+    rectime(None, timestart0, timestop0, log)
+    return result
+
+# ____________________________________________________________
+
+
+COLORS = {
+    None: (248, 248, 248),
+    '': (160, 160, 160),
+    'gc-': (224, 0, 0),
+    'gc-minor': (192, 0, 16),
+    'gc-collect': (255, 0, 0),
+    'jit-': (0, 224, 0),
+    'jit-running': (192, 255, 160),
+    'jit-tracing': (0, 255, 0),
+    'jit-optimize': (160, 255, 0),
+    'jit-backend': (0, 255, 144),
+    'jit-blackhole': (0, 160, 0),
+    }
+SUMMARY = {
+    None: 'normal execution',
+    '': 'other',
+    'gc-': 'gc',
+    'jit-': 'jit',
+    'jit-running': 'jit-running',
+    }
+
+def getcolor(category):
+    while category not in COLORS:
+        category = category[:-1]
+    return COLORS[category]
+
+def getlightercolor((r, g, b)):
+    return ((r*2+255)//3, (g*2+255)//3, (b*2+255)//3)
+
+def getdarkercolor((r, g, b)):
+    return (r*2//3, g*2//3, b*2//3)
+
+def getlabel(text, _cache={}):
+    try:
+        return _cache[text]
+    except KeyError:
+        pass
+    from PIL import Image, ImageDraw
+    if None not in _cache:
+        image = Image.new("RGBA", (1, 1), (0, 0, 0, 0))
+        draw = ImageDraw.Draw(image)
+        _cache[None] = draw
+    else:
+        draw = _cache[None]
+    sx, sy = draw.textsize(text)
+    texthoriz = Image.new("RGBA", (sx, sy), (0, 0, 0, 0))
+    ImageDraw.Draw(texthoriz).text((0, 0), text, fill=(0, 0, 0))
+    textvert = texthoriz.rotate(90)
+    _cache[text] = sx, sy, texthoriz, textvert
+    return _cache[text]
+
+def bevelrect(draw, (x1, y1, x2, y2), color):
+    if x2 <= x1:
+        x2 = x1 + 1   # minimal width
+    elif x2 >= x1 + 4:
+        draw.line((x1, y1+1, x1, y2-1), fill=getlightercolor(color))
+        x1 += 1
+        x2 -= 1
+        draw.line((x2, y1+1, x2, y2-1), fill=getdarkercolor(color))
+    draw.line((x1, y1, x2-1, y1), fill=getlightercolor(color))
+    y1 += 1
+    y2 -= 1
+    draw.line((x1, y2, x2-1, y2), fill=getdarkercolor(color))
+    draw.rectangle((x1, y1, x2-1, y2-1), fill=color)
+
+# ----------
+
+def get_timeline_image(log, width, height):
+    from PIL import Image, ImageDraw
+    timestart0, timestop0 = gettimebounds(log)
+    assert timestop0 > timestart0
+    timefactor = float(width) / (timestop0 - timestart0)
+    #
+    def recdraw(sublist, subheight):
+        firstx1 = None
+        for category1, timestart1, timestop1, subcats in sublist:
+            x1 = int((timestart1 - timestart0) * timefactor)
+            x2 = int((timestop1 - timestart0) * timefactor)
+            y1 = (height - subheight) / 2
+            y2 = y1 + subheight
+            y1 = int(y1)
+            y2 = int(y2)
+            color = getcolor(category1)
+            if firstx1 is None:
+                firstx1 = x1
+            bevelrect(draw, (x1, y1, x2, y2), color)
+            subcats = getsubcategories(subcats)
+            if subcats:
+                x2 = recdraw(subcats, subheight * 0.94) - 1
+            sx, sy, texthoriz, textvert = getlabel(category1)
+            if sx <= x2-x1-8:
+                image.paste(texthoriz, (x1+5, y1+5), texthoriz)
+            elif sy <= x2-x1-2:
+                image.paste(textvert, (x1+1, y1+5), textvert)
+        return firstx1
+    #
+    image = Image.new("RGBA", (width, height), (255, 255, 255, 0))
+    draw = ImageDraw.Draw(image)
+    recdraw(getsubcategories(log), height)
+    return image
+
+# ----------
+
+def render_histogram(times, time0, labels, width, barheight):
+    # Render a histogram showing horizontal time bars are given by the
+    # 'times' dictionary.  Each entry has the label specified by 'labels',
+    # or by default the key used in 'times'.
+    from PIL import Image, ImageDraw
+    times = [(time, key) for (key, time) in times.items()]
+    times.sort()
+    times.reverse()
+    images = []
+    for time, key in times:
+        fraction = float(time) / time0
+        if fraction < 0.01:
+            break
+        color = getcolor(key)
+        image = Image.new("RGBA", (width, barheight), (255, 255, 255, 0))
+        draw = ImageDraw.Draw(image)
+        x2 = int(fraction * width)
+        bevelrect(draw, (0, 0, x2, barheight), color)
+        # draw the labels "x%" and "key"
+        percent = "%.1f%%" % (100.0 * fraction,)
+        s1x, s1y, textpercent, vtextpercent = getlabel(percent)
+        s2x, _, textlabel, _ = getlabel(labels.get(key, key))
+        t1x = 5
+        if t1x + s1x >= x2 - 3:
+            if t1x + s1y < x2 - 3:
+                textpercent = vtextpercent
+                s1x = s1y
+            else:
+                t1x = x2 + 6
+        t2x = t1x + s1x + 12
+        if t2x + s2x >= x2 - 3:
+            t2x = max(t2x, x2 + 8)
+        image.paste(textpercent, (t1x, 5), textpercent)
+        image.paste(textlabel,   (t2x, 5), textlabel)
+        images.append(image)
+    if not images:
+        return None
+    return combine(images, spacing=0, border=1, horizontal=False)
+
+def get_timesummary_single_image(totaltimes, totaltime0, componentdict,
+                                 width, barheight):
+    # Compress the totaltimes dict so that its only entries left are
+    # from componentdict.  We do that by taking the times assigned to
+    # subkeys in totaltimes and adding them to the superkeys specified
+    # in componentdict.
+    totaltimes = totaltimes.copy()
+    for key, value in totaltimes.items():
+        if key in componentdict:
+            continue
+        del totaltimes[key]
+        if key is not None:
+            while key not in componentdict:
+                key = key[:-1]
+            try:
+                totaltimes[key] += value
+            except KeyError:
+                totaltimes[key] = value
+    return render_histogram(totaltimes, totaltime0, componentdict,
+                            width, barheight)
+
+def get_timesummary_image(log, summarywidth, summarybarheight):
+    timestart0, timestop0 = gettimebounds(log)
+    totaltime0 = timestop0 - timestart0
+    totaltimes = gettotaltimes(log)
+    spacing = 50
+    width = (summarywidth - spacing) // 2
+    img1 = get_timesummary_single_image(totaltimes, totaltime0, SUMMARY,
+                                        width, summarybarheight)
+    if None in totaltimes:
+        del totaltimes[None]
+    img2 = render_histogram(totaltimes, totaltime0, {},
+                            width, summarybarheight)
+    if img2 is None:
+        return img1
+    return combine([img1, img2], spacing=spacing, horizontal=True)
+
+# ----------
+
+def combine(imagelist, spacing=50, border=0, horizontal=False):
+    if len(imagelist) <= 1 and not border:
+        return imagelist[0]
+    from PIL import Image, ImageDraw
+    wlist = [image.size[0] for image in imagelist]
+    hlist = [image.size[1] for image in imagelist]
+    if horizontal:
+        w = sum(wlist) + spacing*(len(imagelist)-1)
+        h = max(hlist)
+    else:
+        w = max(wlist)
+        h = sum(hlist) + spacing*(len(imagelist)-1)
+    w += 2*border
+    h += 2*border
+    bigimage = Image.new("RGBA", (w, h), (255, 255, 255, 0))
+    if border:
+        draw = ImageDraw.Draw(bigimage)
+        draw.rectangle((0, 0, w-1, border-1), fill=(0, 0, 0))
+        draw.rectangle((0, h-border, w-1, h-1), fill=(0, 0, 0))
+        draw.rectangle((0, 0, border-1, h-1), fill=(0, 0, 0))
+        draw.rectangle((w-1, 0, w-border, h-1), fill=(0, 0, 0))
+    x = border
+    y = border
+    for image in imagelist:
+        bigimage.paste(image, (x, y))
+        if horizontal:
+            x += image.size[0] + spacing
+        else:
+            y += image.size[1] + spacing
+    return bigimage
+
+def draw_timeline_image(log, output=None, mainwidth=3000, mainheight=150,
+                        summarywidth=850, summarybarheight=40):
+    mainwidth = int(mainwidth)
+    mainheight = int(mainheight)
+    summarywidth = int(summarywidth)
+    summarybarheight = int(summarybarheight)
+    images = []
+    if mainwidth > 0 and mainheight > 0:
+        images.append(get_timeline_image(log, mainwidth, mainheight))
+    if summarywidth > 0 and summarybarheight > 0:
+        images.append(get_timesummary_image(log, summarywidth,
+                                                 summarybarheight))
+    image = combine(images, horizontal=False)
+    if output is None:
+        image.save(sys.stdout, format='png')
+    else:
+        image.save(output)
+
+def print_summary(log, out):
+    totaltimes = gettotaltimes(log)
+    if out == '-':
+        outfile = sys.stdout
+    else:
+        outfile = open(out, "w")
+    l = totaltimes.items()
+    l.sort(cmp=lambda a, b: cmp(b[1], a[1]))
+    total = sum([b for a, b in l])
+    for a, b in l:
+        if a is None:
+            a = 'interpret'
+        s = " " * (50 - len(a))
+        print >>outfile, a, s, str(b*100/total) + "%"
+    if out != '-':
+        outfile.close()
+
+# ____________________________________________________________
+
+
+ACTIONS = {
+    'draw-time': (draw_timeline_image, ['output=',
+                                        'mainwidth=', 'mainheight=',
+                                        'summarywidth=', 'summarybarheight=',
+                                        ]),
+    'print-summary': (print_summary, []),
+    }
+
+if __name__ == '__main__':
+    import getopt
+    if len(sys.argv) < 3:
+        print __doc__
+        sys.exit(2)
+    action = sys.argv[1]
+    func, longopts = ACTIONS[action]
+    options, args = getopt.gnu_getopt(sys.argv[2:], '', longopts)
+    if len(args) != 2:
+        print __doc__
+        sys.exit(2)
+
+    kwds = {}
+    for name, value in options:
+        assert name.startswith('--')
+        kwds[name[2:]] = value
+    log = parse_log_file(args[0])
+    func(log, args[1], **kwds)

rpython/tool/progressbar.py

+# -*- coding: utf-8 -*-
+# Copyright: 2009 Nadia Alramli
+# License: BSD
+"""Draws an animated terminal progress bar
+Usage:
+    p = ProgressBar("blue")
+    p.render(percentage, message)
+"""
+ 
+from pypy.tool import terminal
+import sys
+ 
+class ProgressBar(object):
+    """Terminal progress bar class"""
+    TEMPLATE = (
+     '%(percent)-2s%% %(color)s%(progress)s%(normal)s%(empty)s %(message)s\n'
+    )
+    PADDING = 7
+ 
+    def __init__(self, color=None, width=None, block='.', empty=' '):
+        """
+        color -- color name (BLUE GREEN CYAN RED MAGENTA YELLOW WHITE BLACK)
+        width -- bar width (optinal)
+        block -- progress display character (default '.')
+        empty -- bar display character (default ' ')
+        """
+        if color:
+            self.color = getattr(terminal, color.upper())
+        else:
+            self.color = ''
+        if width and width < terminal.COLUMNS - self.PADDING:
+            self.width = width
+        else:
+            # Adjust to the width of the terminal
+            self.width = terminal.COLUMNS - self.PADDING
+        self.block = block
+        self.empty = empty
+        self.progress = None
+        self.lines = 0
+ 
+    def render(self, percent, message = ''):
+        """Print the progress bar
+        percent -- the progress percentage %
+        message -- message string (optional)
+        """
+        inline_msg_len = 0
+        if message:
+            # The length of the first line in the message
+            inline_msg_len = len(message.splitlines()[0])
+        if inline_msg_len + self.width + self.PADDING > terminal.COLUMNS:
+            # The message is too long to fit in one line.
+            # Adjust the bar width to fit.
+            bar_width = terminal.COLUMNS - inline_msg_len -self.PADDING