Commits

mattip committed c4230f1 Merge

merge default into branch

Comments (0)

Files changed (22)

 *-win-32
 *-win-x86-32
 *-win-x86-64
+slave/pypy-buildbot
+master/pypy-buildbot
 .. -*- mode: rst -*-
 
-Everything has been tested with builbot 0.7.12.  Not sure what happens with
-other versions :-)
+Everything has been tested with builbot 0.8.8.
 
 How to hack the PyPy buildbot
 ==============================
 If you want to run buildbot in production, you need to make sure that the
 function ``pypybuildbot.util.we_are_debugging`` returns ``False`` in your
 environment.  At the moment of writing, debugging is enabled everywhere but on
-wyvern.
+cobra.
 
 You still need to fill ``master/slaveinfo.py`` with the passwords of the
 various slaves you want to use.
 
-Then, to start the buildbot master: ``cd master; make start``
+Then, to start the buildbot master: ``buildbot start <path-to pypy-buildbot/master>``
 
 
 To restart the buildmaster
 
 $ buildbot checkconfig
 
-$ make reconfig
+$ buildbot reconfig
 
 OR
 
-$ make stop
+$ buildbot stop
 
-$ make start
+$ buildbot start
 
 To run a buildslave
 ===================
 
 @app.route('/', methods=['POST'])
 def handle_payload():
-    payload = json.loads(flask.request.form['payload'])
+    open('/tmp/payload', 'w').write(flask.request.form['payload'])
     try:
+        payload = json.loads(flask.request.form['payload'])
         from . import hook
         hook.handle(payload, test=app.testing)
     except:

bot2/pypybuildbot/arm_master.py

                         + crosstranslationjitargs),
     platform='linux-armhf-raring',
     interpreter='pypy',
-    prefix=['schroot', '-c', 'raring'])
+    prefix=['schroot', '-c', 'raring'],
+    trigger='JITLINUXARMHF_RARING_scheduler')
 
 pypyARMJITTranslatedTestFactory = pypybuilds.TranslatedTests(
     translationArgs=(crosstranslationargs
     app_tests=True,
     platform='linux-armhf-raspbian',
 )
+pypyARMHF_RARING_JITTranslatedTestFactory = pypybuilds.TranslatedTests(
+    translationArgs=(crosstranslationargs
+                        + jit_translation_args
+                        + crosstranslationjitargs),
+    lib_python=True,
+    pypyjit=True,
+    app_tests=True,
+    platform='linux-armhf-raring',
+    )
 #
+LINUXARMHF = "own-linux-armhf"
 APPLVLLINUXARM = "pypy-c-app-level-linux-armel"
 APPLVLLINUXARMHF_v7 = "pypy-c-app-level-linux-armhf-v7"
 APPLVLLINUXARMHF_RASPBIAN = "pypy-c-app-level-linux-armhf-raspbian"
 JITLINUXARM = "pypy-c-jit-linux-armel"
 JITLINUXARMHF_v7 = "pypy-c-jit-linux-armhf-v7"
 JITLINUXARMHF_RASPBIAN = "pypy-c-jit-linux-armhf-raspbian"
+JITLINUXARMHF_RARING = "pypy-c-jit-linux-armhf-raring"
 
 JITBACKENDONLYLINUXARMEL = "jitbackendonly-own-linux-armel"
 JITBACKENDONLYLINUXARMHF = "jitbackendonly-own-linux-armhf"
 BUILDJITLINUXARMHF_RASPBIAN = "build-pypy-c-jit-linux-armhf-raspbian"
 BUILDJITLINUXARMHF_RARING = "build-pypy-c-jit-linux-armhf-raring"
 
+builderNames = [
+    LINUXARMHF,
+    APPLVLLINUXARM,
+    APPLVLLINUXARMHF_v7,
+    APPLVLLINUXARMHF_RASPBIAN,
+    JITLINUXARM,
+    JITLINUXARMHF_v7,
+    JITLINUXARMHF_RASPBIAN,
+    JITBACKENDONLYLINUXARMEL,
+    JITBACKENDONLYLINUXARMHF,
+    JITBACKENDONLYLINUXARMHF_v7,
+    BUILDLINUXARM,
+    BUILDJITLINUXARM,
+    BUILDLINUXARMHF_RASPBIAN,
+    BUILDJITLINUXARMHF_RASPBIAN,
+]
 
 schedulers = [
     Nightly("nighly-arm-0-00", [
         BUILDLINUXARM,                 # on hhu-cross-armel, uses 1 core
         BUILDLINUXARMHF_RASPBIAN,      # on hhu-cross-raspbianhf, uses 1 core
 
+        LINUXARMHF,                    # onw tests on greenbox3-node0
+
         JITBACKENDONLYLINUXARMEL,      # on hhu-imx.53
         JITBACKENDONLYLINUXARMHF,
         JITBACKENDONLYLINUXARMHF_v7,   # on cubieboard-bob
         JITLINUXARMHF_RASPBIAN,       # triggered by BUILDJITLINUXARMHF_RASPBIAN
         JITLINUXARMHF_v7,             # triggered by BUILDJITLINUXARMHF_RASPBIAN, on cubieboard-bob
     ]),
+
+    Triggerable("JITLINUXARMHF_RARING_scheduler", [
+        JITLINUXARMHF_RARING,         # triggered by BUILDJITLINUXARMHF_RARING
+    ])
 ]
 
 builders = [
    "locks": [ARMBoardLock.access('counting')],
    },
   ## armv7
+  {"name": LINUXARMHF,
+   "slavenames": ["greenbox3-node0"],
+   "builddir": LINUXARMHF,
+   "factory": pypyOwnTestFactoryARM,
+   "category": 'linux-armhf',
+  },
   {"name": JITBACKENDONLYLINUXARMHF_v7,
    "slavenames": ['cubieboard-bob'],
    "builddir": JITBACKENDONLYLINUXARMHF_v7,
    'category': 'linux-armhf',
    "locks": [ARMBoardLock.access('counting')],
    },
+  {"name": JITLINUXARMHF_RARING,
+   "slavenames": ["greenbox3-node0"],
+   'builddir': JITLINUXARMHF_RARING,
+   'factory': pypyARMHF_RARING_JITTranslatedTestFactory,
+   'category': 'linux-armhf',
+   },
   # Translation Builders for ARM
   {"name": BUILDLINUXARM,
    "slavenames": ['hhu-cross-armel'],

bot2/pypybuildbot/builds.py

+from buildbot.steps.source.mercurial import Mercurial
+from buildbot.process.buildstep import BuildStep
 from buildbot.process import factory
 from buildbot.steps import shell, transfer
 from buildbot.steps.trigger import Trigger
 from buildbot.process.properties import WithProperties
 from buildbot import locks
 from pypybuildbot.util import symlink_force
+from buildbot.status.results import SKIPPED, SUCCESS
 import os
 
 # buildbot supports SlaveLocks, which can be used to limit the amout of builds
 # while the boards can only run one job at the same time
 ARMBoardLock = locks.SlaveLock('arm_boards', maxCount=1)
 
-
-# XXX monkey patch Trigger class, there are to issues with the list of renderables
-# original: Trigger.renderables = [ 'set_propetries', 'scheduler', 'sourceStamp' ]
-Trigger.renderables = [ 'set_properties', 'schedulerNames', 'sourceStamp' ]
+map_branch_name = lambda x: x if x not in ['', None, 'default'] else 'trunk'
 
 class ShellCmd(shell.ShellCommand):
     # our own version that can distinguish abort cases (rc == -1)
 
     def start(self):
         properties = self.build.getProperties()
-        branch = properties['branch']
-        if branch is None:
-            branch = 'trunk'
+        branch = map_branch_name(properties['branch'])
         #masterdest = properties.render(self.masterdest)
         masterdest = os.path.expanduser(self.masterdest)
         if branch.startswith('/'):
     def start(self):
 
         properties = self.build.getProperties()
-        branch = properties['branch']
-        revision = properties['revision']
-
-        if branch is None:
-            branch = 'trunk'
+        branch = map_branch_name(properties['branch'])
+        revision = properties['final_file_name']
         mastersrc = os.path.expanduser(self.mastersrc)
 
         if branch.startswith('/'):
             branch = branch[1:]
         mastersrc = os.path.join(mastersrc, branch)
-        if revision is not None:
+        if revision:
             basename = WithProperties(self.basename).getRenderingFor(self.build)
             basename = basename.replace(':', '-')
         else:
-            basename = self.basename.replace('%(revision)s', 'latest')
+            basename = self.basename.replace('%(final_file_name)s', 'latest')
             assert '%' not in basename
 
         self.mastersrc = os.path.join(mastersrc, basename)
             builder.summary_by_branch_and_revision = {}
         try:
             rev = properties['got_revision']
-            branch = properties['branch']
-            if branch is None:
-                branch = 'trunk'
+            branch = map_branch_name(properties['branch'])
             if branch.endswith('/'):
                 branch = branch[:-1]
         except KeyError:
         builder.saveYourself()
 
 # _______________________________________________________________
-
-class UpdateCheckout(ShellCmd):
-    description = 'hg update'
-    command = 'UNKNOWN'
-
-    def __init__(self, workdir=None, haltOnFailure=True, force_branch=None,
-                 **kwargs):
-        ShellCmd.__init__(self, workdir=workdir, haltOnFailure=haltOnFailure,
-                          **kwargs)
-        self.force_branch = force_branch
-        self.addFactoryArguments(force_branch=force_branch)
-
-    def start(self):
-        if self.force_branch is not None:
-            branch = self.force_branch
-            # Note: We could add a warning to the output if we
-            # ignore the branch set by the user.
-        else:
-            properties = self.build.getProperties()
-            branch = properties['branch'] or 'default'
-        command = ["hg", "update", "--clean", "-r", branch]
-        self.setCommand(command)
-        ShellCmd.start(self)
-
-
+# XXX Currently the build properties got_revision and final_file_name contain
+# the revision number and the changeset-id, CheckGotRevision takes care to set
+# the corresponding build properties
+# rev:changeset for got_revision
+# rev-changeset for final_file_name
+#
+# The rev part of got_revision and filename is used everywhere to sort the
+# builds, i.e. on the summary and download pages.
+#
+# The rev part is strictly local and needs to be removed from the SourceStamp,
+# at least for decoupled builds, which is what ParseRevision does.
+#
+# XXX in general it would be nice to drop the revision-number using only the
+# changeset-id for got_revision and final_file_name and sorting the builds
+# chronologically
 class UpdateGitCheckout(ShellCmd):
     description = 'git checkout'
     command = 'UNKNOWN'
             # ':' should not be part of filenames --- too many issues
             self.build.setProperty('got_revision', got_revision,
                                    'got_revision')
-            self.build.setProperty('final_file_name', final_file_name,
-                                   'got_revision')
+            if not self.build.hasProperty('final_file_name'):
+                self.build.setProperty('final_file_name', final_file_name,
+                                       'got_revision')
 
+class ParseRevision(BuildStep):
+    """Parse the revision property of the source stamp and extract the global
+    part of the revision
+    123:3a34 -> 3a34"""
+    name = "parse_revision"
 
-def update_hg(platform, factory, repourl, workdir, use_branch,
-              force_branch=None):
+    def __init__(self, *args, **kwargs):
+        BuildStep.__init__(self, *args, **kwargs)
+
+    @staticmethod
+    def hideStepIf(results, step):
+        return results==SKIPPED
+
+    @staticmethod
+    def doStepIf(step):
+        revision = step.build.getSourceStamp().revision
+        return isinstance(revision, (unicode, str)) and ':' in revision
+
+    def start(self):
+        stamp = self.build.getSourceStamp()
+        revision = stamp.revision if stamp.revision is not None else ''
+        #
+        if not isinstance(revision, (unicode, str)) or ":" not in revision:
+            self.finished(SKIPPED)
+            return
+        #
+        self.build.setProperty('original_revision', revision, 'parse_revision')
+        self.build.setProperty('final_file_name',
+                                revision.replace(':', '-'), 'parse_revision')
+        #
+        parts = revision.split(':')
+        self.build.setProperty('revision', parts[1], 'parse_revision')
+        stamp.revision = parts[1]
+        self.finished(SUCCESS)
+
+
+def update_hg_old_method(platform, factory, repourl, workdir):
+    # baaaaaah.  Seems that the Mercurial class doesn't support
+    # updating to a different branch than the one specified by
+    # the user (like "default").  This is nonsense if we need
+    # an auxiliary check-out :-(  At least I didn't find how.
     if platform == 'win32':
         command = "if not exist .hg rmdir /q /s ."
     else:
                              command="hg pull",
                              workdir=workdir))
     #
-    if use_branch or force_branch:
-        factory.addStep(UpdateCheckout(workdir=workdir,
-                                       haltOnFailure=True,
-                                       force_branch=force_branch))
-    else:
-        factory.addStep(ShellCmd(description="hg update",
-                command=WithProperties("hg update --clean %(revision)s"),
-                workdir=workdir))
+    # here, update without caring about branches
+    factory.addStep(ShellCmd(description="hg update",
+           command=WithProperties("hg update --clean %(revision)s"),
+           workdir=workdir))
+
+def update_hg(platform, factory, repourl, workdir, use_branch,
+              force_branch=None):
+    if not use_branch:
+        assert force_branch is None
+        update_hg_old_method(platform, factory, repourl, workdir)
+        return
+    factory.addStep(
+            Mercurial(
+                repourl=repourl,
+                mode='full',
+                method='fresh',
+                defaultBranch=force_branch,
+                branchType='inrepo',
+                clobberOnBranchChange=False,
+                workdir=workdir,
+                logEnviron=False))
 
 def update_git(platform, factory, repourl, workdir, use_branch,
               force_branch=None):
         # for debugging
         repourl = '/home/antocuni/pypy/default'
     #
+    factory.addStep(ParseRevision(hideStepIf=ParseRevision.hideStepIf,
+                                  doStepIf=ParseRevision.doStepIf))
+    #
     update_hg(platform, factory, repourl, workdir, use_branch=True,
               force_branch=force_branch)
     #
     factory.addStep(CheckGotRevision(workdir=workdir))
 
+
 def build_name(platform, jit=False, flags=[], placeholder=None):
     if placeholder is None:
         placeholder = '%(final_file_name)s'
             command=['rm', '-rf', 'pypy-c'],
             workdir='.'))
         extension = get_extension(platform)
-        name = build_name(platform, pypyjit, translationArgs, placeholder='%(revision)s') + extension
+        name = build_name(platform, pypyjit, translationArgs, placeholder='%(final_file_name)s') + extension
         self.addStep(PyPyDownload(
             basename=name,
             mastersrc='~/nightly',
             self.addStep(ShellCmd(
                 description="decompress pypy-c",
                 command=['tar', '--extract', '--file=pypy_build'+ extension, '--strip-components=1', '--directory=.'],
-                workdir='pypy-c'))
+                workdir='pypy-c',
+                haltOnFailure=True,
+                ))
 
+        self.addStep(ShellCmd(
+            description="reset permissions",
+            command=['chmod', 'u+rw', '-R', 'build/include'],
+            haltOnFailure=True,
+            workdir='.'))
         # copy pypy-c to the expected location within the pypy source checkout
         self.addStep(ShellCmd(
             description="move pypy-c",
             command=['cp', '-v', 'pypy-c/bin/pypy', 'build/pypy/goal/pypy-c'],
+            haltOnFailure=True,
             workdir='.'))
         # copy generated and copied header files to build/include
         self.addStep(ShellCmd(
             description="move header files",
             command=['cp', '-vr', 'pypy-c/include', 'build'],
+            haltOnFailure=True,
             workdir='.'))
         # copy ctypes_resource_cache generated during translation
         self.addStep(ShellCmd(
+            description="reset permissions",
+            command=['chmod', 'u+rw', '-R', 'build/lib_pypy'],
+            haltOnFailure=True,
+            workdir='.'))
+        self.addStep(ShellCmd(
             description="move ctypes resource cache",
             command=['cp', '-rv', 'pypy-c/lib_pypy/ctypes_config_cache', 'build/lib_pypy'],
+            haltOnFailure=True,
             workdir='.'))
 
         add_translated_tests(self, prefix, platform, app_tests, lib_python, pypyjit)
             command=prefix + ["python", "pypy/tool/release/package.py",
                      ".", WithProperties(name), 'pypy',
                      '.'],
+            haltOnFailure=True,
             workdir='build'))
         nightly = '~/nightly/'
         extension = get_extension(platform)
                                 basename=name + extension,
                                 workdir='.',
                                 blocksize=100 * 1024))
-        if trigger: # if provided trigger schedulers that are depend on this one
+        if trigger: # if provided trigger schedulers that depend on this one
             self.addStep(Trigger(schedulerNames=[trigger]))
 
 
     def __init__(self, platform='linux', host='tannit', postfix=''):
         factory.BuildFactory.__init__(self)
 
-        setup_steps(platform, self)
         #
         repourl = 'https://bitbucket.org/pypy/benchmarks'
         update_hg(platform, self, repourl, 'benchmarks', use_branch=False)
+        #
+        setup_steps(platform, self)
         if host == 'tannit':
             lock = TannitCPU
         elif host == 'speed_python':
         '''
         factory.BuildFactory.__init__(self)
 
+        # check out and update benchmarks
+        repourl = 'https://bitbucket.org/pypy/benchmarks'
+        update_hg(platform, self, repourl, 'benchmarks', use_branch=False)
+
         # checks out and updates the repo
         setup_steps(platform, self, repourl='http://hg.python.org/cpython',
                     force_branch=branch)
 
-        # check out and update benchmarks
-        repourl = 'https://bitbucket.org/pypy/benchmarks'
-        update_hg(platform, self, repourl, 'benchmarks', use_branch=False)
-
         lock = SpeedPythonCPU
 
         self.addStep(ShellCmd(
                                          masterdest=WithProperties(resultfile),
                                          workdir="."))
 
+class PyPyBuildbotTestFactory(factory.BuildFactory):
+    def __init__(self):
+        factory.BuildFactory.__init__(self)
+        # clone
+        self.addStep(
+            Mercurial(
+                repourl='https://bitbucket.org/pypy/buildbot',
+                mode='incremental',
+                method='fresh',
+                defaultBranch='default',
+                branchType='inrepo',
+                clobberOnBranchChange=False,
+                logEnviron=False))
+        # create a virtualenv
+        self.addStep(ShellCmd(
+            description='create virtualenv',
+            haltOnFailure=True,
+            command='virtualenv ../venv'))
+        # install deps
+        self.addStep(ShellCmd(
+            description="install dependencies",
+            haltOnFailure=True,
+            command=('../venv/bin/pip install -r requirements.txt').split()))
+        # run tests
+        self.addStep(PytestCmd(
+            description="pytest buildbot",
+            haltOnFailure=True,
+            command=["../venv/bin/py.test",
+                     "--resultlog=testrun.log",
+                     ],
+            logfiles={'pytestLog': 'testrun.log'}))
+
+
 class NativeNumpyTests(factory.BuildFactory):
     '''
     Download a pypy nightly, install nose and numpy, and run the numpy test suite

bot2/pypybuildbot/ircbot.py

 the customized IRC messages.
 """
 
-import re
-from buildbot.status.words import Contact, IRC, log
+from buildbot.status.words import IRC, log, IRCContact
 
+# see http://www.mirc.com/colors.html
 USE_COLOR_CODES = True
-GREEN  = '\x033'
-RED    = '\x034'
-AZURE  = '\x0311'
-BLUE   = '\x0312'
-PURPLE = '\x0313'
-GRAY   = '\x0315'
-BOLD   = '\x02'
-def color(code, s):
+BOLD = '\x02'
+COLORS = {
+    'WHITE': '\x030',
+    'BLACK': '\x031',
+    'GREEN': '\x033',
+    'RED': '\x034',
+    'AZURE': '\x0311',
+    'BLUE': '\x0312',
+    'PURPLE': '\x0313',
+    'GRAY': '\x0315',
+}
+
+
+def color(s, code=None, bold=False):
     if USE_COLOR_CODES:
-        return '%s%s\x0F' % (code, s)
+        c = BOLD if bold else ''
+        if code in COLORS:
+            c += COLORS[code]
+        return '%s%s\x0F' % (c, s)
     return s
 
-def extract_username(build):
-    regexp = r"The web-page 'force build' button was pressed by '(.*)': .*"
-    match = re.match(regexp, build.getReason())
-    if match:
-        return match.group(1)
-    return None
+
+def get_build_information(build):
+    owner = build.getProperty("owner")
+    reason = build.getProperty("reason")
+    return ": ".join(k for k in (owner, reason) if k)
 
 
 def get_description_for_build(url, build):
-    url = color(GRAY, url) # in gray
+    url = color(url, 'GRAY')  # in gray
     infos = []
-    username = extract_username(build)
-    if username:
-        infos.append(color(BLUE, username)) # in blue
+    buildinfo = get_build_information(build)
+    if buildinfo:
+        infos.append(color(buildinfo, 'BLUE'))  # in blue
     #
-    branch = build.source.branch
+    branch = build.getProperty('branch')
     if branch:
-        infos.append(color(BOLD, branch)) # in bold
+        infos.append(color(branch, bold=True))  # in bold
     #
     if infos:
         return '%s [%s]' % (url, ', '.join(infos))
     else:
         return url
 
+
 def buildStarted(self, builderName, build):
     builder = build.getBuilder()
-    log.msg('[Contact] Builder %r in category %s started' % (builder, builder.category))
+    log.msg('[Contact] Builder %r in category %s started' %
+                                            (builder, builder.category))
 
     # only notify about builders we are interested in
 
-    if (self.channel.categories != None and
-       builder.category not in self.channel.categories):
+    if (self.bot.categories is not None and
+       builder.category not in self.bot.categories):
         log.msg('Not notifying for a build in the wrong category')
         return
 
         log.msg('Not notifying for a build when started-notification disabled')
         return
 
-    buildurl = self.channel.status.getURLForThing(build)
+    buildurl = self.bot.status.getURLForThing(build)
     descr = get_description_for_build(buildurl, build)
     msg = "Started: %s" % descr
     self.send(msg)
     builder = build.getBuilder()
 
     # only notify about builders we are interested in
-    log.msg('[Contact] builder %r in category %s finished' % (builder, builder.category))
+    log.msg('[Contact] builder %r in category %s finished' %
+                                            (builder, builder.category))
 
-    if (self.channel.categories != None and
-        builder.category not in self.channel.categories):
+    if (self.bot.categories is not None and
+        builder.category not in self.bot.categories):
         return
 
     if not self.notify_for_finished(build):
         return
 
-    buildurl = self.channel.status.getURLForThing(build)
+    buildurl = self.bot.status.getURLForThing(build)
     descr = get_description_for_build(buildurl, build)
-    result = self.results_descriptions.get(build.getResults(), "Finished ??")
-    if result == 'Success':
-        result = color(BOLD+GREEN, result)
-    elif result == 'Exception':
-        result = color(BOLD+PURPLE, result)
-    else:
-        result = color(BOLD+RED, result)
+    result, c = self.results_descriptions.get(build.getResults(),
+                                                ("Finished ??", 'RED'))
+    if c not in COLORS:
+        c = 'RED'
+    result = color(result, c, bold=True)
     msg = "%s: %s" % (result, descr)
     self.send(msg)
 
-Contact.buildStarted = buildStarted
-Contact.buildFinished = buildFinished
+IRCContact.buildStarted = buildStarted
+IRCContact.buildFinished = buildFinished
 
 
 ## def send_message(message, test=False):

bot2/pypybuildbot/master.py

 
 import os
-import getpass
-from buildbot.scheduler import Nightly, Triggerable
+from buildbot.scheduler import Nightly
+from buildbot.schedulers.forcesched import ForceScheduler
+from buildbot.schedulers.forcesched import ValidationError
 from buildbot.buildslave import BuildSlave
 from buildbot.status.html import WebStatus
-from buildbot.process.builder import Builder
 #from buildbot import manhole
 from pypybuildbot.pypylist import PyPyList, NumpyStatusList
-from pypybuildbot.ircbot import IRC # side effects
+from pypybuildbot.ircbot import IRC  # side effects
 from pypybuildbot.util import we_are_debugging
 
 # Forbid "force build" with empty user name
-from buildbot.status.web.builder import StatusResourceBuilder
-def my_force(self, req, *args, **kwds):
-    name = req.args.get("username", [""])[0]
-    assert name, "Please write your name in the corresponding field."
-    return _previous_force(self, req, *args, **kwds)
-_previous_force = StatusResourceBuilder.force
-if _previous_force.__name__ == 'force':
-    StatusResourceBuilder.force = my_force
-# Done
+class CustomForceScheduler(ForceScheduler):
+    def force(self, owner, builder_name, **kwargs):
+        if not owner:
+            raise ValidationError, "Please write your name in the corresponding field."
+        return ForceScheduler.force(self, owner, builder_name, **kwargs)
+
 
 if we_are_debugging():
     channel = '#buildbot-test'
 JITLINUX32 = "pypy-c-jit-linux-x86-32"
 JITLINUX64 = "pypy-c-jit-linux-x86-64"
 JITMACOSX64 = "pypy-c-jit-macosx-x86-64"
+JITMACOSX64_2 = "pypy-c-jit-macosx-x86-64-2"
 JITWIN32 = "pypy-c-jit-win-x86-32"
 JITWIN64 = "pypy-c-jit-win-x86-64"
 JITFREEBSD764 = 'pypy-c-jit-freebsd-7-x86-64'
 JITBENCH64_2 = 'jit-benchmark-linux-x86-64-2'
 CPYTHON_64 = "cpython-2-benchmark-x86-64"
 NUMPY_64 = "numpy-compatability-linux-x86-64"
+# buildbot builder
+PYPYBUILDBOT = 'pypy-buildbot'
 
 extra_opts = {'xerxes': {'keepalive_interval': 15},
              'aurora': {'max_builds': 1},
             JITFREEBSD864,             # on ananke
             JITFREEBSD964,             # on exarkun's freebsd
             JITMACOSX64,               # on xerxes
-            ], branch=None, hour=0, minute=0),
+            # buildbot selftest
+            PYPYBUILDBOT               # on cobra
+            ], branch='default', hour=0, minute=0),
 
         Nightly("nightly-2-00", [
             JITBENCH,                  # on tannit32, uses 1 core (in part exclusively)
             JITBENCH64,                # on tannit64, uses 1 core (in part exclusively)
-        ], branch=None, hour=2, minute=0),
+        ], branch='default', hour=2, minute=0),
 
         Nightly("nightly-2-00-py3k", [
             LINUX64,                   # on allegro64, uses all cores
         Nightly("nighly-ppc", [
             JITONLYLINUXPPC64,         # on gcc1
             ], branch='ppc-jit-backend', hour=1, minute=0),
+        CustomForceScheduler('Force Scheduler',
+            builderNames=[
+                        PYPYBUILDBOT,
+                        LINUX32,
+                        LINUX64,
+                        INDIANA32,
+
+                        MACOSX32,
+                        WIN32,
+                        WIN64,
+                        APPLVLLINUX32,
+                        APPLVLLINUX64,
+                        APPLVLWIN32,
+
+                        LIBPYTHON_LINUX32,
+                        LIBPYTHON_LINUX64,
+
+                        JITLINUX32,
+                        JITLINUX64,
+                        JITMACOSX64,
+                        JITMACOSX64_2,
+                        JITWIN32,
+                        JITWIN64,
+                        JITFREEBSD764,
+                        JITFREEBSD864,
+                        JITFREEBSD964,
+                        JITINDIANA32,
+
+                        JITONLYLINUXPPC64,
+                        JITBENCH,
+                        JITBENCH64,
+            ] + ARM.builderNames, properties=[]),
     ] + ARM.schedulers,
 
     'status': [status, ircbot],
                    "category": 'mac32'
                   },
                   {"name" : JITMACOSX64,
-                   "slavenames": ["xerxes"],
+                   "slavenames": ["xerxes", "tosh"],
                    'builddir' : JITMACOSX64,
                    'factory' : pypyJITTranslatedTestFactoryOSX64,
                    'category' : 'mac64',
                    },
+                  {"name" : JITMACOSX64_2,
+                   "slavenames": ["xerxes", "tosh"],
+                   'builddir' : JITMACOSX64_2,
+                   'factory' : pypyJITTranslatedTestFactoryOSX64,
+                   'category' : 'mac64',
+                   },
                   {"name": WIN32,
                    "slavenames": ["aurora", "SalsaSalsa"],
                    "builddir": WIN32,
                    'factory': pypyNumpyCompatability,
                    'category': 'numpy',
                    },
+                  {'name': PYPYBUILDBOT,
+                   'slavenames': ['cobra'],
+                   'builddir': PYPYBUILDBOT,
+                   'factory': pypybuilds.PyPyBuildbotTestFactory(),
+                   'category': 'buildbot',
+                   }
+
                 ] + ARM.builders,
 
     # http://readthedocs.org/docs/buildbot/en/latest/tour.html#debugging-with-manhole

bot2/pypybuildbot/summary.py

 
     def _start_cat_branch(self, cat_branch, fine=False):
         category, branch = cat_branch
-        branch = trunk_name(branch)
+        branch = meta_branch_name(branch)
         category = category_name(category)
 
         self.cur_cat_branch = (category, branch)
         return lambda v: v in membs
 
 def make_subst(v1, v2):
+    if not isinstance(v1, list):
+        v1 = [v1]
     def subst(v):
-        if v == v1:
+        if v in v1:
             return v2
         return v
     return subst
 
-trunk_name = make_subst(None, "<trunk>")
-trunk_value = make_subst("<trunk>", None)
+# Map certain branch names from SourceStamps to a common name shown on the page
+meta_branch_name = make_subst(['default', '', None], '<trunk>')
+# map the meta-branch <trunk> to the actual branch entries from the
+# SourceStamp
+default_value = make_subst('<trunk>', ['default', '', None])
 category_name = make_subst(None, '-')
 nocat_value = make_subst("-", None)
 
 
     def getTitle(self, request):
         status = self.getStatus(request)
-        return "%s: summaries of last %d revisions" % (status.getProjectName(),
-                                                       N)
+        return "%s: summaries of last %d revisions" % (status.getTitle(), N)
 
     @staticmethod
     def _prune_runs(runs, cutnum):
         except KeyError:
             pass
         builder = status.botmaster.builders[builderName]
+        factory = builder.config.factory
         branch = None
-        for _, kw in builder.buildFactory.steps:
+        for step in factory.steps:
+            kw = step.kwargs
             if 'defaultBranch' in kw:
                 if kw.get('explicitBranch'):
                     branch = kw['defaultBranch']
                          only_builder or only_branches)
 
         cat_branches = {}
-
         for builderName in status.getBuilderNames(only_categories):
             if not test_builder(builderName):
                 continue
             for build in builditer:
                 if prune_old and self._age(build) > 7:
                     continue
+                if self._age(build) > 60:   # two months old: prune anyway
+                    continue
                 branch = self._get_branch(status, build)
                 if not test_branch(branch):
                     continue
                 if not test_rev(got_rev):
                     continue
 
+                branch = meta_branch_name(branch)
                 cat_branch = (builderStatus.category, branch)
 
                 runs, no_revision_builds = cat_branches.setdefault(cat_branch,
         only_branches = request.args.get('branch', None)
         only_recentrevs = request.args.get('recentrev', None)
         if only_branches is not None:
-            only_branches = map(trunk_value, only_branches)
+            branches = []
+            for x in map(default_value, only_branches):
+                if isinstance(x, str):
+                    branches.append(x)
+                else:
+                    branches.extend(x)
+            only_branches = branches
         only_builder = request.args.get('builder', None)
         only_builds = None
         if only_builder is not None:
                                          outcome_set_cache.stats()))
 
         if request.args:
-            trunk_vs_any_text = "filter nothing"
-            trunk_vs_any_query = ""
+            default_vs_any_text = "filter nothing"
+            default_vs_any_query = ""
         else:
-            trunk_vs_any_text = "all <trunk>"
-            trunk_vs_any_query = "?branch=<trunk>"
+            default_vs_any_text = "all <trunk>"
+            default_vs_any_query = "?branch=<trunk>"
 
-        trunk_vs_any_anchor = html.a(trunk_vs_any_text,
+        default_vs_any_anchor = html.a(default_vs_any_text,
                                      href="/summary%s" %
-                                     trunk_vs_any_query,
+                                     default_vs_any_query,
                                      class_="failSummary trunkVsAny")
-        trunk_vs_any = html.div(trunk_vs_any_anchor,
+        default_vs_any = html.div(default_vs_any_anchor,
                                 style="position: absolute; right: 5%;")
-        return trunk_vs_any.unicode() + page.render()
+        return default_vs_any.unicode() + page.render()

bot2/pypybuildbot/test/test_builds.py

 
 class FakeProperties(object):
 
-    def __init__(self):
-        from buildbot.process.properties import PropertyMap
-        self.pmap = PropertyMap(self)
-    
+    sources = {}
+
+    def __init__(self, properties=None):
+        if properties is None:
+            self.properties = {'branch':None, 'got_revision': 123,
+                    'final_file_name': '123-ea5ca8'}
+        else:
+            self.properties = properties
+
     def __getitem__(self, item):
-        if item == 'branch':
-            return None
-        if item == 'got_revision':
-            return 123
-        if item == 'final_file_name':
-            return '123-ea5ca8'
-    
+        return self.properties.get(item)
+
+    def __setitem__(self, name, value):
+        self.properties[name] = value
+
     def render(self, x):
         return x
 
+class FakeSourceStamp(object):
+    def __init__(self, properties=None):
+        self.properties = properties if properties is not None else {}
+
+    def __getattr__(self, name):
+        return self.properties.get(name)
+
+    def __setattribute__(self, name, value):
+        self.properties[name] = value
+
 class FakeBuild(object):
     slaveEnvironment = None
 
-    def __init__(self):
-        self.properties = FakeProperties()
-    
+    def __init__(self, properties=None):
+        self.properties = FakeProperties(properties)
+        self.source_stamp = FakeSourceStamp(properties)
+
     def getProperties(self):
         return self.properties
 
+    def setProperty(self, name, value, source):
+        self.properties[name] = value
+        self.properties.sources[name] = source
+
     def getSlaveCommandVersion(self, *args):
         return 3
 
+    def getSourceStamp(self, *args):
+        return self.source_stamp
+
 class FakeStepStatus(object):
     def setText(self, *args):
         pass
 
+    def stepFinished(self, results):
+        self.results = results
+
+    def setHidden(self, *args):
+        pass
+
 class FakeDeferred(object):
+    def callback(*args):
+        pass
     def addCallback(self, *args):
         return FakeDeferred()
     def addErrback(self, *args):
         return FakeDeferred()
 
 def test_Translate():
-    expected = ['translate.py', '--batch', '-O0',
+    expected = ['pypy', '../../rpython/bin/rpython', '--batch', '-O0',
                 'targetpypystandalone', '--no-allworkingmodules']
 
     translateInst = builds.Translate(['-O0'], ['--no-allworkingmodules'])
 
     assert translateInst.command[-len(expected):] == expected
     
-    translateFactory, kw = translateInst.factory
-    rebuiltTranslate = translateFactory(**kw)
+    translateFactory = translateInst._getStepFactory().factory
+    args = translateInst._getStepFactory().args
+    rebuiltTranslate = translateFactory(*args)
                 
     assert rebuiltTranslate.command[-len(expected):] == expected
 
     inst = builds.PyPyUpload(slavesrc='slavesrc', masterdest=str(pth.join('mstr')),
                              basename='base-%(final_file_name)s', workdir='.',
                              blocksize=100)
-    factory, kw = inst.factory
+    factory = inst._getStepFactory().factory
+    kw = inst._getStepFactory().kwargs
     rebuilt = factory(**kw)
     rebuilt.build = FakeBuild()
     rebuilt.step_status = FakeStepStatus()
         step.commandComplete(cmd)
         summary = builder.summary_by_branch_and_revision[('trunk', '123')]
         assert summary.to_tuple() == (2, 2, 4, 0)
+
+
+class TestParseRevision(object):
+
+    def setup_method(self, mth):
+        inst = builds.ParseRevision()
+        factory = inst._getStepFactory().factory
+        kw = inst._getStepFactory().kwargs
+        self.rebuilt = factory(**kw)
+        self.rebuilt.step_status = FakeStepStatus()
+        self.rebuilt.deferred = FakeDeferred()
+
+    def test_has_revision(self):
+        self.rebuilt.build = FakeBuild({'revision':u'123:ea5ca8'})
+        self.rebuilt.start()
+        assert self.rebuilt.build.getProperties()['revision'] == 'ea5ca8'
+        assert self.rebuilt.build.getProperties()['original_revision'] == '123:ea5ca8'
+        assert self.rebuilt.build.getProperties()['final_file_name'] == '123-ea5ca8'
+
+    def test_no_revision(self):
+        self.rebuilt.build = FakeBuild()
+        self.rebuilt.start()
+        assert self.rebuilt.build.getProperties()['revision'] is None
+
+    def test_revision_no_local_part(self):
+        self.rebuilt.build = FakeBuild({'revision':u'ea5ca8'})
+        self.rebuilt.start()
+        assert self.rebuilt.build.getProperties()['revision'] == 'ea5ca8'
+
+    def test_empty_revision(self):
+        self.rebuilt.build = FakeBuild({'revision':u''})
+        self.rebuilt.start()
+        assert self.rebuilt.build.getProperties()['revision'] == ''

bot2/pypybuildbot/test/test_ircbot.py

 from pypybuildbot import ircbot
 
+
 def setup_module(mod):
     ircbot.USE_COLOR_CODES = False
 
+
 def teardown_module(mod):
     ircbot.USE_COLOR_CODES = True
 
+
 class FakeBuild(object):
 
-    def __init__(self, reason=None, source=None):
-        self.reason = reason
-        self.source = source
+    def __init__(self, reason=None, owner=None, branch=None):
+        self.properties = {'owner': owner, 'branch': branch, 'reason': reason}
 
-    def getReason(self):
-        return self.reason
+    def getProperty(self, name):
+        return self.properties.get(name, None)
 
-    def getSourceStamp(self):
-        return self.source
 
-class FakeSource(object):
-
-    def __init__(self, branch):
-        self.branch = branch
-
-def test_extract_username():
-    a = FakeBuild("The web-page 'force build' button was pressed by 'antocuni': foo")
+def test_get_build_information():
+    a = FakeBuild(owner='antocuni',
+            reason="The web-page 'force build' button was pressed")
     b = FakeBuild("The web-page 'force build' button was ...")
-    assert ircbot.extract_username(a) == 'antocuni'
-    assert ircbot.extract_username(b) is None
+    assert ircbot.get_build_information(a) == \
+            "antocuni: The web-page 'force build' button was pressed"
+    assert ircbot.get_build_information(b) == \
+            "The web-page 'force build' button was ..."
 
 
 def test_get_description_for_build():
-    a = FakeBuild('foobar', source=FakeSource(None))
+    a = FakeBuild()
     msg = ircbot.get_description_for_build("http://myurl", a)
     assert msg == "http://myurl"
 
-    a = FakeBuild("The web-page 'force build' button was pressed by 'antocuni': foo",
-                  source=FakeSource(None))
+    a = FakeBuild(owner='antocuni',
+            reason="The web-page 'force build' button was pressed")
     msg = ircbot.get_description_for_build("http://myurl", a)
-    assert msg == "http://myurl [antocuni]"
+    assert msg == "http://myurl [antocuni: " \
+                  + "The web-page 'force build' button was pressed]"
 
-    a = FakeBuild('foobar', source=FakeSource('mybranch'))
+    a = FakeBuild(branch='mybranch')
     msg = ircbot.get_description_for_build("http://myurl", a)
     assert msg == "http://myurl [mybranch]"
 
-    a = FakeBuild("The web-page 'force build' button was pressed by 'antocuni': foo",
-                  source=FakeSource('mybranch'))
+    a = FakeBuild(owner='antocuni', branch='mybranch')
     msg = ircbot.get_description_for_build("http://myurl", a)
     assert msg == "http://myurl [antocuni, mybranch]"

bot2/pypybuildbot/test/test_pypylist.py

         newdir.setmtime(oldtime + ascii * 10)
     pypylist = PyPyList(tmpdir.strpath)
     listener = pypylist.directoryListing()
-    assert listener.dirs == ['trunk', 'mmmm', 'llll',
+    assert listener.dirs == ['trunk', 'llll',
         'kkkk','jjjj','iiii','hhhh','gggg','ffff','eeee',
         'dddd','cccc','bbbb','aaaa']
 
 def load_BuildmasterConfig():
     import os
-    from pypybuildbot import summary, builds
+    from pypybuildbot import summary, builds, arm_master
     def load(name):
         if name == 'pypybuildbot.summary':
             return summary
         elif name == 'pypybuildbot.builds':
             return builds
+        elif name == 'pypybuildbot.arm_master':
+            return arm_master
         else:
             assert False
 

bot2/pypybuildbot/test/test_summary.py

 s a/b.py:test_three
 S a/c.py:test_four
 """)
-        
+
         rev_outcome_set.populate(log)
 
         assert rev_outcome_set.skipped == set([("a.b","test_three"),
 x a/c.py:test_nine
 x a/c.py:test_ten
 """)
-        
+
         rev_outcome_set.populate(log)
         sum = rev_outcome_set.get_summary()
         assert sum.p == 1
         rev_outcome_set = summary.RevisionOutcomeSet('0')
         log = StringIO("")
         rev_outcome_set.populate(log)
-        
+
     def test_populate_longrepr(self):
         rev_outcome_set = summary.RevisionOutcomeSet('50000')
         log = StringIO("""F a/b.py:test_one
 s a/b.py:test_three
  some skip
 """)
-        
+
         rev_outcome_set.populate(log)
 
         assert len(rev_outcome_set.skipped) == 1
 F a/b.py:test_two
  \xc3\xa5 bar
 """)
-        
+
         rev_outcome_set.populate(log)
 
         assert len(rev_outcome_set.failed) == 2
 ! <run>
 ! /a/b/c.py:92
 """)
-        
+
         rev_outcome_set.populate(log)
 
         assert rev_outcome_set.failed == set([
         log = StringIO("""x a/b.py
  EXC
 """)
-        
+
         rev_outcome_set.populate(log)
 
         assert rev_outcome_set.numxfailed == 1
-  
-        
+
+
     def test_absent_outcome(self):
         rev_outcome_set = summary.RevisionOutcomeSet('50000')
 
         def load(x, y):
             calls.append(y)
             return y
-        
+
         cache._load_outcome_set = load
 
         res = cache.get('status', 'a')
         cache.get('status', 'b')
         res = cache.get('status', 'c')
         assert res == 'c'
-        
+
         assert calls == ['a', 'b', 'c']
 
         calls = []
         res = cache.get('status', 'd')
         assert res == 'd'
         assert cache.get('status', 'c') == 'c'
-        assert cache.get('status', 'b') == 'b'        
+        assert cache.get('status', 'b') == 'b'
         assert calls == ['d']
 
         res = cache.get('status', 'a')
 s a/b.py:test_three
 x a/b.py:test_four
 """)
-        
+
         rev_outcome_set_foo.populate(log)
 
 
-        key_bar = ('bar', 7)        
+        key_bar = ('bar', 7)
         rev_outcome_set_bar = summary.RevisionOutcomeSet('50000',
                                                          key_bar)
         log = StringIO(""". a/b.py:test_one
 . a/b.py:test_two
 s a/b.py:test_three
 """)
-        
+
         rev_outcome_set_bar.populate(log)
 
         d = {'foo': rev_outcome_set_foo,
         goutcome = summary.GatherOutcomeSet(d)
 
         assert goutcome.revision == '50000'
-        
+
         assert goutcome.failed == set([('foo', 'a.b', 'test_one')])
 
         assert goutcome.skipped == set([('foo', 'a.b', 'test_three'),
         assert res == ' '
 
         res = goutcome_top.get_longrepr(('what', 'foo', 'a.b', 'test_one'))
-        assert res == ''        
+        assert res == ''
 
 def test_colsizes():
     failed = [('a', 'abc', 'd'), ('ab', 'c', 'xy'),
               ('ab', '', 'cd')]
-    
+
     res = summary.colsizes(failed)
-    
+
     assert res == [2,3,2]
 
 def test__prune_runs():
     res = summary.show_elapsed(0.25)
     assert res == "0.25s"
     res = summary.show_elapsed(1.0)
-    assert res == "1.00s"           
+    assert res == "1.00s"
     res = summary.show_elapsed(1.25)
-    assert res == "1.25s"   
+    assert res == "1.25s"
     res = summary.show_elapsed(4.5)
     assert res == "4.50s"
     res = summary.show_elapsed(5.25)
     assert res == "5s"
     res = summary.show_elapsed(5.5)
-    assert res == "6s"            
+    assert res == "6s"
     res = summary.show_elapsed(2*60+30)
     assert res == "2m30"
     res = summary.show_elapsed(4*60+30)
     res = summary.show_elapsed(61*60)
     assert res == "1h1"
     res = summary.show_elapsed(90*60)
-    assert res == "1h30"                
+    assert res == "1h30"
 
-def _BuilderToStatus(status):
 
-    setup = {'name': 'builder', 'builddir': 'BUILDDIR',
-             'slavebuilddir': 'SLAVEBUILDDIR',
-             'factory': process_factory.BuildFactory() }
-    return process_builder.Builder(setup, status)
+class FakeMasterConfig(object):
+    buildbotURL = "http://buildbot/"
+    logCompressionLimit = 0
+    def __init__(self, builders=None):
+        self.builders = builders
+
+
+class FakeBuilderconfig(object):
+    validNames = 'name factory slavenames builddir slavebuilddir category ' \
+                 'nextSlave nextBuild canStartBuild locks env properties ' \
+                 'mergeRequests description'.split()
+
+    def __init__(self, **kwargs):
+        for kw, item in kwargs.iteritems():
+            assert kw in self.validNames
+            setattr(self, kw, item)
 
 
 class FakeMaster(object):
     basedir = None
-    buildbotURL = "http://buildbot/"
 
     def __init__(self, builders):
         self.botmaster = FakeBotMaster(builders)
+        self.config = FakeMasterConfig()
 
     def subscribeToBuildsetCompletions(self, callback):
         pass
     def subscribeToBuildRequests(self, callback):
         pass
 
+
 class FakeBotMaster(object):
 
     def __init__(self, builders):
             self.builderNames.append(name)
             self.builders[name] = _BuilderToStatus(builder)
 
+
 class FakeSite(object):
 
     def __init__(self, status):
         self.buildbot_service = FakeService(status)
 
+
 class FakeService(object):
-    
+
     def __init__(self, status):
         self.status = status
 
     def getStatus(self):
         return self.status
 
+
 class FakeRequest(object):
 
     def __init__(self, builders, args={}):
         self.site = FakeSite(status)
 
 
+def _BuilderToStatus(status):
+    builder = process_builder.Builder(status.name)
+    builder.builder_status = status
+    builder.builder_status.basedir = 'BASEDIR'
+    builder.config = FakeBuilderconfig(factory=process_factory.BuildFactory())
+    return builder
+
+
 def witness_cat_branch(summary):
     ref = [None]
     recentRuns = summary.recentRuns
         ref[0] = cat_branch
         return cat_branch
     summary.recentRuns = witness
-
     return lambda: ref[0]
 
 class FakeLog(object):
         self.step = step
         self.name = name
         self.cont = cont
-        
+
     def getStep(self):
         return self.step
 
     n = getattr(builder, 'nextBuildNumber', 0)
     t = 1000
     for rev, reslog in builds:
-        build = status_builder.BuildStatus(builder, n)
+        build = status_builder.BuildStatus(builder, builder.master, n)
         build.started = time.time()
         build.setProperty('got_revision', str(rev), None)
         step = build.addStepWithName('pytest')
         step.started = t
         step.finished = t + (n+1)*60
         t = step.finished + 30
+        builder.buildCache.cache[build.number] = build
+        builder.buildStarted(build)
         build.buildFinished()
-        builder.touchBuildCache(build)
         n += 1
     builder.nextBuildNumber = n
-        
+
+
+METABRANCH = '<trunk>'
+
 
 class TestSummary(object):
 
     def setup_method(self, meth):
         summary.outcome_set_cache.clear()
+        self.master = FakeMaster([])
 
     def test_sanity(self):
         s = summary.Summary()
         assert cat_branch == {}
 
     def test_one_build_no_rev(self):
-        builder = status_builder.BuilderStatus('builder0')
-        build = status_builder.BuildStatus(builder, 0)
-        build.started = time.time()
+        builder = status_builder.BuilderStatus('builder0', None, self.master, '')
+        build = status_builder.BuildStatus(builder, self.master, 0)
+        build.buildStarted(builder)
         build.buildFinished()
-        builder.touchBuildCache(build)
-        builder.nextBuildNumber = len(builder.buildCache)
+        builder.nextBuildNumber = len(builder.buildCache.cache)
 
         s = summary.Summary()
-        res = witness_cat_branch(s)        
+        res = witness_cat_branch(s)
         req = FakeRequest([builder])
         out = s.body(req)
         cat_branch = res()
 
-        assert cat_branch == {(None, None): ({}, [build])}
+        assert cat_branch == {(None, METABRANCH): ({}, [build])}
 
     def test_one_build_no_logs(self):
-        builder = status_builder.BuilderStatus('builder0')
-        build = status_builder.BuildStatus(builder, 0)
-        build.started = time.time()        
+        builder = status_builder.BuilderStatus('builder0', None, self.master, '')
+        build = status_builder.BuildStatus(builder, self.master, 0)
+        build.started = time.time()
         build.setProperty('got_revision', '50000', None)
         build.buildFinished()
-        builder.touchBuildCache(build)
-        builder.nextBuildNumber = len(builder.buildCache)
+        builder.buildCache.cache[build.number] = build
+        builder.nextBuildNumber = len(builder.buildCache.cache)
 
         s = summary.Summary()
-        res = witness_cat_branch(s)        
+        res = witness_cat_branch(s)
         req = FakeRequest([builder])
         out = s.body(req)
         cat_branch = res()
-        
-        revs = cat_branch[(None, None)][0]
+
+        revs = cat_branch[(None, METABRANCH)][0]
         assert revs.keys() == ['50000']
 
-        assert '&lt;run&gt;' in out
+        assert 'success' in out
 
     def test_one_build_no_logs_failure(self):
-        builder = status_builder.BuilderStatus('builder0')
-        build = status_builder.BuildStatus(builder, 0)
-        build.started = time.time()        
+        builder = status_builder.BuilderStatus('builder0', None, self.master, '')
+        build = status_builder.BuildStatus(builder, self.master, 0)
+        build.started = time.time()
         build.setProperty('got_revision', '50000', None)
         step = build.addStepWithName('step')
         step.setText(['step', 'borken'])
         step.stepFinished(summary.FAILURE)
         step1 = build.addStepWithName('other')
         step1.setText(['other', 'borken'])
-        step1.stepFinished(summary.FAILURE)        
+        step1.stepFinished(summary.FAILURE)
         build.buildFinished()
-        builder.touchBuildCache(build)
-        builder.nextBuildNumber = len(builder.buildCache)
+        builder.buildCache.cache[build.number] = build
+        builder.nextBuildNumber = len(builder.buildCache.cache)
 
         s = summary.Summary()
-        res = witness_cat_branch(s)        
-        req = FakeRequest([builder])
-        out = s.body(req)
-        cat_branch = res()
-        
-        revs = cat_branch[(None, None)][0]
-        assert revs.keys() == ['50000']
-
-        assert 'step borken' in out
-        assert 'other borken' not in out        
-        
-    def test_one_build(self):
-        builder = status_builder.BuilderStatus('builder0')
-        add_builds(builder, [(60000, "F TEST1\n. b")])
-
-        s = summary.Summary()
-        res = witness_cat_branch(s)        
+        res = witness_cat_branch(s)
         req = FakeRequest([builder])
         out = s.body(req)
         cat_branch = res()
 
-        revs = cat_branch[(None, None)][0]
+        revs = cat_branch[(None, METABRANCH)][0]
+        assert revs.keys() == ['50000']
+
+        assert 'step borken' in out
+        assert 'other borken' not in out
+
+    def test_one_build(self):
+        builder = status_builder.BuilderStatus('builder0', None, self.master, '')
+        add_builds(builder, [(60000, "F TEST1\n. b")])
+
+        s = summary.Summary()
+        res = witness_cat_branch(s)
+        req = FakeRequest([builder])
+        out = s.body(req)
+        cat_branch = res()
+
+        revs = cat_branch[(None, METABRANCH)][0]
         assert revs.keys() == ['60000']
         outcome = revs['60000']['builder0']
         assert outcome.revision == '60000'
         assert 'TEST1' in out
 
     def test_two_builds(self):
-        builder = status_builder.BuilderStatus('builder0')
+        builder = status_builder.BuilderStatus('builder0', None, self.master, '')
         add_builds(builder, [('60000', "F TEST1\n. b"),
                              ('60001', ". TEST1\n. b")])
 
         s = summary.Summary()
-        res = witness_cat_branch(s)        
+        res = witness_cat_branch(s)
         req = FakeRequest([builder])
         out = s.body(req)
         cat_branch = res()
 
-        revs = cat_branch[(None, None)][0]
+        revs = cat_branch[(None, METABRANCH)][0]
         assert sorted(revs.keys()) == ['60000', '60001']
         outcome = revs['60000']['builder0']
         assert outcome.revision == '60000'
 
         assert 'TEST1' in out
         assert ':-)' in out
-        assert '\n <a class="failSummary failed" href="javascript:togglestate(1,1)" id="a1c1">-</a> <span class="failSummary success">+</span>  success' in out
-
+        assert re.search(r'\n <a class="failSummary failed" href="javascript:'
+                         r'togglestate\((\d+),(\d+)\)" id="a\1c\2">-</a> '
+                         r'<span class="failSummary success">\+</span>  success', out) is not None
     def test_two_builds_samerev(self):
-        builder = status_builder.BuilderStatus('builder0')
+        builder = status_builder.BuilderStatus('builder0', None, self.master, '')
         add_builds(builder, [('60000', "F TEST1\n. b"),
-                             ('60000', "F TEST1\n. b")])        
+                             ('60000', "F TEST1\n. b")])
 
         s = summary.Summary()
-        res = witness_cat_branch(s)        
+        res = witness_cat_branch(s)
         req = FakeRequest([builder])
         out = s.body(req)
         cat_branch = res()
 
-        revs = cat_branch[(None, None)][0]
+        revs = cat_branch[(None, METABRANCH)][0]
         assert sorted(revs.keys()) == ['60000']
         outcome = revs['60000']['builder0']
         assert outcome.revision == '60000'
         assert 'TEST1' in out
 
     def test_two_builds_recentrev(self):
-        builder = status_builder.BuilderStatus('builder0')
+        builder = status_builder.BuilderStatus('builder0', None, self.master, '')
         add_builds(builder, [('60000', "F TEST1\n. b"),
                              ('60001', "F TEST1\n. b")])
 
         s = summary.Summary()
-        res = witness_cat_branch(s)        
+        res = witness_cat_branch(s)
         req = FakeRequest([builder])
         req.args = {'recentrev': ['60000']}
         out = s.body(req)
         cat_branch = res()
 
-        revs = cat_branch[(None, None)][0]
+        revs = cat_branch[(None, METABRANCH)][0]
         assert sorted(revs.keys()) == ['60000']
         outcome = revs['60000']['builder0']
         assert outcome.revision == '60000'
         assert 'TEST1' in out
 
     def test_many_builds_query_builder(self):
-        builder = status_builder.BuilderStatus('builder0')
+        builder = status_builder.BuilderStatus('builder0', None, self.master, '')
         add_builds(builder, [('60000', "F TEST1\n. b"),
                              ('60000', ". a\n. b"),
-                             ('60001', "F TEST1\n. b")])        
+                             ('60001', "F TEST1\n. b")])
 
         s = summary.Summary()
-        res = witness_cat_branch(s)        
+        res = witness_cat_branch(s)
         req = FakeRequest([builder])
         req.args={'builder': ['builder0']}
         out = s.body(req)
         cat_branch = res()
 
-        runs = cat_branch[(None, None)][0]
+        runs = cat_branch[(None, METABRANCH)][0]
         assert sorted(runs.keys()) == [(0, '60000'), (1, '60000'), (2, '60001')]
         outcome = runs[(0, '60000')]['builder0']
         assert outcome.revision == '60000'
 
 
     def test_many_builds_query_builder_builds(self):
-        builder = status_builder.BuilderStatus('builder0')
+        builder = status_builder.BuilderStatus('builder0', None, self.master, '')
         add_builds(builder, [('60000', "F TEST1\n. b"),
                              ('60000', ". a\n. b"),
-                             ('60001', "F TEST1\n. b")])        
+                             ('60001', "F TEST1\n. b")])
 
         s = summary.Summary()
-        res = witness_cat_branch(s)        
+        res = witness_cat_branch(s)
         req = FakeRequest([builder])
         req.args={'builder': ['builder0'],
                   'builds': ['0','2-2', '7']}
         out = s.body(req)
         cat_branch = res()
 
-        runs = cat_branch[(None, None)][0]
+        runs = cat_branch[(None, METABRANCH)][0]
         assert sorted(runs.keys()) == [(0, '60000'), (2, '60001')]
         outcome = runs[(0, '60000')]['builder0']
         assert outcome.revision == '60000'
         assert 'TEST1' in out
 
     def test_many_pytestLogs(self):
-        builder = status_builder.BuilderStatus('builder1')
-        build = status_builder.BuildStatus(builder, 0)
+        builder = status_builder.BuilderStatus('builder1', '', self.master, '')
+        build = status_builder.BuildStatus(builder, self.master, 0)
         build.started = time.time()
         build.setProperty('got_revision', '70000', None)
         step = build.addStepWithName('pytest')
         step.logs.extend([FakeLog(step, 'pytestLog', "F TEST1")])
         step.setText(["pytest", "failed"])
-        step.stepFinished(summary.FAILURE)        
+        step.stepFinished(summary.FAILURE)
         step2 = build.addStepWithName('pytest2')
         step2.logs.extend([FakeLog(step, 'pytestLog', ". x\nF TEST2")])
         step2.setText(["pytest2", "aborted"])
         step2.stepFinished(summary.EXCEPTION)
         build.buildFinished()
-        builder.touchBuildCache(build)
-        builder.nextBuildNumber = 1
+        builder.buildCache.cache[build.number] = build
+        builder.nextBuildNumber = len(builder.buildCache.cache)
 
         s = summary.Summary()
         req = FakeRequest([builder])
         assert 'pytest2 aborted' in out
 
     def test_subtle_failures(self):
-        builder = status_builder.BuilderStatus('builder1')
-        build = status_builder.BuildStatus(builder, 0)
+        builder = status_builder.BuilderStatus('builder1', '', self.master, '')
+        build = status_builder.BuildStatus(builder, self.master, 0)
         build.started = time.time()
         build.setProperty('got_revision', '70000', None)
-        step = build.addStepWithName('pytest')        
+        step = build.addStepWithName('pytest')
         step.logs.extend([FakeLog(step, 'pytestLog', ". TEST1")])
         step.setText(["pytest", "failed slave lost"])
-        step.stepFinished(summary.FAILURE)        
+        step.stepFinished(summary.FAILURE)
         build.buildFinished()
-        builder.touchBuildCache(build)
-        builder.nextBuildNumber = 1
+        builder.buildCache.cache[build.number] = build
+        builder.nextBuildNumber = len(builder.buildCache.cache)
 
         s = summary.Summary()
         req = FakeRequest([builder])
         out = s.body(req)
 
-        assert 'pytest failed slave lost' in out        
+        assert 'pytest failed slave lost' in out
 
 
     def test_category_branch_sorting_key(self):
         assert res == (2, '', 2, 'release/1')
 
         res = s._cat_branch_key(('', 'what'))
-        assert res == (2, '', 4, 'what')                
+        assert res == (2, '', 4, 'what')
 
     def test_builders_with_categories(self):
-        builder1 = status_builder.BuilderStatus('builder_foo')
-        builder1.category = 'foo'
-        builder2 = status_builder.BuilderStatus('builder_bar')
-        builder2.category = 'bar'
-        builder3 = status_builder.BuilderStatus('builder_')
-        builder3.category = ''
+        builder1 = status_builder.BuilderStatus('builder_foo', 'foo', self.master, '')
+        builder2 = status_builder.BuilderStatus('builder_bar', 'bar', self.master, '')
+        builder3 = status_builder.BuilderStatus('builder_', '', self.master, '')
 
         add_builds(builder1, [('60000', "F TEST1\n")])
         add_builds(builder2, [('60000', "F TEST2\n")])
-        add_builds(builder3, [('60000', "F TEST3\n")])            
+        add_builds(builder3, [('60000', "F TEST3\n")])
 
         s = summary.Summary(['foo', 'bar'])
         req = FakeRequest([builder1, builder2, builder3])
         assert "{bar}" in out
 
     def test_two_builds_different_rev_digits(self):
-        builder = status_builder.BuilderStatus('builder0')
+        builder = status_builder.BuilderStatus('builder0', '', self.master, '')
         add_builds(builder, [(999, "F TEST1\n. b"),
                              (1000, "F TEST1\n. b")])
 
         assert p999builder0-p999 == p1000builder0-p1000+1
 
     def test_build_times_and_filtering(self):
-        builder1 = status_builder.BuilderStatus('builder1')
-        builder2 = status_builder.BuilderStatus('builder2')
- 
+        builder1 = status_builder.BuilderStatus('builder1', '', self.master, '')
+        builder2 = status_builder.BuilderStatus('builder2', '', self.master, '')
+
         add_builds(builder1, [('60000', "F TEST1\n")])
-        add_builds(builder2, [('50000', ". TEST2\n")])        
+        add_builds(builder2, [('50000', ". TEST2\n")])
         add_builds(builder2, [('60000', "F TEST2\n")])
 
         builder1.getBuild(0).started  = 1228258800 # 3 Dec 2008
         builder1.getBuild(0).finished = 1228258800 # 3 Dec 2008
-        builder2.getBuild(1).started  = 1228431600 # 5 Dec 2008        
+        builder2.getBuild(1).started  = 1228431600 # 5 Dec 2008
         builder2.getBuild(1).finished = 1228431600 # 5 Dec 2008
 
         builder2.getBuild(0).started  = 1227913200 # 29 Nov 2008

bot2/pypybuildbot/util.py

 import socket
 
 def we_are_debugging():
-    return socket.gethostname() not in ("wyvern", "cobra")
+    return socket.gethostname() != 'cobra'
 
 def load(name):
     mod = __import__(name, {}, {}, ['__all__'])

master/public_html/default.css

 	color: #333;
 }
 
+.auth {
+position:absolute;
+top:5px;
+right:40px;
+}
+
+.alert {
+  color: #c30000;
+  background-color: #f2dcdc;
+  padding: 5px 5px 5px 25px;
+  margin-bottom: 20px;
+  border-top:1px solid #ccc;
+  border-bottom:1px solid #ccc;
+  border-color: #c30000;
+  font-size: 20px;
+}
 a:link,a:visited,a:active {
 	color: #444;
 }
 	font-weight: normal;
 	padding: 8px 8px 8px 8px;
 	color: #333333;
+	background-color: #eee;
+	text-align: left;
+}
+
+td.DevBottom {
 	border-bottom-right-radius: 5px;
 	-webkit-border-bottom-right-radius: 5px;
 	-moz-border-radius-bottomright: 5px;
 	border-bottom-left-radius: 5px;
 	-webkit-border-bottom-left-radius: 5px;
 	-moz-border-radius-bottomleft: 5px;
-	background-color: #eee;
-	text-align: left;
 }
 
 td.Alt {
 }
 
 .legend {
-	border-radius: 5px;
-	-webkit-border-radius: 5px;
-	-moz-border-radius: 5px;
+	border-radius: 5px !important;
+	-webkit-border-radius: 5px !important;
+	-moz-border-radius: 5px !important;
 	width: 100px;
 	max-width: 100px;
 	text-align: center;
 	border-color: #A77272;
 }
 
+.failure-again {
+	color: #000;
+	background-color: #eA9;
+	border-color: #A77272;
+}
+
 .warnings {
 	color: #FFFFFF;
 	background-color: #fa3;
 	border-color: #C5C56D;
 }
 
+.paused {
+    color: #FFFFFF;
+    background-color: #8080FF;
+    border-color: #dddddd;
+}
+
 .offline,td.offline {
     color: #FFFFFF;
     background-color: #777777;
 	display: none;
 }
 
+pre {
+	white-space: pre-wrap;
+}
+
 /* change comments (use regular colors here) */
 pre.comments>a:link,pre.comments>a:visited {
 	color: blue;
 pre.comments>a:active {
 	color: purple;
 }
+
+form.command_forcebuild {
+    border-top: 1px solid black;
+    padding: .5em;
+    margin: .5em;
+}
+
+form.command_forcebuild > .row {
+    border-top: 1px dotted gray;
+    padding: .5em 0;
+}
+
+form.command_forcebuild .force-textarea > .label {
+    display: block;
+}
+
+form.command_forcebuild .force-nested > .label {
+    font-weight: bold;
+    display: list-item;
+}
+
+form.command_forcebuild .force-any .force-text {
+    display: inline;
+}

master/public_html/favicon.ico

Added
New image

master/public_html/index.html

 
 <ul>
   <li>the <a href="http://speed.pypy.org/">Performance Plots</a> will give you an overview of performance for recent revisions.</li>
-  <li>the <a href="summary?branch=&lt;trunk&gt;">Summary Display &lt;trunk&gt;</a> will give you a 
<