Commits

Carsten Senger committed 0c6207d

Refactoring, support script params executeable, project, url

* Refactoring: move runner upload code from run_and_store and
perform_upload to main.
* factor out code to guess the executable
* Remove hardcoded values for project, executable and codespeed url
and add them as parameters to the runner script (--update-executable,
--update-project and --upload-urls) and the saveresults script
(--executable, --project and --url)
* adjust parameters of saveresults.save and .send
* remove unused parameter options from save()
* adjust tests

  • Participants
  • Parent commits 69cf045

Comments (0)

Files changed (4)

 from saveresults import save
 from unladen_swallow import perf
 
+BENCHMARK_SET = ['richards', 'slowspitfire', 'django', 'spambayes',
+                 'rietveld', 'html5lib', 'ai']
+BENCHMARK_SET += perf._FindAllBenchmarks(benchmarks.__dict__).keys()
 
-def perform_upload(pypy_c_path, args, force_host, options, res, revision,
-                   changed=True, postfix='', branch='default', project='PyPy',
-                   executable=None):
-    if executable is None:
-        if "--jit" in args:
-            executable = "pypy-c" + postfix
-        else:
-            executable = "pypy-c-jit" + postfix
+# executeablenames
+EX_CPY_PSYCO = "cpython psyco-profile"
+EX_PYPY_C = 'pypy-c'
+EX_PYPY_C_JIT = 'pypy-c-jit'
+
+
+class WrongBenchmark(Exception):
+    pass
+
+
+def guess_executeable(pypy_c_path, postfix, args, changed):
+
+    if ',' in args:
+        args_baseline, args_changed = args.split(',')
+        args = args_changed if changed else args_baseline
 
     if "psyco.sh" in pypy_c_path:
-        executable = "cpython psyco-profile"
-        revision = 100
-        project = 'cpython'
+        executable_prefix = EX_CPY_PSYCO
+    elif '--jit' in args:
+        executable_prefix = EX_PYPY_C
+    else:
+        executable_prefix = EX_PYPY_C_JIT
 
-    host = force_host if force_host else socket.gethostname()
-    print save(project, revision, res, options, executable, host,
-               changed=changed, branch=branch)
+    return executable_prefix + postfix
 
 
 def run_and_store(benchmark_set, result_filename, pypy_c_path, revision=0,
                   options='', branch='default', args='', upload=False,
-                  force_host=None, fast=False, baseline=sys.executable,
-                  full_store=False, postfix=''):
+                  fast=False, baseline=sys.executable, full_store=False):
     funcs = perf.BENCH_FUNCS.copy()
     funcs.update(perf._FindAllBenchmarks(benchmarks.__dict__))
-    opts = ['-b', ','.join(benchmark_set), '--inherit_env=PATH',
+    opts = ['-b', ','.join(benchmark_set),
+            '--inherit_env=PATH',
             '--no_charts']
     if fast:
         opts += ['--fast']
     if args:
         opts += ['--args', args]
     if full_store:
-        opts.append('--no_statistics')
+        opts += ['--no_statistics']
     opts += [baseline, pypy_c_path]
     results = perf.main(opts, funcs)
     f = open(str(result_filename), "w")
-    res = [(name, result.__class__.__name__, result.__dict__)
+    results = [(name, result.__class__.__name__, result.__dict__)
            for name, result in results]
     f.write(json.dumps({
         'revision': revision,
-        'results': res,
+        'results': results,
         'options': options,
         'branch': branch,
         }))
     f.close()
-
-    if upload:
-        if ',' in args:
-            argsbase, argschanged = args.split(',')
-        else:
-            argsbase, argschanged = args, args
-        if 'pypy' in baseline:
-            perform_upload(pypy_c_path, argsbase, force_host, options, res,
-                           revision, changed=False, postfix=postfix,
-                           branch=branch)
-        perform_upload(pypy_c_path, argschanged, force_host, options, res,
-                       revision, changed=True, postfix=postfix, branch=branch)
-
-
-BENCHMARK_SET = ['richards', 'slowspitfire', 'django', 'spambayes',
-                 'rietveld', 'html5lib', 'ai']
-BENCHMARK_SET += perf._FindAllBenchmarks(benchmarks.__dict__).keys()
-
-
-class WrongBenchmark(Exception):
-    pass
+    return results
 
 
 def main(argv):
                             " python, and the arguments after are passed to"
                             " the changed python. If there's no comma, the"
                             " same options are passed to both."))
-    parser.add_option("--upload", default=False, action="store_true",
-                      help="Upload results to speed.pypy.org")
+    parser.add_option("--upload", default=None, action="store_true",
+                      help=("Upload results to speed.pypy.org (unless "
+                            "--upload-url is given)."))
+    parser.add_option("--upload-urls", default="http://speed.pypy.org/",
+                      help=("Comma seperated urls of the codespeed instance to"
+                            " upload to (default: http://speed.pypy.org/)."))
+    parser.add_option("--upload-project", default="PyPy",
+                      help="The project name in codespeed (default: PyPy).")
+    parser.add_option("--upload-executable", default=None,
+                      help=("The executable name in codespeed "
+                            "(guessed if possible and not given)."))
     parser.add_option("--force-host", default=None, action="store",
                       help="Force the hostname")
     parser.add_option("--fast", default=False, action="store_true",
     for benchmark in benchmarks:
         if benchmark not in BENCHMARK_SET:
             raise WrongBenchmark(benchmark)
-    run_and_store(benchmarks, options.output_filename, options.pypy_c,
-                  options.revision, args=options.args, upload=options.upload,
-                  force_host=options.force_host, fast=options.fast,
-                  baseline=options.baseline, full_store=options.full_store,
-                  postfix=options.postfix, branch=options.branch)
+
+    pypy_c_path = options.pypy_c
+    baseline = options.baseline
+    fast = options.fast
+    args = options.args
+    full_store = options.full_store
+    output_filename = options.output_filename
+
+    project = options.upload_project
+    executable = options.upload_executable
+    postfix = options.postfix
+    branch = options.branch
+    revision = options.revision
+
+    force_host = options.force_host
+
+    results = run_and_store(benchmarks, output_filename, pypy_c_path, revision,
+                            args=args, fast=fast, baseline=baseline,
+                            full_store=full_store, branch=branch)
+
+    if options.upload:
+        changed = 'pypy' not in baseline
+        if executable is None:
+            executable = guess_executeable(pypy_c_path, postfix, args, changed)
+
+        if executable == EX_CPY_PSYCO:
+            revision = 100
+            project = 'cpython'
+
+        changed = False
+
+        host = force_host if force_host else socket.gethostname()
+        for url in options.upload_urls.split(','):
+            print save(project, revision, results, executable, host, url,
+                       changed=changed, branch=branch)
+
 
 if __name__ == '__main__':
     main(sys.argv[1:])

File saveresults.py

 import urllib2
 
 
-SPEEDURL = "http://speed.pypy.org/"
-
-
-def save(project, revision, results, options, executeable, host, testing=False,
+def save(project, revision, results, executeable, host, url, testing=False,
          changed=True, branch='default'):
     testparams = []
     #Parse data
         if testing:
             testparams.append(data)
         else:
-            error |= send(data)
+            error |= send(data, url)
 
     if error:
         raise IOError("Saving failed.  See messages above.")
         return 0
 
 
-def send(data):
+def send(data, url):
     #save results
     params = urllib.urlencode(data)
     f = None
         retries = [1, 2, 3, 6]
         while True:
             try:
-                f = urllib2.urlopen(SPEEDURL + 'result/add/', params)
+                f = urllib2.urlopen(url + 'result/add/', params)
                 response = f.read()
                 f.close()
                 break
         print response
         with open('error.html', 'w') as error_file:
             error_file.write(response)
-        print("Server (%s) response written to error.html" % (SPEEDURL,))
+        print("Server (%s) response written to error.html" % (url,))
         print('  Error code: %s\n' % (e,))
         return 1
     print "saved correctly!\n"
         data = simplejson.load(f)
     results = data['results']
     print 'uploading results...',
-    save('PyPy', options.revision, results, '', options.name, options.host,
-         changed=options.changed)
+    save(options.project, options.revision, results, options.executable,
+                options.host, options.url, changed=options.changed)
     print 'done'
 
 
 if __name__ == '__main__':
     parser = optparse.OptionParser(usage="%prog result.json [options]")
     parser.add_option('-r', '--revision', dest='revision',
-                      default=None, type=str)
-    parser.add_option('-n', '--name', dest='name', default=None, type=str)
+                      default=None, type=str, help='VCS revision (required)')
+    parser.add_option('-n', '--name', dest='executable',
+                      default=None, type=str,
+                      help=('Name of the executable for codespeed.'
+                            'Deprecated. Use --e/--executable instead'))
+    parser.add_option('-e', '--executable', dest='executable',
+                      default=None, type=str,
+                      help='Name of the Executable for codespeed (required).')
     parser.add_option('-H', '--host', dest='host', default=None, type=str)
     parser.add_option('-b', '--baseline', dest='changed', default=True,
                       action='store_false',
                       help='upload the results as baseline instead of changed')
+    parser.add_option('-P', '--project', dest='project', default='PyPy')
+    parser.add_option('-u', '--url', dest='url',
+                      default="http://speed.pypy.org/",
+                      help=('Url of the codespeed instance '
+                            '(default: http://speed.pypy.org)'))
     parser.format_description = lambda fmt: __doc__
     parser.description = __doc__
     options, args = parser.parse_args()
-    if (options.revision is None or options.name is None or
+    if (options.revision is None or options.executable is None or
         options.host is None or len(args) != 1):
         parser.print_help()
         sys.exit(2)

File test/test_runner.py

-
 import py
 import json
 import sys
 from runner import run_and_store
 
+
 def test_run_and_store():
     tmpdir = py.test.ensuretemp('bench_runner')
     resfile = tmpdir.join('results')

File test/test_saveresults.py

     
     def test_good_input(self):
         '''Given correct result data, check that every result being saved has the right parameters'''
-        for resultparams in saveresults.save("PyPy", 71212, self.fixture, "", "pypy-c-jit", 'host', True):
+        for resultparams in saveresults.save("PyPy", 71212, self.fixture, "pypy-c-jit", 'host', 'url',
+                                             testing=True):
             assert resultparams['project'] == "PyPy"
             assert resultparams['commitid'] == 71212
             assert resultparams['executable'] == "pypy-c-jit"