diff -r f7283190e896 perf.py --- a/perf.py Thu Nov 12 23:16:25 2015 -0600 +++ b/perf.py Tue Dec 01 12:29:42 2015 -0600 @@ -763,6 +763,8 @@ try: changed_data = benchmark_function(changed_python, options, *args, **kwargs) + if options.raw: + return FormatRawData(changed_data, options) base_data = benchmark_function(base_python, options, *args, **kwargs) except subprocess.CalledProcessError as e: @@ -1058,6 +1060,33 @@ std_changed, delta_std, significant, timeline_link) +class RawBenchmarkResult(dict): + + def __init__(self, average, min=None, max=None, std_dev=None): + self['average'] = average + if min is not None: + self['min'] = min + if max is not None: + self['max'] = max + if std_dev is not None: + self['std_dev'] = std_dev + + def __str__(self): + return '\n'.join('%s: %s' % i for i in sorted(self.items())) + + +def FormatRawData(bm_data, options): + # XXX implement track_memory handling? + times = sorted(bm_data.runtimes) + average = avg(times) + mn = mx = std = None + if len(times) > 1: + mn = times[0] + mx = times[-1] + std = SampleStdDev(times) + return RawBenchmarkResult(average, mn, mx, std) + + def CompareBenchmarkData(base_data, exp_data, options): """Compare performance and memory usage. @@ -1595,6 +1624,8 @@ try: changed_data = MeasureSpitfire(changed_python, options, spitfire_env, extra_args) + if options.raw: + return FormatRawData(changed_data, options) base_data = MeasureSpitfire(base_python, options, spitfire_env, extra_args) except subprocess.CalledProcessError as e: @@ -1999,6 +2030,8 @@ opts = [] changed_data = MeasureStartup(changed_python, opts, num_loops, options.track_memory, options.inherit_env) + if options.raw: + return FormatRawData(changed_data, options) base_data = MeasureStartup(base_python, opts, num_loops, options.track_memory, options.inherit_env) @@ -2016,6 +2049,8 @@ opts = ["-S"] changed_data = MeasureStartup(changed_python, opts, num_loops, options.track_memory, options.inherit_env) + if options.raw: + return FormatRawData(changed_data, options) base_data = MeasureStartup(base_python, opts, num_loops, options.track_memory, options.inherit_env) @@ -2299,6 +2334,8 @@ SLOW_BENCHMARKS = ["hexiom2"] +NON_RAW_BENCHMARKS = ["pybench"] + def _ExpandBenchmarkName(bm_name, bench_groups): """Recursively expand name benchmark names. @@ -2318,7 +2355,7 @@ yield bm_name -def ParseBenchmarksOption(benchmarks_opt, bench_groups, fast=False): +def ParseBenchmarksOption(benchmarks_opt, bench_groups, fast=False, raw=False): """Parses and verifies the --benchmarks option. Args: @@ -2360,6 +2397,12 @@ logging.info("Skipping slow benchmarks (%s) in fast mode", ', '.join(sorted(to_skip))) should_run = should_run - to_skip + if raw: + to_skip = should_run & set(NON_RAW_BENCHMARKS) + if to_skip: + logging.info("Skipping raw-incompatible benchmarks (%s) " + "in raw mode", ', '.join(sorted(to_skip))) + should_run = should_run - to_skip return should_run @@ -2510,12 +2553,20 @@ " Unladen Swallow binaries. This is useful for" " examining many benchmarks for optimization" " effects.")) + parser.add_option("--raw", action="store_true", + help="Run the benchmarks on only one interpreter and " + "return the timing information for each benchmark. " + "Provide only baseline_python, not changed_python.") options, args = parser.parse_args(argv) - if len(args) != 2: + expected = 1 if options.raw else 2 + if len(args) != expected: parser.error("incorrect number of arguments") - base, changed = args + if expected == 1: + base = changed = args[0] + else: + base, changed = args options.base_binary = base options.changed_binary = changed @@ -2525,6 +2576,16 @@ options.experiment_label = options.changed_binary base_args, changed_args = ParsePythonArgsOption(options.args) + if options.raw: + if base_args != changed_args: + parser.error('provide args for only one interpreter in raw mode') + if options.track_memory: + # XXX this might be worth fixing someday? + parser.error('raw mode is not compatible with memory tracking') + if options.diff_instrumentation: + parser.error('raw mode is not compatible with instrumentation') + if options.csv: + parser.error('raw mode does not support csv output') base_cmd_prefix = [base] + base_args changed_cmd_prefix = [changed] + changed_args @@ -2546,7 +2607,7 @@ info("Automatically selected timer: %s", options.timer) should_run = ParseBenchmarksOption(options.benchmarks, bench_groups, - options.fast) + options.fast, options.raw) should_run = FilterBenchmarks(should_run, bench_funcs, base_cmd_prefix, changed_cmd_prefix) @@ -2566,6 +2627,12 @@ print("Report on %s" % " ".join(platform.uname())) if multiprocessing: print("Total CPU cores:", multiprocessing.cpu_count()) + if options.raw: + for name, result in results: + print() + print("###", name, "###") + print(result) + return dict(results) hidden = [] if not options.verbose: shown = [] diff -r f7283190e896 run_and_upload.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/run_and_upload.py Tue Dec 01 12:29:42 2015 -0600 @@ -0,0 +1,80 @@ +from __future__ import print_function + +import argparse +import json +import os +import sys +try: + from urllib.request import urlopen + from urllib.parse import urlencode + from urllib.error import HTTPError +except ImportError: + from urllib2 import urlopen + from urllib import urlencode + from urllib2 import HTTPError + +from perf import main + +try: + from info import connection_info +except ImportError: + sys.exit('Could not import connection information, put it in info.py') + +p = argparse.ArgumentParser( + description='Run benchmarks wih perf.py and upload the results to the ' + 'specified server. Any argument not recognized by this ' + 'script is passed directly to perf.py.' +) +p.add_argument('--python', help='the interpreter to run benchmarks on') +p.add_argument('--revision', help='the revision from which executable was built') +p.add_argument('--executable', default='python', + help='1-3 word description of `python`') +p.add_argument('--environment', default='speed-python', + help='the environment (host) running the benchmark') +p.add_argument('--branch', default='default', + help='the branch from which executable was built') +p.add_argument('--project', default='CPython', + help='the project that produced executable') +ns, args = p.parse_known_args() + +if not any(a.startswith(('-b', '--benchmarks')) for a in args): + args.extend(['-b', 'all']) + +if '--raw' not in args: + args.append('--raw') + +args.append(os.path.abspath(ns.python)) + +results = main(args) + +data = [] +for name, result in results.items(): + if not isinstance(result, dict): + print('Skipping non-dict result for %s: %s' % (name, result)) + continue + data.append( + dict( + commitid=ns.revision, + branch=ns.branch, + project=ns.project, + executable=ns.executable, + environment=ns.environment, + benchmark=name, + result_value=result['average'], + min=result.get('min'), + max=result.get('max'), + std_dev=result.get('std_dev'), + ) + ) + +data = dict(json=json.dumps(data)) + +try: + response = urlopen(data=urlencode(data).encode('utf-8'), **connection_info) + print('Response: "%s"' % response.read().decode()) + response.close() +except HTTPError as e: + print(str(e)) + print(e.read().decode()) + e.close() + sys.exit(1)