284 lines
9.8 KiB
Python
284 lines
9.8 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
# -*- coding: utf-8 -*-
|
||
|
|
# Copyright 2016 The Chromium OS Authors. All rights reserved.
|
||
|
|
# Use of this source code is governed by a BSD-style license that can be
|
||
|
|
# found in the LICENSE file.
|
||
|
|
|
||
|
|
"""Given a specially-formatted JSON object, generates results report(s).
|
||
|
|
|
||
|
|
The JSON object should look like:
|
||
|
|
{"data": BenchmarkData, "platforms": BenchmarkPlatforms}
|
||
|
|
|
||
|
|
BenchmarkPlatforms is a [str], each of which names a platform the benchmark
|
||
|
|
was run on (e.g. peppy, shamu, ...). Note that the order of this list is
|
||
|
|
related with the order of items in BenchmarkData.
|
||
|
|
|
||
|
|
BenchmarkData is a {str: [PlatformData]}. The str is the name of the benchmark,
|
||
|
|
and a PlatformData is a set of data for a given platform. There must be one
|
||
|
|
PlatformData for each benchmark, for each element in BenchmarkPlatforms.
|
||
|
|
|
||
|
|
A PlatformData is a [{str: float}], where each str names a metric we recorded,
|
||
|
|
and the float is the value for that metric. Each element is considered to be
|
||
|
|
the metrics collected from an independent run of this benchmark. NOTE: Each
|
||
|
|
PlatformData is expected to have a "retval" key, with the return value of
|
||
|
|
the benchmark. If the benchmark is successful, said return value should be 0.
|
||
|
|
Otherwise, this will break some of our JSON functionality.
|
||
|
|
|
||
|
|
Putting it all together, a JSON object will end up looking like:
|
||
|
|
{ "platforms": ["peppy", "peppy-new-crosstool"],
|
||
|
|
"data": {
|
||
|
|
"bench_draw_line": [
|
||
|
|
[{"time (ms)": 1.321, "memory (mb)": 128.1, "retval": 0},
|
||
|
|
{"time (ms)": 1.920, "memory (mb)": 128.4, "retval": 0}],
|
||
|
|
[{"time (ms)": 1.221, "memory (mb)": 124.3, "retval": 0},
|
||
|
|
{"time (ms)": 1.423, "memory (mb)": 123.9, "retval": 0}]
|
||
|
|
]
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
Which says that we ran a benchmark on platforms named peppy, and
|
||
|
|
peppy-new-crosstool.
|
||
|
|
We ran one benchmark, named bench_draw_line.
|
||
|
|
It was run twice on each platform.
|
||
|
|
Peppy's runs took 1.321ms and 1.920ms, while peppy-new-crosstool's took 1.221ms
|
||
|
|
and 1.423ms. None of the runs failed to complete.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import division
|
||
|
|
from __future__ import print_function
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import functools
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import sys
|
||
|
|
import traceback
|
||
|
|
|
||
|
|
from results_report import BenchmarkResults
|
||
|
|
from results_report import HTMLResultsReport
|
||
|
|
from results_report import JSONResultsReport
|
||
|
|
from results_report import TextResultsReport
|
||
|
|
|
||
|
|
|
||
|
|
def CountBenchmarks(benchmark_runs):
|
||
|
|
"""Counts the number of iterations for each benchmark in benchmark_runs."""
|
||
|
|
|
||
|
|
# Example input for benchmark_runs:
|
||
|
|
# {"bench": [[run1, run2, run3], [run1, run2, run3, run4]]}
|
||
|
|
def _MaxLen(results):
|
||
|
|
return 0 if not results else max(len(r) for r in results)
|
||
|
|
|
||
|
|
return [(name, _MaxLen(results)) for name, results in benchmark_runs.items()]
|
||
|
|
|
||
|
|
|
||
|
|
def CutResultsInPlace(results, max_keys=50, complain_on_update=True):
|
||
|
|
"""Limits the given benchmark results to max_keys keys in-place.
|
||
|
|
|
||
|
|
This takes the `data` field from the benchmark input, and mutates each
|
||
|
|
benchmark run to contain `max_keys` elements (ignoring special elements, like
|
||
|
|
"retval"). At the moment, it just selects the first `max_keys` keyvals,
|
||
|
|
alphabetically.
|
||
|
|
|
||
|
|
If complain_on_update is true, this will print a message noting that a
|
||
|
|
truncation occurred.
|
||
|
|
|
||
|
|
This returns the `results` object that was passed in, for convenience.
|
||
|
|
|
||
|
|
e.g.
|
||
|
|
>>> benchmark_data = {
|
||
|
|
... "bench_draw_line": [
|
||
|
|
... [{"time (ms)": 1.321, "memory (mb)": 128.1, "retval": 0},
|
||
|
|
... {"time (ms)": 1.920, "memory (mb)": 128.4, "retval": 0}],
|
||
|
|
... [{"time (ms)": 1.221, "memory (mb)": 124.3, "retval": 0},
|
||
|
|
... {"time (ms)": 1.423, "memory (mb)": 123.9, "retval": 0}]
|
||
|
|
... ]
|
||
|
|
... }
|
||
|
|
>>> CutResultsInPlace(benchmark_data, max_keys=1, complain_on_update=False)
|
||
|
|
{
|
||
|
|
'bench_draw_line': [
|
||
|
|
[{'memory (mb)': 128.1, 'retval': 0},
|
||
|
|
{'memory (mb)': 128.4, 'retval': 0}],
|
||
|
|
[{'memory (mb)': 124.3, 'retval': 0},
|
||
|
|
{'memory (mb)': 123.9, 'retval': 0}]
|
||
|
|
]
|
||
|
|
}
|
||
|
|
"""
|
||
|
|
actually_updated = False
|
||
|
|
for bench_results in results.values():
|
||
|
|
for platform_results in bench_results:
|
||
|
|
for i, result in enumerate(platform_results):
|
||
|
|
# Keep the keys that come earliest when sorted alphabetically.
|
||
|
|
# Forcing alphabetical order is arbitrary, but necessary; otherwise,
|
||
|
|
# the keyvals we'd emit would depend on our iteration order through a
|
||
|
|
# map.
|
||
|
|
removable_keys = sorted(k for k in result if k != 'retval')
|
||
|
|
retained_keys = removable_keys[:max_keys]
|
||
|
|
platform_results[i] = {k: result[k] for k in retained_keys}
|
||
|
|
# retval needs to be passed through all of the time.
|
||
|
|
retval = result.get('retval')
|
||
|
|
if retval is not None:
|
||
|
|
platform_results[i]['retval'] = retval
|
||
|
|
actually_updated = actually_updated or \
|
||
|
|
len(retained_keys) != len(removable_keys)
|
||
|
|
|
||
|
|
if actually_updated and complain_on_update:
|
||
|
|
print(
|
||
|
|
'Warning: Some benchmark keyvals have been truncated.', file=sys.stderr)
|
||
|
|
return results
|
||
|
|
|
||
|
|
|
||
|
|
def _PositiveInt(s):
|
||
|
|
i = int(s)
|
||
|
|
if i < 0:
|
||
|
|
raise argparse.ArgumentTypeError('%d is not a positive integer.' % (i,))
|
||
|
|
return i
|
||
|
|
|
||
|
|
|
||
|
|
def _AccumulateActions(args):
|
||
|
|
"""Given program arguments, determines what actions we want to run.
|
||
|
|
|
||
|
|
Returns [(ResultsReportCtor, str)], where ResultsReportCtor can construct a
|
||
|
|
ResultsReport, and the str is the file extension for the given report.
|
||
|
|
"""
|
||
|
|
results = []
|
||
|
|
# The order of these is arbitrary.
|
||
|
|
if args.json:
|
||
|
|
results.append((JSONResultsReport, 'json'))
|
||
|
|
if args.text:
|
||
|
|
results.append((TextResultsReport, 'txt'))
|
||
|
|
if args.email:
|
||
|
|
email_ctor = functools.partial(TextResultsReport, email=True)
|
||
|
|
results.append((email_ctor, 'email'))
|
||
|
|
# We emit HTML if nothing else was specified.
|
||
|
|
if args.html or not results:
|
||
|
|
results.append((HTMLResultsReport, 'html'))
|
||
|
|
return results
|
||
|
|
|
||
|
|
|
||
|
|
# Note: get_contents is a function, because it may be expensive (generating some
|
||
|
|
# HTML reports takes O(seconds) on my machine, depending on the size of the
|
||
|
|
# input data).
|
||
|
|
def WriteFile(output_prefix, extension, get_contents, overwrite, verbose):
|
||
|
|
"""Writes `contents` to a file named "${output_prefix}.${extension}".
|
||
|
|
|
||
|
|
get_contents should be a zero-args function that returns a string (of the
|
||
|
|
contents to write).
|
||
|
|
If output_prefix == '-', this writes to stdout.
|
||
|
|
If overwrite is False, this will not overwrite files.
|
||
|
|
"""
|
||
|
|
if output_prefix == '-':
|
||
|
|
if verbose:
|
||
|
|
print('Writing %s report to stdout' % (extension,), file=sys.stderr)
|
||
|
|
sys.stdout.write(get_contents())
|
||
|
|
return
|
||
|
|
|
||
|
|
file_name = '%s.%s' % (output_prefix, extension)
|
||
|
|
if not overwrite and os.path.exists(file_name):
|
||
|
|
raise IOError('Refusing to write %s -- it already exists' % (file_name,))
|
||
|
|
|
||
|
|
with open(file_name, 'w') as out_file:
|
||
|
|
if verbose:
|
||
|
|
print('Writing %s report to %s' % (extension, file_name), file=sys.stderr)
|
||
|
|
out_file.write(get_contents())
|
||
|
|
|
||
|
|
|
||
|
|
def RunActions(actions, benchmark_results, output_prefix, overwrite, verbose):
|
||
|
|
"""Runs `actions`, returning True if all succeeded."""
|
||
|
|
failed = False
|
||
|
|
|
||
|
|
report_ctor = None # Make the linter happy
|
||
|
|
for report_ctor, extension in actions:
|
||
|
|
try:
|
||
|
|
get_contents = lambda: report_ctor(benchmark_results).GetReport()
|
||
|
|
WriteFile(output_prefix, extension, get_contents, overwrite, verbose)
|
||
|
|
except Exception:
|
||
|
|
# Complain and move along; we may have more actions that might complete
|
||
|
|
# successfully.
|
||
|
|
failed = True
|
||
|
|
traceback.print_exc()
|
||
|
|
return not failed
|
||
|
|
|
||
|
|
|
||
|
|
def PickInputFile(input_name):
|
||
|
|
"""Given program arguments, returns file to read for benchmark input."""
|
||
|
|
return sys.stdin if input_name == '-' else open(input_name)
|
||
|
|
|
||
|
|
|
||
|
|
def _NoPerfReport(_label_name, _benchmark_name, _benchmark_iteration):
|
||
|
|
return {}
|
||
|
|
|
||
|
|
|
||
|
|
def _ParseArgs(argv):
|
||
|
|
parser = argparse.ArgumentParser(description='Turns JSON into results '
|
||
|
|
'report(s).')
|
||
|
|
parser.add_argument(
|
||
|
|
'-v',
|
||
|
|
'--verbose',
|
||
|
|
action='store_true',
|
||
|
|
help='Be a tiny bit more verbose.')
|
||
|
|
parser.add_argument(
|
||
|
|
'-f',
|
||
|
|
'--force',
|
||
|
|
action='store_true',
|
||
|
|
help='Overwrite existing results files.')
|
||
|
|
parser.add_argument(
|
||
|
|
'-o',
|
||
|
|
'--output',
|
||
|
|
default='report',
|
||
|
|
type=str,
|
||
|
|
help='Prefix of the output filename (default: report). '
|
||
|
|
'- means stdout.')
|
||
|
|
parser.add_argument(
|
||
|
|
'-i',
|
||
|
|
'--input',
|
||
|
|
required=True,
|
||
|
|
type=str,
|
||
|
|
help='Where to read the JSON from. - means stdin.')
|
||
|
|
parser.add_argument(
|
||
|
|
'-l',
|
||
|
|
'--statistic-limit',
|
||
|
|
default=0,
|
||
|
|
type=_PositiveInt,
|
||
|
|
help='The maximum number of benchmark statistics to '
|
||
|
|
'display from a single run. 0 implies unlimited.')
|
||
|
|
parser.add_argument(
|
||
|
|
'--json', action='store_true', help='Output a JSON report.')
|
||
|
|
parser.add_argument(
|
||
|
|
'--text', action='store_true', help='Output a text report.')
|
||
|
|
parser.add_argument(
|
||
|
|
'--email',
|
||
|
|
action='store_true',
|
||
|
|
help='Output a text report suitable for email.')
|
||
|
|
parser.add_argument(
|
||
|
|
'--html',
|
||
|
|
action='store_true',
|
||
|
|
help='Output an HTML report (this is the default if no '
|
||
|
|
'other output format is specified).')
|
||
|
|
return parser.parse_args(argv)
|
||
|
|
|
||
|
|
|
||
|
|
def Main(argv):
|
||
|
|
args = _ParseArgs(argv)
|
||
|
|
with PickInputFile(args.input) as in_file:
|
||
|
|
raw_results = json.load(in_file)
|
||
|
|
|
||
|
|
platform_names = raw_results['platforms']
|
||
|
|
results = raw_results['data']
|
||
|
|
if args.statistic_limit:
|
||
|
|
results = CutResultsInPlace(results, max_keys=args.statistic_limit)
|
||
|
|
benches = CountBenchmarks(results)
|
||
|
|
# In crosperf, a label is essentially a platform+configuration. So, a name of
|
||
|
|
# a label and a name of a platform are equivalent for our purposes.
|
||
|
|
bench_results = BenchmarkResults(
|
||
|
|
label_names=platform_names,
|
||
|
|
benchmark_names_and_iterations=benches,
|
||
|
|
run_keyvals=results,
|
||
|
|
read_perf_report=_NoPerfReport)
|
||
|
|
actions = _AccumulateActions(args)
|
||
|
|
ok = RunActions(actions, bench_results, args.output, args.force, args.verbose)
|
||
|
|
return 0 if ok else 1
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
sys.exit(Main(sys.argv[1:]))
|