#!/usr/bin/env python3 # # Copyright (C) 2019 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Sample Usage: # $ python update_profiles.py 500000 ALL --profdata-suffix 2019-04-15 # # Additional/frequently-used arguments: # -b BUG adds a 'Bug: ' to the commit message when adding the profiles. # --do-not-merge adds a 'DO NOT MERGE' tag to the commit message to restrict # automerge of profiles from release branches. # # Try '-h' for a full list of command line arguments. import argparse import os import shutil import subprocess import sys import tempfile import zipfile import utils from android_build_client import AndroidBuildClient class Benchmark(object): def __init__(self, name): self.name = name def apct_test_tag(self): raise NotImplementedError() def profdata_file(self, suffix=''): profdata = os.path.join(self.name, '{}.profdata'.format(self.name)) if suffix: profdata += '.' + suffix return profdata def profraw_files(self): raise NotImplementedError() def merge_profraws(self, profile_dir, output): profraws = [ os.path.join(profile_dir, p) for p in self.profraw_files(profile_dir) ] utils.run_llvm_profdata(profraws, output) class NativeExeBenchmark(Benchmark): def apct_test_tag(self): return 'apct/perf/pgo/profile-collector' def profraw_files(self, profile_dir): if self.name == 'hwui': return [ 'hwuimacro.profraw', 'hwuimacro_64.profraw', 'hwuimicro.profraw', 'hwuimicro_64.profraw', 'skia_nanobench.profraw', 'skia_nanobench_64.profraw' ] elif self.name == 'hwbinder': return ['hwbinder.profraw', 'hwbinder_64.profraw'] class APKBenchmark(Benchmark): def apct_test_tag(self): return 'apct/perf/pgo/apk-profile-collector' def profdata_file(self, suffix=''): profdata = os.path.join('art', '{}_arm_arm64.profdata'.format(self.name)) if suffix: profdata += '.' + suffix return profdata def profraw_files(self, profile_dir): return os.listdir(profile_dir) def BenchmarkFactory(benchmark_name): if benchmark_name == 'dex2oat': return APKBenchmark(benchmark_name) elif benchmark_name in ['hwui', 'hwbinder']: return NativeExeBenchmark(benchmark_name) else: raise RuntimeError('Unknown benchmark ' + benchmark_name) def extract_profiles(build, test_tag, build_client, output_dir): pgo_zip = build_client.download_pgo_zip(build, test_tag, output_dir) zipfile_name = os.path.join(pgo_zip) zip_ref = zipfile.ZipFile(zipfile_name) zip_ref.extractall(output_dir) zip_ref.close() KNOWN_BENCHMARKS = ['ALL', 'dex2oat', 'hwui', 'hwbinder'] def parse_args(): """Parses and returns command line arguments.""" parser = argparse.ArgumentParser() parser.add_argument( 'build', metavar='BUILD', help='Build number to pull from the build server.') parser.add_argument( '-b', '--bug', type=int, help='Bug to reference in commit message.') parser.add_argument( '--use-current-branch', action='store_true', help='Do not repo start a new branch for the update.') parser.add_argument( '--add-do-not-merge', action='store_true', help='Add \'DO NOT MERGE\' to the commit message.') parser.add_argument( '--profdata-suffix', type=str, default='', help='Suffix to append to merged profdata file') parser.add_argument( 'benchmark', metavar='BENCHMARK', help='Update profiles for BENCHMARK. Choices are {}'.format( KNOWN_BENCHMARKS)) parser.add_argument( '--skip-cleanup', '-sc', action='store_true', default=False, help='Skip the cleanup, and leave intermediate files (in /tmp/pgo-profiles-*)' ) return parser.parse_args() def get_current_profile(benchmark): profile = benchmark.profdata_file() dirname, basename = os.path.split(profile) old_profiles = [f for f in os.listdir(dirname) if f.startswith(basename)] if len(old_profiles) == 0: return '' return os.path.join(dirname, old_profiles[0]) def main(): args = parse_args() utils.check_gcertstatus() if args.benchmark == 'ALL': worklist = KNOWN_BENCHMARKS[1:] else: worklist = [args.benchmark] profiles_project = os.path.join(utils.android_build_top(), 'toolchain', 'pgo-profiles') os.chdir(profiles_project) if not args.use_current_branch: branch_name = 'update-profiles-' + args.build utils.check_call(['repo', 'start', branch_name, '.']) build_client = AndroidBuildClient() for benchmark_name in worklist: benchmark = BenchmarkFactory(benchmark_name) # Existing profile file, which gets 'rm'-ed from 'git' down below. current_profile = get_current_profile(benchmark) # Extract profiles to a temporary directory. After extraction, we # expect to find one subdirectory with profraw files under the temporary # directory. extract_dir = tempfile.mkdtemp(prefix='pgo-profiles-' + benchmark_name) extract_profiles(args.build, benchmark.apct_test_tag(), build_client, extract_dir) extract_subdirs = [ os.path.join(extract_dir, sub) for sub in os.listdir(extract_dir) if os.path.isdir(os.path.join(extract_dir, sub)) ] if len(extract_subdirs) != 1: raise RuntimeError( 'Expected one subdir under {}'.format(extract_dir)) # Merge profiles. profdata = benchmark.profdata_file(args.profdata_suffix) benchmark.merge_profraws(extract_subdirs[0], profdata) # Construct 'git' commit message. message_lines = [ 'Update PGO profiles for {}'.format(benchmark_name), '', 'The profiles are from build {}.'.format(args.build), '' ] if args.add_do_not_merge: message_lines[0] = '[DO NOT MERGE] ' + message_lines[0] if args.bug: message_lines.append('') message_lines.append('Bug: http://b/{}'.format(args.bug)) message_lines.append('Test: Build (TH)') message = '\n'.join(message_lines) # Invoke git: Delete current profile, add new profile and commit these # changes. if current_profile: utils.check_call(['git', 'rm', current_profile]) utils.check_call(['git', 'add', profdata]) utils.check_call(['git', 'commit', '-m', message]) if not args.skip_cleanup: shutil.rmtree(extract_dir) if __name__ == '__main__': main()