#!/usr/bin/python3 # # Build performance test script # # Copyright (c) 2016, Intel Corporation. # # SPDX-License-Identifier: GPL-2.0-only # import argparse import errno import fcntl import json import logging import os import re import shutil import sys from datetime import datetime sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib') import scriptpath scriptpath.add_oe_lib_path() scriptpath.add_bitbake_lib_path() import oeqa.buildperf from oeqa.buildperf import (BuildPerfTestLoader, BuildPerfTestResult, BuildPerfTestRunner, KernelDropCaches) from oeqa.utils.commands import runCmd from oeqa.utils.metadata import metadata_from_bb, write_metadata_file # Set-up logging LOG_FORMAT = '[%(asctime)s] %(levelname)s: %(message)s' logging.basicConfig(level=logging.INFO, format=LOG_FORMAT, datefmt='%Y-%m-%d %H:%M:%S') log = logging.getLogger() def acquire_lock(lock_f): """Acquire flock on file""" log.debug("Acquiring lock %s", os.path.abspath(lock_f.name)) try: fcntl.flock(lock_f, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as err: if err.errno == errno.EAGAIN: return False raise log.debug("Lock acquired") return True def pre_run_sanity_check(): """Sanity check of build environment""" build_dir = os.environ.get("BUILDDIR") if not build_dir: log.error("BUILDDIR not set. Please run the build environmnent setup " "script.") return False if os.getcwd() != build_dir: log.error("Please run this script under BUILDDIR (%s)", build_dir) return False ret = runCmd('which bitbake', ignore_status=True) if ret.status: log.error("bitbake command not found") return False return True def setup_file_logging(log_file): """Setup loggin to file""" log_dir = os.path.dirname(log_file) if not os.path.exists(log_dir): os.makedirs(log_dir) formatter = logging.Formatter(LOG_FORMAT) handler = logging.FileHandler(log_file) handler.setFormatter(formatter) log.addHandler(handler) def archive_build_conf(out_dir): """Archive build/conf to test results""" src_dir = os.path.join(os.environ['BUILDDIR'], 'conf') tgt_dir = os.path.join(out_dir, 'build', 'conf') os.makedirs(os.path.dirname(tgt_dir)) shutil.copytree(src_dir, tgt_dir) def update_globalres_file(result_obj, filename, metadata): """Write results to globalres csv file""" # Map test names to time and size columns in globalres # The tuples represent index and length of times and sizes # respectively gr_map = {'test1': ((0, 1), (8, 1)), 'test12': ((1, 1), (None, None)), 'test13': ((2, 1), (9, 1)), 'test2': ((3, 1), (None, None)), 'test3': ((4, 3), (None, None)), 'test4': ((7, 1), (10, 2))} values = ['0'] * 12 for status, test, _ in result_obj.all_results(): if status in ['ERROR', 'SKIPPED']: continue (t_ind, t_len), (s_ind, s_len) = gr_map[test.name] if t_ind is not None: values[t_ind:t_ind + t_len] = test.times if s_ind is not None: values[s_ind:s_ind + s_len] = test.sizes log.debug("Writing globalres log to %s", filename) rev_info = metadata['layers']['meta'] with open(filename, 'a') as fobj: fobj.write('{},{}:{},{},'.format(metadata['hostname'], rev_info['branch'], rev_info['commit'], rev_info['commit'])) fobj.write(','.join(values) + '\n') def parse_args(argv): """Parse command line arguments""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('-D', '--debug', action='store_true', help='Enable debug level logging') parser.add_argument('--globalres-file', type=os.path.abspath, help="Append results to 'globalres' csv file") parser.add_argument('--lock-file', default='./oe-build-perf.lock', metavar='FILENAME', type=os.path.abspath, help="Lock file to use") parser.add_argument('-o', '--out-dir', default='results-{date}', type=os.path.abspath, help="Output directory for test results") parser.add_argument('-x', '--xml', action='store_true', help='Enable JUnit xml output') parser.add_argument('--log-file', default='{out_dir}/oe-build-perf-test.log', help="Log file of this script") parser.add_argument('--run-tests', nargs='+', metavar='TEST', help="List of tests to run") return parser.parse_args(argv) def main(argv=None): """Script entry point""" args = parse_args(argv) # Set-up log file out_dir = args.out_dir.format(date=datetime.now().strftime('%Y%m%d%H%M%S')) setup_file_logging(args.log_file.format(out_dir=out_dir)) if args.debug: log.setLevel(logging.DEBUG) lock_f = open(args.lock_file, 'w') if not acquire_lock(lock_f): log.error("Another instance of this script is running, exiting...") return 1 if not pre_run_sanity_check(): return 1 # Check our capability to drop caches and ask pass if needed KernelDropCaches.check() # Load build perf tests loader = BuildPerfTestLoader() if args.run_tests: suite = loader.loadTestsFromNames(args.run_tests, oeqa.buildperf) else: suite = loader.loadTestsFromModule(oeqa.buildperf) # Save test metadata metadata = metadata_from_bb() log.info("Testing Git revision branch:commit %s:%s (%s)", metadata['layers']['meta']['branch'], metadata['layers']['meta']['commit'], metadata['layers']['meta']['commit_count']) if args.xml: write_metadata_file(os.path.join(out_dir, 'metadata.xml'), metadata) else: with open(os.path.join(out_dir, 'metadata.json'), 'w') as fobj: json.dump(metadata, fobj, indent=2) archive_build_conf(out_dir) runner = BuildPerfTestRunner(out_dir, verbosity=2) # Suppress logger output to stderr so that the output from unittest # is not mixed with occasional logger output log.handlers[0].setLevel(logging.CRITICAL) # Run actual tests result = runner.run(suite) # Restore logger output to stderr log.handlers[0].setLevel(log.level) if args.xml: result.write_results_xml() else: result.write_results_json() result.write_buildstats_json() if args.globalres_file: update_globalres_file(result, args.globalres_file, metadata) if result.wasSuccessful(): return 0 return 2 if __name__ == '__main__': sys.exit(main())