From 89eb37ef1ef8d5deb87fd55c9ea7b2cfa2681b07 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 29 Jun 2016 19:28:31 +0300 Subject: oeqa.buildperf: add BuildPerfTestResult class The new class is derived from unittest.TextTestResult class. It is actually implemented by modifying the old BuildPerfTestRunner class which, in turn, is replaced by a totally new simple implementation derived from unittest.TestRunner. Signed-off-by: Markus Lehtonen Signed-off-by: Ross Burton --- meta/lib/oeqa/buildperf/__init__.py | 4 +- meta/lib/oeqa/buildperf/base.py | 150 +++++++++++++++++++----------------- scripts/oe-build-perf-test | 10 +++ 3 files changed, 90 insertions(+), 74 deletions(-) diff --git a/meta/lib/oeqa/buildperf/__init__.py b/meta/lib/oeqa/buildperf/__init__.py index 7e51726afb..85abf3a25e 100644 --- a/meta/lib/oeqa/buildperf/__init__.py +++ b/meta/lib/oeqa/buildperf/__init__.py @@ -10,9 +10,9 @@ # more details. # """Build performance tests""" -from .base import (perf_test_case, - BuildPerfTestCase, +from .base import (BuildPerfTestCase, BuildPerfTestLoader, + BuildPerfTestResult, BuildPerfTestRunner, KernelDropCaches) from .test_basic import * diff --git a/meta/lib/oeqa/buildperf/base.py b/meta/lib/oeqa/buildperf/base.py index 1ee546dd85..d592bd26b9 100644 --- a/meta/lib/oeqa/buildperf/base.py +++ b/meta/lib/oeqa/buildperf/base.py @@ -76,25 +76,26 @@ def time_cmd(cmd, **kwargs): return ret, timedata -class BuildPerfTestRunner(object): +class BuildPerfTestResult(unittest.TextTestResult): """Runner class for executing the individual tests""" # List of test cases to run test_run_queue = [] - def __init__(self, out_dir): - self.results = {} - self.out_dir = os.path.abspath(out_dir) - if not os.path.exists(self.out_dir): - os.makedirs(self.out_dir) + def __init__(self, out_dir, *args, **kwargs): + super(BuildPerfTestResult, self).__init__(*args, **kwargs) + self.out_dir = out_dir # Get Git parameters try: self.repo = GitRepo('.') except GitError: self.repo = None - self.git_rev, self.git_branch = self.get_git_revision() + self.git_revision, self.git_branch = self.get_git_revision() + self.hostname = socket.gethostname() + self.start_time = self.elapsed_time = None + self.successes = [] log.info("Using Git branch:revision %s:%s", self.git_branch, - self.git_rev) + self.git_revision) def get_git_revision(self): """Get git branch and revision under testing""" @@ -117,79 +118,71 @@ class BuildPerfTestRunner(object): branch = None return str(rev), str(branch) - def run_tests(self): - """Method that actually runs the tests""" - self.results['schema_version'] = 1 - self.results['git_revision'] = self.git_rev - self.results['git_branch'] = self.git_branch - self.results['tester_host'] = socket.gethostname() - start_time = datetime.utcnow() - self.results['start_time'] = start_time - self.results['tests'] = {} - - self.archive_build_conf() - for test_class in self.test_run_queue: - log.info("Executing test %s: %s", test_class.name, - test_class.description) - - test = test_class(self.out_dir) - try: - test.run() - except Exception: - # Catch all exceptions. This way e.g buggy tests won't scrap - # the whole test run - sep = '-' * 5 + ' TRACEBACK ' + '-' * 60 + '\n' - tb_msg = sep + traceback.format_exc() + sep - log.error("Test execution failed with:\n" + tb_msg) - self.results['tests'][test.name] = test.results - - self.results['elapsed_time'] = datetime.utcnow() - start_time - return 0 - - def archive_build_conf(self): - """Archive build/conf to test results""" - src_dir = os.path.join(os.environ['BUILDDIR'], 'conf') - tgt_dir = os.path.join(self.out_dir, 'build', 'conf') - os.makedirs(os.path.dirname(tgt_dir)) - shutil.copytree(src_dir, tgt_dir) + def addSuccess(self, test): + """Record results from successful tests""" + super(BuildPerfTestResult, self).addSuccess(test) + self.successes.append((test, None)) + + def startTest(self, test): + """Pre-test hook""" + test.out_dir = self.out_dir + log.info("Executing test %s: %s", test.name, test.shortDescription()) + self.stream.write(datetime.now().strftime("[%Y-%m-%d %H:%M:%S] ")) + super(BuildPerfTestResult, self).startTest(test) + + def startTestRun(self): + """Pre-run hook""" + self.start_time = datetime.utcnow() + + def stopTestRun(self): + """Pre-run hook""" + self.elapsed_time = datetime.utcnow() - self.start_time + + def all_results(self): + result_map = {'SUCCESS': self.successes, + 'FAIL': self.failures, + 'ERROR': self.errors, + 'EXP_FAIL': self.expectedFailures, + 'UNEXP_SUCCESS': self.unexpectedSuccesses} + for status, tests in result_map.items(): + for test in tests: + yield (status, test) + def update_globalres_file(self, filename): """Write results to globalres csv file""" + # Map test names to time and size columns in globalres + # The tuples represent index and length of times and sizes + # respectively + gr_map = {'test1': ((0, 1), (8, 1)), + 'test12': ((1, 1), (None, None)), + 'test13': ((2, 1), (9, 1)), + 'test2': ((3, 1), (None, None)), + 'test3': ((4, 3), (None, None)), + 'test4': ((7, 1), (10, 2))} + if self.repo: - git_tag_rev = self.repo.run_cmd(['describe', self.git_rev]) + git_tag_rev = self.repo.run_cmd(['describe', self.git_revision]) else: - git_tag_rev = self.git_rev - times = [] - sizes = [] - for test in self.results['tests'].values(): - for measurement in test['measurements']: - res_type = measurement['type'] - values = measurement['values'] - if res_type == BuildPerfTest.SYSRES: - e_sec = values['elapsed_time'].total_seconds() - times.append('{:d}:{:02d}:{:.2f}'.format( - int(e_sec / 3600), - int((e_sec % 3600) / 60), - e_sec % 60)) - elif res_type == BuildPerfTest.DISKUSAGE: - sizes.append(str(values['size'])) - else: - log.warning("Unable to handle '%s' values in " - "globalres.log", res_type) + git_tag_rev = self.git_revision + + values = ['0'] * 12 + for status, test in self.all_results(): + if status not in ['SUCCESS', 'FAILURE', 'EXP_SUCCESS']: + continue + (t_ind, t_len), (s_ind, s_len) = gr_map[test.name] + if t_ind is not None: + values[t_ind:t_ind + t_len] = test.times + if s_ind is not None: + values[s_ind:s_ind + s_len] = test.sizes log.debug("Writing globalres log to %s", filename) with open(filename, 'a') as fobj: - fobj.write('{},{}:{},{},'.format(self.results['tester_host'], - self.results['git_branch'], - self.results['git_revision'], + fobj.write('{},{}:{},{},'.format(self.hostname, + self.git_branch, + self.git_revision, git_tag_rev)) - fobj.write(','.join(times + sizes) + '\n') - - -def perf_test_case(obj): - """Decorator for adding test classes""" - BuildPerfTestRunner.test_run_queue.append(obj) - return obj + fobj.write(','.join(values) + '\n') class BuildPerfTestCase(unittest.TestCase): @@ -330,3 +323,16 @@ class BuildPerfTestCase(unittest.TestCase): class BuildPerfTestLoader(unittest.TestLoader): """Test loader for build performance tests""" sortTestMethodsUsing = None + + +class BuildPerfTestRunner(unittest.TextTestRunner): + """Test loader for build performance tests""" + sortTestMethodsUsing = None + + def __init__(self, out_dir, *args, **kwargs): + super(BuildPerfTestRunner, self).__init__(*args, **kwargs) + self.out_dir = out_dir + + def _makeResult(self): + return BuildPerfTestResult(self.out_dir, self.stream, self.descriptions, + self.verbosity) diff --git a/scripts/oe-build-perf-test b/scripts/oe-build-perf-test index 996996bc62..8142b0332b 100755 --- a/scripts/oe-build-perf-test +++ b/scripts/oe-build-perf-test @@ -19,6 +19,7 @@ import errno import fcntl import logging import os +import shutil import sys from datetime import datetime @@ -78,6 +79,14 @@ def setup_file_logging(log_file): log.addHandler(handler) +def archive_build_conf(out_dir): + """Archive build/conf to test results""" + src_dir = os.path.join(os.environ['BUILDDIR'], 'conf') + tgt_dir = os.path.join(out_dir, 'build', 'conf') + os.makedirs(os.path.dirname(tgt_dir)) + shutil.copytree(src_dir, tgt_dir) + + def parse_args(argv): """Parse command line arguments""" parser = argparse.ArgumentParser( @@ -120,6 +129,7 @@ def main(argv=None): # Run actual tests runner = BuildPerfTestRunner(out_dir) + archive_build_conf(out_dir) ret = runner.run_tests() if not ret: if args.globalres_file: -- cgit 1.2.3-korg