diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2019-02-16 18:13:00 +0000 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2019-02-21 12:31:50 +0000 |
commit | ff2c029b568f70aa9960dde04ddd207829812ea0 (patch) | |
tree | 5d80afe2e19d699d58cb424fe8dd97b3294f47f7 /scripts/lib/resulttool/store.py | |
parent | f24dc9e87085a8fe5410feee10c7a3591fe9d816 (diff) | |
download | openembedded-core-contrib-ff2c029b568f70aa9960dde04ddd207829812ea0.tar.gz |
resulttool: Improvements to allow integration to the autobuilder
This is a combined patch of the various tweaks and improvements I
made to resulttool:
* Avoid subprocess.run() as its a python 3.6 feature and we
have autobuilder workers with 3.5.
* Avoid python keywords as variable names
* Simplify dict accesses using .get()
* Rename resultsutils -> resultutils to match the resultstool ->
resulttool rename
* Formalised the handling of "file_name" to "TESTSERIES" which the code
will now add into the json configuration data if its not present, based
on the directory name.
* When we don't have failed test cases, print something saying so
instead of an empty table
* Tweak the table headers in the report to be more readable (reference
"Test Series" instead if file_id and ID instead of results_id)
* Improve/simplify the max string length handling
* Merge the counts and percentage data into one table in the report
since printing two reports of the same data confuses the user
* Removed the confusing header in the regression report
* Show matches, then regressions, then unmatched runs in the regression
report, also remove chatting unneeded output
* Try harder to "pair" up matching configurations to reduce noise in
the regressions report
* Abstracted the "mapping" table concept used to pairing in the
regression code to general code in resultutils
* Created multiple mappings for results analysis, results storage and
'flattening' results data in a merge
* Simplify the merge command to take a source and a destination,
letting the destination be a directory or a file, removing the need for
an output directory parameter
* Add the 'IMAGE_PKGTYPE' and 'DISTRO' config options to the regression
mappings
* Have the store command place the testresults files in a layout from
the mapping, making commits into the git repo for results storage more
useful for simple comparison purposes
* Set the oe-git-archive tag format appropriately for oeqa results
storage (and simplify the commit messages closer to their defaults)
* Fix oe-git-archive to use the commit/branch data from the results file
* Cleaned up the command option help to match other changes
* Follow the model of git branch/tag processing used by oe-build-perf-report
and use that to read the data using git show to avoid branch change
* Add ptest summary to the report command
* Update the tests to match the above changes
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib/resulttool/store.py')
-rw-r--r-- | scripts/lib/resulttool/store.py | 136 |
1 files changed, 59 insertions, 77 deletions
diff --git a/scripts/lib/resulttool/store.py b/scripts/lib/resulttool/store.py index 2c6fd8492c..6744fb3c05 100644 --- a/scripts/lib/resulttool/store.py +++ b/scripts/lib/resulttool/store.py @@ -1,6 +1,7 @@ -# test result tool - store test results +# resulttool - store test results # # Copyright (c) 2019, Intel Corporation. +# Copyright (c) 2019, Linux Foundation # # This program is free software; you can redistribute it and/or modify it # under the terms and conditions of the GNU General Public License, @@ -11,100 +12,81 @@ # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # more details. # -import datetime import tempfile import os import subprocess +import json +import shutil import scriptpath scriptpath.add_bitbake_lib_path() scriptpath.add_oe_lib_path() -from resulttool.resultsutils import checkout_git_dir -try: - import bb -except ImportError: - pass +import resulttool.resultutils as resultutils +import oeqa.utils.gitarchive as gitarchive -class ResultsGitStore(object): - def _get_output_dir(self): - basepath = os.environ['BUILDDIR'] - return basepath + '/testresults_%s/' % datetime.datetime.now().strftime("%Y%m%d%H%M%S") - - def _create_temporary_workspace_dir(self): - return tempfile.mkdtemp(prefix='testresults.') - - def _remove_temporary_workspace_dir(self, workspace_dir): - return subprocess.run(["rm", "-rf", workspace_dir]) - - def _oe_copy_files(self, source_dir, destination_dir): - from oe.path import copytree - copytree(source_dir, destination_dir) +def store(args, logger): + tempdir = tempfile.mkdtemp(prefix='testresults.') + try: + results = {} + logger.info('Reading files from %s' % args.source) + for root, dirs, files in os.walk(args.source): + for name in files: + f = os.path.join(root, name) + if name == "testresults.json": + resultutils.append_resultsdata(results, f) + elif args.all: + dst = f.replace(args.source, tempdir + "/") + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copyfile(f, dst) + resultutils.save_resultsdata(results, tempdir) - def _copy_files(self, source_dir, destination_dir, copy_ignore=None): - from shutil import copytree - copytree(source_dir, destination_dir, ignore=copy_ignore) + if not results and not args.all: + if args.allow_empty: + logger.info("No results found to store") + return 0 + logger.error("No results found to store") + return 1 - def _store_files_to_git(self, logger, file_dir, git_dir, git_branch, commit_msg_subject, commit_msg_body): - logger.debug('Storing test result into git repository (%s) and branch (%s)' - % (git_dir, git_branch)) - return subprocess.run(["oe-git-archive", - file_dir, - "-g", git_dir, - "-b", git_branch, - "--commit-msg-subject", commit_msg_subject, - "--commit-msg-body", commit_msg_body]) + keywords = {'branch': None, 'commit': None, 'commit_count': None} - def store_to_existing(self, logger, source_dir, git_dir, git_branch): - logger.debug('Storing files to existing git repository and branch') - from shutil import ignore_patterns - dest_dir = self._create_temporary_workspace_dir() - dest_top_dir = os.path.join(dest_dir, 'top_dir') - self._copy_files(git_dir, dest_top_dir, copy_ignore=ignore_patterns('.git')) - self._oe_copy_files(source_dir, dest_top_dir) - self._store_files_to_git(logger, dest_top_dir, git_dir, git_branch, - 'Store as existing git and branch', 'Store as existing git repository and branch') - self._remove_temporary_workspace_dir(dest_dir) - return git_dir + # Find the branch/commit/commit_count and ensure they all match + for suite in results: + for result in results[suite]: + config = results[suite][result]['configuration']['LAYERS']['meta'] + for k in keywords: + if keywords[k] is None: + keywords[k] = config.get(k) + if config.get(k) != keywords[k]: + logger.error("Mismatched source commit/branch/count: %s vs %s" % (config.get(k), keywords[k])) + return 1 - def store_to_existing_with_new_branch(self, logger, source_dir, git_dir, git_branch): - logger.debug('Storing files to existing git repository with new branch') - self._store_files_to_git(logger, source_dir, git_dir, git_branch, - 'Store as existing git with new branch', - 'Store as existing git repository with new branch') - return git_dir + logger.info('Storing test result into git repository %s' % args.git_dir) - def store_to_new(self, logger, source_dir, git_branch): - logger.debug('Storing files to new git repository') - output_dir = self._get_output_dir() - self._store_files_to_git(logger, source_dir, output_dir, git_branch, - 'Store as new', 'Store as new git repository') - return output_dir + gitarchive.gitarchive(tempdir, args.git_dir, False, False, + "Results of {branch}:{commit}", "branch: {branch}\ncommit: {commit}", "{branch}", + False, "{branch}/{commit_count}-g{commit}/{tag_number}", + 'Test run #{tag_number} of {branch}:{commit}', '', + [], [], False, keywords, logger) - def store(self, logger, source_dir, git_dir, git_branch): - if git_dir: - if checkout_git_dir(git_dir, git_branch): - self.store_to_existing(logger, source_dir, git_dir, git_branch) - else: - self.store_to_existing_with_new_branch(logger, source_dir, git_dir, git_branch) - else: - self.store_to_new(logger, source_dir, git_branch) + finally: + subprocess.check_call(["rm", "-rf", tempdir]) -def store(args, logger): - gitstore = ResultsGitStore() - gitstore.store(logger, args.source_dir, args.git_dir, args.git_branch) return 0 def register_commands(subparsers): """Register subcommands from this plugin""" - parser_build = subparsers.add_parser('store', help='store test result files and directories into git repository', - description='store the testresults.json files and related directories ' - 'from the source directory into the destination git repository ' - 'with the given git branch', + parser_build = subparsers.add_parser('store', help='store test results into a git repository', + description='takes a results file or directory of results files and stores ' + 'them into the destination git repository, splitting out the results ' + 'files as configured', group='setup') parser_build.set_defaults(func=store) - parser_build.add_argument('source_dir', - help='source directory that contain the test result files and directories to be stored') - parser_build.add_argument('git_branch', help='git branch used for store') - parser_build.add_argument('-d', '--git-dir', default='', - help='(optional) default store to new <top_dir>/<build>/<testresults_datetime> ' - 'directory unless provided with existing git repository as destination') + parser_build.add_argument('source', + help='source file or directory that contain the test result files to be stored') + parser_build.add_argument('git_dir', + help='the location of the git repository to store the results in') + parser_build.add_argument('-a', '--all', action='store_true', + help='include all files, not just testresults.json files') + parser_build.add_argument('-e', '--allow-empty', action='store_true', + help='don\'t error if no results to store are found') + |