diff options
Diffstat (limited to 'scripts/lib/resulttool/store.py')
-rw-r--r-- | scripts/lib/resulttool/store.py | 160 |
1 files changed, 77 insertions, 83 deletions
diff --git a/scripts/lib/resulttool/store.py b/scripts/lib/resulttool/store.py index 2c6fd8492c..e0951f0a8f 100644 --- a/scripts/lib/resulttool/store.py +++ b/scripts/lib/resulttool/store.py @@ -1,110 +1,104 @@ -# test result tool - store test results +# resulttool - store test results # # Copyright (c) 2019, Intel Corporation. +# Copyright (c) 2019, Linux Foundation # -# This program is free software; you can redistribute it and/or modify it -# under the terms and conditions of the GNU General Public License, -# version 2, as published by the Free Software Foundation. +# SPDX-License-Identifier: GPL-2.0-only # -# This program is distributed in the hope it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -import datetime + import tempfile import os import subprocess +import json +import shutil import scriptpath scriptpath.add_bitbake_lib_path() scriptpath.add_oe_lib_path() -from resulttool.resultsutils import checkout_git_dir -try: - import bb -except ImportError: - pass +import resulttool.resultutils as resultutils +import oeqa.utils.gitarchive as gitarchive -class ResultsGitStore(object): - def _get_output_dir(self): - basepath = os.environ['BUILDDIR'] - return basepath + '/testresults_%s/' % datetime.datetime.now().strftime("%Y%m%d%H%M%S") - - def _create_temporary_workspace_dir(self): - return tempfile.mkdtemp(prefix='testresults.') +def store(args, logger): + tempdir = tempfile.mkdtemp(prefix='testresults.') + try: + configvars = resultutils.extra_configvars.copy() + if args.executed_by: + configvars['EXECUTED_BY'] = args.executed_by + if args.extra_test_env: + configvars['EXTRA_TEST_ENV'] = args.extra_test_env + results = {} + logger.info('Reading files from %s' % args.source) + if resultutils.is_url(args.source) or os.path.isfile(args.source): + resultutils.append_resultsdata(results, args.source, configvars=configvars) + else: + for root, dirs, files in os.walk(args.source): + for name in files: + f = os.path.join(root, name) + if name == "testresults.json": + resultutils.append_resultsdata(results, f, configvars=configvars) + elif args.all: + dst = f.replace(args.source, tempdir + "/") + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copyfile(f, dst) - def _remove_temporary_workspace_dir(self, workspace_dir): - return subprocess.run(["rm", "-rf", workspace_dir]) + revisions = {} - def _oe_copy_files(self, source_dir, destination_dir): - from oe.path import copytree - copytree(source_dir, destination_dir) + if not results and not args.all: + if args.allow_empty: + logger.info("No results found to store") + return 0 + logger.error("No results found to store") + return 1 - def _copy_files(self, source_dir, destination_dir, copy_ignore=None): - from shutil import copytree - copytree(source_dir, destination_dir, ignore=copy_ignore) + # Find the branch/commit/commit_count and ensure they all match + for suite in results: + for result in results[suite]: + config = results[suite][result]['configuration']['LAYERS']['meta'] + revision = (config['commit'], config['branch'], str(config['commit_count'])) + if revision not in revisions: + revisions[revision] = {} + if suite not in revisions[revision]: + revisions[revision][suite] = {} + revisions[revision][suite][result] = results[suite][result] - def _store_files_to_git(self, logger, file_dir, git_dir, git_branch, commit_msg_subject, commit_msg_body): - logger.debug('Storing test result into git repository (%s) and branch (%s)' - % (git_dir, git_branch)) - return subprocess.run(["oe-git-archive", - file_dir, - "-g", git_dir, - "-b", git_branch, - "--commit-msg-subject", commit_msg_subject, - "--commit-msg-body", commit_msg_body]) + logger.info("Found %d revisions to store" % len(revisions)) - def store_to_existing(self, logger, source_dir, git_dir, git_branch): - logger.debug('Storing files to existing git repository and branch') - from shutil import ignore_patterns - dest_dir = self._create_temporary_workspace_dir() - dest_top_dir = os.path.join(dest_dir, 'top_dir') - self._copy_files(git_dir, dest_top_dir, copy_ignore=ignore_patterns('.git')) - self._oe_copy_files(source_dir, dest_top_dir) - self._store_files_to_git(logger, dest_top_dir, git_dir, git_branch, - 'Store as existing git and branch', 'Store as existing git repository and branch') - self._remove_temporary_workspace_dir(dest_dir) - return git_dir + for r in revisions: + results = revisions[r] + keywords = {'commit': r[0], 'branch': r[1], "commit_count": r[2]} + subprocess.check_call(["find", tempdir, "!", "-path", "./.git/*", "-delete"]) + resultutils.save_resultsdata(results, tempdir, ptestlogs=True) - def store_to_existing_with_new_branch(self, logger, source_dir, git_dir, git_branch): - logger.debug('Storing files to existing git repository with new branch') - self._store_files_to_git(logger, source_dir, git_dir, git_branch, - 'Store as existing git with new branch', - 'Store as existing git repository with new branch') - return git_dir + logger.info('Storing test result into git repository %s' % args.git_dir) - def store_to_new(self, logger, source_dir, git_branch): - logger.debug('Storing files to new git repository') - output_dir = self._get_output_dir() - self._store_files_to_git(logger, source_dir, output_dir, git_branch, - 'Store as new', 'Store as new git repository') - return output_dir + gitarchive.gitarchive(tempdir, args.git_dir, False, False, + "Results of {branch}:{commit}", "branch: {branch}\ncommit: {commit}", "{branch}", + False, "{branch}/{commit_count}-g{commit}/{tag_number}", + 'Test run #{tag_number} of {branch}:{commit}', '', + [], [], False, keywords, logger) - def store(self, logger, source_dir, git_dir, git_branch): - if git_dir: - if checkout_git_dir(git_dir, git_branch): - self.store_to_existing(logger, source_dir, git_dir, git_branch) - else: - self.store_to_existing_with_new_branch(logger, source_dir, git_dir, git_branch) - else: - self.store_to_new(logger, source_dir, git_branch) + finally: + subprocess.check_call(["rm", "-rf", tempdir]) -def store(args, logger): - gitstore = ResultsGitStore() - gitstore.store(logger, args.source_dir, args.git_dir, args.git_branch) return 0 def register_commands(subparsers): """Register subcommands from this plugin""" - parser_build = subparsers.add_parser('store', help='store test result files and directories into git repository', - description='store the testresults.json files and related directories ' - 'from the source directory into the destination git repository ' - 'with the given git branch', + parser_build = subparsers.add_parser('store', help='store test results into a git repository', + description='takes a results file or directory of results files and stores ' + 'them into the destination git repository, splitting out the results ' + 'files as configured', group='setup') parser_build.set_defaults(func=store) - parser_build.add_argument('source_dir', - help='source directory that contain the test result files and directories to be stored') - parser_build.add_argument('git_branch', help='git branch used for store') - parser_build.add_argument('-d', '--git-dir', default='', - help='(optional) default store to new <top_dir>/<build>/<testresults_datetime> ' - 'directory unless provided with existing git repository as destination') + parser_build.add_argument('source', + help='source file/directory/URL that contain the test result files to be stored') + parser_build.add_argument('git_dir', + help='the location of the git repository to store the results in') + parser_build.add_argument('-a', '--all', action='store_true', + help='include all files, not just testresults.json files') + parser_build.add_argument('-e', '--allow-empty', action='store_true', + help='don\'t error if no results to store are found') + parser_build.add_argument('-x', '--executed-by', default='', + help='add executed-by configuration to each result file') + parser_build.add_argument('-t', '--extra-test-env', default='', + help='add extra test environment data to each result file configuration') |