diff options
Diffstat (limited to 'scripts/yocto_testresults_query.py')
-rwxr-xr-x | scripts/yocto_testresults_query.py | 131 |
1 files changed, 131 insertions, 0 deletions
diff --git a/scripts/yocto_testresults_query.py b/scripts/yocto_testresults_query.py new file mode 100755 index 0000000000..521ead8473 --- /dev/null +++ b/scripts/yocto_testresults_query.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 + +# Yocto Project test results management tool +# This script is an thin layer over resulttool to manage tes results and regression reports. +# Its main feature is to translate tags or branch names to revisions SHA1, and then to run resulttool +# with those computed revisions +# +# Copyright (C) 2023 OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +import sys +import os +import argparse +import subprocess +import tempfile +import lib.scriptutils as scriptutils + +script_path = os.path.dirname(os.path.realpath(__file__)) +poky_path = os.path.abspath(os.path.join(script_path, "..")) +resulttool = os.path.abspath(os.path.join(script_path, "resulttool")) +logger = scriptutils.logger_create(sys.argv[0]) +testresults_default_url="git://git.yoctoproject.org/yocto-testresults" + +def create_workdir(): + workdir = tempfile.mkdtemp(prefix='yocto-testresults-query.') + logger.info(f"Shallow-cloning testresults in {workdir}") + subprocess.check_call(["git", "clone", testresults_default_url, workdir, "--depth", "1"]) + return workdir + +def get_sha1(pokydir, revision): + try: + rev = subprocess.check_output(["git", "rev-list", "-n", "1", revision], cwd=pokydir).decode('utf-8').strip() + logger.info(f"SHA-1 revision for {revision} in {pokydir} is {rev}") + return rev + except subprocess.CalledProcessError: + logger.error(f"Can not find SHA-1 for {revision} in {pokydir}") + return None + +def get_branch(tag): + # The tags in test results repository, as returned by git rev-list, have the following form: + # refs/tags/<branch>/<count>-g<sha1>/<num> + return '/'.join(tag.split("/")[2:-2]) + +def fetch_testresults(workdir, sha1): + logger.info(f"Fetching test results for {sha1} in {workdir}") + rawtags = subprocess.check_output(["git", "ls-remote", "--refs", "--tags", "origin", f"*{sha1}*"], cwd=workdir).decode('utf-8').strip() + if not rawtags: + raise Exception(f"No reference found for commit {sha1} in {workdir}") + branch = "" + for rev in [rawtag.split()[1] for rawtag in rawtags.splitlines()]: + if not branch: + branch = get_branch(rev) + logger.info(f"Fetching matching revision: {rev}") + subprocess.check_call(["git", "fetch", "--depth", "1", "origin", f"{rev}:{rev}"], cwd=workdir) + return branch + +def compute_regression_report(workdir, basebranch, baserevision, targetbranch, targetrevision, args): + logger.info(f"Running resulttool regression between SHA1 {baserevision} and {targetrevision}") + command = [resulttool, "regression-git", "--branch", basebranch, "--commit", baserevision, "--branch2", targetbranch, "--commit2", targetrevision, workdir] + if args.limit: + command.extend(["-l", args.limit]) + report = subprocess.check_output(command).decode("utf-8") + return report + +def print_report_with_header(report, baseversion, baserevision, targetversion, targetrevision): + print("========================== Regression report ==============================") + print(f'{"=> Target:": <16}{targetversion: <16}({targetrevision})') + print(f'{"=> Base:": <16}{baseversion: <16}({baserevision})') + print("===========================================================================\n") + print(report, end='') + +def regression(args): + logger.info(f"Compute regression report between {args.base} and {args.target}") + if args.testresultsdir: + workdir = args.testresultsdir + else: + workdir = create_workdir() + + try: + baserevision = get_sha1(poky_path, args.base) + targetrevision = get_sha1(poky_path, args.target) + if not baserevision or not targetrevision: + logger.error("One or more revision(s) missing. You might be targeting nonexistant tags/branches, or are in wrong repository (you must use Poky and not oe-core)") + if not args.testresultsdir: + subprocess.check_call(["rm", "-rf", workdir]) + sys.exit(1) + basebranch = fetch_testresults(workdir, baserevision) + targetbranch = fetch_testresults(workdir, targetrevision) + report = compute_regression_report(workdir, basebranch, baserevision, targetbranch, targetrevision, args) + print_report_with_header(report, args.base, baserevision, args.target, targetrevision) + finally: + if not args.testresultsdir: + subprocess.check_call(["rm", "-rf", workdir]) + +def main(): + parser = argparse.ArgumentParser(description="Yocto Project test results helper") + subparsers = parser.add_subparsers( + help="Supported commands for test results helper", + required=True) + parser_regression_report = subparsers.add_parser( + "regression-report", + help="Generate regression report between two fixed revisions. Revisions can be branch name or tag") + parser_regression_report.add_argument( + 'base', + help="Revision or tag against which to compare results (i.e: the older)") + parser_regression_report.add_argument( + 'target', + help="Revision or tag to compare against the base (i.e: the newer)") + parser_regression_report.add_argument( + '-t', + '--testresultsdir', + help=f"An existing test results directory. {sys.argv[0]} will automatically clone it and use default branch if not provided") + parser_regression_report.add_argument( + '-l', + '--limit', + help=f"Maximum number of changes to display per test. Can be set to 0 to print all changes") + parser_regression_report.set_defaults(func=regression) + + args = parser.parse_args() + args.func(args) + +if __name__ == '__main__': + try: + ret = main() + except Exception: + ret = 1 + import traceback + traceback.print_exc() + sys.exit(ret) |