scripts: add new helper for regression report generation

Add yocto-testresults-query script. This is a thin wrapper over resulttool which
is able to translate tags or branch name to specific revisions, and then to work
with those "guessed" revisions with resulttool

Signed-off-by: Alexis Lothoré <alexis.lothore@bootlin.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 106 insertions, 0 deletions

diff --git a/scripts/yocto_testresults_query.py b/scripts/yocto_testresults_query.py
new file mode 100755
index 0000000000..3b478822dc
--- /dev/null
+++ b/scripts/yocto_testresults_query.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+# Yocto Project test results management tool
+# This script is an thin layer over resulttool to manage tes results and regression reports.
+# Its main feature is to translate tags or branch names to revisions SHA1, and then to run resulttool
+# with those computed revisions
+#
+# Copyright (C) 2023 OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+
+import sys
+import os
+import argparse
+import subprocess
+import tempfile
+import lib.scriptutils as scriptutils
+
+script_path = os.path.dirname(os.path.realpath(__file__))
+poky_path = os.path.abspath(os.path.join(script_path, ".."))
+resulttool = os.path.abspath(os.path.join(script_path, "resulttool"))
+logger = scriptutils.logger_create(sys.argv[0])
+testresults_default_url="git://git.yoctoproject.org/yocto-testresults"
+
+def create_workdir():
+    workdir = tempfile.mkdtemp(prefix='yocto-testresults-query.')
+    logger.info(f"Shallow-cloning testresults in {workdir}")
+    subprocess.check_call(["git", "clone", testresults_default_url, workdir, "--depth", "1"])
+    return workdir
+
+def get_sha1(pokydir, revision):
+    rev = subprocess.check_output(["git", "rev-list", "-n", "1", revision], cwd=pokydir).decode('utf-8').strip()
+    logger.info(f"SHA-1 revision for {revision} in {pokydir} is {rev}")
+    return rev
+
+def fetch_testresults(workdir, sha1):
+    logger.info(f"Fetching test results for {sha1} in {workdir}")
+    rawtags = subprocess.check_output(["git", "ls-remote", "--refs", "--tags", "origin", f"*{sha1}*"], cwd=workdir).decode('utf-8').strip()
+    if not rawtags:
+        raise Exception(f"No reference found for commit {sha1} in {workdir}")
+    for rev in [rawtag.split()[1] for rawtag in rawtags.splitlines()]:
+        logger.info(f"Fetching matching revisions: {rev}")
+        subprocess.check_call(["git", "fetch", "--depth", "1", "origin", f"{rev}:{rev}"], cwd=workdir)
+
+def compute_regression_report(workdir, baserevision, targetrevision):
+    logger.info(f"Running resulttool regression between SHA1 {baserevision} and {targetrevision}")
+    report = subprocess.check_output([resulttool, "regression-git", "--commit", baserevision, "--commit2", targetrevision, workdir]).decode("utf-8")
+    return report
+
+def print_report_with_header(report, baseversion, baserevision, targetversion, targetrevision):
+    print("========================== Regression report ==============================")
+    print(f'{"=> Target:": <16}{targetversion: <16}({targetrevision})')
+    print(f'{"=> Base:": <16}{baseversion: <16}({baserevision})')
+    print("===========================================================================\n")
+    print(report, end='')
+
+def regression(args):
+    logger.info(f"Compute regression report between {args.base} and {args.target}")
+    if args.testresultsdir:
+        workdir = args.testresultsdir
+    else:
+        workdir = create_workdir()
+
+    try:
+        baserevision = get_sha1(poky_path, args.base)
+        targetrevision = get_sha1(poky_path, args.target)
+        fetch_testresults(workdir, baserevision)
+        fetch_testresults(workdir, targetrevision)
+        report = compute_regression_report(workdir, baserevision, targetrevision)
+        print_report_with_header(report, args.base, baserevision, args.target, targetrevision)
+    finally:
+        if not args.testresultsdir:
+            subprocess.check_call(["rm", "-rf",  workdir])
+
+def main():
+    parser = argparse.ArgumentParser(description="Yocto Project test results helper")
+    subparsers = parser.add_subparsers(
+        help="Supported commands for test results helper",
+        required=True)
+    parser_regression_report = subparsers.add_parser(
+        "regression-report",
+        help="Generate regression report between two fixed revisions. Revisions can be branch name or tag")
+    parser_regression_report.add_argument(
+        'base',
+        help="Revision or tag against which to compare results (i.e: the older)")
+    parser_regression_report.add_argument(
+        'target',
+        help="Revision or tag to compare against the base (i.e: the newer)")
+    parser_regression_report.add_argument(
+        '-t',
+        '--testresultsdir',
+        help=f"An existing test results directory. {sys.argv[0]} will automatically clone it and use default branch if not provided")
+    parser_regression_report.set_defaults(func=regression)
+
+    args = parser.parse_args()
+    args.func(args)
+
+if __name__ == '__main__':
+    try:
+        ret =  main()
+    except Exception:
+        ret = 1
+        import traceback
+        traceback.print_exc()
+    sys.exit(ret)