summaryrefslogtreecommitdiffstats
path: root/scripts
diff options
context:
space:
mode:
authorAlexis Lothoré <alexis.lothore@bootlin.com>2023-07-21 13:02:06 +0200
committerSteve Sakoman <steve@sakoman.com>2023-08-09 16:17:50 -1000
commit7e393ea6e46060daf3242f2f7721f7f968945122 (patch)
treeeabe1a7a98d7d37c8dca150d25078a03baf01749 /scripts
parent4a9832c55044b894e933179e5c1b6a96634809c0 (diff)
downloadopenembedded-core-contrib-7e393ea6e46060daf3242f2f7721f7f968945122.tar.gz
scripts/resulttool: add mention about new detected tests
Some regression reports show a lot of "PASSED->None" transitions. When such big lot of identical transitions are observed, it could be that tests are now failing, but it could also be that some tests has been renamed. To detect such case, add a log in regression report to report the number of new tests (i.e: tests that are present in target results but not in base result). This new log also allows to know about newly added tests bases Signed-off-by: Alexis Lothoré <alexis.lothore@bootlin.com> Signed-off-by: Alexandre Belloni <alexandre.belloni@bootlin.com> (cherry picked from commit 01b5cefd07e01c7407bc663842b8a8d502358a6d) Signed-off-by: Steve Sakoman <steve@sakoman.com>
Diffstat (limited to 'scripts')
-rw-r--r--scripts/lib/resulttool/regression.py16
1 files changed, 14 insertions, 2 deletions
diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py
index 1facbcd85e1..f80a9182a9a 100644
--- a/scripts/lib/resulttool/regression.py
+++ b/scripts/lib/resulttool/regression.py
@@ -178,6 +178,8 @@ def compare_result(logger, base_name, target_name, base_result, target_result):
base_result = base_result.get('result')
target_result = target_result.get('result')
result = {}
+ new_tests = 0
+
if base_result and target_result:
for k in base_result:
base_testcase = base_result[k]
@@ -189,6 +191,13 @@ def compare_result(logger, base_name, target_name, base_result, target_result):
result[k] = {'base': base_status, 'target': target_status}
else:
logger.error('Failed to retrieved base test case status: %s' % k)
+
+ # Also count new tests that were not present in base results: it
+ # could be newly added tests, but it could also highlights some tests
+ # renames or fixed faulty ptests
+ for k in target_result:
+ if k not in base_result:
+ new_tests += 1
if result:
new_pass_count = sum(test['target'] is not None and test['target'].startswith("PASS") for test in result.values())
# Print a regression report only if at least one test has a regression status (FAIL, SKIPPED, absent...)
@@ -200,10 +209,13 @@ def compare_result(logger, base_name, target_name, base_result, target_result):
if new_pass_count > 0:
resultstring += f' Additionally, {new_pass_count} previously failing test(s) is/are now passing\n'
else:
- resultstring = "Improvement: %s\n %s\n (+%d test(s) passing)" % (base_name, target_name, new_pass_count)
+ resultstring = "Improvement: %s\n %s\n (+%d test(s) passing)\n" % (base_name, target_name, new_pass_count)
result = None
else:
- resultstring = "Match: %s\n %s" % (base_name, target_name)
+ resultstring = "Match: %s\n %s\n" % (base_name, target_name)
+
+ if new_tests > 0:
+ resultstring += f' Additionally, {new_tests} new test(s) is/are present\n'
return result, resultstring
def get_results(logger, source):