diff options
Diffstat (limited to 'meta/lib/oeqa/core/runner.py')
| -rw-r--r-- | meta/lib/oeqa/core/runner.py | 105 |
1 files changed, 85 insertions, 20 deletions
diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py index 930620ea19..a86a706bd9 100644 --- a/meta/lib/oeqa/core/runner.py +++ b/meta/lib/oeqa/core/runner.py @@ -43,6 +43,8 @@ class OETestResult(_TestResult): self.starttime = {} self.endtime = {} self.progressinfo = {} + self.extraresults = {} + self.shownmsg = [] # Inject into tc so that TestDepends decorator can see results tc.results = self @@ -73,6 +75,7 @@ class OETestResult(_TestResult): for (scase, msg) in getattr(self, t): if test.id() == scase.id(): self.tc.logger.info(str(msg)) + self.shownmsg.append(test.id()) break def logSummary(self, component, context_msg=''): @@ -129,19 +132,50 @@ class OETestResult(_TestResult): return 'UNKNOWN', None - def addSuccess(self, test): + def extractExtraResults(self, test, details = None): + extraresults = None + if details is not None and "extraresults" in details: + extraresults = details.get("extraresults", {}) + elif hasattr(test, "extraresults"): + extraresults = test.extraresults + + if extraresults is not None: + for k, v in extraresults.items(): + # handle updating already existing entries (e.g. ptestresults.sections) + if k in self.extraresults: + self.extraresults[k].update(v) + else: + self.extraresults[k] = v + + def addError(self, test, *args, details = None): + self.extractExtraResults(test, details = details) + return super(OETestResult, self).addError(test, *args) + + def addFailure(self, test, *args, details = None): + self.extractExtraResults(test, details = details) + return super(OETestResult, self).addFailure(test, *args) + + def addSuccess(self, test, details = None): #Added so we can keep track of successes too self.successes.append((test, None)) - super(OETestResult, self).addSuccess(test) + self.extractExtraResults(test, details = details) + return super(OETestResult, self).addSuccess(test) + + def addExpectedFailure(self, test, *args, details = None): + self.extractExtraResults(test, details = details) + return super(OETestResult, self).addExpectedFailure(test, *args) + + def addUnexpectedSuccess(self, test, details = None): + self.extractExtraResults(test, details = details) + return super(OETestResult, self).addUnexpectedSuccess(test) def logDetails(self, json_file_dir=None, configuration=None, result_id=None, dump_streams=False): - self.tc.logger.info("RESULTS:") - result = {} + result = self.extraresults logs = {} if hasattr(self.tc, "extraresults"): - result = self.tc.extraresults + result.update(self.tc.extraresults) for case_name in self.tc._registry['cases']: case = self.tc._registry['cases'][case_name] @@ -149,8 +183,10 @@ class OETestResult(_TestResult): (status, log) = self._getTestResultDetails(case) t = "" + duration = 0 if case.id() in self.starttime and case.id() in self.endtime: - t = " (" + "{0:.2f}".format(self.endtime[case.id()] - self.starttime[case.id()]) + "s)" + duration = self.endtime[case.id()] - self.starttime[case.id()] + t = " (" + "{0:.2f}".format(duration) + "s)" if status not in logs: logs[status] = [] @@ -158,12 +194,33 @@ class OETestResult(_TestResult): report = {'status': status} if log: report['log'] = log + # Class setup failures wouldn't enter stopTest so would never display + if case.id() not in self.shownmsg: + self.tc.logger.info("Failure (%s) for %s:\n" % (status, case.id()) + log) + + if duration: + report['duration'] = duration + + alltags = [] + # pull tags from the case class + if hasattr(case, "__oeqa_testtags"): + alltags.extend(getattr(case, "__oeqa_testtags")) + # pull tags from the method itself + test_name = case._testMethodName + if hasattr(case, test_name): + method = getattr(case, test_name) + if hasattr(method, "__oeqa_testtags"): + alltags.extend(getattr(method, "__oeqa_testtags")) + if alltags: + report['oetags'] = alltags + if dump_streams and case.id() in self.logged_output: (stdout, stderr) = self.logged_output[case.id()] report['stdout'] = stdout report['stderr'] = stderr result[case.id()] = report + self.tc.logger.info("RESULTS:") for i in ['PASSED', 'SKIPPED', 'EXPECTEDFAIL', 'ERROR', 'FAILED', 'UNKNOWN']: if i not in logs: continue @@ -178,6 +235,10 @@ class OETestResult(_TestResult): # Override as we unexpected successes aren't failures for us return (len(self.failures) == len(self.errors) == 0) + def hasAnyFailingTest(self): + # Account for expected failures + return not self.wasSuccessful() or len(self.expectedFailures) + class OEListTestsResult(object): def wasSuccessful(self): return True @@ -205,23 +266,20 @@ class OETestRunner(_TestRunner): self._walked_cases = self._walked_cases + 1 def _list_tests_name(self, suite): - from oeqa.core.decorator.oetag import OETestTag - self._walked_cases = 0 def _list_cases(logger, case): - oetag = None - - if hasattr(case, 'decorators'): - for d in case.decorators: - if isinstance(d, OETestTag): - oetag = d.oetag - - logger.info("%s\t\t%s" % (oetag, case.id())) + oetags = [] + if hasattr(case, '__oeqa_testtags'): + oetags = getattr(case, '__oeqa_testtags') + if oetags: + logger.info("%s (%s)" % (case.id(), ",".join(oetags))) + else: + logger.info("%s" % (case.id())) self.tc.logger.info("Listing all available tests:") self._walked_cases = 0 - self.tc.logger.info("id\ttag\t\ttest") + self.tc.logger.info("test (tags)") self.tc.logger.info("-" * 80) self._walk_suite(suite, _list_cases) self.tc.logger.info("-" * 80) @@ -289,10 +347,17 @@ class OETestResultJSONHelper(object): the_file.write(file_content) def dump_testresult_file(self, write_dir, configuration, result_id, test_result): - bb.utils.mkdirhier(write_dir) - lf = bb.utils.lockfile(os.path.join(write_dir, 'jsontestresult.lock')) + try: + import bb + has_bb = True + bb.utils.mkdirhier(write_dir) + lf = bb.utils.lockfile(os.path.join(write_dir, 'jsontestresult.lock')) + except ImportError: + has_bb = False + os.makedirs(write_dir, exist_ok=True) test_results = self._get_existing_testresults_if_available(write_dir) test_results[result_id] = {'configuration': configuration, 'result': test_result} json_testresults = json.dumps(test_results, sort_keys=True, indent=4) self._write_file(write_dir, self.testresult_filename, json_testresults) - bb.utils.unlockfile(lf) + if has_bb: + bb.utils.unlockfile(lf) |