diff options
Diffstat (limited to 'meta/lib/oeqa/core')
36 files changed, 1213 insertions, 898 deletions
diff --git a/meta/lib/oeqa/core/case.py b/meta/lib/oeqa/core/case.py index 917a2aa3f8..bc4446a938 100644 --- a/meta/lib/oeqa/core/case.py +++ b/meta/lib/oeqa/core/case.py @@ -1,6 +1,11 @@ +# # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# +import base64 +import zlib import unittest from oeqa.core.exception import OEQAMissingVariable @@ -29,6 +34,8 @@ class OETestCase(unittest.TestCase): @classmethod def _oeSetUpClass(clss): _validate_td_vars(clss.td, clss.td_vars, "class") + if hasattr(clss, 'setUpHooker') and callable(getattr(clss, 'setUpHooker')): + clss.setUpHooker() clss.setUpClassMethod() @classmethod @@ -36,11 +43,63 @@ class OETestCase(unittest.TestCase): clss.tearDownClassMethod() def _oeSetUp(self): - for d in self.decorators: - d.setUpDecorator() + try: + for d in self.decorators: + d.setUpDecorator() + except: + for d in self.decorators: + d.tearDownDecorator() + raise self.setUpMethod() def _oeTearDown(self): for d in self.decorators: d.tearDownDecorator() self.tearDownMethod() + +class OEPTestResultTestCase: + """ + Mix-in class to provide functions to make interacting with extraresults for + the purposes of storing ptestresult data. + """ + @staticmethod + def _compress_log(log): + logdata = log.encode("utf-8") if isinstance(log, str) else log + logdata = zlib.compress(logdata) + logdata = base64.b64encode(logdata).decode("utf-8") + return {"compressed" : logdata} + + def ptest_rawlog(self, log): + if not hasattr(self, "extraresults"): + self.extraresults = {"ptestresult.sections" : {}} + self.extraresults["ptestresult.rawlogs"] = {"log" : self._compress_log(log)} + + def ptest_section(self, section, duration = None, log = None, logfile = None, exitcode = None): + if not hasattr(self, "extraresults"): + self.extraresults = {"ptestresult.sections" : {}} + + sections = self.extraresults.get("ptestresult.sections") + if section not in sections: + sections[section] = {} + + if log is not None: + sections[section]["log"] = self._compress_log(log) + elif logfile is not None: + with open(logfile, "rb") as f: + sections[section]["log"] = self._compress_log(f.read()) + + if duration is not None: + sections[section]["duration"] = duration + if exitcode is not None: + sections[section]["exitcode"] = exitcode + + def ptest_result(self, section, test, result): + if not hasattr(self, "extraresults"): + self.extraresults = {"ptestresult.sections" : {}} + + sections = self.extraresults.get("ptestresult.sections") + if section not in sections: + sections[section] = {} + resultname = "ptestresult.{}.{}".format(section, test) + self.extraresults[resultname] = {"status" : result} + diff --git a/meta/lib/oeqa/core/cases/example/test_basic.py b/meta/lib/oeqa/core/cases/example/test_basic.py index 11cf3800cc..d77edcdcec 100644 --- a/meta/lib/oeqa/core/cases/example/test_basic.py +++ b/meta/lib/oeqa/core/cases/example/test_basic.py @@ -1,5 +1,7 @@ # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# from oeqa.core.case import OETestCase from oeqa.core.decorator.depends import OETestDepends diff --git a/meta/lib/oeqa/core/context.py b/meta/lib/oeqa/core/context.py index acd547416f..9313271f58 100644 --- a/meta/lib/oeqa/core/context.py +++ b/meta/lib/oeqa/core/context.py @@ -1,5 +1,7 @@ -# Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +## Copyright (C) 2016 Intel Corporation +# +# SPDX-License-Identifier: MIT +# import os import sys @@ -7,6 +9,7 @@ import json import time import logging import collections +import unittest from oeqa.core.loader import OETestLoader from oeqa.core.runner import OETestRunner @@ -27,7 +30,9 @@ class OETestContext(object): self.logger = logger self._registry = {} self._registry['cases'] = collections.OrderedDict() - self._results = {} + + self.results = unittest.TestResult() + unittest.registerResult(self.results) def _read_modules_from_manifest(self, manifest): if not os.path.exists(manifest): @@ -44,28 +49,46 @@ class OETestContext(object): def skipTests(self, skips): if not skips: return + def skipfuncgen(skipmsg): + def func(): + raise unittest.SkipTest(skipmsg) + return func + class_ids = {} for test in self.suites: + if test.__class__ not in class_ids: + class_ids[test.__class__] = '.'.join(test.id().split('.')[:-1]) + for skip in skips: + if (test.id()+'.').startswith(skip+'.'): + setattr(test, 'setUp', skipfuncgen('Skip by the command line argument "%s"' % skip)) + for tclass in class_ids: + cid = class_ids[tclass] for skip in skips: - if test.id().startswith(skip): - setattr(test, 'setUp', lambda: test.skipTest('Skip by the command line argument "%s"' % skip)) + if (cid + '.').startswith(skip + '.'): + setattr(tclass, 'setUpHooker', skipfuncgen('Skip by the command line argument "%s"' % skip)) def loadTests(self, module_paths, modules=[], tests=[], - modules_manifest="", modules_required=[], filters={}): + modules_manifest="", modules_required=[], **kwargs): if modules_manifest: modules = self._read_modules_from_manifest(modules_manifest) self.loader = self.loaderClass(self, module_paths, modules, tests, - modules_required, filters) + modules_required, **kwargs) self.suites = self.loader.discover() - def runTests(self, skips=[]): + def prepareSuite(self, suites, processes): + return suites + + def runTests(self, processes=None, skips=[]): self.runner = self.runnerClass(self, descriptions=False, verbosity=2) - # Dinamically skip those tests specified though arguments + # Dynamically skip those tests specified though arguments self.skipTests(skips) self._run_start_time = time.time() - result = self.runner.run(self.suites) + self._run_end_time = self._run_start_time + if not processes: + self.runner.buffer = True + result = self.runner.run(self.prepareSuite(self.suites, processes)) self._run_end_time = time.time() return result @@ -81,22 +104,27 @@ class OETestContextExecutor(object): name = 'core' help = 'core test component example' description = 'executes core test suite example' + datetime = time.strftime("%Y%m%d%H%M%S") default_cases = [os.path.join(os.path.abspath(os.path.dirname(__file__)), 'cases/example')] default_test_data = os.path.join(default_cases[0], 'data.json') default_tests = None + default_json_result_dir = None def register_commands(self, logger, subparsers): self.parser = subparsers.add_parser(self.name, help=self.help, description=self.description, group='components') - self.default_output_log = '%s-results-%s.log' % (self.name, - time.strftime("%Y%m%d%H%M%S")) + self.default_output_log = '%s-results-%s.log' % (self.name, self.datetime) self.parser.add_argument('--output-log', action='store', default=self.default_output_log, help="results output log, default: %s" % self.default_output_log) + self.parser.add_argument('--json-result-dir', action='store', + default=self.default_json_result_dir, + help="json result output dir, default: %s" % self.default_json_result_dir) + group = self.parser.add_mutually_exclusive_group() group.add_argument('--run-tests', action='store', nargs='+', default=self.default_tests, @@ -132,6 +160,8 @@ class OETestContextExecutor(object): fh = logging.FileHandler(args.output_log) fh.setFormatter(formatter) logger.addHandler(fh) + if getattr(args, 'verbose', False): + logger.setLevel('DEBUG') return logger @@ -159,6 +189,22 @@ class OETestContextExecutor(object): self.module_paths = args.CASES_PATHS + def _get_json_result_dir(self, args): + return args.json_result_dir + + def _get_configuration(self): + td = self.tc_kwargs['init']['td'] + configuration = {'TEST_TYPE': self.name, + 'MACHINE': td.get("MACHINE"), + 'DISTRO': td.get("DISTRO"), + 'IMAGE_BASENAME': td.get("IMAGE_BASENAME"), + 'DATETIME': td.get("DATETIME")} + return configuration + + def _get_result_id(self, configuration): + return '%s_%s_%s_%s' % (configuration['TEST_TYPE'], configuration['IMAGE_BASENAME'], + configuration['MACHINE'], self.datetime) + def _pre_run(self): pass @@ -177,7 +223,16 @@ class OETestContextExecutor(object): else: self._pre_run() rc = self.tc.runTests(**self.tc_kwargs['run']) - rc.logDetails() + + json_result_dir = self._get_json_result_dir(args) + if json_result_dir: + configuration = self._get_configuration() + rc.logDetails(json_result_dir, + configuration, + self._get_result_id(configuration)) + else: + rc.logDetails() + rc.logSummary(self.name) output_link = os.path.join(os.path.dirname(args.output_log), diff --git a/meta/lib/oeqa/core/decorator/__init__.py b/meta/lib/oeqa/core/decorator/__init__.py index 855b6b9d28..93efd30e1d 100644 --- a/meta/lib/oeqa/core/decorator/__init__.py +++ b/meta/lib/oeqa/core/decorator/__init__.py @@ -1,16 +1,19 @@ +# # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# from functools import wraps -from abc import abstractmethod +from abc import ABCMeta decoratorClasses = set() -def registerDecorator(obj): - decoratorClasses.add(obj) - return obj +def registerDecorator(cls): + decoratorClasses.add(cls) + return cls -class OETestDecorator(object): +class OETestDecorator(object, metaclass=ABCMeta): case = None # Reference of OETestCase decorated attrs = None # Attributes to be loaded by decorator implementation @@ -60,12 +63,12 @@ class OETestDiscover(OETestDecorator): def discover(registry): return registry['cases'] -class OETestFilter(OETestDecorator): - - # OETestLoader call it while loading the tests - # in loadTestsFromTestCase method, it needs to - # return a bool, True if needs to be filtered. - # This method must consume the filter used. - @abstractmethod - def filtrate(self, filters): - return False +def OETestTag(*tags): + def decorator(item): + if hasattr(item, "__oeqa_testtags"): + # do not append, create a new list (to handle classes with inheritance) + item.__oeqa_testtags = list(item.__oeqa_testtags) + list(tags) + else: + item.__oeqa_testtags = tags + return item + return decorator diff --git a/meta/lib/oeqa/core/decorator/data.py b/meta/lib/oeqa/core/decorator/data.py index ff7bdd98b7..5444b2cb75 100644 --- a/meta/lib/oeqa/core/decorator/data.py +++ b/meta/lib/oeqa/core/decorator/data.py @@ -1,5 +1,8 @@ +# # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# from oeqa.core.exception import OEQAMissingVariable @@ -10,8 +13,17 @@ def has_feature(td, feature): Checks for feature in DISTRO_FEATURES or IMAGE_FEATURES. """ - if (feature in td.get('DISTRO_FEATURES', '') or - feature in td.get('IMAGE_FEATURES', '')): + if (feature in td.get('DISTRO_FEATURES', '').split() or + feature in td.get('IMAGE_FEATURES', '').split()): + return True + return False + +def has_machine(td, machine): + """ + Checks for MACHINE. + """ + + if (machine == td.get('MACHINE', '')): return True return False @@ -54,6 +66,20 @@ class skipIfNotDataVar(OETestDecorator): self.case.skipTest(self.msg) @registerDecorator +class skipIfInDataVar(OETestDecorator): + """ + Skip test if value is in data store's variable. + """ + + attrs = ('var', 'value', 'msg') + def setUpDecorator(self): + msg = ('Checking if %r value contains %r to skip ' + 'the test' % (self.var, self.value)) + self.logger.debug(msg) + if self.value in (self.case.td.get(self.var)): + self.case.skipTest(self.msg) + +@registerDecorator class skipIfNotInDataVar(OETestDecorator): """ Skip test if value is not in data store's variable. @@ -61,10 +87,10 @@ class skipIfNotInDataVar(OETestDecorator): attrs = ('var', 'value', 'msg') def setUpDecorator(self): - msg = ('Checking if %r value is in %r to run ' + msg = ('Checking if %r value contains %r to run ' 'the test' % (self.var, self.value)) self.logger.debug(msg) - if not self.value in self.case.td.get(self.var): + if not self.value in (self.case.td.get(self.var) or ""): self.case.skipTest(self.msg) @registerDecorator @@ -96,3 +122,109 @@ class skipIfNotFeature(OETestDecorator): self.logger.debug(msg) if not has_feature(self.case.td, self.value): self.case.skipTest(self.msg) + +@registerDecorator +class skipIfFeature(OETestDecorator): + """ + Skip test based on DISTRO_FEATURES. + + value must not be in distro features or it will skip the test + with msg as the reason. + """ + + attrs = ('value', 'msg') + + def setUpDecorator(self): + msg = ('Checking if %s is not in DISTRO_FEATURES ' + 'or IMAGE_FEATURES' % (self.value)) + self.logger.debug(msg) + if has_feature(self.case.td, self.value): + self.case.skipTest(self.msg) + +@registerDecorator +class skipIfNotMachine(OETestDecorator): + """ + Skip test based on MACHINE. + + value must be match MACHINE or it will skip the test + with msg as the reason. + """ + + attrs = ('value', 'msg') + + def setUpDecorator(self): + msg = ('Checking if %s is not this MACHINE' % self.value) + self.logger.debug(msg) + if not has_machine(self.case.td, self.value): + self.case.skipTest(self.msg) + +@registerDecorator +class skipIfMachine(OETestDecorator): + """ + Skip test based on Machine. + + value must not be this machine or it will skip the test + with msg as the reason. + """ + + attrs = ('value', 'msg') + + def setUpDecorator(self): + msg = ('Checking if %s is this MACHINE' % self.value) + self.logger.debug(msg) + if has_machine(self.case.td, self.value): + self.case.skipTest(self.msg) + +@registerDecorator +class skipIfNotQemu(OETestDecorator): + """ + Skip test if MACHINE is not qemu* + """ + def setUpDecorator(self): + self.logger.debug("Checking if not qemu MACHINE") + if not self.case.td.get('MACHINE', '').startswith('qemu'): + self.case.skipTest('Test only runs on qemu machines') + +@registerDecorator +class skipIfNotQemuUsermode(OETestDecorator): + """ + Skip test if MACHINE_FEATURES does not contain qemu-usermode + """ + def setUpDecorator(self): + self.logger.debug("Checking if MACHINE_FEATURES does not contain qemu-usermode") + if 'qemu-usermode' not in self.case.td.get('MACHINE_FEATURES', '').split(): + self.case.skipTest('Test requires qemu-usermode in MACHINE_FEATURES') + +@registerDecorator +class skipIfQemu(OETestDecorator): + """ + Skip test if MACHINE is qemu* + """ + def setUpDecorator(self): + self.logger.debug("Checking if qemu MACHINE") + if self.case.td.get('MACHINE', '').startswith('qemu'): + self.case.skipTest('Test only runs on real hardware') + +@registerDecorator +class skipIfArch(OETestDecorator): + """ + Skip test if HOST_ARCH is present in the tuple specified. + """ + + attrs = ('archs',) + def setUpDecorator(self): + arch = self.case.td['HOST_ARCH'] + if arch in self.archs: + self.case.skipTest('Test skipped on %s' % arch) + +@registerDecorator +class skipIfNotArch(OETestDecorator): + """ + Skip test if HOST_ARCH is not present in the tuple specified. + """ + + attrs = ('archs',) + def setUpDecorator(self): + arch = self.case.td['HOST_ARCH'] + if arch not in self.archs: + self.case.skipTest('Test skipped on %s' % arch) diff --git a/meta/lib/oeqa/core/decorator/depends.py b/meta/lib/oeqa/core/decorator/depends.py index baa04341c7..33f0841cab 100644 --- a/meta/lib/oeqa/core/decorator/depends.py +++ b/meta/lib/oeqa/core/decorator/depends.py @@ -1,9 +1,11 @@ +# # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# from unittest import SkipTest -from oeqa.core.threaded import OETestRunnerThreaded from oeqa.core.exception import OEQADependency from . import OETestDiscover, registerDecorator @@ -64,19 +66,15 @@ def _order_test_case_by_depends(cases, depends): return [cases[case_id] for case_id in cases_ordered] def _skipTestDependency(case, depends): - if isinstance(case.tc.runner, OETestRunnerThreaded): - import threading - results = case.tc._results[threading.get_ident()] - else: - results = case.tc._results - - skipReasons = ['errors', 'failures', 'skipped'] - - for reason in skipReasons: - for test, _ in results[reason]: - if test.id() in depends: - raise SkipTest("Test case %s depends on %s and was in %s." \ - % (case.id(), test.id(), reason)) + for dep in depends: + found = False + for test, _ in case.tc.results.successes: + if test.id() == dep: + found = True + break + if not found: + raise SkipTest("Test case %s depends on %s but it didn't pass/run." \ + % (case.id(), dep)) @registerDecorator class OETestDepends(OETestDiscover): diff --git a/meta/lib/oeqa/core/decorator/oeid.py b/meta/lib/oeqa/core/decorator/oeid.py deleted file mode 100644 index ea8017a55a..0000000000 --- a/meta/lib/oeqa/core/decorator/oeid.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) - -from . import OETestFilter, registerDecorator -from oeqa.core.utils.misc import intToList - -def _idFilter(oeid, filters): - return False if oeid in filters else True - -@registerDecorator -class OETestID(OETestFilter): - attrs = ('oeid',) - - def bind(self, registry, case): - super(OETestID, self).bind(registry, case) - - def filtrate(self, filters): - if filters.get('oeid'): - filterx = intToList(filters['oeid'], 'oeid') - del filters['oeid'] - if _idFilter(self.oeid, filterx): - return True - return False diff --git a/meta/lib/oeqa/core/decorator/oetag.py b/meta/lib/oeqa/core/decorator/oetag.py deleted file mode 100644 index ad38ab78a5..0000000000 --- a/meta/lib/oeqa/core/decorator/oetag.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) - -from . import OETestFilter, registerDecorator -from oeqa.core.utils.misc import strToList - -def _tagFilter(tags, filters): - return False if set(tags) & set(filters) else True - -@registerDecorator -class OETestTag(OETestFilter): - attrs = ('oetag',) - - def bind(self, registry, case): - super(OETestTag, self).bind(registry, case) - self.oetag = strToList(self.oetag, 'oetag') - - def filtrate(self, filters): - if filters.get('oetag'): - filterx = strToList(filters['oetag'], 'oetag') - del filters['oetag'] - if _tagFilter(self.oetag, filterx): - return True - return False diff --git a/meta/lib/oeqa/core/decorator/oetimeout.py b/meta/lib/oeqa/core/decorator/oetimeout.py index f85e7d9792..5e6873ad48 100644 --- a/meta/lib/oeqa/core/decorator/oetimeout.py +++ b/meta/lib/oeqa/core/decorator/oetimeout.py @@ -1,12 +1,11 @@ +# # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) - -from . import OETestDecorator, registerDecorator +# +# SPDX-License-Identifier: MIT +# import signal -from threading import Timer - -from oeqa.core.threaded import OETestRunnerThreaded +from . import OETestDecorator, registerDecorator from oeqa.core.exception import OEQATimeoutError @registerDecorator @@ -14,32 +13,17 @@ class OETimeout(OETestDecorator): attrs = ('oetimeout',) def setUpDecorator(self): - self.logger.debug("Setting up a %d second(s) timeout" % self.oetimeout) - - if isinstance(self.case.tc.runner, OETestRunnerThreaded): - self.timeouted = False - def _timeoutHandler(): - self.timeouted = True - - self.timer = Timer(self.oetimeout, _timeoutHandler) - self.timer.start() - else: - timeout = self.oetimeout - def _timeoutHandler(signum, frame): - raise OEQATimeoutError("Timed out after %s " + timeout = self.oetimeout + def _timeoutHandler(signum, frame): + raise OEQATimeoutError("Timed out after %s " "seconds of execution" % timeout) - self.alarmSignal = signal.signal(signal.SIGALRM, _timeoutHandler) - signal.alarm(self.oetimeout) + self.logger.debug("Setting up a %d second(s) timeout" % self.oetimeout) + self.alarmSignal = signal.signal(signal.SIGALRM, _timeoutHandler) + signal.alarm(self.oetimeout) def tearDownDecorator(self): - if isinstance(self.case.tc.runner, OETestRunnerThreaded): - self.timer.cancel() - self.logger.debug("Removed Timer handler") - if self.timeouted: - raise OEQATimeoutError("Timed out after %s " - "seconds of execution" % self.oetimeout) - else: - signal.alarm(0) + signal.alarm(0) + if hasattr(self, 'alarmSignal'): signal.signal(signal.SIGALRM, self.alarmSignal) self.logger.debug("Removed SIGALRM handler") diff --git a/meta/lib/oeqa/core/exception.py b/meta/lib/oeqa/core/exception.py index 732f2efdeb..05be0ed21f 100644 --- a/meta/lib/oeqa/core/exception.py +++ b/meta/lib/oeqa/core/exception.py @@ -1,5 +1,8 @@ +# # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# class OEQAException(Exception): pass diff --git a/meta/lib/oeqa/core/loader.py b/meta/lib/oeqa/core/loader.py index 975a081ba4..d12d5a055c 100644 --- a/meta/lib/oeqa/core/loader.py +++ b/meta/lib/oeqa/core/loader.py @@ -1,5 +1,8 @@ +# # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# import os import re @@ -13,7 +16,7 @@ from oeqa.core.utils.test import getSuiteModules, getCaseID from oeqa.core.exception import OEQATestNotFound from oeqa.core.case import OETestCase from oeqa.core.decorator import decoratorClasses, OETestDecorator, \ - OETestFilter, OETestDiscover + OETestDiscover # When loading tests, the unittest framework stores any exceptions and # displays them only when the run method is called. @@ -24,7 +27,7 @@ from oeqa.core.decorator import decoratorClasses, OETestDecorator, \ # Generate the function definition because this differ across python versions # Python >= 3.4.4 uses tree parameters instead four but for example Python 3.5.3 # ueses four parameters so isn't incremental. -_failed_test_args = inspect.getargspec(unittest.loader._make_failed_test).args +_failed_test_args = inspect.getfullargspec(unittest.loader._make_failed_test).args exec("""def _make_failed_test(%s): raise exception""" % ', '.join(_failed_test_args)) unittest.loader._make_failed_test = _make_failed_test @@ -34,7 +37,7 @@ def _find_duplicated_modules(suite, directory): if path: raise ImportError("Duplicated %s module found in %s" % (module, path)) -def _built_modules_dict(modules): +def _built_modules_dict(modules, logger): modules_dict = {} if modules == None: @@ -43,7 +46,12 @@ def _built_modules_dict(modules): for module in modules: # Assumption: package and module names do not contain upper case # characters, whereas class names do - m = re.match(r'^([^A-Z]+)(?:\.([A-Z][^.]*)(?:\.([^.]+))?)?$', module) + m = re.match(r'^([0-9a-z_.]+)(?:\.(\w[^.]*)(?:\.([^.]+))?)?$', module, flags=re.ASCII) + if not m: + logger.warn("module '%s' was skipped from selected modules, "\ + "because it doesn't match with module name assumptions: "\ + "package and module names do not contain upper case characters, whereas class names do" % module) + continue module_name, class_name, test_name = m.groups() @@ -53,6 +61,8 @@ def _built_modules_dict(modules): modules_dict[module_name][class_name] = [] if test_name and test_name not in modules_dict[module_name][class_name]: modules_dict[module_name][class_name].append(test_name) + if modules and not modules_dict: + raise OEQATestNotFound("All selected modules were skipped, this would trigger selftest with all tests and -r ignored.") return modules_dict @@ -63,21 +73,15 @@ class OETestLoader(unittest.TestLoader): '_top_level_dir'] def __init__(self, tc, module_paths, modules, tests, modules_required, - filters, *args, **kwargs): + *args, **kwargs): self.tc = tc - self.modules = _built_modules_dict(modules) + self.modules = _built_modules_dict(modules, tc.logger) self.tests = tests self.modules_required = modules_required - self.filters = filters - self.decorator_filters = [d for d in decoratorClasses if \ - issubclass(d, OETestFilter)] - self._validateFilters(self.filters, self.decorator_filters) - self.used_filters = [d for d in self.decorator_filters - for f in self.filters - if f in d.attrs] + self.tags_filter = kwargs.get("tags_filter", None) if isinstance(module_paths, str): module_paths = [module_paths] @@ -99,28 +103,6 @@ class OETestLoader(unittest.TestLoader): setattr(testCaseClass, 'td', self.tc.td) setattr(testCaseClass, 'logger', self.tc.logger) - def _validateFilters(self, filters, decorator_filters): - # Validate if filter isn't empty - for key,value in filters.items(): - if not value: - raise TypeError("Filter %s specified is empty" % key) - - # Validate unique attributes - attr_filters = [attr for clss in decorator_filters \ - for attr in clss.attrs] - dup_attr = [attr for attr in attr_filters - if attr_filters.count(attr) > 1] - if dup_attr: - raise TypeError('Detected duplicated attribute(s) %s in filter' - ' decorators' % ' ,'.join(dup_attr)) - - # Validate if filter is supported - for f in filters: - if f not in attr_filters: - classes = ', '.join([d.__name__ for d in decorator_filters]) - raise TypeError('Found "%s" filter but not declared in any of ' - '%s decorators' % (f, classes)) - def _registerTestCase(self, case): case_id = case.id() self.tc._registry['cases'][case_id] = case @@ -155,7 +137,16 @@ class OETestLoader(unittest.TestLoader): class_name = case.__class__.__name__ test_name = case._testMethodName - if self.modules: + # 'auto' is a reserved key word to run test cases automatically + # warn users if their test case belong to a module named 'auto' + if module_name_small == "auto": + bb.warn("'auto' is a reserved key word for TEST_SUITES. " + "But test case '%s' is detected to belong to auto module. " + "Please condier using a new name for your module." % str(case)) + + # check if case belongs to any specified module + # if 'auto' is specified, such check is skipped + if self.modules and not 'auto' in self.modules: module = None try: module = self.modules[module_name_small] @@ -174,19 +165,20 @@ class OETestLoader(unittest.TestLoader): return True # Decorator filters - if self.filters and isinstance(case, OETestCase): - filters = self.filters.copy() - case_decorators = [cd for cd in case.decorators - if cd.__class__ in self.used_filters] - - # Iterate over case decorators to check if needs to be filtered. - for cd in case_decorators: - if cd.filtrate(filters): - return True - - # Case is missing one or more decorators for all the filters - # being used, so filter test case. - if filters: + if self.tags_filter is not None and callable(self.tags_filter): + alltags = set() + # pull tags from the case class + if hasattr(case, "__oeqa_testtags"): + for t in getattr(case, "__oeqa_testtags"): + alltags.add(t) + # pull tags from the method itself + if hasattr(case, test_name): + method = getattr(case, test_name) + if hasattr(method, "__oeqa_testtags"): + for t in getattr(method, "__oeqa_testtags"): + alltags.add(t) + + if self.tags_filter(alltags): return True return False @@ -245,7 +237,7 @@ class OETestLoader(unittest.TestLoader): for tcName in testCaseNames: case = self._getTestCase(testCaseClass, tcName) # Filer by case id - if not (self.tests and not 'all' in self.tests + if not (self.tests and not 'auto' in self.tests and not getCaseID(case) in self.tests): self._handleTestCaseDecorators(case) @@ -309,14 +301,14 @@ class OETestLoader(unittest.TestLoader): module_name = module.__name__ # Normal test modules are loaded if no modules were specified, - # if module is in the specified module list or if 'all' is in + # if module is in the specified module list or if 'auto' is in # module list. # Underscore modules are loaded only if specified in module list. load_module = True if not module_name.startswith('_') \ and (not self.modules \ or module_name in self.modules \ or module_name_small in self.modules \ - or 'all' in self.modules) \ + or 'auto' in self.modules) \ else False load_underscore = True if module_name.startswith('_') \ @@ -324,6 +316,9 @@ class OETestLoader(unittest.TestLoader): module_name_small in self.modules) \ else False + if any(c.isupper() for c in module.__name__): + raise SystemExit("Module '%s' contains uppercase characters and this isn't supported. Please fix the module name." % module.__name__) + return (load_module, load_underscore) diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py index 13cdf5ba52..a86a706bd9 100644 --- a/meta/lib/oeqa/core/runner.py +++ b/meta/lib/oeqa/core/runner.py @@ -1,22 +1,19 @@ +# # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# import os import time import unittest import logging import re +import json +import sys -xmlEnabled = False -try: - import xmlrunner - from xmlrunner.result import _XMLTestResult as _TestResult - from xmlrunner.runner import XMLTestRunner as _TestRunner - xmlEnabled = True -except ImportError: - # use the base runner instead - from unittest import TextTestResult as _TestResult - from unittest import TextTestRunner as _TestRunner +from unittest import TextTestResult as _TestResult +from unittest import TextTestRunner as _TestRunner class OEStreamLogger(object): def __init__(self, logger): @@ -42,22 +39,44 @@ class OETestResult(_TestResult): def __init__(self, tc, *args, **kwargs): super(OETestResult, self).__init__(*args, **kwargs) + self.successes = [] + self.starttime = {} + self.endtime = {} + self.progressinfo = {} + self.extraresults = {} + self.shownmsg = [] + + # Inject into tc so that TestDepends decorator can see results + tc.results = self + self.tc = tc - self._tc_map_results() + + # stdout and stderr for each test case + self.logged_output = {} def startTest(self, test): - # Allow us to trigger the testcase buffer mode on a per test basis - # so stdout/stderr are only printed upon failure. Enables debugging - # but clean output - if hasattr(test, "buffer"): - self.buffer = test.buffer + # May have been set by concurrencytest + if test.id() not in self.starttime: + self.starttime[test.id()] = time.time() super(OETestResult, self).startTest(test) - def _tc_map_results(self): - self.tc._results['failures'] = self.failures - self.tc._results['errors'] = self.errors - self.tc._results['skipped'] = self.skipped - self.tc._results['expectedFailures'] = self.expectedFailures + def stopTest(self, test): + self.endtime[test.id()] = time.time() + if self.buffer: + self.logged_output[test.id()] = ( + sys.stdout.getvalue(), sys.stderr.getvalue()) + super(OETestResult, self).stopTest(test) + if test.id() in self.progressinfo: + self.tc.logger.info(self.progressinfo[test.id()]) + + # Print the errors/failures early to aid/speed debugging, its a pain + # to wait until selftest finishes to see them. + for t in ['failures', 'errors', 'skipped', 'expectedFailures']: + for (scase, msg) in getattr(self, t): + if test.id() == scase.id(): + self.tc.logger.info(str(msg)) + self.shownmsg.append(test.id()) + break def logSummary(self, component, context_msg=''): elapsed_time = self.tc._run_end_time - self.tc._run_start_time @@ -70,80 +89,155 @@ class OETestResult(_TestResult): msg = "%s - OK - All required tests passed" % component else: msg = "%s - FAIL - Required tests failed" % component - skipped = len(self.tc._results['skipped']) - if skipped: - msg += " (skipped=%d)" % skipped + msg += " (successes=%d, skipped=%d, failures=%d, errors=%d)" % (len(self.successes), len(self.skipped), len(self.failures), len(self.errors)) self.tc.logger.info(msg) - def _getDetailsNotPassed(self, case, type, desc): - found = False - - for (scase, msg) in self.tc._results[type]: - # XXX: When XML reporting is enabled scase is - # xmlrunner.result._TestInfo instance instead of - # string. - if xmlEnabled: - if case.id() == scase.test_id: - found = True - break - scase_str = scase.test_id - else: - if case == scase: + def _getTestResultDetails(self, case): + result_types = {'failures': 'FAILED', 'errors': 'ERROR', 'skipped': 'SKIPPED', + 'expectedFailures': 'EXPECTEDFAIL', 'successes': 'PASSED', + 'unexpectedSuccesses' : 'PASSED'} + + for rtype in result_types: + found = False + for resultclass in getattr(self, rtype): + # unexpectedSuccesses are just lists, not lists of tuples + if isinstance(resultclass, tuple): + scase, msg = resultclass + else: + scase, msg = resultclass, None + if case.id() == scase.id(): found = True break - scase_str = str(scase) + scase_str = str(scase.id()) + + # When fails at module or class level the class name is passed as string + # so figure out to see if match + m = re.search(r"^setUpModule \((?P<module_name>.*)\).*$", scase_str) + if m: + if case.__class__.__module__ == m.group('module_name'): + found = True + break + + m = re.search(r"^setUpClass \((?P<class_name>.*)\).*$", scase_str) + if m: + class_name = "%s.%s" % (case.__class__.__module__, + case.__class__.__name__) + + if class_name == m.group('class_name'): + found = True + break + + if found: + return result_types[rtype], msg + + return 'UNKNOWN', None + + def extractExtraResults(self, test, details = None): + extraresults = None + if details is not None and "extraresults" in details: + extraresults = details.get("extraresults", {}) + elif hasattr(test, "extraresults"): + extraresults = test.extraresults + + if extraresults is not None: + for k, v in extraresults.items(): + # handle updating already existing entries (e.g. ptestresults.sections) + if k in self.extraresults: + self.extraresults[k].update(v) + else: + self.extraresults[k] = v - # When fails at module or class level the class name is passed as string - # so figure out to see if match - m = re.search("^setUpModule \((?P<module_name>.*)\)$", scase_str) - if m: - if case.__class__.__module__ == m.group('module_name'): - found = True - break + def addError(self, test, *args, details = None): + self.extractExtraResults(test, details = details) + return super(OETestResult, self).addError(test, *args) - m = re.search("^setUpClass \((?P<class_name>.*)\)$", scase_str) - if m: - class_name = "%s.%s" % (case.__class__.__module__, - case.__class__.__name__) + def addFailure(self, test, *args, details = None): + self.extractExtraResults(test, details = details) + return super(OETestResult, self).addFailure(test, *args) - if class_name == m.group('class_name'): - found = True - break + def addSuccess(self, test, details = None): + #Added so we can keep track of successes too + self.successes.append((test, None)) + self.extractExtraResults(test, details = details) + return super(OETestResult, self).addSuccess(test) - if found: - return (found, msg) + def addExpectedFailure(self, test, *args, details = None): + self.extractExtraResults(test, details = details) + return super(OETestResult, self).addExpectedFailure(test, *args) - return (found, None) + def addUnexpectedSuccess(self, test, details = None): + self.extractExtraResults(test, details = details) + return super(OETestResult, self).addUnexpectedSuccess(test) + + def logDetails(self, json_file_dir=None, configuration=None, result_id=None, + dump_streams=False): + + result = self.extraresults + logs = {} + if hasattr(self.tc, "extraresults"): + result.update(self.tc.extraresults) - def logDetails(self): - self.tc.logger.info("RESULTS:") for case_name in self.tc._registry['cases']: case = self.tc._registry['cases'][case_name] - result_types = ['failures', 'errors', 'skipped', 'expectedFailures'] - result_desc = ['FAILED', 'ERROR', 'SKIPPED', 'EXPECTEDFAIL'] + (status, log) = self._getTestResultDetails(case) + + t = "" + duration = 0 + if case.id() in self.starttime and case.id() in self.endtime: + duration = self.endtime[case.id()] - self.starttime[case.id()] + t = " (" + "{0:.2f}".format(duration) + "s)" + + if status not in logs: + logs[status] = [] + logs[status].append("RESULTS - %s: %s%s" % (case.id(), status, t)) + report = {'status': status} + if log: + report['log'] = log + # Class setup failures wouldn't enter stopTest so would never display + if case.id() not in self.shownmsg: + self.tc.logger.info("Failure (%s) for %s:\n" % (status, case.id()) + log) + + if duration: + report['duration'] = duration + + alltags = [] + # pull tags from the case class + if hasattr(case, "__oeqa_testtags"): + alltags.extend(getattr(case, "__oeqa_testtags")) + # pull tags from the method itself + test_name = case._testMethodName + if hasattr(case, test_name): + method = getattr(case, test_name) + if hasattr(method, "__oeqa_testtags"): + alltags.extend(getattr(method, "__oeqa_testtags")) + if alltags: + report['oetags'] = alltags + + if dump_streams and case.id() in self.logged_output: + (stdout, stderr) = self.logged_output[case.id()] + report['stdout'] = stdout + report['stderr'] = stderr + result[case.id()] = report - fail = False - desc = None - for idx, name in enumerate(result_types): - (fail, msg) = self._getDetailsNotPassed(case, result_types[idx], - result_desc[idx]) - if fail: - desc = result_desc[idx] - break + self.tc.logger.info("RESULTS:") + for i in ['PASSED', 'SKIPPED', 'EXPECTEDFAIL', 'ERROR', 'FAILED', 'UNKNOWN']: + if i not in logs: + continue + for l in logs[i]: + self.tc.logger.info(l) - oeid = -1 - if hasattr(case, 'decorators'): - for d in case.decorators: - if hasattr(d, 'oeid'): - oeid = d.oeid + if json_file_dir: + tresultjsonhelper = OETestResultJSONHelper() + tresultjsonhelper.dump_testresult_file(json_file_dir, configuration, result_id, result) - if fail: - self.tc.logger.info("RESULTS - %s - Testcase %s: %s" % (case.id(), - oeid, desc)) - else: - self.tc.logger.info("RESULTS - %s - Testcase %s: %s" % (case.id(), - oeid, 'PASSED')) + def wasSuccessful(self): + # Override as we unexpected successes aren't failures for us + return (len(self.failures) == len(self.errors) == 0) + + def hasAnyFailingTest(self): + # Account for expected failures + return not self.wasSuccessful() or len(self.expectedFailures) class OEListTestsResult(object): def wasSuccessful(self): @@ -153,33 +247,14 @@ class OETestRunner(_TestRunner): streamLoggerClass = OEStreamLogger def __init__(self, tc, *args, **kwargs): - if xmlEnabled: - if not kwargs.get('output'): - kwargs['output'] = os.path.join(os.getcwd(), - 'TestResults_%s_%s' % (time.strftime("%Y%m%d%H%M%S"), os.getpid())) - kwargs['stream'] = self.streamLoggerClass(tc.logger) super(OETestRunner, self).__init__(*args, **kwargs) self.tc = tc self.resultclass = OETestResult - # XXX: The unittest-xml-reporting package defines _make_result method instead - # of _makeResult standard on unittest. - if xmlEnabled: - def _make_result(self): - """ - Creates a TestResult object which will be used to store - information about the executed tests. - """ - # override in subclasses if necessary. - return self.resultclass(self.tc, - self.stream, self.descriptions, self.verbosity, self.elapsed_times - ) - else: - def _makeResult(self): - return self.resultclass(self.tc, self.stream, self.descriptions, - self.verbosity) - + def _makeResult(self): + return self.resultclass(self.tc, self.stream, self.descriptions, + self.verbosity) def _walk_suite(self, suite, func): for obj in suite: @@ -191,42 +266,20 @@ class OETestRunner(_TestRunner): self._walked_cases = self._walked_cases + 1 def _list_tests_name(self, suite): - from oeqa.core.decorator.oeid import OETestID - from oeqa.core.decorator.oetag import OETestTag - self._walked_cases = 0 - def _list_cases_without_id(logger, case): - - found_id = False - if hasattr(case, 'decorators'): - for d in case.decorators: - if isinstance(d, OETestID): - found_id = True - - if not found_id: - logger.info('oeid missing for %s' % case.id()) - def _list_cases(logger, case): - oeid = None - oetag = None - - if hasattr(case, 'decorators'): - for d in case.decorators: - if isinstance(d, OETestID): - oeid = d.oeid - elif isinstance(d, OETestTag): - oetag = d.oetag - - logger.info("%s\t%s\t\t%s" % (oeid, oetag, case.id())) - - self.tc.logger.info("Listing test cases that don't have oeid ...") - self._walk_suite(suite, _list_cases_without_id) - self.tc.logger.info("-" * 80) + oetags = [] + if hasattr(case, '__oeqa_testtags'): + oetags = getattr(case, '__oeqa_testtags') + if oetags: + logger.info("%s (%s)" % (case.id(), ",".join(oetags))) + else: + logger.info("%s" % (case.id())) self.tc.logger.info("Listing all available tests:") self._walked_cases = 0 - self.tc.logger.info("id\ttag\t\ttest") + self.tc.logger.info("test (tags)") self.tc.logger.info("-" * 80) self._walk_suite(suite, _list_cases) self.tc.logger.info("-" * 80) @@ -275,3 +328,36 @@ class OETestRunner(_TestRunner): self._list_tests_module(suite) return OEListTestsResult() + +class OETestResultJSONHelper(object): + + testresult_filename = 'testresults.json' + + def _get_existing_testresults_if_available(self, write_dir): + testresults = {} + file = os.path.join(write_dir, self.testresult_filename) + if os.path.exists(file): + with open(file, "r") as f: + testresults = json.load(f) + return testresults + + def _write_file(self, write_dir, file_name, file_content): + file_path = os.path.join(write_dir, file_name) + with open(file_path, 'w') as the_file: + the_file.write(file_content) + + def dump_testresult_file(self, write_dir, configuration, result_id, test_result): + try: + import bb + has_bb = True + bb.utils.mkdirhier(write_dir) + lf = bb.utils.lockfile(os.path.join(write_dir, 'jsontestresult.lock')) + except ImportError: + has_bb = False + os.makedirs(write_dir, exist_ok=True) + test_results = self._get_existing_testresults_if_available(write_dir) + test_results[result_id] = {'configuration': configuration, 'result': test_result} + json_testresults = json.dumps(test_results, sort_keys=True, indent=4) + self._write_file(write_dir, self.testresult_filename, json_testresults) + if has_bb: + bb.utils.unlockfile(lf) diff --git a/meta/lib/oeqa/core/target/__init__.py b/meta/lib/oeqa/core/target/__init__.py index d2468bc257..1382aa9b52 100644 --- a/meta/lib/oeqa/core/target/__init__.py +++ b/meta/lib/oeqa/core/target/__init__.py @@ -1,5 +1,8 @@ +# # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# from abc import abstractmethod diff --git a/meta/lib/oeqa/core/target/qemu.py b/meta/lib/oeqa/core/target/qemu.py index d359bf9fe3..d93b3ac94a 100644 --- a/meta/lib/oeqa/core/target/qemu.py +++ b/meta/lib/oeqa/core/target/qemu.py @@ -1,45 +1,94 @@ +# # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# import os import sys import signal import time +import glob +import subprocess +from collections import defaultdict from .ssh import OESSHTarget from oeqa.utils.qemurunner import QemuRunner -supported_fstypes = ['ext3', 'ext4', 'cpio.gz', 'wic', 'elf'] +supported_fstypes = ['ext3', 'ext4', 'cpio.gz', 'wic'] class OEQemuTarget(OESSHTarget): - def __init__(self, logger, ip, server_ip, timeout=300, user='root', - port=None, machine='', rootfs='', kernel='', kvm=False, - dump_dir='', dump_host_cmds='', display='', bootlog='', - tmpdir='', dir_image='', boottime=60, **kwargs): + def __init__(self, logger, server_ip, timeout=300, user='root', + port=None, machine='', rootfs='', kernel='', kvm=False, slirp=False, + dump_dir='', display='', bootlog='', + tmpdir='', dir_image='', boottime=60, serial_ports=2, + boot_patterns = defaultdict(str), ovmf=False, tmpfsdir=None, **kwargs): - super(OEQemuTarget, self).__init__(logger, ip, server_ip, timeout, + super(OEQemuTarget, self).__init__(logger, None, server_ip, timeout, user, port) - self.ip = ip self.server_ip = server_ip + self.server_port = 0 self.machine = machine self.rootfs = rootfs self.kernel = kernel self.kvm = kvm + self.ovmf = ovmf + self.use_slirp = slirp + self.boot_patterns = boot_patterns + self.dump_dir = dump_dir + self.bootlog = bootlog self.runner = QemuRunner(machine=machine, rootfs=rootfs, tmpdir=tmpdir, deploy_dir_image=dir_image, display=display, logfile=bootlog, boottime=boottime, - use_kvm=kvm, dump_dir=dump_dir, - dump_host_cmds=dump_host_cmds, logger=logger) + use_kvm=kvm, use_slirp=slirp, dump_dir=dump_dir, logger=logger, + serial_ports=serial_ports, boot_patterns = boot_patterns, + use_ovmf=ovmf, tmpfsdir=tmpfsdir) - def start(self, params=None, extra_bootparams=None): - if self.runner.start(params, extra_bootparams=extra_bootparams): + def start(self, params=None, extra_bootparams=None, runqemuparams=''): + if self.use_slirp and not self.server_ip: + self.logger.error("Could not start qemu with slirp without server ip - provide 'TEST_SERVER_IP'") + raise RuntimeError("FAILED to start qemu - check the task log and the boot log") + if self.runner.start(params, extra_bootparams=extra_bootparams, runqemuparams=runqemuparams): self.ip = self.runner.ip - self.server_ip = self.runner.server_ip + if self.use_slirp: + target_ip_port = self.runner.ip.split(':') + if len(target_ip_port) == 2: + target_ip = target_ip_port[0] + port = target_ip_port[1] + self.ip = target_ip + self.ssh = self.ssh + ['-p', port] + self.scp = self.scp + ['-P', port] + else: + self.logger.error("Could not get host machine port to connect qemu with slirp, ssh will not be " + "able to connect to qemu with slirp") + if self.runner.server_ip: + self.server_ip = self.runner.server_ip else: self.stop() - raise RuntimeError("FAILED to start qemu - check the task log and the boot log") + # Display the first 20 lines of top and + # last 20 lines of the bootlog when the + # target is not being booted up. + topfile = glob.glob(self.dump_dir + "/*_qemu/host_*_top") + msg = "\n\n===== start: snippet =====\n\n" + for f in topfile: + msg += "file: %s\n\n" % f + with open(f) as tf: + for x in range(20): + msg += next(tf) + msg += "\n\n===== end: snippet =====\n\n" + blcmd = ["tail", "-20", self.bootlog] + msg += "===== start: snippet =====\n\n" + try: + out = subprocess.check_output(blcmd, stderr=subprocess.STDOUT, timeout=1).decode('utf-8') + msg += "file: %s\n\n" % self.bootlog + msg += out + except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError) as err: + msg += "Error running command: %s\n%s\n" % (blcmd, err) + msg += "\n\n===== end: snippet =====\n" + + raise RuntimeError("FAILED to start qemu - check the task log and the boot log %s" % (msg)) def stop(self): self.runner.stop() diff --git a/meta/lib/oeqa/core/target/ssh.py b/meta/lib/oeqa/core/target/ssh.py index 151b99a77f..09cdd14c75 100644 --- a/meta/lib/oeqa/core/target/ssh.py +++ b/meta/lib/oeqa/core/target/ssh.py @@ -1,5 +1,8 @@ +# # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# import os import time @@ -12,7 +15,7 @@ from . import OETarget class OESSHTarget(OETarget): def __init__(self, logger, ip, server_ip, timeout=300, user='root', - port=None, **kwargs): + port=None, server_port=0, **kwargs): if not logger: logger = logging.getLogger('target') logger.setLevel(logging.INFO) @@ -27,15 +30,21 @@ class OESSHTarget(OETarget): super(OESSHTarget, self).__init__(logger) self.ip = ip self.server_ip = server_ip + self.server_port = server_port self.timeout = timeout self.user = user ssh_options = [ + '-o', 'ServerAliveCountMax=2', + '-o', 'ServerAliveInterval=30', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no', '-o', 'LogLevel=ERROR' ] + scp_options = [ + '-r' + ] self.ssh = ['ssh', '-l', self.user ] + ssh_options - self.scp = ['scp'] + ssh_options + self.scp = ['scp'] + ssh_options + scp_options if port: self.ssh = self.ssh + [ '-p', port ] self.scp = self.scp + [ '-P', port ] @@ -63,7 +72,7 @@ class OESSHTarget(OETarget): return (status, output) - def run(self, command, timeout=None): + def run(self, command, timeout=None, ignore_status=True): """ Runs command in target. @@ -82,8 +91,9 @@ class OESSHTarget(OETarget): else: processTimeout = self.timeout - status, output = self._run(sshCmd, processTimeout, True) - self.logger.debug('Command: %s\nOutput: %s\n' % (command, output)) + status, output = self._run(sshCmd, processTimeout, ignore_status) + self.logger.debug('Command: %s\nStatus: %d Output: %s\n' % (command, status, output)) + return (status, output) def copyTo(self, localSrc, remoteDst): @@ -103,13 +113,16 @@ class OESSHTarget(OETarget): scpCmd = self.scp + [localSrc, remotePath] return self._run(scpCmd, ignore_status=False) - def copyFrom(self, remoteSrc, localDst): + def copyFrom(self, remoteSrc, localDst, warn_on_failure=False): """ Copy file from target. """ remotePath = '%s@%s:%s' % (self.user, self.ip, remoteSrc) scpCmd = self.scp + [remotePath, localDst] - return self._run(scpCmd, ignore_status=False) + (status, output) = self._run(scpCmd, ignore_status=warn_on_failure) + if warn_on_failure and status: + self.logger.warning("Copy returned non-zero exit status %d:\n%s" % (status, output)) + return (status, output) def copyDirTo(self, localSrc, remoteDst): """ @@ -198,27 +211,41 @@ def SSHCall(command, logger, timeout=None, **opts): def run(): nonlocal output nonlocal process + output_raw = b'' starttime = time.time() process = subprocess.Popen(command, **options) + has_timeout = False if timeout: endtime = starttime + timeout eof = False - while time.time() < endtime and not eof: - logger.debug('time: %s, endtime: %s' % (time.time(), endtime)) + os.set_blocking(process.stdout.fileno(), False) + while not has_timeout and not eof: try: + logger.debug('Waiting for process output: time: %s, endtime: %s' % (time.time(), endtime)) if select.select([process.stdout], [], [], 5)[0] != []: - reader = codecs.getreader('utf-8')(process.stdout) - data = reader.read(1024, 1024) + # wait a bit for more data, tries to avoid reading single characters + time.sleep(0.2) + data = process.stdout.read() if not data: - process.stdout.close() eof = True else: - output += data - logger.debug('Partial data from SSH call: %s' % data) + output_raw += data + # ignore errors to capture as much as possible + logger.debug('Partial data from SSH call:\n%s' % data.decode('utf-8', errors='ignore')) endtime = time.time() + timeout except InterruptedError: + logger.debug('InterruptedError') + continue + except BlockingIOError: + logger.debug('BlockingIOError') continue + if time.time() >= endtime: + logger.debug('SSHCall has timeout! Time: %s, endtime: %s' % (time.time(), endtime)) + has_timeout = True + + process.stdout.close() + # process hasn't returned yet if not eof: process.terminate() @@ -226,16 +253,42 @@ def SSHCall(command, logger, timeout=None, **opts): try: process.kill() except OSError: + logger.debug('OSError when killing process') pass endtime = time.time() - starttime lastline = ("\nProcess killed - no output for %d seconds. Total" " running time: %d seconds." % (timeout, endtime)) - logger.debug('Received data from SSH call %s ' % lastline) + logger.debug('Received data from SSH call:\n%s ' % lastline) output += lastline + process.wait() else: - output = process.communicate()[0].decode("utf-8", errors='replace') - logger.debug('Data from SSH call: %s' % output.rstrip()) + output_raw = process.communicate()[0] + + output = output_raw.decode('utf-8', errors='ignore') + logger.debug('Data from SSH call:\n%s' % output.rstrip()) + + # timout or not, make sure process exits and is not hanging + if process.returncode == None: + try: + process.wait(timeout=5) + except TimeoutExpired: + try: + process.kill() + except OSError: + logger.debug('OSError') + pass + process.wait() + + if has_timeout: + # Version of openssh before 8.6_p1 returns error code 0 when killed + # by a signal, when the timeout occurs we will receive a 0 error + # code because the process is been terminated and it's wrong because + # that value means success, but the process timed out. + # Afterwards, from version 8.6_p1 onwards, the returned code is 255. + # Fix this behaviour by checking the return code + if process.returncode == 0: + process.returncode = 255 options = { "stdout": subprocess.PIPE, @@ -243,7 +296,7 @@ def SSHCall(command, logger, timeout=None, **opts): "stdin": None, "shell": False, "bufsize": -1, - "preexec_fn": os.setsid, + "start_new_session": True, } options.update(opts) output = '' @@ -262,6 +315,9 @@ def SSHCall(command, logger, timeout=None, **opts): # whilst running and ensure we don't leave a process behind. if process.poll() is None: process.kill() + if process.returncode == None: + process.wait() logger.debug('Something went wrong, killing SSH process') raise - return (process.wait(), output.rstrip()) + + return (process.returncode, output.rstrip()) diff --git a/meta/lib/oeqa/core/tests/cases/data.py b/meta/lib/oeqa/core/tests/cases/data.py index 88003a6adc..61f88547f7 100644 --- a/meta/lib/oeqa/core/tests/cases/data.py +++ b/meta/lib/oeqa/core/tests/cases/data.py @@ -1,8 +1,11 @@ +# # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# from oeqa.core.case import OETestCase -from oeqa.core.decorator.oetag import OETestTag +from oeqa.core.decorator import OETestTag from oeqa.core.decorator.data import OETestDataDepends class DataTest(OETestCase): diff --git a/meta/lib/oeqa/core/tests/cases/depends.py b/meta/lib/oeqa/core/tests/cases/depends.py index 17cdd90b15..46e7db900d 100644 --- a/meta/lib/oeqa/core/tests/cases/depends.py +++ b/meta/lib/oeqa/core/tests/cases/depends.py @@ -1,5 +1,8 @@ +# # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# from oeqa.core.case import OETestCase from oeqa.core.decorator.depends import OETestDepends diff --git a/meta/lib/oeqa/core/tests/cases/loader/invalid/oeid.py b/meta/lib/oeqa/core/tests/cases/loader/invalid/oeid.py deleted file mode 100644 index 038d445931..0000000000 --- a/meta/lib/oeqa/core/tests/cases/loader/invalid/oeid.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) - -from oeqa.core.case import OETestCase - -class AnotherIDTest(OETestCase): - - def testAnotherIdGood(self): - self.assertTrue(True, msg='How is this possible?') - - def testAnotherIdOther(self): - self.assertTrue(True, msg='How is this possible?') - - def testAnotherIdNone(self): - self.assertTrue(True, msg='How is this possible?') diff --git a/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded.py b/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded.py deleted file mode 100644 index 0fe4cb3f11..0000000000 --- a/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (C) 2017 Intel Corporation -# Released under the MIT license (see COPYING.MIT) - -from oeqa.core.case import OETestCase - -class ThreadedTest(OETestCase): - def test_threaded_no_depends(self): - self.assertTrue(True, msg='How is this possible?') - -class ThreadedTest2(OETestCase): - def test_threaded_same_module(self): - self.assertTrue(True, msg='How is this possible?') diff --git a/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_alone.py b/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_alone.py deleted file mode 100644 index 905f397846..0000000000 --- a/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_alone.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (C) 2017 Intel Corporation -# Released under the MIT license (see COPYING.MIT) - -from oeqa.core.case import OETestCase - -class ThreadedTestAlone(OETestCase): - def test_threaded_alone(self): - self.assertTrue(True, msg='How is this possible?') diff --git a/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_depends.py b/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_depends.py deleted file mode 100644 index 0c158d3bac..0000000000 --- a/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_depends.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (C) 2017 Intel Corporation -# Released under the MIT license (see COPYING.MIT) - -from oeqa.core.case import OETestCase -from oeqa.core.decorator.depends import OETestDepends - -class ThreadedTest3(OETestCase): - @OETestDepends(['threaded.ThreadedTest.test_threaded_no_depends']) - def test_threaded_depends(self): - self.assertTrue(True, msg='How is this possible?') diff --git a/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_module.py b/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_module.py deleted file mode 100644 index 63d17e0401..0000000000 --- a/meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_module.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (C) 2017 Intel Corporation -# Released under the MIT license (see COPYING.MIT) - -from oeqa.core.case import OETestCase - -class ThreadedTestModule(OETestCase): - def test_threaded_module(self): - self.assertTrue(True, msg='How is this possible?') - -class ThreadedTestModule2(OETestCase): - def test_threaded_module2(self): - self.assertTrue(True, msg='How is this possible?') diff --git a/meta/lib/oeqa/core/tests/cases/loader/valid/another.py b/meta/lib/oeqa/core/tests/cases/loader/valid/another.py index c9ffd17773..bedc20c8a6 100644 --- a/meta/lib/oeqa/core/tests/cases/loader/valid/another.py +++ b/meta/lib/oeqa/core/tests/cases/loader/valid/another.py @@ -1,5 +1,8 @@ +# # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# from oeqa.core.case import OETestCase diff --git a/meta/lib/oeqa/core/tests/cases/oeid.py b/meta/lib/oeqa/core/tests/cases/oeid.py deleted file mode 100644 index c2d3d32f2d..0000000000 --- a/meta/lib/oeqa/core/tests/cases/oeid.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) - -from oeqa.core.case import OETestCase -from oeqa.core.decorator.oeid import OETestID - -class IDTest(OETestCase): - - @OETestID(101) - def testIdGood(self): - self.assertTrue(True, msg='How is this possible?') - - @OETestID(102) - def testIdOther(self): - self.assertTrue(True, msg='How is this possible?') - - def testIdNone(self): - self.assertTrue(True, msg='How is this possible?') diff --git a/meta/lib/oeqa/core/tests/cases/oetag.py b/meta/lib/oeqa/core/tests/cases/oetag.py index 0cae02e75c..52f97dfda6 100644 --- a/meta/lib/oeqa/core/tests/cases/oetag.py +++ b/meta/lib/oeqa/core/tests/cases/oetag.py @@ -1,11 +1,13 @@ +# # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# from oeqa.core.case import OETestCase -from oeqa.core.decorator.oetag import OETestTag +from oeqa.core.decorator import OETestTag class TagTest(OETestCase): - @OETestTag('goodTag') def testTagGood(self): self.assertTrue(True, msg='How is this possible?') @@ -14,5 +16,23 @@ class TagTest(OETestCase): def testTagOther(self): self.assertTrue(True, msg='How is this possible?') + @OETestTag('otherTag', 'multiTag') + def testTagOtherMulti(self): + self.assertTrue(True, msg='How is this possible?') + def testTagNone(self): self.assertTrue(True, msg='How is this possible?') + +@OETestTag('classTag') +class TagClassTest(OETestCase): + @OETestTag('otherTag') + def testTagOther(self): + self.assertTrue(True, msg='How is this possible?') + + @OETestTag('otherTag', 'multiTag') + def testTagOtherMulti(self): + self.assertTrue(True, msg='How is this possible?') + + def testTagNone(self): + self.assertTrue(True, msg='How is this possible?') + diff --git a/meta/lib/oeqa/core/tests/cases/timeout.py b/meta/lib/oeqa/core/tests/cases/timeout.py index 870c3157f7..69cf969a67 100644 --- a/meta/lib/oeqa/core/tests/cases/timeout.py +++ b/meta/lib/oeqa/core/tests/cases/timeout.py @@ -1,10 +1,14 @@ +# # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# from time import sleep from oeqa.core.case import OETestCase from oeqa.core.decorator.oetimeout import OETimeout +from oeqa.core.decorator.depends import OETestDepends class TimeoutTest(OETestCase): @@ -16,3 +20,15 @@ class TimeoutTest(OETestCase): def testTimeoutFail(self): sleep(2) self.assertTrue(True, msg='How is this possible?') + + + def testTimeoutSkip(self): + self.skipTest("This test needs to be skipped, so that testTimeoutDepends()'s OETestDepends kicks in") + + @OETestDepends(["timeout.TimeoutTest.testTimeoutSkip"]) + @OETimeout(3) + def testTimeoutDepends(self): + self.assertTrue(False, msg='How is this possible?') + + def testTimeoutUnrelated(self): + sleep(6) diff --git a/meta/lib/oeqa/core/tests/common.py b/meta/lib/oeqa/core/tests/common.py index 1932323409..88cc758ad3 100644 --- a/meta/lib/oeqa/core/tests/common.py +++ b/meta/lib/oeqa/core/tests/common.py @@ -1,5 +1,8 @@ +# # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# import sys import os @@ -27,19 +30,9 @@ class TestBase(unittest.TestCase): directory = os.path.dirname(os.path.abspath(__file__)) self.cases_path = os.path.join(directory, 'cases') - def _testLoader(self, d={}, modules=[], tests=[], filters={}): + def _testLoader(self, d={}, modules=[], tests=[], **kwargs): from oeqa.core.context import OETestContext tc = OETestContext(d, self.logger) tc.loadTests(self.cases_path, modules=modules, tests=tests, - filters=filters) - return tc - - def _testLoaderThreaded(self, d={}, modules=[], - tests=[], filters={}): - from oeqa.core.threaded import OETestContextThreaded - - tc = OETestContextThreaded(d, self.logger) - tc.loadTests(self.cases_path, modules=modules, tests=tests, - filters=filters) - + **kwargs) return tc diff --git a/meta/lib/oeqa/core/tests/test_data.py b/meta/lib/oeqa/core/tests/test_data.py index 320468cbe4..acd726f3a0 100755 --- a/meta/lib/oeqa/core/tests/test_data.py +++ b/meta/lib/oeqa/core/tests/test_data.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# import unittest import logging @@ -20,8 +22,9 @@ class TestData(TestBase): expectedException = "oeqa.core.exception.OEQAMissingVariable" tc = self._testLoader(modules=self.modules) - self.assertEqual(False, tc.runTests().wasSuccessful()) - for test, data in tc._results['errors']: + results = tc.runTests() + self.assertFalse(results.wasSuccessful()) + for test, data in results.errors: expect = False if expectedException in data: expect = True @@ -30,11 +33,12 @@ class TestData(TestBase): def test_data_fail_wrong_variable(self): expectedError = 'AssertionError' - d = {'IMAGE' : 'core-image-sato', 'ARCH' : 'arm'} + d = {'IMAGE' : 'core-image-weston', 'ARCH' : 'arm'} tc = self._testLoader(d=d, modules=self.modules) - self.assertEqual(False, tc.runTests().wasSuccessful()) - for test, data in tc._results['failures']: + results = tc.runTests() + self.assertFalse(results.wasSuccessful()) + for test, data in results.failures: expect = False if expectedError in data: expect = True diff --git a/meta/lib/oeqa/core/tests/test_decorators.py b/meta/lib/oeqa/core/tests/test_decorators.py index cf99e0d72d..5095f39948 100755 --- a/meta/lib/oeqa/core/tests/test_decorators.py +++ b/meta/lib/oeqa/core/tests/test_decorators.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 - +# # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# import signal import unittest @@ -12,58 +14,58 @@ setup_sys_path() from oeqa.core.exception import OEQADependency from oeqa.core.utils.test import getCaseMethod, getSuiteCasesNames, getSuiteCasesIDs -class TestFilterDecorator(TestBase): - - def _runFilterTest(self, modules, filters, expect, msg): - tc = self._testLoader(modules=modules, filters=filters) - test_loaded = set(getSuiteCasesNames(tc.suites)) - self.assertEqual(expect, test_loaded, msg=msg) +class TestTagDecorator(TestBase): + def _runTest(self, modules, filterfn, expect): + tc = self._testLoader(modules = modules, tags_filter = filterfn) + test_loaded = set(getSuiteCasesIDs(tc.suites)) + self.assertEqual(expect, test_loaded) def test_oetag(self): - # Get all cases without filtering. - filter_all = {} - test_all = {'testTagGood', 'testTagOther', 'testTagNone'} - msg_all = 'Failed to get all oetag cases without filtering.' - - # Get cases with 'goodTag'. - filter_good = {'oetag':'goodTag'} - test_good = {'testTagGood'} - msg_good = 'Failed to get just one test filtering with "goodTag" oetag.' - - # Get cases with an invalid tag. - filter_invalid = {'oetag':'invalidTag'} - test_invalid = set() - msg_invalid = 'Failed to filter all test using an invalid oetag.' - - tests = ((filter_all, test_all, msg_all), - (filter_good, test_good, msg_good), - (filter_invalid, test_invalid, msg_invalid)) - - for test in tests: - self._runFilterTest(['oetag'], test[0], test[1], test[2]) - - def test_oeid(self): - # Get all cases without filtering. - filter_all = {} - test_all = {'testIdGood', 'testIdOther', 'testIdNone'} - msg_all = 'Failed to get all oeid cases without filtering.' - - # Get cases with '101' oeid. - filter_good = {'oeid': 101} - test_good = {'testIdGood'} - msg_good = 'Failed to get just one tes filtering with "101" oeid.' - - # Get cases with an invalid id. - filter_invalid = {'oeid':999} - test_invalid = set() - msg_invalid = 'Failed to filter all test using an invalid oeid.' - - tests = ((filter_all, test_all, msg_all), - (filter_good, test_good, msg_good), - (filter_invalid, test_invalid, msg_invalid)) - - for test in tests: - self._runFilterTest(['oeid'], test[0], test[1], test[2]) + # get all cases without any filtering + self._runTest(['oetag'], None, { + 'oetag.TagTest.testTagGood', + 'oetag.TagTest.testTagOther', + 'oetag.TagTest.testTagOtherMulti', + 'oetag.TagTest.testTagNone', + 'oetag.TagClassTest.testTagOther', + 'oetag.TagClassTest.testTagOtherMulti', + 'oetag.TagClassTest.testTagNone', + }) + + # exclude any case with tags + self._runTest(['oetag'], lambda tags: tags, { + 'oetag.TagTest.testTagNone', + }) + + # exclude any case with otherTag + self._runTest(['oetag'], lambda tags: "otherTag" in tags, { + 'oetag.TagTest.testTagGood', + 'oetag.TagTest.testTagNone', + 'oetag.TagClassTest.testTagNone', + }) + + # exclude any case with classTag + self._runTest(['oetag'], lambda tags: "classTag" in tags, { + 'oetag.TagTest.testTagGood', + 'oetag.TagTest.testTagOther', + 'oetag.TagTest.testTagOtherMulti', + 'oetag.TagTest.testTagNone', + }) + + # include any case with classTag + self._runTest(['oetag'], lambda tags: "classTag" not in tags, { + 'oetag.TagClassTest.testTagOther', + 'oetag.TagClassTest.testTagOtherMulti', + 'oetag.TagClassTest.testTagNone', + }) + + # include any case with classTag or no tags + self._runTest(['oetag'], lambda tags: tags and "classTag" not in tags, { + 'oetag.TagTest.testTagNone', + 'oetag.TagClassTest.testTagOther', + 'oetag.TagClassTest.testTagOtherMulti', + 'oetag.TagClassTest.testTagNone', + }) class TestDependsDecorator(TestBase): modules = ['depends'] @@ -131,17 +133,11 @@ class TestTimeoutDecorator(TestBase): msg = "OETestTimeout didn't restore SIGALRM" self.assertIs(alarm_signal, signal.getsignal(signal.SIGALRM), msg=msg) - def test_timeout_thread(self): - tests = ['timeout.TimeoutTest.testTimeoutPass'] - msg = 'Failed to run test using OETestTimeout' - tc = self._testLoaderThreaded(modules=self.modules, tests=tests) + def test_timeout_cancel(self): + tests = ['timeout.TimeoutTest.testTimeoutSkip', 'timeout.TimeoutTest.testTimeoutDepends', 'timeout.TimeoutTest.testTimeoutUnrelated'] + msg = 'Unrelated test failed to complete' + tc = self._testLoader(modules=self.modules, tests=tests) self.assertTrue(tc.runTests().wasSuccessful(), msg=msg) - def test_timeout_threaded_fail(self): - tests = ['timeout.TimeoutTest.testTimeoutFail'] - msg = "OETestTimeout test didn't timeout as expected" - tc = self._testLoaderThreaded(modules=self.modules, tests=tests) - self.assertFalse(tc.runTests().wasSuccessful(), msg=msg) - if __name__ == '__main__': unittest.main() diff --git a/meta/lib/oeqa/core/tests/test_loader.py b/meta/lib/oeqa/core/tests/test_loader.py index e0d917d317..cb38ac845e 100755 --- a/meta/lib/oeqa/core/tests/test_loader.py +++ b/meta/lib/oeqa/core/tests/test_loader.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 - -# Copyright (C) 2016-2017 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# Copyright (C) 2016 Intel Corporation +# +# SPDX-License-Identifier: MIT +# import os import unittest @@ -13,36 +15,12 @@ from oeqa.core.exception import OEQADependency from oeqa.core.utils.test import getSuiteModules, getSuiteCasesIDs class TestLoader(TestBase): - - def test_fail_empty_filter(self): - filters = {'oetag' : ''} - expect = 'Filter oetag specified is empty' - msg = 'Expected TypeError exception for having invalid filter' - try: - # Must throw TypeError because empty filter - tc = self._testLoader(filters=filters) - self.fail(msg) - except TypeError as e: - result = True if expect in str(e) else False - self.assertTrue(result, msg=msg) - - def test_fail_invalid_filter(self): - filters = {'invalid' : 'good'} - expect = 'filter but not declared in any of' - msg = 'Expected TypeError exception for having invalid filter' - try: - # Must throw TypeError because invalid filter - tc = self._testLoader(filters=filters) - self.fail(msg) - except TypeError as e: - result = True if expect in str(e) else False - self.assertTrue(result, msg=msg) - + @unittest.skip("invalid directory is missing oetag.py") def test_fail_duplicated_module(self): cases_path = self.cases_path invalid_path = os.path.join(cases_path, 'loader', 'invalid') self.cases_path = [self.cases_path, invalid_path] - expect = 'Duplicated oeid module found in' + expect = 'Duplicated oetag module found in' msg = 'Expected ImportError exception for having duplicated module' try: # Must throw ImportEror because duplicated module @@ -55,17 +33,16 @@ class TestLoader(TestBase): self.cases_path = cases_path def test_filter_modules(self): - expected_modules = {'oeid', 'oetag'} + expected_modules = {'oetag'} tc = self._testLoader(modules=expected_modules) modules = getSuiteModules(tc.suites) msg = 'Expected just %s modules' % ', '.join(expected_modules) self.assertEqual(modules, expected_modules, msg=msg) def test_filter_cases(self): - modules = ['oeid', 'oetag', 'data'] + modules = ['oetag', 'data'] expected_cases = {'data.DataTest.testDataOk', - 'oetag.TagTest.testTagGood', - 'oeid.IDTest.testIdGood'} + 'oetag.TagTest.testTagGood'} tc = self._testLoader(modules=modules, tests=expected_cases) cases = set(getSuiteCasesIDs(tc.suites)) msg = 'Expected just %s cases' % ', '.join(expected_cases) @@ -74,7 +51,7 @@ class TestLoader(TestBase): def test_import_from_paths(self): cases_path = self.cases_path cases2_path = os.path.join(cases_path, 'loader', 'valid') - expected_modules = {'oeid', 'another'} + expected_modules = {'another'} self.cases_path = [self.cases_path, cases2_path] tc = self._testLoader(modules=expected_modules) modules = getSuiteModules(tc.suites) @@ -82,33 +59,5 @@ class TestLoader(TestBase): msg = 'Expected modules from two different paths' self.assertEqual(modules, expected_modules, msg=msg) - def test_loader_threaded(self): - cases_path = self.cases_path - - self.cases_path = [os.path.join(self.cases_path, 'loader', 'threaded')] - - tc = self._testLoaderThreaded() - self.assertEqual(len(tc.suites), 3, "Expected to be 3 suites") - - case_ids = ['threaded.ThreadedTest.test_threaded_no_depends', - 'threaded.ThreadedTest2.test_threaded_same_module', - 'threaded_depends.ThreadedTest3.test_threaded_depends'] - for case in tc.suites[0]._tests: - self.assertEqual(case.id(), - case_ids[tc.suites[0]._tests.index(case)]) - - case_ids = ['threaded_alone.ThreadedTestAlone.test_threaded_alone'] - for case in tc.suites[1]._tests: - self.assertEqual(case.id(), - case_ids[tc.suites[1]._tests.index(case)]) - - case_ids = ['threaded_module.ThreadedTestModule.test_threaded_module', - 'threaded_module.ThreadedTestModule2.test_threaded_module2'] - for case in tc.suites[2]._tests: - self.assertEqual(case.id(), - case_ids[tc.suites[2]._tests.index(case)]) - - self.cases_path = cases_path - if __name__ == '__main__': unittest.main() diff --git a/meta/lib/oeqa/core/tests/test_runner.py b/meta/lib/oeqa/core/tests/test_runner.py index a3f3861fed..205464cfae 100755 --- a/meta/lib/oeqa/core/tests/test_runner.py +++ b/meta/lib/oeqa/core/tests/test_runner.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 - +# # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# import unittest import logging diff --git a/meta/lib/oeqa/core/threaded.py b/meta/lib/oeqa/core/threaded.py deleted file mode 100644 index 2cafe03a21..0000000000 --- a/meta/lib/oeqa/core/threaded.py +++ /dev/null @@ -1,275 +0,0 @@ -# Copyright (C) 2017 Intel Corporation -# Released under the MIT license (see COPYING.MIT) - -import threading -import multiprocessing -import queue -import time - -from unittest.suite import TestSuite - -from oeqa.core.loader import OETestLoader -from oeqa.core.runner import OEStreamLogger, OETestResult, OETestRunner -from oeqa.core.context import OETestContext - -class OETestLoaderThreaded(OETestLoader): - def __init__(self, tc, module_paths, modules, tests, modules_required, - filters, process_num=0, *args, **kwargs): - super(OETestLoaderThreaded, self).__init__(tc, module_paths, modules, - tests, modules_required, filters, *args, **kwargs) - - self.process_num = process_num - - def discover(self): - suite = super(OETestLoaderThreaded, self).discover() - - if self.process_num <= 0: - self.process_num = min(multiprocessing.cpu_count(), - len(suite._tests)) - - suites = [] - for _ in range(self.process_num): - suites.append(self.suiteClass()) - - def _search_for_module_idx(suites, case): - """ - Cases in the same module needs to be run - in the same thread because PyUnit keeps track - of setUp{Module, Class,} and tearDown{Module, Class,}. - """ - - for idx in range(self.process_num): - suite = suites[idx] - for c in suite._tests: - if case.__module__ == c.__module__: - return idx - - return -1 - - def _search_for_depend_idx(suites, depends): - """ - Dependency cases needs to be run in the same - thread, because OEQA framework look at the state - of dependant test to figure out if skip or not. - """ - - for idx in range(self.process_num): - suite = suites[idx] - - for case in suite._tests: - if case.id() in depends: - return idx - return -1 - - def _get_best_idx(suites): - sizes = [len(suite._tests) for suite in suites] - return sizes.index(min(sizes)) - - def _fill_suites(suite): - idx = -1 - for case in suite: - if isinstance(case, TestSuite): - _fill_suites(case) - else: - idx = _search_for_module_idx(suites, case) - - depends = {} - if 'depends' in self.tc._registry: - depends = self.tc._registry['depends'] - - if idx == -1 and case.id() in depends: - case_depends = depends[case.id()] - idx = _search_for_depend_idx(suites, case_depends) - - if idx == -1: - idx = _get_best_idx(suites) - - suites[idx].addTest(case) - _fill_suites(suite) - - suites_tmp = suites - suites = [] - for suite in suites_tmp: - if len(suite._tests) > 0: - suites.append(suite) - - return suites - -class OEStreamLoggerThreaded(OEStreamLogger): - _lock = threading.Lock() - buffers = {} - - def write(self, msg): - tid = threading.get_ident() - - if not tid in self.buffers: - self.buffers[tid] = "" - - if msg: - self.buffers[tid] += msg - - def finish(self): - tid = threading.get_ident() - - self._lock.acquire() - self.logger.info('THREAD: %d' % tid) - self.logger.info('-' * 70) - for line in self.buffers[tid].split('\n'): - self.logger.info(line) - self._lock.release() - -class OETestResultThreadedInternal(OETestResult): - def _tc_map_results(self): - tid = threading.get_ident() - - # PyUnit generates a result for every test module run, test - # if the thread already has an entry to avoid lose the previous - # test module results. - if not tid in self.tc._results: - self.tc._results[tid] = {} - self.tc._results[tid]['failures'] = self.failures - self.tc._results[tid]['errors'] = self.errors - self.tc._results[tid]['skipped'] = self.skipped - self.tc._results[tid]['expectedFailures'] = self.expectedFailures - -class OETestResultThreaded(object): - _results = {} - _lock = threading.Lock() - - def __init__(self, tc): - self.tc = tc - - def _fill_tc_results(self): - tids = list(self.tc._results.keys()) - fields = ['failures', 'errors', 'skipped', 'expectedFailures'] - - for tid in tids: - result = self.tc._results[tid] - for field in fields: - if not field in self.tc._results: - self.tc._results[field] = [] - self.tc._results[field].extend(result[field]) - - def addResult(self, result, run_start_time, run_end_time): - tid = threading.get_ident() - - self._lock.acquire() - self._results[tid] = {} - self._results[tid]['result'] = result - self._results[tid]['run_start_time'] = run_start_time - self._results[tid]['run_end_time'] = run_end_time - self._results[tid]['result'] = result - self._lock.release() - - def wasSuccessful(self): - wasSuccessful = True - for tid in self._results.keys(): - wasSuccessful = wasSuccessful and \ - self._results[tid]['result'].wasSuccessful() - return wasSuccessful - - def stop(self): - for tid in self._results.keys(): - self._results[tid]['result'].stop() - - def logSummary(self, component, context_msg=''): - elapsed_time = (self.tc._run_end_time - self.tc._run_start_time) - - self.tc.logger.info("SUMMARY:") - self.tc.logger.info("%s (%s) - Ran %d tests in %.3fs" % (component, - context_msg, len(self.tc._registry['cases']), elapsed_time)) - if self.wasSuccessful(): - msg = "%s - OK - All required tests passed" % component - else: - msg = "%s - FAIL - Required tests failed" % component - self.tc.logger.info(msg) - - def logDetails(self): - if list(self._results): - tid = list(self._results)[0] - result = self._results[tid]['result'] - result.logDetails() - -class _Worker(threading.Thread): - """Thread executing tasks from a given tasks queue""" - def __init__(self, tasks, result, stream): - threading.Thread.__init__(self) - self.tasks = tasks - - self.result = result - self.stream = stream - - def run(self): - while True: - try: - func, args, kargs = self.tasks.get(block=False) - except queue.Empty: - break - - try: - run_start_time = time.time() - rc = func(*args, **kargs) - run_end_time = time.time() - self.result.addResult(rc, run_start_time, run_end_time) - self.stream.finish() - except Exception as e: - print(e) - finally: - self.tasks.task_done() - -class _ThreadedPool: - """Pool of threads consuming tasks from a queue""" - def __init__(self, num_workers, num_tasks, stream=None, result=None): - self.tasks = queue.Queue(num_tasks) - self.workers = [] - - for _ in range(num_workers): - worker = _Worker(self.tasks, result, stream) - self.workers.append(worker) - - def start(self): - for worker in self.workers: - worker.start() - - def add_task(self, func, *args, **kargs): - """Add a task to the queue""" - self.tasks.put((func, args, kargs)) - - def wait_completion(self): - """Wait for completion of all the tasks in the queue""" - self.tasks.join() - for worker in self.workers: - worker.join() - -class OETestRunnerThreaded(OETestRunner): - streamLoggerClass = OEStreamLoggerThreaded - - def __init__(self, tc, *args, **kwargs): - super(OETestRunnerThreaded, self).__init__(tc, *args, **kwargs) - self.resultclass = OETestResultThreadedInternal # XXX: XML reporting overrides at __init__ - - def run(self, suites): - result = OETestResultThreaded(self.tc) - - pool = _ThreadedPool(len(suites), len(suites), stream=self.stream, - result=result) - for s in suites: - pool.add_task(super(OETestRunnerThreaded, self).run, s) - pool.start() - pool.wait_completion() - result._fill_tc_results() - - return result - -class OETestContextThreaded(OETestContext): - loaderClass = OETestLoaderThreaded - runnerClass = OETestRunnerThreaded - - def loadTests(self, module_paths, modules=[], tests=[], - modules_manifest="", modules_required=[], filters={}, process_num=0): - if modules_manifest: - modules = self._read_modules_from_manifest(modules_manifest) - - self.loader = self.loaderClass(self, module_paths, modules, tests, - modules_required, filters, process_num) - self.suites = self.loader.discover() diff --git a/meta/lib/oeqa/core/utils/concurrencytest.py b/meta/lib/oeqa/core/utils/concurrencytest.py new file mode 100644 index 0000000000..d10f8f7f04 --- /dev/null +++ b/meta/lib/oeqa/core/utils/concurrencytest.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python3 +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Modified for use in OE by Richard Purdie, 2018 +# +# Modified by: Corey Goldberg, 2013 +# License: GPLv2+ +# +# Original code from: +# Bazaar (bzrlib.tests.__init__.py, v2.6, copied Jun 01 2013) +# Copyright (C) 2005-2011 Canonical Ltd +# License: GPLv2+ + +import os +import sys +import traceback +import unittest +import subprocess +import testtools +import threading +import time +import io +import json +import subunit + +from queue import Queue +from itertools import cycle +from subunit import ProtocolTestCase, TestProtocolClient +from subunit.test_results import AutoTimingTestResultDecorator +from testtools import ThreadsafeForwardingResult, iterate_tests +from testtools.content import Content +from testtools.content_type import ContentType +from oeqa.utils.commands import get_test_layer + +import bb.utils +import oe.path + +_all__ = [ + 'ConcurrentTestSuite', + 'fork_for_tests', + 'partition_tests', +] + +# +# Patch the version from testtools to allow access to _test_start and allow +# computation of timing information and threading progress +# +class BBThreadsafeForwardingResult(ThreadsafeForwardingResult): + + def __init__(self, target, semaphore, threadnum, totalinprocess, totaltests, output, finalresult): + super(BBThreadsafeForwardingResult, self).__init__(target, semaphore) + self.threadnum = threadnum + self.totalinprocess = totalinprocess + self.totaltests = totaltests + self.buffer = True + self.outputbuf = output + self.finalresult = finalresult + self.finalresult.buffer = True + self.target = target + + def _add_result_with_semaphore(self, method, test, *args, **kwargs): + self.semaphore.acquire() + try: + if self._test_start: + self.result.starttime[test.id()] = self._test_start.timestamp() + self.result.threadprogress[self.threadnum].append(test.id()) + totalprogress = sum(len(x) for x in self.result.threadprogress.values()) + self.result.progressinfo[test.id()] = "%s: %s/%s %s/%s (%ss) (%s failed) (%s)" % ( + self.threadnum, + len(self.result.threadprogress[self.threadnum]), + self.totalinprocess, + totalprogress, + self.totaltests, + "{0:.2f}".format(time.time()-self._test_start.timestamp()), + self.target.failed_tests, + test.id()) + finally: + self.semaphore.release() + self.finalresult._stderr_buffer = io.StringIO(initial_value=self.outputbuf.getvalue().decode("utf-8")) + self.finalresult._stdout_buffer = io.StringIO() + super(BBThreadsafeForwardingResult, self)._add_result_with_semaphore(method, test, *args, **kwargs) + +class ProxyTestResult: + # a very basic TestResult proxy, in order to modify add* calls + def __init__(self, target): + self.result = target + self.failed_tests = 0 + + def _addResult(self, method, test, *args, exception = False, **kwargs): + return method(test, *args, **kwargs) + + def addError(self, test, err = None, **kwargs): + self.failed_tests += 1 + self._addResult(self.result.addError, test, err, exception = True, **kwargs) + + def addFailure(self, test, err = None, **kwargs): + self.failed_tests += 1 + self._addResult(self.result.addFailure, test, err, exception = True, **kwargs) + + def addSuccess(self, test, **kwargs): + self._addResult(self.result.addSuccess, test, **kwargs) + + def addExpectedFailure(self, test, err = None, **kwargs): + self._addResult(self.result.addExpectedFailure, test, err, exception = True, **kwargs) + + def addUnexpectedSuccess(self, test, **kwargs): + self._addResult(self.result.addUnexpectedSuccess, test, **kwargs) + + def wasSuccessful(self): + return self.failed_tests == 0 + + def __getattr__(self, attr): + return getattr(self.result, attr) + +class ExtraResultsDecoderTestResult(ProxyTestResult): + def _addResult(self, method, test, *args, exception = False, **kwargs): + if "details" in kwargs and "extraresults" in kwargs["details"]: + if isinstance(kwargs["details"]["extraresults"], Content): + kwargs = kwargs.copy() + kwargs["details"] = kwargs["details"].copy() + extraresults = kwargs["details"]["extraresults"] + data = bytearray() + for b in extraresults.iter_bytes(): + data += b + extraresults = json.loads(data.decode()) + kwargs["details"]["extraresults"] = extraresults + return method(test, *args, **kwargs) + +class ExtraResultsEncoderTestResult(ProxyTestResult): + def _addResult(self, method, test, *args, exception = False, **kwargs): + if hasattr(test, "extraresults"): + extras = lambda : [json.dumps(test.extraresults).encode()] + kwargs = kwargs.copy() + if "details" not in kwargs: + kwargs["details"] = {} + else: + kwargs["details"] = kwargs["details"].copy() + kwargs["details"]["extraresults"] = Content(ContentType("application", "json", {'charset': 'utf8'}), extras) + # if using details, need to encode any exceptions into the details obj, + # testtools does not handle "err" and "details" together. + if "details" in kwargs and exception and (len(args) >= 1 and args[0] is not None): + kwargs["details"]["traceback"] = testtools.content.TracebackContent(args[0], test) + args = [] + return method(test, *args, **kwargs) + +# +# We have to patch subunit since it doesn't understand how to handle addError +# outside of a running test case. This can happen if classSetUp() fails +# for a class of tests. This unfortunately has horrible internal knowledge. +# +def outSideTestaddError(self, offset, line): + """An 'error:' directive has been read.""" + test_name = line[offset:-1].decode('utf8') + self.parser._current_test = subunit.RemotedTestCase(test_name) + self.parser.current_test_description = test_name + self.parser._state = self.parser._reading_error_details + self.parser._reading_error_details.set_simple() + self.parser.subunitLineReceived(line) + +subunit._OutSideTest.addError = outSideTestaddError + +# Like outSideTestaddError above, we need an equivalent for skips +# happening at the setUpClass() level, otherwise we will see "UNKNOWN" +# as a result for concurrent tests +# +def outSideTestaddSkip(self, offset, line): + """A 'skip:' directive has been read.""" + test_name = line[offset:-1].decode('utf8') + self.parser._current_test = subunit.RemotedTestCase(test_name) + self.parser.current_test_description = test_name + self.parser._state = self.parser._reading_skip_details + self.parser._reading_skip_details.set_simple() + self.parser.subunitLineReceived(line) + +subunit._OutSideTest.addSkip = outSideTestaddSkip + +# +# A dummy structure to add to io.StringIO so that the .buffer object +# is available and accepts writes. This allows unittest with buffer=True +# to interact ok with subunit which wants to access sys.stdout.buffer. +# +class dummybuf(object): + def __init__(self, parent): + self.p = parent + def write(self, data): + self.p.write(data.decode("utf-8")) + +# +# Taken from testtools.ConncurrencyTestSuite but modified for OE use +# +class ConcurrentTestSuite(unittest.TestSuite): + + def __init__(self, suite, processes, setupfunc, removefunc, bb_vars): + super(ConcurrentTestSuite, self).__init__([suite]) + self.processes = processes + self.setupfunc = setupfunc + self.removefunc = removefunc + self.bb_vars = bb_vars + + def run(self, result): + testservers, totaltests = fork_for_tests(self.processes, self) + try: + threads = {} + queue = Queue() + semaphore = threading.Semaphore(1) + result.threadprogress = {} + for i, (testserver, testnum, output) in enumerate(testservers): + result.threadprogress[i] = [] + process_result = BBThreadsafeForwardingResult( + ExtraResultsDecoderTestResult(result), + semaphore, i, testnum, totaltests, output, result) + reader_thread = threading.Thread( + target=self._run_test, args=(testserver, process_result, queue)) + threads[testserver] = reader_thread, process_result + reader_thread.start() + while threads: + finished_test = queue.get() + threads[finished_test][0].join() + del threads[finished_test] + except: + for thread, process_result in threads.values(): + process_result.stop() + raise + finally: + for testserver in testservers: + testserver[0]._stream.close() + + def _run_test(self, testserver, process_result, queue): + try: + try: + testserver.run(process_result) + except Exception: + # The run logic itself failed + case = testtools.ErrorHolder( + "broken-runner", + error=sys.exc_info()) + case.run(process_result) + finally: + queue.put(testserver) + +def fork_for_tests(concurrency_num, suite): + testservers = [] + if 'BUILDDIR' in os.environ: + selftestdir = get_test_layer(suite.bb_vars['BBLAYERS']) + + test_blocks = partition_tests(suite, concurrency_num) + # Clear the tests from the original suite so it doesn't keep them alive + suite._tests[:] = [] + totaltests = sum(len(x) for x in test_blocks) + for process_tests in test_blocks: + numtests = len(process_tests) + process_suite = unittest.TestSuite(process_tests) + # Also clear each split list so new suite has only reference + process_tests[:] = [] + c2pread, c2pwrite = os.pipe() + # Clear buffers before fork to avoid duplicate output + sys.stdout.flush() + sys.stderr.flush() + pid = os.fork() + if pid == 0: + ourpid = os.getpid() + try: + newbuilddir = None + stream = os.fdopen(c2pwrite, 'wb') + os.close(c2pread) + + (builddir, newbuilddir) = suite.setupfunc("-st-" + str(ourpid), selftestdir, process_suite) + + # Leave stderr and stdout open so we can see test noise + # Close stdin so that the child goes away if it decides to + # read from stdin (otherwise its a roulette to see what + # child actually gets keystrokes for pdb etc). + newsi = os.open(os.devnull, os.O_RDWR) + os.dup2(newsi, sys.stdin.fileno()) + + # Send stdout/stderr over the stream + os.dup2(c2pwrite, sys.stdout.fileno()) + os.dup2(c2pwrite, sys.stderr.fileno()) + + subunit_client = TestProtocolClient(stream) + subunit_result = AutoTimingTestResultDecorator(subunit_client) + unittest_result = process_suite.run(ExtraResultsEncoderTestResult(subunit_result)) + if ourpid != os.getpid(): + os._exit(0) + if newbuilddir and unittest_result.wasSuccessful(): + suite.removefunc(newbuilddir) + except: + # Don't do anything with process children + if ourpid != os.getpid(): + os._exit(1) + # Try and report traceback on stream, but exit with error + # even if stream couldn't be created or something else + # goes wrong. The traceback is formatted to a string and + # written in one go to avoid interleaving lines from + # multiple failing children. + try: + stream.write(traceback.format_exc().encode('utf-8')) + except: + sys.stderr.write(traceback.format_exc()) + finally: + if newbuilddir: + suite.removefunc(newbuilddir) + stream.flush() + os._exit(1) + stream.flush() + os._exit(0) + else: + os.close(c2pwrite) + stream = os.fdopen(c2pread, 'rb') + # Collect stdout/stderr into an io buffer + output = io.BytesIO() + testserver = ProtocolTestCase(stream, passthrough=output) + testservers.append((testserver, numtests, output)) + return testservers, totaltests + +def partition_tests(suite, count): + # Keep tests from the same class together but allow tests from modules + # to go to different processes to aid parallelisation. + modules = {} + for test in iterate_tests(suite): + m = test.__module__ + "." + test.__class__.__name__ + if m not in modules: + modules[m] = [] + modules[m].append(test) + + # Simply divide the test blocks between the available processes + partitions = [list() for _ in range(count)] + for partition, m in zip(cycle(partitions), modules): + partition.extend(modules[m]) + + # No point in empty threads so drop them + return [p for p in partitions if p] + diff --git a/meta/lib/oeqa/core/utils/misc.py b/meta/lib/oeqa/core/utils/misc.py deleted file mode 100644 index 0b223b5d08..0000000000 --- a/meta/lib/oeqa/core/utils/misc.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) - -def toList(obj, obj_type, obj_name="Object"): - if isinstance(obj, obj_type): - return [obj] - elif isinstance(obj, list): - return obj - else: - raise TypeError("%s must be %s or list" % (obj_name, obj_type)) - -def toSet(obj, obj_type, obj_name="Object"): - if isinstance(obj, obj_type): - return {obj} - elif isinstance(obj, list): - return set(obj) - elif isinstance(obj, set): - return obj - else: - raise TypeError("%s must be %s or set" % (obj_name, obj_type)) - -def strToList(obj, obj_name="Object"): - return toList(obj, str, obj_name) - -def strToSet(obj, obj_name="Object"): - return toSet(obj, str, obj_name) - -def intToList(obj, obj_name="Object"): - return toList(obj, int, obj_name) - -def dataStoteToDict(d, variables): - data = {} - - for v in variables: - data[v] = d.getVar(v) - - return data - -def updateTestData(d, td, variables): - """ - Updates variables with values of data store to test data. - """ - for var in variables: - td[var] = d.getVar(var) diff --git a/meta/lib/oeqa/core/utils/path.py b/meta/lib/oeqa/core/utils/path.py index a21caad5cb..c086dcb0b0 100644 --- a/meta/lib/oeqa/core/utils/path.py +++ b/meta/lib/oeqa/core/utils/path.py @@ -1,5 +1,8 @@ +# # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# import os import sys diff --git a/meta/lib/oeqa/core/utils/test.py b/meta/lib/oeqa/core/utils/test.py index 88d5d13981..d38cab8a51 100644 --- a/meta/lib/oeqa/core/utils/test.py +++ b/meta/lib/oeqa/core/utils/test.py @@ -1,5 +1,8 @@ +# # Copyright (C) 2016 Intel Corporation -# Released under the MIT license (see COPYING.MIT) +# +# SPDX-License-Identifier: MIT +# import os import inspect |