diff options
Diffstat (limited to 'meta/lib/oeqa/utils')
| -rw-r--r-- | meta/lib/oeqa/utils/__init__.py | 70 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/buildproject.py | 63 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/commands.py | 201 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/decorators.py | 11 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/dump.py | 22 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/ftools.py | 4 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/git.py | 24 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/gitarchive.py | 237 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/httpserver.py | 43 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/logparser.py | 249 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/metadata.py | 124 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/network.py | 8 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/nfs.py | 39 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/package_manager.py | 200 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/qemurunner.py | 461 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/qemutinyrunner.py | 20 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/sshcontrol.py | 38 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/subprocesstweak.py | 22 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/targetbuild.py | 33 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/testexport.py | 18 |
20 files changed, 1442 insertions, 445 deletions
diff --git a/meta/lib/oeqa/utils/__init__.py b/meta/lib/oeqa/utils/__init__.py index 8f706f3637..70fbe7b552 100644 --- a/meta/lib/oeqa/utils/__init__.py +++ b/meta/lib/oeqa/utils/__init__.py @@ -1,8 +1,10 @@ +# +# SPDX-License-Identifier: MIT +# # Enable other layers to have modules in the same named directory from pkgutil import extend_path __path__ = extend_path(__path__, __name__) - # Borrowed from CalledProcessError class CommandError(Exception): @@ -36,3 +38,69 @@ def avoid_paths_in_environ(paths): new_path = new_path[:-1] return new_path + +def make_logger_bitbake_compatible(logger): + import logging + + """ + Bitbake logger redifines debug() in order to + set a level within debug, this breaks compatibility + with vainilla logging, so we neeed to redifine debug() + method again also add info() method with INFO + 1 level. + """ + def _bitbake_log_debug(*args, **kwargs): + lvl = logging.DEBUG + + if isinstance(args[0], int): + lvl = args[0] + msg = args[1] + args = args[2:] + else: + msg = args[0] + args = args[1:] + + logger.log(lvl, msg, *args, **kwargs) + + def _bitbake_log_info(msg, *args, **kwargs): + logger.log(logging.INFO + 1, msg, *args, **kwargs) + + logger.debug = _bitbake_log_debug + logger.info = _bitbake_log_info + + return logger + +def load_test_components(logger, executor): + import sys + import os + import importlib + + from oeqa.core.context import OETestContextExecutor + + components = {} + + for path in sys.path: + base_dir = os.path.join(path, 'oeqa') + if os.path.exists(base_dir) and os.path.isdir(base_dir): + for file in os.listdir(base_dir): + comp_name = file + comp_context = os.path.join(base_dir, file, 'context.py') + if os.path.exists(comp_context): + comp_plugin = importlib.import_module('oeqa.%s.%s' % \ + (comp_name, 'context')) + try: + if not issubclass(comp_plugin._executor_class, + OETestContextExecutor): + raise TypeError("Component %s in %s, _executor_class "\ + "isn't derived from OETestContextExecutor."\ + % (comp_name, comp_context)) + + if comp_plugin._executor_class._script_executor \ + != executor: + continue + + components[comp_name] = comp_plugin._executor_class() + except AttributeError: + raise AttributeError("Component %s in %s don't have "\ + "_executor_class defined." % (comp_name, comp_context)) + + return components diff --git a/meta/lib/oeqa/utils/buildproject.py b/meta/lib/oeqa/utils/buildproject.py new file mode 100644 index 0000000000..e6d80cc8dc --- /dev/null +++ b/meta/lib/oeqa/utils/buildproject.py @@ -0,0 +1,63 @@ +# +# Copyright (C) 2013-2016 Intel Corporation +# +# SPDX-License-Identifier: MIT +# + +# Provides a class for automating build tests for projects + +import os +import re +import subprocess +import shutil +import tempfile + +from abc import ABCMeta, abstractmethod + +class BuildProject(metaclass=ABCMeta): + def __init__(self, uri, foldername=None, tmpdir=None, dl_dir=None): + self.uri = uri + self.archive = os.path.basename(uri) + if not tmpdir: + self.tempdirobj = tempfile.TemporaryDirectory(prefix='buildproject-') + tmpdir = self.tempdirobj.name + self.localarchive = os.path.join(tmpdir, self.archive) + self.dl_dir = dl_dir + if foldername: + self.fname = foldername + else: + self.fname = re.sub(r'\.tar\.bz2$|\.tar\.gz$|\.tar\.xz$', '', self.archive) + self.needclean = False + + # Download self.archive to self.localarchive + def _download_archive(self): + + self.needclean = True + if self.dl_dir and os.path.exists(os.path.join(self.dl_dir, self.archive)): + shutil.copyfile(os.path.join(self.dl_dir, self.archive), self.localarchive) + return + + cmd = "wget -O %s %s" % (self.localarchive, self.uri) + subprocess.check_output(cmd, shell=True) + + # This method should provide a way to run a command in the desired environment. + @abstractmethod + def _run(self, cmd): + pass + + # The timeout parameter of target.run is set to 0 to make the ssh command + # run with no timeout. + def run_configure(self, configure_args='', extra_cmds=''): + return self._run('cd %s; gnu-configize; %s ./configure %s' % (self.targetdir, extra_cmds, configure_args)) + + def run_make(self, make_args=''): + return self._run('cd %s; make %s' % (self.targetdir, make_args)) + + def run_install(self, install_args=''): + return self._run('cd %s; make install %s' % (self.targetdir, install_args)) + + def clean(self): + if not self.needclean: + return + self._run('rm -rf %s' % self.targetdir) + subprocess.check_call('rm -f %s' % self.localarchive, shell=True) diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py index 5cd0f7477b..dc1e286dac 100644 --- a/meta/lib/oeqa/utils/commands.py +++ b/meta/lib/oeqa/utils/commands.py @@ -1,6 +1,8 @@ +# # Copyright (c) 2013-2014 Intel Corporation # -# Released under the MIT license (see COPYING.MIT) +# SPDX-License-Identifier: MIT +# # DESCRIPTION # This module is mainly used by scripts/oe-selftest and modules under meta/oeqa/selftest @@ -13,6 +15,7 @@ import sys import signal import subprocess import threading +import time import logging from oeqa.utils import CommandError from oeqa.utils import ftools @@ -25,7 +28,7 @@ except ImportError: pass class Command(object): - def __init__(self, command, bg=False, timeout=None, data=None, **options): + def __init__(self, command, bg=False, timeout=None, data=None, output_log=None, **options): self.defaultopts = { "stdout": subprocess.PIPE, @@ -48,41 +51,106 @@ class Command(object): self.options.update(options) self.status = None + # We collect chunks of output before joining them at the end. + self._output_chunks = [] + self._error_chunks = [] self.output = None self.error = None - self.thread = None + self.threads = [] + self.output_log = output_log self.log = logging.getLogger("utils.commands") def run(self): self.process = subprocess.Popen(self.cmd, **self.options) - def commThread(): - self.output, self.error = self.process.communicate(self.data) - - self.thread = threading.Thread(target=commThread) - self.thread.start() + def readThread(output, stream, logfunc): + if logfunc: + for line in stream: + output.append(line) + logfunc(line.decode("utf-8", errors='replace').rstrip()) + else: + output.append(stream.read()) + + def readStderrThread(): + readThread(self._error_chunks, self.process.stderr, self.output_log.error if self.output_log else None) + + def readStdoutThread(): + readThread(self._output_chunks, self.process.stdout, self.output_log.info if self.output_log else None) + + def writeThread(): + try: + self.process.stdin.write(self.data) + self.process.stdin.close() + except OSError as ex: + # It's not an error when the command does not consume all + # of our data. subprocess.communicate() also ignores that. + if ex.errno != EPIPE: + raise + + # We write in a separate thread because then we can read + # without worrying about deadlocks. The additional thread is + # expected to terminate by itself and we mark it as a daemon, + # so even it should happen to not terminate for whatever + # reason, the main process will still exit, which will then + # kill the write thread. + if self.data: + threading.Thread(target=writeThread, daemon=True).start() + if self.process.stderr: + thread = threading.Thread(target=readStderrThread) + thread.start() + self.threads.append(thread) + if self.output_log: + self.output_log.info('Running: %s' % self.cmd) + thread = threading.Thread(target=readStdoutThread) + thread.start() + self.threads.append(thread) self.log.debug("Running command '%s'" % self.cmd) if not self.bg: - self.thread.join(self.timeout) + if self.timeout is None: + for thread in self.threads: + thread.join() + else: + deadline = time.time() + self.timeout + for thread in self.threads: + timeout = deadline - time.time() + if timeout < 0: + timeout = 0 + thread.join(timeout) self.stop() def stop(self): - if self.thread.isAlive(): - self.process.terminate() + for thread in self.threads: + if thread.isAlive(): + self.process.terminate() # let's give it more time to terminate gracefully before killing it - self.thread.join(5) - if self.thread.isAlive(): + thread.join(5) + if thread.isAlive(): self.process.kill() - self.thread.join() + thread.join() - if not self.output: - self.output = "" - else: - self.output = self.output.decode("utf-8", errors='replace').rstrip() - self.status = self.process.poll() + def finalize_output(data): + if not data: + data = "" + else: + data = b"".join(data) + data = data.decode("utf-8", errors='replace').rstrip() + return data + + self.output = finalize_output(self._output_chunks) + self._output_chunks = None + # self.error used to be a byte string earlier, probably unintentionally. + # Now it is a normal string, just like self.output. + self.error = finalize_output(self._error_chunks) + self._error_chunks = None + # At this point we know that the process has closed stdout/stderr, so + # it is safe and necessary to wait for the actual process completion. + self.status = self.process.wait() + self.process.stdout.close() + if self.process.stderr: + self.process.stderr.close() self.log.debug("Command '%s' returned %d as exit code." % (self.cmd, self.status)) # logging the complete output is insane @@ -97,10 +165,21 @@ class Result(object): pass -def runCmd(command, ignore_status=False, timeout=None, assert_error=True, **options): +def runCmd(command, ignore_status=False, timeout=None, assert_error=True, + native_sysroot=None, limit_exc_output=0, output_log=None, **options): result = Result() - cmd = Command(command, timeout=timeout, **options) + if native_sysroot: + extra_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin" % \ + (native_sysroot, native_sysroot, native_sysroot) + extra_libpaths = "%s/lib:%s/usr/lib" % \ + (native_sysroot, native_sysroot) + nenv = dict(options.get('env', os.environ)) + nenv['PATH'] = extra_paths + ':' + nenv.get('PATH', '') + nenv['LD_LIBRARY_PATH'] = extra_libpaths + ':' + nenv.get('LD_LIBRARY_PATH', '') + options['env'] = nenv + + cmd = Command(command, timeout=timeout, output_log=output_log, **options) cmd.run() result.command = command @@ -110,15 +189,21 @@ def runCmd(command, ignore_status=False, timeout=None, assert_error=True, **opti result.pid = cmd.process.pid if result.status and not ignore_status: + exc_output = result.output + if limit_exc_output > 0: + split = result.output.splitlines() + if len(split) > limit_exc_output: + exc_output = "\n... (last %d lines of output)\n" % limit_exc_output + \ + '\n'.join(split[-limit_exc_output:]) if assert_error: - raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, result.output)) + raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, exc_output)) else: - raise CommandError(result.status, command, result.output) + raise CommandError(result.status, command, exc_output) return result -def bitbake(command, ignore_status=False, timeout=None, postconfig=None, **options): +def bitbake(command, ignore_status=False, timeout=None, postconfig=None, output_log=None, **options): if postconfig: postconfig_file = os.path.join(os.environ.get('BUILDDIR'), 'oeqa-post.conf') @@ -133,7 +218,7 @@ def bitbake(command, ignore_status=False, timeout=None, postconfig=None, **optio cmd = [ "bitbake" ] + [a for a in (command + extra_args.split(" ")) if a not in [""]] try: - return runCmd(cmd, ignore_status, timeout, **options) + return runCmd(cmd, ignore_status, timeout, output_log=output_log, **options) finally: if postconfig: os.remove(postconfig_file) @@ -149,7 +234,9 @@ def get_bb_vars(variables=None, target=None, postconfig=None): """Get values of multiple bitbake variables""" bbenv = get_bb_env(target, postconfig=postconfig) - var_re = re.compile(r'^(export )?(?P<var>\w+)="(?P<value>.*)"$') + if variables is not None: + variables = list(variables) + var_re = re.compile(r'^(export )?(?P<var>\w+(_.*)?)="(?P<value>.*)"$') unset_re = re.compile(r'^unset (?P<var>\w+)$') lastline = None values = {} @@ -206,60 +293,64 @@ def create_temp_layer(templayerdir, templayername, priority=999, recipepathspec= f.write('BBFILE_PATTERN_%s = "^${LAYERDIR}/"\n' % templayername) f.write('BBFILE_PRIORITY_%s = "%d"\n' % (templayername, priority)) f.write('BBFILE_PATTERN_IGNORE_EMPTY_%s = "1"\n' % templayername) - + f.write('LAYERSERIES_COMPAT_%s = "${LAYERSERIES_COMPAT_core}"\n' % templayername) @contextlib.contextmanager -def runqemu(pn, ssh=True): +def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None, qemuparams=None, overrides={}, discard_writes=True): + """ + launch_cmd means directly run the command, don't need set rootfs or env vars. + """ import bb.tinfoil import bb.build + # Need a non-'BitBake' logger to capture the runner output + targetlogger = logging.getLogger('TargetRunner') + targetlogger.setLevel(logging.DEBUG) + handler = logging.StreamHandler(sys.stdout) + targetlogger.addHandler(handler) + tinfoil = bb.tinfoil.Tinfoil() - tinfoil.prepare(False) + tinfoil.prepare(config_only=False, quiet=True) try: tinfoil.logger.setLevel(logging.WARNING) import oeqa.targetcontrol tinfoil.config_data.setVar("TEST_LOG_DIR", "${WORKDIR}/testimage") tinfoil.config_data.setVar("TEST_QEMUBOOT_TIMEOUT", "1000") - import oe.recipeutils - recipefile = oe.recipeutils.pn_to_recipe(tinfoil.cooker, pn) - recipedata = oe.recipeutils.parse_recipe(tinfoil.cooker, recipefile, []) - - # The QemuRunner log is saved out, but we need to ensure it is at the right - # log level (and then ensure that since it's a child of the BitBake logger, - # we disable propagation so we don't then see the log events on the console) - logger = logging.getLogger('BitBake.QemuRunner') - logger.setLevel(logging.DEBUG) - logger.propagate = False - logdir = recipedata.getVar("TEST_LOG_DIR", True) - - qemu = oeqa.targetcontrol.QemuTarget(recipedata) + # Tell QemuTarget() whether need find rootfs/kernel or not + if launch_cmd: + tinfoil.config_data.setVar("FIND_ROOTFS", '0') + else: + tinfoil.config_data.setVar("FIND_ROOTFS", '1') + + recipedata = tinfoil.parse_recipe(pn) + for key, value in overrides.items(): + recipedata.setVar(key, value) + + logdir = recipedata.getVar("TEST_LOG_DIR") + + qemu = oeqa.targetcontrol.QemuTarget(recipedata, targetlogger, image_fstype) finally: # We need to shut down tinfoil early here in case we actually want # to run tinfoil-using utilities with the running QEMU instance. # Luckily QemuTarget doesn't need it after the constructor. tinfoil.shutdown() - # Setup bitbake logger as console handler is removed by tinfoil.shutdown - bblogger = logging.getLogger('BitBake') - bblogger.setLevel(logging.INFO) - console = logging.StreamHandler(sys.stdout) - bbformat = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") - if sys.stdout.isatty(): - bbformat.enable_color() - console.setFormatter(bbformat) - bblogger.addHandler(console) - try: qemu.deploy() try: - qemu.start(ssh=ssh) - except bb.build.FuncFailed: - raise Exception('Failed to start QEMU - see the logs in %s' % logdir) + qemu.start(params=qemuparams, ssh=ssh, runqemuparams=runqemuparams, launch_cmd=launch_cmd, discard_writes=discard_writes) + except Exception as e: + msg = str(e) + '\nFailed to start QEMU - see the logs in %s' % logdir + if os.path.exists(qemu.qemurunnerlog): + with open(qemu.qemurunnerlog, 'r') as f: + msg = msg + "Qemurunner log output from %s:\n%s" % (qemu.qemurunnerlog, f.read()) + raise Exception(msg) yield qemu finally: + targetlogger.removeHandler(handler) try: qemu.stop() except: diff --git a/meta/lib/oeqa/utils/decorators.py b/meta/lib/oeqa/utils/decorators.py index 25f9c54e6b..aabf4110cb 100644 --- a/meta/lib/oeqa/utils/decorators.py +++ b/meta/lib/oeqa/utils/decorators.py @@ -1,6 +1,8 @@ +# # Copyright (C) 2013 Intel Corporation # -# Released under the MIT license (see COPYING.MIT) +# SPDX-License-Identifier: MIT +# # Some custom decorators that can be used by unittests # Most useful is skipUnlessPassed which can be used for @@ -172,18 +174,19 @@ def LogResults(original_class): #check status of tests and record it + tcid = self.id() for (name, msg) in result.errors: - if (self._testMethodName == str(name).split(' ')[0]) and (class_name in str(name).split(' ')[1]): + if tcid == name.id(): local_log.results("Testcase "+str(test_case)+": ERROR") local_log.results("Testcase "+str(test_case)+":\n"+msg) passed = False for (name, msg) in result.failures: - if (self._testMethodName == str(name).split(' ')[0]) and (class_name in str(name).split(' ')[1]): + if tcid == name.id(): local_log.results("Testcase "+str(test_case)+": FAILED") local_log.results("Testcase "+str(test_case)+":\n"+msg) passed = False for (name, msg) in result.skipped: - if (self._testMethodName == str(name).split(' ')[0]) and (class_name in str(name).split(' ')[1]): + if tcid == name.id(): local_log.results("Testcase "+str(test_case)+": SKIPPED") passed = False if passed: diff --git a/meta/lib/oeqa/utils/dump.py b/meta/lib/oeqa/utils/dump.py index 71422a9aea..09a44329e0 100644 --- a/meta/lib/oeqa/utils/dump.py +++ b/meta/lib/oeqa/utils/dump.py @@ -1,3 +1,7 @@ +# +# SPDX-License-Identifier: MIT +# + import os import sys import errno @@ -5,12 +9,6 @@ import datetime import itertools from .commands import runCmd -def get_host_dumper(d): - cmds = d.getVar("testimage_dump_host", True) - parent_dir = d.getVar("TESTIMAGE_DUMP_DIR", True) - return HostDumper(cmds, parent_dir) - - class BaseDumper(object): """ Base class to dump commands from host/target """ @@ -18,7 +16,7 @@ class BaseDumper(object): self.cmds = [] # Some testing doesn't inherit testimage, so it is needed # to set some defaults. - self.parent_dir = parent_dir or "/tmp/oe-saved-tests" + self.parent_dir = parent_dir dft_cmds = """ top -bn1 iostat -x -z -N -d -p ALL 20 2 ps -ef @@ -73,17 +71,19 @@ class HostDumper(BaseDumper): def dump_host(self, dump_dir=""): if dump_dir: self.dump_dir = dump_dir + env = os.environ.copy() + env['PATH'] = '/usr/sbin:/sbin:/usr/bin:/bin' + env['COLUMNS'] = '9999' for cmd in self.cmds: - result = runCmd(cmd, ignore_status=True) + result = runCmd(cmd, ignore_status=True, env=env) self._write_dump(cmd.split()[0], result.output) - class TargetDumper(BaseDumper): """ Class to get dumps from target, it only works with QemuRunner """ - def __init__(self, cmds, parent_dir, qemurunner): + def __init__(self, cmds, parent_dir, runner): super(TargetDumper, self).__init__(cmds, parent_dir) - self.runner = qemurunner + self.runner = runner def dump_target(self, dump_dir=""): if dump_dir: diff --git a/meta/lib/oeqa/utils/ftools.py b/meta/lib/oeqa/utils/ftools.py index a7233d4ca6..3093419cc7 100644 --- a/meta/lib/oeqa/utils/ftools.py +++ b/meta/lib/oeqa/utils/ftools.py @@ -1,3 +1,7 @@ +# +# SPDX-License-Identifier: MIT +# + import os import re import errno diff --git a/meta/lib/oeqa/utils/git.py b/meta/lib/oeqa/utils/git.py index ae85d27663..ea35a766eb 100644 --- a/meta/lib/oeqa/utils/git.py +++ b/meta/lib/oeqa/utils/git.py @@ -1,7 +1,7 @@ # # Copyright (C) 2016 Intel Corporation # -# Released under the MIT license (see COPYING.MIT) +# SPDX-License-Identifier: MIT # """Git repository interactions""" import os @@ -16,8 +16,17 @@ class GitError(Exception): class GitRepo(object): """Class representing a Git repository clone""" def __init__(self, path, is_topdir=False): - self.top_dir = self._run_git_cmd_at(['rev-parse', '--show-toplevel'], - path) + git_dir = self._run_git_cmd_at(['rev-parse', '--git-dir'], path) + git_dir = git_dir if os.path.isabs(git_dir) else os.path.join(path, git_dir) + self.git_dir = os.path.realpath(git_dir) + + if self._run_git_cmd_at(['rev-parse', '--is-bare-repository'], path) == 'true': + self.bare = True + self.top_dir = self.git_dir + else: + self.bare = False + self.top_dir = self._run_git_cmd_at(['rev-parse', '--show-toplevel'], + path) realpath = os.path.realpath(path) if is_topdir and realpath != self.top_dir: raise GitError("{} is not a Git top directory".format(realpath)) @@ -36,9 +45,12 @@ class GitRepo(object): return ret.output.strip() @staticmethod - def init(path): + def init(path, bare=False): """Initialize a new Git repository""" - GitRepo._run_git_cmd_at('init', cwd=path) + cmd = ['init'] + if bare: + cmd.append('--bare') + GitRepo._run_git_cmd_at(cmd, cwd=path) return GitRepo(path, is_topdir=True) def run_cmd(self, git_args, env_update=None): @@ -52,7 +64,7 @@ class GitRepo(object): def rev_parse(self, revision): """Do git rev-parse""" try: - return self.run_cmd(['rev-parse', revision]) + return self.run_cmd(['rev-parse', '--verify', revision]) except GitError: # Revision does not exist return None diff --git a/meta/lib/oeqa/utils/gitarchive.py b/meta/lib/oeqa/utils/gitarchive.py new file mode 100644 index 0000000000..6e8040eb5c --- /dev/null +++ b/meta/lib/oeqa/utils/gitarchive.py @@ -0,0 +1,237 @@ +# +# Helper functions for committing data to git and pushing upstream +# +# Copyright (c) 2017, Intel Corporation. +# Copyright (c) 2019, Linux Foundation +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import os +import re +import sys +from operator import attrgetter +from collections import namedtuple +from oeqa.utils.git import GitRepo, GitError + +class ArchiveError(Exception): + """Internal error handling of this script""" + +def format_str(string, fields): + """Format string using the given fields (dict)""" + try: + return string.format(**fields) + except KeyError as err: + raise ArchiveError("Unable to expand string '{}': unknown field {} " + "(valid fields are: {})".format( + string, err, ', '.join(sorted(fields.keys())))) + + +def init_git_repo(path, no_create, bare, log): + """Initialize local Git repository""" + path = os.path.abspath(path) + if os.path.isfile(path): + raise ArchiveError("Invalid Git repo at {}: path exists but is not a " + "directory".format(path)) + if not os.path.isdir(path) or not os.listdir(path): + if no_create: + raise ArchiveError("No git repo at {}, refusing to create " + "one".format(path)) + if not os.path.isdir(path): + try: + os.mkdir(path) + except (FileNotFoundError, PermissionError) as err: + raise ArchiveError("Failed to mkdir {}: {}".format(path, err)) + if not os.listdir(path): + log.info("Initializing a new Git repo at %s", path) + repo = GitRepo.init(path, bare) + try: + repo = GitRepo(path, is_topdir=True) + except GitError: + raise ArchiveError("Non-empty directory that is not a Git repository " + "at {}\nPlease specify an existing Git repository, " + "an empty directory or a non-existing directory " + "path.".format(path)) + return repo + + +def git_commit_data(repo, data_dir, branch, message, exclude, notes, log): + """Commit data into a Git repository""" + log.info("Committing data into to branch %s", branch) + tmp_index = os.path.join(repo.git_dir, 'index.oe-git-archive') + try: + # Create new tree object from the data + env_update = {'GIT_INDEX_FILE': tmp_index, + 'GIT_WORK_TREE': os.path.abspath(data_dir)} + repo.run_cmd('add .', env_update) + + # Remove files that are excluded + if exclude: + repo.run_cmd(['rm', '--cached'] + [f for f in exclude], env_update) + + tree = repo.run_cmd('write-tree', env_update) + + # Create new commit object from the tree + parent = repo.rev_parse(branch) + if not parent: + parent = repo.rev_parse("origin/" + branch) + git_cmd = ['commit-tree', tree, '-m', message] + if parent: + git_cmd += ['-p', parent] + commit = repo.run_cmd(git_cmd, env_update) + + # Create git notes + for ref, filename in notes: + ref = ref.format(branch_name=branch) + repo.run_cmd(['notes', '--ref', ref, 'add', + '-F', os.path.abspath(filename), commit]) + + # Update branch head + git_cmd = ['update-ref', 'refs/heads/' + branch, commit] + repo.run_cmd(git_cmd) + + # Update current HEAD, if we're on branch 'branch' + if not repo.bare and repo.get_current_branch() == branch: + log.info("Updating %s HEAD to latest commit", repo.top_dir) + repo.run_cmd('reset --hard') + + return commit + finally: + if os.path.exists(tmp_index): + os.unlink(tmp_index) + + +def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern, + keywords): + """Generate tag name and message, with support for running id number""" + keyws = keywords.copy() + # Tag number is handled specially: if not defined, we autoincrement it + if 'tag_number' not in keyws: + # Fill in all other fields than 'tag_number' + keyws['tag_number'] = '{tag_number}' + tag_re = format_str(name_pattern, keyws) + # Replace parentheses for proper regex matching + tag_re = tag_re.replace('(', '\(').replace(')', '\)') + '$' + # Inject regex group pattern for 'tag_number' + tag_re = tag_re.format(tag_number='(?P<tag_number>[0-9]{1,5})') + + keyws['tag_number'] = 0 + for existing_tag in repo.run_cmd('tag').splitlines(): + match = re.match(tag_re, existing_tag) + + if match and int(match.group('tag_number')) >= keyws['tag_number']: + keyws['tag_number'] = int(match.group('tag_number')) + 1 + + tag_name = format_str(name_pattern, keyws) + msg_subj= format_str(msg_subj_pattern.strip(), keyws) + msg_body = format_str(msg_body_pattern, keyws) + return tag_name, msg_subj + '\n\n' + msg_body + +def gitarchive(data_dir, git_dir, no_create, bare, commit_msg_subject, commit_msg_body, branch_name, no_tag, tagname, tag_msg_subject, tag_msg_body, exclude, notes, push, keywords, log): + + if not os.path.isdir(data_dir): + raise ArchiveError("Not a directory: {}".format(data_dir)) + + data_repo = init_git_repo(git_dir, no_create, bare, log) + + # Expand strings early in order to avoid getting into inconsistent + # state (e.g. no tag even if data was committed) + commit_msg = format_str(commit_msg_subject.strip(), keywords) + commit_msg += '\n\n' + format_str(commit_msg_body, keywords) + branch_name = format_str(branch_name, keywords) + tag_name = None + if not no_tag and tagname: + tag_name, tag_msg = expand_tag_strings(data_repo, tagname, + tag_msg_subject, + tag_msg_body, keywords) + + # Commit data + commit = git_commit_data(data_repo, data_dir, branch_name, + commit_msg, exclude, notes, log) + + # Create tag + if tag_name: + log.info("Creating tag %s", tag_name) + data_repo.run_cmd(['tag', '-a', '-m', tag_msg, tag_name, commit]) + + # Push data to remote + if push: + cmd = ['push', '--tags'] + # If no remote is given we push with the default settings from + # gitconfig + if push is not True: + notes_refs = ['refs/notes/' + ref.format(branch_name=branch_name) + for ref, _ in notes] + cmd.extend([push, branch_name] + notes_refs) + log.info("Pushing data to remote") + data_repo.run_cmd(cmd) + +# Container class for tester revisions +TestedRev = namedtuple('TestedRev', 'commit commit_number tags') + +def get_test_runs(log, repo, tag_name, **kwargs): + """Get a sorted list of test runs, matching given pattern""" + # First, get field names from the tag name pattern + field_names = [m.group(1) for m in re.finditer(r'{(\w+)}', tag_name)] + undef_fields = [f for f in field_names if f not in kwargs.keys()] + + # Fields for formatting tag name pattern + str_fields = dict([(f, '*') for f in field_names]) + str_fields.update(kwargs) + + # Get a list of all matching tags + tag_pattern = tag_name.format(**str_fields) + tags = repo.run_cmd(['tag', '-l', tag_pattern]).splitlines() + log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern) + + # Parse undefined fields from tag names + str_fields = dict([(f, r'(?P<{}>[\w\-.()]+)'.format(f)) for f in field_names]) + str_fields['branch'] = r'(?P<branch>[\w\-.()/]+)' + str_fields['commit'] = '(?P<commit>[0-9a-f]{7,40})' + str_fields['commit_number'] = '(?P<commit_number>[0-9]{1,7})' + str_fields['tag_number'] = '(?P<tag_number>[0-9]{1,5})' + # escape parenthesis in fields in order to not messa up the regexp + fixed_fields = dict([(k, v.replace('(', r'\(').replace(')', r'\)')) for k, v in kwargs.items()]) + str_fields.update(fixed_fields) + tag_re = re.compile(tag_name.format(**str_fields)) + + # Parse fields from tags + revs = [] + for tag in tags: + m = tag_re.match(tag) + groups = m.groupdict() + revs.append([groups[f] for f in undef_fields] + [tag]) + + # Return field names and a sorted list of revs + return undef_fields, sorted(revs) + +def get_test_revs(log, repo, tag_name, **kwargs): + """Get list of all tested revisions""" + fields, runs = get_test_runs(log, repo, tag_name, **kwargs) + + revs = {} + commit_i = fields.index('commit') + commit_num_i = fields.index('commit_number') + for run in runs: + commit = run[commit_i] + commit_num = run[commit_num_i] + tag = run[-1] + if not commit in revs: + revs[commit] = TestedRev(commit, commit_num, [tag]) + else: + assert commit_num == revs[commit].commit_number, "Commit numbers do not match" + revs[commit].tags.append(tag) + + # Return in sorted table + revs = sorted(revs.values(), key=attrgetter('commit_number')) + log.debug("Found %d tested revisions:\n %s", len(revs), + "\n ".join(['{} ({})'.format(rev.commit_number, rev.commit) for rev in revs])) + return revs + +def rev_find(revs, attr, val): + """Search from a list of TestedRev""" + for i, rev in enumerate(revs): + if getattr(rev, attr) == val: + return i + raise ValueError("Unable to find '{}' value '{}'".format(attr, val)) + diff --git a/meta/lib/oeqa/utils/httpserver.py b/meta/lib/oeqa/utils/httpserver.py index 7d12331453..58d3c3b3f8 100644 --- a/meta/lib/oeqa/utils/httpserver.py +++ b/meta/lib/oeqa/utils/httpserver.py @@ -1,13 +1,17 @@ +# +# SPDX-License-Identifier: MIT +# + import http.server import multiprocessing import os +import traceback +import signal from socketserver import ThreadingMixIn class HTTPServer(ThreadingMixIn, http.server.HTTPServer): - def server_start(self, root_dir): - import signal - signal.signal(signal.SIGTERM, signal.SIG_DFL) + def server_start(self, root_dir, logger): os.chdir(root_dir) self.serve_forever() @@ -18,19 +22,40 @@ class HTTPRequestHandler(http.server.SimpleHTTPRequestHandler): class HTTPService(object): - def __init__(self, root_dir, host=''): + def __init__(self, root_dir, host='', port=0, logger=None): self.root_dir = root_dir self.host = host - self.port = 0 + self.port = port + self.logger = logger def start(self): + if not os.path.exists(self.root_dir): + self.logger.info("Not starting HTTPService for directory %s which doesn't exist" % (self.root_dir)) + return + self.server = HTTPServer((self.host, self.port), HTTPRequestHandler) if self.port == 0: self.port = self.server.server_port - self.process = multiprocessing.Process(target=self.server.server_start, args=[self.root_dir]) + self.process = multiprocessing.Process(target=self.server.server_start, args=[self.root_dir, self.logger]) + + # The signal handler from testimage.bbclass can cause deadlocks here + # if the HTTPServer is terminated before it can restore the standard + #signal behaviour + orig = signal.getsignal(signal.SIGTERM) + signal.signal(signal.SIGTERM, signal.SIG_DFL) self.process.start() + signal.signal(signal.SIGTERM, orig) + + if self.logger: + self.logger.info("Started HTTPService on %s:%s" % (self.host, self.port)) + def stop(self): - self.server.server_close() - self.process.terminate() - self.process.join() + if hasattr(self, "server"): + self.server.server_close() + if hasattr(self, "process"): + self.process.terminate() + self.process.join() + if self.logger: + self.logger.info("Stopped HTTPService on %s:%s" % (self.host, self.port)) + diff --git a/meta/lib/oeqa/utils/logparser.py b/meta/lib/oeqa/utils/logparser.py index b377dcd271..7313df8ec3 100644 --- a/meta/lib/oeqa/utils/logparser.py +++ b/meta/lib/oeqa/utils/logparser.py @@ -1,125 +1,152 @@ -#!/usr/bin/env python +# +# SPDX-License-Identifier: MIT +# import sys import os import re -from . import ftools - # A parser that can be used to identify weather a line is a test result or a section statement. -class Lparser(object): - - def __init__(self, test_0_pass_regex, test_0_fail_regex, section_0_begin_regex=None, section_0_end_regex=None, **kwargs): - # Initialize the arguments dictionary - if kwargs: - self.args = kwargs - else: - self.args = {} - - # Add the default args to the dictionary - self.args['test_0_pass_regex'] = test_0_pass_regex - self.args['test_0_fail_regex'] = test_0_fail_regex - if section_0_begin_regex: - self.args['section_0_begin_regex'] = section_0_begin_regex - if section_0_end_regex: - self.args['section_0_end_regex'] = section_0_end_regex - - self.test_possible_status = ['pass', 'fail', 'error'] - self.section_possible_status = ['begin', 'end'] - - self.initialized = False - - - # Initialize the parser with the current configuration - def init(self): - - # extra arguments can be added by the user to define new test and section categories. They must follow a pre-defined pattern: <type>_<category_name>_<status>_regex - self.test_argument_pattern = "^test_(.+?)_(%s)_regex" % '|'.join(map(str, self.test_possible_status)) - self.section_argument_pattern = "^section_(.+?)_(%s)_regex" % '|'.join(map(str, self.section_possible_status)) - - # Initialize the test and section regex dictionaries - self.test_regex = {} - self.section_regex ={} - - for arg, value in self.args.items(): - if not value: - raise Exception('The value of provided argument %s is %s. Should have a valid value.' % (key, value)) - is_test = re.search(self.test_argument_pattern, arg) - is_section = re.search(self.section_argument_pattern, arg) - if is_test: - if not is_test.group(1) in self.test_regex: - self.test_regex[is_test.group(1)] = {} - self.test_regex[is_test.group(1)][is_test.group(2)] = re.compile(value) - elif is_section: - if not is_section.group(1) in self.section_regex: - self.section_regex[is_section.group(1)] = {} - self.section_regex[is_section.group(1)][is_section.group(2)] = re.compile(value) - else: - # TODO: Make these call a traceback instead of a simple exception.. - raise Exception("The provided argument name does not correspond to any valid type. Please give one of the following types:\nfor tests: %s\nfor sections: %s" % (self.test_argument_pattern, self.section_argument_pattern)) - - self.initialized = True - - # Parse a line and return a tuple containing the type of result (test/section) and its category, status and name - def parse_line(self, line): - if not self.initialized: - raise Exception("The parser is not initialized..") - - for test_category, test_status_list in self.test_regex.items(): - for test_status, status_regex in test_status_list.items(): - test_name = status_regex.search(line) - if test_name: - return ['test', test_category, test_status, test_name.group(1)] - - for section_category, section_status_list in self.section_regex.items(): - for section_status, status_regex in section_status_list.items(): - section_name = status_regex.search(line) - if section_name: - return ['section', section_category, section_status, section_name.group(1)] - return None - - -class Result(object): - +class PtestParser(object): def __init__(self): - self.result_dict = {} - - def store(self, section, test, status): - if not section in self.result_dict: - self.result_dict[section] = [] - - self.result_dict[section].append((test, status)) - - # sort tests by the test name(the first element of the tuple), for each section. This can be helpful when using git to diff for changes by making sure they are always in the same order. - def sort_tests(self): - for package in self.result_dict: - sorted_results = sorted(self.result_dict[package], key=lambda tup: tup[0]) - self.result_dict[package] = sorted_results + self.results = {} + self.sections = {} + + def parse(self, logfile): + test_regex = {} + test_regex['PASSED'] = re.compile(r"^PASS:(.+)") + test_regex['FAILED'] = re.compile(r"^FAIL:([^(]+)") + test_regex['SKIPPED'] = re.compile(r"^SKIP:(.+)") + + section_regex = {} + section_regex['begin'] = re.compile(r"^BEGIN: .*/(.+)/ptest") + section_regex['end'] = re.compile(r"^END: .*/(.+)/ptest") + section_regex['duration'] = re.compile(r"^DURATION: (.+)") + section_regex['exitcode'] = re.compile(r"^ERROR: Exit status is (.+)") + section_regex['timeout'] = re.compile(r"^TIMEOUT: .*/(.+)/ptest") + + def newsection(): + return { 'name': "No-section", 'log': "" } + + current_section = newsection() + + with open(logfile, errors='replace') as f: + for line in f: + result = section_regex['begin'].search(line) + if result: + current_section['name'] = result.group(1) + continue + + result = section_regex['end'].search(line) + if result: + if current_section['name'] != result.group(1): + bb.warn("Ptest END log section mismatch %s vs. %s" % (current_section['name'], result.group(1))) + if current_section['name'] in self.sections: + bb.warn("Ptest duplicate section for %s" % (current_section['name'])) + self.sections[current_section['name']] = current_section + del self.sections[current_section['name']]['name'] + current_section = newsection() + continue + + result = section_regex['timeout'].search(line) + if result: + if current_section['name'] != result.group(1): + bb.warn("Ptest TIMEOUT log section mismatch %s vs. %s" % (current_section['name'], result.group(1))) + current_section['timeout'] = True + continue + + for t in ['duration', 'exitcode']: + result = section_regex[t].search(line) + if result: + current_section[t] = result.group(1) + continue + + current_section['log'] = current_section['log'] + line + + for t in test_regex: + result = test_regex[t].search(line) + if result: + if current_section['name'] not in self.results: + self.results[current_section['name']] = {} + self.results[current_section['name']][result.group(1).strip()] = t + + return self.results, self.sections # Log the results as files. The file name is the section name and the contents are the tests in that section. - def log_as_files(self, target_dir, test_status): - status_regex = re.compile('|'.join(map(str, test_status))) - if not type(test_status) == type([]): - raise Exception("test_status should be a list. Got " + str(test_status) + " instead.") + def results_as_files(self, target_dir): if not os.path.exists(target_dir): raise Exception("Target directory does not exist: %s" % target_dir) - for section, test_results in self.result_dict.items(): - prefix = '' - for x in test_status: - prefix +=x+'.' - if (section != ''): - prefix += section + for section in self.results: + prefix = 'No-section' + if section: + prefix = section section_file = os.path.join(target_dir, prefix) # purge the file contents if it exists - open(section_file, 'w').close() - for test_result in test_results: - (test_name, status) = test_result - # we log only the tests with status in the test_status list - match_status = status_regex.search(status) - if match_status: - ftools.append_file(section_file, status + ": " + test_name) - - # Not yet implemented! - def log_to_lava(self): - pass + with open(section_file, 'w') as f: + for test_name in sorted(self.results[section]): + status = self.results[section][test_name] + f.write(status + ": " + test_name + "\n") + + +# ltp log parsing +class LtpParser(object): + def __init__(self): + self.results = {} + self.section = {'duration': "", 'log': ""} + + def parse(self, logfile): + test_regex = {} + test_regex['PASSED'] = re.compile(r"PASS") + test_regex['FAILED'] = re.compile(r"FAIL") + test_regex['SKIPPED'] = re.compile(r"SKIP") + + with open(logfile, errors='replace') as f: + for line in f: + for t in test_regex: + result = test_regex[t].search(line) + if result: + self.results[line.split()[0].strip()] = t + + for test in self.results: + result = self.results[test] + self.section['log'] = self.section['log'] + ("%s: %s\n" % (result.strip()[:-2], test.strip())) + + return self.results, self.section + + +# ltp Compliance log parsing +class LtpComplianceParser(object): + def __init__(self): + self.results = {} + self.section = {'duration': "", 'log': ""} + + def parse(self, logfile): + test_regex = {} + test_regex['PASSED'] = re.compile(r"^PASS") + test_regex['FAILED'] = re.compile(r"^FAIL") + test_regex['SKIPPED'] = re.compile(r"(?:UNTESTED)|(?:UNSUPPORTED)") + + section_regex = {} + section_regex['test'] = re.compile(r"^Testing") + + with open(logfile, errors='replace') as f: + for line in f: + result = section_regex['test'].search(line) + if result: + self.name = "" + self.name = line.split()[1].strip() + self.results[self.name] = "PASSED" + failed = 0 + + failed_result = test_regex['FAILED'].search(line) + if failed_result: + failed = line.split()[1].strip() + if int(failed) > 0: + self.results[self.name] = "FAILED" + + for test in self.results: + result = self.results[test] + self.section['log'] = self.section['log'] + ("%s: %s\n" % (result.strip()[:-2], test.strip())) + + return self.results, self.section diff --git a/meta/lib/oeqa/utils/metadata.py b/meta/lib/oeqa/utils/metadata.py new file mode 100644 index 0000000000..8013aa684d --- /dev/null +++ b/meta/lib/oeqa/utils/metadata.py @@ -0,0 +1,124 @@ +# Copyright (C) 2016 Intel Corporation +# +# SPDX-License-Identifier: MIT +# +# Functions to get metadata from the testing host used +# for analytics of test results. + +from collections import OrderedDict +from collections.abc import MutableMapping +from xml.dom.minidom import parseString +from xml.etree.ElementTree import Element, tostring + +from oe.lsb import get_os_release +from oeqa.utils.commands import runCmd, get_bb_vars + + +def metadata_from_bb(): + """ Returns test's metadata as OrderedDict. + + Data will be gathered using bitbake -e thanks to get_bb_vars. + """ + metadata_config_vars = ('MACHINE', 'BB_NUMBER_THREADS', 'PARALLEL_MAKE') + + info_dict = OrderedDict() + hostname = runCmd('hostname') + info_dict['hostname'] = hostname.output + data_dict = get_bb_vars() + + # Distro information + info_dict['distro'] = {'id': data_dict['DISTRO'], + 'version_id': data_dict['DISTRO_VERSION'], + 'pretty_name': '%s %s' % (data_dict['DISTRO'], data_dict['DISTRO_VERSION'])} + + # Host distro information + os_release = get_os_release() + if os_release: + info_dict['host_distro'] = OrderedDict() + for key in ('ID', 'VERSION_ID', 'PRETTY_NAME'): + if key in os_release: + info_dict['host_distro'][key.lower()] = os_release[key] + + info_dict['layers'] = get_layers(data_dict['BBLAYERS']) + info_dict['bitbake'] = git_rev_info(os.path.dirname(bb.__file__)) + + info_dict['config'] = OrderedDict() + for var in sorted(metadata_config_vars): + info_dict['config'][var] = data_dict[var] + return info_dict + +def metadata_from_data_store(d): + """ Returns test's metadata as OrderedDict. + + Data will be collected from the provided data store. + """ + # TODO: Getting metadata from the data store would + # be useful when running within bitbake. + pass + +def git_rev_info(path): + """Get git revision information as a dict""" + info = OrderedDict() + + try: + from git import Repo, InvalidGitRepositoryError, NoSuchPathError + except ImportError: + import subprocess + try: + info['branch'] = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=path).decode('utf-8').strip() + except subprocess.CalledProcessError: + pass + try: + info['commit'] = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=path).decode('utf-8').strip() + except subprocess.CalledProcessError: + pass + try: + info['commit_count'] = int(subprocess.check_output(["git", "rev-list", "--count", "HEAD"], cwd=path).decode('utf-8').strip()) + except subprocess.CalledProcessError: + pass + return info + try: + repo = Repo(path, search_parent_directories=True) + except (InvalidGitRepositoryError, NoSuchPathError): + return info + info['commit'] = repo.head.commit.hexsha + info['commit_count'] = repo.head.commit.count() + try: + info['branch'] = repo.active_branch.name + except TypeError: + info['branch'] = '(nobranch)' + return info + +def get_layers(layers): + """Returns layer information in dict format""" + layer_dict = OrderedDict() + for layer in layers.split(): + layer_name = os.path.basename(layer) + layer_dict[layer_name] = git_rev_info(layer) + return layer_dict + +def write_metadata_file(file_path, metadata): + """ Writes metadata to a XML file in directory. """ + + xml = dict_to_XML('metadata', metadata) + xml_doc = parseString(tostring(xml).decode('UTF-8')) + with open(file_path, 'w') as f: + f.write(xml_doc.toprettyxml()) + +def dict_to_XML(tag, dictionary, **kwargs): + """ Return XML element converting dicts recursively. """ + + elem = Element(tag, **kwargs) + for key, val in dictionary.items(): + if tag == 'layers': + child = (dict_to_XML('layer', val, name=key)) + elif isinstance(val, MutableMapping): + child = (dict_to_XML(key, val)) + else: + if tag == 'config': + child = Element('variable', name=key) + else: + child = Element(key) + child.text = str(val) + elem.append(child) + return elem diff --git a/meta/lib/oeqa/utils/network.py b/meta/lib/oeqa/utils/network.py index 2768f6c5db..59d01723a1 100644 --- a/meta/lib/oeqa/utils/network.py +++ b/meta/lib/oeqa/utils/network.py @@ -1,7 +1,11 @@ +# +# SPDX-License-Identifier: MIT +# + import socket -def get_free_port(): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +def get_free_port(udp = False): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM if not udp else socket.SOCK_DGRAM) s.bind(('', 0)) addr = s.getsockname() s.close() diff --git a/meta/lib/oeqa/utils/nfs.py b/meta/lib/oeqa/utils/nfs.py new file mode 100644 index 0000000000..a37686c914 --- /dev/null +++ b/meta/lib/oeqa/utils/nfs.py @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: MIT +import os +import sys +import tempfile +import contextlib +import socket +from oeqa.utils.commands import bitbake, get_bb_var, Command +from oeqa.utils.network import get_free_port + +@contextlib.contextmanager +def unfs_server(directory, logger = None): + unfs_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "unfs3-native") + if not os.path.exists(os.path.join(unfs_sysroot, "usr", "bin", "unfsd")): + # build native tool + bitbake("unfs3-native -c addto_recipe_sysroot") + + exports = None + cmd = None + try: + # create the exports file + with tempfile.NamedTemporaryFile(delete = False) as exports: + exports.write("{0} (rw,no_root_squash,no_all_squash,insecure)\n".format(directory).encode()) + + # find some ports for the server + nfsport, mountport = get_free_port(udp = True), get_free_port(udp = True) + + nenv = dict(os.environ) + nenv['PATH'] = "{0}/sbin:{0}/usr/sbin:{0}/usr/bin:".format(unfs_sysroot) + nenv.get('PATH', '') + cmd = Command(["unfsd", "-d", "-p", "-N", "-e", exports.name, "-n", str(nfsport), "-m", str(mountport)], + bg = True, env = nenv, output_log = logger) + cmd.run() + yield nfsport, mountport + finally: + if cmd is not None: + cmd.stop() + if exports is not None: + # clean up exports file + os.unlink(exports.name) + diff --git a/meta/lib/oeqa/utils/package_manager.py b/meta/lib/oeqa/utils/package_manager.py index 099ecc9728..2d358f7172 100644 --- a/meta/lib/oeqa/utils/package_manager.py +++ b/meta/lib/oeqa/utils/package_manager.py @@ -1,29 +1,217 @@ +# +# SPDX-License-Identifier: MIT +# + +import os +import json +import shutil + +from oeqa.core.utils.test import getCaseFile, getCaseMethod + def get_package_manager(d, root_path): """ Returns an OE package manager that can install packages in root_path. """ from oe.package_manager import RpmPM, OpkgPM, DpkgPM - pkg_class = d.getVar("IMAGE_PKGTYPE", True) + pkg_class = d.getVar("IMAGE_PKGTYPE") if pkg_class == "rpm": pm = RpmPM(d, root_path, - d.getVar('TARGET_VENDOR', True)) + d.getVar('TARGET_VENDOR'), + filterbydependencies=False) pm.create_configs() elif pkg_class == "ipk": pm = OpkgPM(d, root_path, - d.getVar("IPKGCONF_TARGET", True), - d.getVar("ALL_MULTILIB_PACKAGE_ARCHS", True)) + d.getVar("IPKGCONF_TARGET"), + d.getVar("ALL_MULTILIB_PACKAGE_ARCHS"), + filterbydependencies=False) elif pkg_class == "deb": pm = DpkgPM(d, root_path, - d.getVar('PACKAGE_ARCHS', True), - d.getVar('DPKG_ARCH', True)) + d.getVar('PACKAGE_ARCHS'), + d.getVar('DPKG_ARCH'), + filterbydependencies=False) pm.write_index() pm.update() return pm + +def find_packages_to_extract(test_suite): + """ + Returns packages to extract required by runtime tests. + """ + from oeqa.core.utils.test import getSuiteCasesFiles + + needed_packages = {} + files = getSuiteCasesFiles(test_suite) + + for f in set(files): + json_file = _get_json_file(f) + if json_file: + needed_packages.update(_get_needed_packages(json_file)) + + return needed_packages + +def _get_json_file(module_path): + """ + Returns the path of the JSON file for a module, empty if doesn't exitst. + """ + + json_file = '%s.json' % module_path.rsplit('.', 1)[0] + if os.path.isfile(module_path) and os.path.isfile(json_file): + return json_file + else: + return '' + +def _get_needed_packages(json_file, test=None): + """ + Returns a dict with needed packages based on a JSON file. + + If a test is specified it will return the dict just for that test. + """ + needed_packages = {} + + with open(json_file) as f: + test_packages = json.load(f) + for key,value in test_packages.items(): + needed_packages[key] = value + + if test: + if test in needed_packages: + needed_packages = needed_packages[test] + else: + needed_packages = {} + + return needed_packages + +def extract_packages(d, needed_packages): + """ + Extract packages that will be needed during runtime. + """ + + import bb + import oe.path + + extracted_path = d.getVar('TEST_EXTRACTED_DIR') + + for key,value in needed_packages.items(): + packages = () + if isinstance(value, dict): + packages = (value, ) + elif isinstance(value, list): + packages = value + else: + bb.fatal('Failed to process needed packages for %s; ' + 'Value must be a dict or list' % key) + + for package in packages: + pkg = package['pkg'] + rm = package.get('rm', False) + extract = package.get('extract', True) + + if extract: + #logger.debug(1, 'Extracting %s' % pkg) + dst_dir = os.path.join(extracted_path, pkg) + # Same package used for more than one test, + # don't need to extract again. + if os.path.exists(dst_dir): + continue + + # Extract package and copy it to TEST_EXTRACTED_DIR + pkg_dir = _extract_in_tmpdir(d, pkg) + oe.path.copytree(pkg_dir, dst_dir) + shutil.rmtree(pkg_dir) + + else: + #logger.debug(1, 'Copying %s' % pkg) + _copy_package(d, pkg) + +def _extract_in_tmpdir(d, pkg): + """" + Returns path to a temp directory where the package was + extracted without dependencies. + """ + + from oeqa.utils.package_manager import get_package_manager + + pkg_path = os.path.join(d.getVar('TEST_INSTALL_TMP_DIR'), pkg) + pm = get_package_manager(d, pkg_path) + extract_dir = pm.extract(pkg) + shutil.rmtree(pkg_path) + + return extract_dir + +def _copy_package(d, pkg): + """ + Copy the RPM, DEB or IPK package to dst_dir + """ + + from oeqa.utils.package_manager import get_package_manager + + pkg_path = os.path.join(d.getVar('TEST_INSTALL_TMP_DIR'), pkg) + dst_dir = d.getVar('TEST_PACKAGED_DIR') + pm = get_package_manager(d, pkg_path) + pkg_info = pm.package_info(pkg) + file_path = pkg_info[pkg]['filepath'] + shutil.copy2(file_path, dst_dir) + shutil.rmtree(pkg_path) + +def install_package(test_case): + """ + Installs package in DUT if required. + """ + needed_packages = test_needs_package(test_case) + if needed_packages: + _install_uninstall_packages(needed_packages, test_case, True) + +def uninstall_package(test_case): + """ + Uninstalls package in DUT if required. + """ + needed_packages = test_needs_package(test_case) + if needed_packages: + _install_uninstall_packages(needed_packages, test_case, False) + +def test_needs_package(test_case): + """ + Checks if a test case requires to install/uninstall packages. + """ + test_file = getCaseFile(test_case) + json_file = _get_json_file(test_file) + + if json_file: + test_method = getCaseMethod(test_case) + needed_packages = _get_needed_packages(json_file, test_method) + if needed_packages: + return needed_packages + + return None + +def _install_uninstall_packages(needed_packages, test_case, install=True): + """ + Install/Uninstall packages in the DUT without using a package manager + """ + + if isinstance(needed_packages, dict): + packages = [needed_packages] + elif isinstance(needed_packages, list): + packages = needed_packages + + for package in packages: + pkg = package['pkg'] + rm = package.get('rm', False) + extract = package.get('extract', True) + src_dir = os.path.join(test_case.tc.extract_dir, pkg) + + # Install package + if install and extract: + test_case.tc.target.copyDirTo(src_dir, '/') + + # Uninstall package + elif not install and rm: + test_case.tc.target.deleteDirStructure(src_dir, '/') diff --git a/meta/lib/oeqa/utils/qemurunner.py b/meta/lib/oeqa/utils/qemurunner.py index 1f98682f75..2cada35d48 100644 --- a/meta/lib/oeqa/utils/qemurunner.py +++ b/meta/lib/oeqa/utils/qemurunner.py @@ -1,12 +1,15 @@ +# # Copyright (C) 2013 Intel Corporation # -# Released under the MIT license (see COPYING.MIT) +# SPDX-License-Identifier: MIT +# # This module provides a class for starting qemu images using runqemu. # It's used by testimage.bbclass. import subprocess import os +import sys import time import signal import re @@ -16,10 +19,8 @@ import errno import string import threading import codecs -from oeqa.utils.dump import HostDumper - import logging -logger = logging.getLogger("BitBake.QemuRunner") +from oeqa.utils.dump import HostDumper # Get Unicode non printable control chars control_range = list(range(0,32))+list(range(127,160)) @@ -29,16 +30,19 @@ re_control_char = re.compile('[%s]' % re.escape("".join(control_chars))) class QemuRunner: - def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds, use_kvm): + def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds, + use_kvm, logger, use_slirp=False, serial_ports=2): # Popen object for runqemu self.runqemu = None # pid of the qemu process that runqemu will start self.qemupid = None - # target ip - from the command line + # target ip - from the command line or runqemu output self.ip = None # host ip - where qemu is running self.server_ip = None + # target ip netmask + self.netmask = None self.machine = machine self.rootfs = rootfs @@ -50,9 +54,16 @@ class QemuRunner: self.logged = False self.thread = None self.use_kvm = use_kvm + self.use_slirp = use_slirp + self.serial_ports = serial_ports + self.msg = '' - self.runqemutime = 60 + self.runqemutime = 120 + self.qemu_pidfile = 'pidfile_'+str(os.getpid()) self.host_dumper = HostDumper(dump_host_cmds, dump_dir) + self.monitorpipe = None + + self.logger = logger def create_socket(self): try: @@ -61,7 +72,7 @@ class QemuRunner: sock.bind(("127.0.0.1",0)) sock.listen(2) port = sock.getsockname()[1] - logger.info("Created listening socket for qemu serial console on: 127.0.0.1:%s" % port) + self.logger.debug("Created listening socket for qemu serial console on: 127.0.0.1:%s" % port) return (sock, port) except socket.error: @@ -72,8 +83,9 @@ class QemuRunner: if self.logfile: # It is needed to sanitize the data received from qemu # because is possible to have control characters - msg = msg.decode("utf-8") + msg = msg.decode("utf-8", errors='ignore') msg = re_control_char.sub('', msg) + self.msg += msg with codecs.open(self.logfile, "a", encoding="utf-8") as f: f.write("%s" % msg) @@ -87,67 +99,84 @@ class QemuRunner: def handleSIGCHLD(self, signum, frame): if self.runqemu and self.runqemu.poll(): if self.runqemu.returncode: - logger.info('runqemu exited with code %d' % self.runqemu.returncode) - logger.info("Output from runqemu:\n%s" % self.getOutput(self.runqemu.stdout)) + self.logger.error('runqemu exited with code %d' % self.runqemu.returncode) + self.logger.error('Output from runqemu:\n%s' % self.getOutput(self.runqemu.stdout)) self.stop() self._dump_host() raise SystemExit - def start(self, qemuparams = None, get_ip = True, extra_bootparams = None): + def start(self, qemuparams = None, get_ip = True, extra_bootparams = None, runqemuparams='', launch_cmd=None, discard_writes=True): + env = os.environ.copy() if self.display: - os.environ["DISPLAY"] = self.display + env["DISPLAY"] = self.display # Set this flag so that Qemu doesn't do any grabs as SDL grabs # interact badly with screensavers. - os.environ["QEMU_DONT_GRAB"] = "1" + env["QEMU_DONT_GRAB"] = "1" if not os.path.exists(self.rootfs): - logger.error("Invalid rootfs %s" % self.rootfs) + self.logger.error("Invalid rootfs %s" % self.rootfs) return False if not os.path.exists(self.tmpdir): - logger.error("Invalid TMPDIR path %s" % self.tmpdir) + self.logger.error("Invalid TMPDIR path %s" % self.tmpdir) return False else: - os.environ["OE_TMPDIR"] = self.tmpdir + env["OE_TMPDIR"] = self.tmpdir if not os.path.exists(self.deploy_dir_image): - logger.error("Invalid DEPLOY_DIR_IMAGE path %s" % self.deploy_dir_image) + self.logger.error("Invalid DEPLOY_DIR_IMAGE path %s" % self.deploy_dir_image) return False else: - os.environ["DEPLOY_DIR_IMAGE"] = self.deploy_dir_image + env["DEPLOY_DIR_IMAGE"] = self.deploy_dir_image + if not launch_cmd: + launch_cmd = 'runqemu %s' % ('snapshot' if discard_writes else '') + if self.use_kvm: + self.logger.debug('Using kvm for runqemu') + launch_cmd += ' kvm' + else: + self.logger.debug('Not using kvm for runqemu') + if not self.display: + launch_cmd += ' nographic' + if self.use_slirp: + launch_cmd += ' slirp' + launch_cmd += ' %s %s %s' % (runqemuparams, self.machine, self.rootfs) + + return self.launch(launch_cmd, qemuparams=qemuparams, get_ip=get_ip, extra_bootparams=extra_bootparams, env=env) + + def launch(self, launch_cmd, get_ip = True, qemuparams = None, extra_bootparams = None, env = None): try: - threadsock, threadport = self.create_socket() + if self.serial_ports >= 2: + self.threadsock, threadport = self.create_socket() self.server_socket, self.serverport = self.create_socket() except socket.error as msg: - logger.error("Failed to create listening socket: %s" % msg[1]) + self.logger.error("Failed to create listening socket: %s" % msg[1]) return False - bootparams = 'console=tty1 console=ttyS0,115200n8 printk.time=1' if extra_bootparams: bootparams = bootparams + ' ' + extra_bootparams - self.qemuparams = 'bootparams="{0}" qemuparams="-serial tcp:127.0.0.1:{1}"'.format(bootparams, threadport) - if not self.display: - self.qemuparams = 'nographic ' + self.qemuparams + # Ask QEMU to store the QEMU process PID in file, this way we don't have to parse running processes + # and analyze descendents in order to determine it. + if os.path.exists(self.qemu_pidfile): + os.remove(self.qemu_pidfile) + self.qemuparams = 'bootparams="{0}" qemuparams="-pidfile {1}"'.format(bootparams, self.qemu_pidfile) if qemuparams: self.qemuparams = self.qemuparams[:-1] + " " + qemuparams + " " + '\"' + if self.serial_ports >= 2: + launch_cmd += ' tcpserial=%s:%s %s' % (threadport, self.serverport, self.qemuparams) + else: + launch_cmd += ' tcpserial=%s %s' % (self.serverport, self.qemuparams) + self.origchldhandler = signal.getsignal(signal.SIGCHLD) signal.signal(signal.SIGCHLD, self.handleSIGCHLD) - launch_cmd = 'runqemu snapshot ' - if self.use_kvm: - logger.info('Using kvm for runqemu') - launch_cmd += 'kvm ' - else: - logger.info('Not using kvm for runqemu') - launch_cmd += 'tcpserial=%s %s %s %s' % (self.serverport, self.machine, self.rootfs, self.qemuparams) - logger.info('launchcmd=%s'%(launch_cmd)) + self.logger.debug('launchcmd=%s'%(launch_cmd)) # FIXME: We pass in stdin=subprocess.PIPE here to work around stty # blocking at the end of the runqemu script when using this within # oe-selftest (this makes stty error out immediately). There ought # to be a proper fix but this will suffice for now. - self.runqemu = subprocess.Popen(launch_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, preexec_fn=os.setpgrp) + self.runqemu = subprocess.Popen(launch_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, preexec_fn=os.setpgrp, env=env) output = self.runqemu.stdout # @@ -176,130 +205,183 @@ class QemuRunner: os.killpg(os.getpgid(self.runqemu.pid), signal.SIGTERM) sys.exit(0) - logger.info("runqemu started, pid is %s" % self.runqemu.pid) - logger.info("waiting at most %s seconds for qemu pid" % self.runqemutime) + self.logger.debug("runqemu started, pid is %s" % self.runqemu.pid) + self.logger.debug("waiting at most %s seconds for qemu pid (%s)" % + (self.runqemutime, time.strftime("%D %H:%M:%S"))) endtime = time.time() + self.runqemutime while not self.is_alive() and time.time() < endtime: if self.runqemu.poll(): if self.runqemu.returncode: # No point waiting any longer - logger.info('runqemu exited with code %d' % self.runqemu.returncode) + self.logger.warning('runqemu exited with code %d' % self.runqemu.returncode) self._dump_host() + self.logger.warning("Output from runqemu:\n%s" % self.getOutput(output)) self.stop() - logger.info("Output from runqemu:\n%s" % self.getOutput(output)) return False - time.sleep(1) - - if self.is_alive(): - logger.info("qemu started - qemu procces pid is %s" % self.qemupid) - if get_ip: - cmdline = '' - with open('/proc/%s/cmdline' % self.qemupid) as p: - cmdline = p.read() - # It is needed to sanitize the data received - # because is possible to have control characters - cmdline = re_control_char.sub('', cmdline) - try: - ips = re.findall("((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1]) - if not ips or len(ips) != 3: - raise ValueError - else: - self.ip = ips[0] - self.server_ip = ips[1] - except (IndexError, ValueError): - logger.info("Couldn't get ip from qemu process arguments! Here is the qemu command line used:\n%s\nand output from runqemu:\n%s" % (cmdline, self.getOutput(output))) + time.sleep(0.5) + + if not self.is_alive(): + self.logger.error("Qemu pid didn't appear in %s seconds (%s)" % + (self.runqemutime, time.strftime("%D %H:%M:%S"))) + # Dump all processes to help us to figure out what is going on... + ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command '], stdout=subprocess.PIPE).communicate()[0] + processes = ps.decode("utf-8") + self.logger.debug("Running processes:\n%s" % processes) + self._dump_host() + op = self.getOutput(output) + self.stop() + if op: + self.logger.error("Output from runqemu:\n%s" % op) + else: + self.logger.error("No output from runqemu.\n") + return False + + # We are alive: qemu is running + out = self.getOutput(output) + netconf = False # network configuration is not required by default + self.logger.debug("qemu started in %s seconds - qemu procces pid is %s (%s)" % + (time.time() - (endtime - self.runqemutime), + self.qemupid, time.strftime("%D %H:%M:%S"))) + cmdline = '' + if get_ip: + with open('/proc/%s/cmdline' % self.qemupid) as p: + cmdline = p.read() + # It is needed to sanitize the data received + # because is possible to have control characters + cmdline = re_control_char.sub(' ', cmdline) + try: + if self.use_slirp: + tcp_ports = cmdline.split("hostfwd=tcp::")[1] + host_port = tcp_ports[:tcp_ports.find('-')] + self.ip = "localhost:%s" % host_port + else: + ips = re.findall(r"((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1]) + self.ip = ips[0] + self.server_ip = ips[1] + self.logger.debug("qemu cmdline used:\n{}".format(cmdline)) + except (IndexError, ValueError): + # Try to get network configuration from runqemu output + match = re.match(r'.*Network configuration: ([0-9.]+)::([0-9.]+):([0-9.]+)$.*', + out, re.MULTILINE|re.DOTALL) + if match: + self.ip, self.server_ip, self.netmask = match.groups() + # network configuration is required as we couldn't get it + # from the runqemu command line, so qemu doesn't run kernel + # and guest networking is not configured + netconf = True + else: + self.logger.error("Couldn't get ip from qemu command line and runqemu output! " + "Here is the qemu command line used:\n%s\n" + "and output from runqemu:\n%s" % (cmdline, out)) self._dump_host() self.stop() return False - logger.info("qemu cmdline used:\n{}".format(cmdline)) - logger.info("Target IP: %s" % self.ip) - logger.info("Server IP: %s" % self.server_ip) - self.thread = LoggingThread(self.log, threadsock, logger) + self.logger.debug("Target IP: %s" % self.ip) + self.logger.debug("Server IP: %s" % self.server_ip) + + if self.serial_ports >= 2: + self.thread = LoggingThread(self.log, self.threadsock, self.logger) self.thread.start() if not self.thread.connection_established.wait(self.boottime): - logger.error("Didn't receive a console connection from qemu. " + self.logger.error("Didn't receive a console connection from qemu. " "Here is the qemu command line used:\n%s\nand " - "output from runqemu:\n%s" % (cmdline, - self.getOutput(output))) + "output from runqemu:\n%s" % (cmdline, out)) self.stop_thread() return False - logger.info("Output from runqemu:\n%s", self.getOutput(output)) - logger.info("Waiting at most %d seconds for login banner" % self.boottime) - endtime = time.time() + self.boottime - socklist = [self.server_socket] - reachedlogin = False - stopread = False - qemusock = None - bootlog = '' - data = b'' - while time.time() < endtime and not stopread: - sread, swrite, serror = select.select(socklist, [], [], 5) - for sock in sread: - if sock is self.server_socket: - qemusock, addr = self.server_socket.accept() - qemusock.setblocking(0) - socklist.append(qemusock) - socklist.remove(self.server_socket) - logger.info("Connection from %s:%s" % addr) - else: - data = data + sock.recv(1024) - if data: - try: - data = data.decode("utf-8", errors="surrogateescape") - bootlog += data - data = b'' - if re.search(".* login:", bootlog): - self.server_socket = qemusock - stopread = True - reachedlogin = True - logger.info("Reached login banner") - except UnicodeDecodeError: - continue - else: - socklist.remove(sock) - sock.close() - stopread = True - - if not reachedlogin: - logger.info("Target didn't reached login boot in %d seconds" % self.boottime) - lines = "\n".join(bootlog.splitlines()[-25:]) - logger.info("Last 25 lines of text:\n%s" % lines) - logger.info("Check full boot log: %s" % self.logfile) - self._dump_host() - self.stop() - return False - - # If we are not able to login the tests can continue + self.logger.debug("Output from runqemu:\n%s", out) + self.logger.debug("Waiting at most %d seconds for login banner (%s)" % + (self.boottime, time.strftime("%D %H:%M:%S"))) + endtime = time.time() + self.boottime + socklist = [self.server_socket] + reachedlogin = False + stopread = False + qemusock = None + bootlog = b'' + data = b'' + while time.time() < endtime and not stopread: try: - (status, output) = self.run_serial("root\n", raw=True) - if re.search("root@[a-zA-Z0-9\-]+:~#", output): - self.logged = True - logger.info("Logged as root in serial console") + sread, swrite, serror = select.select(socklist, [], [], 5) + except InterruptedError: + continue + for sock in sread: + if sock is self.server_socket: + qemusock, addr = self.server_socket.accept() + qemusock.setblocking(0) + socklist.append(qemusock) + socklist.remove(self.server_socket) + self.logger.debug("Connection from %s:%s" % addr) else: - logger.info("Couldn't login into serial console" - " as root using blank password") - except: - logger.info("Serial console failed while trying to login") - - else: - logger.info("Qemu pid didn't appeared in %s seconds" % self.runqemutime) + data = data + sock.recv(1024) + if data: + bootlog += data + if self.serial_ports < 2: + # this socket has mixed console/kernel data, log it to logfile + self.log(data) + + data = b'' + if b' login:' in bootlog: + self.server_socket = qemusock + stopread = True + reachedlogin = True + self.logger.debug("Reached login banner in %s seconds (%s)" % + (time.time() - (endtime - self.boottime), + time.strftime("%D %H:%M:%S"))) + else: + # no need to check if reachedlogin unless we support multiple connections + self.logger.debug("QEMU socket disconnected before login banner reached. (%s)" % + time.strftime("%D %H:%M:%S")) + socklist.remove(sock) + sock.close() + stopread = True + + + if not reachedlogin: + if time.time() >= endtime: + self.logger.warning("Target didn't reach login banner in %d seconds (%s)" % + (self.boottime, time.strftime("%D %H:%M:%S"))) + tail = lambda l: "\n".join(l.splitlines()[-25:]) + bootlog = bootlog.decode("utf-8") + # in case bootlog is empty, use tail qemu log store at self.msg + lines = tail(bootlog if bootlog else self.msg) + self.logger.warning("Last 25 lines of text:\n%s" % lines) + self.logger.warning("Check full boot log: %s" % self.logfile) self._dump_host() self.stop() - logger.info("Output from runqemu:\n%s" % self.getOutput(output)) return False - return self.is_alive() + # If we are not able to login the tests can continue + try: + (status, output) = self.run_serial("root\n", raw=True) + if re.search(r"root@[a-zA-Z0-9\-]+:~#", output): + self.logged = True + self.logger.debug("Logged as root in serial console") + if netconf: + # configure guest networking + cmd = "ifconfig eth0 %s netmask %s up\n" % (self.ip, self.netmask) + output = self.run_serial(cmd, raw=True)[1] + if re.search(r"root@[a-zA-Z0-9\-]+:~#", output): + self.logger.debug("configured ip address %s", self.ip) + else: + self.logger.debug("Couldn't configure guest networking") + else: + self.logger.warning("Couldn't login into serial console" + " as root using blank password") + self.logger.warning("The output:\n%s" % output) + except: + self.logger.warning("Serial console failed while trying to login") + return True def stop(self): - self.stop_thread() if hasattr(self, "origchldhandler"): signal.signal(signal.SIGCHLD, self.origchldhandler) + self.stop_thread() + self.stop_qemu_system() if self.runqemu: if hasattr(self, "monitorpid"): os.kill(self.monitorpid, signal.SIGKILL) - logger.info("Sending SIGTERM to runqemu") + self.logger.debug("Sending SIGTERM to runqemu") try: os.killpg(os.getpgid(self.runqemu.pid), signal.SIGTERM) except OSError as e: @@ -309,14 +391,32 @@ class QemuRunner: while self.runqemu.poll() is None and time.time() < endtime: time.sleep(1) if self.runqemu.poll() is None: - logger.info("Sending SIGKILL to runqemu") + self.logger.debug("Sending SIGKILL to runqemu") os.killpg(os.getpgid(self.runqemu.pid), signal.SIGKILL) + self.runqemu.stdin.close() + self.runqemu.stdout.close() self.runqemu = None + if hasattr(self, 'server_socket') and self.server_socket: self.server_socket.close() self.server_socket = None + if hasattr(self, 'threadsock') and self.threadsock: + self.threadsock.close() + self.threadsock = None self.qemupid = None self.ip = None + if os.path.exists(self.qemu_pidfile): + os.remove(self.qemu_pidfile) + if self.monitorpipe: + self.monitorpipe.close() + + def stop_qemu_system(self): + if self.qemupid: + try: + # qemu-system behaves well and a SIGTERM is enough + os.kill(self.qemupid, signal.SIGTERM) + except ProcessLookupError as e: + self.logger.warning('qemu-system ended unexpectedly') def stop_thread(self): if self.thread and self.thread.is_alive(): @@ -324,7 +424,7 @@ class QemuRunner: self.thread.join() def restart(self, qemuparams = None): - logger.info("Restarting qemu process") + self.logger.warning("Restarting qemu process") if self.runqemu.poll() is None: self.stop() if self.start(qemuparams): @@ -332,59 +432,26 @@ class QemuRunner: return False def is_alive(self): - if not self.runqemu: + if not self.runqemu or self.runqemu.poll() is not None: return False - qemu_child = self.find_child(str(self.runqemu.pid)) - if qemu_child: - self.qemupid = qemu_child[0] - if os.path.exists("/proc/" + str(self.qemupid)): - return True + if os.path.isfile(self.qemu_pidfile): + # when handling pidfile, qemu creates the file, stat it, lock it and then write to it + # so it's possible that the file has been created but the content is empty + pidfile_timeout = time.time() + 3 + while time.time() < pidfile_timeout: + with open(self.qemu_pidfile, 'r') as f: + qemu_pid = f.read().strip() + # file created but not yet written contents + if not qemu_pid: + time.sleep(0.5) + continue + else: + if os.path.exists("/proc/" + qemu_pid): + self.qemupid = int(qemu_pid) + return True return False - def find_child(self,parent_pid): - # - # Walk the process tree from the process specified looking for a qemu-system. Return its [pid'cmd] - # - ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command'], stdout=subprocess.PIPE).communicate()[0] - processes = ps.decode("utf-8").split('\n') - nfields = len(processes[0].split()) - 1 - pids = {} - commands = {} - for row in processes[1:]: - data = row.split(None, nfields) - if len(data) != 3: - continue - if data[1] not in pids: - pids[data[1]] = [] - - pids[data[1]].append(data[0]) - commands[data[0]] = data[2] - - if parent_pid not in pids: - return [] - - parents = [] - newparents = pids[parent_pid] - while newparents: - next = [] - for p in newparents: - if p in pids: - for n in pids[p]: - if n not in parents and n not in next: - next.append(n) - if p not in parents: - parents.append(p) - newparents = next - #print("Children matching %s:" % str(parents)) - for p in parents: - # Need to be careful here since runqemu runs "ldd qemu-system-xxxx" - # Also, old versions of ldd (2.11) run "LD_XXXX qemu-system-xxxx" - basecmd = commands[p].split()[0] - basecmd = os.path.basename(basecmd) - if "qemu-system" in basecmd and "-serial tcp" in commands[p]: - return [int(p),commands[p]] - - def run_serial(self, command, raw=False): + def run_serial(self, command, raw=False, timeout=60): # We assume target system have echo to get command status if not raw: command = "%s; echo $?\n" % command @@ -392,20 +459,26 @@ class QemuRunner: data = '' status = 0 self.server_socket.sendall(command.encode('utf-8')) - keepreading = True - while keepreading: - sread, _, _ = select.select([self.server_socket],[],[],5) + start = time.time() + end = start + timeout + while True: + now = time.time() + if now >= end: + data += "<<< run_serial(): command timed out after %d seconds without output >>>\r\n\r\n" % timeout + break + try: + sread, _, _ = select.select([self.server_socket],[],[], end - now) + except InterruptedError: + continue if sread: answer = self.server_socket.recv(1024) if answer: data += answer.decode('utf-8') # Search the prompt to stop - if re.search("[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#", data): - keepreading = False + if re.search(r"[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#", data): + break else: raise Exception("No data on serial console socket") - else: - keepreading = False if data: if raw: @@ -427,7 +500,7 @@ class QemuRunner: def _dump_host(self): self.host_dumper.create_dir("qemu") - logger.warn("Qemu ended unexpectedly, dump data from host" + self.logger.warning("Qemu ended unexpectedly, dump data from host" " is in %s" % self.host_dumper.dump_dir) self.host_dumper.dump_host() @@ -456,17 +529,17 @@ class LoggingThread(threading.Thread): self.teardown() def run(self): - self.logger.info("Starting logging thread") + self.logger.debug("Starting logging thread") self.readpipe, self.writepipe = os.pipe() threading.Thread.run(self) def stop(self): - self.logger.info("Stopping logging thread") + self.logger.debug("Stopping logging thread") if self.running: os.write(self.writepipe, bytes("stop", "utf-8")) def teardown(self): - self.logger.info("Tearing down logging thread") + self.logger.debug("Tearing down logging thread") self.close_socket(self.serversock) if self.readsock is not None: @@ -484,7 +557,7 @@ class LoggingThread(threading.Thread): breakout = False self.running = True - self.logger.info("Starting thread event loop") + self.logger.debug("Starting thread event loop") while not breakout: events = poll.poll() for event in events: @@ -494,19 +567,19 @@ class LoggingThread(threading.Thread): # Event to stop the thread if self.readpipe == event[0]: - self.logger.info("Stop event received") + self.logger.debug("Stop event received") breakout = True break # A connection request was received elif self.serversock.fileno() == event[0]: - self.logger.info("Connection request received") + self.logger.debug("Connection request received") self.readsock, _ = self.serversock.accept() self.readsock.setblocking(0) poll.unregister(self.serversock.fileno()) poll.register(self.readsock.fileno(), event_read_mask) - self.logger.info("Setting connection established event") + self.logger.debug("Setting connection established event") self.connection_established.set() # Actual data to be logged diff --git a/meta/lib/oeqa/utils/qemutinyrunner.py b/meta/lib/oeqa/utils/qemutinyrunner.py index d554f0dbcd..364005bd2d 100644 --- a/meta/lib/oeqa/utils/qemutinyrunner.py +++ b/meta/lib/oeqa/utils/qemutinyrunner.py @@ -1,6 +1,8 @@ +# # Copyright (C) 2015 Intel Corporation # -# Released under the MIT license (see COPYING.MIT) +# SPDX-License-Identifier: MIT +# # This module provides a class for starting qemu images of poky tiny. # It's used by testimage.bbclass. @@ -17,7 +19,7 @@ from .qemurunner import QemuRunner class QemuTinyRunner(QemuRunner): - def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, kernel, boottime): + def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, kernel, boottime, logger): # Popen object for runqemu self.runqemu = None @@ -40,6 +42,7 @@ class QemuTinyRunner(QemuRunner): self.socketfile = "console.sock" self.server_socket = None self.kernel = kernel + self.logger = logger def create_socket(self): @@ -60,7 +63,7 @@ class QemuTinyRunner(QemuRunner): with open(self.logfile, "a") as f: f.write("%s" % msg) - def start(self, qemuparams = None, ssh=True, extra_bootparams=None): + def start(self, qemuparams = None, ssh=True, extra_bootparams=None, runqemuparams='', discard_writes=True): if self.display: os.environ["DISPLAY"] = self.display @@ -107,14 +110,17 @@ class QemuTinyRunner(QemuRunner): return self.is_alive() - def run_serial(self, command): + def run_serial(self, command, timeout=60): self.server_socket.sendall(command+'\n') data = '' status = 0 stopread = False - endtime = time.time()+5 + endtime = time.time()+timeout while time.time()<endtime and not stopread: - sread, _, _ = select.select([self.server_socket],[],[],5) + try: + sread, _, _ = select.select([self.server_socket],[],[],1) + except InterruptedError: + continue for sock in sread: answer = sock.recv(1024) if answer: @@ -124,6 +130,8 @@ class QemuTinyRunner(QemuRunner): stopread = True if not data: status = 1 + if not stopread: + data += "<<< run_serial(): command timed out after %d seconds without output >>>\r\n\r\n" % timeout return (status, str(data)) def find_child(self,parent_pid): diff --git a/meta/lib/oeqa/utils/sshcontrol.py b/meta/lib/oeqa/utils/sshcontrol.py index da485ee408..49a07264c6 100644 --- a/meta/lib/oeqa/utils/sshcontrol.py +++ b/meta/lib/oeqa/utils/sshcontrol.py @@ -1,7 +1,8 @@ -# -*- coding: utf-8 -*- +# # Copyright (C) 2013 Intel Corporation # -# Released under the MIT license (see COPYING.MIT) +# SPDX-License-Identifier: MIT +# # Provides a class for setting up ssh connections, # running commands and copying files to/from a target. @@ -52,17 +53,19 @@ class SSHProcess(object): endtime = self.starttime + timeout eof = False while time.time() < endtime and not eof: - if select.select([self.process.stdout], [], [], 5)[0] != []: - data = os.read(self.process.stdout.fileno(), 1024) - if not data: - self.process.stdout.close() - eof = True - else: - data = data.decode("utf-8") - output += data - self.log(data) - endtime = time.time() + timeout - + try: + if select.select([self.process.stdout], [], [], 5)[0] != []: + data = os.read(self.process.stdout.fileno(), 1024) + if not data: + self.process.stdout.close() + eof = True + else: + data = data.decode("utf-8") + output += data + self.log(data) + endtime = time.time() + timeout + except InterruptedError: + continue # process hasn't returned yet if not eof: @@ -148,12 +151,9 @@ class SSHControl(object): def copy_to(self, localpath, remotepath): if os.path.islink(localpath): - link = os.readlink(localpath) - dst_dir, dst_base = os.path.split(remotepath) - return self.run("cd %s; ln -s %s %s" % (dst_dir, link, dst_base)) - else: - command = self.scp + [localpath, '%s@%s:%s' % (self.user, self.ip, remotepath)] - return self._internal_run(command, ignore_status=False) + localpath = os.path.dirname(localpath) + "/" + os.readlink(localpath) + command = self.scp + [localpath, '%s@%s:%s' % (self.user, self.ip, remotepath)] + return self._internal_run(command, ignore_status=False) def copy_from(self, remotepath, localpath): command = self.scp + ['%s@%s:%s' % (self.user, self.ip, remotepath), localpath] diff --git a/meta/lib/oeqa/utils/subprocesstweak.py b/meta/lib/oeqa/utils/subprocesstweak.py new file mode 100644 index 0000000000..b47975a4bc --- /dev/null +++ b/meta/lib/oeqa/utils/subprocesstweak.py @@ -0,0 +1,22 @@ +# +# SPDX-License-Identifier: MIT +# +import subprocess + +class OETestCalledProcessError(subprocess.CalledProcessError): + def __str__(self): + def strify(o): + if isinstance(o, bytes): + return o.decode("utf-8", errors="replace") + else: + return o + + s = "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode) + if hasattr(self, "output") and self.output: + s = s + "\nStandard Output: " + strify(self.output) + if hasattr(self, "stderr") and self.stderr: + s = s + "\nStandard Error: " + strify(self.stderr) + return s + +def errors_have_output(): + subprocess.CalledProcessError = OETestCalledProcessError diff --git a/meta/lib/oeqa/utils/targetbuild.py b/meta/lib/oeqa/utils/targetbuild.py index 59593f5ef3..1055810ca3 100644 --- a/meta/lib/oeqa/utils/targetbuild.py +++ b/meta/lib/oeqa/utils/targetbuild.py @@ -1,6 +1,8 @@ +# # Copyright (C) 2013 Intel Corporation # -# Released under the MIT license (see COPYING.MIT) +# SPDX-License-Identifier: MIT +# # Provides a class for automating build tests for projects @@ -8,15 +10,21 @@ import os import re import bb.utils import subprocess +import tempfile from abc import ABCMeta, abstractmethod class BuildProject(metaclass=ABCMeta): - def __init__(self, d, uri, foldername=None, tmpdir="/tmp/"): + def __init__(self, d, uri, foldername=None, tmpdir=None): self.d = d self.uri = uri self.archive = os.path.basename(uri) - self.localarchive = os.path.join(tmpdir,self.archive) + if not tmpdir: + tmpdir = self.d.getVar('WORKDIR') + if not tmpdir: + self.tempdirobj = tempfile.TemporaryDirectory(prefix='buildproject-') + tmpdir = self.tempdirobj.name + self.localarchive = os.path.join(tmpdir, self.archive) if foldername: self.fname = foldername else: @@ -24,8 +32,7 @@ class BuildProject(metaclass=ABCMeta): # Download self.archive to self.localarchive def _download_archive(self): - - dl_dir = self.d.getVar("DL_DIR", True) + dl_dir = self.d.getVar("DL_DIR") if dl_dir and os.path.exists(os.path.join(dl_dir, self.archive)): bb.utils.copyfile(os.path.join(dl_dir, self.archive), self.localarchive) return @@ -40,12 +47,12 @@ class BuildProject(metaclass=ABCMeta): cmd = '' for var in exportvars: - val = self.d.getVar(var, True) + val = self.d.getVar(var) if val: cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd) cmd = cmd + "wget -O %s %s" % (self.localarchive, self.uri) - subprocess.check_call(cmd, shell=True) + subprocess.check_output(cmd, shell=True) # This method should provide a way to run a command in the desired environment. @abstractmethod @@ -65,7 +72,7 @@ class BuildProject(metaclass=ABCMeta): def clean(self): self._run('rm -rf %s' % self.targetdir) - subprocess.call('rm -f %s' % self.localarchive, shell=True) + subprocess.check_call('rm -f %s' % self.localarchive, shell=True) pass class TargetBuildProject(BuildProject): @@ -73,7 +80,7 @@ class TargetBuildProject(BuildProject): def __init__(self, target, d, uri, foldername=None): self.target = target self.targetdir = "~/" - BuildProject.__init__(self, d, uri, foldername, tmpdir="/tmp") + BuildProject.__init__(self, d, uri, foldername) def download_archive(self): @@ -103,8 +110,8 @@ class SDKBuildProject(BuildProject): self.testdir = testpath self.targetdir = testpath bb.utils.mkdirhier(testpath) - self.datetime = d.getVar('DATETIME', True) - self.testlogdir = d.getVar("TEST_LOG_DIR", True) + self.datetime = d.getVar('DATETIME') + self.testlogdir = d.getVar("TEST_LOG_DIR") bb.utils.mkdirhier(self.testlogdir) self.logfile = os.path.join(self.testlogdir, "sdk_target_log.%s" % self.datetime) BuildProject.__init__(self, d, uri, foldername, tmpdir=testpath) @@ -114,7 +121,7 @@ class SDKBuildProject(BuildProject): self._download_archive() cmd = 'tar xf %s%s -C %s' % (self.targetdir, self.archive, self.targetdir) - subprocess.check_call(cmd, shell=True) + subprocess.check_output(cmd, shell=True) #Change targetdir to project folder self.targetdir = os.path.join(self.targetdir, self.fname) @@ -132,4 +139,4 @@ class SDKBuildProject(BuildProject): def _run(self, cmd): self.log("Running . %s; " % self.sdkenv + cmd) - return subprocess.call(". %s; " % self.sdkenv + cmd, shell=True) + return subprocess.check_call(". %s; " % self.sdkenv + cmd, shell=True) diff --git a/meta/lib/oeqa/utils/testexport.py b/meta/lib/oeqa/utils/testexport.py index 57be2ca449..e89d130a9c 100644 --- a/meta/lib/oeqa/utils/testexport.py +++ b/meta/lib/oeqa/utils/testexport.py @@ -1,6 +1,8 @@ +# # Copyright (C) 2015 Intel Corporation # -# Released under the MIT license (see COPYING.MIT) +# SPDX-License-Identifier: MIT +# # Provides functions to help with exporting binaries obtained from built targets @@ -72,9 +74,9 @@ def process_binaries(d, params): return extract_bin_command if determine_if_poky_env(): # machine with poky environment - exportpath = d.getVar("TEST_EXPORT_DIR", True) if export_env else d.getVar("DEPLOY_DIR", True) - rpm_deploy_dir = d.getVar("DEPLOY_DIR_RPM", True) - arch = get_dest_folder(d.getVar("TUNE_FEATURES", True), os.listdir(rpm_deploy_dir)) + exportpath = d.getVar("TEST_EXPORT_DIR") if export_env else d.getVar("DEPLOY_DIR") + rpm_deploy_dir = d.getVar("DEPLOY_DIR_RPM") + arch = get_dest_folder(d.getVar("TUNE_FEATURES"), os.listdir(rpm_deploy_dir)) arch_rpm_dir = os.path.join(rpm_deploy_dir, arch) extracted_bin_dir = os.path.join(exportpath,"binaries", arch, "extracted_binaries") packaged_bin_dir = os.path.join(exportpath,"binaries", arch, "packaged_binaries") @@ -92,7 +94,7 @@ def process_binaries(d, params): return "" for item in native_rpm_file_list:# will copy all versions of package. Used version will be selected on remote machine bb.plain("Copying native package file: %s" % item) - sh.copy(os.path.join(rpm_deploy_dir, native_rpm_dir, item), os.path.join(d.getVar("TEST_EXPORT_DIR", True), "binaries", "native")) + sh.copy(os.path.join(rpm_deploy_dir, native_rpm_dir, item), os.path.join(d.getVar("TEST_EXPORT_DIR"), "binaries", "native")) else: # nothing to do here; running tests under bitbake, so we asume native binaries are in sysroots dir. if param_list[1] or param_list[4]: bb.warn("Native binary %s %s%s. Running tests under bitbake environment. Version can't be checked except when the test itself does it" @@ -148,7 +150,7 @@ def process_binaries(d, params): else: # this is for target device if param_list[2] == "rpm": return "No need to extract, this is an .rpm file" - arch = get_dest_folder(d.getVar("TUNE_FEATURES", True), os.listdir(binaries_path)) + arch = get_dest_folder(d.getVar("TUNE_FEATURES"), os.listdir(binaries_path)) extracted_bin_path = os.path.join(binaries_path, arch, "extracted_binaries") extracted_bin_list = [item for item in os.listdir(extracted_bin_path)] packaged_bin_path = os.path.join(binaries_path, arch, "packaged_binaries") @@ -206,9 +208,9 @@ def send_bin_to_DUT(d,params): from oeqa.oetest import oeRuntimeTest param_list = params cleanup_list = list() - bins_dir = os.path.join(d.getVar("TEST_EXPORT_DIR", True), "binaries") if determine_if_poky_env() \ + bins_dir = os.path.join(d.getVar("TEST_EXPORT_DIR"), "binaries") if determine_if_poky_env() \ else os.getenv("bin_dir") - arch = get_dest_folder(d.getVar("TUNE_FEATURES", True), os.listdir(bins_dir)) + arch = get_dest_folder(d.getVar("TUNE_FEATURES"), os.listdir(bins_dir)) arch_rpms_dir = os.path.join(bins_dir, arch, "packaged_binaries") extracted_bin_dir = os.path.join(bins_dir, arch, "extracted_binaries", param_list[0]) |