summaryrefslogtreecommitdiffstats
path: root/meta/lib/oeqa/utils
diff options
context:
space:
mode:
Diffstat (limited to 'meta/lib/oeqa/utils')
-rw-r--r--meta/lib/oeqa/utils/__init__.py32
-rw-r--r--meta/lib/oeqa/utils/buildproject.py15
-rw-r--r--meta/lib/oeqa/utils/commands.py87
-rw-r--r--meta/lib/oeqa/utils/decorators.py89
-rw-r--r--meta/lib/oeqa/utils/dump.py92
-rw-r--r--meta/lib/oeqa/utils/ftools.py6
-rw-r--r--meta/lib/oeqa/utils/git.py2
-rw-r--r--meta/lib/oeqa/utils/gitarchive.py283
-rw-r--r--meta/lib/oeqa/utils/httpserver.py66
-rw-r--r--meta/lib/oeqa/utils/logparser.py282
-rw-r--r--meta/lib/oeqa/utils/metadata.py28
-rw-r--r--meta/lib/oeqa/utils/network.py10
-rw-r--r--meta/lib/oeqa/utils/nfs.py43
-rw-r--r--meta/lib/oeqa/utils/package_manager.py23
-rw-r--r--meta/lib/oeqa/utils/postactions.py98
-rw-r--r--meta/lib/oeqa/utils/qemurunner.py600
-rw-r--r--meta/lib/oeqa/utils/qemutinyrunner.py14
-rw-r--r--meta/lib/oeqa/utils/sshcontrol.py7
-rw-r--r--meta/lib/oeqa/utils/subprocesstweak.py5
-rw-r--r--meta/lib/oeqa/utils/targetbuild.py13
-rw-r--r--meta/lib/oeqa/utils/testexport.py4
21 files changed, 1344 insertions, 455 deletions
diff --git a/meta/lib/oeqa/utils/__init__.py b/meta/lib/oeqa/utils/__init__.py
index d38a323013..53bdcbf266 100644
--- a/meta/lib/oeqa/utils/__init__.py
+++ b/meta/lib/oeqa/utils/__init__.py
@@ -1,3 +1,8 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
# Enable other layers to have modules in the same named directory
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
@@ -40,28 +45,12 @@ def make_logger_bitbake_compatible(logger):
import logging
"""
- Bitbake logger redifines debug() in order to
- set a level within debug, this breaks compatibility
- with vainilla logging, so we neeed to redifine debug()
- method again also add info() method with INFO + 1 level.
+ We need to raise the log level of the info output so unittest
+ messages are visible on the console.
"""
- def _bitbake_log_debug(*args, **kwargs):
- lvl = logging.DEBUG
-
- if isinstance(args[0], int):
- lvl = args[0]
- msg = args[1]
- args = args[2:]
- else:
- msg = args[0]
- args = args[1:]
-
- logger.log(lvl, msg, *args, **kwargs)
-
def _bitbake_log_info(msg, *args, **kwargs):
logger.log(logging.INFO + 1, msg, *args, **kwargs)
- logger.debug = _bitbake_log_debug
logger.info = _bitbake_log_info
return logger
@@ -101,3 +90,10 @@ def load_test_components(logger, executor):
"_executor_class defined." % (comp_name, comp_context))
return components
+
+def get_json_result_dir(d):
+ json_result_dir = os.path.join(d.getVar("LOG_DIR"), 'oeqa')
+ custom_json_result_dir = d.getVar("OEQA_JSON_RESULT_DIR")
+ if custom_json_result_dir:
+ json_result_dir = custom_json_result_dir
+ return json_result_dir \ No newline at end of file
diff --git a/meta/lib/oeqa/utils/buildproject.py b/meta/lib/oeqa/utils/buildproject.py
index 721f35d996..dfb9661868 100644
--- a/meta/lib/oeqa/utils/buildproject.py
+++ b/meta/lib/oeqa/utils/buildproject.py
@@ -1,6 +1,8 @@
+#
# Copyright (C) 2013-2016 Intel Corporation
#
-# Released under the MIT license (see COPYING.MIT)
+# SPDX-License-Identifier: MIT
+#
# Provides a class for automating build tests for projects
@@ -16,17 +18,22 @@ class BuildProject(metaclass=ABCMeta):
def __init__(self, uri, foldername=None, tmpdir=None, dl_dir=None):
self.uri = uri
self.archive = os.path.basename(uri)
+ self.tempdirobj = None
if not tmpdir:
- tmpdir = tempfile.mkdtemp(prefix='buildproject')
+ self.tempdirobj = tempfile.TemporaryDirectory(prefix='buildproject-')
+ tmpdir = self.tempdirobj.name
self.localarchive = os.path.join(tmpdir, self.archive)
self.dl_dir = dl_dir
if foldername:
self.fname = foldername
else:
self.fname = re.sub(r'\.tar\.bz2$|\.tar\.gz$|\.tar\.xz$', '', self.archive)
+ self.needclean = False
# Download self.archive to self.localarchive
def _download_archive(self):
+
+ self.needclean = True
if self.dl_dir and os.path.exists(os.path.join(self.dl_dir, self.archive)):
shutil.copyfile(os.path.join(self.dl_dir, self.archive), self.localarchive)
return
@@ -51,5 +58,9 @@ class BuildProject(metaclass=ABCMeta):
return self._run('cd %s; make install %s' % (self.targetdir, install_args))
def clean(self):
+ if self.tempdirobj:
+ self.tempdirobj.cleanup()
+ if not self.needclean:
+ return
self._run('rm -rf %s' % self.targetdir)
subprocess.check_call('rm -f %s' % self.localarchive, shell=True)
diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py
index cad0bea0be..575e380017 100644
--- a/meta/lib/oeqa/utils/commands.py
+++ b/meta/lib/oeqa/utils/commands.py
@@ -1,16 +1,15 @@
+#
# Copyright (c) 2013-2014 Intel Corporation
#
-# Released under the MIT license (see COPYING.MIT)
+# SPDX-License-Identifier: MIT
+#
# DESCRIPTION
# This module is mainly used by scripts/oe-selftest and modules under meta/oeqa/selftest
# It provides a class and methods for running commands on the host in a convienent way for tests.
-
-
import os
import sys
-import signal
import subprocess
import threading
import time
@@ -19,6 +18,7 @@ from oeqa.utils import CommandError
from oeqa.utils import ftools
import re
import contextlib
+import errno
# Export test doesn't require bb
try:
import bb
@@ -83,7 +83,7 @@ class Command(object):
except OSError as ex:
# It's not an error when the command does not consume all
# of our data. subprocess.communicate() also ignores that.
- if ex.errno != EPIPE:
+ if ex.errno != errno.EPIPE:
raise
# We write in a separate thread because then we can read
@@ -93,7 +93,9 @@ class Command(object):
# reason, the main process will still exit, which will then
# kill the write thread.
if self.data:
- threading.Thread(target=writeThread, daemon=True).start()
+ thread = threading.Thread(target=writeThread, daemon=True)
+ thread.start()
+ self.threads.append(thread)
if self.process.stderr:
thread = threading.Thread(target=readStderrThread)
thread.start()
@@ -113,7 +115,7 @@ class Command(object):
else:
deadline = time.time() + self.timeout
for thread in self.threads:
- timeout = deadline - time.time()
+ timeout = deadline - time.time()
if timeout < 0:
timeout = 0
thread.join(timeout)
@@ -121,11 +123,11 @@ class Command(object):
def stop(self):
for thread in self.threads:
- if thread.isAlive():
+ if thread.is_alive():
self.process.terminate()
# let's give it more time to terminate gracefully before killing it
thread.join(5)
- if thread.isAlive():
+ if thread.is_alive():
self.process.kill()
thread.join()
@@ -146,6 +148,9 @@ class Command(object):
# At this point we know that the process has closed stdout/stderr, so
# it is safe and necessary to wait for the actual process completion.
self.status = self.process.wait()
+ self.process.stdout.close()
+ if self.process.stderr:
+ self.process.stderr.close()
self.log.debug("Command '%s' returned %d as exit code." % (self.cmd, self.status))
# logging the complete output is insane
@@ -160,20 +165,36 @@ class Result(object):
pass
-def runCmd(command, ignore_status=False, timeout=None, assert_error=True,
- native_sysroot=None, limit_exc_output=0, output_log=None, **options):
+def runCmd(command, ignore_status=False, timeout=None, assert_error=True, sync=True,
+ native_sysroot=None, target_sys=None, limit_exc_output=0, output_log=None, **options):
result = Result()
if native_sysroot:
- extra_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin" % \
- (native_sysroot, native_sysroot, native_sysroot)
- nenv = dict(options.get('env', os.environ))
- nenv['PATH'] = extra_paths + ':' + nenv.get('PATH', '')
- options['env'] = nenv
+ new_env = dict(options.get('env', os.environ))
+ paths = new_env["PATH"].split(":")
+ paths = [
+ os.path.join(native_sysroot, "bin"),
+ os.path.join(native_sysroot, "sbin"),
+ os.path.join(native_sysroot, "usr", "bin"),
+ os.path.join(native_sysroot, "usr", "sbin"),
+ ] + paths
+ if target_sys:
+ paths = [os.path.join(native_sysroot, "usr", "bin", target_sys)] + paths
+ new_env["PATH"] = ":".join(paths)
+ options['env'] = new_env
cmd = Command(command, timeout=timeout, output_log=output_log, **options)
cmd.run()
+ # tests can be heavy on IO and if bitbake can't write out its caches, we see timeouts.
+ # call sync around the tests to ensure the IO queue doesn't get too large, taking any IO
+ # hit here rather than in bitbake shutdown.
+ if sync:
+ p = os.environ['PATH']
+ os.environ['PATH'] = "/usr/bin:/bin:/usr/sbin:/sbin:" + p
+ os.system("sync")
+ os.environ['PATH'] = p
+
result.command = command
result.status = cmd.status
result.output = cmd.output
@@ -264,8 +285,10 @@ def get_bb_vars(variables=None, target=None, postconfig=None):
def get_bb_var(var, target=None, postconfig=None):
return get_bb_vars([var], target, postconfig)[var]
-def get_test_layer():
- layers = get_bb_var("BBLAYERS").split()
+def get_test_layer(bblayers=None):
+ if bblayers is None:
+ bblayers = get_bb_var("BBLAYERS")
+ layers = bblayers.split()
testlayer = None
for l in layers:
if '~' in l:
@@ -277,6 +300,7 @@ def get_test_layer():
def create_temp_layer(templayerdir, templayername, priority=999, recipepathspec='recipes-*/*'):
os.makedirs(os.path.join(templayerdir, 'conf'))
+ corenames = get_bb_var('LAYERSERIES_CORENAMES')
with open(os.path.join(templayerdir, 'conf', 'layer.conf'), 'w') as f:
f.write('BBPATH .= ":${LAYERDIR}"\n')
f.write('BBFILES += "${LAYERDIR}/%s/*.bb \\' % recipepathspec)
@@ -285,7 +309,7 @@ def create_temp_layer(templayerdir, templayername, priority=999, recipepathspec=
f.write('BBFILE_PATTERN_%s = "^${LAYERDIR}/"\n' % templayername)
f.write('BBFILE_PRIORITY_%s = "%d"\n' % (templayername, priority))
f.write('BBFILE_PATTERN_IGNORE_EMPTY_%s = "1"\n' % templayername)
-
+ f.write('LAYERSERIES_COMPAT_%s = "%s"\n' % (templayername, corenames))
@contextlib.contextmanager
def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None, qemuparams=None, overrides={}, discard_writes=True):
@@ -307,15 +331,15 @@ def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None,
try:
tinfoil.logger.setLevel(logging.WARNING)
import oeqa.targetcontrol
- tinfoil.config_data.setVar("TEST_LOG_DIR", "${WORKDIR}/testimage")
- tinfoil.config_data.setVar("TEST_QEMUBOOT_TIMEOUT", "1000")
+ recipedata = tinfoil.parse_recipe(pn)
+ recipedata.setVar("TEST_LOG_DIR", "${WORKDIR}/testimage")
+ recipedata.setVar("TEST_QEMUBOOT_TIMEOUT", "1000")
# Tell QemuTarget() whether need find rootfs/kernel or not
if launch_cmd:
- tinfoil.config_data.setVar("FIND_ROOTFS", '0')
+ recipedata.setVar("FIND_ROOTFS", '0')
else:
- tinfoil.config_data.setVar("FIND_ROOTFS", '1')
+ recipedata.setVar("FIND_ROOTFS", '1')
- recipedata = tinfoil.parse_recipe(pn)
for key, value in overrides.items():
recipedata.setVar(key, value)
@@ -332,17 +356,18 @@ def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None,
qemu.deploy()
try:
qemu.start(params=qemuparams, ssh=ssh, runqemuparams=runqemuparams, launch_cmd=launch_cmd, discard_writes=discard_writes)
- except bb.build.FuncFailed:
- raise Exception('Failed to start QEMU - see the logs in %s' % logdir)
+ except Exception as e:
+ msg = str(e) + '\nFailed to start QEMU - see the logs in %s' % logdir
+ if os.path.exists(qemu.qemurunnerlog):
+ with open(qemu.qemurunnerlog, 'r') as f:
+ msg = msg + "Qemurunner log output from %s:\n%s" % (qemu.qemurunnerlog, f.read())
+ raise Exception(msg)
yield qemu
finally:
- try:
- qemu.stop()
- except:
- pass
- targetlogger.removeHandler(handler)
+ targetlogger.removeHandler(handler)
+ qemu.stop()
def updateEnv(env_file):
"""
diff --git a/meta/lib/oeqa/utils/decorators.py b/meta/lib/oeqa/utils/decorators.py
index d876896921..ea90164e5e 100644
--- a/meta/lib/oeqa/utils/decorators.py
+++ b/meta/lib/oeqa/utils/decorators.py
@@ -1,6 +1,8 @@
+#
# Copyright (C) 2013 Intel Corporation
#
-# Released under the MIT license (see COPYING.MIT)
+# SPDX-License-Identifier: MIT
+#
# Some custom decorators that can be used by unittests
# Most useful is skipUnlessPassed which can be used for
@@ -14,91 +16,6 @@ import threading
import signal
from functools import wraps
-#get the "result" object from one of the upper frames provided that one of these upper frames is a unittest.case frame
-class getResults(object):
- def __init__(self):
- #dynamically determine the unittest.case frame and use it to get the name of the test method
- ident = threading.current_thread().ident
- upperf = sys._current_frames()[ident]
- while (upperf.f_globals['__name__'] != 'unittest.case'):
- upperf = upperf.f_back
-
- def handleList(items):
- ret = []
- # items is a list of tuples, (test, failure) or (_ErrorHandler(), Exception())
- for i in items:
- s = i[0].id()
- #Handle the _ErrorHolder objects from skipModule failures
- if "setUpModule (" in s:
- ret.append(s.replace("setUpModule (", "").replace(")",""))
- else:
- ret.append(s)
- # Append also the test without the full path
- testname = s.split('.')[-1]
- if testname:
- ret.append(testname)
- return ret
- self.faillist = handleList(upperf.f_locals['result'].failures)
- self.errorlist = handleList(upperf.f_locals['result'].errors)
- self.skiplist = handleList(upperf.f_locals['result'].skipped)
-
- def getFailList(self):
- return self.faillist
-
- def getErrorList(self):
- return self.errorlist
-
- def getSkipList(self):
- return self.skiplist
-
-class skipIfFailure(object):
-
- def __init__(self,testcase):
- self.testcase = testcase
-
- def __call__(self,f):
- @wraps(f)
- def wrapped_f(*args, **kwargs):
- res = getResults()
- if self.testcase in (res.getFailList() or res.getErrorList()):
- raise unittest.SkipTest("Testcase dependency not met: %s" % self.testcase)
- return f(*args, **kwargs)
- wrapped_f.__name__ = f.__name__
- return wrapped_f
-
-class skipIfSkipped(object):
-
- def __init__(self,testcase):
- self.testcase = testcase
-
- def __call__(self,f):
- @wraps(f)
- def wrapped_f(*args, **kwargs):
- res = getResults()
- if self.testcase in res.getSkipList():
- raise unittest.SkipTest("Testcase dependency not met: %s" % self.testcase)
- return f(*args, **kwargs)
- wrapped_f.__name__ = f.__name__
- return wrapped_f
-
-class skipUnlessPassed(object):
-
- def __init__(self,testcase):
- self.testcase = testcase
-
- def __call__(self,f):
- @wraps(f)
- def wrapped_f(*args, **kwargs):
- res = getResults()
- if self.testcase in res.getSkipList() or \
- self.testcase in res.getFailList() or \
- self.testcase in res.getErrorList():
- raise unittest.SkipTest("Testcase dependency not met: %s" % self.testcase)
- return f(*args, **kwargs)
- wrapped_f.__name__ = f.__name__
- wrapped_f._depends_on = self.testcase
- return wrapped_f
-
class testcase(object):
def __init__(self, test_case):
self.test_case = test_case
diff --git a/meta/lib/oeqa/utils/dump.py b/meta/lib/oeqa/utils/dump.py
index 5a7edc1a86..d4d271369f 100644
--- a/meta/lib/oeqa/utils/dump.py
+++ b/meta/lib/oeqa/utils/dump.py
@@ -1,5 +1,12 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+
import os
import sys
+import json
import errno
import datetime
import itertools
@@ -12,7 +19,8 @@ class BaseDumper(object):
self.cmds = []
# Some testing doesn't inherit testimage, so it is needed
# to set some defaults.
- self.parent_dir = parent_dir or "/tmp/oe-saved-tests"
+ self.parent_dir = parent_dir
+ self.dump_dir = parent_dir
dft_cmds = """ top -bn1
iostat -x -z -N -d -p ALL 20 2
ps -ef
@@ -42,11 +50,11 @@ class BaseDumper(object):
raise err
self.dump_dir = dump_dir
- def _write_dump(self, command, output):
- if isinstance(self, HostDumper):
- prefix = "host"
- elif isinstance(self, TargetDumper):
+ def _construct_filename(self, command):
+ if isinstance(self, TargetDumper):
prefix = "target"
+ elif isinstance(self, MonitorDumper):
+ prefix = "qmp"
else:
prefix = "unknown"
for i in itertools.count():
@@ -54,38 +62,80 @@ class BaseDumper(object):
fullname = os.path.join(self.dump_dir, filename)
if not os.path.exists(fullname):
break
- with open(fullname, 'w') as dump_file:
- dump_file.write(output)
-
-
-class HostDumper(BaseDumper):
- """ Class to get dumps from the host running the tests """
-
- def __init__(self, cmds, parent_dir):
- super(HostDumper, self).__init__(cmds, parent_dir)
+ return fullname
- def dump_host(self, dump_dir=""):
- if dump_dir:
- self.dump_dir = dump_dir
- for cmd in self.cmds:
- result = runCmd(cmd, ignore_status=True)
- self._write_dump(cmd.split()[0], result.output)
+ def _write_dump(self, command, output):
+ fullname = self._construct_filename(command)
+ os.makedirs(os.path.dirname(fullname), exist_ok=True)
+ if isinstance(self, MonitorDumper):
+ with open(fullname, 'w') as json_file:
+ json.dump(output, json_file, indent=4)
+ else:
+ with open(fullname, 'w') as dump_file:
+ dump_file.write(output)
class TargetDumper(BaseDumper):
- """ Class to get dumps from target, it only works with QemuRunner """
+ """ Class to get dumps from target, it only works with QemuRunner.
+ Will give up permanently after 5 errors from running commands over
+ serial console. This helps to end testing when target is really dead, hanging
+ or unresponsive.
+ """
def __init__(self, cmds, parent_dir, runner):
super(TargetDumper, self).__init__(cmds, parent_dir)
self.runner = runner
+ self.errors = 0
def dump_target(self, dump_dir=""):
+ if self.errors >= 5:
+ print("Too many errors when dumping data from target, assuming it is dead! Will not dump data anymore!")
+ return
if dump_dir:
self.dump_dir = dump_dir
for cmd in self.cmds:
# We can continue with the testing if serial commands fail
try:
(status, output) = self.runner.run_serial(cmd)
+ if status == 0:
+ self.errors = self.errors + 1
self._write_dump(cmd.split()[0], output)
except:
+ self.errors = self.errors + 1
print("Tried to dump info from target but "
"serial console failed")
+ print("Failed CMD: %s" % (cmd))
+
+class MonitorDumper(BaseDumper):
+ """ Class to get dumps via the Qemu Monitor, it only works with QemuRunner
+ Will stop completely if there are more than 5 errors when dumping monitor data.
+ This helps to end testing when target is really dead, hanging or unresponsive.
+ """
+
+ def __init__(self, cmds, parent_dir, runner):
+ super(MonitorDumper, self).__init__(cmds, parent_dir)
+ self.runner = runner
+ self.errors = 0
+
+ def dump_monitor(self, dump_dir=""):
+ if self.runner is None:
+ return
+ if dump_dir:
+ self.dump_dir = dump_dir
+ if self.errors >= 5:
+ print("Too many errors when dumping data from qemu monitor, assuming it is dead! Will not dump data anymore!")
+ return
+ for cmd in self.cmds:
+ cmd_name = cmd.split()[0]
+ try:
+ if len(cmd.split()) > 1:
+ cmd_args = cmd.split()[1]
+ if "%s" in cmd_args:
+ filename = self._construct_filename(cmd_name)
+ cmd_data = json.loads(cmd_args % (filename))
+ output = self.runner.run_monitor(cmd_name, cmd_data)
+ else:
+ output = self.runner.run_monitor(cmd_name)
+ self._write_dump(cmd_name, output)
+ except Exception as e:
+ self.errors = self.errors + 1
+ print("Failed to dump QMP CMD: %s with\nException: %s" % (cmd_name, e))
diff --git a/meta/lib/oeqa/utils/ftools.py b/meta/lib/oeqa/utils/ftools.py
index a7233d4ca6..a50aaa84c2 100644
--- a/meta/lib/oeqa/utils/ftools.py
+++ b/meta/lib/oeqa/utils/ftools.py
@@ -1,3 +1,9 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+
import os
import re
import errno
diff --git a/meta/lib/oeqa/utils/git.py b/meta/lib/oeqa/utils/git.py
index 757e3f0cbf..ea35a766eb 100644
--- a/meta/lib/oeqa/utils/git.py
+++ b/meta/lib/oeqa/utils/git.py
@@ -1,7 +1,7 @@
#
# Copyright (C) 2016 Intel Corporation
#
-# Released under the MIT license (see COPYING.MIT)
+# SPDX-License-Identifier: MIT
#
"""Git repository interactions"""
import os
diff --git a/meta/lib/oeqa/utils/gitarchive.py b/meta/lib/oeqa/utils/gitarchive.py
new file mode 100644
index 0000000000..10cb267dfa
--- /dev/null
+++ b/meta/lib/oeqa/utils/gitarchive.py
@@ -0,0 +1,283 @@
+#
+# Helper functions for committing data to git and pushing upstream
+#
+# Copyright (c) 2017, Intel Corporation.
+# Copyright (c) 2019, Linux Foundation
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import os
+import re
+import sys
+from operator import attrgetter
+from collections import namedtuple
+from oeqa.utils.git import GitRepo, GitError
+
+class ArchiveError(Exception):
+ """Internal error handling of this script"""
+
+def format_str(string, fields):
+ """Format string using the given fields (dict)"""
+ try:
+ return string.format(**fields)
+ except KeyError as err:
+ raise ArchiveError("Unable to expand string '{}': unknown field {} "
+ "(valid fields are: {})".format(
+ string, err, ', '.join(sorted(fields.keys()))))
+
+
+def init_git_repo(path, no_create, bare, log):
+ """Initialize local Git repository"""
+ path = os.path.abspath(path)
+ if os.path.isfile(path):
+ raise ArchiveError("Invalid Git repo at {}: path exists but is not a "
+ "directory".format(path))
+ if not os.path.isdir(path) or not os.listdir(path):
+ if no_create:
+ raise ArchiveError("No git repo at {}, refusing to create "
+ "one".format(path))
+ if not os.path.isdir(path):
+ try:
+ os.mkdir(path)
+ except (FileNotFoundError, PermissionError) as err:
+ raise ArchiveError("Failed to mkdir {}: {}".format(path, err))
+ if not os.listdir(path):
+ log.info("Initializing a new Git repo at %s", path)
+ repo = GitRepo.init(path, bare)
+ try:
+ repo = GitRepo(path, is_topdir=True)
+ except GitError:
+ raise ArchiveError("Non-empty directory that is not a Git repository "
+ "at {}\nPlease specify an existing Git repository, "
+ "an empty directory or a non-existing directory "
+ "path.".format(path))
+ return repo
+
+
+def git_commit_data(repo, data_dir, branch, message, exclude, notes, log):
+ """Commit data into a Git repository"""
+ log.info("Committing data into to branch %s", branch)
+ tmp_index = os.path.join(repo.git_dir, 'index.oe-git-archive')
+ try:
+ # Create new tree object from the data
+ env_update = {'GIT_INDEX_FILE': tmp_index,
+ 'GIT_WORK_TREE': os.path.abspath(data_dir)}
+ repo.run_cmd('add .', env_update)
+
+ # Remove files that are excluded
+ if exclude:
+ repo.run_cmd(['rm', '--cached'] + [f for f in exclude], env_update)
+
+ tree = repo.run_cmd('write-tree', env_update)
+
+ # Create new commit object from the tree
+ parent = repo.rev_parse(branch)
+ if not parent:
+ parent = repo.rev_parse("origin/" + branch)
+ git_cmd = ['commit-tree', tree, '-m', message]
+ if parent:
+ git_cmd += ['-p', parent]
+ commit = repo.run_cmd(git_cmd, env_update)
+
+ # Create git notes
+ for ref, filename in notes:
+ ref = ref.format(branch_name=branch)
+ repo.run_cmd(['notes', '--ref', ref, 'add',
+ '-F', os.path.abspath(filename), commit])
+
+ # Update branch head
+ git_cmd = ['update-ref', 'refs/heads/' + branch, commit]
+ repo.run_cmd(git_cmd)
+
+ # Update current HEAD, if we're on branch 'branch'
+ if not repo.bare and repo.get_current_branch() == branch:
+ log.info("Updating %s HEAD to latest commit", repo.top_dir)
+ repo.run_cmd('reset --hard')
+
+ return commit
+ finally:
+ if os.path.exists(tmp_index):
+ os.unlink(tmp_index)
+
+def get_tags(repo, log, pattern=None, url=None):
+ """ Fetch remote tags from current repository
+
+ A pattern can be provided to filter returned tags list
+ An URL can be provided if local repository has no valid remote configured
+ """
+
+ base_cmd = ['ls-remote', '--refs', '--tags', '-q']
+ cmd = base_cmd.copy()
+
+ # First try to fetch tags from repository configured remote
+ cmd.append('origin')
+ if pattern:
+ cmd.append("refs/tags/"+pattern)
+ try:
+ tags_refs = repo.run_cmd(cmd)
+ tags = ["".join(d.split()[1].split('/', 2)[2:]) for d in tags_refs.splitlines()]
+ except GitError as e:
+ # If it fails, retry with repository url if one is provided
+ if url:
+ log.info("No remote repository configured, use provided url")
+ cmd = base_cmd.copy()
+ cmd.append(url)
+ if pattern:
+ cmd.append(pattern)
+ tags_refs = repo.run_cmd(cmd)
+ tags = ["".join(d.split()[1].split('/', 2)[2:]) for d in tags_refs.splitlines()]
+ else:
+ log.info("Read local tags only, some remote tags may be missed")
+ cmd = ["tag"]
+ if pattern:
+ cmd += ["-l", pattern]
+ tags = repo.run_cmd(cmd).splitlines()
+
+ return tags
+
+def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern,
+ url, log, keywords):
+ """Generate tag name and message, with support for running id number"""
+ keyws = keywords.copy()
+ # Tag number is handled specially: if not defined, we autoincrement it
+ if 'tag_number' not in keyws:
+ # Fill in all other fields than 'tag_number'
+ keyws['tag_number'] = '{tag_number}'
+ tag_re = format_str(name_pattern, keyws)
+ # Replace parentheses for proper regex matching
+ tag_re = tag_re.replace('(', '\(').replace(')', '\)') + '$'
+ # Inject regex group pattern for 'tag_number'
+ tag_re = tag_re.format(tag_number='(?P<tag_number>[0-9]{1,5})')
+
+ keyws['tag_number'] = 0
+ for existing_tag in get_tags(repo, log, url=url):
+ match = re.match(tag_re, existing_tag)
+
+ if match and int(match.group('tag_number')) >= keyws['tag_number']:
+ keyws['tag_number'] = int(match.group('tag_number')) + 1
+
+ tag_name = format_str(name_pattern, keyws)
+ msg_subj= format_str(msg_subj_pattern.strip(), keyws)
+ msg_body = format_str(msg_body_pattern, keyws)
+ return tag_name, msg_subj + '\n\n' + msg_body
+
+def gitarchive(data_dir, git_dir, no_create, bare, commit_msg_subject, commit_msg_body, branch_name, no_tag, tagname, tag_msg_subject, tag_msg_body, exclude, notes, push, keywords, log):
+
+ if not os.path.isdir(data_dir):
+ raise ArchiveError("Not a directory: {}".format(data_dir))
+
+ data_repo = init_git_repo(git_dir, no_create, bare, log)
+
+ # Expand strings early in order to avoid getting into inconsistent
+ # state (e.g. no tag even if data was committed)
+ commit_msg = format_str(commit_msg_subject.strip(), keywords)
+ commit_msg += '\n\n' + format_str(commit_msg_body, keywords)
+ branch_name = format_str(branch_name, keywords)
+ tag_name = None
+ if not no_tag and tagname:
+ tag_name, tag_msg = expand_tag_strings(data_repo, tagname,
+ tag_msg_subject,
+ tag_msg_body,
+ push, log, keywords)
+
+ # Commit data
+ commit = git_commit_data(data_repo, data_dir, branch_name,
+ commit_msg, exclude, notes, log)
+
+ # Create tag
+ if tag_name:
+ log.info("Creating tag %s", tag_name)
+ data_repo.run_cmd(['tag', '-a', '-m', tag_msg, tag_name, commit])
+
+ # Push data to remote
+ if push:
+ cmd = ['push', '--tags']
+ # If no remote is given we push with the default settings from
+ # gitconfig
+ if push is not True:
+ notes_refs = ['refs/notes/' + ref.format(branch_name=branch_name)
+ for ref, _ in notes]
+ cmd.extend([push, branch_name] + notes_refs)
+ log.info("Pushing data to remote")
+ data_repo.run_cmd(cmd)
+
+# Container class for tester revisions
+TestedRev = namedtuple('TestedRev', 'commit commit_number tags')
+
+def get_test_runs(log, repo, tag_name, **kwargs):
+ """Get a sorted list of test runs, matching given pattern"""
+ # First, get field names from the tag name pattern
+ field_names = [m.group(1) for m in re.finditer(r'{(\w+)}', tag_name)]
+ undef_fields = [f for f in field_names if f not in kwargs.keys()]
+
+ # Fields for formatting tag name pattern
+ str_fields = dict([(f, '*') for f in field_names])
+ str_fields.update(kwargs)
+
+ # Get a list of all matching tags
+ tag_pattern = tag_name.format(**str_fields)
+ tags = get_tags(repo, log, pattern=tag_pattern)
+ log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern)
+
+ # Parse undefined fields from tag names
+ str_fields = dict([(f, r'(?P<{}>[\w\-.()]+)'.format(f)) for f in field_names])
+ str_fields['branch'] = r'(?P<branch>[\w\-.()/]+)'
+ str_fields['commit'] = '(?P<commit>[0-9a-f]{7,40})'
+ str_fields['commit_number'] = '(?P<commit_number>[0-9]{1,7})'
+ str_fields['tag_number'] = '(?P<tag_number>[0-9]{1,5})'
+ # escape parenthesis in fields in order to not messa up the regexp
+ fixed_fields = dict([(k, v.replace('(', r'\(').replace(')', r'\)')) for k, v in kwargs.items()])
+ str_fields.update(fixed_fields)
+ tag_re = re.compile(tag_name.format(**str_fields))
+
+ # Parse fields from tags
+ revs = []
+ for tag in tags:
+ m = tag_re.match(tag)
+ if not m:
+ continue
+ groups = m.groupdict()
+ revs.append([groups[f] for f in undef_fields] + [tag])
+
+ # Return field names and a sorted list of revs
+ return undef_fields, sorted(revs)
+
+def get_test_revs(log, repo, tag_name, **kwargs):
+ """Get list of all tested revisions"""
+ fields, runs = get_test_runs(log, repo, tag_name, **kwargs)
+
+ revs = {}
+ commit_i = fields.index('commit')
+ commit_num_i = fields.index('commit_number')
+ for run in runs:
+ commit = run[commit_i]
+ commit_num = run[commit_num_i]
+ tag = run[-1]
+ if not commit in revs:
+ revs[commit] = TestedRev(commit, commit_num, [tag])
+ else:
+ if commit_num != revs[commit].commit_number:
+ # Historically we have incorrect commit counts of '1' in the repo so fix these up
+ if int(revs[commit].commit_number) < 5:
+ tags = revs[commit].tags
+ revs[commit] = TestedRev(commit, commit_num, [tags])
+ elif int(commit_num) < 5:
+ pass
+ else:
+ sys.exit("Commit numbers for commit %s don't match (%s vs %s)" % (commit, commit_num, revs[commit].commit_number))
+ revs[commit].tags.append(tag)
+
+ # Return in sorted table
+ revs = sorted(revs.values(), key=attrgetter('commit_number'))
+ log.debug("Found %d tested revisions:\n %s", len(revs),
+ "\n ".join(['{} ({})'.format(rev.commit_number, rev.commit) for rev in revs]))
+ return revs
+
+def rev_find(revs, attr, val):
+ """Search from a list of TestedRev"""
+ for i, rev in enumerate(revs):
+ if getattr(rev, attr) == val:
+ return i
+ raise ValueError("Unable to find '{}' value '{}'".format(attr, val))
+
diff --git a/meta/lib/oeqa/utils/httpserver.py b/meta/lib/oeqa/utils/httpserver.py
index 7d12331453..80752c1377 100644
--- a/meta/lib/oeqa/utils/httpserver.py
+++ b/meta/lib/oeqa/utils/httpserver.py
@@ -1,36 +1,80 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+
import http.server
+import logging
import multiprocessing
import os
+import signal
from socketserver import ThreadingMixIn
class HTTPServer(ThreadingMixIn, http.server.HTTPServer):
- def server_start(self, root_dir):
- import signal
- signal.signal(signal.SIGTERM, signal.SIG_DFL)
+ def server_start(self, root_dir, logger):
os.chdir(root_dir)
+ self.logger = logger
self.serve_forever()
class HTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
def log_message(self, format_str, *args):
- pass
+ self.server.logger.info(format_str, *args)
-class HTTPService(object):
+class HTTPService:
- def __init__(self, root_dir, host=''):
+ def __init__(self, root_dir, host='', port=0, logger=None):
self.root_dir = root_dir
self.host = host
- self.port = 0
+ self.port = port
+ if logger:
+ self.logger = logger.getChild("HTTPService")
+ else:
+ self.logger = logging.getLogger("HTTPService")
def start(self):
+ if not os.path.exists(self.root_dir):
+ self.logger.info("Not starting HTTPService for directory %s which doesn't exist" % (self.root_dir))
+ return
+
self.server = HTTPServer((self.host, self.port), HTTPRequestHandler)
if self.port == 0:
self.port = self.server.server_port
- self.process = multiprocessing.Process(target=self.server.server_start, args=[self.root_dir])
+ self.process = multiprocessing.Process(target=self.server.server_start, args=[self.root_dir, self.logger])
+
+ def handle_error(self, request, client_address):
+ import traceback
+ exception = traceback.format_exc()
+ self.logger.warn("Exception when handling %s: %s" % (request, exception))
+ self.server.handle_error = handle_error
+
+ # The signal handler from testimage.bbclass can cause deadlocks here
+ # if the HTTPServer is terminated before it can restore the standard
+ #signal behaviour
+ orig = signal.getsignal(signal.SIGTERM)
+ signal.signal(signal.SIGTERM, signal.SIG_DFL)
self.process.start()
+ signal.signal(signal.SIGTERM, orig)
+
+ if self.logger:
+ self.logger.info("Started HTTPService for %s on %s:%s" % (self.root_dir, self.host, self.port))
+
def stop(self):
- self.server.server_close()
- self.process.terminate()
- self.process.join()
+ if hasattr(self, "server"):
+ self.server.server_close()
+ if hasattr(self, "process"):
+ self.process.terminate()
+ self.process.join()
+ if self.logger:
+ self.logger.info("Stopped HTTPService on %s:%s" % (self.host, self.port))
+
+if __name__ == "__main__":
+ import sys, logging
+
+ logger = logging.getLogger(__name__)
+ logging.basicConfig(level=logging.DEBUG)
+ httpd = HTTPService(sys.argv[1], port=8888, logger=logger)
+ httpd.start()
diff --git a/meta/lib/oeqa/utils/logparser.py b/meta/lib/oeqa/utils/logparser.py
index 0670627c3c..496d9e0c90 100644
--- a/meta/lib/oeqa/utils/logparser.py
+++ b/meta/lib/oeqa/utils/logparser.py
@@ -1,126 +1,184 @@
-#!/usr/bin/env python
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
-import sys
+import enum
import os
import re
-from . import ftools
-
# A parser that can be used to identify weather a line is a test result or a section statement.
-class Lparser(object):
-
- def __init__(self, test_0_pass_regex, test_0_fail_regex, test_0_skip_regex, section_0_begin_regex=None, section_0_end_regex=None, **kwargs):
- # Initialize the arguments dictionary
- if kwargs:
- self.args = kwargs
- else:
- self.args = {}
-
- # Add the default args to the dictionary
- self.args['test_0_pass_regex'] = test_0_pass_regex
- self.args['test_0_fail_regex'] = test_0_fail_regex
- self.args['test_0_skip_regex'] = test_0_skip_regex
- if section_0_begin_regex:
- self.args['section_0_begin_regex'] = section_0_begin_regex
- if section_0_end_regex:
- self.args['section_0_end_regex'] = section_0_end_regex
-
- self.test_possible_status = ['pass', 'fail', 'error', 'skip']
- self.section_possible_status = ['begin', 'end']
-
- self.initialized = False
-
-
- # Initialize the parser with the current configuration
- def init(self):
-
- # extra arguments can be added by the user to define new test and section categories. They must follow a pre-defined pattern: <type>_<category_name>_<status>_regex
- self.test_argument_pattern = "^test_(.+?)_(%s)_regex" % '|'.join(map(str, self.test_possible_status))
- self.section_argument_pattern = "^section_(.+?)_(%s)_regex" % '|'.join(map(str, self.section_possible_status))
-
- # Initialize the test and section regex dictionaries
- self.test_regex = {}
- self.section_regex ={}
-
- for arg, value in self.args.items():
- if not value:
- raise Exception('The value of provided argument %s is %s. Should have a valid value.' % (key, value))
- is_test = re.search(self.test_argument_pattern, arg)
- is_section = re.search(self.section_argument_pattern, arg)
- if is_test:
- if not is_test.group(1) in self.test_regex:
- self.test_regex[is_test.group(1)] = {}
- self.test_regex[is_test.group(1)][is_test.group(2)] = re.compile(value)
- elif is_section:
- if not is_section.group(1) in self.section_regex:
- self.section_regex[is_section.group(1)] = {}
- self.section_regex[is_section.group(1)][is_section.group(2)] = re.compile(value)
- else:
- # TODO: Make these call a traceback instead of a simple exception..
- raise Exception("The provided argument name does not correspond to any valid type. Please give one of the following types:\nfor tests: %s\nfor sections: %s" % (self.test_argument_pattern, self.section_argument_pattern))
-
- self.initialized = True
-
- # Parse a line and return a tuple containing the type of result (test/section) and its category, status and name
- def parse_line(self, line):
- if not self.initialized:
- raise Exception("The parser is not initialized..")
-
- for test_category, test_status_list in self.test_regex.items():
- for test_status, status_regex in test_status_list.items():
- test_name = status_regex.search(line)
- if test_name:
- return ['test', test_category, test_status, test_name.group(1)]
-
- for section_category, section_status_list in self.section_regex.items():
- for section_status, status_regex in section_status_list.items():
- section_name = status_regex.search(line)
- if section_name:
- return ['section', section_category, section_status, section_name.group(1)]
- return None
-
-
-class Result(object):
-
+class PtestParser(object):
def __init__(self):
- self.result_dict = {}
-
- def store(self, section, test, status):
- if not section in self.result_dict:
- self.result_dict[section] = []
-
- self.result_dict[section].append((test, status))
-
- # sort tests by the test name(the first element of the tuple), for each section. This can be helpful when using git to diff for changes by making sure they are always in the same order.
- def sort_tests(self):
- for package in self.result_dict:
- sorted_results = sorted(self.result_dict[package], key=lambda tup: tup[0])
- self.result_dict[package] = sorted_results
+ self.results = {}
+ self.sections = {}
+
+ def parse(self, logfile):
+ test_regex = {}
+ test_regex['PASSED'] = re.compile(r"^PASS:(.+)")
+ test_regex['FAILED'] = re.compile(r"^FAIL:([^(]+)")
+ test_regex['SKIPPED'] = re.compile(r"^SKIP:(.+)")
+
+ section_regex = {}
+ section_regex['begin'] = re.compile(r"^BEGIN: .*/(.+)/ptest")
+ section_regex['end'] = re.compile(r"^END: .*/(.+)/ptest")
+ section_regex['duration'] = re.compile(r"^DURATION: (.+)")
+ section_regex['exitcode'] = re.compile(r"^ERROR: Exit status is (.+)")
+ section_regex['timeout'] = re.compile(r"^TIMEOUT: .*/(.+)/ptest")
+
+ # Cache markers so we don't take the re.search() hit all the time.
+ markers = ("PASS:", "FAIL:", "SKIP:", "BEGIN:", "END:", "DURATION:", "ERROR: Exit", "TIMEOUT:")
+
+ def newsection():
+ return { 'name': "No-section", 'log': [] }
+
+ current_section = newsection()
+
+ with open(logfile, errors='replace') as f:
+ for line in f:
+ if not line.startswith(markers):
+ current_section['log'].append(line)
+ continue
+
+ result = section_regex['begin'].search(line)
+ if result:
+ current_section['name'] = result.group(1)
+ if current_section['name'] not in self.results:
+ self.results[current_section['name']] = {}
+ continue
+
+ result = section_regex['end'].search(line)
+ if result:
+ if current_section['name'] != result.group(1):
+ bb.warn("Ptest END log section mismatch %s vs. %s" % (current_section['name'], result.group(1)))
+ if current_section['name'] in self.sections:
+ bb.warn("Ptest duplicate section for %s" % (current_section['name']))
+ self.sections[current_section['name']] = current_section
+ del self.sections[current_section['name']]['name']
+ current_section = newsection()
+ continue
+
+ result = section_regex['timeout'].search(line)
+ if result:
+ if current_section['name'] != result.group(1):
+ bb.warn("Ptest TIMEOUT log section mismatch %s vs. %s" % (current_section['name'], result.group(1)))
+ current_section['timeout'] = True
+ continue
+
+ for t in ['duration', 'exitcode']:
+ result = section_regex[t].search(line)
+ if result:
+ current_section[t] = result.group(1)
+ continue
+
+ current_section['log'].append(line)
+
+ for t in test_regex:
+ result = test_regex[t].search(line)
+ if result:
+ try:
+ self.results[current_section['name']][result.group(1).strip()] = t
+ except KeyError:
+ bb.warn("Result with no section: %s - %s" % (t, result.group(1).strip()))
+
+ # Python performance for repeatedly joining long strings is poor, do it all at once at the end.
+ # For 2.1 million lines in a log this reduces 18 hours to 12s.
+ for section in self.sections:
+ self.sections[section]['log'] = "".join(self.sections[section]['log'])
+
+ return self.results, self.sections
# Log the results as files. The file name is the section name and the contents are the tests in that section.
- def log_as_files(self, target_dir, test_status):
- status_regex = re.compile('|'.join(map(str, test_status)))
- if not type(test_status) == type([]):
- raise Exception("test_status should be a list. Got " + str(test_status) + " instead.")
+ def results_as_files(self, target_dir):
if not os.path.exists(target_dir):
raise Exception("Target directory does not exist: %s" % target_dir)
- for section, test_results in self.result_dict.items():
- prefix = ''
- for x in test_status:
- prefix +=x+'.'
+ for section in self.results:
+ prefix = 'No-section'
if section:
- prefix += section
+ prefix = section
section_file = os.path.join(target_dir, prefix)
# purge the file contents if it exists
- open(section_file, 'w').close()
- for test_result in test_results:
- (test_name, status) = test_result
- # we log only the tests with status in the test_status list
- match_status = status_regex.search(status)
- if match_status:
- ftools.append_file(section_file, status + ": " + test_name)
-
- # Not yet implemented!
- def log_to_lava(self):
- pass
+ with open(section_file, 'w') as f:
+ for test_name in sorted(self.results[section]):
+ status = self.results[section][test_name]
+ f.write(status + ": " + test_name + "\n")
+
+
+class LtpParser:
+ """
+ Parse the machine-readable LTP log output into a ptest-friendly data structure.
+ """
+ def parse(self, logfile):
+ results = {}
+ # Aaccumulate the duration here but as the log rounds quick tests down
+ # to 0 seconds this is very much a lower bound. The caller can replace
+ # the value.
+ section = {"duration": 0, "log": ""}
+
+ class LtpExitCode(enum.IntEnum):
+ # Exit codes as defined in ltp/include/tst_res_flags.h
+ TPASS = 0 # Test passed flag
+ TFAIL = 1 # Test failed flag
+ TBROK = 2 # Test broken flag
+ TWARN = 4 # Test warning flag
+ TINFO = 16 # Test information flag
+ TCONF = 32 # Test not appropriate for configuration flag
+
+ with open(logfile, errors="replace") as f:
+ # Lines look like this:
+ # tag=cfs_bandwidth01 stime=1689762564 dur=0 exit=exited stat=32 core=no cu=0 cs=0
+ for line in f:
+ if not line.startswith("tag="):
+ continue
+
+ values = dict(s.split("=") for s in line.strip().split())
+
+ section["duration"] += int(values["dur"])
+ exitcode = int(values["stat"])
+ if values["exit"] == "exited" and exitcode == LtpExitCode.TCONF:
+ # Exited normally with the "invalid configuration" code
+ results[values["tag"]] = "SKIPPED"
+ elif exitcode == LtpExitCode.TPASS:
+ # Successful exit
+ results[values["tag"]] = "PASSED"
+ else:
+ # Other exit
+ results[values["tag"]] = "FAILED"
+
+ return results, section
+
+
+# ltp Compliance log parsing
+class LtpComplianceParser(object):
+ def __init__(self):
+ self.results = {}
+ self.section = {'duration': "", 'log': ""}
+
+ def parse(self, logfile):
+ test_regex = {}
+ test_regex['FAILED'] = re.compile(r"FAIL")
+
+ section_regex = {}
+ section_regex['test'] = re.compile(r"^Executing")
+
+ with open(logfile, errors='replace') as f:
+ name = logfile
+ result = "PASSED"
+ for line in f:
+ regex_result = section_regex['test'].search(line)
+ if regex_result:
+ name = line.split()[1].strip()
+
+ regex_result = test_regex['FAILED'].search(line)
+ if regex_result:
+ result = "FAILED"
+ self.results[name] = result
+
+ for test in self.results:
+ result = self.results[test]
+ print (self.results)
+ self.section['log'] = self.section['log'] + ("%s: %s\n" % (result.strip()[:-2], test.strip()))
+
+ return self.results, self.section
diff --git a/meta/lib/oeqa/utils/metadata.py b/meta/lib/oeqa/utils/metadata.py
index 65bbdc61f4..15ec190c4a 100644
--- a/meta/lib/oeqa/utils/metadata.py
+++ b/meta/lib/oeqa/utils/metadata.py
@@ -1,6 +1,6 @@
# Copyright (C) 2016 Intel Corporation
#
-# Released under the MIT license (see COPYING.MIT)
+# SPDX-License-Identifier: MIT
#
# Functions to get metadata from the testing host used
# for analytics of test results.
@@ -27,9 +27,9 @@ def metadata_from_bb():
data_dict = get_bb_vars()
# Distro information
- info_dict['distro'] = {'id': data_dict['DISTRO'],
- 'version_id': data_dict['DISTRO_VERSION'],
- 'pretty_name': '%s %s' % (data_dict['DISTRO'], data_dict['DISTRO_VERSION'])}
+ info_dict['distro'] = {'id': data_dict.get('DISTRO', 'NODISTRO'),
+ 'version_id': data_dict.get('DISTRO_VERSION', 'NO_DISTRO_VERSION'),
+ 'pretty_name': '%s %s' % (data_dict.get('DISTRO', 'NODISTRO'), data_dict.get('DISTRO_VERSION', 'NO_DISTRO_VERSION'))}
# Host distro information
os_release = get_os_release()
@@ -58,9 +58,25 @@ def metadata_from_data_store(d):
def git_rev_info(path):
"""Get git revision information as a dict"""
- from git import Repo, InvalidGitRepositoryError, NoSuchPathError
-
info = OrderedDict()
+
+ try:
+ from git import Repo, InvalidGitRepositoryError, NoSuchPathError
+ except ImportError:
+ import subprocess
+ try:
+ info['branch'] = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=path).decode('utf-8').strip()
+ except subprocess.CalledProcessError:
+ pass
+ try:
+ info['commit'] = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=path).decode('utf-8').strip()
+ except subprocess.CalledProcessError:
+ pass
+ try:
+ info['commit_count'] = int(subprocess.check_output(["git", "rev-list", "--count", "HEAD"], cwd=path).decode('utf-8').strip())
+ except subprocess.CalledProcessError:
+ pass
+ return info
try:
repo = Repo(path, search_parent_directories=True)
except (InvalidGitRepositoryError, NoSuchPathError):
diff --git a/meta/lib/oeqa/utils/network.py b/meta/lib/oeqa/utils/network.py
index 2768f6c5db..da4ffda9a9 100644
--- a/meta/lib/oeqa/utils/network.py
+++ b/meta/lib/oeqa/utils/network.py
@@ -1,7 +1,13 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+
import socket
-def get_free_port():
- s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+def get_free_port(udp = False):
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM if not udp else socket.SOCK_DGRAM)
s.bind(('', 0))
addr = s.getsockname()
s.close()
diff --git a/meta/lib/oeqa/utils/nfs.py b/meta/lib/oeqa/utils/nfs.py
new file mode 100644
index 0000000000..903469bfee
--- /dev/null
+++ b/meta/lib/oeqa/utils/nfs.py
@@ -0,0 +1,43 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+import os
+import sys
+import tempfile
+import contextlib
+import socket
+from oeqa.utils.commands import bitbake, get_bb_var, Command
+from oeqa.utils.network import get_free_port
+
+@contextlib.contextmanager
+def unfs_server(directory, logger = None, udp = True):
+ unfs_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "unfs3-native")
+ if not os.path.exists(os.path.join(unfs_sysroot, "usr", "bin", "unfsd")):
+ # build native tool
+ bitbake("unfs3-native -c addto_recipe_sysroot")
+
+ exports = None
+ cmd = None
+ try:
+ # create the exports file
+ with tempfile.NamedTemporaryFile(delete = False) as exports:
+ exports.write("{0} (rw,no_root_squash,no_all_squash,insecure)\n".format(directory).encode())
+
+ # find some ports for the server
+ nfsport, mountport = get_free_port(udp), get_free_port(udp)
+
+ nenv = dict(os.environ)
+ nenv['PATH'] = "{0}/sbin:{0}/usr/sbin:{0}/usr/bin:".format(unfs_sysroot) + nenv.get('PATH', '')
+ cmd = Command(["unfsd", "-d", "-p", "-e", exports.name, "-n", str(nfsport), "-m", str(mountport)],
+ bg = True, env = nenv, output_log = logger)
+ cmd.run()
+ yield nfsport, mountport
+ finally:
+ if cmd is not None:
+ cmd.stop()
+ if exports is not None:
+ # clean up exports file
+ os.unlink(exports.name)
+
diff --git a/meta/lib/oeqa/utils/package_manager.py b/meta/lib/oeqa/utils/package_manager.py
index 724afb2b5e..db799b64d6 100644
--- a/meta/lib/oeqa/utils/package_manager.py
+++ b/meta/lib/oeqa/utils/package_manager.py
@@ -1,3 +1,9 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+
import os
import json
import shutil
@@ -8,26 +14,31 @@ def get_package_manager(d, root_path):
"""
Returns an OE package manager that can install packages in root_path.
"""
- from oe.package_manager import RpmPM, OpkgPM, DpkgPM
+ from oe.package_manager.rpm import RpmPM
+ from oe.package_manager.ipk import OpkgPM
+ from oe.package_manager.deb import DpkgPM
pkg_class = d.getVar("IMAGE_PKGTYPE")
if pkg_class == "rpm":
pm = RpmPM(d,
root_path,
- d.getVar('TARGET_VENDOR'))
+ d.getVar('TARGET_VENDOR'),
+ filterbydependencies=False)
pm.create_configs()
elif pkg_class == "ipk":
pm = OpkgPM(d,
root_path,
d.getVar("IPKGCONF_TARGET"),
- d.getVar("ALL_MULTILIB_PACKAGE_ARCHS"))
+ d.getVar("ALL_MULTILIB_PACKAGE_ARCHS"),
+ filterbydependencies=False)
elif pkg_class == "deb":
pm = DpkgPM(d,
root_path,
d.getVar('PACKAGE_ARCHS'),
- d.getVar('DPKG_ARCH'))
+ d.getVar('DPKG_ARCH'),
+ filterbydependencies=False)
pm.write_index()
pm.update()
@@ -108,7 +119,7 @@ def extract_packages(d, needed_packages):
extract = package.get('extract', True)
if extract:
- #logger.debug(1, 'Extracting %s' % pkg)
+ #logger.debug('Extracting %s' % pkg)
dst_dir = os.path.join(extracted_path, pkg)
# Same package used for more than one test,
# don't need to extract again.
@@ -121,7 +132,7 @@ def extract_packages(d, needed_packages):
shutil.rmtree(pkg_dir)
else:
- #logger.debug(1, 'Copying %s' % pkg)
+ #logger.debug('Copying %s' % pkg)
_copy_package(d, pkg)
def _extract_in_tmpdir(d, pkg):
diff --git a/meta/lib/oeqa/utils/postactions.py b/meta/lib/oeqa/utils/postactions.py
new file mode 100644
index 0000000000..ecdddd2d40
--- /dev/null
+++ b/meta/lib/oeqa/utils/postactions.py
@@ -0,0 +1,98 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+
+# Run a set of actions after tests. The runner provides internal data
+# dictionary as well as test context to any action to run.
+
+from oeqa.utils import get_json_result_dir
+
+def create_artifacts_directory(d, tc):
+ import shutil
+
+ local_artifacts_dir = os.path.join(get_json_result_dir(d), "artifacts")
+ if os.path.isdir(local_artifacts_dir):
+ shutil.rmtree(local_artifacts_dir)
+
+ os.makedirs(local_artifacts_dir)
+
+##################################################################
+# Host/target statistics
+##################################################################
+
+def get_target_disk_usage(d, tc):
+ output_file = os.path.join(get_json_result_dir(d), "artifacts", "target_disk_usage.txt")
+ try:
+ (status, output) = tc.target.run('df -h')
+ with open(output_file, 'w') as f:
+ f.write(output)
+ f.write("\n")
+ except Exception as e:
+ bb.warn(f"Can not get target disk usage: {e}")
+
+def get_host_disk_usage(d, tc):
+ import subprocess
+
+ output_file = os.path.join(get_json_result_dir(d), "artifacts", "host_disk_usage.txt")
+ try:
+ with open(output_file, 'w') as f:
+ output = subprocess.run(['df', '-hl'], check=True, text=True, stdout=f, env={})
+ except Exception as e:
+ bb.warn(f"Can not get host disk usage: {e}")
+
+##################################################################
+# Artifacts retrieval
+##################################################################
+
+def get_artifacts_list(target, raw_list):
+ result = []
+ # Passed list may contains patterns in paths, expand them directly on target
+ for raw_path in raw_list.split():
+ cmd = f"for p in {raw_path}; do if [ -e $p ]; then echo $p; fi; done"
+ try:
+ status, output = target.run(cmd)
+ if status != 0 or not output:
+ raise Exception()
+ result += output.split()
+ except:
+ bb.note(f"No file/directory matching path {raw_path}")
+
+ return result
+
+def retrieve_test_artifacts(target, artifacts_list, target_dir):
+ local_artifacts_dir = os.path.join(target_dir, "artifacts")
+ for artifact_path in artifacts_list:
+ if not os.path.isabs(artifact_path):
+ bb.warn(f"{artifact_path} is not an absolute path")
+ continue
+ try:
+ dest_dir = os.path.join(local_artifacts_dir, os.path.dirname(artifact_path[1:]))
+ os.makedirs(dest_dir, exist_ok=True)
+ target.copyFrom(artifact_path, dest_dir)
+ except Exception as e:
+ bb.warn(f"Can not retrieve {artifact_path} from test target: {e}")
+
+def list_and_fetch_failed_tests_artifacts(d, tc):
+ artifacts_list = get_artifacts_list(tc.target, d.getVar("TESTIMAGE_FAILED_QA_ARTIFACTS"))
+ if not artifacts_list:
+ bb.warn("Could not load artifacts list, skip artifacts retrieval")
+ else:
+ retrieve_test_artifacts(tc.target, artifacts_list, get_json_result_dir(d))
+
+
+##################################################################
+# General post actions runner
+##################################################################
+
+def run_failed_tests_post_actions(d, tc):
+ post_actions=[
+ create_artifacts_directory,
+ list_and_fetch_failed_tests_artifacts,
+ get_target_disk_usage,
+ get_host_disk_usage
+ ]
+
+ for action in post_actions:
+ action(d, tc)
diff --git a/meta/lib/oeqa/utils/qemurunner.py b/meta/lib/oeqa/utils/qemurunner.py
index 0631d43218..cda43aad8c 100644
--- a/meta/lib/oeqa/utils/qemurunner.py
+++ b/meta/lib/oeqa/utils/qemurunner.py
@@ -1,6 +1,8 @@
+#
# Copyright (C) 2013 Intel Corporation
#
-# Released under the MIT license (see COPYING.MIT)
+# SPDX-License-Identifier: MIT
+#
# This module provides a class for starting qemu images using runqemu.
# It's used by testimage.bbclass.
@@ -17,8 +19,11 @@ import errno
import string
import threading
import codecs
-import logging
-from oeqa.utils.dump import HostDumper
+import tempfile
+from collections import defaultdict
+from contextlib import contextmanager
+import importlib
+import traceback
# Get Unicode non printable control chars
control_range = list(range(0,32))+list(range(127,160))
@@ -26,12 +31,23 @@ control_chars = [chr(x) for x in control_range
if chr(x) not in string.printable]
re_control_char = re.compile('[%s]' % re.escape("".join(control_chars)))
+def getOutput(o):
+ import fcntl
+ fl = fcntl.fcntl(o, fcntl.F_GETFL)
+ fcntl.fcntl(o, fcntl.F_SETFL, fl | os.O_NONBLOCK)
+ try:
+ return os.read(o.fileno(), 1000000).decode("utf-8")
+ except BlockingIOError:
+ return ""
+
class QemuRunner:
- def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds, use_kvm, logger):
+ def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, use_kvm, logger, use_slirp=False,
+ serial_ports=2, boot_patterns = defaultdict(str), use_ovmf=False, workdir=None, tmpfsdir=None):
# Popen object for runqemu
self.runqemu = None
+ self.runqemu_exited = False
# pid of the qemu process that runqemu will start
self.qemupid = None
# target ip - from the command line or runqemu output
@@ -50,19 +66,49 @@ class QemuRunner:
self.boottime = boottime
self.logged = False
self.thread = None
+ self.threadsock = None
self.use_kvm = use_kvm
+ self.use_ovmf = use_ovmf
+ self.use_slirp = use_slirp
+ self.serial_ports = serial_ports
self.msg = ''
+ self.boot_patterns = boot_patterns
+ self.tmpfsdir = tmpfsdir
- self.runqemutime = 120
- self.qemu_pidfile = 'pidfile_'+str(os.getpid())
- self.host_dumper = HostDumper(dump_host_cmds, dump_dir)
+ self.runqemutime = 300
+ if not workdir:
+ workdir = os.getcwd()
+ self.qemu_pidfile = workdir + '/pidfile_' + str(os.getpid())
+ self.monitorpipe = None
self.logger = logger
+ # Whether we're expecting an exit and should show related errors
+ self.canexit = False
+
+ # Enable testing other OS's
+ # Set commands for target communication, and default to Linux ALWAYS
+ # Other OS's or baremetal applications need to provide their
+ # own implementation passing it through QemuRunner's constructor
+ # or by passing them through TESTIMAGE_BOOT_PATTERNS[flag]
+ # provided variables, where <flag> is one of the mentioned below.
+ accepted_patterns = ['search_reached_prompt', 'send_login_user', 'search_login_succeeded', 'search_cmd_finished']
+ default_boot_patterns = defaultdict(str)
+ # Default to the usual paterns used to communicate with the target
+ default_boot_patterns['search_reached_prompt'] = ' login:'
+ default_boot_patterns['send_login_user'] = 'root\n'
+ default_boot_patterns['search_login_succeeded'] = r"root@[a-zA-Z0-9\-]+:~#"
+ default_boot_patterns['search_cmd_finished'] = r"[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#"
+
+ # Only override patterns that were set e.g. login user TESTIMAGE_BOOT_PATTERNS[send_login_user] = "webserver\n"
+ for pattern in accepted_patterns:
+ if not self.boot_patterns[pattern]:
+ self.boot_patterns[pattern] = default_boot_patterns[pattern]
def create_socket(self):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setblocking(0)
+ sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
sock.bind(("127.0.0.1",0))
sock.listen(2)
port = sock.getsockname()[1]
@@ -73,31 +119,24 @@ class QemuRunner:
sock.close()
raise
- def log(self, msg):
- if self.logfile:
- # It is needed to sanitize the data received from qemu
- # because is possible to have control characters
- msg = msg.decode("utf-8", errors='ignore')
- msg = re_control_char.sub('', msg)
- self.msg += msg
- with codecs.open(self.logfile, "a", encoding="utf-8") as f:
- f.write("%s" % msg)
-
- def getOutput(self, o):
- import fcntl
- fl = fcntl.fcntl(o, fcntl.F_GETFL)
- fcntl.fcntl(o, fcntl.F_SETFL, fl | os.O_NONBLOCK)
- return os.read(o.fileno(), 1000000).decode("utf-8")
+ def decode_qemulog(self, todecode):
+ # Sanitize the data received from qemu as it may contain control characters
+ msg = todecode.decode("utf-8", errors='backslashreplace')
+ msg = re_control_char.sub('', msg)
+ return msg
+ def log(self, msg, extension=""):
+ if self.logfile:
+ with codecs.open(self.logfile + extension, "ab") as f:
+ f.write(msg)
+ self.msg += self.decode_qemulog(msg)
def handleSIGCHLD(self, signum, frame):
if self.runqemu and self.runqemu.poll():
if self.runqemu.returncode:
- self.logger.debug('runqemu exited with code %d' % self.runqemu.returncode)
- self.logger.debug("Output from runqemu:\n%s" % self.getOutput(self.runqemu.stdout))
+ self.logger.error('runqemu exited with code %d' % self.runqemu.returncode)
+ self.logger.error('Output from runqemu:\n%s' % getOutput(self.runqemu.stdout))
self.stop()
- self._dump_host()
- raise SystemExit
def start(self, qemuparams = None, get_ip = True, extra_bootparams = None, runqemuparams='', launch_cmd=None, discard_writes=True):
env = os.environ.copy()
@@ -120,8 +159,11 @@ class QemuRunner:
else:
env["DEPLOY_DIR_IMAGE"] = self.deploy_dir_image
+ if self.tmpfsdir:
+ env["RUNQEMU_TMPFS_DIR"] = self.tmpfsdir
+
if not launch_cmd:
- launch_cmd = 'runqemu %s %s ' % ('snapshot' if discard_writes else '', runqemuparams)
+ launch_cmd = 'runqemu %s' % ('snapshot' if discard_writes else '')
if self.use_kvm:
self.logger.debug('Using kvm for runqemu')
launch_cmd += ' kvm'
@@ -129,19 +171,51 @@ class QemuRunner:
self.logger.debug('Not using kvm for runqemu')
if not self.display:
launch_cmd += ' nographic'
- launch_cmd += ' %s %s' % (self.machine, self.rootfs)
+ if self.use_slirp:
+ launch_cmd += ' slirp'
+ if self.use_ovmf:
+ launch_cmd += ' ovmf'
+ launch_cmd += ' %s %s' % (runqemuparams, self.machine)
+ if self.rootfs.endswith('.vmdk'):
+ self.logger.debug('Bypassing VMDK rootfs for runqemu')
+ else:
+ launch_cmd += ' %s' % (self.rootfs)
return self.launch(launch_cmd, qemuparams=qemuparams, get_ip=get_ip, extra_bootparams=extra_bootparams, env=env)
def launch(self, launch_cmd, get_ip = True, qemuparams = None, extra_bootparams = None, env = None):
+ # use logfile to determine the recipe-sysroot-native path and
+ # then add in the site-packages path components and add that
+ # to the python sys.path so the qmp module can be found.
+ python_path = os.path.dirname(os.path.dirname(self.logfile))
+ python_path += "/recipe-sysroot-native/usr/lib/qemu-python"
+ sys.path.append(python_path)
+ importlib.invalidate_caches()
try:
- threadsock, threadport = self.create_socket()
+ qmp = importlib.import_module("qmp")
+ except Exception as e:
+ self.logger.error("qemurunner: qmp module missing, please ensure it's installed in %s (%s)" % (python_path, str(e)))
+ return False
+ # Path relative to tmpdir used as cwd for qemu below to avoid unix socket path length issues
+ qmp_file = "." + next(tempfile._get_candidate_names())
+ qmp_param = ' -S -qmp unix:./%s,server,wait' % (qmp_file)
+ qmp_port = self.tmpdir + "/" + qmp_file
+ # Create a second socket connection for debugging use,
+ # note this will NOT cause qemu to block waiting for the connection
+ qmp_file2 = "." + next(tempfile._get_candidate_names())
+ qmp_param += ' -qmp unix:./%s,server,nowait' % (qmp_file2)
+ qmp_port2 = self.tmpdir + "/" + qmp_file2
+ self.logger.info("QMP Available for connection at %s" % (qmp_port2))
+
+ try:
+ if self.serial_ports >= 2:
+ self.threadsock, threadport = self.create_socket()
self.server_socket, self.serverport = self.create_socket()
except socket.error as msg:
self.logger.error("Failed to create listening socket: %s" % msg[1])
return False
- bootparams = 'console=tty1 console=ttyS0,115200n8 printk.time=1'
+ bootparams = ' printk.time=1'
if extra_bootparams:
bootparams = bootparams + ' ' + extra_bootparams
@@ -149,23 +223,28 @@ class QemuRunner:
# and analyze descendents in order to determine it.
if os.path.exists(self.qemu_pidfile):
os.remove(self.qemu_pidfile)
- self.qemuparams = 'bootparams="{0}" qemuparams="-serial tcp:127.0.0.1:{1} -pidfile {2}"'.format(bootparams, threadport, self.qemu_pidfile)
+ self.qemuparams = 'bootparams="{0}" qemuparams="-pidfile {1} {2}"'.format(bootparams, self.qemu_pidfile, qmp_param)
+
if qemuparams:
self.qemuparams = self.qemuparams[:-1] + " " + qemuparams + " " + '\"'
- launch_cmd += ' tcpserial=%s %s' % (self.serverport, self.qemuparams)
+ if self.serial_ports >= 2:
+ launch_cmd += ' tcpserial=%s:%s %s' % (threadport, self.serverport, self.qemuparams)
+ else:
+ launch_cmd += ' tcpserial=%s %s' % (self.serverport, self.qemuparams)
self.origchldhandler = signal.getsignal(signal.SIGCHLD)
signal.signal(signal.SIGCHLD, self.handleSIGCHLD)
- self.logger.debug('launchcmd=%s'%(launch_cmd))
+ self.logger.debug('launchcmd=%s' % (launch_cmd))
# FIXME: We pass in stdin=subprocess.PIPE here to work around stty
# blocking at the end of the runqemu script when using this within
# oe-selftest (this makes stty error out immediately). There ought
# to be a proper fix but this will suffice for now.
- self.runqemu = subprocess.Popen(launch_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, preexec_fn=os.setpgrp, env=env)
+ self.runqemu = subprocess.Popen(launch_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, preexec_fn=os.setpgrp, env=env, cwd=self.tmpdir)
output = self.runqemu.stdout
+ launch_time = time.time()
#
# We need the preexec_fn above so that all runqemu processes can easily be killed
@@ -191,57 +270,152 @@ class QemuRunner:
r = os.fdopen(r)
x = r.read()
os.killpg(os.getpgid(self.runqemu.pid), signal.SIGTERM)
- sys.exit(0)
+ os._exit(0)
self.logger.debug("runqemu started, pid is %s" % self.runqemu.pid)
- self.logger.debug("waiting at most %s seconds for qemu pid" % self.runqemutime)
+ self.logger.debug("waiting at most %d seconds for qemu pid (%s)" %
+ (self.runqemutime, time.strftime("%D %H:%M:%S")))
endtime = time.time() + self.runqemutime
while not self.is_alive() and time.time() < endtime:
if self.runqemu.poll():
+ if self.runqemu_exited:
+ self.logger.warning("runqemu during is_alive() test")
+ return False
if self.runqemu.returncode:
# No point waiting any longer
- self.logger.debug('runqemu exited with code %d' % self.runqemu.returncode)
- self._dump_host()
+ self.logger.warning('runqemu exited with code %d' % self.runqemu.returncode)
+ self.logger.warning("Output from runqemu:\n%s" % getOutput(output))
self.stop()
- self.logger.debug("Output from runqemu:\n%s" % self.getOutput(output))
return False
time.sleep(0.5)
+ if self.runqemu_exited:
+ self.logger.warning("runqemu after timeout")
+
+ if self.runqemu.returncode:
+ self.logger.warning('runqemu exited with code %d' % self.runqemu.returncode)
+
if not self.is_alive():
- self.logger.error("Qemu pid didn't appear in %s seconds" % self.runqemutime)
+ self.logger.error("Qemu pid didn't appear in %d seconds (%s)" %
+ (self.runqemutime, time.strftime("%D %H:%M:%S")))
+
+ qemu_pid = None
+ if os.path.isfile(self.qemu_pidfile):
+ with open(self.qemu_pidfile, 'r') as f:
+ qemu_pid = f.read().strip()
+
+ self.logger.error("Status information, poll status: %s, pidfile exists: %s, pidfile contents %s, proc pid exists %s"
+ % (self.runqemu.poll(), os.path.isfile(self.qemu_pidfile), str(qemu_pid), os.path.exists("/proc/" + str(qemu_pid))))
+
# Dump all processes to help us to figure out what is going on...
- ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command '], stdout=subprocess.PIPE).communicate()[0]
+ ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,pri,ni,command '], stdout=subprocess.PIPE).communicate()[0]
processes = ps.decode("utf-8")
self.logger.debug("Running processes:\n%s" % processes)
- self._dump_host()
+ op = getOutput(output)
self.stop()
- op = self.getOutput(output)
if op:
self.logger.error("Output from runqemu:\n%s" % op)
else:
self.logger.error("No output from runqemu.\n")
return False
+ # Create the client socket for the QEMU Monitor Control Socket
+ # This will allow us to read status from Qemu if the the process
+ # is still alive
+ self.logger.debug("QMP Initializing to %s" % (qmp_port))
+ # chdir dance for path length issues with unix sockets
+ origpath = os.getcwd()
+ try:
+ os.chdir(os.path.dirname(qmp_port))
+ try:
+ from qmp.legacy import QEMUMonitorProtocol
+ self.qmp = QEMUMonitorProtocol(os.path.basename(qmp_port))
+ except OSError as msg:
+ self.logger.warning("Failed to initialize qemu monitor socket: %s File: %s" % (msg, msg.filename))
+ return False
+
+ self.logger.debug("QMP Connecting to %s" % (qmp_port))
+ if not os.path.exists(qmp_port) and self.is_alive():
+ self.logger.debug("QMP Port does not exist waiting for it to be created")
+ endtime = time.time() + self.runqemutime
+ while not os.path.exists(qmp_port) and self.is_alive() and time.time() < endtime:
+ self.logger.info("QMP port does not exist yet!")
+ time.sleep(0.5)
+ if not os.path.exists(qmp_port) and self.is_alive():
+ self.logger.warning("QMP Port still does not exist but QEMU is alive")
+ return False
+
+ try:
+ # set timeout value for all QMP calls
+ self.qmp.settimeout(self.runqemutime)
+ self.qmp.connect()
+ connect_time = time.time()
+ self.logger.info("QMP connected to QEMU at %s and took %.2f seconds" %
+ (time.strftime("%D %H:%M:%S"),
+ time.time() - launch_time))
+ except OSError as msg:
+ self.logger.warning("Failed to connect qemu monitor socket: %s File: %s" % (msg, msg.filename))
+ return False
+ except qmp.legacy.QMPError as msg:
+ self.logger.warning("Failed to communicate with qemu monitor: %s" % (msg))
+ return False
+ finally:
+ os.chdir(origpath)
+
+ # We worry that mmap'd libraries may cause page faults which hang the qemu VM for periods
+ # causing failures. Before we "start" qemu, read through it's mapped files to try and
+ # ensure we don't hit page faults later
+ mapdir = "/proc/" + str(self.qemupid) + "/map_files/"
+ try:
+ for f in os.listdir(mapdir):
+ try:
+ linktarget = os.readlink(os.path.join(mapdir, f))
+ if not linktarget.startswith("/") or linktarget.startswith("/dev") or "deleted" in linktarget:
+ continue
+ with open(linktarget, "rb") as readf:
+ data = True
+ while data:
+ data = readf.read(4096)
+ except FileNotFoundError:
+ continue
+ # Centos7 doesn't allow us to read /map_files/
+ except PermissionError:
+ pass
+
+ # Release the qemu process to continue running
+ self.run_monitor('cont')
+ self.logger.info("QMP released QEMU at %s and took %.2f seconds from connect" %
+ (time.strftime("%D %H:%M:%S"),
+ time.time() - connect_time))
+
# We are alive: qemu is running
- out = self.getOutput(output)
+ out = getOutput(output)
netconf = False # network configuration is not required by default
- self.logger.debug("qemu started in %s seconds - qemu procces pid is %s" % (time.time() - (endtime - self.runqemutime), self.qemupid))
+ self.logger.debug("qemu started in %.2f seconds - qemu procces pid is %s (%s)" %
+ (time.time() - (endtime - self.runqemutime),
+ self.qemupid, time.strftime("%D %H:%M:%S")))
+ cmdline = ''
if get_ip:
- cmdline = ''
with open('/proc/%s/cmdline' % self.qemupid) as p:
cmdline = p.read()
# It is needed to sanitize the data received
# because is possible to have control characters
cmdline = re_control_char.sub(' ', cmdline)
try:
- ips = re.findall("((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1])
- self.ip = ips[0]
- self.server_ip = ips[1]
+ if self.use_slirp:
+ tcp_ports = cmdline.split("hostfwd=tcp:")[1]
+ ip, tcp_ports = tcp_ports.split(":")[:2]
+ host_port = tcp_ports[:tcp_ports.find('-')]
+ self.ip = "%s:%s" % (ip, host_port)
+ else:
+ ips = re.findall(r"((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1])
+ self.ip = ips[0]
+ self.server_ip = ips[1]
self.logger.debug("qemu cmdline used:\n{}".format(cmdline))
except (IndexError, ValueError):
# Try to get network configuration from runqemu output
- match = re.match('.*Network configuration: ([0-9.]+)::([0-9.]+):([0-9.]+)$.*',
- out, re.MULTILINE|re.DOTALL)
+ match = re.match(r'.*Network configuration: (?:ip=)*([0-9.]+)::([0-9.]+):([0-9.]+).*',
+ out, re.MULTILINE | re.DOTALL)
if match:
self.ip, self.server_ip, self.netmask = match.groups()
# network configuration is required as we couldn't get it
@@ -252,26 +426,28 @@ class QemuRunner:
self.logger.error("Couldn't get ip from qemu command line and runqemu output! "
"Here is the qemu command line used:\n%s\n"
"and output from runqemu:\n%s" % (cmdline, out))
- self._dump_host()
self.stop()
return False
self.logger.debug("Target IP: %s" % self.ip)
self.logger.debug("Server IP: %s" % self.server_ip)
- self.thread = LoggingThread(self.log, threadsock, self.logger)
+ self.thread = LoggingThread(self.log, self.threadsock, self.logger, self.runqemu.stdout)
self.thread.start()
- if not self.thread.connection_established.wait(self.boottime):
- self.logger.error("Didn't receive a console connection from qemu. "
- "Here is the qemu command line used:\n%s\nand "
- "output from runqemu:\n%s" % (cmdline, out))
- self.stop_thread()
- return False
+
+ if self.serial_ports >= 2:
+ if not self.thread.connection_established.wait(self.boottime):
+ self.logger.error("Didn't receive a console connection from qemu. "
+ "Here is the qemu command line used:\n%s\nand "
+ "output from runqemu:\n%s" % (cmdline, out))
+ self.stop_thread()
+ return False
self.logger.debug("Output from runqemu:\n%s", out)
- self.logger.debug("Waiting at most %d seconds for login banner" % self.boottime)
+ self.logger.debug("Waiting at most %d seconds for login banner (%s)" %
+ (self.boottime, time.strftime("%D %H:%M:%S")))
endtime = time.time() + self.boottime
- socklist = [self.server_socket]
+ filelist = [self.server_socket]
reachedlogin = False
stopread = False
qemusock = None
@@ -279,69 +455,103 @@ class QemuRunner:
data = b''
while time.time() < endtime and not stopread:
try:
- sread, swrite, serror = select.select(socklist, [], [], 5)
+ sread, swrite, serror = select.select(filelist, [], [], 5)
except InterruptedError:
continue
- for sock in sread:
- if sock is self.server_socket:
+ for file in sread:
+ if file is self.server_socket:
qemusock, addr = self.server_socket.accept()
- qemusock.setblocking(0)
- socklist.append(qemusock)
- socklist.remove(self.server_socket)
+ qemusock.setblocking(False)
+ filelist.append(qemusock)
+ filelist.remove(self.server_socket)
self.logger.debug("Connection from %s:%s" % addr)
else:
- data = data + sock.recv(1024)
+ # try to avoid reading only a single character at a time
+ time.sleep(0.1)
+ if hasattr(file, 'read'):
+ read = file.read(1024)
+ elif hasattr(file, 'recv'):
+ read = file.recv(1024)
+ else:
+ self.logger.error('Invalid file type: %s\n%s' % (file))
+ read = b''
+
+ self.logger.debug2('Partial boot log:\n%s' % (read.decode('utf-8', errors='backslashreplace')))
+ data = data + read
if data:
bootlog += data
+ self.log(data, extension = ".2")
data = b''
- if b' login:' in bootlog:
+
+ if bytes(self.boot_patterns['search_reached_prompt'], 'utf-8') in bootlog:
+ self.server_socket.close()
self.server_socket = qemusock
stopread = True
reachedlogin = True
- self.logger.debug("Reached login banner")
+ self.logger.debug("Reached login banner in %.2f seconds (%s)" %
+ (time.time() - (endtime - self.boottime),
+ time.strftime("%D %H:%M:%S")))
else:
- socklist.remove(sock)
- sock.close()
+ # no need to check if reachedlogin unless we support multiple connections
+ self.logger.debug("QEMU socket disconnected before login banner reached. (%s)" %
+ time.strftime("%D %H:%M:%S"))
+ filelist.remove(file)
+ file.close()
stopread = True
-
if not reachedlogin:
- self.logger.debug("Target didn't reached login boot in %d seconds" % self.boottime)
+ if time.time() >= endtime:
+ self.logger.warning("Target didn't reach login banner in %d seconds (%s)" %
+ (self.boottime, time.strftime("%D %H:%M:%S")))
tail = lambda l: "\n".join(l.splitlines()[-25:])
- # in case bootlog is empty, use tail qemu log store at self.msg
- lines = tail(bootlog if bootlog else self.msg)
- self.logger.debug("Last 25 lines of text:\n%s" % lines)
- self.logger.debug("Check full boot log: %s" % self.logfile)
- self._dump_host()
+ bootlog = self.decode_qemulog(bootlog)
+ self.logger.warning("Last 25 lines of login console (%d):\n%s" % (len(bootlog), tail(bootlog)))
+ self.logger.warning("Last 25 lines of all logging (%d):\n%s" % (len(self.msg), tail(self.msg)))
+ self.logger.warning("Check full boot log: %s" % self.logfile)
self.stop()
+ data = True
+ while data:
+ try:
+ time.sleep(1)
+ data = qemusock.recv(1024)
+ self.log(data, extension = ".2")
+ self.logger.warning('Extra log data read: %s\n' % (data.decode('utf-8', errors='backslashreplace')))
+ except Exception as e:
+ self.logger.warning('Extra log data exception %s' % repr(e))
+ data = None
+ self.thread.serial_lock.release()
return False
+ with self.thread.serial_lock:
+ self.thread.set_serialsock(self.server_socket)
+
# If we are not able to login the tests can continue
try:
- (status, output) = self.run_serial("root\n", raw=True)
- if re.search("root@[a-zA-Z0-9\-]+:~#", output):
+ (status, output) = self.run_serial(self.boot_patterns['send_login_user'], raw=True, timeout=120)
+ if re.search(self.boot_patterns['search_login_succeeded'], output):
self.logged = True
- self.logger.debug("Logged as root in serial console")
+ self.logger.debug("Logged in as %s in serial console" % self.boot_patterns['send_login_user'].replace("\n", ""))
if netconf:
# configure guest networking
cmd = "ifconfig eth0 %s netmask %s up\n" % (self.ip, self.netmask)
output = self.run_serial(cmd, raw=True)[1]
- if re.search("root@[a-zA-Z0-9\-]+:~#", output):
+ if re.search(r"root@[a-zA-Z0-9\-]+:~#", output):
self.logger.debug("configured ip address %s", self.ip)
else:
self.logger.debug("Couldn't configure guest networking")
else:
- self.logger.debug("Couldn't login into serial console"
- " as root using blank password")
+ self.logger.warning("Couldn't login into serial console"
+ " as %s using blank password" % self.boot_patterns['send_login_user'].replace("\n", ""))
+ self.logger.warning("The output:\n%s" % output)
except:
- self.logger.debug("Serial console failed while trying to login")
+ self.logger.warning("Serial console failed while trying to login")
return True
def stop(self):
- self.stop_thread()
- self.stop_qemu_system()
if hasattr(self, "origchldhandler"):
signal.signal(signal.SIGCHLD, self.origchldhandler)
+ self.stop_thread()
+ self.stop_qemu_system()
if self.runqemu:
if hasattr(self, "monitorpid"):
os.kill(self.monitorpid, signal.SIGKILL)
@@ -351,20 +561,40 @@ class QemuRunner:
except OSError as e:
if e.errno != errno.ESRCH:
raise
- endtime = time.time() + self.runqemutime
- while self.runqemu.poll() is None and time.time() < endtime:
- time.sleep(1)
- if self.runqemu.poll() is None:
+ try:
+ outs, errs = self.runqemu.communicate(timeout=self.runqemutime)
+ if outs:
+ self.logger.info("Output from runqemu:\n%s", outs.decode("utf-8"))
+ if errs:
+ self.logger.info("Stderr from runqemu:\n%s", errs.decode("utf-8"))
+ except subprocess.TimeoutExpired:
self.logger.debug("Sending SIGKILL to runqemu")
os.killpg(os.getpgid(self.runqemu.pid), signal.SIGKILL)
- self.runqemu = None
+ if not self.runqemu.stdout.closed:
+ self.logger.info("Output from runqemu:\n%s" % getOutput(self.runqemu.stdout))
+ self.runqemu.stdin.close()
+ self.runqemu.stdout.close()
+ self.runqemu_exited = True
+
+ if hasattr(self, 'qmp') and self.qmp:
+ self.qmp.close()
+ self.qmp = None
if hasattr(self, 'server_socket') and self.server_socket:
self.server_socket.close()
self.server_socket = None
+ if hasattr(self, 'threadsock') and self.threadsock:
+ self.threadsock.close()
+ self.threadsock = None
self.qemupid = None
self.ip = None
if os.path.exists(self.qemu_pidfile):
- os.remove(self.qemu_pidfile)
+ try:
+ os.remove(self.qemu_pidfile)
+ except FileNotFoundError as e:
+ # We raced, ignore
+ pass
+ if self.monitorpipe:
+ self.monitorpipe.close()
def stop_qemu_system(self):
if self.qemupid:
@@ -372,15 +602,20 @@ class QemuRunner:
# qemu-system behaves well and a SIGTERM is enough
os.kill(self.qemupid, signal.SIGTERM)
except ProcessLookupError as e:
- self.logger.warn('qemu-system ended unexpectedly')
+ self.logger.warning('qemu-system ended unexpectedly')
def stop_thread(self):
if self.thread and self.thread.is_alive():
self.thread.stop()
self.thread.join()
+ def allowexit(self):
+ self.canexit = True
+ if self.thread:
+ self.thread.allowexit()
+
def restart(self, qemuparams = None):
- self.logger.debug("Restarting qemu process")
+ self.logger.warning("Restarting qemu process")
if self.runqemu.poll() is None:
self.stop()
if self.start(qemuparams):
@@ -388,46 +623,72 @@ class QemuRunner:
return False
def is_alive(self):
- if not self.runqemu:
+ if not self.runqemu or self.runqemu.poll() is not None or self.runqemu_exited:
return False
if os.path.isfile(self.qemu_pidfile):
- f = open(self.qemu_pidfile, 'r')
- qemu_pid = f.read()
- f.close()
- qemupid = int(qemu_pid)
- if os.path.exists("/proc/" + str(qemupid)):
- self.qemupid = qemupid
- return True
+ # when handling pidfile, qemu creates the file, stat it, lock it and then write to it
+ # so it's possible that the file has been created but the content is empty
+ pidfile_timeout = time.time() + 3
+ while time.time() < pidfile_timeout:
+ try:
+ with open(self.qemu_pidfile, 'r') as f:
+ qemu_pid = f.read().strip()
+ except FileNotFoundError:
+ # Can be used to detect shutdown so the pid file can disappear
+ return False
+ # file created but not yet written contents
+ if not qemu_pid:
+ time.sleep(0.5)
+ continue
+ else:
+ if os.path.exists("/proc/" + qemu_pid):
+ self.qemupid = int(qemu_pid)
+ return True
return False
- def run_serial(self, command, raw=False, timeout=5):
+ def run_monitor(self, command, args=None, timeout=60):
+ if hasattr(self, 'qmp') and self.qmp:
+ self.qmp.settimeout(timeout)
+ if args is not None:
+ return self.qmp.cmd_raw(command, args)
+ else:
+ return self.qmp.cmd_raw(command)
+
+ def run_serial(self, command, raw=False, timeout=60):
+ # Returns (status, output) where status is 1 on success and 0 on error
+
# We assume target system have echo to get command status
if not raw:
command = "%s; echo $?\n" % command
data = ''
status = 0
- self.server_socket.sendall(command.encode('utf-8'))
- start = time.time()
- end = start + timeout
- while True:
- now = time.time()
- if now >= end:
- data += "<<< run_serial(): command timed out after %d seconds without output >>>\r\n\r\n" % timeout
- break
- try:
- sread, _, _ = select.select([self.server_socket],[],[], end - now)
- except InterruptedError:
- continue
- if sread:
- answer = self.server_socket.recv(1024)
- if answer:
- data += answer.decode('utf-8')
- # Search the prompt to stop
- if re.search("[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#", data):
- break
- else:
- raise Exception("No data on serial console socket")
+ with self.thread.serial_lock:
+ self.server_socket.sendall(command.encode('utf-8'))
+ start = time.time()
+ end = start + timeout
+ while True:
+ now = time.time()
+ if now >= end:
+ data += "<<< run_serial(): command timed out after %d seconds without output >>>\r\n\r\n" % timeout
+ break
+ try:
+ sread, _, _ = select.select([self.server_socket],[],[], end - now)
+ except InterruptedError:
+ continue
+ if sread:
+ # try to avoid reading single character at a time
+ time.sleep(0.1)
+ answer = self.server_socket.recv(1024)
+ if answer:
+ data += answer.decode('utf-8')
+ # Search the prompt to stop
+ if re.search(self.boot_patterns['search_cmd_finished'], data):
+ break
+ else:
+ if self.canexit:
+ return (1, "")
+ raise Exception("No data on serial console socket, connection closed?")
if data:
if raw:
@@ -446,34 +707,46 @@ class QemuRunner:
status = 1
return (status, str(data))
-
- def _dump_host(self):
- self.host_dumper.create_dir("qemu")
- self.logger.warn("Qemu ended unexpectedly, dump data from host"
- " is in %s" % self.host_dumper.dump_dir)
- self.host_dumper.dump_host()
+@contextmanager
+def nonblocking_lock(lock):
+ locked = lock.acquire(False)
+ try:
+ yield locked
+ finally:
+ if locked:
+ lock.release()
# This class is for reading data from a socket and passing it to logfunc
# to be processed. It's completely event driven and has a straightforward
# event loop. The mechanism for stopping the thread is a simple pipe which
# will wake up the poll and allow for tearing everything down.
class LoggingThread(threading.Thread):
- def __init__(self, logfunc, sock, logger):
+ def __init__(self, logfunc, sock, logger, qemuoutput):
self.connection_established = threading.Event()
+ self.serial_lock = threading.Lock()
+
self.serversock = sock
+ self.serialsock = None
+ self.qemuoutput = qemuoutput
self.logfunc = logfunc
self.logger = logger
self.readsock = None
self.running = False
+ self.canexit = False
self.errorevents = select.POLLERR | select.POLLHUP | select.POLLNVAL
self.readevents = select.POLLIN | select.POLLPRI
threading.Thread.__init__(self, target=self.threadtarget)
+ def set_serialsock(self, serialsock):
+ self.serialsock = serialsock
+
def threadtarget(self):
try:
self.eventloop()
+ except Exception as e:
+ self.logger.warning("Exception %s in logging thread" % traceback.format_exception(e))
finally:
self.teardown()
@@ -489,7 +762,8 @@ class LoggingThread(threading.Thread):
def teardown(self):
self.logger.debug("Tearing down logging thread")
- self.close_socket(self.serversock)
+ if self.serversock:
+ self.close_socket(self.serversock)
if self.readsock is not None:
self.close_socket(self.readsock)
@@ -498,30 +772,37 @@ class LoggingThread(threading.Thread):
self.close_ignore_error(self.writepipe)
self.running = False
+ def allowexit(self):
+ self.canexit = True
+
def eventloop(self):
poll = select.poll()
event_read_mask = self.errorevents | self.readevents
- poll.register(self.serversock.fileno())
+ if self.serversock:
+ poll.register(self.serversock.fileno())
+ serial_registered = False
+ poll.register(self.qemuoutput.fileno())
poll.register(self.readpipe, event_read_mask)
breakout = False
self.running = True
self.logger.debug("Starting thread event loop")
while not breakout:
- events = poll.poll()
- for event in events:
+ events = poll.poll(2)
+ for fd, event in events:
+
# An error occurred, bail out
- if event[1] & self.errorevents:
- raise Exception(self.stringify_event(event[1]))
+ if event & self.errorevents:
+ raise Exception(self.stringify_event(event))
# Event to stop the thread
- if self.readpipe == event[0]:
+ if self.readpipe == fd:
self.logger.debug("Stop event received")
breakout = True
break
# A connection request was received
- elif self.serversock.fileno() == event[0]:
+ elif self.serversock and self.serversock.fileno() == fd:
self.logger.debug("Connection request received")
self.readsock, _ = self.serversock.accept()
self.readsock.setblocking(0)
@@ -532,18 +813,38 @@ class LoggingThread(threading.Thread):
self.connection_established.set()
# Actual data to be logged
- elif self.readsock.fileno() == event[0]:
- data = self.recv(1024)
+ elif self.readsock and self.readsock.fileno() == fd:
+ data = self.recv(1024, self.readsock)
self.logfunc(data)
+ elif self.qemuoutput.fileno() == fd:
+ data = self.qemuoutput.read()
+ self.logger.debug("Data received on qemu stdout %s" % data)
+ self.logfunc(data, ".stdout")
+ elif self.serialsock and self.serialsock.fileno() == fd:
+ if self.serial_lock.acquire(blocking=False):
+ data = self.recv(1024, self.serialsock)
+ self.logger.debug("Data received serial thread %s" % data.decode('utf-8', 'replace'))
+ self.logfunc(data, ".2")
+ self.serial_lock.release()
+ else:
+ serial_registered = False
+ poll.unregister(self.serialsock.fileno())
+
+ if not serial_registered and self.serialsock:
+ with nonblocking_lock(self.serial_lock) as l:
+ if l:
+ serial_registered = True
+ poll.register(self.serialsock.fileno(), event_read_mask)
+
# Since the socket is non-blocking make sure to honor EAGAIN
# and EWOULDBLOCK.
- def recv(self, count):
+ def recv(self, count, sock):
try:
- data = self.readsock.recv(count)
+ data = sock.recv(count)
except socket.error as e:
if e.errno == errno.EAGAIN or e.errno == errno.EWOULDBLOCK:
- return ''
+ return b''
else:
raise
@@ -554,7 +855,9 @@ class LoggingThread(threading.Thread):
# happened. But for this code it counts as an
# error since the connection shouldn't go away
# until qemu exits.
- raise Exception("Console connection closed unexpectedly")
+ if not self.canexit:
+ raise Exception("Console connection closed unexpectedly")
+ return b''
return data
@@ -566,6 +869,9 @@ class LoggingThread(threading.Thread):
val = 'POLLHUP'
elif select.POLLNVAL == event:
val = 'POLLNVAL'
+ else:
+ val = "0x%x" % (event)
+
return val
def close_socket(self, sock):
diff --git a/meta/lib/oeqa/utils/qemutinyrunner.py b/meta/lib/oeqa/utils/qemutinyrunner.py
index 63b5d1648b..20009401ca 100644
--- a/meta/lib/oeqa/utils/qemutinyrunner.py
+++ b/meta/lib/oeqa/utils/qemutinyrunner.py
@@ -1,6 +1,8 @@
+#
# Copyright (C) 2015 Intel Corporation
#
-# Released under the MIT license (see COPYING.MIT)
+# SPDX-License-Identifier: MIT
+#
# This module provides a class for starting qemu images of poky tiny.
# It's used by testimage.bbclass.
@@ -17,7 +19,7 @@ from .qemurunner import QemuRunner
class QemuTinyRunner(QemuRunner):
- def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, kernel, boottime, logger):
+ def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, kernel, boottime, logger, tmpfsdir=None):
# Popen object for runqemu
self.runqemu = None
@@ -35,6 +37,7 @@ class QemuTinyRunner(QemuRunner):
self.deploy_dir_image = deploy_dir_image
self.logfile = logfile
self.boottime = boottime
+ self.tmpfsdir = tmpfsdir
self.runqemutime = 60
self.socketfile = "console.sock"
@@ -81,6 +84,9 @@ class QemuTinyRunner(QemuRunner):
return False
else:
os.environ["DEPLOY_DIR_IMAGE"] = self.deploy_dir_image
+ if self.tmpfsdir:
+ env["RUNQEMU_TMPFS_DIR"] = self.tmpfsdir
+
# Set this flag so that Qemu doesn't do any grabs as SDL grabs interact
# badly with screensavers.
@@ -108,7 +114,7 @@ class QemuTinyRunner(QemuRunner):
return self.is_alive()
- def run_serial(self, command, timeout=5):
+ def run_serial(self, command, timeout=60):
self.server_socket.sendall(command+'\n')
data = ''
status = 0
@@ -136,7 +142,7 @@ class QemuTinyRunner(QemuRunner):
#
# Walk the process tree from the process specified looking for a qemu-system. Return its [pid'cmd]
#
- ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command'], stdout=subprocess.PIPE).communicate()[0]
+ ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,pri,ni,command'], stdout=subprocess.PIPE).communicate()[0]
processes = ps.decode("utf-8").split('\n')
nfields = len(processes[0].split()) - 1
pids = {}
diff --git a/meta/lib/oeqa/utils/sshcontrol.py b/meta/lib/oeqa/utils/sshcontrol.py
index d292893c08..36c2ecb3db 100644
--- a/meta/lib/oeqa/utils/sshcontrol.py
+++ b/meta/lib/oeqa/utils/sshcontrol.py
@@ -1,7 +1,8 @@
-# -*- coding: utf-8 -*-
+#
# Copyright (C) 2013 Intel Corporation
#
-# Released under the MIT license (see COPYING.MIT)
+# SPDX-License-Identifier: MIT
+#
# Provides a class for setting up ssh connections,
# running commands and copying files to/from a target.
@@ -22,7 +23,7 @@ class SSHProcess(object):
"stdin": None,
"shell": False,
"bufsize": -1,
- "preexec_fn": os.setsid,
+ "start_new_session": True,
}
self.options = dict(self.defaultopts)
self.options.update(options)
diff --git a/meta/lib/oeqa/utils/subprocesstweak.py b/meta/lib/oeqa/utils/subprocesstweak.py
index 1f7d11b55c..3e43ed547b 100644
--- a/meta/lib/oeqa/utils/subprocesstweak.py
+++ b/meta/lib/oeqa/utils/subprocesstweak.py
@@ -1,3 +1,8 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
import subprocess
class OETestCalledProcessError(subprocess.CalledProcessError):
diff --git a/meta/lib/oeqa/utils/targetbuild.py b/meta/lib/oeqa/utils/targetbuild.py
index 1202d579fb..09738add1d 100644
--- a/meta/lib/oeqa/utils/targetbuild.py
+++ b/meta/lib/oeqa/utils/targetbuild.py
@@ -1,6 +1,8 @@
+#
# Copyright (C) 2013 Intel Corporation
#
-# Released under the MIT license (see COPYING.MIT)
+# SPDX-License-Identifier: MIT
+#
# Provides a class for automating build tests for projects
@@ -17,11 +19,13 @@ class BuildProject(metaclass=ABCMeta):
self.d = d
self.uri = uri
self.archive = os.path.basename(uri)
+ self.tempdirobj = None
if not tmpdir:
tmpdir = self.d.getVar('WORKDIR')
if not tmpdir:
- tmpdir = tempfile.mkdtemp(prefix='buildproject')
- self.localarchive = os.path.join(tmpdir,self.archive)
+ self.tempdirobj = tempfile.TemporaryDirectory(prefix='buildproject-')
+ tmpdir = self.tempdirobj.name
+ self.localarchive = os.path.join(tmpdir, self.archive)
if foldername:
self.fname = foldername
else:
@@ -68,9 +72,10 @@ class BuildProject(metaclass=ABCMeta):
return self._run('cd %s; make install %s' % (self.targetdir, install_args))
def clean(self):
+ if self.tempdirobj:
+ self.tempdirobj.cleanup()
self._run('rm -rf %s' % self.targetdir)
subprocess.check_call('rm -f %s' % self.localarchive, shell=True)
- pass
class TargetBuildProject(BuildProject):
diff --git a/meta/lib/oeqa/utils/testexport.py b/meta/lib/oeqa/utils/testexport.py
index be2a2110fc..e89d130a9c 100644
--- a/meta/lib/oeqa/utils/testexport.py
+++ b/meta/lib/oeqa/utils/testexport.py
@@ -1,6 +1,8 @@
+#
# Copyright (C) 2015 Intel Corporation
#
-# Released under the MIT license (see COPYING.MIT)
+# SPDX-License-Identifier: MIT
+#
# Provides functions to help with exporting binaries obtained from built targets
generated by cgit 1.2.3-korg (git 2.39.0) at 2024-04-27 11:07:00 +0000