summaryrefslogtreecommitdiffstats
path: root/meta/lib/oeqa/utils
diff options
context:
space:
mode:
Diffstat (limited to 'meta/lib/oeqa/utils')
-rw-r--r--meta/lib/oeqa/utils/__init__.py3
-rw-r--r--meta/lib/oeqa/utils/buildproject.py4
-rw-r--r--meta/lib/oeqa/utils/commands.py45
-rw-r--r--meta/lib/oeqa/utils/decorators.py4
-rw-r--r--meta/lib/oeqa/utils/dump.py11
-rw-r--r--meta/lib/oeqa/utils/ftools.py4
-rw-r--r--meta/lib/oeqa/utils/git.py2
-rw-r--r--meta/lib/oeqa/utils/gitarchive.py237
-rw-r--r--meta/lib/oeqa/utils/httpserver.py8
-rw-r--r--meta/lib/oeqa/utils/logparser.py260
-rw-r--r--meta/lib/oeqa/utils/metadata.py7
-rw-r--r--meta/lib/oeqa/utils/network.py8
-rw-r--r--meta/lib/oeqa/utils/nfs.py39
-rw-r--r--meta/lib/oeqa/utils/package_manager.py8
-rw-r--r--meta/lib/oeqa/utils/qemurunner.py155
-rw-r--r--meta/lib/oeqa/utils/qemutinyrunner.py6
-rw-r--r--meta/lib/oeqa/utils/sshcontrol.py7
-rw-r--r--meta/lib/oeqa/utils/subprocesstweak.py3
-rw-r--r--meta/lib/oeqa/utils/targetbuild.py4
-rw-r--r--meta/lib/oeqa/utils/testexport.py4
20 files changed, 628 insertions, 191 deletions
diff --git a/meta/lib/oeqa/utils/__init__.py b/meta/lib/oeqa/utils/__init__.py
index d38a323013..70fbe7b552 100644
--- a/meta/lib/oeqa/utils/__init__.py
+++ b/meta/lib/oeqa/utils/__init__.py
@@ -1,3 +1,6 @@
+#
+# SPDX-License-Identifier: MIT
+#
# Enable other layers to have modules in the same named directory
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
diff --git a/meta/lib/oeqa/utils/buildproject.py b/meta/lib/oeqa/utils/buildproject.py
index 01a803ab61..e6d80cc8dc 100644
--- a/meta/lib/oeqa/utils/buildproject.py
+++ b/meta/lib/oeqa/utils/buildproject.py
@@ -1,6 +1,8 @@
+#
# Copyright (C) 2013-2016 Intel Corporation
#
-# Released under the MIT license (see COPYING.MIT)
+# SPDX-License-Identifier: MIT
+#
# Provides a class for automating build tests for projects
diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py
index 2e6a2289cd..a71c16ab14 100644
--- a/meta/lib/oeqa/utils/commands.py
+++ b/meta/lib/oeqa/utils/commands.py
@@ -1,6 +1,8 @@
+#
# Copyright (c) 2013-2014 Intel Corporation
#
-# Released under the MIT license (see COPYING.MIT)
+# SPDX-License-Identifier: MIT
+#
# DESCRIPTION
# This module is mainly used by scripts/oe-selftest and modules under meta/oeqa/selftest
@@ -93,7 +95,9 @@ class Command(object):
# reason, the main process will still exit, which will then
# kill the write thread.
if self.data:
- threading.Thread(target=writeThread, daemon=True).start()
+ thread = threading.Thread(target=writeThread, daemon=True)
+ thread.start()
+ self.threads.append(thread)
if self.process.stderr:
thread = threading.Thread(target=readStderrThread)
thread.start()
@@ -121,11 +125,11 @@ class Command(object):
def stop(self):
for thread in self.threads:
- if thread.isAlive():
+ if thread.is_alive():
self.process.terminate()
# let's give it more time to terminate gracefully before killing it
thread.join(5)
- if thread.isAlive():
+ if thread.is_alive():
self.process.kill()
thread.join()
@@ -163,20 +167,32 @@ class Result(object):
pass
-def runCmd(command, ignore_status=False, timeout=None, assert_error=True,
+def runCmd(command, ignore_status=False, timeout=None, assert_error=True, sync=True,
native_sysroot=None, limit_exc_output=0, output_log=None, **options):
result = Result()
if native_sysroot:
extra_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin" % \
(native_sysroot, native_sysroot, native_sysroot)
+ extra_libpaths = "%s/lib:%s/usr/lib" % \
+ (native_sysroot, native_sysroot)
nenv = dict(options.get('env', os.environ))
nenv['PATH'] = extra_paths + ':' + nenv.get('PATH', '')
+ nenv['LD_LIBRARY_PATH'] = extra_libpaths + ':' + nenv.get('LD_LIBRARY_PATH', '')
options['env'] = nenv
cmd = Command(command, timeout=timeout, output_log=output_log, **options)
cmd.run()
+ # tests can be heavy on IO and if bitbake can't write out its caches, we see timeouts.
+ # call sync around the tests to ensure the IO queue doesn't get too large, taking any IO
+ # hit here rather than in bitbake shutdown.
+ if sync:
+ p = os.environ['PATH']
+ os.environ['PATH'] = "/usr/bin:/bin:/usr/sbin:/sbin:" + p
+ os.system("sync")
+ os.environ['PATH'] = p
+
result.command = command
result.status = cmd.status
result.output = cmd.output
@@ -310,15 +326,15 @@ def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None,
try:
tinfoil.logger.setLevel(logging.WARNING)
import oeqa.targetcontrol
- tinfoil.config_data.setVar("TEST_LOG_DIR", "${WORKDIR}/testimage")
- tinfoil.config_data.setVar("TEST_QEMUBOOT_TIMEOUT", "1000")
+ recipedata = tinfoil.parse_recipe(pn)
+ recipedata.setVar("TEST_LOG_DIR", "${WORKDIR}/testimage")
+ recipedata.setVar("TEST_QEMUBOOT_TIMEOUT", "1000")
# Tell QemuTarget() whether need find rootfs/kernel or not
if launch_cmd:
- tinfoil.config_data.setVar("FIND_ROOTFS", '0')
+ recipedata.setVar("FIND_ROOTFS", '0')
else:
- tinfoil.config_data.setVar("FIND_ROOTFS", '1')
+ recipedata.setVar("FIND_ROOTFS", '1')
- recipedata = tinfoil.parse_recipe(pn)
for key, value in overrides.items():
recipedata.setVar(key, value)
@@ -335,8 +351,8 @@ def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None,
qemu.deploy()
try:
qemu.start(params=qemuparams, ssh=ssh, runqemuparams=runqemuparams, launch_cmd=launch_cmd, discard_writes=discard_writes)
- except bb.build.FuncFailed:
- msg = 'Failed to start QEMU - see the logs in %s' % logdir
+ except Exception as e:
+ msg = str(e) + '\nFailed to start QEMU - see the logs in %s' % logdir
if os.path.exists(qemu.qemurunnerlog):
with open(qemu.qemurunnerlog, 'r') as f:
msg = msg + "Qemurunner log output from %s:\n%s" % (qemu.qemurunnerlog, f.read())
@@ -346,10 +362,7 @@ def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None,
finally:
targetlogger.removeHandler(handler)
- try:
- qemu.stop()
- except:
- pass
+ qemu.stop()
def updateEnv(env_file):
"""
diff --git a/meta/lib/oeqa/utils/decorators.py b/meta/lib/oeqa/utils/decorators.py
index d876896921..aabf4110cb 100644
--- a/meta/lib/oeqa/utils/decorators.py
+++ b/meta/lib/oeqa/utils/decorators.py
@@ -1,6 +1,8 @@
+#
# Copyright (C) 2013 Intel Corporation
#
-# Released under the MIT license (see COPYING.MIT)
+# SPDX-License-Identifier: MIT
+#
# Some custom decorators that can be used by unittests
# Most useful is skipUnlessPassed which can be used for
diff --git a/meta/lib/oeqa/utils/dump.py b/meta/lib/oeqa/utils/dump.py
index 5a7edc1a86..09a44329e0 100644
--- a/meta/lib/oeqa/utils/dump.py
+++ b/meta/lib/oeqa/utils/dump.py
@@ -1,3 +1,7 @@
+#
+# SPDX-License-Identifier: MIT
+#
+
import os
import sys
import errno
@@ -12,7 +16,7 @@ class BaseDumper(object):
self.cmds = []
# Some testing doesn't inherit testimage, so it is needed
# to set some defaults.
- self.parent_dir = parent_dir or "/tmp/oe-saved-tests"
+ self.parent_dir = parent_dir
dft_cmds = """ top -bn1
iostat -x -z -N -d -p ALL 20 2
ps -ef
@@ -67,8 +71,11 @@ class HostDumper(BaseDumper):
def dump_host(self, dump_dir=""):
if dump_dir:
self.dump_dir = dump_dir
+ env = os.environ.copy()
+ env['PATH'] = '/usr/sbin:/sbin:/usr/bin:/bin'
+ env['COLUMNS'] = '9999'
for cmd in self.cmds:
- result = runCmd(cmd, ignore_status=True)
+ result = runCmd(cmd, ignore_status=True, env=env)
self._write_dump(cmd.split()[0], result.output)
class TargetDumper(BaseDumper):
diff --git a/meta/lib/oeqa/utils/ftools.py b/meta/lib/oeqa/utils/ftools.py
index a7233d4ca6..3093419cc7 100644
--- a/meta/lib/oeqa/utils/ftools.py
+++ b/meta/lib/oeqa/utils/ftools.py
@@ -1,3 +1,7 @@
+#
+# SPDX-License-Identifier: MIT
+#
+
import os
import re
import errno
diff --git a/meta/lib/oeqa/utils/git.py b/meta/lib/oeqa/utils/git.py
index 757e3f0cbf..ea35a766eb 100644
--- a/meta/lib/oeqa/utils/git.py
+++ b/meta/lib/oeqa/utils/git.py
@@ -1,7 +1,7 @@
#
# Copyright (C) 2016 Intel Corporation
#
-# Released under the MIT license (see COPYING.MIT)
+# SPDX-License-Identifier: MIT
#
"""Git repository interactions"""
import os
diff --git a/meta/lib/oeqa/utils/gitarchive.py b/meta/lib/oeqa/utils/gitarchive.py
new file mode 100644
index 0000000000..6e8040eb5c
--- /dev/null
+++ b/meta/lib/oeqa/utils/gitarchive.py
@@ -0,0 +1,237 @@
+#
+# Helper functions for committing data to git and pushing upstream
+#
+# Copyright (c) 2017, Intel Corporation.
+# Copyright (c) 2019, Linux Foundation
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import os
+import re
+import sys
+from operator import attrgetter
+from collections import namedtuple
+from oeqa.utils.git import GitRepo, GitError
+
+class ArchiveError(Exception):
+ """Internal error handling of this script"""
+
+def format_str(string, fields):
+ """Format string using the given fields (dict)"""
+ try:
+ return string.format(**fields)
+ except KeyError as err:
+ raise ArchiveError("Unable to expand string '{}': unknown field {} "
+ "(valid fields are: {})".format(
+ string, err, ', '.join(sorted(fields.keys()))))
+
+
+def init_git_repo(path, no_create, bare, log):
+ """Initialize local Git repository"""
+ path = os.path.abspath(path)
+ if os.path.isfile(path):
+ raise ArchiveError("Invalid Git repo at {}: path exists but is not a "
+ "directory".format(path))
+ if not os.path.isdir(path) or not os.listdir(path):
+ if no_create:
+ raise ArchiveError("No git repo at {}, refusing to create "
+ "one".format(path))
+ if not os.path.isdir(path):
+ try:
+ os.mkdir(path)
+ except (FileNotFoundError, PermissionError) as err:
+ raise ArchiveError("Failed to mkdir {}: {}".format(path, err))
+ if not os.listdir(path):
+ log.info("Initializing a new Git repo at %s", path)
+ repo = GitRepo.init(path, bare)
+ try:
+ repo = GitRepo(path, is_topdir=True)
+ except GitError:
+ raise ArchiveError("Non-empty directory that is not a Git repository "
+ "at {}\nPlease specify an existing Git repository, "
+ "an empty directory or a non-existing directory "
+ "path.".format(path))
+ return repo
+
+
+def git_commit_data(repo, data_dir, branch, message, exclude, notes, log):
+ """Commit data into a Git repository"""
+ log.info("Committing data into to branch %s", branch)
+ tmp_index = os.path.join(repo.git_dir, 'index.oe-git-archive')
+ try:
+ # Create new tree object from the data
+ env_update = {'GIT_INDEX_FILE': tmp_index,
+ 'GIT_WORK_TREE': os.path.abspath(data_dir)}
+ repo.run_cmd('add .', env_update)
+
+ # Remove files that are excluded
+ if exclude:
+ repo.run_cmd(['rm', '--cached'] + [f for f in exclude], env_update)
+
+ tree = repo.run_cmd('write-tree', env_update)
+
+ # Create new commit object from the tree
+ parent = repo.rev_parse(branch)
+ if not parent:
+ parent = repo.rev_parse("origin/" + branch)
+ git_cmd = ['commit-tree', tree, '-m', message]
+ if parent:
+ git_cmd += ['-p', parent]
+ commit = repo.run_cmd(git_cmd, env_update)
+
+ # Create git notes
+ for ref, filename in notes:
+ ref = ref.format(branch_name=branch)
+ repo.run_cmd(['notes', '--ref', ref, 'add',
+ '-F', os.path.abspath(filename), commit])
+
+ # Update branch head
+ git_cmd = ['update-ref', 'refs/heads/' + branch, commit]
+ repo.run_cmd(git_cmd)
+
+ # Update current HEAD, if we're on branch 'branch'
+ if not repo.bare and repo.get_current_branch() == branch:
+ log.info("Updating %s HEAD to latest commit", repo.top_dir)
+ repo.run_cmd('reset --hard')
+
+ return commit
+ finally:
+ if os.path.exists(tmp_index):
+ os.unlink(tmp_index)
+
+
+def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern,
+ keywords):
+ """Generate tag name and message, with support for running id number"""
+ keyws = keywords.copy()
+ # Tag number is handled specially: if not defined, we autoincrement it
+ if 'tag_number' not in keyws:
+ # Fill in all other fields than 'tag_number'
+ keyws['tag_number'] = '{tag_number}'
+ tag_re = format_str(name_pattern, keyws)
+ # Replace parentheses for proper regex matching
+ tag_re = tag_re.replace('(', '\(').replace(')', '\)') + '$'
+ # Inject regex group pattern for 'tag_number'
+ tag_re = tag_re.format(tag_number='(?P<tag_number>[0-9]{1,5})')
+
+ keyws['tag_number'] = 0
+ for existing_tag in repo.run_cmd('tag').splitlines():
+ match = re.match(tag_re, existing_tag)
+
+ if match and int(match.group('tag_number')) >= keyws['tag_number']:
+ keyws['tag_number'] = int(match.group('tag_number')) + 1
+
+ tag_name = format_str(name_pattern, keyws)
+ msg_subj= format_str(msg_subj_pattern.strip(), keyws)
+ msg_body = format_str(msg_body_pattern, keyws)
+ return tag_name, msg_subj + '\n\n' + msg_body
+
+def gitarchive(data_dir, git_dir, no_create, bare, commit_msg_subject, commit_msg_body, branch_name, no_tag, tagname, tag_msg_subject, tag_msg_body, exclude, notes, push, keywords, log):
+
+ if not os.path.isdir(data_dir):
+ raise ArchiveError("Not a directory: {}".format(data_dir))
+
+ data_repo = init_git_repo(git_dir, no_create, bare, log)
+
+ # Expand strings early in order to avoid getting into inconsistent
+ # state (e.g. no tag even if data was committed)
+ commit_msg = format_str(commit_msg_subject.strip(), keywords)
+ commit_msg += '\n\n' + format_str(commit_msg_body, keywords)
+ branch_name = format_str(branch_name, keywords)
+ tag_name = None
+ if not no_tag and tagname:
+ tag_name, tag_msg = expand_tag_strings(data_repo, tagname,
+ tag_msg_subject,
+ tag_msg_body, keywords)
+
+ # Commit data
+ commit = git_commit_data(data_repo, data_dir, branch_name,
+ commit_msg, exclude, notes, log)
+
+ # Create tag
+ if tag_name:
+ log.info("Creating tag %s", tag_name)
+ data_repo.run_cmd(['tag', '-a', '-m', tag_msg, tag_name, commit])
+
+ # Push data to remote
+ if push:
+ cmd = ['push', '--tags']
+ # If no remote is given we push with the default settings from
+ # gitconfig
+ if push is not True:
+ notes_refs = ['refs/notes/' + ref.format(branch_name=branch_name)
+ for ref, _ in notes]
+ cmd.extend([push, branch_name] + notes_refs)
+ log.info("Pushing data to remote")
+ data_repo.run_cmd(cmd)
+
+# Container class for tester revisions
+TestedRev = namedtuple('TestedRev', 'commit commit_number tags')
+
+def get_test_runs(log, repo, tag_name, **kwargs):
+ """Get a sorted list of test runs, matching given pattern"""
+ # First, get field names from the tag name pattern
+ field_names = [m.group(1) for m in re.finditer(r'{(\w+)}', tag_name)]
+ undef_fields = [f for f in field_names if f not in kwargs.keys()]
+
+ # Fields for formatting tag name pattern
+ str_fields = dict([(f, '*') for f in field_names])
+ str_fields.update(kwargs)
+
+ # Get a list of all matching tags
+ tag_pattern = tag_name.format(**str_fields)
+ tags = repo.run_cmd(['tag', '-l', tag_pattern]).splitlines()
+ log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern)
+
+ # Parse undefined fields from tag names
+ str_fields = dict([(f, r'(?P<{}>[\w\-.()]+)'.format(f)) for f in field_names])
+ str_fields['branch'] = r'(?P<branch>[\w\-.()/]+)'
+ str_fields['commit'] = '(?P<commit>[0-9a-f]{7,40})'
+ str_fields['commit_number'] = '(?P<commit_number>[0-9]{1,7})'
+ str_fields['tag_number'] = '(?P<tag_number>[0-9]{1,5})'
+ # escape parenthesis in fields in order to not messa up the regexp
+ fixed_fields = dict([(k, v.replace('(', r'\(').replace(')', r'\)')) for k, v in kwargs.items()])
+ str_fields.update(fixed_fields)
+ tag_re = re.compile(tag_name.format(**str_fields))
+
+ # Parse fields from tags
+ revs = []
+ for tag in tags:
+ m = tag_re.match(tag)
+ groups = m.groupdict()
+ revs.append([groups[f] for f in undef_fields] + [tag])
+
+ # Return field names and a sorted list of revs
+ return undef_fields, sorted(revs)
+
+def get_test_revs(log, repo, tag_name, **kwargs):
+ """Get list of all tested revisions"""
+ fields, runs = get_test_runs(log, repo, tag_name, **kwargs)
+
+ revs = {}
+ commit_i = fields.index('commit')
+ commit_num_i = fields.index('commit_number')
+ for run in runs:
+ commit = run[commit_i]
+ commit_num = run[commit_num_i]
+ tag = run[-1]
+ if not commit in revs:
+ revs[commit] = TestedRev(commit, commit_num, [tag])
+ else:
+ assert commit_num == revs[commit].commit_number, "Commit numbers do not match"
+ revs[commit].tags.append(tag)
+
+ # Return in sorted table
+ revs = sorted(revs.values(), key=attrgetter('commit_number'))
+ log.debug("Found %d tested revisions:\n %s", len(revs),
+ "\n ".join(['{} ({})'.format(rev.commit_number, rev.commit) for rev in revs]))
+ return revs
+
+def rev_find(revs, attr, val):
+ """Search from a list of TestedRev"""
+ for i, rev in enumerate(revs):
+ if getattr(rev, attr) == val:
+ return i
+ raise ValueError("Unable to find '{}' value '{}'".format(attr, val))
+
diff --git a/meta/lib/oeqa/utils/httpserver.py b/meta/lib/oeqa/utils/httpserver.py
index a48d4994fd..58d3c3b3f8 100644
--- a/meta/lib/oeqa/utils/httpserver.py
+++ b/meta/lib/oeqa/utils/httpserver.py
@@ -1,3 +1,7 @@
+#
+# SPDX-License-Identifier: MIT
+#
+
import http.server
import multiprocessing
import os
@@ -18,10 +22,10 @@ class HTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
class HTTPService(object):
- def __init__(self, root_dir, host='', logger=None):
+ def __init__(self, root_dir, host='', port=0, logger=None):
self.root_dir = root_dir
self.host = host
- self.port = 0
+ self.port = port
self.logger = logger
def start(self):
diff --git a/meta/lib/oeqa/utils/logparser.py b/meta/lib/oeqa/utils/logparser.py
index 0670627c3c..60e16d500e 100644
--- a/meta/lib/oeqa/utils/logparser.py
+++ b/meta/lib/oeqa/utils/logparser.py
@@ -1,126 +1,164 @@
-#!/usr/bin/env python
+#
+# SPDX-License-Identifier: MIT
+#
import sys
import os
import re
-from . import ftools
-
# A parser that can be used to identify weather a line is a test result or a section statement.
-class Lparser(object):
-
- def __init__(self, test_0_pass_regex, test_0_fail_regex, test_0_skip_regex, section_0_begin_regex=None, section_0_end_regex=None, **kwargs):
- # Initialize the arguments dictionary
- if kwargs:
- self.args = kwargs
- else:
- self.args = {}
-
- # Add the default args to the dictionary
- self.args['test_0_pass_regex'] = test_0_pass_regex
- self.args['test_0_fail_regex'] = test_0_fail_regex
- self.args['test_0_skip_regex'] = test_0_skip_regex
- if section_0_begin_regex:
- self.args['section_0_begin_regex'] = section_0_begin_regex
- if section_0_end_regex:
- self.args['section_0_end_regex'] = section_0_end_regex
-
- self.test_possible_status = ['pass', 'fail', 'error', 'skip']
- self.section_possible_status = ['begin', 'end']
-
- self.initialized = False
-
-
- # Initialize the parser with the current configuration
- def init(self):
-
- # extra arguments can be added by the user to define new test and section categories. They must follow a pre-defined pattern: <type>_<category_name>_<status>_regex
- self.test_argument_pattern = "^test_(.+?)_(%s)_regex" % '|'.join(map(str, self.test_possible_status))
- self.section_argument_pattern = "^section_(.+?)_(%s)_regex" % '|'.join(map(str, self.section_possible_status))
-
- # Initialize the test and section regex dictionaries
- self.test_regex = {}
- self.section_regex ={}
-
- for arg, value in self.args.items():
- if not value:
- raise Exception('The value of provided argument %s is %s. Should have a valid value.' % (key, value))
- is_test = re.search(self.test_argument_pattern, arg)
- is_section = re.search(self.section_argument_pattern, arg)
- if is_test:
- if not is_test.group(1) in self.test_regex:
- self.test_regex[is_test.group(1)] = {}
- self.test_regex[is_test.group(1)][is_test.group(2)] = re.compile(value)
- elif is_section:
- if not is_section.group(1) in self.section_regex:
- self.section_regex[is_section.group(1)] = {}
- self.section_regex[is_section.group(1)][is_section.group(2)] = re.compile(value)
- else:
- # TODO: Make these call a traceback instead of a simple exception..
- raise Exception("The provided argument name does not correspond to any valid type. Please give one of the following types:\nfor tests: %s\nfor sections: %s" % (self.test_argument_pattern, self.section_argument_pattern))
-
- self.initialized = True
-
- # Parse a line and return a tuple containing the type of result (test/section) and its category, status and name
- def parse_line(self, line):
- if not self.initialized:
- raise Exception("The parser is not initialized..")
-
- for test_category, test_status_list in self.test_regex.items():
- for test_status, status_regex in test_status_list.items():
- test_name = status_regex.search(line)
- if test_name:
- return ['test', test_category, test_status, test_name.group(1)]
-
- for section_category, section_status_list in self.section_regex.items():
- for section_status, status_regex in section_status_list.items():
- section_name = status_regex.search(line)
- if section_name:
- return ['section', section_category, section_status, section_name.group(1)]
- return None
-
-
-class Result(object):
-
+class PtestParser(object):
def __init__(self):
- self.result_dict = {}
-
- def store(self, section, test, status):
- if not section in self.result_dict:
- self.result_dict[section] = []
-
- self.result_dict[section].append((test, status))
-
- # sort tests by the test name(the first element of the tuple), for each section. This can be helpful when using git to diff for changes by making sure they are always in the same order.
- def sort_tests(self):
- for package in self.result_dict:
- sorted_results = sorted(self.result_dict[package], key=lambda tup: tup[0])
- self.result_dict[package] = sorted_results
+ self.results = {}
+ self.sections = {}
+
+ def parse(self, logfile):
+ test_regex = {}
+ test_regex['PASSED'] = re.compile(r"^PASS:(.+)")
+ test_regex['FAILED'] = re.compile(r"^FAIL:([^(]+)")
+ test_regex['SKIPPED'] = re.compile(r"^SKIP:(.+)")
+
+ section_regex = {}
+ section_regex['begin'] = re.compile(r"^BEGIN: .*/(.+)/ptest")
+ section_regex['end'] = re.compile(r"^END: .*/(.+)/ptest")
+ section_regex['duration'] = re.compile(r"^DURATION: (.+)")
+ section_regex['exitcode'] = re.compile(r"^ERROR: Exit status is (.+)")
+ section_regex['timeout'] = re.compile(r"^TIMEOUT: .*/(.+)/ptest")
+
+ # Cache markers so we don't take the re.search() hit all the time.
+ markers = ("PASS:", "FAIL:", "SKIP:", "BEGIN:", "END:", "DURATION:", "ERROR: Exit", "TIMEOUT:")
+
+ def newsection():
+ return { 'name': "No-section", 'log': [] }
+
+ current_section = newsection()
+
+ with open(logfile, errors='replace') as f:
+ for line in f:
+ if not line.startswith(markers):
+ current_section['log'].append(line)
+ continue
+
+ result = section_regex['begin'].search(line)
+ if result:
+ current_section['name'] = result.group(1)
+ continue
+
+ result = section_regex['end'].search(line)
+ if result:
+ if current_section['name'] != result.group(1):
+ bb.warn("Ptest END log section mismatch %s vs. %s" % (current_section['name'], result.group(1)))
+ if current_section['name'] in self.sections:
+ bb.warn("Ptest duplicate section for %s" % (current_section['name']))
+ self.sections[current_section['name']] = current_section
+ del self.sections[current_section['name']]['name']
+ current_section = newsection()
+ continue
+
+ result = section_regex['timeout'].search(line)
+ if result:
+ if current_section['name'] != result.group(1):
+ bb.warn("Ptest TIMEOUT log section mismatch %s vs. %s" % (current_section['name'], result.group(1)))
+ current_section['timeout'] = True
+ continue
+
+ for t in ['duration', 'exitcode']:
+ result = section_regex[t].search(line)
+ if result:
+ current_section[t] = result.group(1)
+ continue
+
+ current_section['log'].append(line)
+
+ for t in test_regex:
+ result = test_regex[t].search(line)
+ if result:
+ if current_section['name'] not in self.results:
+ self.results[current_section['name']] = {}
+ self.results[current_section['name']][result.group(1).strip()] = t
+
+ # Python performance for repeatedly joining long strings is poor, do it all at once at the end.
+ # For 2.1 million lines in a log this reduces 18 hours to 12s.
+ for section in self.sections:
+ self.sections[section]['log'] = "".join(self.sections[section]['log'])
+
+ return self.results, self.sections
# Log the results as files. The file name is the section name and the contents are the tests in that section.
- def log_as_files(self, target_dir, test_status):
- status_regex = re.compile('|'.join(map(str, test_status)))
- if not type(test_status) == type([]):
- raise Exception("test_status should be a list. Got " + str(test_status) + " instead.")
+ def results_as_files(self, target_dir):
if not os.path.exists(target_dir):
raise Exception("Target directory does not exist: %s" % target_dir)
- for section, test_results in self.result_dict.items():
- prefix = ''
- for x in test_status:
- prefix +=x+'.'
+ for section in self.results:
+ prefix = 'No-section'
if section:
- prefix += section
+ prefix = section
section_file = os.path.join(target_dir, prefix)
# purge the file contents if it exists
- open(section_file, 'w').close()
- for test_result in test_results:
- (test_name, status) = test_result
- # we log only the tests with status in the test_status list
- match_status = status_regex.search(status)
- if match_status:
- ftools.append_file(section_file, status + ": " + test_name)
-
- # Not yet implemented!
- def log_to_lava(self):
- pass
+ with open(section_file, 'w') as f:
+ for test_name in sorted(self.results[section]):
+ status = self.results[section][test_name]
+ f.write(status + ": " + test_name + "\n")
+
+
+# ltp log parsing
+class LtpParser(object):
+ def __init__(self):
+ self.results = {}
+ self.section = {'duration': "", 'log': ""}
+
+ def parse(self, logfile):
+ test_regex = {}
+ test_regex['PASSED'] = re.compile(r"PASS")
+ test_regex['FAILED'] = re.compile(r"FAIL")
+ test_regex['SKIPPED'] = re.compile(r"SKIP")
+
+ with open(logfile, errors='replace') as f:
+ for line in f:
+ for t in test_regex:
+ result = test_regex[t].search(line)
+ if result:
+ self.results[line.split()[0].strip()] = t
+
+ for test in self.results:
+ result = self.results[test]
+ self.section['log'] = self.section['log'] + ("%s: %s\n" % (result.strip()[:-2], test.strip()))
+
+ return self.results, self.section
+
+
+# ltp Compliance log parsing
+class LtpComplianceParser(object):
+ def __init__(self):
+ self.results = {}
+ self.section = {'duration': "", 'log': ""}
+
+ def parse(self, logfile):
+ test_regex = {}
+ test_regex['PASSED'] = re.compile(r"^PASS")
+ test_regex['FAILED'] = re.compile(r"^FAIL")
+ test_regex['SKIPPED'] = re.compile(r"(?:UNTESTED)|(?:UNSUPPORTED)")
+
+ section_regex = {}
+ section_regex['test'] = re.compile(r"^Testing")
+
+ with open(logfile, errors='replace') as f:
+ for line in f:
+ result = section_regex['test'].search(line)
+ if result:
+ self.name = ""
+ self.name = line.split()[1].strip()
+ self.results[self.name] = "PASSED"
+ failed = 0
+
+ failed_result = test_regex['FAILED'].search(line)
+ if failed_result:
+ failed = line.split()[1].strip()
+ if int(failed) > 0:
+ self.results[self.name] = "FAILED"
+
+ for test in self.results:
+ result = self.results[test]
+ self.section['log'] = self.section['log'] + ("%s: %s\n" % (result.strip()[:-2], test.strip()))
+
+ return self.results, self.section
diff --git a/meta/lib/oeqa/utils/metadata.py b/meta/lib/oeqa/utils/metadata.py
index b7def77224..8013aa684d 100644
--- a/meta/lib/oeqa/utils/metadata.py
+++ b/meta/lib/oeqa/utils/metadata.py
@@ -1,6 +1,6 @@
# Copyright (C) 2016 Intel Corporation
#
-# Released under the MIT license (see COPYING.MIT)
+# SPDX-License-Identifier: MIT
#
# Functions to get metadata from the testing host used
# for analytics of test results.
@@ -72,8 +72,11 @@ def git_rev_info(path):
info['commit'] = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=path).decode('utf-8').strip()
except subprocess.CalledProcessError:
pass
+ try:
+ info['commit_count'] = int(subprocess.check_output(["git", "rev-list", "--count", "HEAD"], cwd=path).decode('utf-8').strip())
+ except subprocess.CalledProcessError:
+ pass
return info
-
try:
repo = Repo(path, search_parent_directories=True)
except (InvalidGitRepositoryError, NoSuchPathError):
diff --git a/meta/lib/oeqa/utils/network.py b/meta/lib/oeqa/utils/network.py
index 2768f6c5db..59d01723a1 100644
--- a/meta/lib/oeqa/utils/network.py
+++ b/meta/lib/oeqa/utils/network.py
@@ -1,7 +1,11 @@
+#
+# SPDX-License-Identifier: MIT
+#
+
import socket
-def get_free_port():
- s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+def get_free_port(udp = False):
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM if not udp else socket.SOCK_DGRAM)
s.bind(('', 0))
addr = s.getsockname()
s.close()
diff --git a/meta/lib/oeqa/utils/nfs.py b/meta/lib/oeqa/utils/nfs.py
new file mode 100644
index 0000000000..a37686c914
--- /dev/null
+++ b/meta/lib/oeqa/utils/nfs.py
@@ -0,0 +1,39 @@
+# SPDX-License-Identifier: MIT
+import os
+import sys
+import tempfile
+import contextlib
+import socket
+from oeqa.utils.commands import bitbake, get_bb_var, Command
+from oeqa.utils.network import get_free_port
+
+@contextlib.contextmanager
+def unfs_server(directory, logger = None):
+ unfs_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "unfs3-native")
+ if not os.path.exists(os.path.join(unfs_sysroot, "usr", "bin", "unfsd")):
+ # build native tool
+ bitbake("unfs3-native -c addto_recipe_sysroot")
+
+ exports = None
+ cmd = None
+ try:
+ # create the exports file
+ with tempfile.NamedTemporaryFile(delete = False) as exports:
+ exports.write("{0} (rw,no_root_squash,no_all_squash,insecure)\n".format(directory).encode())
+
+ # find some ports for the server
+ nfsport, mountport = get_free_port(udp = True), get_free_port(udp = True)
+
+ nenv = dict(os.environ)
+ nenv['PATH'] = "{0}/sbin:{0}/usr/sbin:{0}/usr/bin:".format(unfs_sysroot) + nenv.get('PATH', '')
+ cmd = Command(["unfsd", "-d", "-p", "-N", "-e", exports.name, "-n", str(nfsport), "-m", str(mountport)],
+ bg = True, env = nenv, output_log = logger)
+ cmd.run()
+ yield nfsport, mountport
+ finally:
+ if cmd is not None:
+ cmd.stop()
+ if exports is not None:
+ # clean up exports file
+ os.unlink(exports.name)
+
diff --git a/meta/lib/oeqa/utils/package_manager.py b/meta/lib/oeqa/utils/package_manager.py
index 1495f87321..3623299295 100644
--- a/meta/lib/oeqa/utils/package_manager.py
+++ b/meta/lib/oeqa/utils/package_manager.py
@@ -1,3 +1,7 @@
+#
+# SPDX-License-Identifier: MIT
+#
+
import os
import json
import shutil
@@ -8,7 +12,9 @@ def get_package_manager(d, root_path):
"""
Returns an OE package manager that can install packages in root_path.
"""
- from oe.package_manager import RpmPM, OpkgPM, DpkgPM
+ from oe.package_manager.rpm import RpmPM
+ from oe.package_manager.ipk import OpkgPM
+ from oe.package_manager.deb import DpkgPM
pkg_class = d.getVar("IMAGE_PKGTYPE")
if pkg_class == "rpm":
diff --git a/meta/lib/oeqa/utils/qemurunner.py b/meta/lib/oeqa/utils/qemurunner.py
index 7ef506b60a..77ec939ad7 100644
--- a/meta/lib/oeqa/utils/qemurunner.py
+++ b/meta/lib/oeqa/utils/qemurunner.py
@@ -1,6 +1,8 @@
+#
# Copyright (C) 2013 Intel Corporation
#
-# Released under the MIT license (see COPYING.MIT)
+# SPDX-License-Identifier: MIT
+#
# This module provides a class for starting qemu images using runqemu.
# It's used by testimage.bbclass.
@@ -19,6 +21,7 @@ import threading
import codecs
import logging
from oeqa.utils.dump import HostDumper
+from collections import defaultdict
# Get Unicode non printable control chars
control_range = list(range(0,32))+list(range(127,160))
@@ -29,10 +32,11 @@ re_control_char = re.compile('[%s]' % re.escape("".join(control_chars)))
class QemuRunner:
def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds,
- use_kvm, logger, use_slirp=False):
+ use_kvm, logger, use_slirp=False, serial_ports=2, boot_patterns = defaultdict(str), use_ovmf=False, workdir=None):
# Popen object for runqemu
self.runqemu = None
+ self.runqemu_exited = False
# pid of the qemu process that runqemu will start
self.qemupid = None
# target ip - from the command line or runqemu output
@@ -52,15 +56,40 @@ class QemuRunner:
self.logged = False
self.thread = None
self.use_kvm = use_kvm
+ self.use_ovmf = use_ovmf
self.use_slirp = use_slirp
+ self.serial_ports = serial_ports
self.msg = ''
+ self.boot_patterns = boot_patterns
self.runqemutime = 120
- self.qemu_pidfile = 'pidfile_'+str(os.getpid())
+ if not workdir:
+ workdir = os.getcwd()
+ self.qemu_pidfile = workdir + '/pidfile_' + str(os.getpid())
self.host_dumper = HostDumper(dump_host_cmds, dump_dir)
+ self.monitorpipe = None
self.logger = logger
+ # Enable testing other OS's
+ # Set commands for target communication, and default to Linux ALWAYS
+ # Other OS's or baremetal applications need to provide their
+ # own implementation passing it through QemuRunner's constructor
+ # or by passing them through TESTIMAGE_BOOT_PATTERNS[flag]
+ # provided variables, where <flag> is one of the mentioned below.
+ accepted_patterns = ['search_reached_prompt', 'send_login_user', 'search_login_succeeded', 'search_cmd_finished']
+ default_boot_patterns = defaultdict(str)
+ # Default to the usual paterns used to communicate with the target
+ default_boot_patterns['search_reached_prompt'] = b' login:'
+ default_boot_patterns['send_login_user'] = 'root\n'
+ default_boot_patterns['search_login_succeeded'] = r"root@[a-zA-Z0-9\-]+:~#"
+ default_boot_patterns['search_cmd_finished'] = r"[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#"
+
+ # Only override patterns that were set e.g. login user TESTIMAGE_BOOT_PATTERNS[send_login_user] = "webserver\n"
+ for pattern in accepted_patterns:
+ if not self.boot_patterns[pattern]:
+ self.boot_patterns[pattern] = default_boot_patterns[pattern]
+
def create_socket(self):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
@@ -95,11 +124,10 @@ class QemuRunner:
def handleSIGCHLD(self, signum, frame):
if self.runqemu and self.runqemu.poll():
if self.runqemu.returncode:
- self.logger.debug('runqemu exited with code %d' % self.runqemu.returncode)
- self.logger.debug("Output from runqemu:\n%s" % self.getOutput(self.runqemu.stdout))
+ self.logger.error('runqemu exited with code %d' % self.runqemu.returncode)
+ self.logger.error('Output from runqemu:\n%s' % self.getOutput(self.runqemu.stdout))
self.stop()
self._dump_host()
- raise SystemExit
def start(self, qemuparams = None, get_ip = True, extra_bootparams = None, runqemuparams='', launch_cmd=None, discard_writes=True):
env = os.environ.copy()
@@ -123,7 +151,7 @@ class QemuRunner:
env["DEPLOY_DIR_IMAGE"] = self.deploy_dir_image
if not launch_cmd:
- launch_cmd = 'runqemu %s %s ' % ('snapshot' if discard_writes else '', runqemuparams)
+ launch_cmd = 'runqemu %s' % ('snapshot' if discard_writes else '')
if self.use_kvm:
self.logger.debug('Using kvm for runqemu')
launch_cmd += ' kvm'
@@ -133,13 +161,16 @@ class QemuRunner:
launch_cmd += ' nographic'
if self.use_slirp:
launch_cmd += ' slirp'
- launch_cmd += ' %s %s' % (self.machine, self.rootfs)
+ if self.use_ovmf:
+ launch_cmd += ' ovmf'
+ launch_cmd += ' %s %s %s' % (runqemuparams, self.machine, self.rootfs)
return self.launch(launch_cmd, qemuparams=qemuparams, get_ip=get_ip, extra_bootparams=extra_bootparams, env=env)
def launch(self, launch_cmd, get_ip = True, qemuparams = None, extra_bootparams = None, env = None):
try:
- self.threadsock, threadport = self.create_socket()
+ if self.serial_ports >= 2:
+ self.threadsock, threadport = self.create_socket()
self.server_socket, self.serverport = self.create_socket()
except socket.error as msg:
self.logger.error("Failed to create listening socket: %s" % msg[1])
@@ -153,11 +184,14 @@ class QemuRunner:
# and analyze descendents in order to determine it.
if os.path.exists(self.qemu_pidfile):
os.remove(self.qemu_pidfile)
- self.qemuparams = 'bootparams="{0}" qemuparams="-serial tcp:127.0.0.1:{1} -pidfile {2}"'.format(bootparams, threadport, self.qemu_pidfile)
+ self.qemuparams = 'bootparams="{0}" qemuparams="-pidfile {1}"'.format(bootparams, self.qemu_pidfile)
if qemuparams:
self.qemuparams = self.qemuparams[:-1] + " " + qemuparams + " " + '\"'
- launch_cmd += ' tcpserial=%s %s' % (self.serverport, self.qemuparams)
+ if self.serial_ports >= 2:
+ launch_cmd += ' tcpserial=%s:%s %s' % (threadport, self.serverport, self.qemuparams)
+ else:
+ launch_cmd += ' tcpserial=%s %s' % (self.serverport, self.qemuparams)
self.origchldhandler = signal.getsignal(signal.SIGCHLD)
signal.signal(signal.SIGCHLD, self.handleSIGCHLD)
@@ -203,20 +237,34 @@ class QemuRunner:
endtime = time.time() + self.runqemutime
while not self.is_alive() and time.time() < endtime:
if self.runqemu.poll():
+ if self.runqemu_exited:
+ return False
if self.runqemu.returncode:
# No point waiting any longer
- self.logger.debug('runqemu exited with code %d' % self.runqemu.returncode)
+ self.logger.warning('runqemu exited with code %d' % self.runqemu.returncode)
self._dump_host()
- self.logger.debug("Output from runqemu:\n%s" % self.getOutput(output))
+ self.logger.warning("Output from runqemu:\n%s" % self.getOutput(output))
self.stop()
return False
time.sleep(0.5)
+ if self.runqemu_exited:
+ return False
+
if not self.is_alive():
self.logger.error("Qemu pid didn't appear in %s seconds (%s)" %
(self.runqemutime, time.strftime("%D %H:%M:%S")))
+
+ qemu_pid = None
+ if os.path.isfile(self.qemu_pidfile):
+ with open(self.qemu_pidfile, 'r') as f:
+ qemu_pid = f.read().strip()
+
+ self.logger.error("Status information, poll status: %s, pidfile exists: %s, pidfile contents %s, proc pid exists %s"
+ % (self.runqemu.poll(), os.path.isfile(self.qemu_pidfile), str(qemu_pid), os.path.exists("/proc/" + str(qemu_pid))))
+
# Dump all processes to help us to figure out what is going on...
- ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command '], stdout=subprocess.PIPE).communicate()[0]
+ ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,pri,ni,command '], stdout=subprocess.PIPE).communicate()[0]
processes = ps.decode("utf-8")
self.logger.debug("Running processes:\n%s" % processes)
self._dump_host()
@@ -234,8 +282,8 @@ class QemuRunner:
self.logger.debug("qemu started in %s seconds - qemu procces pid is %s (%s)" %
(time.time() - (endtime - self.runqemutime),
self.qemupid, time.strftime("%D %H:%M:%S")))
+ cmdline = ''
if get_ip:
- cmdline = ''
with open('/proc/%s/cmdline' % self.qemupid) as p:
cmdline = p.read()
# It is needed to sanitize the data received
@@ -253,7 +301,7 @@ class QemuRunner:
self.logger.debug("qemu cmdline used:\n{}".format(cmdline))
except (IndexError, ValueError):
# Try to get network configuration from runqemu output
- match = re.match(r'.*Network configuration: ([0-9.]+)::([0-9.]+):([0-9.]+)$.*',
+ match = re.match(r'.*Network configuration: (?:ip=)*([0-9.]+)::([0-9.]+):([0-9.]+)$.*',
out, re.MULTILINE|re.DOTALL)
if match:
self.ip, self.server_ip, self.netmask = match.groups()
@@ -272,14 +320,15 @@ class QemuRunner:
self.logger.debug("Target IP: %s" % self.ip)
self.logger.debug("Server IP: %s" % self.server_ip)
- self.thread = LoggingThread(self.log, self.threadsock, self.logger)
- self.thread.start()
- if not self.thread.connection_established.wait(self.boottime):
- self.logger.error("Didn't receive a console connection from qemu. "
- "Here is the qemu command line used:\n%s\nand "
- "output from runqemu:\n%s" % (cmdline, out))
- self.stop_thread()
- return False
+ if self.serial_ports >= 2:
+ self.thread = LoggingThread(self.log, self.threadsock, self.logger)
+ self.thread.start()
+ if not self.thread.connection_established.wait(self.boottime):
+ self.logger.error("Didn't receive a console connection from qemu. "
+ "Here is the qemu command line used:\n%s\nand "
+ "output from runqemu:\n%s" % (cmdline, out))
+ self.stop_thread()
+ return False
self.logger.debug("Output from runqemu:\n%s", out)
self.logger.debug("Waiting at most %d seconds for login banner (%s)" %
@@ -307,8 +356,12 @@ class QemuRunner:
data = data + sock.recv(1024)
if data:
bootlog += data
+ if self.serial_ports < 2:
+ # this socket has mixed console/kernel data, log it to logfile
+ self.log(data)
+
data = b''
- if b' login:' in bootlog:
+ if self.boot_patterns['search_reached_prompt'] in bootlog:
self.server_socket = qemusock
stopread = True
reachedlogin = True
@@ -326,21 +379,22 @@ class QemuRunner:
if not reachedlogin:
if time.time() >= endtime:
- self.logger.debug("Target didn't reach login banner in %d seconds (%s)" %
+ self.logger.warning("Target didn't reach login banner in %d seconds (%s)" %
(self.boottime, time.strftime("%D %H:%M:%S")))
tail = lambda l: "\n".join(l.splitlines()[-25:])
+ bootlog = bootlog.decode("utf-8")
# in case bootlog is empty, use tail qemu log store at self.msg
lines = tail(bootlog if bootlog else self.msg)
- self.logger.debug("Last 25 lines of text:\n%s" % lines)
- self.logger.debug("Check full boot log: %s" % self.logfile)
+ self.logger.warning("Last 25 lines of text:\n%s" % lines)
+ self.logger.warning("Check full boot log: %s" % self.logfile)
self._dump_host()
self.stop()
return False
# If we are not able to login the tests can continue
try:
- (status, output) = self.run_serial("root\n", raw=True)
- if re.search(r"root@[a-zA-Z0-9\-]+:~#", output):
+ (status, output) = self.run_serial(self.boot_patterns['send_login_user'], raw=True, timeout=120)
+ if re.search(self.boot_patterns['search_login_succeeded'], output):
self.logged = True
self.logger.debug("Logged as root in serial console")
if netconf:
@@ -352,11 +406,11 @@ class QemuRunner:
else:
self.logger.debug("Couldn't configure guest networking")
else:
- self.logger.debug("Couldn't login into serial console"
+ self.logger.warning("Couldn't login into serial console"
" as root using blank password")
- self.logger.debug("The output:\n%s" % output)
+ self.logger.warning("The output:\n%s" % output)
except:
- self.logger.debug("Serial console failed while trying to login")
+ self.logger.warning("Serial console failed while trying to login")
return True
def stop(self):
@@ -381,7 +435,7 @@ class QemuRunner:
os.killpg(os.getpgid(self.runqemu.pid), signal.SIGKILL)
self.runqemu.stdin.close()
self.runqemu.stdout.close()
- self.runqemu = None
+ self.runqemu_exited = True
if hasattr(self, 'server_socket') and self.server_socket:
self.server_socket.close()
@@ -392,7 +446,11 @@ class QemuRunner:
self.qemupid = None
self.ip = None
if os.path.exists(self.qemu_pidfile):
- os.remove(self.qemu_pidfile)
+ try:
+ os.remove(self.qemu_pidfile)
+ except FileNotFoundError as e:
+ # We raced, ignore
+ pass
if self.monitorpipe:
self.monitorpipe.close()
@@ -410,7 +468,7 @@ class QemuRunner:
self.thread.join()
def restart(self, qemuparams = None):
- self.logger.debug("Restarting qemu process")
+ self.logger.warning("Restarting qemu process")
if self.runqemu.poll() is None:
self.stop()
if self.start(qemuparams):
@@ -418,16 +476,23 @@ class QemuRunner:
return False
def is_alive(self):
- if not self.runqemu or self.runqemu.poll() is not None:
+ if not self.runqemu or self.runqemu.poll() is not None or self.runqemu_exited:
return False
if os.path.isfile(self.qemu_pidfile):
- f = open(self.qemu_pidfile, 'r')
- qemu_pid = f.read()
- f.close()
- qemupid = int(qemu_pid)
- if os.path.exists("/proc/" + str(qemupid)):
- self.qemupid = qemupid
- return True
+ # when handling pidfile, qemu creates the file, stat it, lock it and then write to it
+ # so it's possible that the file has been created but the content is empty
+ pidfile_timeout = time.time() + 3
+ while time.time() < pidfile_timeout:
+ with open(self.qemu_pidfile, 'r') as f:
+ qemu_pid = f.read().strip()
+ # file created but not yet written contents
+ if not qemu_pid:
+ time.sleep(0.5)
+ continue
+ else:
+ if os.path.exists("/proc/" + qemu_pid):
+ self.qemupid = int(qemu_pid)
+ return True
return False
def run_serial(self, command, raw=False, timeout=60):
@@ -454,7 +519,7 @@ class QemuRunner:
if answer:
data += answer.decode('utf-8')
# Search the prompt to stop
- if re.search(r"[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#", data):
+ if re.search(self.boot_patterns['search_cmd_finished'], data):
break
else:
raise Exception("No data on serial console socket")
diff --git a/meta/lib/oeqa/utils/qemutinyrunner.py b/meta/lib/oeqa/utils/qemutinyrunner.py
index 5aa99d0686..5c92941c0a 100644
--- a/meta/lib/oeqa/utils/qemutinyrunner.py
+++ b/meta/lib/oeqa/utils/qemutinyrunner.py
@@ -1,6 +1,8 @@
+#
# Copyright (C) 2015 Intel Corporation
#
-# Released under the MIT license (see COPYING.MIT)
+# SPDX-License-Identifier: MIT
+#
# This module provides a class for starting qemu images of poky tiny.
# It's used by testimage.bbclass.
@@ -136,7 +138,7 @@ class QemuTinyRunner(QemuRunner):
#
# Walk the process tree from the process specified looking for a qemu-system. Return its [pid'cmd]
#
- ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command'], stdout=subprocess.PIPE).communicate()[0]
+ ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,pri,ni,command'], stdout=subprocess.PIPE).communicate()[0]
processes = ps.decode("utf-8").split('\n')
nfields = len(processes[0].split()) - 1
pids = {}
diff --git a/meta/lib/oeqa/utils/sshcontrol.py b/meta/lib/oeqa/utils/sshcontrol.py
index d292893c08..36c2ecb3db 100644
--- a/meta/lib/oeqa/utils/sshcontrol.py
+++ b/meta/lib/oeqa/utils/sshcontrol.py
@@ -1,7 +1,8 @@
-# -*- coding: utf-8 -*-
+#
# Copyright (C) 2013 Intel Corporation
#
-# Released under the MIT license (see COPYING.MIT)
+# SPDX-License-Identifier: MIT
+#
# Provides a class for setting up ssh connections,
# running commands and copying files to/from a target.
@@ -22,7 +23,7 @@ class SSHProcess(object):
"stdin": None,
"shell": False,
"bufsize": -1,
- "preexec_fn": os.setsid,
+ "start_new_session": True,
}
self.options = dict(self.defaultopts)
self.options.update(options)
diff --git a/meta/lib/oeqa/utils/subprocesstweak.py b/meta/lib/oeqa/utils/subprocesstweak.py
index 1f7d11b55c..b47975a4bc 100644
--- a/meta/lib/oeqa/utils/subprocesstweak.py
+++ b/meta/lib/oeqa/utils/subprocesstweak.py
@@ -1,3 +1,6 @@
+#
+# SPDX-License-Identifier: MIT
+#
import subprocess
class OETestCalledProcessError(subprocess.CalledProcessError):
diff --git a/meta/lib/oeqa/utils/targetbuild.py b/meta/lib/oeqa/utils/targetbuild.py
index b8db7b2aca..1055810ca3 100644
--- a/meta/lib/oeqa/utils/targetbuild.py
+++ b/meta/lib/oeqa/utils/targetbuild.py
@@ -1,6 +1,8 @@
+#
# Copyright (C) 2013 Intel Corporation
#
-# Released under the MIT license (see COPYING.MIT)
+# SPDX-License-Identifier: MIT
+#
# Provides a class for automating build tests for projects
diff --git a/meta/lib/oeqa/utils/testexport.py b/meta/lib/oeqa/utils/testexport.py
index be2a2110fc..e89d130a9c 100644
--- a/meta/lib/oeqa/utils/testexport.py
+++ b/meta/lib/oeqa/utils/testexport.py
@@ -1,6 +1,8 @@
+#
# Copyright (C) 2015 Intel Corporation
#
-# Released under the MIT license (see COPYING.MIT)
+# SPDX-License-Identifier: MIT
+#
# Provides functions to help with exporting binaries obtained from built targets
generated by cgit 1.2.3-korg (git 2.39.0) at 2024-05-01 18:41:56 +0000