diff options
Diffstat (limited to 'meta/lib/oeqa/utils')
| -rw-r--r-- | meta/lib/oeqa/utils/__init__.py | 20 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/commands.py | 37 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/dump.py | 50 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/httpserver.py | 4 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/logparser.py | 16 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/package_manager.py | 8 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/qemurunner.py | 252 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/qemutinyrunner.py | 8 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/sshcontrol.py | 2 |
9 files changed, 315 insertions, 82 deletions
diff --git a/meta/lib/oeqa/utils/__init__.py b/meta/lib/oeqa/utils/__init__.py index 70fbe7b552..6d1ec4cb99 100644 --- a/meta/lib/oeqa/utils/__init__.py +++ b/meta/lib/oeqa/utils/__init__.py @@ -43,28 +43,12 @@ def make_logger_bitbake_compatible(logger): import logging """ - Bitbake logger redifines debug() in order to - set a level within debug, this breaks compatibility - with vainilla logging, so we neeed to redifine debug() - method again also add info() method with INFO + 1 level. + We need to raise the log level of the info output so unittest + messages are visible on the console. """ - def _bitbake_log_debug(*args, **kwargs): - lvl = logging.DEBUG - - if isinstance(args[0], int): - lvl = args[0] - msg = args[1] - args = args[2:] - else: - msg = args[0] - args = args[1:] - - logger.log(lvl, msg, *args, **kwargs) - def _bitbake_log_info(msg, *args, **kwargs): logger.log(logging.INFO + 1, msg, *args, **kwargs) - logger.debug = _bitbake_log_debug logger.info = _bitbake_log_info return logger diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py index dc1e286dac..024261410e 100644 --- a/meta/lib/oeqa/utils/commands.py +++ b/meta/lib/oeqa/utils/commands.py @@ -95,7 +95,9 @@ class Command(object): # reason, the main process will still exit, which will then # kill the write thread. if self.data: - threading.Thread(target=writeThread, daemon=True).start() + thread = threading.Thread(target=writeThread, daemon=True) + thread.start() + self.threads.append(thread) if self.process.stderr: thread = threading.Thread(target=readStderrThread) thread.start() @@ -123,11 +125,11 @@ class Command(object): def stop(self): for thread in self.threads: - if thread.isAlive(): + if thread.is_alive(): self.process.terminate() # let's give it more time to terminate gracefully before killing it thread.join(5) - if thread.isAlive(): + if thread.is_alive(): self.process.kill() thread.join() @@ -165,23 +167,29 @@ class Result(object): pass -def runCmd(command, ignore_status=False, timeout=None, assert_error=True, +def runCmd(command, ignore_status=False, timeout=None, assert_error=True, sync=True, native_sysroot=None, limit_exc_output=0, output_log=None, **options): result = Result() if native_sysroot: extra_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin" % \ (native_sysroot, native_sysroot, native_sysroot) - extra_libpaths = "%s/lib:%s/usr/lib" % \ - (native_sysroot, native_sysroot) nenv = dict(options.get('env', os.environ)) nenv['PATH'] = extra_paths + ':' + nenv.get('PATH', '') - nenv['LD_LIBRARY_PATH'] = extra_libpaths + ':' + nenv.get('LD_LIBRARY_PATH', '') options['env'] = nenv cmd = Command(command, timeout=timeout, output_log=output_log, **options) cmd.run() + # tests can be heavy on IO and if bitbake can't write out its caches, we see timeouts. + # call sync around the tests to ensure the IO queue doesn't get too large, taking any IO + # hit here rather than in bitbake shutdown. + if sync: + p = os.environ['PATH'] + os.environ['PATH'] = "/usr/bin:/bin:/usr/sbin:/sbin:" + p + os.system("sync") + os.environ['PATH'] = p + result.command = command result.status = cmd.status result.output = cmd.output @@ -315,15 +323,15 @@ def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None, try: tinfoil.logger.setLevel(logging.WARNING) import oeqa.targetcontrol - tinfoil.config_data.setVar("TEST_LOG_DIR", "${WORKDIR}/testimage") - tinfoil.config_data.setVar("TEST_QEMUBOOT_TIMEOUT", "1000") + recipedata = tinfoil.parse_recipe(pn) + recipedata.setVar("TEST_LOG_DIR", "${WORKDIR}/testimage") + recipedata.setVar("TEST_QEMUBOOT_TIMEOUT", "1000") # Tell QemuTarget() whether need find rootfs/kernel or not if launch_cmd: - tinfoil.config_data.setVar("FIND_ROOTFS", '0') + recipedata.setVar("FIND_ROOTFS", '0') else: - tinfoil.config_data.setVar("FIND_ROOTFS", '1') + recipedata.setVar("FIND_ROOTFS", '1') - recipedata = tinfoil.parse_recipe(pn) for key, value in overrides.items(): recipedata.setVar(key, value) @@ -351,10 +359,7 @@ def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None, finally: targetlogger.removeHandler(handler) - try: - qemu.stop() - except: - pass + qemu.stop() def updateEnv(env_file): """ diff --git a/meta/lib/oeqa/utils/dump.py b/meta/lib/oeqa/utils/dump.py index d34e05e2b4..bb067f4846 100644 --- a/meta/lib/oeqa/utils/dump.py +++ b/meta/lib/oeqa/utils/dump.py @@ -4,6 +4,7 @@ import os import sys +import json import errno import datetime import itertools @@ -17,6 +18,7 @@ class BaseDumper(object): # Some testing doesn't inherit testimage, so it is needed # to set some defaults. self.parent_dir = parent_dir + self.dump_dir = parent_dir dft_cmds = """ top -bn1 iostat -x -z -N -d -p ALL 20 2 ps -ef @@ -46,11 +48,13 @@ class BaseDumper(object): raise err self.dump_dir = dump_dir - def _write_dump(self, command, output): + def _construct_filename(self, command): if isinstance(self, HostDumper): prefix = "host" elif isinstance(self, TargetDumper): prefix = "target" + elif isinstance(self, MonitorDumper): + prefix = "qmp" else: prefix = "unknown" for i in itertools.count(): @@ -58,9 +62,16 @@ class BaseDumper(object): fullname = os.path.join(self.dump_dir, filename) if not os.path.exists(fullname): break - with open(fullname, 'w') as dump_file: - dump_file.write(output) + return fullname + def _write_dump(self, command, output): + fullname = self._construct_filename(command) + if isinstance(self, MonitorDumper): + with open(fullname, 'w') as json_file: + json.dump(output, json_file, indent=4) + else: + with open(fullname, 'w') as dump_file: + dump_file.write(output) class HostDumper(BaseDumper): """ Class to get dumps from the host running the tests """ @@ -71,8 +82,11 @@ class HostDumper(BaseDumper): def dump_host(self, dump_dir=""): if dump_dir: self.dump_dir = dump_dir + env = os.environ.copy() + env['PATH'] = '/usr/sbin:/sbin:/usr/bin:/bin' + env['COLUMNS'] = '9999' for cmd in self.cmds: - result = runCmd(cmd, ignore_status=True) + result = runCmd(cmd, ignore_status=True, env=env) self._write_dump(cmd.split()[0], result.output) class TargetDumper(BaseDumper): @@ -93,3 +107,31 @@ class TargetDumper(BaseDumper): except: print("Tried to dump info from target but " "serial console failed") + print("Failed CMD: %s" % (cmd)) + +class MonitorDumper(BaseDumper): + """ Class to get dumps via the Qemu Monitor, it only works with QemuRunner """ + + def __init__(self, cmds, parent_dir, runner): + super(MonitorDumper, self).__init__(cmds, parent_dir) + self.runner = runner + + def dump_monitor(self, dump_dir=""): + if self.runner is None: + return + if dump_dir: + self.dump_dir = dump_dir + for cmd in self.cmds: + cmd_name = cmd.split()[0] + try: + if len(cmd.split()) > 1: + cmd_args = cmd.split()[1] + if "%s" in cmd_args: + filename = self._construct_filename(cmd_name) + cmd_data = json.loads(cmd_args % (filename)) + output = self.runner.run_monitor(cmd_name, cmd_data) + else: + output = self.runner.run_monitor(cmd_name) + self._write_dump(cmd_name, output) + except Exception as e: + print("Failed to dump QMP CMD: %s with\nExecption: %s" % (cmd_name, e)) diff --git a/meta/lib/oeqa/utils/httpserver.py b/meta/lib/oeqa/utils/httpserver.py index aa435590f0..58d3c3b3f8 100644 --- a/meta/lib/oeqa/utils/httpserver.py +++ b/meta/lib/oeqa/utils/httpserver.py @@ -22,10 +22,10 @@ class HTTPRequestHandler(http.server.SimpleHTTPRequestHandler): class HTTPService(object): - def __init__(self, root_dir, host='', logger=None): + def __init__(self, root_dir, host='', port=0, logger=None): self.root_dir = root_dir self.host = host - self.port = 0 + self.port = port self.logger = logger def start(self): diff --git a/meta/lib/oeqa/utils/logparser.py b/meta/lib/oeqa/utils/logparser.py index 7313df8ec3..60e16d500e 100644 --- a/meta/lib/oeqa/utils/logparser.py +++ b/meta/lib/oeqa/utils/logparser.py @@ -25,13 +25,20 @@ class PtestParser(object): section_regex['exitcode'] = re.compile(r"^ERROR: Exit status is (.+)") section_regex['timeout'] = re.compile(r"^TIMEOUT: .*/(.+)/ptest") + # Cache markers so we don't take the re.search() hit all the time. + markers = ("PASS:", "FAIL:", "SKIP:", "BEGIN:", "END:", "DURATION:", "ERROR: Exit", "TIMEOUT:") + def newsection(): - return { 'name': "No-section", 'log': "" } + return { 'name': "No-section", 'log': [] } current_section = newsection() with open(logfile, errors='replace') as f: for line in f: + if not line.startswith(markers): + current_section['log'].append(line) + continue + result = section_regex['begin'].search(line) if result: current_section['name'] = result.group(1) @@ -61,7 +68,7 @@ class PtestParser(object): current_section[t] = result.group(1) continue - current_section['log'] = current_section['log'] + line + current_section['log'].append(line) for t in test_regex: result = test_regex[t].search(line) @@ -70,6 +77,11 @@ class PtestParser(object): self.results[current_section['name']] = {} self.results[current_section['name']][result.group(1).strip()] = t + # Python performance for repeatedly joining long strings is poor, do it all at once at the end. + # For 2.1 million lines in a log this reduces 18 hours to 12s. + for section in self.sections: + self.sections[section]['log'] = "".join(self.sections[section]['log']) + return self.results, self.sections # Log the results as files. The file name is the section name and the contents are the tests in that section. diff --git a/meta/lib/oeqa/utils/package_manager.py b/meta/lib/oeqa/utils/package_manager.py index 2d358f7172..6b67f22fdd 100644 --- a/meta/lib/oeqa/utils/package_manager.py +++ b/meta/lib/oeqa/utils/package_manager.py @@ -12,7 +12,9 @@ def get_package_manager(d, root_path): """ Returns an OE package manager that can install packages in root_path. """ - from oe.package_manager import RpmPM, OpkgPM, DpkgPM + from oe.package_manager.rpm import RpmPM + from oe.package_manager.ipk import OpkgPM + from oe.package_manager.deb import DpkgPM pkg_class = d.getVar("IMAGE_PKGTYPE") if pkg_class == "rpm": @@ -115,7 +117,7 @@ def extract_packages(d, needed_packages): extract = package.get('extract', True) if extract: - #logger.debug(1, 'Extracting %s' % pkg) + #logger.debug('Extracting %s' % pkg) dst_dir = os.path.join(extracted_path, pkg) # Same package used for more than one test, # don't need to extract again. @@ -128,7 +130,7 @@ def extract_packages(d, needed_packages): shutil.rmtree(pkg_dir) else: - #logger.debug(1, 'Copying %s' % pkg) + #logger.debug('Copying %s' % pkg) _copy_package(d, pkg) def _extract_in_tmpdir(d, pkg): diff --git a/meta/lib/oeqa/utils/qemurunner.py b/meta/lib/oeqa/utils/qemurunner.py index fe8b77d97a..d55248c497 100644 --- a/meta/lib/oeqa/utils/qemurunner.py +++ b/meta/lib/oeqa/utils/qemurunner.py @@ -20,7 +20,10 @@ import string import threading import codecs import logging +import tempfile from oeqa.utils.dump import HostDumper +from collections import defaultdict +import importlib # Get Unicode non printable control chars control_range = list(range(0,32))+list(range(127,160)) @@ -31,10 +34,11 @@ re_control_char = re.compile('[%s]' % re.escape("".join(control_chars))) class QemuRunner: def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds, - use_kvm, logger, use_slirp=False): + use_kvm, logger, use_slirp=False, serial_ports=2, boot_patterns = defaultdict(str), use_ovmf=False, workdir=None, tmpfsdir=None): # Popen object for runqemu self.runqemu = None + self.runqemu_exited = False # pid of the qemu process that runqemu will start self.qemupid = None # target ip - from the command line or runqemu output @@ -54,15 +58,42 @@ class QemuRunner: self.logged = False self.thread = None self.use_kvm = use_kvm + self.use_ovmf = use_ovmf self.use_slirp = use_slirp + self.serial_ports = serial_ports self.msg = '' + self.boot_patterns = boot_patterns + self.tmpfsdir = tmpfsdir - self.runqemutime = 120 - self.qemu_pidfile = 'pidfile_'+str(os.getpid()) + self.runqemutime = 300 + if not workdir: + workdir = os.getcwd() + self.qemu_pidfile = workdir + '/pidfile_' + str(os.getpid()) self.host_dumper = HostDumper(dump_host_cmds, dump_dir) self.monitorpipe = None self.logger = logger + # Whether we're expecting an exit and should show related errors + self.canexit = False + + # Enable testing other OS's + # Set commands for target communication, and default to Linux ALWAYS + # Other OS's or baremetal applications need to provide their + # own implementation passing it through QemuRunner's constructor + # or by passing them through TESTIMAGE_BOOT_PATTERNS[flag] + # provided variables, where <flag> is one of the mentioned below. + accepted_patterns = ['search_reached_prompt', 'send_login_user', 'search_login_succeeded', 'search_cmd_finished'] + default_boot_patterns = defaultdict(str) + # Default to the usual paterns used to communicate with the target + default_boot_patterns['search_reached_prompt'] = b' login:' + default_boot_patterns['send_login_user'] = 'root\n' + default_boot_patterns['search_login_succeeded'] = r"root@[a-zA-Z0-9\-]+:~#" + default_boot_patterns['search_cmd_finished'] = r"[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#" + + # Only override patterns that were set e.g. login user TESTIMAGE_BOOT_PATTERNS[send_login_user] = "webserver\n" + for pattern in accepted_patterns: + if not self.boot_patterns[pattern]: + self.boot_patterns[pattern] = default_boot_patterns[pattern] def create_socket(self): try: @@ -92,17 +123,19 @@ class QemuRunner: import fcntl fl = fcntl.fcntl(o, fcntl.F_GETFL) fcntl.fcntl(o, fcntl.F_SETFL, fl | os.O_NONBLOCK) - return os.read(o.fileno(), 1000000).decode("utf-8") + try: + return os.read(o.fileno(), 1000000).decode("utf-8") + except BlockingIOError: + return "" def handleSIGCHLD(self, signum, frame): if self.runqemu and self.runqemu.poll(): if self.runqemu.returncode: - self.logger.warning('runqemu exited with code %d' % self.runqemu.returncode) - self.logger.debug("Output from runqemu:\n%s" % self.getOutput(self.runqemu.stdout)) + self.logger.error('runqemu exited with code %d' % self.runqemu.returncode) + self.logger.error('Output from runqemu:\n%s' % self.getOutput(self.runqemu.stdout)) self.stop() self._dump_host() - raise SystemExit def start(self, qemuparams = None, get_ip = True, extra_bootparams = None, runqemuparams='', launch_cmd=None, discard_writes=True): env = os.environ.copy() @@ -125,6 +158,9 @@ class QemuRunner: else: env["DEPLOY_DIR_IMAGE"] = self.deploy_dir_image + if self.tmpfsdir: + env["RUNQEMU_TMPFS_DIR"] = self.tmpfsdir + if not launch_cmd: launch_cmd = 'runqemu %s' % ('snapshot' if discard_writes else '') if self.use_kvm: @@ -136,19 +172,45 @@ class QemuRunner: launch_cmd += ' nographic' if self.use_slirp: launch_cmd += ' slirp' + if self.use_ovmf: + launch_cmd += ' ovmf' launch_cmd += ' %s %s %s' % (runqemuparams, self.machine, self.rootfs) return self.launch(launch_cmd, qemuparams=qemuparams, get_ip=get_ip, extra_bootparams=extra_bootparams, env=env) def launch(self, launch_cmd, get_ip = True, qemuparams = None, extra_bootparams = None, env = None): + # use logfile to determine the recipe-sysroot-native path and + # then add in the site-packages path components and add that + # to the python sys.path so qmp.py can be found. + python_path = os.path.dirname(os.path.dirname(self.logfile)) + python_path += "/recipe-sysroot-native/usr/lib/python3.9/site-packages" + sys.path.append(python_path) + importlib.invalidate_caches() + try: + qmp = importlib.import_module("qmp") + except: + self.logger.error("qemurunner: qmp.py missing, please ensure it's installed") + return False + # Path relative to tmpdir used as cwd for qemu below to avoid unix socket path length issues + qmp_file = "." + next(tempfile._get_candidate_names()) + qmp_param = ' -S -qmp unix:./%s,server,wait' % (qmp_file) + qmp_port = self.tmpdir + "/" + qmp_file + # Create a second socket connection for debugging use, + # note this will NOT cause qemu to block waiting for the connection + qmp_file2 = "." + next(tempfile._get_candidate_names()) + qmp_param += ' -qmp unix:./%s,server,nowait' % (qmp_file2) + qmp_port2 = self.tmpdir + "/" + qmp_file2 + self.logger.info("QMP Available for connection at %s" % (qmp_port2)) + try: - self.threadsock, threadport = self.create_socket() + if self.serial_ports >= 2: + self.threadsock, threadport = self.create_socket() self.server_socket, self.serverport = self.create_socket() except socket.error as msg: self.logger.error("Failed to create listening socket: %s" % msg[1]) return False - bootparams = 'console=tty1 console=ttyS0,115200n8 printk.time=1' + bootparams = ' printk.time=1' if extra_bootparams: bootparams = bootparams + ' ' + extra_bootparams @@ -156,11 +218,15 @@ class QemuRunner: # and analyze descendents in order to determine it. if os.path.exists(self.qemu_pidfile): os.remove(self.qemu_pidfile) - self.qemuparams = 'bootparams="{0}" qemuparams="-pidfile {1}"'.format(bootparams, self.qemu_pidfile) + self.qemuparams = 'bootparams="{0}" qemuparams="-pidfile {1} {2}"'.format(bootparams, self.qemu_pidfile, qmp_param) + if qemuparams: self.qemuparams = self.qemuparams[:-1] + " " + qemuparams + " " + '\"' - launch_cmd += ' tcpserial=%s:%s %s' % (threadport, self.serverport, self.qemuparams) + if self.serial_ports >= 2: + launch_cmd += ' tcpserial=%s:%s %s' % (threadport, self.serverport, self.qemuparams) + else: + launch_cmd += ' tcpserial=%s %s' % (self.serverport, self.qemuparams) self.origchldhandler = signal.getsignal(signal.SIGCHLD) signal.signal(signal.SIGCHLD, self.handleSIGCHLD) @@ -171,8 +237,9 @@ class QemuRunner: # blocking at the end of the runqemu script when using this within # oe-selftest (this makes stty error out immediately). There ought # to be a proper fix but this will suffice for now. - self.runqemu = subprocess.Popen(launch_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, preexec_fn=os.setpgrp, env=env) + self.runqemu = subprocess.Popen(launch_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, preexec_fn=os.setpgrp, env=env, cwd=self.tmpdir) output = self.runqemu.stdout + launch_time = time.time() # # We need the preexec_fn above so that all runqemu processes can easily be killed @@ -206,6 +273,9 @@ class QemuRunner: endtime = time.time() + self.runqemutime while not self.is_alive() and time.time() < endtime: if self.runqemu.poll(): + if self.runqemu_exited: + self.logger.warning("runqemu during is_alive() test") + return False if self.runqemu.returncode: # No point waiting any longer self.logger.warning('runqemu exited with code %d' % self.runqemu.returncode) @@ -215,11 +285,26 @@ class QemuRunner: return False time.sleep(0.5) + if self.runqemu_exited: + self.logger.warning("runqemu after timeout") + + if self.runqemu.returncode: + self.logger.warning('runqemu exited with code %d' % self.runqemu.returncode) + if not self.is_alive(): self.logger.error("Qemu pid didn't appear in %s seconds (%s)" % (self.runqemutime, time.strftime("%D %H:%M:%S"))) + + qemu_pid = None + if os.path.isfile(self.qemu_pidfile): + with open(self.qemu_pidfile, 'r') as f: + qemu_pid = f.read().strip() + + self.logger.error("Status information, poll status: %s, pidfile exists: %s, pidfile contents %s, proc pid exists %s" + % (self.runqemu.poll(), os.path.isfile(self.qemu_pidfile), str(qemu_pid), os.path.exists("/proc/" + str(qemu_pid)))) + # Dump all processes to help us to figure out what is going on... - ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command '], stdout=subprocess.PIPE).communicate()[0] + ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,pri,ni,command '], stdout=subprocess.PIPE).communicate()[0] processes = ps.decode("utf-8") self.logger.debug("Running processes:\n%s" % processes) self._dump_host() @@ -231,14 +316,80 @@ class QemuRunner: self.logger.error("No output from runqemu.\n") return False + # Create the client socket for the QEMU Monitor Control Socket + # This will allow us to read status from Qemu if the the process + # is still alive + self.logger.debug("QMP Initializing to %s" % (qmp_port)) + # chdir dance for path length issues with unix sockets + origpath = os.getcwd() + try: + os.chdir(os.path.dirname(qmp_port)) + try: + self.qmp = qmp.QEMUMonitorProtocol(os.path.basename(qmp_port)) + except OSError as msg: + self.logger.warning("Failed to initialize qemu monitor socket: %s File: %s" % (msg, msg.filename)) + return False + + self.logger.debug("QMP Connecting to %s" % (qmp_port)) + if not os.path.exists(qmp_port) and self.is_alive(): + self.logger.debug("QMP Port does not exist waiting for it to be created") + endtime = time.time() + self.runqemutime + while not os.path.exists(qmp_port) and self.is_alive() and time.time() < endtime: + self.logger.info("QMP port does not exist yet!") + time.sleep(0.5) + if not os.path.exists(qmp_port) and self.is_alive(): + self.logger.warning("QMP Port still does not exist but QEMU is alive") + return False + + try: + self.qmp.connect() + connect_time = time.time() + self.logger.info("QMP connected to QEMU at %s and took %s seconds" % + (time.strftime("%D %H:%M:%S"), + time.time() - launch_time)) + except OSError as msg: + self.logger.warning("Failed to connect qemu monitor socket: %s File: %s" % (msg, msg.filename)) + return False + except qmp.QMPConnectError as msg: + self.logger.warning("Failed to communicate with qemu monitor: %s" % (msg)) + return False + finally: + os.chdir(origpath) + + # We worry that mmap'd libraries may cause page faults which hang the qemu VM for periods + # causing failures. Before we "start" qemu, read through it's mapped files to try and + # ensure we don't hit page faults later + mapdir = "/proc/" + str(self.qemupid) + "/map_files/" + try: + for f in os.listdir(mapdir): + try: + linktarget = os.readlink(os.path.join(mapdir, f)) + if not linktarget.startswith("/") or linktarget.startswith("/dev") or "deleted" in linktarget: + continue + with open(linktarget, "rb") as readf: + data = True + while data: + data = readf.read(4096) + except FileNotFoundError: + continue + # Centos7 doesn't allow us to read /map_files/ + except PermissionError: + pass + + # Release the qemu process to continue running + self.run_monitor('cont') + self.logger.info("QMP released QEMU at %s and took %s seconds from connect" % + (time.strftime("%D %H:%M:%S"), + time.time() - connect_time)) + # We are alive: qemu is running out = self.getOutput(output) netconf = False # network configuration is not required by default self.logger.debug("qemu started in %s seconds - qemu procces pid is %s (%s)" % (time.time() - (endtime - self.runqemutime), self.qemupid, time.strftime("%D %H:%M:%S"))) + cmdline = '' if get_ip: - cmdline = '' with open('/proc/%s/cmdline' % self.qemupid) as p: cmdline = p.read() # It is needed to sanitize the data received @@ -256,7 +407,7 @@ class QemuRunner: self.logger.debug("qemu cmdline used:\n{}".format(cmdline)) except (IndexError, ValueError): # Try to get network configuration from runqemu output - match = re.match(r'.*Network configuration: ([0-9.]+)::([0-9.]+):([0-9.]+)$.*', + match = re.match(r'.*Network configuration: (?:ip=)*([0-9.]+)::([0-9.]+):([0-9.]+)$.*', out, re.MULTILINE|re.DOTALL) if match: self.ip, self.server_ip, self.netmask = match.groups() @@ -275,14 +426,15 @@ class QemuRunner: self.logger.debug("Target IP: %s" % self.ip) self.logger.debug("Server IP: %s" % self.server_ip) - self.thread = LoggingThread(self.log, self.threadsock, self.logger) - self.thread.start() - if not self.thread.connection_established.wait(self.boottime): - self.logger.error("Didn't receive a console connection from qemu. " - "Here is the qemu command line used:\n%s\nand " - "output from runqemu:\n%s" % (cmdline, out)) - self.stop_thread() - return False + if self.serial_ports >= 2: + self.thread = LoggingThread(self.log, self.threadsock, self.logger) + self.thread.start() + if not self.thread.connection_established.wait(self.boottime): + self.logger.error("Didn't receive a console connection from qemu. " + "Here is the qemu command line used:\n%s\nand " + "output from runqemu:\n%s" % (cmdline, out)) + self.stop_thread() + return False self.logger.debug("Output from runqemu:\n%s", out) self.logger.debug("Waiting at most %d seconds for login banner (%s)" % @@ -310,8 +462,12 @@ class QemuRunner: data = data + sock.recv(1024) if data: bootlog += data + if self.serial_ports < 2: + # this socket has mixed console/kernel data, log it to logfile + self.log(data) + data = b'' - if b' login:' in bootlog: + if self.boot_patterns['search_reached_prompt'] in bootlog: self.server_socket = qemusock stopread = True reachedlogin = True @@ -326,7 +482,6 @@ class QemuRunner: sock.close() stopread = True - if not reachedlogin: if time.time() >= endtime: self.logger.warning("Target didn't reach login banner in %d seconds (%s)" % @@ -343,8 +498,8 @@ class QemuRunner: # If we are not able to login the tests can continue try: - (status, output) = self.run_serial("root\n", raw=True) - if re.search(r"root@[a-zA-Z0-9\-]+:~#", output): + (status, output) = self.run_serial(self.boot_patterns['send_login_user'], raw=True, timeout=120) + if re.search(self.boot_patterns['search_login_succeeded'], output): self.logged = True self.logger.debug("Logged as root in serial console") if netconf: @@ -383,10 +538,15 @@ class QemuRunner: if self.runqemu.poll() is None: self.logger.debug("Sending SIGKILL to runqemu") os.killpg(os.getpgid(self.runqemu.pid), signal.SIGKILL) + if not self.runqemu.stdout.closed: + self.logger.info("Output from runqemu:\n%s" % self.getOutput(self.runqemu.stdout)) self.runqemu.stdin.close() self.runqemu.stdout.close() - self.runqemu = None + self.runqemu_exited = True + if hasattr(self, 'qmp') and self.qmp: + self.qmp.close() + self.qmp = None if hasattr(self, 'server_socket') and self.server_socket: self.server_socket.close() self.server_socket = None @@ -396,7 +556,11 @@ class QemuRunner: self.qemupid = None self.ip = None if os.path.exists(self.qemu_pidfile): - os.remove(self.qemu_pidfile) + try: + os.remove(self.qemu_pidfile) + except FileNotFoundError as e: + # We raced, ignore + pass if self.monitorpipe: self.monitorpipe.close() @@ -413,6 +577,11 @@ class QemuRunner: self.thread.stop() self.thread.join() + def allowexit(self): + self.canexit = True + if self.thread: + self.thread.allowexit() + def restart(self, qemuparams = None): self.logger.warning("Restarting qemu process") if self.runqemu.poll() is None: @@ -422,7 +591,7 @@ class QemuRunner: return False def is_alive(self): - if not self.runqemu or self.runqemu.poll() is not None: + if not self.runqemu or self.runqemu.poll() is not None or self.runqemu_exited: return False if os.path.isfile(self.qemu_pidfile): # when handling pidfile, qemu creates the file, stat it, lock it and then write to it @@ -441,6 +610,13 @@ class QemuRunner: return True return False + def run_monitor(self, command, args=None, timeout=60): + if hasattr(self, 'qmp') and self.qmp: + if args is not None: + return self.qmp.cmd(command, args) + else: + return self.qmp.cmd(command) + def run_serial(self, command, raw=False, timeout=60): # We assume target system have echo to get command status if not raw: @@ -465,10 +641,12 @@ class QemuRunner: if answer: data += answer.decode('utf-8') # Search the prompt to stop - if re.search(r"[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#", data): + if re.search(self.boot_patterns['search_cmd_finished'], data): break else: - raise Exception("No data on serial console socket") + if self.canexit: + return (1, "") + raise Exception("No data on serial console socket, connection closed?") if data: if raw: @@ -506,6 +684,7 @@ class LoggingThread(threading.Thread): self.logger = logger self.readsock = None self.running = False + self.canexit = False self.errorevents = select.POLLERR | select.POLLHUP | select.POLLNVAL self.readevents = select.POLLIN | select.POLLPRI @@ -539,6 +718,9 @@ class LoggingThread(threading.Thread): self.close_ignore_error(self.writepipe) self.running = False + def allowexit(self): + self.canexit = True + def eventloop(self): poll = select.poll() event_read_mask = self.errorevents | self.readevents @@ -584,7 +766,7 @@ class LoggingThread(threading.Thread): data = self.readsock.recv(count) except socket.error as e: if e.errno == errno.EAGAIN or e.errno == errno.EWOULDBLOCK: - return '' + return b'' else: raise @@ -595,7 +777,9 @@ class LoggingThread(threading.Thread): # happened. But for this code it counts as an # error since the connection shouldn't go away # until qemu exits. - raise Exception("Console connection closed unexpectedly") + if not self.canexit: + raise Exception("Console connection closed unexpectedly") + return b'' return data diff --git a/meta/lib/oeqa/utils/qemutinyrunner.py b/meta/lib/oeqa/utils/qemutinyrunner.py index 364005bd2d..20009401ca 100644 --- a/meta/lib/oeqa/utils/qemutinyrunner.py +++ b/meta/lib/oeqa/utils/qemutinyrunner.py @@ -19,7 +19,7 @@ from .qemurunner import QemuRunner class QemuTinyRunner(QemuRunner): - def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, kernel, boottime, logger): + def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, kernel, boottime, logger, tmpfsdir=None): # Popen object for runqemu self.runqemu = None @@ -37,6 +37,7 @@ class QemuTinyRunner(QemuRunner): self.deploy_dir_image = deploy_dir_image self.logfile = logfile self.boottime = boottime + self.tmpfsdir = tmpfsdir self.runqemutime = 60 self.socketfile = "console.sock" @@ -83,6 +84,9 @@ class QemuTinyRunner(QemuRunner): return False else: os.environ["DEPLOY_DIR_IMAGE"] = self.deploy_dir_image + if self.tmpfsdir: + env["RUNQEMU_TMPFS_DIR"] = self.tmpfsdir + # Set this flag so that Qemu doesn't do any grabs as SDL grabs interact # badly with screensavers. @@ -138,7 +142,7 @@ class QemuTinyRunner(QemuRunner): # # Walk the process tree from the process specified looking for a qemu-system. Return its [pid'cmd] # - ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command'], stdout=subprocess.PIPE).communicate()[0] + ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,pri,ni,command'], stdout=subprocess.PIPE).communicate()[0] processes = ps.decode("utf-8").split('\n') nfields = len(processes[0].split()) - 1 pids = {} diff --git a/meta/lib/oeqa/utils/sshcontrol.py b/meta/lib/oeqa/utils/sshcontrol.py index 49a07264c6..36c2ecb3db 100644 --- a/meta/lib/oeqa/utils/sshcontrol.py +++ b/meta/lib/oeqa/utils/sshcontrol.py @@ -23,7 +23,7 @@ class SSHProcess(object): "stdin": None, "shell": False, "bufsize": -1, - "preexec_fn": os.setsid, + "start_new_session": True, } self.options = dict(self.defaultopts) self.options.update(options) |