From 5c04b1ca1e989f569d5755a646734d01a0c56cae Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Fri, 4 Sep 2015 16:59:38 +0100 Subject: oeqa: Test failure/cleanup improvements Currently, if qemu segfaults, the tests merrily continue trying to execute which takes time for them to timeout and is a bit silly. Worse, no logs about the segfault are shown to the user, its silent! This patch tries to unravel the tangled web of issues and ensures that we: * install a SIGCHLD handler which tells the user qemu exited * check if qemu is running, if it isn't fail the test outright * don't leave processes behind in sshcontrol which would hold bitbake.lock and block shutdown Signed-off-by: Richard Purdie --- meta/lib/oeqa/oetest.py | 3 +++ meta/lib/oeqa/targetcontrol.py | 3 +++ meta/lib/oeqa/utils/qemurunner.py | 35 +++++++++++++++++++++++++---------- meta/lib/oeqa/utils/sshcontrol.py | 14 ++++++++++++-- 4 files changed, 43 insertions(+), 12 deletions(-) diff --git a/meta/lib/oeqa/oetest.py b/meta/lib/oeqa/oetest.py index 9724325083..ff62c30268 100644 --- a/meta/lib/oeqa/oetest.py +++ b/meta/lib/oeqa/oetest.py @@ -144,6 +144,9 @@ class oeRuntimeTest(oeTest): self.target = oeRuntimeTest.tc.target super(oeRuntimeTest, self).__init__(methodName) + def setUp(self): + self.assertTrue(self.target.check(), msg = "Qemu not running?") + def tearDown(self): # If a test fails or there is an exception if not exc_info() == (None, None, None): diff --git a/meta/lib/oeqa/targetcontrol.py b/meta/lib/oeqa/targetcontrol.py index 542e259112..edc0d01c1e 100644 --- a/meta/lib/oeqa/targetcontrol.py +++ b/meta/lib/oeqa/targetcontrol.py @@ -188,6 +188,9 @@ class QemuTarget(BaseTarget): bb.error("Qemu log output from %s:\n%s" % (self.qemulog, f.read())) raise bb.build.FuncFailed("%s - FAILED to start qemu - check the task log and the boot log" % self.pn) + def check(self): + return self.runner.is_alive() + def stop(self): self.runner.stop() self.connection = None diff --git a/meta/lib/oeqa/utils/qemurunner.py b/meta/lib/oeqa/utils/qemurunner.py index 3ad747a503..2d485379e8 100644 --- a/meta/lib/oeqa/utils/qemurunner.py +++ b/meta/lib/oeqa/utils/qemurunner.py @@ -64,6 +64,22 @@ class QemuRunner: with open(self.logfile, "a") as f: f.write("%s" % msg) + def getOutput(self, o): + import fcntl + fl = fcntl.fcntl(o, fcntl.F_GETFL) + fcntl.fcntl(o, fcntl.F_SETFL, fl | os.O_NONBLOCK) + return os.read(o.fileno(), 1000000) + + + def handleSIGCHLD(self, signum, frame): + if self.runqemu and self.runqemu.poll(): + if self.runqemu.returncode: + logger.info('runqemu exited with code %d' % self.runqemu.returncode) + logger.info("Output from runqemu:\n%s" % self.getOutput(self.runqemu.stdout)) + self.stop() + self._dump_host() + raise SystemExit + def start(self, qemuparams = None): if self.display: os.environ["DISPLAY"] = self.display @@ -98,11 +114,8 @@ class QemuRunner: if qemuparams: self.qemuparams = self.qemuparams[:-1] + " " + qemuparams + " " + '\"' - def getOutput(o): - import fcntl - fl = fcntl.fcntl(o, fcntl.F_GETFL) - fcntl.fcntl(o, fcntl.F_SETFL, fl | os.O_NONBLOCK) - return os.read(o.fileno(), 1000000) + self.origchldhandler = signal.getsignal(signal.SIGCHLD) + signal.signal(signal.SIGCHLD, self.handleSIGCHLD) launch_cmd = 'runqemu tcpserial=%s %s %s %s' % (self.serverport, self.machine, self.rootfs, self.qemuparams) # FIXME: We pass in stdin=subprocess.PIPE here to work around stty @@ -122,7 +135,7 @@ class QemuRunner: logger.info('runqemu exited with code %d' % self.runqemu.returncode) self._dump_host() self.stop() - logger.info("Output from runqemu:\n%s" % getOutput(output)) + logger.info("Output from runqemu:\n%s" % self.getOutput(output)) return False time.sleep(1) @@ -139,7 +152,7 @@ class QemuRunner: self.ip = ips[0] self.server_ip = ips[1] except IndexError, ValueError: - logger.info("Couldn't get ip from qemu process arguments! Here is the qemu command line used:\n%s\nand output from runqemu:\n%s" % (cmdline, getOutput(output))) + logger.info("Couldn't get ip from qemu process arguments! Here is the qemu command line used:\n%s\nand output from runqemu:\n%s" % (cmdline, self.getOutput(output))) self._dump_host() self.stop() return False @@ -154,7 +167,7 @@ class QemuRunner: logger.error("Didn't receive a console connection from qemu. " "Here is the qemu command line used:\n%s\nand " "output from runqemu:\n%s" % (cmdline, - getOutput(output))) + self.getOutput(output))) self.stop_thread() return False @@ -213,15 +226,15 @@ class QemuRunner: logger.info("Qemu pid didn't appeared in %s seconds" % self.runqemutime) self._dump_host() self.stop() - logger.info("Output from runqemu:\n%s" % getOutput(output)) + logger.info("Output from runqemu:\n%s" % self.getOutput(output)) return False return self.is_alive() def stop(self): - self.stop_thread() if self.runqemu: + signal.signal(signal.SIGCHLD, self.origchldhandler) logger.info("Sending SIGTERM to runqemu") try: os.killpg(self.runqemu.pid, signal.SIGTERM) @@ -255,6 +268,8 @@ class QemuRunner: return False def is_alive(self): + if not self.runqemu: + return False qemu_child = self.find_child(str(self.runqemu.pid)) if qemu_child: self.qemupid = qemu_child[0] diff --git a/meta/lib/oeqa/utils/sshcontrol.py b/meta/lib/oeqa/utils/sshcontrol.py index 6ed48badc8..00f5051053 100644 --- a/meta/lib/oeqa/utils/sshcontrol.py +++ b/meta/lib/oeqa/utils/sshcontrol.py @@ -42,7 +42,7 @@ class SSHProcess(object): with open(self.logfile, "a") as f: f.write("%s" % msg) - def run(self, command, timeout=None, logfile=None): + def _run(self, command, timeout=None, logfile=None): self.logfile = logfile self.starttime = time.time() output = '' @@ -79,8 +79,18 @@ class SSHProcess(object): self.status = self.process.wait() self.output = output.rstrip() - return (self.status, self.output) + def run(self, command, timeout=None, logfile=None): + try: + self._run(command, timeout, logfile) + except: + # Need to guard against a SystemExit or other exception occuring whilst running + # and ensure we don't leave a process behind. + if self.process.poll() is None: + self.process.kill() + self.status = self.process.wait() + raise + return (self.status, self.output) class SSHControl(object): def __init__(self, ip, logfile=None, timeout=300, user='root', port=None): -- cgit 1.2.3-korg