diff options
author | Mikko Rapeli <mikko.rapeli@linaro.org> | 2023-02-09 10:09:32 +0200 |
---|---|---|
committer | Steve Sakoman <steve@sakoman.com> | 2023-02-18 06:41:49 -1000 |
commit | d570e8f3d6ac05be200f5f4acbbd1a15a95fc122 (patch) | |
tree | f5b319bf6bf1588f1edfeae07f53a250d8ccc859 /meta | |
parent | 07db4a45918eb82503b0bbc7003258c0a03bd5fe (diff) | |
download | openembedded-core-d570e8f3d6ac05be200f5f4acbbd1a15a95fc122.tar.gz |
oeqa dump.py: add error counter and stop after 5 failures
If test target qemu machine hangs completely, dump_target() calls
over serial console are taking a long time to time out, possibly
for every failing ssh command execution and a lot of test cases,
and same with dump_monitor().
Instead of trying for ever, count errors and after 5 stop trying
to dump_target() and dump_monitor() completely.
These help to end testing earlier when a test target is completely
deadlocked and all ssh, serial and QMP communication with it are
failing.
Signed-off-by: Mikko Rapeli <mikko.rapeli@linaro.org>
Signed-off-by: Luca Ceresoli <luca.ceresoli@bootlin.com>
(cherry picked from commit d9ad0a055abba983c6cee1dca4d2f0a8a3c48782)
Signed-off-by: Steve Sakoman <steve@sakoman.com>
Diffstat (limited to 'meta')
-rw-r--r-- | meta/lib/oeqa/utils/dump.py | 23 |
1 files changed, 21 insertions, 2 deletions
diff --git a/meta/lib/oeqa/utils/dump.py b/meta/lib/oeqa/utils/dump.py index bcee03b576..d420b497f9 100644 --- a/meta/lib/oeqa/utils/dump.py +++ b/meta/lib/oeqa/utils/dump.py @@ -93,37 +93,55 @@ class HostDumper(BaseDumper): self._write_dump(cmd.split()[0], result.output) class TargetDumper(BaseDumper): - """ Class to get dumps from target, it only works with QemuRunner """ + """ Class to get dumps from target, it only works with QemuRunner. + Will give up permanently after 5 errors from running commands over + serial console. This helps to end testing when target is really dead, hanging + or unresponsive. + """ def __init__(self, cmds, parent_dir, runner): super(TargetDumper, self).__init__(cmds, parent_dir) self.runner = runner + self.errors = 0 def dump_target(self, dump_dir=""): + if self.errors >= 5: + print("Too many errors when dumping data from target, assuming it is dead! Will not dump data anymore!") + return if dump_dir: self.dump_dir = dump_dir for cmd in self.cmds: # We can continue with the testing if serial commands fail try: (status, output) = self.runner.run_serial(cmd) + if status == 0: + self.errors = self.errors + 1 self._write_dump(cmd.split()[0], output) except: + self.errors = self.errors + 1 print("Tried to dump info from target but " "serial console failed") print("Failed CMD: %s" % (cmd)) class MonitorDumper(BaseDumper): - """ Class to get dumps via the Qemu Monitor, it only works with QemuRunner """ + """ Class to get dumps via the Qemu Monitor, it only works with QemuRunner + Will stop completely if there are more than 5 errors when dumping monitor data. + This helps to end testing when target is really dead, hanging or unresponsive. + """ def __init__(self, cmds, parent_dir, runner): super(MonitorDumper, self).__init__(cmds, parent_dir) self.runner = runner + self.errors = 0 def dump_monitor(self, dump_dir=""): if self.runner is None: return if dump_dir: self.dump_dir = dump_dir + if self.errors >= 5: + print("Too many errors when dumping data from qemu monitor, assuming it is dead! Will not dump data anymore!") + return for cmd in self.cmds: cmd_name = cmd.split()[0] try: @@ -137,4 +155,5 @@ class MonitorDumper(BaseDumper): output = self.runner.run_monitor(cmd_name) self._write_dump(cmd_name, output) except Exception as e: + self.errors = self.errors + 1 print("Failed to dump QMP CMD: %s with\nException: %s" % (cmd_name, e)) |