diff options
Diffstat (limited to 'meta/lib')
| -rw-r--r-- | meta/lib/oe/package_manager.py | 36 | ||||
| -rw-r--r-- | meta/lib/oe/sdk.py | 85 | ||||
| -rw-r--r-- | meta/lib/oe/sstatesig.py | 6 | ||||
| -rw-r--r-- | meta/lib/oe/terminal.py | 2 | ||||
| -rw-r--r-- | meta/lib/oeqa/core/target/qemu.py | 2 | ||||
| -rw-r--r-- | meta/lib/oeqa/runtime/cases/buildcpio.py | 3 | ||||
| -rw-r--r-- | meta/lib/oeqa/selftest/runqemu.py | 5 | ||||
| -rw-r--r-- | meta/lib/oeqa/targetcontrol.py | 52 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/commands.py | 25 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/qemurunner.py | 362 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/qemutinyrunner.py | 3 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/sshcontrol.py | 9 |
12 files changed, 315 insertions, 275 deletions
diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py index 518cf8dbe3..1a2914fedc 100644 --- a/meta/lib/oe/package_manager.py +++ b/meta/lib/oe/package_manager.py @@ -370,6 +370,29 @@ class PackageManager(object, metaclass=ABCMeta): pass """ + Install all packages that match a glob. + """ + def install_glob(self, globs, sdk=False): + # TODO don't have sdk here but have a property on the superclass + # (and respect in install_complementary) + if sdk: + pkgdatadir = self.d.expand("${TMPDIR}/pkgdata/${SDK_SYS}") + else: + pkgdatadir = self.d.getVar("PKGDATA_DIR") + + try: + bb.note("Installing globbed packages...") + cmd = ["oe-pkgdata-util", "-p", pkgdatadir, "list-pkgs", globs] + pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode("utf-8") + self.install(pkgs.split(), attempt_only=True) + except subprocess.CalledProcessError as e: + # Return code 1 means no packages matched + if e.returncode != 1: + bb.fatal("Could not compute globbed packages list. Command " + "'%s' returned %d:\n%s" % + (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) + + """ Install complementary packages based upon the list of currently installed packages e.g. locales, *-dev, *-dbg, etc. This will only attempt to install these packages, if they don't exist then no error will occur. Note: every @@ -401,7 +424,7 @@ class PackageManager(object, metaclass=ABCMeta): installed_pkgs.write(output) installed_pkgs.flush() - cmd = [bb.utils.which(os.getenv('PATH'), "oe-pkgdata-util"), + cmd = ["oe-pkgdata-util", "-p", self.d.getVar('PKGDATA_DIR'), "glob", installed_pkgs.name, globs] exclude = self.d.getVar('PACKAGE_EXCLUDE_COMPLEMENTARY') @@ -411,11 +434,11 @@ class PackageManager(object, metaclass=ABCMeta): bb.note("Installing complementary packages ...") bb.note('Running %s' % cmd) complementary_pkgs = subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode("utf-8") + self.install(complementary_pkgs.split(), attempt_only=True) except subprocess.CalledProcessError as e: bb.fatal("Could not compute complementary packages list. Command " "'%s' returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) - self.install(complementary_pkgs.split(), attempt_only=True) def deploy_dir_lock(self): if self.deploy_dir is None: @@ -461,7 +484,8 @@ class RpmPM(PackageManager): task_name='target', providename=None, arch_var=None, - os_var=None): + os_var=None, + rpm_repo_workdir="oe-rootfs-repo"): super(RpmPM, self).__init__(d) self.target_rootfs = target_rootfs self.target_vendor = target_vendor @@ -475,7 +499,7 @@ class RpmPM(PackageManager): else: self.primary_arch = self.d.getVar('MACHINE_ARCH') - self.rpm_repo_dir = oe.path.join(self.d.getVar('WORKDIR'), "oe-rootfs-repo") + self.rpm_repo_dir = oe.path.join(self.d.getVar('WORKDIR'), rpm_repo_workdir) bb.utils.mkdirhier(self.rpm_repo_dir) oe.path.symlink(self.d.getVar('DEPLOY_DIR_RPM'), oe.path.join(self.rpm_repo_dir, "rpm"), True) @@ -1054,7 +1078,7 @@ class OpkgPM(OpkgDpkgPM): output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8") bb.note(output) except subprocess.CalledProcessError as e: - (bb.fatal, bb.note)[attempt_only]("Unable to install packages. " + (bb.fatal, bb.warn)[attempt_only]("Unable to install packages. " "Command '%s' returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) @@ -1353,7 +1377,7 @@ class DpkgPM(OpkgDpkgPM): bb.note("Installing the following packages: %s" % ' '.join(pkgs)) subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: - (bb.fatal, bb.note)[attempt_only]("Unable to install packages. " + (bb.fatal, bb.warn)[attempt_only]("Unable to install packages. " "Command '%s' returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) diff --git a/meta/lib/oe/sdk.py b/meta/lib/oe/sdk.py index deb823b6ec..f89382cd03 100644 --- a/meta/lib/oe/sdk.py +++ b/meta/lib/oe/sdk.py @@ -7,6 +7,51 @@ import shutil import glob import traceback +def generate_locale_archive(d, rootfs): + # Pretty sure we don't need this for SDK archive generation but + # keeping it to be safe... + target_arch = d.getVar('SDK_ARCH') + locale_arch_options = { \ + "arm": ["--uint32-align=4", "--little-endian"], + "armeb": ["--uint32-align=4", "--big-endian"], + "aarch64": ["--uint32-align=4", "--little-endian"], + "aarch64_be": ["--uint32-align=4", "--big-endian"], + "sh4": ["--uint32-align=4", "--big-endian"], + "powerpc": ["--uint32-align=4", "--big-endian"], + "powerpc64": ["--uint32-align=4", "--big-endian"], + "mips": ["--uint32-align=4", "--big-endian"], + "mipsisa32r6": ["--uint32-align=4", "--big-endian"], + "mips64": ["--uint32-align=4", "--big-endian"], + "mipsisa64r6": ["--uint32-align=4", "--big-endian"], + "mipsel": ["--uint32-align=4", "--little-endian"], + "mipsisa32r6el": ["--uint32-align=4", "--little-endian"], + "mips64el": ["--uint32-align=4", "--little-endian"], + "mipsisa64r6el": ["--uint32-align=4", "--little-endian"], + "i586": ["--uint32-align=4", "--little-endian"], + "i686": ["--uint32-align=4", "--little-endian"], + "x86_64": ["--uint32-align=4", "--little-endian"] + } + if target_arch in locale_arch_options: + arch_options = locale_arch_options[target_arch] + else: + bb.error("locale_arch_options not found for target_arch=" + target_arch) + bb.fatal("unknown arch:" + target_arch + " for locale_arch_options") + + localedir = oe.path.join(rootfs, d.getVar("libdir_nativesdk"), "locale") + # Need to set this so cross-localedef knows where the archive is + env = dict(os.environ) + env["LOCALEARCHIVE"] = oe.path.join(localedir, "locale-archive") + + for name in os.listdir(localedir): + path = os.path.join(localedir, name) + if os.path.isdir(path): + try: + cmd = ["cross-localedef", "--verbose"] + cmd += arch_options + cmd += ["--add-to-archive", path] + subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT) + except Exception as e: + bb.fatal("Cannot create locale archive: %s" % e.output) class Sdk(object, metaclass=ABCMeta): def __init__(self, d, manifest_dir): @@ -84,8 +129,32 @@ class Sdk(object, metaclass=ABCMeta): bb.debug(1, "printing the stack trace\n %s" %traceback.format_exc()) bb.warn("cannot remove SDK dir: %s" % path) + def install_locales(self, pm): + # This is only relevant for glibc + if self.d.getVar("TCLIBC") != "glibc": + return + + linguas = self.d.getVar("SDKIMAGE_LINGUAS") + if linguas: + import fnmatch + # Install the binary locales + if linguas == "all": + pm.install_glob("nativesdk-glibc-binary-localedata-*.utf-8", sdk=True) + else: + for lang in linguas.split(): + pm.install("nativesdk-glibc-binary-localedata-%s.utf-8" % lang) + # Generate a locale archive of them + generate_locale_archive(self.d, oe.path.join(self.sdk_host_sysroot, self.sdk_native_path)) + # And now delete the binary locales + pkgs = fnmatch.filter(pm.list_installed(), "nativesdk-glibc-binary-localedata-*.utf-8") + pm.remove(pkgs, with_dependencies=False) + else: + # No linguas so do nothing + pass + + class RpmSdk(Sdk): - def __init__(self, d, manifest_dir=None): + def __init__(self, d, manifest_dir=None, rpm_workdir="oe-sdk-repo"): super(RpmSdk, self).__init__(d, manifest_dir) self.target_manifest = RpmManifest(d, self.manifest_dir, @@ -100,11 +169,17 @@ class RpmSdk(Sdk): 'pkgconfig' ] + rpm_repo_workdir = "oe-sdk-repo" + if "sdk_ext" in d.getVar("BB_RUNTASK"): + rpm_repo_workdir = "oe-sdk-ext-repo" + + self.target_pm = RpmPM(d, self.sdk_target_sysroot, self.d.getVar('TARGET_VENDOR'), 'target', - target_providename + target_providename, + rpm_repo_workdir=rpm_repo_workdir ) sdk_providename = ['/bin/sh', @@ -122,7 +197,8 @@ class RpmSdk(Sdk): 'host', sdk_providename, "SDK_PACKAGE_ARCHS", - "SDK_OS" + "SDK_OS", + rpm_repo_workdir=rpm_repo_workdir ) def _populate_sysroot(self, pm, manifest): @@ -157,6 +233,7 @@ class RpmSdk(Sdk): bb.note("Installing NATIVESDK packages") self._populate_sysroot(self.host_pm, self.host_manifest) + self.install_locales(self.host_pm) execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND")) @@ -238,6 +315,7 @@ class OpkgSdk(Sdk): bb.note("Installing NATIVESDK packages") self._populate_sysroot(self.host_pm, self.host_manifest) + self.install_locales(self.host_pm) execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND")) @@ -322,6 +400,7 @@ class DpkgSdk(Sdk): bb.note("Installing NATIVESDK packages") self._populate_sysroot(self.host_pm, self.host_manifest) + self.install_locales(self.host_pm) execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND")) diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py index f087a019e1..b8dd4c869e 100644 --- a/meta/lib/oe/sstatesig.py +++ b/meta/lib/oe/sstatesig.py @@ -20,8 +20,12 @@ def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCache): def isImage(fn): return "/image.bbclass" in " ".join(dataCache.inherits[fn]) - # Always include our own inter-task dependencies + # (Almost) always include our own inter-task dependencies. + # The exception is the special do_kernel_configme->do_unpack_and_patch + # dependency from archiver.bbclass. if recipename == depname: + if task == "do_kernel_configme" and dep.endswith(".do_unpack_and_patch"): + return False return True # Quilt (patch application) changing isn't likely to affect anything diff --git a/meta/lib/oe/terminal.py b/meta/lib/oe/terminal.py index 89ddb4671a..2f18ec0287 100644 --- a/meta/lib/oe/terminal.py +++ b/meta/lib/oe/terminal.py @@ -224,7 +224,7 @@ def spawn(name, sh_cmd, title=None, env=None, d=None): import time pidfile = tempfile.NamedTemporaryFile(delete = False).name try: - sh_cmd = "oe-gnome-terminal-phonehome " + pidfile + " " + sh_cmd + sh_cmd = bb.utils.which(os.getenv('PATH'), "oe-gnome-terminal-phonehome") + " " + pidfile + " " + sh_cmd pipe = terminal(sh_cmd, title, env, d) output = pipe.communicate()[0] if output: diff --git a/meta/lib/oeqa/core/target/qemu.py b/meta/lib/oeqa/core/target/qemu.py index 2dc521c216..d359bf9fe3 100644 --- a/meta/lib/oeqa/core/target/qemu.py +++ b/meta/lib/oeqa/core/target/qemu.py @@ -31,7 +31,7 @@ class OEQemuTarget(OESSHTarget): deploy_dir_image=dir_image, display=display, logfile=bootlog, boottime=boottime, use_kvm=kvm, dump_dir=dump_dir, - dump_host_cmds=dump_host_cmds) + dump_host_cmds=dump_host_cmds, logger=logger) def start(self, params=None, extra_bootparams=None): if self.runner.start(params, extra_bootparams=extra_bootparams): diff --git a/meta/lib/oeqa/runtime/cases/buildcpio.py b/meta/lib/oeqa/runtime/cases/buildcpio.py index 59edc9c2c1..79b22d04dd 100644 --- a/meta/lib/oeqa/runtime/cases/buildcpio.py +++ b/meta/lib/oeqa/runtime/cases/buildcpio.py @@ -9,8 +9,7 @@ class BuildCpioTest(OERuntimeTestCase): @classmethod def setUpClass(cls): - uri = 'https://ftp.gnu.org/gnu/cpio' - uri = '%s/cpio-2.12.tar.bz2' % uri + uri = 'https://downloads.yoctoproject.org/mirror/sources/cpio-2.12.tar.gz' cls.project = TargetBuildProject(cls.tc.target, uri, dl_dir = cls.tc.td['DL_DIR']) diff --git a/meta/lib/oeqa/selftest/runqemu.py b/meta/lib/oeqa/selftest/runqemu.py index 58c6f96f98..89ee780115 100644 --- a/meta/lib/oeqa/selftest/runqemu.py +++ b/meta/lib/oeqa/selftest/runqemu.py @@ -3,7 +3,6 @@ # import re -import logging from oeqa.selftest.base import oeSelfTest from oeqa.utils.commands import bitbake, runqemu, get_bb_var @@ -21,10 +20,6 @@ class RunqemuTests(oeSelfTest): self.fstypes = "ext4 iso hddimg vmdk qcow2 vdi" self.cmd_common = "runqemu nographic" - # Avoid emit the same record multiple times. - mainlogger = logging.getLogger("BitBake.Main") - mainlogger.propagate = False - self.write_config( """ MACHINE = "%s" diff --git a/meta/lib/oeqa/targetcontrol.py b/meta/lib/oeqa/targetcontrol.py index 3255e3a5c6..f63936c3ec 100644 --- a/meta/lib/oeqa/targetcontrol.py +++ b/meta/lib/oeqa/targetcontrol.py @@ -18,44 +18,18 @@ from oeqa.utils.dump import TargetDumper from oeqa.controllers.testtargetloader import TestTargetLoader from abc import ABCMeta, abstractmethod -logger = logging.getLogger('BitBake.QemuRunner') - -def get_target_controller(d): - testtarget = d.getVar("TEST_TARGET") - # old, simple names - if testtarget == "qemu": - return QemuTarget(d) - elif testtarget == "simpleremote": - return SimpleRemoteTarget(d) - else: - # use the class name - try: - # is it a core class defined here? - controller = getattr(sys.modules[__name__], testtarget) - except AttributeError: - # nope, perhaps a layer defined one - try: - bbpath = d.getVar("BBPATH").split(':') - testtargetloader = TestTargetLoader() - controller = testtargetloader.get_controller_module(testtarget, bbpath) - except ImportError as e: - bb.fatal("Failed to import {0} from available controller modules:\n{1}".format(testtarget,traceback.format_exc())) - except AttributeError as e: - bb.fatal("Invalid TEST_TARGET - " + str(e)) - return controller(d) - - class BaseTarget(object, metaclass=ABCMeta): supported_image_fstypes = [] - def __init__(self, d): + def __init__(self, d, logger): self.connection = None self.ip = None self.server_ip = None self.datetime = d.getVar('DATETIME') self.testdir = d.getVar("TEST_LOG_DIR") self.pn = d.getVar("PN") + self.logger = logger @abstractmethod def deploy(self): @@ -65,7 +39,7 @@ class BaseTarget(object, metaclass=ABCMeta): if os.path.islink(sshloglink): os.unlink(sshloglink) os.symlink(self.sshlog, sshloglink) - logger.info("SSH log file: %s" % self.sshlog) + self.logger.info("SSH log file: %s" % self.sshlog) @abstractmethod def start(self, params=None, ssh=True, extra_bootparams=None): @@ -115,9 +89,9 @@ class QemuTarget(BaseTarget): supported_image_fstypes = ['ext3', 'ext4', 'cpio.gz', 'wic'] - def __init__(self, d, image_fstype=None): + def __init__(self, d, logger, image_fstype=None): - super(QemuTarget, self).__init__(d) + super(QemuTarget, self).__init__(d, logger) self.rootfs = '' self.kernel = '' @@ -145,7 +119,7 @@ class QemuTarget(BaseTarget): self.qemurunnerlog = os.path.join(self.testdir, 'qemurunner_log.%s' % self.datetime) loggerhandler = logging.FileHandler(self.qemurunnerlog) loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) - logger.addHandler(loggerhandler) + self.logger.addHandler(loggerhandler) oe.path.symlink(os.path.basename(self.qemurunnerlog), os.path.join(self.testdir, 'qemurunner_log'), force=True) if d.getVar("DISTRO") == "poky-tiny": @@ -156,7 +130,8 @@ class QemuTarget(BaseTarget): display = d.getVar("BB_ORIGENV", False).getVar("DISPLAY"), logfile = self.qemulog, kernel = self.kernel, - boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT"))) + boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT")), + logger = logger) else: self.runner = QemuRunner(machine=d.getVar("MACHINE"), rootfs=self.rootfs, @@ -167,7 +142,8 @@ class QemuTarget(BaseTarget): boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT")), use_kvm = use_kvm, dump_dir = dump_dir, - dump_host_cmds = d.getVar("testimage_dump_host")) + dump_host_cmds = d.getVar("testimage_dump_host"), + logger = logger) self.target_dumper = TargetDumper(dump_target_cmds, dump_dir, self.runner) @@ -179,8 +155,8 @@ class QemuTarget(BaseTarget): os.unlink(qemuloglink) os.symlink(self.qemulog, qemuloglink) - logger.info("rootfs file: %s" % self.rootfs) - logger.info("Qemu log file: %s" % self.qemulog) + self.logger.info("rootfs file: %s" % self.rootfs) + self.logger.info("Qemu log file: %s" % self.qemulog) super(QemuTarget, self).deploy() def start(self, params=None, ssh=True, extra_bootparams='', runqemuparams='', launch_cmd='', discard_writes=True): @@ -232,14 +208,14 @@ class SimpleRemoteTarget(BaseTarget): self.port = addr.split(":")[1] except IndexError: self.port = None - logger.info("Target IP: %s" % self.ip) + self.logger.info("Target IP: %s" % self.ip) self.server_ip = d.getVar("TEST_SERVER_IP") if not self.server_ip: try: self.server_ip = subprocess.check_output(['ip', 'route', 'get', self.ip ]).split("\n")[0].split()[-1] except Exception as e: bb.fatal("Failed to determine the host IP address (alternatively you can set TEST_SERVER_IP with the IP address of this machine): %s" % e) - logger.info("Server IP: %s" % self.server_ip) + self.logger.info("Server IP: %s" % self.server_ip) def deploy(self): super(SimpleRemoteTarget, self).deploy() diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py index 57286fcb10..ed925a4d92 100644 --- a/meta/lib/oeqa/utils/commands.py +++ b/meta/lib/oeqa/utils/commands.py @@ -233,6 +233,12 @@ def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None, import bb.tinfoil import bb.build + # Need a non-'BitBake' logger to capture the runner output + targetlogger = logging.getLogger('TargetRunner') + targetlogger.setLevel(logging.DEBUG) + handler = logging.StreamHandler(sys.stdout) + targetlogger.addHandler(handler) + tinfoil = bb.tinfoil.Tinfoil() tinfoil.prepare(config_only=False, quiet=True) try: @@ -250,31 +256,15 @@ def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None, for key, value in overrides.items(): recipedata.setVar(key, value) - # The QemuRunner log is saved out, but we need to ensure it is at the right - # log level (and then ensure that since it's a child of the BitBake logger, - # we disable propagation so we don't then see the log events on the console) - logger = logging.getLogger('BitBake.QemuRunner') - logger.setLevel(logging.DEBUG) - logger.propagate = False logdir = recipedata.getVar("TEST_LOG_DIR") - qemu = oeqa.targetcontrol.QemuTarget(recipedata, image_fstype) + qemu = oeqa.targetcontrol.QemuTarget(recipedata, targetlogger, image_fstype) finally: # We need to shut down tinfoil early here in case we actually want # to run tinfoil-using utilities with the running QEMU instance. # Luckily QemuTarget doesn't need it after the constructor. tinfoil.shutdown() - # Setup bitbake logger as console handler is removed by tinfoil.shutdown - bblogger = logging.getLogger('BitBake') - bblogger.setLevel(logging.INFO) - console = logging.StreamHandler(sys.stdout) - bbformat = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") - if sys.stdout.isatty(): - bbformat.enable_color() - console.setFormatter(bbformat) - bblogger.addHandler(console) - try: qemu.deploy() try: @@ -289,6 +279,7 @@ def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None, qemu.stop() except: pass + targetlogger.removeHandler(handler) def updateEnv(env_file): """ diff --git a/meta/lib/oeqa/utils/qemurunner.py b/meta/lib/oeqa/utils/qemurunner.py index ba44b96f53..360aa3fc37 100644 --- a/meta/lib/oeqa/utils/qemurunner.py +++ b/meta/lib/oeqa/utils/qemurunner.py @@ -17,11 +17,8 @@ import errno import string import threading import codecs -from oeqa.utils.dump import HostDumper - import logging -logger = logging.getLogger("BitBake.QemuRunner") -logger.addHandler(logging.StreamHandler()) +from oeqa.utils.dump import HostDumper # Get Unicode non printable control chars control_range = list(range(0,32))+list(range(127,160)) @@ -31,7 +28,7 @@ re_control_char = re.compile('[%s]' % re.escape("".join(control_chars))) class QemuRunner: - def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds, use_kvm): + def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds, use_kvm, logger): # Popen object for runqemu self.runqemu = None @@ -54,10 +51,14 @@ class QemuRunner: self.logged = False self.thread = None self.use_kvm = use_kvm + self.msg = '' - self.runqemutime = 60 + self.runqemutime = 120 + self.qemu_pidfile = 'pidfile_'+str(os.getpid()) self.host_dumper = HostDumper(dump_host_cmds, dump_dir) + self.logger = logger + def create_socket(self): try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -65,7 +66,7 @@ class QemuRunner: sock.bind(("127.0.0.1",0)) sock.listen(2) port = sock.getsockname()[1] - logger.info("Created listening socket for qemu serial console on: 127.0.0.1:%s" % port) + self.logger.debug("Created listening socket for qemu serial console on: 127.0.0.1:%s" % port) return (sock, port) except socket.error: @@ -78,6 +79,7 @@ class QemuRunner: # because is possible to have control characters msg = msg.decode("utf-8", errors='ignore') msg = re_control_char.sub('', msg) + self.msg += msg with codecs.open(self.logfile, "a", encoding="utf-8") as f: f.write("%s" % msg) @@ -91,8 +93,8 @@ class QemuRunner: def handleSIGCHLD(self, signum, frame): if self.runqemu and self.runqemu.poll(): if self.runqemu.returncode: - logger.info('runqemu exited with code %d' % self.runqemu.returncode) - logger.info("Output from runqemu:\n%s" % self.getOutput(self.runqemu.stdout)) + self.logger.debug('runqemu exited with code %d' % self.runqemu.returncode) + self.logger.debug("Output from runqemu:\n%s" % self.getOutput(self.runqemu.stdout)) self.stop() self._dump_host() raise SystemExit @@ -104,15 +106,15 @@ class QemuRunner: # interact badly with screensavers. os.environ["QEMU_DONT_GRAB"] = "1" if not os.path.exists(self.rootfs): - logger.error("Invalid rootfs %s" % self.rootfs) + self.logger.error("Invalid rootfs %s" % self.rootfs) return False if not os.path.exists(self.tmpdir): - logger.error("Invalid TMPDIR path %s" % self.tmpdir) + self.logger.error("Invalid TMPDIR path %s" % self.tmpdir) return False else: os.environ["OE_TMPDIR"] = self.tmpdir if not os.path.exists(self.deploy_dir_image): - logger.error("Invalid DEPLOY_DIR_IMAGE path %s" % self.deploy_dir_image) + self.logger.error("Invalid DEPLOY_DIR_IMAGE path %s" % self.deploy_dir_image) return False else: os.environ["DEPLOY_DIR_IMAGE"] = self.deploy_dir_image @@ -120,10 +122,10 @@ class QemuRunner: if not launch_cmd: launch_cmd = 'runqemu %s %s ' % ('snapshot' if discard_writes else '', runqemuparams) if self.use_kvm: - logger.info('Using kvm for runqemu') + self.logger.debug('Using kvm for runqemu') launch_cmd += ' kvm' else: - logger.info('Not using kvm for runqemu') + self.logger.debug('Not using kvm for runqemu') if not self.display: launch_cmd += ' nographic' launch_cmd += ' %s %s' % (self.machine, self.rootfs) @@ -135,14 +137,18 @@ class QemuRunner: threadsock, threadport = self.create_socket() self.server_socket, self.serverport = self.create_socket() except socket.error as msg: - logger.error("Failed to create listening socket: %s" % msg[1]) + self.logger.error("Failed to create listening socket: %s" % msg[1]) return False bootparams = 'console=tty1 console=ttyS0,115200n8 printk.time=1' if extra_bootparams: bootparams = bootparams + ' ' + extra_bootparams - self.qemuparams = 'bootparams="{0}" qemuparams="-serial tcp:127.0.0.1:{1}"'.format(bootparams, threadport) + # Ask QEMU to store the QEMU process PID in file, this way we don't have to parse running processes + # and analyze descendents in order to determine it. + if os.path.exists(self.qemu_pidfile): + os.remove(self.qemu_pidfile) + self.qemuparams = 'bootparams="{0}" qemuparams="-serial tcp:127.0.0.1:{1} -pidfile {2}"'.format(bootparams, threadport, self.qemu_pidfile) if qemuparams: self.qemuparams = self.qemuparams[:-1] + " " + qemuparams + " " + '\"' @@ -151,7 +157,7 @@ class QemuRunner: self.origchldhandler = signal.getsignal(signal.SIGCHLD) signal.signal(signal.SIGCHLD, self.handleSIGCHLD) - logger.info('launchcmd=%s'%(launch_cmd)) + self.logger.debug('launchcmd=%s'%(launch_cmd)) # FIXME: We pass in stdin=subprocess.PIPE here to work around stty # blocking at the end of the runqemu script when using this within @@ -186,143 +192,149 @@ class QemuRunner: os.killpg(os.getpgid(self.runqemu.pid), signal.SIGTERM) sys.exit(0) - logger.info("runqemu started, pid is %s" % self.runqemu.pid) - logger.info("waiting at most %s seconds for qemu pid" % self.runqemutime) + self.logger.debug("runqemu started, pid is %s" % self.runqemu.pid) + self.logger.debug("waiting at most %s seconds for qemu pid" % self.runqemutime) endtime = time.time() + self.runqemutime while not self.is_alive() and time.time() < endtime: if self.runqemu.poll(): if self.runqemu.returncode: # No point waiting any longer - logger.info('runqemu exited with code %d' % self.runqemu.returncode) + self.logger.debug('runqemu exited with code %d' % self.runqemu.returncode) self._dump_host() self.stop() - logger.info("Output from runqemu:\n%s" % self.getOutput(output)) + self.logger.debug("Output from runqemu:\n%s" % self.getOutput(output)) return False - time.sleep(1) + time.sleep(0.5) + + if not self.is_alive(): + self.logger.error("Qemu pid didn't appear in %s seconds" % self.runqemutime) + # Dump all processes to help us to figure out what is going on... + ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command '], stdout=subprocess.PIPE).communicate()[0] + processes = ps.decode("utf-8") + self.logger.debug("Running processes:\n%s" % processes) + self._dump_host() + self.stop() + op = self.getOutput(output) + if op: + self.logger.error("Output from runqemu:\n%s" % op) + else: + self.logger.error("No output from runqemu.\n") + return False + # We are alive: qemu is running out = self.getOutput(output) netconf = False # network configuration is not required by default - if self.is_alive(): - logger.info("qemu started - qemu procces pid is %s" % self.qemupid) - if get_ip: - cmdline = '' - with open('/proc/%s/cmdline' % self.qemupid) as p: - cmdline = p.read() - # It is needed to sanitize the data received - # because is possible to have control characters - cmdline = re_control_char.sub('', cmdline) - try: - ips = re.findall("((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1]) - self.ip = ips[0] - self.server_ip = ips[1] - logger.info("qemu cmdline used:\n{}".format(cmdline)) - except (IndexError, ValueError): - # Try to get network configuration from runqemu output - match = re.match('.*Network configuration: ([0-9.]+)::([0-9.]+):([0-9.]+)$.*', - out, re.MULTILINE|re.DOTALL) - if match: - self.ip, self.server_ip, self.netmask = match.groups() - # network configuration is required as we couldn't get it - # from the runqemu command line, so qemu doesn't run kernel - # and guest networking is not configured - netconf = True - else: - logger.error("Couldn't get ip from qemu command line and runqemu output! " - "Here is the qemu command line used:\n%s\n" - "and output from runqemu:\n%s" % (cmdline, out)) - self._dump_host() - self.stop() - return False - - logger.info("Target IP: %s" % self.ip) - logger.info("Server IP: %s" % self.server_ip) - - self.thread = LoggingThread(self.log, threadsock, logger) - self.thread.start() - if not self.thread.connection_established.wait(self.boottime): - logger.error("Didn't receive a console connection from qemu. " - "Here is the qemu command line used:\n%s\nand " - "output from runqemu:\n%s" % (cmdline, out)) - self.stop_thread() - return False - - logger.info("Output from runqemu:\n%s", out) - logger.info("Waiting at most %d seconds for login banner" % self.boottime) - endtime = time.time() + self.boottime - socklist = [self.server_socket] - reachedlogin = False - stopread = False - qemusock = None - bootlog = '' - data = b'' - while time.time() < endtime and not stopread: - try: - sread, swrite, serror = select.select(socklist, [], [], 5) - except InterruptedError: - continue - for sock in sread: - if sock is self.server_socket: - qemusock, addr = self.server_socket.accept() - qemusock.setblocking(0) - socklist.append(qemusock) - socklist.remove(self.server_socket) - logger.info("Connection from %s:%s" % addr) - else: - data = data + sock.recv(1024) - if data: - try: - data = data.decode("utf-8", errors="surrogateescape") - bootlog += data - data = b'' - if re.search(".* login:", bootlog): - self.server_socket = qemusock - stopread = True - reachedlogin = True - logger.info("Reached login banner") - except UnicodeDecodeError: - continue - else: - socklist.remove(sock) - sock.close() - stopread = True + self.logger.debug("qemu started in %s seconds - qemu procces pid is %s" % (time.time() - (endtime - self.runqemutime), self.qemupid)) + if get_ip: + cmdline = '' + with open('/proc/%s/cmdline' % self.qemupid) as p: + cmdline = p.read() + # It is needed to sanitize the data received + # because is possible to have control characters + cmdline = re_control_char.sub(' ', cmdline) + try: + ips = re.findall("((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1]) + self.ip = ips[0] + self.server_ip = ips[1] + self.logger.debug("qemu cmdline used:\n{}".format(cmdline)) + except (IndexError, ValueError): + # Try to get network configuration from runqemu output + match = re.match('.*Network configuration: ([0-9.]+)::([0-9.]+):([0-9.]+)$.*', + out, re.MULTILINE|re.DOTALL) + if match: + self.ip, self.server_ip, self.netmask = match.groups() + # network configuration is required as we couldn't get it + # from the runqemu command line, so qemu doesn't run kernel + # and guest networking is not configured + netconf = True + else: + self.logger.error("Couldn't get ip from qemu command line and runqemu output! " + "Here is the qemu command line used:\n%s\n" + "and output from runqemu:\n%s" % (cmdline, out)) + self._dump_host() + self.stop() + return False - if not reachedlogin: - logger.info("Target didn't reached login boot in %d seconds" % self.boottime) - lines = "\n".join(bootlog.splitlines()[-25:]) - logger.info("Last 25 lines of text:\n%s" % lines) - logger.info("Check full boot log: %s" % self.logfile) - self._dump_host() - self.stop() - return False + self.logger.debug("Target IP: %s" % self.ip) + self.logger.debug("Server IP: %s" % self.server_ip) - # If we are not able to login the tests can continue + self.thread = LoggingThread(self.log, threadsock, self.logger) + self.thread.start() + if not self.thread.connection_established.wait(self.boottime): + self.logger.error("Didn't receive a console connection from qemu. " + "Here is the qemu command line used:\n%s\nand " + "output from runqemu:\n%s" % (cmdline, out)) + self.stop_thread() + return False + + self.logger.debug("Output from runqemu:\n%s", out) + self.logger.debug("Waiting at most %d seconds for login banner" % self.boottime) + endtime = time.time() + self.boottime + socklist = [self.server_socket] + reachedlogin = False + stopread = False + qemusock = None + bootlog = b'' + data = b'' + while time.time() < endtime and not stopread: try: - (status, output) = self.run_serial("root\n", raw=True) - if re.search("root@[a-zA-Z0-9\-]+:~#", output): - self.logged = True - logger.info("Logged as root in serial console") - if netconf: - # configure guest networking - cmd = "ifconfig eth0 %s netmask %s up\n" % (self.ip, self.netmask) - output = self.run_serial(cmd, raw=True)[1] - if re.search("root@[a-zA-Z0-9\-]+:~#", output): - logger.info("configured ip address %s", self.ip) - else: - logger.info("Couldn't configure guest networking") + sread, swrite, serror = select.select(socklist, [], [], 5) + except InterruptedError: + continue + for sock in sread: + if sock is self.server_socket: + qemusock, addr = self.server_socket.accept() + qemusock.setblocking(0) + socklist.append(qemusock) + socklist.remove(self.server_socket) + self.logger.debug("Connection from %s:%s" % addr) else: - logger.info("Couldn't login into serial console" - " as root using blank password") - except: - logger.info("Serial console failed while trying to login") - - else: - logger.info("Qemu pid didn't appeared in %s seconds" % self.runqemutime) + data = data + sock.recv(1024) + if data: + bootlog += data + data = b'' + if b' login:' in bootlog: + self.server_socket = qemusock + stopread = True + reachedlogin = True + self.logger.debug("Reached login banner") + else: + socklist.remove(sock) + sock.close() + stopread = True + + + if not reachedlogin: + self.logger.debug("Target didn't reached login boot in %d seconds" % self.boottime) + tail = lambda l: "\n".join(l.splitlines()[-25:]) + # in case bootlog is empty, use tail qemu log store at self.msg + lines = tail(bootlog if bootlog else self.msg) + self.logger.debug("Last 25 lines of text:\n%s" % lines) + self.logger.debug("Check full boot log: %s" % self.logfile) self._dump_host() self.stop() - logger.info("Output from runqemu:\n%s" % self.getOutput(output)) return False - return self.is_alive() + # If we are not able to login the tests can continue + try: + (status, output) = self.run_serial("root\n", raw=True) + if re.search("root@[a-zA-Z0-9\-]+:~#", output): + self.logged = True + self.logger.debug("Logged as root in serial console") + if netconf: + # configure guest networking + cmd = "ifconfig eth0 %s netmask %s up\n" % (self.ip, self.netmask) + output = self.run_serial(cmd, raw=True)[1] + if re.search("root@[a-zA-Z0-9\-]+:~#", output): + self.logger.debug("configured ip address %s", self.ip) + else: + self.logger.debug("Couldn't configure guest networking") + else: + self.logger.debug("Couldn't login into serial console" + " as root using blank password") + except: + self.logger.debug("Serial console failed while trying to login") + return True def stop(self): self.stop_thread() @@ -332,7 +344,7 @@ class QemuRunner: if self.runqemu: if hasattr(self, "monitorpid"): os.kill(self.monitorpid, signal.SIGKILL) - logger.info("Sending SIGTERM to runqemu") + self.logger.debug("Sending SIGTERM to runqemu") try: os.killpg(os.getpgid(self.runqemu.pid), signal.SIGTERM) except OSError as e: @@ -342,7 +354,7 @@ class QemuRunner: while self.runqemu.poll() is None and time.time() < endtime: time.sleep(1) if self.runqemu.poll() is None: - logger.info("Sending SIGKILL to runqemu") + self.logger.debug("Sending SIGKILL to runqemu") os.killpg(os.getpgid(self.runqemu.pid), signal.SIGKILL) self.runqemu = None if hasattr(self, 'server_socket') and self.server_socket: @@ -350,6 +362,8 @@ class QemuRunner: self.server_socket = None self.qemupid = None self.ip = None + if os.path.exists(self.qemu_pidfile): + os.remove(self.qemu_pidfile) def stop_qemu_system(self): if self.qemupid: @@ -357,7 +371,7 @@ class QemuRunner: # qemu-system behaves well and a SIGTERM is enough os.kill(self.qemupid, signal.SIGTERM) except ProcessLookupError as e: - logger.warn('qemu-system ended unexpectedly') + self.logger.warn('qemu-system ended unexpectedly') def stop_thread(self): if self.thread and self.thread.is_alive(): @@ -365,7 +379,7 @@ class QemuRunner: self.thread.join() def restart(self, qemuparams = None): - logger.info("Restarting qemu process") + self.logger.debug("Restarting qemu process") if self.runqemu.poll() is None: self.stop() if self.start(qemuparams): @@ -375,56 +389,16 @@ class QemuRunner: def is_alive(self): if not self.runqemu: return False - qemu_child = self.find_child(str(self.runqemu.pid)) - if qemu_child: - self.qemupid = qemu_child[0] - if os.path.exists("/proc/" + str(self.qemupid)): + if os.path.isfile(self.qemu_pidfile): + f = open(self.qemu_pidfile, 'r') + qemu_pid = f.read() + f.close() + qemupid = int(qemu_pid) + if os.path.exists("/proc/" + str(qemupid)): + self.qemupid = qemupid return True return False - def find_child(self,parent_pid): - # - # Walk the process tree from the process specified looking for a qemu-system. Return its [pid'cmd] - # - ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command'], stdout=subprocess.PIPE).communicate()[0] - processes = ps.decode("utf-8").split('\n') - nfields = len(processes[0].split()) - 1 - pids = {} - commands = {} - for row in processes[1:]: - data = row.split(None, nfields) - if len(data) != 3: - continue - if data[1] not in pids: - pids[data[1]] = [] - - pids[data[1]].append(data[0]) - commands[data[0]] = data[2] - - if parent_pid not in pids: - return [] - - parents = [] - newparents = pids[parent_pid] - while newparents: - next = [] - for p in newparents: - if p in pids: - for n in pids[p]: - if n not in parents and n not in next: - next.append(n) - if p not in parents: - parents.append(p) - newparents = next - #print("Children matching %s:" % str(parents)) - for p in parents: - # Need to be careful here since runqemu runs "ldd qemu-system-xxxx" - # Also, old versions of ldd (2.11) run "LD_XXXX qemu-system-xxxx" - basecmd = commands[p].split()[0] - basecmd = os.path.basename(basecmd) - if "qemu-system" in basecmd and "-serial tcp" in commands[p]: - return [int(p),commands[p]] - def run_serial(self, command, raw=False, timeout=5): # We assume target system have echo to get command status if not raw: @@ -474,7 +448,7 @@ class QemuRunner: def _dump_host(self): self.host_dumper.create_dir("qemu") - logger.warn("Qemu ended unexpectedly, dump data from host" + self.logger.warn("Qemu ended unexpectedly, dump data from host" " is in %s" % self.host_dumper.dump_dir) self.host_dumper.dump_host() @@ -503,17 +477,17 @@ class LoggingThread(threading.Thread): self.teardown() def run(self): - self.logger.info("Starting logging thread") + self.logger.debug("Starting logging thread") self.readpipe, self.writepipe = os.pipe() threading.Thread.run(self) def stop(self): - self.logger.info("Stopping logging thread") + self.logger.debug("Stopping logging thread") if self.running: os.write(self.writepipe, bytes("stop", "utf-8")) def teardown(self): - self.logger.info("Tearing down logging thread") + self.logger.debug("Tearing down logging thread") self.close_socket(self.serversock) if self.readsock is not None: @@ -531,7 +505,7 @@ class LoggingThread(threading.Thread): breakout = False self.running = True - self.logger.info("Starting thread event loop") + self.logger.debug("Starting thread event loop") while not breakout: events = poll.poll() for event in events: @@ -541,19 +515,19 @@ class LoggingThread(threading.Thread): # Event to stop the thread if self.readpipe == event[0]: - self.logger.info("Stop event received") + self.logger.debug("Stop event received") breakout = True break # A connection request was received elif self.serversock.fileno() == event[0]: - self.logger.info("Connection request received") + self.logger.debug("Connection request received") self.readsock, _ = self.serversock.accept() self.readsock.setblocking(0) poll.unregister(self.serversock.fileno()) poll.register(self.readsock.fileno(), event_read_mask) - self.logger.info("Setting connection established event") + self.logger.debug("Setting connection established event") self.connection_established.set() # Actual data to be logged diff --git a/meta/lib/oeqa/utils/qemutinyrunner.py b/meta/lib/oeqa/utils/qemutinyrunner.py index 1bf59007ff..63b5d1648b 100644 --- a/meta/lib/oeqa/utils/qemutinyrunner.py +++ b/meta/lib/oeqa/utils/qemutinyrunner.py @@ -17,7 +17,7 @@ from .qemurunner import QemuRunner class QemuTinyRunner(QemuRunner): - def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, kernel, boottime): + def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, kernel, boottime, logger): # Popen object for runqemu self.runqemu = None @@ -40,6 +40,7 @@ class QemuTinyRunner(QemuRunner): self.socketfile = "console.sock" self.server_socket = None self.kernel = kernel + self.logger = logger def create_socket(self): diff --git a/meta/lib/oeqa/utils/sshcontrol.py b/meta/lib/oeqa/utils/sshcontrol.py index 05d6502550..d292893c08 100644 --- a/meta/lib/oeqa/utils/sshcontrol.py +++ b/meta/lib/oeqa/utils/sshcontrol.py @@ -150,12 +150,9 @@ class SSHControl(object): def copy_to(self, localpath, remotepath): if os.path.islink(localpath): - link = os.readlink(localpath) - dst_dir, dst_base = os.path.split(remotepath) - return self.run("cd %s; ln -s %s %s" % (dst_dir, link, dst_base)) - else: - command = self.scp + [localpath, '%s@%s:%s' % (self.user, self.ip, remotepath)] - return self._internal_run(command, ignore_status=False) + localpath = os.path.dirname(localpath) + "/" + os.readlink(localpath) + command = self.scp + [localpath, '%s@%s:%s' % (self.user, self.ip, remotepath)] + return self._internal_run(command, ignore_status=False) def copy_from(self, remotepath, localpath): command = self.scp + ['%s@%s:%s' % (self.user, self.ip, remotepath), localpath] |