diff options
-rw-r--r-- | meta/classes/populate_sdk_ext.bbclass | 6 | ||||
-rw-r--r-- | meta/conf/distro/include/yocto-uninative.inc | 10 | ||||
-rw-r--r-- | meta/files/toolchain-shar-extract.sh | 11 | ||||
-rw-r--r-- | meta/lib/oe/utils.py | 2 | ||||
-rw-r--r-- | meta/lib/oeqa/core/utils/concurrencytest.py | 2 | ||||
-rw-r--r-- | meta/lib/oeqa/sdkext/testsdk.py | 7 | ||||
-rw-r--r-- | meta/recipes-devtools/createrepo-c/createrepo-c_git.bb | 2 | ||||
-rw-r--r-- | meta/recipes-devtools/rpm/rpm_4.14.2.bb | 2 | ||||
-rw-r--r-- | meta/recipes-kernel/linux/linux-yocto-rt_4.14.bb | 6 | ||||
-rw-r--r-- | meta/recipes-kernel/linux/linux-yocto-tiny_4.14.bb | 6 | ||||
-rw-r--r-- | meta/recipes-kernel/linux/linux-yocto_4.14.bb | 20 | ||||
-rw-r--r-- | meta/recipes-support/iso-codes/iso-codes_4.1.bb | 2 | ||||
-rwxr-xr-x | scripts/runqemu | 128 |
13 files changed, 125 insertions, 79 deletions
diff --git a/meta/classes/populate_sdk_ext.bbclass b/meta/classes/populate_sdk_ext.bbclass index 40b0375e0b..9bab54c6bd 100644 --- a/meta/classes/populate_sdk_ext.bbclass +++ b/meta/classes/populate_sdk_ext.bbclass @@ -574,8 +574,8 @@ sdk_ext_preinst() { exit 1 fi # The relocation script used by buildtools installer requires python - if ! command -v python > /dev/null; then - echo "ERROR: The installer requires python, please install it first" + if ! command -v python3 > /dev/null; then + echo "ERROR: The installer requires python3, please install it first" exit 1 fi missing_utils="" @@ -634,7 +634,7 @@ sdk_ext_postinst() { # current working directory when first ran, nor will it set $1 when # sourcing a script. That is why this has to look so ugly. LOGFILE="$target_sdk_dir/preparing_build_system.log" - sh -c ". buildtools/environment-setup* > $LOGFILE && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE && python $target_sdk_dir/ext-sdk-prepare.py $LOGFILE '${SDK_INSTALL_TARGETS}'" || { echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; } + sh -c ". buildtools/environment-setup* > $LOGFILE && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE && python3 $target_sdk_dir/ext-sdk-prepare.py $LOGFILE '${SDK_INSTALL_TARGETS}'" || { echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; } rm $target_sdk_dir/ext-sdk-prepare.py fi echo done diff --git a/meta/conf/distro/include/yocto-uninative.inc b/meta/conf/distro/include/yocto-uninative.inc index ad75d3e2a3..889695eae3 100644 --- a/meta/conf/distro/include/yocto-uninative.inc +++ b/meta/conf/distro/include/yocto-uninative.inc @@ -6,9 +6,9 @@ # to the distro running on the build machine. # -UNINATIVE_MAXGLIBCVERSION = "2.30" +UNINATIVE_MAXGLIBCVERSION = "2.31" -UNINATIVE_URL ?= "http://downloads.yoctoproject.org/releases/uninative/2.7/" -UNINATIVE_CHECKSUM[aarch64] ?= "e76a45886ee8a0b3904b761c17ac8ff91edf9811ee455f1832d10763ba794dfc" -UNINATIVE_CHECKSUM[i686] ?= "810d027dfb1c7675226afbcec07808770516c969ee7378f6d8240281083f8924" -UNINATIVE_CHECKSUM[x86_64] ?= "9498d8bba047499999a7310ac2576d0796461184965351a56f6d32c888a1f216" +UNINATIVE_URL ?= "http://downloads.yoctoproject.org/releases/uninative/2.8/" +UNINATIVE_CHECKSUM[aarch64] ?= "989187344bf9539b464fb7ed9c223e51f4bdb4c7a677d2c314e6fed393176efe" +UNINATIVE_CHECKSUM[i686] ?= "cc3e45bc8594488b407363e3fa9af5a099279dab2703c64342098719bd674990" +UNINATIVE_CHECKSUM[x86_64] ?= "a09922172c3a439105e0ae6b943daad2d83505b17da0aba97961ff433b8c21ab" diff --git a/meta/files/toolchain-shar-extract.sh b/meta/files/toolchain-shar-extract.sh index 9eabd62630..6c4d14a038 100644 --- a/meta/files/toolchain-shar-extract.sh +++ b/meta/files/toolchain-shar-extract.sh @@ -1,13 +1,8 @@ #!/bin/sh -[ -z "$ENVCLEANED" ] && exec /usr/bin/env -i ENVCLEANED=1 HOME="$HOME" \ - LC_ALL=en_US.UTF-8 \ - TERM=$TERM \ - ICECC_PATH="$ICECC_PATH" \ - http_proxy="$http_proxy" https_proxy="$https_proxy" ftp_proxy="$ftp_proxy" \ - no_proxy="$no_proxy" all_proxy="$all_proxy" GIT_PROXY_COMMAND="$GIT_PROXY_COMMAND" "$0" "$@" -[ -f /etc/environment ] && . /etc/environment -export PATH=`echo "$PATH" | sed -e 's/:\.//' -e 's/::/:/'` +export LC_ALL=en_US.UTF-8 +# Remove invalid PATH elements first (maybe from a previously setup toolchain now deleted +PATH=`python3 -c 'import os; print(":".join(e for e in os.environ["PATH"].split(":") if os.path.exists(e)))'` tweakpath () { case ":${PATH}:" in diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py index 8a584d6ddd..96ebc36b8b 100644 --- a/meta/lib/oe/utils.py +++ b/meta/lib/oe/utils.py @@ -370,7 +370,7 @@ def host_gcc_version(d, taskcontextonly=False): except subprocess.CalledProcessError as e: bb.fatal("Error running %s --version: %s" % (compiler, e.output.decode("utf-8"))) - match = re.match(".* (\d\.\d)\.\d.*", output.split('\n')[0]) + match = re.match(r".* (\d+\.\d+)\.\d+.*", output.split('\n')[0]) if not match: bb.fatal("Can't get compiler version from %s --version output" % compiler) diff --git a/meta/lib/oeqa/core/utils/concurrencytest.py b/meta/lib/oeqa/core/utils/concurrencytest.py index 1a58d35ba0..3e0e5d770c 100644 --- a/meta/lib/oeqa/core/utils/concurrencytest.py +++ b/meta/lib/oeqa/core/utils/concurrencytest.py @@ -194,7 +194,7 @@ def fork_for_tests(concurrency_num, suite): oe.path.copytree(selftestdir, newselftestdir) for e in os.environ: - if builddir in os.environ[e]: + if builddir + "/" in os.environ[e] or os.environ[e].endswith(builddir): os.environ[e] = os.environ[e].replace(builddir, newbuilddir) subprocess.check_output("git init; git add *; git commit -a -m 'initial'", cwd=newselftestdir, shell=True) diff --git a/meta/lib/oeqa/sdkext/testsdk.py b/meta/lib/oeqa/sdkext/testsdk.py index 57b2e0e03f..8ec5262d56 100644 --- a/meta/lib/oeqa/sdkext/testsdk.py +++ b/meta/lib/oeqa/sdkext/testsdk.py @@ -22,11 +22,8 @@ class TestSDKExt(TestSDKBase): subprocesstweak.errors_have_output() - # extensible sdk can be contaminated if native programs are - # in PATH, i.e. use perl-native instead of eSDK one. - paths_to_avoid = [d.getVar('STAGING_DIR'), - d.getVar('BASE_WORKDIR')] - os.environ['PATH'] = avoid_paths_in_environ(paths_to_avoid) + # We need the original PATH for testing the eSDK, not with our manipulations + os.environ['PATH'] = d.getVar("BB_ORIGENV", False).getVar("PATH") tcname = d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.sh") if not os.path.exists(tcname): diff --git a/meta/recipes-devtools/createrepo-c/createrepo-c_git.bb b/meta/recipes-devtools/createrepo-c/createrepo-c_git.bb index 08766b3124..0c5a5f8ce1 100644 --- a/meta/recipes-devtools/createrepo-c/createrepo-c_git.bb +++ b/meta/recipes-devtools/createrepo-c/createrepo-c_git.bb @@ -16,7 +16,7 @@ SRCREV = "434a40babdde57ab405e9a1085a30e9575ea0c92" S = "${WORKDIR}/git" DEPENDS = "expat curl glib-2.0 libxml2 openssl bzip2 zlib file sqlite3 xz rpm" -DEPENDS_append_class-native = " file-replacement-native" +DEPENDS_append_class-native = " file-replacement-native " inherit cmake pkgconfig bash-completion distutils3-base diff --git a/meta/recipes-devtools/rpm/rpm_4.14.2.bb b/meta/recipes-devtools/rpm/rpm_4.14.2.bb index 75ef3f4d03..82519da961 100644 --- a/meta/recipes-devtools/rpm/rpm_4.14.2.bb +++ b/meta/recipes-devtools/rpm/rpm_4.14.2.bb @@ -65,7 +65,7 @@ EXTRA_OECONF_append_libc-musl = " --disable-nls" # # Disable dbus for native, so that rpm doesn't attempt to inhibit shutdown via session dbus even when plugins support is enabled. # Also disable plugins by default for native. -EXTRA_OECONF_append_class-native = " --sysconfdir=/etc --localstatedir=/var --disable-plugins" +EXTRA_OECONF_append_class-native = " --sysconfdir=/etc --localstatedir=/var --disable-plugins " EXTRA_OECONF_append_class-nativesdk = " --sysconfdir=/etc --localstatedir=/var --disable-plugins" BBCLASSEXTEND = "native nativesdk" diff --git a/meta/recipes-kernel/linux/linux-yocto-rt_4.14.bb b/meta/recipes-kernel/linux/linux-yocto-rt_4.14.bb index 0ed290088a..de6f5c98bf 100644 --- a/meta/recipes-kernel/linux/linux-yocto-rt_4.14.bb +++ b/meta/recipes-kernel/linux/linux-yocto-rt_4.14.bb @@ -11,13 +11,13 @@ python () { raise bb.parse.SkipRecipe("Set PREFERRED_PROVIDER_virtual/kernel to linux-yocto-rt to enable it") } -SRCREV_machine ?= "72075349c6af55a7a6d024f0aa241711653fcb97" -SRCREV_meta ?= "1bd749b7ce4240e83024b10fa4a4a6b9de5a5e5f" +SRCREV_machine ?= "3aa9671ae072f45665e72591be5636522c8a6215" +SRCREV_meta ?= "a889c43359ca8bee705601817c50edf3c209bc09" SRC_URI = "git://git.yoctoproject.org/linux-yocto.git;branch=${KBRANCH};name=machine \ git://git.yoctoproject.org/yocto-kernel-cache;type=kmeta;name=meta;branch=yocto-4.14;destsuffix=${KMETA}" -LINUX_VERSION ?= "4.14.143" +LINUX_VERSION ?= "4.14.154" DEPENDS += "${@bb.utils.contains('ARCH', 'x86', 'elfutils-native', '', d)}" DEPENDS += "openssl-native util-linux-native" diff --git a/meta/recipes-kernel/linux/linux-yocto-tiny_4.14.bb b/meta/recipes-kernel/linux/linux-yocto-tiny_4.14.bb index cb463074e1..52c02cad10 100644 --- a/meta/recipes-kernel/linux/linux-yocto-tiny_4.14.bb +++ b/meta/recipes-kernel/linux/linux-yocto-tiny_4.14.bb @@ -4,7 +4,7 @@ KCONFIG_MODE = "--allnoconfig" require recipes-kernel/linux/linux-yocto.inc -LINUX_VERSION ?= "4.14.143" +LINUX_VERSION ?= "4.14.154" DEPENDS += "${@bb.utils.contains('ARCH', 'x86', 'elfutils-native', '', d)}" DEPENDS += "openssl-native util-linux-native" @@ -12,8 +12,8 @@ DEPENDS += "openssl-native util-linux-native" KMETA = "kernel-meta" KCONF_BSP_AUDIT_LEVEL = "2" -SRCREV_machine ?= "3d884bc92763f474cc0728d1feb0becad8ed37d5" -SRCREV_meta ?= "1bd749b7ce4240e83024b10fa4a4a6b9de5a5e5f" +SRCREV_machine ?= "38c3a6549d60a3b4a5ab0cb6a440929ba8502f7f" +SRCREV_meta ?= "a889c43359ca8bee705601817c50edf3c209bc09" PV = "${LINUX_VERSION}+git${SRCPV}" diff --git a/meta/recipes-kernel/linux/linux-yocto_4.14.bb b/meta/recipes-kernel/linux/linux-yocto_4.14.bb index 4a92d27e78..0048735c01 100644 --- a/meta/recipes-kernel/linux/linux-yocto_4.14.bb +++ b/meta/recipes-kernel/linux/linux-yocto_4.14.bb @@ -11,20 +11,20 @@ KBRANCH_qemux86 ?= "v4.14/standard/base" KBRANCH_qemux86-64 ?= "v4.14/standard/base" KBRANCH_qemumips64 ?= "v4.14/standard/mti-malta64" -SRCREV_machine_qemuarm ?= "bd85f4880bb890bf9c45ee6c2fd95f077d2bf67e" -SRCREV_machine_qemuarm64 ?= "445a4787bd489eb6b3d5c172b9842dbe5a34d734" -SRCREV_machine_qemumips ?= "3d07ac9aa6ca729674dfb763563202f18f9eedde" -SRCREV_machine_qemuppc ?= "81ba8dbab3b1bfc371e539956be905809db0e41a" -SRCREV_machine_qemux86 ?= "bc9d4b045fa0254d14ef3a667a200f02cb9af755" -SRCREV_machine_qemux86-64 ?= "bc9d4b045fa0254d14ef3a667a200f02cb9af755" -SRCREV_machine_qemumips64 ?= "3c4acadcbe2ee11043f7d0fce43a5181511d0935" -SRCREV_machine ?= "bc9d4b045fa0254d14ef3a667a200f02cb9af755" -SRCREV_meta ?= "1bd749b7ce4240e83024b10fa4a4a6b9de5a5e5f" +SRCREV_machine_qemuarm ?= "e4e2990af921c2d1544d18efa5f7183f95289cd0" +SRCREV_machine_qemuarm64 ?= "51c9e69ebef5d2d15dfbcdf098269d86e0e38317" +SRCREV_machine_qemumips ?= "e70c76a3fe9cc785619d9e4c8e28cb4d4d76ecaf" +SRCREV_machine_qemuppc ?= "6b6eab44d3a04294c233e0b47d6b7c6cbb6e9ffb" +SRCREV_machine_qemux86 ?= "57278e88a6b0f7c6230f7429cab7e74229f2b7ce" +SRCREV_machine_qemux86-64 ?= "57278e88a6b0f7c6230f7429cab7e74229f2b7ce" +SRCREV_machine_qemumips64 ?= "4e099e87d223bfc1526543a5e4c5383cb2edda70" +SRCREV_machine ?= "57278e88a6b0f7c6230f7429cab7e74229f2b7ce" +SRCREV_meta ?= "a889c43359ca8bee705601817c50edf3c209bc09" SRC_URI = "git://git.yoctoproject.org/linux-yocto.git;name=machine;branch=${KBRANCH}; \ git://git.yoctoproject.org/yocto-kernel-cache;type=kmeta;name=meta;branch=yocto-4.14;destsuffix=${KMETA}" -LINUX_VERSION ?= "4.14.143" +LINUX_VERSION ?= "4.14.154" DEPENDS += "${@bb.utils.contains('ARCH', 'x86', 'elfutils-native', '', d)}" DEPENDS += "openssl-native util-linux-native" diff --git a/meta/recipes-support/iso-codes/iso-codes_4.1.bb b/meta/recipes-support/iso-codes/iso-codes_4.1.bb index 1761ded44e..a70513033d 100644 --- a/meta/recipes-support/iso-codes/iso-codes_4.1.bb +++ b/meta/recipes-support/iso-codes/iso-codes_4.1.bb @@ -5,7 +5,7 @@ BUGTRACKER = "https://salsa.debian.org/iso-codes-team/iso-codes/issues" LICENSE = "LGPLv2.1" LIC_FILES_CHKSUM = "file://COPYING;md5=4fbd65380cdd255951079008b364516c" -SRC_URI = "git://salsa.debian.org/iso-codes-team/iso-codes.git;protocol=http" +SRC_URI = "git://salsa.debian.org/iso-codes-team/iso-codes.git;protocol=http;branch=main;" SRCREV = "164802d5fd8c7a8167816fd86a62b286680619f0" # inherit gettext cannot be used, because it adds gettext-native to BASEDEPENDS which diff --git a/scripts/runqemu b/scripts/runqemu index f83e05728b..b0509672d5 100755 --- a/scripts/runqemu +++ b/scripts/runqemu @@ -157,19 +157,6 @@ def get_first_file(cmds): return f return '' -def check_free_port(host, port): - """ Check whether the port is free or not """ - import socket - from contextlib import closing - - with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: - if sock.connect_ex((host, port)) == 0: - # Port is open, so not free - return False - else: - # Port is not open, so free - return True - class BaseConfig(object): def __init__(self): # The self.d saved vars from self.set(), part of them are from qemuboot.conf @@ -218,8 +205,9 @@ class BaseConfig(object): self.audio_enabled = False self.tcpserial_portnum = '' self.custombiosdir = '' - self.lock = '' - self.lock_descriptor = None + self.taplock = '' + self.taplock_descriptor = None + self.portlocks = {} self.bitbake_e = '' self.snapshot = False self.fstypes = ('ext2', 'ext3', 'ext4', 'jffs2', 'nfs', 'btrfs', @@ -240,30 +228,81 @@ class BaseConfig(object): # avoid cleanup twice self.cleaned = False - def acquire_lock(self, error=True): - logger.debug("Acquiring lockfile %s..." % self.lock) + def acquire_taplock(self, error=True): + logger.debug("Acquiring lockfile %s..." % self.taplock) try: - self.lock_descriptor = open(self.lock, 'w') - fcntl.flock(self.lock_descriptor, fcntl.LOCK_EX|fcntl.LOCK_NB) + self.taplock_descriptor = open(self.taplock, 'w') + fcntl.flock(self.taplock_descriptor, fcntl.LOCK_EX|fcntl.LOCK_NB) except Exception as e: - msg = "Acquiring lockfile %s failed: %s" % (self.lock, e) + msg = "Acquiring lockfile %s failed: %s" % (self.taplock, e) if error: logger.error(msg) else: logger.info(msg) - if self.lock_descriptor: - self.lock_descriptor.close() - self.lock_descriptor = None + if self.taplock_descriptor: + self.taplock_descriptor.close() + self.taplock_descriptor = None return False return True - def release_lock(self): - if self.lock_descriptor: + def release_taplock(self): + if self.taplock_descriptor: logger.debug("Releasing lockfile for tap device '%s'" % self.tap) - fcntl.flock(self.lock_descriptor, fcntl.LOCK_UN) - self.lock_descriptor.close() - os.remove(self.lock) - self.lock_descriptor = None + fcntl.flock(self.taplock_descriptor, fcntl.LOCK_UN) + self.taplock_descriptor.close() + os.remove(self.taplock) + self.taplock_descriptor = None + + def check_free_port(self, host, port, lockdir): + """ Check whether the port is free or not """ + import socket + from contextlib import closing + + lockfile = os.path.join(lockdir, str(port) + '.lock') + if self.acquire_portlock(lockfile): + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: + if sock.connect_ex((host, port)) == 0: + # Port is open, so not free + self.release_portlock(lockfile) + return False + else: + # Port is not open, so free + return True + else: + return False + + def acquire_portlock(self, lockfile, error=True): + logger.debug("Acquiring lockfile %s..." % lockfile) + try: + portlock_descriptor = open(lockfile, 'w') + self.portlocks.update({lockfile: portlock_descriptor}) + fcntl.flock(self.portlocks[lockfile], fcntl.LOCK_EX|fcntl.LOCK_NB) + except Exception as e: + msg = "Acquiring lockfile %s failed: %s" % (lockfile, e) + if error: + logger.error(msg) + else: + logger.info(msg) + if self.portlocks[lockfile]: + self.portlocks[lockfile].close() + del self.portlocks[lockfile] + return False + return True + + def release_portlock(self, lockfile=None): + if lockfile != None: + logger.debug("Releasing lockfile '%s'" % lockfile) + fcntl.flock(self.portlocks[lockfile], fcntl.LOCK_UN) + self.portlocks[lockfile].close() + os.remove(lockfile) + del self.portlocks[lockfile] + elif len(self.portlocks): + for lockfile, descriptor in self.portlocks.items(): + logger.debug("Releasing lockfile '%s'" % lockfile) + fcntl.flock(descriptor, fcntl.LOCK_UN) + descriptor.close() + os.remove(lockfile) + self.portlocks = {} def get(self, key): if key in self.d: @@ -923,10 +962,21 @@ class BaseConfig(object): ports = re.findall('hostfwd=[^-]*:([0-9]+)-[^,-]*', qb_slirp_opt) ports = [int(i) for i in ports] mac = 2 + + lockdir = "/tmp/qemu-port-locks" + if not os.path.exists(lockdir): + # There might be a race issue when multi runqemu processess are + # running at the same time. + try: + os.mkdir(lockdir) + os.chmod(lockdir, 0o777) + except FileExistsError: + pass + # Find a free port to avoid conflicts for p in ports[:]: p_new = p - while not check_free_port('localhost', p_new): + while not self.check_free_port('localhost', p_new, lockdir): p_new += 1 mac += 1 while p_new in ports: @@ -981,8 +1031,8 @@ class BaseConfig(object): if os.path.exists('%s.skip' % lockfile): logger.info('Found %s.skip, skipping %s' % (lockfile, p)) continue - self.lock = lockfile + '.lock' - if self.acquire_lock(error=False): + self.taplock = lockfile + '.lock' + if self.acquire_taplock(error=False): tap = p logger.info("Using preconfigured tap device %s" % tap) logger.info("If this is not intended, touch %s.skip to make runqemu skip %s." %(lockfile, tap)) @@ -1000,8 +1050,8 @@ class BaseConfig(object): cmd = ('sudo', self.qemuifup, str(uid), str(gid), self.bindir_native) tap = subprocess.check_output(cmd).decode('utf-8').strip() lockfile = os.path.join(lockdir, tap) - self.lock = lockfile + '.lock' - self.acquire_lock() + self.taplock = lockfile + '.lock' + self.acquire_taplock() self.cleantap = True logger.debug('Created tap: %s' % tap) @@ -1233,8 +1283,11 @@ class BaseConfig(object): cmds = shlex.split(cmd) logger.info('Running %s\n' % cmd) pass_fds = [] - if self.lock_descriptor: - pass_fds = [self.lock_descriptor.fileno()] + if self.taplock_descriptor: + pass_fds = [self.taplock_descriptor.fileno()] + if len(self.portlocks): + for descriptor in self.portlocks.values(): + pass_fds.append(descriptor.fileno()) process = subprocess.Popen(cmds, stderr=subprocess.PIPE, pass_fds=pass_fds) self.qemupid = process.pid retcode = process.wait() @@ -1256,7 +1309,8 @@ class BaseConfig(object): cmd = ('sudo', self.qemuifdown, self.tap, self.bindir_native) logger.debug('Running %s' % str(cmd)) subprocess.check_call(cmd) - self.release_lock() + self.release_taplock() + self.release_portlock() if self.nfs_running: logger.info("Shutting down the userspace NFS server...") |