diff options
Diffstat (limited to 'meta/lib/oe/utils.py')
| -rw-r--r-- | meta/lib/oe/utils.py | 381 |
1 files changed, 251 insertions, 130 deletions
diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py index 80f0442d0a..14a7d07ef0 100644 --- a/meta/lib/oe/utils.py +++ b/meta/lib/oe/utils.py @@ -1,4 +1,13 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only +# + import subprocess +import multiprocessing +import traceback +import errno def read_file(filename): try: @@ -23,6 +32,13 @@ def conditional(variable, checkvalue, truevalue, falsevalue, d): else: return falsevalue +def vartrue(var, iftrue, iffalse, d): + import oe.types + if oe.types.boolean(d.getVar(var)): + return iftrue + else: + return iffalse + def less_or_equal(variable, checkvalue, truevalue, falsevalue, d): if float(d.getVar(variable)) <= float(checkvalue): return truevalue @@ -69,12 +85,12 @@ def prune_suffix(var, suffixes, d): # See if var ends with any of the suffixes listed and # remove it if found for suffix in suffixes: - if var.endswith(suffix): - var = var.replace(suffix, "") + if suffix and var.endswith(suffix): + var = var[:-len(suffix)] prefix = d.getVar("MLPREFIX") if prefix and var.startswith(prefix): - var = var.replace(prefix, "") + var = var[len(prefix):] return var @@ -156,7 +172,7 @@ def any_distro_features(d, features, truevalue="1", falsevalue=""): """ return bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d) -def parallel_make(d): +def parallel_make(d, makeinst=False): """ Return the integer value for the number of parallel threads to use when building, scraped out of PARALLEL_MAKE. If no parallelization option is @@ -164,7 +180,10 @@ def parallel_make(d): e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an integer. """ - pm = (d.getVar('PARALLEL_MAKE') or '').split() + if makeinst: + pm = (d.getVar('PARALLEL_MAKEINST') or '').split() + else: + pm = (d.getVar('PARALLEL_MAKE') or '').split() # look for '-j' and throw other options (e.g. '-l') away while pm: opt = pm.pop(0) @@ -177,9 +196,9 @@ def parallel_make(d): return int(v) - return None + return '' -def parallel_make_argument(d, fmt, limit=None): +def parallel_make_argument(d, fmt, limit=None, makeinst=False): """ Helper utility to construct a parallel make argument from the number of parallel threads specified in PARALLEL_MAKE. @@ -192,7 +211,7 @@ def parallel_make_argument(d, fmt, limit=None): e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n %d") will return "-n 10" """ - v = parallel_make(d) + v = parallel_make(d, makeinst) if v: if limit: v = min(limit, v) @@ -205,12 +224,12 @@ def packages_filter_out_system(d): PN-dbg PN-doc PN-locale-eb-gb removed. """ pn = d.getVar('PN') - blacklist = [pn + suffix for suffix in ('', '-dbg', '-dev', '-doc', '-locale', '-staticdev')] + pkgfilter = [pn + suffix for suffix in ('', '-dbg', '-dev', '-doc', '-locale', '-staticdev', '-src')] localepkg = pn + "-locale-" pkgs = [] for pkg in d.getVar('PACKAGES').split(): - if pkg not in blacklist and localepkg not in pkg: + if pkg not in pkgfilter and localepkg not in pkg: pkgs.append(pkg) return pkgs @@ -232,59 +251,131 @@ def trim_version(version, num_parts=2): trimmed = ".".join(parts[:num_parts]) return trimmed -def cpu_count(): - import multiprocessing - return multiprocessing.cpu_count() +def cpu_count(at_least=1, at_most=64): + cpus = len(os.sched_getaffinity(0)) + return max(min(cpus, at_most), at_least) def execute_pre_post_process(d, cmds): if cmds is None: return - for cmd in cmds.strip().split(';'): - cmd = cmd.strip() - if cmd != '': - bb.note("Executing %s ..." % cmd) - bb.build.exec_func(cmd, d) - -def multiprocess_exec(commands, function): - import signal - import multiprocessing - - if not commands: - return [] - - def init_worker(): - signal.signal(signal.SIGINT, signal.SIG_IGN) - - fails = [] + cmds = cmds.replace(";", " ") - def failures(res): - fails.append(res) + for cmd in cmds.split(): + bb.note("Executing %s ..." % cmd) + bb.build.exec_func(cmd, d) - nproc = min(multiprocessing.cpu_count(), len(commands)) - pool = bb.utils.multiprocessingpool(nproc, init_worker) +def get_bb_number_threads(d): + return int(d.getVar("BB_NUMBER_THREADS") or os.cpu_count() or 1) - try: - mapresult = pool.map_async(function, commands, error_callback=failures) +def multiprocess_launch(target, items, d, extraargs=None): + max_process = get_bb_number_threads(d) + return multiprocess_launch_mp(target, items, max_process, extraargs) - pool.close() - pool.join() - results = mapresult.get() - except KeyboardInterrupt: - pool.terminate() - pool.join() - raise +# For each item in items, call the function 'target' with item as the first +# argument, extraargs as the other arguments and handle any exceptions in the +# parent thread +def multiprocess_launch_mp(target, items, max_process, extraargs=None): - if fails: - raise fails[0] + class ProcessLaunch(multiprocessing.Process): + def __init__(self, *args, **kwargs): + multiprocessing.Process.__init__(self, *args, **kwargs) + self._pconn, self._cconn = multiprocessing.Pipe() + self._exception = None + self._result = None + def run(self): + try: + ret = self._target(*self._args, **self._kwargs) + self._cconn.send((None, ret)) + except Exception as e: + tb = traceback.format_exc() + self._cconn.send((e, tb)) + + def update(self): + if self._pconn.poll(): + (e, tb) = self._pconn.recv() + if e is not None: + self._exception = (e, tb) + else: + self._result = tb + + @property + def exception(self): + self.update() + return self._exception + + @property + def result(self): + self.update() + return self._result + + launched = [] + errors = [] + results = [] + items = list(items) + while (items and not errors) or launched: + if not errors and items and len(launched) < max_process: + args = (items.pop(),) + if extraargs is not None: + args = args + extraargs + p = ProcessLaunch(target=target, args=args) + p.start() + launched.append(p) + for q in launched: + # Have to manually call update() to avoid deadlocks. The pipe can be full and + # transfer stalled until we try and read the results object but the subprocess won't exit + # as it still has data to write (https://bugs.python.org/issue8426) + q.update() + # The finished processes are joined when calling is_alive() + if not q.is_alive(): + if q.exception: + errors.append(q.exception) + if q.result: + results.append(q.result) + launched.remove(q) + # Paranoia doesn't hurt + for p in launched: + p.join() + if errors: + msg = "" + for (e, tb) in errors: + if isinstance(e, subprocess.CalledProcessError) and e.output: + msg = msg + str(e) + "\n" + msg = msg + "Subprocess output:" + msg = msg + e.output.decode("utf-8", errors="ignore") + else: + msg = msg + str(e) + ": " + str(tb) + "\n" + bb.fatal("Fatal errors occurred in subprocesses:\n%s" % msg) return results def squashspaces(string): import re - return re.sub("\s+", " ", string).strip() + return re.sub(r"\s+", " ", string).strip() + +def rprovides_map(pkgdata_dir, pkg_dict): + # Map file -> pkg provider + rprov_map = {} -def format_pkg_list(pkg_dict, ret_format=None): + for pkg in pkg_dict: + path_to_pkgfile = os.path.join(pkgdata_dir, 'runtime-reverse', pkg) + if not os.path.isfile(path_to_pkgfile): + continue + with open(path_to_pkgfile) as f: + for line in f: + if line.startswith('RPROVIDES') or line.startswith('FILERPROVIDES'): + # List all components provided by pkg. + # Exclude version strings, i.e. those starting with ( + provides = [x for x in line.split()[1:] if not x.startswith('(')] + for prov in provides: + if prov in rprov_map: + rprov_map[prov].append(pkg) + else: + rprov_map[prov] = [pkg] + + return rprov_map + +def format_pkg_list(pkg_dict, ret_format=None, pkgdata_dir=None): output = [] if ret_format == "arch": @@ -297,117 +388,101 @@ def format_pkg_list(pkg_dict, ret_format=None): for pkg in sorted(pkg_dict): output.append("%s %s %s" % (pkg, pkg_dict[pkg]["arch"], pkg_dict[pkg]["ver"])) elif ret_format == "deps": + rprov_map = rprovides_map(pkgdata_dir, pkg_dict) for pkg in sorted(pkg_dict): for dep in pkg_dict[pkg]["deps"]: - output.append("%s|%s" % (pkg, dep)) + if dep in rprov_map: + # There could be multiple providers within the image + for pkg_provider in rprov_map[dep]: + output.append("%s|%s * %s [RPROVIDES]" % (pkg, pkg_provider, dep)) + else: + output.append("%s|%s" % (pkg, dep)) else: for pkg in sorted(pkg_dict): output.append(pkg) - return '\n'.join(output) + output_str = '\n'.join(output) + + if output_str: + # make sure last line is newline terminated + output_str += '\n' + + return output_str + -def host_gcc_version(d): +# Helper function to get the host compiler version +# Do not assume the compiler is gcc +def get_host_compiler_version(d, taskcontextonly=False): import re, subprocess + if taskcontextonly and d.getVar('BB_WORKERCONTEXT') != '1': + return + compiler = d.getVar("BUILD_CC") + # Get rid of ccache since it is not present when parsing. + if compiler.startswith('ccache '): + compiler = compiler[7:] try: env = os.environ.copy() - env["PATH"] = d.getVar("PATH") - output = subprocess.check_output("%s --version" % compiler, shell=True, env=env).decode("utf-8") + # datastore PATH does not contain session PATH as set by environment-setup-... + # this breaks the install-buildtools use-case + # env["PATH"] = d.getVar("PATH") + output = subprocess.check_output("%s --version" % compiler, \ + shell=True, env=env, stderr=subprocess.STDOUT).decode("utf-8") except subprocess.CalledProcessError as e: bb.fatal("Error running %s --version: %s" % (compiler, e.output.decode("utf-8"))) - match = re.match(".* (\d\.\d)\.\d.*", output.split('\n')[0]) + match = re.match(r".* (\d+\.\d+)\.\d+.*", output.split('\n')[0]) if not match: bb.fatal("Can't get compiler version from %s --version output" % compiler) version = match.group(1) - return "-%s" % version if version in ("4.8", "4.9") else "" + return compiler, version -def get_multilib_datastore(variant, d): - localdata = bb.data.createCopy(d) - overrides = localdata.getVar("OVERRIDES", False) + ":virtclass-multilib-" + variant - localdata.setVar("OVERRIDES", overrides) - localdata.setVar("MLPREFIX", variant + "-") - return localdata - -# -# Python 2.7 doesn't have threaded pools (just multiprocessing) -# so implement a version here -# +def host_gcc_version(d, taskcontextonly=False): + import re, subprocess -from queue import Queue -from threading import Thread + if taskcontextonly and d.getVar('BB_WORKERCONTEXT') != '1': + return -class ThreadedWorker(Thread): - """Thread executing tasks from a given tasks queue""" - def __init__(self, tasks, worker_init, worker_end): - Thread.__init__(self) - self.tasks = tasks - self.daemon = True + compiler = d.getVar("BUILD_CC") + # Get rid of ccache since it is not present when parsing. + if compiler.startswith('ccache '): + compiler = compiler[7:] + try: + env = os.environ.copy() + env["PATH"] = d.getVar("PATH") + output = subprocess.check_output("%s --version" % compiler, \ + shell=True, env=env, stderr=subprocess.STDOUT).decode("utf-8") + except subprocess.CalledProcessError as e: + bb.fatal("Error running %s --version: %s" % (compiler, e.output.decode("utf-8"))) - self.worker_init = worker_init - self.worker_end = worker_end + match = re.match(r".* (\d+\.\d+)\.\d+.*", output.split('\n')[0]) + if not match: + bb.fatal("Can't get compiler version from %s --version output" % compiler) - def run(self): - from queue import Empty + version = match.group(1) + return "-%s" % version if version in ("4.8", "4.9") else "" - if self.worker_init is not None: - self.worker_init(self) - while True: - try: - func, args, kargs = self.tasks.get(block=False) - except Empty: - if self.worker_end is not None: - self.worker_end(self) - break +def get_multilib_datastore(variant, d): + localdata = bb.data.createCopy(d) + if variant: + overrides = localdata.getVar("OVERRIDES", False) + ":virtclass-multilib-" + variant + localdata.setVar("OVERRIDES", overrides) + localdata.setVar("MLPREFIX", variant + "-") + else: + origdefault = localdata.getVar("DEFAULTTUNE_MULTILIB_ORIGINAL") + if origdefault: + localdata.setVar("DEFAULTTUNE", origdefault) + overrides = localdata.getVar("OVERRIDES", False).split(":") + overrides = ":".join([x for x in overrides if not x.startswith("virtclass-multilib-")]) + localdata.setVar("OVERRIDES", overrides) + localdata.setVar("MLPREFIX", "") + return localdata - try: - func(self, *args, **kargs) - except Exception as e: - print(e) - finally: - self.tasks.task_done() - -class ThreadedPool: - """Pool of threads consuming tasks from a queue""" - def __init__(self, num_workers, num_tasks, worker_init=None, - worker_end=None): - self.tasks = Queue(num_tasks) - self.workers = [] - - for _ in range(num_workers): - worker = ThreadedWorker(self.tasks, worker_init, worker_end) - self.workers.append(worker) - - def start(self): - for worker in self.workers: - worker.start() - - def add_task(self, func, *args, **kargs): - """Add a task to the queue""" - self.tasks.put((func, args, kargs)) - - def wait_completion(self): - """Wait for completion of all the tasks in the queue""" - self.tasks.join() - for worker in self.workers: - worker.join() - -def write_ld_so_conf(d): - # Some utils like prelink may not have the correct target library paths - # so write an ld.so.conf to help them - ldsoconf = d.expand("${STAGING_DIR_TARGET}${sysconfdir}/ld.so.conf") - if os.path.exists(ldsoconf): - bb.utils.remove(ldsoconf) - bb.utils.mkdirhier(os.path.dirname(ldsoconf)) - with open(ldsoconf, "w") as f: - f.write(d.getVar("base_libdir") + '\n') - f.write(d.getVar("libdir") + '\n') - -class ImageQAFailed(bb.build.FuncFailed): +class ImageQAFailed(Exception): def __init__(self, description, name=None, logfile=None): self.description = description self.name = name @@ -419,3 +494,49 @@ class ImageQAFailed(bb.build.FuncFailed): msg = msg + ' (%s)' % self.description return msg + +def sh_quote(string): + import shlex + return shlex.quote(string) + +def directory_size(root, blocksize=4096): + """ + Calculate the size of the directory, taking into account hard links, + rounding up every size to multiples of the blocksize. + """ + def roundup(size): + """ + Round the size up to the nearest multiple of the block size. + """ + import math + return math.ceil(size / blocksize) * blocksize + + def getsize(filename): + """ + Get the size of the filename, not following symlinks, taking into + account hard links. + """ + stat = os.lstat(filename) + if stat.st_ino not in inodes: + inodes.add(stat.st_ino) + return stat.st_size + else: + return 0 + + inodes = set() + total = 0 + for root, dirs, files in os.walk(root): + total += sum(roundup(getsize(os.path.join(root, name))) for name in files) + total += roundup(getsize(root)) + return total + +# Update the mtime of a file, skip if permission/read-only issues +def touch(filename): + try: + os.utime(filename, None) + except PermissionError: + pass + except OSError as e: + # Handle read-only file systems gracefully + if e.errno != errno.EROFS: + raise e |