""" BitBake 'TaskData' implementation Task data collection and handling """ # Copyright (C) 2006 Richard Purdie # # SPDX-License-Identifier: GPL-2.0-only # import logging import re import bb logger = logging.getLogger("BitBake.TaskData") def re_match_strings(target, strings): """ Whether or not the string 'target' matches any one string of the strings which can be regular expression string """ for name in strings: if name.startswith("^") or name.endswith("$"): if re.match(name, target): return True elif name == target: return True return False class TaskEntry: def __init__(self): self.tdepends = [] self.idepends = [] self.irdepends = [] class TaskData: """ BitBake Task Data implementation """ def __init__(self, abort = True, skiplist = None, allowincomplete = False): self.build_targets = {} self.run_targets = {} self.external_targets = [] self.seenfns = [] self.taskentries = {} self.depids = {} self.rdepids = {} self.consider_msgs_cache = [] self.failed_deps = [] self.failed_rdeps = [] self.failed_fns = [] self.abort = abort self.allowincomplete = allowincomplete self.skiplist = skiplist self.mcdepends = [] def add_tasks(self, fn, dataCache): """ Add tasks for a given fn to the database """ task_deps = dataCache.task_deps[fn] if fn in self.failed_fns: bb.msg.fatal("TaskData", "Trying to re-add a failed file? Something is broken...") # Check if we've already seen this fn if fn in self.seenfns: return self.seenfns.append(fn) self.add_extra_deps(fn, dataCache) def add_mcdepends(task): for dep in task_deps['mcdepends'][task].split(): if len(dep.split(':')) != 5: bb.msg.fatal("TaskData", "Error for %s:%s[%s], multiconfig dependency %s does not contain exactly four ':' characters.\n Task '%s' should be specified in the form 'mc:fromMC:toMC:packagename:task'" % (fn, task, 'mcdepends', dep, 'mcdepends')) if dep not in self.mcdepends: self.mcdepends.append(dep) # Common code for dep_name/depends = 'depends'/idepends and 'rdepends'/irdepends def handle_deps(task, dep_name, depends, seen): if dep_name in task_deps and task in task_deps[dep_name]: ids = [] for dep in task_deps[dep_name][task].split(): if dep: parts = dep.split(":") if len(parts) != 2: bb.msg.fatal("TaskData", "Error for %s:%s[%s], dependency %s in '%s' does not contain exactly one ':' character.\n Task '%s' should be specified in the form 'packagename:task'" % (fn, task, dep_name, dep, task_deps[dep_name][task], dep_name)) ids.append((parts[0], parts[1])) seen(parts[0]) depends.extend(ids) for task in task_deps['tasks']: tid = "%s:%s" % (fn, task) self.taskentries[tid] = TaskEntry() # Work out task dependencies parentids = [] for dep in task_deps['parents'][task]: if dep not in task_deps['tasks']: bb.debug(2, "Not adding dependency of %s on %s since %s does not exist" % (task, dep, dep)) continue parentid = "%s:%s" % (fn, dep) parentids.append(parentid) self.taskentries[tid].tdepends.extend(parentids) # Touch all intertask dependencies handle_deps(task, 'depends', self.taskentries[tid].idepends, self.seen_build_target) handle_deps(task, 'rdepends', self.taskentries[tid].irdepends, self.seen_run_target) if 'mcdepends' in task_deps and task in task_deps['mcdepends']: add_mcdepends(task) # Work out build dependencies if not fn in self.depids: dependids = set() for depend in dataCache.deps[fn]: dependids.add(depend) self.depids[fn] = list(dependids) logger.debug2("Added dependencies %s for %s", str(dataCache.deps[fn]), fn) # Work out runtime dependencies if not fn in self.rdepids: rdependids = set() rdepends = dataCache.rundeps[fn] rrecs = dataCache.runrecs[fn] rdependlist = [] rreclist = [] for package in rdepends: for rdepend in rdepends[package]: rdependlist.append(rdepend) rdependids.add(rdepend) for package in rrecs: for rdepend in rrecs[package]: rreclist.append(rdepend) rdependids.add(rdepend) if rdependlist: logger.debug2("Added runtime dependencies %s for %s", str(rdependlist), fn) if rreclist: logger.debug2("Added runtime recommendations %s for %s", str(rreclist), fn) self.rdepids[fn] = list(rdependids) for dep in self.depids[fn]: self.seen_build_target(dep) if dep in self.failed_deps: self.fail_fn(fn) return for dep in self.rdepids[fn]: self.seen_run_target(dep) if dep in self.failed_rdeps: self.fail_fn(fn) return def add_extra_deps(self, fn, dataCache): func = dataCache.extradepsfunc.get(fn, None) if func: bb.providers.buildWorldTargetList(dataCache) pn = dataCache.pkg_fn[fn] params = {'deps': dataCache.deps[fn], 'world_target': dataCache.world_target, 'pkg_pn': dataCache.pkg_pn, 'self_pn': pn} funcname = '_%s_calculate_extra_depends' % pn.replace('-', '_') paramlist = ','.join(params.keys()) func = 'def %s(%s):\n%s\n\n%s(%s)' % (funcname, paramlist, func, funcname, paramlist) bb.utils.better_exec(func, params) def have_build_target(self, target): """ Have we a build target matching this name? """ if target in self.build_targets and self.build_targets[target]: return True return False def have_runtime_target(self, target): """ Have we a runtime target matching this name? """ if target in self.run_targets and self.run_targets[target]: return True return False def seen_build_target(self, name): """ Maintain a list of build targets """ if name not in self.build_targets: self.build_targets[name] = [] def add_build_target(self, fn, item): """ Add a build target. If already present, append the provider fn to the list """ if item in self.build_targets: if fn in self.build_targets[item]: return self.build_targets[item].append(fn) return self.build_targets[item] = [fn] def seen_run_target(self, name): """ Maintain a list of runtime build targets """ if name not in self.run_targets: self.run_targets[name] = [] def add_runtime_target(self, fn, item): """ Add a runtime target. If already present, append the provider fn to the list """ if item in self.run_targets: if fn in self.run_targets[item]: return self.run_targets[item].append(fn) return self.run_targets[item] = [fn] def mark_external_target(self, target): """ Mark a build target as being externally requested """ if target not in self.external_targets: self.external_targets.append(target) def get_unresolved_build_targets(self, dataCache): """ Return a list of build targets who's providers are unknown. """ unresolved = [] for target in self.build_targets: if re_match_strings(target, dataCache.ignored_dependencies): continue if target in self.failed_deps: continue if not self.build_targets[target]: unresolved.append(target) return unresolved def get_unresolved_run_targets(self, dataCache): """ Return a list of runtime targets who's providers are unknown. """ unresolved = [] for target in self.run_targets: if re_match_strings(target, dataCache.ignored_dependencies): continue if target in self.failed_rdeps: continue if not self.run_targets[target]: unresolved.append(target) return unresolved def get_provider(self, item): """ Return a list of providers of item """ return self.build_targets[item] def get_dependees(self, item): """ Return a list of targets which depend on item """ dependees = [] for fn in self.depids: if item in self.depids[fn]: dependees.append(fn) return dependees def get_rdependees(self, item): """ Return a list of targets which depend on runtime item """ dependees = [] for fn in self.rdepids: if item in self.rdepids[fn]: dependees.append(fn) return dependees def get_reasons(self, item, runtime=False): """ Get the reason(s) for an item not being provided, if any """ reasons = [] if self.skiplist: for fn in self.skiplist: skipitem = self.skiplist[fn] if skipitem.pn == item: reasons.append("%s was skipped: %s" % (skipitem.pn, skipitem.skipreason)) elif runtime and item in skipitem.rprovides: reasons.append("%s RPROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason)) elif not runtime and item in skipitem.provides: reasons.append("%s PROVIDES %s but was skipped: %s" % (skipitem.pn, item, skipitem.skipreason)) return reasons def get_close_matches(self, item, provider_list): import difflib if self.skiplist: skipped = [] for fn in self.skiplist: skipped.append(self.skiplist[fn].pn) full_list = provider_list + skipped else: full_list = provider_list return difflib.get_close_matches(item, full_list, cutoff=0.7) def add_provider(self, cfgData, dataCache, item): try: self.add_provider_internal(cfgData, dataCache, item) except bb.providers.NoProvider: if self.abort: raise self.remove_buildtarget(item) self.mark_external_target(item) def add_provider_internal(self, cfgData, dataCache, item): """ Add the providers of item to the task data Mark entries were specifically added externally as against dependencies added internally during dependency resolution """ if re_match_strings(item, dataCache.ignored_dependencies): return if not item in dataCache.providers: close_matches = self.get_close_matches(item, list(dataCache.providers.keys())) # Is it in RuntimeProviders ? all_p = bb.providers.getRuntimeProviders(dataCache, item) for fn in all_p: new = dataCache.pkg_fn[fn] + " RPROVIDES " + item if new not in close_matches: close_matches.append(new) bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=self.get_reasons(item), close_matches=close_matches), cfgData) raise bb.providers.NoProvider(item) if self.have_build_target(item): return all_p = dataCache.providers[item] eligible, foundUnique = bb.providers.filterProviders(all_p, item, cfgData, dataCache) eligible = [p for p in eligible if not p in self.failed_fns] if not eligible: bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=["No eligible PROVIDERs exist for '%s'" % item]), cfgData) raise bb.providers.NoProvider(item) if len(eligible) > 1 and not foundUnique: if item not in self.consider_msgs_cache: providers_list = [] for fn in eligible: providers_list.append(dataCache.pkg_fn[fn]) bb.event.fire(bb.event.MultipleProviders(item, providers_list), cfgData) self.consider_msgs_cache.append(item) for fn in eligible: if fn in self.failed_fns: continue logger.debug2("adding %s to satisfy %s", fn, item) self.add_build_target(fn, item) self.add_tasks(fn, dataCache) #item = dataCache.pkg_fn[fn] def add_rprovider(self, cfgData, dataCache, item): """ Add the runtime providers of item to the task data (takes item names from RDEPENDS/PACKAGES namespace) """ if re_match_strings(item, dataCache.ignored_dependencies): return if self.have_runtime_target(item): return all_p = bb.providers.getRuntimeProviders(dataCache, item) if not all_p: bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=self.get_reasons(item, True)), cfgData) raise bb.providers.NoRProvider(item) eligible, numberPreferred = bb.providers.filterProvidersRunTime(all_p, item, cfgData, dataCache) eligible = [p for p in eligible if not p in self.failed_fns] if not eligible: bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData) raise bb.providers.NoRProvider(item) if len(eligible) > 1 and numberPreferred == 0: if item not in self.consider_msgs_cache: providers_list = [] for fn in eligible: providers_list.append(dataCache.pkg_fn[fn]) bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData) self.consider_msgs_cache.append(item) if numberPreferred > 1: if item not in self.consider_msgs_cache: providers_list = [] for fn in eligible: providers_list.append(dataCache.pkg_fn[fn]) bb.event.fire(bb.event.MultipleProviders(item, providers_list, runtime=True), cfgData) self.consider_msgs_cache.append(item) raise bb.providers.MultipleRProvider(item) # run through the list until we find one that we can build for fn in eligible: if fn in self.failed_fns: continue logger.debug2("adding '%s' to satisfy runtime '%s'", fn, item) self.add_runtime_target(fn, item) self.add_tasks(fn, dataCache) def fail_fn(self, fn, missing_list=None): """ Mark a file as failed (unbuildable) Remove any references from build and runtime provider lists missing_list, A list of missing requirements for this target """ if fn in self.failed_fns: return if not missing_list: missing_list = [] logger.debug("File '%s' is unbuildable, removing...", fn) self.failed_fns.append(fn) for target in self.build_targets: if fn in self.build_targets[target]: self.build_targets[target].remove(fn) if len(self.build_targets[target]) == 0: self.remove_buildtarget(target, missing_list) for target in self.run_targets: if fn in self.run_targets[target]: self.run_targets[target].remove(fn) if len(self.run_targets[target]) == 0: self.remove_runtarget(target, missing_list) def remove_buildtarget(self, target, missing_list=None): """ Mark a build target as failed (unbuildable) Trigger removal of any files that have this as a dependency """ if not missing_list: missing_list = [target] else: missing_list = [target] + missing_list logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list) self.failed_deps.append(target) dependees = self.get_dependees(target) for fn in dependees: self.fail_fn(fn, missing_list) for tid in self.taskentries: for (idepend, idependtask) in self.taskentries[tid].idepends: if idepend == target: fn = tid.rsplit(":",1)[0] self.fail_fn(fn, missing_list) if self.abort and target in self.external_targets: logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list) raise bb.providers.NoProvider(target) def remove_runtarget(self, target, missing_list=None): """ Mark a run target as failed (unbuildable) Trigger removal of any files that have this as a dependency """ if not missing_list: missing_list = [target] else: missing_list = [target] + missing_list logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list) self.failed_rdeps.append(target) dependees = self.get_rdependees(target) for fn in dependees: self.fail_fn(fn, missing_list) for tid in self.taskentries: for (idepend, idependtask) in self.taskentries[tid].irdepends: if idepend == target: fn = tid.rsplit(":",1)[0] self.fail_fn(fn, missing_list) def add_unresolved(self, cfgData, dataCache): """ Resolve all unresolved build and runtime targets """ logger.info("Resolving any missing task queue dependencies") while True: added = 0 for target in self.get_unresolved_build_targets(dataCache): try: self.add_provider_internal(cfgData, dataCache, target) added = added + 1 except bb.providers.NoProvider: if self.abort and target in self.external_targets and not self.allowincomplete: raise if not self.allowincomplete: self.remove_buildtarget(target) for target in self.get_unresolved_run_targets(dataCache): try: self.add_rprovider(cfgData, dataCache, target) added = added + 1 except (bb.providers.NoRProvider, bb.providers.MultipleRProvider): self.remove_runtarget(target) logger.debug("Resolved " + str(added) + " extra dependencies") if added == 0: break # self.dump_data() def get_providermap(self, prefix=None): provmap = {} for name in self.build_targets: if prefix and not name.startswith(prefix): continue if self.have_build_target(name): provider = self.get_provider(name) if provider: provmap[name] = provider[0] return provmap def get_mcdepends(self): return self.mcdepends def dump_data(self): """ Dump some debug information on the internal data structures """ logger.debug3("build_names:") logger.debug3(", ".join(self.build_targets)) logger.debug3("run_names:") logger.debug3(", ".join(self.run_targets)) logger.debug3("build_targets:") for target in self.build_targets: targets = "None" if target in self.build_targets: targets = self.build_targets[target] logger.debug3(" %s: %s", target, targets) logger.debug3("run_targets:") for target in self.run_targets: targets = "None" if target in self.run_targets: targets = self.run_targets[target] logger.debug3(" %s: %s", target, targets) logger.debug3("tasks:") for tid in self.taskentries: logger.debug3(" %s: %s %s %s", tid, self.taskentries[tid].idepends, self.taskentries[tid].irdepends, self.taskentries[tid].tdepends) logger.debug3("dependency ids (per fn):") for fn in self.depids: logger.debug3(" %s: %s", fn, self.depids[fn]) logger.debug3("runtime dependency ids (per fn):") for fn in self.rdepids: logger.debug3(" %s: %s", fn, self.rdepids[fn])