diff options
Diffstat (limited to 'bitbake/lib/bb/taskdata.py')
| -rw-r--r-- | bitbake/lib/bb/taskdata.py | 412 |
1 files changed, 152 insertions, 260 deletions
diff --git a/bitbake/lib/bb/taskdata.py b/bitbake/lib/bb/taskdata.py index 65628c86f2..d8bdbcabf9 100644 --- a/bitbake/lib/bb/taskdata.py +++ b/bitbake/lib/bb/taskdata.py @@ -37,27 +37,24 @@ def re_match_strings(target, strings): return any(name == target or re.match(name, target) for name in strings) +class TaskEntry: + def __init__(self): + self.tdepends = [] + self.idepends = [] + self.irdepends = [] + class TaskData: """ BitBake Task Data implementation """ def __init__(self, abort = True, tryaltconfigs = False, skiplist = None, allowincomplete = False): - self.build_names_index = [] - self.run_names_index = [] - self.fn_index = [] - self.build_targets = {} self.run_targets = {} self.external_targets = [] - self.tasks_fnid = [] - self.tasks_name = [] - self.tasks_tdepends = [] - self.tasks_idepends = [] - self.tasks_irdepends = [] - # Cache to speed up task ID lookups - self.tasks_lookup = {} + self.seenfns = [] + self.taskentries = {} self.depids = {} self.rdepids = {} @@ -66,7 +63,7 @@ class TaskData: self.failed_deps = [] self.failed_rdeps = [] - self.failed_fnids = [] + self.failed_fns = [] self.abort = abort self.tryaltconfigs = tryaltconfigs @@ -74,88 +71,6 @@ class TaskData: self.skiplist = skiplist - def getbuild_id(self, name): - """ - Return an ID number for the build target name. - If it doesn't exist, create one. - """ - if not name in self.build_names_index: - self.build_names_index.append(name) - return len(self.build_names_index) - 1 - - return self.build_names_index.index(name) - - def getrun_id(self, name): - """ - Return an ID number for the run target name. - If it doesn't exist, create one. - """ - if not name in self.run_names_index: - self.run_names_index.append(name) - return len(self.run_names_index) - 1 - - return self.run_names_index.index(name) - - def getfn_id(self, name): - """ - Return an ID number for the filename. - If it doesn't exist, create one. - """ - if not name in self.fn_index: - self.fn_index.append(name) - return len(self.fn_index) - 1 - - return self.fn_index.index(name) - - def gettask_ids(self, fnid): - """ - Return an array of the ID numbers matching a given fnid. - """ - ids = [] - if fnid in self.tasks_lookup: - for task in self.tasks_lookup[fnid]: - ids.append(self.tasks_lookup[fnid][task]) - return ids - - def gettask_id_fromfnid(self, fnid, task): - """ - Return an ID number for the task matching fnid and task. - """ - if fnid in self.tasks_lookup: - if task in self.tasks_lookup[fnid]: - return self.tasks_lookup[fnid][task] - - return None - - def gettask_id(self, fn, task, create = True): - """ - Return an ID number for the task matching fn and task. - If it doesn't exist, create one by default. - Optionally return None instead. - """ - fnid = self.getfn_id(fn) - - if fnid in self.tasks_lookup: - if task in self.tasks_lookup[fnid]: - return self.tasks_lookup[fnid][task] - - if not create: - return None - - self.tasks_name.append(task) - self.tasks_fnid.append(fnid) - self.tasks_tdepends.append([]) - self.tasks_idepends.append([]) - self.tasks_irdepends.append([]) - - listid = len(self.tasks_name) - 1 - - if fnid not in self.tasks_lookup: - self.tasks_lookup[fnid] = {} - self.tasks_lookup[fnid][task] = listid - - return listid - def add_tasks(self, fn, dataCache): """ Add tasks for a given fn to the database @@ -163,29 +78,31 @@ class TaskData: task_deps = dataCache.task_deps[fn] - fnid = self.getfn_id(fn) - - if fnid in self.failed_fnids: + if fn in self.failed_fns: bb.msg.fatal("TaskData", "Trying to re-add a failed file? Something is broken...") # Check if we've already seen this fn - if fnid in self.tasks_fnid: + if fn in self.seenfns: return + self.seenfns.append(fn) + self.add_extra_deps(fn, dataCache) for task in task_deps['tasks']: + tid = "%s:%s" % (fn, task) + self.taskentries[tid] = TaskEntry() + # Work out task dependencies parentids = [] for dep in task_deps['parents'][task]: if dep not in task_deps['tasks']: bb.debug(2, "Not adding dependeny of %s on %s since %s does not exist" % (task, dep, dep)) continue - parentid = self.gettask_id(fn, dep) + parentid = "%s:%s" % (fn, dep) parentids.append(parentid) - taskid = self.gettask_id(fn, task) - self.tasks_tdepends[taskid].extend(parentids) + self.taskentries[tid].tdepends.extend(parentids) # Touch all intertask dependencies if 'depends' in task_deps and task in task_deps['depends']: @@ -194,29 +111,30 @@ class TaskData: if dep: if ":" not in dep: bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':' character\n. Task 'depends' should be specified in the form 'packagename:task'" % (fn, dep)) - ids.append(((self.getbuild_id(dep.split(":")[0])), dep.split(":")[1])) - self.tasks_idepends[taskid].extend(ids) + ids.append(((dep.split(":")[0]), dep.split(":")[1])) + self.seen_build_target(dep.split(":")[0]) + self.taskentries[tid].idepends.extend(ids) if 'rdepends' in task_deps and task in task_deps['rdepends']: ids = [] for dep in task_deps['rdepends'][task].split(): if dep: if ":" not in dep: bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':' character\n. Task 'rdepends' should be specified in the form 'packagename:task'" % (fn, dep)) - ids.append(((self.getrun_id(dep.split(":")[0])), dep.split(":")[1])) - self.tasks_irdepends[taskid].extend(ids) - + ids.append(((dep.split(":")[0]), dep.split(":")[1])) + self.seen_run_target(dep.split(":")[0]) + self.taskentries[tid].irdepends.extend(ids) # Work out build dependencies - if not fnid in self.depids: - dependids = {} + if not fn in self.depids: + dependids = set() for depend in dataCache.deps[fn]: - dependids[self.getbuild_id(depend)] = None - self.depids[fnid] = dependids.keys() + dependids.add(depend) + self.depids[fn] = list(dependids) logger.debug(2, "Added dependencies %s for %s", str(dataCache.deps[fn]), fn) # Work out runtime dependencies - if not fnid in self.rdepids: - rdependids = {} + if not fn in self.rdepids: + rdependids = set() rdepends = dataCache.rundeps[fn] rrecs = dataCache.runrecs[fn] rdependlist = [] @@ -224,24 +142,26 @@ class TaskData: for package in rdepends: for rdepend in rdepends[package]: rdependlist.append(rdepend) - rdependids[self.getrun_id(rdepend)] = None + rdependids.add(rdepend) for package in rrecs: for rdepend in rrecs[package]: rreclist.append(rdepend) - rdependids[self.getrun_id(rdepend)] = None + rdependids.add(rdepend) if rdependlist: logger.debug(2, "Added runtime dependencies %s for %s", str(rdependlist), fn) if rreclist: logger.debug(2, "Added runtime recommendations %s for %s", str(rreclist), fn) - self.rdepids[fnid] = rdependids.keys() + self.rdepids[fn] = list(rdependids) - for dep in self.depids[fnid]: + for dep in self.depids[fn]: + self.seen_build_target(dep) if dep in self.failed_deps: - self.fail_fnid(fnid) + self.fail_fn(fn) return - for dep in self.rdepids[fnid]: + for dep in self.rdepids[fn]: + self.seen_run_target(dep) if dep in self.failed_rdeps: - self.fail_fnid(fnid) + self.fail_fn(fn) return def add_extra_deps(self, fn, dataCache): @@ -263,9 +183,7 @@ class TaskData: """ Have we a build target matching this name? """ - targetid = self.getbuild_id(target) - - if targetid in self.build_targets: + if target in self.build_targets and self.build_targets[target]: return True return False @@ -273,50 +191,54 @@ class TaskData: """ Have we a runtime target matching this name? """ - targetid = self.getrun_id(target) - - if targetid in self.run_targets: + if target in self.run_targets and self.run_targets[target]: return True return False + def seen_build_target(self, name): + """ + Maintain a list of build targets + """ + if name not in self.build_targets: + self.build_targets[name] = [] + def add_build_target(self, fn, item): """ Add a build target. If already present, append the provider fn to the list """ - targetid = self.getbuild_id(item) - fnid = self.getfn_id(fn) - - if targetid in self.build_targets: - if fnid in self.build_targets[targetid]: + if item in self.build_targets: + if fn in self.build_targets[item]: return - self.build_targets[targetid].append(fnid) + self.build_targets[item].append(fn) return - self.build_targets[targetid] = [fnid] + self.build_targets[item] = [fn] + + def seen_run_target(self, name): + """ + Maintain a list of runtime build targets + """ + if name not in self.run_targets: + self.run_targets[name] = [] def add_runtime_target(self, fn, item): """ Add a runtime target. If already present, append the provider fn to the list """ - targetid = self.getrun_id(item) - fnid = self.getfn_id(fn) - - if targetid in self.run_targets: - if fnid in self.run_targets[targetid]: + if item in self.run_targets: + if fn in self.run_targets[item]: return - self.run_targets[targetid].append(fnid) + self.run_targets[item].append(fn) return - self.run_targets[targetid] = [fnid] + self.run_targets[item] = [fn] - def mark_external_target(self, item): + def mark_external_target(self, target): """ Mark a build target as being externally requested """ - targetid = self.getbuild_id(item) - - if targetid not in self.external_targets: - self.external_targets.append(targetid) + if target not in self.external_targets: + self.external_targets.append(target) def get_unresolved_build_targets(self, dataCache): """ @@ -324,12 +246,12 @@ class TaskData: are unknown. """ unresolved = [] - for target in self.build_names_index: + for target in self.build_targets: if re_match_strings(target, dataCache.ignored_dependencies): continue - if self.build_names_index.index(target) in self.failed_deps: + if target in self.failed_deps: continue - if not self.have_build_target(target): + if not self.build_targets[target]: unresolved.append(target) return unresolved @@ -339,12 +261,12 @@ class TaskData: are unknown. """ unresolved = [] - for target in self.run_names_index: + for target in self.run_targets: if re_match_strings(target, dataCache.ignored_dependencies): continue - if self.run_names_index.index(target) in self.failed_rdeps: + if target in self.failed_rdeps: continue - if not self.have_runtime_target(target): + if not self.run_targets[target]: unresolved.append(target) return unresolved @@ -352,50 +274,26 @@ class TaskData: """ Return a list of providers of item """ - targetid = self.getbuild_id(item) + return self.build_targets[item] - return self.build_targets[targetid] - - def get_dependees(self, itemid): + def get_dependees(self, item): """ Return a list of targets which depend on item """ dependees = [] - for fnid in self.depids: - if itemid in self.depids[fnid]: - dependees.append(fnid) - return dependees - - def get_dependees_str(self, item): - """ - Return a list of targets which depend on item as a user readable string - """ - itemid = self.getbuild_id(item) - dependees = [] - for fnid in self.depids: - if itemid in self.depids[fnid]: - dependees.append(self.fn_index[fnid]) + for fn in self.depids: + if item in self.depids[fn]: + dependees.append(fn) return dependees - def get_rdependees(self, itemid): + def get_rdependees(self, item): """ Return a list of targets which depend on runtime item """ dependees = [] - for fnid in self.rdepids: - if itemid in self.rdepids[fnid]: - dependees.append(fnid) - return dependees - - def get_rdependees_str(self, item): - """ - Return a list of targets which depend on runtime item as a user readable string - """ - itemid = self.getrun_id(item) - dependees = [] - for fnid in self.rdepids: - if itemid in self.rdepids[fnid]: - dependees.append(self.fn_index[fnid]) + for fn in self.rdepids: + if item in self.rdepids[fn]: + dependees.append(fn) return dependees def get_reasons(self, item, runtime=False): @@ -431,7 +329,7 @@ class TaskData: except bb.providers.NoProvider: if self.abort: raise - self.remove_buildtarget(self.getbuild_id(item)) + self.remove_buildtarget(item) self.mark_external_target(item) @@ -453,7 +351,7 @@ class TaskData: new = dataCache.pkg_fn[fn] + " RPROVIDES " + item if new not in close_matches: close_matches.append(new) - bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees_str(item), reasons=self.get_reasons(item), close_matches=close_matches), cfgData) + bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=self.get_reasons(item), close_matches=close_matches), cfgData) raise bb.providers.NoProvider(item) if self.have_build_target(item): @@ -462,10 +360,10 @@ class TaskData: all_p = dataCache.providers[item] eligible, foundUnique = bb.providers.filterProviders(all_p, item, cfgData, dataCache) - eligible = [p for p in eligible if not self.getfn_id(p) in self.failed_fnids] + eligible = [p for p in eligible if not p in self.failed_fns] if not eligible: - bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees_str(item), reasons=["No eligible PROVIDERs exist for '%s'" % item]), cfgData) + bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=["No eligible PROVIDERs exist for '%s'" % item]), cfgData) raise bb.providers.NoProvider(item) if len(eligible) > 1 and foundUnique == False: @@ -477,8 +375,7 @@ class TaskData: self.consider_msgs_cache.append(item) for fn in eligible: - fnid = self.getfn_id(fn) - if fnid in self.failed_fnids: + if fn in self.failed_fns: continue logger.debug(2, "adding %s to satisfy %s", fn, item) self.add_build_target(fn, item) @@ -502,14 +399,14 @@ class TaskData: all_p = bb.providers.getRuntimeProviders(dataCache, item) if not all_p: - bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees_str(item), reasons=self.get_reasons(item, True)), cfgData) + bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=self.get_reasons(item, True)), cfgData) raise bb.providers.NoRProvider(item) eligible, numberPreferred = bb.providers.filterProvidersRunTime(all_p, item, cfgData, dataCache) - eligible = [p for p in eligible if not self.getfn_id(p) in self.failed_fnids] + eligible = [p for p in eligible if not p in self.failed_fns] if not eligible: - bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees_str(item), reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData) + bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData) raise bb.providers.NoRProvider(item) if len(eligible) > 1 and numberPreferred == 0: @@ -531,82 +428,80 @@ class TaskData: # run through the list until we find one that we can build for fn in eligible: - fnid = self.getfn_id(fn) - if fnid in self.failed_fnids: + if fn in self.failed_fns: continue logger.debug(2, "adding '%s' to satisfy runtime '%s'", fn, item) self.add_runtime_target(fn, item) self.add_tasks(fn, dataCache) - def fail_fnid(self, fnid, missing_list=None): + def fail_fn(self, fn, missing_list=None): """ Mark a file as failed (unbuildable) Remove any references from build and runtime provider lists missing_list, A list of missing requirements for this target """ - if fnid in self.failed_fnids: + if fn in self.failed_fns: return if not missing_list: missing_list = [] - logger.debug(1, "File '%s' is unbuildable, removing...", self.fn_index[fnid]) - self.failed_fnids.append(fnid) + logger.debug(1, "File '%s' is unbuildable, removing...", fn) + self.failed_fns.append(fn) for target in self.build_targets: - if fnid in self.build_targets[target]: - self.build_targets[target].remove(fnid) + if fn in self.build_targets[target]: + self.build_targets[target].remove(fn) if len(self.build_targets[target]) == 0: self.remove_buildtarget(target, missing_list) for target in self.run_targets: - if fnid in self.run_targets[target]: - self.run_targets[target].remove(fnid) + if fn in self.run_targets[target]: + self.run_targets[target].remove(fn) if len(self.run_targets[target]) == 0: self.remove_runtarget(target, missing_list) - def remove_buildtarget(self, targetid, missing_list=None): + def remove_buildtarget(self, target, missing_list=None): """ Mark a build target as failed (unbuildable) Trigger removal of any files that have this as a dependency """ if not missing_list: - missing_list = [self.build_names_index[targetid]] + missing_list = [target] else: - missing_list = [self.build_names_index[targetid]] + missing_list - logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", self.build_names_index[targetid], missing_list) - self.failed_deps.append(targetid) - dependees = self.get_dependees(targetid) - for fnid in dependees: - self.fail_fnid(fnid, missing_list) - for taskid in range(len(self.tasks_idepends)): - idepends = self.tasks_idepends[taskid] - for (idependid, idependtask) in idepends: - if idependid == targetid: - self.fail_fnid(self.tasks_fnid[taskid], missing_list) - - if self.abort and targetid in self.external_targets: - target = self.build_names_index[targetid] + missing_list = [target] + missing_list + logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list) + self.failed_deps.append(target) + dependees = self.get_dependees(target) + for fn in dependees: + self.fail_fn(fn, missing_list) + for tid in self.taskentries: + for (idepend, idependtask) in self.taskentries[tid].idepends: + if idepend == target: + fn = tid.rsplit(":",1)[0] + self.fail_fn(fn, missing_list) + + if self.abort and target in self.external_targets: logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list) raise bb.providers.NoProvider(target) - def remove_runtarget(self, targetid, missing_list=None): + def remove_runtarget(self, target, missing_list=None): """ Mark a run target as failed (unbuildable) Trigger removal of any files that have this as a dependency """ if not missing_list: - missing_list = [self.run_names_index[targetid]] + missing_list = [target] else: - missing_list = [self.run_names_index[targetid]] + missing_list - - logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", self.run_names_index[targetid], missing_list) - self.failed_rdeps.append(targetid) - dependees = self.get_rdependees(targetid) - for fnid in dependees: - self.fail_fnid(fnid, missing_list) - for taskid in range(len(self.tasks_irdepends)): - irdepends = self.tasks_irdepends[taskid] - for (idependid, idependtask) in irdepends: - if idependid == targetid: - self.fail_fnid(self.tasks_fnid[taskid], missing_list) + missing_list = [target] + missing_list + + logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list) + self.failed_rdeps.append(target) + dependees = self.get_rdependees(target) + for fn in dependees: + self.fail_fn(fn, missing_list) + for tid in self.taskentries: + for (idepend, idependtask) in self.taskentries[tid].irdepends: + if idepend == target: + fn = tid.rsplit(":",1)[0] + self.fail_fn(fn, missing_list) def add_unresolved(self, cfgData, dataCache): """ @@ -620,17 +515,16 @@ class TaskData: self.add_provider_internal(cfgData, dataCache, target) added = added + 1 except bb.providers.NoProvider: - targetid = self.getbuild_id(target) - if self.abort and targetid in self.external_targets and not self.allowincomplete: + if self.abort and target in self.external_targets and not self.allowincomplete: raise if not self.allowincomplete: - self.remove_buildtarget(targetid) + self.remove_buildtarget(target) for target in self.get_unresolved_run_targets(dataCache): try: self.add_rprovider(cfgData, dataCache, target) added = added + 1 except (bb.providers.NoRProvider, bb.providers.MultipleRProvider): - self.remove_runtarget(self.getrun_id(target)) + self.remove_runtarget(target) logger.debug(1, "Resolved " + str(added) + " extra dependencies") if added == 0: break @@ -638,13 +532,13 @@ class TaskData: def get_providermap(self, prefix=None): provmap = {} - for name in self.build_names_index: + for name in self.build_targets: if prefix and not name.startswith(prefix): continue if self.have_build_target(name): provider = self.get_provider(name) if provider: - provmap[name] = self.fn_index[provider[0]] + provmap[name] = provider[0] return provmap def dump_data(self): @@ -652,39 +546,37 @@ class TaskData: Dump some debug information on the internal data structures """ logger.debug(3, "build_names:") - logger.debug(3, ", ".join(self.build_names_index)) + logger.debug(3, ", ".join(self.build_targets)) logger.debug(3, "run_names:") - logger.debug(3, ", ".join(self.run_names_index)) + logger.debug(3, ", ".join(self.run_targets)) logger.debug(3, "build_targets:") - for buildid in range(len(self.build_names_index)): - target = self.build_names_index[buildid] + for target in self.build_targets: targets = "None" - if buildid in self.build_targets: - targets = self.build_targets[buildid] - logger.debug(3, " (%s)%s: %s", buildid, target, targets) + if target in self.build_targets: + targets = self.build_targets[target] + logger.debug(3, " %s: %s", target, targets) logger.debug(3, "run_targets:") - for runid in range(len(self.run_names_index)): - target = self.run_names_index[runid] + for target in self.run_targets: targets = "None" - if runid in self.run_targets: - targets = self.run_targets[runid] - logger.debug(3, " (%s)%s: %s", runid, target, targets) + if target in self.run_targets: + targets = self.run_targets[target] + logger.debug(3, " %s: %s", target, targets) logger.debug(3, "tasks:") - for task in range(len(self.tasks_name)): - logger.debug(3, " (%s)%s - %s: %s", - task, - self.fn_index[self.tasks_fnid[task]], - self.tasks_name[task], - self.tasks_tdepends[task]) + for tid in self.taskentries: + logger.debug(3, " %s: %s %s %s", + tid, + self.taskentries[tid].idepends, + self.taskentries[tid].irdepends, + self.taskentries[tid].tdepends) logger.debug(3, "dependency ids (per fn):") - for fnid in self.depids: - logger.debug(3, " %s %s: %s", fnid, self.fn_index[fnid], self.depids[fnid]) + for fn in self.depids: + logger.debug(3, " %s: %s", fn, self.depids[fn]) logger.debug(3, "runtime dependency ids (per fn):") - for fnid in self.rdepids: - logger.debug(3, " %s %s: %s", fnid, self.fn_index[fnid], self.rdepids[fnid]) + for fn in self.rdepids: + logger.debug(3, " %s: %s", fn, self.rdepids[fn]) |