diff options
Diffstat (limited to 'lib/bb/cooker.py')
| -rw-r--r-- | lib/bb/cooker.py | 139 |
1 files changed, 82 insertions, 57 deletions
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index e527e2311..8f45233c8 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -525,7 +525,7 @@ class BBCooker: self.parseConfiguration() fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) - fn = self.matchFile(fn) + fn = self.matchFile(fn, mc) fn = bb.cache.realfn2virtual(fn, cls, mc) elif len(pkgs_to_build) == 1: mc = mc_base(pkgs_to_build[0]) @@ -542,7 +542,7 @@ class BBCooker: if fn: try: bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) - envdata = bb_cache.loadDataFull(fn, self.collection.get_file_appends(fn)) + envdata = bb_cache.loadDataFull(fn, self.collections[mc].get_file_appends(fn)) except Exception as e: parselog.exception("Unable to read %s", fn) raise @@ -929,26 +929,33 @@ class BBCooker: logger.info("Task dependencies saved to 'task-depends.dot'") def show_appends_with_no_recipes(self): + appends_without_recipes = {} # Determine which bbappends haven't been applied - - # First get list of recipes, including skipped - recipefns = list(self.recipecaches[''].pkg_fn.keys()) - recipefns.extend(self.skiplist.keys()) - - # Work out list of bbappends that have been applied - applied_appends = [] - for fn in recipefns: - applied_appends.extend(self.collection.get_file_appends(fn)) - - appends_without_recipes = [] - for _, appendfn in self.collection.bbappends: - if not appendfn in applied_appends: - appends_without_recipes.append(appendfn) - - if appends_without_recipes: - msg = 'No recipes available for:\n %s' % '\n '.join(appends_without_recipes) - warn_only = self.data.getVar("BB_DANGLINGAPPENDS_WARNONLY", \ - False) or "no" + for mc in self.multiconfigs: + # First get list of recipes, including skipped + recipefns = list(self.recipecaches[mc].pkg_fn.keys()) + recipefns.extend(self.skiplist.keys()) + + # Work out list of bbappends that have been applied + applied_appends = [] + for fn in recipefns: + applied_appends.extend(self.collections[mc].get_file_appends(fn)) + + appends_without_recipes[mc] = [] + for _, appendfn in self.collections[mc].bbappends: + if not appendfn in applied_appends: + appends_without_recipes[mc].append(appendfn) + + msgs = [] + for mc in sorted(appends_without_recipes.keys()): + if appends_without_recipes[mc]: + msgs.append('No recipes in %s available for:\n %s' % (mc if mc else 'default', + '\n '.join(appends_without_recipes[mc]))) + + if msgs: + msg = "\n".join(msgs) + warn_only = self.databuilder.mcdata[mc].getVar("BB_DANGLINGAPPENDS_WARNONLY", \ + False) or "no" if warn_only.lower() in ("1", "yes", "true"): bb.warn(msg) else: @@ -1249,15 +1256,15 @@ class BBCooker: if siggen_cache: bb.parse.siggen.checksum_cache.mtime_cache.clear() - def matchFiles(self, bf): + def matchFiles(self, bf, mc=''): """ Find the .bb files which match the expression in 'buildfile'. """ if bf.startswith("/") or bf.startswith("../"): bf = os.path.abspath(bf) - self.collection = CookerCollectFiles(self.bbfile_config_priorities) - filelist, masked, searchdirs = self.collection.collect_bbfiles(self.data, self.data) + self.collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)} + filelist, masked, searchdirs = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc]) try: os.stat(bf) bf = os.path.abspath(bf) @@ -1270,12 +1277,12 @@ class BBCooker: matches.append(f) return matches - def matchFile(self, buildfile): + def matchFile(self, buildfile, mc=''): """ Find the .bb file which matches the expression in 'buildfile'. Raise an error if multiple files """ - matches = self.matchFiles(buildfile) + matches = self.matchFiles(buildfile, mc) if len(matches) != 1: if matches: msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches)) @@ -1316,14 +1323,14 @@ class BBCooker: task = "do_%s" % task fn, cls, mc = bb.cache.virtualfn2realfn(buildfile) - fn = self.matchFile(fn) + fn = self.matchFile(fn, mc) self.buildSetVars() self.reset_mtime_caches() bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) - infos = bb_cache.parse(fn, self.collection.get_file_appends(fn)) + infos = bb_cache.parse(fn, self.collections[mc].get_file_appends(fn)) infos = dict(infos) fn = bb.cache.realfn2virtual(fn, cls, mc) @@ -1552,14 +1559,24 @@ class BBCooker: for dep in self.configuration.extra_assume_provided: self.recipecaches[mc].ignored_dependencies.add(dep) - self.collection = CookerCollectFiles(self.bbfile_config_priorities) - (filelist, masked, searchdirs) = self.collection.collect_bbfiles(self.data, self.data) + self.collections = {} + + mcfilelist = {} + total_masked = 0 + searchdirs = set() + for mc in self.multiconfigs: + self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc) + (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc]) + + mcfilelist[mc] = filelist + total_masked += masked + searchdirs |= set(search) # Add inotify watches for directories searched for bb/bbappend files for dirent in searchdirs: self.add_filewatch([[dirent]], dirs=True) - self.parser = CookerParser(self, filelist, masked) + self.parser = CookerParser(self, mcfilelist, total_masked) self.parsecache_valid = True self.state = state.parsing @@ -1571,7 +1588,7 @@ class BBCooker: self.show_appends_with_no_recipes() self.handlePrefProviders() for mc in self.multiconfigs: - self.recipecaches[mc].bbfile_priority = self.collection.collection_priorities(self.recipecaches[mc].pkg_fn, self.data) + self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.data) self.state = state.running # Send an event listing all stamps reachable after parsing @@ -1679,7 +1696,8 @@ class CookerExit(bb.event.Event): class CookerCollectFiles(object): - def __init__(self, priorities): + def __init__(self, priorities, mc=''): + self.mc = mc self.bbappends = [] # Priorities is a list of tupples, with the second element as the pattern. # We need to sort the list with the longest pattern first, and so on to @@ -1846,7 +1864,7 @@ class CookerCollectFiles(object): (bbappend, filename) = b if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])): filelist.append(filename) - return filelist + return tuple(filelist) def collection_priorities(self, pkgfns, d): @@ -1882,7 +1900,8 @@ class CookerCollectFiles(object): for collection, pattern, regex, _ in self.bbfile_config_priorities: if regex in unmatched: if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1': - collectlog.warning("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern)) + collectlog.warning("No bb files in %s matched BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default', + collection, pattern)) return priorities @@ -1978,8 +1997,8 @@ class Parser(multiprocessing.Process): bb.event.LogHandler.filter = origfilter class CookerParser(object): - def __init__(self, cooker, filelist, masked): - self.filelist = filelist + def __init__(self, cooker, mcfilelist, masked): + self.mcfilelist = mcfilelist self.cooker = cooker self.cfgdata = cooker.data self.cfghash = cooker.data_hash @@ -1993,25 +2012,27 @@ class CookerParser(object): self.skipped = 0 self.virtuals = 0 - self.total = len(filelist) self.current = 0 self.process_names = [] self.bb_cache = bb.cache.Cache(self.cfgbuilder, self.cfghash, cooker.caches_array) - self.fromcache = [] - self.willparse = [] - for filename in self.filelist: - appends = self.cooker.collection.get_file_appends(filename) - if not self.bb_cache.cacheValid(filename, appends): - self.willparse.append((filename, appends)) - else: - self.fromcache.append((filename, appends)) - self.toparse = self.total - len(self.fromcache) + self.fromcache = set() + self.willparse = set() + for mc in self.cooker.multiconfigs: + for filename in self.mcfilelist[mc]: + appends = self.cooker.collections[mc].get_file_appends(filename) + if not self.bb_cache.cacheValid(filename, appends): + self.willparse.add((filename, appends)) + else: + self.fromcache.add((filename, appends)) + + self.total = len(self.fromcache) + len(self.willparse) + self.toparse = len(self.willparse) self.progress_chunk = int(max(self.toparse / 100, 1)) self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or - multiprocessing.cpu_count()), len(self.willparse)) + multiprocessing.cpu_count()), self.toparse) self.start() self.haveshutdown = False @@ -2032,7 +2053,7 @@ class CookerParser(object): def chunkify(lst,n): return [lst[i::n] for i in range(n)] - self.jobs = chunkify(self.willparse, self.num_processes) + self.jobs = chunkify(list(self.willparse), self.num_processes) for i in range(0, self.num_processes): parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) @@ -2095,9 +2116,9 @@ class CookerParser(object): print("Processed parsing statistics saved to %s" % (pout)) def load_cached(self): - for filename, appends in self.fromcache: - cached, infos = self.bb_cache.load(filename, appends) - yield not cached, infos + for mc, filename, appends in self.fromcache: + cached, infos = self.bb_cache.load(mc, filename, appends) + yield not cached, mc, infos def parse_generator(self): while True: @@ -2119,7 +2140,7 @@ class CookerParser(object): result = [] parsed = None try: - parsed, result = next(self.results) + parsed, mc, result = next(self.results) except StopIteration: self.shutdown() return False @@ -2181,7 +2202,11 @@ class CookerParser(object): return True def reparse(self, filename): - infos = self.bb_cache.parse(filename, self.cooker.collection.get_file_appends(filename)) - for vfn, info_array in infos: - (fn, cls, mc) = bb.cache.virtualfn2realfn(vfn) - self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array) + to_reparse = set() + for mc in self.cooker.multiconfigs: + to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename))) + + for mc, filename, appends in to_reparse: + infos = self.bb_cache.parse(filename, appends) + for vfn, info_array in infos: + self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array) |