diff options
| -rwxr-xr-x | bin/bitbake | 3 | ||||
| -rwxr-xr-x | bin/bitbake-diffsigs | 12 | ||||
| -rw-r--r-- | lib/bb/cache.py | 21 | ||||
| -rw-r--r-- | lib/bb/cooker.py | 2 | ||||
| -rw-r--r-- | lib/bb/parse/__init__.py | 4 | ||||
| -rw-r--r-- | lib/bb/parse/ast.py | 6 | ||||
| -rw-r--r-- | lib/bb/runqueue.py | 45 | ||||
| -rw-r--r-- | lib/bb/siggen.py | 256 | ||||
| -rw-r--r-- | lib/bb/utils.py | 1 |
9 files changed, 342 insertions, 8 deletions
diff --git a/bin/bitbake b/bin/bitbake index 63b1d1d12..e14c017c1 100755 --- a/bin/bitbake +++ b/bin/bitbake @@ -135,6 +135,9 @@ Default BBFILES are the .bb files in the current directory.""") parser.add_option("-n", "--dry-run", help = "don't execute, just go through the motions", action = "store_true", dest = "dry_run", default = False) + parser.add_option("-S", "--dump-signatures", help = "don't execute, just dump out the signature construction information", + action = "store_true", dest = "dump_signatures", default = False) + parser.add_option("-p", "--parse-only", help = "quit after parsing the BB files (developers only)", action = "store_true", dest = "parse_only", default = False) diff --git a/bin/bitbake-diffsigs b/bin/bitbake-diffsigs new file mode 100755 index 000000000..5eb77ce59 --- /dev/null +++ b/bin/bitbake-diffsigs @@ -0,0 +1,12 @@ +#!/usr/bin/env python +import os +import sys +import warnings +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib')) + +import bb.siggen + +if len(sys.argv) > 2: + bb.siggen.compare_sigfiles(sys.argv[1], sys.argv[2]) +else: + bb.siggen.dump_sigfile(sys.argv[1]) diff --git a/lib/bb/cache.py b/lib/bb/cache.py index d72622d74..2d6eac1a9 100644 --- a/lib/bb/cache.py +++ b/lib/bb/cache.py @@ -43,7 +43,7 @@ except ImportError: logger.info("Importing cPickle failed. " "Falling back to a very slow implementation.") -__cache_version__ = "133" +__cache_version__ = "134" recipe_fields = ( 'pn', @@ -70,6 +70,8 @@ recipe_fields = ( 'nocache', 'variants', 'file_depends', + 'tasks', + 'basetaskhashes', ) @@ -94,20 +96,32 @@ class RecipeInfo(namedtuple('RecipeInfo', recipe_fields)): for pkg in packages) @classmethod + def taskvar(cls, var, tasks, metadata): + return dict((task, cls.getvar("%s_task-%s" % (var, task), metadata)) + for task in tasks) + + @classmethod def getvar(cls, var, metadata): return metadata.getVar(var, True) or '' @classmethod def from_metadata(cls, filename, metadata): + tasks = metadata.getVar('__BBTASKS', False) + pn = cls.getvar('PN', metadata) packages = cls.listvar('PACKAGES', metadata) if not pn in packages: packages.append(pn) + return RecipeInfo( + tasks = tasks, + basetaskhashes = cls.taskvar('BB_BASEHASH', tasks, metadata), + file_depends = metadata.getVar('__depends', False), task_deps = metadata.getVar('_task_deps', False) or {'tasks': [], 'parents': {}}, variants = cls.listvar('__VARIANTS', metadata) + [''], + skipped = cls.getvar('__SKIPPED', metadata), timestamp = bb.parse.cached_mtime(filename), packages = cls.listvar('PACKAGES', metadata), @@ -538,6 +552,8 @@ class CacheData(object): self.task_deps = {} self.stamp = {} self.preferred = {} + self.tasks = {} + self.basetaskhash = {} """ Indirect Cache variables @@ -594,3 +610,6 @@ class CacheData(object): if not info.broken and not info.not_world: self.possible_world.append(fn) + for task, taskhash in info.basetaskhashes.iteritems(): + identifier = '%s.%s' % (fn, task) + self.basetaskhash[identifier] = taskhash diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index 1df2bfe8a..0e341158b 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -485,6 +485,7 @@ class BBCooker: sys.exit(1) data = self.configuration.data + bb.parse.init_parser(data) for f in files: data = _parse(f, data) @@ -523,6 +524,7 @@ class BBCooker: bb.fetch.fetcher_init(self.configuration.data) bb.codeparser.parser_cache_init(self.configuration.data) + bb.parse.init_parser(data) bb.event.fire(bb.event.ConfigParsed(), self.configuration.data) def handleCollections( self, collections ): diff --git a/lib/bb/parse/__init__.py b/lib/bb/parse/__init__.py index 3015d0c8d..7f1562e66 100644 --- a/lib/bb/parse/__init__.py +++ b/lib/bb/parse/__init__.py @@ -30,6 +30,7 @@ import os import logging import bb import bb.utils +import bb.siggen logger = logging.getLogger("BitBake.Parsing") @@ -83,6 +84,9 @@ def init(fn, data): if h['supports'](fn): return h['init'](data) +def init_parser(d): + bb.parse.siggen = bb.siggen.init(d) + def resolve_file(fn, d): if not os.path.isabs(fn): bbpath = bb.data.getVar("BBPATH", d, True) diff --git a/lib/bb/parse/ast.py b/lib/bb/parse/ast.py index 8a02c10f6..c001f1b45 100644 --- a/lib/bb/parse/ast.py +++ b/lib/bb/parse/ast.py @@ -300,7 +300,7 @@ def handleInherit(statements, m): n = __word__.findall(files) statements.append(InheritNode(m.group(1))) -def finalize(fn, d): +def finalize(fn, d, variant = None): for lazykey in bb.data.getVar("__lazy_assigned", d) or (): if bb.data.getVar(lazykey, d) is None: val = bb.data.getVarFlag(lazykey, "defaultval", d) @@ -323,7 +323,7 @@ def finalize(fn, d): tasklist = bb.data.getVar('__BBTASKS', d) or [] bb.build.add_tasks(tasklist, d) - #bb.data.generate_dependencies(d) + bb.parse.siggen.finalise(fn, d, variant) bb.event.fire(bb.event.RecipeParsed(fn), d) @@ -433,7 +433,7 @@ def multi_finalize(fn, d): for variant, variant_d in datastores.iteritems(): if variant: try: - finalize(fn, variant_d) + finalize(fn, variant_d, variant) except bb.parse.SkipPackage: bb.data.setVar("__SKIPPED", True, variant_d) diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py index 087edace3..076d3b8f4 100644 --- a/lib/bb/runqueue.py +++ b/lib/bb/runqueue.py @@ -188,6 +188,7 @@ class RunQueueData: self.runq_task = [] self.runq_depends = [] self.runq_revdeps = [] + self.runq_hash = [] def runq_depends_names(self, ids): import re @@ -489,6 +490,7 @@ class RunQueueData: self.runq_task.append(taskData.tasks_name[task]) self.runq_depends.append(set(depends)) self.runq_revdeps.append(set()) + self.runq_hash.append("") runq_build.append(0) runq_recrdepends.append(recrdepends) @@ -601,6 +603,7 @@ class RunQueueData: del self.runq_depends[listid-delcount] del runq_build[listid-delcount] del self.runq_revdeps[listid-delcount] + del self.runq_hash[listid-delcount] delcount = delcount + 1 maps.append(-1) @@ -692,6 +695,19 @@ class RunQueueData: continue self.runq_setscene.append(task) + # Interate over the task list and call into the siggen code + dealtwith = set() + todeal = set(range(len(self.runq_fnid))) + while len(todeal) > 0: + for task in todeal.copy(): + if len(self.runq_depends[task] - dealtwith) == 0: + dealtwith.add(task) + todeal.remove(task) + procdep = [] + for dep in self.runq_depends[task]: + procdep.append(self.taskData.fn_index[self.runq_fnid[dep]] + "." + self.runq_task[dep]) + self.runq_hash[task] = bb.parse.siggen.get_taskhash(self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task], procdep, self.dataCache) + return len(self.runq_fnid) def dump_data(self, taskQueue): @@ -843,7 +859,9 @@ class RunQueue: fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] if taskname is None: taskname = self.rqdata.runq_task[task] - stampfile = "%s.%s" % (self.rqdata.dataCache.stamp[fn], taskname) + + stampfile = bb.parse.siggen.stampfile(self.rqdata.dataCache.stamp[fn], taskname, self.rqdata.runq_hash[task]) + # If the stamp is missing its not current if not os.access(stampfile, os.F_OK): logger.debug(2, "Stampfile %s not available", stampfile) @@ -863,9 +881,10 @@ class RunQueue: if iscurrent: fn2 = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[dep]] taskname2 = self.rqdata.runq_task[dep] - stampfile2 = "%s.%s" % (self.rqdata.dataCache.stamp[fn2], taskname2) + stampfile2 = bb.parse.siggen.stampfile(self.rqdata.dataCache.stamp[fn2], taskname2, self.rqdata.runq_hash[dep]) + stampfile3 = bb.parse.siggen.stampfile(self.rqdata.dataCache.stamp[fn2], taskname2 + "_setscene", self.rqdata.runq_hash[dep]) t2 = get_timestamp(stampfile2) - t3 = get_timestamp(stampfile2 + "_setscene") + t3 = get_timestamp(stampfile3) if t3 and t3 > t2: continue if fn == fn2 or (fulldeptree and fn2 not in stampwhitelist): @@ -892,7 +911,10 @@ class RunQueue: self.state = runQueueSceneInit if self.state is runQueueSceneInit: - self.rqexe = RunQueueExecuteScenequeue(self) + if self.cooker.configuration.dump_signatures: + self.dump_signatures() + else: + self.rqexe = RunQueueExecuteScenequeue(self) if self.state is runQueueSceneRun: self.rqexe.execute() @@ -933,6 +955,20 @@ class RunQueue: else: self.rqexe.finish() + def dump_signatures(self): + self.state = runQueueComplete + done = set() + bb.note("Reparsing files to collect dependency data") + for task in range(len(self.rqdata.runq_fnid)): + if self.rqdata.runq_fnid[task] not in done: + fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] + the_data = self.cooker.bb_cache.loadDataFull(fn, self.cooker.get_file_appends(fn), self.cooker.configuration.data) + done.add(self.rqdata.runq_fnid[task]) + + bb.parse.siggen.dump_sigs(self.rqdata.dataCache) + + return + class RunQueueExecute: @@ -1024,6 +1060,7 @@ class RunQueueExecute: bb.data.setVar("__RUNQUEUE_DO_NOT_USE_EXTERNALLY2", fn, self.cooker.configuration.data) try: the_data = bb.cache.Cache.loadDataFull(fn, self.cooker.get_file_appends(fn), self.cooker.configuration.data) + the_data.setVar('BB_TASKHASH', self.rqdata.runq_hash[task]) bb.build.exec_task(fn, taskname, the_data) except Exception as exc: logger.critical(str(exc)) diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py new file mode 100644 index 000000000..da8503924 --- /dev/null +++ b/lib/bb/siggen.py @@ -0,0 +1,256 @@ +import hashlib +import logging +import re + +logger = logging.getLogger('BitBake.SigGen') + +try: + import cPickle as pickle +except ImportError: + import pickle + logger.info('Importing cPickle failed. Falling back to a very slow implementation.') + +def init(d): + siggens = [obj for obj in globals().itervalues() + if type(obj) is type and issubclass(obj, SignatureGenerator)] + + desired = bb.data.getVar("BB_SIGNATURE_HANDLER", d, True) or "noop" + for sg in siggens: + if desired == sg.name: + return sg(d) + break + else: + logger.error("Invalid signature generator '%s', using default 'noop'\n" + "Available generators: %s", + ', '.join(obj.name for obj in siggens)) + return SignatureGenerator(d) + +class SignatureGenerator(object): + """ + """ + name = "noop" + + def __init__(self, data): + return + + def finalise(self, fn, d, varient): + return + + def get_taskhash(self, fn, task, deps, dataCache): + return 0 + + def stampfile(self, stampbase, taskname, taskhash): + return "%s.%s" % (stampbase, taskname) + +class SignatureGeneratorBasic(SignatureGenerator): + """ + """ + name = "basic" + + def __init__(self, data): + self.basehash = {} + self.taskhash = {} + self.taskdeps = {} + self.runtaskdeps = {} + self.gendeps = {} + self.lookupcache = {} + self.basewhitelist = (data.getVar("BB_HASHBASE_WHITELIST", True) or "").split() + self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST", True) or None + + if self.taskwhitelist: + self.twl = re.compile(self.taskwhitelist) + else: + self.twl = None + + def _build_data(self, fn, d): + + taskdeps, gendeps = bb.data.generate_dependencies(d) + + basehash = {} + lookupcache = {} + + for task in taskdeps: + data = d.getVar(task, False) + lookupcache[task] = data + for dep in sorted(taskdeps[task]): + if dep in self.basewhitelist: + continue + if dep in lookupcache: + var = lookupcache[dep] + else: + var = d.getVar(dep, False) + lookupcache[dep] = var + if var: + data = data + var + if data is None: + bb.error("Task %s from %s seems to be empty?!" % (task, fn)) + self.basehash[fn + "." + task] = hashlib.md5(data).hexdigest() + + self.taskdeps[fn] = taskdeps + self.gendeps[fn] = gendeps + self.lookupcache[fn] = lookupcache + + return taskdeps + + def finalise(self, fn, d, variant): + + if variant: + fn = "virtual:" + variant + ":" + fn + + taskdeps = self._build_data(fn, d) + + #Slow but can be useful for debugging mismatched basehashes + #for task in self.taskdeps[fn]: + # self.dump_sigtask(fn, task, d.getVar("STAMP", True), False) + + for task in taskdeps: + d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task]) + + def get_taskhash(self, fn, task, deps, dataCache): + k = fn + "." + task + data = dataCache.basetaskhash[k] + self.runtaskdeps[k] = [] + for dep in sorted(deps): + # We only manipulate the dependencies for packages not in the whitelist + if self.twl and not self.twl.search(dataCache.pkg_fn[fn]): + # then process the actual dependencies + dep_fn = re.search("(?P<fn>.*)\..*", dep).group('fn') + if self.twl.search(dataCache.pkg_fn[dep_fn]): + continue + if dep not in self.taskhash: + bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?", dep) + data = data + self.taskhash[dep] + self.runtaskdeps[k].append(dep) + h = hashlib.md5(data).hexdigest() + self.taskhash[k] = h + #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task]) + return h + + def dump_sigtask(self, fn, task, stampbase, runtime): + k = fn + "." + task + if runtime == "customfile": + sigfile = stampbase + elif runtime: + sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[k] + else: + sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[k] + + bb.utils.mkdirhier(os.path.dirname(sigfile)) + + data = {} + data['basewhitelist'] = self.basewhitelist + data['taskwhitelist'] = self.taskwhitelist + data['taskdeps'] = self.taskdeps[fn][task] + data['basehash'] = self.basehash[k] + data['gendeps'] = {} + data['varvals'] = {} + data['varvals'][task] = self.lookupcache[fn][task] + for dep in self.taskdeps[fn][task]: + if dep in self.basewhitelist: + continue + data['gendeps'][dep] = self.gendeps[fn][dep] + data['varvals'][dep] = self.lookupcache[fn][dep] + + if runtime and runtime != "customfile": + data['runtaskdeps'] = self.runtaskdeps[k] + data['runtaskhashes'] = {} + for dep in data['runtaskdeps']: + data['runtaskhashes'][dep] = self.taskhash[dep] + + p = pickle.Pickler(file(sigfile, "wb"), -1) + p.dump(data) + + def dump_sigs(self, dataCache): + for fn in self.taskdeps: + for task in self.taskdeps[fn]: + k = fn + "." + task + if k not in self.taskhash: + continue + if dataCache.basetaskhash[k] != self.basehash[k]: + bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % k) + bb.error("The mismatched hashes were %s and %s" % (dataCache.basetaskhash[k], self.basehash[k])) + self.dump_sigtask(fn, task, dataCache.stamp[fn], True) + +def dump_this_task(outfile, d): + fn = d.getVar("BB_FILENAME", True) + task = "do_" + d.getVar("BB_CURRENTTASK", True) + bb.parse.siggen.dump_sigtask(fn, task, outfile, "customfile") + +def compare_sigfiles(a, b): + p1 = pickle.Unpickler(file(a, "rb")) + a_data = p1.load() + p2 = pickle.Unpickler(file(b, "rb")) + b_data = p2.load() + + def dict_diff(a, b): + sa = set(a.keys()) + sb = set(b.keys()) + common = sa & sb + changed = set() + for i in common: + if a[i] != b[i]: + changed.add(i) + added = sa - sb + removed = sb - sa + return changed, added, removed + + if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']: + print "basewhitelist changed from %s to %s" % (a_data['basewhitelist'], b_data['basewhitelist']) + + if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']: + print "taskwhitelist changed from %s to %s" % (a_data['taskwhitelist'], b_data['taskwhitelist']) + + if a_data['taskdeps'] != b_data['taskdeps']: + print "Task dependencies changed from %s to %s" % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps'])) + + if a_data['basehash'] != b_data['basehash']: + print "basehash changed from %s to %s" % (a_data['basehash'], b_data['basehash']) + + changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps']) + if changed: + for dep in changed: + print "List of dependencies for variable %s changed from %s to %s" % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep]) + if added: + for dep in added: + print "Dependency on variable %s was added" % (dep) + if removed: + for dep in removed: + print "Dependency on Variable %s was removed" % (dep) + + + changed, added, removed = dict_diff(a_data['varvals'], b_data['varvals']) + if changed: + for dep in changed: + print "Variable %s value changed from %s to %s" % (dep, a_data['varvals'][dep], b_data['varvals'][dep]) + if 'runtaskdeps' in a_data and 'runtaskdeps' in b_data and sorted(a_data['runtaskdeps']) != sorted(b_data['runtaskdeps']): + print "Tasks this task depends on changed from %s to %s" % (sorted(a_data['runtaskdeps']), sorted(b_data['runtaskdeps'])) + + if 'runtaskhashes' in a_data: + for dep in a_data['runtaskhashes']: + if a_data['runtaskhashes'][dep] != b_data['runtaskhashes'][dep]: + print "Hash for dependent task %s changed from %s to %s" % (dep, a_data['runtaskhashes'][dep], b_data['runtaskhashes'][dep]) + +def dump_sigfile(a): + p1 = pickle.Unpickler(file(a, "rb")) + a_data = p1.load() + + print "basewhitelist: %s" % (a_data['basewhitelist']) + + print "taskwhitelist: %s" % (a_data['taskwhitelist']) + + print "Task dependencies: %s" % (sorted(a_data['taskdeps'])) + + print "basehash: %s" % (a_data['basehash']) + + for dep in a_data['gendeps']: + print "List of dependencies for variable %s is %s" % (dep, a_data['gendeps'][dep]) + + for dep in a_data['varvals']: + print "Variable %s value is %s" % (dep, a_data['varvals'][dep]) + + if 'runtaskdeps' in a_data: + print "Tasks this task depends on: %s" % (a_data['runtaskdeps']) + + if 'runtaskhashes' in a_data: + for dep in a_data['runtaskhashes']: + print "Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep]) diff --git a/lib/bb/utils.py b/lib/bb/utils.py index b121b121c..a65825c3c 100644 --- a/lib/bb/utils.py +++ b/lib/bb/utils.py @@ -479,6 +479,7 @@ def preserved_envvars_list(): 'BB_PRESERVE_ENV', 'BB_ENV_WHITELIST', 'BB_ENV_EXTRAWHITE', + 'BB_TASKHASH', 'COLORTERM', 'DBUS_SESSION_BUS_ADDRESS', 'DESKTOP_SESSION', |