From 1f5a662176fd174c4793e3e28d1b4e95616b1715 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Sat, 14 Dec 2019 18:17:01 +0000 Subject: siggen: Cache unihash values to avoid cache lookup Add unihash cache of values to speed up cache lookup. This avoids the overhead of the disk based check functions. Signed-off-by: Richard Purdie (cherry picked from commit 5c9cc45b60904a1c355db9bf9c4495f1b25aca37) Signed-off-by: Armin Kuster --- lib/bb/siggen.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py index 209a34288..96807c46c 100644 --- a/lib/bb/siggen.py +++ b/lib/bb/siggen.py @@ -40,6 +40,7 @@ class SignatureGenerator(object): def __init__(self, data): self.basehash = {} self.taskhash = {} + self.unihash = {} self.runtaskdeps = {} self.file_checksum_values = {} self.taints = {} @@ -80,19 +81,19 @@ class SignatureGenerator(object): return def get_taskdata(self): - return (self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash, self.unitaskhashes, self.tidtopn, self.setscenetasks) + return (self.runtaskdeps, self.taskhash, self.unihash, self.file_checksum_values, self.taints, self.basehash, self.unitaskhashes, self.tidtopn, self.setscenetasks) def set_taskdata(self, data): - self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash, self.unitaskhashes, self.tidtopn, self.setscenetasks = data + self.runtaskdeps, self.taskhash, self.unihash, self.file_checksum_values, self.taints, self.basehash, self.unitaskhashes, self.tidtopn, self.setscenetasks = data def reset(self, data): self.__init__(data) def get_taskhashes(self): - return self.taskhash, self.unitaskhashes, self.tidtopn + return self.taskhash, self.unihash, self.unitaskhashes, self.tidtopn def set_taskhashes(self, hashes): - self.taskhash, self.unitaskhashes, self.tidtopn = hashes + self.taskhash, self.unihash, self.unitaskhashes, self.tidtopn = hashes def save_unitaskhashes(self): return @@ -108,6 +109,7 @@ class SignatureGeneratorBasic(SignatureGenerator): def __init__(self, data): self.basehash = {} self.taskhash = {} + self.unihash = {} self.taskdeps = {} self.runtaskdeps = {} self.file_checksum_values = {} @@ -256,7 +258,13 @@ class SignatureGeneratorBasic(SignatureGenerator): data = self.basehash[tid] for dep in self.runtaskdeps[tid]: - data = data + self.get_unihash(dep) + if dep in self.unihash: + if self.unihash[dep] is None: + data = data + self.taskhash[dep] + else: + data = data + self.unihash[dep] + else: + data = data + self.get_unihash(dep) for (f, cs) in self.file_checksum_values[tid]: if cs: @@ -427,6 +435,7 @@ class SignatureGeneratorUniHashMixIn(object): (mc, fn, taskname, taskfn) = bb.runqueue.split_tid_mcfn(tid) key = mc + ":" + self.tidtopn[tid] + ":" + taskname self.unitaskhashes[key] = (self.taskhash[tid], unihash) + self.unihash[tid] = unihash def _get_unihash(self, tid, checkkey=None): if tid not in self.tidtopn: @@ -447,12 +456,14 @@ class SignatureGeneratorUniHashMixIn(object): # If its not a setscene task we can return if self.setscenetasks and tid not in self.setscenetasks: + self.unihash[tid] = None return taskhash # TODO: This cache can grow unbounded. It probably only needs to keep # for each task unihash = self._get_unihash(tid) if unihash is not None: + self.unihash[tid] = unihash return unihash # In the absence of being able to discover a unique hash from the @@ -487,6 +498,7 @@ class SignatureGeneratorUniHashMixIn(object): bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e))) self.set_unihash(tid, unihash) + self.unihash[tid] = unihash return unihash def report_unihash(self, path, task, d): -- cgit 1.2.3-korg