summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2019-12-13 16:07:28 +0000
committerArmin Kuster <akuster808@gmail.com>2019-12-17 22:14:47 -0800
commitc4ce216b4c5a1626764752edd21005750d05a0c2 (patch)
treece5cdb3e5f8ed29dc588b36aaed1e93902d9effc
parent480939538f5887a70d11c2f7cf74446100e25fab (diff)
downloadbitbake-c4ce216b4c5a1626764752edd21005750d05a0c2.tar.gz
siggen: Split get_tashhash for performance
There are two operations happening in get_taskhash, the building of the underlying data and the calculation of the hash. Split these into two funtions since the preparation part doesn't need to rerun when unihash changes, only the calculation does. This split allows sigificant performance improvements for hashequiv in builds where many hashes are equivalent and many hashes are changing. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> (cherry picked from commit 6a32af2808d748819f4af55c443578c8a63062b3)
-rw-r--r--lib/bb/runqueue.py1
-rw-r--r--lib/bb/siggen.py33
2 files changed, 25 insertions, 9 deletions
diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index b3648ddb5..515e9d431 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -1185,6 +1185,7 @@ class RunQueueData:
procdep = []
for dep in self.runtaskentries[tid].depends:
procdep.append(dep)
+ bb.parse.siggen.prep_taskhash(tid, procdep, self.dataCaches[mc_from_tid(tid)])
self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, procdep, self.dataCaches[mc_from_tid(tid)])
self.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid)
diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
index 2fec8599b..e484e5e37 100644
--- a/lib/bb/siggen.py
+++ b/lib/bb/siggen.py
@@ -52,6 +52,9 @@ class SignatureGenerator(object):
def get_unihash(self, tid):
return self.taskhash[tid]
+ def prep_taskhash(self, tid, deps, dataCache):
+ return
+
def get_taskhash(self, tid, deps, dataCache):
self.taskhash[tid] = hashlib.sha256(tid.encode("utf-8")).hexdigest()
return self.taskhash[tid]
@@ -198,12 +201,11 @@ class SignatureGeneratorBasic(SignatureGenerator):
pass
return taint
- def get_taskhash(self, tid, deps, dataCache):
+ def prep_taskhash(self, tid, deps, dataCache):
(mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid)
- data = dataCache.basetaskhash[tid]
- self.basehash[tid] = data
+ self.basehash[tid] = dataCache.basetaskhash[tid]
self.runtaskdeps[tid] = []
self.file_checksum_values[tid] = []
recipename = dataCache.pkg_fn[fn]
@@ -216,7 +218,6 @@ class SignatureGeneratorBasic(SignatureGenerator):
continue
if dep not in self.taskhash:
bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?" % dep)
- data = data + self.get_unihash(dep)
self.runtaskdeps[tid].append(dep)
if task in dataCache.file_checksums[fn]:
@@ -226,27 +227,41 @@ class SignatureGeneratorBasic(SignatureGenerator):
checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename)
for (f,cs) in checksums:
self.file_checksum_values[tid].append((f,cs))
- if cs:
- data = data + cs
taskdep = dataCache.task_deps[fn]
if 'nostamp' in taskdep and task in taskdep['nostamp']:
# Nostamp tasks need an implicit taint so that they force any dependent tasks to run
if tid in self.taints and self.taints[tid].startswith("nostamp:"):
# Don't reset taint value upon every call
- data = data + self.taints[tid][8:]
+ pass
else:
import uuid
taint = str(uuid.uuid4())
- data = data + taint
self.taints[tid] = "nostamp:" + taint
taint = self.read_taint(fn, task, dataCache.stamp[fn])
if taint:
- data = data + taint
self.taints[tid] = taint
logger.warning("%s is tainted from a forced run" % tid)
+ return
+
+ def get_taskhash(self, tid, deps, dataCache):
+
+ data = self.basehash[tid]
+ for dep in self.runtaskdeps[tid]:
+ data = data + self.get_unihash(dep)
+
+ for (f, cs) in self.file_checksum_values[tid]:
+ if cs:
+ data = data + cs
+
+ if tid in self.taints:
+ if self.taints[tid].startswith("nostamp:"):
+ data = data + self.taints[tid][8:]
+ else:
+ data = data + self.taints[tid]
+
h = hashlib.sha256(data.encode("utf-8")).hexdigest()
self.taskhash[tid] = h
#d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])