aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMarkus Lehtonen <markus.lehtonen@linux.intel.com>2016-02-19 14:22:58 +0200
committerMarkus Lehtonen <markus.lehtonen@linux.intel.com>2016-05-10 12:27:13 +0300
commit6e6cfb5ae0f9dae866c4a407a6a381d85b2608b9 (patch)
tree8803537eeeb34c7c47ffe488acc3cab8465c0abe
parente8b54b0363cc180d16191bf1d9d476f8ed21fa5d (diff)
downloadopenembedded-core-contrib-6e6cfb5ae0f9dae866c4a407a6a381d85b2608b9.tar.gz
openembedded-core-contrib-6e6cfb5ae0f9dae866c4a407a6a381d85b2608b9.tar.bz2
openembedded-core-contrib-6e6cfb5ae0f9dae866c4a407a6a381d85b2608b9.zip
SignatureGenerator: add checksum_cache argument to get_taskhash()
Extend the SignatureGenerator API by adding a new argument to get_taskhash() for defining the file checksum cache to use. If the checksum cache argument is not provided, we use the cache in bb.fetch2, as before. This is a step towards removing checksum cache from bb.fetch2. Unfortunately, extending the API is a bit more involved than just adding a new optional argument because it has a sort of cyclic dependecy: the API is defined in bitbake, the implementation may be modified elsewhere (e.g. oe.sstatesig in oe-core layer) by overriding SignatureGenerator's method(s), which are eventually consumed back at bitbake. In bitbake, we must be prepared to use the old interface because oe-core may override our new shiny API with an old one. Thus, we need to inspect the API in RunQueueData and adjust call arguments accordingly. If the signature generator instance implements the new API, we use a checksum cache instance specific to runque. Print a warning if an implementation of the old API is detected. Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
-rw-r--r--lib/bb/runqueue.py30
-rw-r--r--lib/bb/siggen.py9
2 files changed, 31 insertions, 8 deletions
diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index 747ea484e3e..73567408c1a 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -34,6 +34,7 @@ import re
import bb
from bb import msg, data, event
from bb import monitordisk
+from bb.checksum import FileChecksumCache
import subprocess
try:
@@ -219,6 +220,7 @@ class RunQueueData:
BitBake Run Queue implementation
"""
def __init__(self, rq, cooker, cfgData, dataCache, taskData, targets):
+ import inspect
self.cooker = cooker
self.dataCache = dataCache
self.taskData = taskData
@@ -229,6 +231,16 @@ class RunQueueData:
self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST", True) or ""
self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST", True) or "").split()
+ # Check if get_taskhash implements the new API. If so, use our own file
+ # checksum cache
+ self.checksum_cache = None
+ if 'checksum_cache' in inspect.getargspec(bb.parse.siggen.get_taskhash).args:
+ checksum_cache_file = cfgData.getVar("BB_HASH_CHECKSUM_CACHE_FILE", True)
+ self.checksum_cache = FileChecksumCache()
+ self.checksum_cache.init_cache(cfgData, checksum_cache_file)
+ else:
+ logger.warn("%s implements deprecated API of get_taskhash(), "
+ "missing the 'checksum_cache' argument" % bb.parse.siggen.__class__.__name__)
self.reset()
def reset(self):
@@ -884,6 +896,7 @@ class RunQueueData:
if hasattr(bb.parse.siggen, "tasks_resolved"):
bb.parse.siggen.tasks_resolved(virtmap, virtpnmap, self.dataCache)
+
# Iterate over the task list and call into the siggen code
dealtwith = set()
todeal = set(range(len(self.runq_fnid)))
@@ -895,12 +908,19 @@ class RunQueueData:
procdep = []
for dep in self.runq_depends[task]:
procdep.append(self.taskData.fn_index[self.runq_fnid[dep]] + "." + self.runq_task[dep])
- self.runq_hash[task] = bb.parse.siggen.get_taskhash(self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task], procdep, self.dataCache)
-
+ if self.checksum_cache:
+ self.runq_hash[task] = bb.parse.siggen.get_taskhash(self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task], procdep, self.dataCache, self.checksum_cache)
+ else:
+ self.runq_hash[task] = bb.parse.siggen.get_taskhash(self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task], procdep, self.dataCache)
# Write out checksum cache onto disk
- # This relies on the "knowledge" that siggen uses cache of bb.fetch2
- bb.fetch2.fetcher_parse_save()
- bb.fetch2.fetcher_parse_done()
+ if self.checksum_cache:
+ self.checksum_cache.save_extras()
+ self.checksum_cache.save_merge()
+ else:
+ # This relies on the "knowledge" that siggen uses cache of bb.fetch2
+ bb.fetch2.fetcher_parse_save()
+ bb.fetch2.fetcher_parse_done()
+
return len(self.runq_fnid)
def dump_data(self, taskQueue):
diff --git a/lib/bb/siggen.py b/lib/bb/siggen.py
index 9d2c0b6b187..81b42d12f3e 100644
--- a/lib/bb/siggen.py
+++ b/lib/bb/siggen.py
@@ -43,7 +43,7 @@ class SignatureGenerator(object):
def finalise(self, fn, d, varient):
return
- def get_taskhash(self, fn, task, deps, dataCache):
+ def get_taskhash(self, fn, task, deps, dataCache, checksum_cache=None):
return "0"
def stampfile(self, stampbase, file_name, taskname, extrainfo):
@@ -173,7 +173,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
pass
return taint
- def get_taskhash(self, fn, task, deps, dataCache):
+ def get_taskhash(self, fn, task, deps, dataCache, checksum_cache=None):
k = fn + "." + task
data = dataCache.basetaskhash[k]
self.runtaskdeps[k] = []
@@ -190,7 +190,10 @@ class SignatureGeneratorBasic(SignatureGenerator):
self.runtaskdeps[k].append(dep)
if task in dataCache.file_checksums[fn]:
- checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename)
+ if checksum_cache:
+ checksums = checksum_cache.get_checksums(dataCache.file_checksums[fn][task], recipename)
+ else:
+ checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename)
for (f,cs) in checksums:
self.file_checksum_values[k].append((f,cs))
if cs: