From fd79638046891fb5ab6091f7ef3e7ae6df12c39b Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Fri, 2 Aug 2019 15:32:27 +0100 Subject: bitbake: tests/runqueue: Add hashserv+runqueue test Add a test which tests the runqueue adaptations for hash equivalency. (Bitbake rev: 477321d0780df177c1582db119c2bb6795912fc6) Signed-off-by: Richard Purdie --- bitbake/lib/bb/siggen.py | 23 ++++++++++++++---- .../bb/tests/runqueue-tests/classes/base.bbclass | 14 ++++++++++- .../lib/bb/tests/runqueue-tests/conf/bitbake.conf | 2 +- bitbake/lib/bb/tests/runqueue.py | 27 +++++++++++++++++++++- 4 files changed, 59 insertions(+), 7 deletions(-) diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py index 912c92c8be..b503559305 100644 --- a/bitbake/lib/bb/siggen.py +++ b/bitbake/lib/bb/siggen.py @@ -12,6 +12,7 @@ import bb.data import difflib import simplediff from bb.checksum import FileChecksumCache +from bb import runqueue logger = logging.getLogger('BitBake.SigGen') @@ -473,10 +474,12 @@ class SignatureGeneratorUniHashMixIn(object): locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d} - (module, method) = self.method.rsplit('.', 1) - locs['method'] = getattr(importlib.import_module(module), method) - - outhash = bb.utils.better_eval('method(path, sigfile, task, d)', locs) + if "." in self.method: + (module, method) = self.method.rsplit('.', 1) + locs['method'] = getattr(importlib.import_module(module), method) + outhash = bb.utils.better_eval('method(path, sigfile, task, d)', locs) + else: + outhash = bb.utils.better_eval(self.method + '(path, sigfile, task, d)', locs) try: url = '%s/v1/equivalent' % self.server @@ -527,6 +530,18 @@ class SignatureGeneratorUniHashMixIn(object): except OSError: pass + +# +# Dummy class used for bitbake-selftest +# +class SignatureGeneratorTestEquivHash(SignatureGeneratorUniHashMixIn, SignatureGeneratorBasicHash): + name = "TestEquivHash" + def init_rundepcheck(self, data): + super().init_rundepcheck(data) + self.server = "http://" + data.getVar('BB_HASHSERVE') + self.method = "sstate_output_hash" + + def dump_this_task(outfile, d): import bb.parse fn = d.getVar("BB_FILENAME") diff --git a/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass b/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass index 3a0f151c9a..138edc3fa9 100644 --- a/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass +++ b/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass @@ -5,7 +5,8 @@ def stamptask(d): import time thistask = d.expand("${PN}:${BB_CURRENTTASK}") - with open(d.expand("${TOPDIR}/%s.run") % thistask, "a+") as f: + stampname = d.expand("${TOPDIR}/%s.run" % thistask) + with open(stampname, "a+") as f: f.write("\n") if d.getVar("BB_CURRENT_MC") != "default": @@ -13,10 +14,21 @@ def stamptask(d): if thistask in d.getVar("SLOWTASKS").split(): bb.note("Slowing task %s" % thistask) time.sleep(0.5) + if d.getVar("BB_HASHSERVE"): + task = d.getVar("BB_CURRENTTASK") + if task in ['package', 'package_qa', 'packagedata', 'package_write_ipk', 'package_write_rpm', 'populate_lic', 'populate_sysroot']: + bb.parse.siggen.report_unihash(os.getcwd(), d.getVar("BB_CURRENTTASK"), d) with open(d.expand("${TOPDIR}/task.log"), "a+") as f: f.write(thistask + "\n") + +def sstate_output_hash(path, sigfile, task, d): + import hashlib + h = hashlib.sha256() + h.update(d.expand("${PN}:${BB_CURRENTTASK}").encode('utf-8')) + return h.hexdigest() + python do_fetch() { # fetch stamptask(d) diff --git a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf index 96ee1cd5ec..1c61f27607 100644 --- a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf +++ b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf @@ -11,6 +11,6 @@ STAMP = "${TMPDIR}/stamps/${PN}" T = "${TMPDIR}/workdir/${PN}/temp" BB_NUMBER_THREADS = "4" -BB_HASHBASE_WHITELIST = "BB_CURRENT_MC" +BB_HASHBASE_WHITELIST = "BB_CURRENT_MC BB_HASHSERVE" include conf/multiconfig/${BB_CURRENT_MC}.conf diff --git a/bitbake/lib/bb/tests/runqueue.py b/bitbake/lib/bb/tests/runqueue.py index f22ad4bd81..fbdacccfa1 100644 --- a/bitbake/lib/bb/tests/runqueue.py +++ b/bitbake/lib/bb/tests/runqueue.py @@ -25,7 +25,7 @@ class RunQueueTests(unittest.TestCase): a1_sstatevalid = "a1:do_package a1:do_package_qa a1:do_packagedata a1:do_package_write_ipk a1:do_package_write_rpm a1:do_populate_lic a1:do_populate_sysroot" b1_sstatevalid = "b1:do_package b1:do_package_qa b1:do_packagedata b1:do_package_write_ipk b1:do_package_write_rpm b1:do_populate_lic b1:do_populate_sysroot" - def run_bitbakecmd(self, cmd, builddir, sstatevalid="", slowtasks="", extraenv=None): + def run_bitbakecmd(self, cmd, builddir, sstatevalid="", slowtasks="", extraenv=None, cleanup=False): env = os.environ.copy() env["BBPATH"] = os.path.realpath(os.path.join(os.path.dirname(__file__), "runqueue-tests")) env["BB_ENV_EXTRAWHITE"] = "SSTATEVALID SLOWTASKS" @@ -42,6 +42,8 @@ class RunQueueTests(unittest.TestCase): tasks = [] with open(builddir + "/task.log", "r") as f: tasks = [line.rstrip() for line in f] + if cleanup: + os.remove(builddir + "/task.log") return tasks def test_no_setscenevalid(self): @@ -226,3 +228,26 @@ class RunQueueTests(unittest.TestCase): expected.remove(x) self.assertEqual(set(tasks), set(expected)) + + def test_hashserv(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + extraenv = { + "BB_HASHSERVE" : "localhost:0", + "BB_SIGNATURE_HANDLER" : "TestEquivHash" + } + cmd = ["bitbake", "a1", "b1"] + setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene', + 'populate_sysroot_setscene', 'package_qa_setscene'] + sstatevalid = "" + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True) + expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] + self.assertEqual(set(tasks), set(expected)) + cmd = ["bitbake", "a1", "-c", "install", "-f"] + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True) + expected = ['a1:install'] + self.assertEqual(set(tasks), set(expected)) + cmd = ["bitbake", "a1", "b1"] + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True) + expected = ['a1:' + x for x in setscenetasks] + ['b1:' + x for x in setscenetasks] + ['a1:build', 'b1:build'] + self.assertEqual(set(tasks), set(expected)) + -- cgit 1.2.3-korg