diff options
Diffstat (limited to 'bitbake')
-rw-r--r-- | bitbake/lib/bb/runqueue.py | 60 | ||||
-rw-r--r-- | bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass | 19 | ||||
-rw-r--r-- | bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf | 9 | ||||
-rw-r--r-- | bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf | 1 | ||||
-rw-r--r-- | bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf | 1 | ||||
-rw-r--r-- | bitbake/lib/bb/tests/runqueue.py | 19 |
6 files changed, 98 insertions, 11 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 00c71070d2..fa848326d8 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py @@ -68,6 +68,14 @@ def build_tid(mc, fn, taskname): return "mc:" + mc + ":" + fn + ":" + taskname return fn + ":" + taskname +# Index used to pair up potentially matching multiconfig tasks +# We match on PN, taskname and hash being equal +def pending_hash_index(tid, rqdata): + (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) + pn = rqdata.dataCaches[mc].pkg_fn[taskfn] + h = rqdata.runtaskentries[tid].hash + return pn + ":" + "taskname" + h + class RunQueueStats: """ Holds statistics on the tasks handled by the associated runQueue @@ -1717,6 +1725,7 @@ class RunQueueExecute: self.build_stamps = {} self.build_stamps2 = [] self.failed_tids = [] + self.sq_deferred = {} self.stampcache = {} @@ -1921,17 +1930,32 @@ class RunQueueExecute: # Find the next setscene to run for nexttask in self.rqdata.runq_setscene_tids: if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values(): - if nexttask in self.sqdata.unskippable: - logger.debug(2, "Setscene task %s is unskippable" % nexttask) if nexttask not in self.sqdata.unskippable and len(self.sqdata.sq_revdeps[nexttask]) > 0 and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]): if nexttask not in self.rqdata.target_tids: logger.debug(2, "Skipping setscene for task %s" % nexttask) self.sq_task_skip(nexttask) self.scenequeue_notneeded.add(nexttask) + if nexttask in self.sq_deferred: + del self.sq_deferred[nexttask] + return True + if nexttask in self.sq_deferred: + if self.sq_deferred[nexttask] not in self.runq_complete: + continue + logger.debug(1, "Task %s no longer deferred" % nexttask) + del self.sq_deferred[nexttask] + valid = self.rq.validate_hashes(set([nexttask]), self.cooker.data, None, False) + if not valid: + logger.debug(1, "%s didn't become valid, skipping setscene" % nexttask) + self.sq_task_failoutright(nexttask) return True + else: + self.sqdata.outrightfail.remove(nexttask) if nexttask in self.sqdata.outrightfail: + logger.debug(2, 'No package found, so skipping setscene task %s', nexttask) self.sq_task_failoutright(nexttask) return True + if nexttask in self.sqdata.unskippable: + logger.debug(2, "Setscene task %s is unskippable" % nexttask) task = nexttask break if task is not None: @@ -1982,7 +2006,7 @@ class RunQueueExecute: if self.can_start_task(): return True - if not self.sq_live and not self.sqdone: + if not self.sq_live and not self.sqdone and not self.sq_deferred: logger.info("Setscene tasks completed") logger.debug(1, 'We could skip tasks %s', "\n".join(sorted(self.scenequeue_covered))) @@ -2083,6 +2107,13 @@ class RunQueueExecute: self.rq.read_workers() return self.rq.active_fds() + # No more tasks can be run. If we have deferred setscene tasks we should run them. + if self.sq_deferred: + tid = self.sq_deferred.pop(list(self.sq_deferred.keys())[0]) + logger.warning("Runqeueue deadlocked on deferred tasks, forcing task %s" % tid) + self.sq_task_failoutright(tid) + return True + if len(self.failed_tids) != 0: self.rq.state = runQueueFailed return True @@ -2347,7 +2378,7 @@ class SQData(object): # Setscene tasks directly depended upon by the build self.unskippable = set() # List of setscene tasks which aren't present - self.outrightfail = [] + self.outrightfail = set() # A list of normal tasks a setscene task covers self.sq_covered_tasks = {} @@ -2510,7 +2541,9 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq): rqdata.init_progress_reporter.next_stage() + multiconfigs = set() for tid in sqdata.sq_revdeps: + multiconfigs.add(mc_from_tid(tid)) if len(sqdata.sq_revdeps[tid]) == 0: sqrq.sq_buildable.add(tid) @@ -2552,10 +2585,21 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq): for v in valid: valid_new.append(v) - for tid in sqdata.sq_revdeps: - if tid not in valid_new and tid not in noexec: - logger.debug(2, 'No package found, so skipping setscene task %s', tid) - sqdata.outrightfail.append(tid) + hashes = {} + for mc in sorted(multiconfigs): + for tid in sqdata.sq_revdeps: + if mc_from_tid(tid) != mc: + continue + if tid not in valid_new and tid not in noexec and tid not in sqrq.scenequeue_notcovered: + sqdata.outrightfail.add(tid) + + h = pending_hash_index(tid, rqdata) + if h not in hashes: + hashes[h] = tid + else: + sqrq.sq_deferred[tid] = hashes[h] + bb.warn("Deferring %s after %s" % (tid, hashes[h])) + class TaskFailure(Exception): """ diff --git a/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass b/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass index e174c02dd6..cf38d09224 100644 --- a/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass +++ b/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass @@ -4,7 +4,9 @@ SSTATEVALID ??= "" def stamptask(d): import time - thistask = d.expand("${PN}:${BB_CURRENTTASK}") + thistask = d.expand("${PN}:${BB_CURRENTTASK}") + if d.getVar("BB_CURRENT_MC") != "default": + thistask = d.expand("${BB_CURRENT_MC}:${PN}:${BB_CURRENTTASK}") if thistask in d.getVar("SLOWTASKS").split(): bb.note("Slowing task %s" % thistask) time.sleep(0.5) @@ -13,48 +15,63 @@ def stamptask(d): f.write(thistask + "\n") python do_fetch() { + # fetch stamptask(d) } python do_unpack() { + # unpack stamptask(d) } python do_patch() { + # patch stamptask(d) } python do_populate_lic() { + # populate_lic stamptask(d) } python do_prepare_recipe_sysroot() { + # prepare_recipe_sysroot stamptask(d) } python do_configure() { + # configure stamptask(d) } python do_compile() { + # compile stamptask(d) } python do_install() { + # install stamptask(d) } python do_populate_sysroot() { + # populate_sysroot stamptask(d) } python do_package() { + # package stamptask(d) } python do_package_write_ipk() { + # package_write_ipk stamptask(d) } python do_package_write_rpm() { + # package_write_rpm stamptask(d) } python do_packagedata() { + # packagedata stamptask(d) } python do_package_qa() { + # package_qa stamptask(d) } python do_build() { + # build stamptask(d) } do_prepare_recipe_sysroot[deptask] = "do_populate_sysroot" diff --git a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf index 8c7b754dab..96ee1cd5ec 100644 --- a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf +++ b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf @@ -6,6 +6,11 @@ PROVIDES = "${PN}" PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0]}" PF = "${BB_CURRENT_MC}:${PN}" export PATH -STAMP = "${TOPDIR}/stamps/${PN}" -T = "${TOPDIR}/workdir/${PN}/temp" +TMPDIR ??= "${TOPDIR}" +STAMP = "${TMPDIR}/stamps/${PN}" +T = "${TMPDIR}/workdir/${PN}/temp" BB_NUMBER_THREADS = "4" + +BB_HASHBASE_WHITELIST = "BB_CURRENT_MC" + +include conf/multiconfig/${BB_CURRENT_MC}.conf diff --git a/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf b/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf new file mode 100644 index 0000000000..ecf23e1c73 --- /dev/null +++ b/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf @@ -0,0 +1 @@ +TMPDIR = "${TOPDIR}/mc1/" diff --git a/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf b/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf new file mode 100644 index 0000000000..eef338e4cc --- /dev/null +++ b/bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf @@ -0,0 +1 @@ +TMPDIR = "${TOPDIR}/mc2/" diff --git a/bitbake/lib/bb/tests/runqueue.py b/bitbake/lib/bb/tests/runqueue.py index 4a65b5b6e7..f0cea6483f 100644 --- a/bitbake/lib/bb/tests/runqueue.py +++ b/bitbake/lib/bb/tests/runqueue.py @@ -198,3 +198,22 @@ class RunQueueTests(unittest.TestCase): 'b1:packagedata_setscene', 'b1:package_qa_setscene', 'b1:populate_sysroot_setscene'] self.assertEqual(set(tasks), set(expected)) + def test_multiconfig_setscene_optimise(self): + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + extraenv = { + "BBMULTICONFIG" : "mc1 mc2", + "BB_SIGNATURE_HANDLER" : "basic" + } + cmd = ["bitbake", "b1", "mc:mc1:b1", "mc:mc2:b1"] + setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene', + 'populate_sysroot_setscene', 'package_qa_setscene'] + sstatevalid = "" + tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv) + expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] + \ + ['mc1:b1:' + x for x in setscenetasks] + ['mc1:a1:' + x for x in setscenetasks] + \ + ['mc2:b1:' + x for x in setscenetasks] + ['mc2:a1:' + x for x in setscenetasks] + \ + ['mc1:b1:build', 'mc2:b1:build'] + for x in ['mc1:a1:package_qa_setscene', 'mc2:a1:package_qa_setscene', 'a1:build', 'a1:package_qa']: + expected.remove(x) + self.assertEqual(set(tasks), set(expected)) + |