aboutsummaryrefslogtreecommitdiffstats
path: root/lib/bb/runqueue.py
diff options
context:
space:
mode:
Diffstat (limited to 'lib/bb/runqueue.py')
-rw-r--r--lib/bb/runqueue.py110
1 files changed, 65 insertions, 45 deletions
diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index 6c41fe6d..10511a09 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -85,15 +85,19 @@ class RunQueueStats:
"""
Holds statistics on the tasks handled by the associated runQueue
"""
- def __init__(self, total):
+ def __init__(self, total, setscene_total):
self.completed = 0
self.skipped = 0
self.failed = 0
self.active = 0
+ self.setscene_active = 0
+ self.setscene_covered = 0
+ self.setscene_notcovered = 0
+ self.setscene_total = setscene_total
self.total = total
def copy(self):
- obj = self.__class__(self.total)
+ obj = self.__class__(self.total, self.setscene_total)
obj.__dict__.update(self.__dict__)
return obj
@@ -112,6 +116,13 @@ class RunQueueStats:
def taskActive(self):
self.active = self.active + 1
+ def updateCovered(self, covered, notcovered):
+ self.setscene_covered = covered
+ self.setscene_notcovered = notcovered
+
+ def updateActiveSetscene(self, active):
+ self.setscene_active = active
+
# These values indicate the next step due to be run in the
# runQueue state machine
runQueuePrepare = 2
@@ -1735,8 +1746,7 @@ class RunQueueExecute:
self.holdoff_need_update = True
self.sqdone = False
- self.stats = RunQueueStats(len(self.rqdata.runtaskentries))
- self.sq_stats = RunQueueStats(len(self.rqdata.runq_setscene_tids))
+ self.stats = RunQueueStats(len(self.rqdata.runtaskentries), len(self.rqdata.runq_setscene_tids))
for mc in rq.worker:
rq.worker[mc].pipe.setrunqueueexec(self)
@@ -1787,6 +1797,7 @@ class RunQueueExecute:
else:
self.sq_task_complete(task)
self.sq_live.remove(task)
+ self.stats.updateActiveSetscene(len(self.sq_live))
else:
if status != 0:
self.task_fail(task, status, fakerootlog=fakerootlog)
@@ -1820,7 +1831,7 @@ class RunQueueExecute:
def finish(self):
self.rq.state = runQueueCleanUp
- active = self.stats.active + self.sq_stats.active
+ active = self.stats.active + len(self.sq_live)
if active > 0:
bb.event.fire(runQueueExitWait(active), self.cfgData)
self.rq.read_workers()
@@ -1853,7 +1864,7 @@ class RunQueueExecute:
return valid
def can_start_task(self):
- active = self.stats.active + self.sq_stats.active
+ active = self.stats.active + len(self.sq_live)
can_start = active < self.number_tasks
return can_start
@@ -1904,6 +1915,12 @@ class RunQueueExecute:
self.setbuildable(revdep)
logger.debug("Marking task %s as buildable", revdep)
+ for t in self.sq_deferred.copy():
+ if self.sq_deferred[t] == task:
+ logger.debug2("Deferred task %s now buildable" % t)
+ del self.sq_deferred[t]
+ update_scenequeue_data([t], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False)
+
def task_complete(self, task):
self.stats.taskCompleted()
bb.event.fire(runQueueTaskCompleted(task, self.stats, self.rq), self.cfgData)
@@ -1950,7 +1967,7 @@ class RunQueueExecute:
err = False
if not self.sqdone:
logger.debug('We could skip tasks %s', "\n".join(sorted(self.scenequeue_covered)))
- completeevent = sceneQueueComplete(self.sq_stats, self.rq)
+ completeevent = sceneQueueComplete(self.stats, self.rq)
bb.event.fire(completeevent, self.cfgData)
if self.sq_deferred:
logger.error("Scenequeue had deferred entries: %s" % pprint.pformat(self.sq_deferred))
@@ -2061,7 +2078,7 @@ class RunQueueExecute:
self.sq_task_failoutright(task)
return True
- startevent = sceneQueueTaskStarted(task, self.sq_stats, self.rq)
+ startevent = sceneQueueTaskStarted(task, self.stats, self.rq)
bb.event.fire(startevent, self.cfgData)
taskdepdata = self.sq_build_taskdepdata(task)
@@ -2082,7 +2099,7 @@ class RunQueueExecute:
self.build_stamps2.append(self.build_stamps[task])
self.sq_running.add(task)
self.sq_live.add(task)
- self.sq_stats.taskActive()
+ self.stats.updateActiveSetscene(len(self.sq_live))
if self.can_start_task():
return True
@@ -2172,7 +2189,7 @@ class RunQueueExecute:
if self.can_start_task():
return True
- if self.stats.active > 0 or self.sq_stats.active > 0:
+ if self.stats.active > 0 or len(self.sq_live) > 0:
self.rq.read_workers()
return self.rq.active_fds()
@@ -2180,7 +2197,8 @@ class RunQueueExecute:
if self.sq_deferred:
tid = self.sq_deferred.pop(list(self.sq_deferred.keys())[0])
logger.warning("Runqeueue deadlocked on deferred tasks, forcing task %s" % tid)
- self.sq_task_failoutright(tid)
+ if tid not in self.runq_complete:
+ self.sq_task_failoutright(tid)
return True
if len(self.failed_tids) != 0:
@@ -2443,6 +2461,11 @@ class RunQueueExecute:
if update_tasks:
self.sqdone = False
+ for tid in [t[0] for t in update_tasks]:
+ h = pending_hash_index(tid, self.rqdata)
+ if h in self.sqdata.hashes and tid != self.sqdata.hashes[h]:
+ self.sq_deferred[tid] = self.sqdata.hashes[h]
+ bb.note("Deferring %s after %s" % (tid, self.sqdata.hashes[h]))
update_scenequeue_data([t[0] for t in update_tasks], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False)
for (tid, harddepfail, origvalid) in update_tasks:
@@ -2452,6 +2475,7 @@ class RunQueueExecute:
self.sq_task_failoutright(tid)
if changed:
+ self.stats.updateCovered(len(self.scenequeue_covered), len(self.scenequeue_notcovered))
self.holdoff_need_update = True
def scenequeue_updatecounters(self, task, fail=False):
@@ -2485,6 +2509,7 @@ class RunQueueExecute:
new.add(dep)
next = new
+ self.stats.updateCovered(len(self.scenequeue_covered), len(self.scenequeue_notcovered))
self.holdoff_need_update = True
def sq_task_completeoutright(self, task):
@@ -2508,13 +2533,11 @@ class RunQueueExecute:
self.rq.state = runQueueCleanUp
def sq_task_complete(self, task):
- self.sq_stats.taskCompleted()
- bb.event.fire(sceneQueueTaskCompleted(task, self.sq_stats, self.rq), self.cfgData)
+ bb.event.fire(sceneQueueTaskCompleted(task, self.stats, self.rq), self.cfgData)
self.sq_task_completeoutright(task)
def sq_task_fail(self, task, result):
- self.sq_stats.taskFailed()
- bb.event.fire(sceneQueueTaskFailed(task, self.sq_stats, result, self), self.cfgData)
+ bb.event.fire(sceneQueueTaskFailed(task, self.stats, result, self), self.cfgData)
self.scenequeue_notcovered.add(task)
self.scenequeue_updatecounters(task, True)
self.sq_check_taskfail(task)
@@ -2522,8 +2545,6 @@ class RunQueueExecute:
def sq_task_failoutright(self, task):
self.sq_running.add(task)
self.sq_buildable.add(task)
- self.sq_stats.taskSkipped()
- self.sq_stats.taskCompleted()
self.scenequeue_notcovered.add(task)
self.scenequeue_updatecounters(task, True)
@@ -2531,8 +2552,6 @@ class RunQueueExecute:
self.sq_running.add(task)
self.sq_buildable.add(task)
self.sq_task_completeoutright(task)
- self.sq_stats.taskSkipped()
- self.sq_stats.taskCompleted()
def sq_build_taskdepdata(self, task):
def getsetscenedeps(tid):
@@ -2786,6 +2805,19 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
sqdata.stamppresent = set()
sqdata.valid = set()
+ sqdata.hashes = {}
+ sqrq.sq_deferred = {}
+ for mc in sorted(sqdata.multiconfigs):
+ for tid in sorted(sqdata.sq_revdeps):
+ if mc_from_tid(tid) != mc:
+ continue
+ h = pending_hash_index(tid, rqdata)
+ if h not in sqdata.hashes:
+ sqdata.hashes[h] = tid
+ else:
+ sqrq.sq_deferred[tid] = sqdata.hashes[h]
+ bb.note("Deferring %s after %s" % (tid, sqdata.hashes[h]))
+
update_scenequeue_data(sqdata.sq_revdeps, sqdata, rqdata, rq, cooker, stampcache, sqrq, summary=True)
# Compute a list of 'stale' sstate tasks where the current hash does not match the one
@@ -2850,32 +2882,20 @@ def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, s
sqdata.valid |= rq.validate_hashes(tocheck, cooker.data, len(sqdata.stamppresent), False, summary=summary)
- sqdata.hashes = {}
- sqrq.sq_deferred = {}
- for mc in sorted(sqdata.multiconfigs):
- for tid in sorted(sqdata.sq_revdeps):
- if mc_from_tid(tid) != mc:
- continue
- if tid in sqdata.stamppresent:
- continue
- if tid in sqdata.valid:
- continue
- if tid in sqdata.noexec:
- continue
- if tid in sqrq.scenequeue_notcovered:
- continue
- if tid in sqrq.scenequeue_covered:
- continue
-
- h = pending_hash_index(tid, rqdata)
- if h not in sqdata.hashes:
- if tid in tids:
- sqdata.outrightfail.add(tid)
- sqdata.hashes[h] = tid
- else:
- sqrq.sq_deferred[tid] = sqdata.hashes[h]
- bb.note("Deferring %s after %s" % (tid, sqdata.hashes[h]))
-
+ for tid in tids:
+ if tid in sqdata.stamppresent:
+ continue
+ if tid in sqdata.valid:
+ continue
+ if tid in sqdata.noexec:
+ continue
+ if tid in sqrq.scenequeue_covered:
+ continue
+ if tid in sqrq.scenequeue_notcovered:
+ continue
+ if tid in sqrq.sq_deferred:
+ continue
+ sqdata.outrightfail.add(tid)
class TaskFailure(Exception):
"""