aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMark Hatle <mark.hatle@windriver.com>2018-05-14 10:21:39 -0400
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-06-15 09:12:33 +0100
commit055865047c63b9c3b213b47a1884924ce0adeda0 (patch)
treedd074f787ffd17a4dddeb2651d97115286067846
parent26d5ea9bb892bd6a2e1fd29a9023e0b0644edc16 (diff)
downloadbitbake-055865047c63b9c3b213b47a1884924ce0adeda0.zip
bitbake-055865047c63b9c3b213b47a1884924ce0adeda0.tar.gz
bitbake-055865047c63b9c3b213b47a1884924ce0adeda0.tar.bz2
runqueue.py: Initial implementation of per task process limits
On high core machines, in do_fetch, it is possible to DDoS your own machine. A method to limit any arbitrary task type to a certain number of simultaneous threads is needed. (Similar to how BB_NUMBER_THREADS works in the general case.) The format of this new limitation is: do_fetch[number_threads] = "2" This should be set globally. If it is set in individual recipes it could result in unpredictable behavior. Note: a value for number_threads > BB_NUMBER_THREADS will have no effect. Signed-off-by: Mark Hatle <mark.hatle@windriver.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--lib/bb/runqueue.py21
1 files changed, 21 insertions, 0 deletions
diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py
index a937a0b..2d9e18d 100644
--- a/lib/bb/runqueue.py
+++ b/lib/bb/runqueue.py
@@ -134,6 +134,7 @@ class RunQueueScheduler(object):
self.prio_map = [self.rqdata.runtaskentries.keys()]
self.buildable = []
+ self.skip_maxthread = {}
self.stamps = {}
for tid in self.rqdata.runtaskentries:
(mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
@@ -150,8 +151,25 @@ class RunQueueScheduler(object):
self.buildable = [x for x in self.buildable if x not in self.rq.runq_running]
if not self.buildable:
return None
+
+ # Filter out tasks that have a max number of threads that have been exceeded
+ skip_buildable = {}
+ for running in self.rq.runq_running.difference(self.rq.runq_complete):
+ rtaskname = taskname_from_tid(running)
+ if rtaskname not in self.skip_maxthread:
+ self.skip_maxthread[rtaskname] = self.rq.cfgData.getVarFlag(rtaskname, "number_threads")
+ if not self.skip_maxthread[rtaskname]:
+ continue
+ if rtaskname in skip_buildable:
+ skip_buildable[rtaskname] += 1
+ else:
+ skip_buildable[rtaskname] = 1
+
if len(self.buildable) == 1:
tid = self.buildable[0]
+ taskname = taskname_from_tid(tid)
+ if taskname in skip_buildable and skip_buildable[taskname] >= int(self.skip_maxthread[taskname]):
+ return None
stamp = self.stamps[tid]
if stamp not in self.rq.build_stamps.values():
return tid
@@ -164,6 +182,9 @@ class RunQueueScheduler(object):
best = None
bestprio = None
for tid in self.buildable:
+ taskname = taskname_from_tid(tid)
+ if taskname in skip_buildable and skip_buildable[taskname] >= int(self.skip_maxthread[taskname]):
+ continue
prio = self.rev_prio_map[tid]
if bestprio is None or bestprio > prio:
stamp = self.stamps[tid]