diff options
Diffstat (limited to 'lib/bb/runqueue.py')
| -rw-r--r-- | lib/bb/runqueue.py | 65 |
1 files changed, 65 insertions, 0 deletions
diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py index f34f1568e..203ef8c15 100644 --- a/lib/bb/runqueue.py +++ b/lib/bb/runqueue.py @@ -24,6 +24,7 @@ import pickle from multiprocessing import Process import shlex import pprint +import time bblogger = logging.getLogger("BitBake") logger = logging.getLogger("BitBake.RunQueue") @@ -159,6 +160,46 @@ class RunQueueScheduler(object): self.buildable.append(tid) self.rev_prio_map = None + self.is_pressure_usable() + + def is_pressure_usable(self): + """ + If monitoring pressure, return True if pressure files can be open and read. For example + openSUSE /proc/pressure/* files have readable file permissions but when read the error EOPNOTSUPP (Operation not supported) + is returned. + """ + if self.rq.max_cpu_pressure or self.rq.max_io_pressure: + try: + with open("/proc/pressure/cpu") as cpu_pressure_fds, open("/proc/pressure/io") as io_pressure_fds: + self.prev_cpu_pressure = cpu_pressure_fds.readline().split()[4].split("=")[1] + self.prev_io_pressure = io_pressure_fds.readline().split()[4].split("=")[1] + self.prev_pressure_time = time.time() + self.check_pressure = True + except: + bb.warn("The /proc/pressure files can't be read. Continuing build without monitoring pressure") + self.check_pressure = False + else: + self.check_pressure = False + + def exceeds_max_pressure(self): + """ + Monitor the difference in total pressure at least once per second, if + BB_PRESSURE_MAX_{CPU|IO} are set, return True if above threshold. + """ + if self.check_pressure: + with open("/proc/pressure/cpu") as cpu_pressure_fds, open("/proc/pressure/io") as io_pressure_fds: + # extract "total" from /proc/pressure/{cpu|io} + curr_cpu_pressure = cpu_pressure_fds.readline().split()[4].split("=")[1] + curr_io_pressure = io_pressure_fds.readline().split()[4].split("=")[1] + exceeds_cpu_pressure = self.rq.max_cpu_pressure and (float(curr_cpu_pressure) - float(self.prev_cpu_pressure)) > self.rq.max_cpu_pressure + exceeds_io_pressure = self.rq.max_io_pressure and (float(curr_io_pressure) - float(self.prev_io_pressure)) > self.rq.max_io_pressure + now = time.time() + if now - self.prev_pressure_time > 1.0: + self.prev_cpu_pressure = curr_cpu_pressure + self.prev_io_pressure = curr_io_pressure + self.prev_pressure_time = now + return (exceeds_cpu_pressure or exceeds_io_pressure) + return False def next_buildable_task(self): """ @@ -172,6 +213,12 @@ class RunQueueScheduler(object): if not buildable: return None + # Bitbake requires that at least one task be active. Only check for pressure if + # this is the case, otherwise the pressure limitation could result in no tasks + # being active and no new tasks started thereby, at times, breaking the scheduler. + if self.rq.stats.active and self.exceeds_max_pressure(): + return None + # Filter out tasks that have a max number of threads that have been exceeded skip_buildable = {} for running in self.rq.runq_running.difference(self.rq.runq_complete): @@ -1699,6 +1746,8 @@ class RunQueueExecute: self.number_tasks = int(self.cfgData.getVar("BB_NUMBER_THREADS") or 1) self.scheduler = self.cfgData.getVar("BB_SCHEDULER") or "speed" + self.max_cpu_pressure = self.cfgData.getVar("BB_PRESSURE_MAX_CPU") + self.max_io_pressure = self.cfgData.getVar("BB_PRESSURE_MAX_IO") self.sq_buildable = set() self.sq_running = set() @@ -1733,6 +1782,22 @@ class RunQueueExecute: if self.number_tasks <= 0: bb.fatal("Invalid BB_NUMBER_THREADS %s" % self.number_tasks) + lower_limit = 1.0 + upper_limit = 1000000.0 + if self.max_cpu_pressure: + self.max_cpu_pressure = float(self.max_cpu_pressure) + if self.max_cpu_pressure < lower_limit: + bb.fatal("Invalid BB_PRESSURE_MAX_CPU %s, minimum value is %s." % (self.max_cpu_pressure, lower_limit)) + if self.max_cpu_pressure > upper_limit: + bb.warn("Your build will be largely unregulated since BB_PRESSURE_MAX_CPU is set to %s. It is very unlikely that such high pressure will be experienced." % (self.max_cpu_pressure)) + + if self.max_io_pressure: + self.max_io_pressure = float(self.max_io_pressure) + if self.max_io_pressure < lower_limit: + bb.fatal("Invalid BB_PRESSURE_MAX_IO %s, minimum value is %s." % (self.max_io_pressure, lower_limit)) + if self.max_io_pressure > upper_limit: + bb.warn("Your build will be largely unregulated since BB_PRESSURE_MAX_IO is set to %s. It is very unlikely that such high pressure will be experienced." % (self.max_io_pressure)) + # List of setscene tasks which we've covered self.scenequeue_covered = set() # List of tasks which are covered (including setscene ones) |