diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2018-12-28 16:40:33 +0000 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2019-01-08 11:15:58 +0000 |
commit | ae79868861568d673a70472e85a4bde9e2d84a8f (patch) | |
tree | b0bff3cdacb40b63626c433f52ab594fbffddf23 /lib | |
parent | b2d39fc37fcf3c81a562ec1ef4f8b4c1a493fc57 (diff) | |
download | bitbake-contrib-ae79868861568d673a70472e85a4bde9e2d84a8f.tar.gz |
cooker: Split recipes to parse amongst threads ahead of time
We have two choices, split the recipes amongst the parsing threads in
blocks ahead of time, or have a queue which parsers pull from when idle.
The optimum approach depends on how similar the pieces are. For the single
recipe reparse case, there is currently a significant wait for the feeder
thread to start (around 0.25s in a 2s command).
Its possible splitting into blocks in advance may be unluckly for some other
workloads but experimentally it seems to work better overall for me at least.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'lib')
-rw-r--r-- | lib/bb/cooker.py | 17 |
1 files changed, 6 insertions, 11 deletions
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index d1d2868d6..e6b8d880a 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -1941,11 +1941,8 @@ class Parser(multiprocessing.Process): result = pending.pop() else: try: - job = self.jobs.get(timeout=0.25) - except queue.Empty: - continue - - if job is None: + job = self.jobs.pop() + except IndexError: break result = self.parse(*job) @@ -2032,12 +2029,12 @@ class CookerParser(object): self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes) self.result_queue = multiprocessing.Queue() - self.jobs = multiprocessing.Queue() - for j in self.willparse: - self.jobs.put(j) + def chunkify(lst,n): + return [lst[i::n] for i in range(n)] + self.jobs = chunkify(self.willparse, self.num_processes) for i in range(0, self.num_processes): - parser = Parser(self.jobs, self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) + parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) parser.start() self.process_names.append(parser.name) self.processes.append(parser) @@ -2065,8 +2062,6 @@ class CookerParser(object): for process in self.processes: self.parser_quit.put(None) - self.jobs.cancel_join_thread() - for process in self.processes: if force: process.join(.1) |