diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2018-12-28 10:23:05 +0000 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2019-01-08 11:15:58 +0000 |
commit | b2d39fc37fcf3c81a562ec1ef4f8b4c1a493fc57 (patch) | |
tree | ce7e6b2d802531327bd2a090e3709d0eab529981 /lib/bb/cooker.py | |
parent | 08c314eac231ac9292c8c95f1c5a6fc2023fe749 (diff) | |
download | bitbake-b2d39fc37fcf3c81a562ec1ef4f8b4c1a493fc57.tar.gz |
cooker: Remove Feeder() since its no longer needed with moderm multiprocessing
There used to be many bugs in multiprocessing and we implemented our own
feeder process to avoid them. Now that we have python 3.x, these are fixed
and just using the standard Queue mechanism appears to work fine. We can
therefore drop the unneeded code and simplify.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'lib/bb/cooker.py')
-rw-r--r-- | lib/bb/cooker.py | 42 |
1 files changed, 5 insertions, 37 deletions
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index db52964c3..d1d2868d6 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -1895,35 +1895,6 @@ class ParsingFailure(Exception): self.recipe = recipe Exception.__init__(self, realexception, recipe) -class Feeder(multiprocessing.Process): - def __init__(self, jobs, to_parsers, quit): - self.quit = quit - self.jobs = jobs - self.to_parsers = to_parsers - multiprocessing.Process.__init__(self) - - def run(self): - while True: - try: - quit = self.quit.get_nowait() - except queue.Empty: - pass - else: - if quit == 'cancel': - self.to_parsers.cancel_join_thread() - break - - try: - job = self.jobs.pop() - except IndexError: - break - - try: - self.to_parsers.put(job, timeout=0.5) - except queue.Full: - self.jobs.insert(0, job) - continue - class Parser(multiprocessing.Process): def __init__(self, jobs, results, quit, init, profile): self.jobs = jobs @@ -2058,12 +2029,13 @@ class CookerParser(object): multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1) multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1) - self.feeder_quit = multiprocessing.Queue(maxsize=1) self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes) - self.jobs = multiprocessing.Queue(maxsize=self.num_processes) self.result_queue = multiprocessing.Queue() - self.feeder = Feeder(self.willparse, self.jobs, self.feeder_quit) - self.feeder.start() + + self.jobs = multiprocessing.Queue() + for j in self.willparse: + self.jobs.put(j) + for i in range(0, self.num_processes): parser = Parser(self.jobs, self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) parser.start() @@ -2086,12 +2058,9 @@ class CookerParser(object): self.total) bb.event.fire(event, self.cfgdata) - self.feeder_quit.put(None) for process in self.processes: self.parser_quit.put(None) else: - self.feeder_quit.put('cancel') - self.parser_quit.cancel_join_thread() for process in self.processes: self.parser_quit.put(None) @@ -2104,7 +2073,6 @@ class CookerParser(object): process.terminate() else: process.join() - self.feeder.join() sync = threading.Thread(target=self.bb_cache.sync) sync.start() |