aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2018-12-28 10:23:05 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-01-08 11:15:58 +0000
commitb2d39fc37fcf3c81a562ec1ef4f8b4c1a493fc57 (patch)
treece7e6b2d802531327bd2a090e3709d0eab529981
parent08c314eac231ac9292c8c95f1c5a6fc2023fe749 (diff)
downloadbitbake-contrib-b2d39fc37fcf3c81a562ec1ef4f8b4c1a493fc57.tar.gz
cooker: Remove Feeder() since its no longer needed with moderm multiprocessing
There used to be many bugs in multiprocessing and we implemented our own feeder process to avoid them. Now that we have python 3.x, these are fixed and just using the standard Queue mechanism appears to work fine. We can therefore drop the unneeded code and simplify. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--lib/bb/cooker.py42
1 files changed, 5 insertions, 37 deletions
diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index db52964c3..d1d2868d6 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -1895,35 +1895,6 @@ class ParsingFailure(Exception):
self.recipe = recipe
Exception.__init__(self, realexception, recipe)
-class Feeder(multiprocessing.Process):
- def __init__(self, jobs, to_parsers, quit):
- self.quit = quit
- self.jobs = jobs
- self.to_parsers = to_parsers
- multiprocessing.Process.__init__(self)
-
- def run(self):
- while True:
- try:
- quit = self.quit.get_nowait()
- except queue.Empty:
- pass
- else:
- if quit == 'cancel':
- self.to_parsers.cancel_join_thread()
- break
-
- try:
- job = self.jobs.pop()
- except IndexError:
- break
-
- try:
- self.to_parsers.put(job, timeout=0.5)
- except queue.Full:
- self.jobs.insert(0, job)
- continue
-
class Parser(multiprocessing.Process):
def __init__(self, jobs, results, quit, init, profile):
self.jobs = jobs
@@ -2058,12 +2029,13 @@ class CookerParser(object):
multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1)
multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1)
- self.feeder_quit = multiprocessing.Queue(maxsize=1)
self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes)
- self.jobs = multiprocessing.Queue(maxsize=self.num_processes)
self.result_queue = multiprocessing.Queue()
- self.feeder = Feeder(self.willparse, self.jobs, self.feeder_quit)
- self.feeder.start()
+
+ self.jobs = multiprocessing.Queue()
+ for j in self.willparse:
+ self.jobs.put(j)
+
for i in range(0, self.num_processes):
parser = Parser(self.jobs, self.result_queue, self.parser_quit, init, self.cooker.configuration.profile)
parser.start()
@@ -2086,12 +2058,9 @@ class CookerParser(object):
self.total)
bb.event.fire(event, self.cfgdata)
- self.feeder_quit.put(None)
for process in self.processes:
self.parser_quit.put(None)
else:
- self.feeder_quit.put('cancel')
-
self.parser_quit.cancel_join_thread()
for process in self.processes:
self.parser_quit.put(None)
@@ -2104,7 +2073,6 @@ class CookerParser(object):
process.terminate()
else:
process.join()
- self.feeder.join()
sync = threading.Thread(target=self.bb_cache.sync)
sync.start()