From 7f2f9b3ff011b340b5d23bb7c47b12c357dc9f02 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Fri, 1 Sep 2017 14:24:31 +0100 Subject: lib/oe/utils: Handle exceptions in multiprocess_exec Currently exceptions that happen in pool commands are ignored. Any errors would be printed on the console but everything else is silent. Switch to use pool.map_async which allows for an error_callback which we can use to detect exceptions and make sure these errors are handled. Signed-off-by: Richard Purdie --- meta/lib/oe/utils.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'meta/lib/oe/utils.py') diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py index 822d0cd586..643ab78df7 100644 --- a/meta/lib/oe/utils.py +++ b/meta/lib/oe/utils.py @@ -224,25 +224,30 @@ def multiprocess_exec(commands, function): def init_worker(): signal.signal(signal.SIGINT, signal.SIG_IGN) + fails = [] + + def failures(res): + fails.append(res) + nproc = min(multiprocessing.cpu_count(), len(commands)) pool = bb.utils.multiprocessingpool(nproc, init_worker) - imap = pool.imap(function, commands) try: - res = list(imap) + mapresult = pool.map_async(function, commands, error_callback=failures) + pool.close() pool.join() - results = [] - for result in res: - if result is not None: - results.append(result) - return results - + results = mapresult.get() except KeyboardInterrupt: pool.terminate() pool.join() raise + if fails: + raise fails[0] + + return results + def squashspaces(string): import re return re.sub("\s+", " ", string).strip() -- cgit 1.2.3-korg