diff options
author | Jianxun Zhang <jianxun.zhang@linux.intel.com> | 2016-12-21 12:27:37 -0800 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2017-01-17 11:29:28 +0000 |
commit | 7eb02e837ed253a2ae69023595c25660b4d56b7c (patch) | |
tree | 3eb6a36ac22a10787419b7cb369dfc4ba3bf2e55 | |
parent | 62591d97221714678a3a02c5c42fffe9e538f5f9 (diff) | |
download | openembedded-core-contrib-7eb02e837ed253a2ae69023595c25660b4d56b7c.tar.gz |
bitbake: use multiple processes to dump signatures.
This change significantly shortens the time on reparsing stage
of '-S' option.
Each file is reparsed and then dumped within a dedicated
process. The maximum number of the running processes is not
greater than the value of BB_NUMBER_PARSE_THREADS if it is set.
The dump_sigs() in class SignatureGeneratorBasic is _replaced_
by a new dump_sigfn() interface, so calls from the outside and
subclasses are dispatched to the implementation in the base
class of SignatureGeneratorBasic.
Fixes [YOCTO #10352]
(Bitbake rev: 99d3703edd77a21770b366c6ad65a3c0f5183493)
Signed-off-by: Jianxun Zhang <jianxun.zhang@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r-- | bitbake/lib/bb/runqueue.py | 32 | ||||
-rw-r--r-- | bitbake/lib/bb/siggen.py | 4 |
2 files changed, 29 insertions, 7 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 48c6a79ffb..d42eb81664 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py @@ -36,6 +36,7 @@ from bb import msg, data, event from bb import monitordisk import subprocess import pickle +from multiprocessing import Process bblogger = logging.getLogger("BitBake") logger = logging.getLogger("BitBake.RunQueue") @@ -1303,15 +1304,36 @@ class RunQueue: else: self.rqexe.finish() + def rq_dump_sigfn(self, fn, options): + bb_cache = bb.cache.NoCache(self.cooker.databuilder) + the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn)) + siggen = bb.parse.siggen + dataCaches = self.rqdata.dataCaches + siggen.dump_sigfn(fn, dataCaches, options) + def dump_signatures(self, options): - done = set() + fns = set() bb.note("Reparsing files to collect dependency data") - bb_cache = bb.cache.NoCache(self.cooker.databuilder) + for tid in self.rqdata.runtaskentries: fn = fn_from_tid(tid) - if fn not in done: - the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn)) - done.add(fn) + fns.add(fn) + + max_process = int(self.cfgData.getVar("BB_NUMBER_PARSE_THREADS") or os.cpu_count() or 1) + # We cannot use the real multiprocessing.Pool easily due to some local data + # that can't be pickled. This is a cheap multi-process solution. + launched = [] + while fns: + if len(launched) < max_process: + p = Process(target=self.rq_dump_sigfn, args=(fns.pop(), options)) + p.start() + launched.append(p) + for q in launched: + # The finished processes are joined when calling is_alive() + if not q.is_alive(): + launched.remove(q) + for p in launched: + p.join() bb.parse.siggen.dump_sigs(self.rqdata.dataCaches, options) diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py index 4226c80c85..3ceeef1ccc 100644 --- a/bitbake/lib/bb/siggen.py +++ b/bitbake/lib/bb/siggen.py @@ -307,8 +307,8 @@ class SignatureGeneratorBasic(SignatureGenerator): pass raise err - def dump_sigs(self, dataCaches, options): - for fn in self.taskdeps: + def dump_sigfn(self, fn, dataCaches, options): + if fn in self.taskdeps: for task in self.taskdeps[fn]: tid = fn + ":" + task (mc, _, _) = bb.runqueue.split_tid(tid) |