diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2022-11-16 23:11:38 +0000 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2022-11-20 08:31:24 +0000 |
commit | 3a82acdcf40bdccd933c4dcef3d7e480f0d7ad3a (patch) | |
tree | ad3cd3473a402e402712130383e669033a317ec7 /bin | |
parent | c97c1f1c127ef3f8fbbd1b4e187ab58bfb0a73e5 (diff) | |
download | bitbake-3a82acdcf40bdccd933c4dcef3d7e480f0d7ad3a.tar.gz |
worker/runqueue: Reduce initial data transfer in workerdata
When setting up the worker we were transfering large amounts of data
which aren't needed until task execution time.
Defer the fakeroot and taskdeps data until they're needed for a specific
task. This will duplicate some information when executing different tasks
for a given recipe but as is is spread over the build run, it shouldn't
be an issue overall.
Also take the opportunity to clean up the silly length argument lists
that were being passed around at the expense of extra dictionary keys.
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bin')
-rwxr-xr-x | bin/bitbake-worker | 32 |
1 files changed, 22 insertions, 10 deletions
diff --git a/bin/bitbake-worker b/bin/bitbake-worker index d54044f36..3799b170c 100755 --- a/bin/bitbake-worker +++ b/bin/bitbake-worker @@ -145,7 +145,16 @@ def sigterm_handler(signum, frame): os.killpg(0, signal.SIGTERM) sys.exit() -def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskhash, unihash, appends, taskdepdata, extraconfigdata, quieterrors=False, dry_run_exec=False): +def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask): + + fn = runtask['fn'] + task = runtask['task'] + taskname = runtask['taskname'] + taskhash = runtask['taskhash'] + unihash = runtask['unihash'] + appends = runtask['appends'] + taskdepdata = runtask['taskdepdata'] + quieterrors = runtask['quieterrors'] # We need to setup the environment BEFORE the fork, since # a fork() or exec*() activates PSEUDO... @@ -157,8 +166,7 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskha uid = os.getuid() gid = os.getgid() - - taskdep = workerdata["taskdeps"][fn] + taskdep = runtask['taskdep'] if 'umask' in taskdep and taskname in taskdep['umask']: umask = taskdep['umask'][taskname] elif workerdata["umask"]: @@ -170,24 +178,24 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskha except TypeError: pass - dry_run = cfg.dry_run or dry_run_exec + dry_run = cfg.dry_run or runtask['dry_run'] # We can't use the fakeroot environment in a dry run as it possibly hasn't been built if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not dry_run: fakeroot = True - envvars = (workerdata["fakerootenv"][fn] or "").split() + envvars = (runtask['fakerootenv'] or "").split() for key, value in (var.split('=') for var in envvars): envbackup[key] = os.environ.get(key) os.environ[key] = value fakeenv[key] = value - fakedirs = (workerdata["fakerootdirs"][fn] or "").split() + fakedirs = (runtask['fakerootdirs'] or "").split() for p in fakedirs: bb.utils.mkdirhier(p) logger.debug2('Running %s:%s under fakeroot, fakedirs: %s' % (fn, taskname, ', '.join(fakedirs))) else: - envvars = (workerdata["fakerootnoenv"][fn] or "").split() + envvars = (runtask['fakerootnoenv'] or "").split() for key, value in (var.split('=') for var in envvars): envbackup[key] = os.environ.get(key) os.environ[key] = value @@ -474,11 +482,15 @@ class BitbakeWorker(object): sys.exit(0) def handle_runtask(self, data): - fn, task, taskname, taskhash, unihash, quieterrors, appends, taskdepdata, dry_run_exec = pickle.loads(data) - workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname)) + runtask = pickle.loads(data) - pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, taskhash, unihash, appends, taskdepdata, self.extraconfigdata, quieterrors, dry_run_exec) + fn = runtask['fn'] + task = runtask['task'] + taskname = runtask['taskname'] + + workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname)) + pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, self.extraconfigdata, runtask) self.build_pids[pid] = task self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout) |