diff options
| author |   Etienne Cordonnier <ecordonnier@snap.com> | 2023-09-20 09:41:48 +0200 |
|---|---|---|
| committer |   Richard Purdie <richard.purdie@linuxfoundation.org> | 2023-09-22 07:45:38 +0100 |
| commit | 2302b5316091dff189e6c3f546341b2274ed9d0a (patch) | |
| tree | d125a82b1e83370e36f5498f83cd6900aa716c70 | |
| parent | b90520eedb1dbc7f6a3928d089fe74fafb864eb5 (diff) | |
| download | bitbake-contrib-2302b5316091dff189e6c3f546341b2274ed9d0a.tar.gz | |
bitbake-worker/runqueue: Avoid unnecessary bytes object copies
declaring queue=b"" creates an object of types bytes().
bytes() is an immutable object, and therefore doing "self.queue = self.queue + r"
creates a new object containing "self.queue" concatenated with "r".
On my test setup, we are passing 180MB of data of "workerdata" to the bitbake-worker,
so those copies significantly slow down the initialization of the bitbake-worker.
Rather use bytearray() which a mutable type, and use extend() to avoid copies.
In my test setup, byterray.extend() is 10.000 times faster than copying the queue,
for a queue size of 180MB.
Signed-off-by: Etienne Cordonnier <ecordonnier@snap.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
| -rwxr-xr-x | bin/bitbake-worker | 14 | ||||
| -rw-r--r-- | lib/bb/runqueue.py | 4 |
2 files changed, 9 insertions, 9 deletions
diff --git a/bin/bitbake-worker b/bin/bitbake-worker index 451e6926b..609e276fe 100755 --- a/bin/bitbake-worker +++ b/bin/bitbake-worker @@ -91,19 +91,19 @@ def worker_fire_prepickled(event): worker_thread_exit = False def worker_flush(worker_queue): - worker_queue_int = b"" + worker_queue_int = bytearray() global worker_pipe, worker_thread_exit while True: try: - worker_queue_int = worker_queue_int + worker_queue.get(True, 1) + worker_queue_int.extend(worker_queue.get(True, 1)) except queue.Empty: pass while (worker_queue_int or not worker_queue.empty()): try: (_, ready, _) = select.select([], [worker_pipe], [], 1) if not worker_queue.empty(): - worker_queue_int = worker_queue_int + worker_queue.get() + worker_queue_int.extend(worker_queue.get()) written = os.write(worker_pipe, worker_queue_int) worker_queue_int = worker_queue_int[written:] except (IOError, OSError) as e: @@ -346,12 +346,12 @@ class runQueueWorkerPipe(): if pipeout: pipeout.close() bb.utils.nonblockingfd(self.input) - self.queue = b"" + self.queue = bytearray() def read(self): start = len(self.queue) try: - self.queue = self.queue + (self.input.read(102400) or b"") + self.queue.extend(self.input.read(102400) or b"") except (OSError, IOError) as e: if e.errno != errno.EAGAIN: raise @@ -379,7 +379,7 @@ class BitbakeWorker(object): def __init__(self, din): self.input = din bb.utils.nonblockingfd(self.input) - self.queue = b"" + self.queue = bytearray() self.cookercfg = None self.databuilder = None self.data = None @@ -413,7 +413,7 @@ class BitbakeWorker(object): if len(r) == 0: # EOF on pipe, server must have terminated self.sigterm_exception(signal.SIGTERM, None) - self.queue = self.queue + r + self.queue.extend(r) except (OSError, IOError): pass if len(self.queue): diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py index c88d7129c..c40a3be21 100644 --- a/lib/bb/runqueue.py +++ b/lib/bb/runqueue.py @@ -3159,7 +3159,7 @@ class runQueuePipe(): if pipeout: pipeout.close() bb.utils.nonblockingfd(self.input) - self.queue = b"" + self.queue = bytearray() self.d = d self.rq = rq self.rqexec = rqexec @@ -3178,7 +3178,7 @@ class runQueuePipe(): start = len(self.queue) try: - self.queue = self.queue + (self.input.read(102400) or b"") + self.queue.extend(self.input.read(102400) or b"") except (OSError, IOError) as e: if e.errno != errno.EAGAIN: raise |