Message ID | 20230922085146.2997927-1-ecordonnier@snap.com |
---|---|
State | Accepted, archived |
Commit | 9993a89e5b97dda5f3657e5a7cc3a4fa94ff7111 |
Headers | show |
Series | [kirkstone,2.0] bitbake-worker/runqueue: Avoid unnecessary bytes object copies | expand |
Adding Steve in CC. -- Etienne On Fri, Sep 22, 2023 at 10:51 AM <ecordonnier@snap.com> wrote: > From: Etienne Cordonnier <ecordonnier@snap.com> > > declaring queue=b"" creates an object of types bytes(). > bytes() is an immutable object, and therefore doing "self.queue = > self.queue + r" > creates a new object containing "self.queue" concatenated with "r". > > On my test setup, we are passing 180MB of data of "workerdata" to the > bitbake-worker, > so those copies significantly slow down the initialization of the > bitbake-worker. > > Rather use bytearray() which a mutable type, and use extend() to avoid > copies. > In my test setup, byterray.extend() is 10.000 times faster than copying > the queue, > for a queue size of 180MB. > > Signed-off-by: Etienne Cordonnier <ecordonnier@snap.com> > Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> > --- > bin/bitbake-worker | 14 +++++++------- > lib/bb/runqueue.py | 4 ++-- > 2 files changed, 9 insertions(+), 9 deletions(-) > > diff --git a/bin/bitbake-worker b/bin/bitbake-worker > index 5e62bc20..e02f2532 100755 > --- a/bin/bitbake-worker > +++ b/bin/bitbake-worker > @@ -91,19 +91,19 @@ def worker_fire_prepickled(event): > worker_thread_exit = False > > def worker_flush(worker_queue): > - worker_queue_int = b"" > + worker_queue_int = bytearray() > global worker_pipe, worker_thread_exit > > while True: > try: > - worker_queue_int = worker_queue_int + worker_queue.get(True, > 1) > + worker_queue_int.extend(worker_queue.get(True, 1)) > except queue.Empty: > pass > while (worker_queue_int or not worker_queue.empty()): > try: > (_, ready, _) = select.select([], [worker_pipe], [], 1) > if not worker_queue.empty(): > - worker_queue_int = worker_queue_int + > worker_queue.get() > + worker_queue_int.extend(worker_queue.get()) > written = os.write(worker_pipe, worker_queue_int) > worker_queue_int = worker_queue_int[written:] > except (IOError, OSError) as e: > @@ -338,12 +338,12 @@ class runQueueWorkerPipe(): > if pipeout: > pipeout.close() > bb.utils.nonblockingfd(self.input) > - self.queue = b"" > + self.queue = bytearray() > > def read(self): > start = len(self.queue) > try: > - self.queue = self.queue + (self.input.read(102400) or b"") > + self.queue.extend(self.input.read(102400) or b"") > except (OSError, IOError) as e: > if e.errno != errno.EAGAIN: > raise > @@ -371,7 +371,7 @@ class BitbakeWorker(object): > def __init__(self, din): > self.input = din > bb.utils.nonblockingfd(self.input) > - self.queue = b"" > + self.queue = bytearray() > self.cookercfg = None > self.databuilder = None > self.data = None > @@ -405,7 +405,7 @@ class BitbakeWorker(object): > if len(r) == 0: > # EOF on pipe, server must have terminated > self.sigterm_exception(signal.SIGTERM, None) > - self.queue = self.queue + r > + self.queue.extend(r) > except (OSError, IOError): > pass > if len(self.queue): > diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py > index 56952070..46ff30a7 100644 > --- a/lib/bb/runqueue.py > +++ b/lib/bb/runqueue.py > @@ -3101,7 +3101,7 @@ class runQueuePipe(): > if pipeout: > pipeout.close() > bb.utils.nonblockingfd(self.input) > - self.queue = b"" > + self.queue = bytearray() > self.d = d > self.rq = rq > self.rqexec = rqexec > @@ -3120,7 +3120,7 @@ class runQueuePipe(): > > start = len(self.queue) > try: > - self.queue = self.queue + (self.input.read(102400) or b"") > + self.queue.extend(self.input.read(102400) or b"") > except (OSError, IOError) as e: > if e.errno != errno.EAGAIN: > raise > -- > 2.36.1.vfs.0.0 > >
diff --git a/bin/bitbake-worker b/bin/bitbake-worker index 5e62bc20..e02f2532 100755 --- a/bin/bitbake-worker +++ b/bin/bitbake-worker @@ -91,19 +91,19 @@ def worker_fire_prepickled(event): worker_thread_exit = False def worker_flush(worker_queue): - worker_queue_int = b"" + worker_queue_int = bytearray() global worker_pipe, worker_thread_exit while True: try: - worker_queue_int = worker_queue_int + worker_queue.get(True, 1) + worker_queue_int.extend(worker_queue.get(True, 1)) except queue.Empty: pass while (worker_queue_int or not worker_queue.empty()): try: (_, ready, _) = select.select([], [worker_pipe], [], 1) if not worker_queue.empty(): - worker_queue_int = worker_queue_int + worker_queue.get() + worker_queue_int.extend(worker_queue.get()) written = os.write(worker_pipe, worker_queue_int) worker_queue_int = worker_queue_int[written:] except (IOError, OSError) as e: @@ -338,12 +338,12 @@ class runQueueWorkerPipe(): if pipeout: pipeout.close() bb.utils.nonblockingfd(self.input) - self.queue = b"" + self.queue = bytearray() def read(self): start = len(self.queue) try: - self.queue = self.queue + (self.input.read(102400) or b"") + self.queue.extend(self.input.read(102400) or b"") except (OSError, IOError) as e: if e.errno != errno.EAGAIN: raise @@ -371,7 +371,7 @@ class BitbakeWorker(object): def __init__(self, din): self.input = din bb.utils.nonblockingfd(self.input) - self.queue = b"" + self.queue = bytearray() self.cookercfg = None self.databuilder = None self.data = None @@ -405,7 +405,7 @@ class BitbakeWorker(object): if len(r) == 0: # EOF on pipe, server must have terminated self.sigterm_exception(signal.SIGTERM, None) - self.queue = self.queue + r + self.queue.extend(r) except (OSError, IOError): pass if len(self.queue): diff --git a/lib/bb/runqueue.py b/lib/bb/runqueue.py index 56952070..46ff30a7 100644 --- a/lib/bb/runqueue.py +++ b/lib/bb/runqueue.py @@ -3101,7 +3101,7 @@ class runQueuePipe(): if pipeout: pipeout.close() bb.utils.nonblockingfd(self.input) - self.queue = b"" + self.queue = bytearray() self.d = d self.rq = rq self.rqexec = rqexec @@ -3120,7 +3120,7 @@ class runQueuePipe(): start = len(self.queue) try: - self.queue = self.queue + (self.input.read(102400) or b"") + self.queue.extend(self.input.read(102400) or b"") except (OSError, IOError) as e: if e.errno != errno.EAGAIN: raise