diff mbox series

[2/2] server/process: Improve idle thread exception handling

Message ID 20230220100733.453039-2-richard.purdie@linuxfoundation.org
State Accepted, archived
Commit 358b5b02d5de1ab0f98104c4ec4953e46999b9a5
Headers show
Series [1/2] cooker: Ensure lock is held with changing notifier | expand

Commit Message

Richard Purdie Feb. 20, 2023, 10:07 a.m. UTC 
If the inotifier code has an exception, bitbake currently hangs. Catch any
exception and exit if seen. Also check the idle thread is alive and exit
if it disappears. This should stop bitbake hanging if such a situation arises
in future such as this example:

3323260 21:48:31.554468 Running command ['getVariable', 'BBINCLUDELOGS']
Exception in thread Thread-1 (idle_thread):
Traceback (most recent call last):
  File "/usr/lib64/python3.10/threading.py", line 1016, in _bootstrap_inner
    self.run()
  File "/usr/lib64/python3.10/threading.py", line 953, in run
    self._target(*self._args, **self._kwargs)
  File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/server/process.py", line 408, in idle_thread
    self.cooker.process_inotify_updates()
  File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/cooker.py", line 256, in process_inotify_updates
    n.read_events()
  File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/pyinotify.py", line 1207, in read_events
    if fcntl.ioctl(self._fd, termios.FIONREAD, buf_, 1) == -1:
OSError: [Errno 9] Bad file descriptor
3323260 21:48:32.206995 Command Completed (socket: True)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/server/process.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)
diff mbox series

Patch

diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py
index 916ee0a0e5..db417c8428 100644
--- a/lib/bb/server/process.py
+++ b/lib/bb/server/process.py
@@ -405,7 +405,11 @@  class ProcessServer():
             nextsleep = 0.1
             fds = []
 
-            self.cooker.process_inotify_updates()
+            try:
+                self.cooker.process_inotify_updates()
+            except Exception as exc:
+                serverlog("Exception %s in inofify updates broke the idle_thread, exiting" % traceback.format_exc())
+                self.quit = True
 
             with bb.utils.lock_timeout(self._idlefuncsLock):
                 items = list(self._idlefuns.items())
@@ -473,6 +477,10 @@  class ProcessServer():
         if not self.idle:
             self.idle = threading.Thread(target=self.idle_thread)
             self.idle.start()
+        elif self.idle and not self.idle.is_alive():
+            serverlog("Idle thread terminated, main thread exiting too")
+            bb.error("Idle thread terminated, main thread exiting too")
+            self.quit = True
 
         if nextsleep is not None:
             if self.xmlrpc: