@@ -84,7 +84,7 @@ class Command:
if not hasattr(command_method, 'readonly') or not getattr(command_method, 'readonly'):
return None, "Not able to execute not readonly commands in readonly mode"
try:
- self.cooker.process_inotify_updates()
+ self.cooker.process_inotify_updates_apply()
if getattr(command_method, 'needconfig', True):
self.cooker.updateCacheSync()
result = command_method(self, commandline)
@@ -109,7 +109,7 @@ class Command:
def runAsyncCommand(self):
try:
- self.cooker.process_inotify_updates()
+ self.cooker.process_inotify_updates_apply()
if self.cooker.state in (bb.cooker.state.error, bb.cooker.state.shutdown, bb.cooker.state.forceshutdown):
# updateCache will trigger a shutdown of the parser
# and then raise BBHandledException triggering an exit
@@ -220,6 +220,8 @@ class BBCooker:
bb.debug(1, "BBCooker startup complete %s" % time.time())
sys.stdout.flush()
+ self.inotify_threadlock = threading.Lock()
+
def init_configdata(self):
if not hasattr(self, "data"):
self.initConfigurationData()
@@ -248,11 +250,18 @@ class BBCooker:
self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
def process_inotify_updates(self):
- for n in [self.confignotifier, self.notifier]:
- if n and n.check_events(timeout=0):
- # read notified events and enqueue them
- n.read_events()
- n.process_events()
+ with self.inotify_threadlock:
+ for n in [self.confignotifier, self.notifier]:
+ if n and n.check_events(timeout=0):
+ # read notified events and enqueue them
+ n.read_events()
+
+ def process_inotify_updates_apply(self):
+ with self.inotify_threadlock:
+ for n in [self.confignotifier, self.notifier]:
+ if n and n.check_events(timeout=0):
+ n.read_events()
+ n.process_events()
def config_notifications(self, event):
if event.maskname == "IN_Q_OVERFLOW":
@@ -88,6 +88,7 @@ class ProcessServer():
self.maxuiwait = 30
self.xmlrpc = False
+ self.idle = None
self._idlefuns = {}
self.bitbake_lock = lock
@@ -148,6 +149,7 @@ class ProcessServer():
self.cooker.pre_serve()
bb.utils.set_process_name("Cooker")
+ bb.event.enable_threadlock()
ready = []
newconnections = []
@@ -278,6 +280,9 @@ class ProcessServer():
ready = self.idle_commands(.1, fds)
+ if self.idle:
+ self.idle.join()
+
serverlog("Exiting (socket: %s)" % os.path.exists(self.sockname))
# Remove the socket file so we don't get any more connections to avoid races
# The build directory could have been renamed so if the file isn't the one we created
@@ -352,33 +357,44 @@ class ProcessServer():
msg.append(":\n%s" % procs)
serverlog("".join(msg))
+ def idle_thread(self):
+ while not self.quit:
+ nextsleep = 0.1
+ fds = []
+ for function, data in list(self._idlefuns.items()):
+ try:
+ retval = function(self, data, False)
+ if retval is False:
+ del self._idlefuns[function]
+ nextsleep = None
+ elif retval is True:
+ nextsleep = None
+ elif isinstance(retval, float) and nextsleep:
+ if (retval < nextsleep):
+ nextsleep = retval
+ elif nextsleep is None:
+ continue
+ else:
+ fds = fds + retval
+ except SystemExit:
+ raise
+ except Exception as exc:
+ if not isinstance(exc, bb.BBHandledException):
+ logger.exception('Running idle function')
+ del self._idlefuns[function]
+ self.quit = True
+
+ if nextsleep is not None:
+ select.select(fds,[],[],nextsleep)[0]
+
def idle_commands(self, delay, fds=None):
nextsleep = delay
if not fds:
fds = []
- for function, data in list(self._idlefuns.items()):
- try:
- retval = function(self, data, False)
- if retval is False:
- del self._idlefuns[function]
- nextsleep = None
- elif retval is True:
- nextsleep = None
- elif isinstance(retval, float) and nextsleep:
- if (retval < nextsleep):
- nextsleep = retval
- elif nextsleep is None:
- continue
- else:
- fds = fds + retval
- except SystemExit:
- raise
- except Exception as exc:
- if not isinstance(exc, bb.BBHandledException):
- logger.exception('Running idle function')
- del self._idlefuns[function]
- self.quit = True
+ if not self.idle:
+ self.idle = threading.Thread(target=self.idle_thread)
+ self.idle.start()
# Create new heartbeat event?
now = time.time()
When bitbake is off running heavier "idle" commands, it doesn't service it's command socket which means stopping/interrupting it is hard. It also means we can't "ping" from the UI to know if it is still alive. For those reasons, split idle command execution into it's own thread. The commands are generally already self containted so this is easier than expected. We do have to be careful to only handle inotify poll() from a single thread at a time. It also means we always have to use a thread lock when sending events since both the idle thread and the command thread may generate log messages (and hence events). The patch does depend on a couple of previous fixes to the builtins locking in event.py and the heartbeat enable/disable changes. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> --- lib/bb/command.py | 4 +-- lib/bb/cooker.py | 19 +++++++++---- lib/bb/server/process.py | 60 +++++++++++++++++++++++++--------------- 3 files changed, 54 insertions(+), 29 deletions(-)