diff mbox series

[4/5] cooker: Reset and rebuild inotify watches

Message ID 20220403102144.1679700-4-richard.purdie@linuxfoundation.org
State Accepted, archived
Commit 3df322a200c28b45af1f2c92478c85eb7d20c38b
Headers show
Series [1/5] parse: Ensure any existing siggen is closed down first | expand

Commit Message

Richard Purdie April 3, 2022, 10:21 a.m. UTC 
The recent inotify changes can cause entire build trees to be monitored
which is suboptimal for performance.

Rather than trying increasingly convoluted tricks to try and handle add/removed
directories, rebuild the inotify watch when we reparse the configuration or
metadata.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/cooker.py | 54 ++++++++++++++++++++++++++----------------------
 1 file changed, 29 insertions(+), 25 deletions(-)
diff mbox series

Patch

diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 5a941f879f..7685db11f3 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -188,27 +188,15 @@  class BBCooker:
         bb.debug(1, "BBCooker starting %s" % time.time())
         sys.stdout.flush()
 
-        self.configwatcher = pyinotify.WatchManager()
-        bb.debug(1, "BBCooker pyinotify1 %s" % time.time())
-        sys.stdout.flush()
+        self.configwatcher = None
+        self.confignotifier = None
 
-        self.configwatcher.bbseen = set()
-        self.configwatcher.bbwatchedfiles = set()
-        self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications)
-        bb.debug(1, "BBCooker pyinotify2 %s" % time.time())
-        sys.stdout.flush()
         self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \
                          pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \
                          pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO
-        self.watcher = pyinotify.WatchManager()
-        bb.debug(1, "BBCooker pyinotify3 %s" % time.time())
-        sys.stdout.flush()
-        self.watcher.bbseen = set()
-        self.watcher.bbwatchedfiles = set()
-        self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
 
-        bb.debug(1, "BBCooker pyinotify complete %s" % time.time())
-        sys.stdout.flush()
+        self.watcher = None
+        self.notifier = None
 
         # If being called by something like tinfoil, we need to clean cached data
         # which may now be invalid
@@ -259,9 +247,29 @@  class BBCooker:
             sys.stdout.flush()
             self.handlePRServ()
 
+    def setupConfigWatcher(self):
+        if self.configwatcher:
+            self.configwatcher.close()
+            self.confignotifier = None
+            self.configwatcher = None
+        self.configwatcher = pyinotify.WatchManager()
+        self.configwatcher.bbseen = set()
+        self.configwatcher.bbwatchedfiles = set()
+        self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications)
+
+    def setupParserWatcher(self):
+        if self.watcher:
+            self.watcher.close()
+            self.notifier = None
+            self.watcher = None
+        self.watcher = pyinotify.WatchManager()
+        self.watcher.bbseen = set()
+        self.watcher.bbwatchedfiles = set()
+        self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
+
     def process_inotify_updates(self):
         for n in [self.confignotifier, self.notifier]:
-            if n.check_events(timeout=0):
+            if n and n.check_events(timeout=0):
                 # read notified events and enqeue them
                 n.read_events()
                 n.process_events()
@@ -281,10 +289,6 @@  class BBCooker:
                     self.configwatcher.bbseen.remove(event.pathname)
                 # Could remove all entries starting with the directory but for now...
                 bb.parse.clear_cache()
-            if "IN_CREATE" in event.maskname:
-                self.add_filewatch([[event.pathname]], watcher=self.configwatcher, dirs=True)
-            elif "IN_DELETE" in event.maskname and event.pathname in self.configwatcher.bbseen:
-                self.configwatcher.bbseen.remove(event.pathname)
         if not event.pathname in self.inotify_modified_files:
             self.inotify_modified_files.append(event.pathname)
         self.baseconfig_valid = False
@@ -304,10 +308,6 @@  class BBCooker:
                     self.watcher.bbseen.remove(event.pathname)
                 # Could remove all entries starting with the directory but for now...
                 bb.parse.clear_cache()
-            if "IN_CREATE" in event.maskname:
-                self.add_filewatch([[event.pathname]], dirs=True)
-            elif "IN_DELETE" in event.maskname and event.pathname in self.watcher.bbseen:
-                self.watcher.bbseen.remove(event.pathname)
         if not event.pathname in self.inotify_modified_files:
             self.inotify_modified_files.append(event.pathname)
         self.parsecache_valid = False
@@ -377,6 +377,8 @@  class BBCooker:
             if mod not in self.orig_sysmodules:
                 del sys.modules[mod]
 
+        self.setupConfigWatcher()
+
         # Need to preserve BB_CONSOLELOG over resets
         consolelog = None
         if hasattr(self, "data"):
@@ -1658,6 +1660,8 @@  class BBCooker:
             self.updateCacheSync()
 
         if self.state != state.parsing and not self.parsecache_valid:
+            self.setupParserWatcher()
+
             bb.parse.siggen.reset(self.data)
             self.parseConfiguration ()
             if CookerFeatures.SEND_SANITYEVENTS in self.featureset: