diff mbox series

[3/3] codeparser: Avoid log bufer overhead in cache case

Message ID 20220317122225.996597-3-richard.purdie@linuxfoundation.org
State Accepted, archived
Commit ac868167ad854f9bb32dcb2e63528870547805a7
Headers show
Series [1/3] data_smart: Skip commonly accessed variables from variable data context lookup | expand

Commit Message

Richard Purdie March 17, 2022, 12:22 p.m. UTC 
Creating the new log instances triggers a lot of python logging overhead
in a commonly called function (about 600,000 for parsing OE-Core).

We only need the log functionality if we're parsing, not if we just hit
from the cache. Therefore defer the log setup overhead until we know it
is a cache miss.

Whilst this complicates the code slightly, the performance gain is worth
it as for parsing OE-Core we drop 60 million funciton calls (from 225
overall).

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/codeparser.py | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)
diff mbox series

Patch

diff --git a/lib/bb/codeparser.py b/lib/bb/codeparser.py
index 0cec452c00..3b3c3b41ff 100644
--- a/lib/bb/codeparser.py
+++ b/lib/bb/codeparser.py
@@ -195,6 +195,10 @@  class BufferedLogger(Logger):
                 self.target.handle(record)
         self.buffer = []
 
+class DummyLogger():
+    def flush(self):
+        return
+
 class PythonParser():
     getvars = (".getVar", ".appendVar", ".prependVar", "oe.utils.conditional")
     getvarflags = (".getVarFlag", ".appendVarFlag", ".prependVarFlag")
@@ -276,7 +280,9 @@  class PythonParser():
         self.contains = {}
         self.execs = set()
         self.references = set()
-        self.log = BufferedLogger('BitBake.Data.PythonParser', logging.DEBUG, log)
+        self._log = log
+        # Defer init as expensive
+        self.log = DummyLogger()
 
         self.unhandled_message = "in call of %s, argument '%s' is not a string literal"
         self.unhandled_message = "while parsing %s, %s" % (name, self.unhandled_message)
@@ -303,6 +309,9 @@  class PythonParser():
                 self.contains[i] = set(codeparsercache.pythoncacheextras[h].contains[i])
             return
 
+        # Need to parse so take the hit on the real log buffer
+        self.log = BufferedLogger('BitBake.Data.PythonParser', logging.DEBUG, self._log)
+
         # We can't add to the linenumbers for compile, we can pad to the correct number of blank lines though
         node = "\n" * int(lineno) + node
         code = compile(check_indent(str(node)), filename, "exec",
@@ -321,7 +330,11 @@  class ShellParser():
         self.funcdefs = set()
         self.allexecs = set()
         self.execs = set()
-        self.log = BufferedLogger('BitBake.Data.%s' % name, logging.DEBUG, log)
+        self._name = name
+        self._log = log
+        # Defer init as expensive
+        self.log = DummyLogger()
+
         self.unhandled_template = "unable to handle non-literal command '%s'"
         self.unhandled_template = "while parsing %s, %s" % (name, self.unhandled_template)
 
@@ -340,6 +353,9 @@  class ShellParser():
             self.execs = set(codeparsercache.shellcacheextras[h].execs)
             return self.execs
 
+        # Need to parse so take the hit on the real log buffer
+        self.log = BufferedLogger('BitBake.Data.%s' % self._name, logging.DEBUG, self._log)
+
         self._parse_shell(value)
         self.execs = set(cmd for cmd in self.allexecs if cmd not in self.funcdefs)