[bitbake-devel,v3,3/3] utils.py: get_file_layer(): Improve performance

Submitted by Robert Yang on Sept. 9, 2020, 11:55 a.m. | Patch ID: 176200

Details

Message ID 6581d0399348b1ffb3b387e6b40325b04678b8c8.1599652176.git.liezhi.yang@windriver.com
State New
Headers show

Commit Message

Robert Yang Sept. 9, 2020, 11:55 a.m.
The following code costs a lot of time when there are lot of layers and recipes:

     for collection in collections:
         collection_res[collection] = d.getVar('BBFILE_PATTERN_%s' % collection) or ''

My build has more than 100 layers and 3000 recipes, which calls d.getVar() 300K
(3000 * 100) times and makes 'bitbake-layers show-recipes' very slow, add a
keyword argument to get_file_layer() can fix the problem, it can save about 90%
time in my build (6min -> 40s).

Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
---
 bitbake/lib/bb/utils.py       | 12 +++++++++---
 bitbake/lib/bblayers/query.py | 12 ++++++++++--
 2 files changed, 19 insertions(+), 5 deletions(-)

Patch hide | download patch | download mbox

diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py
index 5722fced97..fc78c5436b 100644
--- a/bitbake/lib/bb/utils.py
+++ b/bitbake/lib/bb/utils.py
@@ -1458,14 +1458,20 @@  def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None):
 
     return (notadded, notremoved)
 
-
-def get_file_layer(filename, d):
-    """Determine the collection (as defined by a layer's layer.conf file) containing the specified file"""
+def get_collection_res(d):
     collections = (d.getVar('BBFILE_COLLECTIONS') or '').split()
     collection_res = {}
     for collection in collections:
         collection_res[collection] = d.getVar('BBFILE_PATTERN_%s' % collection) or ''
 
+    return collection_res
+
+
+def get_file_layer(filename, d, collection_res={}):
+    """Determine the collection (as defined by a layer's layer.conf file) containing the specified file"""
+    if not collection_res:
+        collection_res = get_collection_res(d)
+
     def path_to_layer(path):
         # Use longest path so we handle nested layers
         matchlen = 0
diff --git a/bitbake/lib/bblayers/query.py b/bitbake/lib/bblayers/query.py
index ee2db0efed..f5e3c84747 100644
--- a/bitbake/lib/bblayers/query.py
+++ b/bitbake/lib/bblayers/query.py
@@ -21,6 +21,10 @@  def plugin_init(plugins):
 
 
 class QueryPlugin(LayerPlugin):
+    def __init__(self):
+        super(QueryPlugin, self).__init__()
+        self.collection_res = {}
+
     def do_show_layers(self, args):
         """show current configured layers."""
         logger.plain("%s  %s  %s" % ("layer".ljust(20), "path".ljust(40), "priority"))
@@ -222,7 +226,6 @@  skipped recipes will also be listed, with a " (skipped)" suffix.
                             multilayer = True
                         if prov[0] != pref[0]:
                             same_ver = False
-
                     if (multilayer or not show_overlayed_only) and (same_ver or not show_same_ver_only):
                         if not items_listed:
                             logger.plain('=== %s ===' % title)
@@ -243,8 +246,13 @@  skipped recipes will also be listed, with a " (skipped)" suffix.
         else:
             return '?'
 
+    def get_collection_res(self):
+        if not self.collection_res:
+            self.collection_res = bb.utils.get_collection_res(self.tinfoil.config_data)
+        return self.collection_res
+
     def get_file_layerdir(self, filename):
-        layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data)
+        layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data, self.get_collection_res())
         return self.bbfile_collections.get(layer, None)
 
     def remove_layer_prefix(self, f):