From patchwork Wed Dec 21 23:35:05 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 17111 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 3BE03C4332F for ; Wed, 21 Dec 2022 23:35:17 +0000 (UTC) Received: from mail-wm1-f50.google.com (mail-wm1-f50.google.com [209.85.128.50]) by mx.groups.io with SMTP id smtpd.web10.33951.1671665710298879048 for ; Wed, 21 Dec 2022 15:35:10 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=dOUxTtYW; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.50, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f50.google.com with SMTP id f13-20020a1cc90d000000b003d08c4cf679so102295wmb.5 for ; Wed, 21 Dec 2022 15:35:10 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=content-transfer-encoding:mime-version:message-id:date:subject:to :from:from:to:cc:subject:date:message-id:reply-to; bh=M17YPV9+LXFC21btQGaTy1+O0Rg0A/VlmpfX4t3Vehs=; b=dOUxTtYWrwqbaiuJzeTvNayADRXz9pyKkvf+hR3yvMqW36Ad/uptHHrV6wenLVB2mP fkrh55OthR1MPTXsS/olgc1x5EccWI6+l3RQlsgZMknG9koKeD9FiE9jAlVZ9FbUMeKp KODUxRVcVRGxEXBLG1dHERB9SxSrfc8r3lTl4= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:message-id:date:subject:to :from:x-gm-message-state:from:to:cc:subject:date:message-id:reply-to; bh=M17YPV9+LXFC21btQGaTy1+O0Rg0A/VlmpfX4t3Vehs=; b=bGVFPAHHpmZBvUGqCTZvqDIKYyZUMNnIgBCZmVUjSmW5FiDHVobQSdhzv64dqNC9zS m/LrQ9Q8mQ/rIHttJUG//xIT0CulUG07zoQSqaIFPhz25YeHSME6apZqgweqcxqoi6rS 80L2wWGrTuvtud+E+m4euzKkwexWWbiYxZfq3SrSRedXAhQdaV+z/B5AHNLAYZTOIQzp IvgfuUVXWko8uy4NAA/RbX7TJ8Vtl31oCVd5o/cEJf3gvfducwvo/APMHPWuQuL93j8t UX7Lexr9rFIPpu38zDWZv0gsgW8OXXURXOsLpKLlN+YSwRvW9aSMsRlPXFPV9YiMfOhN NAcw== X-Gm-Message-State: AFqh2kr6fEO1EebExEgkajOuz/Eoe4+QqL21yOmzU7fn6reeqNMUFyBF preiNLheHqBAE/DZ17QUZaS0rrifubidjQEW X-Google-Smtp-Source: AMrXdXvSpCifX5t2Ch+NBcqbHKJC+J17ik1w/lLDqEP/lkbAcIVXk5+K1ZlmWONIifjSa3v6zAY9nA== X-Received: by 2002:a7b:c3c6:0:b0:3d2:3376:6f37 with SMTP id t6-20020a7bc3c6000000b003d233766f37mr2815280wmj.10.1671665708503; Wed, 21 Dec 2022 15:35:08 -0800 (PST) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:a8db:6d5d:f54f:c6b2]) by smtp.gmail.com with ESMTPSA id g15-20020a05600c310f00b003b47e75b401sm4063967wmo.37.2022.12.21.15.35.07 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 21 Dec 2022 15:35:08 -0800 (PST) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 1/3] event: Always use threadlock Date: Wed, 21 Dec 2022 23:35:05 +0000 Message-Id: <20221221233507.519249-1-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.37.2 MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Wed, 21 Dec 2022 23:35:17 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/14227 With the move to a server idle thread, we always need threading. The existing accessor functions could end up turning this off! I was going to hold the lock whilst changing it, check if the value was already set, cache the result and also fix the event code to always release the lock with a try/finally. Instead, disable the existing functions and use a with: block to handle the lock, keeping things much simpler. Signed-off-by: Richard Purdie --- lib/bb/event.py | 73 +++++++++++++++++++--------------------- lib/bb/server/process.py | 1 - lib/bb/tests/event.py | 17 +--------- 3 files changed, 35 insertions(+), 56 deletions(-) diff --git a/lib/bb/event.py b/lib/bb/event.py index db90724444..7826541a64 100644 --- a/lib/bb/event.py +++ b/lib/bb/event.py @@ -68,16 +68,15 @@ _catchall_handlers = {} _eventfilter = None _uiready = False _thread_lock = threading.Lock() -_thread_lock_enabled = False _heartbeat_enabled = False def enable_threadlock(): - global _thread_lock_enabled - _thread_lock_enabled = True + # Always needed now + return def disable_threadlock(): - global _thread_lock_enabled - _thread_lock_enabled = False + # Always needed now + return def enable_heartbeat(): global _heartbeat_enabled @@ -179,36 +178,30 @@ def print_ui_queue(): def fire_ui_handlers(event, d): global _thread_lock - global _thread_lock_enabled if not _uiready: # No UI handlers registered yet, queue up the messages ui_queue.append(event) return - if _thread_lock_enabled: - _thread_lock.acquire() - - errors = [] - for h in _ui_handlers: - #print "Sending event %s" % event - try: - if not _ui_logfilters[h].filter(event): - continue - # We use pickle here since it better handles object instances - # which xmlrpc's marshaller does not. Events *must* be serializable - # by pickle. - if hasattr(_ui_handlers[h].event, "sendpickle"): - _ui_handlers[h].event.sendpickle((pickle.dumps(event))) - else: - _ui_handlers[h].event.send(event) - except: - errors.append(h) - for h in errors: - del _ui_handlers[h] - - if _thread_lock_enabled: - _thread_lock.release() + with _thread_lock: + errors = [] + for h in _ui_handlers: + #print "Sending event %s" % event + try: + if not _ui_logfilters[h].filter(event): + continue + # We use pickle here since it better handles object instances + # which xmlrpc's marshaller does not. Events *must* be serializable + # by pickle. + if hasattr(_ui_handlers[h].event, "sendpickle"): + _ui_handlers[h].event.sendpickle((pickle.dumps(event))) + else: + _ui_handlers[h].event.send(event) + except: + errors.append(h) + for h in errors: + del _ui_handlers[h] def fire(event, d): """Fire off an Event""" @@ -322,21 +315,23 @@ def set_eventfilter(func): _eventfilter = func def register_UIHhandler(handler, mainui=False): - bb.event._ui_handler_seq = bb.event._ui_handler_seq + 1 - _ui_handlers[_ui_handler_seq] = handler - level, debug_domains = bb.msg.constructLogOptions() - _ui_logfilters[_ui_handler_seq] = UIEventFilter(level, debug_domains) - if mainui: - global _uiready - _uiready = _ui_handler_seq - return _ui_handler_seq + with _thread_lock: + bb.event._ui_handler_seq = bb.event._ui_handler_seq + 1 + _ui_handlers[_ui_handler_seq] = handler + level, debug_domains = bb.msg.constructLogOptions() + _ui_logfilters[_ui_handler_seq] = UIEventFilter(level, debug_domains) + if mainui: + global _uiready + _uiready = _ui_handler_seq + return _ui_handler_seq def unregister_UIHhandler(handlerNum, mainui=False): if mainui: global _uiready _uiready = False - if handlerNum in _ui_handlers: - del _ui_handlers[handlerNum] + with _thread_lock: + if handlerNum in _ui_handlers: + del _ui_handlers[handlerNum] return def get_uihandler(): diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py index 12dfb6ea19..51eb882092 100644 --- a/lib/bb/server/process.py +++ b/lib/bb/server/process.py @@ -150,7 +150,6 @@ class ProcessServer(): self.cooker.pre_serve() bb.utils.set_process_name("Cooker") - bb.event.enable_threadlock() ready = [] newconnections = [] diff --git a/lib/bb/tests/event.py b/lib/bb/tests/event.py index 4de4cced5e..d959f2d95d 100644 --- a/lib/bb/tests/event.py +++ b/lib/bb/tests/event.py @@ -451,10 +451,9 @@ class EventHandlingTest(unittest.TestCase): and disable threadlocks tests """ bb.event.fire(bb.event.OperationStarted(), None) - def test_enable_threadlock(self): + def test_event_threadlock(self): """ Test enable_threadlock method """ self._set_threadlock_test_mockups() - bb.event.enable_threadlock() self._set_and_run_threadlock_test_workers() # Calls to UI handlers should be in order as all the registered # handlers for the event coming from the first worker should be @@ -462,20 +461,6 @@ class EventHandlingTest(unittest.TestCase): self.assertEqual(self._threadlock_test_calls, ["w1_ui1", "w1_ui2", "w2_ui1", "w2_ui2"]) - - def test_disable_threadlock(self): - """ Test disable_threadlock method """ - self._set_threadlock_test_mockups() - bb.event.disable_threadlock() - self._set_and_run_threadlock_test_workers() - # Calls to UI handlers should be intertwined together. Thanks to the - # delay in the registered handlers for the event coming from the first - # worker, the event coming from the second worker starts being - # processed before finishing handling the first worker event. - self.assertEqual(self._threadlock_test_calls, - ["w1_ui1", "w2_ui1", "w1_ui2", "w2_ui2"]) - - class EventClassesTest(unittest.TestCase): """ Event classes test class """ From patchwork Wed Dec 21 23:35:06 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 17110 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 3F186C41535 for ; Wed, 21 Dec 2022 23:35:17 +0000 (UTC) Received: from mail-wm1-f48.google.com (mail-wm1-f48.google.com [209.85.128.48]) by mx.groups.io with SMTP id smtpd.web11.33950.1671665710834077800 for ; Wed, 21 Dec 2022 15:35:11 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=CpaeC2c+; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.48, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f48.google.com with SMTP id i82-20020a1c3b55000000b003d1e906ca23so13456wma.3 for ; Wed, 21 Dec 2022 15:35:10 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=z14wn+fdObnCb4A98XEBqYC6JuHymdEstH4/N/oFOos=; b=CpaeC2c+KQ7WJsvjSDAgqFCk1I0xWRlA0aH9hweuM9N4Nehr2gFH3D9sWDY6T2nb5x 0e5u99Nmh+QMuIjHHcRqDCIANvrTqyAKouhRTHy9S13l2G5Z8y3COt6lM96BGIr+21pP BxuevBI/iVbYTqR8grDWZbb5qBV91YRJnvnNM= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=z14wn+fdObnCb4A98XEBqYC6JuHymdEstH4/N/oFOos=; b=LBK5RUzUK+wTLcGGVJsPayipruzscrMfXn3KVzRaUBVaZOvRkS8jSwRfqtETz0wM9x jDsOI+8KTIWFucTlcDh8VgS4aZCWIGPTQEgt6G/rhu4tINJC25RkGGoP1ntnwYdhSgOU I1dfCv/Laj0cd/yqP/V8hmWboQTWzze0eT66TKfRQas31LVG+OBNQfmesoV/XyXs0c7M iFjApcOHnXeG2k4Au7KQHZImNE9dWQVxlVnVyN/YxRbVPW16SJYcKAkwQ39ILItzZ5Vo Pca4gC9Bm/UsdB7hP6qY/hD2R1MZXjYGQ/5nbmsKGRm5aWfkLtZwah++zrn345uKjyzc Yt5Q== X-Gm-Message-State: AFqh2kr+Hurct1W5W67xSAGANAkHe1E4SaNdJR5pJqO7wqEklosliN65 AwMirAyUOBJV5OWP9o6DevOFFzcPs1oUwm8H X-Google-Smtp-Source: AMrXdXu7faXsI2i/CA0FQn7HCZYD7KDYv7ddM7WHTCtC3lS2yhctL2C6TGVfrb2U26YIgPBcvFTN/w== X-Received: by 2002:a1c:4c12:0:b0:3c6:e63e:89a6 with SMTP id z18-20020a1c4c12000000b003c6e63e89a6mr2861752wmf.2.1671665709205; Wed, 21 Dec 2022 15:35:09 -0800 (PST) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:a8db:6d5d:f54f:c6b2]) by smtp.gmail.com with ESMTPSA id g15-20020a05600c310f00b003b47e75b401sm4063967wmo.37.2022.12.21.15.35.08 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 21 Dec 2022 15:35:08 -0800 (PST) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 2/3] server/process: Improve exception logging Date: Wed, 21 Dec 2022 23:35:06 +0000 Message-Id: <20221221233507.519249-2-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.37.2 In-Reply-To: <20221221233507.519249-1-richard.purdie@linuxfoundation.org> References: <20221221233507.519249-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Wed, 21 Dec 2022 23:35:17 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/14228 Currently if either idle functions loop suffers a traceback, it is silently dropped and there is no log message to say what happened. This change at least means the traceback is in the cooker log, making some debugging possible. Signed-off-by: Richard Purdie --- lib/bb/server/process.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py index 51eb882092..a3f2829560 100644 --- a/lib/bb/server/process.py +++ b/lib/bb/server/process.py @@ -395,6 +395,7 @@ class ProcessServer(): if not isinstance(exc, bb.BBHandledException): logger.exception('Running idle function') del self._idlefuns[function] + serverlog("Exception %s broke the idle_thread, exiting" % traceback.format_exc()) self.quit = True if nextsleep is not None: @@ -424,6 +425,7 @@ class ProcessServer(): except Exception as exc: if not isinstance(exc, bb.BBHandledException): logger.exception('Running heartbeat function') + serverlog("Exception %s broke in idle_commands, exiting" % traceback.format_exc()) self.quit = True if nextsleep and bb.event._heartbeat_enabled and now + nextsleep > self.next_heartbeat: # Shorten timeout so that we we wake up in time for From patchwork Wed Dec 21 23:35:07 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 17112 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 3D207C10F1B for ; Wed, 21 Dec 2022 23:35:17 +0000 (UTC) Received: from mail-wm1-f49.google.com (mail-wm1-f49.google.com [209.85.128.49]) by mx.groups.io with SMTP id smtpd.web11.33951.1671665711595801268 for ; Wed, 21 Dec 2022 15:35:11 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=VsYvfQ3x; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.49, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f49.google.com with SMTP id c65-20020a1c3544000000b003cfffd00fc0so2712810wma.1 for ; Wed, 21 Dec 2022 15:35:11 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:from:to:cc:subject:date:message-id :reply-to; bh=+0ONtpbjCTH0su7cgVw1Ip0Nhv9RztuUYIq5L/Jh9fE=; b=VsYvfQ3xCkH72bENeXXuVg8T4IUKZSOFOU/5o4rlBGfQV7rTm91BZvua/MZ94AxE01 JJH0eWtkykk5VVpIqSn7FAcqH5vShfUISyZWpAlspPRj1PkWbh2413cR4p2h7+rSCpKI PFjneKXJfTwxG5os5NJNCMFT9Jcp2kYXcwE1U= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=+0ONtpbjCTH0su7cgVw1Ip0Nhv9RztuUYIq5L/Jh9fE=; b=cxL7Nbd+Sku3huwvgihoYev6ySuBdsZ39dfCdkCFSLTXF11x88pmB1t4GG1itag3Br jurF33upubazV6wVO8p6sZTj8BDVWLN2Noc8eL7lOoKGjhF6erIEZ5wi4f5VvMXmauCV uq1cZI6NN6m/ohF+DrNjr1XZ+7u9zOANR8nnG7PyJQpGhrv59L3Thhd7SL6oy4TsMzg9 UM0CWl7GDfsRribW7Ys+f+jaugdSr3Y1fI5e7mq2ZsP1FOnvZZ+EON6QOsJi9IQUdakK y9kyCKhqS0rBXNXda6eADBc2CRDxIMIhwk2M41y6BmGWwhww25zwjKQcAuwcHAz384Hp 2ppg== X-Gm-Message-State: AFqh2kqMVVitM7z4Y/wXxs9hfXHtwJeQrJh8sllICF3N2SW/3vzcQQn1 CNXRlmww46NHgkjQb+QjEYL3kDFcyF7nGUUe X-Google-Smtp-Source: AMrXdXuMbmNtr9WJ071MfYoLz5J6l+B9BetfeOXFh0NHCQHOXzYprvknrxR3huOE/rT86zDX32yPtg== X-Received: by 2002:a7b:c408:0:b0:3d5:64bf:ccc2 with SMTP id k8-20020a7bc408000000b003d564bfccc2mr2738000wmi.32.1671665709831; Wed, 21 Dec 2022 15:35:09 -0800 (PST) Received: from max.int.rpsys.net ([2001:8b0:aba:5f3c:a8db:6d5d:f54f:c6b2]) by smtp.gmail.com with ESMTPSA id g15-20020a05600c310f00b003b47e75b401sm4063967wmo.37.2022.12.21.15.35.09 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 21 Dec 2022 15:35:09 -0800 (PST) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 3/3] cooker/cookerdata: Rework the way the datastores are reset Date: Wed, 21 Dec 2022 23:35:07 +0000 Message-Id: <20221221233507.519249-3-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.37.2 In-Reply-To: <20221221233507.519249-1-richard.purdie@linuxfoundation.org> References: <20221221233507.519249-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Wed, 21 Dec 2022 23:35:17 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/14229 As far as I could tell, the current code could result in some strange situations where some data was set back to the original data store copy but the multiconfig data was not restored. There are also some changes made to the datastore which did not persist. The data store was also being reset at every client reset, which seems a little excessive if we can reset it to the original condition properly. Move the __depends -> __base_depends rename into databuilder along with the __bbclasstype change so these are saved in the original data. Tweak the databuilder code to be clearer and easier to follow on which copies are where, then save copies of all the mc datastores. Finally, drop the cache invalidation upon reset for the base config as we shouldn't need that now (oe-selftest -r tinfoil works with memory resident bitbake which was the original reproducer requiring that change). Signed-off-by: Richard Purdie --- lib/bb/cooker.py | 9 +++------ lib/bb/cookerdata.py | 31 +++++++++++++++++++------------ 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index 1daa587ac5..43e0842284 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -409,9 +409,7 @@ class BBCooker: self.disableDataTracking() for mc in self.databuilder.mcdata.values(): - mc.renameVar("__depends", "__base_depends") self.add_filewatch(mc.getVar("__base_depends", False), self.configwatcher) - mc.setVar("__bbclasstype", "recipe") self.baseconfig_valid = True self.parsecache_valid = False @@ -445,10 +443,8 @@ class BBCooker: upstream=upstream, ) self.hashserv.serve_as_process() - self.data.setVar("BB_HASHSERVE", self.hashservaddr) - self.databuilder.origdata.setVar("BB_HASHSERVE", self.hashservaddr) - self.databuilder.data.setVar("BB_HASHSERVE", self.hashservaddr) for mc in self.databuilder.mcdata: + self.databuilder.mcorigdata[mc].setVar("BB_HASHSERVE", self.hashservaddr) self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.hashservaddr) bb.parse.init_parser(self.data) @@ -1797,8 +1793,9 @@ class BBCooker: if hasattr(self, "data"): self.databuilder.reset() self.data = self.databuilder.data + # In theory tinfoil could have modified the base data before parsing, + # ideally need to track if anything did modify the datastore self.parsecache_valid = False - self.baseconfig_valid = False class CookerExit(bb.event.Event): diff --git a/lib/bb/cookerdata.py b/lib/bb/cookerdata.py index 650ae05ec9..b4e0c4216b 100644 --- a/lib/bb/cookerdata.py +++ b/lib/bb/cookerdata.py @@ -263,6 +263,7 @@ class CookerDataBuilder(object): self.mcdata = {} def parseBaseConfiguration(self, worker=False): + mcdata = {} data_hash = hashlib.sha256() try: self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles) @@ -288,18 +289,18 @@ class CookerDataBuilder(object): bb.parse.init_parser(self.data) data_hash.update(self.data.get_hash().encode('utf-8')) - self.mcdata[''] = self.data + mcdata[''] = self.data multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split() for config in multiconfig: if config[0].isdigit(): bb.fatal("Multiconfig name '%s' is invalid as multiconfigs cannot start with a digit" % config) - mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config) - bb.event.fire(bb.event.ConfigParsed(), mcdata) - self.mcdata[config] = mcdata - data_hash.update(mcdata.get_hash().encode('utf-8')) + parsed_mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config) + bb.event.fire(bb.event.ConfigParsed(), parsed_mcdata) + mcdata[config] = parsed_mcdata + data_hash.update(parsed_mcdata.get_hash().encode('utf-8')) if multiconfig: - bb.event.fire(bb.event.MultiConfigParsed(self.mcdata), self.data) + bb.event.fire(bb.event.MultiConfigParsed(mcdata), self.data) self.data_hash = data_hash.hexdigest() except (SyntaxError, bb.BBHandledException): @@ -332,17 +333,23 @@ class CookerDataBuilder(object): if issues: raise bb.BBHandledException() + for mc in mcdata: + mcdata[mc].renameVar("__depends", "__base_depends") + mcdata[mc].setVar("__bbclasstype", "recipe") + # Create a copy so we can reset at a later date when UIs disconnect - self.origdata = self.data - self.data = bb.data.createCopy(self.origdata) - self.mcdata[''] = self.data + self.mcorigdata = mcdata + for mc in mcdata: + self.mcdata[mc] = bb.data.createCopy(mcdata[mc]) + self.data = self.mcdata[''] def reset(self): # We may not have run parseBaseConfiguration() yet - if not hasattr(self, 'origdata'): + if not hasattr(self, 'mcorigdata'): return - self.data = bb.data.createCopy(self.origdata) - self.mcdata[''] = self.data + for mc in self.mcorigdata: + self.mcdata[mc] = bb.data.createCopy(self.mcorigdata[mc]) + self.data = self.mcdata[''] def _findLayerConf(self, data): return findConfigFile("bblayers.conf", data)