From patchwork Tue Mar 29 14:27:50 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 6013 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 4D64DC4332F for ; Tue, 29 Mar 2022 14:28:02 +0000 (UTC) Received: from mail-wr1-f42.google.com (mail-wr1-f42.google.com [209.85.221.42]) by mx.groups.io with SMTP id smtpd.web12.7341.1648564081034558569 for ; Tue, 29 Mar 2022 07:28:01 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=hKv7j19b; spf=pass (domain: linuxfoundation.org, ip: 209.85.221.42, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wr1-f42.google.com with SMTP id h4so25070156wrc.13 for ; Tue, 29 Mar 2022 07:28:00 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=from:to:subject:date:message-id:mime-version :content-transfer-encoding; bh=B7qKs0RXdT8tcnKNLNvHNimytnJmiwAooEs5nIYh7Dc=; b=hKv7j19bDqV1c7o6MGoPABEVvW/n3ljbuw5yZW7bL5NRIYIAAzKr3W8EQygnlFj71t 1meyIJXY23OkxLApAHq/EQBvb+oC/bZ4N565AFK/vgCxZSqY2Ukm9+vB1mTPLDLrxVpZ PJMGT6MWPMRbF4n9bLDFZkNp2tajN3SsWWGUA= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:subject:date:message-id:mime-version :content-transfer-encoding; bh=B7qKs0RXdT8tcnKNLNvHNimytnJmiwAooEs5nIYh7Dc=; b=5RLPcqhpXF9nhWT56xPi2ccUzIqHpgLeS2tI6fFIIlWkaq4Z8vQLUvyIMyIIEAG4/u wZxox9FjVb8jv8aretCj6eH7FAJ8aOj+wXXx9RNJCVMiIQZS4bm+bpVmaIFwGoS4KJlO nCE5IOFVU7E+umse5dMbdRP5UIw8occxyOWQDThrIgsAbPB82Axtn6g6TzRWZjC7Okba Vv8izNkXeWhaYgbDK6k3ICLPqydvXakPSx5iGGmtv2lWr3nSb9/8mYwkCn01+UoKqM3q TqIN4EPV4xEz3vpaQsvDQi12rWI1v36ET2Ev0/SdPqS90PJdNbYmS1mM2tVPmnrleHvp lKvg== X-Gm-Message-State: AOAM533wu9+SNZkD9Ohs2USw47HPzBHLA3mviwPzQw0dpbmW/tCh21ZI eQuFlJ91CTDySkb14PlyjwrvLxRgPrU/G5PP X-Google-Smtp-Source: ABdhPJxbpakXZVV7+SvIFoif0YrVTxAS9K4nUWsKL8Y8T4C/LPUWoquJJYKnlJWQC2cgzEE33Nyiaw== X-Received: by 2002:adf:dfc2:0:b0:1f0:262a:d831 with SMTP id q2-20020adfdfc2000000b001f0262ad831mr31152605wrn.589.1648564077012; Tue, 29 Mar 2022 07:27:57 -0700 (PDT) Received: from hex.int.rpsys.net ([2001:8b0:aba:5f3c:a144:3266:4a07:b254]) by smtp.gmail.com with ESMTPSA id bg42-20020a05600c3caa00b00380deeaae72sm2529043wmb.1.2022.03.29.07.27.56 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 29 Mar 2022 07:27:56 -0700 (PDT) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 1/6] cooker: Fix exception handling in parsers Date: Tue, 29 Mar 2022 15:27:50 +0100 Message-Id: <20220329142755.1473185-1-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.32.0 MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 29 Mar 2022 14:28:02 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/13542 We shouldn't be generating exception inside a generator. Rearrange the code to improve the handling of this. Also fix the misconverted code from when multiconfig was added and pass the exception as "result". Signed-off-by: Richard Purdie --- lib/bb/cooker.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index eac956aa97..c4d720a6b6 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -2087,12 +2087,12 @@ class Parser(multiprocessing.Process): tb = sys.exc_info()[2] exc.recipe = filename exc.traceback = list(bb.exceptions.extract_traceback(tb, context=3)) - return True, exc + return True, None, exc # Need to turn BaseExceptions into Exceptions here so we gracefully shutdown # and for example a worker thread doesn't just exit on its own in response to # a SystemExit event for example. except BaseException as exc: - return True, ParsingFailure(exc, filename) + return True, None, ParsingFailure(exc, filename) finally: bb.event.LogHandler.filter = origfilter @@ -2252,11 +2252,7 @@ class CookerParser(object): pass else: empty = False - value = result[1] - if isinstance(value, BaseException): - raise value - else: - yield result + yield result if not (self.parsed >= self.toparse): raise bb.parse.ParseError("Not all recipes parsed, parser thread killed/died? Exiting.", None) @@ -2267,6 +2263,9 @@ class CookerParser(object): parsed = None try: parsed, mc, result = next(self.results) + if isinstance(result, BaseException): + # Turn exceptions back into exceptions + raise result except StopIteration: self.shutdown() return False From patchwork Tue Mar 29 14:27:51 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 6012 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 5B0D2C433EF for ; Tue, 29 Mar 2022 14:28:01 +0000 (UTC) Received: from mail-wm1-f52.google.com (mail-wm1-f52.google.com [209.85.128.52]) by mx.groups.io with SMTP id smtpd.web08.7268.1648564079918028520 for ; Tue, 29 Mar 2022 07:28:00 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=NpuR0kQG; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.52, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f52.google.com with SMTP id 125-20020a1c0283000000b0038d043aac51so1365803wmc.0 for ; Tue, 29 Mar 2022 07:27:59 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=from:to:subject:date:message-id:in-reply-to:references:mime-version :content-transfer-encoding; bh=sYndZs6A1eX88S/gUxvFN/QfdvLC6ew57JSoQWAy1vE=; b=NpuR0kQGOKGAUy8NX7A4jn7eMflwtWgaVpvofNflUunbeHyaB9ZmvQGDQVXp4PFn0U owHOOslG5+OysmtZbLicyqB/2w1KZL8waEGVg1wwOynpLF2PqcdWFsISfnfpjyEK+qjn 61NrsMoVoKYFpgDLzd3H2oJvlonX8/sHMR3kA= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=sYndZs6A1eX88S/gUxvFN/QfdvLC6ew57JSoQWAy1vE=; b=xdBoFfin+sGbIsUV+Hxr1VPJbIUiJjGAG4Pb7KjLRWRm1T6O4EaJl/omQTMoDRDOuD Mo1Ogd3/6HRFqemdGQkhkSMaxOEapfY6GgirPk+Rb/sjLaSpytXKf5QqFwZCSKjSqS14 e+SFH1wLA5t923neYONmoewbSNtupZEOkGnsSe6kW88z77AJnIYEpfjIjW8ijZXxbiam EYqxv69GXVCsEbWIWanq/DGspou/F08X4zaHApB0XGim+c8DR8L+mH+99dcEa2vPcKF7 HsjK8u4qiCg3kPHCy4iSI5CUl5lT8MKya6bjBwq0JIWPcABOeNqUZqzqONIHzQvBgYM6 DyIw== X-Gm-Message-State: AOAM533Qf5SG3/OjdWaPZqVjjfGerQnbTjTehijt2EvV3KCjoHGvArvz SqSx3KkPgv4Wi6/LjOwnELcK2a/gX2V28K2c X-Google-Smtp-Source: ABdhPJzcgHCR5xvQjZGLtX6DON+t+EXI/edihyVBxvNeGGqVu5o6TtZRGo8c4WGt4azbbDXXgectsg== X-Received: by 2002:a05:600c:3b24:b0:38c:dbdb:a35c with SMTP id m36-20020a05600c3b2400b0038cdbdba35cmr7316445wms.175.1648564077824; Tue, 29 Mar 2022 07:27:57 -0700 (PDT) Received: from hex.int.rpsys.net ([2001:8b0:aba:5f3c:a144:3266:4a07:b254]) by smtp.gmail.com with ESMTPSA id bg42-20020a05600c3caa00b00380deeaae72sm2529043wmb.1.2022.03.29.07.27.57 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 29 Mar 2022 07:27:57 -0700 (PDT) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 2/6] cooker: Fix main loop starvation when parsing Date: Tue, 29 Mar 2022 15:27:51 +0100 Message-Id: <20220329142755.1473185-2-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.32.0 In-Reply-To: <20220329142755.1473185-1-richard.purdie@linuxfoundation.org> References: <20220329142755.1473185-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 29 Mar 2022 14:28:01 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/13540 When parsing, the parser isn't servicing the main loop so a Ctrl+C in the UI won't be seen on the cooker/server side. Fix this by returning when queue timeouts occur. This helps where there is a hung or slow parsing thread. Signed-off-by: Richard Purdie --- lib/bb/cooker.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index c4d720a6b6..fb71a968f2 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -2249,7 +2249,7 @@ class CookerParser(object): result = self.result_queue.get(timeout=0.25) except queue.Empty: empty = True - pass + yield None, None, None else: empty = False yield result @@ -2266,6 +2266,10 @@ class CookerParser(object): if isinstance(result, BaseException): # Turn exceptions back into exceptions raise result + if parsed is None: + # Timeout, loop back through the main loop + return True + except StopIteration: self.shutdown() return False From patchwork Tue Mar 29 14:27:52 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 6014 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 4B790C433F5 for ; Tue, 29 Mar 2022 14:28:02 +0000 (UTC) Received: from mail-wr1-f53.google.com (mail-wr1-f53.google.com [209.85.221.53]) by mx.groups.io with SMTP id smtpd.web10.7458.1648564080772424932 for ; Tue, 29 Mar 2022 07:28:01 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=h8CyEOUW; spf=pass (domain: linuxfoundation.org, ip: 209.85.221.53, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wr1-f53.google.com with SMTP id w21so20533795wra.2 for ; Tue, 29 Mar 2022 07:28:00 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=from:to:subject:date:message-id:in-reply-to:references:mime-version :content-transfer-encoding; bh=JVbUVfXAYv56QT8+yrD8AwyJtTP1eOGwcft83b87yRo=; b=h8CyEOUWSdc9VMgXCDEfRnyd3qyUw5z2bAPS7rIGPz8RFJ7PejxpAdIKimWleFUUmW /0S4r7dI8njlub6oofn50Q2JgxVd7+b2VtVr6a0WlJeri0NGUdQ1lnvqPNJyDiQdi+nd WwpYzOve1BJxu9LM1t5DOOoyb07EPwgMzIi88= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=JVbUVfXAYv56QT8+yrD8AwyJtTP1eOGwcft83b87yRo=; b=RxNIeJLlN2uhAqH3d9en8n0jRksptZUIJeB45J2V8RdVL0Wuo6qOrgK+My9rhgRh7r wHm48qwlbI6HsfftK8v3FEb6UV3ibKGtUX2uq1lrmxkE1GStfPY0/0UcsP0fN5Ss7pp1 /IbPV5kGVNyyzi/PlmY8wUKjJSDrYyomOdNmzftQj50vxc0ik/nVsXYcBI8VYmCB+xip Xo3AuowYYym/mqmeCav0oc6JbIYf9cj0ue8JgaJ6RYFUl4Tzj+JLxqH5d2k8tlxCg1XV pIYVjETdZWO6BZytai5OFmvkoJZjkD09n90tar3diLGEejVq5IhgrT1fmpC604J4RFGW XXHA== X-Gm-Message-State: AOAM533ZsTKVaBvk3y0BeckBJgv9LANJJZUBvaTERL79gFiLhgjFv4nk lB65cVHdB7XypNw3NLG/EclcNRJrlNzbqAL3 X-Google-Smtp-Source: ABdhPJz4+kl14KPD8xMVPT0881PxiXsBRNevyhJ9KtIKmeeunZAPJefK5QDbnf4tfsqOrzNwNkZQ6A== X-Received: by 2002:a5d:62c9:0:b0:205:c0ef:7cb4 with SMTP id o9-20020a5d62c9000000b00205c0ef7cb4mr15231349wrv.688.1648564079079; Tue, 29 Mar 2022 07:27:59 -0700 (PDT) Received: from hex.int.rpsys.net ([2001:8b0:aba:5f3c:a144:3266:4a07:b254]) by smtp.gmail.com with ESMTPSA id bg42-20020a05600c3caa00b00380deeaae72sm2529043wmb.1.2022.03.29.07.27.57 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 29 Mar 2022 07:27:58 -0700 (PDT) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 3/6] cooker: Improve exception handling in parsing process Date: Tue, 29 Mar 2022 15:27:52 +0100 Message-Id: <20220329142755.1473185-3-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.32.0 In-Reply-To: <20220329142755.1473185-1-richard.purdie@linuxfoundation.org> References: <20220329142755.1473185-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 29 Mar 2022 14:28:02 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/13541 If an exception occurs in the parsing process, ensure the cleanup is called via all codepaths using a try/finally. Signed-off-by: Richard Purdie --- lib/bb/cooker.py | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index fb71a968f2..2264b18c54 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -2041,32 +2041,32 @@ class Parser(multiprocessing.Process): self.init() pending = [] - while True: - try: - self.quit.get_nowait() - except queue.Empty: - pass - else: - self.results.close() - self.results.join_thread() - break - - if pending: - result = pending.pop() - else: + try: + while True: try: - job = self.jobs.pop() - except IndexError: - self.results.close() - self.results.join_thread() + self.quit.get_nowait() + except queue.Empty: + pass + else: break - result = self.parse(*job) - # Clear the siggen cache after parsing to control memory usage, its huge - bb.parse.siggen.postparsing_clean_cache() - try: - self.results.put(result, timeout=0.25) - except queue.Full: - pending.append(result) + + if pending: + result = pending.pop() + else: + try: + job = self.jobs.pop() + except IndexError: + break + result = self.parse(*job) + # Clear the siggen cache after parsing to control memory usage, its huge + bb.parse.siggen.postparsing_clean_cache() + try: + self.results.put(result, timeout=0.25) + except queue.Full: + pending.append(result) + finally: + self.results.close() + self.results.join_thread() def parse(self, mc, cache, filename, appends): try: From patchwork Tue Mar 29 14:27:53 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 6015 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 4B556C433FE for ; Tue, 29 Mar 2022 14:28:03 +0000 (UTC) Received: from mail-wr1-f48.google.com (mail-wr1-f48.google.com [209.85.221.48]) by mx.groups.io with SMTP id smtpd.web08.7270.1648564082528197667 for ; Tue, 29 Mar 2022 07:28:02 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=ag3+smXu; spf=pass (domain: linuxfoundation.org, ip: 209.85.221.48, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wr1-f48.google.com with SMTP id r7so23973509wrc.0 for ; Tue, 29 Mar 2022 07:28:02 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=from:to:subject:date:message-id:in-reply-to:references:mime-version :content-transfer-encoding; bh=W9XSP6IqEHkSqSFcBt5RQaoQhdrVyChCox5y3wFHTOU=; b=ag3+smXuspf/QUiHErq1WqY01ik2NU+GRGWT6QBWkmAvGCPi8jFa98x80h7IzpFRBu hrws03+prbubvWsMp2BIoeLR/bQcTPuI2F13ZPC4LPGVAyTAdP3BkBSvBdyAsKNOemM3 y7b8fGTo/NR+KznmbgN6GtAxhNEZYZaYYV3Ow= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=W9XSP6IqEHkSqSFcBt5RQaoQhdrVyChCox5y3wFHTOU=; b=RrYHr38jBzusIa+ELwV24Z3zSHtFj5YaWnnyxgD68U4c0UGGiga4FQSUa8xtavs6Me WmEMfJYhEe/cJchFDM5YQLON0SG4iIHwU/UwgAJXalHbVNhAU+az/w5w+xUvBlvcZ+As 4aErhcweBsAi5UeLNQiczj1OLbxpKDD4wmY5UtTieoM9DCCg7LBT5S1jehWxXwKa9QCc xEZ9ozYHnFGAfcNSnn5mV+JlSFO2TI5W7TnsRpsGQt6wK5YfP0Zh6UwNOtGvQYHPi7JP J9MWBAKcQ/HTja26B/V09dNxwvpgHX2W7OdUtXUIBNgHRrQNfoVJJm7czedIb6Zc8NEB fZ3A== X-Gm-Message-State: AOAM531Z7ZYTzc/VGWc6zwwun0FQLJ2kAZqC6G+za/OxKmL2aXhIpp9P V4krhtIkewLGBBqXFkbWqvkn+9GaywuvE0t/ X-Google-Smtp-Source: ABdhPJwfvZqyJ4gMZRWb/6cqEiDp3O3oMHLYmlbWqe5dPWpg17hZ1xwJ8RhJz3JsjaVsKymV95T1Qg== X-Received: by 2002:adf:eb48:0:b0:203:f854:86cc with SMTP id u8-20020adfeb48000000b00203f85486ccmr30904378wrn.102.1648564080821; Tue, 29 Mar 2022 07:28:00 -0700 (PDT) Received: from hex.int.rpsys.net ([2001:8b0:aba:5f3c:a144:3266:4a07:b254]) by smtp.gmail.com with ESMTPSA id bg42-20020a05600c3caa00b00380deeaae72sm2529043wmb.1.2022.03.29.07.27.59 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 29 Mar 2022 07:27:59 -0700 (PDT) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 4/6] cooker: Simplify parser init function handling Date: Tue, 29 Mar 2022 15:27:53 +0100 Message-Id: <20220329142755.1473185-4-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.32.0 In-Reply-To: <20220329142755.1473185-1-richard.purdie@linuxfoundation.org> References: <20220329142755.1473185-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 29 Mar 2022 14:28:03 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/13543 Not sure why this is so convoluted but we should simplify it! Signed-off-by: Richard Purdie --- lib/bb/cooker.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index 2264b18c54..d6fcd9e05c 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -2009,11 +2009,10 @@ class ParsingFailure(Exception): Exception.__init__(self, realexception, recipe) class Parser(multiprocessing.Process): - def __init__(self, jobs, results, quit, init, profile): + def __init__(self, jobs, results, quit, profile): self.jobs = jobs self.results = results self.quit = quit - self.init = init multiprocessing.Process.__init__(self) self.context = bb.utils.get_context().copy() self.handlers = bb.event.get_class_handlers().copy() @@ -2037,8 +2036,12 @@ class Parser(multiprocessing.Process): prof.dump_stats(logfile) def realrun(self): - if self.init: - self.init() + signal.signal(signal.SIGTERM, signal.SIG_DFL) + signal.signal(signal.SIGHUP, signal.SIG_DFL) + signal.signal(signal.SIGINT, signal.SIG_IGN) + bb.utils.set_process_name(multiprocessing.current_process().name) + multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1) + multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1) pending = [] try: @@ -2143,13 +2146,6 @@ class CookerParser(object): self.processes = [] if self.toparse: bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata) - def init(): - signal.signal(signal.SIGTERM, signal.SIG_DFL) - signal.signal(signal.SIGHUP, signal.SIG_DFL) - signal.signal(signal.SIGINT, signal.SIG_IGN) - bb.utils.set_process_name(multiprocessing.current_process().name) - multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1) - multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1) self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes) self.result_queue = multiprocessing.Queue() @@ -2159,7 +2155,7 @@ class CookerParser(object): self.jobs = chunkify(list(self.willparse), self.num_processes) for i in range(0, self.num_processes): - parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) + parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, self.cooker.configuration.profile) parser.start() self.process_names.append(parser.name) self.processes.append(parser) From patchwork Tue Mar 29 14:27:54 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 6016 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 49499C433EF for ; Tue, 29 Mar 2022 14:28:05 +0000 (UTC) Received: from mail-wm1-f54.google.com (mail-wm1-f54.google.com [209.85.128.54]) by mx.groups.io with SMTP id smtpd.web09.7297.1648564083975695724 for ; Tue, 29 Mar 2022 07:28:04 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=TfSJzzdD; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.54, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f54.google.com with SMTP id r7so10430763wmq.2 for ; Tue, 29 Mar 2022 07:28:03 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=from:to:subject:date:message-id:in-reply-to:references:mime-version :content-transfer-encoding; bh=6Qxp89SqDLZp4dijR+R8YJGNs9BkPF09FXA/ws0Ig/c=; b=TfSJzzdDGQ2sC0m88ntpoiLc/BW9UtOqjIqMaqAnzevgFMJzOMsAq8hBs4qXKCv7Cv SCmCqdJRNWaJL8zS3h3NEpE0LL5DAbrn9Ukhy2qBdoIn4wuqwlVrXmXOPV4JTz026IxP DScRCSbB386jQJMg9a8M4zXzJBUPrRDk/hblY= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=6Qxp89SqDLZp4dijR+R8YJGNs9BkPF09FXA/ws0Ig/c=; b=QfEJSeXAPFDjP6dJg90AqPaEQ4nxwjkSE/pqinlNbzkKftsAmjYTLyYcQEJ81hOwA0 meDDnIhTX52tDDW/fg4L8UpLrRcKql7BfqxlYU+fiN7jTece1hFXmlK/No5FXhVe+bOH H134anUotAOk8CQyypH6P80Ze9yO4t47DMjfiTHFY+vhCGblZCegtsuuHsN68DBbkbmq lYybpmV2JKXsm6iv7eI2Ee4Daft82Ld+scAztcRr4Tg2hbMhbALtIjOakYJXfnkB8Bc0 Wfm2ikhVDr+5A6PHWLee5BufbtsyOgMez/0qd6wTvD0BkNh2pgP899S9q2/1FPI9CZB4 d85g== X-Gm-Message-State: AOAM5318SCFmFG4NngiLix3/fglP0cY5kbw7gA0UQXf4fIvHW38jvSH1 lNHs9EgqAsojF7I77U18yArEyJ0L8HsHECR7 X-Google-Smtp-Source: ABdhPJwRz5+MA0/tbDffn1jE1iprEzcl0oRjfMPdYitdPED62Tzwc7Jwc1Bw8XnoW48p0PjGzeW9hw== X-Received: by 2002:a05:600c:4f15:b0:38c:b729:4838 with SMTP id l21-20020a05600c4f1500b0038cb7294838mr7302621wmq.132.1648564082105; Tue, 29 Mar 2022 07:28:02 -0700 (PDT) Received: from hex.int.rpsys.net ([2001:8b0:aba:5f3c:a144:3266:4a07:b254]) by smtp.gmail.com with ESMTPSA id bg42-20020a05600c3caa00b00380deeaae72sm2529043wmb.1.2022.03.29.07.28.00 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 29 Mar 2022 07:28:01 -0700 (PDT) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 5/6] cooker/process: Fix signal handling lockups Date: Tue, 29 Mar 2022 15:27:54 +0100 Message-Id: <20220329142755.1473185-5-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.32.0 In-Reply-To: <20220329142755.1473185-1-richard.purdie@linuxfoundation.org> References: <20220329142755.1473185-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 29 Mar 2022 14:28:05 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/13544 If a parser process is terminated while holding a write lock, then it will lead to a deadlock (see https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Process.terminate). With SIGTERM, we don't want to terminate holding the lock. We also don't want a SIGINT to cause a partial write to the event stream. I tried using signal masks to avoid this but it doesn't work, see https://bugs.python.org/issue47139 Instead, add a signal handler and catch the calls around the critical section. We also need a thread lock to ensure other threads in the same process don't handle the signal until all the threads are not in the lock. Signed-off-by: Richard Purdie --- lib/bb/cooker.py | 28 ++++++++++++++++++++++++++-- lib/bb/server/process.py | 22 ++++++++++++++++++++-- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index d6fcd9e05c..7c0c5d4efa 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -2017,6 +2017,22 @@ class Parser(multiprocessing.Process): self.context = bb.utils.get_context().copy() self.handlers = bb.event.get_class_handlers().copy() self.profile = profile + self.queue_signals = False + self.signal_received = [] + self.signal_threadlock = threading.Lock() + + def catch_sig(self, signum, frame): + if self.queue_signals: + self.signal_received.append(signum) + else: + self.handle_sig(signum, frame) + + def handle_sig(self, signum, frame): + if signum == signal.SIGTERM: + signal.signal(signal.SIGTERM, signal.SIG_DFL) + os.kill(os.getpid(), signal.SIGTERM) + elif signum == signal.SIGINT: + signal.default_int_handler(signum, frame) def run(self): @@ -2036,9 +2052,17 @@ class Parser(multiprocessing.Process): prof.dump_stats(logfile) def realrun(self): - signal.signal(signal.SIGTERM, signal.SIG_DFL) + # Signal handling here is hard. We must not terminate any process or thread holding the write + # lock for the event stream as it will not be released, ever, and things will hang. + # Python handles signals in the main thread/process but they can be raised from any thread and + # we want to defer processing of any SIGTERM/SIGINT signal until we're outside the critical section + # and don't hold the lock (see server/process.py). We therefore always catch the signals (so any + # new thread should also do so) and we defer handling but we handle with the local thread lock + # held (a threading lock, not a multiprocessing one) so that no other thread in the process + # can be in the critical section. + signal.signal(signal.SIGTERM, self.catch_sig) signal.signal(signal.SIGHUP, signal.SIG_DFL) - signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGINT, self.catch_sig) bb.utils.set_process_name(multiprocessing.current_process().name) multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1) multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1) diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py index 7c587a9110..ce53fdc678 100644 --- a/lib/bb/server/process.py +++ b/lib/bb/server/process.py @@ -20,6 +20,7 @@ import os import sys import time import select +import signal import socket import subprocess import errno @@ -737,11 +738,28 @@ class ConnectionWriter(object): # Why bb.event needs this I have no idea self.event = self - def send(self, obj): - obj = multiprocessing.reduction.ForkingPickler.dumps(obj) + def _send(self, obj): with self.wlock: self.writer.send_bytes(obj) + def send(self, obj): + obj = multiprocessing.reduction.ForkingPickler.dumps(obj) + # See notes/code in CookerParser + # We must not terminate holding this lock else processes will hang. + # For SIGTERM, raising afterwards avoids this. + # For SIGINT, we don't want to have written partial data to the pipe. + # pthread_sigmask block/unblock would be nice but doesn't work, https://bugs.python.org/issue47139 + process = multiprocessing.current_process() + if process and hasattr(process, "queue_signals"): + with process.signal_threadlock: + process.queue_signals = True + self._send(obj) + process.queue_signals = False + for sig in process.signal_received.pop(): + process.handle_sig(sig, None) + else: + self._send(obj) + def fileno(self): return self.writer.fileno() From patchwork Tue Mar 29 14:27:55 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Purdie X-Patchwork-Id: 6017 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 4F04FC433F5 for ; Tue, 29 Mar 2022 14:28:06 +0000 (UTC) Received: from mail-wm1-f42.google.com (mail-wm1-f42.google.com [209.85.128.42]) by mx.groups.io with SMTP id smtpd.web11.7261.1648564084815790718 for ; Tue, 29 Mar 2022 07:28:05 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linuxfoundation.org header.s=google header.b=O5KdfOqe; spf=pass (domain: linuxfoundation.org, ip: 209.85.128.42, mailfrom: richard.purdie@linuxfoundation.org) Received: by mail-wm1-f42.google.com with SMTP id 125-20020a1c0283000000b0038d043aac51so1365945wmc.0 for ; Tue, 29 Mar 2022 07:28:04 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxfoundation.org; s=google; h=from:to:subject:date:message-id:in-reply-to:references:mime-version :content-transfer-encoding; bh=P67altbjq8eoAsQk9mhKF+xIRrDWZMZHmcEdGF6jcTU=; b=O5KdfOqeVJrK02fKsFE14/Ukg32L/KuqRT9yzpUH9Dk8e91p2BpY92Tg6h1Pgd+dPt PGK7X8c8D49YYNXQuRikll9Cf39OfHI/sFJ57QRhuT+s8Hc0ZJS3n9FOzcsxwoxMYGdH udUXZVeUMAv73dShvCk6KMXaDVQq4JWiAYOVo= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=x-gm-message-state:from:to:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=P67altbjq8eoAsQk9mhKF+xIRrDWZMZHmcEdGF6jcTU=; b=zml+kmX/AmtQWKnmjDuDJL1wnT8r9QwuIcKwGcgIEKivUd+Cv3ZR14IDhwStIBsGzI Qjt3pyxW9MZj/7DkwG/onB/FlaEaYA2rYZzRL7EcsXWM9z3F1hfDgpGovWTBwqRduKOz VRlWPMwKvv+khXlSzWJD1ndQuTEms65R4GfDRRn0cPueKiMxMZK2Gjw6E5QoWg1TzQbX h2+9VLGTMVT5S3JSZt/2qQ0sxbj0WWgS7H5fQB6y6XA02+tgO0HRZ1uvAM5zEU3Klt69 v7MbHEwczD/g/Ih1L4DjhrErVmwjGjuiMcUsobZ+RAlkAXEMtYwEo0wOWm7f71K0RNj7 pQuQ== X-Gm-Message-State: AOAM530kzFs+8UNXqfaf858msP/rNl249Tqj5L1cIsAGbBzwXcKsRGEe jAhaCeZY9r4YTIoTZg1uw4EmJBCRc1IMYu/m X-Google-Smtp-Source: ABdhPJzWjdmQ8zlwUKzz65eCtVOukcKcbcAa4Pcqp+mM4HnheXefA85sfGPku8CQ5J4O9k++dZuVEA== X-Received: by 2002:a1c:7c03:0:b0:38c:804d:d477 with SMTP id x3-20020a1c7c03000000b0038c804dd477mr7323185wmc.32.1648564082964; Tue, 29 Mar 2022 07:28:02 -0700 (PDT) Received: from hex.int.rpsys.net ([2001:8b0:aba:5f3c:a144:3266:4a07:b254]) by smtp.gmail.com with ESMTPSA id bg42-20020a05600c3caa00b00380deeaae72sm2529043wmb.1.2022.03.29.07.28.02 for (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 29 Mar 2022 07:28:02 -0700 (PDT) From: Richard Purdie To: bitbake-devel@lists.openembedded.org Subject: [PATCH 6/6] cooker: Rework force parser shutdown Date: Tue, 29 Mar 2022 15:27:55 +0100 Message-Id: <20220329142755.1473185-6-richard.purdie@linuxfoundation.org> X-Mailer: git-send-email 2.32.0 In-Reply-To: <20220329142755.1473185-1-richard.purdie@linuxfoundation.org> References: <20220329142755.1473185-1-richard.purdie@linuxfoundation.org> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 29 Mar 2022 14:28:06 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/13545 The "force" option to parser shutdown was often the cause of lockups and there is no good reason we should have two different behaviours. Change and unify the codepaths to always: * Wait for longer for a controlled shutdown of a process (0.5s). Usually it will be much faster if it has finished so the delay doesn't really matter. * Send processes a SIGINT * Failing that, send a SIGTERM * Call .close() if available to release zombies This means we no longer need the "force" parameter to the function so it is removed. Signed-off-by: Richard Purdie --- lib/bb/cooker.py | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py index 7c0c5d4efa..f435b18c87 100644 --- a/lib/bb/cooker.py +++ b/lib/bb/cooker.py @@ -1613,7 +1613,7 @@ class BBCooker: if self.state in (state.shutdown, state.forceshutdown, state.error): if hasattr(self.parser, 'shutdown'): - self.parser.shutdown(clean=False, force = True) + self.parser.shutdown(clean=False) self.parser.final_cleanup() raise bb.BBHandledException() @@ -1741,7 +1741,7 @@ class BBCooker: self.state = state.shutdown if self.parser: - self.parser.shutdown(clean=not force, force=force) + self.parser.shutdown(clean=not force) self.parser.final_cleanup() def finishcommand(self): @@ -2186,7 +2186,7 @@ class CookerParser(object): self.results = itertools.chain(self.results, self.parse_generator()) - def shutdown(self, clean=True, force=False): + def shutdown(self, clean=True): if not self.toparse: return if self.haveshutdown: @@ -2215,11 +2215,24 @@ class CookerParser(object): break for process in self.processes: - if force: - process.join(.1) + process.join(0.5) + + for process in self.processes: + if process.exitcode is None: + os.kill(process.pid, signal.SIGINT) + + for process in self.processes: + process.join(0.5) + + for process in self.processes: + if process.exitcode is None: process.terminate() - else: - process.join() + + for process in self.processes: + process.join() + # Added in 3.7, cleans up zombies + if hasattr(process, "close"): + process.close() self.parser_quit.close() # Allow data left in the cancel queue to be discarded @@ -2296,18 +2309,18 @@ class CookerParser(object): except bb.BBHandledException as exc: self.error += 1 logger.debug('Failed to parse recipe: %s' % exc.recipe) - self.shutdown(clean=False, force=True) + self.shutdown(clean=False) return False except ParsingFailure as exc: self.error += 1 logger.error('Unable to parse %s: %s' % (exc.recipe, bb.exceptions.to_string(exc.realexception))) - self.shutdown(clean=False, force=True) + self.shutdown(clean=False) return False except bb.parse.ParseError as exc: self.error += 1 logger.error(str(exc)) - self.shutdown(clean=False, force=True) + self.shutdown(clean=False) return False except bb.data_smart.ExpansionError as exc: self.error += 1 @@ -2316,7 +2329,7 @@ class CookerParser(object): tb = list(itertools.dropwhile(lambda e: e.filename.startswith(bbdir), exc.traceback)) logger.error('ExpansionError during parsing %s', value.recipe, exc_info=(etype, value, tb)) - self.shutdown(clean=False, force=True) + self.shutdown(clean=False) return False except Exception as exc: self.error += 1 @@ -2328,7 +2341,7 @@ class CookerParser(object): # Most likely, an exception occurred during raising an exception import traceback logger.error('Exception during parse: %s' % traceback.format_exc()) - self.shutdown(clean=False, force=True) + self.shutdown(clean=False) return False self.current += 1