From patchwork Tue Feb 28 18:10:45 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 20286 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 8658CC7EE33 for ; Tue, 28 Feb 2023 18:10:40 +0000 (UTC) Received: from relay11.mail.gandi.net (relay11.mail.gandi.net [217.70.178.231]) by mx.groups.io with SMTP id smtpd.web10.31355.1677607837255983553 for ; Tue, 28 Feb 2023 10:10:39 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=kt/CaPQy; spf=pass (domain: bootlin.com, ip: 217.70.178.231, mailfrom: alexis.lothore@bootlin.com) Received: (Authenticated sender: alexis.lothore@bootlin.com) by mail.gandi.net (Postfix) with ESMTPSA id A6660100005; Tue, 28 Feb 2023 18:10:35 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1677607836; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=Jdhv5J5uXoxFm30YIAvsGV8jjaRhkTuxANhDW0rUmIk=; b=kt/CaPQyyTmktzpTlOt3r1DhBuvuj6EJW1dnVd7LCNIHe7003PED8qsf7KhqHlkDKP6RLv UiJ6pQyfjhvvr5MuH1SDIalwvOjUZ15wnGpC4sHUEvKrheXO8/Cvrnt6QDZyIy5Hfrt78V aBDu8HpGLgd/a0OpoCnoX+eJFRwAMpZL1PaIXN8n/2qEKqqn9do49BIPYbRReE8soH6Lwx KjNEo8Ctpfbv/6r7PVaIVR4j2FVETeW+DMWT5Qnn1Rrhd4oyZTc8Q9LGLa/N7Vc73srntb 6BN7cNo/0eKOtVdjSo86INZwkd1y8cIe7iVpPNVwvHK5fhaUsgcrz5BVt570Ew== From: alexis.lothore@bootlin.com To: openembedded-core@lists.openembedded.org Cc: alexandre.belloni@bootlin.com, thomas.petazzoni@bootlin.com Subject: [PATCH 1/6] scripts/resulttool: call fixup_ptest_names in regression_common Date: Tue, 28 Feb 2023 19:10:45 +0100 Message-Id: <20230228181052.4191521-2-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.39.1 In-Reply-To: <20230228181052.4191521-1-alexis.lothore@bootlin.com> References: <20230228181052.4191521-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 28 Feb 2023 18:10:40 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/177850 From: Alexis Lothoré ptests names not only need to be fixed for regression based on git testresults but also for testsresults provided "manually" Move ptests naming fixup in regression_common to share the fixup between both regression use cases Signed-off-by: Alexis Lothoré --- scripts/lib/resulttool/regression.py | 41 ++++++++++++++-------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py index 74fd5f38951..ad377c596b6 100644 --- a/scripts/lib/resulttool/regression.py +++ b/scripts/lib/resulttool/regression.py @@ -206,12 +206,32 @@ def regression(args, logger): regression_common(args, logger, base_results, target_results) +# Some test case naming is poor and contains random strings, particularly lttng/babeltrace. +# Truncating the test names works since they contain file and line number identifiers +# which allows us to match them without the random components. +def fixup_ptest_names(results, logger): + for r in results: + for i in results[r]: + tests = list(results[r][i]['result'].keys()) + for test in tests: + new = None + if test.startswith(("ptestresult.lttng-tools.", "ptestresult.babeltrace.", "ptestresult.babeltrace2")) and "_-_" in test: + new = test.split("_-_")[0] + elif test.startswith(("ptestresult.curl.")) and "__" in test: + new = test.split("__")[0] + if new: + results[r][i]['result'][new] = results[r][i]['result'][test] + del results[r][i]['result'][test] + def regression_common(args, logger, base_results, target_results): if args.base_result_id: base_results = resultutils.filter_resultsdata(base_results, args.base_result_id) if args.target_result_id: target_results = resultutils.filter_resultsdata(target_results, args.target_result_id) + fixup_ptest_names(base_results, logger) + fixup_ptest_names(target_results, logger) + matches = [] regressions = [] notfound = [] @@ -248,24 +268,6 @@ def regression_common(args, logger, base_results, target_results): return 0 -# Some test case naming is poor and contains random strings, particularly lttng/babeltrace. -# Truncating the test names works since they contain file and line number identifiers -# which allows us to match them without the random components. -def fixup_ptest_names(results, logger): - for r in results: - for i in results[r]: - tests = list(results[r][i]['result'].keys()) - for test in tests: - new = None - if test.startswith(("ptestresult.lttng-tools.", "ptestresult.babeltrace.", "ptestresult.babeltrace2")) and "_-_" in test: - new = test.split("_-_")[0] - elif test.startswith(("ptestresult.curl.")) and "__" in test: - new = test.split("__")[0] - if new: - results[r][i]['result'][new] = results[r][i]['result'][test] - del results[r][i]['result'][test] - - def regression_git(args, logger): base_results = {} target_results = {} @@ -327,9 +329,6 @@ def regression_git(args, logger): base_results = resultutils.git_get_result(repo, revs[index1][2]) target_results = resultutils.git_get_result(repo, revs[index2][2]) - fixup_ptest_names(base_results, logger) - fixup_ptest_names(target_results, logger) - regression_common(args, logger, base_results, target_results) return 0 From patchwork Tue Feb 28 18:10:47 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 20283 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 85078C7EE32 for ; Tue, 28 Feb 2023 18:10:40 +0000 (UTC) Received: from relay11.mail.gandi.net (relay11.mail.gandi.net [217.70.178.231]) by mx.groups.io with SMTP id smtpd.web10.31358.1677607838541612569 for ; Tue, 28 Feb 2023 10:10:38 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=Nztw5QnO; spf=pass (domain: bootlin.com, ip: 217.70.178.231, mailfrom: alexis.lothore@bootlin.com) Received: (Authenticated sender: alexis.lothore@bootlin.com) by mail.gandi.net (Postfix) with ESMTPSA id 9948210000A; Tue, 28 Feb 2023 18:10:36 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1677607837; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=Unk9yl6gWTtxlaAQ7t1O99MjRSqZ/vYRLkLlE4rgY2Y=; b=Nztw5QnOsqdtmAaHLOVgLxQ9NdV48p+uLvgdOn1MPsM/EtzDICFaumsKm+tWrZk+r19H0b 76d0SlVi62Wf7WQiZI6Mb2tOhi0dmlQEkNPKf7CCAtrpeR7t0/88MYH25jQ21On7RtHolt dYmcrx5l+0F6STXj+uTIRgaV4yo/D2DOUm9zz8tLfUml67pkFHEcQTC79F6kij9Nio3YiZ 1mSNAKSqemQUWNA+RQ7HnL80cI/58l5p4H3hTZQVuAA4f33OkBtGEIaWhvYws9lGDe2oZl uYnCys2Hjfk6HR7RL8gNsE4txG7XhAm37APWGti9c6olBbbITswRIWp12HT4uA== From: alexis.lothore@bootlin.com To: openembedded-core@lists.openembedded.org Cc: alexandre.belloni@bootlin.com, thomas.petazzoni@bootlin.com Subject: [PATCH 2/6] oeqa/selftest/resulttool: fix ptest tests Date: Tue, 28 Feb 2023 19:10:47 +0100 Message-Id: <20230228181052.4191521-4-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.39.1 In-Reply-To: <20230228181052.4191521-1-alexis.lothore@bootlin.com> References: <20230228181052.4191521-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 28 Feb 2023 18:10:40 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/177849 From: Alexis Lothoré There are multiple issues with the tests for the ptest fixup mechanism introduced in c45d58f003e8d8b323169ca9d479dc49c43a9974: - the feature does not impact regression.can_be_compared but directly the comparison step in regression.regression_common - the "status" field was wrong ("STATUS"), which prevents tests comparison - there is a typo babeltrace2 ptest result name Signed-off-by: Alexis Lothoré --- .../oeqa/selftest/cases/resulttooltests.py | 80 ++++++++++--------- 1 file changed, 44 insertions(+), 36 deletions(-) diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py b/meta/lib/oeqa/selftest/cases/resulttooltests.py index f059991c19c..8489ae8eb52 100644 --- a/meta/lib/oeqa/selftest/cases/resulttooltests.py +++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py @@ -305,39 +305,47 @@ class ResultToolTests(OESelftestTestCase): msg="incorrect ltpresult filtering, matching ltpresult content should be compared") def test_can_match_non_static_ptest_names(self): - base_configuration = {"configuration": { - "TEST_TYPE": "runtime", - "MACHINE": "qemux86" - }, "result": { - "ptestresult.lttng-tools.foo_-_bar_-_moo": { - "STATUS": "PASSED" - }, - "ptestresult.babeltrace.bar_-_moo_-_foo": { - "STATUS": "PASSED" - }, - "ptestresult.babletrace2.moo_-_foo_-_bar": { - "STATUS": "PASSED" - }, - "ptestresult.curl.test_0000__foo_out_of_bar": { - "STATUS": "PASSED" - } - }} - target_configuration = {"configuration": { - "TEST_TYPE": "runtime", - "MACHINE": "qemux86" - }, "result": { - "ptestresult.lttng-tools.xxx_-_yyy_-_zzz": { - "STATUS": "PASSED" - }, - "ptestresult.babeltrace.yyy_-_zzz_-_xxx": { - "STATUS": "PASSED" - }, - "ptestresult.babletrace2.zzz_-_xxx_-_yyy": { - "STATUS": "PASSED" - }, - "ptestresult.curl.test_0000__xxx_out_of_yyy": { - "STATUS": "PASSED" - } - }} - self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration), - msg="incorrect ptests filtering, tests shoould be compared if prefixes match") + base_configuration = {"a": { + "conf_X": { + "configuration": { + "TEST_TYPE": "runtime", + "MACHINE": "qemux86" + }, "result": { + "ptestresult.lttng-tools.foo_-_bar_-_moo": { + "status": "PASSED" + }, + "ptestresult.babeltrace.bar_-_moo_-_foo": { + "status": "PASSED" + }, + "ptestresult.babeltrace2.moo_-_foo_-_bar": { + "status": "PASSED" + }, + "ptestresult.curl.test_0000__foo_out_of_bar": { + "status": "PASSED" + } + }}}} + target_configuration = {"a": { + "conf_Y": { + "configuration": { + "TEST_TYPE": "runtime", + "MACHINE": "qemux86" + }, "result": { + "ptestresult.lttng-tools.foo_-_yyy_-_zzz": { + "status": "PASSED" + }, + "ptestresult.babeltrace.bar_-_zzz_-_xxx": { + "status": "PASSED" + }, + "ptestresult.babeltrace2.moo_-_xxx_-_yyy": { + "status": "PASSED" + }, + "ptestresult.curl.test_0000__xxx_out_of_yyy": { + "status": "PASSED" + } + }}}} + regression.fixup_ptest_names(base_configuration, self.logger) + regression.fixup_ptest_names(target_configuration, self.logger) + result, resultstring = regression.compare_result( + self.logger, "A", "B", base_configuration["a"]["conf_X"], target_configuration["a"]["conf_Y"]) + self.assertDictEqual( + result, {}, msg=f"ptests should be compared: {resultstring}") From patchwork Tue Feb 28 18:10:49 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 20285 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 84E0AC7EE30 for ; Tue, 28 Feb 2023 18:10:40 +0000 (UTC) Received: from relay11.mail.gandi.net (relay11.mail.gandi.net [217.70.178.231]) by mx.groups.io with SMTP id smtpd.web11.31408.1677607839763643802 for ; Tue, 28 Feb 2023 10:10:40 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=FnoPgO1n; spf=pass (domain: bootlin.com, ip: 217.70.178.231, mailfrom: alexis.lothore@bootlin.com) Received: (Authenticated sender: alexis.lothore@bootlin.com) by mail.gandi.net (Postfix) with ESMTPSA id 24B50100009; Tue, 28 Feb 2023 18:10:38 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1677607838; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=V8dilXzrQKXmmIkVD1sEsCfDNJ9/I0s++z3sVjqhAEw=; b=FnoPgO1nE/vFJoRv31bpi9uJqbFJWxPdRuRWRi3XJDIIab7qacNzsnuIM+NxJyZqKO6798 Bz3hDEz4F3iF1GMRBQjcxcU1s3zJp2SzYPPY0xo0n0rhmYp6v3y7BX8g+6H+8d1pN7KJNB KZcGRmwuPB8cnalFLUTGLYwy8S4BgdmHw1Gyp4WYg8riHy585G1bnEPbSmTtYvToSLAD1j KKX5QufEsJoxrMkqxCLNO9LbTF4QmzbohrYjpxF7pNQecYDSBd8VqiPOXgC7TaG0KZDu56 yXlV+rCwPCEivGgOOrvyCWn5afBeYbFk7tGyKnBBYKXG2jj4yb0txxA0o6lJrQ== From: alexis.lothore@bootlin.com To: openembedded-core@lists.openembedded.org Cc: alexandre.belloni@bootlin.com, thomas.petazzoni@bootlin.com Subject: [PATCH 3/6] oeqa/selftest/resulttool: fix fake data Date: Tue, 28 Feb 2023 19:10:49 +0100 Message-Id: <20230228181052.4191521-6-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.39.1 In-Reply-To: <20230228181052.4191521-1-alexis.lothore@bootlin.com> References: <20230228181052.4191521-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 28 Feb 2023 18:10:40 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/177851 From: Alexis Lothoré resulttool searches for "status" field, not "STATUS", in results. This fix is more to avoid confusion than fixing anything, since the updated tests are about regression.can_be_compared, which does not check for "status" Signed-off-by: Alexis Lothoré --- meta/lib/oeqa/selftest/cases/resulttooltests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py b/meta/lib/oeqa/selftest/cases/resulttooltests.py index 8489ae8eb52..eeed276b8de 100644 --- a/meta/lib/oeqa/selftest/cases/resulttooltests.py +++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py @@ -274,14 +274,14 @@ class ResultToolTests(OESelftestTestCase): "MACHINE": "qemux86" }, "result": { "ltpresult_foo": { - "STATUS": "PASSED" + "status": "PASSED" }}} target_configuration = {"configuration": { "TEST_TYPE": "runtime", "MACHINE": "qemux86_64" }, "result": { "bar": { - "STATUS": "PASSED" + "status": "PASSED" }}} self.assertFalse(regression.can_be_compared(self.logger, base_configuration, target_configuration), msg="incorrect ltpresult filtering, mismatching ltpresult content should not be compared") @@ -292,14 +292,14 @@ class ResultToolTests(OESelftestTestCase): "MACHINE": "qemux86" }, "result": { "ltpresult_foo": { - "STATUS": "PASSED" + "status": "PASSED" }}} target_configuration = {"configuration": { "TEST_TYPE": "runtime", "MACHINE": "qemux86" }, "result": { "ltpresult_foo": { - "STATUS": "PASSED" + "status": "PASSED" }}} self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration), msg="incorrect ltpresult filtering, matching ltpresult content should be compared") From patchwork Tue Feb 28 18:10:50 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 20288 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 4BDA8C64EC7 for ; Tue, 28 Feb 2023 18:10:50 +0000 (UTC) Received: from relay11.mail.gandi.net (relay11.mail.gandi.net [217.70.178.231]) by mx.groups.io with SMTP id smtpd.web11.31410.1677607840433532012 for ; Tue, 28 Feb 2023 10:10:40 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=OhzbuErN; spf=pass (domain: bootlin.com, ip: 217.70.178.231, mailfrom: alexis.lothore@bootlin.com) Received: (Authenticated sender: alexis.lothore@bootlin.com) by mail.gandi.net (Postfix) with ESMTPSA id ACA37100010; Tue, 28 Feb 2023 18:10:38 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1677607839; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=e0EdXF/+9/LWXG6RqA/gnOp4+mIVgxx5r+STd6Q+44Y=; b=OhzbuErNZt+zC23VMEaqNBqDv36XkPPZGvdq8WZbut9DyzLmO6rrmBGDfAtoRUX+ZVRaG2 gdAvfucgeSkL2oHK2GREBVixLXU/UIvyqT7XwsuGbMRsiQkO467C4j4KPRSqcIv504RKxd i3lD2xUxhAxIVaa9wv1tou00BTZB4Ec8XVo4AXUvCAHvuybnupfjjaArvv0ogdP9g1wJLP TkEp3/7DccZ1ZDU/5/vY0noyenTSS5pUuYqPrQ8ZeXT17djAwF+kK/4BoaiaoeHVJkPO1k 00Duw478YThGCuZcE2gOjSUECFDcWPR6cnqwHKjAJ5L/ZnmkSxj2CyuWOvkQcA== From: alexis.lothore@bootlin.com To: openembedded-core@lists.openembedded.org Cc: alexandre.belloni@bootlin.com, thomas.petazzoni@bootlin.com Subject: [PATCH 4/6] scripts/resulttool: fix ptests results containing a non reproducible path Date: Tue, 28 Feb 2023 19:10:50 +0100 Message-Id: <20230228181052.4191521-7-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.39.1 In-Reply-To: <20230228181052.4191521-1-alexis.lothore@bootlin.com> References: <20230228181052.4191521-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 28 Feb 2023 18:10:50 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/177852 From: Alexis Lothoré Some ptests results may be wrongly sampled, resulting in some part of the error being propagated in test name. For example: "ptestresult.binutils-ld.in testcase /home/pokybuild/yocto-worker/qemumips/build/build-st-1666126/tmp/work/mips32r2-poky-linux/binutils-cross-testsuite/2.40-r0/git/ld/testsuite/ld-ctf/ctf.exp" "ptestresult.gcc.Couldn't create remote directory /tmp/runtest.455781 on ssh" "ptestresult.gcc-libstdc++-v3.Couldn't create remote directory /tmp/runtest.3814266 on target" While the root errors must be fixed in corresponding ptests packages for those tests, the test results history must still be usable even with this issue Add new filters to detect if temporary test directories (build-st-*, /tmp/runtime.*) are present in name. If so, truncate test name. As a side effect, it will aggregate multiple failing errors into one, but the regression will still be raised Signed-off-by: Alexis Lothoré --- scripts/lib/resulttool/regression.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py index ad377c596b6..d7f0b16ca98 100644 --- a/scripts/lib/resulttool/regression.py +++ b/scripts/lib/resulttool/regression.py @@ -219,6 +219,12 @@ def fixup_ptest_names(results, logger): new = test.split("_-_")[0] elif test.startswith(("ptestresult.curl.")) and "__" in test: new = test.split("__")[0] + elif test.startswith(("ptestresult.dbus.")) and "__" in test: + new = test.split("__")[0] + elif test.startswith("ptestresult.binutils") and "build-st-" in test: + new = test.split(" ")[0] + elif test.startswith("ptestresult.gcc") and "/tmp/runtest." in test: + new = ".".join(test.split(".")[:2]) if new: results[r][i]['result'][new] = results[r][i]['result'][test] del results[r][i]['result'][test] From patchwork Tue Feb 28 18:10:51 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 20289 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 531B7C64ED6 for ; Tue, 28 Feb 2023 18:10:50 +0000 (UTC) Received: from relay11.mail.gandi.net (relay11.mail.gandi.net [217.70.178.231]) by mx.groups.io with SMTP id smtpd.web11.31411.1677607841001630554 for ; Tue, 28 Feb 2023 10:10:41 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=M6Ankg+K; spf=pass (domain: bootlin.com, ip: 217.70.178.231, mailfrom: alexis.lothore@bootlin.com) Received: (Authenticated sender: alexis.lothore@bootlin.com) by mail.gandi.net (Postfix) with ESMTPSA id 5DCB310000A; Tue, 28 Feb 2023 18:10:39 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1677607839; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=phWTC9wjG66QZ40Qrn3NnJZufKBUmaxcJFDHSb9wLeA=; b=M6Ankg+KmjZM8m1MzSCBr+0ws7dqWr7N0+hYPOMaXFGESoPAGlW74As34kF6wIHfCDY2AZ jchX7Zdl4TFBxfHjni1NWqV+gCjhJtgfNICl5RTsaHgG+B06xKo5PuhDBX3/pzo94vYkg0 K9R0/D5q6WTUSWswgAUyB0H8wkmUQlgjBkS2eVC1vd5BcwwaD5awEnIl8FikiebtgSnDDh RTaKT3CzppDZDGHyew4XS3MlCVry4ATVXnZ7CTK9XprXffhnE2RWo8D8l8cGDo/e7hTWOP wK4D9jwI13N1OyN2wcbPnbqIeybodC460UySgmqsWDZaPQG93Cd/j4z24BkOLg== From: alexis.lothore@bootlin.com To: openembedded-core@lists.openembedded.org Cc: alexandre.belloni@bootlin.com, thomas.petazzoni@bootlin.com Subject: [PATCH 5/6] oeqa/selftest/resulttool: add test for error propagation in test name filtering Date: Tue, 28 Feb 2023 19:10:51 +0100 Message-Id: <20230228181052.4191521-8-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.39.1 In-Reply-To: <20230228181052.4191521-1-alexis.lothore@bootlin.com> References: <20230228181052.4191521-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 28 Feb 2023 18:10:50 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/177853 From: Alexis Lothoré Signed-off-by: Alexis Lothoré --- .../oeqa/selftest/cases/resulttooltests.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py b/meta/lib/oeqa/selftest/cases/resulttooltests.py index eeed276b8de..c3303f3fbbe 100644 --- a/meta/lib/oeqa/selftest/cases/resulttooltests.py +++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py @@ -322,6 +322,18 @@ class ResultToolTests(OESelftestTestCase): }, "ptestresult.curl.test_0000__foo_out_of_bar": { "status": "PASSED" + }, + "ptestresult.dbus.test_0000__foo_out_of_bar,_remaining:_00:02,_took_0.032s,_duration:_03:32_": { + "status": "PASSED" + }, + "ptestresult.binutils-ld.in testcase /foo/build-st-bar/moo/ctf.exp": { + "status": "PASSED" + }, + "ptestresult.gcc-libstdc++-v3.Couldn't create remote directory /tmp/runtest.30975 on target": { + "status": "PASSED" + }, + "ptestresult.gcc-libgomp.Couldn't create remote directory /tmp/runtest.3657621 on": { + "status": "PASSED" } }}}} target_configuration = {"a": { @@ -341,6 +353,18 @@ class ResultToolTests(OESelftestTestCase): }, "ptestresult.curl.test_0000__xxx_out_of_yyy": { "status": "PASSED" + }, + "ptestresult.dbus.test_0000__yyy_out_of_zzz,_remaining:_00:03,_took_0.034s,_duration:_03:30_": { + "status": "PASSED" + }, + "ptestresult.binutils-ld.in testcase /xxx/build-st-yyy/zzz/ctf.exp": { + "status": "PASSED" + }, + "ptestresult.gcc-libstdc++-v3.Couldn't create remote directory /tmp/runtest.45678 on target": { + "status": "PASSED" + }, + "ptestresult.gcc-libgomp.Couldn't create remote directory /tmp/runtest.3657621 on": { + "status": "PASSED" } }}}} regression.fixup_ptest_names(base_configuration, self.logger) From patchwork Tue Feb 28 18:10:52 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 20287 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 53357C7EE30 for ; Tue, 28 Feb 2023 18:10:50 +0000 (UTC) Received: from relay11.mail.gandi.net (relay11.mail.gandi.net [217.70.178.231]) by mx.groups.io with SMTP id smtpd.web11.31412.1677607841424292878 for ; Tue, 28 Feb 2023 10:10:41 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=APv6p0VI; spf=pass (domain: bootlin.com, ip: 217.70.178.231, mailfrom: alexis.lothore@bootlin.com) Received: (Authenticated sender: alexis.lothore@bootlin.com) by mail.gandi.net (Postfix) with ESMTPSA id CBC83100012; Tue, 28 Feb 2023 18:10:39 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1677607840; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=+tiNaNPBfVSQfRsh8p6X53eS0F+1MHfIqXiSXbSubz0=; b=APv6p0VI2n60WDEgD9A8R4FyOIbnTZ3K2MnfnAjyob2zkjEF67uKM1+txuVE5zY3d+k6sa /cmleev5M+s0Mb+z+8fFbZSxWhA/FR+RQTVCX8zevKNRU6Wf1XRX+Zp94rmB10tOYgsaGE JnZtAV6FNfLNOG4FDb6h1ngby5vRoIjlIy2yo+fqdXzF9kSMdwoTM1LlNqK0J8mxaN84Xc 1x6u1NVphtfTSxHKAzZeG2D8H0Y4LHqbF26BDCcN3XruWgTOhXbIQBAa1ZuDlJPka1Orf9 upBsxIHJwbWDXC3yEEhVTOtpRhM35ZW46pCtndQqTM02tN2Yq86jwssNiDrZRQ== From: alexis.lothore@bootlin.com To: openembedded-core@lists.openembedded.org Cc: alexandre.belloni@bootlin.com, thomas.petazzoni@bootlin.com Subject: [PATCH 6/6] scripts/resulttool: do not count newly passing tests as regressions Date: Tue, 28 Feb 2023 19:10:52 +0100 Message-Id: <20230228181052.4191521-9-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.39.1 In-Reply-To: <20230228181052.4191521-1-alexis.lothore@bootlin.com> References: <20230228181052.4191521-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 28 Feb 2023 18:10:50 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/177855 From: Alexis Lothoré resulttool regression module simply compare a base test status to a target test result status. This approach raises many false positives since all XXX -> PASS transitions (XXX being any status different from PASS) are flagged as regression. - Do not list XXX -> PASS transitions in regression report, instead count them and print a summary of "newly passing tests" - If an inspected pair has only "newly passing tests", do not print detailed list and print it as "Improvement" instead of "Regression" Updated output example looks like the following: [...] Improvement: oeselftest_fedora-37_qemux86-64_20230127010225 oeselftest_ubuntu-22.04_qemux86-64_20230226120516 (+1 test(s) passing) [...] Match: oeselftest_almalinux-8.7_qemuarm64_20230127015830 oeselftest_almalinux-8.7_qemuarm64_20230227015258 [...] Regression: oeselftest_almalinux-9.1_qemumips_20230127000217 oeselftest_opensuseleap-15.4_qemumips_20230226130046 ptestresult.glibc-user.debug/tst-read-chk-cancel: PASS -> None ptestresult.glibc-user.nptl/tst-mutexpi4: PASS -> FAIL ptestresult.glibc-user.nptl/tst-mutexpi5a: PASS -> FAIL Additionally, 44 previously failing test(s) is/are now passing Signed-off-by: Alexis Lothoré --- scripts/lib/resulttool/regression.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py index d7f0b16ca98..1facbcd85e1 100644 --- a/scripts/lib/resulttool/regression.py +++ b/scripts/lib/resulttool/regression.py @@ -190,11 +190,20 @@ def compare_result(logger, base_name, target_name, base_result, target_result): else: logger.error('Failed to retrieved base test case status: %s' % k) if result: - resultstring = "Regression: %s\n %s\n" % (base_name, target_name) - for k in sorted(result): - resultstring += ' %s: %s -> %s\n' % (k, result[k]['base'], result[k]['target']) + new_pass_count = sum(test['target'] is not None and test['target'].startswith("PASS") for test in result.values()) + # Print a regression report only if at least one test has a regression status (FAIL, SKIPPED, absent...) + if new_pass_count < len(result): + resultstring = "Regression: %s\n %s\n" % (base_name, target_name) + for k in sorted(result): + if not result[k]['target'] or not result[k]['target'].startswith("PASS"): + resultstring += ' %s: %s -> %s\n' % (k, result[k]['base'], result[k]['target']) + if new_pass_count > 0: + resultstring += f' Additionally, {new_pass_count} previously failing test(s) is/are now passing\n' + else: + resultstring = "Improvement: %s\n %s\n (+%d test(s) passing)" % (base_name, target_name, new_pass_count) + result = None else: - resultstring = "Match: %s\n %s" % (base_name, target_name) + resultstring = "Match: %s\n %s" % (base_name, target_name) return result, resultstring def get_results(logger, source): @@ -269,9 +278,9 @@ def regression_common(args, logger, base_results, target_results): else: notfound.append("%s not found in target" % a) print("\n".join(sorted(matches))) + print("\n") print("\n".join(sorted(regressions))) print("\n".join(sorted(notfound))) - return 0 def regression_git(args, logger):