From patchwork Tue Feb 14 16:53:06 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 19549 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 7FC60C05027 for ; Tue, 14 Feb 2023 16:53:25 +0000 (UTC) Received: from relay9-d.mail.gandi.net (relay9-d.mail.gandi.net [217.70.183.199]) by mx.groups.io with SMTP id smtpd.web10.12611.1676393595280499853 for ; Tue, 14 Feb 2023 08:53:15 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=XCNuUC2j; spf=pass (domain: bootlin.com, ip: 217.70.183.199, mailfrom: alexis.lothore@bootlin.com) Received: (Authenticated sender: alexis.lothore@bootlin.com) by mail.gandi.net (Postfix) with ESMTPSA id 18D2DFF803; Tue, 14 Feb 2023 16:53:13 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1676393593; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=DtmO4vNPFXkxmi3ZhP/VUsDvy6y16eA+0xJZgfY3RmU=; b=XCNuUC2j+XzMHRVe8ywPHj8W4s/Zz/PVymvp+7IA5kFf5EBY85zH+p886W9LhXJkNQG/Oc RKGZT+vwH09qyw+mn3wQrYVU6s8YFeMwoSFTdlTvbi11t2VpHuJdHv+QAWy7meuDDFNsSM xOS/AupUifvQ8OdNeKmZ+nuMP3kri8dhAHdHxshtuy9Gtje+LwGrVKJ6K3HDoGiFqgMenD 6fjD10Cl8FqfRIDLzzc+12DOwi1CxpowuJkQ/PXls/GsMpAJIDNXPf7HsapDy2of92j5ca de3BuhjwEtH5m2HJ7qTfGJ9hqsVwTVgu/xykH2zNcciHcIC71oxAYwa5ypR5pg== From: alexis.lothore@bootlin.com To: openembedded-core@lists.openembedded.org Cc: alexandre.belloni@bootlin.com, thomas.petazzoni@bootlin.com Subject: [PATCH v2 1/4] scripts/oe-selftest: append metadata to tests results Date: Tue, 14 Feb 2023 17:53:06 +0100 Message-Id: <20230214165309.63527-2-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.39.1 In-Reply-To: <20230214165309.63527-1-alexis.lothore@bootlin.com> References: <20230214165309.63527-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 14 Feb 2023 16:53:25 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/177163 From: Alexis Lothoré Many stored results TEST_TYPE are set to "oeselftest", however all those tests are not run with the same sets of parameters, so those tests results may not be comparable. Attach relevant parameters as tests metadata to allow identifying tests configuration so we can compare tests only when they are run with the same parameters. Signed-off-by: Alexis Lothoré --- meta/lib/oeqa/selftest/context.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/meta/lib/oeqa/selftest/context.py b/meta/lib/oeqa/selftest/context.py index c7dd03ce37..8cc46283ed 100644 --- a/meta/lib/oeqa/selftest/context.py +++ b/meta/lib/oeqa/selftest/context.py @@ -22,6 +22,17 @@ from oeqa.core.exception import OEQAPreRun, OEQATestNotFound from oeqa.utils.commands import runCmd, get_bb_vars, get_test_layer +OESELFTEST_METADATA=["run_all_tests", "run_tests", "skips", "machine", "select_tags", "exclude_tags"] + +def get_oeselftest_metadata(args): + result = {} + raw_args = vars(args) + for metadata in OESELFTEST_METADATA: + if metadata in raw_args: + result[metadata] = raw_args[metadata] + + return result + class NonConcurrentTestSuite(unittest.TestSuite): def __init__(self, suite, processes, setupfunc, removefunc): super().__init__([suite]) @@ -334,12 +345,14 @@ class OESelftestTestContextExecutor(OETestContextExecutor): import platform from oeqa.utils.metadata import metadata_from_bb metadata = metadata_from_bb() + oeselftest_metadata = get_oeselftest_metadata(args) configuration = {'TEST_TYPE': 'oeselftest', 'STARTTIME': args.test_start_time, 'MACHINE': self.tc.td["MACHINE"], 'HOST_DISTRO': oe.lsb.distro_identifier().replace(' ', '-'), 'HOST_NAME': metadata['hostname'], - 'LAYERS': metadata['layers']} + 'LAYERS': metadata['layers'], + 'OESELFTEST_METADATA':oeselftest_metadata} return configuration def get_result_id(self, configuration): From patchwork Tue Feb 14 16:53:07 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 19551 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 88C3EC64EC7 for ; Tue, 14 Feb 2023 16:53:25 +0000 (UTC) Received: from relay9-d.mail.gandi.net (relay9-d.mail.gandi.net [217.70.183.199]) by mx.groups.io with SMTP id smtpd.web11.12509.1676393595245742296 for ; Tue, 14 Feb 2023 08:53:15 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=HAzGL+XO; spf=pass (domain: bootlin.com, ip: 217.70.183.199, mailfrom: alexis.lothore@bootlin.com) Received: (Authenticated sender: alexis.lothore@bootlin.com) by mail.gandi.net (Postfix) with ESMTPSA id 75AD5FF807; Tue, 14 Feb 2023 16:53:13 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1676393593; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=4JKrDMxPGITgj6t0yAr3ej8p6IEuXtwEKin0/+p6/yk=; b=HAzGL+XOjOYyxuoyy+I1wRSb1KrOs24gZHUpK+8xfH9PGACmYviJjn7m06QISbZZHaAhFM H68joeQkNV7rVW4LFqPumeh6+n7j8UAJ6o4eoA2mkZ+DZpaL5Sm+FG/0IGtQ0of+KP2rDv mhWnLgrjrxKOMHNLrB6u2HVpQugZlw8qeGQ0h4Hvr+qeiOR50mMpaC5vhxhQCYReS0u8Ct 835Cpm99HrCRVKDefNLPc2/ByK2z+8sRtnwQCpv+42A0Zy2f7vrnSKAf4To+eWVscQGmbS hsQV66Ox2ePNQV1EcHTct0JLl8cz0y/umNmiw8uQ2/KT5fCcl54Wdw87mf1QOQ== From: alexis.lothore@bootlin.com To: openembedded-core@lists.openembedded.org Cc: alexandre.belloni@bootlin.com, thomas.petazzoni@bootlin.com Subject: [PATCH v2 2/4] oeqa/selftest/resulttooltests: fix minor typo Date: Tue, 14 Feb 2023 17:53:07 +0100 Message-Id: <20230214165309.63527-3-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.39.1 In-Reply-To: <20230214165309.63527-1-alexis.lothore@bootlin.com> References: <20230214165309.63527-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 14 Feb 2023 16:53:25 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/177162 From: Alexis Lothoré Signed-off-by: Alexis Lothoré --- meta/lib/oeqa/selftest/cases/resulttooltests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py b/meta/lib/oeqa/selftest/cases/resulttooltests.py index c2e76f1a44..efdfd98af3 100644 --- a/meta/lib/oeqa/selftest/cases/resulttooltests.py +++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py @@ -71,7 +71,7 @@ class ResultToolTests(OESelftestTestCase): self.assertTrue('target_result1' in results['runtime/mydistro/qemux86/image'], msg="Pair not correct:%s" % results) self.assertTrue('target_result3' in results['runtime/mydistro/qemux86-64/image'], msg="Pair not correct:%s" % results) - def test_regrresion_can_get_regression_result(self): + def test_regression_can_get_regression_result(self): base_result_data = {'result': {'test1': {'status': 'PASSED'}, 'test2': {'status': 'PASSED'}, 'test3': {'status': 'FAILED'}, From patchwork Tue Feb 14 16:53:08 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 19550 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 88866C6379F for ; Tue, 14 Feb 2023 16:53:25 +0000 (UTC) Received: from relay9-d.mail.gandi.net (relay9-d.mail.gandi.net [217.70.183.199]) by mx.groups.io with SMTP id smtpd.web11.12510.1676393595460063016 for ; Tue, 14 Feb 2023 08:53:15 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=AzYOA+ih; spf=pass (domain: bootlin.com, ip: 217.70.183.199, mailfrom: alexis.lothore@bootlin.com) Received: (Authenticated sender: alexis.lothore@bootlin.com) by mail.gandi.net (Postfix) with ESMTPSA id D495CFF80B; Tue, 14 Feb 2023 16:53:13 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1676393594; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=heB1vGl9ENAmE+gwjcV10BuYSuD6f+MBxv3U3Vd+yzU=; b=AzYOA+ihH3/D6+C2kwwNsSe7MBPEsDdwRnPc6OG5OJX9QVwqy2NYOhPhyBw7eTjRfpzqUs AoET3f7PGU2/CxP+ZJYzYbMdHGcUZFNiwu1yC4A46H455I2shI5SwSLXQXPFhlSXefdBhb u7/SZXl5MFHrLw8xMovSCVca9GNbt1cGgNutYC8jqaRp1cw1FOkFOJXRiNHc6vMIqKVTUI PWGkl4+4FNpFBqh8RVxPUsIlQ3TtmjdvesUjWRySrcS5qBQr3e3qyxgD0uPyL9+0lZpY4o VNFGFpiWO/fS/DHD6j0J2fs88bQlaj8JVe4ATeSeZeLimx0UjsBZNFgXizgQhA== From: alexis.lothore@bootlin.com To: openembedded-core@lists.openembedded.org Cc: alexandre.belloni@bootlin.com, thomas.petazzoni@bootlin.com Subject: [PATCH v2 3/4] scripts/resulttool/regression: add metadata filtering for oeselftest Date: Tue, 14 Feb 2023 17:53:08 +0100 Message-Id: <20230214165309.63527-4-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.39.1 In-Reply-To: <20230214165309.63527-1-alexis.lothore@bootlin.com> References: <20230214165309.63527-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 14 Feb 2023 16:53:25 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/177164 From: Alexis Lothoré When generating regression reports, many false positive can be observed since some tests results are compared while the corresponding tests sets are not the same, as it can be seen for example for oeselftest tests (oeselftest is run multiple time but with different parameters, resulting in different tests sets) Add a filtering mechanism in resulttool regression module to enable a better matching between tests. The METADATA_MATCH_TABLE defines that when the TEST_TYPE is "oeselftest", then resulttool should filter pairs based on OESELFTEST_METADATA appended to test configuration. If metadata is absent from "base" test results, tests are marked "comparable" to preserve compatibility with test results which still do not have those new metadata. Signed-off-by: Alexis Lothoré --- scripts/lib/resulttool/regression.py | 34 ++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py index 9f952951b3..64d1eeee37 100644 --- a/scripts/lib/resulttool/regression.py +++ b/scripts/lib/resulttool/regression.py @@ -12,6 +12,36 @@ import json from oeqa.utils.git import GitRepo import oeqa.utils.gitarchive as gitarchive +METADATA_MATCH_TABLE={ + "oeselftest": "OESELFTEST_METADATA" +} + + +def metadata_matches(base_configuration, target_configuration): + """ + For passed base and target, check test type. If test type matches one of + properties described in METADATA_MATCH_TABLE, compare metadata if it is + present in base. Return true if metadata matches, or if base lacks some + data (either TEST_TYPE or the corresponding metadata) + """ + test_type=base_configuration.get('TEST_TYPE') + metadata_key=METADATA_MATCH_TABLE.get(test_type) + if metadata_key not in base_configuration: + return True + + if target_configuration.get(metadata_key) != base_configuration[metadata_key]: + return False + + return True + +def can_be_compared(base_configuration, target_configuration): + """ + Some tests are not relevant to be compared, for example some oeselftest + run with different tests sets or parameters. Return true if tests can be + compared + """ + return metadata_matches(base_configuration, target_configuration) + def compare_result(logger, base_name, target_name, base_result, target_result): base_result = base_result.get('result') target_result = target_result.get('result') @@ -62,6 +92,8 @@ def regression_common(args, logger, base_results, target_results): # removing any pairs which match for c in base.copy(): for b in target.copy(): + if not can_be_compared(base_results[a][c]['configuration'], target_results[a][b]['configuration']): + continue res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b]) if not res: matches.append(resstr) @@ -71,6 +103,8 @@ def regression_common(args, logger, base_results, target_results): # Should only now see regressions, we may not be able to match multiple pairs directly for c in base: for b in target: + if not can_be_compared(base_results[a][c]['configuration'], target_results[a][b]['configuration']): + continue res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b]) if res: regressions.append(resstr) From patchwork Tue Feb 14 16:53:09 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 19552 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 931CDC61DA4 for ; Tue, 14 Feb 2023 16:53:25 +0000 (UTC) Received: from relay9-d.mail.gandi.net (relay9-d.mail.gandi.net [217.70.183.199]) by mx.groups.io with SMTP id smtpd.web11.12512.1676393595802970884 for ; Tue, 14 Feb 2023 08:53:16 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=ByxqXsN7; spf=pass (domain: bootlin.com, ip: 217.70.183.199, mailfrom: alexis.lothore@bootlin.com) Received: (Authenticated sender: alexis.lothore@bootlin.com) by mail.gandi.net (Postfix) with ESMTPSA id 41B9EFF80F; Tue, 14 Feb 2023 16:53:14 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1676393594; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=Rztb2D/2vZXUDPzmD2X0J80X5XwXr5Xp+MPcecsbTas=; b=ByxqXsN7mTsKf2f5o7tvKyVkNLVAoVrRGdIlyG2ksUfOKEnMo6RNhu1Iu9AiYwAJJnkSD2 /wjKTIBcKdo7s2B3IcAnPWI4GhH57sjNrNIxZroTVtf+uwt1nfxUblP3klrTztt0IEkK3k 9ZAGsKdxkdIwq2JQ8AKiqeQGIGJ8w+i/4qyoim4fWQ/sQBZsOs3sXqBD9JW7uCQQo3mA1g Hn4xnEJuZ7b+1S9Ypv4P49ZgZYXVwjEGQErje6EMfKVqfQke0yBziD57r1di5zn5AdTEjM f1p1bFILTwGgfa9Qa1yKRmiXc0GRJgsuyISxI1bocpzhgZRXqIxAGihzSrDdTQ== From: alexis.lothore@bootlin.com To: openembedded-core@lists.openembedded.org Cc: alexandre.belloni@bootlin.com, thomas.petazzoni@bootlin.com Subject: [PATCH v2 4/4] oeqa/selftest/resulttool: add test for metadata filtering on regression Date: Tue, 14 Feb 2023 17:53:09 +0100 Message-Id: <20230214165309.63527-5-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.39.1 In-Reply-To: <20230214165309.63527-1-alexis.lothore@bootlin.com> References: <20230214165309.63527-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 14 Feb 2023 16:53:25 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/177165 From: Alexis Lothoré Introduce new tests for the metadata-based filtering added for oeselftest results Signed-off-by: Alexis Lothoré --- .../oeqa/selftest/cases/resulttooltests.py | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py b/meta/lib/oeqa/selftest/cases/resulttooltests.py index efdfd98af3..e93796e145 100644 --- a/meta/lib/oeqa/selftest/cases/resulttooltests.py +++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py @@ -98,3 +98,124 @@ class ResultToolTests(OESelftestTestCase): resultutils.append_resultsdata(results, ResultToolTests.target_results_data, configmap=resultutils.flatten_map) self.assertEqual(len(results[''].keys()), 5, msg="Flattened results not correct %s" % str(results)) + def test_results_without_metadata_can_be_compared(self): + base_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86"} + target_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86"} + self.assertTrue(regression.can_be_compared(base_configuration, target_configuration), + msg="incorrect metadata filtering, tests without metadata should be compared") + + def test_target_result_with_missing_metadata_can_not_be_compared(self): + base_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["toolchain-user", "toolchain-system"], + "exclude_tags": None}} + target_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86"} + self.assertFalse(regression.can_be_compared(base_configuration, target_configuration), + msg="incorrect metadata filtering, tests should not be compared") + + def test_results_with_matching_metadata_can_be_compared(self): + base_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["toolchain-user", "toolchain-system"], + "exclude_tags": None}} + target_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["toolchain-user", "toolchain-system"], + "exclude_tags": None}} + self.assertTrue(regression.can_be_compared(base_configuration, target_configuration), + msg="incorrect metadata filtering, tests with matching metadata should be compared") + + def test_results_with_mismatching_metadata_can_not_be_compared(self): + base_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["toolchain-user", "toolchain-system"], + "exclude_tags": None}} + target_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["machine"], + "exclude_tags": None}} + self.assertFalse(regression.can_be_compared(base_configuration, target_configuration), + msg="incorrect metadata filtering, tests with mismatching metadata should not be compared") + + def test_metadata_matching_is_only_checked_for_relevant_test_type(self): + base_configuration = {"TEST_TYPE": "runtime", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["toolchain-user", "toolchain-system"], + "exclude_tags": None}} + target_configuration = {"TEST_TYPE": "runtime", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["machine"], + "exclude_tags": None}} + self.assertTrue(regression.can_be_compared(base_configuration, target_configuration), + msg="incorrect metadata filtering, %s tests should be compared" % base_configuration['TEST_TYPE'])