From patchwork Fri Feb 24 16:45:50 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 20120 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 9578AC7EE31 for ; Fri, 24 Feb 2023 16:46:01 +0000 (UTC) Received: from relay11.mail.gandi.net (relay11.mail.gandi.net [217.70.178.231]) by mx.groups.io with SMTP id smtpd.web11.22869.1677257155218692058 for ; Fri, 24 Feb 2023 08:45:55 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=Cb1iBAyB; spf=pass (domain: bootlin.com, ip: 217.70.178.231, mailfrom: alexis.lothore@bootlin.com) Received: (Authenticated sender: alexis.lothore@bootlin.com) by mail.gandi.net (Postfix) with ESMTPSA id 99832100003; Fri, 24 Feb 2023 16:45:53 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1677257153; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=Wny86nfBwibXwGz6ynB3IsyCgb9YKCAKsUpGs7je1uM=; b=Cb1iBAyBWPMC1Skz+jhHUnFyNmZftFG8g9ZNKnvlVZjv/d+ceQBJ7AJXXLlLgHxqMfmg/T TMqaArvOJ4OzljOSwRxiO9pdK6sNkTset2Gy25GthyuOoyerGOB5haQDBzvfaTktcF/t6T y05/chKGe7hDTyUAfVwcUuiFOGYshjzNEfboAWCe+F1KRWrsoy6M5jGz03AdVw38Hddl0a nqFGb3Z2Z2vzDyE0DQkz6N6MQCDYx7qUS95OByymLeNf4e6JMCA7QRd65OXtlLWWirmLcC VgY593npvonaxdhrg5uYLR1P+K7QzMqL0Zb90w0/I7BH7U+8ZB2b7HF/gviG3w== From: alexis.lothore@bootlin.com To: openembedded-core@lists.openembedded.org Cc: alexandre.belloni@bootlin.com, thomas.petazzoni@bootlin.com Subject: [PATCH v3 1/6] scripts/oe-selftest: append metadata to tests results Date: Fri, 24 Feb 2023 17:45:50 +0100 Message-Id: <20230224164555.67634-2-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.39.1 In-Reply-To: <20230224164555.67634-1-alexis.lothore@bootlin.com> References: <20230224164555.67634-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 24 Feb 2023 16:46:01 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/177699 From: Alexis Lothoré Many stored results TEST_TYPE are set to "oeselftest", however all those tests are not run with the same sets of parameters, so those tests results may not be comparable. Attach relevant parameters as tests metadata to allow identifying tests configuration so we can compare tests only when they are run with the same parameters. Signed-off-by: Alexis Lothoré --- meta/lib/oeqa/selftest/context.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/meta/lib/oeqa/selftest/context.py b/meta/lib/oeqa/selftest/context.py index c7dd03ce378..8cc46283ed0 100644 --- a/meta/lib/oeqa/selftest/context.py +++ b/meta/lib/oeqa/selftest/context.py @@ -22,6 +22,17 @@ from oeqa.core.exception import OEQAPreRun, OEQATestNotFound from oeqa.utils.commands import runCmd, get_bb_vars, get_test_layer +OESELFTEST_METADATA=["run_all_tests", "run_tests", "skips", "machine", "select_tags", "exclude_tags"] + +def get_oeselftest_metadata(args): + result = {} + raw_args = vars(args) + for metadata in OESELFTEST_METADATA: + if metadata in raw_args: + result[metadata] = raw_args[metadata] + + return result + class NonConcurrentTestSuite(unittest.TestSuite): def __init__(self, suite, processes, setupfunc, removefunc): super().__init__([suite]) @@ -334,12 +345,14 @@ class OESelftestTestContextExecutor(OETestContextExecutor): import platform from oeqa.utils.metadata import metadata_from_bb metadata = metadata_from_bb() + oeselftest_metadata = get_oeselftest_metadata(args) configuration = {'TEST_TYPE': 'oeselftest', 'STARTTIME': args.test_start_time, 'MACHINE': self.tc.td["MACHINE"], 'HOST_DISTRO': oe.lsb.distro_identifier().replace(' ', '-'), 'HOST_NAME': metadata['hostname'], - 'LAYERS': metadata['layers']} + 'LAYERS': metadata['layers'], + 'OESELFTEST_METADATA':oeselftest_metadata} return configuration def get_result_id(self, configuration): From patchwork Fri Feb 24 16:45:51 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 20118 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 7B0FDC7EE2F for ; Fri, 24 Feb 2023 16:46:01 +0000 (UTC) Received: from relay11.mail.gandi.net (relay11.mail.gandi.net [217.70.178.231]) by mx.groups.io with SMTP id smtpd.web10.22896.1677257156543631533 for ; Fri, 24 Feb 2023 08:45:56 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=fAlOvI/H; spf=pass (domain: bootlin.com, ip: 217.70.178.231, mailfrom: alexis.lothore@bootlin.com) Received: (Authenticated sender: alexis.lothore@bootlin.com) by mail.gandi.net (Postfix) with ESMTPSA id 0EFB2100005; Fri, 24 Feb 2023 16:45:53 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1677257154; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=kkS0ToEK+wb2MZzobdybJxW7l75nGkeqns2KTP7XgpI=; b=fAlOvI/HvhCbRwinBz9wt4zYzym7UDsV7re+jeyTXg5cUKM3L8NIJeSR0wOJwgvoSktsn/ YTocizoJyMFUVbp/1IWRRqwzR1lE2e+p7wIEr6AyfoO/BfG4h15Npx7Y6uzwfhZp0VJcnq 6XKWx5StXz/4A7RiftoNJZ84AoapWr5ADTBrqX7oGg6n5xRVr/VKi3JNMS0TEcY/7oEU/y RuK9QfRUrDHKYrSTDXSr5HwLvlAMdbU8EVz/628o+hsuXnDL0Ub3LgkTvUj/6CN/Iy0b8R 9bkMMJwTCSuPR5Sn23UcyeQN4tpfpbwNIKpaO5abMMz34IzEDWv3pCFth43n1g== From: alexis.lothore@bootlin.com To: openembedded-core@lists.openembedded.org Cc: alexandre.belloni@bootlin.com, thomas.petazzoni@bootlin.com Subject: [PATCH v3 2/6] scripts/resulttool/regression: remove unused import Date: Fri, 24 Feb 2023 17:45:51 +0100 Message-Id: <20230224164555.67634-3-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.39.1 In-Reply-To: <20230224164555.67634-1-alexis.lothore@bootlin.com> References: <20230224164555.67634-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 24 Feb 2023 16:46:01 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/177701 From: Alexis Lothoré Signed-off-by: Alexis Lothoré --- scripts/lib/resulttool/regression.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py index 9f952951b3f..d0b0c318051 100644 --- a/scripts/lib/resulttool/regression.py +++ b/scripts/lib/resulttool/regression.py @@ -7,7 +7,6 @@ # import resulttool.resultutils as resultutils -import json from oeqa.utils.git import GitRepo import oeqa.utils.gitarchive as gitarchive From patchwork Fri Feb 24 16:45:52 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 20122 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 8C805C7EE32 for ; Fri, 24 Feb 2023 16:46:01 +0000 (UTC) Received: from relay11.mail.gandi.net (relay11.mail.gandi.net [217.70.178.231]) by mx.groups.io with SMTP id smtpd.web10.22895.1677257156185575893 for ; Fri, 24 Feb 2023 08:45:56 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=MRRA48d5; spf=pass (domain: bootlin.com, ip: 217.70.178.231, mailfrom: alexis.lothore@bootlin.com) Received: (Authenticated sender: alexis.lothore@bootlin.com) by mail.gandi.net (Postfix) with ESMTPSA id 8209010000A; Fri, 24 Feb 2023 16:45:54 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1677257154; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=gyYFjNF2fWM7E86SJekuEZr9LW9SvHkTJ7vErRQ4YeE=; b=MRRA48d57WsO+iJvkqCzFSsZrtPN5gXga7j4IZeHnQYZftzvjR6DN8mgvgRNEpHwGWvttH MABbPvwn10K++4Hn7GQM3yDBOEAahLwWhzSdXExcmFOs19rSVLcCqEQ3HfIakwwToyTHNu qf+m6lzA4uglAMC+kPbno2Nlwq5mlNjXuciPLAY7GsC9Por4CSAjT2S6w3YjpnKAtP1TnL kFFQrLJeLpEQc/IN1KMkHXl8M54PBYMeH4CcTunw3Q4o/FfCZxOu14wyMhMW1eEZlN29Dw 23sOZy0P/VzNesuh1ydiucvW+gyaWAlUOBbH4Bg16SwbwhrY3RNwjBOnflrxSw== From: alexis.lothore@bootlin.com To: openembedded-core@lists.openembedded.org Cc: alexandre.belloni@bootlin.com, thomas.petazzoni@bootlin.com Subject: [PATCH v3 3/6] scripts/resulttool/regression: add metadata filtering for oeselftest Date: Fri, 24 Feb 2023 17:45:52 +0100 Message-Id: <20230224164555.67634-4-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.39.1 In-Reply-To: <20230224164555.67634-1-alexis.lothore@bootlin.com> References: <20230224164555.67634-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 24 Feb 2023 16:46:01 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/177700 From: Alexis Lothoré When generating regression reports, many false positive can be observed since some tests results are compared while the corresponding tests sets are not the same, as it can be seen for example for oeselftest tests (oeselftest is run multiple time but with different parameters, resulting in different tests sets) Add a filtering mechanism in resulttool regression module to enable a better matching between tests. The METADATA_MATCH_TABLE defines that when the TEST_TYPE is "oeselftest", then resulttool should filter pairs based on OESELFTEST_METADATA appended to test configuration. If metadata is absent from test results, in order to keep compatibility with older results, add a "guessing" mechanism to generate the missing OESELFTEST_METADATA. The guessed data is tightly coupled to the autobuilder configuration, where all oe-selftest executions are described Signed-off-by: Alexis Lothoré --- scripts/lib/resulttool/regression.py | 162 +++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py index d0b0c318051..1b0c8335a39 100644 --- a/scripts/lib/resulttool/regression.py +++ b/scripts/lib/resulttool/regression.py @@ -11,6 +11,164 @@ import resulttool.resultutils as resultutils from oeqa.utils.git import GitRepo import oeqa.utils.gitarchive as gitarchive +METADATA_MATCH_TABLE = { + "oeselftest": "OESELFTEST_METADATA" +} + +OESELFTEST_METADATA_GUESS_TABLE={ + "trigger-build-posttrigger": { + "run_all_tests": False, + "run_tests":["buildoptions.SourceMirroring.test_yocto_source_mirror"], + "skips": None, + "machine": None, + "select_tags":None, + "exclude_tags": None + }, + "reproducible": { + "run_all_tests": False, + "run_tests":["reproducible"], + "skips": None, + "machine": None, + "select_tags":None, + "exclude_tags": None + }, + "arch-qemu-quick": { + "run_all_tests": True, + "run_tests":None, + "skips": None, + "machine": None, + "select_tags":["machine"], + "exclude_tags": None + }, + "arch-qemu-full-x86-or-x86_64": { + "run_all_tests": True, + "run_tests":None, + "skips": None, + "machine": None, + "select_tags":["machine", "toolchain-system"], + "exclude_tags": None + }, + "arch-qemu-full-others": { + "run_all_tests": True, + "run_tests":None, + "skips": None, + "machine": None, + "select_tags":["machine", "toolchain-user"], + "exclude_tags": None + }, + "selftest": { + "run_all_tests": True, + "run_tests":None, + "skips": ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror", "reproducible"], + "machine": None, + "select_tags":None, + "exclude_tags": ["machine", "toolchain-system", "toolchain-user"] + }, + "bringup": { + "run_all_tests": True, + "run_tests":None, + "skips": ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror"], + "machine": None, + "select_tags":None, + "exclude_tags": ["machine", "toolchain-system", "toolchain-user"] + } +} + +def test_has_at_least_one_matching_tag(test, tag_list): + return "oetags" in test and any(oetag in tag_list for oetag in test["oetags"]) + +def all_tests_have_at_least_one_matching_tag(results, tag_list): + return all(test_has_at_least_one_matching_tag(test_result, tag_list) or test_name.startswith("ptestresult") for (test_name, test_result) in results.items()) + +def any_test_have_any_matching_tag(results, tag_list): + return any(test_has_at_least_one_matching_tag(test, tag_list) for test in results.values()) + +def have_skipped_test(result, test_prefix): + return all( result[test]['status'] == "SKIPPED" for test in result if test.startswith(test_prefix)) + +def have_all_tests_skipped(result, test_prefixes_list): + return all(have_skipped_test(result, test_prefix) for test_prefix in test_prefixes_list) + +def guess_oeselftest_metadata(results): + """ + When an oeselftest test result is lacking OESELFTEST_METADATA, we can try to guess it based on results content. + Check results for specific values (absence/presence of oetags, number and name of executed tests...), + and if it matches one of known configuration from autobuilder configuration, apply guessed OSELFTEST_METADATA + to it to allow proper test filtering. + This guessing process is tightly coupled to config.json in autobuilder. It should trigger less and less, + as new tests will have OESELFTEST_METADATA properly appended at test reporting time + """ + + if len(results) == 1 and "buildoptions.SourceMirroring.test_yocto_source_mirror" in results: + return OESELFTEST_METADATA_GUESS_TABLE['trigger-build-posttrigger'] + elif all(result.startswith("reproducible") for result in results): + return OESELFTEST_METADATA_GUESS_TABLE['reproducible'] + elif all_tests_have_at_least_one_matching_tag(results, ["machine"]): + return OESELFTEST_METADATA_GUESS_TABLE['arch-qemu-quick'] + elif all_tests_have_at_least_one_matching_tag(results, ["machine", "toolchain-system"]): + return OESELFTEST_METADATA_GUESS_TABLE['arch-qemu-full-x86-or-x86_64'] + elif all_tests_have_at_least_one_matching_tag(results, ["machine", "toolchain-user"]): + return OESELFTEST_METADATA_GUESS_TABLE['arch-qemu-full-others'] + elif not any_test_have_any_matching_tag(results, ["machine", "toolchain-user", "toolchain-system"]): + if have_all_tests_skipped(results, ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror", "reproducible"]): + return OESELFTEST_METADATA_GUESS_TABLE['selftest'] + elif have_all_tests_skipped(results, ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror"]): + return OESELFTEST_METADATA_GUESS_TABLE['bringup'] + + return None + + +def metadata_matches(base_configuration, target_configuration): + """ + For passed base and target, check test type. If test type matches one of + properties described in METADATA_MATCH_TABLE, compare metadata if it is + present in base. Return true if metadata matches, or if base lacks some + data (either TEST_TYPE or the corresponding metadata) + """ + test_type = base_configuration.get('TEST_TYPE') + if test_type not in METADATA_MATCH_TABLE: + return True + + metadata_key = METADATA_MATCH_TABLE.get(test_type) + if target_configuration.get(metadata_key) != base_configuration.get(metadata_key): + return False + + return True + + +def machine_matches(base_configuration, target_configuration): + return base_configuration.get('MACHINE') == target_configuration.get('MACHINE') + + +def can_be_compared(logger, base, target): + """ + Some tests are not relevant to be compared, for example some oeselftest + run with different tests sets or parameters. Return true if tests can be + compared + """ + base_configuration = base['configuration'] + target_configuration = target['configuration'] + + # Older test results lack proper OESELFTEST_METADATA: if not present, try to guess it based on tests results. + if base_configuration.get('TEST_TYPE') == 'oeselftest' and 'OESELFTEST_METADATA' not in base_configuration: + guess = guess_oeselftest_metadata(base['result']) + if guess is None: + logger.error(f"ERROR: did not manage to guess oeselftest metadata for {base_configuration['STARTTIME']}") + else: + logger.debug(f"Enriching {base_configuration['STARTTIME']} with {guess}") + base_configuration['OESELFTEST_METADATA'] = guess + if target_configuration.get('TEST_TYPE') == 'oeselftest' and 'OESELFTEST_METADATA' not in target_configuration: + guess = guess_oeselftest_metadata(target['result']) + if guess is None: + logger.error(f"ERROR: did not manage to guess oeselftest metadata for {target_configuration['STARTTIME']}") + else: + logger.debug(f"Enriching {target_configuration['STARTTIME']} with {guess}") + target_configuration['OESELFTEST_METADATA'] = guess + + return metadata_matches(base_configuration, target_configuration) \ + and machine_matches(base_configuration, target_configuration) + + def compare_result(logger, base_name, target_name, base_result, target_result): base_result = base_result.get('result') target_result = target_result.get('result') @@ -61,6 +219,8 @@ def regression_common(args, logger, base_results, target_results): # removing any pairs which match for c in base.copy(): for b in target.copy(): + if not can_be_compared(logger, base_results[a][c], target_results[a][b]): + continue res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b]) if not res: matches.append(resstr) @@ -70,6 +230,8 @@ def regression_common(args, logger, base_results, target_results): # Should only now see regressions, we may not be able to match multiple pairs directly for c in base: for b in target: + if not can_be_compared(logger, base_results[a][c], target_results[a][b]): + continue res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b]) if res: regressions.append(resstr) From patchwork Fri Feb 24 16:45:53 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 20121 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 883A9C7EE2E for ; Fri, 24 Feb 2023 16:46:01 +0000 (UTC) Received: from relay11.mail.gandi.net (relay11.mail.gandi.net [217.70.178.231]) by mx.groups.io with SMTP id smtpd.web10.22897.1677257156704770231 for ; Fri, 24 Feb 2023 08:45:57 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=RZ0n1+vt; spf=pass (domain: bootlin.com, ip: 217.70.178.231, mailfrom: alexis.lothore@bootlin.com) Received: (Authenticated sender: alexis.lothore@bootlin.com) by mail.gandi.net (Postfix) with ESMTPSA id 034CE10000B; Fri, 24 Feb 2023 16:45:54 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1677257155; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=Z1WghUF7crZbHE9wA60204eEHLs7uZX5P4MhxYWnPHU=; b=RZ0n1+vt4fGSOBlI3m3OHQLWqS8H4h6Rna/hREjlYweiWdb5eaMDAxwmVdc/1dKa0BIT0Q RFMq2yBoUkqRQXCW5ekjJcMXS8VaKcELCwjSL2g6jTEmk87ZG6RsIGkgp7ED/8+kMXZZ6X rXsWwZvHGfDiwfVT/tL2QIe6vszD9js5Kuk+c77V9U1piDieJigfI4KVdlZzQKFUvEKcgo Kk6A9AlnyNlBLT293I/oj3aT7pG5MfziZl/YJT4Vz+344Njv/1ZRoqvFJbVKh4Xx+8qi5C 7Jk/GZvwZqqLfVjtfwlrr258rf42LS7/lcvbpHCjOMyUcK3VJAFCh3zPzWz5pw== From: alexis.lothore@bootlin.com To: openembedded-core@lists.openembedded.org Cc: alexandre.belloni@bootlin.com, thomas.petazzoni@bootlin.com Subject: [PATCH v3 4/6] oeqa/selftest/resulttool: add test for metadata filtering on regression Date: Fri, 24 Feb 2023 17:45:53 +0100 Message-Id: <20230224164555.67634-5-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.39.1 In-Reply-To: <20230224164555.67634-1-alexis.lothore@bootlin.com> References: <20230224164555.67634-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 24 Feb 2023 16:46:01 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/177702 From: Alexis Lothoré Introduce new tests for the metadata-based filtering added for oeselftest results Signed-off-by: Alexis Lothoré --- .../oeqa/selftest/cases/resulttooltests.py | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py b/meta/lib/oeqa/selftest/cases/resulttooltests.py index efdfd98af3c..75d406c122d 100644 --- a/meta/lib/oeqa/selftest/cases/resulttooltests.py +++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py @@ -98,3 +98,140 @@ class ResultToolTests(OESelftestTestCase): resultutils.append_resultsdata(results, ResultToolTests.target_results_data, configmap=resultutils.flatten_map) self.assertEqual(len(results[''].keys()), 5, msg="Flattened results not correct %s" % str(results)) + def test_results_without_metadata_can_be_compared(self): + base_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86"} + target_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86"} + self.assertTrue(regression.can_be_compared(base_configuration, target_configuration), + msg="incorrect metadata filtering, tests without metadata should be compared") + + def test_target_result_with_missing_metadata_can_not_be_compared(self): + base_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["toolchain-user", "toolchain-system"], + "exclude_tags": None}} + target_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86"} + self.assertFalse(regression.can_be_compared(base_configuration, target_configuration), + msg="incorrect metadata filtering, tests should not be compared") + + def test_results_with_matching_metadata_can_be_compared(self): + base_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["toolchain-user", "toolchain-system"], + "exclude_tags": None}} + target_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["toolchain-user", "toolchain-system"], + "exclude_tags": None}} + self.assertTrue(regression.can_be_compared(base_configuration, target_configuration), + msg="incorrect metadata filtering, tests with matching metadata should be compared") + + def test_results_with_mismatching_metadata_can_not_be_compared(self): + base_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["toolchain-user", "toolchain-system"], + "exclude_tags": None}} + target_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["machine"], + "exclude_tags": None}} + self.assertFalse(regression.can_be_compared(base_configuration, target_configuration), + msg="incorrect metadata filtering, tests with mismatching metadata should not be compared") + + def test_metadata_matching_is_only_checked_for_relevant_test_type(self): + base_configuration = {"TEST_TYPE": "runtime", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["toolchain-user", "toolchain-system"], + "exclude_tags": None}} + target_configuration = {"TEST_TYPE": "runtime", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["machine"], + "exclude_tags": None}} + self.assertTrue(regression.can_be_compared(base_configuration, target_configuration), + msg="incorrect metadata filtering, %s tests should be compared" % base_configuration['TEST_TYPE']) + + def test_machine_matches(self): + base_configuration = {"TEST_TYPE": "runtime", + "MACHINE": "qemux86"} + target_configuration = {"TEST_TYPE": "runtime", + "MACHINE": "qemux86"} + self.assertTrue(regression.can_be_compared(base_configuration, target_configuration), + msg="incorrect machine filtering, identical machine tests should be compared") + + def test_machine_mismatches(self): + base_configuration = {"TEST_TYPE": "runtime", + "MACHINE": "qemux86"} + target_configuration = {"TEST_TYPE": "runtime", + "MACHINE": "qemux86_64"} + self.assertFalse(regression.can_be_compared(base_configuration, target_configuration), + msg="incorrect machine filtering, mismatching machine tests should not be compared") From patchwork Fri Feb 24 16:45:54 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 20119 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 89031C7EE30 for ; Fri, 24 Feb 2023 16:46:01 +0000 (UTC) Received: from relay11.mail.gandi.net (relay11.mail.gandi.net [217.70.178.231]) by mx.groups.io with SMTP id smtpd.web10.22898.1677257157158790165 for ; Fri, 24 Feb 2023 08:45:57 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=k7GDG95+; spf=pass (domain: bootlin.com, ip: 217.70.178.231, mailfrom: alexis.lothore@bootlin.com) Received: (Authenticated sender: alexis.lothore@bootlin.com) by mail.gandi.net (Postfix) with ESMTPSA id 7223410000E; Fri, 24 Feb 2023 16:45:55 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1677257155; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=TezxmH1GgEU1xPlN5nCiDf3Flcs8r8bBSfLbH/KfJsg=; b=k7GDG95+9cuSJDlTS+QyuReM3YYMvSgfVaewBojoBRnWOHQ9muzIdHxUbDuAns4RhPK4IW evaggdtbMVRA3/d/SBzHovOZ7q9eBTHHWZoLTU3qzIJEDfFVgGsnxca6gKFBuynr7u81EJ sogTy7zb2Ta/1iTCf8VBNBczbAj3cVEWYF5kHP2LJ6hNAz5E0C9BoJQnA4ZXHOA166wGpH xaXckB00bHG31ZYlIesAA1XQV4szfWRKPwdFKvhnWzWZ8T42umxXSrn0AfoCB2XWATLgnd IGqZdzf0WMXVjV4dAqhtzw4TQAAh1+/Ul9BAjSAoJoOfsne2h9aEyNfHer6fSw== From: alexis.lothore@bootlin.com To: openembedded-core@lists.openembedded.org Cc: alexandre.belloni@bootlin.com, thomas.petazzoni@bootlin.com Subject: [PATCH v3 5/6] scripts: add new helper for regression report generation Date: Fri, 24 Feb 2023 17:45:54 +0100 Message-Id: <20230224164555.67634-6-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.39.1 In-Reply-To: <20230224164555.67634-1-alexis.lothore@bootlin.com> References: <20230224164555.67634-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 24 Feb 2023 16:46:01 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/177703 From: Alexis Lothoré Add yocto-testresults-query script. This is a thin wrapper over resulttool which is able to translate tags or branch name to specific revisions, and then to work with those "guessed" revisions with resulttool Signed-off-by: Alexis Lothoré --- scripts/yocto_testresults_query.py | 106 +++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100755 scripts/yocto_testresults_query.py diff --git a/scripts/yocto_testresults_query.py b/scripts/yocto_testresults_query.py new file mode 100755 index 00000000000..fee3855c6d8 --- /dev/null +++ b/scripts/yocto_testresults_query.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 + +# Yocto Project test results management tool +# This script is an thin layer over resulttool to manage tes results and regression reports. +# Its main feature is to translate tags or branch names to revisions SHA1, and then to run resulttool +# with those computed revisions +# +# Copyright (C) 2023 OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +import sys +import os +import argparse +import subprocess +import tempfile +import lib.scriptutils as scriptutils + +script_path = os.path.dirname(os.path.realpath(__file__)) +poky_path = os.path.abspath(os.path.join(script_path, "..")) +resulttool = os.path.abspath(os.path.join(script_path, "resulttool")) +logger = scriptutils.logger_create(sys.argv[0]) +testresults_default_url="git://git.yoctoproject.org/yocto-testresults" + +def create_workdir(): + workdir = tempfile.mkdtemp(prefix='yocto-testresults-query.') + logger.info(f"Shallow-cloning testresults in {workdir}") + subprocess.check_call(["git", "clone", testresults_default_url, workdir, "--depth", "1"]) + return workdir + +def get_sha1(pokydir, revision): + rev = subprocess.check_output(["git", "rev-list", "-n", "1", revision], cwd=pokydir).decode('utf-8').strip() + logger.info(f"SHA-1 revision for {revision} in {pokydir} is {rev}") + return rev + +def fetch_testresults(workdir, sha1): + logger.info(f"Fetching test results for {sha1} in {workdir}") + rawtags = subprocess.check_output(["git", "ls-remote", "--refs", "--tags", "origin", f"*{sha1}*"], cwd=workdir).decode('utf-8').strip() + if not rawtags: + raise Exception(f"No reference found for commit {sha1} in {workdir}") + for rev in [rawtag.split()[1] for rawtag in rawtags.splitlines()]: + logger.info(f"Fetching matching revisions: {rev}") + subprocess.check_call(["git", "fetch", "--depth", "1", "origin", f"{rev}:{rev}"], cwd=workdir) + +def compute_regression_report(workdir, baserevision, targetrevision): + logger.info(f"Running resulttool regression between SHA1 {baserevision} and {targetrevision}") + report = subprocess.check_output([resulttool, "regression-git", "--commit", baserevision, "--commit2", targetrevision, workdir]).decode("utf-8") + return report + +def print_report_with_header(report, baseversion, baserevision, targetversion, targetrevision): + print("========================== Regression report ==============================") + print(f'{"=> Target:": <16}{targetversion: <16}({targetrevision})') + print(f'{"=> Base:": <16}{baseversion: <16}({baserevision})') + print("===========================================================================\n") + print(report, end='') + +def regression(args): + logger.info(f"Compute regression report between {args.base} and {args.target}") + if args.testresultsdir: + workdir = args.testresultsdir + else: + workdir = create_workdir() + + try: + baserevision = get_sha1(poky_path, args.base) + targetrevision = get_sha1(poky_path, args.target) + fetch_testresults(workdir, baserevision) + fetch_testresults(workdir, targetrevision) + report = compute_regression_report(workdir, baserevision, targetrevision) + print_report_with_header(report, args.base, baserevision, args.target, targetrevision) + finally: + if not args.testresultsdir: + subprocess.check_call(["rm", "-rf", workdir]) + +def main(): + parser = argparse.ArgumentParser(description="Yocto Project test results helper") + subparsers = parser.add_subparsers( + help="Supported commands for test results helper", + required=True) + parser_regression_report = subparsers.add_parser( + "regression-report", + help="Generate regression report between two fixed revisions. Revisions can be branch name or tag") + parser_regression_report.add_argument( + 'base', + help="Revision or tag against which to compare results (i.e: the older)") + parser_regression_report.add_argument( + 'target', + help="Revision or tag to compare against the base (i.e: the newer)") + parser_regression_report.add_argument( + '-t', + '--testresultsdir', + help=f"An existing test results directory. {sys.argv[0]} will automatically clone it and use default branch if not provided") + parser_regression_report.set_defaults(func=regression) + + args = parser.parse_args() + args.func(args) + +if __name__ == '__main__': + try: + ret = main() + except Exception: + ret = 1 + import traceback + traceback.print_exc() + sys.exit(ret) \ No newline at end of file From patchwork Fri Feb 24 16:45:55 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 20117 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 7871CC7EE23 for ; Fri, 24 Feb 2023 16:46:01 +0000 (UTC) Received: from relay11.mail.gandi.net (relay11.mail.gandi.net [217.70.178.231]) by mx.groups.io with SMTP id smtpd.web11.22871.1677257157557812787 for ; Fri, 24 Feb 2023 08:45:57 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=ooNH8LnW; spf=pass (domain: bootlin.com, ip: 217.70.178.231, mailfrom: alexis.lothore@bootlin.com) Received: (Authenticated sender: alexis.lothore@bootlin.com) by mail.gandi.net (Postfix) with ESMTPSA id E90D9100008; Fri, 24 Feb 2023 16:45:55 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1677257156; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=8b/VdszHhGr2cAhz112sW2HJEtRicSFp0i4Lzvd2UpU=; b=ooNH8LnWbehs3re4m3DXCdMXfl/Utla9El9JyjeKz6QSYiKUzcWBS5BlIPcOU8naHHAvmV t4H+1K0/KL6QRO46wez8z4Jb0KrCaiGXt76vXtI1sYz8ciMB5fnUFFQ4h2sjB2+0Rt3Fr2 qekYpX4uxYNlVlFPdFutcBRzGMJ/OKqLVzGdJqx0c+mF0D469hDH+cYSt1eH4Nbx9hMWs6 eS1CQ2Ld8TjXMyxu/LUcK91va/TriKFcv3QzN7AusAGd09vmZfZVed67WZj7xoFMsGT3Nm GbE2W4LjmW3KQPlVFf+MLZkUpckxwEtCvxB3/AX6d/TazBFL8IAFU6IcUk9SHw== From: alexis.lothore@bootlin.com To: openembedded-core@lists.openembedded.org Cc: alexandre.belloni@bootlin.com, thomas.petazzoni@bootlin.com Subject: [PATCH v3 6/6] oeqa/selftest: add test for yocto_testresults_query.py Date: Fri, 24 Feb 2023 17:45:55 +0100 Message-Id: <20230224164555.67634-7-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.39.1 In-Reply-To: <20230224164555.67634-1-alexis.lothore@bootlin.com> References: <20230224164555.67634-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 24 Feb 2023 16:46:01 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/177704 From: Alexis Lothoré Add some tests for new yocto_testresults_query.py helper. First test is taken from yocto-autobuilder-helper feature which has moved in yocto_testresults_query Signed-off-by: Alexis Lothoré --- .../cases/yoctotestresultsquerytests.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 meta/lib/oeqa/selftest/cases/yoctotestresultsquerytests.py diff --git a/meta/lib/oeqa/selftest/cases/yoctotestresultsquerytests.py b/meta/lib/oeqa/selftest/cases/yoctotestresultsquerytests.py new file mode 100644 index 00000000000..312edb64319 --- /dev/null +++ b/meta/lib/oeqa/selftest/cases/yoctotestresultsquerytests.py @@ -0,0 +1,39 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +import os +import sys +import subprocess +import shutil +from oeqa.selftest.case import OESelftestTestCase +from yocto_testresults_query import get_sha1, create_workdir +basepath = os.path.abspath(os.path.dirname(__file__) + '/../../../../../') +lib_path = basepath + '/scripts/lib' +sys.path = sys.path + [lib_path] + + +class TestResultsQueryTests(OESelftestTestCase): + def test_get_sha1(self): + test_data_get_sha1 = [ + {"input": "yocto-4.0", "expected": "00cfdde791a0176c134f31e5a09eff725e75b905"}, + {"input": "4.1_M1", "expected": "95066dde6861ee08fdb505ab3e0422156cc24fae"}, + ] + for data in test_data_get_sha1: + test_name = data["input"] + with self.subTest(f"Test SHA1 from {test_name}"): + self.assertEqual( + get_sha1(basepath, data["input"]), data["expected"]) + + def test_create_workdir(self): + workdir = create_workdir() + try: + url = subprocess.check_output( + ["git", "-C", workdir, "remote", "get-url", "origin"]).strip().decode("utf-8") + except: + shutil.rmtree(workdir, ignore_errors=True) + self.fail(f"Can not execute git commands in {workdir}") + shutil.rmtree(workdir) + self.assertEqual(url, "git://git.yoctoproject.org/yocto-testresults")