diff mbox series

[4/4] oeqa/selftest/resulttool: add test for metadata filtering on regression

Message ID 20230214135214.42413-5-alexis.lothore@bootlin.com
State Accepted, archived
Commit b84302ef56f2516742a496aef43b89d4c3decd37
Headers show
Series scripts/resulttool/regression: add metadata filtering | expand

Commit Message

Alexis Lothoré Feb. 14, 2023, 1:52 p.m. UTC 
Introduce new tests for the metadata-based filtering added for oeselftest
results

Signed-off-by: Alexis Lothoré <alexis.lothore@bootlin.com>
---
 .../oeqa/selftest/cases/resulttooltests.py    | 121 ++++++++++++++++++
 1 file changed, 121 insertions(+)
diff mbox series

Patch

diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py b/meta/lib/oeqa/selftest/cases/resulttooltests.py
index efdfd98af3..e93796e145 100644
--- a/meta/lib/oeqa/selftest/cases/resulttooltests.py
+++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py
@@ -98,3 +98,124 @@  class ResultToolTests(OESelftestTestCase):
         resultutils.append_resultsdata(results, ResultToolTests.target_results_data, configmap=resultutils.flatten_map)
         self.assertEqual(len(results[''].keys()), 5, msg="Flattened results not correct %s" % str(results))
 
+    def test_results_without_metadata_can_be_compared(self):
+        base_configuration = {"TEST_TYPE": "oeselftest",
+                              "TESTSERIES": "series1",
+                              "IMAGE_BASENAME": "image",
+                              "IMAGE_PKGTYPE": "ipk",
+                              "DISTRO": "mydistro",
+                              "MACHINE": "qemux86"}
+        target_configuration = {"TEST_TYPE": "oeselftest",
+                                "TESTSERIES": "series1",
+                                "IMAGE_BASENAME": "image",
+                                "IMAGE_PKGTYPE": "ipk",
+                                "DISTRO": "mydistro",
+                                "MACHINE": "qemux86"}
+        self.assertTrue(regression.can_be_compared(base_configuration, target_configuration),
+                        msg="incorrect metadata filtering, tests without metadata should be compared")
+
+    def test_target_result_with_missing_metadata_can_not_be_compared(self):
+        base_configuration = {"TEST_TYPE": "oeselftest",
+                              "TESTSERIES": "series1",
+                              "IMAGE_BASENAME": "image",
+                              "IMAGE_PKGTYPE": "ipk",
+                              "DISTRO": "mydistro",
+                              "MACHINE": "qemux86",
+                              "OESELFTEST_METADATA": {"run_all_tests": True,
+                                                      "run_tests": None,
+                                                      "skips": None,
+                                                      "machine": None,
+                                                      "select_tags": ["toolchain-user", "toolchain-system"],
+                                                      "exclude_tags": None}}
+        target_configuration = {"TEST_TYPE": "oeselftest",
+                                "TESTSERIES": "series1",
+                                "IMAGE_BASENAME": "image",
+                                "IMAGE_PKGTYPE": "ipk",
+                                "DISTRO": "mydistro",
+                                "MACHINE": "qemux86"}
+        self.assertFalse(regression.can_be_compared(base_configuration, target_configuration),
+                         msg="incorrect metadata filtering, tests should not be compared")
+
+    def test_results_with_matching_metadata_can_be_compared(self):
+        base_configuration = {"TEST_TYPE": "oeselftest",
+                              "TESTSERIES": "series1",
+                              "IMAGE_BASENAME": "image",
+                              "IMAGE_PKGTYPE": "ipk",
+                              "DISTRO": "mydistro",
+                              "MACHINE": "qemux86",
+                              "OESELFTEST_METADATA": {"run_all_tests": True,
+                                                      "run_tests": None,
+                                                      "skips": None,
+                                                      "machine": None,
+                                                      "select_tags": ["toolchain-user", "toolchain-system"],
+                                                      "exclude_tags": None}}
+        target_configuration = {"TEST_TYPE": "oeselftest",
+                                "TESTSERIES": "series1",
+                                "IMAGE_BASENAME": "image",
+                                "IMAGE_PKGTYPE": "ipk",
+                                "DISTRO": "mydistro",
+                                "MACHINE": "qemux86",
+                                "OESELFTEST_METADATA": {"run_all_tests": True,
+                                                        "run_tests": None,
+                                                        "skips": None,
+                                                        "machine": None,
+                                                        "select_tags": ["toolchain-user", "toolchain-system"],
+                                                        "exclude_tags": None}}
+        self.assertTrue(regression.can_be_compared(base_configuration, target_configuration),
+                        msg="incorrect metadata filtering, tests with matching metadata should be compared")
+
+    def test_results_with_mismatching_metadata_can_not_be_compared(self):
+        base_configuration = {"TEST_TYPE": "oeselftest",
+                              "TESTSERIES": "series1",
+                              "IMAGE_BASENAME": "image",
+                              "IMAGE_PKGTYPE": "ipk",
+                              "DISTRO": "mydistro",
+                              "MACHINE": "qemux86",
+                              "OESELFTEST_METADATA": {"run_all_tests": True,
+                                                      "run_tests": None,
+                                                      "skips": None,
+                                                      "machine": None,
+                                                      "select_tags": ["toolchain-user", "toolchain-system"],
+                                                      "exclude_tags": None}}
+        target_configuration = {"TEST_TYPE": "oeselftest",
+                                "TESTSERIES": "series1",
+                                "IMAGE_BASENAME": "image",
+                                "IMAGE_PKGTYPE": "ipk",
+                                "DISTRO": "mydistro",
+                                "MACHINE": "qemux86",
+                                "OESELFTEST_METADATA": {"run_all_tests": True,
+                                                        "run_tests": None,
+                                                        "skips": None,
+                                                        "machine": None,
+                                                        "select_tags": ["machine"],
+                                                        "exclude_tags": None}}
+        self.assertFalse(regression.can_be_compared(base_configuration, target_configuration),
+                         msg="incorrect metadata filtering, tests with mismatching metadata should not be compared")
+
+    def test_metadata_matching_is_only_checked_for_relevant_test_type(self):
+        base_configuration = {"TEST_TYPE": "runtime",
+                              "TESTSERIES": "series1",
+                              "IMAGE_BASENAME": "image",
+                              "IMAGE_PKGTYPE": "ipk",
+                              "DISTRO": "mydistro",
+                              "MACHINE": "qemux86",
+                              "OESELFTEST_METADATA": {"run_all_tests": True,
+                                                      "run_tests": None,
+                                                      "skips": None,
+                                                      "machine": None,
+                                                      "select_tags": ["toolchain-user", "toolchain-system"],
+                                                      "exclude_tags": None}}
+        target_configuration = {"TEST_TYPE": "runtime",
+                                "TESTSERIES": "series1",
+                                "IMAGE_BASENAME": "image",
+                                "IMAGE_PKGTYPE": "ipk",
+                                "DISTRO": "mydistro",
+                                "MACHINE": "qemux86",
+                                "OESELFTEST_METADATA": {"run_all_tests": True,
+                                                        "run_tests": None,
+                                                        "skips": None,
+                                                        "machine": None,
+                                                        "select_tags": ["machine"],
+                                                        "exclude_tags": None}}
+        self.assertTrue(regression.can_be_compared(base_configuration, target_configuration),
+                         msg="incorrect metadata filtering, %s tests should be compared" % base_configuration['TEST_TYPE'])