diff mbox series

[2/6] oeqa/selftest/resulttool: fix ptest tests

Message ID 20230228181052.4191521-4-alexis.lothore@bootlin.com
State New
Headers show
Series keep reducing regression reports noise | expand

Commit Message

Alexis Lothoré Feb. 28, 2023, 6:10 p.m. UTC 
From: Alexis Lothoré <alexis.lothore@bootlin.com>

There are multiple issues with the tests for the ptest fixup mechanism
introduced in c45d58f003e8d8b323169ca9d479dc49c43a9974:
- the feature does not impact regression.can_be_compared but directly the
  comparison step in regression.regression_common
- the "status" field was wrong ("STATUS"), which prevents tests comparison
- there is a typo babeltrace2 ptest result name

Signed-off-by: Alexis Lothoré <alexis.lothore@bootlin.com>
---
 .../oeqa/selftest/cases/resulttooltests.py    | 80 ++++++++++---------
 1 file changed, 44 insertions(+), 36 deletions(-)
diff mbox series

Patch

diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py b/meta/lib/oeqa/selftest/cases/resulttooltests.py
index f059991c19c..8489ae8eb52 100644
--- a/meta/lib/oeqa/selftest/cases/resulttooltests.py
+++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py
@@ -305,39 +305,47 @@  class ResultToolTests(OESelftestTestCase):
                         msg="incorrect ltpresult filtering, matching ltpresult content should be compared")
 
     def test_can_match_non_static_ptest_names(self):
-        base_configuration = {"configuration": {
-            "TEST_TYPE": "runtime",
-            "MACHINE": "qemux86"
-        }, "result": {
-            "ptestresult.lttng-tools.foo_-_bar_-_moo": {
-                "STATUS": "PASSED"
-            },
-            "ptestresult.babeltrace.bar_-_moo_-_foo": {
-                "STATUS": "PASSED"
-            },
-            "ptestresult.babletrace2.moo_-_foo_-_bar": {
-                "STATUS": "PASSED"
-            },
-            "ptestresult.curl.test_0000__foo_out_of_bar": {
-                "STATUS": "PASSED"
-            }
-        }}
-        target_configuration = {"configuration": {
-            "TEST_TYPE": "runtime",
-            "MACHINE": "qemux86"
-        }, "result": {
-            "ptestresult.lttng-tools.xxx_-_yyy_-_zzz": {
-                "STATUS": "PASSED"
-            },
-            "ptestresult.babeltrace.yyy_-_zzz_-_xxx": {
-                "STATUS": "PASSED"
-            },
-            "ptestresult.babletrace2.zzz_-_xxx_-_yyy": {
-                "STATUS": "PASSED"
-            },
-            "ptestresult.curl.test_0000__xxx_out_of_yyy": {
-                "STATUS": "PASSED"
-            }
-            }}
-        self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration),
-                        msg="incorrect ptests filtering, tests shoould be compared if prefixes match")
+        base_configuration = {"a": {
+            "conf_X": {
+                "configuration": {
+                    "TEST_TYPE": "runtime",
+                    "MACHINE": "qemux86"
+                }, "result": {
+                    "ptestresult.lttng-tools.foo_-_bar_-_moo": {
+                        "status": "PASSED"
+                    },
+                    "ptestresult.babeltrace.bar_-_moo_-_foo": {
+                        "status": "PASSED"
+                    },
+                    "ptestresult.babeltrace2.moo_-_foo_-_bar": {
+                        "status": "PASSED"
+                    },
+                    "ptestresult.curl.test_0000__foo_out_of_bar": {
+                        "status": "PASSED"
+                    }
+                }}}}
+        target_configuration = {"a": {
+            "conf_Y": {
+                "configuration": {
+                    "TEST_TYPE": "runtime",
+                    "MACHINE": "qemux86"
+                }, "result": {
+                    "ptestresult.lttng-tools.foo_-_yyy_-_zzz": {
+                        "status": "PASSED"
+                    },
+                    "ptestresult.babeltrace.bar_-_zzz_-_xxx": {
+                        "status": "PASSED"
+                    },
+                    "ptestresult.babeltrace2.moo_-_xxx_-_yyy": {
+                        "status": "PASSED"
+                    },
+                    "ptestresult.curl.test_0000__xxx_out_of_yyy": {
+                        "status": "PASSED"
+                    }
+                }}}}
+        regression.fixup_ptest_names(base_configuration, self.logger)
+        regression.fixup_ptest_names(target_configuration, self.logger)
+        result, resultstring = regression.compare_result(
+            self.logger, "A", "B", base_configuration["a"]["conf_X"], target_configuration["a"]["conf_Y"])
+        self.assertDictEqual(
+            result, {}, msg=f"ptests should be compared: {resultstring}")