| Message ID | 20230224164555.67634-5-alexis.lothore@bootlin.com |
|---|---|
| State | Accepted, archived |
| Commit | b84302ef56f2516742a496aef43b89d4c3decd37 |
| Headers | show |
| Series | scripts/resulttool/regression: add metadata filtering | expand |
On Fri, 2023-02-24 at 17:45 +0100, Alexis Lothoré via lists.openembedded.org wrote: > From: Alexis Lothoré <alexis.lothore@bootlin.com> > > Introduce new tests for the metadata-based filtering added for oeselftest > results > > Signed-off-by: Alexis Lothoré <alexis.lothore@bootlin.com> > --- > .../oeqa/selftest/cases/resulttooltests.py | 137 ++++++++++++++++++ > 1 file changed, 137 insertions(+) > > diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py b/meta/lib/oeqa/selftest/cases/resulttooltests.py > index efdfd98af3c..75d406c122d 100644 > --- a/meta/lib/oeqa/selftest/cases/resulttooltests.py > +++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py > @@ -98,3 +98,140 @@ class ResultToolTests(OESelftestTestCase): > resultutils.append_resultsdata(results, ResultToolTests.target_results_data, configmap=resultutils.flatten_map) > self.assertEqual(len(results[''].keys()), 5, msg="Flattened results not correct %s" % str(results)) > > + def test_results_without_metadata_can_be_compared(self): > + base_configuration = {"TEST_TYPE": "oeselftest", > + "TESTSERIES": "series1", > + "IMAGE_BASENAME": "image", > + "IMAGE_PKGTYPE": "ipk", > + "DISTRO": "mydistro", > + "MACHINE": "qemux86"} > + target_configuration = {"TEST_TYPE": "oeselftest", > + "TESTSERIES": "series1", > + "IMAGE_BASENAME": "image", > + "IMAGE_PKGTYPE": "ipk", > + "DISTRO": "mydistro", > + "MACHINE": "qemux86"} > + self.assertTrue(regression.can_be_compared(base_configuration, target_configuration), > + msg="incorrect metadata filtering, tests without metadata should be compared") > + > + def test_target_result_with_missing_metadata_can_not_be_compared(self): > + base_configuration = {"TEST_TYPE": "oeselftest", > + "TESTSERIES": "series1", > + "IMAGE_BASENAME": "image", > + "IMAGE_PKGTYPE": "ipk", > + "DISTRO": "mydistro", > + "MACHINE": "qemux86", > + "OESELFTEST_METADATA": {"run_all_tests": True, > + "run_tests": None, > + "skips": None, > + "machine": None, > + "select_tags": ["toolchain-user", "toolchain-system"], > + "exclude_tags": None}} > + target_configuration = {"TEST_TYPE": "oeselftest", > + "TESTSERIES": "series1", > + "IMAGE_BASENAME": "image", > + "IMAGE_PKGTYPE": "ipk", > + "DISTRO": "mydistro", > + "MACHINE": "qemux86"} > + self.assertFalse(regression.can_be_compared(base_configuration, target_configuration), > + msg="incorrect metadata filtering, tests should not be compared") > + > + def test_results_with_matching_metadata_can_be_compared(self): > + base_configuration = {"TEST_TYPE": "oeselftest", > + "TESTSERIES": "series1", > + "IMAGE_BASENAME": "image", > + "IMAGE_PKGTYPE": "ipk", > + "DISTRO": "mydistro", > + "MACHINE": "qemux86", > + "OESELFTEST_METADATA": {"run_all_tests": True, > + "run_tests": None, > + "skips": None, > + "machine": None, > + "select_tags": ["toolchain-user", "toolchain-system"], > + "exclude_tags": None}} > + target_configuration = {"TEST_TYPE": "oeselftest", > + "TESTSERIES": "series1", > + "IMAGE_BASENAME": "image", > + "IMAGE_PKGTYPE": "ipk", > + "DISTRO": "mydistro", > + "MACHINE": "qemux86", > + "OESELFTEST_METADATA": {"run_all_tests": True, > + "run_tests": None, > + "skips": None, > + "machine": None, > + "select_tags": ["toolchain-user", "toolchain-system"], > + "exclude_tags": None}} > + self.assertTrue(regression.can_be_compared(base_configuration, target_configuration), > + msg="incorrect metadata filtering, tests with matching metadata should be compared") > + > + def test_results_with_mismatching_metadata_can_not_be_compared(self): > + base_configuration = {"TEST_TYPE": "oeselftest", > + "TESTSERIES": "series1", > + "IMAGE_BASENAME": "image", > + "IMAGE_PKGTYPE": "ipk", > + "DISTRO": "mydistro", > + "MACHINE": "qemux86", > + "OESELFTEST_METADATA": {"run_all_tests": True, > + "run_tests": None, > + "skips": None, > + "machine": None, > + "select_tags": ["toolchain-user", "toolchain-system"], > + "exclude_tags": None}} > + target_configuration = {"TEST_TYPE": "oeselftest", > + "TESTSERIES": "series1", > + "IMAGE_BASENAME": "image", > + "IMAGE_PKGTYPE": "ipk", > + "DISTRO": "mydistro", > + "MACHINE": "qemux86", > + "OESELFTEST_METADATA": {"run_all_tests": True, > + "run_tests": None, > + "skips": None, > + "machine": None, > + "select_tags": ["machine"], > + "exclude_tags": None}} > + self.assertFalse(regression.can_be_compared(base_configuration, target_configuration), > + msg="incorrect metadata filtering, tests with mismatching metadata should not be compared") > + > + def test_metadata_matching_is_only_checked_for_relevant_test_type(self): > + base_configuration = {"TEST_TYPE": "runtime", > + "TESTSERIES": "series1", > + "IMAGE_BASENAME": "image", > + "IMAGE_PKGTYPE": "ipk", > + "DISTRO": "mydistro", > + "MACHINE": "qemux86", > + "OESELFTEST_METADATA": {"run_all_tests": True, > + "run_tests": None, > + "skips": None, > + "machine": None, > + "select_tags": ["toolchain-user", "toolchain-system"], > + "exclude_tags": None}} > + target_configuration = {"TEST_TYPE": "runtime", > + "TESTSERIES": "series1", > + "IMAGE_BASENAME": "image", > + "IMAGE_PKGTYPE": "ipk", > + "DISTRO": "mydistro", > + "MACHINE": "qemux86", > + "OESELFTEST_METADATA": {"run_all_tests": True, > + "run_tests": None, > + "skips": None, > + "machine": None, > + "select_tags": ["machine"], > + "exclude_tags": None}} > + self.assertTrue(regression.can_be_compared(base_configuration, target_configuration), > + msg="incorrect metadata filtering, %s tests should be compared" % base_configuration['TEST_TYPE']) > + > + def test_machine_matches(self): > + base_configuration = {"TEST_TYPE": "runtime", > + "MACHINE": "qemux86"} > + target_configuration = {"TEST_TYPE": "runtime", > + "MACHINE": "qemux86"} > + self.assertTrue(regression.can_be_compared(base_configuration, target_configuration), > + msg="incorrect machine filtering, identical machine tests should be compared") > + > + def test_machine_mismatches(self): > + base_configuration = {"TEST_TYPE": "runtime", > + "MACHINE": "qemux86"} > + target_configuration = {"TEST_TYPE": "runtime", > + "MACHINE": "qemux86_64"} > + self.assertFalse(regression.can_be_compared(base_configuration, target_configuration), > + msg="incorrect machine filtering, mismatching machine tests should not be compared") I love the fact this has tests but they don't work: https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/4854 then with the obvious error fixed to add self.logger: https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/4858/steps/14/logs/stdio Cheers, Richard
diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py b/meta/lib/oeqa/selftest/cases/resulttooltests.py index efdfd98af3c..75d406c122d 100644 --- a/meta/lib/oeqa/selftest/cases/resulttooltests.py +++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py @@ -98,3 +98,140 @@ class ResultToolTests(OESelftestTestCase): resultutils.append_resultsdata(results, ResultToolTests.target_results_data, configmap=resultutils.flatten_map) self.assertEqual(len(results[''].keys()), 5, msg="Flattened results not correct %s" % str(results)) + def test_results_without_metadata_can_be_compared(self): + base_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86"} + target_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86"} + self.assertTrue(regression.can_be_compared(base_configuration, target_configuration), + msg="incorrect metadata filtering, tests without metadata should be compared") + + def test_target_result_with_missing_metadata_can_not_be_compared(self): + base_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["toolchain-user", "toolchain-system"], + "exclude_tags": None}} + target_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86"} + self.assertFalse(regression.can_be_compared(base_configuration, target_configuration), + msg="incorrect metadata filtering, tests should not be compared") + + def test_results_with_matching_metadata_can_be_compared(self): + base_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["toolchain-user", "toolchain-system"], + "exclude_tags": None}} + target_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["toolchain-user", "toolchain-system"], + "exclude_tags": None}} + self.assertTrue(regression.can_be_compared(base_configuration, target_configuration), + msg="incorrect metadata filtering, tests with matching metadata should be compared") + + def test_results_with_mismatching_metadata_can_not_be_compared(self): + base_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["toolchain-user", "toolchain-system"], + "exclude_tags": None}} + target_configuration = {"TEST_TYPE": "oeselftest", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["machine"], + "exclude_tags": None}} + self.assertFalse(regression.can_be_compared(base_configuration, target_configuration), + msg="incorrect metadata filtering, tests with mismatching metadata should not be compared") + + def test_metadata_matching_is_only_checked_for_relevant_test_type(self): + base_configuration = {"TEST_TYPE": "runtime", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["toolchain-user", "toolchain-system"], + "exclude_tags": None}} + target_configuration = {"TEST_TYPE": "runtime", + "TESTSERIES": "series1", + "IMAGE_BASENAME": "image", + "IMAGE_PKGTYPE": "ipk", + "DISTRO": "mydistro", + "MACHINE": "qemux86", + "OESELFTEST_METADATA": {"run_all_tests": True, + "run_tests": None, + "skips": None, + "machine": None, + "select_tags": ["machine"], + "exclude_tags": None}} + self.assertTrue(regression.can_be_compared(base_configuration, target_configuration), + msg="incorrect metadata filtering, %s tests should be compared" % base_configuration['TEST_TYPE']) + + def test_machine_matches(self): + base_configuration = {"TEST_TYPE": "runtime", + "MACHINE": "qemux86"} + target_configuration = {"TEST_TYPE": "runtime", + "MACHINE": "qemux86"} + self.assertTrue(regression.can_be_compared(base_configuration, target_configuration), + msg="incorrect machine filtering, identical machine tests should be compared") + + def test_machine_mismatches(self): + base_configuration = {"TEST_TYPE": "runtime", + "MACHINE": "qemux86"} + target_configuration = {"TEST_TYPE": "runtime", + "MACHINE": "qemux86_64"} + self.assertFalse(regression.can_be_compared(base_configuration, target_configuration), + msg="incorrect machine filtering, mismatching machine tests should not be compared")