@@ -12,6 +12,36 @@ import json
from oeqa.utils.git import GitRepo
import oeqa.utils.gitarchive as gitarchive
+METADATA_MATCH_TABLE={
+ "oeselftest": "OESELFTEST_METADATA"
+}
+
+
+def metadata_matches(base_configuration, target_configuration):
+ """
+ For passed base and target, check test type. If test type matches one of
+ properties described in METADATA_MATCH_TABLE, compare metadata if it is
+ present in base. Return true if metadata matches, or if base lacks some
+ data (either TEST_TYPE or the corresponding metadata)
+ """
+ test_type=base_configuration.get('TEST_TYPE')
+ metadata_key=METADATA_MATCH_TABLE.get(test_type)
+ if metadata_key not in base_configuration:
+ return True
+
+ if target_configuration.get(metadata_key) != base_configuration[metadata_key]:
+ return False
+
+ return True
+
+def can_be_compared(base_configuration, target_configuration):
+ """
+ Some tests are not relevant to be compared, for example some oeselftest
+ run with different tests sets or parameters. Return true if tests can be
+ compared
+ """
+ return metadata_matches(base_configuration, target_configuration)
+
def compare_result(logger, base_name, target_name, base_result, target_result):
base_result = base_result.get('result')
target_result = target_result.get('result')
@@ -62,6 +92,8 @@ def regression_common(args, logger, base_results, target_results):
# removing any pairs which match
for c in base.copy():
for b in target.copy():
+ if not can_be_compared(base_results[a][c]['configuration'], target_results[a][b]['configuration']):
+ continue
res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b])
if not res:
matches.append(resstr)
@@ -71,6 +103,8 @@ def regression_common(args, logger, base_results, target_results):
# Should only now see regressions, we may not be able to match multiple pairs directly
for c in base:
for b in target:
+ if not can_be_compared(base_results[a][c]['configuration'], target_results[a][b]['configuration']):
+ continue
res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b])
if res:
regressions.append(resstr)