@@ -190,11 +190,20 @@ def compare_result(logger, base_name, target_name, base_result, target_result):
else:
logger.error('Failed to retrieved base test case status: %s' % k)
if result:
- resultstring = "Regression: %s\n %s\n" % (base_name, target_name)
- for k in sorted(result):
- resultstring += ' %s: %s -> %s\n' % (k, result[k]['base'], result[k]['target'])
+ new_pass_count = sum(test['target'] is not None and test['target'].startswith("PASS") for test in result.values())
+ # Print a regression report only if at least one test has a regression status (FAIL, SKIPPED, absent...)
+ if new_pass_count < len(result):
+ resultstring = "Regression: %s\n %s\n" % (base_name, target_name)
+ for k in sorted(result):
+ if not result[k]['target'] or not result[k]['target'].startswith("PASS"):
+ resultstring += ' %s: %s -> %s\n' % (k, result[k]['base'], result[k]['target'])
+ if new_pass_count > 0:
+ resultstring += f' Additionally, {new_pass_count} previously failing test(s) is/are now passing\n'
+ else:
+ resultstring = "Improvement: %s\n %s\n (+%d test(s) passing)" % (base_name, target_name, new_pass_count)
+ result = None
else:
- resultstring = "Match: %s\n %s" % (base_name, target_name)
+ resultstring = "Match: %s\n %s" % (base_name, target_name)
return result, resultstring
def get_results(logger, source):
@@ -269,9 +278,9 @@ def regression_common(args, logger, base_results, target_results):
else:
notfound.append("%s not found in target" % a)
print("\n".join(sorted(matches)))
+ print("\n")
print("\n".join(sorted(regressions)))
print("\n".join(sorted(notfound)))
-
return 0
def regression_git(args, logger):