From patchwork Sun Oct 22 17:49:35 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 32724 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 72167C25B42 for ; Sun, 22 Oct 2023 17:49:05 +0000 (UTC) Received: from relay4-d.mail.gandi.net (relay4-d.mail.gandi.net [217.70.183.196]) by mx.groups.io with SMTP id smtpd.web10.103460.1697996934731249052 for ; Sun, 22 Oct 2023 10:48:55 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=KLavJQr4; spf=pass (domain: bootlin.com, ip: 217.70.183.196, mailfrom: alexis.lothore@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id F17A1E0004; Sun, 22 Oct 2023 17:48:52 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1697996933; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=BQdKol0rV2WRIfYQSV58X1fgrx9/ULYzMBwSPZhE9hk=; b=KLavJQr4ErM1mpvK1Zs5/bckKowNOV0uQKswxUQ4Dw9cFipi+8OCv0j/dDh7kP4e+2c918 EY1PT3562s3RaO2JoGYQ/Idzgt4W+WIVIv61QHzL6T9o9qGeJU7as5ZtrlO/BtZQsrNtjE ZEQbWVUF3ns6dnBmPGwc1m/ssHT56Yu26dp/fhjMprQLvOWwzGP/RHiCFgMrHWR67tZp8f xLZxhRufrxO/ngAduvvS/gU9WT8tkOaM0Ga5SUc5WX7AM07PFNkeBdhO8vl+QkbM7s8iYu D7jkYkoJlCk0bmccmy18NfG+Suq6nxzV9Ibp9a1TPSnZV3j4vTbzrQ/el2W+wA== From: =?utf-8?q?Alexis_Lothor=C3=A9?= To: Cc: Thomas Petazzoni , Alexandre Belloni Subject: [OE-Core][PATCH v2 1/4] scripts/resulttool: limit the number of changes displayed per test Date: Sun, 22 Oct 2023 19:49:35 +0200 Message-ID: <20231022174938.7012-2-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.42.0 In-Reply-To: <20231022174938.7012-1-alexis.lothore@bootlin.com> References: <20231022174938.7012-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: alexis.lothore@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Sun, 22 Oct 2023 17:49:05 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/189608 From: Alexis Lothoré Most of the changes list generated in regression reports fall in one of the two following categories: - there is only a few (<10) changes listed and the info is valuable/relevant - the list is huge (> 100 ? 1000 ?) and basically tells us that the whole tests category suffers the same status (test missing, test failing, test skipped, etc) Prevent those huge, worthless lists by limiting the output for each test group: - current default limit is arbitrarily set to 50 - limit can still be overriden with a new "-l"/"--limit" flag, either with custom value, or with 0 to print the whole lists of changes - limit is applied per test family: currently it distinguishes only types of ptests, but it can be adapted to other kind of tests Signed-off-by: Alexis Lothoré --- Changes in v2: - set the display limit per test category instead of one general limit - add a "per group" count, additionally to the general regression count - print total regressions count and "per group" regressions count on top of detailed report instead of bottom - fix resulttool selftests --- scripts/lib/resulttool/regression.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py index 3d64b8f4af7c..d98504f4ced8 100644 --- a/scripts/lib/resulttool/regression.py +++ b/scripts/lib/resulttool/regression.py @@ -78,6 +78,8 @@ STATUS_STRINGS = { "None": "No matching test result" } +REGRESSIONS_DISPLAY_LIMIT=50 + def test_has_at_least_one_matching_tag(test, tag_list): return "oetags" in test and any(oetag in tag_list for oetag in test["oetags"]) @@ -181,11 +183,15 @@ def get_status_str(raw_status): raw_status_lower = raw_status.lower() if raw_status else "None" return STATUS_STRINGS.get(raw_status_lower, raw_status) -def compare_result(logger, base_name, target_name, base_result, target_result): +def compare_result(logger, base_name, target_name, base_result, target_result, display_limit=None): base_result = base_result.get('result') target_result = target_result.get('result') result = {} new_tests = 0 + regressions = {} + resultstring = "" + + display_limit = int(display_limit) if display_limit else REGRESSIONS_DISPLAY_LIMIT if base_result and target_result: for k in base_result: @@ -212,7 +218,17 @@ def compare_result(logger, base_name, target_name, base_result, target_result): resultstring = "Regression: %s\n %s\n" % (base_name, target_name) for k in sorted(result): if not result[k]['target'] or not result[k]['target'].startswith("PASS"): - resultstring += ' %s: %s -> %s\n' % (k, get_status_str(result[k]['base']), get_status_str(result[k]['target'])) + # Differentiate each ptest kind when listing regressions + key = '.'.join(k.split('.')[:2]) if k.startswith('ptest') else k + # Append new regression to corresponding test family + regressions[key] = regressions.setdefault(key, []) + [' %s: %s -> %s\n' % (k, get_status_str(result[k]['base']), get_status_str(result[k]['target']))] + resultstring += f" Total: {sum([len(regressions[r]) for r in regressions])} new regression(s):\n" + for k in regressions: + resultstring += f" {len(regressions[k])} regression(s) for {k}\n" + count_to_print=min([display_limit, len(regressions[k])]) if display_limit > 0 else len(regressions[k]) + resultstring += ''.join(regressions[k][:count_to_print]) + if count_to_print < len(regressions[k]): + resultstring+=' [...]\n' if new_pass_count > 0: resultstring += f' Additionally, {new_pass_count} previously failing test(s) is/are now passing\n' else: @@ -280,7 +296,7 @@ def regression_common(args, logger, base_results, target_results): for b in target.copy(): if not can_be_compared(logger, base_results[a][c], target_results[a][b]): continue - res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b]) + res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b], args.limit) if not res: matches.append(resstr) base.remove(c) @@ -291,7 +307,7 @@ def regression_common(args, logger, base_results, target_results): for b in target: if not can_be_compared(logger, base_results[a][c], target_results[a][b]): continue - res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b]) + res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b], args.limit) if res: regressions.append(resstr) else: @@ -403,4 +419,5 @@ def register_commands(subparsers): parser_build.add_argument('--commit-number', help="Revision number to search for, redundant if --commit is specified") parser_build.add_argument('--commit2', help="Revision to compare with") parser_build.add_argument('--commit-number2', help="Revision number to compare with, redundant if --commit2 is specified") + parser_build.add_argument('-l', '--limit', default=REGRESSIONS_DISPLAY_LIMIT, help="Maximum number of changes to display per test. Can be set to 0 to print all changes") From patchwork Sun Oct 22 17:49:36 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 32723 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 6A81EC25B41 for ; Sun, 22 Oct 2023 17:49:05 +0000 (UTC) Received: from relay4-d.mail.gandi.net (relay4-d.mail.gandi.net [217.70.183.196]) by mx.groups.io with SMTP id smtpd.web11.103870.1697996935285741517 for ; Sun, 22 Oct 2023 10:48:55 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=H/gGDV6d; spf=pass (domain: bootlin.com, ip: 217.70.183.196, mailfrom: alexis.lothore@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id 43EB7E0005; Sun, 22 Oct 2023 17:48:53 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1697996933; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=0w8jGuwDSZmm1L1l8TrSM3cxHpdmo2eGQ7iyyRCjt0Q=; b=H/gGDV6dmPpKs3uHy77Yoo6asc9Cz3RwFDY7Sni6ZN8OHMD1m4bdSiDiOSQC0Zxf+dkh3T BhKOVGRxYN7+mmb3/6g979p1lpgGN4uNCD5tMBrgUWrpcLr2D9rkoidIGrkAc8cRKAzr9M TXBC5Pg2DyJTzNJvZB/YWllsfRms4+hlw60GEVelbtViBUopuofGMiUU/CO0eNmJZL21Rp KoChRHMfHg30A2pTKFUskYP2RaRNA2F7BzMovJyLwL7dxWSKuVOONq2ClTUvcr4+Gx4ugb oqdAq3EJf7XjGuXLkr/CIUtOZIceYDSadjNo6SDFTgX1MpJJdo9PvG0Ts7FKLg== From: =?utf-8?q?Alexis_Lothor=C3=A9?= To: Cc: Thomas Petazzoni , Alexandre Belloni Subject: [OE-Core][PATCH v2 2/4] scripts/resulttool: rearrange regressions report order Date: Sun, 22 Oct 2023 19:49:36 +0200 Message-ID: <20231022174938.7012-3-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.42.0 In-Reply-To: <20231022174938.7012-1-alexis.lothore@bootlin.com> References: <20231022174938.7012-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: alexis.lothore@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Sun, 22 Oct 2023 17:49:05 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/189609 From: Alexis Lothoré Regressions reports currently reports matching pairs and improved pairs first, then regressions. Change order to print regressions first, which is the most valuable info in the report, and then print improvements and matches at the bottom. Signed-off-by: Alexis Lothoré --- scripts/lib/resulttool/regression.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py index d98504f4ced8..560d102e172c 100644 --- a/scripts/lib/resulttool/regression.py +++ b/scripts/lib/resulttool/regression.py @@ -80,6 +80,9 @@ STATUS_STRINGS = { REGRESSIONS_DISPLAY_LIMIT=50 +MISSING_TESTS_BANNER = "-------------------------- Missing tests --------------------------" +ADDITIONAL_DATA_BANNER = "--------------------- Matches and improvements --------------------" + def test_has_at_least_one_matching_tag(test, tag_list): return "oetags" in test and any(oetag in tag_list for oetag in test["oetags"]) @@ -312,10 +315,11 @@ def regression_common(args, logger, base_results, target_results): regressions.append(resstr) else: notfound.append("%s not found in target" % a) - print("\n".join(sorted(matches))) - print("\n") print("\n".join(sorted(regressions))) + print("\n" + MISSING_TESTS_BANNER + "\n") print("\n".join(sorted(notfound))) + print("\n" + ADDITIONAL_DATA_BANNER + "\n") + print("\n".join(sorted(matches))) return 0 def regression_git(args, logger): From patchwork Sun Oct 22 17:49:37 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 32722 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 97AFDCDB474 for ; Sun, 22 Oct 2023 17:49:05 +0000 (UTC) Received: from relay4-d.mail.gandi.net (relay4-d.mail.gandi.net [217.70.183.196]) by mx.groups.io with SMTP id smtpd.web10.103461.1697996935013503866 for ; Sun, 22 Oct 2023 10:48:55 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=CzY1/J8o; spf=pass (domain: bootlin.com, ip: 217.70.183.196, mailfrom: alexis.lothore@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id 7FCE4E0006; Sun, 22 Oct 2023 17:48:53 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1697996933; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=8qFDqs0CIE77srTuCv14ZOAThd/Pwa+890knaw2wJbk=; b=CzY1/J8oFX1lrxLptCjvwJUMWmLl5v/NZZX0y8XW6EGanZy7SCz9GOK63g9NV5InuhoHr1 6R2hPGlVgfcX/a7/K3HPaji1gyJkCYe7o6lzD5qVyCCN2NMsnVJtwytFIpZ+z2vMv2BPbf ZqJq1FjndgQo6pCEpCQFKoStvvyJKeVBYPBwcwlrcqbbIpyaAQXzI7Mmgvuh7arz3bpRhQ I4/wXDzyh3Q215Tn5pZQrVCWrJlBMIc8xATBJohsR8mZD7fXt94LENaNQfHuH1DUQ+A/eh /Wt2YtsVEpVgFE1hB8wBOdmVosp3F1AHY2388w71BNwCunYrWg92mSaHv1lsow== From: =?utf-8?q?Alexis_Lothor=C3=A9?= To: Cc: Thomas Petazzoni , Alexandre Belloni Subject: [OE-Core][PATCH v2 3/4] scripts/resulttool: make additional info more compact Date: Sun, 22 Oct 2023 19:49:37 +0200 Message-ID: <20231022174938.7012-4-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.42.0 In-Reply-To: <20231022174938.7012-1-alexis.lothore@bootlin.com> References: <20231022174938.7012-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: alexis.lothore@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Sun, 22 Oct 2023 17:49:05 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/189607 From: Alexis Lothoré Since "matched" and "improved" tests are not as important as regressions, reduce the place they take in the regression report: - merge "matched" and "improved" tests, while removing the label - add a single line of additional info per pair Those changes make the "Matches and improvements" look like the following sample: oeselftest_almalinux-9.2_qemux86-64_20230910083156 oeselftest_almalinux-8.8_qemux86-64_20231018010951 -> +7 test(s) present oeselftest_almalinux-9.2_qemux86-64_20230911010538 oeselftest_debian-11_qemux86-64_20231017150459 oeselftest_debian-11_qemux86-64_20230910012927 oeselftest_debian-11_qemux86-64_20231017151319 -> +7 test(s) present [...] Signed-off-by: Alexis Lothoré --- scripts/lib/resulttool/regression.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py index 560d102e172c..8fbe5a54783a 100644 --- a/scripts/lib/resulttool/regression.py +++ b/scripts/lib/resulttool/regression.py @@ -186,6 +186,18 @@ def get_status_str(raw_status): raw_status_lower = raw_status.lower() if raw_status else "None" return STATUS_STRINGS.get(raw_status_lower, raw_status) +def get_additional_info_line(new_pass_count, new_tests): + result=[] + if new_tests: + result.append(f'+{new_tests} test(s) present') + if new_pass_count: + result.append(f'+{new_pass_count} test(s) now passing') + + if not result: + return "" + + return ' -> ' + ', '.join(result) + '\n' + def compare_result(logger, base_name, target_name, base_result, target_result, display_limit=None): base_result = base_result.get('result') target_result = target_result.get('result') @@ -193,6 +205,8 @@ def compare_result(logger, base_name, target_name, base_result, target_result, d new_tests = 0 regressions = {} resultstring = "" + new_tests = 0 + new_pass_count = 0 display_limit = int(display_limit) if display_limit else REGRESSIONS_DISPLAY_LIMIT @@ -234,14 +248,19 @@ def compare_result(logger, base_name, target_name, base_result, target_result, d resultstring+=' [...]\n' if new_pass_count > 0: resultstring += f' Additionally, {new_pass_count} previously failing test(s) is/are now passing\n' + if new_tests > 0: + resultstring += f' Additionally, {new_tests} new test(s) is/are present\n' else: - resultstring = "Improvement: %s\n %s\n (+%d test(s) passing)\n" % (base_name, target_name, new_pass_count) + resultstring = "%s\n%s\n" % (base_name, target_name) result = None else: - resultstring = "Match: %s\n %s\n" % (base_name, target_name) + resultstring = "%s\n%s\n" % (base_name, target_name) + + if not result: + additional_info = get_additional_info_line(new_pass_count, new_tests) + if additional_info: + resultstring += additional_info - if new_tests > 0: - resultstring += f' Additionally, {new_tests} new test(s) is/are present\n' return result, resultstring def get_results(logger, source): From patchwork Sun Oct 22 17:49:38 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 32721 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 5B8C9C00A8F for ; Sun, 22 Oct 2023 17:49:05 +0000 (UTC) Received: from relay4-d.mail.gandi.net (relay4-d.mail.gandi.net [217.70.183.196]) by mx.groups.io with SMTP id smtpd.web11.103869.1697996935260497683 for ; Sun, 22 Oct 2023 10:48:55 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=QyYV+nay; spf=pass (domain: bootlin.com, ip: 217.70.183.196, mailfrom: alexis.lothore@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id BD684E0007; Sun, 22 Oct 2023 17:48:53 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1697996933; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=HURJjrAvaMDxabzmJkUVya+TJLu3DmRBEJkm04hgF3s=; b=QyYV+naysT4l18yQcQYqztPCVMJ3Z3imPSav/uNiEFyWAdWc/GLSzMmvuUrsOCel8viWA8 vdnNkjDW3LuXIqhatSwQssYcD4W4zOpqsl4vG/Yc9qY8m9I9j0UJo6nBjzh0GZqwxZCfV0 Rj6bjM+t7RLMGH1aCY7HxCrj+mu9ciSGbHW/TyNjbEAh+DGvniZOz/pi1SvogoriMjDxIJ 5niGahwdKG+oCcz75hTi7ZLGAEa7ht82zTJX0hSgYBBKhkTyeWJX2eonjO6SuWs+/CGD4u GkzsOCw1g4cCZ1d9NNFPaOg8s8rBxqNlV9wwbr5UShNQVZek2glhCgyP57Ivrg== From: =?utf-8?q?Alexis_Lothor=C3=A9?= To: Cc: Thomas Petazzoni , Alexandre Belloni Subject: [OE-Core][PATCH v2 4/4] scripts/yocto_testresults_query: add option to change display limit Date: Sun, 22 Oct 2023 19:49:38 +0200 Message-ID: <20231022174938.7012-5-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.42.0 In-Reply-To: <20231022174938.7012-1-alexis.lothore@bootlin.com> References: <20231022174938.7012-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: alexis.lothore@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Sun, 22 Oct 2023 17:49:05 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/189610 From: Alexis Lothoré Add a "-l"/"--limit" option to allow changing the display limit in resulttool. - If no value is passed, resulttool uses its default value. - If 0 is passed, the display limit is removed and every regression will be displayed - If a custom value is passed, this value overrides the vlaue configured in resulttool Signed-off-by: Alexis Lothoré --- scripts/yocto_testresults_query.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/yocto_testresults_query.py b/scripts/yocto_testresults_query.py index a5073736aab5..521ead8473ad 100755 --- a/scripts/yocto_testresults_query.py +++ b/scripts/yocto_testresults_query.py @@ -56,9 +56,12 @@ def fetch_testresults(workdir, sha1): subprocess.check_call(["git", "fetch", "--depth", "1", "origin", f"{rev}:{rev}"], cwd=workdir) return branch -def compute_regression_report(workdir, basebranch, baserevision, targetbranch, targetrevision): +def compute_regression_report(workdir, basebranch, baserevision, targetbranch, targetrevision, args): logger.info(f"Running resulttool regression between SHA1 {baserevision} and {targetrevision}") - report = subprocess.check_output([resulttool, "regression-git", "--branch", basebranch, "--commit", baserevision, "--branch2", targetbranch, "--commit2", targetrevision, workdir]).decode("utf-8") + command = [resulttool, "regression-git", "--branch", basebranch, "--commit", baserevision, "--branch2", targetbranch, "--commit2", targetrevision, workdir] + if args.limit: + command.extend(["-l", args.limit]) + report = subprocess.check_output(command).decode("utf-8") return report def print_report_with_header(report, baseversion, baserevision, targetversion, targetrevision): @@ -85,7 +88,7 @@ def regression(args): sys.exit(1) basebranch = fetch_testresults(workdir, baserevision) targetbranch = fetch_testresults(workdir, targetrevision) - report = compute_regression_report(workdir, basebranch, baserevision, targetbranch, targetrevision) + report = compute_regression_report(workdir, basebranch, baserevision, targetbranch, targetrevision, args) print_report_with_header(report, args.base, baserevision, args.target, targetrevision) finally: if not args.testresultsdir: @@ -109,6 +112,10 @@ def main(): '-t', '--testresultsdir', help=f"An existing test results directory. {sys.argv[0]} will automatically clone it and use default branch if not provided") + parser_regression_report.add_argument( + '-l', + '--limit', + help=f"Maximum number of changes to display per test. Can be set to 0 to print all changes") parser_regression_report.set_defaults(func=regression) args = parser.parse_args()