From patchwork Fri Oct 27 15:29:39 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 33032 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 3F6F5C25B48 for ; Fri, 27 Oct 2023 15:29:47 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web11.9784.1698420584599374409 for ; Fri, 27 Oct 2023 08:29:44 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 2ADBA1424; Fri, 27 Oct 2023 08:30:25 -0700 (PDT) Received: from oss-tx204.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 1B1493F64C; Fri, 27 Oct 2023 08:29:42 -0700 (PDT) From: ross.burton@arm.com To: openembedded-core@lists.openembedded.org Cc: nd@arm.com Subject: [PATCH 1/3] scripts/patchreview: rework patch detection Date: Fri, 27 Oct 2023 16:29:39 +0100 Message-Id: <20231027152941.232042-1-ross.burton@arm.com> X-Mailer: git-send-email 2.34.1 MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 27 Oct 2023 15:29:47 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/189747 From: Ross Burton A previous patch[1] added the ability to allow the search pattern for patches to be changed, so that patchreview can be used across the entire meta-oe repository by changing the patterns. However, this means the caller needs to write long patterns when calling patchreview. Instead, we can see if the specified directory contains a layer by checking if conf/layer.conf exists. If it does, then search for patches inside this directory. If it doesn't, assume that the specified directory is a repository that contains sublayers (such as meta-openembedded) and look through each of the directories that match the pattern meta-*. This means patchreview can both scan either a single layer (eg .../poky/meta) or a repository of sublayers (eg .../meta-openembedded). [1] oe-core 599046ea9302af0cf856d3fcd827f6a2be75b7e1 Signed-off-by: Ross Burton --- scripts/contrib/patchreview.py | 36 +++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/scripts/contrib/patchreview.py b/scripts/contrib/patchreview.py index 43de105adc2..af66e32e02e 100755 --- a/scripts/contrib/patchreview.py +++ b/scripts/contrib/patchreview.py @@ -41,7 +41,7 @@ def blame_patch(patch): "--format=%s (%aN <%aE>)", "--", patch)).decode("utf-8").splitlines() -def patchreview(path, patches): +def patchreview(patches): import re, os.path # General pattern: start of line, optional whitespace, tag with optional @@ -56,11 +56,10 @@ def patchreview(path, patches): for patch in patches: - fullpath = os.path.join(path, patch) result = Result() - results[fullpath] = result + results[patch] = result - content = open(fullpath, encoding='ascii', errors='ignore').read() + content = open(patch, encoding='ascii', errors='ignore').read() # Find the Signed-off-by tag match = sob_re.search(content) @@ -198,21 +197,40 @@ def histogram(results): for k in bars: print("%-20s %s (%d)" % (k.capitalize() if k else "No status", bars[k], counts[k])) +def gather_patches(candidate): + # candidate can either be the path to a layer directly (eg meta-intel), or a + # repository that contains other layers (meta-arm). We can determine what by + # looking for a conf/layer.conf file. If that file exists then it's a layer, + # otherwise its a repository of layers and we can assume they're called + # meta-*. + + if (candidate / "conf" / "layer.conf").exists(): + print(f"{candidate} is a layer") + scan = [candidate] + else: + print(f"{candidate} is not a layer, checking for sub-layers") + scan = [d for d in candidate.iterdir() if d.is_dir() and (d.name == "meta" or d.name.startswith("meta-"))] + print(f"Found layers {' '.join((d.name for d in scan))}") + + patches = [] + for directory in scan: + filenames = subprocess.check_output(("git", "-C", directory, "ls-files", "recipes-*/**/*.patch", "recipes-*/**/*.diff"), universal_newlines=True).split() + patches += [os.path.join(directory, f) for f in filenames] + return patches if __name__ == "__main__": - import argparse, subprocess, os + import argparse, subprocess, os, pathlib args = argparse.ArgumentParser(description="Patch Review Tool") args.add_argument("-b", "--blame", action="store_true", help="show blame for malformed patches") args.add_argument("-v", "--verbose", action="store_true", help="show per-patch results") args.add_argument("-g", "--histogram", action="store_true", help="show patch histogram") args.add_argument("-j", "--json", help="update JSON") - args.add_argument("-p", "--pattern", nargs=1, action="extend", default=["recipes-*/**/*.patch", "recipes-*/**/*.diff"], help="pattern to search recipes patch") - args.add_argument("directory", help="directory to scan") + args.add_argument("directory", type=pathlib.Path, metavar="DIRECTORY", help="directory to scan (layer, or repository of layers)") args = args.parse_args() - patches = subprocess.check_output(("git", "-C", args.directory, "ls-files") + tuple(args.pattern)).decode("utf-8").split() - results = patchreview(args.directory, patches) + patches = gather_patches(args.directory) + results = patchreview(patches) analyse(results, want_blame=args.blame, verbose=args.verbose) if args.json: From patchwork Fri Oct 27 15:29:40 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 33033 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 40C7AC25B6F for ; Fri, 27 Oct 2023 15:29:47 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web10.9798.1698420584635552516 for ; Fri, 27 Oct 2023 08:29:45 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id D31D6143D; Fri, 27 Oct 2023 08:30:25 -0700 (PDT) Received: from oss-tx204.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id C76063F64C; Fri, 27 Oct 2023 08:29:43 -0700 (PDT) From: ross.burton@arm.com To: openembedded-core@lists.openembedded.org Cc: nd@arm.com Subject: [PATCH 2/3] scripts/contrib/patchreview: add commit and recipe count fields to JSON Date: Fri, 27 Oct 2023 16:29:40 +0100 Message-Id: <20231027152941.232042-2-ross.burton@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20231027152941.232042-1-ross.burton@arm.com> References: <20231027152941.232042-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 27 Oct 2023 15:29:47 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/189748 From: Ross Burton The autobuilder scripts post-process the generated JSON to inject recipe and commit counts into the data. We can do this easily in patchreview instead. Signed-off-by: Ross Burton --- scripts/contrib/patchreview.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/scripts/contrib/patchreview.py b/scripts/contrib/patchreview.py index af66e32e02e..36038d06d2e 100755 --- a/scripts/contrib/patchreview.py +++ b/scripts/contrib/patchreview.py @@ -197,7 +197,7 @@ def histogram(results): for k in bars: print("%-20s %s (%d)" % (k.capitalize() if k else "No status", bars[k], counts[k])) -def gather_patches(candidate): +def find_layers(candidate): # candidate can either be the path to a layer directly (eg meta-intel), or a # repository that contains other layers (meta-arm). We can determine what by # looking for a conf/layer.conf file. If that file exists then it's a layer, @@ -205,19 +205,26 @@ def gather_patches(candidate): # meta-*. if (candidate / "conf" / "layer.conf").exists(): - print(f"{candidate} is a layer") - scan = [candidate] + return [candidate.absolute()] else: - print(f"{candidate} is not a layer, checking for sub-layers") - scan = [d for d in candidate.iterdir() if d.is_dir() and (d.name == "meta" or d.name.startswith("meta-"))] - print(f"Found layers {' '.join((d.name for d in scan))}") + return [d.absolute() for d in candidate.iterdir() if d.is_dir() and (d.name == "meta" or d.name.startswith("meta-"))] +# TODO these don't actually handle dynamic-layers/ + +def gather_patches(layers): patches = [] - for directory in scan: + for directory in layers: filenames = subprocess.check_output(("git", "-C", directory, "ls-files", "recipes-*/**/*.patch", "recipes-*/**/*.diff"), universal_newlines=True).split() patches += [os.path.join(directory, f) for f in filenames] return patches +def count_recipes(layers): + count = 0 + for directory in layers: + output = subprocess.check_output(["git", "-C", directory, "ls-files", "recipes-*/**/*.bb"], universal_newlines=True) + count += len(output.splitlines()) + return count + if __name__ == "__main__": import argparse, subprocess, os, pathlib @@ -229,7 +236,9 @@ if __name__ == "__main__": args.add_argument("directory", type=pathlib.Path, metavar="DIRECTORY", help="directory to scan (layer, or repository of layers)") args = args.parse_args() - patches = gather_patches(args.directory) + layers = find_layers(args.directory) + print(f"Found layers {' '.join((d.name for d in layers))}") + patches = gather_patches(layers) results = patchreview(patches) analyse(results, want_blame=args.blame, verbose=args.verbose) @@ -242,8 +251,11 @@ if __name__ == "__main__": row = collections.Counter() row["total"] = len(results) - row["date"] = subprocess.check_output(["git", "-C", args.directory, "show", "-s", "--pretty=format:%cd", "--date=format:%s"]).decode("utf-8").strip() - row["commit"] = subprocess.check_output(["git", "-C", args.directory, "show", "-s", "--pretty=format:%H"]).decode("utf-8").strip() + row["date"] = subprocess.check_output(["git", "-C", args.directory, "show", "-s", "--pretty=format:%cd", "--date=format:%s"], universal_newlines=True).strip() + row["commit"] = subprocess.check_output(["git", "-C", args.directory, "show-ref", "--hash", "HEAD"], universal_newlines=True).strip() + row['commit_count'] = subprocess.check_output(["git", "-C", args.directory, "rev-list", "--count", "HEAD"], universal_newlines=True).strip() + row['recipe_count'] = count_recipes(layers) + for r in results.values(): if r.upstream_status in status_values: row[r.upstream_status] += 1 From patchwork Fri Oct 27 15:29:41 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 33034 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 4E1CDC25B47 for ; Fri, 27 Oct 2023 15:29:47 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web10.9800.1698420585696344424 for ; Fri, 27 Oct 2023 08:29:45 -0700 Authentication-Results: mx.groups.io; dkim=none (message not signed); spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 869E91424; Fri, 27 Oct 2023 08:30:26 -0700 (PDT) Received: from oss-tx204.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 7A64C3F64C; Fri, 27 Oct 2023 08:29:44 -0700 (PDT) From: ross.burton@arm.com To: openembedded-core@lists.openembedded.org Cc: nd@arm.com Subject: [PATCH 3/3] scripts/contrib/patchreview: consolidate imports Date: Fri, 27 Oct 2023 16:29:41 +0100 Message-Id: <20231027152941.232042-3-ross.burton@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20231027152941.232042-1-ross.burton@arm.com> References: <20231027152941.232042-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 27 Oct 2023 15:29:47 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/189749 From: Ross Burton Move most imports to the top of the file. Signed-off-by: Ross Burton --- scripts/contrib/patchreview.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/contrib/patchreview.py b/scripts/contrib/patchreview.py index 36038d06d2e..f95cadab0c6 100755 --- a/scripts/contrib/patchreview.py +++ b/scripts/contrib/patchreview.py @@ -5,6 +5,15 @@ # SPDX-License-Identifier: GPL-2.0-only # +import argparse +import collections +import json +import os +import os.path +import pathlib +import re +import subprocess + # TODO # - option to just list all broken files # - test suite @@ -35,14 +44,12 @@ def blame_patch(patch): From a patch filename, return a list of "commit summary (author name )" strings representing the history. """ - import subprocess return subprocess.check_output(("git", "log", "--follow", "--find-renames", "--diff-filter=A", "--format=%s (%aN <%aE>)", "--", patch)).decode("utf-8").splitlines() def patchreview(patches): - import re, os.path # General pattern: start of line, optional whitespace, tag with optional # hyphen or spaces, maybe a colon, some whitespace, then the value, all case @@ -192,6 +199,7 @@ Patches in Pending state: %s""" % (total_patches, def histogram(results): from toolz import recipes, dicttoolz import math + counts = recipes.countby(lambda r: r.upstream_status, results.values()) bars = dicttoolz.valmap(lambda v: "#" * int(math.ceil(float(v) / len(results) * 100)), counts) for k in bars: @@ -226,8 +234,6 @@ def count_recipes(layers): return count if __name__ == "__main__": - import argparse, subprocess, os, pathlib - args = argparse.ArgumentParser(description="Patch Review Tool") args.add_argument("-b", "--blame", action="store_true", help="show blame for malformed patches") args.add_argument("-v", "--verbose", action="store_true", help="show per-patch results") @@ -243,7 +249,6 @@ if __name__ == "__main__": analyse(results, want_blame=args.blame, verbose=args.verbose) if args.json: - import json, os.path, collections if os.path.isfile(args.json): data = json.load(open(args.json)) else: