scripts: add buildstats-summary

This script will write a summary of the buildstats to the terminal,
sorted by start time or duration, optionally hiding short tasks, and
highlighting long running tasks.

Signed-off-by: Ross Burton <ross.burton@arm.com>
---
 scripts/buildstats-summary | 126 +++++++++++++++++++++++++++++++++++++
 1 file changed, 126 insertions(+)
 create mode 100755 scripts/buildstats-summary

This looks useful, I tried it out and it worked in one case but failed in
few others with

% buildstats-summary build/tmp/buildstats/20230319021406
Traceback (most recent call last):
  File "/mnt/b/yoe/master/sources/poky/scripts/buildstats-summary",
line 126, in <module>
    sys.exit(main())
  File "/mnt/b/yoe/master/sources/poky/scripts/buildstats-summary",
line 119, in main
    bs = read_buildstats(args.buildstats)
  File "/mnt/b/yoe/master/sources/poky/scripts/buildstats-summary",
line 54, in read_buildstats
    return buildstats.BuildStats.from_dir(path)
  File "/mnt/b/yoe/master/sources/poky/scripts/lib/buildstats.py",
line 256, in from_dir
    build_started, build_elapsed = buildstats.parse_top_build_stats(top_stats)
  File "/mnt/b/yoe/master/sources/poky/scripts/lib/buildstats.py",
line 245, in parse_top_build_stats
    return start, elapsed
UnboundLocalError: local variable 'elapsed' referenced before assignment

On Tue, Mar 21, 2023 at 10:39 AM Ross Burton <ross.burton@arm.com> wrote:
>
> This script will write a summary of the buildstats to the terminal,
> sorted by start time or duration, optionally hiding short tasks, and
> highlighting long running tasks.
>
> Signed-off-by: Ross Burton <ross.burton@arm.com>
> ---
>  scripts/buildstats-summary | 126 +++++++++++++++++++++++++++++++++++++
>  1 file changed, 126 insertions(+)
>  create mode 100755 scripts/buildstats-summary
>
> diff --git a/scripts/buildstats-summary b/scripts/buildstats-summary
> new file mode 100755
> index 00000000000..89348318afa
> --- /dev/null
> +++ b/scripts/buildstats-summary
> @@ -0,0 +1,126 @@
> +#! /usr/bin/python3
> +#
> +# Dump a summary of the specified buildstats to the terminal, filtering and
> +# sorting by walltime.
> +#
> +# SPDX-License-Identifier: GPL-2.0-only
> +
> +import argparse
> +import dataclasses
> +import datetime
> +import enum
> +import os
> +import pathlib
> +import sys
> +
> +scripts_path = os.path.dirname(os.path.realpath(__file__))
> +sys.path.append(os.path.join(scripts_path, "lib"))
> +import buildstats
> +
> +
> +@dataclasses.dataclass
> +class Task:
> +    recipe: str
> +    task: str
> +    start: datetime.datetime
> +    duration: datetime.timedelta
> +
> +
> +class Sorting(enum.Enum):
> +    start = 1
> +    duration = 2
> +
> +    # argparse integration
> +    def __str__(self) -> str:
> +        return self.name
> +
> +    def __repr__(self) -> str:
> +        return self.name
> +
> +    @staticmethod
> +    def from_string(s: str):
> +        try:
> +            return Sorting[s]
> +        except KeyError:
> +            return s
> +
> +
> +def read_buildstats(path: pathlib.Path) -> buildstats.BuildStats:
> +    if not path.exists():
> +        raise Exception(f"No such file or directory: {path}")
> +    if path.is_file():
> +        return buildstats.BuildStats.from_file_json(path)
> +    if (path / "build_stats").is_file():
> +        return buildstats.BuildStats.from_dir(path)
> +    raise Exception(f"Cannot find buildstats in {path}")
> +
> +
> +def dump_buildstats(args, bs: buildstats.BuildStats):
> +    tasks = []
> +    for recipe in bs.values():
> +        for task, stats in recipe.tasks.items():
> +            t = Task(
> +                recipe.name,
> +                task,
> +                datetime.datetime.fromtimestamp(stats["start_time"]),
> +                datetime.timedelta(seconds=int(stats.walltime)),
> +            )
> +            tasks.append(t)
> +
> +    tasks.sort(key=lambda t: getattr(t, args.sort.name))
> +
> +    minimum = datetime.timedelta(seconds=args.shortest)
> +    highlight = datetime.timedelta(seconds=args.highlight)
> +
> +    for t in tasks:
> +        if t.duration >= minimum:
> +            line = f"{t.duration}    {t.recipe}:{t.task}"
> +            if t.duration >= highlight:
> +                print(f"\033[1m{line}\033[0m")
> +            else:
> +                print(line)
> +
> +
> +def main(argv=None) -> int:
> +    parser = argparse.ArgumentParser(
> +        formatter_class=argparse.ArgumentDefaultsHelpFormatter
> +    )
> +
> +    parser.add_argument(
> +        "buildstats", metavar="BUILDSTATS", help="Buildstats file", type=pathlib.Path
> +    )
> +    parser.add_argument(
> +        "--sort",
> +        "-s",
> +        type=Sorting.from_string,
> +        choices=list(Sorting),
> +        default=Sorting.start,
> +        help="Sort tasks",
> +    )
> +    parser.add_argument(
> +        "--shortest",
> +        "-t",
> +        type=int,
> +        default=1,
> +        metavar="SECS",
> +        help="Hide tasks shorter than SECS seconds",
> +    )
> +    parser.add_argument(
> +        "--highlight",
> +        "-g",
> +        type=int,
> +        default=60,
> +        metavar="SECS",
> +        help="Highlight tasks longer than SECS seconds",
> +    )
> +
> +    args = parser.parse_args(argv)
> +
> +    bs = read_buildstats(args.buildstats)
> +    dump_buildstats(args, bs)
> +
> +    return 0
> +
> +
> +if __name__ == "__main__":
> +    sys.exit(main())
> --
> 2.34.1
>
>
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#178892): https://lists.openembedded.org/g/openembedded-core/message/178892
> Mute This Topic: https://lists.openembedded.org/mt/97760470/1997914
> Group Owner: openembedded-core+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [raj.khem@gmail.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>

here is the link for faulty one

https://uclibc.org/~kraj/20230319021406.tar.xz

On Tue, Mar 21, 2023 at 10:55 AM Khem Raj <raj.khem@gmail.com> wrote:
>
> This looks useful, I tried it out and it worked in one case but failed in
> few others with
>
> % buildstats-summary build/tmp/buildstats/20230319021406
> Traceback (most recent call last):
>   File "/mnt/b/yoe/master/sources/poky/scripts/buildstats-summary",
> line 126, in <module>
>     sys.exit(main())
>   File "/mnt/b/yoe/master/sources/poky/scripts/buildstats-summary",
> line 119, in main
>     bs = read_buildstats(args.buildstats)
>   File "/mnt/b/yoe/master/sources/poky/scripts/buildstats-summary",
> line 54, in read_buildstats
>     return buildstats.BuildStats.from_dir(path)
>   File "/mnt/b/yoe/master/sources/poky/scripts/lib/buildstats.py",
> line 256, in from_dir
>     build_started, build_elapsed = buildstats.parse_top_build_stats(top_stats)
>   File "/mnt/b/yoe/master/sources/poky/scripts/lib/buildstats.py",
> line 245, in parse_top_build_stats
>     return start, elapsed
> UnboundLocalError: local variable 'elapsed' referenced before assignment
>
> On Tue, Mar 21, 2023 at 10:39 AM Ross Burton <ross.burton@arm.com> wrote:
> >
> > This script will write a summary of the buildstats to the terminal,
> > sorted by start time or duration, optionally hiding short tasks, and
> > highlighting long running tasks.
> >
> > Signed-off-by: Ross Burton <ross.burton@arm.com>
> > ---
> >  scripts/buildstats-summary | 126 +++++++++++++++++++++++++++++++++++++
> >  1 file changed, 126 insertions(+)
> >  create mode 100755 scripts/buildstats-summary
> >
> > diff --git a/scripts/buildstats-summary b/scripts/buildstats-summary
> > new file mode 100755
> > index 00000000000..89348318afa
> > --- /dev/null
> > +++ b/scripts/buildstats-summary
> > @@ -0,0 +1,126 @@
> > +#! /usr/bin/python3
> > +#
> > +# Dump a summary of the specified buildstats to the terminal, filtering and
> > +# sorting by walltime.
> > +#
> > +# SPDX-License-Identifier: GPL-2.0-only
> > +
> > +import argparse
> > +import dataclasses
> > +import datetime
> > +import enum
> > +import os
> > +import pathlib
> > +import sys
> > +
> > +scripts_path = os.path.dirname(os.path.realpath(__file__))
> > +sys.path.append(os.path.join(scripts_path, "lib"))
> > +import buildstats
> > +
> > +
> > +@dataclasses.dataclass
> > +class Task:
> > +    recipe: str
> > +    task: str
> > +    start: datetime.datetime
> > +    duration: datetime.timedelta
> > +
> > +
> > +class Sorting(enum.Enum):
> > +    start = 1
> > +    duration = 2
> > +
> > +    # argparse integration
> > +    def __str__(self) -> str:
> > +        return self.name
> > +
> > +    def __repr__(self) -> str:
> > +        return self.name
> > +
> > +    @staticmethod
> > +    def from_string(s: str):
> > +        try:
> > +            return Sorting[s]
> > +        except KeyError:
> > +            return s
> > +
> > +
> > +def read_buildstats(path: pathlib.Path) -> buildstats.BuildStats:
> > +    if not path.exists():
> > +        raise Exception(f"No such file or directory: {path}")
> > +    if path.is_file():
> > +        return buildstats.BuildStats.from_file_json(path)
> > +    if (path / "build_stats").is_file():
> > +        return buildstats.BuildStats.from_dir(path)
> > +    raise Exception(f"Cannot find buildstats in {path}")
> > +
> > +
> > +def dump_buildstats(args, bs: buildstats.BuildStats):
> > +    tasks = []
> > +    for recipe in bs.values():
> > +        for task, stats in recipe.tasks.items():
> > +            t = Task(
> > +                recipe.name,
> > +                task,
> > +                datetime.datetime.fromtimestamp(stats["start_time"]),
> > +                datetime.timedelta(seconds=int(stats.walltime)),
> > +            )
> > +            tasks.append(t)
> > +
> > +    tasks.sort(key=lambda t: getattr(t, args.sort.name))
> > +
> > +    minimum = datetime.timedelta(seconds=args.shortest)
> > +    highlight = datetime.timedelta(seconds=args.highlight)
> > +
> > +    for t in tasks:
> > +        if t.duration >= minimum:
> > +            line = f"{t.duration}    {t.recipe}:{t.task}"
> > +            if t.duration >= highlight:
> > +                print(f"\033[1m{line}\033[0m")
> > +            else:
> > +                print(line)
> > +
> > +
> > +def main(argv=None) -> int:
> > +    parser = argparse.ArgumentParser(
> > +        formatter_class=argparse.ArgumentDefaultsHelpFormatter
> > +    )
> > +
> > +    parser.add_argument(
> > +        "buildstats", metavar="BUILDSTATS", help="Buildstats file", type=pathlib.Path
> > +    )
> > +    parser.add_argument(
> > +        "--sort",
> > +        "-s",
> > +        type=Sorting.from_string,
> > +        choices=list(Sorting),
> > +        default=Sorting.start,
> > +        help="Sort tasks",
> > +    )
> > +    parser.add_argument(
> > +        "--shortest",
> > +        "-t",
> > +        type=int,
> > +        default=1,
> > +        metavar="SECS",
> > +        help="Hide tasks shorter than SECS seconds",
> > +    )
> > +    parser.add_argument(
> > +        "--highlight",
> > +        "-g",
> > +        type=int,
> > +        default=60,
> > +        metavar="SECS",
> > +        help="Highlight tasks longer than SECS seconds",
> > +    )
> > +
> > +    args = parser.parse_args(argv)
> > +
> > +    bs = read_buildstats(args.buildstats)
> > +    dump_buildstats(args, bs)
> > +
> > +    return 0
> > +
> > +
> > +if __name__ == "__main__":
> > +    sys.exit(main())
> > --
> > 2.34.1
> >
> >
> > -=-=-=-=-=-=-=-=-=-=-=-
> > Links: You receive all messages sent to this group.
> > View/Reply Online (#178892): https://lists.openembedded.org/g/openembedded-core/message/178892
> > Mute This Topic: https://lists.openembedded.org/mt/97760470/1997914
> > Group Owner: openembedded-core+owner@lists.openembedded.org
> > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [raj.khem@gmail.com]
> > -=-=-=-=-=-=-=-=-=-=-=-
> >

On 21 Mar 2023, at 17:55, Khem Raj <raj.khem@gmail.com> wrote:
> 
> This looks useful, I tried it out and it worked in one case but failed in
> few others with
> 
> % buildstats-summary build/tmp/buildstats/20230319021406
> Traceback (most recent call last):
>  File "/mnt/b/yoe/master/sources/poky/scripts/buildstats-summary",
> line 126, in <module>
>    sys.exit(main())
>  File "/mnt/b/yoe/master/sources/poky/scripts/buildstats-summary",
> line 119, in main
>    bs = read_buildstats(args.buildstats)
>  File "/mnt/b/yoe/master/sources/poky/scripts/buildstats-summary",
> line 54, in read_buildstats
>    return buildstats.BuildStats.from_dir(path)
>  File "/mnt/b/yoe/master/sources/poky/scripts/lib/buildstats.py",
> line 256, in from_dir
>    build_started, build_elapsed = buildstats.parse_top_build_stats(top_stats)
>  File "/mnt/b/yoe/master/sources/poky/scripts/lib/buildstats.py",
> line 245, in parse_top_build_stats
>    return start, elapsed
> UnboundLocalError: local variable 'elapsed' referenced before assignment

Looks like that build was either in progress or bitbake was killed and it didn’t finish writing the files.

I’ll make it default the elapsed time to 0 to handle this case.

Ross