Message ID | 20200914181903.499094-1-ross.burton@arm.com |
---|---|
State | New |
Headers | show |
diff --git a/meta-arm/recipes-bsp/scp-firmware/files/0001-tools-gen_module_code-atomically-rewrite-the-generat.patch b/meta-arm/recipes-bsp/scp-firmware/files/0001-tools-gen_module_code-atomically-rewrite-the-generat.patch new file mode 100644 index 0000000..05bb025 --- /dev/null +++ b/meta-arm/recipes-bsp/scp-firmware/files/0001-tools-gen_module_code-atomically-rewrite-the-generat.patch @@ -0,0 +1,69 @@ +Upstream-Status: Backport +Signed-off-by: Ross Burton <ross.burton@arm.com> + +From 34e1c04534607f5605255f39fb46e26261fc9c4e Mon Sep 17 00:00:00 2001 +From: Ross Burton <ross.burton@arm.com> +Date: Tue, 8 Sep 2020 11:49:08 +0100 +Subject: [PATCH] tools/gen_module_code: atomically rewrite the generated files + +The gen_module rule in rules.mk is marked as .PHONY, so make will +execute it whenever it is mentioned. This results in gen_module_code +being executed 64 times for a Juno build. + +However in heavily parallel builds there's a good chance that +gen_module_code is writing a file whilst the compiler is reading it +because make also doesn't know what files are generated by +gen_module_code. + +The correct fix is to adjust the Makefiles so that the dependencies are +correct but this isn't trivial, so band-aid the problem by atomically +writing the generated files. + +Change-Id: I82d44f9ea6537a91002e1f80de8861d208571630 +Signed-off-by: Ross Burton <ross.burton@arm.com> +--- + tools/gen_module_code.py | 19 ++++++++++++++----- + 1 file changed, 14 insertions(+), 5 deletions(-) + +diff --git a/tools/gen_module_code.py b/tools/gen_module_code.py +index 7b395384..ee099b71 100755 +--- a/tools/gen_module_code.py ++++ b/tools/gen_module_code.py +@@ -17,6 +17,7 @@ + import argparse + import os + import sys ++import tempfile + + DEFAULT_PATH = 'build/' + +@@ -53,13 +54,21 @@ TEMPLATE_C = "/* This file was auto generated using {} */\n" \ + + def generate_file(path, filename, content): + full_filename = os.path.join(path, filename) +- with open(full_filename, 'a+') as f: +- f.seek(0) +- if f.read() != content: ++ ++ try: ++ with open(full_filename) as f: ++ rewrite = f.read() != content ++ except FileNotFoundError: ++ rewrite = True ++ ++ if rewrite: ++ with tempfile.NamedTemporaryFile(prefix="gen-module-code", ++ dir=path, ++ delete=False, ++ mode="wt") as f: + print("[GEN] {}...".format(full_filename)) +- f.seek(0) +- f.truncate() + f.write(content) ++ os.replace(f.name, full_filename) + + + def generate_header(path, modules): +-- +2.28.0 + diff --git a/meta-arm/recipes-bsp/scp-firmware/scp-firmware_2.6.0.bb b/meta-arm/recipes-bsp/scp-firmware/scp-firmware_2.6.0.bb index 4de0912..f9510e4 100644 --- a/meta-arm/recipes-bsp/scp-firmware/scp-firmware_2.6.0.bb +++ b/meta-arm/recipes-bsp/scp-firmware/scp-firmware_2.6.0.bb @@ -6,7 +6,8 @@ LICENSE = "BSD-3-Clause & Apache-2.0" LIC_FILES_CHKSUM = "file://license.md;beginline=5;md5=9db9e3d2fb8d9300a6c3d15101b19731 \ file://cmsis/LICENSE.txt;md5=e3fc50a88d0a364313df4b21ef20c29e" -SRC_URI = "gitsm://github.com/ARM-software/SCP-firmware.git;protocol=https" +SRC_URI = "gitsm://github.com/ARM-software/SCP-firmware.git;protocol=https \ + file://0001-tools-gen_module_code-atomically-rewrite-the-generat.patch" SRCREV = "db19910aca6d1032eb0329e5fbb70a92b997f6f2" PROVIDES += "virtual/control-processor-firmware"
On Mon, Sep 14, 2020 at 07:19:03PM +0100, Ross Burton wrote: > Backport a patch to fix a race in parallel builds. > > Change-Id: I4b33b819f2da302369ca77429f25d5c15672c5f3 > Signed-off-by: Ross Burton <ross.burton@arm.com> Pulled into master. Thanks, Jon > --- > ..._code-atomically-rewrite-the-generat.patch | 69 +++++++++++++++++++ > .../scp-firmware/scp-firmware_2.6.0.bb | 3 +- > 2 files changed, 71 insertions(+), 1 deletion(-) > create mode 100644 meta-arm/recipes-bsp/scp-firmware/files/0001-tools-gen_module_code-atomically-rewrite-the-generat.patch > > diff --git a/meta-arm/recipes-bsp/scp-firmware/files/0001-tools-gen_module_code-atomically-rewrite-the-generat.patch b/meta-arm/recipes-bsp/scp-firmware/files/0001-tools-gen_module_code-atomically-rewrite-the-generat.patch > new file mode 100644 > index 0000000..05bb025 > --- /dev/null > +++ b/meta-arm/recipes-bsp/scp-firmware/files/0001-tools-gen_module_code-atomically-rewrite-the-generat.patch > @@ -0,0 +1,69 @@ > +Upstream-Status: Backport > +Signed-off-by: Ross Burton <ross.burton@arm.com> > + > +From 34e1c04534607f5605255f39fb46e26261fc9c4e Mon Sep 17 00:00:00 2001 > +From: Ross Burton <ross.burton@arm.com> > +Date: Tue, 8 Sep 2020 11:49:08 +0100 > +Subject: [PATCH] tools/gen_module_code: atomically rewrite the generated files > + > +The gen_module rule in rules.mk is marked as .PHONY, so make will > +execute it whenever it is mentioned. This results in gen_module_code > +being executed 64 times for a Juno build. > + > +However in heavily parallel builds there's a good chance that > +gen_module_code is writing a file whilst the compiler is reading it > +because make also doesn't know what files are generated by > +gen_module_code. > + > +The correct fix is to adjust the Makefiles so that the dependencies are > +correct but this isn't trivial, so band-aid the problem by atomically > +writing the generated files. > + > +Change-Id: I82d44f9ea6537a91002e1f80de8861d208571630 > +Signed-off-by: Ross Burton <ross.burton@arm.com> > +--- > + tools/gen_module_code.py | 19 ++++++++++++++----- > + 1 file changed, 14 insertions(+), 5 deletions(-) > + > +diff --git a/tools/gen_module_code.py b/tools/gen_module_code.py > +index 7b395384..ee099b71 100755 > +--- a/tools/gen_module_code.py > ++++ b/tools/gen_module_code.py > +@@ -17,6 +17,7 @@ > + import argparse > + import os > + import sys > ++import tempfile > + > + DEFAULT_PATH = 'build/' > + > +@@ -53,13 +54,21 @@ TEMPLATE_C = "/* This file was auto generated using {} */\n" \ > + > + def generate_file(path, filename, content): > + full_filename = os.path.join(path, filename) > +- with open(full_filename, 'a+') as f: > +- f.seek(0) > +- if f.read() != content: > ++ > ++ try: > ++ with open(full_filename) as f: > ++ rewrite = f.read() != content > ++ except FileNotFoundError: > ++ rewrite = True > ++ > ++ if rewrite: > ++ with tempfile.NamedTemporaryFile(prefix="gen-module-code", > ++ dir=path, > ++ delete=False, > ++ mode="wt") as f: > + print("[GEN] {}...".format(full_filename)) > +- f.seek(0) > +- f.truncate() > + f.write(content) > ++ os.replace(f.name, full_filename) > + > + > + def generate_header(path, modules): > +-- > +2.28.0 > + > diff --git a/meta-arm/recipes-bsp/scp-firmware/scp-firmware_2.6.0.bb b/meta-arm/recipes-bsp/scp-firmware/scp-firmware_2.6.0.bb > index 4de0912..f9510e4 100644 > --- a/meta-arm/recipes-bsp/scp-firmware/scp-firmware_2.6.0.bb > +++ b/meta-arm/recipes-bsp/scp-firmware/scp-firmware_2.6.0.bb > @@ -6,7 +6,8 @@ LICENSE = "BSD-3-Clause & Apache-2.0" > LIC_FILES_CHKSUM = "file://license.md;beginline=5;md5=9db9e3d2fb8d9300a6c3d15101b19731 \ > file://cmsis/LICENSE.txt;md5=e3fc50a88d0a364313df4b21ef20c29e" > > -SRC_URI = "gitsm://github.com/ARM-software/SCP-firmware.git;protocol=https" > +SRC_URI = "gitsm://github.com/ARM-software/SCP-firmware.git;protocol=https \ > + file://0001-tools-gen_module_code-atomically-rewrite-the-generat.patch" > SRCREV = "db19910aca6d1032eb0329e5fbb70a92b997f6f2" > > PROVIDES += "virtual/control-processor-firmware" > -- > 2.28.0 > > -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#1131): https://lists.yoctoproject.org/g/meta-arm/message/1131 Mute This Topic: https://lists.yoctoproject.org/mt/76848757/3617530 Group Owner: meta-arm+owner@lists.yoctoproject.org Unsubscribe: https://lists.yoctoproject.org/g/meta-arm/unsub [oe-patchwork@oe-patch.openembedded.org] -=-=-=-=-=-=-=-=-=-=-=-
Backport a patch to fix a race in parallel builds. Change-Id: I4b33b819f2da302369ca77429f25d5c15672c5f3 Signed-off-by: Ross Burton <ross.burton@arm.com> --- ..._code-atomically-rewrite-the-generat.patch | 69 +++++++++++++++++++ .../scp-firmware/scp-firmware_2.6.0.bb | 3 +- 2 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 meta-arm/recipes-bsp/scp-firmware/files/0001-tools-gen_module_code-atomically-rewrite-the-generat.patch