[1/8] package: Switch debug source handling to use prefix map

Reproducible builds are no longer a configuration option but are required.
We also rely on the prefix mapping capability of the compilers now.

As such, rewrite the source locating code to use the prefix maps instead
of taking a guess about WORKDIR which isn't correct for kernels, gcc,
externalsrc and probably more.

Instead, iterate the maps to locate any matching source code, keeping
in mind that multiple maps may map to one target location.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 meta/classes-global/package.bbclass | 68 ++++++++++++-----------------
 1 file changed, 28 insertions(+), 40 deletions(-)

Looks like a good series, thanks for your work on this. LGTM.

On Tue, Aug 16, 2022 at 1:57 PM Richard Purdie <
richard.purdie@linuxfoundation.org> wrote:

> Reproducible builds are no longer a configuration option but are required.
> We also rely on the prefix mapping capability of the compilers now.
>
> As such, rewrite the source locating code to use the prefix maps instead
> of taking a guess about WORKDIR which isn't correct for kernels, gcc,
> externalsrc and probably more.
>
> Instead, iterate the maps to locate any matching source code, keeping
> in mind that multiple maps may map to one target location.
>
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
>  meta/classes-global/package.bbclass | 68 ++++++++++++-----------------
>  1 file changed, 28 insertions(+), 40 deletions(-)
>
> diff --git a/meta/classes-global/package.bbclass
> b/meta/classes-global/package.bbclass
> index 418400da8c9..2d985d8affd 100644
> --- a/meta/classes-global/package.bbclass
> +++ b/meta/classes-global/package.bbclass
> @@ -565,26 +565,16 @@ def copydebugsources(debugsrcdir, sources, d):
>          objcopy = d.getVar("OBJCOPY")
>          workdir = d.getVar("WORKDIR")
>          sdir = d.getVar("S")
> -        sparentdir = os.path.dirname(os.path.dirname(sdir))
> -        sbasedir = os.path.basename(os.path.dirname(sdir)) + "/" +
> os.path.basename(sdir)
> -        workparentdir = os.path.dirname(os.path.dirname(workdir))
> -        workbasedir = os.path.basename(os.path.dirname(workdir)) + "/" +
> os.path.basename(workdir)
> -
> -        # If S isnt based on WORKDIR we can infer our sources are located
> elsewhere,
> -        # e.g. using externalsrc; use S as base for our dirs
> -        if workdir in sdir or 'work-shared' in sdir:
> -            basedir = workbasedir
> -            parentdir = workparentdir
> -        else:
> -            basedir = sbasedir
> -            parentdir = sparentdir
> +        cflags = d.expand("${CFLAGS}")
>
> -        # If build path exists in sourcefile, it means toolchain did not
> use
> -        # -fdebug-prefix-map to compile
> -        if checkbuildpath(sourcefile, d):
> -            localsrc_prefix = parentdir + "/"
> -        else:
> -            localsrc_prefix = "/usr/src/debug/"
> +        prefixmap = {}
> +        for flag in cflags.split():
> +            if not flag.startswith("-fdebug-prefix-map"):
> +                continue
> +            if "recipe-sysroot" in flag:
> +                continue
> +            flag = flag.split("=")
> +            prefixmap[flag[1]] = flag[2]
>
>          nosuchdir = []
>          basepath = dvar
> @@ -595,28 +585,26 @@ def copydebugsources(debugsrcdir, sources, d):
>          bb.utils.mkdirhier(basepath)
>          cpath.updatecache(basepath)
>
> -        # Ignore files from the recipe sysroots (target and native)
> -        processdebugsrc =  "LC_ALL=C ; sort -z -u '%s' | egrep -v -z
> '((<internal>|<built-in>)$|/.*recipe-sysroot.*/)' | "
> -        # We need to ignore files that are not actually ours
> -        # we do this by only paying attention to items from this package
> -        processdebugsrc += "fgrep -zw '%s' | "
> -        # Remove prefix in the source paths
> -        processdebugsrc += "sed 's#%s##g' | "
> -        processdebugsrc += "(cd '%s' ; cpio -pd0mlL --no-preserve-owner
> '%s%s' 2>/dev/null)"
> -
> -        cmd = processdebugsrc % (sourcefile, basedir, localsrc_prefix,
> parentdir, dvar, debugsrcdir)
> -        try:
> -            subprocess.check_output(cmd, shell=True,
> stderr=subprocess.STDOUT)
> -        except subprocess.CalledProcessError:
> -            # Can "fail" if internal headers/transient sources are
> attempted
> -            pass
> -
> -        # cpio seems to have a bug with -lL together and symbolic links
> are just copied, not dereferenced.
> -        # Work around this by manually finding and copying any symbolic
> links that made it through.
> -        cmd = "find %s%s -type l -print0 -delete | sed s#%s%s/##g | (cd
> '%s' ; cpio -pd0mL --no-preserve-owner '%s%s')" % \
> -                (dvar, debugsrcdir, dvar, debugsrcdir, parentdir, dvar,
> debugsrcdir)
> -        subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
> +        for pmap in prefixmap:
> +            # Ignore files from the recipe sysroots (target and native)
> +            cmd =  "LC_ALL=C ; sort -z -u '%s' | egrep -v -z
> '((<internal>|<built-in>)$|/.*recipe-sysroot.*/)' | " % sourcefile
> +            # We need to ignore files that are not actually ours
> +            # we do this by only paying attention to items from this
> package
> +            cmd += "fgrep -zw '%s' | " % prefixmap[pmap]
> +            # Remove prefix in the source paths
> +            cmd += "sed 's#%s/##g' | " % (prefixmap[pmap])
> +            cmd += "(cd '%s' ; cpio -pd0mlL --no-preserve-owner '%s%s'
> 2>/dev/null)" % (pmap, dvar, prefixmap[pmap])
>
> +            try:
> +                subprocess.check_output(cmd, shell=True,
> stderr=subprocess.STDOUT)
> +            except subprocess.CalledProcessError:
> +                # Can "fail" if internal headers/transient sources are
> attempted
> +                pass
> +            # cpio seems to have a bug with -lL together and symbolic
> links are just copied, not dereferenced.
> +            # Work around this by manually finding and copying any
> symbolic links that made it through.
> +            cmd = "find %s%s -type l -print0 -delete | sed s#%s%s/##g |
> (cd '%s' ; cpio -pd0mL --no-preserve-owner '%s%s')" % \
> +                    (dvar, prefixmap[pmap], dvar, prefixmap[pmap], pmap,
> dvar, prefixmap[pmap])
> +            subprocess.check_output(cmd, shell=True,
> stderr=subprocess.STDOUT)
>
>          # debugsources.list may be polluted from the host if we used
> externalsrc,
>          # cpio uses copy-pass and may have just created a directory
> structure
> --
> 2.34.1
>
>
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#169455):
> https://lists.openembedded.org/g/openembedded-core/message/169455
> Mute This Topic: https://lists.openembedded.org/mt/93068145/3617123
> Group Owner: openembedded-core+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [
> kergoth@gmail.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>
>

Series looks good now to me. I think we should upstream the gcc
patches as well since its
a good change to have everywhere. I am seeing one issue in curl-config
now getting buildpaths [1]
emitted into it, but it could be clang only problem too, I will look
more into it

[1] http://sprunge.us/JmSaGG

On Tue, Aug 16, 2022 at 2:24 PM Christopher Larson <kergoth@gmail.com> wrote:
>
> Looks like a good series, thanks for your work on this. LGTM.
>
> On Tue, Aug 16, 2022 at 1:57 PM Richard Purdie <richard.purdie@linuxfoundation.org> wrote:
>>
>> Reproducible builds are no longer a configuration option but are required.
>> We also rely on the prefix mapping capability of the compilers now.
>>
>> As such, rewrite the source locating code to use the prefix maps instead
>> of taking a guess about WORKDIR which isn't correct for kernels, gcc,
>> externalsrc and probably more.
>>
>> Instead, iterate the maps to locate any matching source code, keeping
>> in mind that multiple maps may map to one target location.
>>
>> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
>> ---
>>  meta/classes-global/package.bbclass | 68 ++++++++++++-----------------
>>  1 file changed, 28 insertions(+), 40 deletions(-)
>>
>> diff --git a/meta/classes-global/package.bbclass b/meta/classes-global/package.bbclass
>> index 418400da8c9..2d985d8affd 100644
>> --- a/meta/classes-global/package.bbclass
>> +++ b/meta/classes-global/package.bbclass
>> @@ -565,26 +565,16 @@ def copydebugsources(debugsrcdir, sources, d):
>>          objcopy = d.getVar("OBJCOPY")
>>          workdir = d.getVar("WORKDIR")
>>          sdir = d.getVar("S")
>> -        sparentdir = os.path.dirname(os.path.dirname(sdir))
>> -        sbasedir = os.path.basename(os.path.dirname(sdir)) + "/" + os.path.basename(sdir)
>> -        workparentdir = os.path.dirname(os.path.dirname(workdir))
>> -        workbasedir = os.path.basename(os.path.dirname(workdir)) + "/" + os.path.basename(workdir)
>> -
>> -        # If S isnt based on WORKDIR we can infer our sources are located elsewhere,
>> -        # e.g. using externalsrc; use S as base for our dirs
>> -        if workdir in sdir or 'work-shared' in sdir:
>> -            basedir = workbasedir
>> -            parentdir = workparentdir
>> -        else:
>> -            basedir = sbasedir
>> -            parentdir = sparentdir
>> +        cflags = d.expand("${CFLAGS}")
>>
>> -        # If build path exists in sourcefile, it means toolchain did not use
>> -        # -fdebug-prefix-map to compile
>> -        if checkbuildpath(sourcefile, d):
>> -            localsrc_prefix = parentdir + "/"
>> -        else:
>> -            localsrc_prefix = "/usr/src/debug/"
>> +        prefixmap = {}
>> +        for flag in cflags.split():
>> +            if not flag.startswith("-fdebug-prefix-map"):
>> +                continue
>> +            if "recipe-sysroot" in flag:
>> +                continue
>> +            flag = flag.split("=")
>> +            prefixmap[flag[1]] = flag[2]
>>
>>          nosuchdir = []
>>          basepath = dvar
>> @@ -595,28 +585,26 @@ def copydebugsources(debugsrcdir, sources, d):
>>          bb.utils.mkdirhier(basepath)
>>          cpath.updatecache(basepath)
>>
>> -        # Ignore files from the recipe sysroots (target and native)
>> -        processdebugsrc =  "LC_ALL=C ; sort -z -u '%s' | egrep -v -z '((<internal>|<built-in>)$|/.*recipe-sysroot.*/)' | "
>> -        # We need to ignore files that are not actually ours
>> -        # we do this by only paying attention to items from this package
>> -        processdebugsrc += "fgrep -zw '%s' | "
>> -        # Remove prefix in the source paths
>> -        processdebugsrc += "sed 's#%s##g' | "
>> -        processdebugsrc += "(cd '%s' ; cpio -pd0mlL --no-preserve-owner '%s%s' 2>/dev/null)"
>> -
>> -        cmd = processdebugsrc % (sourcefile, basedir, localsrc_prefix, parentdir, dvar, debugsrcdir)
>> -        try:
>> -            subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
>> -        except subprocess.CalledProcessError:
>> -            # Can "fail" if internal headers/transient sources are attempted
>> -            pass
>> -
>> -        # cpio seems to have a bug with -lL together and symbolic links are just copied, not dereferenced.
>> -        # Work around this by manually finding and copying any symbolic links that made it through.
>> -        cmd = "find %s%s -type l -print0 -delete | sed s#%s%s/##g | (cd '%s' ; cpio -pd0mL --no-preserve-owner '%s%s')" % \
>> -                (dvar, debugsrcdir, dvar, debugsrcdir, parentdir, dvar, debugsrcdir)
>> -        subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
>> +        for pmap in prefixmap:
>> +            # Ignore files from the recipe sysroots (target and native)
>> +            cmd =  "LC_ALL=C ; sort -z -u '%s' | egrep -v -z '((<internal>|<built-in>)$|/.*recipe-sysroot.*/)' | " % sourcefile
>> +            # We need to ignore files that are not actually ours
>> +            # we do this by only paying attention to items from this package
>> +            cmd += "fgrep -zw '%s' | " % prefixmap[pmap]
>> +            # Remove prefix in the source paths
>> +            cmd += "sed 's#%s/##g' | " % (prefixmap[pmap])
>> +            cmd += "(cd '%s' ; cpio -pd0mlL --no-preserve-owner '%s%s' 2>/dev/null)" % (pmap, dvar, prefixmap[pmap])
>>
>> +            try:
>> +                subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
>> +            except subprocess.CalledProcessError:
>> +                # Can "fail" if internal headers/transient sources are attempted
>> +                pass
>> +            # cpio seems to have a bug with -lL together and symbolic links are just copied, not dereferenced.
>> +            # Work around this by manually finding and copying any symbolic links that made it through.
>> +            cmd = "find %s%s -type l -print0 -delete | sed s#%s%s/##g | (cd '%s' ; cpio -pd0mL --no-preserve-owner '%s%s')" % \
>> +                    (dvar, prefixmap[pmap], dvar, prefixmap[pmap], pmap, dvar, prefixmap[pmap])
>> +            subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
>>
>>          # debugsources.list may be polluted from the host if we used externalsrc,
>>          # cpio uses copy-pass and may have just created a directory structure
>> --
>> 2.34.1
>>
>>
>>
>>
>
>
> --
> Christopher Larson
> chris_larson@mentor.com, chris.larson@siemens.com, kergoth@gmail.com
> Principal Software Engineer, Embedded Linux Solutions, Siemens Digital Industries Software
>
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#169463): https://lists.openembedded.org/g/openembedded-core/message/169463
> Mute This Topic: https://lists.openembedded.org/mt/93068145/1997914
> Group Owner: openembedded-core+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [raj.khem@gmail.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>