From patchwork Wed Feb 28 06:21:39 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Philip Lorenz X-Patchwork-Id: 40199 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 5921CC47DD9 for ; Wed, 28 Feb 2024 06:22:10 +0000 (UTC) Received: from esa8.hc324-48.eu.iphmx.com (esa8.hc324-48.eu.iphmx.com [207.54.65.242]) by mx.groups.io with SMTP id smtpd.web11.7498.1709101328540302000 for ; Tue, 27 Feb 2024 22:22:09 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@bmw.de header.s=mailing1 header.b=QZMpUc6+; spf=pass (domain: bmw.de, ip: 207.54.65.242, mailfrom: prvs=781c072f5=philip.lorenz@bmw.de) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bmw.de; i=@bmw.de; q=dns/txt; s=mailing1; t=1709101328; x=1740637328; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=zKIekixgu1vBG5yu+CWRso3lDgMIF7c6aY6bg3RQ5BI=; b=QZMpUc6+qPSMxKxTRBinCvLaw3VjX0kq/dXhVjwvtsNDmpW2IiJxbDUE 5Xmx7VQPK5Gx8PW0fcZbHKhPm806eZmswRzf00wH60Ts/WHFJvPGoN3ur 6Nhad5cCI6MrQ4dUYG+TZNObjt7GeDlYnWwsOaPC4at3VxbpUGyOt+wff 8=; X-CSE-ConnectionGUID: N8tKU7JqTvmYjexV8BG4gw== X-CSE-MsgGUID: eBJpshRGSBaU5wQ+aNVu9A== Received: from esagw3.bmwgroup.com (HELO esagw3.muc) ([160.46.252.35]) by esa8.hc324-48.eu.iphmx.com with ESMTP/TLS; 28 Feb 2024 07:22:06 +0100 Received: from esabb1.muc ([160.50.100.31]) by esagw3.muc with ESMTP/TLS; 28 Feb 2024 07:22:06 +0100 Received: from smucmp10e.bmwgroup.net (HELO SMUCMP10E.europe.bmw.corp) ([10.30.13.87]) by esabb1.muc with ESMTP/TLS; 28 Feb 2024 07:22:06 +0100 Received: from localhost.localdomain (10.30.85.210) by SMUCMP10E.europe.bmw.corp (10.30.13.87) with Microsoft SMTP Server (version=TLS; Wed, 28 Feb 2024 07:22:06 +0100 From: Philip Lorenz To: CC: Philip Lorenz Subject: [RFC PATCH 1/1] package.bbclass: Expose list of split out debug files Date: Wed, 28 Feb 2024 07:21:39 +0100 Message-ID: <20240228062139.473528-2-philip.lorenz@bmw.de> X-Mailer: git-send-email 2.43.2 In-Reply-To: <20240228062139.473528-1-philip.lorenz@bmw.de> References: <20240228062139.473528-1-philip.lorenz@bmw.de> MIME-Version: 1.0 X-ClientProxiedBy: smucmp18h.europe.bmw.corp (10.30.13.166) To SMUCMP10E.europe.bmw.corp (10.30.13.87) List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Wed, 28 Feb 2024 06:22:10 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/196323 A packaging hook installed via PACKAGEFUNCS may want to access the list of debug symbol files produced during packaging. Correctly determining the list of debug files based on existing variables is non-trivial, so this patch introduces the PKGDEBUGFILES variable which holds the path to all files generated during stripping. Signed-off-by: Philip Lorenz --- meta/classes-global/package.bbclass | 4 ++++ meta/lib/oe/package.py | 19 ++++++++++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/meta/classes-global/package.bbclass b/meta/classes-global/package.bbclass index aa1eb5e901..f2d358459f 100644 --- a/meta/classes-global/package.bbclass +++ b/meta/classes-global/package.bbclass @@ -67,6 +67,10 @@ PACKAGE_DEPENDS += "rpm-native dwarfsrcfiles-native" # tools at rootfs build time. PACKAGE_WRITE_DEPS ??= "" +# List of files containing debug symbols. The paths are rooted at their +# destination path (e.g. /usr/lib/.debug instead of ${PKGD}/usr/lib/.debug) +PKGDEBUGFILES = "" + def legitimize_package_name(s): return oe.package.legitimize_package_name(s) diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py index 587810bdaf..921a958ed1 100644 --- a/meta/lib/oe/package.py +++ b/meta/lib/oe/package.py @@ -781,7 +781,7 @@ def splitdebuginfo(file, dvar, dv, d): # target system binary, the other contains any debugging information. The # two files are linked to reference each other. # - # return a mapping of files:debugsources + # return a mapping of files:debugfile:debugsources src = file[len(dvar):] dest = dv["libdir"] + os.path.dirname(src) + dv["dir"] + "/" + os.path.basename(src) + dv["append"] @@ -791,7 +791,7 @@ def splitdebuginfo(file, dvar, dv, d): if file.endswith(".ko") and file.find("/lib/modules/") != -1: if oe.package.is_kernel_module_signed(file): bb.debug(1, "Skip strip on signed module %s" % file) - return (file, sources) + return (file, file, sources) # Split the file... bb.utils.mkdirhier(os.path.dirname(debugfile)) @@ -821,7 +821,7 @@ def splitdebuginfo(file, dvar, dv, d): if newmode: os.chmod(file, origmode) - return (file, sources) + return (file, debugfile, sources) def splitstaticdebuginfo(file, dvar, dv, d): # Unlike the function above, there is no way to split a static library @@ -830,7 +830,7 @@ def splitstaticdebuginfo(file, dvar, dv, d): # We will then strip (preserving symbols) the static library in the # typical location. # - # return a mapping of files:debugsources + # return a mapping of files:debugfile:debugsources src = file[len(dvar):] dest = dv["staticlibdir"] + os.path.dirname(src) + dv["staticdir"] + "/" + os.path.basename(src) + dv["staticappend"] @@ -861,7 +861,7 @@ def splitstaticdebuginfo(file, dvar, dv, d): if newmode: os.chmod(file, origmode) - return (file, sources) + return (file, debugfile, sources) def inject_minidebuginfo(file, dvar, dv, d): # Extract just the symbols from debuginfo into minidebuginfo, @@ -1175,13 +1175,14 @@ def process_split_and_strip_files(d): results = oe.utils.multiprocess_launch(splitstaticdebuginfo, staticlibs, d, extraargs=(dvar, dv, d)) else: for file in staticlibs: - results.append( (file,source_info(file, d)) ) + results.append( (file,file,source_info(file, d)) ) - d.setVar("PKGDEBUGSOURCES", {strip_pkgd_prefix(f): sorted(s) for f, s in results}) + d.setVar("PKGDEBUGSOURCES", {strip_pkgd_prefix(f): sorted(s) for f, _, s in results}) + d.setVar("PKGDEBUGFILES", [strip_pkgd_prefix(d) for _, d, _ in results]) sources = set() - for r in results: - sources.update(r[1]) + for _, _, sourcefile in results: + sources.update(sourcefile) # Hardlink our debug symbols to the other hardlink copies for ref in inodes: From patchwork Tue Mar 5 16:18:29 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Philip Lorenz X-Patchwork-Id: 40483 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 9C3AEC54E41 for ; Tue, 5 Mar 2024 16:19:35 +0000 (UTC) Received: from esa10.hc324-48.eu.iphmx.com (esa10.hc324-48.eu.iphmx.com [207.54.69.29]) by mx.groups.io with SMTP id smtpd.web11.27959.1709655569000723432 for ; Tue, 05 Mar 2024 08:19:30 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@bmw.de header.s=mailing1 header.b=nmyt+0d2; spf=pass (domain: bmw.de, ip: 207.54.69.29, mailfrom: prvs=787e54d6c=philip.lorenz@bmw.de) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bmw.de; i=@bmw.de; q=dns/txt; s=mailing1; t=1709655568; x=1741191568; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=sbNc9C4bar3q7KPeXsON0n3/fXSHAlJwog1+2wTYLuc=; b=nmyt+0d231Lb5KoP0b2oxbkDcvttXw/paIeavnWqolY1cuwEWsodwVZM OdoQFZ/BiasbEaPaqPvL6vKRBXTT74vvuu4dPwmTtuG2u+F7QbVUoWUWP 76UzZRxA11LVgt8iyN86396WL3KVs5TZctLTx9DMnNT4jAQKk4qSwoGmK s=; X-CSE-ConnectionGUID: v+MnCwzGQemdTPyJ1j1e3A== X-CSE-MsgGUID: JHrdzF3JTEqKC0vg9H5Izw== Received: from esagw2.bmwgroup.com (HELO esagw2.muc) ([160.46.252.38]) by esa10.hc324-48.eu.iphmx.com with ESMTP/TLS; 05 Mar 2024 17:19:26 +0100 Received: from esabb2.muc ([160.50.100.34]) by esagw2.muc with ESMTP/TLS; 05 Mar 2024 17:19:26 +0100 Received: from smucmp10e.bmwgroup.net (HELO SMUCMP10E.europe.bmw.corp) ([10.30.13.87]) by esabb2.muc with ESMTP/TLS; 05 Mar 2024 17:19:26 +0100 Received: from localhost.localdomain (10.30.85.204) by SMUCMP10E.europe.bmw.corp (2a03:1e80:a15:58f::2027) with Microsoft SMTP Server (version=TLS; Tue, 5 Mar 2024 17:19:26 +0100 From: Philip Lorenz To: CC: Philip Lorenz Subject: [RFC PATCH v2 2/3] package: Expose list of split out debug files Date: Tue, 5 Mar 2024 17:18:29 +0100 Message-ID: <20240305161856.4163138-3-philip.lorenz@bmw.de> X-Mailer: git-send-email 2.44.0 In-Reply-To: <20240228062139.473528-1-philip.lorenz@bmw.de> References: <20240228062139.473528-1-philip.lorenz@bmw.de> MIME-Version: 1.0 X-ClientProxiedBy: SMUCMP10F.europe.bmw.corp (2a03:1e80:a15:58f::202c) To SMUCMP10E.europe.bmw.corp (2a03:1e80:a15:58f::2027) List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 05 Mar 2024 16:19:35 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/196638 As correctly determining the list of debug files based on existing variables is non-trivial, this patch introduces the PKGDEBUGFILES variable which holds the path to all files generated during stripping. This list may then be used for further processing, such as extracting the GNU build ID of all files containing debug symbols. Signed-off-by: Philip Lorenz --- meta/classes-global/package.bbclass | 4 ++++ meta/lib/oe/package.py | 19 ++++++++++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/meta/classes-global/package.bbclass b/meta/classes-global/package.bbclass index aa1eb5e901c..f2d358459f8 100644 --- a/meta/classes-global/package.bbclass +++ b/meta/classes-global/package.bbclass @@ -67,6 +67,10 @@ PACKAGE_DEPENDS += "rpm-native dwarfsrcfiles-native" # tools at rootfs build time. PACKAGE_WRITE_DEPS ??= "" +# List of files containing debug symbols. The paths are rooted at their +# destination path (e.g. /usr/lib/.debug instead of ${PKGD}/usr/lib/.debug) +PKGDEBUGFILES = "" + def legitimize_package_name(s): return oe.package.legitimize_package_name(s) diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py index 587810bdafd..921a958ed1f 100644 --- a/meta/lib/oe/package.py +++ b/meta/lib/oe/package.py @@ -781,7 +781,7 @@ def splitdebuginfo(file, dvar, dv, d): # target system binary, the other contains any debugging information. The # two files are linked to reference each other. # - # return a mapping of files:debugsources + # return a mapping of files:debugfile:debugsources src = file[len(dvar):] dest = dv["libdir"] + os.path.dirname(src) + dv["dir"] + "/" + os.path.basename(src) + dv["append"] @@ -791,7 +791,7 @@ def splitdebuginfo(file, dvar, dv, d): if file.endswith(".ko") and file.find("/lib/modules/") != -1: if oe.package.is_kernel_module_signed(file): bb.debug(1, "Skip strip on signed module %s" % file) - return (file, sources) + return (file, file, sources) # Split the file... bb.utils.mkdirhier(os.path.dirname(debugfile)) @@ -821,7 +821,7 @@ def splitdebuginfo(file, dvar, dv, d): if newmode: os.chmod(file, origmode) - return (file, sources) + return (file, debugfile, sources) def splitstaticdebuginfo(file, dvar, dv, d): # Unlike the function above, there is no way to split a static library @@ -830,7 +830,7 @@ def splitstaticdebuginfo(file, dvar, dv, d): # We will then strip (preserving symbols) the static library in the # typical location. # - # return a mapping of files:debugsources + # return a mapping of files:debugfile:debugsources src = file[len(dvar):] dest = dv["staticlibdir"] + os.path.dirname(src) + dv["staticdir"] + "/" + os.path.basename(src) + dv["staticappend"] @@ -861,7 +861,7 @@ def splitstaticdebuginfo(file, dvar, dv, d): if newmode: os.chmod(file, origmode) - return (file, sources) + return (file, debugfile, sources) def inject_minidebuginfo(file, dvar, dv, d): # Extract just the symbols from debuginfo into minidebuginfo, @@ -1175,13 +1175,14 @@ def process_split_and_strip_files(d): results = oe.utils.multiprocess_launch(splitstaticdebuginfo, staticlibs, d, extraargs=(dvar, dv, d)) else: for file in staticlibs: - results.append( (file,source_info(file, d)) ) + results.append( (file,file,source_info(file, d)) ) - d.setVar("PKGDEBUGSOURCES", {strip_pkgd_prefix(f): sorted(s) for f, s in results}) + d.setVar("PKGDEBUGSOURCES", {strip_pkgd_prefix(f): sorted(s) for f, _, s in results}) + d.setVar("PKGDEBUGFILES", [strip_pkgd_prefix(d) for _, d, _ in results]) sources = set() - for r in results: - sources.update(r[1]) + for _, _, sourcefile in results: + sources.update(sourcefile) # Hardlink our debug symbols to the other hardlink copies for ref in inodes: From patchwork Tue Mar 5 16:18:30 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Philip Lorenz X-Patchwork-Id: 40485 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 80B7AC54E41 for ; Tue, 5 Mar 2024 16:19:55 +0000 (UTC) Received: from esa1.hc324-48.eu.iphmx.com (esa1.hc324-48.eu.iphmx.com [207.54.68.119]) by mx.groups.io with SMTP id smtpd.web11.27972.1709655584462247589 for ; Tue, 05 Mar 2024 08:19:47 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@bmw.de header.s=mailing1 header.b=m0CCXrQG; spf=pass (domain: bmw.de, ip: 207.54.68.119, mailfrom: prvs=787e54d6c=philip.lorenz@bmw.de) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bmw.de; i=@bmw.de; q=dns/txt; s=mailing1; t=1709655584; x=1741191584; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=y8PwzUzSHc5sGecDaDNnmrlxv9z3bejaUbZGe3KmFcQ=; b=m0CCXrQGihcdGmGiD3zrM9+NVcnWBxavQWYCapl4MiNZB0kshGx0WcVJ zdsz58aglCbudr1tA3y+I86CaUmrlfDCJ/SpqwvC7en45z0wgmYyd35gL QO5eE9ceSI89oYEjOHyLDnViWSPrl/AB9bESFDNp02bIbE0q4HE+ea3u5 M=; X-CSE-ConnectionGUID: UPEj3ElJSlCQcAORxwnQjA== X-CSE-MsgGUID: oUBuVthhSumWe+On69CQsw== Received: from esagw5.bmwgroup.com (HELO esagw5.muc) ([160.46.252.46]) by esa1.hc324-48.eu.iphmx.com with ESMTP/TLS; 05 Mar 2024 17:19:29 +0100 Received: from esabb4.muc ([160.50.100.33]) by esagw5.muc with ESMTP/TLS; 05 Mar 2024 17:19:26 +0100 Received: from smucmp10e.bmwgroup.net (HELO SMUCMP10E.europe.bmw.corp) ([10.30.13.87]) by esabb4.muc with ESMTP/TLS; 05 Mar 2024 17:19:26 +0100 Received: from localhost.localdomain (10.30.85.204) by SMUCMP10E.europe.bmw.corp (2a03:1e80:a15:58f::2027) with Microsoft SMTP Server (version=TLS; Tue, 5 Mar 2024 17:19:26 +0100 From: Philip Lorenz To: CC: Philip Lorenz Subject: [RFC PATCH v2 3/3] packagedata: Extract GNU build ID during pkgdata creation Date: Tue, 5 Mar 2024 17:18:30 +0100 Message-ID: <20240305161856.4163138-4-philip.lorenz@bmw.de> X-Mailer: git-send-email 2.44.0 In-Reply-To: <20240228062139.473528-1-philip.lorenz@bmw.de> References: <20240228062139.473528-1-philip.lorenz@bmw.de> MIME-Version: 1.0 X-ClientProxiedBy: SMUCMP10F.europe.bmw.corp (2a03:1e80:a15:58f::202c) To SMUCMP10E.europe.bmw.corp (2a03:1e80:a15:58f::2027) List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 05 Mar 2024 16:19:55 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/196640 Extract the GNU build ID from all files containing debug symbols and store it within the "extended" package information of pkgdata as an attribute of the files contained within a package - e.g. { "files_info": { "/lib/.debug/ld-linux-x86-64.so.2": { "gnu_build_id": "a165bf2a6c9c6a0818450293bd6bb66a316eaa4f", "size":1170552 } } } Tools can then consume this data for different purposes such as building an build ID index into the generated package feed or to preseed the database of a debug info server such as debuginfod. Sequentially reading out the GNU build ID of ~8000 debug symbol files (from a core-image-minimal build) took approximately 90 seconds on my machine. Given that the read out in a typical build will be highly parallel, I deemed this figure low enough to simply enable it without an additional configuration flag. Signed-off-by: Philip Lorenz --- meta/lib/oe/packagedata.py | 25 +++++++++++++++++++++++++ meta/lib/oeqa/selftest/cases/pkgdata.py | 2 ++ 2 files changed, 27 insertions(+) diff --git a/meta/lib/oe/packagedata.py b/meta/lib/oe/packagedata.py index 2d1d6ddeb75..3404c2a5cd2 100644 --- a/meta/lib/oe/packagedata.py +++ b/meta/lib/oe/packagedata.py @@ -255,6 +255,27 @@ fi fd.write("PACKAGES: %s\n" % packages) pkgdebugsource = d.getVar("PKGDEBUGSOURCES") or [] + pkgdebugfiles = d.getVar("PKGDEBUGFILES") or [] + + pkgd = d.getVar("PKGD") + readelf = d.getVar("READELF") + + def extract_gnu_build_id(file): + import subprocess + + cmd = "%s -n '%s' | grep '^ Build ID: '" % (readelf, pkgd + file) + try: + result = subprocess.check_output(cmd, shell=True) + # If grep hadn't matched it would've returned a non-zero exit code + # and the CalledProcessError would've been raised. It is therefore + # safe to assume that the output has the format " Build ID: " + gnu_build_id = result[result.rfind(b" ") + 1:].rstrip().decode() + return (file, gnu_build_id) + except subprocess.CalledProcessError: + return (None, None) + + pkg_debug_build_ids = { debug_file: build_id \ + for debug_file, build_id in oe.utils.multiprocess_launch(extract_gnu_build_id, pkgdebugfiles, d) if debug_file} pn = d.getVar('PN') global_variants = (d.getVar('MULTILIB_GLOBAL_VARIANTS') or "").split() @@ -300,6 +321,10 @@ fi if fpath in pkgdebugsource: extended_data["files_info"][fpath]['debugsrc'] = pkgdebugsource[fpath] del pkgdebugsource[fpath] + if fpath in pkg_debug_build_ids: + extended_data["files_info"][fpath]['gnu_build_id'] = pkg_debug_build_ids[fpath] + del pkg_debug_build_ids[fpath] + d.setVar('FILES_INFO:' + pkg , json.dumps(files, sort_keys=True)) diff --git a/meta/lib/oeqa/selftest/cases/pkgdata.py b/meta/lib/oeqa/selftest/cases/pkgdata.py index 6c5b7a84f47..8b993261055 100644 --- a/meta/lib/oeqa/selftest/cases/pkgdata.py +++ b/meta/lib/oeqa/selftest/cases/pkgdata.py @@ -241,3 +241,5 @@ class OePkgdataUtilTests(OESelftestTestCase): file_info = files_info[libz_file_name] self.assertIn('size', file_info, "Couldn't find key 'size' in '%s'" % file_info) + self.assertIn('gnu_build_id', file_info, "Couldn't find key 'gnu_build_id' in '%s'" % file_info) + self.assertGreater(len(file_info['gnu_build_id']), 0)