From patchwork Thu Jun 29 15:26:54 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Peter Hoyes X-Patchwork-Id: 26665 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 96537EB64D9 for ; Thu, 29 Jun 2023 15:27:19 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web11.52.1688052429521546785 for ; Thu, 29 Jun 2023 08:27:09 -0700 Authentication-Results: mx.groups.io; dkim=missing; spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: peter.hoyes@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 35157C14; Thu, 29 Jun 2023 08:27:52 -0700 (PDT) Received: from e125920.cambridge.arm.com (unknown [10.1.199.64]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id AE7923F64C; Thu, 29 Jun 2023 08:27:07 -0700 (PDT) From: Peter Hoyes To: openembedded-core@lists.openembedded.org Cc: luca.ceresoli@bootlin.com, Ross.Burton@arm.com, richard.purdie@linuxfoundation.org, Peter Hoyes Subject: [PATCH v3] rootfs-postcommands: Set vardeps for do_write_test_data Date: Thu, 29 Jun 2023 16:26:54 +0100 Message-Id: <20230629152654.2048044-1-peter.hoyes@arm.com> X-Mailer: git-send-email 2.34.1 MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 29 Jun 2023 15:27:19 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/183637 From: Peter Hoyes The testdata.json file generated by testdata.bbclass currently contains almost all Bitbake variables. However only a small number of variables are actually used and the testdata.json is not automatically updated when the variables are updated. Introduce the testdata_vars bbclass which defines TESTDATA_EXPORT_VARS to explicitly set the variables to be collected for testdata.json and populate with all variables used by runtime and sdk tests in OE-core. Use this variable to set the vardeps of do_write_test_data. Modify export2json to take an explicit list of keys to export, instead of exporting everything in the datastore. Adapt write_sdk_test_data and meta-ide-support to use the same list of testdata variables. Add a demonstrative OE selftest to rootfspostcommandstests. Signed-off-by: Peter Hoyes --- Changes from v2: * Move TESTDATA_EXPORT_VARS to common testdata_vars.bbclass * Update other export2json call sites (eSDK and meta-ide-support) Tested with: * oe-selftest -r rootfspostcommandstests * oe-selftest -r sstatetests.SStateHashSameSigs * bitbake core-image-sato -c testimage * bitbake core-image-sato -c testsdkext * bitbake buildtools-tarball buildtools-extended-tarball * bitbake meta-ide-support meta/classes-recipe/populate_sdk_base.bbclass | 5 +++- .../rootfs-postcommands.bbclass | 6 +++- meta/classes-recipe/testdata_vars.bbclass | 30 +++++++++++++++++++ meta/lib/oe/data.py | 17 +---------- .../selftest/cases/rootfspostcommandstests.py | 19 ++++++++++++ meta/recipes-core/meta/meta-ide-support.bb | 5 +++- 6 files changed, 63 insertions(+), 19 deletions(-) create mode 100644 meta/classes-recipe/testdata_vars.bbclass diff --git a/meta/classes-recipe/populate_sdk_base.bbclass b/meta/classes-recipe/populate_sdk_base.bbclass index b23ea26d40..ffb06f2062 100644 --- a/meta/classes-recipe/populate_sdk_base.bbclass +++ b/meta/classes-recipe/populate_sdk_base.bbclass @@ -134,11 +134,14 @@ sdk_prune_dirs () { done } +inherit testdata_vars + python write_sdk_test_data() { from oe.data import export2json testdata = "%s/%s.testdata.json" % (d.getVar('SDKDEPLOYDIR'), d.getVar('TOOLCHAIN_OUTPUTNAME')) bb.utils.mkdirhier(os.path.dirname(testdata)) - export2json(d, testdata) + keys = d.getVar('TESTDATA_EXPORT_VARS').split() + export2json(d, keys, testdata) } python write_host_sdk_manifest () { diff --git a/meta/classes-recipe/rootfs-postcommands.bbclass b/meta/classes-recipe/rootfs-postcommands.bbclass index 652601b95f..26eddc8c35 100644 --- a/meta/classes-recipe/rootfs-postcommands.bbclass +++ b/meta/classes-recipe/rootfs-postcommands.bbclass @@ -454,6 +454,8 @@ rootfs_sysroot_relativelinks () { } # Generated test data json file +inherit testdata_vars + python write_image_test_data() { from oe.data import export2json @@ -461,8 +463,9 @@ python write_image_test_data() { link_name = d.getVar('IMAGE_LINK_NAME') testdata_name = os.path.join(deploy_dir, "%s.testdata.json" % d.getVar('IMAGE_NAME')) + keys = d.getVar('TESTDATA_EXPORT_VARS').split() searchString = "%s/"%(d.getVar("TOPDIR")).replace("//","/") - export2json(d, testdata_name, searchString=searchString, replaceString="") + export2json(d, keys, testdata_name, searchString=searchString, replaceString="") if os.path.exists(testdata_name) and link_name: testdata_link = os.path.join(deploy_dir, "%s.testdata.json" % link_name) @@ -471,6 +474,7 @@ python write_image_test_data() { os.remove(testdata_link) os.symlink(os.path.basename(testdata_name), testdata_link) } +write_image_test_data[vardeps] += "${TESTDATA_EXPORT_VARS}" write_image_test_data[vardepsexclude] += "TOPDIR" # Check for unsatisfied recommendations (RRECOMMENDS) diff --git a/meta/classes-recipe/testdata_vars.bbclass b/meta/classes-recipe/testdata_vars.bbclass new file mode 100644 index 0000000000..cecbc6d7ca --- /dev/null +++ b/meta/classes-recipe/testdata_vars.bbclass @@ -0,0 +1,30 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +# The variables exported to testdata.json for testimage and other similar tasks +TESTDATA_EXPORT_VARS ?= "\ + ARCH \ + DEPLOY_DIR \ + DEPLOY_DIR_DEB \ + DISTRO_FEATURES \ + DL_DIR \ + IMAGE \ + IMAGE_FEATURES \ + libdir \ + MACHINE \ + MULTILIB_VARIANTS \ + PACKAGE_FEED_GPG_NAME \ + PTEST_EXPECT_FAILURE \ + QEMU_USE_KVM \ + SDK_DEPLOY \ + SDKPATH \ + T \ + TEST_LOG_DIR \ + TOOLCHAINEXT_OUTPUTNAME \ + TUNE_PKGARCH \ + VIRTUAL-RUNTIME_init_manager \ + WORKDIR \ +" diff --git a/meta/lib/oe/data.py b/meta/lib/oe/data.py index 37121cfad2..731f2b5b46 100644 --- a/meta/lib/oe/data.py +++ b/meta/lib/oe/data.py @@ -23,23 +23,8 @@ def typed_value(key, d): except (TypeError, ValueError) as exc: bb.msg.fatal("Data", "%s: %s" % (key, str(exc))) -def export2json(d, json_file, expand=True, searchString="",replaceString=""): +def export2json(d, keys2export, json_file, expand=True, searchString="",replaceString=""): data2export = {} - keys2export = [] - - for key in d.keys(): - if key.startswith("_"): - continue - elif key.startswith("BB"): - continue - elif key.startswith("B_pn"): - continue - elif key.startswith("do_"): - continue - elif d.getVarFlag(key, "func"): - continue - - keys2export.append(key) for key in keys2export: try: diff --git a/meta/lib/oeqa/selftest/cases/rootfspostcommandstests.py b/meta/lib/oeqa/selftest/cases/rootfspostcommandstests.py index 44e2c09a6f..fec13ce213 100644 --- a/meta/lib/oeqa/selftest/cases/rootfspostcommandstests.py +++ b/meta/lib/oeqa/selftest/cases/rootfspostcommandstests.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: MIT +import json import os import oe import unittest @@ -95,3 +96,21 @@ class ShadowUtilsTidyFiles(OESelftestTestCase): unsorted.append(file) if (unsorted): raise Exception("The following files were not sorted by ID as expected: %s" % unsorted) + + +class TestDataTests(OESelftestTestCase): + def test_vardeps(self): + """ + Test that variables changes are reflected in testdata.json + """ + test_image = "core-image-minimal" + self.write_config('TESTDATA_EXPORT_VARS:append = " TEST_VARIABLE"\nTEST_VARIABLE = "VALUE1"') + bitbake(test_image) + self.append_config('TEST_VARIABLE = "VALUE2"') + bitbake(test_image) + + vars = get_bb_vars(('DEPLOY_DIR_IMAGE', 'IMAGE_LINK_NAME'), test_image) + testdata_json = "%s/%s.testdata.json" % (vars['DEPLOY_DIR_IMAGE'], vars['IMAGE_LINK_NAME']) + with open(testdata_json, 'r') as tf: + testdata_vars = json.load(tf) + self.assertEqual(testdata_vars['TEST_VARIABLE'], 'VALUE2') diff --git a/meta/recipes-core/meta/meta-ide-support.bb b/meta/recipes-core/meta/meta-ide-support.bb index 7f349f673d..3ee796d124 100644 --- a/meta/recipes-core/meta/meta-ide-support.bb +++ b/meta/recipes-core/meta/meta-ide-support.bb @@ -22,13 +22,16 @@ python () { addtask populate_ide_support before do_deploy after do_install +inherit testdata_vars + python do_write_test_data() { from oe.data import export2json out_dir = d.getVar('B') testdata_name = os.path.join(out_dir, "%s.testdata.json" % d.getVar('PN')) - export2json(d, testdata_name) + keys = d.getVar('TESTDATA_EXPORT_VARS').split() + export2json(d, keys, testdata_name) } addtask write_test_data before do_deploy after do_install