From patchwork Tue Nov 7 15:26:36 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: jhatnedap@gmail.com X-Patchwork-Id: 34020 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id A380CC4332F for ; Tue, 7 Nov 2023 15:27:44 +0000 (UTC) Received: from mail-ej1-f45.google.com (mail-ej1-f45.google.com [209.85.218.45]) by mx.groups.io with SMTP id smtpd.web10.13757.1699370860672907499 for ; Tue, 07 Nov 2023 07:27:41 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@gmail.com header.s=20230601 header.b=Cje5Shgi; spf=pass (domain: gmail.com, ip: 209.85.218.45, mailfrom: jhatnedap@gmail.com) Received: by mail-ej1-f45.google.com with SMTP id a640c23a62f3a-9ae2cc4d17eso911602466b.1 for ; Tue, 07 Nov 2023 07:27:40 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1699370859; x=1699975659; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:reply-to:references :in-reply-to:message-id:date:subject:cc:to:from:from:to:cc:subject :date:message-id:reply-to; bh=WHUzNc75luqM+RvtqCwEQPKMNhYLJDM89tc8rVRUPhI=; b=Cje5Shgio8ud9pgo9nLiFNKmR90SuNx1YP3viaMkVxSZcmuzQvotnz6shIhS1z91bc paSeQTw+bw7crZK25AEWvQA9p9oSVb6WIfXfZKllXlMH/b+7BdJDZpFhFp5cKyXm6xEA Z+x18xZRklzL/zLfYNK7AAXwd/IE/z2ccv6Wzwp6qrCg8/fH6A0i4EnkFFc4vU34ihD+ eu7kABodxRr0BNyCDSuVjuyG8FGPabbn2aQ3pBIuz3ywsbrPzfdb85PvHu1dKr/iiFXw qfSGgmZ2EwMqVW5Q2LWyxPV4FYthepuQEdNaUvrb35mg9r2ALJ2SaJCn90n81Cmnk4QE r3bQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1699370859; x=1699975659; h=content-transfer-encoding:mime-version:reply-to:references :in-reply-to:message-id:date:subject:cc:to:from:x-gm-message-state :from:to:cc:subject:date:message-id:reply-to; bh=WHUzNc75luqM+RvtqCwEQPKMNhYLJDM89tc8rVRUPhI=; b=U/LLQTSWT9f+gBh4TokzWKasmctaxWg/ucbBRjbScuNgtLWjqhDIJU14nJmcUlnjwK 2lsiz1V4hDnwJxtC1amdQJGWUQXGzj86BS+TpQqE0owJcX3zzDMDri6MxkIYctrt2IBc 938fYgYb6o/mMrQLObYe3XJNm+cPh2TLO5mLp9mCK52ZBao+fg3BJJTJtQyj1DkYghrn f1rHWba6MAZMJuD48W+dIXQn4a8P/dqgakj2b1afET3ijaRzU2OfE9SfQqQoai9HbOvZ +lc3UynI9GUyJuplJtQ1lqQwsGUdFoDvjbFmSsN8hJPhhjDB2vAyiP+27eMie2MNtGvi 3CxQ== X-Gm-Message-State: AOJu0YyIzPuN9JF4G8fcIfQBWqVCFm4BAJyFw3mkMR15FeU4TgeIjNbK jPgaSyceXncG9UvQ1NgGnIvONAgDg2I= X-Google-Smtp-Source: AGHT+IH9ohhzgypMxbIB2iiWyqjG8Txp/0cGBJbblhl/k3yO0i3YAwApCXeAYhmjrxMdsykzpC+2Cg== X-Received: by 2002:a17:907:31ce:b0:9c7:6087:49a7 with SMTP id xf14-20020a17090731ce00b009c7608749a7mr17712264ejb.43.1699370858842; Tue, 07 Nov 2023 07:27:38 -0800 (PST) Received: from NVC4181.nedap.local ([149.3.168.10]) by smtp.gmail.com with ESMTPSA id bx7-20020a170906a1c700b009c5c5c2c5a4sm1146574ejb.219.2023.11.07.07.27.38 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 07 Nov 2023 07:27:38 -0800 (PST) From: jhatnedap@gmail.com X-Google-Original-From: jermain.horsman@nedap.com To: openembedded-core@lists.openembedded.org Cc: Jermain Horsman Subject: [RFC 1/7] scripts/oe-setup-layers: Allow setup using branches instead of fixed revisions Date: Tue, 7 Nov 2023 16:26:36 +0100 Message-ID: <20231107152641.1043-3-jermain.horsman@nedap.com> X-Mailer: git-send-email 2.42.0.windows.2 In-Reply-To: <20231107152641.1043-2-jermain.horsman@nedap.com> References: <20231107152641.1043-2-jermain.horsman@nedap.com> Reply-To: jermain.horsman@nedap.com MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 07 Nov 2023 15:27:44 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/190291 From: Jermain Horsman These changes allow for situations where one or more layers are checked out using a branch instead of a revision, care is taken to make sure this works when using multiple remotes. All changes made are backwards compatible with older setup-layer json files. Signed-off-by: Jermain Horsman --- scripts/oe-setup-layers | 91 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 84 insertions(+), 7 deletions(-) diff --git a/scripts/oe-setup-layers b/scripts/oe-setup-layers index 6d49688a32..ac9a9f139b 100755 --- a/scripts/oe-setup-layers +++ b/scripts/oe-setup-layers @@ -37,6 +37,16 @@ def _is_repo_at_rev(repodir, rev): pass return False +def _is_repo_at_head(repodir): + try: + curr_branch = subprocess.check_output('git -C {repodir} for-each-ref --format=\'%(refname:short)\' "$(git -C {repodir} symbolic-ref -q HEAD)"'.format(repodir=repodir), + shell=True, stderr=subprocess.DEVNULL) + if not curr_branch.strip().decode("utf-8"): + return True + except subprocess.CalledProcessError: + pass + return False + def _is_repo_at_remote_uri(repodir, remote, uri): try: curr_uri = subprocess.check_output("git -C %s remote get-url %s" % (repodir, remote), shell=True, stderr=subprocess.DEVNULL) @@ -46,6 +56,39 @@ def _is_repo_at_remote_uri(repodir, remote, uri): pass return False +def _is_repo_at_default_remote(repodir, default_remote): + try: + curr_default_remote = subprocess.check_output('git -C {repodir} config checkout.defaultRemote'.format(repodir=repodir), shell=True, stderr=subprocess.DEVNULL) + if curr_default_remote.strip().decode("utf-8") == default_remote: + return True + except subprocess.CalledProcessError: + pass + return False + +def _is_repo_at_branch(repodir, branch): + try: + curr_branch = subprocess.check_output('git -C {repodir} for-each-ref --format=\'%(refname:short)\' "$(git -C {repodir} symbolic-ref -q HEAD)"'.format(repodir=repodir), + shell=True, stderr=subprocess.DEVNULL) + if curr_branch.strip().decode("utf-8") == branch: + return True + except subprocess.CalledProcessError: + pass + return False + +def _is_repo_branch_latest(repodir): + try: + curr_rev = subprocess.check_output('git -C {repodir} rev-parse HEAD'.format(repodir=repodir), + shell=True, stderr=subprocess.DEVNULL) + upstream_branch = subprocess.check_output('git -C {repodir} for-each-ref --format=\'%(upstream:short)\' "$(git -C {repodir} symbolic-ref -q HEAD)"'.format(repodir=repodir), + shell=True, stderr=subprocess.DEVNULL) + upstream_rev = subprocess.check_output('git -C {repodir} rev-parse {upstream_branch}'.format(repodir=repodir, upstream_branch=upstream_branch.strip().decode("utf-8")), + shell=True, stderr=subprocess.DEVNULL) + if curr_rev.strip().decode("utf-8") == upstream_rev.strip().decode("utf-8"): + return True + except subprocess.CalledProcessError: + pass + return False + def _contains_submodules(repodir): return os.path.exists(os.path.join(repodir,".gitmodules")) @@ -63,10 +106,17 @@ def _do_checkout(args, json): r_remote = r_data['git-remote'] rev = r_remote['rev'] desc = r_remote['describe'] - if not desc: - desc = rev[:10] branch = r_remote['branch'] remotes = r_remote['remotes'] + default_remote = None + if not rev: + # note: default-remote is required if rev is not set, this key might not exist in older json files though + default_remote = r_remote['default-remote'] + if not desc: + desc = '{}/{}'.format(default_remote, branch) + else: + if not desc: + desc = rev[:10] print('\nSetting up source {}, revision {}, branch {}'.format(r_name, desc, branch)) if not _is_repo_git_repo(repodir): @@ -84,16 +134,43 @@ def _do_checkout(args, json): print("Running '{}' in {}".format(cmd, repodir)) subprocess.check_output(cmd, shell=True, cwd=repodir) - if not _is_repo_at_rev(repodir, rev): + if rev: + if not _is_repo_at_rev(repodir, rev) or not _is_repo_at_head(repodir): + cmd = "git fetch -q --all || true" + print("Running '{}' in {}".format(cmd, repodir)) + subprocess.check_output(cmd, shell=True, cwd=repodir) + + cmd = 'git checkout -q {}'.format(rev) + print("Running '{}' in {}".format(cmd, repodir)) + subprocess.check_output(cmd, shell=True, cwd=repodir) + + if _contains_submodules(repodir): + print("Repo {} contains submodules, use 'git submodule update' to ensure they are up to date".format(repodir)) + else: + check_submodule = False + cmd = "git fetch -q --all || true" print("Running '{}' in {}".format(cmd, repodir)) subprocess.check_output(cmd, shell=True, cwd=repodir) - cmd = 'git checkout -q {}'.format(rev) - print("Running '{}' in {}".format(cmd, repodir)) - subprocess.check_output(cmd, shell=True, cwd=repodir) + if not _is_repo_at_default_remote(repodir, default_remote): + cmd = 'git config checkout.defaultRemote {}'.format(default_remote) + print("Running '{}' in {}".format(cmd, repodir)) + subprocess.check_output(cmd, shell=True, cwd=repodir) + + if not _is_repo_at_branch(repodir, branch): + cmd = 'git checkout -q {}'.format(branch) + print("Running '{}' in {}".format(cmd, repodir)) + subprocess.check_output(cmd, shell=True, cwd=repodir) + check_submodule = True + + if not _is_repo_branch_latest(repodir): + cmd = 'git pull' + print("Running '{}' in {}".format(cmd, repodir)) + subprocess.check_output(cmd, shell=True, cwd=repodir) + check_submodule = True - if _contains_submodules(repodir): + if check_submodule and _contains_submodules(repodir): print("Repo {} contains submodules, use 'git submodule update' to ensure they are up to date".format(repodir)) parser = argparse.ArgumentParser(description="A self contained python script that fetches all the needed layers and sets them to correct revisions using data in a json format from a separate file. The json data can be created from an active build directory with 'bitbake-layers create-layers-setup destdir' and there's a sample file and a schema in meta/files/") From patchwork Tue Nov 7 15:26:37 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: jhatnedap@gmail.com X-Patchwork-Id: 34022 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id A043AC4332F for ; Tue, 7 Nov 2023 15:27:54 +0000 (UTC) Received: from mail-ej1-f49.google.com (mail-ej1-f49.google.com [209.85.218.49]) by mx.groups.io with SMTP id smtpd.web11.13567.1699370864675365944 for ; Tue, 07 Nov 2023 07:27:44 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@gmail.com header.s=20230601 header.b=bXtVg+RQ; spf=pass (domain: gmail.com, ip: 209.85.218.49, mailfrom: jhatnedap@gmail.com) Received: by mail-ej1-f49.google.com with SMTP id a640c23a62f3a-9e2838bcb5eso113643766b.0 for ; Tue, 07 Nov 2023 07:27:44 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1699370863; x=1699975663; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:reply-to:references :in-reply-to:message-id:date:subject:cc:to:from:from:to:cc:subject :date:message-id:reply-to; bh=e/cZ/9Mo+MI4jon5yOj1vuIgOr8TwKVVqvSz7TzeEJU=; b=bXtVg+RQ392SsZGrFv6p859baqENGfIysWD4DVJvgDa+AhZkf6gtRzJ00Uw2Sy+c+O CuXRky9JOTl8lDhwkOdx8Oot27vJKfBAJirBYfJ7hTm2Wwd/9u3StSd2M7fYsV917wcE K3YW5sqZepJVgkNRZftIVIRcTCMvhrhnAKwK0YYK1u/zgGq44Vc0vSws4jDcXCHvu2xh kCGLJps8YeG4C0O2MM6FU2WGwAO53mKtdC8c71v8O+WhyYPJ+cG2JbTmAIVl+avymnZq lDWZ5dtPCF3dhZBp5dMPLTt+H771+vBIw2EnMdsiIJLt8SwEO/gbhI3QZPQWh8CjM7PX bSRQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1699370863; x=1699975663; h=content-transfer-encoding:mime-version:reply-to:references :in-reply-to:message-id:date:subject:cc:to:from:x-gm-message-state :from:to:cc:subject:date:message-id:reply-to; bh=e/cZ/9Mo+MI4jon5yOj1vuIgOr8TwKVVqvSz7TzeEJU=; b=GYtrCEk4tG5dywvA2hxw8JBCLaWcWYAU7SzFdNAfJbUoMUtJFuc0/Jg9d0qdHroPFY Wxg87w6Qq5qlUzTYDYBPWTRa+7na8Hps+v5ptG5ikWKOWV+lixIZbEEavFDviQNg0zOh S9LGnnVkRBQOwSpXGLF00rfyfjqZJa8Gc72uVFngEHAvruQqTRfnvLHIykcU/LmKZ4sA pND67o60jiK7LvqyDr5L6nHNJ12+LRaYCeWofsVx7K1vARHNi53b20SJevkLJcbMPHKR yVtLTkNvWpURvDlYgtmqj7i7+yZxHclbCSK3buogfB9jj/+gQf2p+IRArwQtHarJg70T 2kKA== X-Gm-Message-State: AOJu0YxU1/biBBCFBSw6MiZXfyU1/kTiTmOt2WtGYEWoqLHnkX42ose8 n6JAUscAU76v7Eh7dtMF/j9kUGQWrQc= X-Google-Smtp-Source: AGHT+IHRRIxlGKjwOkMOEzQbEgUlfg3PJhSAMmtTvjcIx2WMD7Xhk0ztROLXFwb4WxsSHLMlI/y+Wg== X-Received: by 2002:a17:906:d54b:b0:9a5:b878:7336 with SMTP id cr11-20020a170906d54b00b009a5b8787336mr18591632ejc.7.1699370863151; Tue, 07 Nov 2023 07:27:43 -0800 (PST) Received: from NVC4181.nedap.local ([149.3.168.10]) by smtp.gmail.com with ESMTPSA id bx7-20020a170906a1c700b009c5c5c2c5a4sm1146574ejb.219.2023.11.07.07.27.42 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 07 Nov 2023 07:27:42 -0800 (PST) From: jhatnedap@gmail.com X-Google-Original-From: jermain.horsman@nedap.com To: openembedded-core@lists.openembedded.org Cc: Jermain Horsman Subject: [RFC 2/7] lib/oe/buildcfg.py: Add additional git functions Date: Tue, 7 Nov 2023 16:26:37 +0100 Message-ID: <20231107152641.1043-4-jermain.horsman@nedap.com> X-Mailer: git-send-email 2.42.0.windows.2 In-Reply-To: <20231107152641.1043-2-jermain.horsman@nedap.com> References: <20231107152641.1043-2-jermain.horsman@nedap.com> Reply-To: jermain.horsman@nedap.com MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 07 Nov 2023 15:27:54 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/190292 From: Jermain Horsman Includes a git describe and git toplevel function and functions to get info for git remotes. Signed-off-by: Jermain Horsman --- meta/lib/oe/buildcfg.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/meta/lib/oe/buildcfg.py b/meta/lib/oe/buildcfg.py index b3fe510309..f9cb0cc74c 100644 --- a/meta/lib/oe/buildcfg.py +++ b/meta/lib/oe/buildcfg.py @@ -28,6 +28,35 @@ def get_metadata_git_revision(path): rev = '' return rev.strip() +def get_metadata_git_toplevel(path): + try: + toplevel, _ = bb.process.run('git rev-parse --show-toplevel', cwd=path) + except bb.process.ExecutionError: + return None + return toplevel.strip() + +def get_metadata_git_remotes(path): + try: + remotes_list, _ = bb.process.run('git remote', cwd=path) + remotes = remotes_list.split() + except bb.process.ExecutionError: + remotes = [] + return remotes + +def get_metadata_git_remote_url(path, remote): + try: + uri, _ = bb.process.run('git remote get-url {remote}'.format(remote=remote), cwd=path) + except bb.process.ExecutionError: + uri = '' + return uri.strip() + +def get_metadata_git_describe(path): + try: + describe, _ = bb.process.run('git describe --tags', cwd=path) + except bb.process.ExecutionError: + return "" + return describe.strip() + def is_layer_modified(path): try: subprocess.check_output("""cd %s; export PSEUDO_UNLOAD=1; set -e; From patchwork Tue Nov 7 15:26:38 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: jhatnedap@gmail.com X-Patchwork-Id: 34021 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id A8322C4167D for ; Tue, 7 Nov 2023 15:27:54 +0000 (UTC) Received: from mail-ej1-f46.google.com (mail-ej1-f46.google.com [209.85.218.46]) by mx.groups.io with SMTP id smtpd.web10.13762.1699370871546965525 for ; Tue, 07 Nov 2023 07:27:51 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@gmail.com header.s=20230601 header.b=K6wBbjwy; spf=pass (domain: gmail.com, ip: 209.85.218.46, mailfrom: jhatnedap@gmail.com) Received: by mail-ej1-f46.google.com with SMTP id a640c23a62f3a-9a6190af24aso891705366b.0 for ; Tue, 07 Nov 2023 07:27:51 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1699370870; x=1699975670; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:reply-to:references :in-reply-to:message-id:date:subject:cc:to:from:from:to:cc:subject :date:message-id:reply-to; bh=RFvbbG1puZVRW/plYLyDK792tCE0Y7pCAM6JPh/jJiY=; b=K6wBbjwyFbIypSKsRFI6r1YAZgSUolEsv/TyW48DNeWo/BQu27eh+qt908iOjiEdGj eTq40eBTWo2itGRbaIxbv1kzdm2i4vsicbAok60vykxe9qWz5ivwjjLnOZToj8j8S+Di PpVNBB2V8SF6uWc0eO1kt7qzMIkNw9pJ3BtVOLhdIlb3LrkWl/wBEBklAxnVBctYCF0+ XeBX7dZnW+S80UTnjnt39vh9Qv+WBlmu80P4hFuuVS/IqP9OkW2LROC656T3imHJUQvu HoLxUQZg5QA6mqtefNNcAKZRL7cOY1nJhnKo1doTRkkl0jQqYm2eJVUzO8qy+ZN/dVJQ ntXw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1699370870; x=1699975670; h=content-transfer-encoding:mime-version:reply-to:references :in-reply-to:message-id:date:subject:cc:to:from:x-gm-message-state :from:to:cc:subject:date:message-id:reply-to; bh=RFvbbG1puZVRW/plYLyDK792tCE0Y7pCAM6JPh/jJiY=; b=uF6gt8SUdLxfZCXIM2f5LSF03Aghm6x+uitlFsto5EwsNP1YtLRV3Sh+fkt53MwKrM AILZ7+YcES7cNI1/cuo4tV/6OAANb+KY5ZfJibpP9D6M+2j9xuGUtwScvYb54OE1S0XZ onXLPWT3jP+9VIzhDgZwTvQvSW3WhsMgjOO2nfiYt3+mUrunwYWWk6l0Vel2M3KALfk2 b0fToG/NdKV/HtaKNW8be0cDpVBOA4KLRl/CKKjwQ3DEU2uzn9xptSkEtAAO4//6YGn6 xWQcr8w8rOopXdTP7W0174in/f7dhdv/0YdxAIWGY7rmsdjWSPd63hYK+IZRl9frrAC5 YH0w== X-Gm-Message-State: AOJu0YxhF73RWqGiHRcJESbvYfc8Xvr0GwGwCslnrETsqLpVP+b06UwG G1agBT1Z0rIT2o5mMaT7B1myPhHMhgg= X-Google-Smtp-Source: AGHT+IFutcOM7dX8FNYZeIHLwg0W7Pou5SVxud+2BWScjQA1vqx/vhKa/rw8837Q2DPgznd3iE1qfQ== X-Received: by 2002:a17:906:fd89:b0:9ce:24d0:8a01 with SMTP id xa9-20020a170906fd8900b009ce24d08a01mr16919403ejb.60.1699370870036; Tue, 07 Nov 2023 07:27:50 -0800 (PST) Received: from NVC4181.nedap.local ([149.3.168.10]) by smtp.gmail.com with ESMTPSA id bx7-20020a170906a1c700b009c5c5c2c5a4sm1146574ejb.219.2023.11.07.07.27.49 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 07 Nov 2023 07:27:49 -0800 (PST) From: jhatnedap@gmail.com X-Google-Original-From: jermain.horsman@nedap.com To: openembedded-core@lists.openembedded.org Cc: Jermain Horsman Subject: [RFC 3/7] bitbake-layers: Add ability to update layer repository configuration Date: Tue, 7 Nov 2023 16:26:38 +0100 Message-ID: <20231107152641.1043-5-jermain.horsman@nedap.com> X-Mailer: git-send-email 2.42.0.windows.2 In-Reply-To: <20231107152641.1043-2-jermain.horsman@nedap.com> References: <20231107152641.1043-2-jermain.horsman@nedap.com> Reply-To: jermain.horsman@nedap.com MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 07 Nov 2023 15:27:54 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/190293 From: Jermain Horsman This allows for modifications of an existing configuration file using current revisions. Signed-off-by: Jermain Horsman --- meta/lib/bblayers/updatesetup.py | 78 ++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 meta/lib/bblayers/updatesetup.py diff --git a/meta/lib/bblayers/updatesetup.py b/meta/lib/bblayers/updatesetup.py new file mode 100644 index 0000000000..1f8760f2d3 --- /dev/null +++ b/meta/lib/bblayers/updatesetup.py @@ -0,0 +1,78 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import json +import logging +import os +import sys + +from bblayers.common import LayerPlugin + +logger = logging.getLogger('bitbake-layers') + +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +import oe.buildcfg + +def plugin_init(plugins): + return UpdateSetupPlugin() + +class UpdateSetupPlugin(LayerPlugin): + + def get_repo_path(self, args): + layers = (self.tinfoil.config_data.getVar("BBLAYERS") or "").split() + for layer in layers: + if os.path.basename(layer) == 'workspace': + continue + + repo_path = oe.buildcfg.get_metadata_git_toplevel(layer) + if os.path.basename(repo_path) != args.repo: + continue + + return repo_path + + def do_update_setup(self, args): + """ Modify an existing configuration file using current revisions. """ + with open(args.json, "r+") as json_file: + setup_layers_json = json.load(json_file) + if not 'sources' in setup_layers_json.keys(): + logger.error("File does not contain valid setup-layers config") + return + + sources = setup_layers_json['sources'] + if not args.repo in sources.keys(): + logger.error("Repository {} does not exist in setup-layers config".format(args.repo)) + return + + repo = self.get_repo_path(args) + + branch = oe.buildcfg.get_metadata_git_branch(repo) + + if args.branch: + rev = '' + describe = '' + else: + rev = oe.buildcfg.get_metadata_git_revision(repo) + describe = oe.buildcfg.get_metadata_git_describe(repo) + + layer_remote = setup_layers_json['sources'][args.repo]['git-remote'] + layer_remote['branch'] = branch + layer_remote['rev'] = rev + layer_remote['describe'] = describe + + json_file.seek(0) + json_file.truncate() + json.dump(setup_layers_json, json_file, sort_keys=True, indent=4) + + def register_commands(self, sp): + parser_update_layers = self.add_command(sp, 'update-layers-setup', self.do_update_setup, parserecipes=False) + parser_update_layers.add_argument('json', + help='File location of the setup-layers json config.') + parser_update_layers.add_argument('repo', + help='Name of the repository to be updated.') + parser_update_layers.add_argument('--branch', '-b', + help='store branch', + action='store_true') From patchwork Tue Nov 7 15:26:39 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: jhatnedap@gmail.com X-Patchwork-Id: 34023 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id A4B28C4167B for ; Tue, 7 Nov 2023 15:28:04 +0000 (UTC) Received: from mail-ej1-f42.google.com (mail-ej1-f42.google.com [209.85.218.42]) by mx.groups.io with SMTP id smtpd.web11.13572.1699370874267993966 for ; Tue, 07 Nov 2023 07:27:54 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@gmail.com header.s=20230601 header.b=Zrt6DWwN; spf=pass (domain: gmail.com, ip: 209.85.218.42, mailfrom: jhatnedap@gmail.com) Received: by mail-ej1-f42.google.com with SMTP id a640c23a62f3a-9c53e8b7cf4so876847066b.1 for ; Tue, 07 Nov 2023 07:27:54 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1699370873; x=1699975673; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:reply-to:references :in-reply-to:message-id:date:subject:cc:to:from:from:to:cc:subject :date:message-id:reply-to; bh=0MzlrWhoEy88S2X7kCj/GVUa6prl8cejg2XsuMDScj8=; b=Zrt6DWwN6DKjnZlBlbpb4szR248Nd5X5GrkAWf3dh/hTuKr9nLgBUU1SiBjRJ9Yf34 bWyvn+yiNz9ST0WWT6e4rPBtXoQeBIxG2jah4yKXxKAQ0NYsxsPNjC2PdCcEp+GtLSGK 1u5KZYqFDRdq4cMYEGeber0EtZ6vB7w0w7UvmGDltYjKN+5mHgVpxs2boqulKeII1Uy9 9u7ICz2WvdirDMVcLVgW0uI/spSExKqupGuy8RM+8YnB0XYb69SQcOT8AjBv6eukEQ+s UZtEXUpfNukIc9q4UrpoCkf0WdnrCjhdYwlLpYn9dc45o0TTWQd4ZcuVfbKhiSWGsZYr b/mQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1699370873; x=1699975673; h=content-transfer-encoding:mime-version:reply-to:references :in-reply-to:message-id:date:subject:cc:to:from:x-gm-message-state :from:to:cc:subject:date:message-id:reply-to; bh=0MzlrWhoEy88S2X7kCj/GVUa6prl8cejg2XsuMDScj8=; b=cuOiC8ishXQJkiNPm/vxjK7rP+6ODvWgVkltTs6AqXVUb2keXg99TKWj/vDSi5wiQU dJ92eQUrTL6dydEbVa/TXSkK+5w45FRFzHN0Aq4qCTnUUufcr7aWJI6y6I4QTdrwXe3y 7T0N9UGVor87cGYPPqa22+gdDu0QkRBR1k5JQwKxnJt9BSG3ssNAq9VQWxszDTddZWrl mUp/pwRqinvYUgBnaHilSHxT1P8xsjEc0Oc4v5aFV48AJqf+qwFItRVmTn+3bNpCHQoa fH6lBlPq01Y/vK+6Z/Rj5rPtBm5eo6zyHHJlLMKo+eIHGinlRdyZBQPAlwkDbr0YXbHK 75SQ== X-Gm-Message-State: AOJu0YwB/mALvvlUbVXOmzy66tn6YWSUh4yKHGR0saBYphtUrkqn08HW TTUxhBMHs1hfaJkIv8Yv5R2ClwfvgzU= X-Google-Smtp-Source: AGHT+IFmM2QwKCKI7tC/Tg4mAEFL0q0fAROJbCSDqhxCX4+6D2vt+zpQZG6r0QnK2QVmP7bvbOEl8A== X-Received: by 2002:a17:907:1c86:b0:9df:bc50:2508 with SMTP id nb6-20020a1709071c8600b009dfbc502508mr6662457ejc.57.1699370872728; Tue, 07 Nov 2023 07:27:52 -0800 (PST) Received: from NVC4181.nedap.local ([149.3.168.10]) by smtp.gmail.com with ESMTPSA id bx7-20020a170906a1c700b009c5c5c2c5a4sm1146574ejb.219.2023.11.07.07.27.52 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 07 Nov 2023 07:27:52 -0800 (PST) From: jhatnedap@gmail.com X-Google-Original-From: jermain.horsman@nedap.com To: openembedded-core@lists.openembedded.org Cc: Jermain Horsman Subject: [RFC 4/7] bblayers/makesetup.py: Use new functions from oe.buildcfg Date: Tue, 7 Nov 2023 16:26:39 +0100 Message-ID: <20231107152641.1043-6-jermain.horsman@nedap.com> X-Mailer: git-send-email 2.42.0.windows.2 In-Reply-To: <20231107152641.1043-2-jermain.horsman@nedap.com> References: <20231107152641.1043-2-jermain.horsman@nedap.com> Reply-To: jermain.horsman@nedap.com MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 07 Nov 2023 15:28:04 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/190294 From: Jermain Horsman Refactor to use git functions from library. Signed-off-by: Jermain Horsman --- meta/lib/bblayers/makesetup.py | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/meta/lib/bblayers/makesetup.py b/meta/lib/bblayers/makesetup.py index 5fb6f1469e..d24470dc83 100644 --- a/meta/lib/bblayers/makesetup.py +++ b/meta/lib/bblayers/makesetup.py @@ -26,25 +26,12 @@ def plugin_init(plugins): class MakeSetupPlugin(LayerPlugin): - def _get_repo_path(self, layer_path): - repo_path, _ = bb.process.run('git rev-parse --show-toplevel', cwd=layer_path) - return repo_path.strip() - - def _get_remotes(self, repo_path): + def _get_remotes_with_url(self, repo_path): remotes = {} - remotes_list,_ = bb.process.run('git remote', cwd=repo_path) - for r in remotes_list.split(): - uri,_ = bb.process.run('git remote get-url {r}'.format(r=r), cwd=repo_path) - remotes[r] = {'uri':uri.strip()} + for r in oe.buildcfg.get_metadata_git_remotes(repo_path): + remotes[r] = {'uri':oe.buildcfg.get_metadata_git_remote_url(repo_path, r)} return remotes - def _get_describe(self, repo_path): - try: - describe,_ = bb.process.run('git describe --tags', cwd=repo_path) - except bb.process.ExecutionError: - return "" - return describe.strip() - def _is_submodule(self, repo_path): # This is slightly brittle: git does not offer a way to tell whether # a given repo dir is a submodule checkout, so we need to rely on .git @@ -58,10 +45,7 @@ class MakeSetupPlugin(LayerPlugin): available here. """ repos = {} layers = oe.buildcfg.get_layer_revisions(self.tinfoil.config_data) - try: - destdir_repo = self._get_repo_path(destdir) - except bb.process.ExecutionError: - destdir_repo = None + destdir_repo = oe.buildcfg.get_metadata_git_toplevel(destdir) for (l_path, l_name, l_branch, l_rev, l_ismodified) in layers: if l_name == 'workspace': @@ -69,12 +53,16 @@ class MakeSetupPlugin(LayerPlugin): if l_ismodified: logger.error("Layer {name} in {path} has uncommitted modifications or is not in a git repository.".format(name=l_name,path=l_path)) return - repo_path = self._get_repo_path(l_path) + repo_path = oe.buildcfg.get_metadata_git_toplevel(l_path) if self._is_submodule(repo_path): continue if repo_path not in repos.keys(): - repos[repo_path] = {'path':os.path.basename(repo_path),'git-remote':{'rev':l_rev, 'branch':l_branch, 'remotes':self._get_remotes(repo_path), 'describe':self._get_describe(repo_path)}} + repos[repo_path] = {'path':os.path.basename(repo_path),'git-remote':{ + 'rev':l_rev, + 'branch':l_branch, + 'remotes':self._get_remotes_with_url(repo_path), + 'describe':oe.buildcfg.get_metadata_git_describe(repo_path)}} if repo_path == destdir_repo: repos[repo_path]['contains_this_file'] = True if not repos[repo_path]['git-remote']['remotes'] and not repos[repo_path]['contains_this_file']: From patchwork Tue Nov 7 15:26:40 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: jhatnedap@gmail.com X-Patchwork-Id: 34025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id C2417C00141 for ; Tue, 7 Nov 2023 15:28:04 +0000 (UTC) Received: from mail-ej1-f53.google.com (mail-ej1-f53.google.com [209.85.218.53]) by mx.groups.io with SMTP id smtpd.web10.13765.1699370876337448445 for ; Tue, 07 Nov 2023 07:27:56 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@gmail.com header.s=20230601 header.b=GIiM3sDo; spf=pass (domain: gmail.com, ip: 209.85.218.53, mailfrom: jhatnedap@gmail.com) Received: by mail-ej1-f53.google.com with SMTP id a640c23a62f3a-9d10972e63eso862339466b.2 for ; Tue, 07 Nov 2023 07:27:56 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1699370875; x=1699975675; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:reply-to:references :in-reply-to:message-id:date:subject:cc:to:from:from:to:cc:subject :date:message-id:reply-to; bh=jEov84liuOIYpydN2gCeN6fg+lXT8deOyhMLeA1Of9o=; b=GIiM3sDorlaXOBa1KyuC1R8u1XxGR6MIUSHw6tBoD7HD5JBAblRzCDTw2l47s7tlK3 TbnnRTnKUPp41qAcrmISanjtDYMIUMGTON3ucFeLfoXprEMhM70WTyJxfsnD7NbSBpvj +68oP0togjHpfWsC6OSXO5rPMQwIqDdv7ts/tYI720yD8OMFhxGvdhHooWvYrS51E0sy acVKjwZ7/AHKLarq/FDxiXF3Mkj/vWcJ7Ex8fLR+QFa2PyDd2GNkeuAi2OIXeJkChgfj F82V0EEfJjjquAx5MGvOQtsMA2daQNShaDpLDLVZ85gfHM0XT6FkxgtTxyBUEokeQKKB 7yTA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1699370875; x=1699975675; h=content-transfer-encoding:mime-version:reply-to:references :in-reply-to:message-id:date:subject:cc:to:from:x-gm-message-state :from:to:cc:subject:date:message-id:reply-to; bh=jEov84liuOIYpydN2gCeN6fg+lXT8deOyhMLeA1Of9o=; b=tahCsTSb7yei/U5gndNooLWXJlGT/OSGj9yNJ6V8s/962gd0rxslJzjCa74fx6391L 4vAkDolB4xs6kMGInd8Gjahw4ndy8VUpTDRIWLGHyvyCanDgUVTiFccSCD8/Q+jJz1VE KbvIFNjSDTEzp6EgSvGXSlbSKntEheFQ6gae1xt14dCVfS07hLWY4RIyry7C62VkBR4Y 6xkVn5KtYGpYkGjeS4+bNRGHvFiSwGQqyDbQFfl06rSFVKyjCH5AGPAqu2Ex3G82crbg EYXyf6AnwWxklyvtEYvG68R4nfVoddO8hanr9mLrNlU2LnUSrdu6+l14eV5crg2u9Q06 tP/w== X-Gm-Message-State: AOJu0YycnwZnimzxEFYCiOAge+qIWnPzhh633Z+6HAs1MeDmTZZ6GYpe vqjKg9wrroczBGTc00L1xQtQ62wRDu4= X-Google-Smtp-Source: AGHT+IGC5Jz3slPl3vu1WeA3jf/Fh+Dt94VWBWGYhPLDui7sTatv2tVqdkfWS7Lq9HBTjSsjx3xrYQ== X-Received: by 2002:a17:906:ef0e:b0:9c7:4d3e:1e50 with SMTP id f14-20020a170906ef0e00b009c74d3e1e50mr19752804ejs.76.1699370874815; Tue, 07 Nov 2023 07:27:54 -0800 (PST) Received: from NVC4181.nedap.local ([149.3.168.10]) by smtp.gmail.com with ESMTPSA id bx7-20020a170906a1c700b009c5c5c2c5a4sm1146574ejb.219.2023.11.07.07.27.54 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 07 Nov 2023 07:27:54 -0800 (PST) From: jhatnedap@gmail.com X-Google-Original-From: jermain.horsman@nedap.com To: openembedded-core@lists.openembedded.org Cc: Jermain Horsman Subject: [RFC 5/7] lib/oe/buildcfg.py: Add additional git functions Date: Tue, 7 Nov 2023 16:26:40 +0100 Message-ID: <20231107152641.1043-7-jermain.horsman@nedap.com> X-Mailer: git-send-email 2.42.0.windows.2 In-Reply-To: <20231107152641.1043-2-jermain.horsman@nedap.com> References: <20231107152641.1043-2-jermain.horsman@nedap.com> Reply-To: jermain.horsman@nedap.com MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 07 Nov 2023 15:28:04 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/190295 From: Jermain Horsman Includes a function to determine the default remote. Signed-off-by: Jermain Horsman --- meta/lib/oe/buildcfg.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/meta/lib/oe/buildcfg.py b/meta/lib/oe/buildcfg.py index f9cb0cc74c..2ca654e56e 100644 --- a/meta/lib/oe/buildcfg.py +++ b/meta/lib/oe/buildcfg.py @@ -50,6 +50,22 @@ def get_metadata_git_remote_url(path, remote): uri = '' return uri.strip() +def get_metadata_git_default_remote(path): + remotes = get_metadata_git_remotes(path) + if len(remotes) == 1: + default_remote = remotes[0] + else: + try: + default_remote, _ = bb.process.run('git config --local checkout.defaultRemote', cwd=path) + except bb.process.ExecutionError: + default_remote = "" + if not default_remote: + try: + default_remote, _ = bb.process.run('git config checkout.defaultRemote', cwd=path) + except bb.process.ExecutionError: + default_remote = "" + return default_remote.strip() + def get_metadata_git_describe(path): try: describe, _ = bb.process.run('git describe --tags', cwd=path) From patchwork Tue Nov 7 15:26:41 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: jhatnedap@gmail.com X-Patchwork-Id: 34024 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 9F87EC4332F for ; Tue, 7 Nov 2023 15:28:04 +0000 (UTC) Received: from mail-ej1-f51.google.com (mail-ej1-f51.google.com [209.85.218.51]) by mx.groups.io with SMTP id smtpd.web11.13576.1699370878491875333 for ; Tue, 07 Nov 2023 07:27:58 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@gmail.com header.s=20230601 header.b=CghnVDuO; spf=pass (domain: gmail.com, ip: 209.85.218.51, mailfrom: jhatnedap@gmail.com) Received: by mail-ej1-f51.google.com with SMTP id a640c23a62f3a-9e213f198dfso158277566b.2 for ; Tue, 07 Nov 2023 07:27:58 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1699370877; x=1699975677; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:reply-to:references :in-reply-to:message-id:date:subject:cc:to:from:from:to:cc:subject :date:message-id:reply-to; bh=0FLSRWZTjFkpTUJKPjPtD5DTeSfoel+H42Tsh2HXR0w=; b=CghnVDuO4NVYdHTcHofuy8uQXkrQogvCZ0RRS3eJ11kq1M8AfGAc3K+pwQvD6IVsyz nrB+7AwdkojC/pzHQryLcRfR7rFgPVZELFs78WTuaSJZc8mpVyAK/y8ueNpY1GCWUuwe br+GSyyjNQs74yJ3550GiV23JKARXuon7WzaqCcbHY0mfAg1ZYF2CAgOLE20xWhcTsl+ gymo7aXjKQGA0/htkRZT6AJ2AJ1Jyb0XcWzucoIkYgB6HZnLgf26vsEYe5JRmuyxd7As UR9Xi14dv4yr7LCjL149ez0Tm7Z3+IVCJcS2achMUntesvoCFrIhXQLSUS6bqPvBlmyE e2CQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1699370877; x=1699975677; h=content-transfer-encoding:mime-version:reply-to:references :in-reply-to:message-id:date:subject:cc:to:from:x-gm-message-state :from:to:cc:subject:date:message-id:reply-to; bh=0FLSRWZTjFkpTUJKPjPtD5DTeSfoel+H42Tsh2HXR0w=; b=KhJFYABlmz7QLT+iqJEpDaPR59SGL+QOjbrlYcHcDH6gOlr/W2G500ze7/D/kgKr33 JaPT1vAKjEH+6d6E2ainpSnw4agg1mHYCDxWa7T34lDZJkVLSIjFqf3oAv+K2MtLyfpT shYk+fTkWR2qDH05O7bvMGPpP/FN6ANNnhWWCCyzmovKoO8U43fjDYpOqQb85Z5RYGHR O61Z7xR0D186m1PlkjltS5bddHhLO18Z2dQVfC/u3DtWs1n8o0BSaiVF1YkLe6FAupY8 p1fO24SwhacPiG9K/ana1uCZp2BQjDjcg/CvjbPxrn5Mf+Q8DTiR1i+hVEPv0tcW++Ip Oh+g== X-Gm-Message-State: AOJu0YxckX8qsIhF9RFndNMfYy4WQzFhPRfU+0Xx2/LKjn6Z6ar71irw AzomHZeGPLupIAPTOhg23Klk5OiHII4= X-Google-Smtp-Source: AGHT+IHDwELP2djgRJbLe6TSqS4NqEGKSoXZRPLMQwAV3/7gOXSGCcZTL37rN2od+s4cCZKdtNrevg== X-Received: by 2002:a17:907:7255:b0:9bf:10f3:e435 with SMTP id ds21-20020a170907725500b009bf10f3e435mr16467778ejc.1.1699370876964; Tue, 07 Nov 2023 07:27:56 -0800 (PST) Received: from NVC4181.nedap.local ([149.3.168.10]) by smtp.gmail.com with ESMTPSA id bx7-20020a170906a1c700b009c5c5c2c5a4sm1146574ejb.219.2023.11.07.07.27.56 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 07 Nov 2023 07:27:56 -0800 (PST) From: jhatnedap@gmail.com X-Google-Original-From: jermain.horsman@nedap.com To: openembedded-core@lists.openembedded.org Cc: Jermain Horsman Subject: [RFC 6/7] bitbake-layers: Add default remote to layers-setup.json Date: Tue, 7 Nov 2023 16:26:41 +0100 Message-ID: <20231107152641.1043-8-jermain.horsman@nedap.com> X-Mailer: git-send-email 2.42.0.windows.2 In-Reply-To: <20231107152641.1043-2-jermain.horsman@nedap.com> References: <20231107152641.1043-2-jermain.horsman@nedap.com> Reply-To: jermain.horsman@nedap.com MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 07 Nov 2023 15:28:04 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/190296 From: Jermain Horsman Having a default remote present allows for the checkout of a branch in the case multiple remotes are available. This requires an update to the json-schema and associated checks. Signed-off-by: Jermain Horsman --- meta/files/layers.example.json | 5 ++++- meta/files/layers.schema.json | 8 ++++++-- meta/lib/bblayers/makesetup.py | 1 + meta/lib/bblayers/setupwriters/oe-setup-layers.py | 2 +- scripts/oe-setup-layers | 2 +- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/meta/files/layers.example.json b/meta/files/layers.example.json index 0a6a6a7b48..06ba196cf6 100644 --- a/meta/files/layers.example.json +++ b/meta/files/layers.example.json @@ -4,6 +4,7 @@ "contains_this_file": true, "git-remote": { "branch": "master", + "default-remote": "remote-alex", "describe": "", "remotes": { "remote-alex": { @@ -17,6 +18,7 @@ "meta-intel": { "git-remote": { "branch": "master", + "default-remote": "origin", "describe": "15.0-hardknott-3.3-310-g0a96edae", "remotes": { "origin": { @@ -30,6 +32,7 @@ "poky": { "git-remote": { "branch": "akanavin/setup-layers", + "default-remote": "poky-contrib", "describe": "4.1_M1-374-g9dda719b2a", "remotes": { "origin": { @@ -44,5 +47,5 @@ "path": "poky" } }, - "version": "1.0" + "version": "1.1" } diff --git a/meta/files/layers.schema.json b/meta/files/layers.schema.json index 659ee8da49..06a34656e5 100644 --- a/meta/files/layers.schema.json +++ b/meta/files/layers.schema.json @@ -7,8 +7,8 @@ ], "properties": { "version": { - "description": "The version of this document; currently '1.0'", - "enum": ["1.0"] + "description": "The version of this document; currently '1.1'", + "enum": ["1.1"] }, "sources": { "description": "The dict of layer sources", @@ -41,6 +41,10 @@ "description": "The git branch to fetch (optional)", "type": "string" }, + "default-remote": { + "description": "The default git remote", + "type": "string" + }, "rev": { "description": "The git revision to checkout", "type": "string" diff --git a/meta/lib/bblayers/makesetup.py b/meta/lib/bblayers/makesetup.py index d24470dc83..0d0e42e1d0 100644 --- a/meta/lib/bblayers/makesetup.py +++ b/meta/lib/bblayers/makesetup.py @@ -62,6 +62,7 @@ class MakeSetupPlugin(LayerPlugin): 'rev':l_rev, 'branch':l_branch, 'remotes':self._get_remotes_with_url(repo_path), + 'default-remote':oe.buildcfg.get_metadata_git_default_remote(repo_path), 'describe':oe.buildcfg.get_metadata_git_describe(repo_path)}} if repo_path == destdir_repo: repos[repo_path]['contains_this_file'] = True diff --git a/meta/lib/bblayers/setupwriters/oe-setup-layers.py b/meta/lib/bblayers/setupwriters/oe-setup-layers.py index d5bc19a8cb..4e0c780eab 100644 --- a/meta/lib/bblayers/setupwriters/oe-setup-layers.py +++ b/meta/lib/bblayers/setupwriters/oe-setup-layers.py @@ -36,7 +36,7 @@ class OeSetupLayersWriter(): if not os.path.exists(args.destdir): os.makedirs(args.destdir) repos = parent.make_repo_config(args.destdir) - json = {"version":"1.0","sources":repos} + json = {"version":"1.1","sources":repos} if not repos: raise Exception("Could not determine layer sources") output = args.output_prefix or "setup-layers" diff --git a/scripts/oe-setup-layers b/scripts/oe-setup-layers index ac9a9f139b..4622c264f9 100755 --- a/scripts/oe-setup-layers +++ b/scripts/oe-setup-layers @@ -191,7 +191,7 @@ args = parser.parse_args() with open(args.jsondata) as f: json_f = json.load(f) -supported_versions = ["1.0"] +supported_versions = ["1.0", "1.1"] if json_f["version"] not in supported_versions: raise Exception("File {} has version {}, which is not in supported versions: {}".format(args.jsondata, json_f["version"], supported_versions)) From patchwork Tue Nov 7 15:26:42 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: jhatnedap@gmail.com X-Patchwork-Id: 34026 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id B1B2EC4167D for ; Tue, 7 Nov 2023 15:28:04 +0000 (UTC) Received: from mail-ej1-f46.google.com (mail-ej1-f46.google.com [209.85.218.46]) by mx.groups.io with SMTP id smtpd.web10.13767.1699370880679359769 for ; Tue, 07 Nov 2023 07:28:01 -0800 Authentication-Results: mx.groups.io; dkim=pass header.i=@gmail.com header.s=20230601 header.b=AZJbbpqb; spf=pass (domain: gmail.com, ip: 209.85.218.46, mailfrom: jhatnedap@gmail.com) Received: by mail-ej1-f46.google.com with SMTP id a640c23a62f3a-9dd5879a126so638760566b.3 for ; Tue, 07 Nov 2023 07:28:00 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1699370879; x=1699975679; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:reply-to:references :in-reply-to:message-id:date:subject:cc:to:from:from:to:cc:subject :date:message-id:reply-to; bh=/3i3Qi2QpIrZh9W4yO//xltt7a8m3UUu5ChkwsiZUzY=; b=AZJbbpqbniFc5SyKNhUefHx+Jewd8ZNQCtDAFB0qOcABBfZ/TJu9dkp/alSYWniZvy IpK+LQtmEnJmKMt0pKaQtrbWVylFSPf+E1s23N58DLwiXCdkIVgNJjj9gyCswDAr+Zxx jWdVRJmJJQ3QSBIdlHy3P06Dzqve+PTgTqaYUisXrWudcoTKcJpoRBYpKdEa6wNtPBur T33YLK7viAalnKICY0/OdfAiBxJc0JvcW93sONWDlwnBBhbC1XqDpq8WYVg8/OnnbM7p vnmwA/Dvd/DTmRO9jX8gvsHChS2vD3wL56V+2I8HLeYr30CQMhRsSoRqAE0Rt7nHguCe xRAQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1699370879; x=1699975679; h=content-transfer-encoding:mime-version:reply-to:references :in-reply-to:message-id:date:subject:cc:to:from:x-gm-message-state :from:to:cc:subject:date:message-id:reply-to; bh=/3i3Qi2QpIrZh9W4yO//xltt7a8m3UUu5ChkwsiZUzY=; b=o+vNyuEJJ/ZUhIHoklBxLL6GriVBYyUfkp0Y+jIzCnGxttGGONjA4Ld5OSDRPtImnA 1lfrd2uzZrRFuFtfqVvbxfPUqAsu0G1Ab42720UAqCOiMqLriEyR5/2DwzD0eM0FAaaO XOU1dyxfu23EEydMm+FH9MhuFB8RuvPJ4wRrkUoI3GjbQPcRKuqs2DLrLsSmuPsLY4sT C5AUHJLrnr9zvJNFKWGK764GH0lDqrSvuylVRZ5vJX3orAKFf+nZc88GtMt8RE9vzW8X waDiuIClDm5LvOVQiMLi+BpmgK9ENcZdQ4WeZRb9AscyK3n5+4ZTnadlpkNNzd56FivV 0CDw== X-Gm-Message-State: AOJu0Yw4Ac+fQkMorULq8cQgU0xtUTC6Bg2P/BLMNH+6K7HjGbp/EJYC 8H9rvwCWn234zmfY9MBjmdH77Co0IaM= X-Google-Smtp-Source: AGHT+IEsi6dKKqWvm5xzk6ujdBEhv2YzTqT4W4vjdxVsutBHxa9G+tKu4seyiYXIXiUWBSKHMrG9WA== X-Received: by 2002:a17:907:6e86:b0:9be:d55a:81c5 with SMTP id sh6-20020a1709076e8600b009bed55a81c5mr17219000ejc.60.1699370879114; Tue, 07 Nov 2023 07:27:59 -0800 (PST) Received: from NVC4181.nedap.local ([149.3.168.10]) by smtp.gmail.com with ESMTPSA id bx7-20020a170906a1c700b009c5c5c2c5a4sm1146574ejb.219.2023.11.07.07.27.58 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 07 Nov 2023 07:27:58 -0800 (PST) From: jhatnedap@gmail.com X-Google-Original-From: jermain.horsman@nedap.com To: openembedded-core@lists.openembedded.org Cc: Jermain Horsman Subject: [RFC 7/7] bitbake-layers: Add test case update-layer-setup Date: Tue, 7 Nov 2023 16:26:42 +0100 Message-ID: <20231107152641.1043-9-jermain.horsman@nedap.com> X-Mailer: git-send-email 2.42.0.windows.2 In-Reply-To: <20231107152641.1043-2-jermain.horsman@nedap.com> References: <20231107152641.1043-2-jermain.horsman@nedap.com> Reply-To: jermain.horsman@nedap.com MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Tue, 07 Nov 2023 15:28:04 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/190297 From: Jermain Horsman The test creates a layer setup, modifies it to use a branch, attempts a checkout using the modified file, it modifies the layers setup again to use a revision restoring it to the original state. A final checkout is skipped as the revision might not be present, additionally this functionality is covered by the setup test. Signed-off-by: Jermain Horsman --- meta/lib/oeqa/selftest/cases/bblayers.py | 58 +++++++++++++++++++++--- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/meta/lib/oeqa/selftest/cases/bblayers.py b/meta/lib/oeqa/selftest/cases/bblayers.py index 8faa060234..bac4d5bbb5 100644 --- a/meta/lib/oeqa/selftest/cases/bblayers.py +++ b/meta/lib/oeqa/selftest/cases/bblayers.py @@ -142,6 +142,14 @@ class BitbakeLayers(OESelftestTestCase): jsonschema = os.path.join(get_bb_var('COREBASE'), 'meta/files/layers.schema.json') result = runCmd("{} {} -i {} {}".format(python, jsonvalidator, json, jsonschema)) + def validate_layerscheckout(self, path): + testcheckoutdir = os.path.join(self.builddir, 'test-layer-checkout') + result = runCmd('{}/setup-layers --destdir {}'.format(path, testcheckoutdir)) + # May not necessarily be named 'poky' or 'openembedded-core' + oecoredir = os.listdir(testcheckoutdir)[0] + testcheckoutfile = os.path.join(testcheckoutdir, oecoredir, "oe-init-build-env") + self.assertTrue(os.path.exists(testcheckoutfile), "File {} not found in test layer checkout".format(testcheckoutfile)) + def test_validate_examplelayersjson(self): json = os.path.join(get_bb_var('COREBASE'), "meta/files/layers.example.json") self.validate_layersjson(json) @@ -161,9 +169,47 @@ class BitbakeLayers(OESelftestTestCase): with open(jsonfile, 'w') as f: json.dump(data, f) - testcheckoutdir = os.path.join(self.builddir, 'test-layer-checkout') - result = runCmd('{}/setup-layers --destdir {}'.format(self.testlayer_path, testcheckoutdir)) - # May not necessarily be named 'poky' or 'openembedded-core' - oecoredir = os.listdir(testcheckoutdir)[0] - testcheckoutfile = os.path.join(testcheckoutdir, oecoredir, "oe-init-build-env") - self.assertTrue(os.path.exists(testcheckoutfile), "File {} not found in test layer checkout".format(testcheckoutfile)) + self.validate_layerscheckout(self.testlayer_path) + + def test_bitbakelayers_updatelayer(self): + result = runCmd('bitbake-layers create-layers-setup {}'.format(self.testlayer_path)) + jsonfile = os.path.join(self.testlayer_path, "setup-layers.json") + self.validate_layersjson(jsonfile) + + import json + with open(jsonfile) as f: + data = json.load(f) + test_repo = None + for s in data['sources']: + if 'contains_this_file' not in data['sources'][s].keys(): + test_repo = s + test_rev = data['sources'][s]['git-remote']['rev'] + test_desc = data['sources'][s]['git-remote']['describe'] + break + + self.assertTrue(test_repo, "No test repo found") + + result = runCmd('bitbake-layers update-layers-setup {json} {repo} --branch'.format(json=jsonfile, repo=test_repo)) + self.validate_layersjson(jsonfile) + + with open(jsonfile) as f: + data = json.load(f) + + curr_rev = data['sources'][test_repo]['git-remote']['rev'] + curr_desc = data['sources'][test_repo]['git-remote']['describe'] + + self.assertEqual(curr_rev, '', "Revision not cleared: '{}'".format(curr_rev)) + self.assertEqual(curr_desc, '', "Describe not cleared: '{}'".format(curr_desc)) + + self.validate_layerscheckout(self.testlayer_path) + + result = runCmd('bitbake-layers update-layers-setup {json} {repo}'.format(json=jsonfile, repo=test_repo)) + self.validate_layersjson(jsonfile) + + with open(jsonfile) as f: + data = json.load(f) + + curr_rev = data['sources'][test_repo]['git-remote']['rev'] + curr_desc = data['sources'][test_repo]['git-remote']['describe'] + self.assertEqual(curr_rev, test_rev, "Revision incorrect: '{}', expected '{}'".format(curr_rev, test_rev)) + self.assertEqual(curr_desc, test_desc, "Describe incorrect: '{}', expected '{}'".format(curr_desc, test_desc))