diff mbox series

[RFC,1/7] scripts/oe-setup-layers: Allow setup using branches instead of fixed revisions

Message ID 20231107152641.1043-3-jermain.horsman@nedap.com
State New
Headers show
Series bitbake-layers: Add update-layers-setup | expand

Commit Message

jhatnedap@gmail.com Nov. 7, 2023, 3:26 p.m. UTC 
From: Jermain Horsman <jermain.horsman@nedap.com>

These changes allow for situations where one or more layers are checked out
using a branch instead of a revision, care is taken to make sure this works
when using multiple remotes.
All changes made are backwards compatible with older setup-layer json files.

Signed-off-by: Jermain Horsman <jermain.horsman@nedap.com>
---
 scripts/oe-setup-layers | 91 +++++++++++++++++++++++++++++++++++++----
 1 file changed, 84 insertions(+), 7 deletions(-)

Comments

Alexander Kanavin Nov. 8, 2023, 10:51 a.m. UTC | #1 
Is it possible to simply run a single 'git checkout' command to get
the repo checkout to what the json prescribes? I have to say I really
don't like the combinatorial explosion of possible code paths here.
Can't anymore understand what is going on.

Alex




On Tue, 7 Nov 2023 at 16:27, Jermain Horsman <jermain.horsman@nedap.com> wrote:
>
> From: Jermain Horsman <jermain.horsman@nedap.com>
>
> These changes allow for situations where one or more layers are checked out
> using a branch instead of a revision, care is taken to make sure this works
> when using multiple remotes.
> All changes made are backwards compatible with older setup-layer json files.
>
> Signed-off-by: Jermain Horsman <jermain.horsman@nedap.com>
> ---
>  scripts/oe-setup-layers | 91 +++++++++++++++++++++++++++++++++++++----
>  1 file changed, 84 insertions(+), 7 deletions(-)
>
> diff --git a/scripts/oe-setup-layers b/scripts/oe-setup-layers
> index 6d49688a32..ac9a9f139b 100755
> --- a/scripts/oe-setup-layers
> +++ b/scripts/oe-setup-layers
> @@ -37,6 +37,16 @@ def _is_repo_at_rev(repodir, rev):
>          pass
>      return False
>
> +def _is_repo_at_head(repodir):
> +    try:
> +        curr_branch = subprocess.check_output('git -C {repodir} for-each-ref --format=\'%(refname:short)\' "$(git -C {repodir} symbolic-ref -q HEAD)"'.format(repodir=repodir),
> +                                            shell=True, stderr=subprocess.DEVNULL)
> +        if not curr_branch.strip().decode("utf-8"):
> +            return True
> +    except subprocess.CalledProcessError:
> +        pass
> +    return False
> +
>  def _is_repo_at_remote_uri(repodir, remote, uri):
>      try:
>          curr_uri = subprocess.check_output("git -C %s remote get-url %s" % (repodir, remote), shell=True, stderr=subprocess.DEVNULL)
> @@ -46,6 +56,39 @@ def _is_repo_at_remote_uri(repodir, remote, uri):
>          pass
>      return False
>
> +def _is_repo_at_default_remote(repodir, default_remote):
> +    try:
> +        curr_default_remote = subprocess.check_output('git -C {repodir} config checkout.defaultRemote'.format(repodir=repodir), shell=True, stderr=subprocess.DEVNULL)
> +        if curr_default_remote.strip().decode("utf-8") == default_remote:
> +            return True
> +    except subprocess.CalledProcessError:
> +        pass
> +    return False
> +
> +def _is_repo_at_branch(repodir, branch):
> +    try:
> +        curr_branch = subprocess.check_output('git -C {repodir} for-each-ref --format=\'%(refname:short)\' "$(git -C {repodir} symbolic-ref -q HEAD)"'.format(repodir=repodir),
> +                                            shell=True, stderr=subprocess.DEVNULL)
> +        if curr_branch.strip().decode("utf-8") == branch:
> +            return True
> +    except subprocess.CalledProcessError:
> +        pass
> +    return False
> +
> +def _is_repo_branch_latest(repodir):
> +    try:
> +        curr_rev = subprocess.check_output('git -C {repodir} rev-parse HEAD'.format(repodir=repodir),
> +                                            shell=True, stderr=subprocess.DEVNULL)
> +        upstream_branch = subprocess.check_output('git -C {repodir} for-each-ref --format=\'%(upstream:short)\' "$(git -C {repodir} symbolic-ref -q HEAD)"'.format(repodir=repodir),
> +                                            shell=True, stderr=subprocess.DEVNULL)
> +        upstream_rev = subprocess.check_output('git -C {repodir} rev-parse {upstream_branch}'.format(repodir=repodir, upstream_branch=upstream_branch.strip().decode("utf-8")),
> +                                            shell=True, stderr=subprocess.DEVNULL)
> +        if curr_rev.strip().decode("utf-8") == upstream_rev.strip().decode("utf-8"):
> +            return True
> +    except subprocess.CalledProcessError:
> +        pass
> +    return False
> +
>  def _contains_submodules(repodir):
>      return os.path.exists(os.path.join(repodir,".gitmodules"))
>
> @@ -63,10 +106,17 @@ def _do_checkout(args, json):
>          r_remote = r_data['git-remote']
>          rev = r_remote['rev']
>          desc = r_remote['describe']
> -        if not desc:
> -            desc = rev[:10]
>          branch = r_remote['branch']
>          remotes = r_remote['remotes']
> +        default_remote = None
> +        if not rev:
> +            # note: default-remote is required if rev is not set, this key might not exist in older json files though
> +            default_remote = r_remote['default-remote']
> +            if not desc:
> +                desc = '{}/{}'.format(default_remote, branch)
> +        else:
> +            if not desc:
> +                desc = rev[:10]
>
>          print('\nSetting up source {}, revision {}, branch {}'.format(r_name, desc, branch))
>          if not _is_repo_git_repo(repodir):
> @@ -84,16 +134,43 @@ def _do_checkout(args, json):
>                  print("Running '{}' in {}".format(cmd, repodir))
>                  subprocess.check_output(cmd, shell=True, cwd=repodir)
>
> -        if not _is_repo_at_rev(repodir, rev):
> +        if rev:
> +            if not _is_repo_at_rev(repodir, rev) or not _is_repo_at_head(repodir):
> +                cmd = "git fetch -q --all || true"
> +                print("Running '{}' in {}".format(cmd, repodir))
> +                subprocess.check_output(cmd, shell=True, cwd=repodir)
> +
> +                cmd = 'git checkout -q {}'.format(rev)
> +                print("Running '{}' in {}".format(cmd, repodir))
> +                subprocess.check_output(cmd, shell=True, cwd=repodir)
> +
> +                if _contains_submodules(repodir):
> +                    print("Repo {} contains submodules, use 'git submodule update' to ensure they are up to date".format(repodir))
> +        else:
> +            check_submodule = False
> +
>              cmd = "git fetch -q --all || true"
>              print("Running '{}' in {}".format(cmd, repodir))
>              subprocess.check_output(cmd, shell=True, cwd=repodir)
>
> -            cmd = 'git checkout -q {}'.format(rev)
> -            print("Running '{}' in {}".format(cmd, repodir))
> -            subprocess.check_output(cmd, shell=True, cwd=repodir)
> +            if not _is_repo_at_default_remote(repodir, default_remote):
> +                cmd = 'git config checkout.defaultRemote {}'.format(default_remote)
> +                print("Running '{}' in {}".format(cmd, repodir))
> +                subprocess.check_output(cmd, shell=True, cwd=repodir)
> +
> +            if not _is_repo_at_branch(repodir, branch):
> +                cmd = 'git checkout -q {}'.format(branch)
> +                print("Running '{}' in {}".format(cmd, repodir))
> +                subprocess.check_output(cmd, shell=True, cwd=repodir)
> +                check_submodule = True
> +
> +            if not _is_repo_branch_latest(repodir):
> +                cmd = 'git pull'
> +                print("Running '{}' in {}".format(cmd, repodir))
> +                subprocess.check_output(cmd, shell=True, cwd=repodir)
> +                check_submodule = True
>
> -            if _contains_submodules(repodir):
> +            if check_submodule and _contains_submodules(repodir):
>                  print("Repo {} contains submodules, use 'git submodule update' to ensure they are up to date".format(repodir))
>
>  parser = argparse.ArgumentParser(description="A self contained python script that fetches all the needed layers and sets them to correct revisions using data in a json format from a separate file. The json data can be created from an active build directory with 'bitbake-layers create-layers-setup destdir' and there's a sample file and a schema in meta/files/")
> --
> 2.42.0.windows.2
>
>
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#190291): https://lists.openembedded.org/g/openembedded-core/message/190291
> Mute This Topic: https://lists.openembedded.org/mt/102444602/1686489
> Group Owner: openembedded-core+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [alex.kanavin@gmail.com]
> -=-=-=-=-=-=-=-=-=-=-=-
>
diff mbox series

Patch

diff --git a/scripts/oe-setup-layers b/scripts/oe-setup-layers
index 6d49688a32..ac9a9f139b 100755
--- a/scripts/oe-setup-layers
+++ b/scripts/oe-setup-layers
@@ -37,6 +37,16 @@  def _is_repo_at_rev(repodir, rev):
         pass
     return False
 
+def _is_repo_at_head(repodir):
+    try:
+        curr_branch = subprocess.check_output('git -C {repodir} for-each-ref --format=\'%(refname:short)\' "$(git -C {repodir} symbolic-ref -q HEAD)"'.format(repodir=repodir),
+                                            shell=True, stderr=subprocess.DEVNULL)
+        if not curr_branch.strip().decode("utf-8"):
+            return True
+    except subprocess.CalledProcessError:
+        pass
+    return False
+
 def _is_repo_at_remote_uri(repodir, remote, uri):
     try:
         curr_uri = subprocess.check_output("git -C %s remote get-url %s" % (repodir, remote), shell=True, stderr=subprocess.DEVNULL)
@@ -46,6 +56,39 @@  def _is_repo_at_remote_uri(repodir, remote, uri):
         pass
     return False
 
+def _is_repo_at_default_remote(repodir, default_remote):
+    try:
+        curr_default_remote = subprocess.check_output('git -C {repodir} config checkout.defaultRemote'.format(repodir=repodir), shell=True, stderr=subprocess.DEVNULL)
+        if curr_default_remote.strip().decode("utf-8") == default_remote:
+            return True
+    except subprocess.CalledProcessError:
+        pass
+    return False
+
+def _is_repo_at_branch(repodir, branch):
+    try:
+        curr_branch = subprocess.check_output('git -C {repodir} for-each-ref --format=\'%(refname:short)\' "$(git -C {repodir} symbolic-ref -q HEAD)"'.format(repodir=repodir),
+                                            shell=True, stderr=subprocess.DEVNULL)
+        if curr_branch.strip().decode("utf-8") == branch:
+            return True
+    except subprocess.CalledProcessError:
+        pass
+    return False
+
+def _is_repo_branch_latest(repodir):
+    try:
+        curr_rev = subprocess.check_output('git -C {repodir} rev-parse HEAD'.format(repodir=repodir),
+                                            shell=True, stderr=subprocess.DEVNULL)
+        upstream_branch = subprocess.check_output('git -C {repodir} for-each-ref --format=\'%(upstream:short)\' "$(git -C {repodir} symbolic-ref -q HEAD)"'.format(repodir=repodir),
+                                            shell=True, stderr=subprocess.DEVNULL)
+        upstream_rev = subprocess.check_output('git -C {repodir} rev-parse {upstream_branch}'.format(repodir=repodir, upstream_branch=upstream_branch.strip().decode("utf-8")),
+                                            shell=True, stderr=subprocess.DEVNULL)
+        if curr_rev.strip().decode("utf-8") == upstream_rev.strip().decode("utf-8"):
+            return True
+    except subprocess.CalledProcessError:
+        pass
+    return False
+
 def _contains_submodules(repodir):
     return os.path.exists(os.path.join(repodir,".gitmodules"))
 
@@ -63,10 +106,17 @@  def _do_checkout(args, json):
         r_remote = r_data['git-remote']
         rev = r_remote['rev']
         desc = r_remote['describe']
-        if not desc:
-            desc = rev[:10]
         branch = r_remote['branch']
         remotes = r_remote['remotes']
+        default_remote = None
+        if not rev:
+            # note: default-remote is required if rev is not set, this key might not exist in older json files though
+            default_remote = r_remote['default-remote']
+            if not desc:
+                desc = '{}/{}'.format(default_remote, branch)
+        else:
+            if not desc:
+                desc = rev[:10]
 
         print('\nSetting up source {}, revision {}, branch {}'.format(r_name, desc, branch))
         if not _is_repo_git_repo(repodir):
@@ -84,16 +134,43 @@  def _do_checkout(args, json):
                 print("Running '{}' in {}".format(cmd, repodir))
                 subprocess.check_output(cmd, shell=True, cwd=repodir)
 
-        if not _is_repo_at_rev(repodir, rev):
+        if rev:
+            if not _is_repo_at_rev(repodir, rev) or not _is_repo_at_head(repodir):
+                cmd = "git fetch -q --all || true"
+                print("Running '{}' in {}".format(cmd, repodir))
+                subprocess.check_output(cmd, shell=True, cwd=repodir)
+
+                cmd = 'git checkout -q {}'.format(rev)
+                print("Running '{}' in {}".format(cmd, repodir))
+                subprocess.check_output(cmd, shell=True, cwd=repodir)
+
+                if _contains_submodules(repodir):
+                    print("Repo {} contains submodules, use 'git submodule update' to ensure they are up to date".format(repodir))
+        else:
+            check_submodule = False
+
             cmd = "git fetch -q --all || true"
             print("Running '{}' in {}".format(cmd, repodir))
             subprocess.check_output(cmd, shell=True, cwd=repodir)
 
-            cmd = 'git checkout -q {}'.format(rev)
-            print("Running '{}' in {}".format(cmd, repodir))
-            subprocess.check_output(cmd, shell=True, cwd=repodir)
+            if not _is_repo_at_default_remote(repodir, default_remote):
+                cmd = 'git config checkout.defaultRemote {}'.format(default_remote)
+                print("Running '{}' in {}".format(cmd, repodir))
+                subprocess.check_output(cmd, shell=True, cwd=repodir)
+
+            if not _is_repo_at_branch(repodir, branch):
+                cmd = 'git checkout -q {}'.format(branch)
+                print("Running '{}' in {}".format(cmd, repodir))
+                subprocess.check_output(cmd, shell=True, cwd=repodir)
+                check_submodule = True
+
+            if not _is_repo_branch_latest(repodir):
+                cmd = 'git pull'
+                print("Running '{}' in {}".format(cmd, repodir))
+                subprocess.check_output(cmd, shell=True, cwd=repodir)
+                check_submodule = True
 
-            if _contains_submodules(repodir):
+            if check_submodule and _contains_submodules(repodir):
                 print("Repo {} contains submodules, use 'git submodule update' to ensure they are up to date".format(repodir))
 
 parser = argparse.ArgumentParser(description="A self contained python script that fetches all the needed layers and sets them to correct revisions using data in a json format from a separate file. The json data can be created from an active build directory with 'bitbake-layers create-layers-setup destdir' and there's a sample file and a schema in meta/files/")