Message ID | 20231107152641.1043-3-jermain.horsman@nedap.com |
---|---|
State | New |
Headers | show |
Series | bitbake-layers: Add update-layers-setup | expand |
Is it possible to simply run a single 'git checkout' command to get the repo checkout to what the json prescribes? I have to say I really don't like the combinatorial explosion of possible code paths here. Can't anymore understand what is going on. Alex On Tue, 7 Nov 2023 at 16:27, Jermain Horsman <jermain.horsman@nedap.com> wrote: > > From: Jermain Horsman <jermain.horsman@nedap.com> > > These changes allow for situations where one or more layers are checked out > using a branch instead of a revision, care is taken to make sure this works > when using multiple remotes. > All changes made are backwards compatible with older setup-layer json files. > > Signed-off-by: Jermain Horsman <jermain.horsman@nedap.com> > --- > scripts/oe-setup-layers | 91 +++++++++++++++++++++++++++++++++++++---- > 1 file changed, 84 insertions(+), 7 deletions(-) > > diff --git a/scripts/oe-setup-layers b/scripts/oe-setup-layers > index 6d49688a32..ac9a9f139b 100755 > --- a/scripts/oe-setup-layers > +++ b/scripts/oe-setup-layers > @@ -37,6 +37,16 @@ def _is_repo_at_rev(repodir, rev): > pass > return False > > +def _is_repo_at_head(repodir): > + try: > + curr_branch = subprocess.check_output('git -C {repodir} for-each-ref --format=\'%(refname:short)\' "$(git -C {repodir} symbolic-ref -q HEAD)"'.format(repodir=repodir), > + shell=True, stderr=subprocess.DEVNULL) > + if not curr_branch.strip().decode("utf-8"): > + return True > + except subprocess.CalledProcessError: > + pass > + return False > + > def _is_repo_at_remote_uri(repodir, remote, uri): > try: > curr_uri = subprocess.check_output("git -C %s remote get-url %s" % (repodir, remote), shell=True, stderr=subprocess.DEVNULL) > @@ -46,6 +56,39 @@ def _is_repo_at_remote_uri(repodir, remote, uri): > pass > return False > > +def _is_repo_at_default_remote(repodir, default_remote): > + try: > + curr_default_remote = subprocess.check_output('git -C {repodir} config checkout.defaultRemote'.format(repodir=repodir), shell=True, stderr=subprocess.DEVNULL) > + if curr_default_remote.strip().decode("utf-8") == default_remote: > + return True > + except subprocess.CalledProcessError: > + pass > + return False > + > +def _is_repo_at_branch(repodir, branch): > + try: > + curr_branch = subprocess.check_output('git -C {repodir} for-each-ref --format=\'%(refname:short)\' "$(git -C {repodir} symbolic-ref -q HEAD)"'.format(repodir=repodir), > + shell=True, stderr=subprocess.DEVNULL) > + if curr_branch.strip().decode("utf-8") == branch: > + return True > + except subprocess.CalledProcessError: > + pass > + return False > + > +def _is_repo_branch_latest(repodir): > + try: > + curr_rev = subprocess.check_output('git -C {repodir} rev-parse HEAD'.format(repodir=repodir), > + shell=True, stderr=subprocess.DEVNULL) > + upstream_branch = subprocess.check_output('git -C {repodir} for-each-ref --format=\'%(upstream:short)\' "$(git -C {repodir} symbolic-ref -q HEAD)"'.format(repodir=repodir), > + shell=True, stderr=subprocess.DEVNULL) > + upstream_rev = subprocess.check_output('git -C {repodir} rev-parse {upstream_branch}'.format(repodir=repodir, upstream_branch=upstream_branch.strip().decode("utf-8")), > + shell=True, stderr=subprocess.DEVNULL) > + if curr_rev.strip().decode("utf-8") == upstream_rev.strip().decode("utf-8"): > + return True > + except subprocess.CalledProcessError: > + pass > + return False > + > def _contains_submodules(repodir): > return os.path.exists(os.path.join(repodir,".gitmodules")) > > @@ -63,10 +106,17 @@ def _do_checkout(args, json): > r_remote = r_data['git-remote'] > rev = r_remote['rev'] > desc = r_remote['describe'] > - if not desc: > - desc = rev[:10] > branch = r_remote['branch'] > remotes = r_remote['remotes'] > + default_remote = None > + if not rev: > + # note: default-remote is required if rev is not set, this key might not exist in older json files though > + default_remote = r_remote['default-remote'] > + if not desc: > + desc = '{}/{}'.format(default_remote, branch) > + else: > + if not desc: > + desc = rev[:10] > > print('\nSetting up source {}, revision {}, branch {}'.format(r_name, desc, branch)) > if not _is_repo_git_repo(repodir): > @@ -84,16 +134,43 @@ def _do_checkout(args, json): > print("Running '{}' in {}".format(cmd, repodir)) > subprocess.check_output(cmd, shell=True, cwd=repodir) > > - if not _is_repo_at_rev(repodir, rev): > + if rev: > + if not _is_repo_at_rev(repodir, rev) or not _is_repo_at_head(repodir): > + cmd = "git fetch -q --all || true" > + print("Running '{}' in {}".format(cmd, repodir)) > + subprocess.check_output(cmd, shell=True, cwd=repodir) > + > + cmd = 'git checkout -q {}'.format(rev) > + print("Running '{}' in {}".format(cmd, repodir)) > + subprocess.check_output(cmd, shell=True, cwd=repodir) > + > + if _contains_submodules(repodir): > + print("Repo {} contains submodules, use 'git submodule update' to ensure they are up to date".format(repodir)) > + else: > + check_submodule = False > + > cmd = "git fetch -q --all || true" > print("Running '{}' in {}".format(cmd, repodir)) > subprocess.check_output(cmd, shell=True, cwd=repodir) > > - cmd = 'git checkout -q {}'.format(rev) > - print("Running '{}' in {}".format(cmd, repodir)) > - subprocess.check_output(cmd, shell=True, cwd=repodir) > + if not _is_repo_at_default_remote(repodir, default_remote): > + cmd = 'git config checkout.defaultRemote {}'.format(default_remote) > + print("Running '{}' in {}".format(cmd, repodir)) > + subprocess.check_output(cmd, shell=True, cwd=repodir) > + > + if not _is_repo_at_branch(repodir, branch): > + cmd = 'git checkout -q {}'.format(branch) > + print("Running '{}' in {}".format(cmd, repodir)) > + subprocess.check_output(cmd, shell=True, cwd=repodir) > + check_submodule = True > + > + if not _is_repo_branch_latest(repodir): > + cmd = 'git pull' > + print("Running '{}' in {}".format(cmd, repodir)) > + subprocess.check_output(cmd, shell=True, cwd=repodir) > + check_submodule = True > > - if _contains_submodules(repodir): > + if check_submodule and _contains_submodules(repodir): > print("Repo {} contains submodules, use 'git submodule update' to ensure they are up to date".format(repodir)) > > parser = argparse.ArgumentParser(description="A self contained python script that fetches all the needed layers and sets them to correct revisions using data in a json format from a separate file. The json data can be created from an active build directory with 'bitbake-layers create-layers-setup destdir' and there's a sample file and a schema in meta/files/") > -- > 2.42.0.windows.2 > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#190291): https://lists.openembedded.org/g/openembedded-core/message/190291 > Mute This Topic: https://lists.openembedded.org/mt/102444602/1686489 > Group Owner: openembedded-core+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [alex.kanavin@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- >
diff --git a/scripts/oe-setup-layers b/scripts/oe-setup-layers index 6d49688a32..ac9a9f139b 100755 --- a/scripts/oe-setup-layers +++ b/scripts/oe-setup-layers @@ -37,6 +37,16 @@ def _is_repo_at_rev(repodir, rev): pass return False +def _is_repo_at_head(repodir): + try: + curr_branch = subprocess.check_output('git -C {repodir} for-each-ref --format=\'%(refname:short)\' "$(git -C {repodir} symbolic-ref -q HEAD)"'.format(repodir=repodir), + shell=True, stderr=subprocess.DEVNULL) + if not curr_branch.strip().decode("utf-8"): + return True + except subprocess.CalledProcessError: + pass + return False + def _is_repo_at_remote_uri(repodir, remote, uri): try: curr_uri = subprocess.check_output("git -C %s remote get-url %s" % (repodir, remote), shell=True, stderr=subprocess.DEVNULL) @@ -46,6 +56,39 @@ def _is_repo_at_remote_uri(repodir, remote, uri): pass return False +def _is_repo_at_default_remote(repodir, default_remote): + try: + curr_default_remote = subprocess.check_output('git -C {repodir} config checkout.defaultRemote'.format(repodir=repodir), shell=True, stderr=subprocess.DEVNULL) + if curr_default_remote.strip().decode("utf-8") == default_remote: + return True + except subprocess.CalledProcessError: + pass + return False + +def _is_repo_at_branch(repodir, branch): + try: + curr_branch = subprocess.check_output('git -C {repodir} for-each-ref --format=\'%(refname:short)\' "$(git -C {repodir} symbolic-ref -q HEAD)"'.format(repodir=repodir), + shell=True, stderr=subprocess.DEVNULL) + if curr_branch.strip().decode("utf-8") == branch: + return True + except subprocess.CalledProcessError: + pass + return False + +def _is_repo_branch_latest(repodir): + try: + curr_rev = subprocess.check_output('git -C {repodir} rev-parse HEAD'.format(repodir=repodir), + shell=True, stderr=subprocess.DEVNULL) + upstream_branch = subprocess.check_output('git -C {repodir} for-each-ref --format=\'%(upstream:short)\' "$(git -C {repodir} symbolic-ref -q HEAD)"'.format(repodir=repodir), + shell=True, stderr=subprocess.DEVNULL) + upstream_rev = subprocess.check_output('git -C {repodir} rev-parse {upstream_branch}'.format(repodir=repodir, upstream_branch=upstream_branch.strip().decode("utf-8")), + shell=True, stderr=subprocess.DEVNULL) + if curr_rev.strip().decode("utf-8") == upstream_rev.strip().decode("utf-8"): + return True + except subprocess.CalledProcessError: + pass + return False + def _contains_submodules(repodir): return os.path.exists(os.path.join(repodir,".gitmodules")) @@ -63,10 +106,17 @@ def _do_checkout(args, json): r_remote = r_data['git-remote'] rev = r_remote['rev'] desc = r_remote['describe'] - if not desc: - desc = rev[:10] branch = r_remote['branch'] remotes = r_remote['remotes'] + default_remote = None + if not rev: + # note: default-remote is required if rev is not set, this key might not exist in older json files though + default_remote = r_remote['default-remote'] + if not desc: + desc = '{}/{}'.format(default_remote, branch) + else: + if not desc: + desc = rev[:10] print('\nSetting up source {}, revision {}, branch {}'.format(r_name, desc, branch)) if not _is_repo_git_repo(repodir): @@ -84,16 +134,43 @@ def _do_checkout(args, json): print("Running '{}' in {}".format(cmd, repodir)) subprocess.check_output(cmd, shell=True, cwd=repodir) - if not _is_repo_at_rev(repodir, rev): + if rev: + if not _is_repo_at_rev(repodir, rev) or not _is_repo_at_head(repodir): + cmd = "git fetch -q --all || true" + print("Running '{}' in {}".format(cmd, repodir)) + subprocess.check_output(cmd, shell=True, cwd=repodir) + + cmd = 'git checkout -q {}'.format(rev) + print("Running '{}' in {}".format(cmd, repodir)) + subprocess.check_output(cmd, shell=True, cwd=repodir) + + if _contains_submodules(repodir): + print("Repo {} contains submodules, use 'git submodule update' to ensure they are up to date".format(repodir)) + else: + check_submodule = False + cmd = "git fetch -q --all || true" print("Running '{}' in {}".format(cmd, repodir)) subprocess.check_output(cmd, shell=True, cwd=repodir) - cmd = 'git checkout -q {}'.format(rev) - print("Running '{}' in {}".format(cmd, repodir)) - subprocess.check_output(cmd, shell=True, cwd=repodir) + if not _is_repo_at_default_remote(repodir, default_remote): + cmd = 'git config checkout.defaultRemote {}'.format(default_remote) + print("Running '{}' in {}".format(cmd, repodir)) + subprocess.check_output(cmd, shell=True, cwd=repodir) + + if not _is_repo_at_branch(repodir, branch): + cmd = 'git checkout -q {}'.format(branch) + print("Running '{}' in {}".format(cmd, repodir)) + subprocess.check_output(cmd, shell=True, cwd=repodir) + check_submodule = True + + if not _is_repo_branch_latest(repodir): + cmd = 'git pull' + print("Running '{}' in {}".format(cmd, repodir)) + subprocess.check_output(cmd, shell=True, cwd=repodir) + check_submodule = True - if _contains_submodules(repodir): + if check_submodule and _contains_submodules(repodir): print("Repo {} contains submodules, use 'git submodule update' to ensure they are up to date".format(repodir)) parser = argparse.ArgumentParser(description="A self contained python script that fetches all the needed layers and sets them to correct revisions using data in a json format from a separate file. The json data can be created from an active build directory with 'bitbake-layers create-layers-setup destdir' and there's a sample file and a schema in meta/files/")