diff mbox series

[v2,2/3] oeqa/utils/gitarchive: fix tag computation when creating archive

Message ID 20230818141712.189294-3-alexis.lothore@bootlin.com
State Accepted, archived
Commit 9cbbe9689866158825a7ae774b7965b41ff5c461
Headers show
Series oeqa/utils/gitarchive: fix tag name computation | expand

Commit Message

Alexis Lothoré Aug. 18, 2023, 2:17 p.m. UTC
From: Alexis Lothoré <alexis.lothore@bootlin.com>

Sporadic errors have been observed in autobuilder when trying to store new
tests results:

error: failed to push some refs to 'push.yoctoproject.org:yocto-testresults'
hint: Updates were rejected because the tag already exists in the remote.

The new tag name is generated by gitarchive based on known tags from the
repository (learnt with git tag). In autobuilder case, this repository is a
shallow clone, so git tag only returns most recent tags, which mean we
could miss some older tags which exist in remote but not locally. In this
case, gitarchive will likely create a tag which already exists in remote,
and so will fail to push

Fix this tag duplication by using git ls-remote to learn about existing
tags instead of git tag. To do so, create a helper ("get_tags") which
manages both nominal case (target directory is a git repository with a
proper remote) and fallback case (target directory is not from a clone, no
remote has been configured)

Fixes [YOCTO #15140]

Signed-off-by: Alexis Lothoré <alexis.lothore@bootlin.com>
---
V2:
- mutualize ls-remote usage in a single helper
- add a fallback when no remote is properly configured: use push url if
  provided
---
 meta/lib/oeqa/utils/gitarchive.py | 36 +++++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)
diff mbox series

Patch

diff --git a/meta/lib/oeqa/utils/gitarchive.py b/meta/lib/oeqa/utils/gitarchive.py
index 6e8040eb5c96..1a28a09efef7 100644
--- a/meta/lib/oeqa/utils/gitarchive.py
+++ b/meta/lib/oeqa/utils/gitarchive.py
@@ -100,9 +100,36 @@  def git_commit_data(repo, data_dir, branch, message, exclude, notes, log):
         if os.path.exists(tmp_index):
             os.unlink(tmp_index)
 
+def get_tags(repo, pattern=None, url=None):
+    """ Fetch remote tags from current repository
+
+    A pattern can be provided to filter returned tags list
+    An URL can be provided if local repository has no valid remote configured
+    """
+
+    base_cmd = ['ls-remote', '--refs', '--tags', '-q']
+    cmd = base_cmd.copy()
+
+    # First try to fetch tags from repository configured remote
+    cmd.append('origin')
+    if pattern:
+        cmd.append(pattern)
+    try:
+        tags_refs = repo.run_cmd(cmd)
+    except GitError as e:
+        # If it fails, retry with repository url if one is provided
+        if not url:
+            raise(e)
+        cmd = base_cmd.copy()
+        cmd.append(url)
+        if pattern:
+            cmd.append(pattern)
+        tags_refs = repo.run_cmd(cmd)
+
+    return ["".join(d.split()[1].split('/', 2)[2:]) for d in tags_refs.splitlines()]
 
 def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern,
-                       keywords):
+                       url, keywords):
     """Generate tag name and message, with support for running id number"""
     keyws = keywords.copy()
     # Tag number is handled specially: if not defined, we autoincrement it
@@ -116,7 +143,7 @@  def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern,
         tag_re = tag_re.format(tag_number='(?P<tag_number>[0-9]{1,5})')
 
         keyws['tag_number'] = 0
-        for existing_tag in repo.run_cmd('tag').splitlines():
+        for existing_tag in get_tags(repo, url=url):
             match = re.match(tag_re, existing_tag)
 
             if match and int(match.group('tag_number')) >= keyws['tag_number']:
@@ -143,7 +170,8 @@  def gitarchive(data_dir, git_dir, no_create, bare, commit_msg_subject, commit_ms
     if not no_tag and tagname:
         tag_name, tag_msg = expand_tag_strings(data_repo, tagname,
                                                tag_msg_subject,
-                                               tag_msg_body, keywords)
+                                               tag_msg_body,
+                                               push, keywords)
 
     # Commit data
     commit = git_commit_data(data_repo, data_dir, branch_name,
@@ -181,7 +209,7 @@  def get_test_runs(log, repo, tag_name, **kwargs):
 
     # Get a list of all matching tags
     tag_pattern = tag_name.format(**str_fields)
-    tags = repo.run_cmd(['tag', '-l', tag_pattern]).splitlines()
+    tags = get_tags(repo, pattern=tag_pattern)
     log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern)
 
     # Parse undefined fields from tag names