From patchwork Wed Aug 23 10:35:19 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 29310 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 5F7B1EE49B4 for ; Wed, 23 Aug 2023 10:35:06 +0000 (UTC) Received: from relay6-d.mail.gandi.net (relay6-d.mail.gandi.net [217.70.183.198]) by mx.groups.io with SMTP id smtpd.web10.8321.1692786899536257438 for ; Wed, 23 Aug 2023 03:35:00 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=AZHKwxz+; spf=pass (domain: bootlin.com, ip: 217.70.183.198, mailfrom: alexis.lothore@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id 331F7C0009; Wed, 23 Aug 2023 10:34:57 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1692786897; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=XLg3TJZFPqEZfZt3cKKHgjLIl6DkoIt+PAgIbVyerSs=; b=AZHKwxz+nHFfN9H8ZG+HmoLmcaSMZ4VMBgTuTD349J4l4ajxRQS6qHBeY9+hdHW02kZrOf 54aoVQJ8CsiV2nCaz2tMuTmJI+sDF/bLLaLeMRCs/tkWItVmLqpuMW9adWwXXDUyZt7gHV 0hDFUiZMOSvCA7+BLsXM7zuvImF1vJq6nNDJOpRWYSj8fFS3CQljOxd4acdYx1irH7B2f5 8QkANE+SSKlAN9xss9qYcGLkWG039/Ec6GEoSggz6VxkHrdTYnkEfiSmITMF4iVlFhJcxv 4eycoTWnBF3aeBoQEbzgfpeflxLykfh0B6aesgF6cVC+vVOZuDEycwxKy1UgsQ== From: =?utf-8?q?Alexis_Lothor=C3=A9?= To: Cc: Thomas Petazzoni , Alexandre Belloni Subject: [OE-Core][PATCH 1/2] oeqa/utils/gitarchive: allow to pass a logger to get_tags Date: Wed, 23 Aug 2023 12:35:19 +0200 Message-ID: <20230823103520.72792-2-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.41.0 In-Reply-To: <20230823103520.72792-1-alexis.lothore@bootlin.com> References: <20230823103520.72792-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: alexis.lothore@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Wed, 23 Aug 2023 10:35:06 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/186569 From: Alexis Lothoré Propagate a "log" parameter to get_tags in order to know what method is used to retrieve existing tags Signed-off-by: Alexis Lothoré --- meta/lib/oeqa/selftest/cases/gitarchivetests.py | 13 ++++++++++--- meta/lib/oeqa/utils/gitarchive.py | 11 ++++++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/meta/lib/oeqa/selftest/cases/gitarchivetests.py b/meta/lib/oeqa/selftest/cases/gitarchivetests.py index 11b88daab860..e19fd4f28088 100644 --- a/meta/lib/oeqa/selftest/cases/gitarchivetests.py +++ b/meta/lib/oeqa/selftest/cases/gitarchivetests.py @@ -14,6 +14,7 @@ from oeqa.utils.git import GitError import tempfile import shutil import scriptutils +import logging from oeqa.selftest.case import OESelftestTestCase logger = scriptutils.logger_create('resulttool') @@ -58,6 +59,12 @@ class GitArchiveTests(OESelftestTestCase): TEST_COMMIT="0f7d5df" TEST_COMMIT_COUNT="42" + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.log = logging.getLogger('gitarchivetests') + cls.log.setLevel(logging.DEBUG) + def test_create_first_test_tag(self): path, git_obj = create_fake_repository(False) keywords = {'commit': self.TEST_COMMIT, 'branch': self.TEST_BRANCH, "commit_count": self.TEST_COMMIT_COUNT} @@ -101,7 +108,7 @@ class GitArchiveTests(OESelftestTestCase): url = 'git://git.yoctoproject.org/poky' path, git_obj = create_fake_repository(False, None, False) - tags = ga.get_tags(git_obj, pattern="yocto-*", url=url) + tags = ga.get_tags(git_obj, self.log, pattern="yocto-*", url=url) """Test for some well established tags (released tags)""" self.assertIn("yocto-4.0", tags) self.assertIn("yocto-4.1", tags) @@ -114,7 +121,7 @@ class GitArchiveTests(OESelftestTestCase): """Test for some well established tags (released tags)""" with self.assertRaises(GitError): - tags = ga.get_tags(git_obj, pattern="yocto-*") + tags = ga.get_tags(git_obj, self.log, pattern="yocto-*") delete_fake_repository(path) def test_get_tags_without_valid_remote_and_wrong_url(self): @@ -123,5 +130,5 @@ class GitArchiveTests(OESelftestTestCase): """Test for some well established tags (released tags)""" with self.assertRaises(GitError): - tags = ga.get_tags(git_obj, pattern="yocto-*", url=url) + tags = ga.get_tags(git_obj, self.log, pattern="yocto-*", url=url) delete_fake_repository(path) diff --git a/meta/lib/oeqa/utils/gitarchive.py b/meta/lib/oeqa/utils/gitarchive.py index 1a28a09efef7..c15a44ce9e4c 100644 --- a/meta/lib/oeqa/utils/gitarchive.py +++ b/meta/lib/oeqa/utils/gitarchive.py @@ -100,7 +100,7 @@ def git_commit_data(repo, data_dir, branch, message, exclude, notes, log): if os.path.exists(tmp_index): os.unlink(tmp_index) -def get_tags(repo, pattern=None, url=None): +def get_tags(repo, log, pattern=None, url=None): """ Fetch remote tags from current repository A pattern can be provided to filter returned tags list @@ -120,6 +120,7 @@ def get_tags(repo, pattern=None, url=None): # If it fails, retry with repository url if one is provided if not url: raise(e) + log.info("No remote repository configured, use provided url") cmd = base_cmd.copy() cmd.append(url) if pattern: @@ -129,7 +130,7 @@ def get_tags(repo, pattern=None, url=None): return ["".join(d.split()[1].split('/', 2)[2:]) for d in tags_refs.splitlines()] def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern, - url, keywords): + url, log, keywords): """Generate tag name and message, with support for running id number""" keyws = keywords.copy() # Tag number is handled specially: if not defined, we autoincrement it @@ -143,7 +144,7 @@ def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern, tag_re = tag_re.format(tag_number='(?P[0-9]{1,5})') keyws['tag_number'] = 0 - for existing_tag in get_tags(repo, url=url): + for existing_tag in get_tags(repo, log, url=url): match = re.match(tag_re, existing_tag) if match and int(match.group('tag_number')) >= keyws['tag_number']: @@ -171,7 +172,7 @@ def gitarchive(data_dir, git_dir, no_create, bare, commit_msg_subject, commit_ms tag_name, tag_msg = expand_tag_strings(data_repo, tagname, tag_msg_subject, tag_msg_body, - push, keywords) + push, log, keywords) # Commit data commit = git_commit_data(data_repo, data_dir, branch_name, @@ -209,7 +210,7 @@ def get_test_runs(log, repo, tag_name, **kwargs): # Get a list of all matching tags tag_pattern = tag_name.format(**str_fields) - tags = get_tags(repo, pattern=tag_pattern) + tags = get_tags(repo, log, pattern=tag_pattern) log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern) # Parse undefined fields from tag names From patchwork Wed Aug 23 10:35:20 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Alexis_Lothor=C3=A9?= X-Patchwork-Id: 29309 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 5E1CFEE49A0 for ; Wed, 23 Aug 2023 10:35:06 +0000 (UTC) Received: from relay6-d.mail.gandi.net (relay6-d.mail.gandi.net [217.70.183.198]) by mx.groups.io with SMTP id smtpd.web10.8322.1692786899544084752 for ; Wed, 23 Aug 2023 03:35:00 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=fQPJbqT9; spf=pass (domain: bootlin.com, ip: 217.70.183.198, mailfrom: alexis.lothore@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id 875C5C0004; Wed, 23 Aug 2023 10:34:57 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1692786897; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=5iQNfqb/fmznFoPFeNcZguQj1zFqX/5wlTaZV/fxMBY=; b=fQPJbqT9jf+aMh/S1SXHr84TEiPvYokSlZM+UW7m5o+YHKMwry00+zsYL63v4/US8yEoK7 nv4wjvVdq/zdjPbkOSf0Vs51Rs2S+MfhQO5fW2m0Odgjvd+2iFhwzr78ZetAYEiWPX21bm c1bfj0NeK7P24EjYv05BPwxKM0jBPpim49dG8V0HlVOiaxaK6T4bEJ8e9/ifDrkpWy/1L8 aD7EnSmD64Wg6P2QlKCD7amEhMtP3lMyqWfTxQLWhBqDQn45kIy5F2TdSh0sCWMmRRysbZ h8AH2X+HoTDAVYNowRaIXgQ3/37/ltX+SMU+IrTX9M9nLJwEpNgF52qiTcP42w== From: =?utf-8?q?Alexis_Lothor=C3=A9?= To: Cc: Thomas Petazzoni , Alexandre Belloni Subject: [OE-Core][PATCH 2/2] oeqa/utils/gitarchive: fall back to local tags when listing existing tags Date: Wed, 23 Aug 2023 12:35:20 +0200 Message-ID: <20230823103520.72792-3-alexis.lothore@bootlin.com> X-Mailer: git-send-email 2.41.0 In-Reply-To: <20230823103520.72792-1-alexis.lothore@bootlin.com> References: <20230823103520.72792-1-alexis.lothore@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: alexis.lothore@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Wed, 23 Aug 2023 10:35:06 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/186568 From: Alexis Lothoré e9cff55e73cc has switched tag listing from bare "git tag" to "git ls-remote" to make sure not to miss remote tags which are not fetched locally. This mechanism first checks for configured remote repository, next for possibly passed url, and then fails if none worked. However there are still cases where no remote repository is configured and no url is provided (for instance: buildperf tests use an empty git directory to store tests). Fix those cases by putting back the old behavior (local tags check) as last resort, with at least a warning for future diagnostics if we still encounter tagging issues Fixes: e9cff55e73cc ("oeqa/utils/gitarchive: fix tag computation when creating archive") Signed-off-by: Alexis Lothoré --- .../oeqa/selftest/cases/gitarchivetests.py | 14 ++++++----- meta/lib/oeqa/utils/gitarchive.py | 25 ++++++++++++------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/meta/lib/oeqa/selftest/cases/gitarchivetests.py b/meta/lib/oeqa/selftest/cases/gitarchivetests.py index e19fd4f28088..71382089c124 100644 --- a/meta/lib/oeqa/selftest/cases/gitarchivetests.py +++ b/meta/lib/oeqa/selftest/cases/gitarchivetests.py @@ -115,13 +115,15 @@ class GitArchiveTests(OESelftestTestCase): self.assertIn("yocto-4.2", tags) delete_fake_repository(path) - def test_get_tags_without_valid_remote_neither_url(self): - url = 'git://git.yoctoproject.org/poky' - path, git_obj = create_fake_repository(False, None, False) + def test_get_tags_with_only_local_tag(self): + fake_tags_list=["main/10-g0f7d5df/0", "main/10-g0f7d5df/1", "foo/20-g2468f5d/0"] + path, git_obj = create_fake_repository(True, fake_tags_list, False) - """Test for some well established tags (released tags)""" - with self.assertRaises(GitError): - tags = ga.get_tags(git_obj, self.log, pattern="yocto-*") + """No remote is configured and no url is passed: get_tags must fall + back to local tags + """ + tags = ga.get_tags(git_obj, self.log) + self.assertCountEqual(tags, fake_tags_list) delete_fake_repository(path) def test_get_tags_without_valid_remote_and_wrong_url(self): diff --git a/meta/lib/oeqa/utils/gitarchive.py b/meta/lib/oeqa/utils/gitarchive.py index c15a44ce9e4c..ac36ecb3a9ca 100644 --- a/meta/lib/oeqa/utils/gitarchive.py +++ b/meta/lib/oeqa/utils/gitarchive.py @@ -116,18 +116,25 @@ def get_tags(repo, log, pattern=None, url=None): cmd.append(pattern) try: tags_refs = repo.run_cmd(cmd) + tags = ["".join(d.split()[1].split('/', 2)[2:]) for d in tags_refs.splitlines()] except GitError as e: # If it fails, retry with repository url if one is provided - if not url: - raise(e) - log.info("No remote repository configured, use provided url") - cmd = base_cmd.copy() - cmd.append(url) - if pattern: - cmd.append(pattern) - tags_refs = repo.run_cmd(cmd) + if url: + log.info("No remote repository configured, use provided url") + cmd = base_cmd.copy() + cmd.append(url) + if pattern: + cmd.append(pattern) + tags_refs = repo.run_cmd(cmd) + tags = ["".join(d.split()[1].split('/', 2)[2:]) for d in tags_refs.splitlines()] + else: + log.warning("Read local tags only, some remote tags may be missed") + cmd = ["tag"] + if pattern: + cmd += ["-l", pattern] + tags = repo.run_cmd(cmd).splitlines() - return ["".join(d.split()[1].split('/', 2)[2:]) for d in tags_refs.splitlines()] + return tags def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern, url, log, keywords):