From patchwork Thu Apr 11 10:10:29 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Sven X-Patchwork-Id: 42210 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id D4BD0CD1284 for ; Thu, 11 Apr 2024 10:10:55 +0000 (UTC) Received: from mail.schwermer.no (mail.schwermer.no [49.12.228.226]) by mx.groups.io with SMTP id smtpd.web10.14261.1712830245179628270 for ; Thu, 11 Apr 2024 03:10:46 -0700 Authentication-Results: mx.groups.io; dkim=fail reason="dkim: body hash did not verify" header.i=@svenschwermer.de header.s=mail header.b=S2COhHg4; spf=pass (domain: svenschwermer.de, ip: 49.12.228.226, mailfrom: sven@svenschwermer.de) X-Virus-Scanned: Yes From: Sven Schwermer DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=svenschwermer.de; s=mail; t=1712830241; bh=onW6WT5JeyeHi1N4Hx9WyFYzbwVpf+8SLeWK+IYGU2Y=; h=From:To:Cc:Subject; b=S2COhHg4AIxg6HR0l2gTGVacXc0iryco4bcwFGL0desmTKXApveIqbCpTP/Rlc6NN c69IvVfy6bCfHZOF+63NPyOIYYhrPhvZaJzd82ClJlXgrj2qHYOMKFV63V7m56avKH Uu90Ao3aQrZDm4/w9OvuVaPmLl9VusWgs8wH4qbkfEDELrOUte+ZuDAriVDnh6Hpso PFqf+hKtsoPYCHO8ezAM9jvjD7lG/CIRCQIHxuPG2cOP6Vb6B7dy4Efi5QYcAhkHAr tdJnQd/WgK++bC+1VsDabXbSYazVOpQsOweiN6aYOje6SufvqrnAlP1SyCIc7JTt6i +dalF3P32qLEQ== To: openembedded-core@lists.openembedded.org Cc: lukas.funke@weidmueller.com, uvv.mail@gmail.com, Sven Schwermer Subject: [PATCH 1/2] recipetool: Handle unclean response in go resolver Date: Thu, 11 Apr 2024 12:10:29 +0200 Message-ID: <20240411101030.2846823-1-sven@svenschwermer.de> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 11 Apr 2024 10:10:55 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/198124 From: Sven Schwermer It appears that some go modules repond with a 404 error when trying to resolve them dynamically. The response body may still contain the go-import meta tag. An example for such behaviour is gonum.org/v1/gonum. Signed-off-by: Sven Schwermer --- scripts/lib/recipetool/create_go.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/lib/recipetool/create_go.py b/scripts/lib/recipetool/create_go.py index c560831442..0fb7115e26 100644 --- a/scripts/lib/recipetool/create_go.py +++ b/scripts/lib/recipetool/create_go.py @@ -16,7 +16,7 @@ from html.parser import HTMLParser from recipetool.create import RecipeHandler, handle_license_vars from recipetool.create import guess_license, tidy_licenses, fixup_license from recipetool.create import determine_from_url -from urllib.error import URLError +from urllib.error import URLError, HTTPError import bb.utils import json @@ -251,15 +251,18 @@ class GoRecipeHandler(RecipeHandler): req = urllib.request.Request(url) try: - resp = urllib.request.urlopen(req) - + body = urllib.request.urlopen(req).read() + except HTTPError as http_err: + logger.warning( + "Unclean status when fetching page from [%s]: %s", url, str(http_err)) + body = http_err.fp.read() except URLError as url_err: logger.warning( "Failed to fetch page from [%s]: %s", url, str(url_err)) return None parser = GoImportHTMLParser() - parser.feed(resp.read().decode('utf-8')) + parser.feed(body.decode('utf-8')) parser.close() return GoImport(parser.import_prefix, parser.vcs, parser.repourl, None) From patchwork Thu Apr 11 10:10:30 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Sven X-Patchwork-Id: 42211 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id DB14CCD1292 for ; Thu, 11 Apr 2024 10:10:55 +0000 (UTC) Received: from mail.schwermer.no (mail.schwermer.no [49.12.228.226]) by mx.groups.io with SMTP id smtpd.web11.14125.1712830245475750470 for ; Thu, 11 Apr 2024 03:10:46 -0700 Authentication-Results: mx.groups.io; dkim=fail reason="dkim: body hash did not verify" header.i=@svenschwermer.de header.s=mail header.b=TZuuSwjC; spf=pass (domain: svenschwermer.de, ip: 49.12.228.226, mailfrom: sven@svenschwermer.de) X-Virus-Scanned: Yes From: Sven Schwermer DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=svenschwermer.de; s=mail; t=1712830241; bh=jpnaUZwAhlNq+TTwzXreSSMypn7rOZnPT+w9I5axLBo=; h=From:To:Cc:Subject:In-Reply-To:References; b=TZuuSwjCYmFPQliyjodhTuPVt1Glj2xJ5UHbwd63Jt0U4eRtyN9JY0gGEeJDcCNdJ 3MtxabagiV+do5Om4O/0xA7hsdX3cu4BSIY/Cp8JJYjB3CMGhWilKE7T8vI+1C2F70 dXh3Qzb5+XRl7M8nEakeEC61dEAaNYllDNRBm0H8sNUc/SwZkLWSUyMe97NXWEG3Jw MXvSKXLvXd5W1e9OAHU1uEnUxjCiLMu0lESuikexxNw/qg6x6YEEUDLp6dd20mWg9o IlODHitBph2Faa4PhoIaDnSxkVxu+hOGW5E2pdfpLqJReYY2kPOghDA09c1i6n0eF5 miz4PVuXZBQ4Q== To: openembedded-core@lists.openembedded.org Cc: lukas.funke@weidmueller.com, uvv.mail@gmail.com, Sven Schwermer Subject: [PATCH 2/2] recipetool: Handle several go-import tags in go resolver Date: Thu, 11 Apr 2024 12:10:30 +0200 Message-ID: <20240411101030.2846823-2-sven@svenschwermer.de> In-Reply-To: <20240411101030.2846823-1-sven@svenschwermer.de> References: <20240411101030.2846823-1-sven@svenschwermer.de> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 11 Apr 2024 10:10:55 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/198125 From: Sven Schwermer When dynamically resolving go modules, the HTML page may contain several go-import meta tags. We must handle all and pick the correct one based on the module name. An example for such a behaviour is gonum.org/v1/gonum: Signed-off-by: Sven Schwermer --- scripts/lib/recipetool/create_go.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/scripts/lib/recipetool/create_go.py b/scripts/lib/recipetool/create_go.py index 0fb7115e26..a85a2f2786 100644 --- a/scripts/lib/recipetool/create_go.py +++ b/scripts/lib/recipetool/create_go.py @@ -225,7 +225,7 @@ class GoRecipeHandler(RecipeHandler): def __init__(self): super().__init__() - self.__srv = [] + self.__srv = {} def handle_starttag(self, tag, attrs): if tag == 'meta' and list( @@ -233,19 +233,14 @@ class GoRecipeHandler(RecipeHandler): content = list( filter(lambda a: (a[0] == 'content'), attrs)) if content: - self.__srv = content[0][1].split() + srv = content[0][1].split() + self.__srv[srv[0]] = srv - @property - def import_prefix(self): - return self.__srv[0] if len(self.__srv) else None - - @property - def vcs(self): - return self.__srv[1] if len(self.__srv) else None - - @property - def repourl(self): - return self.__srv[2] if len(self.__srv) else None + def go_import(self, modulepath): + if modulepath in self.__srv: + srv = self.__srv[modulepath] + return GoImport(srv[0], srv[1], srv[2], None) + return None url = url.geturl() + "?go-get=1" req = urllib.request.Request(url) @@ -265,7 +260,7 @@ class GoRecipeHandler(RecipeHandler): parser.feed(body.decode('utf-8')) parser.close() - return GoImport(parser.import_prefix, parser.vcs, parser.repourl, None) + return parser.go_import(modulepath) def __resolve_from_golang_proxy(self, modulepath, version): """