| Message ID | 20231204155934.836992-6-jstephan@baylibre.com |
|---|---|
| State | Accepted, archived |
| Commit | 097a43846cd99a7d74d004efc57f583ce78970a4 |
| Headers | show |
| Series | Devtool/Recipetool: adding pypi support | expand |
On Mon, Dec 4, 2023 at 7:59 AM Julien Stephan <jstephan@baylibre.com> wrote: > Today, we can use devtool/recipetool to create recipes for python projects > using the github url or the direct release tarball of the project, but the > create_buildsys_python plugin doesn't support the pypi class, since we > cannot > know from the extracted source if the package is available on pypi or not. > > By implementing the new optional process_url callback, we can detect > that the url is a pypi one (i.e 'https://pypi.org/project/<package>') > and retrieve the release tarball location. > Also detect if the url points to a release tarball hosted on > "files.pythonhosted.iorg" (i.e https://files.pythonhosted.org/packages/.. > .) > > In both cases, adds the pypi class, remove 'S' and 'SRC_URIxxx' > variables from the created recipe as they will be handled by the pypi class > and add the PYPI_PACKAGE variable > > This helps to produce cleaner recipes when package is hosted on pypi. > > If the url points to a github url or a release tarball not coming from > "files.pythonhosted.org", the created recipe is the same as before. > One can also use the newly added "--no-pypi" switch to NOT inherit > from pypi class on matching url, to keep legacy behaviour. > > To create a recipe for a pypi package, one can now use one of the > new following syntax (using recipetool create / devtool add): > > * recipetool create https://pypi.org/project/<package> > * recipetool create https://pypi.org/project/<package>/<version> > * recipetool create https://pypi.org/project/<package> --version <version> > > or the old syntax: > * recipetool create https://files.pythonhosted.org/packages/<...> > > Signed-off-by: Julien Stephan <jstephan@baylibre.com> > --- > scripts/lib/devtool/standard.py | 3 + > scripts/lib/recipetool/create.py | 1 + > .../lib/recipetool/create_buildsys_python.py | 72 +++++++++++++++++++ > 3 files changed, 76 insertions(+) > > diff --git a/scripts/lib/devtool/standard.py > b/scripts/lib/devtool/standard.py > index d53fb810071..f18ebaa70d3 100644 > --- a/scripts/lib/devtool/standard.py > +++ b/scripts/lib/devtool/standard.py > @@ -147,6 +147,8 @@ def add(args, config, basepath, workspace): > extracmdopts += ' -a' > if args.npm_dev: > extracmdopts += ' --npm-dev' > + if args.no_pypi: > + extracmdopts += ' --no-pypi' > if args.mirrors: > extracmdopts += ' --mirrors' > if args.srcrev: > @@ -2260,6 +2262,7 @@ def register_commands(subparsers, context): > group.add_argument('--no-same-dir', help='Force build in a separate > build directory', action="store_true") > parser_add.add_argument('--fetch', '-f', help='Fetch the specified > URI and extract it to create the source tree (deprecated - pass as > positional argument instead)', metavar='URI') > parser_add.add_argument('--npm-dev', help='For npm, also fetch > devDependencies', action="store_true") > + parser_add.add_argument('--no-pypi', help='Do not inherit pypi > class', action="store_true") > parser_add.add_argument('--version', '-V', help='Version to use > within recipe (PV)') > parser_add.add_argument('--no-git', '-g', help='If fetching source, > do not set up source tree as a git repository', action="store_true") > group = parser_add.add_mutually_exclusive_group() > diff --git a/scripts/lib/recipetool/create.py > b/scripts/lib/recipetool/create.py > index 5c5ac7ae403..963aa91421e 100644 > --- a/scripts/lib/recipetool/create.py > +++ b/scripts/lib/recipetool/create.py > @@ -1413,6 +1413,7 @@ def register_commands(subparsers): > parser_create.add_argument('-B', '--srcbranch', help='Branch in > source repository if fetching from an SCM such as git (default master)') > parser_create.add_argument('--keep-temp', action="store_true", > help='Keep temporary directory (for debugging)') > parser_create.add_argument('--npm-dev', action="store_true", > help='For npm, also fetch devDependencies') > + parser_create.add_argument('--no-pypi', action="store_true", help='Do > not inherit pypi class') > parser_create.add_argument('--devtool', action="store_true", > help=argparse.SUPPRESS) > parser_create.add_argument('--mirrors', action="store_true", > help='Enable PREMIRRORS and MIRRORS for source tree fetching (disabled by > default).') > parser_create.set_defaults(func=create_recipe) > diff --git a/scripts/lib/recipetool/create_buildsys_python.py > b/scripts/lib/recipetool/create_buildsys_python.py > index b620e3271b1..5e07222ece1 100644 > --- a/scripts/lib/recipetool/create_buildsys_python.py > +++ b/scripts/lib/recipetool/create_buildsys_python.py > @@ -18,7 +18,11 @@ import os > import re > import sys > import subprocess > +import json > +import urllib.request > from recipetool.create import RecipeHandler > +from urllib.parse import urldefrag > +from recipetool.create import determine_from_url > > logger = logging.getLogger('recipetool') > > @@ -111,6 +115,74 @@ class PythonRecipeHandler(RecipeHandler): > def __init__(self): > pass > > + def process_url(self, args, classes, handled, extravalues): > + """ > + Convert any pypi url https://pypi.org/project/<package>/<version> > into https://files.pythonhosted.org/packages/source/... > + which corresponds to the archive location, and add pypi class > + """ > + > + if 'url' in handled: > + return None > + > + fetch_uri = None > + source = args.source > + required_version = args.version if args.version else None > + match = re.match(r'https?://pypi.org/project/([ > <http://pypi.org/project/(%5B>^/]+)(?:/([^/]+))?/?$', > urldefrag(source)[0]) > + if match: > + package = match.group(1) > + version = match.group(2) if match.group(2) else > required_version > + > + json_url = f"https://pypi.org/pypi/%s/json" % package > + response = urllib.request.urlopen(json_url) > + if response.status == 200: > + data = json.loads(response.read()) > + if not version: > + # grab latest version > + version = data["info"]["version"] > + pypi_package = data["info"]["name"] > + for release in reversed(data["releases"][version]): > + if release["packagetype"] == "sdist": > + fetch_uri = release["url"] > + break > + else: > + logger.warning("Cannot handle pypi url %s: cannot fetch > package information using %s", source, json_url) > + return None > + else: > + match = re.match(r'^https?:// > files.pythonhosted.org/packages.*/(.*)-.*$', source) > + if match: > + fetch_uri = source > + pypi_package = match.group(1) > + _, version = determine_from_url(fetch_uri) > + > + if match and not args.no_pypi: > + if required_version and version != required_version: > + raise Exception("Version specified using --version/-V > (%s) and version specified in the url (%s) do not match" % > (required_version, version)) > + # This is optionnal if BPN looks like "python-<pypi_package>" > or "python3-<pypi_package>" (see pypi.bbclass) > + # but at this point we cannot know because because user can > specify the output name of the recipe on the command line > + extravalues["PYPI_PACKAGE"] = pypi_package > + # If the tarball extension is not 'tar.gz' (default value in > pypi.bblcass) whe should set PYPI_PACKAGE_EXT in the recipe > + pypi_package_ext = re.match(r'.*%s-%s\.(.*)$' % > (pypi_package, version), fetch_uri) > + if pypi_package_ext: > + pypi_package_ext = pypi_package_ext.group(1) > + if pypi_package_ext != "tar.gz": > + extravalues["PYPI_PACKAGE_EXT"] = pypi_package_ext > + > + # Pypi class will handle S and SRC_URIxxx variables, so > remove them > + # TODO: allow oe.recipeutils.patch_recipe_lines() to accept > regexp so we can simplify the following to: > + # extravalues['SRC_URI(?:\[.*?\])?'] = None > + extravalues['S'] = None > + extravalues['SRC_URI'] = None > + extravalues['SRC_URI[md5sum]'] = None > + extravalues['SRC_URI[sha1sum]'] = None > + extravalues['SRC_URI[sha256sum]'] = None > + extravalues['SRC_URI[sha384sum]'] = None > + extravalues['SRC_URI[sha512sum]'] = None > + this was not correct. The pypi.bbclass does not create the SRC_URI[sha256sum]. It does create the SRC_URI. So new recipes will not have any check on the validity of the downloaded tarball. The only test case which checked for this was with —nopypi https://git.yoctoproject.org/poky-contrib/tree/meta/lib/oeqa/selftest/cases/recipetool.py#n752 > + classes.append('pypi') > + > + handled.append('url') > + return fetch_uri > + > def handle_classifier_license(self, classifiers, > existing_licenses=""): > > licenses = [] > -- > 2.42.0 > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#191759): > https://lists.openembedded.org/g/openembedded-core/message/191759 > Mute This Topic: https://lists.openembedded.org/mt/102972952/924729 > Group Owner: openembedded-core+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [ > ticotimo@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- > >
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py index d53fb810071..f18ebaa70d3 100644 --- a/scripts/lib/devtool/standard.py +++ b/scripts/lib/devtool/standard.py @@ -147,6 +147,8 @@ def add(args, config, basepath, workspace): extracmdopts += ' -a' if args.npm_dev: extracmdopts += ' --npm-dev' + if args.no_pypi: + extracmdopts += ' --no-pypi' if args.mirrors: extracmdopts += ' --mirrors' if args.srcrev: @@ -2260,6 +2262,7 @@ def register_commands(subparsers, context): group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI') parser_add.add_argument('--npm-dev', help='For npm, also fetch devDependencies', action="store_true") + parser_add.add_argument('--no-pypi', help='Do not inherit pypi class', action="store_true") parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)') parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true") group = parser_add.add_mutually_exclusive_group() diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py index 5c5ac7ae403..963aa91421e 100644 --- a/scripts/lib/recipetool/create.py +++ b/scripts/lib/recipetool/create.py @@ -1413,6 +1413,7 @@ def register_commands(subparsers): parser_create.add_argument('-B', '--srcbranch', help='Branch in source repository if fetching from an SCM such as git (default master)') parser_create.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') parser_create.add_argument('--npm-dev', action="store_true", help='For npm, also fetch devDependencies') + parser_create.add_argument('--no-pypi', action="store_true", help='Do not inherit pypi class') parser_create.add_argument('--devtool', action="store_true", help=argparse.SUPPRESS) parser_create.add_argument('--mirrors', action="store_true", help='Enable PREMIRRORS and MIRRORS for source tree fetching (disabled by default).') parser_create.set_defaults(func=create_recipe) diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py index b620e3271b1..5e07222ece1 100644 --- a/scripts/lib/recipetool/create_buildsys_python.py +++ b/scripts/lib/recipetool/create_buildsys_python.py @@ -18,7 +18,11 @@ import os import re import sys import subprocess +import json +import urllib.request from recipetool.create import RecipeHandler +from urllib.parse import urldefrag +from recipetool.create import determine_from_url logger = logging.getLogger('recipetool') @@ -111,6 +115,74 @@ class PythonRecipeHandler(RecipeHandler): def __init__(self): pass + def process_url(self, args, classes, handled, extravalues): + """ + Convert any pypi url https://pypi.org/project/<package>/<version> into https://files.pythonhosted.org/packages/source/... + which corresponds to the archive location, and add pypi class + """ + + if 'url' in handled: + return None + + fetch_uri = None + source = args.source + required_version = args.version if args.version else None + match = re.match(r'https?://pypi.org/project/([^/]+)(?:/([^/]+))?/?$', urldefrag(source)[0]) + if match: + package = match.group(1) + version = match.group(2) if match.group(2) else required_version + + json_url = f"https://pypi.org/pypi/%s/json" % package + response = urllib.request.urlopen(json_url) + if response.status == 200: + data = json.loads(response.read()) + if not version: + # grab latest version + version = data["info"]["version"] + pypi_package = data["info"]["name"] + for release in reversed(data["releases"][version]): + if release["packagetype"] == "sdist": + fetch_uri = release["url"] + break + else: + logger.warning("Cannot handle pypi url %s: cannot fetch package information using %s", source, json_url) + return None + else: + match = re.match(r'^https?://files.pythonhosted.org/packages.*/(.*)-.*$', source) + if match: + fetch_uri = source + pypi_package = match.group(1) + _, version = determine_from_url(fetch_uri) + + if match and not args.no_pypi: + if required_version and version != required_version: + raise Exception("Version specified using --version/-V (%s) and version specified in the url (%s) do not match" % (required_version, version)) + # This is optionnal if BPN looks like "python-<pypi_package>" or "python3-<pypi_package>" (see pypi.bbclass) + # but at this point we cannot know because because user can specify the output name of the recipe on the command line + extravalues["PYPI_PACKAGE"] = pypi_package + # If the tarball extension is not 'tar.gz' (default value in pypi.bblcass) whe should set PYPI_PACKAGE_EXT in the recipe + pypi_package_ext = re.match(r'.*%s-%s\.(.*)$' % (pypi_package, version), fetch_uri) + if pypi_package_ext: + pypi_package_ext = pypi_package_ext.group(1) + if pypi_package_ext != "tar.gz": + extravalues["PYPI_PACKAGE_EXT"] = pypi_package_ext + + # Pypi class will handle S and SRC_URIxxx variables, so remove them + # TODO: allow oe.recipeutils.patch_recipe_lines() to accept regexp so we can simplify the following to: + # extravalues['SRC_URI(?:\[.*?\])?'] = None + extravalues['S'] = None + extravalues['SRC_URI'] = None + extravalues['SRC_URI[md5sum]'] = None + extravalues['SRC_URI[sha1sum]'] = None + extravalues['SRC_URI[sha256sum]'] = None + extravalues['SRC_URI[sha384sum]'] = None + extravalues['SRC_URI[sha512sum]'] = None + + classes.append('pypi') + + handled.append('url') + return fetch_uri + def handle_classifier_license(self, classifiers, existing_licenses=""): licenses = []
Today, we can use devtool/recipetool to create recipes for python projects using the github url or the direct release tarball of the project, but the create_buildsys_python plugin doesn't support the pypi class, since we cannot know from the extracted source if the package is available on pypi or not. By implementing the new optional process_url callback, we can detect that the url is a pypi one (i.e 'https://pypi.org/project/<package>') and retrieve the release tarball location. Also detect if the url points to a release tarball hosted on "files.pythonhosted.iorg" (i.e https://files.pythonhosted.org/packages/...) In both cases, adds the pypi class, remove 'S' and 'SRC_URIxxx' variables from the created recipe as they will be handled by the pypi class and add the PYPI_PACKAGE variable This helps to produce cleaner recipes when package is hosted on pypi. If the url points to a github url or a release tarball not coming from "files.pythonhosted.org", the created recipe is the same as before. One can also use the newly added "--no-pypi" switch to NOT inherit from pypi class on matching url, to keep legacy behaviour. To create a recipe for a pypi package, one can now use one of the new following syntax (using recipetool create / devtool add): * recipetool create https://pypi.org/project/<package> * recipetool create https://pypi.org/project/<package>/<version> * recipetool create https://pypi.org/project/<package> --version <version> or the old syntax: * recipetool create https://files.pythonhosted.org/packages/<...> Signed-off-by: Julien Stephan <jstephan@baylibre.com> --- scripts/lib/devtool/standard.py | 3 + scripts/lib/recipetool/create.py | 1 + .../lib/recipetool/create_buildsys_python.py | 72 +++++++++++++++++++ 3 files changed, 76 insertions(+)