From patchwork Mon Mar 14 14:39:22 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 5191 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 07738C433F5 for ; Mon, 14 Mar 2022 14:39:29 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web11.27047.1647268767467165826 for ; Mon, 14 Mar 2022 07:39:28 -0700 Authentication-Results: mx.groups.io; dkim=missing; spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 49572D6E for ; Mon, 14 Mar 2022 07:39:26 -0700 (PDT) Received: from oss-tx204.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id ECA133F7D7 for ; Mon, 14 Mar 2022 07:39:25 -0700 (PDT) From: Ross Burton To: openembedded-core@lists.openembedded.org Subject: [RFC PATCH v2 1/2] python3-installer: add installer module Date: Mon, 14 Mar 2022 14:39:22 +0000 Message-Id: <20220314143923.939188-1-ross.burton@arm.com> X-Mailer: git-send-email 2.25.1 MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Mon, 14 Mar 2022 14:39:29 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/163150 Add a recipe for Installer, a minimal library/tool to install Python Wheels. Unlike PIP, it explicitly only installs wheels and does nothing else. Signed-off-by: Ross Burton --- meta/conf/distro/include/maintainers.inc | 1 + .../python3-installer/interpreter.patch | 61 +++++++++++++++++++ .../python/python3-installer_0.5.1.bb | 22 +++++++ 3 files changed, 84 insertions(+) create mode 100644 meta/recipes-devtools/python/python3-installer/interpreter.patch create mode 100644 meta/recipes-devtools/python/python3-installer_0.5.1.bb diff --git a/meta/conf/distro/include/maintainers.inc b/meta/conf/distro/include/maintainers.inc index 3fceb6701b..1c605b0ecb 100644 --- a/meta/conf/distro/include/maintainers.inc +++ b/meta/conf/distro/include/maintainers.inc @@ -607,6 +607,7 @@ RECIPE_MAINTAINER:pn-python3-idna = "Bruce Ashfield " RECIPE_MAINTAINER:pn-python3-importlib-metadata = "Tim Orling " RECIPE_MAINTAINER:pn-python3-iniconfig = "Tim Orling " RECIPE_MAINTAINER:pn-python3-iniparse = "Oleksandr Kravchuk " +RECIPE_MAINTAINER:pn-python3-installer = "Ross Burton " RECIPE_MAINTAINER:pn-python3-jinja2 = "Richard Purdie " RECIPE_MAINTAINER:pn-python3-jsonpointer = "Bruce Ashfield " RECIPE_MAINTAINER:pn-python3-jsonschema = "Bruce Ashfield " diff --git a/meta/recipes-devtools/python/python3-installer/interpreter.patch b/meta/recipes-devtools/python/python3-installer/interpreter.patch new file mode 100644 index 0000000000..ef10ef1b45 --- /dev/null +++ b/meta/recipes-devtools/python/python3-installer/interpreter.patch @@ -0,0 +1,61 @@ +Let us override the hashbang directly (possibly upstreamable), and don't +play games with hashbangs: for now assume that even hashbangs with spaces +are simple (assume the spaces are only used to separate arguments) and +we don't have long hashbangs. + +Upstream-Status: Inappropriate +Signed-off-by: Ross Burton + +diff --git a/src/installer/__main__.py b/src/installer/__main__.py +index 3357ec5..d2fd8d2 100644 +--- a/src/installer/__main__.py ++++ b/src/installer/__main__.py +@@ -23,6 +23,13 @@ def _get_main_parser() -> argparse.ArgumentParser: + type=str, + help="destination directory (prefix to prepend to each file)", + ) ++ parser.add_argument( ++ "--interpreter", ++ "-i", ++ type=str, ++ default=sys.executable, ++ help=f"interpreter (defaults to {sys.executable})", ++ ) + parser.add_argument( + "--compile-bytecode", + action="append", +@@ -73,7 +80,7 @@ def _main(cli_args: Sequence[str], program: Optional[str] = None) -> None: + with WheelFile.open(args.wheel) as source: + destination = SchemeDictionaryDestination( + scheme_dict=_get_scheme_dict(source.distribution), +- interpreter=sys.executable, ++ interpreter=args.interpreter, + script_kind=get_launcher_kind(), + bytecode_optimization_levels=bytecode_levels, + destdir=args.destdir, +diff --git a/src/installer/scripts.py b/src/installer/scripts.py +index 7e3c8fc..ba6ed5a 100644 +--- a/src/installer/scripts.py ++++ b/src/installer/scripts.py +@@ -59,20 +59,7 @@ def _build_shebang(executable: str, forlauncher: bool) -> bytes: + https://bitbucket.org/pypa/distlib/src/58cd5c6/distlib/scripts.py#lines-124 + """ + executable_bytes = executable.encode("utf-8") +- if forlauncher: # The launcher can just use the command as-is. +- return b"#!" + executable_bytes +- if _is_executable_simple(executable_bytes): +- return b"#!" + executable_bytes +- +- # Shebang support for an executable with a space in it is under-specified +- # and platform-dependent, so we use a clever hack to generate a script to +- # run in ``/bin/sh`` that should work on all reasonably modern platforms. +- # Read the following message to understand how the hack works: +- # https://github.com/pradyunsg/installer/pull/4#issuecomment-623668717 +- +- quoted = shlex.quote(executable).encode("utf-8") +- # I don't understand a lick what this is trying to do. +- return b"#!/bin/sh\n'''exec' " + quoted + b' "$0" "$@"\n' + b"' '''" ++ return b"#!" + executable_bytes + + + class InvalidScript(ValueError): diff --git a/meta/recipes-devtools/python/python3-installer_0.5.1.bb b/meta/recipes-devtools/python/python3-installer_0.5.1.bb new file mode 100644 index 0000000000..12d9fce249 --- /dev/null +++ b/meta/recipes-devtools/python/python3-installer_0.5.1.bb @@ -0,0 +1,22 @@ +SUMMARY = "Library and tool for installing Python wheels" +DESCRIPTION = "A low-level library for installing a Python package from a wheel distribution." +HOMEPAGE = "https://installer.readthedocs.io/" +BUGTRACKER = "https://github.com/pypa/installer/issues" + +LICENSE = "MIT" +LIC_FILES_CHKSUM = "file://LICENSE;md5=5038641aec7a77451e31da828ebfae00" + +SRC_URI += "file://interpreter.patch" + +SRC_URI[sha256sum] = "f970995ec2bb815e2fdaf7977b26b2091e1e386f0f42eafd5ac811953dc5d445" + +inherit pypi flit_core + +DEPENDS:remove:class-native = "python3-installer-native" +DEPENDS:append:class-native = " unzip-native" + +do_install:class-native () { + python_pep517_do_bootstrap_install +} + +BBCLASSEXTEND = "native nativesdk" From patchwork Mon Mar 14 14:39:23 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ross Burton X-Patchwork-Id: 5192 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 0696EC433EF for ; Mon, 14 Mar 2022 14:39:29 +0000 (UTC) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mx.groups.io with SMTP id smtpd.web10.26857.1647268767856121871 for ; Mon, 14 Mar 2022 07:39:28 -0700 Authentication-Results: mx.groups.io; dkim=missing; spf=pass (domain: arm.com, ip: 217.140.110.172, mailfrom: ross.burton@arm.com) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id F2BDDED1 for ; Mon, 14 Mar 2022 07:39:26 -0700 (PDT) Received: from oss-tx204.lab.cambridge.arm.com (usa-sjc-imap-foss1.foss.arm.com [10.121.207.14]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id 7F4203F7D7 for ; Mon, 14 Mar 2022 07:39:26 -0700 (PDT) From: Ross Burton To: openembedded-core@lists.openembedded.org Subject: [RFC PATCH v2 2/2] python_pep517: use installer instead of pip Date: Mon, 14 Mar 2022 14:39:23 +0000 Message-Id: <20220314143923.939188-2-ross.burton@arm.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20220314143923.939188-1-ross.burton@arm.com> References: <20220314143923.939188-1-ross.burton@arm.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Mon, 14 Mar 2022 14:39:29 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/163151 Instead of battling pip to install a wheel, use installer. Installer does one thing, so it's faster and easier to work with. This means setuptools, pip, and wheel are no longer part of the bootstrap phase, so they can be built normally. To avoid sysroot file conflicts these three recipes can't install .pyc files to the native sysroot. We currently patch pypa/installer to allow us to override the interpreter used, which means we can drop the interpreter seding. We don't need to recompile any Python which is found in $bindir as Python doesn't actually load those files. Across a build of oe-core, the only differences between using pip and installer are: - the .dist-info/RECORD files are ordered differently - the .dist-info/REQUESTED and INSTALLER files are not created - the hashbang in native scripts is "/usr/bin/env nativepython" instead of pointing directly at the native sysroot python3. Signed-off-by: Ross Burton --- meta/classes/python_pep517.bbclass | 27 +++----------- .../python/python3-flit-core_3.7.1.bb | 2 +- .../python/python3-pip_22.0.3.bb | 35 +++---------------- .../python/python3-setuptools_59.5.0.bb | 12 +++---- .../python/python3-wheel_0.37.1.bb | 26 +++----------- 5 files changed, 18 insertions(+), 84 deletions(-) diff --git a/meta/classes/python_pep517.bbclass b/meta/classes/python_pep517.bbclass index 756333bcf2..ac7c1e44c6 100644 --- a/meta/classes/python_pep517.bbclass +++ b/meta/classes/python_pep517.bbclass @@ -1,7 +1,7 @@ # Common infrastructure for Python packages that use PEP-517 compliant packaging. # https://www.python.org/dev/peps/pep-0517/ -DEPENDS:append = " python3-pip-native" +DEPENDS:append = " python3-installer-native" # Where to execute the build process from PEP517_SOURCE_PATH ?= "${S}" @@ -11,19 +11,11 @@ PEP517_SOURCE_PATH ?= "${S}" # a recipe wants to install prebuilt wheels. PEP517_WHEEL_PATH ?= "${WORKDIR}/dist" -PIP_INSTALL_ARGS = "\ - -vvvv \ - --ignore-installed \ - --no-cache \ - --no-deps \ - --no-index \ - --root=${D} \ - --prefix=${prefix} \ -" - PEP517_INSTALL_PYTHON = "python3" PEP517_INSTALL_PYTHON:class-native = "nativepython3" +INSTALL_WHEEL_COMPILE_BYTECODE ?= "--compile-bytecode=0" + python_pep517_do_install () { COUNT=$(find ${PEP517_WHEEL_PATH} -name '*.whl' | wc -l) if test $COUNT -eq 0; then @@ -32,18 +24,7 @@ python_pep517_do_install () { bbfatal More than one wheel found in ${PEP517_WHEEL_PATH}, this should not happen fi - nativepython3 -m pip install ${PIP_INSTALL_ARGS} ${PEP517_WHEEL_PATH}/*.whl - - cd ${D} - for i in ${D}${bindir}/* ${D}${sbindir}/*; do - if [ -f "$i" ]; then - sed -i -e "1s,#!.*nativepython3,#!${USRBINPATH}/env ${PEP517_INSTALL_PYTHON}," $i - sed -i -e "s:${PYTHON}:${USRBINPATH}/env\ ${PEP517_INSTALL_PYTHON}:g" $i - sed -i -e "s:${STAGING_BINDIR_NATIVE}:${bindir}:g" $i - # Not everything we find may be Python, so ignore errors - nativepython3 -mpy_compile $(realpath --relative-to=${D} $i) || true - fi - done + nativepython3 -m installer ${INSTALL_WHEEL_COMPILE_BYTECODE} --interpreter "${USRBINPATH}/env ${PEP517_INSTALL_PYTHON}" --destdir=${D} ${PEP517_WHEEL_PATH}/*.whl } # A manual do_install that just uses unzip for bootstrapping purposes. Callers should DEPEND on unzip-native. diff --git a/meta/recipes-devtools/python/python3-flit-core_3.7.1.bb b/meta/recipes-devtools/python/python3-flit-core_3.7.1.bb index b12b8e42d7..dc815acf08 100644 --- a/meta/recipes-devtools/python/python3-flit-core_3.7.1.bb +++ b/meta/recipes-devtools/python/python3-flit-core_3.7.1.bb @@ -13,7 +13,7 @@ SRC_URI[sha256sum] = "3c9bd9c140515bfe62dd938c6610d10d6efb9e35cc647fc614fe5fb3a5 inherit pypi flit_core # Need to install by hand as there's a dependency loop -DEPENDS:remove:class-native = " python3-pip-native" +DEPENDS:remove:class-native = " python3-installer-native" DEPENDS:append:class-native = " unzip-native" # We need the full flit tarball diff --git a/meta/recipes-devtools/python/python3-pip_22.0.3.bb b/meta/recipes-devtools/python/python3-pip_22.0.3.bb index bfeeee789c..9ca8fbc1e5 100644 --- a/meta/recipes-devtools/python/python3-pip_22.0.3.bb +++ b/meta/recipes-devtools/python/python3-pip_22.0.3.bb @@ -6,43 +6,14 @@ LIC_FILES_CHKSUM = "file://LICENSE.txt;md5=63ec52baf95163b597008bb46db68030" inherit pypi setuptools_build_meta -DEPENDS += "python3" - -# To avoid a dependency loop; we bootstrap -native -DEPENDS:remove:class-native = "python3-pip-native" -DEPENDS:append:class-native = " unzip-native" - SRC_URI += "file://0001-change-shebang-to-python3.patch" SRC_URI += "file://no_shebang_mangling.patch" SRC_URI += "file://reproducible.patch" SRC_URI[sha256sum] = "f29d589df8c8ab99c060e68ad294c4a9ed896624f6368c5349d70aa581b333d0" -do_install:class-native() { - python_pep517_do_bootstrap_install - - # pip install would normally generate [console_scripts] in ${bindir} - install -d ${D}/${bindir} - # We will skip the ${bindir}/pip variant as we would just remove it in the do_install:append - cat << EOF >> ${D}/${bindir}/pip3 | tee ${D}/${bindir}/pip${PYTHON_BASEVERSION} -#!/bin/sh -'''exec' ${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN} "\$0" "\$@" -' ''' -# -*- coding: utf-8 -*- -import re -import sys -from pip._internal.cli.main import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) -EOF - chmod 0755 ${D}${bindir}/pip3 ${D}${bindir}/pip${PYTHON_BASEVERSION} -} - do_install:append() { - if [ -e ${D}/${bindir}/pip ]; then - rm ${D}/${bindir}/pip - fi + rm -f ${D}/${bindir}/pip } RDEPENDS:${PN} = "\ @@ -59,3 +30,7 @@ RDEPENDS:${PN} = "\ " BBCLASSEXTEND = "native nativesdk" + +# This used to use the bootstrap install which didn't compile. Until we bump the +# tmpdir version we can't compile the native otherwise the sysroot unpack fails +INSTALL_WHEEL_COMPILE_BYTECODE:class-native = "--no-compile-bytecode" diff --git a/meta/recipes-devtools/python/python3-setuptools_59.5.0.bb b/meta/recipes-devtools/python/python3-setuptools_59.5.0.bb index d93972b9a0..35bec19e33 100644 --- a/meta/recipes-devtools/python/python3-setuptools_59.5.0.bb +++ b/meta/recipes-devtools/python/python3-setuptools_59.5.0.bb @@ -17,14 +17,6 @@ SRC_URI[sha256sum] = "d144f85102f999444d06f9c0e8c737fd0194f10f2f7e5fdb77573f6e2f DEPENDS += "${PYTHON_PN}" -# Avoid dependency loop; we bootstrap -native -DEPENDS:remove:class-native = "python3-pip-native python3-setuptools-native" -DEPENDS:append:class-native = " unzip-native" - -do_install:class-native() { - python_pep517_do_bootstrap_install -} - RDEPENDS:${PN} = "\ ${PYTHON_PN}-2to3 \ ${PYTHON_PN}-compile \ @@ -59,3 +51,7 @@ RDEPENDS:${PYTHON_PN}-pkg-resources = "\ ${PYTHON_PN}-plistlib \ ${PYTHON_PN}-pprint \ " + +# This used to use the bootstrap install which didn't compile. Until we bump the +# tmpdir version we can't compile the native otherwise the sysroot unpack fails +INSTALL_WHEEL_COMPILE_BYTECODE:class-native = "--no-compile-bytecode" diff --git a/meta/recipes-devtools/python/python3-wheel_0.37.1.bb b/meta/recipes-devtools/python/python3-wheel_0.37.1.bb index a339e65824..efd6c2f968 100644 --- a/meta/recipes-devtools/python/python3-wheel_0.37.1.bb +++ b/meta/recipes-devtools/python/python3-wheel_0.37.1.bb @@ -10,26 +10,8 @@ inherit flit_core pypi SRC_URI += " file://0001-Backport-pyproject.toml-from-flit-backend-branch.patch" -DEPENDS:remove:class-native = "python3-pip-native" - -do_install:class-native () { - python_pep517_do_bootstrap_install - - # pip install would normally generate [project.scripts] in ${bindir} - install -d ${D}/${bindir} - cat << EOF >> ${D}/${bindir}/wheel -#!/bin/sh -'''exec' ${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN} "\$0" "\$@" -' ''' -# -*- coding: utf-8 -*- -import re -import sys -from wheel.cli import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) -EOF - chmod 0755 ${D}${bindir}/wheel -} - BBCLASSEXTEND = "native nativesdk" + +# This used to use the bootstrap install which didn't compile. Until we bump the +# tmpdir version we can't compile the native otherwise the sysroot unpack fails +INSTALL_WHEEL_COMPILE_BYTECODE:class-native = "--no-compile-bytecode"