diff --git a/packaging/pep517_backend/__init__.py b/packaging/pep517_backend/__init__.py new file mode 100644 index 00000000000..1d3bc14c823 --- /dev/null +++ b/packaging/pep517_backend/__init__.py @@ -0,0 +1 @@ +"""PEP 517 build backend for optionally pre-building docs before setuptools.""" diff --git a/packaging/pep517_backend/_backend.py b/packaging/pep517_backend/_backend.py new file mode 100644 index 00000000000..9adbc50f2fe --- /dev/null +++ b/packaging/pep517_backend/_backend.py @@ -0,0 +1,108 @@ +"""PEP 517 build backend wrapper for optionally pre-building docs for sdist.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from configparser import ConfigParser +from importlib.metadata import import_module +from pathlib import Path + +from setuptools.build_meta import ( + build_sdist as _setuptools_build_sdist, + get_requires_for_build_sdist as _setuptools_get_requires_for_build_sdist, +) + + +__all__ = ( # noqa: WPS317, WPS410 + 'build_sdist', 'get_requires_for_build_sdist', +) + + +def _make_in_tree_ansible_importable() -> None: + """Add the library directory to module lookup paths.""" + lib_path = str(Path.cwd() / 'lib/') + os.environ['PYTHONPATH'] = lib_path # NOTE: for subprocesses + sys.path.insert(0, lib_path) # NOTE: for the current runtime session + + +def _get_package_distribution_version() -> str: + """Retrieve the current version number from setuptools config.""" + setup_cfg_path = Path.cwd() / 'setup.cfg' + setup_cfg = ConfigParser() + setup_cfg.read_string(setup_cfg_path.read_text()) + cfg_version = setup_cfg.get('metadata', 'version') + importable_version_str = cfg_version.removeprefix('attr: ') + version_mod_str, version_var_str = importable_version_str.rsplit('.', 1) + return getattr(import_module(version_mod_str), version_var_str) + + +def _generate_rst_in_templates() -> Path: + """Create ``*.1.rst.in`` files out of CLI Python modules.""" + generate_man_cmd = ( + sys.executable, + 'hacking/build-ansible.py', + 'generate-man', + '--template-file=docs/templates/man.j2', + '--output-dir=docs/man/man1/', + '--output-format=man', + *Path('lib/ansible/cli/').glob('*.py'), + ) + subprocess.check_call(tuple(map(str, generate_man_cmd))) + return Path('docs/man/man1/').glob('*.1.rst.in') + + +def _convert_rst_in_template_to_manpage(rst_in, version_number) -> None: + """Render pre-made ``*.1.rst.in`` templates into manpages. + + This includes pasting the hardcoded version into the resulting files. + The resulting ``in``-files are wiped in the process. + """ + templated_rst_doc = rst_in.with_suffix('') + templated_rst_doc.write_text( + rst_in.read_text().replace('%VERSION%', version_number)) + + rst_in.unlink() + + rst2man_cmd = ( + sys.executable, + Path(sys.executable).parent / 'rst2man.py', + templated_rst_doc, + templated_rst_doc.with_suffix(''), + ) + subprocess.check_call(tuple(map(str, rst2man_cmd))) + templated_rst_doc.unlink() + + +def build_sdist( # noqa: WPS210, WPS430 + sdist_directory: os.PathLike, + config_settings: dict[str, str] | None = None, +) -> str: + build_manpages_requested = '--build-manpages' in ( + config_settings or {} + ) + if build_manpages_requested: + Path('docs/man/man1/').mkdir(exist_ok=True, parents=True) + _make_in_tree_ansible_importable() + version_number = _get_package_distribution_version() + for rst_in in _generate_rst_in_templates(): + _convert_rst_in_template_to_manpage(rst_in, version_number) + + return _setuptools_build_sdist( + sdist_directory=sdist_directory, + config_settings=config_settings, + ) + + +def get_requires_for_build_sdist( + config_settings: dict[str, str] | None = None, +) -> list[str]: + return _setuptools_get_requires_for_build_sdist( + config_settings=config_settings, + ) + [ + 'docutils', # provides `rst2man` + 'jinja2', # used in `hacking/build-ansible.py generate-man` + 'straight.plugin', # used in `hacking/build-ansible.py` for subcommand + 'pyyaml', # needed for importing in-tree `ansible-core` from `lib/` + ] diff --git a/packaging/pep517_backend/hooks.py b/packaging/pep517_backend/hooks.py new file mode 100644 index 00000000000..b834338a6dc --- /dev/null +++ b/packaging/pep517_backend/hooks.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- + +"""PEP 517 build backend for optionally pre-building docs before setuptools.""" + +from setuptools.build_meta import * # Re-exporting PEP 517 hooks # pylint: disable=unused-wildcard-import,wildcard-import + +from ._backend import ( # noqa: WPS436 # Re-exporting PEP 517 hooks + build_sdist, get_requires_for_build_sdist, +) diff --git a/pyproject.toml b/pyproject.toml index 38c5a47d622..4a583ee5a39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,4 @@ [build-system] requires = ["setuptools >= 39.2.0", "wheel"] -build-backend = "setuptools.build_meta" +backend-path = ["packaging"] # requires 'Pip>=20' or 'pep517>=0.6.0' +build-backend = "pep517_backend.hooks" # wraps `setuptools.build_meta` diff --git a/test/integration/targets/canonical-pep517-self-packaging/aliases b/test/integration/targets/canonical-pep517-self-packaging/aliases new file mode 100644 index 00000000000..8278ec8bcc7 --- /dev/null +++ b/test/integration/targets/canonical-pep517-self-packaging/aliases @@ -0,0 +1,2 @@ +shippable/posix/group3 +context/controller diff --git a/test/integration/targets/canonical-pep517-self-packaging/minimum-build-constraints.txt b/test/integration/targets/canonical-pep517-self-packaging/minimum-build-constraints.txt new file mode 100644 index 00000000000..ea5d808482f --- /dev/null +++ b/test/integration/targets/canonical-pep517-self-packaging/minimum-build-constraints.txt @@ -0,0 +1,16 @@ +# Lowest supporting Python 3.9 and 3.10: +setuptools == 57.0.0; python_version == "3.9" or python_version == "3.10" + +# Lowest supporting Python 3.11: +setuptools == 60.0.0; python_version >= "3.11" + + +# An arbitrary old version that was released before Python 3.9.0: +wheel == 0.33.6 + +# Conditional dependencies: +docutils == 0.16 +Jinja2 == 3.0.0 +MarkupSafe == 2.0.0 +PyYAML == 5.3 +straight.plugin == 1.4.2 diff --git a/test/integration/targets/canonical-pep517-self-packaging/modernish-build-constraints.txt b/test/integration/targets/canonical-pep517-self-packaging/modernish-build-constraints.txt new file mode 100644 index 00000000000..7f744afde89 --- /dev/null +++ b/test/integration/targets/canonical-pep517-self-packaging/modernish-build-constraints.txt @@ -0,0 +1,11 @@ +setuptools == 67.4.0 + +# Wheel-only build dependency +wheel == 0.38.4 + +# Conditional dependencies: +docutils == 0.19 +Jinja2 == 3.1.2 +MarkupSafe == 2.1.2 +PyYAML == 6.0 +straight.plugin == 1.5.0 # WARNING: v1.5.0 doesn't have a Git tag / src diff --git a/test/integration/targets/canonical-pep517-self-packaging/runme.sh b/test/integration/targets/canonical-pep517-self-packaging/runme.sh new file mode 100755 index 00000000000..028348f8fe6 --- /dev/null +++ b/test/integration/targets/canonical-pep517-self-packaging/runme.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +if [[ "${ANSIBLE_DEBUG}" == true ]] # `ansible-test` invoked with `--debug` +then + PYTEST_VERY_VERBOSE_FLAG=-vvvvv + SET_DEBUG_MODE=-x +else + ANSIBLE_DEBUG=false + PYTEST_VERY_VERBOSE_FLAG= + SET_DEBUG_MODE=+x +fi + + +set -eEuo pipefail + +source virtualenv.sh + +set "${SET_DEBUG_MODE}" + +export PIP_DISABLE_PIP_VERSION_CHECK=true +export PIP_NO_PYTHON_VERSION_WARNING=true +export PIP_NO_WARN_SCRIPT_LOCATION=true + +python -Im pip install 'pytest ~= 7.2.0' +python -Im pytest ${PYTEST_VERY_VERBOSE_FLAG} \ + --basetemp="${OUTPUT_DIR}/pytest-tmp" \ + --color=yes \ + --showlocals \ + -p no:forked \ + -p no:mock \ + -ra diff --git a/test/integration/targets/canonical-pep517-self-packaging/runme_test.py b/test/integration/targets/canonical-pep517-self-packaging/runme_test.py new file mode 100644 index 00000000000..0fea1da6739 --- /dev/null +++ b/test/integration/targets/canonical-pep517-self-packaging/runme_test.py @@ -0,0 +1,265 @@ +"""Smoke tests for the in-tree PEP 517 backend.""" + +from __future__ import annotations + +from filecmp import dircmp +from os import environ +from pathlib import Path +from shutil import rmtree +from subprocess import check_call, check_output, PIPE +from sys import executable as current_interpreter +from tarfile import TarFile +import typing as t + +import pytest + + +DIST_NAME = 'ansible_core' +DIST_FILENAME_BASE = 'ansible-core' +OUTPUT_DIR = Path(environ['OUTPUT_DIR']).resolve().absolute() +SRC_ROOT_DIR = OUTPUT_DIR.parents[3] +GENERATED_MANPAGES_SUBDIR = SRC_ROOT_DIR / 'docs' / 'man' / 'man1' +LOWEST_SUPPORTED_BUILD_DEPS_FILE = ( + Path(__file__).parent / 'minimum-build-constraints.txt' +).resolve().absolute() +MODERNISH_BUILD_DEPS_FILE = ( + Path(__file__).parent / 'modernish-build-constraints.txt' +).resolve().absolute() +RELEASE_MODULE = SRC_ROOT_DIR / 'lib' / 'ansible' / 'release.py' +VERSION_LINE_PREFIX = "__version__ = '" +PKG_DIST_VERSION = next( + line[len(VERSION_LINE_PREFIX):-1] + for line in RELEASE_MODULE.read_text().splitlines() + if line.startswith(VERSION_LINE_PREFIX) +) +EXPECTED_SDIST_NAME_BASE = f'{DIST_FILENAME_BASE}-{PKG_DIST_VERSION}' +EXPECTED_SDIST_NAME = f'{EXPECTED_SDIST_NAME_BASE}.tar.gz' +EXPECTED_WHEEL_NAME = f'{DIST_NAME}-{PKG_DIST_VERSION}-py3-none-any.whl' + + +def wipe_generated_manpages() -> None: + """Ensure man1 pages aren't present in the source checkout.""" + # Cleaning up the gitignored manpages... + if not GENERATED_MANPAGES_SUBDIR.exists(): + return + + rmtree(GENERATED_MANPAGES_SUBDIR) + # Removed the generated manpages... + + +def contains_man1_pages(sdist_tarball: Path) -> Path: + """Check if the man1 pages are present in given tarball.""" + with sdist_tarball.open(mode='rb') as tarball_fd: + with TarFile.gzopen(fileobj=tarball_fd, name=None) as tarball: + try: + tarball.getmember( + name=f'{EXPECTED_SDIST_NAME_BASE}/docs/man/man1', + ) + except KeyError: + return False + + return True + + +def unpack_sdist(sdist_tarball: Path, target_directory: Path) -> Path: + """Unarchive given tarball. + + :returns: Path of the package source checkout. + """ + with sdist_tarball.open(mode='rb') as tarball_fd: + with TarFile.gzopen(fileobj=tarball_fd, name=None) as tarball: + tarball.extractall(path=target_directory) + return target_directory / EXPECTED_SDIST_NAME_BASE + + +def assert_dirs_equal(*dir_paths: t.List[Path]) -> None: + dir_comparison = dircmp(*dir_paths) + assert not dir_comparison.left_only + assert not dir_comparison.right_only + assert not dir_comparison.diff_files + assert not dir_comparison.funny_files + + +def normalize_unpacked_rebuilt_sdist(sdist_path: Path) -> None: + top_pkg_info_path = sdist_path / 'PKG-INFO' + nested_pkg_info_path = ( + sdist_path / 'lib' / f'{DIST_NAME}.egg-info' / 'PKG-INFO' + ) + entry_points_path = nested_pkg_info_path.parent / 'entry_points.txt' + + # setuptools v39 write out two trailing empty lines and an unknown platform + # while the recent don't + top_pkg_info_path.write_text( + top_pkg_info_path.read_text().replace( + 'Classifier: Development Status :: 5', + 'Platform: UNKNOWN\nClassifier: Development Status :: 5', + ) + '\n\n' + ) + nested_pkg_info_path.write_text( + nested_pkg_info_path.read_text().replace( + 'Classifier: Development Status :: 5', + 'Platform: UNKNOWN\nClassifier: Development Status :: 5', + ) + '\n\n' + ) + + # setuptools v39 write out one trailing empty line while the recent don't + entry_points_path.write_text(entry_points_path.read_text() + '\n') + + +@pytest.fixture +def venv_python_exe(tmp_path: Path) -> t.Iterator[Path]: + venv_path = tmp_path / 'pytest-managed-venv' + mkvenv_cmd = ( + current_interpreter, '-m', 'venv', str(venv_path), + ) + check_call(mkvenv_cmd, env={}, stderr=PIPE, stdout=PIPE) + yield venv_path / 'bin' / 'python' + rmtree(venv_path) + + +def build_dists( + python_exe: Path, *cli_args: t.Iterable[str], + env_vars: t.Dict[str, str], +) -> str: + full_cmd = str(python_exe), '-m', 'build', *cli_args + return check_output(full_cmd, env=env_vars, stderr=PIPE) + + +def pip_install( + python_exe: Path, *cli_args: t.Iterable[str], + env_vars: t.Dict[str, str] = None, +) -> str: + if env_vars is None: + env_vars = {} + full_cmd = str(python_exe), '-m', 'pip', 'install', *cli_args + return check_output(full_cmd, env=env_vars, stderr=PIPE) + + +def test_dist_rebuilds_with_manpages_premutations( + venv_python_exe: Path, tmp_path: Path, +) -> None: + """Test a series of sdist rebuilds under different conditions. + + This check builds sdists right from the Git checkout with and without + the manpages. It also does this using different versions of the setuptools + PEP 517 build backend being pinned. Finally, it builds a wheel out of one + of the rebuilt sdists. + As intermediate assertions, this test makes simple smoke tests along + the way. + """ + pip_install(venv_python_exe, 'build ~= 0.10.0') + + # Test building an sdist without manpages from the Git checkout + tmp_dir_sdist_without_manpages = tmp_path / 'sdist-without-manpages' + wipe_generated_manpages() + build_dists( + venv_python_exe, '--sdist', + f'--outdir={tmp_dir_sdist_without_manpages!s}', + str(SRC_ROOT_DIR), + env_vars={ + 'PIP_CONSTRAINT': str(MODERNISH_BUILD_DEPS_FILE), + }, + ) + tmp_path_sdist_without_manpages = ( + tmp_dir_sdist_without_manpages / EXPECTED_SDIST_NAME + ) + assert tmp_path_sdist_without_manpages.exists() + assert not contains_man1_pages(tmp_path_sdist_without_manpages) + sdist_without_manpages_path = unpack_sdist( + tmp_path_sdist_without_manpages, + tmp_dir_sdist_without_manpages / 'src', + ) + + # Test building an sdist with manpages from the Git checkout + # and lowest supported build deps + wipe_generated_manpages() + tmp_dir_sdist_with_manpages = tmp_path / 'sdist-with-manpages' + build_dists( + venv_python_exe, '--sdist', + '--config-setting=--build-manpages', + f'--outdir={tmp_dir_sdist_with_manpages!s}', + str(SRC_ROOT_DIR), + env_vars={ + 'PIP_CONSTRAINT': str(LOWEST_SUPPORTED_BUILD_DEPS_FILE), + }, + ) + tmp_path_sdist_with_manpages = ( + tmp_dir_sdist_with_manpages / EXPECTED_SDIST_NAME + ) + assert tmp_path_sdist_with_manpages.exists() + assert contains_man1_pages(tmp_path_sdist_with_manpages) + sdist_with_manpages_path = unpack_sdist( + tmp_path_sdist_with_manpages, + tmp_dir_sdist_with_manpages / 'src', + ) + + # Test re-building an sdist with manpages from the + # sdist contents that does not include the manpages + tmp_dir_rebuilt_sdist = tmp_path / 'rebuilt-sdist' + build_dists( + venv_python_exe, '--sdist', + '--config-setting=--build-manpages', + f'--outdir={tmp_dir_rebuilt_sdist!s}', + str(sdist_without_manpages_path), + env_vars={ + 'PIP_CONSTRAINT': str(MODERNISH_BUILD_DEPS_FILE), + }, + ) + tmp_path_rebuilt_sdist = tmp_dir_rebuilt_sdist / EXPECTED_SDIST_NAME + # Checking that the expected sdist got created + # from the previous unpacked sdist... + assert tmp_path_rebuilt_sdist.exists() + assert contains_man1_pages(tmp_path_rebuilt_sdist) + rebuilt_sdist_path = unpack_sdist( + tmp_path_rebuilt_sdist, + tmp_dir_rebuilt_sdist / 'src', + ) + assert rebuilt_sdist_path.exists() + assert rebuilt_sdist_path.is_dir() + normalize_unpacked_rebuilt_sdist(rebuilt_sdist_path) + assert_dirs_equal(rebuilt_sdist_path, sdist_with_manpages_path) + + # Test building a wheel from the rebuilt sdist with manpages contents + # and lowest supported build deps + tmp_dir_rebuilt_wheel = tmp_path / 'rebuilt-wheel' + build_dists( + venv_python_exe, '--wheel', + f'--outdir={tmp_dir_rebuilt_wheel!s}', + str(sdist_with_manpages_path), + env_vars={ + 'PIP_CONSTRAINT': str(LOWEST_SUPPORTED_BUILD_DEPS_FILE), + }, + ) + tmp_path_rebuilt_wheel = tmp_dir_rebuilt_wheel / EXPECTED_WHEEL_NAME + # Checking that the expected wheel got created... + assert tmp_path_rebuilt_wheel.exists() + + +def test_pep660_editable_install_smoke(venv_python_exe: Path) -> None: + """Smoke-test PEP 660 editable install. + + This verifies that the in-tree build backend wrapper + does not break any required interfaces. + """ + pip_install(venv_python_exe, '-e', str(SRC_ROOT_DIR)) + + pip_show_cmd = ( + str(venv_python_exe), '-m', + 'pip', 'show', DIST_FILENAME_BASE, + ) + installed_ansible_meta = check_output( + pip_show_cmd, + env={}, stderr=PIPE, text=True, + ).splitlines() + assert f'Name: {DIST_FILENAME_BASE}' in installed_ansible_meta + assert f'Version: {PKG_DIST_VERSION}' in installed_ansible_meta + + pip_runtime_version_cmd = ( + str(venv_python_exe), '-c', + 'from ansible import __version__; print(__version__)', + ) + runtime_ansible_version = check_output( + pip_runtime_version_cmd, + env={}, stderr=PIPE, text=True, + ).strip() + assert runtime_ansible_version == PKG_DIST_VERSION