[backport-2.14] 📦 Integrate manpage builds into PEP 517 build backend (#80129)

This patch creates a thin wrapper around the `setuptools`' PEP 517
build backend in-tree. It features an ability to request generating
the manpage files in the process of building a source distribution.
This toggle is implemented using the `config_settings` mechanism of
PEP 517.
One must explicitly pass it a CLI option to the build front-end to
trigger said behavior. The packagers are expected to use the
following call:

    python -m build --config-setting=--build-manpages

This option has no effect on building wheels.

🧪 The change includes integration tests

This test runs building and re-building sdists and wheels with and
without the `--build-manpages` config setting under the
oldest-supported and new `setuptools` pinned.

It is intended to preserve the interoperability of the packaging setup
across Python runtimes.

An extra smoke test also verifies that non PEP 517 interfaces remain functional.

PR #79606

Co-authored-by: Matt Clay <matt@mystile.com>
(cherry picked from commit 56036013cd)

packaging/pep517_backend/__init__.py

@@ -0,0 +1 @@
+"""PEP 517 build backend for optionally pre-building docs before setuptools."""

packaging/pep517_backend/_backend.py

@@ -0,0 +1,108 @@
+"""PEP 517 build backend wrapper for optionally pre-building docs for sdist."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from configparser import ConfigParser
+from importlib.metadata import import_module
+from pathlib import Path
+
+from setuptools.build_meta import (
+    build_sdist as _setuptools_build_sdist,
+    get_requires_for_build_sdist as _setuptools_get_requires_for_build_sdist,
+)
+
+
+__all__ = (  # noqa: WPS317, WPS410
+    'build_sdist', 'get_requires_for_build_sdist',
+)
+
+
+def _make_in_tree_ansible_importable() -> None:
+    """Add the library directory to module lookup paths."""
+    lib_path = str(Path.cwd() / 'lib/')
+    os.environ['PYTHONPATH'] = lib_path  # NOTE: for subprocesses
+    sys.path.insert(0, lib_path)  # NOTE: for the current runtime session
+
+
+def _get_package_distribution_version() -> str:
+    """Retrieve the current version number from setuptools config."""
+    setup_cfg_path = Path.cwd() / 'setup.cfg'
+    setup_cfg = ConfigParser()
+    setup_cfg.read_string(setup_cfg_path.read_text())
+    cfg_version = setup_cfg.get('metadata', 'version')
+    importable_version_str = cfg_version.removeprefix('attr: ')
+    version_mod_str, version_var_str = importable_version_str.rsplit('.', 1)
+    return getattr(import_module(version_mod_str), version_var_str)
+
+
+def _generate_rst_in_templates() -> Path:
+    """Create ``*.1.rst.in`` files out of CLI Python modules."""
+    generate_man_cmd = (
+        sys.executable,
+        'hacking/build-ansible.py',
+        'generate-man',
+        '--template-file=docs/templates/man.j2',
+        '--output-dir=docs/man/man1/',
+        '--output-format=man',
+        *Path('lib/ansible/cli/').glob('*.py'),
+    )
+    subprocess.check_call(tuple(map(str, generate_man_cmd)))
+    return Path('docs/man/man1/').glob('*.1.rst.in')
+
+
+def _convert_rst_in_template_to_manpage(rst_in, version_number) -> None:
+    """Render pre-made ``*.1.rst.in`` templates into manpages.
+
+    This includes pasting the hardcoded version into the resulting files.
+    The resulting ``in``-files are wiped in the process.
+    """
+    templated_rst_doc = rst_in.with_suffix('')
+    templated_rst_doc.write_text(
+        rst_in.read_text().replace('%VERSION%', version_number))
+
+    rst_in.unlink()
+
+    rst2man_cmd = (
+        sys.executable,
+        Path(sys.executable).parent / 'rst2man.py',
+        templated_rst_doc,
+        templated_rst_doc.with_suffix(''),
+    )
+    subprocess.check_call(tuple(map(str, rst2man_cmd)))
+    templated_rst_doc.unlink()
+
+
+def build_sdist(  # noqa: WPS210, WPS430
+         sdist_directory: os.PathLike,
+         config_settings: dict[str, str] | None = None,
+) -> str:
+    build_manpages_requested = '--build-manpages' in (
+        config_settings or {}
+    )
+    if build_manpages_requested:
+        Path('docs/man/man1/').mkdir(exist_ok=True, parents=True)
+        _make_in_tree_ansible_importable()
+        version_number = _get_package_distribution_version()
+        for rst_in in _generate_rst_in_templates():
+            _convert_rst_in_template_to_manpage(rst_in, version_number)
+
+    return _setuptools_build_sdist(
+        sdist_directory=sdist_directory,
+        config_settings=config_settings,
+    )
+
+
+def get_requires_for_build_sdist(
+        config_settings: dict[str, str] | None = None,
+) -> list[str]:
+    return _setuptools_get_requires_for_build_sdist(
+        config_settings=config_settings,
+    ) + [
+        'docutils',  # provides `rst2man`
+        'jinja2',  # used in `hacking/build-ansible.py generate-man`
+        'straight.plugin',  # used in `hacking/build-ansible.py` for subcommand
+        'pyyaml',  # needed for importing in-tree `ansible-core` from `lib/`
+    ]

packaging/pep517_backend/hooks.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+
+"""PEP 517 build backend for optionally pre-building docs before setuptools."""
+
+from setuptools.build_meta import *  # Re-exporting PEP 517 hooks  # pylint: disable=unused-wildcard-import,wildcard-import
+
+from ._backend import (  # noqa: WPS436  # Re-exporting PEP 517 hooks
+    build_sdist, get_requires_for_build_sdist,
+)

pyproject.toml

@@ -1,3 +1,4 @@
 [build-system]
 requires = ["setuptools >= 39.2.0", "wheel"]
-build-backend = "setuptools.build_meta"
+backend-path = ["packaging"]  # requires 'Pip>=20' or 'pep517>=0.6.0'
+build-backend = "pep517_backend.hooks"  # wraps `setuptools.build_meta`

test/integration/targets/canonical-pep517-self-packaging/aliases

@@ -0,0 +1,2 @@
+shippable/posix/group3
+context/controller

test/integration/targets/canonical-pep517-self-packaging/minimum-build-constraints.txt

@@ -0,0 +1,16 @@
+# Lowest supporting Python 3.9 and 3.10:
+setuptools == 57.0.0; python_version == "3.9" or python_version == "3.10"
+
+# Lowest supporting Python 3.11:
+setuptools == 60.0.0; python_version >= "3.11"
+
+
+# An arbitrary old version that was released before Python 3.9.0:
+wheel == 0.33.6
+
+# Conditional dependencies:
+docutils == 0.16
+Jinja2 == 3.0.0
+MarkupSafe == 2.0.0
+PyYAML == 5.3
+straight.plugin == 1.4.2

test/integration/targets/canonical-pep517-self-packaging/modernish-build-constraints.txt

@@ -0,0 +1,11 @@
+setuptools == 67.4.0
+
+# Wheel-only build dependency
+wheel == 0.38.4
+
+# Conditional dependencies:
+docutils == 0.19
+Jinja2 == 3.1.2
+MarkupSafe == 2.1.2
+PyYAML == 6.0
+straight.plugin == 1.5.0  # WARNING: v1.5.0 doesn't have a Git tag / src

test/integration/targets/canonical-pep517-self-packaging/runme.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+if [[ "${ANSIBLE_DEBUG}" == true ]]  # `ansible-test` invoked with `--debug`
+then
+    PYTEST_VERY_VERBOSE_FLAG=-vvvvv
+    SET_DEBUG_MODE=-x
+else
+    ANSIBLE_DEBUG=false
+    PYTEST_VERY_VERBOSE_FLAG=
+    SET_DEBUG_MODE=+x
+fi
+
+
+set -eEuo pipefail
+
+source virtualenv.sh
+
+set "${SET_DEBUG_MODE}"
+
+export PIP_DISABLE_PIP_VERSION_CHECK=true
+export PIP_NO_PYTHON_VERSION_WARNING=true
+export PIP_NO_WARN_SCRIPT_LOCATION=true
+
+python -Im pip install 'pytest ~= 7.2.0'
+python -Im pytest ${PYTEST_VERY_VERBOSE_FLAG} \
+  --basetemp="${OUTPUT_DIR}/pytest-tmp" \
+  --color=yes \
+  --showlocals \
+  -p no:forked \
+  -p no:mock \
+  -ra

test/integration/targets/canonical-pep517-self-packaging/runme_test.py

@@ -0,0 +1,265 @@
+"""Smoke tests for the in-tree PEP 517 backend."""
+
+from __future__ import annotations
+
+from filecmp import dircmp
+from os import environ
+from pathlib import Path
+from shutil import rmtree
+from subprocess import check_call, check_output, PIPE
+from sys import executable as current_interpreter
+from tarfile import TarFile
+import typing as t
+
+import pytest
+
+
+DIST_NAME = 'ansible_core'
+DIST_FILENAME_BASE = 'ansible-core'
+OUTPUT_DIR = Path(environ['OUTPUT_DIR']).resolve().absolute()
+SRC_ROOT_DIR = OUTPUT_DIR.parents[3]
+GENERATED_MANPAGES_SUBDIR = SRC_ROOT_DIR / 'docs' / 'man' / 'man1'
+LOWEST_SUPPORTED_BUILD_DEPS_FILE = (
+    Path(__file__).parent / 'minimum-build-constraints.txt'
+).resolve().absolute()
+MODERNISH_BUILD_DEPS_FILE = (
+    Path(__file__).parent / 'modernish-build-constraints.txt'
+).resolve().absolute()
+RELEASE_MODULE = SRC_ROOT_DIR / 'lib' / 'ansible' / 'release.py'
+VERSION_LINE_PREFIX = "__version__ = '"
+PKG_DIST_VERSION = next(
+    line[len(VERSION_LINE_PREFIX):-1]
+    for line in RELEASE_MODULE.read_text().splitlines()
+    if line.startswith(VERSION_LINE_PREFIX)
+)
+EXPECTED_SDIST_NAME_BASE = f'{DIST_FILENAME_BASE}-{PKG_DIST_VERSION}'
+EXPECTED_SDIST_NAME = f'{EXPECTED_SDIST_NAME_BASE}.tar.gz'
+EXPECTED_WHEEL_NAME = f'{DIST_NAME}-{PKG_DIST_VERSION}-py3-none-any.whl'
+
+
+def wipe_generated_manpages() -> None:
+    """Ensure man1 pages aren't present in the source checkout."""
+    # Cleaning up the gitignored manpages...
+    if not GENERATED_MANPAGES_SUBDIR.exists():
+        return
+
+    rmtree(GENERATED_MANPAGES_SUBDIR)
+    # Removed the generated manpages...
+
+
+def contains_man1_pages(sdist_tarball: Path) -> Path:
+    """Check if the man1 pages are present in given tarball."""
+    with sdist_tarball.open(mode='rb') as tarball_fd:
+        with TarFile.gzopen(fileobj=tarball_fd, name=None) as tarball:
+            try:
+                tarball.getmember(
+                    name=f'{EXPECTED_SDIST_NAME_BASE}/docs/man/man1',
+                )
+            except KeyError:
+                return False
+
+    return True
+
+
+def unpack_sdist(sdist_tarball: Path, target_directory: Path) -> Path:
+    """Unarchive given tarball.
+
+    :returns: Path of the package source checkout.
+    """
+    with sdist_tarball.open(mode='rb') as tarball_fd:
+        with TarFile.gzopen(fileobj=tarball_fd, name=None) as tarball:
+            tarball.extractall(path=target_directory)
+    return target_directory / EXPECTED_SDIST_NAME_BASE
+
+
+def assert_dirs_equal(*dir_paths: t.List[Path]) -> None:
+    dir_comparison = dircmp(*dir_paths)
+    assert not dir_comparison.left_only
+    assert not dir_comparison.right_only
+    assert not dir_comparison.diff_files
+    assert not dir_comparison.funny_files
+
+
+def normalize_unpacked_rebuilt_sdist(sdist_path: Path) -> None:
+    top_pkg_info_path = sdist_path / 'PKG-INFO'
+    nested_pkg_info_path = (
+        sdist_path / 'lib' / f'{DIST_NAME}.egg-info' / 'PKG-INFO'
+    )
+    entry_points_path = nested_pkg_info_path.parent / 'entry_points.txt'
+
+    # setuptools v39 write out two trailing empty lines and an unknown platform
+    # while the recent don't
+    top_pkg_info_path.write_text(
+        top_pkg_info_path.read_text().replace(
+            'Classifier: Development Status :: 5',
+            'Platform: UNKNOWN\nClassifier: Development Status :: 5',
+        ) + '\n\n'
+    )
+    nested_pkg_info_path.write_text(
+        nested_pkg_info_path.read_text().replace(
+            'Classifier: Development Status :: 5',
+            'Platform: UNKNOWN\nClassifier: Development Status :: 5',
+        ) + '\n\n'
+    )
+
+    # setuptools v39 write out one trailing empty line while the recent don't
+    entry_points_path.write_text(entry_points_path.read_text() + '\n')
+
+
+@pytest.fixture
+def venv_python_exe(tmp_path: Path) -> t.Iterator[Path]:
+    venv_path = tmp_path / 'pytest-managed-venv'
+    mkvenv_cmd = (
+        current_interpreter, '-m', 'venv', str(venv_path),
+    )
+    check_call(mkvenv_cmd, env={}, stderr=PIPE, stdout=PIPE)
+    yield venv_path / 'bin' / 'python'
+    rmtree(venv_path)
+
+
+def build_dists(
+        python_exe: Path, *cli_args: t.Iterable[str],
+        env_vars: t.Dict[str, str],
+) -> str:
+    full_cmd = str(python_exe), '-m', 'build', *cli_args
+    return check_output(full_cmd, env=env_vars, stderr=PIPE)
+
+
+def pip_install(
+        python_exe: Path, *cli_args: t.Iterable[str],
+        env_vars: t.Dict[str, str] = None,
+) -> str:
+    if env_vars is None:
+        env_vars = {}
+    full_cmd = str(python_exe), '-m', 'pip', 'install', *cli_args
+    return check_output(full_cmd, env=env_vars, stderr=PIPE)
+
+
+def test_dist_rebuilds_with_manpages_premutations(
+        venv_python_exe: Path, tmp_path: Path,
+) -> None:
+    """Test a series of sdist rebuilds under different conditions.
+
+    This check builds sdists right from the Git checkout with and without
+    the manpages. It also does this using different versions of the setuptools
+    PEP 517 build backend being pinned. Finally, it builds a wheel out of one
+    of the rebuilt sdists.
+    As intermediate assertions, this test makes simple smoke tests along
+    the way.
+    """
+    pip_install(venv_python_exe, 'build ~= 0.10.0')
+
+    # Test building an sdist without manpages from the Git checkout
+    tmp_dir_sdist_without_manpages = tmp_path / 'sdist-without-manpages'
+    wipe_generated_manpages()
+    build_dists(
+        venv_python_exe, '--sdist',
+        f'--outdir={tmp_dir_sdist_without_manpages!s}',
+        str(SRC_ROOT_DIR),
+        env_vars={
+            'PIP_CONSTRAINT': str(MODERNISH_BUILD_DEPS_FILE),
+        },
+    )
+    tmp_path_sdist_without_manpages = (
+        tmp_dir_sdist_without_manpages / EXPECTED_SDIST_NAME
+    )
+    assert tmp_path_sdist_without_manpages.exists()
+    assert not contains_man1_pages(tmp_path_sdist_without_manpages)
+    sdist_without_manpages_path = unpack_sdist(
+        tmp_path_sdist_without_manpages,
+        tmp_dir_sdist_without_manpages / 'src',
+    )
+
+    # Test building an sdist with manpages from the Git checkout
+    # and lowest supported build deps
+    wipe_generated_manpages()
+    tmp_dir_sdist_with_manpages = tmp_path / 'sdist-with-manpages'
+    build_dists(
+        venv_python_exe, '--sdist',
+        '--config-setting=--build-manpages',
+        f'--outdir={tmp_dir_sdist_with_manpages!s}',
+        str(SRC_ROOT_DIR),
+        env_vars={
+            'PIP_CONSTRAINT': str(LOWEST_SUPPORTED_BUILD_DEPS_FILE),
+        },
+    )
+    tmp_path_sdist_with_manpages = (
+        tmp_dir_sdist_with_manpages / EXPECTED_SDIST_NAME
+    )
+    assert tmp_path_sdist_with_manpages.exists()
+    assert contains_man1_pages(tmp_path_sdist_with_manpages)
+    sdist_with_manpages_path = unpack_sdist(
+        tmp_path_sdist_with_manpages,
+        tmp_dir_sdist_with_manpages / 'src',
+    )
+
+    # Test re-building an sdist with manpages from the
+    # sdist contents that does not include the manpages
+    tmp_dir_rebuilt_sdist = tmp_path / 'rebuilt-sdist'
+    build_dists(
+        venv_python_exe, '--sdist',
+        '--config-setting=--build-manpages',
+        f'--outdir={tmp_dir_rebuilt_sdist!s}',
+        str(sdist_without_manpages_path),
+        env_vars={
+            'PIP_CONSTRAINT': str(MODERNISH_BUILD_DEPS_FILE),
+        },
+    )
+    tmp_path_rebuilt_sdist = tmp_dir_rebuilt_sdist / EXPECTED_SDIST_NAME
+    # Checking that the expected sdist got created
+    # from the previous unpacked sdist...
+    assert tmp_path_rebuilt_sdist.exists()
+    assert contains_man1_pages(tmp_path_rebuilt_sdist)
+    rebuilt_sdist_path = unpack_sdist(
+        tmp_path_rebuilt_sdist,
+        tmp_dir_rebuilt_sdist / 'src',
+    )
+    assert rebuilt_sdist_path.exists()
+    assert rebuilt_sdist_path.is_dir()
+    normalize_unpacked_rebuilt_sdist(rebuilt_sdist_path)
+    assert_dirs_equal(rebuilt_sdist_path, sdist_with_manpages_path)
+
+    # Test building a wheel from the rebuilt sdist with manpages contents
+    # and lowest supported build deps
+    tmp_dir_rebuilt_wheel = tmp_path / 'rebuilt-wheel'
+    build_dists(
+        venv_python_exe, '--wheel',
+        f'--outdir={tmp_dir_rebuilt_wheel!s}',
+        str(sdist_with_manpages_path),
+        env_vars={
+            'PIP_CONSTRAINT': str(LOWEST_SUPPORTED_BUILD_DEPS_FILE),
+        },
+    )
+    tmp_path_rebuilt_wheel = tmp_dir_rebuilt_wheel / EXPECTED_WHEEL_NAME
+    # Checking that the expected wheel got created...
+    assert tmp_path_rebuilt_wheel.exists()
+
+
+def test_pep660_editable_install_smoke(venv_python_exe: Path) -> None:
+    """Smoke-test PEP 660 editable install.
+
+    This verifies that the in-tree build backend wrapper
+    does not break any required interfaces.
+    """
+    pip_install(venv_python_exe, '-e', str(SRC_ROOT_DIR))
+
+    pip_show_cmd = (
+        str(venv_python_exe), '-m',
+        'pip', 'show', DIST_FILENAME_BASE,
+    )
+    installed_ansible_meta = check_output(
+        pip_show_cmd,
+        env={}, stderr=PIPE, text=True,
+    ).splitlines()
+    assert f'Name: {DIST_FILENAME_BASE}' in installed_ansible_meta
+    assert f'Version: {PKG_DIST_VERSION}' in installed_ansible_meta
+
+    pip_runtime_version_cmd = (
+        str(venv_python_exe), '-c',
+        'from ansible import __version__; print(__version__)',
+    )
+    runtime_ansible_version = check_output(
+        pip_runtime_version_cmd,
+        env={}, stderr=PIPE, text=True,
+    ).strip()
+    assert runtime_ansible_version == PKG_DIST_VERSION