📦 Switch sdist build-system to pure setuptools (#80255)

This patch modifies the in-tree build backend to build sdists that swap
out pointers to it in the `pyproject.toml`'s `[build-system]` section.

The effect of this is that the first build from source (for example,
from a Git checkout) uses our PEP 517 in-tree build backend. But the
produced tarball has `build-backend` set to `setuptools.build_meta`
which is the native build backend of `setuptools`. So any following
builds from that sdist will skip using the in-tree build backend,
calling the setuptools' one.
The good news is that if the first build generated the manpages, they
will be included and won't go anywhere even though, a different build
system is in place.

Combined with #80253, this will make sure not to modify the current
source checkout on that first build.

Co-authored-by: Matt Clay <matt@mystile.com>

packaging/pep517_backend/_backend.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import os
+import re
 import subprocess
 import sys
 import typing as t
@@ -131,6 +132,19 @@ def build_sdist(  # noqa: WPS210, WPS430
                 )
                 rst_in.unlink()
 
+        Path('pyproject.toml').write_text(
+            re.sub(
+                r"""(?x)
+                backend-path\s=\s\[  # value is a list of double-quoted strings
+                    [^]]+
+                ].*\n
+                build-backend\s=\s"[^"]+".*\n  # value is double-quoted
+                """,
+                'build-backend = "setuptools.build_meta"\n',
+                Path('pyproject.toml').read_text(),
+            )
+        )
+
         built_sdist_basename = _setuptools_build_sdist(
             sdist_directory=sdist_directory,
             config_settings=config_settings,

test/integration/targets/canonical-pep517-self-packaging/runme_test.py

@@ -3,14 +3,28 @@
 from __future__ import annotations
 
 from filecmp import dircmp
-from os import environ
+from os import chdir, environ, PathLike
 from pathlib import Path
 from shutil import rmtree
 from subprocess import check_call, check_output, PIPE
-from sys import executable as current_interpreter
+from sys import executable as current_interpreter, version_info
 from tarfile import TarFile
 import typing as t
 
+try:
+    from contextlib import chdir as _chdir_cm
+except ImportError:
+    from contextlib import contextmanager as _contextmanager
+
+    @_contextmanager
+    def _chdir_cm(path: PathLike) -> t.Iterator[None]:
+        original_wd = Path.cwd()
+        chdir(path)
+        try:
+            yield
+        finally:
+            chdir(original_wd)
+
 import pytest
 
 
@@ -36,6 +50,8 @@ EXPECTED_SDIST_NAME_BASE = f'{DIST_FILENAME_BASE}-{PKG_DIST_VERSION}'
 EXPECTED_SDIST_NAME = f'{EXPECTED_SDIST_NAME_BASE}.tar.gz'
 EXPECTED_WHEEL_NAME = f'{DIST_NAME}-{PKG_DIST_VERSION}-py3-none-any.whl'
 
+IS_PYTHON310_PLUS = version_info[:2] >= (3, 10)
+
 
 def wipe_generated_manpages() -> None:
     """Ensure man1 pages aren't present in the source checkout."""
@@ -117,22 +133,109 @@ def venv_python_exe(tmp_path: Path) -> t.Iterator[Path]:
     rmtree(venv_path)
 
 
+def run_with_venv_python(
+        python_exe: Path, *cli_args: t.Iterable[str],
+        env_vars: t.Dict[str, str] = None,
+) -> str:
+    if env_vars is None:
+        env_vars = {}
+    full_cmd = str(python_exe), *cli_args
+    return check_output(full_cmd, env=env_vars, stderr=PIPE)
+
+
 def build_dists(
         python_exe: Path, *cli_args: t.Iterable[str],
         env_vars: t.Dict[str, str],
 ) -> str:
-    full_cmd = str(python_exe), '-m', 'build', *cli_args
-    return check_output(full_cmd, env=env_vars, stderr=PIPE)
+    return run_with_venv_python(
+        python_exe, '-m', 'build',
+        *cli_args, env_vars=env_vars,
+    )
 
 
 def pip_install(
         python_exe: Path, *cli_args: t.Iterable[str],
         env_vars: t.Dict[str, str] = None,
 ) -> str:
-    if env_vars is None:
-        env_vars = {}
-    full_cmd = str(python_exe), '-m', 'pip', 'install', *cli_args
-    return check_output(full_cmd, env=env_vars, stderr=PIPE)
+    return run_with_venv_python(
+        python_exe, '-m', 'pip', 'install',
+        *cli_args, env_vars=env_vars,
+    )
+
+
+def test_installing_sdist_build_with_modern_deps_to_old_env(
+        venv_python_exe: Path, tmp_path: Path,
+) -> None:
+    pip_install(venv_python_exe, 'build ~= 0.10.0')
+    tmp_dir_sdist_w_modern_tools = tmp_path / 'sdist-w-modern-tools'
+    build_dists(
+        venv_python_exe, '--sdist',
+        '--config-setting=--build-manpages',
+        f'--outdir={tmp_dir_sdist_w_modern_tools!s}',
+        str(SRC_ROOT_DIR),
+        env_vars={
+            'PIP_CONSTRAINT': str(MODERNISH_BUILD_DEPS_FILE),
+        },
+    )
+    tmp_path_sdist_w_modern_tools = (
+        tmp_dir_sdist_w_modern_tools / EXPECTED_SDIST_NAME
+    )
+
+    # Downgrading pip, because v20+ supports in-tree build backends
+    pip_install(venv_python_exe, 'pip ~= 19.3.1')
+
+    # Smoke test — installing an sdist with pip that does not support
+    # in-tree build backends.
+    pip_install(
+        venv_python_exe, str(tmp_path_sdist_w_modern_tools), '--no-deps',
+    )
+
+    # Downgrading pip, because versions that support PEP 517 don't allow
+    # disabling it with `--no-use-pep517` when `build-backend` is set in
+    # the `[build-system]` section of `pyproject.toml`, considering this
+    # an explicit opt-in.
+    if not IS_PYTHON310_PLUS:
+        pip_install(venv_python_exe, 'pip == 18.0')
+
+    # Smoke test — installing an sdist with pip that does not support invoking
+    # PEP 517 interface at all.
+    # In this scenario, pip will run `setup.py install` since `wheel` is not in
+    # the environment.
+    if IS_PYTHON310_PLUS:
+        tmp_dir_unpacked_sdist_root = tmp_path / 'unpacked-sdist'
+        tmp_dir_unpacked_sdist_path = tmp_dir_unpacked_sdist_root / EXPECTED_SDIST_NAME_BASE
+        with TarFile.gzopen(tmp_path_sdist_w_modern_tools) as sdist_fd:
+            sdist_fd.extractall(path=tmp_dir_unpacked_sdist_root)
+
+        with _chdir_cm(tmp_dir_unpacked_sdist_path):
+            run_with_venv_python(
+                venv_python_exe, 'setup.py', 'sdist',
+                env_vars={'PATH': environ['PATH']},
+            )
+            run_with_venv_python(
+                venv_python_exe, 'setup.py', 'install',
+                env_vars={'PATH': environ['PATH']},
+            )
+    else:
+        pip_install(
+            venv_python_exe, str(tmp_path_sdist_w_modern_tools), '--no-deps',
+        )
+
+    # Smoke test — installing an sdist with pip that does not support invoking
+    # PEP 517 interface at all.
+    # With `wheel` present, pip will run `setup.py bdist_wheel` and then,
+    # unpack the result.
+    pip_install(venv_python_exe, 'wheel')
+    if IS_PYTHON310_PLUS:
+        with _chdir_cm(tmp_dir_unpacked_sdist_path):
+            run_with_venv_python(
+                venv_python_exe, 'setup.py', 'bdist_wheel',
+                env_vars={'PATH': environ['PATH']},
+            )
+    else:
+        pip_install(
+            venv_python_exe, str(tmp_path_sdist_w_modern_tools), '--no-deps',
+        )
 
 
 def test_dist_rebuilds_with_manpages_premutations(
@@ -209,7 +312,19 @@ def test_dist_rebuilds_with_manpages_premutations(
     # Checking that the expected sdist got created
     # from the previous unpacked sdist...
     assert tmp_path_rebuilt_sdist.exists()
-    assert contains_man1_pages(tmp_path_rebuilt_sdist)
+    # NOTE: The following assertion is disabled due to the fact that, when
+    # NOTE: building an sdist from the original source checkout, the build
+    # NOTE: backend replaces itself with pure setuptools in the resulting
+    # NOTE: sdist, and the following rebuilds from that sdist are no longer
+    # NOTE: able to process the custom config settings that are implemented in
+    # NOTE: the in-tree build backend. It is expected that said
+    # NOTE: `pyproject.toml` mutation change will be reverted once all of the
+    # NOTE: supported `ansible-core` versions ship wheels, meaning that the
+    # NOTE: end-users won't be building the distribution from sdist on install.
+    # NOTE: Another case, when it can be reverted is declaring pip below v20
+    # NOTE: unsupported — it is the first version to support in-tree build
+    # NOTE: backends natively.
+    # assert contains_man1_pages(tmp_path_rebuilt_sdist)  # FIXME: See #80255
     rebuilt_sdist_path = unpack_sdist(
         tmp_path_rebuilt_sdist,
         tmp_dir_rebuilt_sdist / 'src',