[stable-2.16] release.py - Auto-update setuptools upper bound (#83713) (#83743)

When releases are prepared, the upper bound on setuptools in pyproject.toml will be automatically updated
to the latest version available on PyPI. This version will then be tested by the package-data sanity test
during the release process and will be used to build the release.

This change ensures that a released version of ansible-core can be built in the future if a new setuptools
release includes breaking changes that would prevent building a functional package. If a downstream package
maintainer requires a newer setuptools version than the upper bound permits, they can patch pyproject.toml
as needed. Since ansible-core releases support specific Python versions, lack of support for new setuptools
releases will have no effect on support for future Python versions.
(cherry picked from commit 4e69d83fac)

packaging/release.py

@@ -369,6 +369,7 @@ ANSIBLE_DIR = ANSIBLE_LIB_DIR / "ansible"
 ANSIBLE_BIN_DIR = CHECKOUT_DIR / "bin"
 ANSIBLE_RELEASE_FILE = ANSIBLE_DIR / "release.py"
 ANSIBLE_REQUIREMENTS_FILE = CHECKOUT_DIR / "requirements.txt"
+ANSIBLE_PYPROJECT_TOML_FILE = CHECKOUT_DIR / "pyproject.toml"
 
 DIST_DIR = CHECKOUT_DIR / "dist"
 VENV_DIR = DIST_DIR / ".venv" / "release"
@@ -708,6 +709,35 @@ twine
     return env
 
 
+def get_pypi_project(repository: str, project: str, version: Version | None = None) -> dict[str, t.Any]:
+    """Return the project JSON from PyPI for the specified repository, project and version (optional)."""
+    endpoint = PYPI_ENDPOINTS[repository]
+
+    if version:
+        url = f"{endpoint}/{project}/{version}/json"
+    else:
+        url = f"{endpoint}/{project}/json"
+
+    opener = urllib.request.build_opener()
+    response: http.client.HTTPResponse
+
+    try:
+        with opener.open(url) as response:
+            data = json.load(response)
+    except urllib.error.HTTPError as ex:
+        if version:
+            target = f'{project!r} version {version}'
+        else:
+            target = f'{project!r}'
+
+        if ex.status == http.HTTPStatus.NOT_FOUND:
+            raise ApplicationError(f"Could not find {target} on PyPI.") from None
+
+        raise RuntimeError(f"Failed to get {target} from PyPI.") from ex
+
+    return data
+
+
 def get_ansible_version(version: str | None = None, /, commit: str | None = None, mode: VersionMode = VersionMode.DEFAULT) -> Version:
     """Parse and return the current ansible-core version, the provided version or the version from the provided commit."""
     if version and commit:
@@ -802,6 +832,38 @@ def set_ansible_version(current_version: Version, requested_version: Version) ->
     ANSIBLE_RELEASE_FILE.write_text(updated)
 
 
+def get_latest_setuptools_version() -> Version:
+    """Return the latest setuptools version found on PyPI."""
+    data = get_pypi_project('pypi', 'setuptools')
+    version = Version(data['info']['version'])
+
+    return version
+
+
+def set_setuptools_upper_bound(requested_version: Version) -> None:
+    """Set the upper bound on setuptools in pyproject.toml."""
+    current = ANSIBLE_PYPROJECT_TOML_FILE.read_text()
+    pattern = re.compile(r'^(?P<begin>requires = \["setuptools >= )(?P<lower>[^,]+)(?P<middle>, <= )(?P<upper>[^"]+)(?P<end>".*)$', re.MULTILINE)
+    match = pattern.search(current)
+
+    if not match:
+        raise ApplicationError(f"Unable to find the 'requires' entry in: {ANSIBLE_PYPROJECT_TOML_FILE.relative_to(CHECKOUT_DIR)}")
+
+    current_version = Version(match.group('upper'))
+
+    if requested_version == current_version:
+        return
+
+    display.show(f"Updating setuptools upper bound from {current_version} to {requested_version} ...")
+
+    updated = pattern.sub(fr'\g<begin>\g<lower>\g<middle>{requested_version}\g<end>', current)
+
+    if current == updated:
+        raise RuntimeError("Failed to set the setuptools upper bound.")
+
+    ANSIBLE_PYPROJECT_TOML_FILE.write_text(updated)
+
+
 def create_reproducible_sdist(original_path: pathlib.Path, output_path: pathlib.Path, mtime: int) -> None:
     """Read the specified sdist and write out a new copy with uniform file metadata at the specified location."""
     with tarfile.open(original_path) as original_archive:
@@ -878,21 +940,7 @@ def calculate_digest(path: pathlib.Path) -> str:
 @functools.cache
 def get_release_artifact_details(repository: str, version: Version, validate: bool) -> list[ReleaseArtifact]:
     """Return information about the release artifacts hosted on PyPI."""
-    endpoint = PYPI_ENDPOINTS[repository]
+    data = get_pypi_project(repository, 'ansible-core', version)
-    url = f"{endpoint}/ansible-core/{version}/json"
-
-    opener = urllib.request.build_opener()
-    response: http.client.HTTPResponse
-
-    try:
-        with opener.open(url) as response:
-            data = json.load(response)
-    except urllib.error.HTTPError as ex:
-        if ex.status == http.HTTPStatus.NOT_FOUND:
-            raise ApplicationError(f"Version {version} not found on PyPI.") from None
-
-        raise RuntimeError(f"Failed to get {version} from PyPI: {ex}") from ex
-
     artifacts = [describe_release_artifact(version, item, validate) for item in data["urls"]]
 
     expected_artifact_types = {"bdist_wheel", "sdist"}
@@ -1138,6 +1186,7 @@ command = CommandFramework(
     mailto=dict(name="--mailto", action="store_true", help="write announcement to mailto link instead of console"),
     validate=dict(name="--no-validate", action="store_false", help="disable validation of PyPI artifacts against local ones"),
     prompt=dict(name="--no-prompt", action="store_false", help="disable interactive prompt before publishing with twine"),
+    setuptools=dict(name='--no-setuptools', action="store_false", help="disable updating setuptools upper bound"),
     allow_tag=dict(action="store_true", help="allow an existing release tag (for testing)"),
     allow_stale=dict(action="store_true", help="allow a stale checkout (for testing)"),
     allow_dirty=dict(action="store_true", help="allow untracked files and files with changes (for testing)"),
@@ -1198,6 +1247,7 @@ def prepare(final: bool = False, pre: str | None = None, version: str | None = N
     """Prepare a release."""
     command.run(
         update_version,
+        update_setuptools,
         check_state,
         generate_summary,
         generate_changelog,
@@ -1218,6 +1268,16 @@ def update_version(final: bool = False, pre: str | None = None, version: str | N
     set_ansible_version(current_version, requested_version)
 
 
+@command
+def update_setuptools(setuptools: bool) -> None:
+    """Update the setuptools upper bound in pyproject.toml."""
+    if not setuptools:
+        return
+
+    requested_version = get_latest_setuptools_version()
+    set_setuptools_upper_bound(requested_version)
+
+
 @command
 def generate_summary() -> None:
     """Generate a summary changelog fragment for this release."""

pyproject.toml

@@ -1,3 +1,3 @@
 [build-system]
-requires = ["setuptools >= 66.1.0"]  # minimum setuptools version supporting Python 3.12
+requires = ["setuptools >= 66.1.0, <= 72.1.0"]  # lower bound to support controller Python versions, upper bound for latest version tested at release
 build-backend = "setuptools.build_meta"

test/sanity/code-smell/package-data.py

@@ -95,7 +95,7 @@ def clean_repository(complete_file_list: list[str]) -> t.Generator[str, None, No
 def build(source_dir: str, tmp_dir: str) -> tuple[pathlib.Path, pathlib.Path]:
     """Create a sdist and wheel."""
     create = subprocess.run(
-        [sys.executable, '-m', 'build', '--no-isolation', '--outdir', tmp_dir],
+        [sys.executable, '-m', 'build', '--outdir', tmp_dir],
         stdin=subprocess.DEVNULL,
         capture_output=True,
         text=True,