ansible-test - Improve requirements handling. (#77825)

The requirements for virtualenv and coverage are now kept in a requirements file for easier container builds.

The test-constraints sanity test has been updated to make sure the requirements file is kept up-to-date.

test/lib/ansible_test/_data/requirements/ansible-test.txt

@@ -0,0 +1,4 @@
+# The test-constraints sanity test verifies this file, but changes must be made manually to keep it in up-to-date.
+virtualenv == 16.7.12 ; python_version < '3'
+coverage == 6.3.3 ; python_version >= '3.7' and python_version <= '3.11'
+coverage == 4.5.4 ; python_version >= '2.6' and python_version <= '3.6'

test/lib/ansible_test/_internal/coverage_util.py

@@ -63,12 +63,12 @@ class CoverageVersion:
 
 
 COVERAGE_VERSIONS = (
+    # IMPORTANT: Keep this in sync with the ansible-test.txt requirements file.
     CoverageVersion('6.3.3', 7, (3, 7), (3, 11)),
-    CoverageVersion('4.5.4', 0, (2, 6), (3, 7)),
+    CoverageVersion('4.5.4', 0, (2, 6), (3, 6)),
 )
 """
 This tuple specifies the coverage version to use for Python version ranges.
-When versions overlap, the latest version of coverage (listed first) will be used.
 """
 
 CONTROLLER_COVERAGE_VERSION = COVERAGE_VERSIONS[0]
@@ -92,6 +92,9 @@ def get_coverage_version(version: str) -> CoverageVersion:
     if not supported_versions:
         raise InternalError(f'Python {version} has no matching entry in COVERAGE_VERSIONS.')
 
+    if len(supported_versions) > 1:
+        raise InternalError(f'Python {version} has multiple matching entries in COVERAGE_VERSIONS.')
+
     coverage_version = supported_versions[0]
 
     return coverage_version

test/lib/ansible_test/_internal/python_requirements.py

@@ -62,6 +62,8 @@ from .coverage_util import (
 QUIET_PIP_SCRIPT_PATH = os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'setup', 'quiet_pip.py')
 REQUIREMENTS_SCRIPT_PATH = os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'setup', 'requirements.py')
 
+# IMPORTANT: Keep this in sync with the ansible-test.txt requirements file.
+VIRTUALENV_VERSION = '16.7.12'
 
 # Pip Abstraction
 
@@ -211,7 +213,7 @@ def collect_requirements(
     if virtualenv:
         # sanity tests on Python 2.x install virtualenv when it is too old or is not already installed and the `--requirements` option is given
         # the last version of virtualenv with no dependencies is used to minimize the changes made outside a virtual environment
-        commands.extend(collect_package_install(packages=['virtualenv==16.7.12'], constraints=False))
+        commands.extend(collect_package_install(packages=[f'virtualenv=={VIRTUALENV_VERSION}'], constraints=False))
 
     if coverage:
         commands.extend(collect_package_install(packages=[f'coverage=={get_coverage_version(python.version).coverage_version}'], constraints=False))

test/sanity/code-smell/test-constraints.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+import pathlib
 import re
 import sys
 
@@ -14,9 +15,15 @@ def main():
         if path == 'test/lib/ansible_test/_data/requirements/ansible.txt':
             # This file is an exact copy of the ansible requirements.txt and should not conflict with other constraints.
             continue
+
         with open(path, 'r') as path_fd:
             requirements[path] = parse_requirements(path_fd.read().splitlines())
 
+        if path == 'test/lib/ansible_test/_data/requirements/ansible-test.txt':
+            # Special handling is required for ansible-test's requirements file.
+            check_ansible_test(path, requirements.pop(path))
+            continue
+
     frozen_sanity = {}
     non_sanity_requirements = set()
 
@@ -65,6 +72,33 @@ def main():
                     req[0], req[1], req[3].start('constraints') + 1, req[3].group('constraints'), name))
 
 
+def check_ansible_test(path: str, requirements: list[tuple[int, str, re.Match]]) -> None:
+    sys.path.insert(0, str(pathlib.Path(__file__).parent.parent.parent.joinpath('lib')))
+
+    from ansible_test._internal.python_requirements import VIRTUALENV_VERSION
+    from ansible_test._internal.coverage_util import COVERAGE_VERSIONS
+    from ansible_test._internal.util import version_to_str
+
+    expected_lines = set([
+        f"virtualenv == {VIRTUALENV_VERSION} ; python_version < '3'",
+    ] + [
+        f"coverage == {item.coverage_version} ; python_version >= '{version_to_str(item.min_python)}' and python_version <= '{version_to_str(item.max_python)}'"
+        for item in COVERAGE_VERSIONS
+    ])
+
+    for idx, requirement in enumerate(requirements):
+        lineno, line, match = requirement
+
+        if line in expected_lines:
+            expected_lines.remove(line)
+            continue
+
+        print('%s:%d:%d: unexpected line: %s' % (path, lineno, 1, line))
+
+    for expected_line in sorted(expected_lines):
+        print('%s:%d:%d: missing line: %s' % (path, requirements[-1][0] + 1, 1, expected_line))
+
+
 def parse_requirements(lines):
     # see https://www.python.org/dev/peps/pep-0508/#names
     pattern = re.compile(r'^(?P<name>[A-Z0-9][A-Z0-9._-]*[A-Z0-9]|[A-Z0-9])(?P<extras> *\[[^]]*])?(?P<constraints>[^;#]*)(?P<markers>[^#]*)(?P<comment>.*)$',