ansible-test - Pre-build PyYAML wheels (#81300)

This works around Cython failures when attempting to install PyYAML >= 5.4 <= 6.0.

changelogs/fragments/ansible-test-pyyaml-build.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - ansible-test - Pre-build a PyYAML wheel before installing requirements to avoid a potential Cython build failure.

hacking/update-sanity-requirements.py

@@ -7,11 +7,15 @@ from __future__ import annotations
 import argparse
 import dataclasses
 import pathlib
+import re
 import subprocess
 import tempfile
 import typing as t
 import venv
 
+import packaging.version
+import packaging.specifiers
+
 try:
     import argcomplete
 except ImportError:
@@ -59,7 +63,22 @@ class SanityTest:
 
             pip_freeze = subprocess.run(pip + ['freeze'] + freeze_options, env=env, check=True, capture_output=True, text=True)
 
-        requirements = f'# edit "{self.source_path.name}" and generate with: {SELF} --test {self.name}\n{pip_freeze.stdout}'
+        self.write_requirements(pip_freeze.stdout)
+
+    def update_pre_build(self) -> None:
+        """Update requirements in place with current pre-build instructions."""
+        requirements = pathlib.Path(self.requirements_path).read_text()
+        lines = requirements.splitlines(keepends=True)
+        lines = [line for line in lines if not line.startswith('#')]
+        requirements = ''.join(lines)
+
+        self.write_requirements(requirements)
+
+    def write_requirements(self, requirements: str) -> None:
+        """Write the given test requirements to the requirements file for this test."""
+        pre_build = pre_build_instructions(requirements)
+
+        requirements = f'# edit "{self.source_path.name}" and generate with: {SELF} --test {self.name}\n{pre_build}{requirements}'
 
         with open(self.requirements_path, 'w') as requirement_file:
             requirement_file.write(requirements)
@@ -73,6 +92,38 @@ class SanityTest:
         )
 
 
+def pre_build_instructions(requirements: str) -> str:
+    """Parse the given requirements and return any applicable pre-build instructions."""
+    parsed_requirements = requirements.splitlines()
+
+    package_versions = {
+        match.group('package').lower(): match.group('version') for match
+        in (re.search('^(?P<package>.*)==(?P<version>.*)$', requirement) for requirement in parsed_requirements)
+        if match
+    }
+
+    instructions: list[str] = []
+
+    build_constraints = (
+        ('pyyaml', '>= 5.4, <= 6.0', ('Cython < 3.0',)),
+    )
+
+    for package, specifier, constraints in build_constraints:
+        version_string = package_versions.get(package)
+
+        if version_string:
+            version = packaging.version.Version(version_string)
+            specifier_set = packaging.specifiers.SpecifierSet(specifier)
+
+            if specifier_set.contains(version):
+                instructions.append(f'# pre-build requirement: {package} == {version}\n')
+
+                for constraint in constraints:
+                    instructions.append(f'# pre-build constraint: {constraint}\n')
+
+    return ''.join(instructions)
+
+
 def main() -> None:
     tests = find_tests()
 
@@ -86,6 +137,12 @@ def main() -> None:
         help='test requirements to update'
     )
 
+    parser.add_argument(
+        '--pre-build-only',
+        action='store_true',
+        help='apply pre-build instructions to existing requirements',
+    )
+
     if argcomplete:
         argcomplete.autocomplete(parser)
 
@@ -96,7 +153,11 @@ def main() -> None:
 
     for test in tests:
         print(f'===[ {test.name} ]===', flush=True)
-        test.freeze_requirements()
+
+        if args.pre_build_only:
+            test.update_pre_build()
+        else:
+            test.freeze_requirements()
 
 
 def find_tests() -> t.List[SanityTest]:

test/integration/targets/canonical-pep517-self-packaging/modernish-build-constraints.txt

@@ -7,4 +7,4 @@ wheel == 0.38.4
 docutils == 0.19
 Jinja2 == 3.1.2
 MarkupSafe == 2.1.2
-PyYAML == 6.0
+PyYAML == 6.0.1

test/lib/ansible_test/_data/requirements/sanity.ansible-doc.txt

@@ -1,4 +1,6 @@
 # edit "sanity.ansible-doc.in" and generate with: hacking/update-sanity-requirements.py --test ansible-doc
+# pre-build requirement: pyyaml == 6.0
+# pre-build constraint: Cython < 3.0
 Jinja2==3.1.2
 MarkupSafe==2.1.2
 packaging==23.0

test/lib/ansible_test/_data/requirements/sanity.changelog.txt

@@ -1,4 +1,6 @@
 # edit "sanity.changelog.in" and generate with: hacking/update-sanity-requirements.py --test changelog
+# pre-build requirement: pyyaml == 6.0
+# pre-build constraint: Cython < 3.0
 antsibull-changelog==0.19.0
 docutils==0.18.1
 packaging==23.0

test/lib/ansible_test/_data/requirements/sanity.import.plugin.txt

@@ -1,4 +1,6 @@
 # edit "sanity.import.plugin.in" and generate with: hacking/update-sanity-requirements.py --test import.plugin
+# pre-build requirement: pyyaml == 6.0
+# pre-build constraint: Cython < 3.0
 Jinja2==3.1.2
 MarkupSafe==2.1.2
 PyYAML==6.0

test/lib/ansible_test/_data/requirements/sanity.import.txt

@@ -1,2 +1,4 @@
 # edit "sanity.import.in" and generate with: hacking/update-sanity-requirements.py --test import
+# pre-build requirement: pyyaml == 6.0
+# pre-build constraint: Cython < 3.0
 PyYAML==6.0

test/lib/ansible_test/_data/requirements/sanity.integration-aliases.txt

@@ -1,2 +1,4 @@
 # edit "sanity.integration-aliases.in" and generate with: hacking/update-sanity-requirements.py --test integration-aliases
+# pre-build requirement: pyyaml == 6.0
+# pre-build constraint: Cython < 3.0
 PyYAML==6.0

test/lib/ansible_test/_data/requirements/sanity.pylint.txt

@@ -1,4 +1,6 @@
 # edit "sanity.pylint.in" and generate with: hacking/update-sanity-requirements.py --test pylint
+# pre-build requirement: pyyaml == 6.0
+# pre-build constraint: Cython < 3.0
 astroid==2.15.4
 dill==0.3.6
 isort==5.12.0

test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt

@@ -1,3 +1,5 @@
 # edit "sanity.runtime-metadata.in" and generate with: hacking/update-sanity-requirements.py --test runtime-metadata
+# pre-build requirement: pyyaml == 6.0
+# pre-build constraint: Cython < 3.0
 PyYAML==6.0
 voluptuous==0.13.1

test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt

@@ -1,4 +1,6 @@
 # edit "sanity.validate-modules.in" and generate with: hacking/update-sanity-requirements.py --test validate-modules
+# pre-build requirement: pyyaml == 6.0
+# pre-build constraint: Cython < 3.0
 antsibull-docs-parser==1.0.0
 Jinja2==3.1.2
 MarkupSafe==2.1.2

test/lib/ansible_test/_data/requirements/sanity.yamllint.txt

@@ -1,4 +1,6 @@
 # edit "sanity.yamllint.in" and generate with: hacking/update-sanity-requirements.py --test yamllint
+# pre-build requirement: pyyaml == 6.0
+# pre-build constraint: Cython < 3.0
 pathspec==0.11.1
 PyYAML==6.0
 yamllint==1.30.0

test/lib/ansible_test/_internal/commands/sanity/__init__.py

@@ -71,6 +71,7 @@ from ...executor import (
 )
 
 from ...python_requirements import (
+    PipCommand,
     PipInstall,
     collect_requirements,
     run_pip,
@@ -1157,7 +1158,7 @@ def create_sanity_virtualenv(
     # The path to the virtual environment must be kept short to avoid the 127 character shebang length limit on Linux.
     # If the limit is exceeded, generated entry point scripts from pip installed packages will fail with syntax errors.
     virtualenv_install = json.dumps([command.serialize() for command in commands], indent=4)
-    virtualenv_hash = hashlib.sha256(to_bytes(virtualenv_install)).hexdigest()[:8]
+    virtualenv_hash = hash_pip_commands(commands)
     virtualenv_cache = os.path.join(os.path.expanduser('~/.ansible/test/venv'))
     virtualenv_path = os.path.join(virtualenv_cache, label, f'{python.version}', virtualenv_hash)
     virtualenv_marker = os.path.join(virtualenv_path, 'marker.txt')
@@ -1200,6 +1201,39 @@ def create_sanity_virtualenv(
     return virtualenv_python
 
 
+def hash_pip_commands(commands: list[PipCommand]) -> str:
+    """Return a short hash unique to the given list of pip commands, suitable for identifying the resulting sanity test environment."""
+    serialized_commands = json.dumps([make_pip_command_hashable(command) for command in commands], indent=4)
+
+    return hashlib.sha256(to_bytes(serialized_commands)).hexdigest()[:8]
+
+
+def make_pip_command_hashable(command: PipCommand) -> tuple[str, dict[str, t.Any]]:
+    """Return a serialized version of the given pip command that is suitable for hashing."""
+    if isinstance(command, PipInstall):
+        # The pre-build instructions for pip installs must be omitted, so they do not affect the hash.
+        # This is allows the pre-build commands to be added without breaking sanity venv caching.
+        # It is safe to omit these from the hash since they only affect packages used during builds, not what is installed in the venv.
+        command = PipInstall(
+            requirements=[omit_pre_build_from_requirement(*req) for req in command.requirements],
+            constraints=list(command.constraints),
+            packages=list(command.packages),
+        )
+
+    return command.serialize()
+
+
+def omit_pre_build_from_requirement(path: str, requirements: str) -> tuple[str, str]:
+    """Return the given requirements with pre-build instructions omitted."""
+    lines = requirements.splitlines(keepends=True)
+
+    # CAUTION: This code must be kept in sync with the code which processes pre-build instructions in:
+    #          test/lib/ansible_test/_util/target/setup/requirements.py
+    lines = [line for line in lines if not line.startswith('# pre-build ')]
+
+    return path, ''.join(lines)
+
+
 def check_sanity_virtualenv_yaml(python: VirtualPythonConfig) -> t.Optional[bool]:
     """Return True if PyYAML has libyaml support for the given sanity virtual environment, False if it does not and None if it was not found."""
     virtualenv_path = os.path.dirname(os.path.dirname(python.path))

test/lib/ansible_test/_util/target/setup/requirements.py

@@ -134,6 +134,14 @@ def install(pip, options):  # type: (str, t.Dict[str, t.Any]) -> None
         options.extend(packages)
 
         for path, content in requirements:
+            if path.split(os.sep)[0] in ('test', 'requirements'):
+                # Support for pre-build is currently limited to requirements embedded in ansible-test and those used by ansible-core.
+                # Requirements from ansible-core can be found in the 'test' and 'requirements' directories.
+                # This feature will probably be extended to support collections after further testing.
+                # Requirements from collections can be found in the 'tests' directory.
+                for pre_build in parse_pre_build_instructions(content):
+                    pre_build.execute(pip)
+
             write_text_file(os.path.join(tempdir, path), content, True)
             options.extend(['-r', path])
 
@@ -150,6 +158,61 @@ def install(pip, options):  # type: (str, t.Dict[str, t.Any]) -> None
         remove_tree(tempdir)
 
 
+class PreBuild:
+    """Parsed pre-build instructions."""
+
+    def __init__(self, requirement):  # type: (str) -> None
+        self.requirement = requirement
+        self.constraints = []  # type: list[str]
+
+    def execute(self, pip):  # type: (str) -> None
+        """Execute these pre-build instructions."""
+        tempdir = tempfile.mkdtemp(prefix='ansible-test-', suffix='-pre-build')
+
+        try:
+            options = common_pip_options()
+            options.append(self.requirement)
+
+            constraints = '\n'.join(self.constraints) + '\n'
+            constraints_path = os.path.join(tempdir, 'constraints.txt')
+
+            write_text_file(constraints_path, constraints, True)
+
+            env = common_pip_environment()
+            env.update(PIP_CONSTRAINT=constraints_path)
+
+            command = [sys.executable, pip, 'wheel'] + options
+
+            execute_command(command, env=env, cwd=tempdir)
+        finally:
+            remove_tree(tempdir)
+
+
+def parse_pre_build_instructions(requirements):  # type: (str) -> list[PreBuild]
+    """Parse the given pip requirements and return a list of extracted pre-build instructions."""
+    # CAUTION: This code must be kept in sync with the sanity test hashing code in:
+    #          test/lib/ansible_test/_internal/commands/sanity/__init__.py
+
+    pre_build_prefix = '# pre-build '
+    pre_build_requirement_prefix = pre_build_prefix + 'requirement: '
+    pre_build_constraint_prefix = pre_build_prefix + 'constraint: '
+
+    lines = requirements.splitlines()
+    pre_build_lines = [line for line in lines if line.startswith(pre_build_prefix)]
+
+    instructions = []  # type: list[PreBuild]
+
+    for line in pre_build_lines:
+        if line.startswith(pre_build_requirement_prefix):
+            instructions.append(PreBuild(line[len(pre_build_requirement_prefix):]))
+        elif line.startswith(pre_build_constraint_prefix):
+            instructions[-1].constraints.append(line[len(pre_build_constraint_prefix):])
+        else:
+            raise RuntimeError('Unsupported pre-build comment: ' + line)
+
+    return instructions
+
+
 def uninstall(pip, options):  # type: (str, t.Dict[str, t.Any]) -> None
     """Perform a pip uninstall."""
     packages = options['packages']

test/sanity/code-smell/deprecated-config.requirements.txt

@@ -1,4 +1,6 @@
 # edit "deprecated-config.requirements.in" and generate with: hacking/update-sanity-requirements.py --test deprecated-config
+# pre-build requirement: pyyaml == 6.0
+# pre-build constraint: Cython < 3.0
 Jinja2==3.1.2
 MarkupSafe==2.1.2
 PyYAML==6.0

test/sanity/code-smell/package-data.requirements.txt

@@ -1,4 +1,6 @@
 # edit "package-data.requirements.in" and generate with: hacking/update-sanity-requirements.py --test package-data
+# pre-build requirement: pyyaml == 6.0
+# pre-build constraint: Cython < 3.0
 antsibull-changelog==0.19.0
 build==0.10.0
 docutils==0.17.1