diff --git a/changelogs/fragments/ansible-test-pyyaml-build.yml b/changelogs/fragments/ansible-test-pyyaml-build.yml new file mode 100644 index 00000000000..5e971b2a515 --- /dev/null +++ b/changelogs/fragments/ansible-test-pyyaml-build.yml @@ -0,0 +1,2 @@ +bugfixes: + - ansible-test - Pre-build a PyYAML wheel before installing requirements to avoid a potential Cython build failure. diff --git a/hacking/update-sanity-requirements.py b/hacking/update-sanity-requirements.py index 63eaec786a1..5861590beaf 100755 --- a/hacking/update-sanity-requirements.py +++ b/hacking/update-sanity-requirements.py @@ -7,11 +7,15 @@ from __future__ import annotations import argparse import dataclasses import pathlib +import re import subprocess import tempfile import typing as t import venv +import packaging.version +import packaging.specifiers + try: import argcomplete except ImportError: @@ -59,7 +63,22 @@ class SanityTest: pip_freeze = subprocess.run(pip + ['freeze'] + freeze_options, env=env, check=True, capture_output=True, text=True) - requirements = f'# edit "{self.source_path.name}" and generate with: {SELF} --test {self.name}\n{pip_freeze.stdout}' + self.write_requirements(pip_freeze.stdout) + + def update_pre_build(self) -> None: + """Update requirements in place with current pre-build instructions.""" + requirements = pathlib.Path(self.requirements_path).read_text() + lines = requirements.splitlines(keepends=True) + lines = [line for line in lines if not line.startswith('#')] + requirements = ''.join(lines) + + self.write_requirements(requirements) + + def write_requirements(self, requirements: str) -> None: + """Write the given test requirements to the requirements file for this test.""" + pre_build = pre_build_instructions(requirements) + + requirements = f'# edit "{self.source_path.name}" and generate with: {SELF} --test {self.name}\n{pre_build}{requirements}' with open(self.requirements_path, 'w') as requirement_file: requirement_file.write(requirements) @@ -73,6 +92,38 @@ class SanityTest: ) +def pre_build_instructions(requirements: str) -> str: + """Parse the given requirements and return any applicable pre-build instructions.""" + parsed_requirements = requirements.splitlines() + + package_versions = { + match.group('package').lower(): match.group('version') for match + in (re.search('^(?P.*)==(?P.*)$', requirement) for requirement in parsed_requirements) + if match + } + + instructions: list[str] = [] + + build_constraints = ( + ('pyyaml', '>= 5.4, <= 6.0', ('Cython < 3.0',)), + ) + + for package, specifier, constraints in build_constraints: + version_string = package_versions.get(package) + + if version_string: + version = packaging.version.Version(version_string) + specifier_set = packaging.specifiers.SpecifierSet(specifier) + + if specifier_set.contains(version): + instructions.append(f'# pre-build requirement: {package} == {version}\n') + + for constraint in constraints: + instructions.append(f'# pre-build constraint: {constraint}\n') + + return ''.join(instructions) + + def main() -> None: tests = find_tests() @@ -86,6 +137,12 @@ def main() -> None: help='test requirements to update' ) + parser.add_argument( + '--pre-build-only', + action='store_true', + help='apply pre-build instructions to existing requirements', + ) + if argcomplete: argcomplete.autocomplete(parser) @@ -96,7 +153,11 @@ def main() -> None: for test in tests: print(f'===[ {test.name} ]===', flush=True) - test.freeze_requirements() + + if args.pre_build_only: + test.update_pre_build() + else: + test.freeze_requirements() def find_tests() -> t.List[SanityTest]: diff --git a/test/integration/targets/canonical-pep517-self-packaging/modernish-build-constraints.txt b/test/integration/targets/canonical-pep517-self-packaging/modernish-build-constraints.txt index 9b8e9d0aa6b..5de287375c1 100644 --- a/test/integration/targets/canonical-pep517-self-packaging/modernish-build-constraints.txt +++ b/test/integration/targets/canonical-pep517-self-packaging/modernish-build-constraints.txt @@ -7,4 +7,4 @@ wheel == 0.38.4 docutils == 0.19 Jinja2 == 3.1.2 MarkupSafe == 2.1.2 -PyYAML == 6.0 +PyYAML == 6.0.1 diff --git a/test/lib/ansible_test/_data/requirements/sanity.ansible-doc.txt b/test/lib/ansible_test/_data/requirements/sanity.ansible-doc.txt index d3289d0c48f..35c90f9b06b 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.ansible-doc.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.ansible-doc.txt @@ -1,4 +1,6 @@ # edit "sanity.ansible-doc.in" and generate with: hacking/update-sanity-requirements.py --test ansible-doc +# pre-build requirement: pyyaml == 6.0 +# pre-build constraint: Cython < 3.0 Jinja2==3.1.2 MarkupSafe==2.1.2 packaging==23.0 diff --git a/test/lib/ansible_test/_data/requirements/sanity.changelog.txt b/test/lib/ansible_test/_data/requirements/sanity.changelog.txt index 7cbe3b261f1..4a7161c8bc4 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.changelog.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.changelog.txt @@ -1,4 +1,6 @@ # edit "sanity.changelog.in" and generate with: hacking/update-sanity-requirements.py --test changelog +# pre-build requirement: pyyaml == 6.0 +# pre-build constraint: Cython < 3.0 antsibull-changelog==0.19.0 docutils==0.18.1 packaging==23.0 diff --git a/test/lib/ansible_test/_data/requirements/sanity.import.plugin.txt b/test/lib/ansible_test/_data/requirements/sanity.import.plugin.txt index e7e5ae5a539..d4c2cbe5a84 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.import.plugin.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.import.plugin.txt @@ -1,4 +1,6 @@ # edit "sanity.import.plugin.in" and generate with: hacking/update-sanity-requirements.py --test import.plugin +# pre-build requirement: pyyaml == 6.0 +# pre-build constraint: Cython < 3.0 Jinja2==3.1.2 MarkupSafe==2.1.2 PyYAML==6.0 diff --git a/test/lib/ansible_test/_data/requirements/sanity.import.txt b/test/lib/ansible_test/_data/requirements/sanity.import.txt index e9645ea2dad..4fda120dfe4 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.import.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.import.txt @@ -1,2 +1,4 @@ # edit "sanity.import.in" and generate with: hacking/update-sanity-requirements.py --test import +# pre-build requirement: pyyaml == 6.0 +# pre-build constraint: Cython < 3.0 PyYAML==6.0 diff --git a/test/lib/ansible_test/_data/requirements/sanity.integration-aliases.txt b/test/lib/ansible_test/_data/requirements/sanity.integration-aliases.txt index ba3a50284fd..51cc1ca3b28 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.integration-aliases.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.integration-aliases.txt @@ -1,2 +1,4 @@ # edit "sanity.integration-aliases.in" and generate with: hacking/update-sanity-requirements.py --test integration-aliases +# pre-build requirement: pyyaml == 6.0 +# pre-build constraint: Cython < 3.0 PyYAML==6.0 diff --git a/test/lib/ansible_test/_data/requirements/sanity.pylint.txt b/test/lib/ansible_test/_data/requirements/sanity.pylint.txt index 18b5f189131..1932994e73b 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.pylint.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.pylint.txt @@ -1,4 +1,6 @@ # edit "sanity.pylint.in" and generate with: hacking/update-sanity-requirements.py --test pylint +# pre-build requirement: pyyaml == 6.0 +# pre-build constraint: Cython < 3.0 astroid==2.15.4 dill==0.3.6 isort==5.12.0 diff --git a/test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt b/test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt index 3953b77cbdf..b2b705670da 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt @@ -1,3 +1,5 @@ # edit "sanity.runtime-metadata.in" and generate with: hacking/update-sanity-requirements.py --test runtime-metadata +# pre-build requirement: pyyaml == 6.0 +# pre-build constraint: Cython < 3.0 PyYAML==6.0 voluptuous==0.13.1 diff --git a/test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt b/test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt index 7a7bee594af..cda0dfe6736 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt @@ -1,4 +1,6 @@ # edit "sanity.validate-modules.in" and generate with: hacking/update-sanity-requirements.py --test validate-modules +# pre-build requirement: pyyaml == 6.0 +# pre-build constraint: Cython < 3.0 antsibull-docs-parser==1.0.0 Jinja2==3.1.2 MarkupSafe==2.1.2 diff --git a/test/lib/ansible_test/_data/requirements/sanity.yamllint.txt b/test/lib/ansible_test/_data/requirements/sanity.yamllint.txt index 05ca0ec218a..96d411948e9 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.yamllint.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.yamllint.txt @@ -1,4 +1,6 @@ # edit "sanity.yamllint.in" and generate with: hacking/update-sanity-requirements.py --test yamllint +# pre-build requirement: pyyaml == 6.0 +# pre-build constraint: Cython < 3.0 pathspec==0.11.1 PyYAML==6.0 yamllint==1.30.0 diff --git a/test/lib/ansible_test/_internal/commands/sanity/__init__.py b/test/lib/ansible_test/_internal/commands/sanity/__init__.py index 3b61ca817db..9b675e4a61b 100644 --- a/test/lib/ansible_test/_internal/commands/sanity/__init__.py +++ b/test/lib/ansible_test/_internal/commands/sanity/__init__.py @@ -71,6 +71,7 @@ from ...executor import ( ) from ...python_requirements import ( + PipCommand, PipInstall, collect_requirements, run_pip, @@ -1157,7 +1158,7 @@ def create_sanity_virtualenv( # The path to the virtual environment must be kept short to avoid the 127 character shebang length limit on Linux. # If the limit is exceeded, generated entry point scripts from pip installed packages will fail with syntax errors. virtualenv_install = json.dumps([command.serialize() for command in commands], indent=4) - virtualenv_hash = hashlib.sha256(to_bytes(virtualenv_install)).hexdigest()[:8] + virtualenv_hash = hash_pip_commands(commands) virtualenv_cache = os.path.join(os.path.expanduser('~/.ansible/test/venv')) virtualenv_path = os.path.join(virtualenv_cache, label, f'{python.version}', virtualenv_hash) virtualenv_marker = os.path.join(virtualenv_path, 'marker.txt') @@ -1200,6 +1201,39 @@ def create_sanity_virtualenv( return virtualenv_python +def hash_pip_commands(commands: list[PipCommand]) -> str: + """Return a short hash unique to the given list of pip commands, suitable for identifying the resulting sanity test environment.""" + serialized_commands = json.dumps([make_pip_command_hashable(command) for command in commands], indent=4) + + return hashlib.sha256(to_bytes(serialized_commands)).hexdigest()[:8] + + +def make_pip_command_hashable(command: PipCommand) -> tuple[str, dict[str, t.Any]]: + """Return a serialized version of the given pip command that is suitable for hashing.""" + if isinstance(command, PipInstall): + # The pre-build instructions for pip installs must be omitted, so they do not affect the hash. + # This is allows the pre-build commands to be added without breaking sanity venv caching. + # It is safe to omit these from the hash since they only affect packages used during builds, not what is installed in the venv. + command = PipInstall( + requirements=[omit_pre_build_from_requirement(*req) for req in command.requirements], + constraints=list(command.constraints), + packages=list(command.packages), + ) + + return command.serialize() + + +def omit_pre_build_from_requirement(path: str, requirements: str) -> tuple[str, str]: + """Return the given requirements with pre-build instructions omitted.""" + lines = requirements.splitlines(keepends=True) + + # CAUTION: This code must be kept in sync with the code which processes pre-build instructions in: + # test/lib/ansible_test/_util/target/setup/requirements.py + lines = [line for line in lines if not line.startswith('# pre-build ')] + + return path, ''.join(lines) + + def check_sanity_virtualenv_yaml(python: VirtualPythonConfig) -> t.Optional[bool]: """Return True if PyYAML has libyaml support for the given sanity virtual environment, False if it does not and None if it was not found.""" virtualenv_path = os.path.dirname(os.path.dirname(python.path)) diff --git a/test/lib/ansible_test/_util/target/setup/requirements.py b/test/lib/ansible_test/_util/target/setup/requirements.py index 4fe9a6c5a33..b145fde5013 100644 --- a/test/lib/ansible_test/_util/target/setup/requirements.py +++ b/test/lib/ansible_test/_util/target/setup/requirements.py @@ -134,6 +134,14 @@ def install(pip, options): # type: (str, t.Dict[str, t.Any]) -> None options.extend(packages) for path, content in requirements: + if path.split(os.sep)[0] in ('test', 'requirements'): + # Support for pre-build is currently limited to requirements embedded in ansible-test and those used by ansible-core. + # Requirements from ansible-core can be found in the 'test' and 'requirements' directories. + # This feature will probably be extended to support collections after further testing. + # Requirements from collections can be found in the 'tests' directory. + for pre_build in parse_pre_build_instructions(content): + pre_build.execute(pip) + write_text_file(os.path.join(tempdir, path), content, True) options.extend(['-r', path]) @@ -150,6 +158,61 @@ def install(pip, options): # type: (str, t.Dict[str, t.Any]) -> None remove_tree(tempdir) +class PreBuild: + """Parsed pre-build instructions.""" + + def __init__(self, requirement): # type: (str) -> None + self.requirement = requirement + self.constraints = [] # type: list[str] + + def execute(self, pip): # type: (str) -> None + """Execute these pre-build instructions.""" + tempdir = tempfile.mkdtemp(prefix='ansible-test-', suffix='-pre-build') + + try: + options = common_pip_options() + options.append(self.requirement) + + constraints = '\n'.join(self.constraints) + '\n' + constraints_path = os.path.join(tempdir, 'constraints.txt') + + write_text_file(constraints_path, constraints, True) + + env = common_pip_environment() + env.update(PIP_CONSTRAINT=constraints_path) + + command = [sys.executable, pip, 'wheel'] + options + + execute_command(command, env=env, cwd=tempdir) + finally: + remove_tree(tempdir) + + +def parse_pre_build_instructions(requirements): # type: (str) -> list[PreBuild] + """Parse the given pip requirements and return a list of extracted pre-build instructions.""" + # CAUTION: This code must be kept in sync with the sanity test hashing code in: + # test/lib/ansible_test/_internal/commands/sanity/__init__.py + + pre_build_prefix = '# pre-build ' + pre_build_requirement_prefix = pre_build_prefix + 'requirement: ' + pre_build_constraint_prefix = pre_build_prefix + 'constraint: ' + + lines = requirements.splitlines() + pre_build_lines = [line for line in lines if line.startswith(pre_build_prefix)] + + instructions = [] # type: list[PreBuild] + + for line in pre_build_lines: + if line.startswith(pre_build_requirement_prefix): + instructions.append(PreBuild(line[len(pre_build_requirement_prefix):])) + elif line.startswith(pre_build_constraint_prefix): + instructions[-1].constraints.append(line[len(pre_build_constraint_prefix):]) + else: + raise RuntimeError('Unsupported pre-build comment: ' + line) + + return instructions + + def uninstall(pip, options): # type: (str, t.Dict[str, t.Any]) -> None """Perform a pip uninstall.""" packages = options['packages'] diff --git a/test/sanity/code-smell/deprecated-config.requirements.txt b/test/sanity/code-smell/deprecated-config.requirements.txt index 4439d998444..b434439eba9 100644 --- a/test/sanity/code-smell/deprecated-config.requirements.txt +++ b/test/sanity/code-smell/deprecated-config.requirements.txt @@ -1,4 +1,6 @@ # edit "deprecated-config.requirements.in" and generate with: hacking/update-sanity-requirements.py --test deprecated-config +# pre-build requirement: pyyaml == 6.0 +# pre-build constraint: Cython < 3.0 Jinja2==3.1.2 MarkupSafe==2.1.2 PyYAML==6.0 diff --git a/test/sanity/code-smell/package-data.requirements.txt b/test/sanity/code-smell/package-data.requirements.txt index c4b6119bf77..ba714afa911 100644 --- a/test/sanity/code-smell/package-data.requirements.txt +++ b/test/sanity/code-smell/package-data.requirements.txt @@ -1,4 +1,6 @@ # edit "package-data.requirements.in" and generate with: hacking/update-sanity-requirements.py --test package-data +# pre-build requirement: pyyaml == 6.0 +# pre-build constraint: Cython < 3.0 antsibull-changelog==0.19.0 build==0.10.0 docutils==0.17.1