You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
ansible/test/lib/ansible_test/_internal/python_requirements.py

590 lines
20 KiB
Python

"""Python requirements management"""
from __future__ import annotations
import base64
import dataclasses
import json
import os
import re
import typing as t
from .encoding import (
to_text,
to_bytes,
)
from .io import (
read_text_file,
)
from .util import (
ANSIBLE_TEST_DATA_ROOT,
ANSIBLE_TEST_TARGET_ROOT,
ANSIBLE_TEST_TOOLS_ROOT,
ApplicationError,
SubprocessError,
display,
find_executable,
raw_command,
str_to_version,
version_to_str,
)
from .util_common import (
check_pyyaml,
create_result_directories,
)
from .config import (
EnvironmentConfig,
IntegrationConfig,
UnitsConfig,
)
from .data import (
data_context,
)
from .host_configs import (
PosixConfig,
PythonConfig,
VirtualPythonConfig,
)
from .connections import (
LocalConnection,
Connection,
)
from .coverage_util import (
get_coverage_version,
)
QUIET_PIP_SCRIPT_PATH = os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'setup', 'quiet_pip.py')
REQUIREMENTS_SCRIPT_PATH = os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'setup', 'requirements.py')
# IMPORTANT: Keep this in sync with the ansible-test.txt requirements file.
VIRTUALENV_VERSION = '16.7.12'
# Pip Abstraction
class PipUnavailableError(ApplicationError):
"""Exception raised when pip is not available."""
def __init__(self, python: PythonConfig) -> None:
super().__init__(f'Python {python.version} at "{python.path}" does not have pip available.')
@dataclasses.dataclass(frozen=True)
class PipCommand:
"""Base class for pip commands."""
def serialize(self) -> tuple[str, dict[str, t.Any]]:
"""Return a serialized representation of this command."""
name = type(self).__name__[3:].lower()
return name, self.__dict__
@dataclasses.dataclass(frozen=True)
class PipInstall(PipCommand):
"""Details required to perform a pip install."""
requirements: list[tuple[str, str]]
constraints: list[tuple[str, str]]
packages: list[str]
def has_package(self, name: str) -> bool:
"""Return True if the specified package will be installed, otherwise False."""
name = name.lower()
return (any(name in package.lower() for package in self.packages) or
any(name in contents.lower() for path, contents in self.requirements))
@dataclasses.dataclass(frozen=True)
class PipUninstall(PipCommand):
"""Details required to perform a pip uninstall."""
packages: list[str]
ignore_errors: bool
@dataclasses.dataclass(frozen=True)
class PipVersion(PipCommand):
"""Details required to get the pip version."""
@dataclasses.dataclass(frozen=True)
class PipBootstrap(PipCommand):
"""Details required to bootstrap pip."""
pip_version: str
packages: list[str]
# Entry Points
def install_requirements(
args: EnvironmentConfig,
python: PythonConfig,
ansible: bool = False,
command: bool = False,
coverage: bool = False,
virtualenv: bool = False,
controller: bool = True,
connection: t.Optional[Connection] = None,
) -> None:
"""Install requirements for the given Python using the specified arguments."""
create_result_directories(args)
if not requirements_allowed(args, controller):
return
if command and isinstance(args, (UnitsConfig, IntegrationConfig)) and args.coverage:
coverage = True
cryptography = False
if ansible:
try:
ansible_cache = install_requirements.ansible_cache # type: ignore[attr-defined]
except AttributeError:
ansible_cache = install_requirements.ansible_cache = {} # type: ignore[attr-defined]
ansible_installed = ansible_cache.get(python.path)
if ansible_installed:
ansible = False
else:
ansible_cache[python.path] = True
# Install the latest cryptography version that the current requirements can support if it is not already available.
# This avoids downgrading cryptography when OS packages provide a newer version than we are able to install using pip.
# If not installed here, later install commands may try to install a version of cryptography which cannot be installed.
cryptography = not is_cryptography_available(python.path)
commands = collect_requirements(
python=python,
controller=controller,
ansible=ansible,
cryptography=cryptography,
command=args.command if command else None,
coverage=coverage,
virtualenv=virtualenv,
minimize=False,
sanity=None,
)
if not commands:
return
run_pip(args, python, commands, connection)
# false positive: pylint: disable=no-member
if any(isinstance(command, PipInstall) and command.has_package('pyyaml') for command in commands):
check_pyyaml(python)
def collect_bootstrap(python: PythonConfig) -> list[PipCommand]:
"""Return the details necessary to bootstrap pip into an empty virtual environment."""
infrastructure_packages = get_venv_packages(python)
pip_version = infrastructure_packages['pip']
packages = [f'{name}=={version}' for name, version in infrastructure_packages.items()]
bootstrap = PipBootstrap(
pip_version=pip_version,
packages=packages,
)
return [bootstrap]
def collect_requirements(
python: PythonConfig,
controller: bool,
ansible: bool,
cryptography: bool,
coverage: bool,
virtualenv: bool,
minimize: bool,
command: t.Optional[str],
sanity: t.Optional[str],
) -> list[PipCommand]:
"""Collect requirements for the given Python using the specified arguments."""
commands: list[PipCommand] = []
if virtualenv:
# sanity tests on Python 2.x install virtualenv when it is too old or is not already installed and the `--requirements` option is given
# the last version of virtualenv with no dependencies is used to minimize the changes made outside a virtual environment
commands.extend(collect_package_install(packages=[f'virtualenv=={VIRTUALENV_VERSION}'], constraints=False))
if coverage:
commands.extend(collect_package_install(packages=[f'coverage=={get_coverage_version(python.version).coverage_version}'], constraints=False))
if cryptography:
commands.extend(collect_package_install(packages=get_cryptography_requirements(python)))
if ansible or command:
commands.extend(collect_general_install(command, ansible))
if sanity:
commands.extend(collect_sanity_install(sanity))
if command == 'units':
commands.extend(collect_units_install())
if command in ('integration', 'windows-integration', 'network-integration'):
commands.extend(collect_integration_install(command, controller))
if (sanity or minimize) and any(isinstance(command, PipInstall) for command in commands):
# bootstrap the managed virtual environment, which will have been created without any installed packages
# sanity tests which install no packages skip this step
commands = collect_bootstrap(python) + commands
# most infrastructure packages can be removed from sanity test virtual environments after they've been created
# removing them reduces the size of environments cached in containers
uninstall_packages = list(get_venv_packages(python))
if not minimize:
# installed packages may have run-time dependencies on setuptools
uninstall_packages.remove('setuptools')
commands.extend(collect_uninstall(packages=uninstall_packages))
return commands
def run_pip(
args: EnvironmentConfig,
python: PythonConfig,
commands: list[PipCommand],
connection: t.Optional[Connection],
) -> None:
"""Run the specified pip commands for the given Python, and optionally the specified host."""
connection = connection or LocalConnection(args)
script = prepare_pip_script(commands)
if isinstance(args, IntegrationConfig):
# Integration tests can involve two hosts (controller and target).
# The connection type can be used to disambiguate between the two.
context = " (controller)" if isinstance(connection, LocalConnection) else " (target)"
else:
context = ""
if isinstance(python, VirtualPythonConfig):
context += " [venv]"
# The interpreter path is not included below.
# It can be seen by running ansible-test with increased verbosity (showing all commands executed).
display.info(f'Installing requirements for Python {python.version}{context}')
if not args.explain:
try:
connection.run([python.path], data=script, capture=False)
except SubprocessError:
script = prepare_pip_script([PipVersion()])
try:
connection.run([python.path], data=script, capture=True)
except SubprocessError as ex:
if 'pip is unavailable:' in ex.stdout + ex.stderr:
raise PipUnavailableError(python)
raise
# Collect
def collect_general_install(
command: t.Optional[str] = None,
ansible: bool = False,
) -> list[PipInstall]:
"""Return details necessary for the specified general-purpose pip install(s)."""
requirements_paths: list[tuple[str, str]] = []
constraints_paths: list[tuple[str, str]] = []
if ansible:
path = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'requirements', 'ansible.txt')
requirements_paths.append((ANSIBLE_TEST_DATA_ROOT, path))
if command:
path = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'requirements', f'{command}.txt')
requirements_paths.append((ANSIBLE_TEST_DATA_ROOT, path))
return collect_install(requirements_paths, constraints_paths)
def collect_package_install(packages: list[str], constraints: bool = True) -> list[PipInstall]:
"""Return the details necessary to install the specified packages."""
return collect_install([], [], packages, constraints=constraints)
def collect_sanity_install(sanity: str) -> list[PipInstall]:
"""Return the details necessary for the specified sanity pip install(s)."""
requirements_paths: list[tuple[str, str]] = []
constraints_paths: list[tuple[str, str]] = []
path = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'requirements', f'sanity.{sanity}.txt')
requirements_paths.append((ANSIBLE_TEST_DATA_ROOT, path))
if data_context().content.is_ansible:
path = os.path.join(data_context().content.sanity_path, 'code-smell', f'{sanity}.requirements.txt')
requirements_paths.append((data_context().content.root, path))
return collect_install(requirements_paths, constraints_paths, constraints=False)
def collect_units_install() -> list[PipInstall]:
"""Return details necessary for the specified units pip install(s)."""
requirements_paths: list[tuple[str, str]] = []
constraints_paths: list[tuple[str, str]] = []
path = os.path.join(data_context().content.unit_path, 'requirements.txt')
requirements_paths.append((data_context().content.root, path))
path = os.path.join(data_context().content.unit_path, 'constraints.txt')
constraints_paths.append((data_context().content.root, path))
return collect_install(requirements_paths, constraints_paths)
def collect_integration_install(command: str, controller: bool) -> list[PipInstall]:
"""Return details necessary for the specified integration pip install(s)."""
requirements_paths: list[tuple[str, str]] = []
constraints_paths: list[tuple[str, str]] = []
# Support for prefixed files was added to ansible-test in ansible-core 2.12 when split controller/target testing was implemented.
# Previous versions of ansible-test only recognize non-prefixed files.
# If a prefixed file exists (even if empty), it takes precedence over the non-prefixed file.
prefixes = ('controller.' if controller else 'target.', '')
for prefix in prefixes:
path = os.path.join(data_context().content.integration_path, f'{prefix}requirements.txt')
if os.path.exists(path):
requirements_paths.append((data_context().content.root, path))
break
for prefix in prefixes:
path = os.path.join(data_context().content.integration_path, f'{command}.{prefix}requirements.txt')
if os.path.exists(path):
requirements_paths.append((data_context().content.root, path))
break
for prefix in prefixes:
path = os.path.join(data_context().content.integration_path, f'{prefix}constraints.txt')
if os.path.exists(path):
constraints_paths.append((data_context().content.root, path))
break
return collect_install(requirements_paths, constraints_paths)
def collect_install(
requirements_paths: list[tuple[str, str]],
constraints_paths: list[tuple[str, str]],
packages: t.Optional[list[str]] = None,
constraints: bool = True,
) -> list[PipInstall]:
"""Build a pip install list from the given requirements, constraints and packages."""
# listing content constraints first gives them priority over constraints provided by ansible-test
constraints_paths = list(constraints_paths)
if constraints:
constraints_paths.append((ANSIBLE_TEST_DATA_ROOT, os.path.join(ANSIBLE_TEST_DATA_ROOT, 'requirements', 'constraints.txt')))
requirements = [(os.path.relpath(path, root), read_text_file(path)) for root, path in requirements_paths if usable_pip_file(path)]
constraints = [(os.path.relpath(path, root), read_text_file(path)) for root, path in constraints_paths if usable_pip_file(path)]
packages = packages or []
if requirements or packages:
installs = [PipInstall(
requirements=requirements,
constraints=constraints,
packages=packages,
)]
else:
installs = []
return installs
def collect_uninstall(packages: list[str], ignore_errors: bool = False) -> list[PipUninstall]:
"""Return the details necessary for the specified pip uninstall."""
uninstall = PipUninstall(
packages=packages,
ignore_errors=ignore_errors,
)
return [uninstall]
# Support
def get_venv_packages(python: PythonConfig) -> dict[str, str]:
"""Return a dictionary of Python packages needed for a consistent virtual environment specific to the given Python version."""
# NOTE: This same information is needed for building the base-test-container image.
# See: https://github.com/ansible/base-test-container/blob/main/files/installer.py
default_packages = dict(
pip='21.3.1',
setuptools='60.8.2',
wheel='0.37.1',
)
override_packages = {
'2.7': dict(
pip='20.3.4', # 21.0 requires Python 3.6+
setuptools='44.1.1', # 45.0.0 requires Python 3.5+
wheel=None,
),
'3.5': dict(
pip='20.3.4', # 21.0 requires Python 3.6+
setuptools='50.3.2', # 51.0.0 requires Python 3.6+
wheel=None,
),
'3.6': dict(
pip='21.3.1', # 22.0 requires Python 3.7+
setuptools='59.6.0', # 59.7.0 requires Python 3.7+
wheel=None,
),
}
packages = {name: version or default_packages[name] for name, version in override_packages.get(python.version, default_packages).items()}
return packages
def requirements_allowed(args: EnvironmentConfig, controller: bool) -> bool:
"""
Return True if requirements can be installed, otherwise return False.
Requirements are only allowed if one of the following conditions is met:
The user specified --requirements manually.
The install will occur on the controller and the controller or controller Python is managed by ansible-test.
The install will occur on the target and the target or target Python is managed by ansible-test.
"""
if args.requirements:
return True
if controller:
return args.controller.is_managed or args.controller.python.is_managed
target = args.only_targets(PosixConfig)[0]
return target.is_managed or target.python.is_managed
def prepare_pip_script(commands: list[PipCommand]) -> str:
"""Generate a Python script to perform the requested pip commands."""
data = [command.serialize() for command in commands]
display.info(f'>>> Requirements Commands\n{json.dumps(data, indent=4)}', verbosity=3)
args = dict(
script=read_text_file(QUIET_PIP_SCRIPT_PATH),
verbosity=display.verbosity,
commands=data,
)
payload = to_text(base64.b64encode(to_bytes(json.dumps(args))))
path = REQUIREMENTS_SCRIPT_PATH
template = read_text_file(path)
script = template.format(payload=payload)
display.info(f'>>> Python Script from Template ({path})\n{script.strip()}', verbosity=4)
return script
def usable_pip_file(path: t.Optional[str]) -> bool:
"""Return True if the specified pip file is usable, otherwise False."""
return bool(path) and os.path.exists(path) and bool(os.path.getsize(path))
# Cryptography
def is_cryptography_available(python: str) -> bool:
"""Return True if cryptography is available for the given python."""
try:
raw_command([python, '-c', 'import cryptography'], capture=True)
except SubprocessError:
return False
return True
def get_cryptography_requirements(python: PythonConfig) -> list[str]:
"""
Return the correct cryptography and pyopenssl requirements for the given python version.
The version of cryptography installed depends on the python version and openssl version.
"""
openssl_version = get_openssl_version(python)
if openssl_version and openssl_version < (1, 1, 0):
# cryptography 3.2 requires openssl 1.1.x or later
# see https://cryptography.io/en/latest/changelog.html#v3-2
cryptography = 'cryptography < 3.2'
# pyopenssl 20.0.0 requires cryptography 3.2 or later
pyopenssl = 'pyopenssl < 20.0.0'
else:
# cryptography 3.4+ builds require a working rust toolchain
# systems bootstrapped using ansible-core-ci can access additional wheels through the spare-tire package index
cryptography = 'cryptography'
# any future installation of pyopenssl is free to use any compatible version of cryptography
pyopenssl = ''
requirements = [
cryptography,
pyopenssl,
]
requirements = [requirement for requirement in requirements if requirement]
return requirements
def get_openssl_version(python: PythonConfig) -> t.Optional[tuple[int, ...]]:
"""Return the openssl version."""
if not python.version.startswith('2.'):
# OpenSSL version checking only works on Python 3.x.
# This should be the most accurate, since it is the Python we will be using.
version = json.loads(raw_command([python.path, os.path.join(ANSIBLE_TEST_TOOLS_ROOT, 'sslcheck.py')], capture=True)[0])['version']
if version:
display.info(f'Detected OpenSSL version {version_to_str(version)} under Python {python.version}.', verbosity=1)
return tuple(version)
# Fall back to detecting the OpenSSL version from the CLI.
# This should provide an adequate solution on Python 2.x.
openssl_path = find_executable('openssl', required=False)
if openssl_path:
try:
result = raw_command([openssl_path, 'version'], capture=True)[0]
except SubprocessError:
result = ''
match = re.search(r'^OpenSSL (?P<version>[0-9]+\.[0-9]+\.[0-9]+)', result)
if match:
version = str_to_version(match.group('version'))
display.info(f'Detected OpenSSL version {version_to_str(version)} using the openssl CLI.', verbosity=1)
return version
display.info('Unable to detect OpenSSL version.', verbosity=1)
return None