"""Execute Ansible tests.""" from __future__ import (absolute_import, division, print_function) __metaclass__ = type import atexit import json import os import re from . import types as t from .io import ( make_dirs, read_text_file, write_text_file, ) from .util import ( ApplicationWarning, ApplicationError, SubprocessError, display, find_executable, raw_command, generate_pip_command, find_python, cmd_quote, ANSIBLE_TEST_DATA_ROOT, ANSIBLE_TEST_TOOLS_ROOT, str_to_version, version_to_str, ) from .util_common import ( intercept_command, run_command, ResultType, CommonConfig, ) from .docker_util import ( docker_pull, docker_run, docker_inspect, ) from .ansible_util import ( ansible_environment, check_pyyaml, ) from .ci import ( get_ci_provider, ) from .classification import ( categorize_changes, ) from .config import ( TestConfig, EnvironmentConfig, IntegrationConfig, ShellConfig, UnitsConfig, SanityConfig, ) from .metadata import ( ChangeDescription, ) from .data import ( data_context, ) from .http import ( urlparse, ) def create_shell_command(command): """ :type command: list[str] :rtype: list[str] """ optional_vars = ( 'TERM', ) cmd = ['/usr/bin/env'] cmd += ['%s=%s' % (var, os.environ[var]) for var in optional_vars if var in os.environ] cmd += command return cmd def get_openssl_version(args, python, python_version): # type: (EnvironmentConfig, str, str) -> t.Optional[t.Tuple[int, ...]] """Return the openssl version.""" if not python_version.startswith('2.'): # OpenSSL version checking only works on Python 3.x. # This should be the most accurate, since it is the Python we will be using. version = json.loads(run_command(args, [python, os.path.join(ANSIBLE_TEST_TOOLS_ROOT, 'sslcheck.py')], capture=True, always=True)[0])['version'] if version: display.info('Detected OpenSSL version %s under Python %s.' % (version_to_str(version), python_version), verbosity=1) return tuple(version) # Fall back to detecting the OpenSSL version from the CLI. # This should provide an adequate solution on Python 2.x. openssl_path = find_executable('openssl', required=False) if openssl_path: try: result = raw_command([openssl_path, 'version'], capture=True)[0] except SubprocessError: result = '' match = re.search(r'^OpenSSL (?P[0-9]+\.[0-9]+\.[0-9]+)', result) if match: version = str_to_version(match.group('version')) display.info('Detected OpenSSL version %s using the openssl CLI.' % version_to_str(version), verbosity=1) return version display.info('Unable to detect OpenSSL version.', verbosity=1) return None def is_cryptography_available(python): # type: (str) -> bool """Return True if cryptography is available for the given python.""" try: raw_command([python, '-c', 'import cryptography'], capture=True) except SubprocessError: return False return True def get_setuptools_version(args, python): # type: (EnvironmentConfig, str) -> t.Tuple[int] """Return the setuptools version for the given python.""" try: return str_to_version(raw_command([python, '-c', 'import setuptools; print(setuptools.__version__)'], capture=True)[0]) except SubprocessError: if args.explain: return tuple() # ignore errors in explain mode in case setuptools is not aleady installed raise def install_cryptography(args, python, python_version, pip): # type: (EnvironmentConfig, str, str, t.List[str]) -> None """ Install cryptography for the specified environment. """ # make sure ansible-test's basic requirements are met before continuing # this is primarily to ensure that pip is new enough to facilitate further requirements installation install_ansible_test_requirements(args, pip) # make sure setuptools is available before trying to install cryptography # the installed version of setuptools affects the version of cryptography to install run_command(args, generate_pip_install(pip, '', packages=['setuptools'])) # skip cryptography install if it is already available # this avoids downgrading cryptography when OS packages provide a newer version than we are able to install using pip if is_cryptography_available(python): return # install the latest cryptography version that the current requirements can support # use a custom constraints file to avoid the normal constraints file overriding the chosen version of cryptography # if not installed here later install commands may try to install an unsupported version due to the presence of older setuptools # this is done instead of upgrading setuptools to allow tests to function with older distribution provided versions of setuptools run_command(args, generate_pip_install(pip, '', packages=[get_cryptography_requirement(args, python, python_version)], constraints=os.path.join(ANSIBLE_TEST_DATA_ROOT, 'cryptography-constraints.txt'))) def get_cryptography_requirement(args, python, python_version): # type: (EnvironmentConfig, str, str) -> str """ Return the correct cryptography requirement for the given python version. The version of cryptography installed depends on the python version, setuptools version and openssl version. """ setuptools_version = get_setuptools_version(args, python) openssl_version = get_openssl_version(args, python, python_version) if setuptools_version >= (18, 5): if python_version == '2.6': # cryptography 2.2+ requires python 2.7+ # see https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst#22---2018-03-19 cryptography = 'cryptography < 2.2' elif openssl_version and openssl_version < (1, 1, 0): # cryptography 3.2 requires openssl 1.1.x or later # see https://cryptography.io/en/latest/changelog.html#v3-2 cryptography = 'cryptography < 3.2' else: # cryptography 3.4+ fails to install on many systems # this is a temporary work-around until a more permanent solution is available cryptography = 'cryptography < 3.4' else: # cryptography 2.1+ requires setuptools 18.5+ # see https://github.com/pyca/cryptography/blob/62287ae18383447585606b9d0765c0f1b8a9777c/setup.py#L26 cryptography = 'cryptography < 2.1' return cryptography def install_command_requirements(args, python_version=None, context=None, enable_pyyaml_check=False, extra_requirements=None): """ :type args: EnvironmentConfig :type python_version: str | None :type context: str | None :type enable_pyyaml_check: bool :type extra_requirements: list[str] | None """ if not args.explain: make_dirs(ResultType.COVERAGE.path) make_dirs(ResultType.DATA.path) if isinstance(args, ShellConfig): if args.raw: return if not args.requirements: return if isinstance(args, ShellConfig): return packages = [] if isinstance(args, TestConfig): if args.coverage: packages.append('coverage') if args.junit: packages.append('junit-xml') if not python_version: python_version = args.python_version python = find_python(python_version) pip = generate_pip_command(python) # skip packages which have aleady been installed for python_version try: package_cache = install_command_requirements.package_cache except AttributeError: package_cache = install_command_requirements.package_cache = {} installed_packages = package_cache.setdefault(python_version, set()) skip_packages = [package for package in packages if package in installed_packages] for package in skip_packages: packages.remove(package) installed_packages.update(packages) if args.command != 'sanity': install_cryptography(args, python, python_version, pip) commands = [generate_pip_install(pip, args.command, packages=packages, context=context)] if extra_requirements: for extra_requirement in extra_requirements: commands.append(generate_pip_install(pip, extra_requirement)) commands = [cmd for cmd in commands if cmd] if not commands: return # no need to detect changes or run pip check since we are not making any changes # only look for changes when more than one requirements file is needed detect_pip_changes = len(commands) > 1 # first pass to install requirements, changes expected unless environment is already set up install_ansible_test_requirements(args, pip) changes = run_pip_commands(args, pip, commands, detect_pip_changes) if changes: # second pass to check for conflicts in requirements, changes are not expected here changes = run_pip_commands(args, pip, commands, detect_pip_changes) if changes: raise ApplicationError('Conflicts detected in requirements. The following commands reported changes during verification:\n%s' % '\n'.join((' '.join(cmd_quote(c) for c in cmd) for cmd in changes))) if args.pip_check: # ask pip to check for conflicts between installed packages try: run_command(args, pip + ['check', '--disable-pip-version-check'], capture=True) except SubprocessError as ex: if ex.stderr.strip() == 'ERROR: unknown command "check"': display.warning('Cannot check pip requirements for conflicts because "pip check" is not supported.') else: raise if enable_pyyaml_check: # pyyaml may have been one of the requirements that was installed, so perform an optional check for it check_pyyaml(args, python_version, required=False) def install_ansible_test_requirements(args, pip): # type: (EnvironmentConfig, t.List[str]) -> None """Install requirements for ansible-test for the given pip if not already installed.""" try: installed = install_command_requirements.installed except AttributeError: installed = install_command_requirements.installed = set() if tuple(pip) in installed: return # make sure basic ansible-test requirements are met, including making sure that pip is recent enough to support constraints # virtualenvs created by older distributions may include very old pip versions, such as those created in the centos6 test container (pip 6.0.8) run_command(args, generate_pip_install(pip, 'ansible-test', use_constraints=False)) installed.add(tuple(pip)) def run_pip_commands(args, pip, commands, detect_pip_changes=False): """ :type args: EnvironmentConfig :type pip: list[str] :type commands: list[list[str]] :type detect_pip_changes: bool :rtype: list[list[str]] """ changes = [] after_list = pip_list(args, pip) if detect_pip_changes else None for cmd in commands: if not cmd: continue before_list = after_list run_command(args, cmd) after_list = pip_list(args, pip) if detect_pip_changes else None if before_list != after_list: changes.append(cmd) return changes def pip_list(args, pip): """ :type args: EnvironmentConfig :type pip: list[str] :rtype: str """ stdout = run_command(args, pip + ['list'], capture=True)[0] return stdout def generate_pip_install(pip, command, packages=None, constraints=None, use_constraints=True, context=None): """ :type pip: list[str] :type command: str :type packages: list[str] | None :type constraints: str | None :type use_constraints: bool :type context: str | None :rtype: list[str] | None """ constraints = constraints or os.path.join(ANSIBLE_TEST_DATA_ROOT, 'requirements', 'constraints.txt') requirements = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'requirements', '%s.txt' % ('%s.%s' % (command, context) if context else command)) content_constraints = None options = [] if os.path.exists(requirements) and os.path.getsize(requirements): options += ['-r', requirements] if command == 'sanity' and data_context().content.is_ansible: requirements = os.path.join(data_context().content.sanity_path, 'code-smell', '%s.requirements.txt' % context) if os.path.exists(requirements) and os.path.getsize(requirements): options += ['-r', requirements] if command == 'units': requirements = os.path.join(data_context().content.unit_path, 'requirements.txt') if os.path.exists(requirements) and os.path.getsize(requirements): options += ['-r', requirements] content_constraints = os.path.join(data_context().content.unit_path, 'constraints.txt') if command in ('integration', 'windows-integration', 'network-integration'): requirements = os.path.join(data_context().content.integration_path, 'requirements.txt') if os.path.exists(requirements) and os.path.getsize(requirements): options += ['-r', requirements] requirements = os.path.join(data_context().content.integration_path, '%s.requirements.txt' % command) if os.path.exists(requirements) and os.path.getsize(requirements): options += ['-r', requirements] content_constraints = os.path.join(data_context().content.integration_path, 'constraints.txt') if command.startswith('integration.cloud.'): content_constraints = os.path.join(data_context().content.integration_path, 'constraints.txt') if packages: options += packages if not options: return None if use_constraints: if content_constraints and os.path.exists(content_constraints) and os.path.getsize(content_constraints): # listing content constraints first gives them priority over constraints provided by ansible-test options.extend(['-c', content_constraints]) options.extend(['-c', constraints]) return pip + ['install', '--disable-pip-version-check'] + options def parse_inventory(args, inventory_path): # type: (IntegrationConfig, str) -> t.Dict[str, t.Any] """Return a dict parsed from the given inventory file.""" cmd = ['ansible-inventory', '-i', inventory_path, '--list'] env = ansible_environment(args) inventory = json.loads(intercept_command(args, cmd, '', env, capture=True, disable_coverage=True)[0]) return inventory def get_hosts(inventory, group_name): # type: (t.Dict[str, t.Any], str) -> t.Dict[str, t.Dict[str, t.Any]] """Return a dict of hosts from the specified group in the given inventory.""" hostvars = inventory.get('_meta', {}).get('hostvars', {}) group = inventory.get(group_name, {}) host_names = group.get('hosts', []) hosts = dict((name, hostvars[name]) for name in host_names) return hosts def run_pypi_proxy(args): # type: (EnvironmentConfig) -> t.Tuple[t.Optional[str], t.Optional[str]] """Run a PyPI proxy container, returning the container ID and proxy endpoint.""" use_proxy = False if args.docker_raw == 'centos6': use_proxy = True # python 2.6 is the only version available if args.docker_raw == 'default': if args.python == '2.6': use_proxy = True # python 2.6 requested elif not args.python and isinstance(args, (SanityConfig, UnitsConfig, ShellConfig)): use_proxy = True # multiple versions (including python 2.6) can be used if args.docker_raw and args.pypi_proxy: use_proxy = True # manual override to force proxy usage if not use_proxy: return None, None proxy_image = 'quay.io/ansible/pypi-test-container:1.0.0' port = 3141 options = [ '--detach', ] docker_pull(args, proxy_image) container_id = docker_run(args, proxy_image, options=options) container = docker_inspect(args, container_id) container_ip = container.get_ip_address() if not container_ip: raise Exception('PyPI container IP not available.') endpoint = 'http://%s:%d/root/pypi/+simple/' % (container_ip, port) return container_id, endpoint def configure_pypi_proxy(args): # type: (CommonConfig) -> None """Configure the environment to use a PyPI proxy, if present.""" if not isinstance(args, EnvironmentConfig): return if args.pypi_endpoint: configure_pypi_block_access() configure_pypi_proxy_pip(args) configure_pypi_proxy_easy_install(args) def configure_pypi_block_access(): # type: () -> None """Block direct access to PyPI to ensure proxy configurations are always used.""" if os.getuid() != 0: display.warning('Skipping custom hosts block for PyPI for non-root user.') return hosts_path = '/etc/hosts' hosts_block = ''' 127.0.0.1 pypi.org pypi.python.org files.pythonhosted.org ''' def hosts_cleanup(): display.info('Removing custom PyPI hosts entries: %s' % hosts_path, verbosity=1) with open(hosts_path) as hosts_file_read: content = hosts_file_read.read() content = content.replace(hosts_block, '') with open(hosts_path, 'w') as hosts_file_write: hosts_file_write.write(content) display.info('Injecting custom PyPI hosts entries: %s' % hosts_path, verbosity=1) display.info('Config: %s\n%s' % (hosts_path, hosts_block), verbosity=3) with open(hosts_path, 'a') as hosts_file: hosts_file.write(hosts_block) atexit.register(hosts_cleanup) def configure_pypi_proxy_pip(args): # type: (EnvironmentConfig) -> None """Configure a custom index for pip based installs.""" pypi_hostname = urlparse(args.pypi_endpoint)[1].split(':')[0] pip_conf_path = os.path.expanduser('~/.pip/pip.conf') pip_conf = ''' [global] index-url = {0} trusted-host = {1} '''.format(args.pypi_endpoint, pypi_hostname).strip() def pip_conf_cleanup(): display.info('Removing custom PyPI config: %s' % pip_conf_path, verbosity=1) os.remove(pip_conf_path) if os.path.exists(pip_conf_path): raise ApplicationError('Refusing to overwrite existing file: %s' % pip_conf_path) display.info('Injecting custom PyPI config: %s' % pip_conf_path, verbosity=1) display.info('Config: %s\n%s' % (pip_conf_path, pip_conf), verbosity=3) write_text_file(pip_conf_path, pip_conf, True) atexit.register(pip_conf_cleanup) def configure_pypi_proxy_easy_install(args): # type: (EnvironmentConfig) -> None """Configure a custom index for easy_install based installs.""" pydistutils_cfg_path = os.path.expanduser('~/.pydistutils.cfg') pydistutils_cfg = ''' [easy_install] index_url = {0} '''.format(args.pypi_endpoint).strip() if os.path.exists(pydistutils_cfg_path): raise ApplicationError('Refusing to overwrite existing file: %s' % pydistutils_cfg_path) def pydistutils_cfg_cleanup(): display.info('Removing custom PyPI config: %s' % pydistutils_cfg_path, verbosity=1) os.remove(pydistutils_cfg_path) display.info('Injecting custom PyPI config: %s' % pydistutils_cfg_path, verbosity=1) display.info('Config: %s\n%s' % (pydistutils_cfg_path, pydistutils_cfg), verbosity=3) write_text_file(pydistutils_cfg_path, pydistutils_cfg, True) atexit.register(pydistutils_cfg_cleanup) def get_changes_filter(args): """ :type args: TestConfig :rtype: list[str] """ paths = detect_changes(args) if not args.metadata.change_description: if paths: changes = categorize_changes(args, paths, args.command) else: changes = ChangeDescription() args.metadata.change_description = changes if paths is None: return [] # change detection not enabled, do not filter targets if not paths: raise NoChangesDetected() if args.metadata.change_description.targets is None: raise NoTestsForChanges() return args.metadata.change_description.targets def detect_changes(args): """ :type args: TestConfig :rtype: list[str] | None """ if args.changed: paths = get_ci_provider().detect_changes(args) elif args.changed_from or args.changed_path: paths = args.changed_path or [] if args.changed_from: paths += read_text_file(args.changed_from).splitlines() else: return None # change detection not enabled if paths is None: return None # act as though change detection not enabled, do not filter targets display.info('Detected changes in %d file(s).' % len(paths)) for path in paths: display.info(path, verbosity=1) return paths def get_python_version(args, configs, name): """ :type args: EnvironmentConfig :type configs: dict[str, dict[str, str]] :type name: str """ config = configs.get(name, {}) config_python = config.get('python') if not config or not config_python: if args.python: return args.python display.warning('No Python version specified. ' 'Use completion config or the --python option to specify one.', unique=True) return '' # failure to provide a version may result in failures or reduced functionality later supported_python_versions = config_python.split(',') default_python_version = supported_python_versions[0] if args.python and args.python not in supported_python_versions: raise ApplicationError('Python %s is not supported by %s. Supported Python version(s) are: %s' % ( args.python, name, ', '.join(sorted(supported_python_versions)))) python_version = args.python or default_python_version return python_version def get_python_interpreter(args, configs, name): """ :type args: EnvironmentConfig :type configs: dict[str, dict[str, str]] :type name: str """ if args.python_interpreter: return args.python_interpreter config = configs.get(name, {}) if not config: if args.python: guess = 'python%s' % args.python else: guess = 'python' display.warning('Using "%s" as the Python interpreter. ' 'Use completion config or the --python-interpreter option to specify the path.' % guess, unique=True) return guess python_version = get_python_version(args, configs, name) python_dir = config.get('python_dir', '/usr/bin') python_interpreter = os.path.join(python_dir, 'python%s' % python_version) python_interpreter = config.get('python%s' % python_version, python_interpreter) return python_interpreter class NoChangesDetected(ApplicationWarning): """Exception when change detection was performed, but no changes were found.""" def __init__(self): super(NoChangesDetected, self).__init__('No changes detected.') class NoTestsForChanges(ApplicationWarning): """Exception when changes detected, but no tests trigger as a result.""" def __init__(self): super(NoTestsForChanges, self).__init__('No tests found for detected changes.') class Delegate(Exception): """Trigger command delegation.""" def __init__(self, exclude=None, require=None): """ :type exclude: list[str] | None :type require: list[str] | None """ super(Delegate, self).__init__() self.exclude = exclude or [] self.require = require or [] class AllTargetsSkipped(ApplicationWarning): """All targets skipped.""" def __init__(self): super(AllTargetsSkipped, self).__init__('All targets skipped.')