ansible/test/lib/ansible_test/_internal/ansible_util.py

"""Miscellaneous utility functions and classes specific to ansible cli tools."""
from __future__ import annotations

import json
import os
import shutil
import typing as t

from .constants import (
    ANSIBLE_BIN_SYMLINK_MAP,
    SOFT_RLIMIT_NOFILE,
)

from .io import (
    write_text_file,
)

from .util import (
    common_environment,
    ApplicationError,
    ANSIBLE_LIB_ROOT,
    ANSIBLE_TEST_ROOT,
    ANSIBLE_TEST_DATA_ROOT,
    ANSIBLE_ROOT,
    ANSIBLE_SOURCE_ROOT,
    ANSIBLE_TEST_TOOLS_ROOT,
    MODE_FILE_EXECUTE,
    get_ansible_version,
    raw_command,
    verified_chmod,
)

from .util_common import (
    create_temp_dir,
    ResultType,
    intercept_python,
    get_injector_path,
)

from .config import (
    IntegrationConfig,
    PosixIntegrationConfig,
    EnvironmentConfig,
    CommonConfig,
)

from .data import (
    data_context,
)

from .python_requirements import (
    install_requirements,
)

from .host_configs import (
    PythonConfig,
)

from .thread import (
    mutex,
)


def parse_inventory(args: EnvironmentConfig, inventory_path: str) -> dict[str, t.Any]:
    """Return a dict parsed from the given inventory file."""
    cmd = ['ansible-inventory', '-i', inventory_path, '--list']
    env = ansible_environment(args)
    inventory = json.loads(intercept_python(args, args.controller_python, cmd, env, capture=True, always=True)[0])
    return inventory


def get_hosts(inventory: dict[str, t.Any], group_name: str) -> dict[str, dict[str, t.Any]]:
    """Return a dict of hosts from the specified group in the given inventory."""
    hostvars = inventory.get('_meta', {}).get('hostvars', {})
    group = inventory.get(group_name, {})
    host_names = group.get('hosts', [])
    hosts = dict((name, hostvars.get(name, {})) for name in host_names)
    return hosts


def ansible_environment(args: CommonConfig, color: bool = True, ansible_config: t.Optional[str] = None) -> dict[str, str]:
    """Return a dictionary of environment variables to use when running Ansible commands."""
    env = common_environment()
    path = env['PATH']

    ansible_bin_path = get_ansible_bin_path(args)

    if not path.startswith(ansible_bin_path + os.path.pathsep):
        path = ansible_bin_path + os.path.pathsep + path

    if not ansible_config:
        # use the default empty configuration unless one has been provided
        ansible_config = args.get_ansible_config()

    if not args.explain and not os.path.exists(ansible_config):
        raise ApplicationError('Configuration not found: %s' % ansible_config)

    ansible = dict(
        ANSIBLE_PYTHON_MODULE_RLIMIT_NOFILE=str(SOFT_RLIMIT_NOFILE),
        ANSIBLE_FORCE_COLOR='%s' % 'true' if args.color and color else 'false',
        ANSIBLE_FORCE_HANDLERS='true',  # allow cleanup handlers to run when tests fail
        ANSIBLE_HOST_PATTERN_MISMATCH='error',  # prevent tests from unintentionally passing when hosts are not found
        ANSIBLE_INVENTORY='/dev/null',  # force tests to provide inventory
        ANSIBLE_DEPRECATION_WARNINGS='false',
        ANSIBLE_HOST_KEY_CHECKING='false',
        ANSIBLE_RETRY_FILES_ENABLED='false',
        ANSIBLE_CONFIG=ansible_config,
        ANSIBLE_LIBRARY='/dev/null',
        ANSIBLE_DEVEL_WARNING='false',  # Don't show warnings that CI is running devel
        PYTHONPATH=get_ansible_python_path(args),
        PAGER='/bin/cat',
        PATH=path,
        # give TQM worker processes time to report code coverage results
        # without this the last task in a play may write no coverage file, an empty file, or an incomplete file
        # enabled even when not using code coverage to surface warnings when worker processes do not exit cleanly
        ANSIBLE_WORKER_SHUTDOWN_POLL_COUNT='100',
        ANSIBLE_WORKER_SHUTDOWN_POLL_DELAY='0.1',
        # ansible-test specific environment variables require an 'ANSIBLE_TEST_' prefix to distinguish them from ansible-core env vars defined by config
        ANSIBLE_TEST_ANSIBLE_LIB_ROOT=ANSIBLE_LIB_ROOT,  # used by the coverage injector
    )

    if isinstance(args, IntegrationConfig) and args.coverage:
        # standard path injection is not effective for the persistent connection helper, instead the location must be configured
        # it only requires the injector for code coverage
        # the correct python interpreter is already selected using the sys.executable used to invoke ansible
        ansible.update(
            _ANSIBLE_CONNECTION_PATH=os.path.join(get_injector_path(), 'ansible_connection_cli_stub.py'),
        )

    if isinstance(args, PosixIntegrationConfig):
        ansible.update(
            ANSIBLE_PYTHON_INTERPRETER='/set/ansible_python_interpreter/in/inventory',  # force tests to set ansible_python_interpreter in inventory
        )

    env.update(ansible)

    if args.debug:
        env.update(
            ANSIBLE_DEBUG='true',
            ANSIBLE_LOG_PATH=os.path.join(ResultType.LOGS.name, 'debug.log'),
        )

    if data_context().content.collection:
        env.update(
            ANSIBLE_COLLECTIONS_PATH=data_context().content.collection.root,
        )

    if data_context().content.is_ansible:
        env.update(configure_plugin_paths(args))

    return env


def configure_plugin_paths(args: CommonConfig) -> dict[str, str]:
    """Return environment variables with paths to plugins relevant for the current command."""
    if not isinstance(args, IntegrationConfig):
        return {}

    support_path = os.path.join(ANSIBLE_SOURCE_ROOT, 'test', 'support', args.command)

    # provide private copies of collections for integration tests
    collection_root = os.path.join(support_path, 'collections')

    env = dict(
        ANSIBLE_COLLECTIONS_PATH=collection_root,
    )

    # provide private copies of plugins for integration tests
    plugin_root = os.path.join(support_path, 'plugins')

    plugin_list = [
        'action',
        'become',
        'cache',
        'callback',
        'cliconf',
        'connection',
        'filter',
        'httpapi',
        'inventory',
        'lookup',
        'netconf',
        # 'shell' is not configurable
        'strategy',
        'terminal',
        'test',
        'vars',
    ]

    # most plugins follow a standard naming convention
    plugin_map = dict(('%s_plugins' % name, name) for name in plugin_list)

    # these plugins do not follow the standard naming convention
    plugin_map.update(
        doc_fragment='doc_fragments',
        library='modules',
        module_utils='module_utils',
    )

    env.update(dict(('ANSIBLE_%s' % key.upper(), os.path.join(plugin_root, value)) for key, value in plugin_map.items()))

    # only configure directories which exist
    env = dict((key, value) for key, value in env.items() if os.path.isdir(value))

    return env


@mutex
def get_ansible_bin_path(args: CommonConfig) -> str:
    """
    Return a directory usable for PATH, containing only the ansible entry points.
    If a temporary directory is required, it will be cached for the lifetime of the process and cleaned up at exit.
    """
    try:
        return get_ansible_bin_path.bin_path  # type: ignore[attr-defined]
    except AttributeError:
        pass

    if ANSIBLE_SOURCE_ROOT:
        # when running from source there is no need for a temporary directory since we already have known entry point scripts
        bin_path = os.path.join(ANSIBLE_ROOT, 'bin')
    else:
        # when not running from source the installed entry points cannot be relied upon
        # doing so would require using the interpreter specified by those entry points, which conflicts with using our interpreter and injector
        # instead a temporary directory is created which contains only ansible entry points
        # symbolic links cannot be used since the files are likely not executable
        bin_path = create_temp_dir(prefix='ansible-test-', suffix='-bin')
        bin_links = {os.path.join(bin_path, name): get_cli_path(path) for name, path in ANSIBLE_BIN_SYMLINK_MAP.items()}

        if not args.explain:
            for dst, src in bin_links.items():
                shutil.copy(src, dst)
                verified_chmod(dst, MODE_FILE_EXECUTE)

    get_ansible_bin_path.bin_path = bin_path  # type: ignore[attr-defined]

    return bin_path


def get_cli_path(path: str) -> str:
    """Return the absolute path to the CLI script from the given path which is relative to the `bin` directory of the original source tree layout."""
    path_rewrite = {
        '../lib/ansible/': ANSIBLE_LIB_ROOT,
        '../test/lib/ansible_test/': ANSIBLE_TEST_ROOT,
    }

    for prefix, destination in path_rewrite.items():
        if path.startswith(prefix):
            return os.path.join(destination, path[len(prefix):])

    raise RuntimeError(path)


@mutex
def get_ansible_python_path(args: CommonConfig) -> str:
    """
    Return a directory usable for PYTHONPATH, containing only the ansible package.
    If a temporary directory is required, it will be cached for the lifetime of the process and cleaned up at exit.
    """
    try:
        return get_ansible_python_path.python_path  # type: ignore[attr-defined]
    except AttributeError:
        pass

    if ANSIBLE_SOURCE_ROOT:
        # when running from source there is no need for a temporary directory to isolate the ansible package
        python_path = os.path.dirname(ANSIBLE_LIB_ROOT)
    else:
        # when not running from source the installed directory is unsafe to add to PYTHONPATH
        # doing so would expose many unwanted packages on sys.path
        # instead a temporary directory is created which contains only ansible using a symlink
        python_path = create_temp_dir(prefix='ansible-test-')

        os.symlink(ANSIBLE_LIB_ROOT, os.path.join(python_path, 'ansible'))

    if not args.explain:
        generate_egg_info(python_path)

    get_ansible_python_path.python_path = python_path  # type: ignore[attr-defined]

    return python_path


def generate_egg_info(path: str) -> None:
    """Generate an egg-info in the specified base directory."""
    # minimal PKG-INFO stub following the format defined in PEP 241
    # required for older setuptools versions to avoid a traceback when importing pkg_resources from packages like cryptography
    # newer setuptools versions are happy with an empty directory
    # including a stub here means we don't need to locate the existing file or run any tools to generate it when running from source
    pkg_info = '''
Metadata-Version: 1.0
Name: ansible
Version: %s
Platform: UNKNOWN
Summary: Radically simple IT automation
Author-email: info@ansible.com
License: GPLv3+
''' % get_ansible_version()

    pkg_info_path = os.path.join(path, 'ansible_core.egg-info', 'PKG-INFO')

    if os.path.exists(pkg_info_path):
        return

    write_text_file(pkg_info_path, pkg_info.lstrip(), create_directories=True)


class CollectionDetail:
    """Collection detail."""

    def __init__(self) -> None:
        self.version: t.Optional[str] = None


class CollectionDetailError(ApplicationError):
    """An error occurred retrieving collection detail."""

    def __init__(self, reason: str) -> None:
        super().__init__('Error collecting collection detail: %s' % reason)
        self.reason = reason


def get_collection_detail(python: PythonConfig) -> CollectionDetail:
    """Return collection detail."""
    collection = data_context().content.collection
    directory = os.path.join(collection.root, collection.directory)

    stdout = raw_command([python.path, os.path.join(ANSIBLE_TEST_TOOLS_ROOT, 'collection_detail.py'), directory], capture=True)[0]
    result = json.loads(stdout)
    error = result.get('error')

    if error:
        raise CollectionDetailError(error)

    version = result.get('version')

    detail = CollectionDetail()
    detail.version = str(version) if version is not None else None

    return detail


def run_playbook(
    args: EnvironmentConfig,
    inventory_path: str,
    playbook: str,
    capture: bool,
    variables: t.Optional[dict[str, t.Any]] = None,
) -> None:
    """Run the specified playbook using the given inventory file and playbook variables."""
    playbook_path = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'playbooks', playbook)
    cmd = ['ansible-playbook', '-i', inventory_path, playbook_path]

    if variables:
        cmd.extend(['-e', json.dumps(variables)])

    if args.verbosity:
        cmd.append('-%s' % ('v' * args.verbosity))

    install_requirements(args, args.controller_python, ansible=True)  # run_playbook()
    env = ansible_environment(args)
    intercept_python(args, args.controller_python, cmd, env, capture=capture)