ansible/test/lib/ansible_test/_internal/commands/sanity/integration_aliases.py

"""Sanity test to check integration test aliases."""
from __future__ import annotations

import dataclasses
import json
import textwrap
import os
import re
import typing as t

from . import (
    SanitySingleVersion,
    SanityMessage,
    SanityFailure,
    SanitySuccess,
    SanityTargets,
    SANITY_ROOT,
)

from ...test import (
    TestResult,
)

from ...config import (
    SanityConfig,
)

from ...target import (
    filter_targets,
    walk_posix_integration_targets,
    walk_windows_integration_targets,
    walk_integration_targets,
    walk_module_targets,
    CompletionTarget,
    IntegrationTargetType,
)

from ..integration.cloud import (
    get_cloud_platforms,
)

from ...io import (
    read_text_file,
)

from ...util import (
    display,
    raw_command,
)

from ...util_common import (
    write_json_test_results,
    ResultType,
)

from ...host_configs import (
    PythonConfig,
)


class IntegrationAliasesTest(SanitySingleVersion):
    """Sanity test to evaluate integration test aliases."""
    CI_YML = '.azure-pipelines/azure-pipelines.yml'
    TEST_ALIAS_PREFIX = 'shippable'  # this will be changed at some point in the future

    DISABLED = 'disabled/'
    UNSTABLE = 'unstable/'
    UNSUPPORTED = 'unsupported/'

    EXPLAIN_URL = 'https://docs.ansible.com/ansible-core/devel/dev_guide/testing/sanity/integration-aliases.html'

    TEMPLATE_DISABLED = """
    The following integration tests are **disabled** [[explain]({explain_url}#disabled)]:

    {tests}

    Consider fixing the integration tests before or alongside changes.
    """

    TEMPLATE_UNSTABLE = """
    The following integration tests are **unstable** [[explain]({explain_url}#unstable)]:

    {tests}

    Tests may need to be restarted due to failures unrelated to changes.
    """

    TEMPLATE_UNSUPPORTED = """
    The following integration tests are **unsupported** [[explain]({explain_url}#unsupported)]:

    {tests}

    Consider running the tests manually or extending test infrastructure to add support.
    """

    TEMPLATE_UNTESTED = """
    The following modules have **no integration tests** [[explain]({explain_url}#untested)]:

    {tests}

    Consider adding integration tests before or alongside changes.
    """

    ansible_only = True

    def __init__(self):
        super().__init__()

        self._ci_config: t.Dict[str, t.Any] = {}
        self._ci_test_groups: t.Dict[str, t.List[int]] = {}

    @property
    def can_ignore(self) -> bool:
        """True if the test supports ignore entries."""
        return False

    @property
    def no_targets(self) -> bool:
        """True if the test does not use test targets. Mutually exclusive with all_targets."""
        return True

    def load_ci_config(self, python: PythonConfig) -> t.Dict[str, t.Any]:
        """Load and return the CI YAML configuration."""
        if not self._ci_config:
            self._ci_config = self.load_yaml(python, self.CI_YML)

        return self._ci_config

    @property
    def ci_test_groups(self) -> t.Dict[str, t.List[int]]:
        """Return a dictionary of CI test names and their group(s)."""
        if not self._ci_test_groups:
            test_groups: t.Dict[str, t.Set[int]] = {}

            for stage in self._ci_config['stages']:
                for job in stage['jobs']:
                    if job.get('template') != 'templates/matrix.yml':
                        continue

                    parameters = job['parameters']

                    groups = parameters.get('groups', [])
                    test_format = parameters.get('testFormat', '{0}')
                    test_group_format = parameters.get('groupFormat', '{0}/{{1}}')

                    for target in parameters['targets']:
                        test = target.get('test') or target.get('name')

                        if groups:
                            tests_formatted = [test_group_format.format(test_format).format(test, group) for group in groups]
                        else:
                            tests_formatted = [test_format.format(test)]

                        for test_formatted in tests_formatted:
                            parts = test_formatted.split('/')
                            key = parts[0]

                            if key in ('sanity', 'units'):
                                continue

                            try:
                                group = int(parts[-1])
                            except ValueError:
                                continue

                            if group < 1 or group > 99:
                                continue

                            group_set = test_groups.setdefault(key, set())
                            group_set.add(group)

            self._ci_test_groups = dict((key, sorted(value)) for key, value in test_groups.items())

        return self._ci_test_groups

    def format_test_group_alias(self, name: str, fallback: str = '') -> str:
        """Return a test group alias using the given name and fallback."""
        group_numbers = self.ci_test_groups.get(name, None)

        if group_numbers:
            if min(group_numbers) != 1:
                display.warning('Min test group "%s" in %s is %d instead of 1.' % (name, self.CI_YML, min(group_numbers)), unique=True)

            if max(group_numbers) != len(group_numbers):
                display.warning('Max test group "%s" in %s is %d instead of %d.' % (name, self.CI_YML, max(group_numbers), len(group_numbers)), unique=True)

            if max(group_numbers) > 9:
                alias = '%s/%s/group(%s)/' % (self.TEST_ALIAS_PREFIX, name, '|'.join(str(i) for i in range(min(group_numbers), max(group_numbers) + 1)))
            elif len(group_numbers) > 1:
                alias = '%s/%s/group[%d-%d]/' % (self.TEST_ALIAS_PREFIX, name, min(group_numbers), max(group_numbers))
            else:
                alias = '%s/%s/group%d/' % (self.TEST_ALIAS_PREFIX, name, min(group_numbers))
        elif fallback:
            alias = '%s/%s/group%d/' % (self.TEST_ALIAS_PREFIX, fallback, 1)
        else:
            raise Exception('cannot find test group "%s" in %s' % (name, self.CI_YML))

        return alias

    def load_yaml(self, python: PythonConfig, path: str) -> t.Dict[str, t.Any]:
        """Load the specified YAML file and return the contents."""
        yaml_to_json_path = os.path.join(SANITY_ROOT, self.name, 'yaml_to_json.py')
        return json.loads(raw_command([python.path, yaml_to_json_path], data=read_text_file(path), capture=True)[0])

    def test(self, args: SanityConfig, targets: SanityTargets, python: PythonConfig) -> TestResult:
        if args.explain:
            return SanitySuccess(self.name)

        if not os.path.isfile(self.CI_YML):
            return SanityFailure(self.name, messages=[SanityMessage(
                message='file missing',
                path=self.CI_YML,
            )])

        results = Results(
            comments=[],
            labels={},
        )

        self.load_ci_config(python)
        self.check_changes(args, results)

        write_json_test_results(ResultType.BOT, 'data-sanity-ci.json', results.__dict__)

        messages = []

        messages += self.check_posix_targets(args)
        messages += self.check_windows_targets()

        if messages:
            return SanityFailure(self.name, messages=messages)

        return SanitySuccess(self.name)

    def check_posix_targets(self, args: SanityConfig) -> t.List[SanityMessage]:
        """Check POSIX integration test targets and return messages with any issues found."""
        posix_targets = tuple(walk_posix_integration_targets())

        clouds = get_cloud_platforms(args, posix_targets)
        cloud_targets = ['cloud/%s/' % cloud for cloud in clouds]

        all_cloud_targets = tuple(filter_targets(posix_targets, ['cloud/'], directories=False, errors=False))
        invalid_cloud_targets = tuple(filter_targets(all_cloud_targets, cloud_targets, include=False, directories=False, errors=False))

        messages = []

        for target in invalid_cloud_targets:
            for alias in target.aliases:
                if alias.startswith('cloud/') and alias != 'cloud/':
                    if any(alias.startswith(cloud_target) for cloud_target in cloud_targets):
                        continue

                    messages.append(SanityMessage('invalid alias `%s`' % alias, '%s/aliases' % target.path))

        messages += self.check_ci_group(
            targets=tuple(filter_targets(posix_targets, ['cloud/', '%s/generic/' % self.TEST_ALIAS_PREFIX], include=False, directories=False, errors=False)),
            find=self.format_test_group_alias('linux').replace('linux', 'posix'),
            find_incidental=['%s/posix/incidental/' % self.TEST_ALIAS_PREFIX],
        )

        messages += self.check_ci_group(
            targets=tuple(filter_targets(posix_targets, ['%s/generic/' % self.TEST_ALIAS_PREFIX], directories=False, errors=False)),
            find=self.format_test_group_alias('generic'),
        )

        for cloud in clouds:
            if cloud == 'httptester':
                find = self.format_test_group_alias('linux').replace('linux', 'posix')
                find_incidental = ['%s/posix/incidental/' % self.TEST_ALIAS_PREFIX]
            else:
                find = self.format_test_group_alias(cloud, 'generic')
                find_incidental = ['%s/%s/incidental/' % (self.TEST_ALIAS_PREFIX, cloud), '%s/cloud/incidental/' % self.TEST_ALIAS_PREFIX]

            messages += self.check_ci_group(
                targets=tuple(filter_targets(posix_targets, ['cloud/%s/' % cloud], directories=False, errors=False)),
                find=find,
                find_incidental=find_incidental,
            )

        target_type_groups = {
            IntegrationTargetType.TARGET: (1, 2),
            IntegrationTargetType.CONTROLLER: (3, 4, 5),
            IntegrationTargetType.CONFLICT: (),
            IntegrationTargetType.UNKNOWN: (),
        }

        for target in posix_targets:
            if f'{self.TEST_ALIAS_PREFIX}/posix/' not in target.aliases:
                continue

            found_groups = [alias for alias in target.aliases if re.search(f'^{self.TEST_ALIAS_PREFIX}/posix/group[0-9]+/$', alias)]
            expected_groups = [f'{self.TEST_ALIAS_PREFIX}/posix/group{group}/' for group in target_type_groups[target.target_type]]
            valid_groups = [group for group in found_groups if group in expected_groups]
            invalid_groups = [group for group in found_groups if not any(group.startswith(expected_group) for expected_group in expected_groups)]

            if not valid_groups:
                messages.append(SanityMessage(f'Target of type {target.target_type.name} must be in at least one of these groups: {", ".join(expected_groups)}',
                                              f'{target.path}/aliases'))

            if invalid_groups:
                messages.append(SanityMessage(f'Target of type {target.target_type.name} cannot be in these groups: {", ".join(invalid_groups)}',
                                              f'{target.path}/aliases'))

        return messages

    def check_windows_targets(self):
        """
        :rtype: list[SanityMessage]
        """
        windows_targets = tuple(walk_windows_integration_targets())

        messages = []

        messages += self.check_ci_group(
            targets=windows_targets,
            find=self.format_test_group_alias('windows'),
            find_incidental=['%s/windows/incidental/' % self.TEST_ALIAS_PREFIX],
        )

        return messages

    def check_ci_group(
            self,
            targets: t.Tuple[CompletionTarget, ...],
            find: str,
            find_incidental: t.Optional[t.List[str]] = None,
    ) -> t.List[SanityMessage]:
        """Check the CI groups set in the provided targets and return a list of messages with any issues found."""
        all_paths = set(target.path for target in targets)
        supported_paths = set(target.path for target in filter_targets(targets, [find], directories=False, errors=False))
        unsupported_paths = set(target.path for target in filter_targets(targets, [self.UNSUPPORTED], directories=False, errors=False))

        if find_incidental:
            incidental_paths = set(target.path for target in filter_targets(targets, find_incidental, directories=False, errors=False))
        else:
            incidental_paths = set()

        unassigned_paths = all_paths - supported_paths - unsupported_paths - incidental_paths
        conflicting_paths = supported_paths & unsupported_paths

        unassigned_message = 'missing alias `%s` or `%s`' % (find.strip('/'), self.UNSUPPORTED.strip('/'))
        conflicting_message = 'conflicting alias `%s` and `%s`' % (find.strip('/'), self.UNSUPPORTED.strip('/'))

        messages = []

        for path in unassigned_paths:
            messages.append(SanityMessage(unassigned_message, '%s/aliases' % path))

        for path in conflicting_paths:
            messages.append(SanityMessage(conflicting_message, '%s/aliases' % path))

        return messages

    def check_changes(self, args: SanityConfig, results: Results) -> None:
        """Check changes and store results in the provided result dictionary."""
        integration_targets = list(walk_integration_targets())
        module_targets = list(walk_module_targets())

        integration_targets_by_name = dict((target.name, target) for target in integration_targets)
        module_names_by_path = dict((target.path, target.module) for target in module_targets)

        disabled_targets = []
        unstable_targets = []
        unsupported_targets = []

        for command in [command for command in args.metadata.change_description.focused_command_targets if 'integration' in command]:
            for target in args.metadata.change_description.focused_command_targets[command]:
                if self.DISABLED in integration_targets_by_name[target].aliases:
                    disabled_targets.append(target)
                elif self.UNSTABLE in integration_targets_by_name[target].aliases:
                    unstable_targets.append(target)
                elif self.UNSUPPORTED in integration_targets_by_name[target].aliases:
                    unsupported_targets.append(target)

        untested_modules = []

        for path in args.metadata.change_description.no_integration_paths:
            module = module_names_by_path.get(path)

            if module:
                untested_modules.append(module)

        comments = [
            self.format_comment(self.TEMPLATE_DISABLED, disabled_targets),
            self.format_comment(self.TEMPLATE_UNSTABLE, unstable_targets),
            self.format_comment(self.TEMPLATE_UNSUPPORTED, unsupported_targets),
            self.format_comment(self.TEMPLATE_UNTESTED, untested_modules),
        ]

        comments = [comment for comment in comments if comment]

        labels = dict(
            needs_tests=bool(untested_modules),
            disabled_tests=bool(disabled_targets),
            unstable_tests=bool(unstable_targets),
            unsupported_tests=bool(unsupported_targets),
        )

        results.comments += comments
        results.labels.update(labels)

    def format_comment(self, template: str, targets: t.List[str]) -> t.Optional[str]:
        """Format and return a comment based on the given template and targets, or None if there are no targets."""
        if not targets:
            return None

        tests = '\n'.join('- %s' % target for target in targets)

        data = dict(
            explain_url=self.EXPLAIN_URL,
            tests=tests,
        )

        message = textwrap.dedent(template).strip().format(**data)

        return message


@dataclasses.dataclass
class Results:
    """Check results."""
    comments: t.List[str]
    labels: t.Dict[str, bool]