ansible/test/lib/ansible_test/_internal/test.py

"""Classes for storing and processing test results."""
from __future__ import annotations

import datetime
import re
import typing as t

from .util import (
    display,
    get_ansible_version,
)

from .util_common import (
    write_text_test_results,
    write_json_test_results,
    ResultType,
)

from .metadata import (
    Metadata,
)

from .config import (
    TestConfig,
)

from . import junit_xml


def calculate_best_confidence(choices, metadata):  # type: (t.Tuple[t.Tuple[str, int], ...], Metadata) -> int
    """Return the best confidence value available from the given choices and metadata."""
    best_confidence = 0

    for path, line in choices:
        confidence = calculate_confidence(path, line, metadata)
        best_confidence = max(confidence, best_confidence)

    return best_confidence


def calculate_confidence(path, line, metadata):  # type: (str, int, Metadata) -> int
    """Return the confidence level for a test result associated with the given file path and line number."""
    ranges = metadata.changes.get(path)

    # no changes were made to the file
    if not ranges:
        return 0

    # changes were made to the same file and line
    if any(r[0] <= line <= r[1] in r for r in ranges):
        return 100

    # changes were made to the same file and the line number is unknown
    if line == 0:
        return 75

    # changes were made to the same file and the line number is different
    return 50


class TestResult:
    """Base class for test results."""
    def __init__(self, command, test, python_version=None):  # type: (str, str, t.Optional[str]) -> None
        self.command = command
        self.test = test
        self.python_version = python_version
        self.name = self.test or self.command

        if self.python_version:
            self.name += '-python-%s' % self.python_version

    def write(self, args):  # type: (TestConfig) -> None
        """Write the test results to various locations."""
        self.write_console()
        self.write_bot(args)

        if args.lint:
            self.write_lint()

        if args.junit:
            self.write_junit(args)

    def write_console(self):  # type: () -> None
        """Write results to console."""

    def write_lint(self):  # type: () -> None
        """Write lint results to stdout."""

    def write_bot(self, args):  # type: (TestConfig) -> None
        """Write results to a file for ansibullbot to consume."""

    def write_junit(self, args):  # type: (TestConfig) -> None
        """Write results to a junit XML file."""

    def create_result_name(self, extension):  # type: (str) -> str
        """Return the name of the result file using the given extension."""
        name = 'ansible-test-%s' % self.command

        if self.test:
            name += '-%s' % self.test

        if self.python_version:
            name += '-python-%s' % self.python_version

        name += extension

        return name

    def save_junit(self, args, test_case):  # type: (TestConfig, junit_xml.TestCase) -> None
        """Save the given test case results to disk as JUnit XML."""
        suites = junit_xml.TestSuites(
            suites=[
                junit_xml.TestSuite(
                    name='ansible-test',
                    cases=[test_case],
                    timestamp=datetime.datetime.utcnow(),
                ),
            ],
        )

        report = suites.to_pretty_xml()

        if args.explain:
            return

        write_text_test_results(ResultType.JUNIT, self.create_result_name('.xml'), report)


class TestTimeout(TestResult):
    """Test timeout."""
    def __init__(self, timeout_duration):  # type: (int) -> None
        super().__init__(command='timeout', test='')

        self.timeout_duration = timeout_duration

    def write(self, args):  # type: (TestConfig) -> None
        """Write the test results to various locations."""
        message = 'Tests were aborted after exceeding the %d minute time limit.' % self.timeout_duration

        # Include a leading newline to improve readability on Shippable "Tests" tab.
        # Without this, the first line becomes indented.
        output = '''
One or more of the following situations may be responsible:

- Code changes have resulted in tests that hang or run for an excessive amount of time.
- Tests have been added which exceed the time limit when combined with existing tests.
- Test infrastructure and/or external dependencies are operating slower than normal.'''

        if args.coverage:
            output += '\n- Additional overhead from collecting code coverage has resulted in tests exceeding the time limit.'

        output += '\n\nConsult the console log for additional details on where the timeout occurred.'

        timestamp = datetime.datetime.utcnow()

        suites = junit_xml.TestSuites(
            suites=[
                junit_xml.TestSuite(
                    name='ansible-test',
                    timestamp=timestamp,
                    cases=[
                        junit_xml.TestCase(
                            name='timeout',
                            classname='timeout',
                            errors=[
                                junit_xml.TestError(
                                    message=message,
                                ),
                            ],
                        ),
                    ],
                )
            ],
        )

        report = suites.to_pretty_xml()

        write_text_test_results(ResultType.JUNIT, self.create_result_name('.xml'), report)


class TestSuccess(TestResult):
    """Test success."""
    def write_junit(self, args):  # type: (TestConfig) -> None
        """Write results to a junit XML file."""
        test_case = junit_xml.TestCase(classname=self.command, name=self.name)

        self.save_junit(args, test_case)


class TestSkipped(TestResult):
    """Test skipped."""
    def __init__(self, command, test, python_version=None):  # type: (str, str, t.Optional[str]) -> None
        super().__init__(command, test, python_version)

        self.reason = None  # type: t.Optional[str]

    def write_console(self):  # type: () -> None
        """Write results to console."""
        if self.reason:
            display.warning(self.reason)
        else:
            display.info('No tests applicable.', verbosity=1)

    def write_junit(self, args):  # type: (TestConfig) -> None
        """Write results to a junit XML file."""
        test_case = junit_xml.TestCase(
            classname=self.command,
            name=self.name,
            skipped=self.reason or 'No tests applicable.',
        )

        self.save_junit(args, test_case)


class TestFailure(TestResult):
    """Test failure."""
    def __init__(
            self,
            command,  # type: str
            test,  # type: str
            python_version=None,  # type: t.Optional[str]
            messages=None,  # type: t.Optional[t.Sequence[TestMessage]]
            summary=None,  # type: t.Optional[str]
    ):
        super().__init__(command, test, python_version)

        if messages:
            messages = sorted(messages)
        else:
            messages = []

        self.messages = messages
        self.summary = summary

    def write(self, args):  # type: (TestConfig) -> None
        """Write the test results to various locations."""
        if args.metadata.changes:
            self.populate_confidence(args.metadata)

        super().write(args)

    def write_console(self):  # type: () -> None
        """Write results to console."""
        if self.summary:
            display.error(self.summary)
        else:
            if self.python_version:
                specifier = ' on python %s' % self.python_version
            else:
                specifier = ''

            display.error('Found %d %s issue(s)%s which need to be resolved:' % (len(self.messages), self.test or self.command, specifier))

            for message in self.messages:
                display.error(message.format(show_confidence=True))

            doc_url = self.find_docs()
            if doc_url:
                display.info('See documentation for help: %s' % doc_url)

    def write_lint(self):  # type: () -> None
        """Write lint results to stdout."""
        if self.summary:
            command = self.format_command()
            message = 'The test `%s` failed. See stderr output for details.' % command
            path = ''
            message = TestMessage(message, path)
            print(message)  # display goes to stderr, this should be on stdout
        else:
            for message in self.messages:
                print(message)  # display goes to stderr, this should be on stdout

    def write_junit(self, args):  # type: (TestConfig) -> None
        """Write results to a junit XML file."""
        title = self.format_title()
        output = self.format_block()

        test_case = junit_xml.TestCase(
            classname=self.command,
            name=self.name,
            failures=[
                junit_xml.TestFailure(
                    message=title,
                    output=output,
                ),
            ],
        )

        self.save_junit(args, test_case)

    def write_bot(self, args):  # type: (TestConfig) -> None
        """Write results to a file for ansibullbot to consume."""
        docs = self.find_docs()
        message = self.format_title(help_link=docs)
        output = self.format_block()

        if self.messages:
            verified = all((m.confidence or 0) >= 50 for m in self.messages)
        else:
            verified = False

        bot_data = dict(
            verified=verified,
            docs=docs,
            results=[
                dict(
                    message=message,
                    output=output,
                ),
            ],
        )

        if args.explain:
            return

        write_json_test_results(ResultType.BOT, self.create_result_name('.json'), bot_data)

    def populate_confidence(self, metadata):  # type: (Metadata) -> None
        """Populate test result confidence using the provided metadata."""
        for message in self.messages:
            if message.confidence is None:
                message.confidence = calculate_confidence(message.path, message.line, metadata)

    def format_command(self):  # type: () -> str
        """Return a string representing the CLI command associated with the test failure."""
        command = 'ansible-test %s' % self.command

        if self.test:
            command += ' --test %s' % self.test

        if self.python_version:
            command += ' --python %s' % self.python_version

        return command

    def find_docs(self):
        """
        :rtype: str
        """
        if self.command != 'sanity':
            return None  # only sanity tests have docs links

        # Use the major.minor version for the URL only if this a release that
        # matches the pattern 2.4.0, otherwise, use 'devel'
        ansible_version = get_ansible_version()
        url_version = 'devel'
        if re.search(r'^[0-9.]+$', ansible_version):
            url_version = '.'.join(ansible_version.split('.')[:2])

        testing_docs_url = 'https://docs.ansible.com/ansible-core/%s/dev_guide/testing' % url_version

        url = '%s/%s/' % (testing_docs_url, self.command)

        if self.test:
            url += '%s.html' % self.test

        return url

    def format_title(self, help_link=None):  # type: (t.Optional[str]) -> str
        """Return a string containing a title/heading for this test failure, including an optional help link to explain the test."""
        command = self.format_command()

        if self.summary:
            reason = 'the error'
        else:
            reason = '1 error' if len(self.messages) == 1 else '%d errors' % len(self.messages)

        if help_link:
            help_link_markup = ' [[explain](%s)]' % help_link
        else:
            help_link_markup = ''

        title = 'The test `%s`%s failed with %s:' % (command, help_link_markup, reason)

        return title

    def format_block(self):  # type: () -> str
        """Format the test summary or messages as a block of text and return the result."""
        if self.summary:
            block = self.summary
        else:
            block = '\n'.join(m.format() for m in self.messages)

        message = block.strip()

        # Hack to remove ANSI color reset code from SubprocessError messages.
        message = message.replace(display.clear, '')

        return message


class TestMessage:
    """Single test message for one file."""
    def __init__(
            self,
            message,  # type: str
            path,  # type: str
            line=0,  # type: int
            column=0,  # type: int
            level='error',  # type: str
            code=None,  # type: t.Optional[str]
            confidence=None,  # type: t.Optional[int]
    ):
        self.__path = path
        self.__line = line
        self.__column = column
        self.__level = level
        self.__code = code
        self.__message = message

        self.confidence = confidence

    @property
    def path(self):  # type: () -> str
        """Return the path."""
        return self.__path

    @property
    def line(self):  # type: () -> int
        """Return the line number, or 0 if none is available."""
        return self.__line

    @property
    def column(self):  # type: () -> int
        """Return the column number, or 0 if none is available."""
        return self.__column

    @property
    def level(self):  # type: () -> str
        """Return the level."""
        return self.__level

    @property
    def code(self):  # type: () -> t.Optional[str]
        """Return the code, if any."""
        return self.__code

    @property
    def message(self):  # type: () -> str
        """Return the message."""
        return self.__message

    @property
    def tuple(self):  # type: () -> t.Tuple[str, int, int, str, t.Optional[str], str]
        """Return a tuple with all the immutable values of this test message."""
        return self.__path, self.__line, self.__column, self.__level, self.__code, self.__message

    def __lt__(self, other):
        return self.tuple < other.tuple

    def __le__(self, other):
        return self.tuple <= other.tuple

    def __eq__(self, other):
        return self.tuple == other.tuple

    def __ne__(self, other):
        return self.tuple != other.tuple

    def __gt__(self, other):
        return self.tuple > other.tuple

    def __ge__(self, other):
        return self.tuple >= other.tuple

    def __hash__(self):
        return hash(self.tuple)

    def __str__(self):
        return self.format()

    def format(self, show_confidence=False):  # type: (bool) -> str
        """Return a string representation of this message, optionally including the confidence level."""
        if self.__code:
            msg = '%s: %s' % (self.__code, self.__message)
        else:
            msg = self.__message

        if show_confidence and self.confidence is not None:
            msg += ' (%d%%)' % self.confidence

        return '%s:%s:%s: %s' % (self.__path, self.__line, self.__column, msg)