|
|
|
@ -16,6 +16,10 @@ from .util_common import (
|
|
|
|
|
ResultType,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
from .metadata import (
|
|
|
|
|
Metadata,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
from .config import (
|
|
|
|
|
TestConfig,
|
|
|
|
|
)
|
|
|
|
@ -23,12 +27,8 @@ from .config import (
|
|
|
|
|
from . import junit_xml
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def calculate_best_confidence(choices, metadata):
|
|
|
|
|
"""
|
|
|
|
|
:type choices: tuple[tuple[str, int]]
|
|
|
|
|
:type metadata: Metadata
|
|
|
|
|
:rtype: int
|
|
|
|
|
"""
|
|
|
|
|
def calculate_best_confidence(choices, metadata): # type: (t.Tuple[t.Tuple[str, int], ...], Metadata) -> int
|
|
|
|
|
"""Return the best confidence value available from the given choices and metadata."""
|
|
|
|
|
best_confidence = 0
|
|
|
|
|
|
|
|
|
|
for path, line in choices:
|
|
|
|
@ -38,13 +38,8 @@ def calculate_best_confidence(choices, metadata):
|
|
|
|
|
return best_confidence
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def calculate_confidence(path, line, metadata):
|
|
|
|
|
"""
|
|
|
|
|
:type path: str
|
|
|
|
|
:type line: int
|
|
|
|
|
:type metadata: Metadata
|
|
|
|
|
:rtype: int
|
|
|
|
|
"""
|
|
|
|
|
def calculate_confidence(path, line, metadata): # type: (str, int, Metadata) -> int
|
|
|
|
|
"""Return the confidence level for a test result associated with the given file path and line number."""
|
|
|
|
|
ranges = metadata.changes.get(path)
|
|
|
|
|
|
|
|
|
|
# no changes were made to the file
|
|
|
|
@ -65,12 +60,7 @@ def calculate_confidence(path, line, metadata):
|
|
|
|
|
|
|
|
|
|
class TestResult:
|
|
|
|
|
"""Base class for test results."""
|
|
|
|
|
def __init__(self, command, test, python_version=None):
|
|
|
|
|
"""
|
|
|
|
|
:type command: str
|
|
|
|
|
:type test: str
|
|
|
|
|
:type python_version: str
|
|
|
|
|
"""
|
|
|
|
|
def __init__(self, command, test, python_version=None): # type: (str, str, t.Optional[str]) -> None
|
|
|
|
|
self.command = command
|
|
|
|
|
self.test = test
|
|
|
|
|
self.python_version = python_version
|
|
|
|
@ -90,27 +80,20 @@ class TestResult:
|
|
|
|
|
if args.junit:
|
|
|
|
|
self.write_junit(args)
|
|
|
|
|
|
|
|
|
|
def write_console(self):
|
|
|
|
|
def write_console(self): # type: () -> None
|
|
|
|
|
"""Write results to console."""
|
|
|
|
|
|
|
|
|
|
def write_lint(self):
|
|
|
|
|
def write_lint(self): # type: () -> None
|
|
|
|
|
"""Write lint results to stdout."""
|
|
|
|
|
|
|
|
|
|
def write_bot(self, args):
|
|
|
|
|
"""
|
|
|
|
|
:type args: TestConfig
|
|
|
|
|
"""
|
|
|
|
|
def write_bot(self, args): # type: (TestConfig) -> None
|
|
|
|
|
"""Write results to a file for ansibullbot to consume."""
|
|
|
|
|
|
|
|
|
|
def write_junit(self, args):
|
|
|
|
|
"""
|
|
|
|
|
:type args: TestConfig
|
|
|
|
|
"""
|
|
|
|
|
def write_junit(self, args): # type: (TestConfig) -> None
|
|
|
|
|
"""Write results to a junit XML file."""
|
|
|
|
|
|
|
|
|
|
def create_result_name(self, extension):
|
|
|
|
|
"""
|
|
|
|
|
:type extension: str
|
|
|
|
|
:rtype: str
|
|
|
|
|
"""
|
|
|
|
|
def create_result_name(self, extension): # type: (str) -> str
|
|
|
|
|
"""Return the name of the result file using the given extension."""
|
|
|
|
|
name = 'ansible-test-%s' % self.command
|
|
|
|
|
|
|
|
|
|
if self.test:
|
|
|
|
@ -145,18 +128,13 @@ class TestResult:
|
|
|
|
|
|
|
|
|
|
class TestTimeout(TestResult):
|
|
|
|
|
"""Test timeout."""
|
|
|
|
|
def __init__(self, timeout_duration):
|
|
|
|
|
"""
|
|
|
|
|
:type timeout_duration: int
|
|
|
|
|
"""
|
|
|
|
|
def __init__(self, timeout_duration): # type: (int) -> None
|
|
|
|
|
super().__init__(command='timeout', test='')
|
|
|
|
|
|
|
|
|
|
self.timeout_duration = timeout_duration
|
|
|
|
|
|
|
|
|
|
def write(self, args):
|
|
|
|
|
"""
|
|
|
|
|
:type args: TestConfig
|
|
|
|
|
"""
|
|
|
|
|
def write(self, args): # type: (TestConfig) -> None
|
|
|
|
|
"""Write the test results to various locations."""
|
|
|
|
|
message = 'Tests were aborted after exceeding the %d minute time limit.' % self.timeout_duration
|
|
|
|
|
|
|
|
|
|
# Include a leading newline to improve readability on Shippable "Tests" tab.
|
|
|
|
@ -202,10 +180,8 @@ One or more of the following situations may be responsible:
|
|
|
|
|
|
|
|
|
|
class TestSuccess(TestResult):
|
|
|
|
|
"""Test success."""
|
|
|
|
|
def write_junit(self, args):
|
|
|
|
|
"""
|
|
|
|
|
:type args: TestConfig
|
|
|
|
|
"""
|
|
|
|
|
def write_junit(self, args): # type: (TestConfig) -> None
|
|
|
|
|
"""Write results to a junit XML file."""
|
|
|
|
|
test_case = junit_xml.TestCase(classname=self.command, name=self.name)
|
|
|
|
|
|
|
|
|
|
self.save_junit(args, test_case)
|
|
|
|
@ -213,27 +189,20 @@ class TestSuccess(TestResult):
|
|
|
|
|
|
|
|
|
|
class TestSkipped(TestResult):
|
|
|
|
|
"""Test skipped."""
|
|
|
|
|
def __init__(self, command, test, python_version=None):
|
|
|
|
|
"""
|
|
|
|
|
:type command: str
|
|
|
|
|
:type test: str
|
|
|
|
|
:type python_version: str
|
|
|
|
|
"""
|
|
|
|
|
def __init__(self, command, test, python_version=None): # type: (str, str, t.Optional[str]) -> None
|
|
|
|
|
super().__init__(command, test, python_version)
|
|
|
|
|
|
|
|
|
|
self.reason = None # type: t.Optional[str]
|
|
|
|
|
|
|
|
|
|
def write_console(self):
|
|
|
|
|
def write_console(self): # type: () -> None
|
|
|
|
|
"""Write results to console."""
|
|
|
|
|
if self.reason:
|
|
|
|
|
display.warning(self.reason)
|
|
|
|
|
else:
|
|
|
|
|
display.info('No tests applicable.', verbosity=1)
|
|
|
|
|
|
|
|
|
|
def write_junit(self, args):
|
|
|
|
|
"""
|
|
|
|
|
:type args: TestConfig
|
|
|
|
|
"""
|
|
|
|
|
def write_junit(self, args): # type: (TestConfig) -> None
|
|
|
|
|
"""Write results to a junit XML file."""
|
|
|
|
|
test_case = junit_xml.TestCase(
|
|
|
|
|
classname=self.command,
|
|
|
|
|
name=self.name,
|
|
|
|
@ -245,14 +214,14 @@ class TestSkipped(TestResult):
|
|
|
|
|
|
|
|
|
|
class TestFailure(TestResult):
|
|
|
|
|
"""Test failure."""
|
|
|
|
|
def __init__(self, command, test, python_version=None, messages=None, summary=None):
|
|
|
|
|
"""
|
|
|
|
|
:type command: str
|
|
|
|
|
:type test: str
|
|
|
|
|
:type python_version: str | None
|
|
|
|
|
:type messages: list[TestMessage] | None
|
|
|
|
|
:type summary: unicode | None
|
|
|
|
|
"""
|
|
|
|
|
def __init__(
|
|
|
|
|
self,
|
|
|
|
|
command, # type: str
|
|
|
|
|
test, # type: str
|
|
|
|
|
python_version=None, # type: t.Optional[str]
|
|
|
|
|
messages=None, # type: t.Optional[t.List[TestMessage]]
|
|
|
|
|
summary=None, # type: t.Optional[str]
|
|
|
|
|
):
|
|
|
|
|
super().__init__(command, test, python_version)
|
|
|
|
|
|
|
|
|
|
if messages:
|
|
|
|
@ -263,16 +232,14 @@ class TestFailure(TestResult):
|
|
|
|
|
self.messages = messages
|
|
|
|
|
self.summary = summary
|
|
|
|
|
|
|
|
|
|
def write(self, args):
|
|
|
|
|
"""
|
|
|
|
|
:type args: TestConfig
|
|
|
|
|
"""
|
|
|
|
|
def write(self, args): # type: (TestConfig) -> None
|
|
|
|
|
"""Write the test results to various locations."""
|
|
|
|
|
if args.metadata.changes:
|
|
|
|
|
self.populate_confidence(args.metadata)
|
|
|
|
|
|
|
|
|
|
super().write(args)
|
|
|
|
|
|
|
|
|
|
def write_console(self):
|
|
|
|
|
def write_console(self): # type: () -> None
|
|
|
|
|
"""Write results to console."""
|
|
|
|
|
if self.summary:
|
|
|
|
|
display.error(self.summary)
|
|
|
|
@ -291,7 +258,7 @@ class TestFailure(TestResult):
|
|
|
|
|
if doc_url:
|
|
|
|
|
display.info('See documentation for help: %s' % doc_url)
|
|
|
|
|
|
|
|
|
|
def write_lint(self):
|
|
|
|
|
def write_lint(self): # type: () -> None
|
|
|
|
|
"""Write lint results to stdout."""
|
|
|
|
|
if self.summary:
|
|
|
|
|
command = self.format_command()
|
|
|
|
@ -303,10 +270,8 @@ class TestFailure(TestResult):
|
|
|
|
|
for message in self.messages:
|
|
|
|
|
print(message)
|
|
|
|
|
|
|
|
|
|
def write_junit(self, args):
|
|
|
|
|
"""
|
|
|
|
|
:type args: TestConfig
|
|
|
|
|
"""
|
|
|
|
|
def write_junit(self, args): # type: (TestConfig) -> None
|
|
|
|
|
"""Write results to a junit XML file."""
|
|
|
|
|
title = self.format_title()
|
|
|
|
|
output = self.format_block()
|
|
|
|
|
|
|
|
|
@ -323,10 +288,8 @@ class TestFailure(TestResult):
|
|
|
|
|
|
|
|
|
|
self.save_junit(args, test_case)
|
|
|
|
|
|
|
|
|
|
def write_bot(self, args):
|
|
|
|
|
"""
|
|
|
|
|
:type args: TestConfig
|
|
|
|
|
"""
|
|
|
|
|
def write_bot(self, args): # type: (TestConfig) -> None
|
|
|
|
|
"""Write results to a file for ansibullbot to consume."""
|
|
|
|
|
docs = self.find_docs()
|
|
|
|
|
message = self.format_title(help_link=docs)
|
|
|
|
|
output = self.format_block()
|
|
|
|
@ -352,18 +315,14 @@ class TestFailure(TestResult):
|
|
|
|
|
|
|
|
|
|
write_json_test_results(ResultType.BOT, self.create_result_name('.json'), bot_data)
|
|
|
|
|
|
|
|
|
|
def populate_confidence(self, metadata):
|
|
|
|
|
"""
|
|
|
|
|
:type metadata: Metadata
|
|
|
|
|
"""
|
|
|
|
|
def populate_confidence(self, metadata): # type: (Metadata) -> None
|
|
|
|
|
"""Populate test result confidence using the provided metadata."""
|
|
|
|
|
for message in self.messages:
|
|
|
|
|
if message.confidence is None:
|
|
|
|
|
message.confidence = calculate_confidence(message.path, message.line, metadata)
|
|
|
|
|
|
|
|
|
|
def format_command(self):
|
|
|
|
|
"""
|
|
|
|
|
:rtype: str
|
|
|
|
|
"""
|
|
|
|
|
def format_command(self): # type: () -> str
|
|
|
|
|
"""Return a string representing the CLI command associated with the test failure."""
|
|
|
|
|
command = 'ansible-test %s' % self.command
|
|
|
|
|
|
|
|
|
|
if self.test:
|
|
|
|
@ -397,11 +356,8 @@ class TestFailure(TestResult):
|
|
|
|
|
|
|
|
|
|
return url
|
|
|
|
|
|
|
|
|
|
def format_title(self, help_link=None):
|
|
|
|
|
"""
|
|
|
|
|
:type help_link: str | None
|
|
|
|
|
:rtype: str
|
|
|
|
|
"""
|
|
|
|
|
def format_title(self, help_link=None): # type: (t.Optional[str]) -> str
|
|
|
|
|
"""Return a string containing a title/heading for this test failure, including an optional help link to explain the test."""
|
|
|
|
|
command = self.format_command()
|
|
|
|
|
|
|
|
|
|
if self.summary:
|
|
|
|
@ -418,10 +374,8 @@ class TestFailure(TestResult):
|
|
|
|
|
|
|
|
|
|
return title
|
|
|
|
|
|
|
|
|
|
def format_block(self):
|
|
|
|
|
"""
|
|
|
|
|
:rtype: str
|
|
|
|
|
"""
|
|
|
|
|
def format_block(self): # type: () -> str
|
|
|
|
|
"""Format the test summary or messages as a block of text and return the result."""
|
|
|
|
|
if self.summary:
|
|
|
|
|
block = self.summary
|
|
|
|
|
else:
|
|
|
|
@ -437,16 +391,16 @@ class TestFailure(TestResult):
|
|
|
|
|
|
|
|
|
|
class TestMessage:
|
|
|
|
|
"""Single test message for one file."""
|
|
|
|
|
def __init__(self, message, path, line=0, column=0, level='error', code=None, confidence=None):
|
|
|
|
|
"""
|
|
|
|
|
:type message: str
|
|
|
|
|
:type path: str
|
|
|
|
|
:type line: int
|
|
|
|
|
:type column: int
|
|
|
|
|
:type level: str
|
|
|
|
|
:type code: str | None
|
|
|
|
|
:type confidence: int | None
|
|
|
|
|
"""
|
|
|
|
|
def __init__(
|
|
|
|
|
self,
|
|
|
|
|
message, # type: str
|
|
|
|
|
path, # type: str
|
|
|
|
|
line=0, # type: int
|
|
|
|
|
column=0, # type: int
|
|
|
|
|
level='error', # type: str
|
|
|
|
|
code=None, # type: t.Optional[str]
|
|
|
|
|
confidence=None, # type: t.Optional[int]
|
|
|
|
|
):
|
|
|
|
|
self.__path = path
|
|
|
|
|
self.__line = line
|
|
|
|
|
self.__column = column
|
|
|
|
@ -515,11 +469,8 @@ class TestMessage:
|
|
|
|
|
def __str__(self):
|
|
|
|
|
return self.format()
|
|
|
|
|
|
|
|
|
|
def format(self, show_confidence=False):
|
|
|
|
|
"""
|
|
|
|
|
:type show_confidence: bool
|
|
|
|
|
:rtype: str
|
|
|
|
|
"""
|
|
|
|
|
def format(self, show_confidence=False): # type: (bool) -> str
|
|
|
|
|
"""Return a string representation of this message, optionally including the confidence level."""
|
|
|
|
|
if self.__code:
|
|
|
|
|
msg = '%s: %s' % (self.__code, self.__message)
|
|
|
|
|
else:
|
|
|
|
|