diff --git a/docs/docsite/rst/dev_guide/testing/compile/index.rst b/docs/docsite/rst/dev_guide/testing/compile/index.rst new file mode 100644 index 00000000000..f49022bad51 --- /dev/null +++ b/docs/docsite/rst/dev_guide/testing/compile/index.rst @@ -0,0 +1,4 @@ +Compile Tests +============= + +See :doc:`testing_compile` for more information. diff --git a/test/runner/lib/config.py b/test/runner/lib/config.py index 7bc5018b06c..e04fbc99741 100644 --- a/test/runner/lib/config.py +++ b/test/runner/lib/config.py @@ -140,6 +140,7 @@ class IntegrationConfig(TestConfig): self.start_at_task = args.start_at_task # type: str self.allow_destructive = args.allow_destructive if 'allow_destructive' in args else False # type: bool self.retry_on_error = args.retry_on_error # type: bool + self.continue_on_error = args.continue_on_error # type: bool self.debug_strategy = args.debug_strategy # type: bool self.changed_all_target = args.changed_all_target # type: str self.tags = args.tags @@ -182,6 +183,7 @@ class NetworkIntegrationConfig(IntegrationConfig): super(NetworkIntegrationConfig, self).__init__(args, 'network-integration') self.platform = args.platform # type: list [str] + self.inventory = args.inventory # type: str class UnitsConfig(TestConfig): diff --git a/test/runner/lib/executor.py b/test/runner/lib/executor.py index 361cb6b1a6f..073b5d6daad 100644 --- a/test/runner/lib/executor.py +++ b/test/runner/lib/executor.py @@ -252,10 +252,23 @@ def command_network_integration(args): """ :type args: NetworkIntegrationConfig """ - filename = 'test/integration/inventory.networking' + default_filename = 'test/integration/inventory.networking' - if not args.explain and not args.platform and not os.path.isfile(filename): - raise ApplicationError('Use the --platform option or provide an inventory file (see %s.template).' % filename) + if args.inventory: + filename = os.path.join('test/integration', args.inventory) + else: + filename = default_filename + + if not args.explain and not args.platform and not os.path.exists(filename): + if args.inventory: + filename = os.path.abspath(filename) + + raise ApplicationError( + 'Inventory not found: %s\n' + 'Use --inventory to specify the inventory path.\n' + 'Use --platform to provision resources and generate an inventory file.\n' + 'See also inventory template: %s.template' % (filename, default_filename) + ) internal_targets = command_integration_filter(args, walk_network_integration_targets()) platform_targets = set(a for t in internal_targets for a in t.aliases if a.startswith('network/')) @@ -500,6 +513,8 @@ def command_integration_filtered(args, targets): :type targets: tuple[IntegrationTarget] """ found = False + passed = [] + failed = [] targets_iter = iter(targets) @@ -568,7 +583,14 @@ def command_integration_filtered(args, targets): display.verbosity = args.verbosity = 6 original_environment.validate(target.name, throw=True) - except: + passed.append(target) + except Exception as ex: + failed.append(target) + + if args.continue_on_error: + display.error(ex) + continue + display.notice('To resume at this test target, use the option: --start-at %s' % target.name) next_target = next(targets_iter, None) @@ -580,6 +602,10 @@ def command_integration_filtered(args, targets): finally: display.verbosity = args.verbosity = verbosity + if failed: + raise ApplicationError('The %d integration test(s) listed below (out of %d) failed. See error output above for details:\n%s' % ( + len(failed), len(passed) + len(failed), '\n'.join(target.name for target in failed))) + def integration_environment(args, target, cmd): """ @@ -642,7 +668,7 @@ def command_integration_role(args, target, start_at_task): hosts = 'windows' gather_facts = False elif isinstance(args, NetworkIntegrationConfig): - inventory = 'inventory.networking' + inventory = args.inventory or 'inventory.networking' hosts = target.name[:target.name.find('_')] gather_facts = False if hosts == 'net': diff --git a/test/runner/lib/test.py b/test/runner/lib/test.py index 70edb703eb5..4342b5bd33d 100644 --- a/test/runner/lib/test.py +++ b/test/runner/lib/test.py @@ -4,6 +4,7 @@ from __future__ import absolute_import, print_function import datetime import json +import os from lib.util import ( display, @@ -260,6 +261,7 @@ class TestFailure(TestResult): """ message = self.format_title() output = self.format_block() + docs = self.find_docs() if self.messages: verified = all((m.confidence or 0) >= 50 for m in self.messages) @@ -268,6 +270,7 @@ class TestFailure(TestResult): bot_data = dict( verified=verified, + docs=docs, results=[ dict( message=message, @@ -307,6 +310,25 @@ class TestFailure(TestResult): return command + def find_docs(self): + """ + :rtype: str + """ + testing_docs_url = 'https://docs.ansible.com/ansible/devel/dev_guide/testing' + testing_docs_dir = 'docs/docsite/rst/dev_guide/testing' + + url = '%s/%s/' % (testing_docs_url, self.command) + path = os.path.join(testing_docs_dir, self.command) + + if self.test: + url += '%s.html' % self.test + path = os.path.join(path, '%s.rst' % self.test) + + if os.path.exists(path): + return url + + return None + def format_title(self): """ :rtype: str diff --git a/test/runner/test.py b/test/runner/test.py index f1e2534a465..002595f3bbf 100755 --- a/test/runner/test.py +++ b/test/runner/test.py @@ -216,6 +216,10 @@ def parse_args(): action='store_true', help='retry failed test with increased verbosity') + integration.add_argument('--continue-on-error', + action='store_true', + help='continue after failed test') + integration.add_argument('--debug-strategy', action='store_true', help='run test playbooks using the debug strategy') @@ -251,6 +255,10 @@ def parse_args(): action='append', help='network platform/version').completer = complete_network_platform + network_integration.add_argument('--inventory', + metavar='PATH', + help='path to inventory used for tests') + windows_integration = subparsers.add_parser('windows-integration', parents=[integration], help='windows integration tests')