Improve handling of integration test aliases. (#38698)

* Include change classification data in metadata.
* Add support for disabled tests.
* Add support for unstable tests.
* Add support for unsupported tests.
* Overhaul integration aliases sanity test.
* Update Shippable scripts to handle unstable tests.
* Mark unstable Azure tests.
* Mark unstable Windows tests.
* Mark disabled tests.
pull/38714/head
Matt Clay 7 years ago committed by GitHub
parent 26fa3adeab
commit 8a223009ca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -1,2 +1,4 @@
cloud/azure cloud/azure
destructive destructive
posix/ci/cloud/group2/azure
unstable

@ -1,3 +1,4 @@
cloud/azure cloud/azure
posix/ci/cloud/azure posix/ci/cloud/group2/azure
destructive destructive
disabled

@ -1,3 +1,4 @@
cloud/azure cloud/azure
posix/ci/cloud/azure posix/ci/cloud/group2/azure
destructive destructive
disabled

@ -1,2 +1,4 @@
cloud/azure cloud/azure
posix/ci/cloud/group3/azure
unstable
destructive destructive

@ -1,2 +1,4 @@
cloud/azure cloud/azure
destructive destructive
posix/ci/cloud/group2/azure
unstable

@ -1,2 +1,4 @@
cloud/azure cloud/azure
destructive destructive
posix/ci/cloud/group2/azure
unstable

@ -1,3 +1,5 @@
cloud/azure cloud/azure
posix/ci/cloud/group2/azure
unstable
destructive destructive
azure_rm_securitygroup_facts azure_rm_securitygroup_facts

@ -1,2 +1,4 @@
cloud/azure cloud/azure
destructive destructive
posix/ci/cloud/group2/azure
unstable

@ -1,2 +1,4 @@
cloud/azure cloud/azure
destructive destructive
posix/ci/cloud/group2/azure
unstable

@ -1,2 +1,4 @@
cloud/azure cloud/azure
posix/ci/cloud/group3/azure
unstable
destructive destructive

@ -1,2 +1,4 @@
cloud/aws cloud/aws
posix/ci/cloud/group4/aws
disabled
ec2_ami_facts ec2_ami_facts

@ -1 +1,3 @@
cloud/aws cloud/aws
posix/ci/cloud/group4/aws
disabled

@ -1 +1,3 @@
cloud/aws cloud/aws
posix/ci/cloud/group4/aws
disabled

@ -1 +1,3 @@
destructive destructive
posix/ci/group1
disabled

@ -1,4 +1,6 @@
destructive destructive
posix/ci/group1
disabled
skip/osx skip/osx
skip/freebsd skip/freebsd
skip/rhel skip/rhel

@ -1 +1,3 @@
posix/ci/group1
skip/python3 skip/python3
disabled

@ -1,2 +1,4 @@
cloud/vcenter cloud/vcenter
destructive destructive
posix/ci/cloud/group4/vcenter
disabled

@ -0,0 +1,2 @@
windows/ci/group2
unstable

@ -0,0 +1,2 @@
windows/ci/group3
unstable

@ -1 +1,2 @@
windows/ci/group1
disabled

@ -0,0 +1,2 @@
windows/ci/group3
unstable

@ -0,0 +1,2 @@
windows/ci/group1
unstable

@ -1 +1,3 @@
destructive destructive
posix/ci/group1
disabled

@ -2,6 +2,7 @@
from __future__ import absolute_import, print_function from __future__ import absolute_import, print_function
import collections
import os import os
import re import re
import time import time
@ -33,13 +34,19 @@ from lib.config import (
IntegrationConfig, IntegrationConfig,
) )
from lib.metadata import (
ChangeDescription,
)
FOCUSED_TARGET = '__focused__'
def categorize_changes(args, paths, verbose_command=None): def categorize_changes(args, paths, verbose_command=None):
""" """
:type args: TestConfig :type args: TestConfig
:type paths: list[str] :type paths: list[str]
:type verbose_command: str :type verbose_command: str
:rtype paths: dict[str, list[str]] :rtype: ChangeDescription
""" """
mapper = PathMapper(args) mapper = PathMapper(args)
@ -51,12 +58,20 @@ def categorize_changes(args, paths, verbose_command=None):
'network-integration': set(), 'network-integration': set(),
} }
focused_commands = collections.defaultdict(set)
deleted_paths = set()
original_paths = set()
additional_paths = set() additional_paths = set()
no_integration_paths = set()
for path in paths: for path in paths:
if not os.path.exists(path): if not os.path.exists(path):
deleted_paths.add(path)
continue continue
original_paths.add(path)
dependent_paths = mapper.get_dependent_paths(path) dependent_paths = mapper.get_dependent_paths(path)
if not dependent_paths: if not dependent_paths:
@ -80,18 +95,28 @@ def categorize_changes(args, paths, verbose_command=None):
tests = mapper.classify(path) tests = mapper.classify(path)
if tests is None: if tests is None:
focused_target = False
display.info('%s -> all' % path, verbosity=1) display.info('%s -> all' % path, verbosity=1)
tests = all_tests(args) # not categorized, run all tests tests = all_tests(args) # not categorized, run all tests
display.warning('Path not categorized: %s' % path) display.warning('Path not categorized: %s' % path)
else: else:
focused_target = tests.pop(FOCUSED_TARGET, False) and path in original_paths
tests = dict((key, value) for key, value in tests.items() if value) tests = dict((key, value) for key, value in tests.items() if value)
if focused_target and not any('integration' in command for command in tests):
no_integration_paths.add(path) # path triggers no integration tests
if verbose_command: if verbose_command:
result = '%s: %s' % (verbose_command, tests.get(verbose_command) or 'none') result = '%s: %s' % (verbose_command, tests.get(verbose_command) or 'none')
# identify targeted integration tests (those which only target a single integration command) # identify targeted integration tests (those which only target a single integration command)
if 'integration' in verbose_command and tests.get(verbose_command): if 'integration' in verbose_command and tests.get(verbose_command):
if not any('integration' in command for command in tests if command != verbose_command): if not any('integration' in command for command in tests if command != verbose_command):
if focused_target:
result += ' (focused)'
result += ' (targeted)' result += ' (targeted)'
else: else:
result = '%s' % tests result = '%s' % tests
@ -101,6 +126,9 @@ def categorize_changes(args, paths, verbose_command=None):
for command, target in tests.items(): for command, target in tests.items():
commands[command].add(target) commands[command].add(target)
if focused_target:
focused_commands[command].add(target)
for command in commands: for command in commands:
commands[command].discard('none') commands[command].discard('none')
@ -108,8 +136,21 @@ def categorize_changes(args, paths, verbose_command=None):
commands[command] = set(['all']) commands[command] = set(['all'])
commands = dict((c, sorted(commands[c])) for c in commands if commands[c]) commands = dict((c, sorted(commands[c])) for c in commands if commands[c])
focused_commands = dict((c, sorted(focused_commands[c])) for c in focused_commands)
for command in commands:
if commands[command] == ['all']:
commands[command] = [] # changes require testing all targets, do not filter targets
changes = ChangeDescription()
changes.command = verbose_command
changes.changed_paths = sorted(original_paths)
changes.deleted_paths = sorted(deleted_paths)
changes.regular_command_targets = commands
changes.focused_command_targets = focused_commands
changes.no_integration_paths = sorted(no_integration_paths)
return commands return changes
class PathMapper(object): class PathMapper(object):
@ -278,6 +319,7 @@ class PathMapper(object):
'integration': self.posix_integration_by_module.get(module_name) if ext == '.py' else None, 'integration': self.posix_integration_by_module.get(module_name) if ext == '.py' else None,
'windows-integration': self.windows_integration_by_module.get(module_name) if ext == '.ps1' else None, 'windows-integration': self.windows_integration_by_module.get(module_name) if ext == '.ps1' else None,
'network-integration': self.network_integration_by_module.get(module_name), 'network-integration': self.network_integration_by_module.get(module_name),
FOCUSED_TARGET: True,
} }
return minimal return minimal
@ -459,6 +501,7 @@ class PathMapper(object):
'integration': target.name if 'posix/' in target.aliases else None, 'integration': target.name if 'posix/' in target.aliases else None,
'windows-integration': target.name if 'windows/' in target.aliases else None, 'windows-integration': target.name if 'windows/' in target.aliases else None,
'network-integration': target.name if 'network/' in target.aliases else None, 'network-integration': target.name if 'network/' in target.aliases else None,
FOCUSED_TARGET: True,
} }
if path.startswith('test/integration/'): if path.startswith('test/integration/'):

@ -162,6 +162,10 @@ class IntegrationConfig(TestConfig):
self.start_at_task = args.start_at_task # type: str self.start_at_task = args.start_at_task # type: str
self.allow_destructive = args.allow_destructive # type: bool self.allow_destructive = args.allow_destructive # type: bool
self.allow_root = args.allow_root # type: bool self.allow_root = args.allow_root # type: bool
self.allow_disabled = args.allow_disabled # type: bool
self.allow_unstable = args.allow_unstable # type: bool
self.allow_unstable_changed = args.allow_unstable_changed # type: bool
self.allow_unsupported = args.allow_unsupported # type: bool
self.retry_on_error = args.retry_on_error # type: bool self.retry_on_error = args.retry_on_error # type: bool
self.continue_on_error = args.continue_on_error # type: bool self.continue_on_error = args.continue_on_error # type: bool
self.debug_strategy = args.debug_strategy # type: bool self.debug_strategy = args.debug_strategy # type: bool

@ -88,6 +88,10 @@ from lib.config import (
WindowsIntegrationConfig, WindowsIntegrationConfig,
) )
from lib.metadata import (
ChangeDescription,
)
SUPPORTED_PYTHON_VERSIONS = ( SUPPORTED_PYTHON_VERSIONS = (
'2.6', '2.6',
'2.7', '2.7',
@ -1029,23 +1033,24 @@ def get_changes_filter(args):
""" """
paths = detect_changes(args) paths = detect_changes(args)
if not args.metadata.change_description:
if paths:
changes = categorize_changes(args, paths, args.command)
else:
changes = ChangeDescription()
args.metadata.change_description = changes
if paths is None: if paths is None:
return [] # change detection not enabled, do not filter targets return [] # change detection not enabled, do not filter targets
if not paths: if not paths:
raise NoChangesDetected() raise NoChangesDetected()
commands = categorize_changes(args, paths, args.command) if args.metadata.change_description.targets is None:
targets = commands.get(args.command)
if targets is None:
raise NoTestsForChanges() raise NoTestsForChanges()
if targets == ['all']: return args.metadata.change_description.targets
return [] # changes require testing all targets, do not filter targets
return targets
def detect_changes(args): def detect_changes(args):
@ -1175,6 +1180,49 @@ def get_integration_filter(args, targets):
return get_integration_local_filter(args, targets) return get_integration_local_filter(args, targets)
def common_integration_filter(args, targets, exclude):
"""
:type args: IntegrationConfig
:type targets: tuple[IntegrationTarget]
:type exclude: list[str]
"""
override_disabled = set(target for target in args.include if target.startswith('disabled/'))
if not args.allow_disabled:
skip = 'disabled/'
override = [target.name for target in targets if override_disabled & set(target.aliases)]
skipped = [target.name for target in targets if skip in target.aliases and target.name not in override]
if skipped:
exclude.extend(skipped)
display.warning('Excluding tests marked "%s" which require --allow-disabled or prefixing with "disabled/": %s'
% (skip.rstrip('/'), ', '.join(skipped)))
override_unsupported = set(target for target in args.include if target.startswith('unsupported/'))
if not args.allow_unsupported:
skip = 'unsupported/'
override = [target.name for target in targets if override_unsupported & set(target.aliases)]
skipped = [target.name for target in targets if skip in target.aliases and target.name not in override]
if skipped:
exclude.extend(skipped)
display.warning('Excluding tests marked "%s" which require --allow-unsupported or prefixing with "unsupported/": %s'
% (skip.rstrip('/'), ', '.join(skipped)))
override_unstable = set(target for target in args.include if target.startswith('unstable/'))
if args.allow_unstable_changed:
override_unstable |= set(args.metadata.change_description.focused_targets or [])
if not args.allow_unstable:
skip = 'unstable/'
override = [target.name for target in targets if override_unstable & set(target.aliases)]
skipped = [target.name for target in targets if skip in target.aliases and target.name not in override]
if skipped:
exclude.extend(skipped)
display.warning('Excluding tests marked "%s" which require --allow-unstable or prefixing with "unstable/": %s'
% (skip.rstrip('/'), ', '.join(skipped)))
def get_integration_local_filter(args, targets): def get_integration_local_filter(args, targets):
""" """
:type args: IntegrationConfig :type args: IntegrationConfig
@ -1183,6 +1231,8 @@ def get_integration_local_filter(args, targets):
""" """
exclude = [] exclude = []
common_integration_filter(args, targets, exclude)
if not args.allow_root and os.getuid() != 0: if not args.allow_root and os.getuid() != 0:
skip = 'needs/root/' skip = 'needs/root/'
skipped = [target.name for target in targets if skip in target.aliases] skipped = [target.name for target in targets if skip in target.aliases]
@ -1225,6 +1275,8 @@ def get_integration_docker_filter(args, targets):
""" """
exclude = [] exclude = []
common_integration_filter(args, targets, exclude)
if not args.docker_privileged: if not args.docker_privileged:
skip = 'needs/privileged/' skip = 'needs/privileged/'
skipped = [target.name for target in targets if skip in target.aliases] skipped = [target.name for target in targets if skip in target.aliases]
@ -1271,6 +1323,8 @@ def get_integration_remote_filter(args, targets):
exclude = [] exclude = []
common_integration_filter(args, targets, exclude)
skip = 'skip/%s/' % platform skip = 'skip/%s/' % platform
skipped = [target.name for target in targets if skip in target.aliases] skipped = [target.name for target in targets if skip in target.aliases]
if skipped: if skipped:

@ -21,6 +21,7 @@ class Metadata(object):
self.changes = {} # type: dict [str, tuple[tuple[int, int]] self.changes = {} # type: dict [str, tuple[tuple[int, int]]
self.cloud_config = None # type: dict [str, str] self.cloud_config = None # type: dict [str, str]
self.instance_config = None # type: list[dict[str, str]] self.instance_config = None # type: list[dict[str, str]]
self.change_description = None # type: ChangeDescription
if is_shippable(): if is_shippable():
self.ci_provider = 'shippable' self.ci_provider = 'shippable'
@ -57,6 +58,7 @@ class Metadata(object):
cloud_config=self.cloud_config, cloud_config=self.cloud_config,
instance_config=self.instance_config, instance_config=self.instance_config,
ci_provider=self.ci_provider, ci_provider=self.ci_provider,
change_description=self.change_description.to_dict(),
) )
def to_file(self, path): def to_file(self, path):
@ -92,5 +94,60 @@ class Metadata(object):
metadata.cloud_config = data['cloud_config'] metadata.cloud_config = data['cloud_config']
metadata.instance_config = data['instance_config'] metadata.instance_config = data['instance_config']
metadata.ci_provider = data['ci_provider'] metadata.ci_provider = data['ci_provider']
metadata.change_description = ChangeDescription.from_dict(data['change_description'])
return metadata return metadata
class ChangeDescription(object):
"""Description of changes."""
def __init__(self):
self.command = '' # type: str
self.changed_paths = [] # type: list[str]
self.deleted_paths = [] # type: list[str]
self.regular_command_targets = {} # type: dict[str, list[str]]
self.focused_command_targets = {} # type: dict[str, list[str]]
self.no_integration_paths = [] # type: list[str]
@property
def targets(self):
"""
:rtype: list[str] | None
"""
return self.regular_command_targets.get(self.command)
@property
def focused_targets(self):
"""
:rtype: list[str] | None
"""
return self.focused_command_targets.get(self.command)
def to_dict(self):
"""
:rtype: dict[str, any]
"""
return dict(
command=self.command,
changed_paths=self.changed_paths,
deleted_paths=self.deleted_paths,
regular_command_targets=self.regular_command_targets,
focused_command_targets=self.focused_command_targets,
no_integration_paths=self.no_integration_paths,
)
@staticmethod
def from_dict(data):
"""
:param data: dict[str, any]
:rtype: ChangeDescription
"""
changes = ChangeDescription()
changes.command = data['command']
changes.changed_paths = data['changed_paths']
changes.deleted_paths = data['deleted_paths']
changes.regular_command_targets = data['regular_command_targets']
changes.focused_command_targets = data['focused_command_targets']
changes.no_integration_paths = data['no_integration_paths']
return changes

@ -0,0 +1,247 @@
"""Sanity test to check integration test aliases."""
from __future__ import absolute_import, print_function
import json
import textwrap
from lib.sanity import (
SanitySingleVersion,
SanityMessage,
SanityFailure,
SanitySuccess,
SanityTargets,
)
from lib.config import (
SanityConfig,
)
from lib.target import (
filter_targets,
walk_posix_integration_targets,
walk_windows_integration_targets,
walk_integration_targets,
walk_module_targets,
)
from lib.cloud import (
get_cloud_platforms,
)
class IntegrationAliasesTest(SanitySingleVersion):
"""Sanity test to evaluate integration test aliases."""
DISABLED = 'disabled/'
UNSTABLE = 'unstable/'
UNSUPPORTED = 'unsupported/'
EXPLAIN_URL = 'https://docs.ansible.com/ansible/devel/dev_guide/testing/sanity/integration-aliases.html'
TEMPLATE_DISABLED = """
The following integration tests are **disabled** [[explain]({explain_url}#disabled)]:
{tests}
Consider fixing the integration tests before or alongside changes.
"""
TEMPLATE_UNSTABLE = """
The following integration tests are **unstable** [[explain]({explain_url}#unstable)]:
{tests}
Tests may need to be restarted due to failures unrelated to changes.
"""
TEMPLATE_UNSUPPORTED = """
The following integration tests are **unsupported** [[explain]({explain_url}#unsupported)]:
{tests}
Consider running the tests manually or extending test infrastructure to add support.
"""
TEMPLATE_UNTESTED = """
The following modules have **no integration tests** [[explain]({explain_url}#untested)]:
{tests}
Consider adding integration tests before or alongside changes.
"""
def test(self, args, targets):
"""
:type args: SanityConfig
:type targets: SanityTargets
:rtype: TestResult
"""
if args.explain:
return SanitySuccess(self.name)
results = dict(
comments=[],
labels={},
)
self.check_changes(args, results)
with open('test/results/bot/data-sanity-ci.json', 'w') as results_fd:
json.dump(results, results_fd, sort_keys=True, indent=4)
messages = []
messages += self.check_posix_targets(args)
messages += self.check_windows_targets()
if messages:
return SanityFailure(self.name, messages=messages)
return SanitySuccess(self.name)
def check_posix_targets(self, args):
"""
:type args: SanityConfig
:rtype: list[SanityMessage]
"""
posix_targets = tuple(walk_posix_integration_targets())
clouds = get_cloud_platforms(args, posix_targets)
cloud_targets = ['cloud/%s/' % cloud for cloud in clouds]
all_cloud_targets = tuple(filter_targets(posix_targets, ['cloud/'], include=True, directories=False, errors=False))
invalid_cloud_targets = tuple(filter_targets(all_cloud_targets, cloud_targets, include=False, directories=False, errors=False))
messages = []
for target in invalid_cloud_targets:
for alias in target.aliases:
if alias.startswith('cloud/') and alias != 'cloud/':
if any(alias.startswith(cloud_target) for cloud_target in cloud_targets):
continue
messages.append(SanityMessage('invalid alias `%s`' % alias, '%s/aliases' % target.path))
messages += self.check_ci_group(
targets=tuple(filter_targets(posix_targets, ['cloud/'], include=False, directories=False, errors=False)),
find='posix/ci/group[1-3]/',
)
for cloud in clouds:
messages += self.check_ci_group(
targets=tuple(filter_targets(posix_targets, ['cloud/%s/' % cloud], include=True, directories=False, errors=False)),
find='posix/ci/cloud/group[1-5]/%s/' % cloud,
)
return messages
def check_windows_targets(self):
"""
:rtype: list[SanityMessage]
"""
windows_targets = tuple(walk_windows_integration_targets())
messages = []
messages += self.check_ci_group(
targets=windows_targets,
find='windows/ci/group[1-3]/',
)
return messages
def check_ci_group(self, targets, find):
"""
:type targets: tuple[CompletionTarget]
:type find: str
:rtype: list[SanityMessage]
"""
all_paths = set(t.path for t in targets)
supported_paths = set(t.path for t in filter_targets(targets, [find], include=True, directories=False, errors=False))
unsupported_paths = set(t.path for t in filter_targets(targets, [self.UNSUPPORTED], include=True, directories=False, errors=False))
unassigned_paths = all_paths - supported_paths - unsupported_paths
conflicting_paths = supported_paths & unsupported_paths
unassigned_message = 'missing alias `%s` or `%s`' % (find.strip('/'), self.UNSUPPORTED.strip('/'))
conflicting_message = 'conflicting alias `%s` and `%s`' % (find.strip('/'), self.UNSUPPORTED.strip('/'))
messages = []
for path in unassigned_paths:
messages.append(SanityMessage(unassigned_message, '%s/aliases' % path))
for path in conflicting_paths:
messages.append(SanityMessage(conflicting_message, '%s/aliases' % path))
return messages
def check_changes(self, args, results):
"""
:type args: SanityConfig
:type results: dict[str, any]
"""
integration_targets = list(walk_integration_targets())
module_targets = list(walk_module_targets())
integration_targets_by_name = dict((t.name, t) for t in integration_targets)
module_names_by_path = dict((t.path, t.module) for t in module_targets)
disabled_targets = []
unstable_targets = []
unsupported_targets = []
for command in [command for command in args.metadata.change_description.focused_command_targets if 'integration' in command]:
for target in args.metadata.change_description.focused_command_targets[command]:
if self.DISABLED in integration_targets_by_name[target].aliases:
disabled_targets.append(target)
elif self.UNSTABLE in integration_targets_by_name[target].aliases:
unstable_targets.append(target)
elif self.UNSUPPORTED in integration_targets_by_name[target].aliases:
unsupported_targets.append(target)
untested_modules = []
for path in args.metadata.change_description.no_integration_paths:
module = module_names_by_path.get(path)
if module:
untested_modules.append(module)
comments = [
self.format_comment(self.TEMPLATE_DISABLED, disabled_targets),
self.format_comment(self.TEMPLATE_UNSTABLE, unstable_targets),
self.format_comment(self.TEMPLATE_UNSUPPORTED, unsupported_targets),
self.format_comment(self.TEMPLATE_UNTESTED, untested_modules),
]
comments = [comment for comment in comments if comment]
labels = dict(
needs_tests=bool(untested_modules),
disabled_tests=bool(disabled_targets),
unstable_tests=bool(unstable_targets),
unsupported_tests=bool(unsupported_targets),
)
results['comments'] += comments
results['labels'].update(labels)
def format_comment(self, template, targets):
"""
:type template: str
:type targets: list[str]
:rtype: str | None
"""
if not targets:
return None
tests = '\n'.join('- %s' % target for target in targets)
data = dict(
explain_url=self.EXPLAIN_URL,
tests=tests,
)
message = textwrap.dedent(template).strip().format(**data)
return message

@ -229,6 +229,22 @@ def parse_args():
action='store_true', action='store_true',
help='allow tests requiring root when not root') help='allow tests requiring root when not root')
integration.add_argument('--allow-disabled',
action='store_true',
help='allow tests which have been marked as disabled')
integration.add_argument('--allow-unstable',
action='store_true',
help='allow tests which have been marked as unstable')
integration.add_argument('--allow-unstable-changed',
action='store_true',
help='allow tests which have been marked as unstable when focused changes are detected')
integration.add_argument('--allow-unsupported',
action='store_true',
help='allow tests which have been marked as unsupported')
integration.add_argument('--retry-on-error', integration.add_argument('--retry-on-error',
action='store_true', action='store_true',
help='retry failed test with increased verbosity') help='retry failed test with increased verbosity')

@ -1,43 +0,0 @@
#!/usr/bin/env python
import os
import textwrap
def main():
targets_dir = 'test/integration/targets'
with open('test/integration/target-prefixes.network', 'r') as prefixes_fd:
network_prefixes = prefixes_fd.read().splitlines()
for target in sorted(os.listdir(targets_dir)):
target_dir = os.path.join(targets_dir, target)
aliases_path = os.path.join(target_dir, 'aliases')
files = sorted(os.listdir(target_dir))
# aliases already defined
if os.path.exists(aliases_path):
continue
# don't require aliases for support directories
if any(os.path.splitext(f)[0] == 'test' and os.access(os.path.join(target_dir, f), os.X_OK) for f in files):
continue
# don't require aliases for setup_ directories
if target.startswith('setup_'):
continue
# don't require aliases for prepare_ directories
if target.startswith('prepare_'):
continue
# TODO: remove this exclusion once the `ansible-test network-integration` command is working properly
# don't require aliases for network modules
if any(target.startswith('%s_' % prefix) for prefix in network_prefixes):
continue
print('%s: missing integration test `aliases` file' % aliases_path)
if __name__ == '__main__':
main()

@ -12,6 +12,6 @@ target="posix/ci/cloud/group${args[3]}/"
stage="${S:-prod}" stage="${S:-prod}"
# shellcheck disable=SC2086 # shellcheck disable=SC2086
ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
--remote-terminate always --remote-stage "${stage}" \ --remote-terminate always --remote-stage "${stage}" \
--docker "${image}" --python "${python}" --changed-all-target "${target}smoketest/" --docker "${image}" --python "${python}" --changed-all-target "${target}smoketest/"

@ -18,6 +18,6 @@ stage="${S:-prod}"
provider="${P:-default}" provider="${P:-default}"
# shellcheck disable=SC2086 # shellcheck disable=SC2086
ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
--exclude "posix/ci/cloud/" \ --exclude "posix/ci/cloud/" \
--remote "${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}" --remote "${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}"

@ -14,6 +14,6 @@ else
fi fi
# shellcheck disable=SC2086 # shellcheck disable=SC2086
ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
--exclude "posix/ci/cloud/" \ --exclude "posix/ci/cloud/" \
--docker "${image}" --docker "${image}"

@ -3,7 +3,8 @@
set -o pipefail set -o pipefail
# shellcheck disable=SC2086 # shellcheck disable=SC2086
ansible-test network-integration --explain ${CHANGED:+"$CHANGED"} 2>&1 | { grep ' network-integration: .* (targeted)$' || true; } > /tmp/network.txt ansible-test network-integration --explain ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} 2>&1 \
| { grep ' network-integration: .* (targeted)$' || true; } > /tmp/network.txt
if [ "${COVERAGE}" ]; then if [ "${COVERAGE}" ]; then
# when on-demand coverage is enabled, force tests to run for all network platforms # when on-demand coverage is enabled, force tests to run for all network platforms
@ -49,7 +50,8 @@ for version in "${python_versions[@]}"; do
fi fi
# shellcheck disable=SC2086 # shellcheck disable=SC2086
ansible-test network-integration --color -v --retry-on-error "${target}" --docker default --python "${version}" \ ansible-test network-integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} "${platforms[@]}" \ "${platforms[@]}" \
--docker default --python "${version}" \
--remote-terminate "${terminate}" --remote-stage "${stage}" --remote-provider "${provider}" --remote-terminate "${terminate}" --remote-stage "${stage}" --remote-provider "${provider}"
done done

@ -18,6 +18,6 @@ stage="${S:-prod}"
provider="${P:-default}" provider="${P:-default}"
# shellcheck disable=SC2086 # shellcheck disable=SC2086
ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
--exclude "posix/ci/cloud/" \ --exclude "posix/ci/cloud/" \
--remote "${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}" --remote "${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}"

@ -18,6 +18,6 @@ stage="${S:-prod}"
provider="${P:-default}" provider="${P:-default}"
# shellcheck disable=SC2086 # shellcheck disable=SC2086
ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
--exclude "posix/ci/cloud/" \ --exclude "posix/ci/cloud/" \
--remote "${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}" --remote "${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}"

@ -53,6 +53,14 @@ else
export CHANGED="--changed" export CHANGED="--changed"
fi fi
if [ "${IS_PULL_REQUEST:-}" == "true" ]; then
# run unstable tests which are targeted by focused changes on PRs
export UNSTABLE="--allow-unstable-changed"
else
# do not run unstable tests outside PRs
export UNSTABLE=""
fi
# remove empty core/extras module directories from PRs created prior to the repo-merge # remove empty core/extras module directories from PRs created prior to the repo-merge
find lib/ansible/modules -type d -empty -print -delete find lib/ansible/modules -type d -empty -print -delete

@ -24,7 +24,8 @@ python_versions=(
single_version=2012-R2 single_version=2012-R2
# shellcheck disable=SC2086 # shellcheck disable=SC2086
ansible-test windows-integration "${target}" --explain ${CHANGED:+"$CHANGED"} 2>&1 | { grep ' windows-integration: .* (targeted)$' || true; } > /tmp/windows.txt ansible-test windows-integration "${target}" --explain ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} 2>&1 \
| { grep ' windows-integration: .* (targeted)$' || true; } > /tmp/windows.txt
if [ -s /tmp/windows.txt ] || [ "${CHANGED:+$CHANGED}" == "" ]; then if [ -s /tmp/windows.txt ] || [ "${CHANGED:+$CHANGED}" == "" ]; then
echo "Detected changes requiring integration tests specific to Windows:" echo "Detected changes requiring integration tests specific to Windows:"
@ -84,7 +85,8 @@ for version in "${python_versions[@]}"; do
fi fi
# shellcheck disable=SC2086 # shellcheck disable=SC2086
ansible-test windows-integration --color -v --retry-on-error "${ci}" --docker default --python "${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \ ansible-test windows-integration --color -v --retry-on-error "${ci}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
"${platforms[@]}" --changed-all-target "${changed_all_target}" \ "${platforms[@]}" --changed-all-target "${changed_all_target}" \
--docker default --python "${version}" \
--remote-terminate "${terminate}" --remote-stage "${stage}" --remote-provider "${provider}" --remote-terminate "${terminate}" --remote-stage "${stage}" --remote-provider "${provider}"
done done

Loading…
Cancel
Save