From 8f4c8844c45990d7ab6bde116ab8f1884c629eda Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Tue, 5 Sep 2017 15:32:26 -0700 Subject: [PATCH] Divide Windows integration tests into 2 groups. (#29029) * Divide Windows integration tests into 2 groups. * Support `none` for `--changed-all-target`. * Run 2 separate Windows groups on Shippable. * Only run smoketest and minimal for the group1 job. --- shippable.yml | 3 ++- .../targets/connection_winrm/aliases | 2 +- .../targets/win_async_wrapper/aliases | 2 +- .../targets/win_chocolatey/aliases | 2 +- test/integration/targets/win_command/aliases | 2 +- test/integration/targets/win_dsc/aliases | 2 +- .../targets/win_environment/aliases | 2 +- test/integration/targets/win_eventlog/aliases | 2 +- .../targets/win_eventlog_entry/aliases | 2 +- test/integration/targets/win_feature/aliases | 2 +- test/integration/targets/win_file/aliases | 2 +- .../targets/win_firewall_rule/aliases | 2 +- test/integration/targets/win_group/aliases | 2 +- .../targets/win_group_membership/aliases | 2 +- .../targets/win_module_utils/aliases | 1 + test/integration/targets/win_path/aliases | 2 +- .../targets/win_rabbitmq_plugin/aliases | 2 +- test/integration/targets/win_region/aliases | 2 +- test/integration/targets/win_say/aliases | 2 +- .../targets/win_scheduled_task/aliases | 2 +- test/integration/targets/win_service/aliases | 2 +- test/integration/targets/win_shell/aliases | 2 +- test/integration/targets/win_template/aliases | 2 +- test/integration/targets/win_timezone/aliases | 2 +- test/integration/targets/win_user/aliases | 2 +- test/runner/lib/classification.py | 2 ++ test/utils/shippable/windows.sh | 21 ++++++++++++++++--- 27 files changed, 46 insertions(+), 27 deletions(-) diff --git a/shippable.yml b/shippable.yml index ad748a0820e..80b83c3e197 100644 --- a/shippable.yml +++ b/shippable.yml @@ -22,7 +22,8 @@ matrix: - env: TEST=rhel/7.4 - - env: TEST=windows + - env: TEST=windows/1 + - env: TEST=windows/2 - env: TEST=network diff --git a/test/integration/targets/connection_winrm/aliases b/test/integration/targets/connection_winrm/aliases index 8ad5ddbe289..0589efba855 100644 --- a/test/integration/targets/connection_winrm/aliases +++ b/test/integration/targets/connection_winrm/aliases @@ -1,2 +1,2 @@ -windows/ci/group3 +windows/ci/group1 windows/ci/smoketest diff --git a/test/integration/targets/win_async_wrapper/aliases b/test/integration/targets/win_async_wrapper/aliases index 7b014a9025c..33acd5ebc98 100644 --- a/test/integration/targets/win_async_wrapper/aliases +++ b/test/integration/targets/win_async_wrapper/aliases @@ -1,3 +1,3 @@ async_status -windows/ci/group3 +windows/ci/group1 windows/ci/smoketest diff --git a/test/integration/targets/win_chocolatey/aliases b/test/integration/targets/win_chocolatey/aliases index c6d61981670..ee0ed5974e9 100644 --- a/test/integration/targets/win_chocolatey/aliases +++ b/test/integration/targets/win_chocolatey/aliases @@ -1 +1 @@ -windows/ci/group3 +windows/ci/group2 diff --git a/test/integration/targets/win_command/aliases b/test/integration/targets/win_command/aliases index c6d61981670..ee0ed5974e9 100644 --- a/test/integration/targets/win_command/aliases +++ b/test/integration/targets/win_command/aliases @@ -1 +1 @@ -windows/ci/group3 +windows/ci/group2 diff --git a/test/integration/targets/win_dsc/aliases b/test/integration/targets/win_dsc/aliases index c6d61981670..ee0ed5974e9 100644 --- a/test/integration/targets/win_dsc/aliases +++ b/test/integration/targets/win_dsc/aliases @@ -1 +1 @@ -windows/ci/group3 +windows/ci/group2 diff --git a/test/integration/targets/win_environment/aliases b/test/integration/targets/win_environment/aliases index c6d61981670..ee0ed5974e9 100644 --- a/test/integration/targets/win_environment/aliases +++ b/test/integration/targets/win_environment/aliases @@ -1 +1 @@ -windows/ci/group3 +windows/ci/group2 diff --git a/test/integration/targets/win_eventlog/aliases b/test/integration/targets/win_eventlog/aliases index c6d61981670..10e03fc2bf7 100644 --- a/test/integration/targets/win_eventlog/aliases +++ b/test/integration/targets/win_eventlog/aliases @@ -1 +1 @@ -windows/ci/group3 +windows/ci/group1 diff --git a/test/integration/targets/win_eventlog_entry/aliases b/test/integration/targets/win_eventlog_entry/aliases index c6d61981670..ee0ed5974e9 100644 --- a/test/integration/targets/win_eventlog_entry/aliases +++ b/test/integration/targets/win_eventlog_entry/aliases @@ -1 +1 @@ -windows/ci/group3 +windows/ci/group2 diff --git a/test/integration/targets/win_feature/aliases b/test/integration/targets/win_feature/aliases index c6d61981670..ee0ed5974e9 100644 --- a/test/integration/targets/win_feature/aliases +++ b/test/integration/targets/win_feature/aliases @@ -1 +1 @@ -windows/ci/group3 +windows/ci/group2 diff --git a/test/integration/targets/win_file/aliases b/test/integration/targets/win_file/aliases index c72bb881dbd..0589efba855 100644 --- a/test/integration/targets/win_file/aliases +++ b/test/integration/targets/win_file/aliases @@ -1,2 +1,2 @@ -windows/ci/group2 +windows/ci/group1 windows/ci/smoketest diff --git a/test/integration/targets/win_firewall_rule/aliases b/test/integration/targets/win_firewall_rule/aliases index c6d61981670..ee0ed5974e9 100644 --- a/test/integration/targets/win_firewall_rule/aliases +++ b/test/integration/targets/win_firewall_rule/aliases @@ -1 +1 @@ -windows/ci/group3 +windows/ci/group2 diff --git a/test/integration/targets/win_group/aliases b/test/integration/targets/win_group/aliases index c72bb881dbd..0589efba855 100644 --- a/test/integration/targets/win_group/aliases +++ b/test/integration/targets/win_group/aliases @@ -1,2 +1,2 @@ -windows/ci/group2 +windows/ci/group1 windows/ci/smoketest diff --git a/test/integration/targets/win_group_membership/aliases b/test/integration/targets/win_group_membership/aliases index c6d61981670..ee0ed5974e9 100644 --- a/test/integration/targets/win_group_membership/aliases +++ b/test/integration/targets/win_group_membership/aliases @@ -1 +1 @@ -windows/ci/group3 +windows/ci/group2 diff --git a/test/integration/targets/win_module_utils/aliases b/test/integration/targets/win_module_utils/aliases index 9b589423c21..0589efba855 100644 --- a/test/integration/targets/win_module_utils/aliases +++ b/test/integration/targets/win_module_utils/aliases @@ -1 +1,2 @@ +windows/ci/group1 windows/ci/smoketest diff --git a/test/integration/targets/win_path/aliases b/test/integration/targets/win_path/aliases index c72bb881dbd..0589efba855 100644 --- a/test/integration/targets/win_path/aliases +++ b/test/integration/targets/win_path/aliases @@ -1,2 +1,2 @@ -windows/ci/group2 +windows/ci/group1 windows/ci/smoketest diff --git a/test/integration/targets/win_rabbitmq_plugin/aliases b/test/integration/targets/win_rabbitmq_plugin/aliases index c6d61981670..ee0ed5974e9 100644 --- a/test/integration/targets/win_rabbitmq_plugin/aliases +++ b/test/integration/targets/win_rabbitmq_plugin/aliases @@ -1 +1 @@ -windows/ci/group3 +windows/ci/group2 diff --git a/test/integration/targets/win_region/aliases b/test/integration/targets/win_region/aliases index c6d61981670..ee0ed5974e9 100644 --- a/test/integration/targets/win_region/aliases +++ b/test/integration/targets/win_region/aliases @@ -1 +1 @@ -windows/ci/group3 +windows/ci/group2 diff --git a/test/integration/targets/win_say/aliases b/test/integration/targets/win_say/aliases index c6d61981670..ee0ed5974e9 100644 --- a/test/integration/targets/win_say/aliases +++ b/test/integration/targets/win_say/aliases @@ -1 +1 @@ -windows/ci/group3 +windows/ci/group2 diff --git a/test/integration/targets/win_scheduled_task/aliases b/test/integration/targets/win_scheduled_task/aliases index c6d61981670..10e03fc2bf7 100644 --- a/test/integration/targets/win_scheduled_task/aliases +++ b/test/integration/targets/win_scheduled_task/aliases @@ -1 +1 @@ -windows/ci/group3 +windows/ci/group1 diff --git a/test/integration/targets/win_service/aliases b/test/integration/targets/win_service/aliases index 8ad5ddbe289..0589efba855 100644 --- a/test/integration/targets/win_service/aliases +++ b/test/integration/targets/win_service/aliases @@ -1,2 +1,2 @@ -windows/ci/group3 +windows/ci/group1 windows/ci/smoketest diff --git a/test/integration/targets/win_shell/aliases b/test/integration/targets/win_shell/aliases index 8ad5ddbe289..0589efba855 100644 --- a/test/integration/targets/win_shell/aliases +++ b/test/integration/targets/win_shell/aliases @@ -1,2 +1,2 @@ -windows/ci/group3 +windows/ci/group1 windows/ci/smoketest diff --git a/test/integration/targets/win_template/aliases b/test/integration/targets/win_template/aliases index c72bb881dbd..0589efba855 100644 --- a/test/integration/targets/win_template/aliases +++ b/test/integration/targets/win_template/aliases @@ -1,2 +1,2 @@ -windows/ci/group2 +windows/ci/group1 windows/ci/smoketest diff --git a/test/integration/targets/win_timezone/aliases b/test/integration/targets/win_timezone/aliases index c6d61981670..10e03fc2bf7 100644 --- a/test/integration/targets/win_timezone/aliases +++ b/test/integration/targets/win_timezone/aliases @@ -1 +1 @@ -windows/ci/group3 +windows/ci/group1 diff --git a/test/integration/targets/win_user/aliases b/test/integration/targets/win_user/aliases index c6d61981670..ee0ed5974e9 100644 --- a/test/integration/targets/win_user/aliases +++ b/test/integration/targets/win_user/aliases @@ -1 +1 @@ -windows/ci/group3 +windows/ci/group2 diff --git a/test/runner/lib/classification.py b/test/runner/lib/classification.py index c111232f4da..42fcfd56e53 100644 --- a/test/runner/lib/classification.py +++ b/test/runner/lib/classification.py @@ -98,6 +98,8 @@ def categorize_changes(args, paths, verbose_command=None): commands[command].add(target) for command in commands: + commands[command].discard('none') + if any(t == 'all' for t in commands[command]): commands[command] = set(['all']) diff --git a/test/utils/shippable/windows.sh b/test/utils/shippable/windows.sh index 3fe673cc7a9..52054c8cd6c 100755 --- a/test/utils/shippable/windows.sh +++ b/test/utils/shippable/windows.sh @@ -2,6 +2,11 @@ set -o pipefail +declare -a args +IFS='/:' read -ra args <<< "${TEST}" + +target="windows/ci/group${args[1]}/" + # python versions to test in order # python 2.7 runs full tests while other versions run minimal tests python_versions=( @@ -12,7 +17,7 @@ python_versions=( ) # shellcheck disable=SC2086 -ansible-test windows-integration --explain ${CHANGED:+"$CHANGED"} 2>&1 | { grep ' windows-integration: .* (targeted)$' || true; } > /tmp/windows.txt +ansible-test windows-integration "${target}" --explain ${CHANGED:+"$CHANGED"} 2>&1 | { grep ' windows-integration: .* (targeted)$' || true; } > /tmp/windows.txt if [ -s /tmp/windows.txt ]; then echo "Detected changes requiring integration tests specific to Windows:" @@ -48,13 +53,23 @@ for version in "${python_versions[@]}"; do if [ "${CHANGED}" ]; then # with change detection enabled run tests for anything changed # use the smoketest tests for any change that triggers all tests - ci="windows/ci/" - changed_all_target="windows/ci/smoketest/" + ci="${target}" + if [ "${target}" == "windows/ci/group1/" ]; then + # only run smoketest tests for group1 + changed_all_target="windows/ci/smoketest/" + else + # smoketest tests already covered by group1 + changed_all_target="none" + fi else + # only run smoketest tests for group1 + if [ "${target}" != "windows/ci/group1/" ]; then continue; fi # without change detection enabled run only smoketest tests ci="windows/ci/smoketest/" fi else + # only run minimal tests for group1 + if [ "${target}" != "windows/ci/group1/" ]; then continue; fi # minimal tests for other python versions ci="windows/ci/minimal/" fi