Divide Windows integration tests into 2 groups. (#29029)

* Divide Windows integration tests into 2 groups. * Support `none` for `--changed-all-target`. * Run 2 separate Windows groups on Shippable. * Only run smoketest and minimal for the group1 job.
7 years ago · 8f4c8844c4
parent 02a362e7de
commit 8f4c8844c4
27 changed files with 46 additions and 27 deletions
--- a/shippable.yml
+++ b/shippable.yml
@ -22,7 +22,8 @@ matrix:

    - env: TEST=rhel/7.4

-    - env: TEST=windows
+    - env: TEST=windows/1
+    - env: TEST=windows/2

    - env: TEST=network

--- a/test/integration/targets/connection_winrm/aliases
+++ b/test/integration/targets/connection_winrm/aliases
@ -1,2 +1,2 @@
-windows/ci/group3
+windows/ci/group1
 windows/ci/smoketest
--- a/test/integration/targets/win_async_wrapper/aliases
+++ b/test/integration/targets/win_async_wrapper/aliases
@ -1,3 +1,3 @@
 async_status
-windows/ci/group3
+windows/ci/group1
 windows/ci/smoketest
--- a/test/integration/targets/win_chocolatey/aliases
+++ b/test/integration/targets/win_chocolatey/aliases
@ -1 +1 @@
-windows/ci/group3
+windows/ci/group2
--- a/test/integration/targets/win_command/aliases
+++ b/test/integration/targets/win_command/aliases
@ -1 +1 @@
-windows/ci/group3
+windows/ci/group2
--- a/test/integration/targets/win_dsc/aliases
+++ b/test/integration/targets/win_dsc/aliases
@ -1 +1 @@
-windows/ci/group3
+windows/ci/group2
--- a/test/integration/targets/win_environment/aliases
+++ b/test/integration/targets/win_environment/aliases
@ -1 +1 @@
-windows/ci/group3
+windows/ci/group2
--- a/test/integration/targets/win_eventlog/aliases
+++ b/test/integration/targets/win_eventlog/aliases
@ -1 +1 @@
-windows/ci/group3
+windows/ci/group1
--- a/test/integration/targets/win_eventlog_entry/aliases
+++ b/test/integration/targets/win_eventlog_entry/aliases
@ -1 +1 @@
-windows/ci/group3
+windows/ci/group2
--- a/test/integration/targets/win_feature/aliases
+++ b/test/integration/targets/win_feature/aliases
@ -1 +1 @@
-windows/ci/group3
+windows/ci/group2
--- a/test/integration/targets/win_file/aliases
+++ b/test/integration/targets/win_file/aliases
@ -1,2 +1,2 @@
-windows/ci/group2
+windows/ci/group1
 windows/ci/smoketest
--- a/test/integration/targets/win_firewall_rule/aliases
+++ b/test/integration/targets/win_firewall_rule/aliases
@ -1 +1 @@
-windows/ci/group3
+windows/ci/group2
--- a/test/integration/targets/win_group/aliases
+++ b/test/integration/targets/win_group/aliases
@ -1,2 +1,2 @@
-windows/ci/group2
+windows/ci/group1
 windows/ci/smoketest
--- a/test/integration/targets/win_group_membership/aliases
+++ b/test/integration/targets/win_group_membership/aliases
@ -1 +1 @@
-windows/ci/group3
+windows/ci/group2
--- a/test/integration/targets/win_module_utils/aliases
+++ b/test/integration/targets/win_module_utils/aliases
@ -1 +1,2 @@
+windows/ci/group1
 windows/ci/smoketest
--- a/test/integration/targets/win_path/aliases
+++ b/test/integration/targets/win_path/aliases
@ -1,2 +1,2 @@
-windows/ci/group2
+windows/ci/group1
 windows/ci/smoketest
--- a/test/integration/targets/win_rabbitmq_plugin/aliases
+++ b/test/integration/targets/win_rabbitmq_plugin/aliases
@ -1 +1 @@
-windows/ci/group3
+windows/ci/group2
--- a/test/integration/targets/win_region/aliases
+++ b/test/integration/targets/win_region/aliases
@ -1 +1 @@
-windows/ci/group3
+windows/ci/group2
--- a/test/integration/targets/win_say/aliases
+++ b/test/integration/targets/win_say/aliases
@ -1 +1 @@
-windows/ci/group3
+windows/ci/group2
--- a/test/integration/targets/win_scheduled_task/aliases
+++ b/test/integration/targets/win_scheduled_task/aliases
@ -1 +1 @@
-windows/ci/group3
+windows/ci/group1
--- a/test/integration/targets/win_service/aliases
+++ b/test/integration/targets/win_service/aliases
@ -1,2 +1,2 @@
-windows/ci/group3
+windows/ci/group1
 windows/ci/smoketest
--- a/test/integration/targets/win_shell/aliases
+++ b/test/integration/targets/win_shell/aliases
@ -1,2 +1,2 @@
-windows/ci/group3
+windows/ci/group1
 windows/ci/smoketest
--- a/test/integration/targets/win_template/aliases
+++ b/test/integration/targets/win_template/aliases
@ -1,2 +1,2 @@
-windows/ci/group2
+windows/ci/group1
 windows/ci/smoketest
--- a/test/integration/targets/win_timezone/aliases
+++ b/test/integration/targets/win_timezone/aliases
@ -1 +1 @@
-windows/ci/group3
+windows/ci/group1
--- a/test/integration/targets/win_user/aliases
+++ b/test/integration/targets/win_user/aliases
@ -1 +1 @@
-windows/ci/group3
+windows/ci/group2
--- a/test/runner/lib/classification.py
+++ b/test/runner/lib/classification.py
@ -98,6 +98,8 @@ def categorize_changes(args, paths, verbose_command=None):
            commands[command].add(target)

    for command in commands:
+        commands[command].discard('none')
+
        if any(t == 'all' for t in commands[command]):
            commands[command] = set(['all'])

--- a/test/utils/shippable/windows.sh
+++ b/test/utils/shippable/windows.sh
@ -2,6 +2,11 @@

 set -o pipefail

+declare -a args
+IFS='/:' read -ra args <<< "${TEST}"
+
+target="windows/ci/group${args[1]}/"
+
 # python versions to test in order
 # python 2.7 runs full tests while other versions run minimal tests
 python_versions=(
@ -12,7 +17,7 @@ python_versions=(
 )

 # shellcheck disable=SC2086
-ansible-test windows-integration --explain ${CHANGED:+"$CHANGED"} 2>&1 | { grep ' windows-integration: .* (targeted)$' || true; } > /tmp/windows.txt
+ansible-test windows-integration "${target}" --explain ${CHANGED:+"$CHANGED"} 2>&1 | { grep ' windows-integration: .* (targeted)$' || true; } > /tmp/windows.txt

 if [ -s /tmp/windows.txt ]; then
    echo "Detected changes requiring integration tests specific to Windows:"
@ -48,13 +53,23 @@ for version in "${python_versions[@]}"; do
        if [ "${CHANGED}" ]; then
            # with change detection enabled run tests for anything changed
            # use the smoketest tests for any change that triggers all tests
-            ci="windows/ci/"
-            changed_all_target="windows/ci/smoketest/"
+            ci="${target}"
+            if [ "${target}" == "windows/ci/group1/" ]; then
+                # only run smoketest tests for group1
+                changed_all_target="windows/ci/smoketest/"
+            else
+                # smoketest tests already covered by group1
+                changed_all_target="none"
+            fi
        else
+            # only run smoketest tests for group1
+            if [ "${target}" != "windows/ci/group1/" ]; then continue; fi
            # without change detection enabled run only smoketest tests
            ci="windows/ci/smoketest/"
        fi
    else
+        # only run minimal tests for group1
+        if [ "${target}" != "windows/ci/group1/" ]; then continue; fi
        # minimal tests for other python versions
        ci="windows/ci/minimal/"
    fi