Use subset of Windows tests for "all" tests. (#26830)

shippable.yml

@@ -22,9 +22,7 @@ matrix:
 
     - env: TEST=rhel/7.3
 
-    - env: TEST=windows/1
-    - env: TEST=windows/2
-    - env: TEST=windows/3
+    - env: TEST=windows
 
     - env: TEST=network
 

test/runner/lib/classification.py

@@ -23,14 +23,20 @@ from lib.import_analysis import (
     get_python_module_utils_imports,
 )
 
+from lib.config import (
+    TestConfig,
+    IntegrationConfig,
+)
+
 
-def categorize_changes(paths, verbose_command=None):
+def categorize_changes(args, paths, verbose_command=None):
     """
+    :type args: TestConfig
     :type paths: list[str]
     :type verbose_command: str
     :rtype paths: dict[str, list[str]]
     """
-    mapper = PathMapper()
+    mapper = PathMapper(args)
 
     commands = {
         'sanity': set(),
@@ -71,7 +77,7 @@ def categorize_changes(paths, verbose_command=None):
 
         if tests is None:
             display.info('%s -> all' % path, verbosity=1)
-            tests = all_tests()  # not categorized, run all tests
+            tests = all_tests(args)  # not categorized, run all tests
             display.warning('Path not categorized: %s' % path)
         else:
             tests = dict((key, value) for key, value in tests.items() if value)
@@ -102,7 +108,13 @@ def categorize_changes(paths, verbose_command=None):
 
 class PathMapper(object):
     """Map file paths to test commands and targets."""
-    def __init__(self):
+    def __init__(self, args):
+        """
+        :type args: TestConfig
+        """
+        self.args = args
+        self.integration_all_target = get_integration_all_target(self.args)
+
         self.integration_targets = list(walk_integration_targets())
         self.module_targets = list(walk_module_targets())
         self.compile_targets = list(walk_compile_targets())
@@ -250,7 +262,7 @@ class PathMapper(object):
         if path.startswith('lib/ansible/module_utils/'):
             if ext == '.ps1':
                 return {
-                    'windows-integration': 'all',
+                    'windows-integration': self.integration_all_target,
                 }
 
             if ext == '.py':
@@ -259,9 +271,9 @@ class PathMapper(object):
         if path.startswith('lib/ansible/plugins/connection/'):
             if name == '__init__':
                 return {
-                    'integration': 'all',
-                    'windows-integration': 'all',
-                    'network-integration': 'all',
+                    'integration': self.integration_all_target,
+                    'windows-integration': self.integration_all_target,
+                    'network-integration': self.integration_all_target,
                     'units': 'test/units/plugins/connection/',
                 }
 
@@ -279,20 +291,20 @@ class PathMapper(object):
 
             if name == 'winrm':
                 return {
-                    'windows-integration': 'all',
+                    'windows-integration': self.integration_all_target,
                     'units': units_path,
                 }
 
             if name == 'local':
                 return {
-                    'integration': 'all',
-                    'network-integration': 'all',
+                    'integration': self.integration_all_target,
+                    'network-integration': self.integration_all_target,
                     'units': units_path,
                 }
 
             if name == 'network_cli':
                 return {
-                    'network-integration': 'all',
+                    'network-integration': self.integration_all_target,
                     'units': units_path,
                 }
 
@@ -321,7 +333,7 @@ class PathMapper(object):
                     }
 
                 return {
-                    'network-integration': 'all',
+                    'network-integration': self.integration_all_target,
                     'units': 'all',
                 }
 
@@ -331,7 +343,7 @@ class PathMapper(object):
             }
 
         if path.startswith('lib/ansible/'):
-            return all_tests()  # broad impact, run all tests
+            return all_tests(self.args)  # broad impact, run all tests
 
         if path.startswith('packaging/'):
             return minimal
@@ -358,9 +370,9 @@ class PathMapper(object):
                     return minimal  # already expanded using get_dependent_paths
 
                 return {
-                    'integration': 'all',
-                    'windows-integration': 'all',
-                    'network-integration': 'all',
+                    'integration': self.integration_all_target,
+                    'windows-integration': self.integration_all_target,
+                    'network-integration': self.integration_all_target,
                 }
 
             return {
@@ -377,9 +389,9 @@ class PathMapper(object):
                 return minimal  # network integration test playbook not used by ansible-test
 
             return {
-                'integration': 'all',
-                'windows-integration': 'all',
-                'network-integration': 'all',
+                'integration': self.integration_all_target,
+                'windows-integration': self.integration_all_target,
+                'network-integration': self.integration_all_target,
             }
 
         if path.startswith('test/sanity/'):
@@ -413,13 +425,13 @@ class PathMapper(object):
                     'integration': cloud_target,
                 }
 
-            return all_tests()  # test infrastructure, run all tests
+            return all_tests(self.args)  # test infrastructure, run all tests
 
         if path.startswith('test/runner/'):
-            return all_tests()  # test infrastructure, run all tests
+            return all_tests(self.args)  # test infrastructure, run all tests
 
         if path.startswith('test/utils/shippable/'):
-            return all_tests()  # test infrastructure, run all tests
+            return all_tests(self.args)  # test infrastructure, run all tests
 
         if path.startswith('test/utils/'):
             return minimal
@@ -448,7 +460,7 @@ class PathMapper(object):
                     'shippable.yml',
                     '.coveragerc',
             ):
-                return all_tests()  # test infrastructure, run all tests
+                return all_tests(self.args)  # test infrastructure, run all tests
 
             if path == '.yamllint':
                 return {
@@ -461,15 +473,29 @@ class PathMapper(object):
         return None  # unknown, will result in fall-back to run all tests
 
 
-def all_tests():
+def all_tests(args):
     """
+    :type args: TestConfig
     :rtype: dict[str, str]
     """
+    integration_all_target = get_integration_all_target(args)
+
     return {
         'sanity': 'all',
         'compile': 'all',
         'units': 'all',
-        'integration': 'all',
-        'windows-integration': 'all',
-        'network-integration': 'all',
+        'integration': integration_all_target,
+        'windows-integration': integration_all_target,
+        'network-integration': integration_all_target,
     }
+
+
+def get_integration_all_target(args):
+    """
+    :type args: TestConfig
+    :rtype: str
+    """
+    if isinstance(args, IntegrationConfig):
+        return args.changed_all_target
+
+    return 'all'

test/runner/lib/config.py

@@ -141,6 +141,7 @@ class IntegrationConfig(TestConfig):
         self.allow_destructive = args.allow_destructive if 'allow_destructive' in args else False  # type: bool
         self.retry_on_error = args.retry_on_error  # type: bool
         self.debug_strategy = args.debug_strategy  # type: bool
+        self.changed_all_target = args.changed_all_target  # type: str
         self.tags = args.tags
         self.skip_tags = args.skip_tags
         self.diff = args.diff

test/runner/lib/executor.py

@@ -975,7 +975,7 @@ def get_changes_filter(args):
     if not paths:
         raise NoChangesDetected()
 
-    commands = categorize_changes(paths, args.command)
+    commands = categorize_changes(args, paths, args.command)
 
     targets = commands.get(args.command)
 

test/runner/test.py

@@ -220,6 +220,11 @@ def parse_args():
                              action='store_true',
                              help='run test playbooks using the debug strategy')
 
+    integration.add_argument('--changed-all-target',
+                             metavar='TARGET',
+                             default='all',
+                             help='target to run when all tests are needed')
+
     subparsers = parser.add_subparsers(metavar='COMMAND')
     subparsers.required = True  # work-around for python 3 bug which makes subparsers optional
 

test/utils/shippable/windows.sh

@@ -2,11 +2,6 @@
 
 set -o pipefail
 
-declare -a args
-IFS='/:' read -ra args <<< "${TEST}"
-
-job="${args[1]}"
-
 # python versions to test in order
 # python 2.7 runs full tests while other versions run minimal tests
 python_versions=(
@@ -23,15 +18,8 @@ if [ -s /tmp/windows.txt ]; then
     echo "Detected changes requiring integration tests specific to Windows:"
     cat /tmp/windows.txt
 
-    if [ "${job}" != "1" ]; then
-        echo "Nothing to do, all Windows tests will run under TEST=windows/1 instead."
-        exit 0
-    fi
-
     echo "Running Windows integration tests for multiple versions concurrently."
 
-    target="windows/ci/"
-
     platforms=(
         --windows 2008-SP2
         --windows 2008-R2_SP1
@@ -42,8 +30,6 @@ else
     echo "No changes requiring integration tests specific to Windows were detected."
     echo "Running Windows integration tests for a single version only."
 
-    target="windows/ci/group${job}/"
-
     platforms=(
         --windows 2012-R2_RTM
     )
@@ -55,17 +41,25 @@ for version in "${python_versions[@]}"; do
     # clean up between test runs until we switch from --tox to --docker
     rm -rf ~/.ansible/{cp,pc,tmp}/
 
-    if [ "${job}" == "1" ] || [ "${version}" == "2.7" ]; then
-        if [ "${version}" == "2.7" ]; then
-            # full tests for python 2.7
-            ci="${target}"
+    changed_all_target="all"
+
+    if [ "${version}" == "2.7" ]; then
+        # smoketest tests for python 2.7
+        if [ "${CHANGED}" ]; then
+            # with change detection enabled run tests for anything changed
+            # use the smoketest tests for any change that triggers all tests
+            ci="windows/ci/"
+            changed_all_target="windows/ci/smoketest/"
         else
-            # minimal tests for other python versions
-            ci="windows/ci/minimal/"
+            # without change detection enabled run only smoketest tests
+            ci="windows/ci/smoketest/"
         fi
-
-        # shellcheck disable=SC2086
-        ansible-test windows-integration --color -v --retry-on-error "${ci}" --tox --python "${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
-            "${platforms[@]}"
+    else
+        # minimal tests for other python versions
+        ci="windows/ci/minimal/"
     fi
+
+    # shellcheck disable=SC2086
+    ansible-test windows-integration --color -v --retry-on-error "${ci}" --tox --python "${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+        "${platforms[@]}" --changed-all-target "${changed_all_target}"
 done