Support private test plugins in ansible-test. (#67660)

* Support private test plugins in ansible-test.

This feature is exclusively used for testing Ansible itself and is not available to collections.

Content in the newly supported directories will be added in follow-up PRs.

* Add new directory to path classification.

* Support new paths in shebang test.

* Add new directory to manifest.

MANIFEST.in

@@ -26,6 +26,7 @@ recursive-include test/lib/ansible_test/_data *.cfg *.ini *.json *.ps1 *.psd1 *.
 recursive-include test/lib/ansible_test/_data/injector ansible ansible-config ansible-connection ansible-console ansible-doc ansible-galaxy ansible-playbook ansible-pull ansible-test ansible-vault pytest
 recursive-include test/lib/ansible_test/_data/sanity/validate-modules validate-modules
 recursive-include test/sanity *.json *.py *.txt
+recursive-include test/support *.py *.ps1 *.psm1 *.cs
 exclude test/sanity/code-smell/botmeta.*
 recursive-include test/units *
 include Makefile

test/lib/ansible_test/_data/sanity/code-smell/shebang.py

@@ -3,6 +3,7 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import os
+import re
 import stat
 import sys
 
@@ -64,6 +65,10 @@ def main():
 
             if path.startswith('lib/ansible/modules/'):
                 is_module = True
+            elif re.search('^test/support/[^/]+/plugins/modules/', path):
+                is_module = True
+            elif re.search('^test/support/[^/]+/collections/ansible_collections/[^/]+/[^/]+/plugins/modules/', path):
+                is_module = True
             elif path.startswith('test/lib/ansible_test/_data/'):
                 pass
             elif path.startswith('lib/') or path.startswith('test/lib/'):

test/lib/ansible_test/_internal/ansible_util.py

@@ -5,6 +5,8 @@ __metaclass__ = type
 import json
 import os
 
+from . import types as t
+
 from .constants import (
     SOFT_RLIMIT_NOFILE,
 )
@@ -103,6 +105,65 @@ def ansible_environment(args, color=True, ansible_config=None):
             ANSIBLE_COLLECTIONS_PATHS=data_context().content.collection.root,
         ))
 
+    if data_context().content.is_ansible:
+        env.update(configure_plugin_paths(args))
+
+    return env
+
+
+def configure_plugin_paths(args):  # type: (CommonConfig) -> t.Dict[str, str]
+    """Return environment variables with paths to plugins relevant for the current command."""
+    # temporarily require opt-in to this feature
+    # once collection migration has occurred this feature should always be enabled
+    if not isinstance(args, IntegrationConfig) or not args.enable_test_support:
+        return {}
+
+    support_path = os.path.join(ANSIBLE_SOURCE_ROOT, 'test', 'support', args.command)
+
+    # provide private copies of collections for integration tests
+    collection_root = os.path.join(support_path, 'collections')
+
+    env = dict(
+        ANSIBLE_COLLECTIONS_PATHS=collection_root,
+    )
+
+    # provide private copies of plugins for integration tests
+    plugin_root = os.path.join(support_path, 'plugins')
+
+    plugin_list = [
+        'action',
+        'become',
+        'cache',
+        'callback',
+        'cliconf',
+        'connection',
+        'filter',
+        'httpapi',
+        'inventory',
+        'lookup',
+        'netconf',
+        # 'shell' is not configurable
+        'strategy',
+        'terminal',
+        'test',
+        'vars',
+    ]
+
+    # most plugins follow a standard naming convention
+    plugin_map = dict(('%s_plugins' % name, name) for name in plugin_list)
+
+    # these plugins do not follow the standard naming convention
+    plugin_map.update(
+        doc_fragment='doc_fragments',
+        library='modules',
+        module_utils='module_utils',
+    )
+
+    env.update(dict(('ANSIBLE_%s' % key.upper(), os.path.join(plugin_root, value)) for key, value in plugin_map.items()))
+
+    # only configure directories which exist
+    env = dict((key, value) for key, value in env.items() if os.path.isdir(value))
+
     return env
 
 

test/lib/ansible_test/_internal/classification.py

@@ -809,6 +809,9 @@ class PathMapper:
         if path.startswith('test/lib/'):
             return all_tests(self.args)  # test infrastructure, run all tests
 
+        if path.startswith('test/support/'):
+            return all_tests(self.args)  # test infrastructure, run all tests
+
         if path.startswith('test/utils/shippable/'):
             if dirname == 'test/utils/shippable':
                 test_map = {

test/lib/ansible_test/_internal/cli.py

@@ -384,6 +384,10 @@ def parse_args():
                              action='store_true',
                              help='avoid unicode characters in temporary directory (use only for verifying broken tests)')
 
+    integration.add_argument('--enable-test-support',
+                             action='store_true',
+                             help=argparse.SUPPRESS)  # temporary option for side-by-side testing -- remove after collection migration
+
     subparsers = parser.add_subparsers(metavar='COMMAND')
     subparsers.required = True  # work-around for python 3 bug which makes subparsers optional
 

test/lib/ansible_test/_internal/config.py

@@ -256,6 +256,7 @@ class IntegrationConfig(TestConfig):
         self.diff = args.diff
         self.no_temp_workdir = args.no_temp_workdir
         self.no_temp_unicode = args.no_temp_unicode
+        self.enable_test_support = args.enable_test_support
 
         if self.get_delegated_completion().get('temp-unicode', 'enabled') == 'disabled':
             self.no_temp_unicode = True