Update ansible-test default containers. (#69819)

* Move ansible-test completion code.

* Fix a few type hints.

* Change docker completion based on context.

Collections now use version 2.0 of the default-test-container.
This is an updated version of the earlier 1.x default-test-container with ansible-base and cloud specific requirements removed.

Testing of ansible-base now uses version 1.0 of a new ansible-base-test-container.
This container is similar to the earlier 1.x default-test-container, but with unnecessary duplication of requirements across Python versions removed.

Collections which have tests that depend on requirements no longer present in the default test container should specify them in their test requirements files:

* tests/integration/requirements.txt
* tests/unit/requirements.txt

* Bump test container versions

Co-authored-by: Jordan Borean <jborean93@gmail.com>

changelogs/fragments/ansible-test-docker-context.yml

@@ -0,0 +1,2 @@
+major_changes:
+    - ansible-test now uses a different ``default`` test container for Ansible Collections

test/lib/ansible_test/_data/completion/docker.txt

@@ -1,4 +1,5 @@
-default name=quay.io/ansible/default-test-container:1.14 python=3.6,2.6,2.7,3.5,3.7,3.8,3.9 seccomp=unconfined
+default name=quay.io/ansible/default-test-container:2.1 python=3.6,2.6,2.7,3.5,3.7,3.8,3.9 seccomp=unconfined context=collection
+default name=quay.io/ansible/ansible-base-test-container:1.1 python=3.6,2.6,2.7,3.5,3.7,3.8,3.9 seccomp=unconfined context=ansible-base
 centos6 name=quay.io/ansible/centos6-test-container:1.16.0 python=2.6 seccomp=unconfined
 centos7 name=quay.io/ansible/centos7-test-container:1.16.0 python=2.7 seccomp=unconfined
 centos8 name=quay.io/ansible/centos8-test-container:1.16.0 python=3.6 seccomp=unconfined

test/lib/ansible_test/_internal/ci/local.py

@@ -164,7 +164,7 @@ class InvalidBranch(ApplicationError):
 
 class LocalChanges:
     """Change information for local work."""
-    def __init__(self, args):  # type: (CommonConfig) -> None
+    def __init__(self, args):  # type: (TestConfig) -> None
         self.args = args
         self.git = Git()
 

test/lib/ansible_test/_internal/ci/shippable.py

@@ -206,7 +206,7 @@ class ShippableAuthHelper(OpenSSLAuthHelper):
 
 class ShippableChanges:
     """Change information for Shippable build."""
-    def __init__(self, args):  # type: (CommonConfig) -> None
+    def __init__(self, args):  # type: (TestConfig) -> None
         self.args = args
         self.git = Git()
 

test/lib/ansible_test/_internal/cli.py

@@ -18,9 +18,6 @@ from .util import (
     ApplicationError,
     display,
     raw_command,
-    get_docker_completion,
-    get_network_completion,
-    get_remote_completion,
     generate_pip_command,
     read_lines_without_comments,
     MAXFD,
@@ -91,6 +88,9 @@ from .data import (
 )
 
 from .util_common import (
+    get_docker_completion,
+    get_network_completion,
+    get_remote_completion,
     CommonConfig,
 )
 

test/lib/ansible_test/_internal/config.py

@@ -8,15 +8,15 @@ import sys
 from . import types as t
 
 from .util import (
-    docker_qualify_image,
     find_python,
     generate_pip_command,
-    get_docker_completion,
-    get_remote_completion,
     ApplicationError,
 )
 
 from .util_common import (
+    docker_qualify_image,
+    get_docker_completion,
+    get_remote_completion,
     CommonConfig,
 )
 

test/lib/ansible_test/_internal/delegation.py

@@ -21,8 +21,6 @@ from .executor import (
     start_httptester,
     get_python_interpreter,
     get_python_version,
-    get_docker_completion,
-    get_remote_completion,
 )
 
 from .config import (
@@ -60,6 +58,8 @@ from .util_common import (
     run_command,
     ResultType,
     create_interpreter_wrapper,
+    get_docker_completion,
+    get_remote_completion,
 )
 
 from .docker_util import (

test/lib/ansible_test/_internal/executor.py

@@ -59,9 +59,6 @@ from .util import (
     get_available_port,
     generate_pip_command,
     find_python,
-    get_docker_completion,
-    get_network_settings,
-    get_remote_completion,
     cmd_quote,
     ANSIBLE_LIB_ROOT,
     ANSIBLE_TEST_DATA_ROOT,
@@ -74,6 +71,9 @@ from .util import (
 )
 
 from .util_common import (
+    get_docker_completion,
+    get_network_settings,
+    get_remote_completion,
     get_python_path,
     intercept_command,
     named_temporary_file,

test/lib/ansible_test/_internal/manage_ci.py

@@ -12,11 +12,11 @@ from .util import (
     cmd_quote,
     display,
     ANSIBLE_TEST_DATA_ROOT,
-    get_network_settings,
 )
 
 from .util_common import (
     intercept_command,
+    get_network_settings,
     run_command,
 )
 

test/lib/ansible_test/_internal/util.py

@@ -62,9 +62,6 @@ except AttributeError:
     C = None
 
 
-DOCKER_COMPLETION = {}  # type: t.Dict[str, t.Dict[str, str]]
-REMOTE_COMPLETION = {}  # type: t.Dict[str, t.Dict[str, str]]
-NETWORK_COMPLETION = {}  # type: t.Dict[str, t.Dict[str, str]]
 PYTHON_PATHS = {}  # type: t.Dict[str, str]
 
 try:
@@ -122,57 +119,6 @@ SUPPORTED_PYTHON_VERSIONS = (
 )
 
 
-def get_docker_completion():
-    """
-    :rtype: dict[str, dict[str, str]]
-    """
-    return get_parameterized_completion(DOCKER_COMPLETION, 'docker')
-
-
-def get_remote_completion():
-    """
-    :rtype: dict[str, dict[str, str]]
-    """
-    return get_parameterized_completion(REMOTE_COMPLETION, 'remote')
-
-
-def get_network_completion():
-    """
-    :rtype: dict[str, dict[str, str]]
-    """
-    return get_parameterized_completion(NETWORK_COMPLETION, 'network')
-
-
-def get_parameterized_completion(cache, name):
-    """
-    :type cache: dict[str, dict[str, str]]
-    :type name: str
-    :rtype: dict[str, dict[str, str]]
-    """
-    if not cache:
-        images = read_lines_without_comments(os.path.join(ANSIBLE_TEST_DATA_ROOT, 'completion', '%s.txt' % name), remove_blank_lines=True)
-
-        cache.update(dict(kvp for kvp in [parse_parameterized_completion(i) for i in images] if kvp))
-
-    return cache
-
-
-def parse_parameterized_completion(value):
-    """
-    :type value: str
-    :rtype: tuple[str, dict[str, str]]
-    """
-    values = value.split()
-
-    if not values:
-        return None
-
-    name = values[0]
-    data = dict((kvp[0], kvp[1] if len(kvp) > 1 else '') for kvp in [item.split('=', 1) for item in values[1:]])
-
-    return name, data
-
-
 def remove_file(path):
     """
     :type path: str
@@ -745,40 +691,6 @@ class MissingEnvironmentVariable(ApplicationError):
         self.name = name
 
 
-class NetworkPlatformSettings:
-    """Settings required for provisioning a network platform."""
-    def __init__(self, collection, inventory_vars):  # type: (str, t.Type[str, str]) -> None
-        self.collection = collection
-        self.inventory_vars = inventory_vars
-
-
-def get_network_settings(args, platform, version):  # type: (NetworkIntegrationConfig, str, str) -> NetworkPlatformSettings
-    """Returns settings for the given network platform and version."""
-    platform_version = '%s/%s' % (platform, version)
-    completion = get_network_completion().get(platform_version, {})
-    collection = args.platform_collection.get(platform, completion.get('collection'))
-
-    settings = NetworkPlatformSettings(
-        collection,
-        dict(
-            ansible_connection=args.platform_connection.get(platform, completion.get('connection')),
-            ansible_network_os='%s.%s' % (collection, platform) if collection else platform,
-        )
-    )
-
-    return settings
-
-
-def docker_qualify_image(name):
-    """
-    :type name: str
-    :rtype: str
-    """
-    config = get_docker_completion().get(name, {})
-
-    return config.get('name', name)
-
-
 def parse_to_list_of_dict(pattern, value):
     """
     :type pattern: str

test/lib/ansible_test/_internal/util_common.py

@@ -26,6 +26,7 @@ from .util import (
     MODE_FILE_EXECUTE,
     PYTHON_PATHS,
     raw_command,
+    read_lines_without_comments,
     ANSIBLE_TEST_DATA_ROOT,
     ApplicationError,
 )
@@ -43,6 +44,10 @@ from .provider.layout import (
     LayoutMessages,
 )
 
+DOCKER_COMPLETION = {}  # type: t.Dict[str, t.Dict[str, str]]
+REMOTE_COMPLETION = {}  # type: t.Dict[str, t.Dict[str, str]]
+NETWORK_COMPLETION = {}  # type: t.Dict[str, t.Dict[str, str]]
+
 
 class ResultType:
     """Test result type."""
@@ -110,6 +115,93 @@ class CommonConfig:
         return os.path.join(ANSIBLE_TEST_DATA_ROOT, 'ansible.cfg')
 
 
+class NetworkPlatformSettings:
+    """Settings required for provisioning a network platform."""
+    def __init__(self, collection, inventory_vars):  # type: (str, t.Type[str, str]) -> None
+        self.collection = collection
+        self.inventory_vars = inventory_vars
+
+
+def get_docker_completion():
+    """
+    :rtype: dict[str, dict[str, str]]
+    """
+    return get_parameterized_completion(DOCKER_COMPLETION, 'docker')
+
+
+def get_remote_completion():
+    """
+    :rtype: dict[str, dict[str, str]]
+    """
+    return get_parameterized_completion(REMOTE_COMPLETION, 'remote')
+
+
+def get_network_completion():
+    """
+    :rtype: dict[str, dict[str, str]]
+    """
+    return get_parameterized_completion(NETWORK_COMPLETION, 'network')
+
+
+def get_parameterized_completion(cache, name):
+    """
+    :type cache: dict[str, dict[str, str]]
+    :type name: str
+    :rtype: dict[str, dict[str, str]]
+    """
+    if not cache:
+        if data_context().content.collection:
+            context = 'collection'
+        else:
+            context = 'ansible-base'
+
+        images = read_lines_without_comments(os.path.join(ANSIBLE_TEST_DATA_ROOT, 'completion', '%s.txt' % name), remove_blank_lines=True)
+
+        cache.update(dict(kvp for kvp in [parse_parameterized_completion(i) for i in images] if kvp and kvp[1].get('context', context) == context))
+
+    return cache
+
+
+def parse_parameterized_completion(value):  # type: (str) -> t.Optional[t.Tuple[str, t.Dict[str, str]]]
+    """Parse the given completion entry, returning the entry name and a dictionary of key/value settings."""
+    values = value.split()
+
+    if not values:
+        return None
+
+    name = values[0]
+    data = dict((kvp[0], kvp[1] if len(kvp) > 1 else '') for kvp in [item.split('=', 1) for item in values[1:]])
+
+    return name, data
+
+
+def docker_qualify_image(name):
+    """
+    :type name: str
+    :rtype: str
+    """
+    config = get_docker_completion().get(name, {})
+
+    return config.get('name', name)
+
+
+def get_network_settings(args, platform, version):  # type: (NetworkIntegrationConfig, str, str) -> NetworkPlatformSettings
+    """Returns settings for the given network platform and version."""
+    platform_version = '%s/%s' % (platform, version)
+    completion = get_network_completion().get(platform_version, {})
+    collection = args.platform_collection.get(platform, completion.get('collection'))
+
+    settings = NetworkPlatformSettings(
+        collection,
+        dict(
+            ansible_connection=args.platform_connection.get(platform, completion.get('connection')),
+            ansible_network_os='%s.%s' % (collection, platform) if collection else platform,
+        )
+    )
+
+    return settings
+
+
 def handle_layout_messages(messages):  # type: (t.Optional[LayoutMessages]) -> None
     """Display the given layout messages."""
     if not messages: