From a81dd4f06a4a424496f55d89ebd52962c3204847 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Tue, 9 Jun 2020 15:40:56 -0700 Subject: [PATCH] Update ansible-test default containers. (#69819) * Move ansible-test completion code. * Fix a few type hints. * Change docker completion based on context. Collections now use version 2.0 of the default-test-container. This is an updated version of the earlier 1.x default-test-container with ansible-base and cloud specific requirements removed. Testing of ansible-base now uses version 1.0 of a new ansible-base-test-container. This container is similar to the earlier 1.x default-test-container, but with unnecessary duplication of requirements across Python versions removed. Collections which have tests that depend on requirements no longer present in the default test container should specify them in their test requirements files: * tests/integration/requirements.txt * tests/unit/requirements.txt * Bump test container versions Co-authored-by: Jordan Borean --- .../fragments/ansible-test-docker-context.yml | 2 + .../ansible_test/_data/completion/docker.txt | 3 +- test/lib/ansible_test/_internal/ci/local.py | 2 +- .../ansible_test/_internal/ci/shippable.py | 2 +- test/lib/ansible_test/_internal/cli.py | 6 +- test/lib/ansible_test/_internal/config.py | 6 +- test/lib/ansible_test/_internal/delegation.py | 4 +- test/lib/ansible_test/_internal/executor.py | 6 +- test/lib/ansible_test/_internal/manage_ci.py | 2 +- test/lib/ansible_test/_internal/util.py | 88 ------------------ .../lib/ansible_test/_internal/util_common.py | 92 +++++++++++++++++++ 11 files changed, 110 insertions(+), 103 deletions(-) create mode 100644 changelogs/fragments/ansible-test-docker-context.yml diff --git a/changelogs/fragments/ansible-test-docker-context.yml b/changelogs/fragments/ansible-test-docker-context.yml new file mode 100644 index 00000000000..0bd6e5ae1ff --- /dev/null +++ b/changelogs/fragments/ansible-test-docker-context.yml @@ -0,0 +1,2 @@ +major_changes: + - ansible-test now uses a different ``default`` test container for Ansible Collections diff --git a/test/lib/ansible_test/_data/completion/docker.txt b/test/lib/ansible_test/_data/completion/docker.txt index c87e357214c..1e4a059b296 100644 --- a/test/lib/ansible_test/_data/completion/docker.txt +++ b/test/lib/ansible_test/_data/completion/docker.txt @@ -1,4 +1,5 @@ -default name=quay.io/ansible/default-test-container:1.14 python=3.6,2.6,2.7,3.5,3.7,3.8,3.9 seccomp=unconfined +default name=quay.io/ansible/default-test-container:2.1 python=3.6,2.6,2.7,3.5,3.7,3.8,3.9 seccomp=unconfined context=collection +default name=quay.io/ansible/ansible-base-test-container:1.1 python=3.6,2.6,2.7,3.5,3.7,3.8,3.9 seccomp=unconfined context=ansible-base centos6 name=quay.io/ansible/centos6-test-container:1.16.0 python=2.6 seccomp=unconfined centos7 name=quay.io/ansible/centos7-test-container:1.16.0 python=2.7 seccomp=unconfined centos8 name=quay.io/ansible/centos8-test-container:1.16.0 python=3.6 seccomp=unconfined diff --git a/test/lib/ansible_test/_internal/ci/local.py b/test/lib/ansible_test/_internal/ci/local.py index fbb850b8306..5f605c863e7 100644 --- a/test/lib/ansible_test/_internal/ci/local.py +++ b/test/lib/ansible_test/_internal/ci/local.py @@ -164,7 +164,7 @@ class InvalidBranch(ApplicationError): class LocalChanges: """Change information for local work.""" - def __init__(self, args): # type: (CommonConfig) -> None + def __init__(self, args): # type: (TestConfig) -> None self.args = args self.git = Git() diff --git a/test/lib/ansible_test/_internal/ci/shippable.py b/test/lib/ansible_test/_internal/ci/shippable.py index 99f6137237f..f9f0a1928f9 100644 --- a/test/lib/ansible_test/_internal/ci/shippable.py +++ b/test/lib/ansible_test/_internal/ci/shippable.py @@ -206,7 +206,7 @@ class ShippableAuthHelper(OpenSSLAuthHelper): class ShippableChanges: """Change information for Shippable build.""" - def __init__(self, args): # type: (CommonConfig) -> None + def __init__(self, args): # type: (TestConfig) -> None self.args = args self.git = Git() diff --git a/test/lib/ansible_test/_internal/cli.py b/test/lib/ansible_test/_internal/cli.py index ddeb6820746..2ee6df73923 100644 --- a/test/lib/ansible_test/_internal/cli.py +++ b/test/lib/ansible_test/_internal/cli.py @@ -18,9 +18,6 @@ from .util import ( ApplicationError, display, raw_command, - get_docker_completion, - get_network_completion, - get_remote_completion, generate_pip_command, read_lines_without_comments, MAXFD, @@ -91,6 +88,9 @@ from .data import ( ) from .util_common import ( + get_docker_completion, + get_network_completion, + get_remote_completion, CommonConfig, ) diff --git a/test/lib/ansible_test/_internal/config.py b/test/lib/ansible_test/_internal/config.py index b0d153c5c1a..7d3ab9e27db 100644 --- a/test/lib/ansible_test/_internal/config.py +++ b/test/lib/ansible_test/_internal/config.py @@ -8,15 +8,15 @@ import sys from . import types as t from .util import ( - docker_qualify_image, find_python, generate_pip_command, - get_docker_completion, - get_remote_completion, ApplicationError, ) from .util_common import ( + docker_qualify_image, + get_docker_completion, + get_remote_completion, CommonConfig, ) diff --git a/test/lib/ansible_test/_internal/delegation.py b/test/lib/ansible_test/_internal/delegation.py index 9dafe53d2e5..23ee4a6a755 100644 --- a/test/lib/ansible_test/_internal/delegation.py +++ b/test/lib/ansible_test/_internal/delegation.py @@ -21,8 +21,6 @@ from .executor import ( start_httptester, get_python_interpreter, get_python_version, - get_docker_completion, - get_remote_completion, ) from .config import ( @@ -60,6 +58,8 @@ from .util_common import ( run_command, ResultType, create_interpreter_wrapper, + get_docker_completion, + get_remote_completion, ) from .docker_util import ( diff --git a/test/lib/ansible_test/_internal/executor.py b/test/lib/ansible_test/_internal/executor.py index 9642d995417..05614a4a283 100644 --- a/test/lib/ansible_test/_internal/executor.py +++ b/test/lib/ansible_test/_internal/executor.py @@ -59,9 +59,6 @@ from .util import ( get_available_port, generate_pip_command, find_python, - get_docker_completion, - get_network_settings, - get_remote_completion, cmd_quote, ANSIBLE_LIB_ROOT, ANSIBLE_TEST_DATA_ROOT, @@ -74,6 +71,9 @@ from .util import ( ) from .util_common import ( + get_docker_completion, + get_network_settings, + get_remote_completion, get_python_path, intercept_command, named_temporary_file, diff --git a/test/lib/ansible_test/_internal/manage_ci.py b/test/lib/ansible_test/_internal/manage_ci.py index 2915313521f..a1da541171b 100644 --- a/test/lib/ansible_test/_internal/manage_ci.py +++ b/test/lib/ansible_test/_internal/manage_ci.py @@ -12,11 +12,11 @@ from .util import ( cmd_quote, display, ANSIBLE_TEST_DATA_ROOT, - get_network_settings, ) from .util_common import ( intercept_command, + get_network_settings, run_command, ) diff --git a/test/lib/ansible_test/_internal/util.py b/test/lib/ansible_test/_internal/util.py index 9dcd2b20d55..fd7e13d1e09 100644 --- a/test/lib/ansible_test/_internal/util.py +++ b/test/lib/ansible_test/_internal/util.py @@ -62,9 +62,6 @@ except AttributeError: C = None -DOCKER_COMPLETION = {} # type: t.Dict[str, t.Dict[str, str]] -REMOTE_COMPLETION = {} # type: t.Dict[str, t.Dict[str, str]] -NETWORK_COMPLETION = {} # type: t.Dict[str, t.Dict[str, str]] PYTHON_PATHS = {} # type: t.Dict[str, str] try: @@ -122,57 +119,6 @@ SUPPORTED_PYTHON_VERSIONS = ( ) -def get_docker_completion(): - """ - :rtype: dict[str, dict[str, str]] - """ - return get_parameterized_completion(DOCKER_COMPLETION, 'docker') - - -def get_remote_completion(): - """ - :rtype: dict[str, dict[str, str]] - """ - return get_parameterized_completion(REMOTE_COMPLETION, 'remote') - - -def get_network_completion(): - """ - :rtype: dict[str, dict[str, str]] - """ - return get_parameterized_completion(NETWORK_COMPLETION, 'network') - - -def get_parameterized_completion(cache, name): - """ - :type cache: dict[str, dict[str, str]] - :type name: str - :rtype: dict[str, dict[str, str]] - """ - if not cache: - images = read_lines_without_comments(os.path.join(ANSIBLE_TEST_DATA_ROOT, 'completion', '%s.txt' % name), remove_blank_lines=True) - - cache.update(dict(kvp for kvp in [parse_parameterized_completion(i) for i in images] if kvp)) - - return cache - - -def parse_parameterized_completion(value): - """ - :type value: str - :rtype: tuple[str, dict[str, str]] - """ - values = value.split() - - if not values: - return None - - name = values[0] - data = dict((kvp[0], kvp[1] if len(kvp) > 1 else '') for kvp in [item.split('=', 1) for item in values[1:]]) - - return name, data - - def remove_file(path): """ :type path: str @@ -745,40 +691,6 @@ class MissingEnvironmentVariable(ApplicationError): self.name = name -class NetworkPlatformSettings: - """Settings required for provisioning a network platform.""" - def __init__(self, collection, inventory_vars): # type: (str, t.Type[str, str]) -> None - self.collection = collection - self.inventory_vars = inventory_vars - - -def get_network_settings(args, platform, version): # type: (NetworkIntegrationConfig, str, str) -> NetworkPlatformSettings - """Returns settings for the given network platform and version.""" - platform_version = '%s/%s' % (platform, version) - completion = get_network_completion().get(platform_version, {}) - collection = args.platform_collection.get(platform, completion.get('collection')) - - settings = NetworkPlatformSettings( - collection, - dict( - ansible_connection=args.platform_connection.get(platform, completion.get('connection')), - ansible_network_os='%s.%s' % (collection, platform) if collection else platform, - ) - ) - - return settings - - -def docker_qualify_image(name): - """ - :type name: str - :rtype: str - """ - config = get_docker_completion().get(name, {}) - - return config.get('name', name) - - def parse_to_list_of_dict(pattern, value): """ :type pattern: str diff --git a/test/lib/ansible_test/_internal/util_common.py b/test/lib/ansible_test/_internal/util_common.py index 0739fd9ea81..1ac2e60dd17 100644 --- a/test/lib/ansible_test/_internal/util_common.py +++ b/test/lib/ansible_test/_internal/util_common.py @@ -26,6 +26,7 @@ from .util import ( MODE_FILE_EXECUTE, PYTHON_PATHS, raw_command, + read_lines_without_comments, ANSIBLE_TEST_DATA_ROOT, ApplicationError, ) @@ -43,6 +44,10 @@ from .provider.layout import ( LayoutMessages, ) +DOCKER_COMPLETION = {} # type: t.Dict[str, t.Dict[str, str]] +REMOTE_COMPLETION = {} # type: t.Dict[str, t.Dict[str, str]] +NETWORK_COMPLETION = {} # type: t.Dict[str, t.Dict[str, str]] + class ResultType: """Test result type.""" @@ -110,6 +115,93 @@ class CommonConfig: return os.path.join(ANSIBLE_TEST_DATA_ROOT, 'ansible.cfg') +class NetworkPlatformSettings: + """Settings required for provisioning a network platform.""" + def __init__(self, collection, inventory_vars): # type: (str, t.Type[str, str]) -> None + self.collection = collection + self.inventory_vars = inventory_vars + + +def get_docker_completion(): + """ + :rtype: dict[str, dict[str, str]] + """ + return get_parameterized_completion(DOCKER_COMPLETION, 'docker') + + +def get_remote_completion(): + """ + :rtype: dict[str, dict[str, str]] + """ + return get_parameterized_completion(REMOTE_COMPLETION, 'remote') + + +def get_network_completion(): + """ + :rtype: dict[str, dict[str, str]] + """ + return get_parameterized_completion(NETWORK_COMPLETION, 'network') + + +def get_parameterized_completion(cache, name): + """ + :type cache: dict[str, dict[str, str]] + :type name: str + :rtype: dict[str, dict[str, str]] + """ + if not cache: + if data_context().content.collection: + context = 'collection' + else: + context = 'ansible-base' + + images = read_lines_without_comments(os.path.join(ANSIBLE_TEST_DATA_ROOT, 'completion', '%s.txt' % name), remove_blank_lines=True) + + cache.update(dict(kvp for kvp in [parse_parameterized_completion(i) for i in images] if kvp and kvp[1].get('context', context) == context)) + + return cache + + +def parse_parameterized_completion(value): # type: (str) -> t.Optional[t.Tuple[str, t.Dict[str, str]]] + """Parse the given completion entry, returning the entry name and a dictionary of key/value settings.""" + values = value.split() + + if not values: + return None + + name = values[0] + data = dict((kvp[0], kvp[1] if len(kvp) > 1 else '') for kvp in [item.split('=', 1) for item in values[1:]]) + + return name, data + + +def docker_qualify_image(name): + """ + :type name: str + :rtype: str + """ + config = get_docker_completion().get(name, {}) + + return config.get('name', name) + + +def get_network_settings(args, platform, version): # type: (NetworkIntegrationConfig, str, str) -> NetworkPlatformSettings + """Returns settings for the given network platform and version.""" + platform_version = '%s/%s' % (platform, version) + completion = get_network_completion().get(platform_version, {}) + collection = args.platform_collection.get(platform, completion.get('collection')) + + settings = NetworkPlatformSettings( + collection, + dict( + ansible_connection=args.platform_connection.get(platform, completion.get('connection')), + ansible_network_os='%s.%s' % (collection, platform) if collection else platform, + ) + ) + + return settings + + def handle_layout_messages(messages): # type: (t.Optional[LayoutMessages]) -> None """Display the given layout messages.""" if not messages: