Update ansible-test handling of delegation paths. (#69056)

.gitignore

@@ -105,5 +105,3 @@ test/units/.coverage.*
 /SYMLINK_CACHE.json
 changelogs/.plugin-cache.yaml
 .ansible-test-timeout.json
-# ansible-test temporary metadata file for use with delgation
-/metadata-*.json

changelogs/fragments/ansible-test-delegation-paths.yml

@@ -0,0 +1,2 @@
+minor_changes:
+    - ansible-test now places the ansible source and collections content in separate directories when using the ``--docker`` or ``--remote`` options.

test/lib/ansible_test/_internal/cloud/__init__.py

@@ -197,12 +197,7 @@ class CloudBase(ABC):
         def config_callback(files):  # type: (t.List[t.Tuple[str, str]]) -> None
             """Add the config file to the payload file list."""
             if self._get_cloud_config(self._CONFIG_PATH, ''):
-                if data_context().content.collection:
-                    working_path = data_context().content.collection.directory
-                else:
-                    working_path = ''
-
-                pair = (self.config_path, os.path.join(working_path, os.path.relpath(self.config_path, data_context().content.root)))
+                pair = (self.config_path, os.path.relpath(self.config_path, data_context().content.root))
 
                 if pair not in files:
                     display.info('Including %s config: %s -> %s' % (self.platform, pair[0], pair[1]), verbosity=3)

test/lib/ansible_test/_internal/config.py

@@ -209,13 +209,8 @@ class TestConfig(EnvironmentConfig):
             """Add the metadata file to the payload file list."""
             config = self
 
-            if data_context().content.collection:
-                working_path = data_context().content.collection.directory
-            else:
-                working_path = ''
-
             if self.metadata_path:
-                files.append((os.path.abspath(config.metadata_path), os.path.join(working_path, config.metadata_path)))
+                files.append((os.path.abspath(config.metadata_path), config.metadata_path))
 
         data_context().register_payload_callback(metadata_callback)
 

test/lib/ansible_test/_internal/core_ci.py

@@ -617,13 +617,8 @@ class SshKey:
             Add the SSH keys to the payload file list.
             They are either outside the source tree or in the cache dir which is ignored by default.
             """
-            if data_context().content.collection:
-                working_path = data_context().content.collection.directory
-            else:
-                working_path = ''
-
-            files.append((key, os.path.join(working_path, os.path.relpath(key_dst, data_context().content.root))))
-            files.append((pub, os.path.join(working_path, os.path.relpath(pub_dst, data_context().content.root))))
+            files.append((key, os.path.relpath(key_dst, data_context().content.root)))
+            files.append((pub, os.path.relpath(pub_dst, data_context().content.root)))
 
         data_context().register_payload_callback(ssh_key_callback)
 

test/lib/ansible_test/_internal/delegation.py

@@ -117,8 +117,10 @@ def delegate(args, exclude, require, integration_targets):
     :rtype: bool
     """
     if isinstance(args, TestConfig):
-        with tempfile.NamedTemporaryFile(prefix='metadata-', suffix='.json', dir=data_context().content.root) as metadata_fd:
-            args.metadata_path = os.path.basename(metadata_fd.name)
+        make_dirs(ResultType.TMP.path)
+
+        with tempfile.NamedTemporaryFile(prefix='metadata-', suffix='.json', dir=ResultType.TMP.path) as metadata_fd:
+            args.metadata_path = os.path.join(ResultType.TMP.relative_path, os.path.basename(metadata_fd.name))
             args.metadata.to_file(args.metadata_path)
 
             try:
@@ -244,16 +246,17 @@ def delegate_docker(args, exclude, require, integration_targets):
 
     python_interpreter = get_python_interpreter(args, get_docker_completion(), args.docker_raw)
 
-    install_root = '/root/ansible'
+    pwd = '/root'
+    ansible_root = os.path.join(pwd, 'ansible')
 
     if data_context().content.collection:
-        content_root = os.path.join(install_root, data_context().content.collection.directory)
+        content_root = os.path.join(pwd, data_context().content.collection.directory)
     else:
-        content_root = install_root
+        content_root = ansible_root
 
     remote_results_root = os.path.join(content_root, data_context().content.results_path)
 
-    cmd = generate_command(args, python_interpreter, os.path.join(install_root, 'bin'), content_root, options, exclude, require)
+    cmd = generate_command(args, python_interpreter, os.path.join(ansible_root, 'bin'), content_root, options, exclude, require)
 
     if isinstance(args, TestConfig):
         if args.coverage and not args.coverage_label:
@@ -321,9 +324,8 @@ def delegate_docker(args, exclude, require, integration_targets):
             # write temporary files to /root since /tmp isn't ready immediately on container start
             docker_put(args, test_id, os.path.join(ANSIBLE_TEST_DATA_ROOT, 'setup', 'docker.sh'), '/root/docker.sh')
             docker_exec(args, test_id, ['/bin/bash', '/root/docker.sh'])
-            docker_put(args, test_id, local_source_fd.name, '/root/ansible.tgz')
-            docker_exec(args, test_id, ['mkdir', '/root/ansible'])
-            docker_exec(args, test_id, ['tar', 'oxzf', '/root/ansible.tgz', '-C', '/root/ansible'])
+            docker_put(args, test_id, local_source_fd.name, '/root/test.tgz')
+            docker_exec(args, test_id, ['tar', 'oxzf', '/root/test.tgz', '-C', '/root'])
 
             # docker images are only expected to have a single python version available
             if isinstance(args, UnitsConfig) and not args.python:
@@ -440,14 +442,14 @@ def delegate_remote(args, exclude, require, integration_targets):
 
             python_interpreter = get_python_interpreter(args, get_remote_completion(), args.remote)
 
-            install_root = os.path.join(pwd, 'ansible')
+            ansible_root = os.path.join(pwd, 'ansible')
 
             if data_context().content.collection:
-                content_root = os.path.join(install_root, data_context().content.collection.directory)
+                content_root = os.path.join(pwd, data_context().content.collection.directory)
             else:
-                content_root = install_root
+                content_root = ansible_root
 
-            cmd = generate_command(args, python_interpreter, os.path.join(install_root, 'bin'), content_root, options, exclude, require)
+            cmd = generate_command(args, python_interpreter, os.path.join(ansible_root, 'bin'), content_root, options, exclude, require)
 
             if httptester_id:
                 cmd += ['--inject-httptester']

test/lib/ansible_test/_internal/executor.py

@@ -972,12 +972,7 @@ def command_integration_filter(args,  # type: TIntegrationConfig
             Add the integration config vars file to the payload file list.
             This will preserve the file during delegation even if the file is ignored by source control.
             """
-            if data_context().content.collection:
-                working_path = data_context().content.collection.directory
-            else:
-                working_path = ''
-
-            files.append((vars_file_src, os.path.join(working_path, data_context().content.integration_vars_path)))
+            files.append((vars_file_src, data_context().content.integration_vars_path))
 
         data_context().register_payload_callback(integration_config_callback)
 

test/lib/ansible_test/_internal/integration/__init__.py

@@ -170,12 +170,7 @@ def delegate_inventory(args, inventory_path_src):  # type: (IntegrationConfig, s
         Add the inventory file to the payload file list.
         This will preserve the file during delegation even if it is ignored or is outside the content and install roots.
         """
-        if data_context().content.collection:
-            working_path = data_context().content.collection.directory
-        else:
-            working_path = ''
-
-        inventory_path = os.path.join(working_path, get_inventory_relative_path(args))
+        inventory_path = get_inventory_relative_path(args)
         inventory_tuple = inventory_path_src, inventory_path
 
         if os.path.isfile(inventory_path_src) and inventory_tuple not in files:

test/lib/ansible_test/_internal/manage_ci.py

@@ -277,7 +277,7 @@ class ManagePosixCI:
             # being different and -z not being recognized. This pattern works
             # with both versions of tar.
             self.ssh(
-                'rm -rf ~/ansible && mkdir ~/ansible && cd ~/ansible && gunzip --stdout %s | tar oxf - && rm %s' %
+                'rm -rf ~/ansible ~/ansible_collections && cd ~/ && gunzip --stdout %s | tar oxf - && rm %s' %
                 (remote_source_path, remote_source_path)
             )
 

test/lib/ansible_test/_internal/payload.py

@@ -75,7 +75,7 @@ def create_payload(args, dst_path):  # type: (CommonConfig, str) -> None
         files = [f for f in files if
                  is_subdir(f[1], 'bin/') or
                  is_subdir(f[1], 'lib/ansible/') or
-                 (is_subdir(f[1], 'test/lib/ansible_test/') and not is_subdir(f[1], 'test/lib/ansible_test/tests/'))]
+                 is_subdir(f[1], 'test/lib/ansible_test/')]
 
         if not isinstance(args, (ShellConfig, IntegrationConfig)):
             # exclude built-in ansible modules when they are not needed
@@ -83,12 +83,35 @@ def create_payload(args, dst_path):  # type: (CommonConfig, str) -> None
 
         collection_layouts = data_context().create_collection_layouts()
 
+        content_files = []
+        extra_files = []
+
         for layout in collection_layouts:
-            # include files from each collection in the same collection root as the content being tested
-            files.extend((os.path.join(layout.root, path), os.path.join(layout.collection.directory, path)) for path in layout.all_files())
+            if layout == data_context().content:
+                # include files from the current collection (layout.collection.directory will be added later)
+                content_files.extend((os.path.join(layout.root, path), path) for path in data_context().content.all_files())
+            else:
+                # include files from each collection in the same collection root as the content being tested
+                extra_files.extend((os.path.join(layout.root, path), os.path.join(layout.collection.directory, path)) for path in layout.all_files())
+    else:
+        # when testing ansible itself the ansible source is the content
+        content_files = files
+        # there are no extra files when testing ansible itself
+        extra_files = []
 
     for callback in data_context().payload_callbacks:
-        callback(files)
+        # execute callbacks only on the content paths
+        # this is done before placing them in the appropriate subdirectory (see below)
+        callback(content_files)
+
+    # place ansible source files under the 'ansible' directory on the delegated host
+    files = [(src, os.path.join('ansible', dst)) for src, dst in files]
+
+    if data_context().content.collection:
+        # place collection files under the 'ansible_collections/{namespace}/{collection}' directory on the delegated host
+        files.extend((src, os.path.join(data_context().content.collection.directory, dst)) for src, dst in content_files)
+        # extra files already have the correct destination path
+        files.extend(extra_files)
 
     # maintain predictable file order
     files = sorted(set(files))

test/sanity/code-smell/package-data.py

@@ -27,7 +27,6 @@ def assemble_files_to_ship(complete_file_list):
         'hacking/shippable/*',
         'hacking/tests/*',
         'hacking/ticket_stubs/*',
-        'metadata-*.json',  # ansible-test with --docker produces this tmp file.
         'test/sanity/code-smell/botmeta.*',
         'test/utils/*',
         'test/utils/*/*',