ansible-test - Always use managed entry points (#81537)

changelogs/fragments/ansible-test-entry-points.yml

@@ -0,0 +1,3 @@
+bugfixes:
+  - ansible-test - Always use ansible-test managed entry points for ansible-core CLI tools when not running from source.
+                   This fixes issues where CLI entry points created during install are not compatible with ansible-test.

test/integration/targets/ansible-test-installed/aliases

@@ -0,0 +1,4 @@
+shippable/posix/group3  # runs in the distro test containers
+shippable/generic/group1  # runs in the default test container
+context/controller
+needs/target/collection

test/integration/targets/ansible-test-installed/ansible_collections/ns/col/tests/integration/targets/installed/aliases

@@ -0,0 +1 @@
+context/controller

test/integration/targets/ansible-test-installed/ansible_collections/ns/col/tests/integration/targets/installed/runme.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# This test ensures that the bin entry points created by ansible-test work
+# when ansible-test is running from an install instead of from source.
+
+set -eux
+
+# The third PATH entry is the injected bin directory created by ansible-test.
+bin_dir="$(python -c 'import os; print(os.environ["PATH"].split(":")[2])')"
+
+while IFS= read -r name
+do
+    bin="${bin_dir}/${name}"
+
+    entry_point="${name//ansible-/}"
+    entry_point="${entry_point//ansible/adhoc}"
+
+    echo "=== ${name} (${entry_point})=${bin} ==="
+
+    if [ "${name}" == "ansible-test" ]; then
+        echo "skipped - ansible-test does not support self-testing from an install"
+    else
+        "${bin}" --version | tee /dev/stderr | grep -Eo "(^${name}\ \[core\ .*|executable location = ${bin}$)"
+    fi
+done < entry-points.txt

test/integration/targets/ansible-test-installed/runme.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+base_dir="$(dirname "$(dirname "$(dirname "$(dirname "${OUTPUT_DIR}")")")")"
+bin_dir="${base_dir}/bin"
+
+source ../collection/setup.sh
+source virtualenv.sh
+
+unset PYTHONPATH
+
+# find the bin entry points to test
+ls "${bin_dir}" > tests/integration/targets/installed/entry-points.txt
+
+# deps are already installed, using --no-deps to avoid re-installing them
+pip install "${base_dir}" --disable-pip-version-check --no-deps
+
+# verify entry point generation without delegation
+ansible-test integration --color --truncate 0 "${@}"
+
+# verify entry point generation with same-host delegation
+ansible-test integration --venv --color --truncate 0 "${@}"

test/lib/ansible_test/_internal/ansible_util.py

@@ -3,9 +3,11 @@ from __future__ import annotations
 
 import json
 import os
+import shutil
 import typing as t
 
 from .constants import (
+    ANSIBLE_BIN_SYMLINK_MAP,
     SOFT_RLIMIT_NOFILE,
 )
 
@@ -17,12 +19,15 @@ from .util import (
     common_environment,
     ApplicationError,
     ANSIBLE_LIB_ROOT,
+    ANSIBLE_TEST_ROOT,
     ANSIBLE_TEST_DATA_ROOT,
-    ANSIBLE_BIN_PATH,
+    ANSIBLE_ROOT,
     ANSIBLE_SOURCE_ROOT,
     ANSIBLE_TEST_TOOLS_ROOT,
+    MODE_FILE_EXECUTE,
     get_ansible_version,
     raw_command,
+    verified_chmod,
 )
 
 from .util_common import (
@@ -78,8 +83,10 @@ def ansible_environment(args: CommonConfig, color: bool = True, ansible_config:
     env = common_environment()
     path = env['PATH']
 
-    if not path.startswith(ANSIBLE_BIN_PATH + os.path.pathsep):
+    ansible_bin_path = get_ansible_bin_path(args)
-        path = ANSIBLE_BIN_PATH + os.path.pathsep + path
+
+    if not path.startswith(ansible_bin_path + os.path.pathsep):
+        path = ansible_bin_path + os.path.pathsep + path
 
     if not ansible_config:
         # use the default empty configuration unless one has been provided
@@ -196,6 +203,52 @@ def configure_plugin_paths(args: CommonConfig) -> dict[str, str]:
     return env
 
 
+@mutex
+def get_ansible_bin_path(args: CommonConfig) -> str:
+    """
+    Return a directory usable for PATH, containing only the ansible entry points.
+    If a temporary directory is required, it will be cached for the lifetime of the process and cleaned up at exit.
+    """
+    try:
+        return get_ansible_bin_path.bin_path  # type: ignore[attr-defined]
+    except AttributeError:
+        pass
+
+    if ANSIBLE_SOURCE_ROOT:
+        # when running from source there is no need for a temporary directory since we already have known entry point scripts
+        bin_path = os.path.join(ANSIBLE_ROOT, 'bin')
+    else:
+        # when not running from source the installed entry points cannot be relied upon
+        # doing so would require using the interpreter specified by those entry points, which conflicts with using our interpreter and injector
+        # instead a temporary directory is created which contains only ansible entry points
+        # symbolic links cannot be used since the files are likely not executable
+        bin_path = create_temp_dir(prefix='ansible-test-', suffix='-bin')
+        bin_links = {os.path.join(bin_path, name): get_cli_path(path) for name, path in ANSIBLE_BIN_SYMLINK_MAP.items()}
+
+        if not args.explain:
+            for dst, src in bin_links.items():
+                shutil.copy(src, dst)
+                verified_chmod(dst, MODE_FILE_EXECUTE)
+
+    get_ansible_bin_path.bin_path = bin_path  # type: ignore[attr-defined]
+
+    return bin_path
+
+
+def get_cli_path(path: str) -> str:
+    """Return the absolute path to the CLI script from the given path which is relative to the `bin` directory of the original source tree layout."""
+    path_rewrite = {
+        '../lib/ansible/': ANSIBLE_LIB_ROOT,
+        '../test/lib/ansible_test/': ANSIBLE_TEST_ROOT,
+    }
+
+    for prefix, destination in path_rewrite.items():
+        if path.startswith(prefix):
+            return os.path.join(destination, path[len(prefix):])
+
+    raise RuntimeError(path)
+
+
 @mutex
 def get_ansible_python_path(args: CommonConfig) -> str:
     """

test/lib/ansible_test/_internal/commands/sanity/bin_symlinks.py

@@ -32,7 +32,7 @@ from ...payload import (
 )
 
 from ...util import (
-    ANSIBLE_BIN_PATH,
+    ANSIBLE_SOURCE_ROOT,
 )
 
 
@@ -52,7 +52,7 @@ class BinSymlinksTest(SanityVersionNeutral):
         return True
 
     def test(self, args: SanityConfig, targets: SanityTargets) -> TestResult:
-        bin_root = ANSIBLE_BIN_PATH
+        bin_root = os.path.join(ANSIBLE_SOURCE_ROOT, 'bin')
         bin_names = os.listdir(bin_root)
         bin_paths = sorted(os.path.join(bin_root, path) for path in bin_names)
 

test/lib/ansible_test/_internal/constants.py

@@ -33,6 +33,7 @@ SECCOMP_CHOICES = [
 # This bin symlink map must exactly match the contents of the bin directory.
 # It is necessary for payload creation to reconstruct the bin directory when running ansible-test from an installed version of ansible.
 # It is also used to construct the injector directory at runtime.
+# It is also used to construct entry points when not running ansible-test from source.
 ANSIBLE_BIN_SYMLINK_MAP = {
     'ansible': '../lib/ansible/cli/adhoc.py',
     'ansible-config': '../lib/ansible/cli/config.py',

test/lib/ansible_test/_internal/delegation.py

@@ -33,7 +33,6 @@ from .util import (
     SubprocessError,
     display,
     filter_args,
-    ANSIBLE_BIN_PATH,
     ANSIBLE_LIB_ROOT,
     ANSIBLE_TEST_ROOT,
     OutputStream,
@@ -44,6 +43,10 @@ from .util_common import (
     process_scoped_temporary_directory,
 )
 
+from .ansible_util import (
+    get_ansible_bin_path,
+)
+
 from .containers import (
     support_container_context,
     ContainerDatabase,
@@ -145,7 +148,7 @@ def delegate_command(args: EnvironmentConfig, host_state: HostState, exclude: li
             con.extract_archive(chdir=working_directory, src=payload_file)
     else:
         content_root = working_directory
-        ansible_bin_path = ANSIBLE_BIN_PATH
+        ansible_bin_path = get_ansible_bin_path(args)
 
     command = generate_command(args, host_state.controller_profile.python, ansible_bin_path, content_root, exclude, require)
 

test/lib/ansible_test/_internal/util.py

@@ -69,14 +69,12 @@ ANSIBLE_TEST_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 # assume running from install
 ANSIBLE_ROOT = os.path.dirname(ANSIBLE_TEST_ROOT)
-ANSIBLE_BIN_PATH = os.path.dirname(os.path.abspath(sys.argv[0]))
 ANSIBLE_LIB_ROOT = os.path.join(ANSIBLE_ROOT, 'ansible')
 ANSIBLE_SOURCE_ROOT = None
 
 if not os.path.exists(ANSIBLE_LIB_ROOT):
     # running from source
     ANSIBLE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(ANSIBLE_TEST_ROOT)))
-    ANSIBLE_BIN_PATH = os.path.join(ANSIBLE_ROOT, 'bin')
     ANSIBLE_LIB_ROOT = os.path.join(ANSIBLE_ROOT, 'lib', 'ansible')
     ANSIBLE_SOURCE_ROOT = ANSIBLE_ROOT