From e7f64ed9d5da541c265c17b13df65a21884d9815 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Tue, 22 Aug 2023 12:27:41 -0700 Subject: [PATCH] [stable-2.14] ansible-test - Always use managed entry points (#81537) (#81540) (cherry picked from commit 390e508d27db7a51eece36bb6d9698b63a5b638a) --- .../fragments/ansible-test-entry-points.yml | 3 + .../targets/ansible-test-installed/aliases | 4 ++ .../integration/targets/installed/aliases | 1 + .../integration/targets/installed/runme.sh | 24 ++++++++ .../targets/ansible-test-installed/runme.sh | 21 +++++++ .../ansible_test/_internal/ansible_util.py | 59 ++++++++++++++++++- .../_internal/commands/sanity/bin_symlinks.py | 4 +- test/lib/ansible_test/_internal/constants.py | 1 + test/lib/ansible_test/_internal/delegation.py | 7 ++- test/lib/ansible_test/_internal/util.py | 2 - 10 files changed, 117 insertions(+), 9 deletions(-) create mode 100644 changelogs/fragments/ansible-test-entry-points.yml create mode 100644 test/integration/targets/ansible-test-installed/aliases create mode 100644 test/integration/targets/ansible-test-installed/ansible_collections/ns/col/tests/integration/targets/installed/aliases create mode 100755 test/integration/targets/ansible-test-installed/ansible_collections/ns/col/tests/integration/targets/installed/runme.sh create mode 100755 test/integration/targets/ansible-test-installed/runme.sh diff --git a/changelogs/fragments/ansible-test-entry-points.yml b/changelogs/fragments/ansible-test-entry-points.yml new file mode 100644 index 00000000000..e770a2cfc86 --- /dev/null +++ b/changelogs/fragments/ansible-test-entry-points.yml @@ -0,0 +1,3 @@ +bugfixes: + - ansible-test - Always use ansible-test managed entry points for ansible-core CLI tools when not running from source. + This fixes issues where CLI entry points created during install are not compatible with ansible-test. diff --git a/test/integration/targets/ansible-test-installed/aliases b/test/integration/targets/ansible-test-installed/aliases new file mode 100644 index 00000000000..7741d444515 --- /dev/null +++ b/test/integration/targets/ansible-test-installed/aliases @@ -0,0 +1,4 @@ +shippable/posix/group3 # runs in the distro test containers +shippable/generic/group1 # runs in the default test container +context/controller +needs/target/collection diff --git a/test/integration/targets/ansible-test-installed/ansible_collections/ns/col/tests/integration/targets/installed/aliases b/test/integration/targets/ansible-test-installed/ansible_collections/ns/col/tests/integration/targets/installed/aliases new file mode 100644 index 00000000000..1af1cf90b6a --- /dev/null +++ b/test/integration/targets/ansible-test-installed/ansible_collections/ns/col/tests/integration/targets/installed/aliases @@ -0,0 +1 @@ +context/controller diff --git a/test/integration/targets/ansible-test-installed/ansible_collections/ns/col/tests/integration/targets/installed/runme.sh b/test/integration/targets/ansible-test-installed/ansible_collections/ns/col/tests/integration/targets/installed/runme.sh new file mode 100755 index 00000000000..9de3820a31c --- /dev/null +++ b/test/integration/targets/ansible-test-installed/ansible_collections/ns/col/tests/integration/targets/installed/runme.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# This test ensures that the bin entry points created by ansible-test work +# when ansible-test is running from an install instead of from source. + +set -eux + +# The third PATH entry is the injected bin directory created by ansible-test. +bin_dir="$(python -c 'import os; print(os.environ["PATH"].split(":")[2])')" + +while IFS= read -r name +do + bin="${bin_dir}/${name}" + + entry_point="${name//ansible-/}" + entry_point="${entry_point//ansible/adhoc}" + + echo "=== ${name} (${entry_point})=${bin} ===" + + if [ "${name}" == "ansible-test" ]; then + echo "skipped - ansible-test does not support self-testing from an install" + else + "${bin}" --version | tee /dev/stderr | grep -Eo "(^${name}\ \[core\ .*|executable location = ${bin}$)" + fi +done < entry-points.txt diff --git a/test/integration/targets/ansible-test-installed/runme.sh b/test/integration/targets/ansible-test-installed/runme.sh new file mode 100755 index 00000000000..8315357ebc6 --- /dev/null +++ b/test/integration/targets/ansible-test-installed/runme.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +base_dir="$(dirname "$(dirname "$(dirname "$(dirname "${OUTPUT_DIR}")")")")" +bin_dir="${base_dir}/bin" + +source ../collection/setup.sh +source virtualenv.sh + +unset PYTHONPATH + +# find the bin entry points to test +ls "${bin_dir}" > tests/integration/targets/installed/entry-points.txt + +# deps are already installed, using --no-deps to avoid re-installing them +pip install "${base_dir}" --disable-pip-version-check --no-deps + +# verify entry point generation without delegation +ansible-test integration --color --truncate 0 "${@}" + +# verify entry point generation with same-host delegation +ansible-test integration --venv --color --truncate 0 "${@}" diff --git a/test/lib/ansible_test/_internal/ansible_util.py b/test/lib/ansible_test/_internal/ansible_util.py index be88ccd8750..885489f4ba9 100644 --- a/test/lib/ansible_test/_internal/ansible_util.py +++ b/test/lib/ansible_test/_internal/ansible_util.py @@ -3,9 +3,11 @@ from __future__ import annotations import json import os +import shutil import typing as t from .constants import ( + ANSIBLE_BIN_SYMLINK_MAP, SOFT_RLIMIT_NOFILE, ) @@ -17,12 +19,15 @@ from .util import ( common_environment, ApplicationError, ANSIBLE_LIB_ROOT, + ANSIBLE_TEST_ROOT, ANSIBLE_TEST_DATA_ROOT, - ANSIBLE_BIN_PATH, + ANSIBLE_ROOT, ANSIBLE_SOURCE_ROOT, ANSIBLE_TEST_TOOLS_ROOT, + MODE_FILE_EXECUTE, get_ansible_version, raw_command, + verified_chmod, ) from .util_common import ( @@ -78,8 +83,10 @@ def ansible_environment(args: CommonConfig, color: bool = True, ansible_config: env = common_environment() path = env['PATH'] - if not path.startswith(ANSIBLE_BIN_PATH + os.path.pathsep): - path = ANSIBLE_BIN_PATH + os.path.pathsep + path + ansible_bin_path = get_ansible_bin_path(args) + + if not path.startswith(ansible_bin_path + os.path.pathsep): + path = ansible_bin_path + os.path.pathsep + path if not ansible_config: # use the default empty configuration unless one has been provided @@ -196,6 +203,52 @@ def configure_plugin_paths(args: CommonConfig) -> dict[str, str]: return env +@mutex +def get_ansible_bin_path(args: CommonConfig) -> str: + """ + Return a directory usable for PATH, containing only the ansible entry points. + If a temporary directory is required, it will be cached for the lifetime of the process and cleaned up at exit. + """ + try: + return get_ansible_bin_path.bin_path # type: ignore[attr-defined] + except AttributeError: + pass + + if ANSIBLE_SOURCE_ROOT: + # when running from source there is no need for a temporary directory since we already have known entry point scripts + bin_path = os.path.join(ANSIBLE_ROOT, 'bin') + else: + # when not running from source the installed entry points cannot be relied upon + # doing so would require using the interpreter specified by those entry points, which conflicts with using our interpreter and injector + # instead a temporary directory is created which contains only ansible entry points + # symbolic links cannot be used since the files are likely not executable + bin_path = create_temp_dir(prefix='ansible-test-', suffix='-bin') + bin_links = {os.path.join(bin_path, name): get_cli_path(path) for name, path in ANSIBLE_BIN_SYMLINK_MAP.items()} + + if not args.explain: + for dst, src in bin_links.items(): + shutil.copy(src, dst) + verified_chmod(dst, MODE_FILE_EXECUTE) + + get_ansible_bin_path.bin_path = bin_path # type: ignore[attr-defined] + + return bin_path + + +def get_cli_path(path: str) -> str: + """Return the absolute path to the CLI script from the given path which is relative to the `bin` directory of the original source tree layout.""" + path_rewrite = { + '../lib/ansible/': ANSIBLE_LIB_ROOT, + '../test/lib/ansible_test/': ANSIBLE_TEST_ROOT, + } + + for prefix, destination in path_rewrite.items(): + if path.startswith(prefix): + return os.path.join(destination, path[len(prefix):]) + + raise RuntimeError(path) + + @mutex def get_ansible_python_path(args: CommonConfig) -> str: """ diff --git a/test/lib/ansible_test/_internal/commands/sanity/bin_symlinks.py b/test/lib/ansible_test/_internal/commands/sanity/bin_symlinks.py index 8f4fe8a4c56..6c7618d168e 100644 --- a/test/lib/ansible_test/_internal/commands/sanity/bin_symlinks.py +++ b/test/lib/ansible_test/_internal/commands/sanity/bin_symlinks.py @@ -32,7 +32,7 @@ from ...payload import ( ) from ...util import ( - ANSIBLE_BIN_PATH, + ANSIBLE_SOURCE_ROOT, ) @@ -52,7 +52,7 @@ class BinSymlinksTest(SanityVersionNeutral): return True def test(self, args: SanityConfig, targets: SanityTargets) -> TestResult: - bin_root = ANSIBLE_BIN_PATH + bin_root = os.path.join(ANSIBLE_SOURCE_ROOT, 'bin') bin_names = os.listdir(bin_root) bin_paths = sorted(os.path.join(bin_root, path) for path in bin_names) diff --git a/test/lib/ansible_test/_internal/constants.py b/test/lib/ansible_test/_internal/constants.py index b6072fbee0c..fdf2d954e60 100644 --- a/test/lib/ansible_test/_internal/constants.py +++ b/test/lib/ansible_test/_internal/constants.py @@ -33,6 +33,7 @@ SECCOMP_CHOICES = [ # This bin symlink map must exactly match the contents of the bin directory. # It is necessary for payload creation to reconstruct the bin directory when running ansible-test from an installed version of ansible. # It is also used to construct the injector directory at runtime. +# It is also used to construct entry points when not running ansible-test from source. ANSIBLE_BIN_SYMLINK_MAP = { 'ansible': '../lib/ansible/cli/adhoc.py', 'ansible-config': '../lib/ansible/cli/config.py', diff --git a/test/lib/ansible_test/_internal/delegation.py b/test/lib/ansible_test/_internal/delegation.py index 7114f2abd48..f9e544558d3 100644 --- a/test/lib/ansible_test/_internal/delegation.py +++ b/test/lib/ansible_test/_internal/delegation.py @@ -33,7 +33,6 @@ from .util import ( SubprocessError, display, filter_args, - ANSIBLE_BIN_PATH, ANSIBLE_LIB_ROOT, ANSIBLE_TEST_ROOT, OutputStream, @@ -44,6 +43,10 @@ from .util_common import ( process_scoped_temporary_directory, ) +from .ansible_util import ( + get_ansible_bin_path, +) + from .containers import ( support_container_context, ContainerDatabase, @@ -145,7 +148,7 @@ def delegate_command(args: EnvironmentConfig, host_state: HostState, exclude: li con.extract_archive(chdir=working_directory, src=payload_file) else: content_root = working_directory - ansible_bin_path = ANSIBLE_BIN_PATH + ansible_bin_path = get_ansible_bin_path(args) command = generate_command(args, host_state.controller_profile.python, ansible_bin_path, content_root, exclude, require) diff --git a/test/lib/ansible_test/_internal/util.py b/test/lib/ansible_test/_internal/util.py index a5a9fabaeda..1859be5bd16 100644 --- a/test/lib/ansible_test/_internal/util.py +++ b/test/lib/ansible_test/_internal/util.py @@ -74,14 +74,12 @@ ANSIBLE_TEST_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # assume running from install ANSIBLE_ROOT = os.path.dirname(ANSIBLE_TEST_ROOT) -ANSIBLE_BIN_PATH = os.path.dirname(os.path.abspath(sys.argv[0])) ANSIBLE_LIB_ROOT = os.path.join(ANSIBLE_ROOT, 'ansible') ANSIBLE_SOURCE_ROOT = None if not os.path.exists(ANSIBLE_LIB_ROOT): # running from source ANSIBLE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(ANSIBLE_TEST_ROOT))) - ANSIBLE_BIN_PATH = os.path.join(ANSIBLE_ROOT, 'bin') ANSIBLE_LIB_ROOT = os.path.join(ANSIBLE_ROOT, 'lib', 'ansible') ANSIBLE_SOURCE_ROOT = ANSIBLE_ROOT