diff --git a/MANIFEST.in b/MANIFEST.in index 5fbecb45970..b87e4c388b5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -27,7 +27,6 @@ recursive-include test/integration * recursive-include test/lib/ansible_test/config *.yml *.template recursive-include test/lib/ansible_test/_data *.cfg *.ini *.ps1 *.txt *.yml coveragerc recursive-include test/lib/ansible_test/_util *.cfg *.json *.ps1 *.psd1 *.py *.sh *.txt *.yml -recursive-include test/lib/ansible_test/_util/target/injector ansible ansible-config ansible-connection ansible-console ansible-doc ansible-galaxy ansible-inventory ansible-playbook ansible-pull ansible-test ansible-vault pytest recursive-include test/lib/ansible_test/_util/controller/sanity/validate-modules validate-modules recursive-include test/sanity *.json *.py *.txt recursive-include test/support *.py *.ps1 *.psm1 *.cs diff --git a/changelogs/fragments/ansible-test-injector.yml b/changelogs/fragments/ansible-test-injector.yml new file mode 100644 index 00000000000..4e13f1069a1 --- /dev/null +++ b/changelogs/fragments/ansible-test-injector.yml @@ -0,0 +1,2 @@ +minor_changes: + - ansible-test - The "injector" scripts are now generated at runtime to avoid issues with symlinks and shebangs. diff --git a/test/lib/ansible_test/_internal/ansible_util.py b/test/lib/ansible_test/_internal/ansible_util.py index 1d98ec2b9fd..8de04af36ab 100644 --- a/test/lib/ansible_test/_internal/ansible_util.py +++ b/test/lib/ansible_test/_internal/ansible_util.py @@ -20,7 +20,6 @@ from .util import ( ANSIBLE_TEST_DATA_ROOT, ANSIBLE_BIN_PATH, ANSIBLE_SOURCE_ROOT, - ANSIBLE_TEST_TARGET_ROOT, ANSIBLE_TEST_TOOLS_ROOT, get_ansible_version, ) @@ -30,6 +29,7 @@ from .util_common import ( run_command, ResultType, intercept_python, + get_injector_path, ) from .config import ( @@ -117,7 +117,7 @@ def ansible_environment(args, color=True, ansible_config=None): # ansible-connection only requires the injector for code coverage # the correct python interpreter is already selected using the sys.executable used to invoke ansible ansible.update(dict( - ANSIBLE_CONNECTION_PATH=os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'injector', 'ansible-connection'), + ANSIBLE_CONNECTION_PATH=os.path.join(get_injector_path(), 'ansible-connection'), )) if isinstance(args, PosixIntegrationConfig): diff --git a/test/lib/ansible_test/_internal/bootstrap.py b/test/lib/ansible_test/_internal/bootstrap.py index 6f675aeb8f8..9eb26de7d2e 100644 --- a/test/lib/ansible_test/_internal/bootstrap.py +++ b/test/lib/ansible_test/_internal/bootstrap.py @@ -15,6 +15,7 @@ from .util import ( from .util_common import ( ShellScriptTemplate, + set_shebang, ) from .core_ci import ( @@ -48,7 +49,10 @@ class Bootstrap: def get_script(self): # type: () -> str """Return a shell script to bootstrap the specified host.""" path = os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'setup', 'bootstrap.sh') + content = read_text_file(path) + content = set_shebang(content, '/bin/sh') + template = ShellScriptTemplate(content) variables = self.get_variables() diff --git a/test/lib/ansible_test/_internal/commands/sanity/bin_symlinks.py b/test/lib/ansible_test/_internal/commands/sanity/bin_symlinks.py index 067cc0de928..5dc582fa0ee 100644 --- a/test/lib/ansible_test/_internal/commands/sanity/bin_symlinks.py +++ b/test/lib/ansible_test/_internal/commands/sanity/bin_symlinks.py @@ -12,6 +12,10 @@ from . import ( SanityTargets, ) +from ...constants import ( + __file__ as symlink_map_full_path, +) + from ...test import ( TestResult, ) @@ -26,12 +30,10 @@ from ...data import ( from ...payload import ( ANSIBLE_BIN_SYMLINK_MAP, - __file__ as symlink_map_full_path, ) from ...util import ( ANSIBLE_BIN_PATH, - ANSIBLE_TEST_TARGET_ROOT, ) @@ -54,9 +56,6 @@ class BinSymlinksTest(SanityVersionNeutral): bin_names = os.listdir(bin_root) bin_paths = sorted(os.path.join(bin_root, path) for path in bin_names) - injector_root = os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'injector') - injector_names = os.listdir(injector_root) - errors = [] # type: t.List[t.Tuple[str, str]] symlink_map_path = os.path.relpath(symlink_map_full_path, data_context().content.root) @@ -95,10 +94,6 @@ class BinSymlinksTest(SanityVersionNeutral): bin_path = os.path.join(bin_root, bin_name) errors.append((bin_path, 'missing symlink to "%s" defined in ANSIBLE_BIN_SYMLINK_MAP in file "%s"' % (dest, symlink_map_path))) - if bin_name not in injector_names: - injector_path = os.path.join(injector_root, bin_name) - errors.append((injector_path, 'missing symlink to "python.py"')) - messages = [SanityMessage(message=message, path=os.path.relpath(path, data_context().content.root), confidence=100) for path, message in errors] if errors: diff --git a/test/lib/ansible_test/_internal/payload.py b/test/lib/ansible_test/_internal/payload.py index 7168f6de57b..d92f9f6589f 100644 --- a/test/lib/ansible_test/_internal/payload.py +++ b/test/lib/ansible_test/_internal/payload.py @@ -9,6 +9,10 @@ import tempfile import time import typing as t +from .constants import ( + ANSIBLE_BIN_SYMLINK_MAP, +) + from .config import ( IntegrationConfig, ShellConfig, @@ -33,22 +37,6 @@ from .util_common import ( tarfile.pwd = None tarfile.grp = None -# this bin symlink map must exactly match the contents of the bin directory -# it is necessary for payload creation to reconstruct the bin directory when running ansible-test from an installed version of ansible -ANSIBLE_BIN_SYMLINK_MAP = { - 'ansible': '../lib/ansible/cli/scripts/ansible_cli_stub.py', - 'ansible-config': 'ansible', - 'ansible-connection': '../lib/ansible/cli/scripts/ansible_connection_cli_stub.py', - 'ansible-console': 'ansible', - 'ansible-doc': 'ansible', - 'ansible-galaxy': 'ansible', - 'ansible-inventory': 'ansible', - 'ansible-playbook': 'ansible', - 'ansible-pull': 'ansible', - 'ansible-test': '../test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py', - 'ansible-vault': 'ansible', -} - def create_payload(args, dst_path): # type: (CommonConfig, str) -> None """Create a payload for delegation.""" diff --git a/test/lib/ansible_test/_internal/util_common.py b/test/lib/ansible_test/_internal/util_common.py index 1850d8049f1..37e4cd64724 100644 --- a/test/lib/ansible_test/_internal/util_common.py +++ b/test/lib/ansible_test/_internal/util_common.py @@ -12,15 +12,21 @@ import tempfile import textwrap import typing as t +from .constants import ( + ANSIBLE_BIN_SYMLINK_MAP, +) + from .encoding import ( to_bytes, ) from .util import ( + cache, display, remove_tree, MODE_DIRECTORY, MODE_FILE_EXECUTE, + MODE_FILE, PYTHON_PATHS, raw_command, ANSIBLE_TEST_DATA_ROOT, @@ -32,6 +38,7 @@ from .util import ( from .io import ( make_dirs, + read_text_file, write_text_file, write_json_file, ) @@ -226,6 +233,72 @@ def write_text_test_results(category, name, content): # type: (ResultType, str, write_text_file(path, content, create_directories=True) +@cache +def get_injector_path(): # type: () -> str + """Return the path to a directory which contains a `python.py` executable and associated injector scripts.""" + injector_path = tempfile.mkdtemp(prefix='ansible-test-', suffix='-injector', dir='/tmp') + + display.info(f'Initializing "{injector_path}" as the temporary injector directory.', verbosity=1) + + injector_names = sorted(list(ANSIBLE_BIN_SYMLINK_MAP) + [ + 'importer.py', + 'pytest', + ]) + + scripts = ( + ('python.py', '/usr/bin/env python', MODE_FILE_EXECUTE), + ('virtualenv.sh', '/usr/bin/env bash', MODE_FILE), + ) + + source_path = os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'injector') + + for name in injector_names: + os.symlink('python.py', os.path.join(injector_path, name)) + + for name, shebang, mode in scripts: + src = os.path.join(source_path, name) + dst = os.path.join(injector_path, name) + + script = read_text_file(src) + script = set_shebang(script, shebang) + + write_text_file(dst, script) + os.chmod(dst, mode) + + os.chmod(injector_path, MODE_DIRECTORY) + + def cleanup_injector(): + """Remove the temporary injector directory.""" + remove_tree(injector_path) + + atexit.register(cleanup_injector) + + return injector_path + + +def set_shebang(script, executable): # type: (str, str) -> str + """Return the given script with the specified executable used for the shebang.""" + prefix = '#!' + shebang = prefix + executable + + overwrite = ( + prefix, + '# auto-shebang', + '# shellcheck shell=', + ) + + lines = script.splitlines() + + if any(lines[0].startswith(value) for value in overwrite): + lines[0] = shebang + else: + lines.insert(0, shebang) + + script = '\n'.join(lines) + + return script + + def get_python_path(interpreter): # type: (str) -> str """Return the path to a directory which contains a `python` executable that runs the specified interpreter.""" python_path = PYTHON_PATHS.get(interpreter) @@ -318,7 +391,7 @@ def intercept_python( """ env = env.copy() cmd = list(cmd) - inject_path = os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'injector') + inject_path = get_injector_path() # make sure scripts (including injector.py) find the correct Python interpreter if isinstance(python, VirtualPythonConfig): diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/shebang.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/shebang.py index 6f210651c6b..401af1aee68 100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/shebang.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/shebang.py @@ -68,8 +68,8 @@ def main(): is_module = True elif re.search('^test/support/[^/]+/collections/ansible_collections/[^/]+/[^/]+/plugins/modules/', path): is_module = True - elif path.startswith('test/lib/ansible_test/_util/target/'): - pass + elif path == 'test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py': + pass # ansible-test entry point must be executable and have a shebang elif path.startswith('lib/') or path.startswith('test/lib/'): if executable: print('%s:%d:%d: should not be executable' % (path, 0, 0)) diff --git a/test/lib/ansible_test/_util/target/common/constants.py b/test/lib/ansible_test/_util/target/common/constants.py index 9902b046a69..3c02eb2ec5d 100644 --- a/test/lib/ansible_test/_util/target/common/constants.py +++ b/test/lib/ansible_test/_util/target/common/constants.py @@ -43,3 +43,20 @@ SECCOMP_CHOICES = [ 'default', 'unconfined', ] + +# This bin symlink map must exactly match the contents of the bin directory. +# It is necessary for payload creation to reconstruct the bin directory when running ansible-test from an installed version of ansible. +# It is also used to construct the injector directory at runtime. +ANSIBLE_BIN_SYMLINK_MAP = { + 'ansible': '../lib/ansible/cli/scripts/ansible_cli_stub.py', + 'ansible-config': 'ansible', + 'ansible-connection': '../lib/ansible/cli/scripts/ansible_connection_cli_stub.py', + 'ansible-console': 'ansible', + 'ansible-doc': 'ansible', + 'ansible-galaxy': 'ansible', + 'ansible-inventory': 'ansible', + 'ansible-playbook': 'ansible', + 'ansible-pull': 'ansible', + 'ansible-test': '../test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py', + 'ansible-vault': 'ansible', +} diff --git a/test/lib/ansible_test/_util/target/injector/ansible b/test/lib/ansible_test/_util/target/injector/ansible deleted file mode 120000 index 6bbbfe4d919..00000000000 --- a/test/lib/ansible_test/_util/target/injector/ansible +++ /dev/null @@ -1 +0,0 @@ -python.py \ No newline at end of file diff --git a/test/lib/ansible_test/_util/target/injector/ansible-config b/test/lib/ansible_test/_util/target/injector/ansible-config deleted file mode 120000 index 6bbbfe4d919..00000000000 --- a/test/lib/ansible_test/_util/target/injector/ansible-config +++ /dev/null @@ -1 +0,0 @@ -python.py \ No newline at end of file diff --git a/test/lib/ansible_test/_util/target/injector/ansible-connection b/test/lib/ansible_test/_util/target/injector/ansible-connection deleted file mode 120000 index 6bbbfe4d919..00000000000 --- a/test/lib/ansible_test/_util/target/injector/ansible-connection +++ /dev/null @@ -1 +0,0 @@ -python.py \ No newline at end of file diff --git a/test/lib/ansible_test/_util/target/injector/ansible-console b/test/lib/ansible_test/_util/target/injector/ansible-console deleted file mode 120000 index 6bbbfe4d919..00000000000 --- a/test/lib/ansible_test/_util/target/injector/ansible-console +++ /dev/null @@ -1 +0,0 @@ -python.py \ No newline at end of file diff --git a/test/lib/ansible_test/_util/target/injector/ansible-doc b/test/lib/ansible_test/_util/target/injector/ansible-doc deleted file mode 120000 index 6bbbfe4d919..00000000000 --- a/test/lib/ansible_test/_util/target/injector/ansible-doc +++ /dev/null @@ -1 +0,0 @@ -python.py \ No newline at end of file diff --git a/test/lib/ansible_test/_util/target/injector/ansible-galaxy b/test/lib/ansible_test/_util/target/injector/ansible-galaxy deleted file mode 120000 index 6bbbfe4d919..00000000000 --- a/test/lib/ansible_test/_util/target/injector/ansible-galaxy +++ /dev/null @@ -1 +0,0 @@ -python.py \ No newline at end of file diff --git a/test/lib/ansible_test/_util/target/injector/ansible-inventory b/test/lib/ansible_test/_util/target/injector/ansible-inventory deleted file mode 120000 index 6bbbfe4d919..00000000000 --- a/test/lib/ansible_test/_util/target/injector/ansible-inventory +++ /dev/null @@ -1 +0,0 @@ -python.py \ No newline at end of file diff --git a/test/lib/ansible_test/_util/target/injector/ansible-playbook b/test/lib/ansible_test/_util/target/injector/ansible-playbook deleted file mode 120000 index 6bbbfe4d919..00000000000 --- a/test/lib/ansible_test/_util/target/injector/ansible-playbook +++ /dev/null @@ -1 +0,0 @@ -python.py \ No newline at end of file diff --git a/test/lib/ansible_test/_util/target/injector/ansible-pull b/test/lib/ansible_test/_util/target/injector/ansible-pull deleted file mode 120000 index 6bbbfe4d919..00000000000 --- a/test/lib/ansible_test/_util/target/injector/ansible-pull +++ /dev/null @@ -1 +0,0 @@ -python.py \ No newline at end of file diff --git a/test/lib/ansible_test/_util/target/injector/ansible-test b/test/lib/ansible_test/_util/target/injector/ansible-test deleted file mode 120000 index 6bbbfe4d919..00000000000 --- a/test/lib/ansible_test/_util/target/injector/ansible-test +++ /dev/null @@ -1 +0,0 @@ -python.py \ No newline at end of file diff --git a/test/lib/ansible_test/_util/target/injector/ansible-vault b/test/lib/ansible_test/_util/target/injector/ansible-vault deleted file mode 120000 index 6bbbfe4d919..00000000000 --- a/test/lib/ansible_test/_util/target/injector/ansible-vault +++ /dev/null @@ -1 +0,0 @@ -python.py \ No newline at end of file diff --git a/test/lib/ansible_test/_util/target/injector/importer.py b/test/lib/ansible_test/_util/target/injector/importer.py deleted file mode 120000 index 6bbbfe4d919..00000000000 --- a/test/lib/ansible_test/_util/target/injector/importer.py +++ /dev/null @@ -1 +0,0 @@ -python.py \ No newline at end of file diff --git a/test/lib/ansible_test/_util/target/injector/pytest b/test/lib/ansible_test/_util/target/injector/pytest deleted file mode 120000 index 6bbbfe4d919..00000000000 --- a/test/lib/ansible_test/_util/target/injector/pytest +++ /dev/null @@ -1 +0,0 @@ -python.py \ No newline at end of file diff --git a/test/lib/ansible_test/_util/target/injector/python.py b/test/lib/ansible_test/_util/target/injector/python.py old mode 100755 new mode 100644 index 1063d1f011b..67516396670 --- a/test/lib/ansible_test/_util/target/injector/python.py +++ b/test/lib/ansible_test/_util/target/injector/python.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +# auto-shebang """Provides an entry point for python scripts and python modules on the controller with the current python interpreter and optional code coverage collection.""" from __future__ import (absolute_import, division, print_function) __metaclass__ = type @@ -46,21 +46,24 @@ def main(): sys.exit('ERROR: Use `python -c` instead of `python.py -c` to avoid errors when code coverage is collected.') elif name == 'pytest': args += ['-m', 'pytest'] + elif name == 'importer.py': + args += [find_program(name, False)] else: - args += [find_executable(name)] + args += [find_program(name, True)] args += sys.argv[1:] os.execv(args[0], args) -def find_executable(name): +def find_program(name, executable): # type: (str, bool) -> str """ - :type name: str - :rtype: str + Find and return the full path to the named program, optionally requiring it to be executable. + Raises an exception if the program is not found. """ path = os.environ.get('PATH', os.path.defpath) seen = set([os.path.abspath(__file__)]) + mode = os.F_OK | os.X_OK if executable else os.F_OK for base in path.split(os.path.pathsep): candidate = os.path.abspath(os.path.join(base, name)) @@ -70,7 +73,7 @@ def find_executable(name): seen.add(candidate) - if os.path.exists(candidate) and os.access(candidate, os.F_OK | os.X_OK): + if os.path.exists(candidate) and os.access(candidate, mode): return candidate raise Exception('Executable "%s" not found in path: %s' % (name, path)) diff --git a/test/lib/ansible_test/_util/target/injector/virtualenv.sh b/test/lib/ansible_test/_util/target/injector/virtualenv.sh index cb19a7ce475..5dcbe0e058a 100644 --- a/test/lib/ansible_test/_util/target/injector/virtualenv.sh +++ b/test/lib/ansible_test/_util/target/injector/virtualenv.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +# shellcheck shell=bash # Create and activate a fresh virtual environment with `source virtualenv.sh`. rm -rf "${OUTPUT_DIR}/venv" diff --git a/test/lib/ansible_test/_util/target/sanity/import/importer.py b/test/lib/ansible_test/_util/target/sanity/import/importer.py old mode 100755 new mode 100644 index f0659d9b4e9..778643bb58d --- a/test/lib/ansible_test/_util/target/sanity/import/importer.py +++ b/test/lib/ansible_test/_util/target/sanity/import/importer.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Import the given python module(s) and report error(s) encountered.""" from __future__ import (absolute_import, division, print_function) __metaclass__ = type diff --git a/test/lib/ansible_test/_util/target/setup/bootstrap.sh b/test/lib/ansible_test/_util/target/setup/bootstrap.sh index 36ca68f4942..2d31945e751 100644 --- a/test/lib/ansible_test/_util/target/setup/bootstrap.sh +++ b/test/lib/ansible_test/_util/target/setup/bootstrap.sh @@ -1,4 +1,4 @@ -#!/bin/sh +# shellcheck shell=sh set -eu