ansible-test - Create injector scripts at runtime. (#75761)

* ansible-test - Create injector scripts at runtime.
* Set bootstrap.sh shebang at runtime.
* Remove shebang and execute bit from importer.
* Update shebang sanity test.
* Preserve line numbers.

MANIFEST.in

@@ -27,7 +27,6 @@ recursive-include test/integration *
 recursive-include test/lib/ansible_test/config *.yml *.template
 recursive-include test/lib/ansible_test/_data *.cfg *.ini *.ps1 *.txt *.yml coveragerc
 recursive-include test/lib/ansible_test/_util *.cfg *.json *.ps1 *.psd1 *.py *.sh *.txt *.yml
-recursive-include test/lib/ansible_test/_util/target/injector ansible ansible-config ansible-connection ansible-console ansible-doc ansible-galaxy ansible-inventory ansible-playbook ansible-pull ansible-test ansible-vault pytest
 recursive-include test/lib/ansible_test/_util/controller/sanity/validate-modules validate-modules
 recursive-include test/sanity *.json *.py *.txt
 recursive-include test/support *.py *.ps1 *.psm1 *.cs

changelogs/fragments/ansible-test-injector.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - ansible-test - The "injector" scripts are now generated at runtime to avoid issues with symlinks and shebangs.

test/lib/ansible_test/_internal/ansible_util.py

@@ -20,7 +20,6 @@ from .util import (
     ANSIBLE_TEST_DATA_ROOT,
     ANSIBLE_BIN_PATH,
     ANSIBLE_SOURCE_ROOT,
-    ANSIBLE_TEST_TARGET_ROOT,
     ANSIBLE_TEST_TOOLS_ROOT,
     get_ansible_version,
 )
@@ -30,6 +29,7 @@ from .util_common import (
     run_command,
     ResultType,
     intercept_python,
+    get_injector_path,
 )
 
 from .config import (
@@ -117,7 +117,7 @@ def ansible_environment(args, color=True, ansible_config=None):
         # ansible-connection only requires the injector for code coverage
         # the correct python interpreter is already selected using the sys.executable used to invoke ansible
         ansible.update(dict(
-            ANSIBLE_CONNECTION_PATH=os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'injector', 'ansible-connection'),
+            ANSIBLE_CONNECTION_PATH=os.path.join(get_injector_path(), 'ansible-connection'),
         ))
 
     if isinstance(args, PosixIntegrationConfig):

test/lib/ansible_test/_internal/bootstrap.py

@@ -15,6 +15,7 @@ from .util import (
 
 from .util_common import (
     ShellScriptTemplate,
+    set_shebang,
 )
 
 from .core_ci import (
@@ -48,7 +49,10 @@ class Bootstrap:
     def get_script(self):  # type: () -> str
         """Return a shell script to bootstrap the specified host."""
         path = os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'setup', 'bootstrap.sh')
+
         content = read_text_file(path)
+        content = set_shebang(content, '/bin/sh')
+
         template = ShellScriptTemplate(content)
 
         variables = self.get_variables()

test/lib/ansible_test/_internal/commands/sanity/bin_symlinks.py

@@ -12,6 +12,10 @@ from . import (
     SanityTargets,
 )
 
+from ...constants import (
+    __file__ as symlink_map_full_path,
+)
+
 from ...test import (
     TestResult,
 )
@@ -26,12 +30,10 @@ from ...data import (
 
 from ...payload import (
     ANSIBLE_BIN_SYMLINK_MAP,
-    __file__ as symlink_map_full_path,
 )
 
 from ...util import (
     ANSIBLE_BIN_PATH,
-    ANSIBLE_TEST_TARGET_ROOT,
 )
 
 
@@ -54,9 +56,6 @@ class BinSymlinksTest(SanityVersionNeutral):
         bin_names = os.listdir(bin_root)
         bin_paths = sorted(os.path.join(bin_root, path) for path in bin_names)
 
-        injector_root = os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'injector')
-        injector_names = os.listdir(injector_root)
-
         errors = []  # type: t.List[t.Tuple[str, str]]
 
         symlink_map_path = os.path.relpath(symlink_map_full_path, data_context().content.root)
@@ -95,10 +94,6 @@ class BinSymlinksTest(SanityVersionNeutral):
                 bin_path = os.path.join(bin_root, bin_name)
                 errors.append((bin_path, 'missing symlink to "%s" defined in ANSIBLE_BIN_SYMLINK_MAP in file "%s"' % (dest, symlink_map_path)))
 
-            if bin_name not in injector_names:
-                injector_path = os.path.join(injector_root, bin_name)
-                errors.append((injector_path, 'missing symlink to "python.py"'))
-
         messages = [SanityMessage(message=message, path=os.path.relpath(path, data_context().content.root), confidence=100) for path, message in errors]
 
         if errors:

test/lib/ansible_test/_internal/payload.py

@@ -9,6 +9,10 @@ import tempfile
 import time
 import typing as t
 
+from .constants import (
+    ANSIBLE_BIN_SYMLINK_MAP,
+)
+
 from .config import (
     IntegrationConfig,
     ShellConfig,
@@ -33,22 +37,6 @@ from .util_common import (
 tarfile.pwd = None
 tarfile.grp = None
 
-# this bin symlink map must exactly match the contents of the bin directory
-# it is necessary for payload creation to reconstruct the bin directory when running ansible-test from an installed version of ansible
-ANSIBLE_BIN_SYMLINK_MAP = {
-    'ansible': '../lib/ansible/cli/scripts/ansible_cli_stub.py',
-    'ansible-config': 'ansible',
-    'ansible-connection': '../lib/ansible/cli/scripts/ansible_connection_cli_stub.py',
-    'ansible-console': 'ansible',
-    'ansible-doc': 'ansible',
-    'ansible-galaxy': 'ansible',
-    'ansible-inventory': 'ansible',
-    'ansible-playbook': 'ansible',
-    'ansible-pull': 'ansible',
-    'ansible-test': '../test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py',
-    'ansible-vault': 'ansible',
-}
-
 
 def create_payload(args, dst_path):  # type: (CommonConfig, str) -> None
     """Create a payload for delegation."""

test/lib/ansible_test/_internal/util_common.py

@@ -12,15 +12,21 @@ import tempfile
 import textwrap
 import typing as t
 
+from .constants import (
+    ANSIBLE_BIN_SYMLINK_MAP,
+)
+
 from .encoding import (
     to_bytes,
 )
 
 from .util import (
+    cache,
     display,
     remove_tree,
     MODE_DIRECTORY,
     MODE_FILE_EXECUTE,
+    MODE_FILE,
     PYTHON_PATHS,
     raw_command,
     ANSIBLE_TEST_DATA_ROOT,
@@ -32,6 +38,7 @@ from .util import (
 
 from .io import (
     make_dirs,
+    read_text_file,
     write_text_file,
     write_json_file,
 )
@@ -226,6 +233,72 @@ def write_text_test_results(category, name, content):  # type: (ResultType, str,
     write_text_file(path, content, create_directories=True)
 
 
+@cache
+def get_injector_path():  # type: () -> str
+    """Return the path to a directory which contains a `python.py` executable and associated injector scripts."""
+    injector_path = tempfile.mkdtemp(prefix='ansible-test-', suffix='-injector', dir='/tmp')
+
+    display.info(f'Initializing "{injector_path}" as the temporary injector directory.', verbosity=1)
+
+    injector_names = sorted(list(ANSIBLE_BIN_SYMLINK_MAP) + [
+        'importer.py',
+        'pytest',
+    ])
+
+    scripts = (
+        ('python.py', '/usr/bin/env python', MODE_FILE_EXECUTE),
+        ('virtualenv.sh', '/usr/bin/env bash', MODE_FILE),
+    )
+
+    source_path = os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'injector')
+
+    for name in injector_names:
+        os.symlink('python.py', os.path.join(injector_path, name))
+
+    for name, shebang, mode in scripts:
+        src = os.path.join(source_path, name)
+        dst = os.path.join(injector_path, name)
+
+        script = read_text_file(src)
+        script = set_shebang(script, shebang)
+
+        write_text_file(dst, script)
+        os.chmod(dst, mode)
+
+    os.chmod(injector_path, MODE_DIRECTORY)
+
+    def cleanup_injector():
+        """Remove the temporary injector directory."""
+        remove_tree(injector_path)
+
+    atexit.register(cleanup_injector)
+
+    return injector_path
+
+
+def set_shebang(script, executable):  # type: (str, str) -> str
+    """Return the given script with the specified executable used for the shebang."""
+    prefix = '#!'
+    shebang = prefix + executable
+
+    overwrite = (
+        prefix,
+        '# auto-shebang',
+        '# shellcheck shell=',
+    )
+
+    lines = script.splitlines()
+
+    if any(lines[0].startswith(value) for value in overwrite):
+        lines[0] = shebang
+    else:
+        lines.insert(0, shebang)
+
+    script = '\n'.join(lines)
+
+    return script
+
+
 def get_python_path(interpreter):  # type: (str) -> str
     """Return the path to a directory which contains a `python` executable that runs the specified interpreter."""
     python_path = PYTHON_PATHS.get(interpreter)
@@ -318,7 +391,7 @@ def intercept_python(
     """
     env = env.copy()
     cmd = list(cmd)
-    inject_path = os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'injector')
+    inject_path = get_injector_path()
 
     # make sure scripts (including injector.py) find the correct Python interpreter
     if isinstance(python, VirtualPythonConfig):

test/lib/ansible_test/_util/controller/sanity/code-smell/shebang.py

@@ -68,8 +68,8 @@ def main():
                 is_module = True
             elif re.search('^test/support/[^/]+/collections/ansible_collections/[^/]+/[^/]+/plugins/modules/', path):
                 is_module = True
-            elif path.startswith('test/lib/ansible_test/_util/target/'):
-                pass
+            elif path == 'test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py':
+                pass  # ansible-test entry point must be executable and have a shebang
             elif path.startswith('lib/') or path.startswith('test/lib/'):
                 if executable:
                     print('%s:%d:%d: should not be executable' % (path, 0, 0))

test/lib/ansible_test/_util/target/common/constants.py

@@ -43,3 +43,20 @@ SECCOMP_CHOICES = [
     'default',
     'unconfined',
 ]
+
+# This bin symlink map must exactly match the contents of the bin directory.
+# It is necessary for payload creation to reconstruct the bin directory when running ansible-test from an installed version of ansible.
+# It is also used to construct the injector directory at runtime.
+ANSIBLE_BIN_SYMLINK_MAP = {
+    'ansible': '../lib/ansible/cli/scripts/ansible_cli_stub.py',
+    'ansible-config': 'ansible',
+    'ansible-connection': '../lib/ansible/cli/scripts/ansible_connection_cli_stub.py',
+    'ansible-console': 'ansible',
+    'ansible-doc': 'ansible',
+    'ansible-galaxy': 'ansible',
+    'ansible-inventory': 'ansible',
+    'ansible-playbook': 'ansible',
+    'ansible-pull': 'ansible',
+    'ansible-test': '../test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py',
+    'ansible-vault': 'ansible',
+}

test/lib/ansible_test/_util/target/injector/ansible

@@ -1 +0,0 @@
-python.py

test/lib/ansible_test/_util/target/injector/ansible-config

@@ -1 +0,0 @@
-python.py

test/lib/ansible_test/_util/target/injector/ansible-connection

@@ -1 +0,0 @@
-python.py

test/lib/ansible_test/_util/target/injector/ansible-console

@@ -1 +0,0 @@
-python.py

test/lib/ansible_test/_util/target/injector/ansible-doc

@@ -1 +0,0 @@
-python.py

test/lib/ansible_test/_util/target/injector/ansible-galaxy

@@ -1 +0,0 @@
-python.py

test/lib/ansible_test/_util/target/injector/ansible-inventory

@@ -1 +0,0 @@
-python.py

test/lib/ansible_test/_util/target/injector/ansible-playbook

@@ -1 +0,0 @@
-python.py

test/lib/ansible_test/_util/target/injector/ansible-pull

@@ -1 +0,0 @@
-python.py

test/lib/ansible_test/_util/target/injector/ansible-test

@@ -1 +0,0 @@
-python.py

test/lib/ansible_test/_util/target/injector/ansible-vault

@@ -1 +0,0 @@
-python.py

test/lib/ansible_test/_util/target/injector/importer.py

@@ -1 +0,0 @@
-python.py

test/lib/ansible_test/_util/target/injector/pytest

@@ -1 +0,0 @@
-python.py

test/lib/ansible_test/_util/target/injector/python.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+# auto-shebang
 """Provides an entry point for python scripts and python modules on the controller with the current python interpreter and optional code coverage collection."""
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
@@ -46,21 +46,24 @@ def main():
             sys.exit('ERROR: Use `python -c` instead of `python.py -c` to avoid errors when code coverage is collected.')
     elif name == 'pytest':
         args += ['-m', 'pytest']
+    elif name == 'importer.py':
+        args += [find_program(name, False)]
     else:
-        args += [find_executable(name)]
+        args += [find_program(name, True)]
 
     args += sys.argv[1:]
 
     os.execv(args[0], args)
 
 
-def find_executable(name):
+def find_program(name, executable):  # type: (str, bool) -> str
     """
-    :type name: str
-    :rtype: str
+    Find and return the full path to the named program, optionally requiring it to be executable.
+    Raises an exception if the program is not found.
     """
     path = os.environ.get('PATH', os.path.defpath)
     seen = set([os.path.abspath(__file__)])
+    mode = os.F_OK | os.X_OK if executable else os.F_OK
 
     for base in path.split(os.path.pathsep):
         candidate = os.path.abspath(os.path.join(base, name))
@@ -70,7 +73,7 @@ def find_executable(name):
 
         seen.add(candidate)
 
-        if os.path.exists(candidate) and os.access(candidate, os.F_OK | os.X_OK):
+        if os.path.exists(candidate) and os.access(candidate, mode):
             return candidate
 
     raise Exception('Executable "%s" not found in path: %s' % (name, path))

test/lib/ansible_test/_util/target/injector/virtualenv.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+# shellcheck shell=bash
 # Create and activate a fresh virtual environment with `source virtualenv.sh`.
 
 rm -rf "${OUTPUT_DIR}/venv"

test/lib/ansible_test/_util/target/sanity/import/importer.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 """Import the given python module(s) and report error(s) encountered."""
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type

test/lib/ansible_test/_util/target/setup/bootstrap.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+# shellcheck shell=sh
 
 set -eu