[stable-2.9] Fix issues with ansible-test --venv option. (#62033)

* Fix ansible-test venv activation.

When using the ansible-test --venv option, an execv wrapper for each python interpreter is now used instead of a symbolic link.

* Fix ansible-test execv wrapper generation.

Use the currently running Python interpreter for the shebang in the execv wrapper instead of the selected interpreter.

This allows the wrapper to work when the selected interpreter is a script instead of a binary.

* Fix ansible-test sanity requirements install.

When running sanity tests on multiple Python versions, install requirements for all versions used instead of only the default version.

* Fix ansible-test --venv when installed.

When running ansible-test from an install, the --venv delegation option needs to make sure the ansible-test code is available in the created virtual environment.

Exposing system site packages does not work because the virtual environment may be for a different Python version than the one on which ansible-test is installed.
(cherry picked from commit c77ab11051)

Co-authored-by: Matt Clay <matt@mystile.com>

changelogs/fragments/ansible-test-execv-wrapper-shebang.yml

@@ -0,0 +1,2 @@
+bugfixes:
+    - ansible-test now properly handles creation of Python execv wrappers when the selected interpreter is a script

changelogs/fragments/ansible-test-sanity-requirements.yml

@@ -0,0 +1,2 @@
+bugfixes:
+    - ansible-test now properly installs requirements for multiple Python versions when running sanity tests

changelogs/fragments/ansible-test-venv-activation.yml

@@ -0,0 +1,2 @@
+bugfixes:
+    - ansible-test now properly activates virtual environments created using the --venv option

changelogs/fragments/ansible-test-venv-pythonpath.yml

@@ -0,0 +1,2 @@
+bugfixes:
+    - ansible-test now properly registers its own code in a virtual environment when running from an install

test/lib/ansible_test/_internal/delegation.py

@@ -48,12 +48,16 @@ from .util import (
     display,
     ANSIBLE_BIN_PATH,
     ANSIBLE_TEST_DATA_ROOT,
+    ANSIBLE_LIB_ROOT,
+    ANSIBLE_TEST_ROOT,
     tempdir,
+    make_dirs,
 )
 
 from .util_common import (
     run_command,
     ResultType,
+    create_interpreter_wrapper,
 )
 
 from .docker_util import (
@@ -244,7 +248,7 @@ def delegate_venv(args,  # type: EnvironmentConfig
 
     with tempdir() as inject_path:
         for version, path in venvs.items():
-            os.symlink(os.path.join(path, 'bin', 'python'), os.path.join(inject_path, 'python%s' % version))
+            create_interpreter_wrapper(os.path.join(path, 'bin', 'python'), os.path.join(inject_path, 'python%s' % version))
 
         python_interpreter = os.path.join(inject_path, 'python%s' % args.python_version)
 
@@ -255,11 +259,18 @@ def delegate_venv(args,  # type: EnvironmentConfig
                 cmd += ['--coverage-label', 'venv']
 
         env = common_environment()
-        env.update(
-            PATH=inject_path + os.pathsep + env['PATH'],
-        )
 
-        run_command(args, cmd, env=env)
+        with tempdir() as library_path:
+            # expose ansible and ansible_test to the virtual environment (only required when running from an install)
+            os.symlink(ANSIBLE_LIB_ROOT, os.path.join(library_path, 'ansible'))
+            os.symlink(ANSIBLE_TEST_ROOT, os.path.join(library_path, 'ansible_test'))
+
+            env.update(
+                PATH=inject_path + os.pathsep + env['PATH'],
+                PYTHONPATH=library_path,
+            )
+
+            run_command(args, cmd, env=env)
 
 
 def delegate_docker(args, exclude, require, integration_targets):

test/lib/ansible_test/_internal/sanity/__init__.py

@@ -89,8 +89,6 @@ def command_sanity(args):
     if args.delegate:
         raise Delegate(require=changes, exclude=args.exclude)
 
-    install_command_requirements(args)
-
     tests = sanity_get_tests()
 
     if args.test:
@@ -108,6 +106,8 @@ def command_sanity(args):
     total = 0
     failed = []
 
+    requirements_installed = set()  # type: t.Set[str]
+
     for test in tests:
         if args.list_tests:
             display.info(test.name)
@@ -180,6 +180,10 @@ def command_sanity(args):
                 sanity_targets = SanityTargets(tuple(all_targets), tuple(usable_targets))
 
                 if usable_targets or test.no_targets:
+                    if version not in requirements_installed:
+                        requirements_installed.add(version)
+                        install_command_requirements(args, version)
+
                     if isinstance(test, SanityCodeSmellTest):
                         result = test.test(args, sanity_targets, version)
                     elif isinstance(test, SanityMultipleVersion):

test/lib/ansible_test/_internal/util_common.py

@@ -7,6 +7,7 @@ import contextlib
 import json
 import os
 import shutil
+import sys
 import tempfile
 import textwrap
 
@@ -204,31 +205,40 @@ def get_python_path(args, interpreter):
     else:
         display.info('Injecting "%s" as a execv wrapper for the "%s" interpreter.' % (injected_interpreter, interpreter), verbosity=1)
 
-        code = textwrap.dedent('''
+        create_interpreter_wrapper(interpreter, injected_interpreter)
-        #!%s
 
-        from __future__ import absolute_import
+    os.chmod(python_path, MODE_DIRECTORY)
 
-        from os import execv
+    if not PYTHON_PATHS:
-        from sys import argv
+        atexit.register(cleanup_python_paths)
 
-        python = '%s'
+    PYTHON_PATHS[interpreter] = python_path
 
-        execv(python, [python] + argv[1:])
+    return python_path
-        ''' % (interpreter, interpreter)).lstrip()
 
-        write_text_file(injected_interpreter, code)
 
-        os.chmod(injected_interpreter, MODE_FILE_EXECUTE)
+def create_interpreter_wrapper(interpreter, injected_interpreter):  # type: (str, str) -> None
+    """Create a wrapper for the given Python interpreter at the specified path."""
+    # sys.executable is used for the shebang to guarantee it is a binary instead of a script
+    # injected_interpreter could be a script from the system or our own wrapper created for the --venv option
+    shebang_interpreter = sys.executable
 
-    os.chmod(python_path, MODE_DIRECTORY)
+    code = textwrap.dedent('''
+    #!%s
 
-    if not PYTHON_PATHS:
+    from __future__ import absolute_import
-        atexit.register(cleanup_python_paths)
 
-    PYTHON_PATHS[interpreter] = python_path
+    from os import execv
+    from sys import argv
 
-    return python_path
+    python = '%s'
+
+    execv(python, [python] + argv[1:])
+    ''' % (shebang_interpreter, interpreter)).lstrip()
+
+    write_text_file(injected_interpreter, code)
+
+    os.chmod(injected_interpreter, MODE_FILE_EXECUTE)
 
 
 def cleanup_python_paths():