ansible-test - Fix vendoring support (#80074)

- Support loading of vendored Python packages.
- Exclude vendored Python packages from payloads.

changelogs/fragments/ansible-test-vendoring-support.yml

@@ -0,0 +1,3 @@
+bugfixes:
+  - ansible-test - Support loading of vendored Python packages from ansible-core.
+  - ansible-test - Exclude ansible-core vendored Python packages from ansible-test payloads.

test/integration/targets/ansible-test-vendoring/aliases

@@ -0,0 +1,5 @@
+shippable/posix/group3  # runs in the distro test containers
+shippable/generic/group1  # runs in the default test container
+context/controller
+needs/target/collection
+destructive  # adds and then removes packages into lib/ansible/_vendor/

test/integration/targets/ansible-test-vendoring/ansible_collections/ns/col/tests/config.yml

@@ -0,0 +1,4 @@
+# This config file is included to cause ansible-test to import the `packaging` module.
+
+modules:
+  python_requires: default

test/integration/targets/ansible-test-vendoring/runme.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# Run import sanity tests which require modifications to the source directory.
+
+vendor_dir="$(python -c 'import pathlib, ansible._vendor; print(pathlib.Path(ansible._vendor.__file__).parent)')"
+
+mkdir "${vendor_dir}/packaging/"  # intended to fail if packaging is already present (to avoid deleting it later)
+
+cleanup() {
+    rm -rf "${vendor_dir}/packaging/"
+}
+
+trap cleanup EXIT
+
+# Verify that packages installed in the vendor directory are loaded by ansible-test.
+# This is done by injecting a broken `packaging` package, which should cause ansible-test to fail.
+
+echo 'raise Exception("intentional failure from ansible-test-vendoring integration test")' > "${vendor_dir}/packaging/__init__.py"
+
+if ansible-test sanity --test import --color --truncate 0 "${@}" > output.log 2>&1; then
+    echo "ansible-test did not exit with a non-zero status"
+    cat output.log
+    exit 1
+fi
+
+if ! grep '^Exception: intentional failure from ansible-test-vendoring integration test$' output.log; then
+    echo "ansible-test did not fail with the expected output"
+    cat output.log
+    exit 1
+fi
+

test/lib/ansible_test/_internal/payload.py

@@ -48,6 +48,14 @@ def create_payload(args: CommonConfig, dst_path: str) -> None:
     permissions: dict[str, int] = {}
     filters: dict[str, t.Callable[[tarfile.TarInfo], t.Optional[tarfile.TarInfo]]] = {}
 
+    # Exclude vendored files from the payload.
+    # They may not be compatible with the delegated environment.
+    files = [
+        (abs_path, rel_path) for abs_path, rel_path in files
+        if not rel_path.startswith('lib/ansible/_vendor/')
+        or rel_path == 'lib/ansible/_vendor/__init__.py'
+    ]
+
     def apply_permissions(tar_info: tarfile.TarInfo, mode: int) -> t.Optional[tarfile.TarInfo]:
         """
         Apply the specified permissions to the given file.

test/lib/ansible_test/_internal/util.py

@@ -23,10 +23,14 @@ import time
 import functools
 import shlex
 import typing as t
+import warnings
 
 from struct import unpack, pack
 from termios import TIOCGWINSZ
 
+# CAUTION: Avoid third-party imports in this module whenever possible.
+#          Any third-party imports occurring here will result in an error if they are vendored by ansible-core.
+
 try:
     from typing_extensions import TypeGuard  # TypeGuard was added in Python 3.10
 except ImportError:
@@ -339,6 +343,17 @@ def get_ansible_version() -> str:
     return ansible_version
 
 
+def _enable_vendoring() -> None:
+    """Enable support for loading Python packages vendored by ansible-core."""
+    # Load the vendoring code by file path, since ansible may not be in our sys.path.
+    # Convert warnings into errors, to avoid problems from surfacing later.
+
+    with warnings.catch_warnings():
+        warnings.filterwarnings('error')
+
+        load_module(os.path.join(ANSIBLE_LIB_ROOT, '_vendor', '__init__.py'), 'ansible_vendor')
+
+
 @cache
 def get_available_python_versions() -> dict[str, str]:
     """Return a dictionary indicating which supported Python versions are available."""
@@ -1154,3 +1169,5 @@ def type_guard(sequence: c.Sequence[t.Any], guard_type: t.Type[C]) -> TypeGuard[
 
 
 display = Display()  # pylint: disable=locally-disabled, invalid-name
+
+_enable_vendoring()