From a1ad7100dfb4f673f567453e3ea4ca812dc75924 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Thu, 21 Mar 2024 21:06:25 -0700 Subject: [PATCH] ansible-test - Handle externally managed Python (#82883) Remove EXTERNALLY-MANAGED marker in ansible-test managed environments Additional changes: - Test the Alpine 3.19 container - Stop testing Alpine 3.18 container and remote - Add missing changelog entry the Alpine 3.19 container --- .azure-pipelines/azure-pipelines.yml | 10 +++--- ...ansible-test-externally-managed-python.yml | 4 +++ .../fragments/ansible_test_alpine_3.19.yml | 1 + test/lib/ansible_test/_internal/bootstrap.py | 4 +-- .../ansible_test/_internal/host_profiles.py | 9 ++--- .../_util/target/setup/bootstrap.sh | 33 ++++++++++++++++--- 6 files changed, 45 insertions(+), 16 deletions(-) create mode 100644 changelogs/fragments/ansible-test-externally-managed-python.yml diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml index c1ff7c70f28..a7897f9c492 100644 --- a/.azure-pipelines/azure-pipelines.yml +++ b/.azure-pipelines/azure-pipelines.yml @@ -108,8 +108,6 @@ stages: - template: templates/matrix.yml # context/controller (ansible-test container management) parameters: targets: - - name: Alpine 3.18 - test: alpine/3.18 - name: Alpine 3.19 test: alpine/3.19 - name: Fedora 39 @@ -127,8 +125,8 @@ stages: parameters: testFormat: linux/{0} targets: - - name: Alpine 3 - test: alpine3 + - name: Alpine 3.19 + test: alpine319 - name: Fedora 39 test: fedora39 - name: Ubuntu 20.04 @@ -142,8 +140,8 @@ stages: parameters: testFormat: linux/{0} targets: - - name: Alpine 3 - test: alpine3 + - name: Alpine 3.19 + test: alpine319 - name: Fedora 39 test: fedora39 - name: Ubuntu 22.04 diff --git a/changelogs/fragments/ansible-test-externally-managed-python.yml b/changelogs/fragments/ansible-test-externally-managed-python.yml new file mode 100644 index 00000000000..fb6609dafff --- /dev/null +++ b/changelogs/fragments/ansible-test-externally-managed-python.yml @@ -0,0 +1,4 @@ +minor_changes: + - ansible-test - Containers and remotes managed by ansible-test will have their Python ``EXTERNALLY-MANAGED`` marker (PEP668) removed. + This provides backwards compatibility for existing tests running in newer environments which mark their Python as externally managed. + A future version of ansible-test may change this behavior, requiring tests to be adapted to such environments. diff --git a/changelogs/fragments/ansible_test_alpine_3.19.yml b/changelogs/fragments/ansible_test_alpine_3.19.yml index b73c214ffac..fa3052cc354 100644 --- a/changelogs/fragments/ansible_test_alpine_3.19.yml +++ b/changelogs/fragments/ansible_test_alpine_3.19.yml @@ -1,2 +1,3 @@ minor_changes: - ansible-test - Add Alpine 3.19 to remotes. + - ansible-test - Add Alpine 3.19 container. diff --git a/test/lib/ansible_test/_internal/bootstrap.py b/test/lib/ansible_test/_internal/bootstrap.py index b0cfb601d94..a9dd6370ee3 100644 --- a/test/lib/ansible_test/_internal/bootstrap.py +++ b/test/lib/ansible_test/_internal/bootstrap.py @@ -28,7 +28,7 @@ class Bootstrap: """Base class for bootstrapping systems.""" controller: bool - python_versions: list[str] + python_interpreters: dict[str, str] ssh_key: SshKey @property @@ -41,7 +41,7 @@ class Bootstrap: return dict( bootstrap_type=self.bootstrap_type, controller='yes' if self.controller else '', - python_versions=self.python_versions, + python_interpreters=[f'{key}:{value}' for key, value in self.python_interpreters.items()], ssh_key_type=self.ssh_key.KEY_TYPE, ssh_private_key=self.ssh_key.key_contents, ssh_public_key=self.ssh_key.pub_contents, diff --git a/test/lib/ansible_test/_internal/host_profiles.py b/test/lib/ansible_test/_internal/host_profiles.py index 09812456ca0..39fe7d209ab 100644 --- a/test/lib/ansible_test/_internal/host_profiles.py +++ b/test/lib/ansible_test/_internal/host_profiles.py @@ -958,7 +958,7 @@ class DockerProfile(ControllerHostProfile[DockerConfig], SshTargetHostProfile[Do """Perform out-of-band setup before delegation.""" bootstrapper = BootstrapDocker( controller=self.controller, - python_versions=[self.python.version], + python_interpreters={self.python.version: self.python.path}, ssh_key=SshKey(self.args), ) @@ -1214,8 +1214,9 @@ class PosixRemoteProfile(ControllerHostProfile[PosixRemoteConfig], RemoteProfile def configure(self) -> None: """Perform in-band configuration. Executed before delegation for the controller and after delegation for targets.""" # a target uses a single python version, but a controller may include additional versions for targets running on the controller - python_versions = [self.python.version] + [target.python.version for target in self.targets if isinstance(target, ControllerConfig)] - python_versions = sorted_versions(list(set(python_versions))) + python_interpreters = {self.python.version: self.python.path} + python_interpreters.update({target.python.version: target.python.path for target in self.targets if isinstance(target, ControllerConfig)}) + python_interpreters = {version: python_interpreters[version] for version in sorted_versions(list(python_interpreters.keys()))} core_ci = self.wait_for_instance() pwd = self.wait_until_ready() @@ -1226,7 +1227,7 @@ class PosixRemoteProfile(ControllerHostProfile[PosixRemoteConfig], RemoteProfile controller=self.controller, platform=self.config.platform, platform_version=self.config.version, - python_versions=python_versions, + python_interpreters=python_interpreters, ssh_key=core_ci.ssh_key, ) diff --git a/test/lib/ansible_test/_util/target/setup/bootstrap.sh b/test/lib/ansible_test/_util/target/setup/bootstrap.sh index 279ce4036df..fe2c85830ba 100644 --- a/test/lib/ansible_test/_util/target/setup/bootstrap.sh +++ b/test/lib/ansible_test/_util/target/setup/bootstrap.sh @@ -2,6 +2,16 @@ set -eu +remove_externally_managed_marker() +{ + "${python_interpreter}" -c ' +import pathlib +import sysconfig +path = pathlib.Path(sysconfig.get_path("stdlib")) / "EXTERNALLY-MANAGED" +path.unlink(missing_ok=True) +' +} + install_ssh_keys() { if [ ! -f "${ssh_private_key_path}" ]; then @@ -340,14 +350,27 @@ bootstrap_docker() { # Required for newer mysql-server packages to install/upgrade on Ubuntu 16.04. rm -f /usr/sbin/policy-rc.d + + for key_value in ${python_interpreters}; do + IFS=':' read -r python_version python_interpreter << EOF +${key_value} +EOF + + echo "Bootstrapping Python ${python_version} at: ${python_interpreter}" + + remove_externally_managed_marker + done } bootstrap_remote() { - for python_version in ${python_versions}; do - echo "Bootstrapping Python ${python_version}" + for key_value in ${python_interpreters}; do + IFS=':' read -r python_version python_interpreter << EOF +${key_value} +EOF + + echo "Bootstrapping Python ${python_version} at: ${python_interpreter}" - python_interpreter="python${python_version}" python_package_version="$(echo "${python_version}" | tr -d '.')" case "${platform}" in @@ -358,6 +381,8 @@ bootstrap_remote() "rhel") bootstrap_remote_rhel ;; "ubuntu") bootstrap_remote_ubuntu ;; esac + + remove_externally_managed_marker done } @@ -384,7 +409,7 @@ bootstrap_type=#{bootstrap_type} controller=#{controller} platform=#{platform} platform_version=#{platform_version} -python_versions=#{python_versions} +python_interpreters=#{python_interpreters} ssh_key_type=#{ssh_key_type} ssh_private_key=#{ssh_private_key} ssh_public_key=#{ssh_public_key}