From 48dfe8e21524a84e87d4d1f6ec49edb9c2d7c68c Mon Sep 17 00:00:00 2001 From: Sloane Hertel <19572925+s-hertel@users.noreply.github.com> Date: Fri, 21 Jul 2023 13:23:14 -0400 Subject: [PATCH] add Python 3.12 support to ansible-test (#80834) * update docker containers versions to use newer ansible-test ref in the pre-built venvs * Allow invoking ansible-test with Python 3.12 * Add python3.12 to the INTERPRETER_PYTHON_FALLBACK * changelog * add Python 3.12 as a non-default Python version for the test containers * Update mypy ignores for Python 3.12 * Add Python 3.12 to CI matrix for unit tests, generic tests, and galaxy * Update unit test for using the Python 2 collection loader path with Python 3. Skip the existing test on Python 3.12, since find_module is removed. Suppress the pre-existing deprecation warnings using the Python 2 codepath with Python 3. Add a test for Python >= 3.12, which doesn't call find_module. * Ignore sanity test errors on systems without libselinux present. --- .azure-pipelines/azure-pipelines.yml | 3 +++ .../fragments/ansible-test-python-3.12.yml | 4 ++++ lib/ansible/config/base.yml | 1 + .../ansible_test/_data/completion/docker.txt | 6 ++--- .../_internal/python_requirements.py | 4 ++-- .../_util/target/common/constants.py | 1 + .../_util/target/setup/bootstrap.sh | 2 +- test/sanity/ignore.txt | 4 ++++ .../test_collection_loader.py | 22 +++++++++++++++++-- 9 files changed, 39 insertions(+), 8 deletions(-) create mode 100644 changelogs/fragments/ansible-test-python-3.12.yml diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml index 888f1cb3970..5d4dbf257d1 100644 --- a/.azure-pipelines/azure-pipelines.yml +++ b/.azure-pipelines/azure-pipelines.yml @@ -61,6 +61,7 @@ stages: - test: 3.9 - test: '3.10' - test: 3.11 + - test: 3.12 - stage: Windows dependsOn: [] jobs: @@ -175,6 +176,7 @@ stages: targets: - test: '3.10' - test: 3.11 + - test: 3.12 - stage: Generic dependsOn: [] jobs: @@ -185,6 +187,7 @@ stages: targets: - test: '3.10' - test: 3.11 + - test: 3.12 - stage: Incidental_Windows displayName: Incidental Windows dependsOn: [] diff --git a/changelogs/fragments/ansible-test-python-3.12.yml b/changelogs/fragments/ansible-test-python-3.12.yml new file mode 100644 index 00000000000..7a8b3592d9e --- /dev/null +++ b/changelogs/fragments/ansible-test-python-3.12.yml @@ -0,0 +1,4 @@ +minor_changes: + - ansible-test - Add support for testing with Python 3.12. + - ansible-test - Update pip to ``23.1.2`` and setuptools to ``67.7.2``. + - Add ``python3.12`` to the default ``INTERPRETER_PYTHON_FALLBACK`` list. diff --git a/lib/ansible/config/base.yml b/lib/ansible/config/base.yml index 52d63a8dcd0..ae846330fd5 100644 --- a/lib/ansible/config/base.yml +++ b/lib/ansible/config/base.yml @@ -1556,6 +1556,7 @@ _INTERPRETER_PYTHON_DISTRO_MAP: INTERPRETER_PYTHON_FALLBACK: name: Ordered list of Python interpreters to check for in discovery default: + - python3.12 - python3.11 - python3.10 - python3.9 diff --git a/test/lib/ansible_test/_data/completion/docker.txt b/test/lib/ansible_test/_data/completion/docker.txt index 9488d15c21b..7381cdfd056 100644 --- a/test/lib/ansible_test/_data/completion/docker.txt +++ b/test/lib/ansible_test/_data/completion/docker.txt @@ -1,6 +1,6 @@ -base image=quay.io/ansible/base-test-container:4.1.0 python=3.11,2.7,3.6,3.7,3.8,3.9,3.10 -default image=quay.io/ansible/default-test-container:8.2.0 python=3.11,2.7,3.6,3.7,3.8,3.9,3.10 context=collection -default image=quay.io/ansible/ansible-core-test-container:8.2.0 python=3.11,2.7,3.6,3.7,3.8,3.9,3.10 context=ansible-core +base image=quay.io/ansible/base-test-container:5.0.0 python=3.11,2.7,3.6,3.7,3.8,3.9,3.10,3.12 +default image=quay.io/ansible/default-test-container:8.3.0 python=3.11,2.7,3.6,3.7,3.8,3.9,3.10,3.12 context=collection +default image=quay.io/ansible/ansible-core-test-container:8.3.0 python=3.11,2.7,3.6,3.7,3.8,3.9,3.10,3.12 context=ansible-core alpine3 image=quay.io/ansible/alpine3-test-container:5.0.0 python=3.10 cgroup=none audit=none centos7 image=quay.io/ansible/centos7-test-container:5.0.0 python=2.7 cgroup=v1-only fedora38 image=quay.io/ansible/fedora38-test-container:6.1.0 python=3.11 diff --git a/test/lib/ansible_test/_internal/python_requirements.py b/test/lib/ansible_test/_internal/python_requirements.py index 44765f43868..a17cacb3b0a 100644 --- a/test/lib/ansible_test/_internal/python_requirements.py +++ b/test/lib/ansible_test/_internal/python_requirements.py @@ -434,8 +434,8 @@ def get_venv_packages(python: PythonConfig) -> dict[str, str]: # See: https://github.com/ansible/base-test-container/blob/main/files/installer.py default_packages = dict( - pip='21.3.1', - setuptools='60.8.2', + pip='23.1.2', + setuptools='67.7.2', wheel='0.37.1', ) diff --git a/test/lib/ansible_test/_util/target/common/constants.py b/test/lib/ansible_test/_util/target/common/constants.py index da42299b0e4..36a5a2c4342 100644 --- a/test/lib/ansible_test/_util/target/common/constants.py +++ b/test/lib/ansible_test/_util/target/common/constants.py @@ -16,4 +16,5 @@ REMOTE_ONLY_PYTHON_VERSIONS = ( CONTROLLER_PYTHON_VERSIONS = ( '3.10', '3.11', + '3.12', ) diff --git a/test/lib/ansible_test/_util/target/setup/bootstrap.sh b/test/lib/ansible_test/_util/target/setup/bootstrap.sh index 90f80ad963e..ed9816dca23 100644 --- a/test/lib/ansible_test/_util/target/setup/bootstrap.sh +++ b/test/lib/ansible_test/_util/target/setup/bootstrap.sh @@ -53,7 +53,7 @@ install_pip() { pip_bootstrap_url="https://ci-files.testing.ansible.com/ansible-test/get-pip-20.3.4.py" ;; *) - pip_bootstrap_url="https://ci-files.testing.ansible.com/ansible-test/get-pip-21.3.1.py" + pip_bootstrap_url="https://ci-files.testing.ansible.com/ansible-test/get-pip-23.1.2.py" ;; esac diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index 345c9850d2a..ea0252ed5c6 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -5,10 +5,13 @@ lib/ansible/executor/powershell/async_wrapper.ps1 pslint:PSCustomUseLiteralPath lib/ansible/executor/powershell/exec_wrapper.ps1 pslint:PSCustomUseLiteralPath lib/ansible/galaxy/collection/__init__.py mypy-3.10:attr-defined # inline ignore has no effect lib/ansible/galaxy/collection/__init__.py mypy-3.11:attr-defined # inline ignore has no effect +lib/ansible/galaxy/collection/__init__.py mypy-3.12:attr-defined # inline ignore has no effect lib/ansible/galaxy/collection/gpg.py mypy-3.10:arg-type lib/ansible/galaxy/collection/gpg.py mypy-3.11:arg-type +lib/ansible/galaxy/collection/gpg.py mypy-3.12:arg-type lib/ansible/parsing/yaml/constructor.py mypy-3.10:type-var # too many occurrences to ignore inline lib/ansible/parsing/yaml/constructor.py mypy-3.11:type-var # too many occurrences to ignore inline +lib/ansible/parsing/yaml/constructor.py mypy-3.12:type-var # too many occurrences to ignore inline lib/ansible/keyword_desc.yml no-unwanted-files lib/ansible/modules/apt.py validate-modules:parameter-invalid lib/ansible/modules/apt_repository.py validate-modules:parameter-invalid @@ -55,6 +58,7 @@ lib/ansible/module_utils/compat/selinux.py import-3.8!skip # pass/fail depends o lib/ansible/module_utils/compat/selinux.py import-3.9!skip # pass/fail depends on presence of libselinux.so lib/ansible/module_utils/compat/selinux.py import-3.10!skip # pass/fail depends on presence of libselinux.so lib/ansible/module_utils/compat/selinux.py import-3.11!skip # pass/fail depends on presence of libselinux.so +lib/ansible/module_utils/compat/selinux.py import-3.12!skip # pass/fail depends on presence of libselinux.so lib/ansible/module_utils/distro/_distro.py future-import-boilerplate # ignore bundled lib/ansible/module_utils/distro/_distro.py metaclass-boilerplate # ignore bundled lib/ansible/module_utils/distro/_distro.py no-assert diff --git a/test/units/utils/collection_loader/test_collection_loader.py b/test/units/utils/collection_loader/test_collection_loader.py index 9be59307a9e..feaaf97a9c8 100644 --- a/test/units/utils/collection_loader/test_collection_loader.py +++ b/test/units/utils/collection_loader/test_collection_loader.py @@ -29,8 +29,16 @@ def teardown(*args, **kwargs): # BEGIN STANDALONE TESTS - these exercise behaviors of the individual components without the import machinery -@pytest.mark.skipif(not PY3, reason='Testing Python 2 codepath (find_module) on Python 3') -def test_find_module_py3(): +@pytest.mark.filterwarnings( + 'ignore:' + r'find_module\(\) is deprecated and slated for removal in Python 3\.12; use find_spec\(\) instead' + ':DeprecationWarning', + 'ignore:' + r'FileFinder\.find_loader\(\) is deprecated and slated for removal in Python 3\.12; use find_spec\(\) instead' + ':DeprecationWarning', +) +@pytest.mark.skipif(not PY3 or sys.version_info >= (3, 12), reason='Testing Python 2 codepath (find_module) on Python 3, <= 3.11') +def test_find_module_py3_lt_312(): dir_to_a_file = os.path.dirname(ping_module.__file__) path_hook_finder = _AnsiblePathHookFinder(_AnsibleCollectionFinder(), dir_to_a_file) @@ -40,6 +48,16 @@ def test_find_module_py3(): assert path_hook_finder.find_module('missing') is None +@pytest.mark.skipif(sys.version_info < (3, 12), reason='Testing Python 2 codepath (find_module) on Python >= 3.12') +def test_find_module_py3_gt_311(): + dir_to_a_file = os.path.dirname(ping_module.__file__) + path_hook_finder = _AnsiblePathHookFinder(_AnsibleCollectionFinder(), dir_to_a_file) + + # setuptools may fall back to find_module on Python 3 if find_spec returns None + # see https://github.com/pypa/setuptools/pull/2918 + assert path_hook_finder.find_spec('missing') is None + + def test_finder_setup(): # ensure scalar path is listified f = _AnsibleCollectionFinder(paths='/bogus/bogus')