add Python 3.12 support to ansible-test (#80834)

* update docker containers versions to use newer ansible-test ref in the pre-built venvs

* Allow invoking ansible-test with Python 3.12

* Add python3.12 to the INTERPRETER_PYTHON_FALLBACK

* changelog

* add Python 3.12 as a non-default Python version for the test containers

* Update mypy ignores for Python 3.12

* Add Python 3.12 to CI matrix for unit tests, generic tests, and galaxy

* Update unit test for using the Python 2 collection loader path with Python 3.

Skip the existing test on Python 3.12, since find_module is removed.

Suppress the pre-existing deprecation warnings using the Python 2
codepath with Python 3.

Add a test for Python >= 3.12, which doesn't call find_module.

* Ignore sanity test errors on systems without libselinux present.

.azure-pipelines/azure-pipelines.yml

@@ -61,6 +61,7 @@ stages:
             - test: 3.9
             - test: '3.10'
             - test: 3.11
+            - test: 3.12
   - stage: Windows
     dependsOn: []
     jobs:
@@ -175,6 +176,7 @@ stages:
           targets:
             - test: '3.10'
             - test: 3.11
+            - test: 3.12
   - stage: Generic
     dependsOn: []
     jobs:
@@ -185,6 +187,7 @@ stages:
           targets:
             - test: '3.10'
             - test: 3.11
+            - test: 3.12
   - stage: Incidental_Windows
     displayName: Incidental Windows
     dependsOn: []

changelogs/fragments/ansible-test-python-3.12.yml

@@ -0,0 +1,4 @@
+minor_changes:
+  - ansible-test - Add support for testing with Python 3.12.
+  - ansible-test - Update pip to ``23.1.2`` and setuptools to ``67.7.2``.
+  - Add ``python3.12`` to the default ``INTERPRETER_PYTHON_FALLBACK`` list.

lib/ansible/config/base.yml

@@ -1556,6 +1556,7 @@ _INTERPRETER_PYTHON_DISTRO_MAP:
 INTERPRETER_PYTHON_FALLBACK:
   name: Ordered list of Python interpreters to check for in discovery
   default:
+  - python3.12
   - python3.11
   - python3.10
   - python3.9

test/lib/ansible_test/_data/completion/docker.txt

@@ -1,6 +1,6 @@
-base image=quay.io/ansible/base-test-container:4.1.0 python=3.11,2.7,3.6,3.7,3.8,3.9,3.10
+base image=quay.io/ansible/base-test-container:5.0.0 python=3.11,2.7,3.6,3.7,3.8,3.9,3.10,3.12
-default image=quay.io/ansible/default-test-container:8.2.0 python=3.11,2.7,3.6,3.7,3.8,3.9,3.10 context=collection
+default image=quay.io/ansible/default-test-container:8.3.0 python=3.11,2.7,3.6,3.7,3.8,3.9,3.10,3.12 context=collection
-default image=quay.io/ansible/ansible-core-test-container:8.2.0 python=3.11,2.7,3.6,3.7,3.8,3.9,3.10 context=ansible-core
+default image=quay.io/ansible/ansible-core-test-container:8.3.0 python=3.11,2.7,3.6,3.7,3.8,3.9,3.10,3.12 context=ansible-core
 alpine3 image=quay.io/ansible/alpine3-test-container:5.0.0 python=3.10 cgroup=none audit=none
 centos7 image=quay.io/ansible/centos7-test-container:5.0.0 python=2.7 cgroup=v1-only
 fedora38 image=quay.io/ansible/fedora38-test-container:6.1.0 python=3.11

test/lib/ansible_test/_internal/python_requirements.py

@@ -434,8 +434,8 @@ def get_venv_packages(python: PythonConfig) -> dict[str, str]:
     #       See: https://github.com/ansible/base-test-container/blob/main/files/installer.py
 
     default_packages = dict(
-        pip='21.3.1',
+        pip='23.1.2',
-        setuptools='60.8.2',
+        setuptools='67.7.2',
         wheel='0.37.1',
     )
 

test/lib/ansible_test/_util/target/common/constants.py

@@ -16,4 +16,5 @@ REMOTE_ONLY_PYTHON_VERSIONS = (
 CONTROLLER_PYTHON_VERSIONS = (
     '3.10',
     '3.11',
+    '3.12',
 )

test/lib/ansible_test/_util/target/setup/bootstrap.sh

@@ -53,7 +53,7 @@ install_pip() {
                 pip_bootstrap_url="https://ci-files.testing.ansible.com/ansible-test/get-pip-20.3.4.py"
                 ;;
             *)
-                pip_bootstrap_url="https://ci-files.testing.ansible.com/ansible-test/get-pip-21.3.1.py"
+                pip_bootstrap_url="https://ci-files.testing.ansible.com/ansible-test/get-pip-23.1.2.py"
                 ;;
         esac
 

test/sanity/ignore.txt

@@ -5,10 +5,13 @@ lib/ansible/executor/powershell/async_wrapper.ps1 pslint:PSCustomUseLiteralPath
 lib/ansible/executor/powershell/exec_wrapper.ps1 pslint:PSCustomUseLiteralPath
 lib/ansible/galaxy/collection/__init__.py mypy-3.10:attr-defined  # inline ignore has no effect
 lib/ansible/galaxy/collection/__init__.py mypy-3.11:attr-defined  # inline ignore has no effect
+lib/ansible/galaxy/collection/__init__.py mypy-3.12:attr-defined  # inline ignore has no effect
 lib/ansible/galaxy/collection/gpg.py mypy-3.10:arg-type
 lib/ansible/galaxy/collection/gpg.py mypy-3.11:arg-type
+lib/ansible/galaxy/collection/gpg.py mypy-3.12:arg-type
 lib/ansible/parsing/yaml/constructor.py mypy-3.10:type-var  # too many occurrences to ignore inline
 lib/ansible/parsing/yaml/constructor.py mypy-3.11:type-var  # too many occurrences to ignore inline
+lib/ansible/parsing/yaml/constructor.py mypy-3.12:type-var  # too many occurrences to ignore inline
 lib/ansible/keyword_desc.yml no-unwanted-files
 lib/ansible/modules/apt.py validate-modules:parameter-invalid
 lib/ansible/modules/apt_repository.py validate-modules:parameter-invalid
@@ -55,6 +58,7 @@ lib/ansible/module_utils/compat/selinux.py import-3.8!skip # pass/fail depends o
 lib/ansible/module_utils/compat/selinux.py import-3.9!skip # pass/fail depends on presence of libselinux.so
 lib/ansible/module_utils/compat/selinux.py import-3.10!skip # pass/fail depends on presence of libselinux.so
 lib/ansible/module_utils/compat/selinux.py import-3.11!skip # pass/fail depends on presence of libselinux.so
+lib/ansible/module_utils/compat/selinux.py import-3.12!skip # pass/fail depends on presence of libselinux.so
 lib/ansible/module_utils/distro/_distro.py future-import-boilerplate # ignore bundled
 lib/ansible/module_utils/distro/_distro.py metaclass-boilerplate # ignore bundled
 lib/ansible/module_utils/distro/_distro.py no-assert

test/units/utils/collection_loader/test_collection_loader.py

@@ -29,8 +29,16 @@ def teardown(*args, **kwargs):
 # BEGIN STANDALONE TESTS - these exercise behaviors of the individual components without the import machinery
 
 
-@pytest.mark.skipif(not PY3, reason='Testing Python 2 codepath (find_module) on Python 3')
+@pytest.mark.filterwarnings(
-def test_find_module_py3():
+    'ignore:'
+    r'find_module\(\) is deprecated and slated for removal in Python 3\.12; use find_spec\(\) instead'
+    ':DeprecationWarning',
+    'ignore:'
+    r'FileFinder\.find_loader\(\) is deprecated and slated for removal in Python 3\.12; use find_spec\(\) instead'
+    ':DeprecationWarning',
+)
+@pytest.mark.skipif(not PY3 or sys.version_info >= (3, 12), reason='Testing Python 2 codepath (find_module) on Python 3, <= 3.11')
+def test_find_module_py3_lt_312():
     dir_to_a_file = os.path.dirname(ping_module.__file__)
     path_hook_finder = _AnsiblePathHookFinder(_AnsibleCollectionFinder(), dir_to_a_file)
 
@@ -40,6 +48,16 @@ def test_find_module_py3():
     assert path_hook_finder.find_module('missing') is None
 
 
+@pytest.mark.skipif(sys.version_info < (3, 12), reason='Testing Python 2 codepath (find_module) on Python >= 3.12')
+def test_find_module_py3_gt_311():
+    dir_to_a_file = os.path.dirname(ping_module.__file__)
+    path_hook_finder = _AnsiblePathHookFinder(_AnsibleCollectionFinder(), dir_to_a_file)
+
+    # setuptools may fall back to find_module on Python 3 if find_spec returns None
+    # see https://github.com/pypa/setuptools/pull/2918
+    assert path_hook_finder.find_spec('missing') is None
+
+
 def test_finder_setup():
     # ensure scalar path is listified
     f = _AnsibleCollectionFinder(paths='/bogus/bogus')