Remove Python 3.9 support for the controller (#80973)

* Remove obsolete Python <=3.9 controller code
* Remove Python 3.9 test controller bootstrapping
* Update test requirements

Co-authored-by: Matt Clay <matt@mystile.com>

.azure-pipelines/azure-pipelines.yml

@@ -180,7 +180,6 @@ stages:
           nameFormat: Python {0}
           testFormat: galaxy/{0}/1
           targets:
-            - test: 3.9
             - test: '3.10'
             - test: 3.11
   - stage: Generic
@@ -191,7 +190,6 @@ stages:
           nameFormat: Python {0}
           testFormat: generic/{0}/1
           targets:
-            - test: 3.9
             - test: '3.10'
             - test: 3.11
   - stage: Incidental_Windows

changelogs/fragments/remove-python3.9-controller-support.yml

@@ -0,0 +1,2 @@
+removed_features:
+  - Removed Python 3.9 as a supported version on the controller. Python 3.10 or newer is required.

hacking/README.md

@@ -5,7 +5,7 @@ env-setup
 ---------
 
 The 'env-setup' script modifies your environment to allow you to run
-ansible from a git checkout using python >= 3.8. 
+ansible from a git checkout using python >= 3.10.
 
 First, set up your environment to run from the checkout:
 

lib/ansible/cli/__init__.py

@@ -13,9 +13,9 @@ import sys
 
 # Used for determining if the system is running a new enough python version
 # and should only restrict on our documented minimum versions
-if sys.version_info < (3, 9):
+if sys.version_info < (3, 10):
     raise SystemExit(
-        'ERROR: Ansible requires Python 3.9 or newer on the controller. '
+        'ERROR: Ansible requires Python 3.10 or newer on the controller. '
         'Current version: %s' % ''.join(sys.version.splitlines())
     )
 

lib/ansible/galaxy/api.py

@@ -11,7 +11,6 @@ import functools
 import hashlib
 import json
 import os
-import socket
 import stat
 import tarfile
 import time
@@ -66,7 +65,7 @@ def should_retry_error(exception):
 
         # Handle common URL related errors such as TimeoutError, and BadStatusLine
         # Note: socket.timeout is only required for Py3.9
-        if isinstance(orig_exc, (TimeoutError, BadStatusLine, IncompleteRead, socket.timeout)):
+        if isinstance(orig_exc, (TimeoutError, BadStatusLine, IncompleteRead)):
             return True
 
     return False

lib/ansible/utils/encrypt.py

@@ -152,8 +152,7 @@ class CryptHash(BaseHash):
 
         saltstring += "$%s" % salt
 
-        # crypt.crypt on Python < 3.9 returns None if it cannot parse saltstring
+        # crypt.crypt throws OSError on Python >= 3.9 if it cannot parse saltstring.
-        # On Python >= 3.9, it throws OSError.
         try:
             result = crypt.crypt(secret, saltstring)
             orig_exc = None

lib/ansible/utils/shlex.py

@@ -20,15 +20,7 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import shlex
-from ansible.module_utils.six import PY3
-from ansible.module_utils.common.text.converters import to_bytes, to_text
 
 
-if PY3:
+# shlex.split() wants Unicode (i.e. ``str``) input on Python 3
-    # shlex.split() wants Unicode (i.e. ``str``) input on Python 3
+shlex_split = shlex.split
-    shlex_split = shlex.split
-else:
-    # shlex.split() wants bytes (i.e. ``str``) input on Python 2
-    def shlex_split(s, comments=False, posix=True):
-        return map(to_text, shlex.split(to_bytes(s), comments, posix))
-    shlex_split.__doc__ = shlex.split.__doc__

lib/ansible/utils/vars.py

@@ -29,7 +29,7 @@ from json import dumps
 from ansible import constants as C
 from ansible import context
 from ansible.errors import AnsibleError, AnsibleOptionsError
-from ansible.module_utils.six import string_types, PY3
+from ansible.module_utils.six import string_types
 from ansible.module_utils.common.text.converters import to_native, to_text
 from ansible.parsing.splitter import parse_kv
 
@@ -241,13 +241,7 @@ def _isidentifier_PY3(ident):
     if not isinstance(ident, string_types):
         return False
 
-    # NOTE Python 3.7 offers str.isascii() so switch over to using it once
+    if not ident.isascii():
-    # we stop supporting 3.5 and 3.6 on the controller
-    try:
-        # Python 2 does not allow non-ascii characters in identifiers so unify
-        # the behavior for Python 3
-        ident.encode('ascii')
-    except UnicodeEncodeError:
         return False
 
     if not ident.isidentifier():
@@ -259,26 +253,7 @@ def _isidentifier_PY3(ident):
     return True
 
 
-def _isidentifier_PY2(ident):
+isidentifier = _isidentifier_PY3
-    if not isinstance(ident, string_types):
-        return False
-
-    if not ident:
-        return False
-
-    if C.INVALID_VARIABLE_NAMES.search(ident):
-        return False
-
-    if keyword.iskeyword(ident) or ident in ADDITIONAL_PY2_KEYWORDS:
-        return False
-
-    return True
-
-
-if PY3:
-    isidentifier = _isidentifier_PY3
-else:
-    isidentifier = _isidentifier_PY2
 
 
 isidentifier.__doc__ = """Determine if string is valid identifier.

requirements.txt

@@ -7,9 +7,6 @@ jinja2 >= 3.0.0
 PyYAML >= 5.1  # PyYAML 5.1 is required for Python 3.8+ support
 cryptography
 packaging
-# importlib.resources in stdlib for py3.9 is lacking native hooks for
-# importlib.resources.files
-importlib_resources >= 5.0, < 5.1; python_version < '3.10'
 # NOTE: resolvelib 0.x version bumps should be considered major/breaking
 # NOTE: and we should update the upper cap with care, at least until 1.0
 # NOTE: Ref: https://github.com/sarugaku/resolvelib/issues/69

setup.cfg

@@ -27,7 +27,6 @@ classifiers =
     Natural Language :: English
     Operating System :: POSIX
     Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3.11
     Programming Language :: Python :: 3 :: Only
@@ -37,7 +36,7 @@ classifiers =
 
 [options]
 zip_safe = False
-python_requires = >=3.9
+python_requires = >=3.10
 # keep ansible-test as a verbatim script to work with editable installs, since it needs to do its
 # own package redirection magic that's beyond the scope of the normal `ansible` path redirection
 # done by setuptools `develop`

test/integration/targets/canonical-pep517-self-packaging/minimum-build-constraints.txt

@@ -1,11 +1,11 @@
-# Lowest supporting Python 3.9 and 3.10:
+# Lowest supporting Python 3.10:
-setuptools == 57.0.0; python_version == "3.9" or python_version == "3.10"
+setuptools == 57.0.0; python_version == "3.10"
 
 # Lowest supporting Python 3.11:
 setuptools == 60.0.0; python_version >= "3.11"
 
 
-# An arbitrary old version that was released before Python 3.9.0:
+# An arbitrary old version that was released before Python 3.10.0:
 wheel == 0.33.6
 
 # Conditional dependencies:

test/lib/ansible_test/_data/requirements/ansible.txt

@@ -7,9 +7,6 @@ jinja2 >= 3.0.0
 PyYAML >= 5.1  # PyYAML 5.1 is required for Python 3.8+ support
 cryptography
 packaging
-# importlib.resources in stdlib for py3.9 is lacking native hooks for
-# importlib.resources.files
-importlib_resources >= 5.0, < 5.1; python_version < '3.10'
 # NOTE: resolvelib 0.x version bumps should be considered major/breaking
 # NOTE: and we should update the upper cap with care, at least until 1.0
 # NOTE: Ref: https://github.com/sarugaku/resolvelib/issues/69

test/lib/ansible_test/_data/requirements/constraints.txt

@@ -12,5 +12,4 @@ pyparsing < 3.0.0 ; python_version < '3.5' # pyparsing 3 and later require pytho
 mock >= 2.0.0 # needed for features backported from Python 3.6 unittest.mock (assert_called, assert_called_once...)
 pytest-mock >= 1.4.0 # needed for mock_use_standalone_module pytest option
 setuptools < 45 ; python_version == '2.7' # setuptools 45 and later require python 3.5 or later
-pyspnego >= 0.1.6 ; python_version >= '3.10' # bug in older releases breaks on Python 3.10
 wheel < 0.38.0 ; python_version < '3.7' # wheel 0.38.0 and later require python 3.7 or later

test/lib/ansible_test/_internal/util.py

@@ -31,11 +31,6 @@ from termios import TIOCGWINSZ
 # CAUTION: Avoid third-party imports in this module whenever possible.
 #          Any third-party imports occurring here will result in an error if they are vendored by ansible-core.
 
-try:
-    from typing_extensions import TypeGuard  # TypeGuard was added in Python 3.10
-except ImportError:
-    TypeGuard = None
-
 from .locale_util import (
     LOCALE_WARNING,
     CONFIGURED_LOCALE,
@@ -1157,7 +1152,7 @@ def verify_sys_executable(path: str) -> t.Optional[str]:
     return expected_executable
 
 
-def type_guard(sequence: c.Sequence[t.Any], guard_type: t.Type[C]) -> TypeGuard[c.Sequence[C]]:
+def type_guard(sequence: c.Sequence[t.Any], guard_type: t.Type[C]) -> t.TypeGuard[c.Sequence[C]]:
     """
     Raises an exception if any item in the given sequence does not match the specified guard type.
     Use with assert so that type checkers are aware of the type guard.

test/lib/ansible_test/_util/target/common/constants.py

@@ -10,10 +10,10 @@ REMOTE_ONLY_PYTHON_VERSIONS = (
     '3.6',
     '3.7',
     '3.8',
+    '3.9',
 )
 
 CONTROLLER_PYTHON_VERSIONS = (
-    '3.9',
     '3.10',
     '3.11',
 )

test/lib/ansible_test/_util/target/setup/bootstrap.sh

@@ -163,8 +163,6 @@ bootstrap_remote_freebsd()
         # Declare platform/python version combinations which do not have supporting OS packages available.
         # For these combinations ansible-test will use pip to install the requirements instead.
         case "${platform_version}/${python_version}" in
-            "12.4/3.9")
-                ;;
             *)
                 jinja2_pkg=""  # not available
                 cryptography_pkg=""  # not available
@@ -331,22 +329,14 @@ bootstrap_remote_rhel_9()
 
     # Jinja2 is not installed with an OS package since the provided version is too old.
     # Instead, ansible-test will install it using pip.
+    # packaging and resolvelib are missing for Python 3.11 (and possible later) so we just
+    # skip them and let ansible-test install them from PyPI.
     if [ "${controller}" ]; then
         packages="
             ${packages}
             ${py_pkg_prefix}-cryptography
             ${py_pkg_prefix}-pyyaml
             "
-
-        # The following OS packages are missing for 3.11 (and possibly later) so we just
-        # skip them and let ansible-test install them from PyPI.
-        if [ "${python_version}" = "3.9" ]; then
-            packages="
-                ${packages}
-                ${py_pkg_prefix}-packaging
-                ${py_pkg_prefix}-resolvelib
-            "
-        fi
     fi
 
     while true; do
@@ -425,14 +415,6 @@ bootstrap_remote_ubuntu()
         echo "Failed to install packages. Sleeping before trying again..."
         sleep 10
     done
-
-    if [ "${controller}" ]; then
-        if [ "${platform_version}/${python_version}" = "20.04/3.9" ]; then
-            # Install pyyaml using pip so libyaml support is available on Python 3.9.
-            # The OS package install (which is installed by default) only has a .so file for Python 3.8.
-            pip_install "--upgrade pyyaml"
-        fi
-    fi
 }
 
 bootstrap_docker()

test/sanity/ignore.txt

@@ -9,13 +9,10 @@ lib/ansible/config/base.yml no-unwanted-files
 lib/ansible/executor/powershell/async_watchdog.ps1 pslint:PSCustomUseLiteralPath
 lib/ansible/executor/powershell/async_wrapper.ps1 pslint:PSCustomUseLiteralPath
 lib/ansible/executor/powershell/exec_wrapper.ps1 pslint:PSCustomUseLiteralPath
-lib/ansible/galaxy/collection/__init__.py mypy-3.9:attr-defined  # inline ignore has no effect
 lib/ansible/galaxy/collection/__init__.py mypy-3.10:attr-defined  # inline ignore has no effect
 lib/ansible/galaxy/collection/__init__.py mypy-3.11:attr-defined  # inline ignore has no effect
-lib/ansible/galaxy/collection/gpg.py mypy-3.9:arg-type
 lib/ansible/galaxy/collection/gpg.py mypy-3.10:arg-type
 lib/ansible/galaxy/collection/gpg.py mypy-3.11:arg-type
-lib/ansible/parsing/yaml/constructor.py mypy-3.9:type-var  # too many occurrences to ignore inline
 lib/ansible/parsing/yaml/constructor.py mypy-3.10:type-var  # too many occurrences to ignore inline
 lib/ansible/parsing/yaml/constructor.py mypy-3.11:type-var  # too many occurrences to ignore inline
 lib/ansible/keyword_desc.yml no-unwanted-files

test/units/requirements.txt

@@ -1,4 +1,4 @@
-bcrypt ; python_version >= '3.9'  # controller only
+bcrypt ; python_version >= '3.10'  # controller only
-passlib ; python_version >= '3.9'  # controller only
+passlib ; python_version >= '3.10'  # controller only
-pexpect ; python_version >= '3.9'  # controller only
+pexpect ; python_version >= '3.10'  # controller only
-pywinrm ; python_version >= '3.9'  # controller only
+pywinrm ; python_version >= '3.10'  # controller only