From 67b78a17c458a69e6b5b4c21c737219d37ed9f82 Mon Sep 17 00:00:00 2001 From: Sloane Hertel <19572925+s-hertel@users.noreply.github.com> Date: Mon, 10 Jul 2023 12:01:47 -0400 Subject: [PATCH] Remove Python 3.9 support for the controller (#80973) * Remove obsolete Python <=3.9 controller code * Remove Python 3.9 test controller bootstrapping * Update test requirements Co-authored-by: Matt Clay --- .azure-pipelines/azure-pipelines.yml | 2 -- .../remove-python3.9-controller-support.yml | 2 ++ hacking/README.md | 2 +- lib/ansible/cli/__init__.py | 4 +-- lib/ansible/galaxy/api.py | 3 +- lib/ansible/utils/encrypt.py | 3 +- lib/ansible/utils/shlex.py | 12 ++----- lib/ansible/utils/vars.py | 31 ++----------------- requirements.txt | 3 -- setup.cfg | 3 +- .../minimum-build-constraints.txt | 6 ++-- .../_data/requirements/ansible.txt | 3 -- .../_data/requirements/constraints.txt | 1 - test/lib/ansible_test/_internal/util.py | 7 +---- .../_util/target/common/constants.py | 2 +- .../_util/target/setup/bootstrap.sh | 22 ++----------- test/sanity/ignore.txt | 3 -- test/units/requirements.txt | 8 ++--- 18 files changed, 24 insertions(+), 93 deletions(-) create mode 100644 changelogs/fragments/remove-python3.9-controller-support.yml diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml index d41152612df..44f481e3d23 100644 --- a/.azure-pipelines/azure-pipelines.yml +++ b/.azure-pipelines/azure-pipelines.yml @@ -180,7 +180,6 @@ stages: nameFormat: Python {0} testFormat: galaxy/{0}/1 targets: - - test: 3.9 - test: '3.10' - test: 3.11 - stage: Generic @@ -191,7 +190,6 @@ stages: nameFormat: Python {0} testFormat: generic/{0}/1 targets: - - test: 3.9 - test: '3.10' - test: 3.11 - stage: Incidental_Windows diff --git a/changelogs/fragments/remove-python3.9-controller-support.yml b/changelogs/fragments/remove-python3.9-controller-support.yml new file mode 100644 index 00000000000..632aa471e82 --- /dev/null +++ b/changelogs/fragments/remove-python3.9-controller-support.yml @@ -0,0 +1,2 @@ +removed_features: + - Removed Python 3.9 as a supported version on the controller. Python 3.10 or newer is required. diff --git a/hacking/README.md b/hacking/README.md index eaec6c4e8bb..d9f4c896198 100644 --- a/hacking/README.md +++ b/hacking/README.md @@ -5,7 +5,7 @@ env-setup --------- The 'env-setup' script modifies your environment to allow you to run -ansible from a git checkout using python >= 3.8. +ansible from a git checkout using python >= 3.10. First, set up your environment to run from the checkout: diff --git a/lib/ansible/cli/__init__.py b/lib/ansible/cli/__init__.py index 678e5d54433..91d6a969618 100644 --- a/lib/ansible/cli/__init__.py +++ b/lib/ansible/cli/__init__.py @@ -13,9 +13,9 @@ import sys # Used for determining if the system is running a new enough python version # and should only restrict on our documented minimum versions -if sys.version_info < (3, 9): +if sys.version_info < (3, 10): raise SystemExit( - 'ERROR: Ansible requires Python 3.9 or newer on the controller. ' + 'ERROR: Ansible requires Python 3.10 or newer on the controller. ' 'Current version: %s' % ''.join(sys.version.splitlines()) ) diff --git a/lib/ansible/galaxy/api.py b/lib/ansible/galaxy/api.py index 5b049914782..e1cbc83a93c 100644 --- a/lib/ansible/galaxy/api.py +++ b/lib/ansible/galaxy/api.py @@ -11,7 +11,6 @@ import functools import hashlib import json import os -import socket import stat import tarfile import time @@ -66,7 +65,7 @@ def should_retry_error(exception): # Handle common URL related errors such as TimeoutError, and BadStatusLine # Note: socket.timeout is only required for Py3.9 - if isinstance(orig_exc, (TimeoutError, BadStatusLine, IncompleteRead, socket.timeout)): + if isinstance(orig_exc, (TimeoutError, BadStatusLine, IncompleteRead)): return True return False diff --git a/lib/ansible/utils/encrypt.py b/lib/ansible/utils/encrypt.py index 04ae8dff91e..819a0a9cad0 100644 --- a/lib/ansible/utils/encrypt.py +++ b/lib/ansible/utils/encrypt.py @@ -152,8 +152,7 @@ class CryptHash(BaseHash): saltstring += "$%s" % salt - # crypt.crypt on Python < 3.9 returns None if it cannot parse saltstring - # On Python >= 3.9, it throws OSError. + # crypt.crypt throws OSError on Python >= 3.9 if it cannot parse saltstring. try: result = crypt.crypt(secret, saltstring) orig_exc = None diff --git a/lib/ansible/utils/shlex.py b/lib/ansible/utils/shlex.py index db78d3d5f7b..8f50ffd9824 100644 --- a/lib/ansible/utils/shlex.py +++ b/lib/ansible/utils/shlex.py @@ -20,15 +20,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type import shlex -from ansible.module_utils.six import PY3 -from ansible.module_utils.common.text.converters import to_bytes, to_text -if PY3: - # shlex.split() wants Unicode (i.e. ``str``) input on Python 3 - shlex_split = shlex.split -else: - # shlex.split() wants bytes (i.e. ``str``) input on Python 2 - def shlex_split(s, comments=False, posix=True): - return map(to_text, shlex.split(to_bytes(s), comments, posix)) - shlex_split.__doc__ = shlex.split.__doc__ +# shlex.split() wants Unicode (i.e. ``str``) input on Python 3 +shlex_split = shlex.split diff --git a/lib/ansible/utils/vars.py b/lib/ansible/utils/vars.py index 889f5943ae5..66f872f4f4e 100644 --- a/lib/ansible/utils/vars.py +++ b/lib/ansible/utils/vars.py @@ -29,7 +29,7 @@ from json import dumps from ansible import constants as C from ansible import context from ansible.errors import AnsibleError, AnsibleOptionsError -from ansible.module_utils.six import string_types, PY3 +from ansible.module_utils.six import string_types from ansible.module_utils.common.text.converters import to_native, to_text from ansible.parsing.splitter import parse_kv @@ -241,13 +241,7 @@ def _isidentifier_PY3(ident): if not isinstance(ident, string_types): return False - # NOTE Python 3.7 offers str.isascii() so switch over to using it once - # we stop supporting 3.5 and 3.6 on the controller - try: - # Python 2 does not allow non-ascii characters in identifiers so unify - # the behavior for Python 3 - ident.encode('ascii') - except UnicodeEncodeError: + if not ident.isascii(): return False if not ident.isidentifier(): @@ -259,26 +253,7 @@ def _isidentifier_PY3(ident): return True -def _isidentifier_PY2(ident): - if not isinstance(ident, string_types): - return False - - if not ident: - return False - - if C.INVALID_VARIABLE_NAMES.search(ident): - return False - - if keyword.iskeyword(ident) or ident in ADDITIONAL_PY2_KEYWORDS: - return False - - return True - - -if PY3: - isidentifier = _isidentifier_PY3 -else: - isidentifier = _isidentifier_PY2 +isidentifier = _isidentifier_PY3 isidentifier.__doc__ = """Determine if string is valid identifier. diff --git a/requirements.txt b/requirements.txt index a07aa985063..5eaf9f2cbc2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,9 +7,6 @@ jinja2 >= 3.0.0 PyYAML >= 5.1 # PyYAML 5.1 is required for Python 3.8+ support cryptography packaging -# importlib.resources in stdlib for py3.9 is lacking native hooks for -# importlib.resources.files -importlib_resources >= 5.0, < 5.1; python_version < '3.10' # NOTE: resolvelib 0.x version bumps should be considered major/breaking # NOTE: and we should update the upper cap with care, at least until 1.0 # NOTE: Ref: https://github.com/sarugaku/resolvelib/issues/69 diff --git a/setup.cfg b/setup.cfg index e020ee3b15c..2f1815127aa 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,7 +27,6 @@ classifiers = Natural Language :: English Operating System :: POSIX Programming Language :: Python :: 3 - Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Programming Language :: Python :: 3 :: Only @@ -37,7 +36,7 @@ classifiers = [options] zip_safe = False -python_requires = >=3.9 +python_requires = >=3.10 # keep ansible-test as a verbatim script to work with editable installs, since it needs to do its # own package redirection magic that's beyond the scope of the normal `ansible` path redirection # done by setuptools `develop` diff --git a/test/integration/targets/canonical-pep517-self-packaging/minimum-build-constraints.txt b/test/integration/targets/canonical-pep517-self-packaging/minimum-build-constraints.txt index 3ba47aeb4b6..a6ce0dcbbb6 100644 --- a/test/integration/targets/canonical-pep517-self-packaging/minimum-build-constraints.txt +++ b/test/integration/targets/canonical-pep517-self-packaging/minimum-build-constraints.txt @@ -1,11 +1,11 @@ -# Lowest supporting Python 3.9 and 3.10: -setuptools == 57.0.0; python_version == "3.9" or python_version == "3.10" +# Lowest supporting Python 3.10: +setuptools == 57.0.0; python_version == "3.10" # Lowest supporting Python 3.11: setuptools == 60.0.0; python_version >= "3.11" -# An arbitrary old version that was released before Python 3.9.0: +# An arbitrary old version that was released before Python 3.10.0: wheel == 0.33.6 # Conditional dependencies: diff --git a/test/lib/ansible_test/_data/requirements/ansible.txt b/test/lib/ansible_test/_data/requirements/ansible.txt index a07aa985063..5eaf9f2cbc2 100644 --- a/test/lib/ansible_test/_data/requirements/ansible.txt +++ b/test/lib/ansible_test/_data/requirements/ansible.txt @@ -7,9 +7,6 @@ jinja2 >= 3.0.0 PyYAML >= 5.1 # PyYAML 5.1 is required for Python 3.8+ support cryptography packaging -# importlib.resources in stdlib for py3.9 is lacking native hooks for -# importlib.resources.files -importlib_resources >= 5.0, < 5.1; python_version < '3.10' # NOTE: resolvelib 0.x version bumps should be considered major/breaking # NOTE: and we should update the upper cap with care, at least until 1.0 # NOTE: Ref: https://github.com/sarugaku/resolvelib/issues/69 diff --git a/test/lib/ansible_test/_data/requirements/constraints.txt b/test/lib/ansible_test/_data/requirements/constraints.txt index 67081b13673..dd837e3bbcb 100644 --- a/test/lib/ansible_test/_data/requirements/constraints.txt +++ b/test/lib/ansible_test/_data/requirements/constraints.txt @@ -12,5 +12,4 @@ pyparsing < 3.0.0 ; python_version < '3.5' # pyparsing 3 and later require pytho mock >= 2.0.0 # needed for features backported from Python 3.6 unittest.mock (assert_called, assert_called_once...) pytest-mock >= 1.4.0 # needed for mock_use_standalone_module pytest option setuptools < 45 ; python_version == '2.7' # setuptools 45 and later require python 3.5 or later -pyspnego >= 0.1.6 ; python_version >= '3.10' # bug in older releases breaks on Python 3.10 wheel < 0.38.0 ; python_version < '3.7' # wheel 0.38.0 and later require python 3.7 or later diff --git a/test/lib/ansible_test/_internal/util.py b/test/lib/ansible_test/_internal/util.py index a5a9fabaeda..2cd957f6e0e 100644 --- a/test/lib/ansible_test/_internal/util.py +++ b/test/lib/ansible_test/_internal/util.py @@ -31,11 +31,6 @@ from termios import TIOCGWINSZ # CAUTION: Avoid third-party imports in this module whenever possible. # Any third-party imports occurring here will result in an error if they are vendored by ansible-core. -try: - from typing_extensions import TypeGuard # TypeGuard was added in Python 3.10 -except ImportError: - TypeGuard = None - from .locale_util import ( LOCALE_WARNING, CONFIGURED_LOCALE, @@ -1157,7 +1152,7 @@ def verify_sys_executable(path: str) -> t.Optional[str]: return expected_executable -def type_guard(sequence: c.Sequence[t.Any], guard_type: t.Type[C]) -> TypeGuard[c.Sequence[C]]: +def type_guard(sequence: c.Sequence[t.Any], guard_type: t.Type[C]) -> t.TypeGuard[c.Sequence[C]]: """ Raises an exception if any item in the given sequence does not match the specified guard type. Use with assert so that type checkers are aware of the type guard. diff --git a/test/lib/ansible_test/_util/target/common/constants.py b/test/lib/ansible_test/_util/target/common/constants.py index 812b919c6bc..da42299b0e4 100644 --- a/test/lib/ansible_test/_util/target/common/constants.py +++ b/test/lib/ansible_test/_util/target/common/constants.py @@ -10,10 +10,10 @@ REMOTE_ONLY_PYTHON_VERSIONS = ( '3.6', '3.7', '3.8', + '3.9', ) CONTROLLER_PYTHON_VERSIONS = ( - '3.9', '3.10', '3.11', ) diff --git a/test/lib/ansible_test/_util/target/setup/bootstrap.sh b/test/lib/ansible_test/_util/target/setup/bootstrap.sh index cbbe59466ca..90f80ad963e 100644 --- a/test/lib/ansible_test/_util/target/setup/bootstrap.sh +++ b/test/lib/ansible_test/_util/target/setup/bootstrap.sh @@ -163,8 +163,6 @@ bootstrap_remote_freebsd() # Declare platform/python version combinations which do not have supporting OS packages available. # For these combinations ansible-test will use pip to install the requirements instead. case "${platform_version}/${python_version}" in - "12.4/3.9") - ;; *) jinja2_pkg="" # not available cryptography_pkg="" # not available @@ -331,22 +329,14 @@ bootstrap_remote_rhel_9() # Jinja2 is not installed with an OS package since the provided version is too old. # Instead, ansible-test will install it using pip. + # packaging and resolvelib are missing for Python 3.11 (and possible later) so we just + # skip them and let ansible-test install them from PyPI. if [ "${controller}" ]; then packages=" ${packages} ${py_pkg_prefix}-cryptography ${py_pkg_prefix}-pyyaml " - - # The following OS packages are missing for 3.11 (and possibly later) so we just - # skip them and let ansible-test install them from PyPI. - if [ "${python_version}" = "3.9" ]; then - packages=" - ${packages} - ${py_pkg_prefix}-packaging - ${py_pkg_prefix}-resolvelib - " - fi fi while true; do @@ -425,14 +415,6 @@ bootstrap_remote_ubuntu() echo "Failed to install packages. Sleeping before trying again..." sleep 10 done - - if [ "${controller}" ]; then - if [ "${platform_version}/${python_version}" = "20.04/3.9" ]; then - # Install pyyaml using pip so libyaml support is available on Python 3.9. - # The OS package install (which is installed by default) only has a .so file for Python 3.8. - pip_install "--upgrade pyyaml" - fi - fi } bootstrap_docker() diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index 923e45723ce..bbd78a1b691 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -9,13 +9,10 @@ lib/ansible/config/base.yml no-unwanted-files lib/ansible/executor/powershell/async_watchdog.ps1 pslint:PSCustomUseLiteralPath lib/ansible/executor/powershell/async_wrapper.ps1 pslint:PSCustomUseLiteralPath lib/ansible/executor/powershell/exec_wrapper.ps1 pslint:PSCustomUseLiteralPath -lib/ansible/galaxy/collection/__init__.py mypy-3.9:attr-defined # inline ignore has no effect lib/ansible/galaxy/collection/__init__.py mypy-3.10:attr-defined # inline ignore has no effect lib/ansible/galaxy/collection/__init__.py mypy-3.11:attr-defined # inline ignore has no effect -lib/ansible/galaxy/collection/gpg.py mypy-3.9:arg-type lib/ansible/galaxy/collection/gpg.py mypy-3.10:arg-type lib/ansible/galaxy/collection/gpg.py mypy-3.11:arg-type -lib/ansible/parsing/yaml/constructor.py mypy-3.9:type-var # too many occurrences to ignore inline lib/ansible/parsing/yaml/constructor.py mypy-3.10:type-var # too many occurrences to ignore inline lib/ansible/parsing/yaml/constructor.py mypy-3.11:type-var # too many occurrences to ignore inline lib/ansible/keyword_desc.yml no-unwanted-files diff --git a/test/units/requirements.txt b/test/units/requirements.txt index 1822adaa1f9..c77c55cdd06 100644 --- a/test/units/requirements.txt +++ b/test/units/requirements.txt @@ -1,4 +1,4 @@ -bcrypt ; python_version >= '3.9' # controller only -passlib ; python_version >= '3.9' # controller only -pexpect ; python_version >= '3.9' # controller only -pywinrm ; python_version >= '3.9' # controller only +bcrypt ; python_version >= '3.10' # controller only +passlib ; python_version >= '3.10' # controller only +pexpect ; python_version >= '3.10' # controller only +pywinrm ; python_version >= '3.10' # controller only