From 724800cd3f39867943adb198bcbb8ab2523809c1 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Tue, 21 Sep 2021 13:43:31 -0500 Subject: [PATCH] Python 3.8 Controller Minimum (#74013) --- .../fragments/74013-controller-py38-min.yml | 3 ++ .../installation_guide/intro_installation.rst | 2 +- .../porting_guide_core_2.12.rst | 14 ++--- lib/ansible/cli/scripts/ansible_cli_stub.py | 45 ++++++++-------- lib/ansible/config/base.yml | 9 ---- lib/ansible/config/manager.py | 13 ++--- lib/ansible/executor/task_queue_manager.py | 5 +- lib/ansible/parsing/vault/__init__.py | 12 ++--- lib/ansible/parsing/yaml/dumper.py | 16 ++---- lib/ansible/parsing/yaml/objects.py | 19 +------ lib/ansible/playbook/play_context.py | 6 +-- lib/ansible/plugins/callback/__init__.py | 10 +--- lib/ansible/template/native_helpers.py | 5 +- lib/ansible/template/safe_eval.py | 51 ++----------------- setup.py | 8 +-- .../ansible_test/_internal/ansible_util.py | 1 - test/sanity/ignore.txt | 1 - 17 files changed, 56 insertions(+), 164 deletions(-) create mode 100644 changelogs/fragments/74013-controller-py38-min.yml diff --git a/changelogs/fragments/74013-controller-py38-min.yml b/changelogs/fragments/74013-controller-py38-min.yml new file mode 100644 index 00000000000..402c0f3c046 --- /dev/null +++ b/changelogs/fragments/74013-controller-py38-min.yml @@ -0,0 +1,3 @@ +major_changes: +- Python Controller Requirement - Python 3.8 or newer is required for the control node (the machine that runs Ansible) + (https://github.com/ansible/ansible/pull/74013) diff --git a/docs/docsite/rst/installation_guide/intro_installation.rst b/docs/docsite/rst/installation_guide/intro_installation.rst index 7c3aa26cf10..a6fa962bde4 100644 --- a/docs/docsite/rst/installation_guide/intro_installation.rst +++ b/docs/docsite/rst/installation_guide/intro_installation.rst @@ -22,7 +22,7 @@ Before you install Ansible, review the requirements for a control node. Before y Control node requirements ------------------------- -For your control node (the machine that runs Ansible), you can use any machine with Python 2 (version 2.7) or Python 3 (versions 3.5 and higher) installed. ansible-core 2.11 and Ansible 4.0.0 will make Python 3.8 a soft dependency for the control node, but will function with the aforementioned requirements. ansible-core 2.12 and Ansible 5.0.0 will require Python 3.8 or newer to function on the control node. Starting with ansible-core 2.11, the project will only be packaged for Python 3.8 and newer. +For your control node (the machine that runs Ansible), you can use any machine with Python 3.8 or newer installed. This includes Red Hat, Debian, CentOS, macOS, any of the BSDs, and so on. Windows is not supported for the control node, read more about this in `Matt Davis's blog post `_. diff --git a/docs/docsite/rst/porting_guides/porting_guide_core_2.12.rst b/docs/docsite/rst/porting_guides/porting_guide_core_2.12.rst index 3961a553e2c..fbfbc43c1bd 100644 --- a/docs/docsite/rst/porting_guides/porting_guide_core_2.12.rst +++ b/docs/docsite/rst/porting_guides/porting_guide_core_2.12.rst @@ -1,15 +1,15 @@ -.. _porting_2.12_guide: +.. _porting_2.12_guide_core: -************************** -Ansible 2.12 Porting Guide -************************** +******************************* +Ansible-core 2.12 Porting Guide +******************************* -This section discusses the behavioral changes between Ansible 2.11 and Ansible 2.12. +This section discusses the behavioral changes between ``ansible-core`` 2.11 and ``ansible-core`` 2.12. It is intended to assist in updating your playbooks, plugins and other parts of your Ansible infrastructure so they will work with this version of Ansible. -We suggest you read this page along with `Ansible Changelog for 2.12 `_ to understand what updates you may need to make. +We suggest you read this page along with `ansible-core Changelog for 2.12 `_ to understand what updates you may need to make. This document is part of a collection on porting. The complete list of porting guides can be found at :ref:`porting guides `. @@ -45,9 +45,9 @@ See :ref:`interpreter discovery documentation ` for more Command Line ============ +* Python 3.8 on the controller node is a hard requirement for this release. The command line scripts will not function with a lower Python version. * ``ansible-vault`` no longer supports ``PyCrypto`` and requires ``cryptography``. - Deprecated ========== diff --git a/lib/ansible/cli/scripts/ansible_cli_stub.py b/lib/ansible/cli/scripts/ansible_cli_stub.py index d7ebbb79b0d..622152c4155 100755 --- a/lib/ansible/cli/scripts/ansible_cli_stub.py +++ b/lib/ansible/cli/scripts/ansible_cli_stub.py @@ -29,19 +29,23 @@ import shutil import sys import traceback +# Used for determining if the system is running a new enough python version +# and should only restrict on our documented minimum versions +_PY38_MIN = sys.version_info[:2] >= (3, 8) +if not _PY38_MIN: + raise SystemExit( + 'ERROR: Ansible requires Python 3.8 or newer on the controller. ' + 'Current version: %s' % ''.join(sys.version.splitlines()) + ) + + +# These lines appear after the PY38 check, to ensure the "friendly" error happens before +# any invalid syntax appears in other files that may get imported from ansible import context from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError from ansible.module_utils._text import to_text - -# Used for determining if the system is running a new enough python version -# and should only restrict on our documented minimum versions -_PY38_MIN = sys.version_info[:2] >= (3, 8) -_PY3_MIN = sys.version_info[:2] >= (3, 5) -_PY2_MIN = (2, 6) <= sys.version_info[:2] < (3,) -_PY_MIN = _PY3_MIN or _PY2_MIN -if not _PY_MIN: - raise SystemExit('ERROR: Ansible requires a minimum of Python2 version 2.6 or Python3 version 3.5. Current version: %s' % ''.join(sys.version.splitlines())) +from pathlib import Path class LastResort(object): @@ -67,19 +71,10 @@ if __name__ == '__main__': initialize_locale() cli = None - me = os.path.basename(sys.argv[0]) + me = Path(sys.argv[0]).name try: display = Display() - if C.CONTROLLER_PYTHON_WARNING and not _PY38_MIN: - display.deprecated( - ( - 'Ansible will require Python 3.8 or newer on the controller starting with Ansible 2.12. ' - 'Current version: %s' % ''.join(sys.version.splitlines()) - ), - version='2.12', - collection_name='ansible.builtin', - ) display.debug("starting run") sub = None @@ -111,16 +106,16 @@ if __name__ == '__main__': else: raise - b_ansible_dir = os.path.expanduser(os.path.expandvars(b"~/.ansible")) + ansible_dir = Path("~/.ansible").expanduser() try: - os.mkdir(b_ansible_dir, 0o700) + ansible_dir.mkdir(mode=0o700) except OSError as exc: if exc.errno != errno.EEXIST: - display.warning("Failed to create the directory '%s': %s" - % (to_text(b_ansible_dir, errors='surrogate_or_replace'), - to_text(exc, errors='surrogate_or_replace'))) + display.warning( + "Failed to create the directory '%s': %s" % (ansible_dir, to_text(exc, errors='surrogate_or_replace')) + ) else: - display.debug("Created the '%s' directory" % to_text(b_ansible_dir, errors='surrogate_or_replace')) + display.debug("Created the '%s' directory" % ansible_dir) try: args = [to_text(a, errors='surrogate_or_strict') for a in sys.argv] diff --git a/lib/ansible/config/base.yml b/lib/ansible/config/base.yml index a5070956545..35a6ad2fca9 100644 --- a/lib/ansible/config/base.yml +++ b/lib/ansible/config/base.yml @@ -558,15 +558,6 @@ CALLABLE_ACCEPT_LIST: section: defaults version_added: '2.11' type: list -CONTROLLER_PYTHON_WARNING: - name: Running Older than Python 3.8 Warning - default: True - description: Toggle to control showing warnings related to running a Python version - older than Python 3.8 on the controller - env: [{name: ANSIBLE_CONTROLLER_PYTHON_WARNING}] - ini: - - {key: controller_python_warning, section: defaults} - type: boolean DEFAULT_CALLBACK_PLUGIN_PATH: name: Callback Plugins Path default: ~/.ansible/plugins/callback:/usr/share/ansible/plugins/callback diff --git a/lib/ansible/config/manager.py b/lib/ansible/config/manager.py index 5adad38b21a..ee9979b0b58 100644 --- a/lib/ansible/config/manager.py +++ b/lib/ansible/config/manager.py @@ -19,7 +19,7 @@ from ansible.errors import AnsibleOptionsError, AnsibleError from ansible.module_utils._text import to_text, to_bytes, to_native from ansible.module_utils.common._collections_compat import Mapping, Sequence from ansible.module_utils.common.yaml import yaml_load -from ansible.module_utils.six import PY3, string_types +from ansible.module_utils.six import string_types from ansible.module_utils.six.moves import configparser from ansible.module_utils.parsing.convert_bool import boolean from ansible.parsing.quoting import unquote @@ -323,21 +323,14 @@ class ConfigManager(object): ftype = get_config_type(cfile) if cfile is not None: if ftype == 'ini': - kwargs = {} - if PY3: - kwargs['inline_comment_prefixes'] = (';',) - self._parsers[cfile] = configparser.ConfigParser(**kwargs) + self._parsers[cfile] = configparser.ConfigParser(inline_comment_prefixes=(';',)) with open(to_bytes(cfile), 'rb') as f: try: cfg_text = to_text(f.read(), errors='surrogate_or_strict') except UnicodeError as e: raise AnsibleOptionsError("Error reading config file(%s) because the config file was not utf8 encoded: %s" % (cfile, to_native(e))) try: - if PY3: - self._parsers[cfile].read_string(cfg_text) - else: - cfg_file = io.StringIO(cfg_text) - self._parsers[cfile].readfp(cfg_file) + self._parsers[cfile].read_string(cfg_text) except configparser.Error as e: raise AnsibleOptionsError("Error reading config file (%s): %s" % (cfile, to_native(e))) # FIXME: this should eventually handle yaml config files diff --git a/lib/ansible/executor/task_queue_manager.py b/lib/ansible/executor/task_queue_manager.py index 48ebf1aaff2..bc701e50627 100644 --- a/lib/ansible/executor/task_queue_manager.py +++ b/lib/ansible/executor/task_queue_manager.py @@ -32,7 +32,7 @@ from ansible.errors import AnsibleError from ansible.executor.play_iterator import PlayIterator from ansible.executor.stats import AggregateStats from ansible.executor.task_result import TaskResult -from ansible.module_utils.six import PY3, string_types +from ansible.module_utils.six import string_types from ansible.module_utils._text import to_text, to_native from ansible.playbook.play_context import PlayContext from ansible.playbook.task import Task @@ -60,8 +60,7 @@ class CallbackSend: class FinalQueue(multiprocessing.queues.Queue): def __init__(self, *args, **kwargs): - if PY3: - kwargs['ctx'] = multiprocessing_context + kwargs['ctx'] = multiprocessing_context super(FinalQueue, self).__init__(*args, **kwargs) def send_callback(self, method_name, *args, **kwargs): diff --git a/lib/ansible/parsing/vault/__init__.py b/lib/ansible/parsing/vault/__init__.py index 11e54d0496f..1b4d8113ba7 100644 --- a/lib/ansible/parsing/vault/__init__.py +++ b/lib/ansible/parsing/vault/__init__.py @@ -54,7 +54,7 @@ except ImportError: from ansible.errors import AnsibleError, AnsibleAssertionError from ansible import constants as C -from ansible.module_utils.six import PY3, binary_type +from ansible.module_utils.six import binary_type # Note: on py2, this zip is izip not the list based zip() builtin from ansible.module_utils.six.moves import zip from ansible.module_utils._text import to_bytes, to_text, to_native @@ -1015,10 +1015,7 @@ class VaultEditor: try: if filename == '-': - if PY3: - data = sys.stdin.buffer.read() - else: - data = sys.stdin.read() + data = sys.stdin.buffer.read() else: with open(filename, "rb") as fh: data = fh.read() @@ -1258,10 +1255,7 @@ class VaultAES256: result = 0 for b_x, b_y in zip(b_a, b_b): - if PY3: - result |= b_x ^ b_y - else: - result |= ord(b_x) ^ ord(b_y) + result |= b_x ^ b_y return result == 0 @classmethod diff --git a/lib/ansible/parsing/yaml/dumper.py b/lib/ansible/parsing/yaml/dumper.py index 72731b2d128..65d35781f79 100644 --- a/lib/ansible/parsing/yaml/dumper.py +++ b/lib/ansible/parsing/yaml/dumper.py @@ -21,7 +21,7 @@ __metaclass__ = type import yaml -from ansible.module_utils.six import PY3, text_type, binary_type +from ansible.module_utils.six import text_type, binary_type from ansible.module_utils.common.yaml import SafeDumper from ansible.parsing.yaml.objects import AnsibleUnicode, AnsibleSequence, AnsibleMapping, AnsibleVaultEncryptedUnicode from ansible.utils.unsafe_proxy import AnsibleUnsafeText, AnsibleUnsafeBytes @@ -46,18 +46,12 @@ def represent_vault_encrypted_unicode(self, data): return self.represent_scalar(u'!vault', data._ciphertext.decode(), style='|') -if PY3: - def represent_unicode(self, data): - return yaml.representer.SafeRepresenter.represent_str(self, text_type(data)) +def represent_unicode(self, data): + return yaml.representer.SafeRepresenter.represent_str(self, text_type(data)) - def represent_binary(self, data): - return yaml.representer.SafeRepresenter.represent_binary(self, binary_type(data)) -else: - def represent_unicode(self, data): - return yaml.representer.SafeRepresenter.represent_unicode(self, text_type(data)) - def represent_binary(self, data): - return yaml.representer.SafeRepresenter.represent_str(self, binary_type(data)) +def represent_binary(self, data): + return yaml.representer.SafeRepresenter.represent_binary(self, binary_type(data)) def represent_undefined(self, data): diff --git a/lib/ansible/parsing/yaml/objects.py b/lib/ansible/parsing/yaml/objects.py index 3da84471a12..3d183f5895d 100644 --- a/lib/ansible/parsing/yaml/objects.py +++ b/lib/ansible/parsing/yaml/objects.py @@ -58,17 +58,7 @@ class AnsibleBaseYAMLObject(object): ansible_pos = property(_get_ansible_position, _set_ansible_position) -# try to always use orderddict with yaml, after py3.6 the dict type already does this -odict = dict -if sys.version_info[:2] < (3, 7): - # if python 2.7 or py3 < 3.7 - try: - from collections import OrderedDict as odict - except ImportError: - pass - - -class AnsibleMapping(AnsibleBaseYAMLObject, odict): +class AnsibleMapping(AnsibleBaseYAMLObject, dict): ''' sub class for dictionaries ''' pass @@ -314,12 +304,7 @@ class AnsibleVaultEncryptedUnicode(Sequence, AnsibleBaseYAMLObject): def lstrip(self, chars=None): return self.data.lstrip(chars) - try: - # PY3 - maketrans = str.maketrans - except AttributeError: - # PY2 - maketrans = string.maketrans + maketrans = str.maketrans def partition(self, sep): return self.data.partition(sep) diff --git a/lib/ansible/playbook/play_context.py b/lib/ansible/playbook/play_context.py index 9058792f458..a3b70f5a5f0 100644 --- a/lib/ansible/playbook/play_context.py +++ b/lib/ansible/playbook/play_context.py @@ -199,10 +199,8 @@ class PlayContext(Base): # loop through a subset of attributes on the task object and set # connection fields based on their values for attr in TASK_ATTRIBUTE_OVERRIDES: - if hasattr(task, attr): - attr_val = getattr(task, attr) - if attr_val is not None: - setattr(new_info, attr, attr_val) + if (attr_val := getattr(task, attr, None)) is not None: + setattr(new_info, attr, attr_val) # next, use the MAGIC_VARIABLE_MAPPING dictionary to update this # connection info object with 'magic' variables from the variable list. diff --git a/lib/ansible/plugins/callback/__init__.py b/lib/ansible/plugins/callback/__init__.py index 38a1daa76df..3b4503cb89a 100644 --- a/lib/ansible/plugins/callback/__init__.py +++ b/lib/ansible/plugins/callback/__init__.py @@ -22,12 +22,11 @@ __metaclass__ = type import difflib import json import sys - +from collections import OrderedDict from copy import deepcopy from ansible import constants as C from ansible.module_utils.common._collections_compat import MutableMapping -from ansible.module_utils.six import PY3 from ansible.module_utils._text import to_text from ansible.parsing.ajson import AnsibleJSONEncoder from ansible.plugins import AnsiblePlugin, get_plugin_class @@ -35,13 +34,6 @@ from ansible.utils.color import stringc from ansible.utils.display import Display from ansible.vars.clean import strip_internal_keys, module_response_deepcopy -if PY3: - # OrderedDict is needed for a backwards compat shim on Python3.x only - # https://github.com/ansible/ansible/pull/49512 - from collections import OrderedDict -else: - OrderedDict = None - global_display = Display() diff --git a/lib/ansible/template/native_helpers.py b/lib/ansible/template/native_helpers.py index d2632051cab..8886fc1b0b7 100644 --- a/lib/ansible/template/native_helpers.py +++ b/lib/ansible/template/native_helpers.py @@ -15,7 +15,7 @@ from jinja2.runtime import StrictUndefined from ansible.module_utils._text import to_text from ansible.module_utils.common.collections import is_sequence, Mapping from ansible.module_utils.common.text.converters import container_to_text -from ansible.module_utils.six import PY2, text_type, string_types +from ansible.module_utils.six import text_type, string_types from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode from ansible.utils.native_jinja import NativeJinjaText @@ -84,9 +84,6 @@ def ansible_native_concat(nodes): try: out = literal_eval(out) - if PY2: - # ensure bytes are not returned back into Ansible from templating - out = container_to_text(out) return out except (ValueError, SyntaxError, MemoryError): return out diff --git a/lib/ansible/template/safe_eval.py b/lib/ansible/template/safe_eval.py index 3ed46addaf9..b7389dcecf8 100644 --- a/lib/ansible/template/safe_eval.py +++ b/lib/ansible/template/safe_eval.py @@ -23,9 +23,8 @@ import sys from ansible import constants as C from ansible.module_utils.common.text.converters import container_to_text, to_native -from ansible.module_utils.six import string_types, PY2 +from ansible.module_utils.six import string_types from ansible.module_utils.six.moves import builtins -from ansible.plugins.loader import filter_loader, test_loader def safe_eval(expr, locals=None, include_exceptions=False): @@ -64,6 +63,7 @@ def safe_eval(expr, locals=None, include_exceptions=False): ast.BinOp, # ast.Call, ast.Compare, + ast.Constant, ast.Dict, ast.Div, ast.Expression, @@ -72,6 +72,7 @@ def safe_eval(expr, locals=None, include_exceptions=False): ast.Mult, ast.Num, ast.Name, + ast.Set, ast.Str, ast.Sub, ast.USub, @@ -80,47 +81,7 @@ def safe_eval(expr, locals=None, include_exceptions=False): ) ) - # AST node types were expanded after 2.6 - if sys.version_info[:2] >= (2, 7): - SAFE_NODES.update( - set( - (ast.Set,) - ) - ) - - # And in Python 3.4 too - if sys.version_info[:2] >= (3, 4): - SAFE_NODES.update( - set( - (ast.NameConstant,) - ) - ) - - # And in Python 3.6 too, although not encountered until Python 3.8, see https://bugs.python.org/issue32892 - if sys.version_info[:2] >= (3, 6): - SAFE_NODES.update( - set( - (ast.Constant,) - ) - ) - - filter_list = [] - for filter_ in filter_loader.all(): - try: - filter_list.extend(filter_.filters().keys()) - except Exception: - # This is handled and displayed in JinjaPluginIntercept._load_ansible_plugins - continue - - test_list = [] - for test in test_loader.all(): - try: - test_list.extend(test.tests().keys()) - except Exception: - # This is handled and displayed in JinjaPluginIntercept._load_ansible_plugins - continue - - CALL_ENABLED = C.CALLABLE_ACCEPT_LIST + filter_list + test_list + CALL_ENABLED = [] class CleansingNodeVisitor(ast.NodeVisitor): def generic_visit(self, node, inside_call=False): @@ -153,10 +114,6 @@ def safe_eval(expr, locals=None, include_exceptions=False): # callables (and other identifiers) are recognized. this is in # addition to the filtering of builtins done in CleansingNodeVisitor result = eval(compiled, OUR_GLOBALS, dict(locals)) - if PY2: - # On Python 2 u"{'key': 'value'}" is evaluated to {'key': 'value'}, - # ensure it is converted to {u'key': u'value'}. - result = container_to_text(result) if include_exceptions: return (result, None) diff --git a/setup.py b/setup.py index 1cf9f08ff17..11f88c6afaa 100644 --- a/setup.py +++ b/setup.py @@ -333,7 +333,7 @@ static_setup_params = dict( license='GPLv3+', # Ansible will also make use of a system copy of python-six and # python-selectors2 if installed but use a Bundled copy if it's not. - python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*', + python_requires='>=3.8', package_dir={'': 'lib', 'ansible_test': 'test/lib/ansible_test'}, packages=find_packages('lib') + find_packages('test/lib'), @@ -347,14 +347,10 @@ static_setup_params = dict( 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)', 'Natural Language :: English', 'Operating System :: POSIX', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Topic :: System :: Installation/Setup', 'Topic :: System :: Systems Administration', 'Topic :: Utilities', diff --git a/test/lib/ansible_test/_internal/ansible_util.py b/test/lib/ansible_test/_internal/ansible_util.py index a3446743f2b..1d98ec2b9fd 100644 --- a/test/lib/ansible_test/_internal/ansible_util.py +++ b/test/lib/ansible_test/_internal/ansible_util.py @@ -101,7 +101,6 @@ def ansible_environment(args, color=True, ansible_config=None): ANSIBLE_CONFIG=ansible_config, ANSIBLE_LIBRARY='/dev/null', ANSIBLE_DEVEL_WARNING='false', # Don't show warnings that CI is running devel - ANSIBLE_CONTROLLER_PYTHON_WARNING='false', # Don't show warnings in CI for old controller Python ANSIBLE_JINJA2_NATIVE_WARNING='false', # Don't show warnings in CI for old Jinja for native PYTHONPATH=get_ansible_python_path(args), PAGER='/bin/cat', diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index 76baca07455..4a3bb6206df 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -8,7 +8,6 @@ examples/scripts/my_test_info.py shebang # example module but not in a normal mo examples/scripts/upgrade_to_ps3.ps1 pslint:PSCustomUseLiteralPath examples/scripts/upgrade_to_ps3.ps1 pslint:PSUseApprovedVerbs lib/ansible/cli/console.py pylint:disallowed-name -lib/ansible/cli/scripts/ansible_cli_stub.py pylint:ansible-deprecated-version lib/ansible/cli/scripts/ansible_cli_stub.py shebang lib/ansible/cli/scripts/ansible_connection_cli_stub.py shebang lib/ansible/config/base.yml no-unwanted-files