From 357fe38766e0db4c3d75b1eaf946e32b10597db6 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 11 Aug 2024 09:27:15 +0100 Subject: [PATCH] Ansible 10 (ansible-core 2.17) support Notably - Python 2.7 and 3.6 are no longer supported by Ansible on targets - The yum module has been removed, and redirected to dnf - _INTERPRETER_PYTHON_DISTRO_MAP has been neutered. Interpreter discovery always favours specific `python3.` interpreters in decending version order, then generic `python3` or `python`. - Add the ability for an action plugin to call self._execute_module(*, ignore_unknown_opts=True) to execute a module with options that may not be supported for the version being called. https://docs.ansible.com/ansible/devel/porting_guides/porting_guide_10.html https://github.com/ansible-community/ansible-build-data/blob/main/10/CHANGELOG-v10.md https://github.com/ansible/ansible/blob/stable-2.17/changelogs/CHANGELOG-v2.17.rst fixes #1074, refs #1082 Co-authored-by: Claude Becker --- .ci/azure-pipelines.yml | 9 +- ansible_mitogen/loaders.py | 2 +- ansible_mitogen/mixins.py | 12 +- docs/ansible_detailed.rst | 21 ++-- docs/changelog.rst | 1 + .../ansible_2_8_tests.yml | 106 ++++++++++-------- .../interpreter_discovery/complex_args.yml | 7 ++ .../runner/custom_bash_hashbang_argument.yml | 6 + .../transport_config/python_path.yml | 18 ++- ...ssue_1066__add_host__host_key_checking.yml | 4 + .../issue_776__load_plugins_called_twice.yml | 14 ++- tox.ini | 17 ++- 12 files changed, 147 insertions(+), 70 deletions(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 51908e92..a3ffa9b4 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -32,10 +32,14 @@ jobs: tox.env: py312-mode_localhost-ansible9 Van_312_9: tox.env: py312-mode_localhost-ansible9-strategy_linear + Loc_312_10: + tox.env: py312-mode_localhost-ansible10 + Van_312_10: + tox.env: py312-mode_localhost-ansible10-strategy_linear - job: Linux pool: - # https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2004-Readme.md + # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md vmImage: ubuntu-20.04 steps: - template: azure-pipelines-steps.yml @@ -152,3 +156,6 @@ jobs: Ans_312_9: python.version: '3.12' tox.env: py312-mode_ansible-ansible9 + Ans_312_10: + python.version: '3.12' + tox.env: py312-mode_ansible-ansible10 diff --git a/ansible_mitogen/loaders.py b/ansible_mitogen/loaders.py index 9729b8a1..9d9876ac 100644 --- a/ansible_mitogen/loaders.py +++ b/ansible_mitogen/loaders.py @@ -49,7 +49,7 @@ __all__ = [ ANSIBLE_VERSION_MIN = (2, 10) -ANSIBLE_VERSION_MAX = (2, 16) +ANSIBLE_VERSION_MAX = (2, 17) NEW_VERSION_MSG = ( "Your Ansible version (%s) is too recent. The most recent version\n" diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 9cc97a48..0ba41aad 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -357,7 +357,9 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): def _execute_module(self, module_name=None, module_args=None, tmp=None, task_vars=None, persist_files=False, - delete_remote_tmp=True, wrap_async=False): + delete_remote_tmp=True, wrap_async=False, + ignore_unknown_opts=False, + ): """ Collect up a module's execution environment then use it to invoke target.run_module() or helpers.run_module_async() in the target @@ -370,7 +372,13 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): if task_vars is None: task_vars = {} - self._update_module_args(module_name, module_args, task_vars) + if ansible_mitogen.utils.ansible_version[:2] >= (2, 17): + self._update_module_args( + module_name, module_args, task_vars, + ignore_unknown_opts=ignore_unknown_opts, + ) + else: + self._update_module_args(module_name, module_args, task_vars) env = {} self._compute_environment_string(env) self._set_temp_file_args(module_args, wrap_async) diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index e6f7b42c..7d960ed9 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -165,7 +165,9 @@ Noteworthy Differences +-----------------+-----------------+ | 8 | 3.9 - 3.12 | +-----------------+-----------------+ - | 9 | 3.10 - 3.12 | + | 9 | | + +-----------------+ 3.10 - 3.12 | + | 10 | | +-----------------+-----------------+ Verify your installation is running one of these versions by checking @@ -271,15 +273,14 @@ Noteworthy Differences * "Module Replacer" style modules are not supported. These rarely appear in practice, and light web searches failed to reveal many examples of them. -.. - * The ``ansible_python_interpreter`` variable is parsed using a restrictive - :mod:`shell-like ` syntax, permitting values such as ``/usr/bin/env - FOO=bar python`` or ``source /opt/rh/rh-python36/enable && python``, which - occur in practice. Jinja2 templating is also supported for complex task-level - interpreter settings. Ansible `documents this - `_ - as an absolute path, however the implementation passes it unquoted through - the shell, permitting arbitrary code to be injected. +* The ``ansible_python_interpreter`` variable is parsed using a restrictive + :mod:`shell-like ` syntax, permitting values such as ``/usr/bin/env + FOO=bar python`` or ``source /opt/rh/rh-python36/enable && python``. + Jinja2 templating is also supported for complex task-level + interpreter settings. Ansible documents `ansible_python_interpreter + `_ + as an absolute path and releases since June 2024 (e.g. Ansible 10.1) + reflect this. Older Ansible releases passed it to the shell unquoted. .. * Configurations will break that rely on the `hashbang argument splitting diff --git a/docs/changelog.rst b/docs/changelog.rst index c1754b9e..33ff9f9b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,6 +23,7 @@ Unreleased * :gh:issue:`1097` Respect `ansible_facts.discovered_interpreter_python` when executing non new-style modules (e.g. JSONARGS style, WANT_JSON style). +* :gh:issue:`1074` Support Ansible 10 (ansible-core 2.17) v0.3.8 (2024-07-30) diff --git a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml index 6ac9bada..0c7d30c9 100644 --- a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml +++ b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml @@ -4,6 +4,55 @@ - name: integration/interpreter_discovery/ansible_2_8_tests.yml hosts: test-targets gather_facts: true + vars: + DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_lt_2_12: + centos: + '6': /usr/bin/python + '7': /usr/bin/python + '8': /usr/libexec/platform-python + debian: + '9': /usr/bin/python + '10': /usr/bin/python3 + '11': /usr/bin/python + 'NA': /usr/bin/python # Debian 11, Ansible <= 7 (ansible-core <= 2.14) + 'bullseye/sid': /usr/bin/python # Debian 11, Ansible 8 - 9 (ansible-core 2.15 - 2.16) + ubuntu: + '16': /usr/bin/python3 + '18': /usr/bin/python3 + '20': /usr/bin/python3 + + DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_2_12_to_2_16: + centos: + '6': /usr/bin/python + '7': /usr/bin/python + '8': /usr/libexec/platform-python + debian: + '9': /usr/bin/python + '10': /usr/bin/python3 + '11': /usr/bin/python3.9 + 'NA': /usr/bin/python3.9 # Debian 11, Ansible <= 7 (ansible-core <= 2.14) + 'bullseye/sid': /usr/bin/python3.9 # Debian 11, Ansible 8 - 9 (ansible-core 2.15 - 2.16) + ubuntu: + '16': /usr/bin/python3 + '18': /usr/bin/python3 + '20': /usr/bin/python3 + + DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_ge_2_17: + debian: + '10': /usr/bin/python3.7 + '11': /usr/bin/python3.9 + 'bullseye/sid': /usr/bin/python3.9 + ubuntu: + '20': /usr/bin/python3.8 + + discovered_interpreter_expected: >- + {%- if ansible_version.full is version('2.12', '<', strict=True) -%} + {{ DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_lt_2_12[distro][distro_major] }} + {%- elif ansible_version.full is version('2.17', '<', strict=True) -%} + {{ DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_2_12_to_2_16[distro][distro_major] }} + {%- else -%} + {{ DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_ge_2_17[distro][distro_major] }} + {%- endif -%} tasks: - name: can only run these tests on ansible >= 2.8.0 block: @@ -18,9 +67,9 @@ - name: snag some facts to validate for later set_fact: - distro: '{{ ansible_distribution | default("unknown") | lower }}' - distro_version: '{{ ansible_distribution_version | default("unknown") }}' - os_family: '{{ ansible_os_family | default("unknown") }}' + distro: '{{ ansible_facts.distribution | lower }}' + distro_major: '{{ ansible_facts.distribution_major_version }}' + system: '{{ ansible_facts.system }}' - name: test that python discovery is working and that fact persistence makes it only run once block: @@ -159,50 +208,19 @@ - ansible_facts['ansible_bogus_interpreter'] | default('nope') == 'nope' - ansible_facts['discovered_interpreter_bogus'] | default('nope') == 'nope' - - name: fedora assertions + - name: Check discovered interpreter matches expected assert: that: - - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python3' - fail_msg: auto_out={{auto_out}} + - auto_out.ansible_facts.discovered_interpreter_python == discovered_interpreter_expected + fail_msg: >- + distro={{ distro }} + distro_major= {{ distro_major }} + system={{ system }} + auto_out={{ auto_out }} + discovered_interpreter_expected={{ discovered_interpreter_expected }} + ansible_version.full={{ ansible_version.full }} when: - - distro == 'fedora' - - distro_version is version('23.0', '>=', strict=True) - - - name: rhel < 8 assertions - assert: - that: - - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python' - fail_msg: auto_out={{auto_out}} - when: - - distro in ('redhat', 'centos') - - distro_version is version('8.0', '<', strict=true) - - - name: rhel 8+ assertions - assert: - that: - - auto_out.ansible_facts.discovered_interpreter_python == '/usr/libexec/platform-python' - fail_msg: auto_out={{auto_out}} - when: - - distro in ('redhat', 'centos') - - distro_version is version('8.0', '>=', strict=true) - - - name: ubuntu < 16.04 assertions - assert: - that: - - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python' - fail_msg: auto_out={{auto_out}} - when: - - distro == 'ubuntu' - - distro_version is version('16.04', '<', strict=true) - - - name: ubuntu 16.04+ assertions - assert: - that: - - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python3' - fail_msg: auto_out={{auto_out}} - when: - - distro == 'ubuntu' - - distro_version is version('16.04', '>=', strict=True) + - system in ['Linux'] always: - meta: clear_facts diff --git a/tests/ansible/integration/interpreter_discovery/complex_args.yml b/tests/ansible/integration/interpreter_discovery/complex_args.yml index ea7e2d63..af2b5e46 100644 --- a/tests/ansible/integration/interpreter_discovery/complex_args.yml +++ b/tests/ansible/integration/interpreter_discovery/complex_args.yml @@ -9,6 +9,13 @@ https_proxy: "{{ lookup('env', 'https_proxy') | default(omit) }}" no_proxy: "{{ lookup('env', 'no_proxy') | default(omit) }}" tasks: + # Ansible releases after June 2024 quote ansible_python_interpreter + # https://github.com/ansible/ansible/pull/83365 + - meta: end_play + when: + - not is_mitogen + - ansible_version.full is version('2.17.1', '>=', strict=True) + - name: create temp file to source file: path: /tmp/fake diff --git a/tests/ansible/integration/runner/custom_bash_hashbang_argument.yml b/tests/ansible/integration/runner/custom_bash_hashbang_argument.yml index ebb7966e..82535d56 100644 --- a/tests/ansible/integration/runner/custom_bash_hashbang_argument.yml +++ b/tests/ansible/integration/runner/custom_bash_hashbang_argument.yml @@ -2,6 +2,12 @@ - name: integration/runner/custom_bash_hashbang_argument.yml hosts: test-targets tasks: + # Ansible releases after June 2024 quote ansible_python_interpreter + # https://github.com/ansible/ansible/pull/83365 + - meta: end_play + when: + - not is_mitogen + - ansible_version.full is version('2.17.1', '>=', strict=True) - custom_bash_old_style_module: foo: true diff --git a/tests/ansible/integration/transport_config/python_path.yml b/tests/ansible/integration/transport_config/python_path.yml index 0f04ed59..ecc24374 100644 --- a/tests/ansible/integration/transport_config/python_path.yml +++ b/tests/ansible/integration/transport_config/python_path.yml @@ -2,12 +2,17 @@ # Each case is followed by mitogen_via= case to test hostvars method. -# When no ansible_python_interpreter is set, ansible 2.8+ automatically -# tries to detect the desired interpreter, falling back to "/usr/bin/python" if necessary +# If ansible_python_interpreter isn't set, Ansible 2.8+ tries to connect and +# detect the interpreter. If that fails (e.g. connection timeout) +# - Ansible 2.8 - 9 (ansible-core 2.8 - 2.16) assumes "/usr/bin/python" +# - Ansible 10+ (ansible-core 2.17+) marks the target unreachable - name: integration/transport_config/python_path.yml hosts: tc-python-path-unset tasks: - include_tasks: ../_mitogen_only.yml + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path @@ -19,6 +24,9 @@ vars: {mitogen_via: tc-python-path-unset} tasks: - include_tasks: ../_mitogen_only.yml + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path @@ -45,6 +53,9 @@ vars: {mitogen_via: tc-python-path-hostvar} tasks: - include_tasks: ../_mitogen_only.yml + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path @@ -71,6 +82,9 @@ vars: {mitogen_via: localhost} tasks: - include_tasks: ../_mitogen_only.yml + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path diff --git a/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml b/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml index 4b4609b6..dd754f84 100644 --- a/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml +++ b/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml @@ -28,6 +28,10 @@ become: false serial: 1 tasks: + # FIXME https://github.com/mitogen-hq/mitogen/issues/1096 + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) - meta: reset_connection # The host key might be in ~/.ssh/known_hosts. If it's removed then no diff --git a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml index 44ff6863..ef573276 100755 --- a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml +++ b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml @@ -3,13 +3,25 @@ - name: regression/issue_776__load_plugins_called_twice.yml hosts: test-targets become: "{{ groups.linux is defined and inventory_hostname in groups.linux }}" - gather_facts: yes + # Delayed until after the end_play, due to Python version requirements + gather_facts: false tags: - issue_776 vars: ansible_python_interpreter: "{{ pkg_mgr_python_interpreter }}" package: rsync # Chosen to exist in all tested distros/package managers tasks: + # The package management modules require using the same Python version + # as the target's package manager libraries. This is sometimes in conflict + # with Ansible requirements, e.g. Ansible 10 (ansible-core 2.17) does not + # support Python 2.x on targets. + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) + + - name: Gather facts manually + setup: + - name: Switch to archived package repositories copy: dest: "{{ item.dest }}" diff --git a/tox.ini b/tox.ini index 5dcb1bf2..08f4c371 100644 --- a/tox.ini +++ b/tox.ini @@ -10,9 +10,9 @@ # 2.4 2.3? <= 3.7.1 <= 1.3.7 <= 1.1 <= 2.1.3 <= 1.4 <= 1.8 # 2.5 <= 3.7.1 <= 1.4.22 <= 1.3.1 <= 2.1.3 <= 2.8.7 <= 1.6.1 <= 1.9.1 # 2.6 <= 2.6.20 <= 2.12 <= 4.5.4 <= 1.6.11 <= 2.10.3 <= 9.0.3 <= 5.9.0 <= 3.2.5 <= 2.9.1 <= 15.2.0 -# 2.7 <= 2.11 <= 5.5 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.15² +# 2.7 <= 2.11 <= 2.16 <= 5.5 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.15² # 3.5 <= 2.11 <= 2.15 <= 5.5 <= 2.2.28 <= 2.11.3 <= 20 <= 5.9.5 <= 6.1.0 <= 3.28 <= 20.15² -# 3.6 <= 2.11 <= 6.2 <= 3.2.20 <= 3.0.3 <= 21 <= 7.0.1 <= 3.28 <= 20.17² +# 3.6 <= 2.11 <= 2.16 <= 6.2 <= 3.2.20 <= 3.0.3 <= 21 <= 7.0.1 <= 3.28 <= 20.17² # 3.7 <= 2.12 <= 7.2.7 <= 3.2.20 <= 7.4.4 <= 4.8.0 # 3.8 <= 2.12 # 3.9 <= 2.15 @@ -46,18 +46,14 @@ # ansible == 7.x ansible-core ~= 2.14.0 # ansible == 8.x ansible-core ~= 2.15.0 # ansible == 9.x ansible-core ~= 2.16.0 - -# pip --no-python-version-warning -# pip --disable-pip-version-check - -# TODO distros=-py3 +# ansible == 10.x ansible-core ~= 2.17.0 [tox] envlist = init, py{27,36}-mode_ansible-ansible{2.10,3,4}, py{311}-mode_ansible-ansible{2.10,3,4,5}, - py{312}-mode_ansible-ansible{6,7,8,9}, + py{312}-mode_ansible-ansible{6,7,8,9,10}, py{27,36,312}-mode_mitogen-distro_centos{6,7,8}, py{27,36,312}-mode_mitogen-distro_debian{9,10,11}, py{27,36,312}-mode_mitogen-distro_ubuntu{1604,1804,2004}, @@ -86,6 +82,7 @@ deps = ansible7: ansible~=7.0 ansible8: ansible~=8.0 ansible9: ansible~=9.0 + ansible10: ansible~=10.0 install_command = python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages} commands_pre = @@ -129,8 +126,10 @@ setenv = ansible6: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 ansible7: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 ansible8: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 - # Ansible >= 9 (ansible-core >= 2.16) require Python 2.7 or >= 3.6 on targets + # Ansible 9 (ansible-core 2.16) requires Python 2.7 or >= 3.6 on targets ansible9: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1804 ubuntu2004 + # Ansible 10 (ansible-core 2.17) requires Python >= 3.7 on targets + ansible10: DISTROS=debian10-py3 debian11-py3 ubuntu2004-py3 distros_centos: DISTROS=centos6 centos7 centos8 distros_centos5: DISTROS=centos5 distros_centos6: DISTROS=centos6