From 8613f685abdfe5cbe192221eb14ec26a33d3683b Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sat, 10 Aug 2024 19:37:25 +0100 Subject: [PATCH 1/6] tests: Skip AWS ECR login outside of CI jobs To avoid rate limiting errors, CI (currently Azure Devops) logs into the container registry (currently AWS ECR). Outside CI this is unnnecessary and makes it harder to run the tests, because very few people have access to a suitable AWS secret token. Following this change `aws ecr-public get-login-password` will only be run if the environment variable $TF_BUILD==True. This is set by Azure Pipelines jobs. If the CI platform is changed then another indicator should be used. https://adamj.eu/tech/2020/03/09/detect-if-your-tests-are-running-on-ci/ --- .ci/ansible_install.py | 2 +- .ci/debops_common_install.py | 2 +- .ci/mitogen_install.py | 2 +- .ci/mitogen_py24_install.py | 2 +- tox.ini | 3 +++ 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.ci/ansible_install.py b/.ci/ansible_install.py index 900303f6..3b217ff2 100755 --- a/.ci/ansible_install.py +++ b/.ci/ansible_install.py @@ -4,7 +4,7 @@ import ci_lib batches = [ [ - 'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws', + 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', ] ] diff --git a/.ci/debops_common_install.py b/.ci/debops_common_install.py index afafe39a..565f9488 100755 --- a/.ci/debops_common_install.py +++ b/.ci/debops_common_install.py @@ -10,7 +10,7 @@ ci_lib.run_batches([ 'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"', ], [ - 'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws', + 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', ], ]) diff --git a/.ci/mitogen_install.py b/.ci/mitogen_install.py index b0b1eb14..23ff384b 100755 --- a/.ci/mitogen_install.py +++ b/.ci/mitogen_install.py @@ -7,7 +7,7 @@ batches = [ if ci_lib.have_docker(): batches.append([ - 'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws', + 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', ]) diff --git a/.ci/mitogen_py24_install.py b/.ci/mitogen_py24_install.py index bd6ecb24..8af90405 100755 --- a/.ci/mitogen_py24_install.py +++ b/.ci/mitogen_py24_install.py @@ -4,7 +4,7 @@ import ci_lib batches = [ [ - 'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws', + 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', ], [ 'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv', diff --git a/tox.ini b/tox.ini index a0b0e1ae..5dcb1bf2 100644 --- a/tox.ini +++ b/tox.ini @@ -104,6 +104,9 @@ passenv = AWS_DEFAULT_REGION AWS_SECRET_ACCESS_KEY HOME + # Azure DevOps, TF_BUILD is set to 'True' when running in a build task + # https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables + TF_BUILD setenv = # See also azure-pipelines.yml ANSIBLE_STRATEGY = mitogen_linear From 9185805bf2b6467cb22622e4428531faa1fae1fa Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sat, 10 Aug 2024 21:38:51 +0100 Subject: [PATCH 2/6] ansible_mitogen: cast ansible_python_interpreter value This was the last remaining use of `mitogen.utils.cast()`. I missed it in #1046. --- ansible_mitogen/planner.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 0b1b7aab..b557272f 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -54,6 +54,7 @@ import mitogen.select import ansible_mitogen.loaders import ansible_mitogen.parsing import ansible_mitogen.target +import ansible_mitogen.utils.unsafe LOG = logging.getLogger(__name__) @@ -220,7 +221,7 @@ class ScriptPlanner(BinaryPlanner): variable, render it and return it. :param str path: - Absolute UNIX path to original interpreter. + Absolute path to original interpreter (e.g. '/usr/bin/python'). :returns: Shell fragment prefix used to execute the script via "/bin/sh -c". @@ -232,9 +233,12 @@ class ScriptPlanner(BinaryPlanner): try: template = self._inv.task_vars[key] except KeyError: - return path + pass + else: + configured_interpreter = self._inv.templar.template(template) + return ansible_mitogen.utils.unsafe.cast(configured_interpreter) - return mitogen.utils.cast(self._inv.templar.template(template)) + return path def _get_interpreter(self): path, arg = ansible_mitogen.parsing.parse_hashbang( From 40695f413b0a278b7d689462cf6e9ac8a218d243 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sat, 10 Aug 2024 22:05:22 +0100 Subject: [PATCH 3/6] ansible_mitogen: Respect ansible_facts.discovered_interpreter_python more fixes #1097 --- ansible_mitogen/planner.py | 23 +++++++++++++++---- docs/changelog.rst | 2 ++ .../module_utils/adjacent_to_role.yml | 2 +- .../roles/modrole/library/uses_external3.py | 2 +- .../library/uses_custom_known_hosts.py | 2 +- .../modules/custom_python_json_args_module.py | 2 +- .../lib/modules/custom_python_os_getcwd.py | 2 +- .../modules/custom_python_want_json_module.py | 2 +- 8 files changed, 26 insertions(+), 11 deletions(-) diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index b557272f..0a91039a 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -216,9 +216,12 @@ class ScriptPlanner(BinaryPlanner): """ def _rewrite_interpreter(self, path): """ - Given the original interpreter binary extracted from the script's - interpreter line, look up the associated `ansible_*_interpreter` - variable, render it and return it. + Given the interpreter path (from the script's hashbang line), return + the desired interpreter path. This tries, in order + + 1. Look up & render the `ansible_*_interpreter` variable, if set + 2. Look up the `discovered_interpreter_*` fact, if present + 3. The unmodified path from the hashbang line. :param str path: Absolute path to original interpreter (e.g. '/usr/bin/python'). @@ -229,7 +232,8 @@ class ScriptPlanner(BinaryPlanner): involved here, the vanilla implementation uses it and that use is exploited in common playbooks. """ - key = u'ansible_%s_interpreter' % os.path.basename(path).strip() + interpreter_name = os.path.basename(path).strip() + key = u'ansible_%s_interpreter' % interpreter_name try: template = self._inv.task_vars[key] except KeyError: @@ -238,6 +242,14 @@ class ScriptPlanner(BinaryPlanner): configured_interpreter = self._inv.templar.template(template) return ansible_mitogen.utils.unsafe.cast(configured_interpreter) + key = u'discovered_interpreter_%s' % interpreter_name + try: + discovered_interpreter = self._inv.task_vars['ansible_facts'][key] + except KeyError: + pass + else: + return ansible_mitogen.utils.unsafe.cast(discovered_interpreter) + return path def _get_interpreter(self): @@ -253,7 +265,8 @@ class ScriptPlanner(BinaryPlanner): if arg: fragment += ' ' + arg - return fragment, path.startswith('python') + is_python = path.startswith('python') + return fragment, is_python def get_kwargs(self, **kwargs): interpreter_fragment, is_python = self._get_interpreter() diff --git a/docs/changelog.rst b/docs/changelog.rst index ec122abf..c1754b9e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,8 @@ To avail of fixes in an unreleased version, please download a ZIP file Unreleased ---------- +* :gh:issue:`1097` Respect `ansible_facts.discovered_interpreter_python` when + executing non new-style modules (e.g. JSONARGS style, WANT_JSON style). v0.3.8 (2024-07-30) diff --git a/tests/ansible/integration/module_utils/adjacent_to_role.yml b/tests/ansible/integration/module_utils/adjacent_to_role.yml index c4fb813f..93499790 100644 --- a/tests/ansible/integration/module_utils/adjacent_to_role.yml +++ b/tests/ansible/integration/module_utils/adjacent_to_role.yml @@ -1,7 +1,7 @@ # external2 is loaded from config path. # external1 is loaded from integration/module_utils/roles/modrole/module_utils/.. -- name: integration/module_utils/adjacent_to_playbook.yml +- name: integration/module_utils/adjacent_to_role.yml hosts: test-targets roles: - modrole diff --git a/tests/ansible/integration/module_utils/roles/modrole/library/uses_external3.py b/tests/ansible/integration/module_utils/roles/modrole/library/uses_external3.py index faf8a9a5..78f2d71d 100644 --- a/tests/ansible/integration/module_utils/roles/modrole/library/uses_external3.py +++ b/tests/ansible/integration/module_utils/roles/modrole/library/uses_external3.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python from ansible.module_utils.basic import AnsibleModule from ansible.module_utils import external3 diff --git a/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py b/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py index 4fc4e04c..688fc45a 100644 --- a/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py +++ b/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python import json import ansible.module_utils.basic diff --git a/tests/ansible/lib/modules/custom_python_json_args_module.py b/tests/ansible/lib/modules/custom_python_json_args_module.py index 61640579..846037ec 100755 --- a/tests/ansible/lib/modules/custom_python_json_args_module.py +++ b/tests/ansible/lib/modules/custom_python_json_args_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible Python JSONARGS module. I should receive an encoding string. json_arguments = """<>""" diff --git a/tests/ansible/lib/modules/custom_python_os_getcwd.py b/tests/ansible/lib/modules/custom_python_os_getcwd.py index c5e264ae..d465ac9e 100644 --- a/tests/ansible/lib/modules/custom_python_os_getcwd.py +++ b/tests/ansible/lib/modules/custom_python_os_getcwd.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # #591: call os.getcwd() before AnsibleModule ever gets a chance to fix up the # process environment. diff --git a/tests/ansible/lib/modules/custom_python_want_json_module.py b/tests/ansible/lib/modules/custom_python_want_json_module.py index f5e33862..23eeeb55 100755 --- a/tests/ansible/lib/modules/custom_python_want_json_module.py +++ b/tests/ansible/lib/modules/custom_python_want_json_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible Python WANT_JSON module. I should receive a JSON-encoded file. import json From 863f923f1410c7fa34dc8a7cd0f8e0f695a7b5a4 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 11 Aug 2024 01:05:10 +0100 Subject: [PATCH 4/6] tests: Bypass interpreter discovery on non-existant connection delegation targets By setting ansible_python_interpreter for these fictious hosts we avoid Ansible trying and failing to connect to them in a attempt to populate ansible_facts.discovered_interpreter_python. This speeds up these tests by avoiding a timeout. It is also a necessary pre-requisite for Ansible 10 (ansible-core 2.17). In that release no hardcoded fallback is used, failure to determine a valid Python interpreter is a fatal error. refs #1074 --- tests/ansible/hosts/become_same_user.hosts | 3 +++ .../ansible/hosts/connection_delegation.hosts | 4 +++- .../delegate_to_template.yml | 4 ++-- .../osa_container_standalone.yml | 2 +- .../osa_delegate_to_self.yml | 2 +- .../stack_construction.yml | 24 +++++++++---------- 6 files changed, 22 insertions(+), 17 deletions(-) diff --git a/tests/ansible/hosts/become_same_user.hosts b/tests/ansible/hosts/become_same_user.hosts index ac744ed7..249246cb 100644 --- a/tests/ansible/hosts/become_same_user.hosts +++ b/tests/ansible/hosts/become_same_user.hosts @@ -1,6 +1,9 @@ # code: language=ini # vim: syntax=dosini +[become_same_user] # become_same_user.yml bsu-joe ansible_user=joe +[become_same_user:vars] +ansible_python_interpreter=python3000 diff --git a/tests/ansible/hosts/connection_delegation.hosts b/tests/ansible/hosts/connection_delegation.hosts index 4ae861b0..7de798a1 100644 --- a/tests/ansible/hosts/connection_delegation.hosts +++ b/tests/ansible/hosts/connection_delegation.hosts @@ -4,7 +4,7 @@ # Connection delegation scenarios. It's impossible to connect to them, but their would-be # config can be inspected using "mitogen_get_stack" action. - +[cd] # Normal inventory host, no aliasing. cd-normal ansible_connection=mitogen_doas ansible_user=normal-user @@ -23,6 +23,8 @@ cd-newuser-normal-normal mitogen_via=cd-normal ansible_user=newuser-normal-norma # doas:newuser via host. cd-newuser-doas-normal mitogen_via=cd-normal ansible_connection=mitogen_doas ansible_user=newuser-doas-normal-user +[cd:vars] +ansible_python_interpreter = python3000 [connection-delegation-test] cd-bastion diff --git a/tests/ansible/integration/connection_delegation/delegate_to_template.yml b/tests/ansible/integration/connection_delegation/delegate_to_template.yml index 0c3f35c2..f9ad9e0b 100644 --- a/tests/ansible/integration/connection_delegation/delegate_to_template.yml +++ b/tests/ansible/integration/connection_delegation/delegate_to_template.yml @@ -40,7 +40,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - 'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"], + 'python_path': ['python3000'], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, @@ -68,7 +68,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - 'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"], + 'python_path': ['python3000'], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, diff --git a/tests/ansible/integration/connection_delegation/osa_container_standalone.yml b/tests/ansible/integration/connection_delegation/osa_container_standalone.yml index 98788d39..df295fd0 100644 --- a/tests/ansible/integration/connection_delegation/osa_container_standalone.yml +++ b/tests/ansible/integration/connection_delegation/osa_container_standalone.yml @@ -19,7 +19,7 @@ 'kind': 'lxc', 'lxc_info_path': null, 'machinectl_path': null, - 'python_path': ['/usr/bin/python'], + 'python_path': ['python3000'], 'remote_name': null, 'username': null, }, diff --git a/tests/ansible/integration/connection_delegation/osa_delegate_to_self.yml b/tests/ansible/integration/connection_delegation/osa_delegate_to_self.yml index f97d1c01..53addbfe 100644 --- a/tests/ansible/integration/connection_delegation/osa_delegate_to_self.yml +++ b/tests/ansible/integration/connection_delegation/osa_delegate_to_self.yml @@ -23,7 +23,7 @@ 'lxc_info_path': null, 'lxc_path': null, 'machinectl_path': null, - 'python_path': ["/usr/bin/python"], + 'python_path': ["python3000"], 'username': 'ansible-cfg-remote-user', }, 'method': 'setns', diff --git a/tests/ansible/integration/connection_delegation/stack_construction.yml b/tests/ansible/integration/connection_delegation/stack_construction.yml index 784d2af2..8cf064bb 100644 --- a/tests/ansible/integration/connection_delegation/stack_construction.yml +++ b/tests/ansible/integration/connection_delegation/stack_construction.yml @@ -40,7 +40,7 @@ "connect_timeout": 30, "doas_path": null, "password": null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, "username": "normal-user", }, @@ -73,7 +73,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, @@ -115,7 +115,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, @@ -150,7 +150,7 @@ 'connect_timeout': 30, 'doas_path': null, 'password': null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, 'username': 'normal-user', }, @@ -168,7 +168,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, @@ -210,7 +210,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, @@ -238,7 +238,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, @@ -272,7 +272,7 @@ 'connect_timeout': 30, 'doas_path': null, 'password': null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, 'username': 'normal-user', }, @@ -290,7 +290,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, @@ -333,7 +333,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, @@ -388,7 +388,7 @@ 'connect_timeout': 30, 'doas_path': null, 'password': null, - 'python_path': ["/usr/bin/python"], + 'python_path': ["python3000"], 'remote_name': null, 'username': 'normal-user', }, @@ -399,7 +399,7 @@ 'connect_timeout': 30, 'doas_path': null, 'password': null, - 'python_path': ["/usr/bin/python"], + 'python_path': ["python3000"], 'remote_name': null, 'username': 'newuser-doas-normal-user', }, From 85b1b4070a6521b48265d4ef2d2d6d1cb9d061d1 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 11 Aug 2024 02:12:16 +0100 Subject: [PATCH 5/6] tests: Respect configured or detected Python more often Relying on the virtualenv default or hardcoding "python" results in a Python 2.x virtualenv on some targets (e.g. debian10-test). This caused a failure when testing with Ansible >= 10 (ansible-core >= 2.17), which have dropped Python 2.x support. refs #1074 --- .../integration/interpreter_discovery/complex_args.yml | 3 +-- tests/ansible/integration/stub_connections/kubectl.yml | 1 - tests/ansible/integration/stub_connections/lxc.yml | 1 - tests/ansible/integration/stub_connections/lxd.yml | 1 - tests/ansible/integration/stub_connections/mitogen_doas.yml | 1 - tests/ansible/integration/stub_connections/mitogen_sudo.yml | 1 - .../ansible/regression/issue_152__virtualenv_python_fails.yml | 4 +++- 7 files changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/ansible/integration/interpreter_discovery/complex_args.yml b/tests/ansible/integration/interpreter_discovery/complex_args.yml index a3755ebf..ea7e2d63 100644 --- a/tests/ansible/integration/interpreter_discovery/complex_args.yml +++ b/tests/ansible/integration/interpreter_discovery/complex_args.yml @@ -25,8 +25,7 @@ # special_python: source /tmp/fake && python - name: set python using sourced file set_fact: - # Avoid 2.x vs 3.x cross-compatiblity issues (that I can't remember the exact details of). - special_python: "source /tmp/fake || true && python{{ ansible_facts.python.version.major }}" + special_python: "source /tmp/fake || true && {{ ansible_facts.python.executable }}" - name: run get_url with specially-sourced python uri: diff --git a/tests/ansible/integration/stub_connections/kubectl.yml b/tests/ansible/integration/stub_connections/kubectl.yml index e3e927c0..13e522d2 100644 --- a/tests/ansible/integration/stub_connections/kubectl.yml +++ b/tests/ansible/integration/stub_connections/kubectl.yml @@ -12,7 +12,6 @@ - custom_python_detect_environment: vars: ansible_connection: kubectl - ansible_python_interpreter: python # avoid Travis virtualenv breakage mitogen_kubectl_path: stub-kubectl.py register: out diff --git a/tests/ansible/integration/stub_connections/lxc.yml b/tests/ansible/integration/stub_connections/lxc.yml index ae71a603..3f1a2282 100644 --- a/tests/ansible/integration/stub_connections/lxc.yml +++ b/tests/ansible/integration/stub_connections/lxc.yml @@ -8,7 +8,6 @@ - custom_python_detect_environment: vars: ansible_connection: lxc - ansible_python_interpreter: python # avoid Travis virtualenv breakage mitogen_lxc_attach_path: stub-lxc-attach.py register: out diff --git a/tests/ansible/integration/stub_connections/lxd.yml b/tests/ansible/integration/stub_connections/lxd.yml index a2f8f80a..fd811fb7 100644 --- a/tests/ansible/integration/stub_connections/lxd.yml +++ b/tests/ansible/integration/stub_connections/lxd.yml @@ -8,7 +8,6 @@ - custom_python_detect_environment: vars: ansible_connection: lxd - ansible_python_interpreter: python # avoid Travis virtualenv breakage mitogen_lxc_path: stub-lxc.py register: out diff --git a/tests/ansible/integration/stub_connections/mitogen_doas.yml b/tests/ansible/integration/stub_connections/mitogen_doas.yml index 338bc09b..a8b77edd 100644 --- a/tests/ansible/integration/stub_connections/mitogen_doas.yml +++ b/tests/ansible/integration/stub_connections/mitogen_doas.yml @@ -8,7 +8,6 @@ - custom_python_detect_environment: vars: ansible_connection: mitogen_doas - ansible_python_interpreter: python # avoid Travis virtualenv breakage ansible_doas_exe: stub-doas.py ansible_user: someuser register: out diff --git a/tests/ansible/integration/stub_connections/mitogen_sudo.yml b/tests/ansible/integration/stub_connections/mitogen_sudo.yml index 0ab18f3b..bcf0f22f 100644 --- a/tests/ansible/integration/stub_connections/mitogen_sudo.yml +++ b/tests/ansible/integration/stub_connections/mitogen_sudo.yml @@ -8,7 +8,6 @@ - custom_python_detect_environment: vars: ansible_connection: mitogen_sudo - ansible_python_interpreter: python # avoid Travis virtualenv breakage ansible_user: root ansible_become_exe: stub-sudo.py ansible_become_flags: -H --type=sometype --role=somerole diff --git a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml index 610eaf33..e0439fdd 100644 --- a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml +++ b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml @@ -12,7 +12,9 @@ https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}" no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}" PATH: "{{ lookup('env', 'PATH') }}" - shell: virtualenv /tmp/issue_152_virtualenv + command: + cmd: virtualenv -p "{{ ansible_facts.python.executable }}" /tmp/issue_152_virtualenv + creates: /tmp/issue_152_virtualenv when: - lout.python.version.full is version('2.7', '>=', strict=True) From 357fe38766e0db4c3d75b1eaf946e32b10597db6 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 11 Aug 2024 09:27:15 +0100 Subject: [PATCH 6/6] Ansible 10 (ansible-core 2.17) support Notably - Python 2.7 and 3.6 are no longer supported by Ansible on targets - The yum module has been removed, and redirected to dnf - _INTERPRETER_PYTHON_DISTRO_MAP has been neutered. Interpreter discovery always favours specific `python3.` interpreters in decending version order, then generic `python3` or `python`. - Add the ability for an action plugin to call self._execute_module(*, ignore_unknown_opts=True) to execute a module with options that may not be supported for the version being called. https://docs.ansible.com/ansible/devel/porting_guides/porting_guide_10.html https://github.com/ansible-community/ansible-build-data/blob/main/10/CHANGELOG-v10.md https://github.com/ansible/ansible/blob/stable-2.17/changelogs/CHANGELOG-v2.17.rst fixes #1074, refs #1082 Co-authored-by: Claude Becker --- .ci/azure-pipelines.yml | 9 +- ansible_mitogen/loaders.py | 2 +- ansible_mitogen/mixins.py | 12 +- docs/ansible_detailed.rst | 21 ++-- docs/changelog.rst | 1 + .../ansible_2_8_tests.yml | 106 ++++++++++-------- .../interpreter_discovery/complex_args.yml | 7 ++ .../runner/custom_bash_hashbang_argument.yml | 6 + .../transport_config/python_path.yml | 18 ++- ...ssue_1066__add_host__host_key_checking.yml | 4 + .../issue_776__load_plugins_called_twice.yml | 14 ++- tox.ini | 17 ++- 12 files changed, 147 insertions(+), 70 deletions(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 51908e92..a3ffa9b4 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -32,10 +32,14 @@ jobs: tox.env: py312-mode_localhost-ansible9 Van_312_9: tox.env: py312-mode_localhost-ansible9-strategy_linear + Loc_312_10: + tox.env: py312-mode_localhost-ansible10 + Van_312_10: + tox.env: py312-mode_localhost-ansible10-strategy_linear - job: Linux pool: - # https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2004-Readme.md + # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md vmImage: ubuntu-20.04 steps: - template: azure-pipelines-steps.yml @@ -152,3 +156,6 @@ jobs: Ans_312_9: python.version: '3.12' tox.env: py312-mode_ansible-ansible9 + Ans_312_10: + python.version: '3.12' + tox.env: py312-mode_ansible-ansible10 diff --git a/ansible_mitogen/loaders.py b/ansible_mitogen/loaders.py index 9729b8a1..9d9876ac 100644 --- a/ansible_mitogen/loaders.py +++ b/ansible_mitogen/loaders.py @@ -49,7 +49,7 @@ __all__ = [ ANSIBLE_VERSION_MIN = (2, 10) -ANSIBLE_VERSION_MAX = (2, 16) +ANSIBLE_VERSION_MAX = (2, 17) NEW_VERSION_MSG = ( "Your Ansible version (%s) is too recent. The most recent version\n" diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 9cc97a48..0ba41aad 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -357,7 +357,9 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): def _execute_module(self, module_name=None, module_args=None, tmp=None, task_vars=None, persist_files=False, - delete_remote_tmp=True, wrap_async=False): + delete_remote_tmp=True, wrap_async=False, + ignore_unknown_opts=False, + ): """ Collect up a module's execution environment then use it to invoke target.run_module() or helpers.run_module_async() in the target @@ -370,7 +372,13 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): if task_vars is None: task_vars = {} - self._update_module_args(module_name, module_args, task_vars) + if ansible_mitogen.utils.ansible_version[:2] >= (2, 17): + self._update_module_args( + module_name, module_args, task_vars, + ignore_unknown_opts=ignore_unknown_opts, + ) + else: + self._update_module_args(module_name, module_args, task_vars) env = {} self._compute_environment_string(env) self._set_temp_file_args(module_args, wrap_async) diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index e6f7b42c..7d960ed9 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -165,7 +165,9 @@ Noteworthy Differences +-----------------+-----------------+ | 8 | 3.9 - 3.12 | +-----------------+-----------------+ - | 9 | 3.10 - 3.12 | + | 9 | | + +-----------------+ 3.10 - 3.12 | + | 10 | | +-----------------+-----------------+ Verify your installation is running one of these versions by checking @@ -271,15 +273,14 @@ Noteworthy Differences * "Module Replacer" style modules are not supported. These rarely appear in practice, and light web searches failed to reveal many examples of them. -.. - * The ``ansible_python_interpreter`` variable is parsed using a restrictive - :mod:`shell-like ` syntax, permitting values such as ``/usr/bin/env - FOO=bar python`` or ``source /opt/rh/rh-python36/enable && python``, which - occur in practice. Jinja2 templating is also supported for complex task-level - interpreter settings. Ansible `documents this - `_ - as an absolute path, however the implementation passes it unquoted through - the shell, permitting arbitrary code to be injected. +* The ``ansible_python_interpreter`` variable is parsed using a restrictive + :mod:`shell-like ` syntax, permitting values such as ``/usr/bin/env + FOO=bar python`` or ``source /opt/rh/rh-python36/enable && python``. + Jinja2 templating is also supported for complex task-level + interpreter settings. Ansible documents `ansible_python_interpreter + `_ + as an absolute path and releases since June 2024 (e.g. Ansible 10.1) + reflect this. Older Ansible releases passed it to the shell unquoted. .. * Configurations will break that rely on the `hashbang argument splitting diff --git a/docs/changelog.rst b/docs/changelog.rst index c1754b9e..33ff9f9b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,6 +23,7 @@ Unreleased * :gh:issue:`1097` Respect `ansible_facts.discovered_interpreter_python` when executing non new-style modules (e.g. JSONARGS style, WANT_JSON style). +* :gh:issue:`1074` Support Ansible 10 (ansible-core 2.17) v0.3.8 (2024-07-30) diff --git a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml index 6ac9bada..0c7d30c9 100644 --- a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml +++ b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml @@ -4,6 +4,55 @@ - name: integration/interpreter_discovery/ansible_2_8_tests.yml hosts: test-targets gather_facts: true + vars: + DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_lt_2_12: + centos: + '6': /usr/bin/python + '7': /usr/bin/python + '8': /usr/libexec/platform-python + debian: + '9': /usr/bin/python + '10': /usr/bin/python3 + '11': /usr/bin/python + 'NA': /usr/bin/python # Debian 11, Ansible <= 7 (ansible-core <= 2.14) + 'bullseye/sid': /usr/bin/python # Debian 11, Ansible 8 - 9 (ansible-core 2.15 - 2.16) + ubuntu: + '16': /usr/bin/python3 + '18': /usr/bin/python3 + '20': /usr/bin/python3 + + DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_2_12_to_2_16: + centos: + '6': /usr/bin/python + '7': /usr/bin/python + '8': /usr/libexec/platform-python + debian: + '9': /usr/bin/python + '10': /usr/bin/python3 + '11': /usr/bin/python3.9 + 'NA': /usr/bin/python3.9 # Debian 11, Ansible <= 7 (ansible-core <= 2.14) + 'bullseye/sid': /usr/bin/python3.9 # Debian 11, Ansible 8 - 9 (ansible-core 2.15 - 2.16) + ubuntu: + '16': /usr/bin/python3 + '18': /usr/bin/python3 + '20': /usr/bin/python3 + + DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_ge_2_17: + debian: + '10': /usr/bin/python3.7 + '11': /usr/bin/python3.9 + 'bullseye/sid': /usr/bin/python3.9 + ubuntu: + '20': /usr/bin/python3.8 + + discovered_interpreter_expected: >- + {%- if ansible_version.full is version('2.12', '<', strict=True) -%} + {{ DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_lt_2_12[distro][distro_major] }} + {%- elif ansible_version.full is version('2.17', '<', strict=True) -%} + {{ DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_2_12_to_2_16[distro][distro_major] }} + {%- else -%} + {{ DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_ge_2_17[distro][distro_major] }} + {%- endif -%} tasks: - name: can only run these tests on ansible >= 2.8.0 block: @@ -18,9 +67,9 @@ - name: snag some facts to validate for later set_fact: - distro: '{{ ansible_distribution | default("unknown") | lower }}' - distro_version: '{{ ansible_distribution_version | default("unknown") }}' - os_family: '{{ ansible_os_family | default("unknown") }}' + distro: '{{ ansible_facts.distribution | lower }}' + distro_major: '{{ ansible_facts.distribution_major_version }}' + system: '{{ ansible_facts.system }}' - name: test that python discovery is working and that fact persistence makes it only run once block: @@ -159,50 +208,19 @@ - ansible_facts['ansible_bogus_interpreter'] | default('nope') == 'nope' - ansible_facts['discovered_interpreter_bogus'] | default('nope') == 'nope' - - name: fedora assertions + - name: Check discovered interpreter matches expected assert: that: - - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python3' - fail_msg: auto_out={{auto_out}} + - auto_out.ansible_facts.discovered_interpreter_python == discovered_interpreter_expected + fail_msg: >- + distro={{ distro }} + distro_major= {{ distro_major }} + system={{ system }} + auto_out={{ auto_out }} + discovered_interpreter_expected={{ discovered_interpreter_expected }} + ansible_version.full={{ ansible_version.full }} when: - - distro == 'fedora' - - distro_version is version('23.0', '>=', strict=True) - - - name: rhel < 8 assertions - assert: - that: - - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python' - fail_msg: auto_out={{auto_out}} - when: - - distro in ('redhat', 'centos') - - distro_version is version('8.0', '<', strict=true) - - - name: rhel 8+ assertions - assert: - that: - - auto_out.ansible_facts.discovered_interpreter_python == '/usr/libexec/platform-python' - fail_msg: auto_out={{auto_out}} - when: - - distro in ('redhat', 'centos') - - distro_version is version('8.0', '>=', strict=true) - - - name: ubuntu < 16.04 assertions - assert: - that: - - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python' - fail_msg: auto_out={{auto_out}} - when: - - distro == 'ubuntu' - - distro_version is version('16.04', '<', strict=true) - - - name: ubuntu 16.04+ assertions - assert: - that: - - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python3' - fail_msg: auto_out={{auto_out}} - when: - - distro == 'ubuntu' - - distro_version is version('16.04', '>=', strict=True) + - system in ['Linux'] always: - meta: clear_facts diff --git a/tests/ansible/integration/interpreter_discovery/complex_args.yml b/tests/ansible/integration/interpreter_discovery/complex_args.yml index ea7e2d63..af2b5e46 100644 --- a/tests/ansible/integration/interpreter_discovery/complex_args.yml +++ b/tests/ansible/integration/interpreter_discovery/complex_args.yml @@ -9,6 +9,13 @@ https_proxy: "{{ lookup('env', 'https_proxy') | default(omit) }}" no_proxy: "{{ lookup('env', 'no_proxy') | default(omit) }}" tasks: + # Ansible releases after June 2024 quote ansible_python_interpreter + # https://github.com/ansible/ansible/pull/83365 + - meta: end_play + when: + - not is_mitogen + - ansible_version.full is version('2.17.1', '>=', strict=True) + - name: create temp file to source file: path: /tmp/fake diff --git a/tests/ansible/integration/runner/custom_bash_hashbang_argument.yml b/tests/ansible/integration/runner/custom_bash_hashbang_argument.yml index ebb7966e..82535d56 100644 --- a/tests/ansible/integration/runner/custom_bash_hashbang_argument.yml +++ b/tests/ansible/integration/runner/custom_bash_hashbang_argument.yml @@ -2,6 +2,12 @@ - name: integration/runner/custom_bash_hashbang_argument.yml hosts: test-targets tasks: + # Ansible releases after June 2024 quote ansible_python_interpreter + # https://github.com/ansible/ansible/pull/83365 + - meta: end_play + when: + - not is_mitogen + - ansible_version.full is version('2.17.1', '>=', strict=True) - custom_bash_old_style_module: foo: true diff --git a/tests/ansible/integration/transport_config/python_path.yml b/tests/ansible/integration/transport_config/python_path.yml index 0f04ed59..ecc24374 100644 --- a/tests/ansible/integration/transport_config/python_path.yml +++ b/tests/ansible/integration/transport_config/python_path.yml @@ -2,12 +2,17 @@ # Each case is followed by mitogen_via= case to test hostvars method. -# When no ansible_python_interpreter is set, ansible 2.8+ automatically -# tries to detect the desired interpreter, falling back to "/usr/bin/python" if necessary +# If ansible_python_interpreter isn't set, Ansible 2.8+ tries to connect and +# detect the interpreter. If that fails (e.g. connection timeout) +# - Ansible 2.8 - 9 (ansible-core 2.8 - 2.16) assumes "/usr/bin/python" +# - Ansible 10+ (ansible-core 2.17+) marks the target unreachable - name: integration/transport_config/python_path.yml hosts: tc-python-path-unset tasks: - include_tasks: ../_mitogen_only.yml + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path @@ -19,6 +24,9 @@ vars: {mitogen_via: tc-python-path-unset} tasks: - include_tasks: ../_mitogen_only.yml + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path @@ -45,6 +53,9 @@ vars: {mitogen_via: tc-python-path-hostvar} tasks: - include_tasks: ../_mitogen_only.yml + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path @@ -71,6 +82,9 @@ vars: {mitogen_via: localhost} tasks: - include_tasks: ../_mitogen_only.yml + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path diff --git a/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml b/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml index 4b4609b6..dd754f84 100644 --- a/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml +++ b/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml @@ -28,6 +28,10 @@ become: false serial: 1 tasks: + # FIXME https://github.com/mitogen-hq/mitogen/issues/1096 + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) - meta: reset_connection # The host key might be in ~/.ssh/known_hosts. If it's removed then no diff --git a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml index 44ff6863..ef573276 100755 --- a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml +++ b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml @@ -3,13 +3,25 @@ - name: regression/issue_776__load_plugins_called_twice.yml hosts: test-targets become: "{{ groups.linux is defined and inventory_hostname in groups.linux }}" - gather_facts: yes + # Delayed until after the end_play, due to Python version requirements + gather_facts: false tags: - issue_776 vars: ansible_python_interpreter: "{{ pkg_mgr_python_interpreter }}" package: rsync # Chosen to exist in all tested distros/package managers tasks: + # The package management modules require using the same Python version + # as the target's package manager libraries. This is sometimes in conflict + # with Ansible requirements, e.g. Ansible 10 (ansible-core 2.17) does not + # support Python 2.x on targets. + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) + + - name: Gather facts manually + setup: + - name: Switch to archived package repositories copy: dest: "{{ item.dest }}" diff --git a/tox.ini b/tox.ini index 5dcb1bf2..08f4c371 100644 --- a/tox.ini +++ b/tox.ini @@ -10,9 +10,9 @@ # 2.4 2.3? <= 3.7.1 <= 1.3.7 <= 1.1 <= 2.1.3 <= 1.4 <= 1.8 # 2.5 <= 3.7.1 <= 1.4.22 <= 1.3.1 <= 2.1.3 <= 2.8.7 <= 1.6.1 <= 1.9.1 # 2.6 <= 2.6.20 <= 2.12 <= 4.5.4 <= 1.6.11 <= 2.10.3 <= 9.0.3 <= 5.9.0 <= 3.2.5 <= 2.9.1 <= 15.2.0 -# 2.7 <= 2.11 <= 5.5 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.15² +# 2.7 <= 2.11 <= 2.16 <= 5.5 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.15² # 3.5 <= 2.11 <= 2.15 <= 5.5 <= 2.2.28 <= 2.11.3 <= 20 <= 5.9.5 <= 6.1.0 <= 3.28 <= 20.15² -# 3.6 <= 2.11 <= 6.2 <= 3.2.20 <= 3.0.3 <= 21 <= 7.0.1 <= 3.28 <= 20.17² +# 3.6 <= 2.11 <= 2.16 <= 6.2 <= 3.2.20 <= 3.0.3 <= 21 <= 7.0.1 <= 3.28 <= 20.17² # 3.7 <= 2.12 <= 7.2.7 <= 3.2.20 <= 7.4.4 <= 4.8.0 # 3.8 <= 2.12 # 3.9 <= 2.15 @@ -46,18 +46,14 @@ # ansible == 7.x ansible-core ~= 2.14.0 # ansible == 8.x ansible-core ~= 2.15.0 # ansible == 9.x ansible-core ~= 2.16.0 - -# pip --no-python-version-warning -# pip --disable-pip-version-check - -# TODO distros=-py3 +# ansible == 10.x ansible-core ~= 2.17.0 [tox] envlist = init, py{27,36}-mode_ansible-ansible{2.10,3,4}, py{311}-mode_ansible-ansible{2.10,3,4,5}, - py{312}-mode_ansible-ansible{6,7,8,9}, + py{312}-mode_ansible-ansible{6,7,8,9,10}, py{27,36,312}-mode_mitogen-distro_centos{6,7,8}, py{27,36,312}-mode_mitogen-distro_debian{9,10,11}, py{27,36,312}-mode_mitogen-distro_ubuntu{1604,1804,2004}, @@ -86,6 +82,7 @@ deps = ansible7: ansible~=7.0 ansible8: ansible~=8.0 ansible9: ansible~=9.0 + ansible10: ansible~=10.0 install_command = python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages} commands_pre = @@ -129,8 +126,10 @@ setenv = ansible6: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 ansible7: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 ansible8: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 - # Ansible >= 9 (ansible-core >= 2.16) require Python 2.7 or >= 3.6 on targets + # Ansible 9 (ansible-core 2.16) requires Python 2.7 or >= 3.6 on targets ansible9: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1804 ubuntu2004 + # Ansible 10 (ansible-core 2.17) requires Python >= 3.7 on targets + ansible10: DISTROS=debian10-py3 debian11-py3 ubuntu2004-py3 distros_centos: DISTROS=centos6 centos7 centos8 distros_centos5: DISTROS=centos5 distros_centos6: DISTROS=centos6