From 63e39c1ac5becd9e3d2c635c8466d51e25813a46 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 2 Jun 2022 13:51:05 +0100 Subject: [PATCH 1/6] ci: Remove traces of Ansible < 2.10 (not supported in 0.3.x) --- .ci/azure-pipelines.yml | 37 ------------------------------------- tox.ini | 4 ---- 2 files changed, 41 deletions(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 6f45397b..f8c6c66b 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -190,43 +190,6 @@ jobs: python.version: '3.10' tox.env: py310-mode_mitogen-distro_ubuntu2004 - #DebOps_2460_27_27: - #python.version: '2.7' - #MODE: debops_common - #VER: 2.4.6.0 - - #DebOps_262_36_27: - #python.version: '3.6' - #MODE: debops_common - #VER: 2.6.2 - - #Ansible_2460_26: - #python.version: '2.7' - #MODE: ansible - #VER: 2.4.6.0 - - #Ansible_262_26: - #python.version: '2.7' - #MODE: ansible - #VER: 2.6.2 - - #Ansible_2460_36: - #python.version: '3.6' - #MODE: ansible - #VER: 2.4.6.0 - - #Ansible_262_36: - #python.version: '3.6' - #MODE: ansible - #VER: 2.6.2 - - #Vanilla_262_27: - #python.version: '2.7' - #MODE: ansible - #VER: 2.6.2 - #DISTROS: debian - #STRATEGY: linear - Ans_27_210: python.version: '2.7' tox.env: py27-mode_ansible-ansible2.10 diff --git a/tox.ini b/tox.ini index bcacba03..dd0bc214 100644 --- a/tox.ini +++ b/tox.ini @@ -57,10 +57,6 @@ basepython = deps = -r{toxinidir}/tests/requirements.txt mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt - ansible2.3: ansible==2.3.3.0 - ansible2.4: ansible==2.4.6.0 - ansible2.8: ansible==2.8.3 - ansible2.9: ansible==2.9.6 ansible2.10: ansible-base<2.10.14 ansible2.10: ansible==2.10.0 ansible3: ansible-base<2.10.14 From db114d3bb2eb3bc1894aacf96874050fd2772162 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 2 Jun 2022 13:53:36 +0100 Subject: [PATCH 2/6] ci: Bump Ansible releases used in tests --- tox.ini | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index dd0bc214..6b3381f5 100644 --- a/tox.ini +++ b/tox.ini @@ -57,12 +57,10 @@ basepython = deps = -r{toxinidir}/tests/requirements.txt mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt - ansible2.10: ansible-base<2.10.14 - ansible2.10: ansible==2.10.0 - ansible3: ansible-base<2.10.14 + ansible2.10: ansible==2.10.7 ansible3: ansible==3.4.0 - ansible4: ansible==4.8.0 - ansible5: ansible==5.0.1 + ansible4: ansible==4.10.0 + ansible5: ansible==5.8.0 install_command = python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages} commands_pre = From 195b40008710eeefd1666c9d2906eb0f628e8be2 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 2 Jun 2022 17:45:44 +0100 Subject: [PATCH 3/6] ci: Drop Ansible 3 tests to free up CI capacity --- .ci/azure-pipelines.yml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index f8c6c66b..f1e29813 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -33,9 +33,6 @@ jobs: Loc_27_210: python.version: '2.7' tox.env: py27-mode_localhost-ansible2.10 - Loc_27_3: - python.version: '2.7' - tox.env: py27-mode_localhost-ansible3 Loc_27_4: python.version: '2.7' tox.env: py27-mode_localhost-ansible4 @@ -46,11 +43,6 @@ jobs: tox.env: py27-mode_localhost-ansible2.10 STRATEGY: linear ANSIBLE_SKIP_TAGS: resource_intensive - Van_27_3: - python.version: '2.7' - tox.env: py27-mode_localhost-ansible3 - STRATEGY: linear - ANSIBLE_SKIP_TAGS: resource_intensive Van_27_4: python.version: '2.7' tox.env: py27-mode_localhost-ansible4 @@ -79,8 +71,6 @@ jobs: # TODO: test python3, python3 tests are broken Loc_27_210: tox.env: py27-mode_localhost-ansible2.10 - Loc_27_3: - tox.env: py27-mode_localhost-ansible3 Loc_27_4: tox.env: py27-mode_localhost-ansible4 @@ -89,10 +79,6 @@ jobs: tox.env: py27-mode_localhost-ansible2.10 STRATEGY: linear ANSIBLE_SKIP_TAGS: resource_intensive - Van_27_3: - tox.env: py27-mode_localhost-ansible3 - STRATEGY: linear - ANSIBLE_SKIP_TAGS: resource_intensive Van_27_4: tox.env: py27-mode_localhost-ansible4 STRATEGY: linear @@ -193,9 +179,6 @@ jobs: Ans_27_210: python.version: '2.7' tox.env: py27-mode_ansible-ansible2.10 - Ans_27_3: - python.version: '2.7' - tox.env: py27-mode_ansible-ansible3 Ans_27_4: python.version: '2.7' tox.env: py27-mode_ansible-ansible4 @@ -203,9 +186,6 @@ jobs: Ans_36_210: python.version: '3.6' tox.env: py36-mode_ansible-ansible2.10 - Ans_36_3: - python.version: '3.6' - tox.env: py36-mode_ansible-ansible3 Ans_36_4: python.version: '3.6' tox.env: py36-mode_ansible-ansible4 From e8ad12e881a04e7b49048645ec88a3afbc304f14 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 2 Jun 2022 17:48:10 +0100 Subject: [PATCH 4/6] Ansible 6 support fixes #929 --- .ci/azure-pipelines.yml | 3 +++ ansible_mitogen/loaders.py | 2 +- docs/ansible_detailed.rst | 2 +- docs/changelog.rst | 1 + tox.ini | 6 +++++- 5 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index f1e29813..98f4805d 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -202,3 +202,6 @@ jobs: Ans_310_5: python.version: '3.10' tox.env: py310-mode_ansible-ansible5 + Ans_310_6: + python.version: '3.10' + tox.env: py310-mode_ansible-ansible6 diff --git a/ansible_mitogen/loaders.py b/ansible_mitogen/loaders.py index cd05feac..24f3d2a1 100644 --- a/ansible_mitogen/loaders.py +++ b/ansible_mitogen/loaders.py @@ -48,7 +48,7 @@ __all__ = [ ANSIBLE_VERSION_MIN = (2, 10) -ANSIBLE_VERSION_MAX = (2, 12) +ANSIBLE_VERSION_MAX = (2, 13) NEW_VERSION_MSG = ( "Your Ansible version (%s) is too recent. The most recent version\n" diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index d3298074..dd569a76 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -148,7 +148,7 @@ Noteworthy Differences * Mitogen 0.2.x supports Ansible 2.3-2.9; with Python 2.6, 2.7, or 3.6. Mitogen 0.3.1+ supports - Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.10 - - Ansible 5; with Python 3.8-3.10 + - Ansible 5 and 6; with Python 3.8-3.10 Verify your installation is running one of these versions by checking ``ansible --version`` output. diff --git a/docs/changelog.rst b/docs/changelog.rst index fdb69923..c620ed15 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -20,6 +20,7 @@ To avail of fixes in an unreleased version, please download a ZIP file v0.3.4.dev0 ------------------- +* :gh:issue:`929` Support Ansible 6 and ansible-core 2.13 * :gh:issue:`832` Fix runtime error when using the ansible.builtin.dnf module multiple times v0.3.3 (2022-06-03) diff --git a/tox.ini b/tox.ini index 6b3381f5..43170f9f 100644 --- a/tox.ini +++ b/tox.ini @@ -26,6 +26,7 @@ # ansible == 3.* ansible-base ~= 2.10.0 # ansible == 4.* ansible-core ~= 2.11.0 # ansible == 5.* ansible-core ~= 2.12.0 +# ansible == 6.* ansible-core ~= 2.13.0 # pip --no-python-version-warning # pip --disable-pip-version-check @@ -36,7 +37,7 @@ envlist = init, py{27,36}-mode_ansible-ansible{2.10,3,4}, - py{310}-mode_ansible-ansible{2.10,3,4,5}, + py{310}-mode_ansible-ansible{2.10,3,4,5,6}, py{27,36,310}-mode_mitogen-distro_centos{6,7,8}, py{27,36,310}-mode_mitogen-distro_debian{9,10,11}, py{27,36,310}-mode_mitogen-distro_ubuntu{1604,1804,2004}, @@ -61,6 +62,7 @@ deps = ansible3: ansible==3.4.0 ansible4: ansible==4.10.0 ansible5: ansible==5.8.0 + ansible6: ansible==6.0.0 install_command = python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages} commands_pre = @@ -97,6 +99,8 @@ setenv = distro_ubuntu1804: DISTRO=ubuntu1804 distro_ubuntu2004: DISTRO=ubuntu2004 # Note the plural, only applicable to MODE=ansible + # Ansible 6 (ansible-core 2.13) requires Python >= 2.7 or >= 3.5 on targets + ansible6: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 distros_centos: DISTROS=centos6 centos7 centos8 distros_centos5: DISTROS=centos5 distros_centos6: DISTROS=centos6 From ad4b686836c2e1b8b8611fa5d467ad018ef75cf8 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 2 Jun 2022 23:44:38 +0100 Subject: [PATCH 5/6] master.PkgutilMethod: Skip module loaders that raise ValueError e.g. in Ansible 6, ansible-core 2.13 ``` [mux 2717] 23:39:11.342416 D mitogen: PkgutilMethod(): _AnsibleCollectionLoader(path=None).get_file_name('ansible.plugins') failed: ValueError('_AnsibleCollectionLoader(path=None) cannot find files for ansible.plugins, only ansible_collections.ansible.builtin.plugins') ``` --- mitogen/master.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/master.py b/mitogen/master.py index 3a163a8b..4fb535f0 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -536,7 +536,7 @@ class PkgutilMethod(FinderMethod): try: path = loader.get_filename(fullname) - except (AttributeError, ImportError): + except (AttributeError, ImportError, ValueError): # - get_filename() may throw ImportError if pkgutil.find_loader() # picks a "parent" package's loader for some crap that's been # stuffed in sys.modules, for example in the case of urllib3: From f1503874de82353cbed8b51408d20fdfa899f8f7 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 4 Jul 2022 21:50:58 +0100 Subject: [PATCH 6/6] ansible_mitogen: Correct ansible_become_pass/ansible_become_password precendence Until Ansible 2.9 it looks like ansible_become_password had higher priority. From Ansible 2.10 ansible_become_pass has higher priority [1]. Mitogen was not respecting this. I may need to rework this further, instatiating the become plugin may have slowed down execution. [1] Based on testing with ``` [ubuntus] become-pass-pass ansible_become_pass=1234 become-pass-password ansible_become_password=1234 become-pass-both ansible_become_password=wrong ansible_become_pass=1234 [ubuntus:vars] ansible_host=ubuntu2004.local ansible_user=ubuntu ``` ``` - hosts: ubuntus gather_facts: false become: true tasks: - ping: ``` --- ansible_mitogen/loaders.py | 2 ++ ansible_mitogen/transport_config.py | 10 +++++++--- tests/ansible/hosts/transport_config.hosts | 2 +- .../integration/transport_config/become_pass.yml | 9 ++++----- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/ansible_mitogen/loaders.py b/ansible_mitogen/loaders.py index 24f3d2a1..1f4d8fc6 100644 --- a/ansible_mitogen/loaders.py +++ b/ansible_mitogen/loaders.py @@ -39,6 +39,7 @@ import ansible_mitogen.utils __all__ = [ 'action_loader', + 'become_loader', 'connection_loader', 'module_loader', 'module_utils_loader', @@ -90,6 +91,7 @@ assert_supported_release() from ansible.plugins.loader import action_loader +from ansible.plugins.loader import become_loader from ansible.plugins.loader import connection_loader from ansible.plugins.loader import module_loader from ansible.plugins.loader import module_utils_loader diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index b488b851..cc4e4a79 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -79,6 +79,7 @@ try: except ImportError: from ansible.vars.unsafe_proxy import AnsibleUnsafeText +import ansible_mitogen.loaders import mitogen.core @@ -435,7 +436,10 @@ class PlayContextSpec(Spec): return self._play_context.become_user def become_pass(self): - return optional_secret(self._play_context.become_pass) + become_method = self.become_method() + become_plugin = ansible_mitogen.loaders.become_loader.get(become_method) + become_pass = become_plugin.get_option('become_pass', hostvars=self._task_vars) + return optional_secret(become_pass) def password(self): return optional_secret(self._play_context.password) @@ -652,8 +656,8 @@ class MitogenViaSpec(Spec): def become_pass(self): return optional_secret( - self._host_vars.get('ansible_become_password') or - self._host_vars.get('ansible_become_pass') + self._host_vars.get('ansible_become_pass') or + self._host_vars.get('ansible_become_password') ) def password(self): diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts index 05e0d4f1..dc21c332 100644 --- a/tests/ansible/hosts/transport_config.hosts +++ b/tests/ansible/hosts/transport_config.hosts @@ -47,7 +47,7 @@ tc-become-user-set ansible_become_user=ansi-become-user tc-become-pass-unset tc-become-pass-password ansible_become_password=apassword tc-become-pass-pass ansible_become_pass=apass -tc-become-pass-both ansible_become_password=a.b.c ansible_become_pass=c.b.a +tc-become-pass-both ansible_become_pass=bpass ansible_become_password=bpassword # port() tc-port-unset diff --git a/tests/ansible/integration/transport_config/become_pass.yml b/tests/ansible/integration/transport_config/become_pass.yml index 5cbbdf19..317e0522 100644 --- a/tests/ansible/integration/transport_config/become_pass.yml +++ b/tests/ansible/integration/transport_config/become_pass.yml @@ -119,9 +119,6 @@ fail_msg: out={{out}} - -# ansible_become_pass & ansible_become_password set, password used to take precedence -# but it's possible since https://github.com/ansible/ansible/pull/69629/files#r428376864, now it doesn't - hosts: tc-become-pass-both become: true tasks: @@ -132,7 +129,9 @@ - out.result|length == 2 - out.result[0].method == "ssh" - out.result[1].method == "sudo" - - out.result[1].kwargs.password == "c.b.a" + # Ansible >= 2.10 builtin become plugins (e.g. sudo, su) give priority + # to ansible_become_pass over ansible_become_password. + - out.result[1].kwargs.password == "bpass" fail_msg: out={{out}} @@ -147,6 +146,6 @@ - out.result|length == 3 - out.result[0].method == "ssh" - out.result[1].method == "sudo" - - out.result[1].kwargs.password == "a.b.c" + - out.result[1].kwargs.password == "bpass" - out.result[2].method == "ssh" fail_msg: out={{out}}