From c61c063b4f9b2b63dcaa86443631a268c9f72870 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 10 Nov 2021 22:34:32 +0000 Subject: [PATCH] Support for Ansible 3 & 4 fixes #834 Co-authored-by: Claude Becker (@upekkha) Co-authored-by: Dolph Mathews (@dolph) --- .ci/azure-pipelines-steps.yml | 2 +- .ci/azure-pipelines.yml | 37 ++++++++--------- .ci/ci_lib.py | 41 ++++++++++--------- .ci/debops_common_tests.py | 8 +--- .ci/localhost_ansible_tests.py | 14 ++++--- ansible_mitogen/loaders.py | 4 +- ansible_mitogen/transport_config.py | 32 ++++----------- docs/ansible_detailed.rst | 7 ++-- docs/changelog.rst | 1 + setup.py | 1 + .../async/result_shell_echo_hi.yml | 5 ++- .../integration/become/sudo_flags_failure.yml | 12 +++--- .../integration/become/sudo_nonexistent.yml | 4 +- tox.ini | 26 ++++++++---- 14 files changed, 94 insertions(+), 100 deletions(-) diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml index 3249da41..fbf3d8fa 100644 --- a/.ci/azure-pipelines-steps.yml +++ b/.ci/azure-pipelines-steps.yml @@ -13,7 +13,7 @@ steps: - script: python -mpip install tox displayName: Install tooling -- script: tox -e $(tox.env) +- script: tox -e "$(tox.env)" displayName: "Run tests" env: AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index dd9e9c68..fbfaa2f1 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -25,14 +25,14 @@ jobs: tox.env: py27-mode_mitogen # TODO: test python3, python3 tests are broken - Ans210_27: + Local_Py27: python.version: '2.7' - tox.env: py27-mode_localhost-ansible2.10 + tox.env: py27-mode_localhost-ansible{2.10,3,4} # NOTE: this hangs when ran in Ubuntu 18.04 - Vanilla_210_27: + Vanilla_Py27: python.version: '2.7' - tox.env: py27-mode_localhost-ansible2.10 + tox.env: py27-mode_localhost-ansible{2.10,3,4} STRATEGY: linear ANSIBLE_SKIP_TAGS: resource_intensive @@ -45,22 +45,17 @@ jobs: - template: azure-pipelines-steps.yml strategy: matrix: - Mito27Debian_27: + Mito_Py27: python.version: '2.7' - tox.env: py27-mode_mitogen-distro_debian9 + tox.env: py27-mode_mitogen-distro_{centos6,centos7,centos8,debian9,debian10,debian11,ubuntu1604,ubuntu1804,ubuntu2004} - Mito36CentOS6_26: + Mito_Py36: python.version: '3.6' - tox.env: py36-mode_mitogen-distro_centos6 + tox.env: py36-mode_mitogen-distro_{centos6,centos7,centos8,debian9,debian10,debian11,ubuntu1604,ubuntu1804,ubuntu2004} - Mito39Debian_27: + Mito_Py39: python.version: '3.9' - tox.env: py39-mode_mitogen-distro_debian9 - - #Py26CentOS7: - #python.version: '2.7' - #MODE: mitogen - #DISTRO: centos6 + tox.env: py39-mode_mitogen-distro_{centos6,centos7,centos8,debian9,debian10,debian11,ubuntu1604,ubuntu1804,ubuntu2004} #DebOps_2460_27_27: #python.version: '2.7' @@ -99,14 +94,14 @@ jobs: #DISTROS: debian #STRATEGY: linear - Ansible_210_27: + Ansible_Py27: python.version: '2.7' - tox.env: py27-mode_ansible-ansible2.10 + tox.env: py27-mode_ansible-ansible{2.10,3,4} - Ansible_210_36: + Ansible_Py36: python.version: '3.6' - tox.env: py36-mode_ansible-ansible2.10 + tox.env: py36-mode_ansible-ansible{2.10,3,4} - Ansible_210_39: + Ansible_Py39: python.version: '3.9' - tox.env: py39-mode_ansible-ansible2.10 + tox.env: py39-mode_ansible-ansible{2.10,3,4} diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 1d9b0d73..65b7265a 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -22,6 +22,14 @@ os.chdir( ) ) +_print = print +def print(*args, **kwargs): + file = kwargs.get('file', sys.stdout) + flush = kwargs.pop('flush', False) + _print(*args, **kwargs) + if flush: + file.flush() + # # check_output() monkeypatch cutpasted from testlib.py @@ -71,24 +79,22 @@ def _argv(s, *args): def run(s, *args, **kwargs): - """ Run a command, with arguments, and print timing information + """ Run a command, with arguments >>> rc = run('echo "%s %s"', 'foo', 'bar') - Running: ['/usr/bin/time', '--', 'echo', 'foo bar'] + Running: ['echo', 'foo bar'] foo bar - 0.00user 0.00system 0:00.00elapsed ?%CPU (0avgtext+0avgdata 1964maxresident)k - 0inputs+0outputs (0major+71minor)pagefaults 0swaps - Finished running: ['/usr/bin/time', '--', 'echo', 'foo bar'] + Finished running: ['echo', 'foo bar'] >>> rc 0 """ - argv = ['/usr/bin/time', '--'] + _argv(s, *args) - print('Running: %s' % (argv,)) + argv = _argv(s, *args) + print('Running: %s' % (argv,), flush=True) try: ret = subprocess.check_call(argv, **kwargs) - print('Finished running: %s' % (argv,)) + print('Finished running: %s' % (argv,), flush=True) except Exception: - print('Exception occurred while running: %s' % (argv,)) + print('Exception occurred while running: %s' % (argv,), file=sys.stderr, flush=True) raise return ret @@ -155,7 +161,7 @@ def get_output(s, *args, **kwargs): 'foo bar\n' """ argv = _argv(s, *args) - print('Running: %s' % (argv,)) + print('Running: %s' % (argv,), flush=True) return subprocess.check_output(argv, **kwargs) @@ -368,12 +374,10 @@ def start_containers(containers): def verify_procs(hostname, old, new): oldpids = set(pid for pid, _ in old) if any(pid not in oldpids for pid, _ in new): - print('%r had stray processes running:' % (hostname,)) + print('%r had stray processes running:' % (hostname,), file=sys.stderr, flush=True) for pid, line in new: if pid not in oldpids: - print('New process:', line) - - print() + print('New process:', line, flush=True) return False return True @@ -397,13 +401,10 @@ def check_stray_processes(old, containers=None): def dump_file(path): - print() - print('--- %s ---' % (path,)) - print() + print('--- %s ---' % (path,), flush=True) with open(path, 'r') as fp: - print(fp.read().rstrip()) - print('---') - print() + print(fp.read().rstrip(), flush=True) + print('---', flush=True) # SSH passes these through to the container when run interactively, causing diff --git a/.ci/debops_common_tests.py b/.ci/debops_common_tests.py index 97631704..7db8a797 100755 --- a/.ci/debops_common_tests.py +++ b/.ci/debops_common_tests.py @@ -1,8 +1,6 @@ #!/usr/bin/env python -from __future__ import print_function import os -import shutil import sys import ci_lib @@ -60,11 +58,7 @@ with ci_lib.Fold('job_setup'): for container in containers ) - print() - print(' echo --- ansible/inventory/hosts: ---') - ci_lib.run('cat ansible/inventory/hosts') - print('---') - print() + ci_lib.dump_file('ansible/inventory/hosts') # Now we have real host key checking, we need to turn it off os.environ['ANSIBLE_HOST_KEY_CHECKING'] = 'False' diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index 13c77d96..a54d45d9 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -33,15 +33,19 @@ with ci_lib.Fold('job_setup'): with ci_lib.Fold('machine_prep'): # generate a new ssh key for localhost ssh - os.system("ssh-keygen -P '' -m pem -f ~/.ssh/id_rsa") - os.system("cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys") + if not os.path.exists(os.path.expanduser("~/.ssh/id_rsa")): + os.system("ssh-keygen -P '' -m pem -f ~/.ssh/id_rsa") + os.system("cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys") + # also generate it for the sudo user - os.system("sudo ssh-keygen -P '' -m pem -f /var/root/.ssh/id_rsa") - os.system("sudo cat /var/root/.ssh/id_rsa.pub | sudo tee -a /var/root/.ssh/authorized_keys") + if os.system("sudo [ -f /var/root/.ssh/id_rsa ]") != 0: + os.system("sudo ssh-keygen -P '' -m pem -f /var/root/.ssh/id_rsa") + os.system("sudo cat /var/root/.ssh/id_rsa.pub | sudo tee -a /var/root/.ssh/authorized_keys") + os.chmod(os.path.expanduser('~/.ssh'), int('0700', 8)) os.chmod(os.path.expanduser('~/.ssh/authorized_keys'), int('0600', 8)) # run chmod through sudo since it's owned by root - os.system('sudo chmod 600 /var/root/.ssh') + os.system('sudo chmod 700 /var/root/.ssh') os.system('sudo chmod 600 /var/root/.ssh/authorized_keys') if os.path.expanduser('~mitogen__user1') == '~mitogen__user1': diff --git a/ansible_mitogen/loaders.py b/ansible_mitogen/loaders.py index c00915d5..b4b98962 100644 --- a/ansible_mitogen/loaders.py +++ b/ansible_mitogen/loaders.py @@ -45,7 +45,7 @@ __all__ = [ import ansible ANSIBLE_VERSION_MIN = (2, 10) -ANSIBLE_VERSION_MAX = (2, 10) +ANSIBLE_VERSION_MAX = (2, 11) NEW_VERSION_MSG = ( "Your Ansible version (%s) is too recent. The most recent version\n" @@ -79,7 +79,7 @@ def assert_supported_release(): if v[:2] > ANSIBLE_VERSION_MAX: raise ansible.errors.AnsibleError( - NEW_VERSION_MSG % (ansible.__version__, ANSIBLE_VERSION_MAX) + NEW_VERSION_MSG % (v, ANSIBLE_VERSION_MAX) ) diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index 2a7a1e58..4babbde3 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -451,7 +451,7 @@ class PlayContextSpec(Spec): return self._play_context.private_key_file def ssh_executable(self): - return self._play_context.ssh_executable + return C.config.get_config_value("ssh_executable", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})) def timeout(self): return self._play_context.timeout @@ -467,9 +467,9 @@ class PlayContextSpec(Spec): return [ mitogen.core.to_text(term) for s in ( - getattr(self._play_context, 'ssh_args', ''), - getattr(self._play_context, 'ssh_common_args', ''), - getattr(self._play_context, 'ssh_extra_args', '') + C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})), + C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})), + C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})) ) for term in ansible.utils.shlex.shlex_split(s or '') ] @@ -679,10 +679,7 @@ class MitogenViaSpec(Spec): ) def ssh_executable(self): - return ( - self._host_vars.get('ansible_ssh_executable') or - C.ANSIBLE_SSH_EXECUTABLE - ) + return C.config.get_config_value("ssh_executable", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})) def timeout(self): # TODO: must come from PlayContext too. @@ -699,22 +696,9 @@ class MitogenViaSpec(Spec): return [ mitogen.core.to_text(term) for s in ( - ( - self._host_vars.get('ansible_ssh_args') or - getattr(C, 'ANSIBLE_SSH_ARGS', None) or - os.environ.get('ANSIBLE_SSH_ARGS') - # TODO: ini entry. older versions. - ), - ( - self._host_vars.get('ansible_ssh_common_args') or - os.environ.get('ANSIBLE_SSH_COMMON_ARGS') - # TODO: ini entry. - ), - ( - self._host_vars.get('ansible_ssh_extra_args') or - os.environ.get('ANSIBLE_SSH_EXTRA_ARGS') - # TODO: ini entry. - ), + C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})), + C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})), + C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})) ) for term in ansible.utils.shlex.shlex_split(s) if s diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index c9bbcf51..f17e876e 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -145,9 +145,10 @@ Testimonials Noteworthy Differences ---------------------- -* Ansible 2.3-2.9 are supported along with Python 2.6, 2.7, 3.6 and 3.7. Verify - your installation is running one of these versions by checking ``ansible - --version`` output. +* Mitogen 0.2.x supports Ansible 2.3-2.9; with Python 2.6, 2.7, or 3.6. + Mitogen 0.3.1+ supports Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.9. + Verify your installation is running one of these versions by checking + ``ansible --version`` output. * The ``raw`` action executes as a regular Mitogen connection, which requires Python on the target, precluding its use for installing Python. This will be diff --git a/docs/changelog.rst b/docs/changelog.rst index 1dbe99a8..7a1ae039 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,7 @@ To avail of fixes in an unreleased version, please download a ZIP file v0.3.1.dev0 (unreleased) ------------------------ +* :gh:issue:`834` Support for Ansible 3 and 4 (ansible-core 2.11) * :gh:issue:`869` Continuous Integration tests are now run with Tox * :gh:issue:`869` Continuous Integration tests now cover CentOS 6 & 8, Debian 9 & 11, Ubuntu 16.04 & 20.04 * :gh:issue:`860` Add initial support for podman connection (w/o Ansible support yet) diff --git a/setup.py b/setup.py index 8d3c75df..8275a07e 100644 --- a/setup.py +++ b/setup.py @@ -64,6 +64,7 @@ setup( zip_safe = False, classifiers = [ 'Environment :: Console', + 'Frameworks :: Ansible', 'Intended Audience :: System Administrators', 'License :: OSI Approved :: BSD License', 'Operating System :: MacOS :: MacOS X', diff --git a/tests/ansible/integration/async/result_shell_echo_hi.yml b/tests/ansible/integration/async/result_shell_echo_hi.yml index d7905e2e..c0cb283c 100644 --- a/tests/ansible/integration/async/result_shell_echo_hi.yml +++ b/tests/ansible/integration/async/result_shell_echo_hi.yml @@ -30,7 +30,10 @@ - async_out.invocation.module_args.creates == None - async_out.invocation.module_args.executable == None - async_out.invocation.module_args.removes == None - - async_out.invocation.module_args.warn == True + # In Ansible 4 (ansible-core 2.11) the warn parameter is deprecated and defaults to false. + # It's scheduled for removal in ansible-core 2.13. + - (ansible_version.full is version("2.11", "<", strict=True) and async_out.invocation.module_args.warn == True) + or (ansible_version.full is version("2.11", ">=", strict=True) and async_out.invocation.module_args.warn == False) - async_out.rc == 0 - async_out.start.startswith("20") - async_out.stderr == "there" diff --git a/tests/ansible/integration/become/sudo_flags_failure.yml b/tests/ansible/integration/become/sudo_flags_failure.yml index 971890dd..fdbb712c 100644 --- a/tests/ansible/integration/become/sudo_flags_failure.yml +++ b/tests/ansible/integration/become/sudo_flags_failure.yml @@ -15,12 +15,12 @@ assert: that: - out.failed - - | - ('sudo: no such option: --derps' in out.msg) or - ("sudo: invalid option -- '-'" in out.module_stderr) or - ("sudo: unrecognized option `--derps'" in out.module_stderr) or - ("sudo: unrecognized option `--derps'" in out.module_stdout) or - ("sudo: unrecognized option '--derps'" in out.module_stderr) + - >- + 'sudo: no such option: --derps' in out.msg + or out.module_stdout is match("sudo: invalid option -- '-'") + or out.module_stderr is match("sudo: invalid option -- '-'") + or out.module_stdout is match("sudo: unrecognized option [`']--derps'") + or out.module_stderr is match("sudo: unrecognized option [`']--derps'") fail_msg: out={{out}} tags: - sudo diff --git a/tests/ansible/integration/become/sudo_nonexistent.yml b/tests/ansible/integration/become/sudo_nonexistent.yml index d0e43214..bc3de7b8 100644 --- a/tests/ansible/integration/become/sudo_nonexistent.yml +++ b/tests/ansible/integration/become/sudo_nonexistent.yml @@ -22,8 +22,8 @@ # sudo-1.8.6p3-29.el6_10.3 on RHEL & CentOS 6.10 (final release) # removed user/group error messages, as defence against CVE-2019-14287. - >- - ('sudo: unknown user: slartibartfast' in out.module_stderr | default(out.msg)) - or ('chown: slartibartfast: illegal user name' in out.module_stderr | default(out.msg)) + 'sudo: unknown user: slartibartfast' in out.module_stdout | default(out.msg) + or 'sudo: unknown user: slartibartfast' in out.module_stderr | default(out.msg) or (ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_version == '6.10') fail_msg: out={{out}} when: diff --git a/tox.ini b/tox.ini index d6a7eccc..6e51d7ed 100644 --- a/tox.ini +++ b/tox.ini @@ -9,16 +9,23 @@ # Last version to support each python version # -# tox vir'env pip ansible ansible coverage -# control target +# Python tox virt'env pip A cntllr A target coverage # ========== ======== ======== ======== ======== ======== ======== # python2.4 1.4 1.8 1.1 2.3? # python2.5 1.6.1 1.9.1 1.3.1 ??? -# python2.6 2.9.1 15.2.0 9.0.3 2.6.20 4.5.4 -# python2.7 20.3 2.10 -# python3.5 2.10 -# python3.6 2.10 -# python3.7 2.10 +# python2.6 2.9.1 15.2.0 9.0.3 2.6.20 2.13 4.5.4 +# python2.7 20.3 2.11 +# python3.5 2.11 +# python3.6 2.11 +# python3.7 2.11 + +# Ansible Dependency +# ================== ====================== +# ansible <= 2.9 +# ansible == 2.10.* ansible-base ~= 2.10.0 +# ansible == 3.* ansible-base ~= 2.10.0 +# ansible == 4.* ansible-core ~= 2.11.0 +# ansible == 5.* ansible-core ~= 2.12.0 # pip --no-python-version-warning # pip --disable-pip-version-check @@ -28,7 +35,7 @@ [tox] envlist = init, - py{27,36,39}-mode_ansible-ansible2.10, + py{27,36,39}-mode_ansible-ansible{2.10,3,4}, py{27,36,39}-mode_mitogen-distro_centos{6,7,8}, py{27,36,39}-mode_mitogen-distro_debian{9,10,11}, py{27,36,39}-mode_mitogen-distro_ubuntu{1604,1804,2004}, @@ -54,6 +61,9 @@ deps = ansible2.9: ansible=2.9.6 ansible2.10: ansible-base<2.10.14 ansible2.10: ansible==2.10.0 + ansible3: ansible-base<2.10.14 + ansible3: ansible==3.4.0 + ansible4: ansible==4.8.0 install_command = python -m pip --no-python-version-warning install {opts} {packages} commands_pre =