Support for Ansible 3 & 4

fixes #834

Co-authored-by: Claude Becker (@upekkha)
Co-authored-by: Dolph Mathews (@dolph)

.ci/azure-pipelines-steps.yml

@@ -13,7 +13,7 @@ steps:
 - script: python -mpip install tox
   displayName: Install tooling
 
-- script: tox -e $(tox.env)
+- script: tox -e "$(tox.env)"
   displayName: "Run tests"
   env:
     AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)

.ci/azure-pipelines.yml

@@ -25,14 +25,14 @@ jobs:
         tox.env: py27-mode_mitogen
 
       # TODO: test python3, python3 tests are broken
-      Ans210_27:
+      Local_Py27:
         python.version: '2.7'
-        tox.env: py27-mode_localhost-ansible2.10
+        tox.env: py27-mode_localhost-ansible{2.10,3,4}
 
       # NOTE: this hangs when ran in Ubuntu 18.04
-      Vanilla_210_27:
+      Vanilla_Py27:
         python.version: '2.7'
-        tox.env: py27-mode_localhost-ansible2.10
+        tox.env: py27-mode_localhost-ansible{2.10,3,4}
         STRATEGY: linear
         ANSIBLE_SKIP_TAGS: resource_intensive
 
@@ -45,22 +45,17 @@ jobs:
   - template: azure-pipelines-steps.yml
   strategy:
     matrix:
-      Mito27Debian_27:
+      Mito_Py27:
         python.version: '2.7'
-        tox.env: py27-mode_mitogen-distro_debian9
+        tox.env: py27-mode_mitogen-distro_{centos6,centos7,centos8,debian9,debian10,debian11,ubuntu1604,ubuntu1804,ubuntu2004}
 
-      Mito36CentOS6_26:
+      Mito_Py36:
         python.version: '3.6'
-        tox.env: py36-mode_mitogen-distro_centos6
+        tox.env: py36-mode_mitogen-distro_{centos6,centos7,centos8,debian9,debian10,debian11,ubuntu1604,ubuntu1804,ubuntu2004}
 
-      Mito39Debian_27:
+      Mito_Py39:
         python.version: '3.9'
-        tox.env: py39-mode_mitogen-distro_debian9
-
-      #Py26CentOS7:
-        #python.version: '2.7'
-        #MODE: mitogen
-        #DISTRO: centos6
+        tox.env: py39-mode_mitogen-distro_{centos6,centos7,centos8,debian9,debian10,debian11,ubuntu1604,ubuntu1804,ubuntu2004}
 
       #DebOps_2460_27_27:
         #python.version: '2.7'
@@ -99,14 +94,14 @@ jobs:
         #DISTROS: debian
         #STRATEGY: linear
 
-      Ansible_210_27:
+      Ansible_Py27:
         python.version: '2.7'
-        tox.env: py27-mode_ansible-ansible2.10
+        tox.env: py27-mode_ansible-ansible{2.10,3,4}
 
-      Ansible_210_36:
+      Ansible_Py36:
         python.version: '3.6'
-        tox.env: py36-mode_ansible-ansible2.10
+        tox.env: py36-mode_ansible-ansible{2.10,3,4}
 
-      Ansible_210_39:
+      Ansible_Py39:
         python.version: '3.9'
-        tox.env: py39-mode_ansible-ansible2.10
+        tox.env: py39-mode_ansible-ansible{2.10,3,4}

.ci/ci_lib.py

@@ -22,6 +22,14 @@ os.chdir(
     )
 )
 
+_print = print
+def print(*args, **kwargs):
+    file = kwargs.get('file', sys.stdout)
+    flush = kwargs.pop('flush', False)
+    _print(*args, **kwargs)
+    if flush:
+        file.flush()
+
 
 #
 # check_output() monkeypatch cutpasted from testlib.py
@@ -71,24 +79,22 @@ def _argv(s, *args):
 
 
 def run(s, *args, **kwargs):
-    """ Run a command, with arguments, and print timing information
+    """ Run a command, with arguments
 
     >>> rc = run('echo "%s %s"', 'foo', 'bar')
-    Running: ['/usr/bin/time', '--', 'echo', 'foo bar']
+    Running: ['echo', 'foo bar']
     foo bar
-    0.00user 0.00system 0:00.00elapsed ?%CPU (0avgtext+0avgdata 1964maxresident)k
-    0inputs+0outputs (0major+71minor)pagefaults 0swaps
-    Finished running: ['/usr/bin/time', '--', 'echo', 'foo bar']
+    Finished running: ['echo', 'foo bar']
     >>> rc
     0
     """
-    argv = ['/usr/bin/time', '--'] + _argv(s, *args)
-    print('Running: %s' % (argv,))
+    argv = _argv(s, *args)
+    print('Running: %s' % (argv,), flush=True)
     try:
         ret = subprocess.check_call(argv, **kwargs)
-        print('Finished running: %s' % (argv,))
+        print('Finished running: %s' % (argv,), flush=True)
     except Exception:
-        print('Exception occurred while running: %s' % (argv,))
+        print('Exception occurred while running: %s' % (argv,), file=sys.stderr, flush=True)
         raise
 
     return ret
@@ -155,7 +161,7 @@ def get_output(s, *args, **kwargs):
     'foo bar\n'
     """
     argv = _argv(s, *args)
-    print('Running: %s' % (argv,))
+    print('Running: %s' % (argv,), flush=True)
     return subprocess.check_output(argv, **kwargs)
 
 
@@ -368,12 +374,10 @@ def start_containers(containers):
 def verify_procs(hostname, old, new):
     oldpids = set(pid for pid, _ in old)
     if any(pid not in oldpids for pid, _ in new):
-        print('%r had stray processes running:' % (hostname,))
+        print('%r had stray processes running:' % (hostname,), file=sys.stderr, flush=True)
         for pid, line in new:
             if pid not in oldpids:
-                print('New process:', line)
-
-        print()
+                print('New process:', line, flush=True)
         return False
 
     return True
@@ -397,13 +401,10 @@ def check_stray_processes(old, containers=None):
 
 
 def dump_file(path):
-    print()
-    print('--- %s ---' % (path,))
-    print()
+    print('--- %s ---' % (path,), flush=True)
     with open(path, 'r') as fp:
-        print(fp.read().rstrip())
-    print('---')
-    print()
+        print(fp.read().rstrip(), flush=True)
+    print('---', flush=True)
 
 
 # SSH passes these through to the container when run interactively, causing

.ci/debops_common_tests.py

@@ -1,8 +1,6 @@
 #!/usr/bin/env python
 
-from __future__ import print_function
 import os
-import shutil
 import sys
 
 import ci_lib
@@ -60,11 +58,7 @@ with ci_lib.Fold('job_setup'):
             for container in containers
         )
 
-    print()
-    print(' echo --- ansible/inventory/hosts: ---')
-    ci_lib.run('cat ansible/inventory/hosts')
-    print('---')
-    print()
+    ci_lib.dump_file('ansible/inventory/hosts')
 
     # Now we have real host key checking, we need to turn it off
     os.environ['ANSIBLE_HOST_KEY_CHECKING'] = 'False'

.ci/localhost_ansible_tests.py

@@ -33,15 +33,19 @@ with ci_lib.Fold('job_setup'):
 
 with ci_lib.Fold('machine_prep'):
     # generate a new ssh key for localhost ssh
-    os.system("ssh-keygen -P '' -m pem -f ~/.ssh/id_rsa")
-    os.system("cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys")
+    if not os.path.exists(os.path.expanduser("~/.ssh/id_rsa")):
+        os.system("ssh-keygen -P '' -m pem -f ~/.ssh/id_rsa")
+        os.system("cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys")
+
     # also generate it for the sudo user
-    os.system("sudo ssh-keygen -P '' -m pem -f /var/root/.ssh/id_rsa")
-    os.system("sudo cat /var/root/.ssh/id_rsa.pub | sudo tee -a /var/root/.ssh/authorized_keys")
+    if os.system("sudo [ -f /var/root/.ssh/id_rsa ]") != 0:
+        os.system("sudo ssh-keygen -P '' -m pem -f /var/root/.ssh/id_rsa")
+        os.system("sudo cat /var/root/.ssh/id_rsa.pub | sudo tee -a /var/root/.ssh/authorized_keys")
+
     os.chmod(os.path.expanduser('~/.ssh'), int('0700', 8))
     os.chmod(os.path.expanduser('~/.ssh/authorized_keys'), int('0600', 8))
     # run chmod through sudo since it's owned by root
-    os.system('sudo chmod 600 /var/root/.ssh')
+    os.system('sudo chmod 700 /var/root/.ssh')
     os.system('sudo chmod 600 /var/root/.ssh/authorized_keys')
 
     if os.path.expanduser('~mitogen__user1') == '~mitogen__user1':

ansible_mitogen/loaders.py

@@ -45,7 +45,7 @@ __all__ = [
 import ansible
 
 ANSIBLE_VERSION_MIN = (2, 10)
-ANSIBLE_VERSION_MAX = (2, 10)
+ANSIBLE_VERSION_MAX = (2, 11)
 
 NEW_VERSION_MSG = (
     "Your Ansible version (%s) is too recent. The most recent version\n"
@@ -79,7 +79,7 @@ def assert_supported_release():
 
     if v[:2] > ANSIBLE_VERSION_MAX:
         raise ansible.errors.AnsibleError(
-            NEW_VERSION_MSG % (ansible.__version__, ANSIBLE_VERSION_MAX)
+            NEW_VERSION_MSG % (v, ANSIBLE_VERSION_MAX)
         )
 
 

ansible_mitogen/transport_config.py

@@ -451,7 +451,7 @@ class PlayContextSpec(Spec):
         return self._play_context.private_key_file
 
     def ssh_executable(self):
-        return self._play_context.ssh_executable
+        return C.config.get_config_value("ssh_executable", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
 
     def timeout(self):
         return self._play_context.timeout
@@ -467,9 +467,9 @@ class PlayContextSpec(Spec):
         return [
             mitogen.core.to_text(term)
             for s in (
-                getattr(self._play_context, 'ssh_args', ''),
-                getattr(self._play_context, 'ssh_common_args', ''),
-                getattr(self._play_context, 'ssh_extra_args', '')
+                C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
+                C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
+                C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
             )
             for term in ansible.utils.shlex.shlex_split(s or '')
         ]
@@ -679,10 +679,7 @@ class MitogenViaSpec(Spec):
         )
 
     def ssh_executable(self):
-        return (
-            self._host_vars.get('ansible_ssh_executable') or
-            C.ANSIBLE_SSH_EXECUTABLE
-        )
+        return C.config.get_config_value("ssh_executable", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
 
     def timeout(self):
         # TODO: must come from PlayContext too.
@@ -699,22 +696,9 @@ class MitogenViaSpec(Spec):
         return [
             mitogen.core.to_text(term)
             for s in (
-                (
-                    self._host_vars.get('ansible_ssh_args') or
-                    getattr(C, 'ANSIBLE_SSH_ARGS', None) or
-                    os.environ.get('ANSIBLE_SSH_ARGS')
-                    # TODO: ini entry. older versions.
-                ),
-                (
-                    self._host_vars.get('ansible_ssh_common_args') or
-                    os.environ.get('ANSIBLE_SSH_COMMON_ARGS')
-                    # TODO: ini entry.
-                ),
-                (
-                    self._host_vars.get('ansible_ssh_extra_args') or
-                    os.environ.get('ANSIBLE_SSH_EXTRA_ARGS')
-                    # TODO: ini entry.
-                ),
+                C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
+                C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
+                C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
             )
             for term in ansible.utils.shlex.shlex_split(s)
             if s

docs/ansible_detailed.rst

@@ -145,9 +145,10 @@ Testimonials
 Noteworthy Differences
 ----------------------
 
-* Ansible 2.3-2.9 are supported along with Python 2.6, 2.7, 3.6 and 3.7. Verify
-  your installation is running one of these versions by checking ``ansible
-  --version`` output.
+* Mitogen 0.2.x supports Ansible 2.3-2.9; with Python 2.6, 2.7, or 3.6.
+  Mitogen 0.3.1+ supports Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.9.
+  Verify your installation is running one of these versions by checking
+  ``ansible --version`` output.
 
 * The ``raw`` action executes as a regular Mitogen connection, which requires
   Python on the target, precluding its use for installing Python. This will be

docs/changelog.rst

@@ -21,6 +21,7 @@ To avail of fixes in an unreleased version, please download a ZIP file
 v0.3.1.dev0 (unreleased)
 ------------------------
 
+* :gh:issue:`834` Support for Ansible 3 and 4 (ansible-core 2.11)
 * :gh:issue:`869` Continuous Integration tests are now run with Tox
 * :gh:issue:`869` Continuous Integration tests now cover CentOS 6 & 8, Debian 9 & 11, Ubuntu 16.04 & 20.04
 * :gh:issue:`860` Add initial support for podman connection (w/o Ansible support yet)

setup.py

@@ -64,6 +64,7 @@ setup(
     zip_safe = False,
     classifiers = [
         'Environment :: Console',
+        'Frameworks :: Ansible',
         'Intended Audience :: System Administrators',
         'License :: OSI Approved :: BSD License',
         'Operating System :: MacOS :: MacOS X',

tests/ansible/integration/async/result_shell_echo_hi.yml

@@ -30,7 +30,10 @@
         - async_out.invocation.module_args.creates == None
         - async_out.invocation.module_args.executable == None
         - async_out.invocation.module_args.removes == None
-        - async_out.invocation.module_args.warn == True
+        # In Ansible 4 (ansible-core 2.11) the warn parameter is deprecated and defaults to false.
+        # It's scheduled for removal in ansible-core 2.13.
+        - (ansible_version.full is version("2.11", "<", strict=True) and async_out.invocation.module_args.warn == True)
+          or (ansible_version.full is version("2.11", ">=", strict=True) and async_out.invocation.module_args.warn == False)
         - async_out.rc == 0
         - async_out.start.startswith("20")
         - async_out.stderr == "there"

tests/ansible/integration/become/sudo_flags_failure.yml

@@ -15,12 +15,12 @@
       assert:
         that:
           - out.failed
-          - |
-            ('sudo: no such option: --derps' in out.msg) or
-            ("sudo: invalid option -- '-'" in out.module_stderr) or
-            ("sudo: unrecognized option `--derps'" in out.module_stderr) or
-            ("sudo: unrecognized option `--derps'" in out.module_stdout) or
-            ("sudo: unrecognized option '--derps'" in out.module_stderr)
+          - >-
+            'sudo: no such option: --derps' in out.msg
+            or out.module_stdout is match("sudo: invalid option -- '-'")
+            or out.module_stderr is match("sudo: invalid option -- '-'")
+            or out.module_stdout is match("sudo: unrecognized option [`']--derps'")
+            or out.module_stderr is match("sudo: unrecognized option [`']--derps'")
         fail_msg: out={{out}}
   tags:
     - sudo

tests/ansible/integration/become/sudo_nonexistent.yml

@@ -22,8 +22,8 @@
           # sudo-1.8.6p3-29.el6_10.3 on RHEL & CentOS 6.10 (final release)
           # removed user/group error messages, as defence against CVE-2019-14287.
           - >-
-            ('sudo: unknown user: slartibartfast' in out.module_stderr | default(out.msg))
-            or ('chown: slartibartfast: illegal user name' in out.module_stderr | default(out.msg))
+            'sudo: unknown user: slartibartfast' in out.module_stdout | default(out.msg)
+            or 'sudo: unknown user: slartibartfast' in out.module_stderr | default(out.msg)
             or (ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_version == '6.10')
         fail_msg: out={{out}}
       when:

tox.ini

@@ -9,16 +9,23 @@
 
 # Last version to support each python version
 #
-#            tox      vir'env  pip      ansible  ansible  coverage
-#                                       control  target
+# Python     tox      virt'env pip      A cntllr A target coverage
 # ========== ======== ======== ======== ======== ======== ========
 # python2.4  1.4      1.8      1.1               2.3?
 # python2.5  1.6.1    1.9.1    1.3.1    ???
-# python2.6  2.9.1    15.2.0   9.0.3    2.6.20            4.5.4
-# python2.7                    20.3     2.10
-# python3.5                             2.10
-# python3.6                             2.10
-# python3.7                             2.10
+# python2.6  2.9.1    15.2.0   9.0.3    2.6.20   2.13     4.5.4
+# python2.7                    20.3     2.11
+# python3.5                             2.11
+# python3.6                             2.11
+# python3.7                             2.11
+
+# Ansible            Dependency
+# ================== ======================
+# ansible <= 2.9
+# ansible == 2.10.*  ansible-base ~= 2.10.0
+# ansible == 3.*     ansible-base ~= 2.10.0
+# ansible == 4.*     ansible-core ~= 2.11.0
+# ansible == 5.*     ansible-core ~= 2.12.0
 
 # pip --no-python-version-warning
 # pip --disable-pip-version-check
@@ -28,7 +35,7 @@
 [tox]
 envlist =
     init,
-    py{27,36,39}-mode_ansible-ansible2.10,
+    py{27,36,39}-mode_ansible-ansible{2.10,3,4},
     py{27,36,39}-mode_mitogen-distro_centos{6,7,8},
     py{27,36,39}-mode_mitogen-distro_debian{9,10,11},
     py{27,36,39}-mode_mitogen-distro_ubuntu{1604,1804,2004},
@@ -54,6 +61,9 @@ deps =
     ansible2.9: ansible=2.9.6
     ansible2.10: ansible-base<2.10.14
     ansible2.10: ansible==2.10.0
+    ansible3: ansible-base<2.10.14
+    ansible3: ansible==3.4.0
+    ansible4: ansible==4.8.0
 install_command =
     python -m pip --no-python-version-warning install {opts} {packages}
 commands_pre =