Merge remote-tracking branch 'origin/issue587'

* origin/issue587:
  issue #587: docs: update Changelog.
  issue #587: disable SSH key setup, it breaks unit tests
  issue #587: attempt to fix Mac Azure job
  issue #587: Add 2.8.0/Py2.7 job to Azure
  issue #587: use deadsnakes PPA Python because VSTS version is junk
  issue #587: workaround for ansible/ansible#56629
  issue #587: enable spawn_reverse_shell and tidy up Azure step names
  issue #587: ensure Azure worker has a working SSH configuration
  issue #587: import spawn_reverse_shell.py script.
  issue #587: support pausing ansible_tests if flag file exists
  issue #587: "state: absent" was removed in 2.8.0
  issue #587: consistent become_exe() behaviour for older Ansibles.
  issue #587: update stub_connections/ test to use new doas var
  issue #587: update MODULE FAILURE message format for post >2.7
  issue #587: fix syntax error due to presence of comment
  issue #587: update mitogen_doas doc to match varible change.
  issue #587: disable deprecation_warnings for CI.
  issue #587: mitogen_doas should not become_exe for doas_path
  issue #587: 2.8 whitespace handling was improved.
  issue #587: 2.8 PlayContext lacks sudo_flags attribute.
  issue #587: 2.8 PluginLoader.get() introduced new collection_list kwarg
  issue #587: 2.8 PlayContext.connection no longer contains connection name
  issue #587: bump max Ansible version

.ci/ansible_tests.py

@@ -3,6 +3,7 @@
 
 import glob
 import os
+import signal
 import sys
 
 import ci_lib
@@ -13,6 +14,12 @@ TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible')
 HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts')
 
 
+def pause_if_interactive():
+    if os.path.exists('/tmp/interactive'):
+        while True:
+            signal.pause()
+
+
 with ci_lib.Fold('unit_tests'):
     os.environ['SKIP_MITOGEN'] = '1'
     ci_lib.run('./run_tests -v')
@@ -59,5 +66,11 @@ with ci_lib.Fold('job_setup'):
 
 with ci_lib.Fold('ansible'):
     playbook = os.environ.get('PLAYBOOK', 'all.yml')
-    run('./run_ansible_playbook.py %s -i "%s" %s',
-        playbook, HOSTS_DIR, ' '.join(sys.argv[1:]))
+    try:
+        run('./run_ansible_playbook.py %s -i "%s" %s',
+            playbook, HOSTS_DIR, ' '.join(sys.argv[1:]))
+    except:
+        pause_if_interactive()
+        raise
+
+pause_if_interactive()

.ci/azure-pipelines-steps.yml

@@ -5,16 +5,27 @@ parameters:
   sign: false
 
 steps:
-- task: UsePythonVersion@0
-  inputs:
-    versionSpec: '$(python.version)'
-    architecture: 'x64'
+- script: "PYTHONVERSION=$(python.version) .ci/prep_azure.py"
+  displayName: "Run prep_azure.py"
 
-- script: .ci/prep_azure.py
-  displayName: "Install requirements."
+# The VSTS-shipped Pythons available via UsePythonVErsion are pure garbage,
+# broken symlinks, incorrect permissions and missing codecs. So we use the
+# deadsnakes PPA to get sane Pythons, and setup a virtualenv to install our
+# stuff into. The virtualenv can probably be removed again, but this was a
+# hard-fought battle and for now I am tired of this crap.
+- script: |
+    sudo ln -fs /usr/bin/python$(python.version) /usr/bin/python
+    /usr/bin/python -m pip install -U virtualenv setuptools wheel
+    /usr/bin/python -m virtualenv /tmp/venv -p /usr/bin/python$(python.version)
+    echo "##vso[task.prependpath]/tmp/venv/bin"
+
+  displayName: activate venv
+
+- script: .ci/spawn_reverse_shell.py
+  displayName: "Spawn reverse shell"
 
 - script: .ci/$(MODE)_install.py
-  displayName: "Install requirements."
+  displayName: "Run $(MODE)_install.py"
 
 - script: .ci/$(MODE)_tests.py
-  displayName: Run tests.
+  displayName: "Run $(MODE)_tests.py"

.ci/azure-pipelines.yml

@@ -87,3 +87,9 @@ jobs:
         #VER: 2.6.2
         #DISTROS: debian
         #STRATEGY: linear
+
+      Vanilla_280_27:
+        python.version: '2.7'
+        MODE: ansible
+        VER: 2.8.0
+        DISTROS: debian

.ci/prep_azure.py

@@ -7,19 +7,43 @@ import ci_lib
 
 batches = []
 
+if 0 and os.uname()[0] == 'Linux':
+    batches += [
+        [
+            "sudo chown `whoami`: ~",
+            "chmod u=rwx,g=rx,o= ~",
+
+            "sudo mkdir /var/run/sshd",
+            "sudo /etc/init.d/ssh start",
+
+            "mkdir -p ~/.ssh",
+            "chmod u=rwx,go= ~/.ssh",
+
+            "ssh-keyscan -H localhost >> ~/.ssh/known_hosts",
+            "chmod u=rw,go= ~/.ssh/known_hosts",
+
+            "cat tests/data/docker/mitogen__has_sudo_pubkey.key > ~/.ssh/id_rsa",
+            "chmod u=rw,go= ~/.ssh/id_rsa",
+
+            "cat tests/data/docker/mitogen__has_sudo_pubkey.key.pub > ~/.ssh/authorized_keys",
+            "chmod u=rw,go=r ~/.ssh/authorized_keys",
+        ]
+    ]
+
 if ci_lib.have_apt():
     batches.append([
         'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync',
         'sudo add-apt-repository ppa:deadsnakes/ppa',
         'sudo apt-get update',
-        'sudo apt-get -y install python2.6 python2.6-dev libsasl2-dev libldap2-dev',
+        'sudo apt-get -y install '
+            'python{pv} '
+            'python{pv}-dev '
+            'libsasl2-dev '
+            'libldap2-dev '
+            .format(pv=os.environ['PYTHONVERSION'])
     ])
 
 
-#batches.append([
-    #'pip install -r dev_requirements.txt',
-#])
-
 if ci_lib.have_docker():
     batches.extend(
         ['docker pull %s' % (ci_lib.image_for_distro(distro),)]

.ci/spawn_reverse_shell.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+
+"""
+Allow poking around Azure while the job is running.
+"""
+
+import os
+import pty
+import socket
+import subprocess
+import sys
+import time
+
+
+if os.fork():
+    sys.exit(0)
+
+
+def try_once():
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    s.connect(("k3.botanicus.net", 9494))
+    open('/tmp/interactive', 'w').close()
+
+    os.dup2(s.fileno(), 0)
+    os.dup2(s.fileno(), 1)
+    os.dup2(s.fileno(), 2)
+    p = pty.spawn("/bin/sh")
+
+
+while True:
+    try:
+        try_once()
+    except:
+        time.sleep(5)
+        continue
+

ansible_mitogen/connection.py

@@ -356,7 +356,7 @@ def _connect_mitogen_doas(spec):
             'username': spec.remote_user(),
             'password': spec.password(),
             'python_path': spec.python_path(),
-            'doas_path': spec.become_exe(),
+            'doas_path': spec.ansible_doas_exe(),
             'connect_timeout': spec.timeout(),
             'remote_name': get_remote_name(spec),
         }

ansible_mitogen/strategy.py

@@ -40,9 +40,15 @@ import ansible_mitogen.process
 import ansible
 import ansible.executor.process.worker
 
+try:
+    # 2.8+ has a standardized "unset" object.
+    from ansible.utils.sentinel import Sentinel
+except ImportError:
+    Sentinel = None
+
 
 ANSIBLE_VERSION_MIN = '2.3'
-ANSIBLE_VERSION_MAX = '2.7'
+ANSIBLE_VERSION_MAX = '2.8'
 NEW_VERSION_MSG = (
     "Your Ansible version (%s) is too recent. The most recent version\n"
     "supported by Mitogen for Ansible is %s.x. Please check the Mitogen\n"
@@ -115,7 +121,11 @@ def wrap_action_loader__get(name, *args, **kwargs):
     This is used instead of static subclassing as it generalizes to third party
     action modules outside the Ansible tree.
     """
-    klass = action_loader__get(name, class_only=True)
+    get_kwargs = {'class_only': True}
+    if ansible.__version__ >= '2.8':
+        get_kwargs['collection_list'] = kwargs.pop('collection_list', None)
+
+    klass = action_loader__get(name, **get_kwargs)
     if klass:
         bases = (ansible_mitogen.mixins.ActionModuleMixin, klass)
         adorned_klass = type(str(name), bases, {})
@@ -261,14 +271,17 @@ class StrategyMixin(object):
             name=task.action,
             mod_type='',
         )
-        ansible_mitogen.loaders.connection_loader.get(
-            name=play_context.connection,
-            class_only=True,
-        )
         ansible_mitogen.loaders.action_loader.get(
             name=task.action,
             class_only=True,
         )
+        if play_context.connection is not Sentinel:
+            # 2.8 appears to defer computing this until inside the worker.
+            # TODO: figure out where it has moved.
+            ansible_mitogen.loaders.connection_loader.get(
+                name=play_context.connection,
+                class_only=True,
+            )
 
         return super(StrategyMixin, self)._queue_task(
             host=host,

ansible_mitogen/transport_config.py

@@ -294,6 +294,12 @@ class Spec(with_metaclass(abc.ABCMeta, object)):
         Connection-specific arguments.
         """
 
+    @abc.abstractmethod
+    def ansible_doas_exe(self):
+        """
+        Value of "ansible_doas_exe" variable.
+        """
+
 
 class PlayContextSpec(Spec):
     """
@@ -372,7 +378,15 @@ class PlayContextSpec(Spec):
         ]
 
     def become_exe(self):
-        return self._play_context.become_exe
+        # In Ansible 2.8, PlayContext.become_exe always has a default value due
+        # to the new options mechanism. Previously it was only set if a value
+        # ("somewhere") had been specified for the task.
+        # For consistency in the tests, here we make older Ansibles behave like
+        # newer Ansibles.
+        exe = self._play_context.become_exe
+        if exe is None and self._play_context.become_method == 'sudo':
+            exe = 'sudo'
+        return exe
 
     def sudo_args(self):
         return [
@@ -380,8 +394,9 @@ class PlayContextSpec(Spec):
             for term in ansible.utils.shlex.shlex_split(
                 first_true((
                     self._play_context.become_flags,
-                    self._play_context.sudo_flags,
-                    # Ansible 2.3.
+                    # Ansible <=2.7.
+                    getattr(self._play_context, 'sudo_flags', ''),
+                    # Ansible <=2.3.
                     getattr(C, 'DEFAULT_BECOME_FLAGS', ''),
                     getattr(C, 'DEFAULT_SUDO_FLAGS', '')
                 ), default='')
@@ -424,6 +439,12 @@ class PlayContextSpec(Spec):
     def extra_args(self):
         return self._connection.get_extra_args()
 
+    def ansible_doas_exe(self):
+        return (
+            self._connection.get_task_var('ansible_doas_exe') or
+            os.environ.get('ANSIBLE_DOAS_EXE')
+        )
+
 
 class MitogenViaSpec(Spec):
     """
@@ -634,3 +655,9 @@ class MitogenViaSpec(Spec):
 
     def extra_args(self):
         return []  # TODO
+
+    def ansible_doas_exe(self):
+        return (
+            self._host_vars.get('ansible_doas_exe') or
+            os.environ.get('ANSIBLE_DOAS_EXE')
+        )

docs/ansible_detailed.rst

@@ -731,7 +731,7 @@ as a become method.
 When used as a become method:
 
 * ``ansible_python_interpreter``
-* ``ansible_become_exe``: path to ``doas`` binary.
+* ``ansible_become_exe`` / ``ansible_doas_exe``: path to ``doas`` binary.
 * ``ansible_become_user`` (default: ``root``)
 * ``ansible_become_pass`` (default: assume passwordless)
 * ``mitogen_mask_remote_name``: if :data:`True`, mask the identity of the
@@ -746,6 +746,7 @@ When used as the ``mitogen_doas`` connection method:
 * The inventory hostname has no special meaning.
 * ``ansible_user``: username to use.
 * ``ansible_password``: password to use.
+* ``ansible_doas_exe``: path to ``doas`` binary.
 * ``ansible_python_interpreter``
 
 

docs/changelog.rst

@@ -21,6 +21,24 @@ v0.2.8 (unreleased)
 To avail of fixes in an unreleased version, please download a ZIP file
 `directly from GitHub <https://github.com/dw/mitogen/>`_.
 
+Enhancements
+^^^^^^^^^^^^
+
+* `#587 <https://github.com/dw/mitogen/issues/587>`_: partial support for
+  Ansible 2.8 is now available. This implementation does not yet support the
+  new `become plugins
+  <https://docs.ansible.com/ansible/latest/plugins/become.html>`_
+  functionality, which will be addressed in a future release.
+
+
+Thanks!
+~~~~~~~
+
+Mitogen would not be possible without the support of users. A huge thanks for
+bug reports, testing, features and fixes in this release contributed by
+`Orion Poplawski <https://github.com/opoplawski>`_, and
+`Ulrich Schreiner <https://github.com/ulrichSchreiner>`_.
+
 
 v0.2.7 (2019-05-19)
 -------------------

tests/ansible/ansible.cfg

@@ -13,6 +13,9 @@ retry_files_enabled = False
 display_args_to_stdout = True
 forks = 100
 
+# We use lots of deprecated functionality to support older versions.
+deprecation_warnings = False
+
 # issue #434; hosts/delegate_to; integration/delegate_to
 remote_user = ansible-cfg-remote-user
 

tests/ansible/integration/action/synchronize.yml

@@ -7,6 +7,10 @@
     ansible_user: mitogen__has_sudo_pubkey
     ansible_become_pass: has_sudo_pubkey_password
     ansible_ssh_private_key_file: /tmp/synchronize-action-key
+
+    # https://github.com/ansible/ansible/issues/56629
+    ansible_ssh_pass: ''
+    ansible_password: ''
   tasks:
     # must copy git file to set proper file mode.
     - copy:

tests/ansible/integration/async/runner_one_job.yml

@@ -37,7 +37,12 @@
       - result1.ansible_job_id == job1.ansible_job_id
       - result1.attempts <= 100000
       - result1.changed == True
-      - result1.cmd == "sleep 1;\n echo alldone"
+      # ansible/b72e989e1837ccad8dcdc926c43ccbc4d8cdfe44
+      - |
+        (ansible_version.full >= '2.8' and
+         result1.cmd == "sleep 1;\necho alldone\n") or
+        (ansible_version.full < '2.8' and
+         result1.cmd == "sleep 1;\n echo alldone")
       - result1.delta|length == 14
       - result1.start|length == 26
       - result1.finished == 1

tests/ansible/integration/connection_delegation/local_action.yml

@@ -27,7 +27,7 @@
               'remote_name': null,
               'password': null,
               'username': 'root',
-              'sudo_path': null,
+              'sudo_path': 'sudo',
               'sudo_args': ['-H', '-S', '-n'],
             },
             'method': 'sudo',

tests/ansible/integration/runner/crashy_new_style_module.yml

@@ -12,7 +12,14 @@
         that:
         - not out.changed
         - out.rc == 1
-        - out.msg == "MODULE FAILURE"
+        # ansible/62d8c8fde6a76d9c567ded381e9b34dad69afcd6
+        - |
+          (ansible_version.full < '2.7' and out.msg == "MODULE FAILURE") or
+          (ansible_version.full >= '2.7' and
+           out.msg == (
+            "MODULE FAILURE\n" +
+            "See stdout/stderr for the exact error"
+          ))
         - out.module_stdout == ""
         - "'Traceback (most recent call last)' in out.module_stderr"
         - "\"NameError: name 'kaboom' is not defined\" in out.module_stderr"

tests/ansible/integration/stub_connections/mitogen_doas.yml

@@ -10,7 +10,7 @@
   - custom_python_detect_environment:
     vars:
       ansible_connection: mitogen_doas
-      ansible_become_exe: stub-doas.py
+      ansible_doas_exe: stub-doas.py
       ansible_user: someuser
     register: out
 

tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml

@@ -10,5 +10,4 @@
 
   - assert:
       that:
-        - out.state == 'absent'
         - out.msg == 'file (/usr/bin/does-not-exist) is absent, cannot continue'