Merge commit 'e334b50d9d5bc5174d1e07c83c6617db6406193e' into release-v0.3.8

.ci/ansible_tests.py

@@ -6,6 +6,7 @@ import glob
 import os
 import signal
 import sys
+import textwrap
 
 import ci_lib
 
@@ -74,7 +75,15 @@ with ci_lib.Fold('job_setup'):
             fp.write('\n[%s]\n' % family)
             fp.writelines('%s\n' % name for name in hostnames)
 
-        fp.write('\n[linux:children]\ntest-targets\n')
+        fp.write(textwrap.dedent(
+            '''
+            [linux:children]
+            test-targets
+
+            [linux_containers:children]
+            test-targets
+            '''
+        ))
 
     ci_lib.dump_file(inventory_path)
 

.ci/azure-pipelines.yml

@@ -16,32 +16,21 @@ trigger:
       - docs-master
 
 jobs:
-- job: mac11
+- job: mac12
   # vanilla Ansible is really slow
   timeoutInMinutes: 120
   steps:
   - template: azure-pipelines-steps.yml
   pool:
-    # https://github.com/actions/runner-images/blob/main/images/macos/macos-11-Readme.md
-    vmImage: macOS-11
+    # https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md
+    vmImage: macOS-12
   strategy:
     matrix:
-      Mito_27:
-        tox.env: py27-mode_mitogen
       Mito_312:
-        python.version: '3.12'
         tox.env: py312-mode_mitogen
-
-      Loc_27_210:
-        tox.env: py27-mode_localhost-ansible2.10
       Loc_312_9:
-        python.version: '3.12'
         tox.env: py312-mode_localhost-ansible9
-
-      Van_27_210:
-        tox.env: py27-mode_localhost-ansible2.10-strategy_linear
       Van_312_9:
-        python.version: '3.12'
         tox.env: py312-mode_localhost-ansible9-strategy_linear
 
 - job: Linux

.ci/localhost_ansible_tests.py

@@ -93,5 +93,5 @@ with ci_lib.Fold('machine_prep'):
 with ci_lib.Fold('ansible'):
     os.chdir(TESTS_DIR)
     playbook = os.environ.get('PLAYBOOK', 'all.yml')
-    ci_lib.run('./run_ansible_playbook.py %s -l target %s',
+    ci_lib.run('./run_ansible_playbook.py %s %s',
         playbook, ' '.join(sys.argv[1:]))

ansible_mitogen/connection.py

@@ -119,7 +119,7 @@ def _connect_ssh(spec):
     """
     Return ContextService arguments for an SSH connection.
     """
-    if C.HOST_KEY_CHECKING:
+    if spec.host_key_checking():
         check_host_keys = 'enforce'
     else:
         check_host_keys = 'ignore'

ansible_mitogen/process.py

@@ -281,11 +281,11 @@ def get_cpu_count(default=None):
 
 class Broker(mitogen.master.Broker):
     """
-    WorkerProcess maintains at most 2 file descriptors, therefore does not need
+    WorkerProcess maintains fewer file descriptors, therefore does not need
     the exuberant syscall expense of EpollPoller, so override it and restore
     the poll() poller.
     """
-    poller_class = mitogen.core.Poller
+    poller_class = mitogen.parent.POLLER_LIGHTWEIGHT
 
 
 class Binding(object):

ansible_mitogen/transport_config.py

@@ -67,6 +67,7 @@ import ansible.utils.shlex
 import ansible.constants as C
 
 from ansible.module_utils.six import with_metaclass
+from ansible.module_utils.parsing.convert_bool import boolean
 
 # this was added in Ansible >= 2.8.0; fallback to the default interpreter if necessary
 try:
@@ -79,7 +80,6 @@ try:
 except ImportError:
     from ansible.vars.unsafe_proxy import AnsibleUnsafeText
 
-import ansible_mitogen.loaders
 import mitogen.core
 
 
@@ -246,6 +246,12 @@ class Spec(with_metaclass(abc.ABCMeta, object)):
         Path to the Python interpreter on the target machine.
         """
 
+    @abc.abstractmethod
+    def host_key_checking(self):
+        """
+        Whether or not to check the keys of the target machine
+        """
+
     @abc.abstractmethod
     def private_key_file(self):
         """
@@ -436,9 +442,18 @@ class PlayContextSpec(Spec):
         return self._play_context.become_user
 
     def become_pass(self):
-        become_method = self.become_method()
-        become_plugin = ansible_mitogen.loaders.become_loader.get(become_method)
-        become_pass = become_plugin.get_option('become_pass', hostvars=self._task_vars)
+        # become_pass is owned/provided by the active become plugin. However
+        # PlayContext is intertwined with it. Known complications
+        # - ansible_become_password is higher priority than ansible_become_pass,
+        #   `play_context.become_pass` doesn't obey this (atleast with Mitgeon).
+        # - `meta: reset_connection` runs `connection.reset()` but
+        #   `ansible_mitogen.connection.Connection.reset()` recreates the
+        #   connection object, setting `connection.become = None`.
+        become_plugin = self._connection.become
+        try:
+            become_pass = become_plugin.get_option('become_pass', playcontext=self._play_context)
+        except AttributeError:
+            become_pass = self._play_context.become_pass
         return optional_secret(become_pass)
 
     def password(self):
@@ -458,6 +473,14 @@ class PlayContextSpec(Spec):
             action=self._action,
             rediscover_python=rediscover_python)
 
+    def host_key_checking(self):
+        def candidates():
+            yield self._connection.get_task_var('ansible_ssh_host_key_checking')
+            yield self._connection.get_task_var('ansible_host_key_checking')
+            yield C.HOST_KEY_CHECKING
+        val = next((v for v in candidates() if v is not None), True)
+        return boolean(val)
+
     def private_key_file(self):
         return self._play_context.private_key_file
 
@@ -684,6 +707,14 @@ class MitogenViaSpec(Spec):
             action=self._action,
             rediscover_python=rediscover_python)
 
+    def host_key_checking(self):
+        def candidates():
+            yield self._host_vars.get('ansible_ssh_host_key_checking')
+            yield self._host_vars.get('ansible_host_key_checking')
+            yield C.HOST_KEY_CHECKING
+        val = next((v for v in candidates() if v is not None), True)
+        return boolean(val)
+
     def private_key_file(self):
         # TODO: must come from PlayContext too.
         return (

docs/ansible_detailed.rst

@@ -247,6 +247,15 @@ Noteworthy Differences
   part of the core library, and should therefore be straightforward to fix as
   part of 0.2.x.
 
+* Connection and become timeouts are applied differently. Mitogen may consider
+  a connection to have timed out, when Ansible would have waited longer or
+  indefinately. For example if SSH authentication completes within the
+  timeout, but execution of login scripts exceeds it - then Mitogen will
+  consider the task to have timed out and that host to have failed.
+
+..
+    tests/ansible/integration/ssh/timeouts.yml covers (some of) this behaviour.
+
 ..
     * SSH and ``become`` are treated distinctly when applying timeouts, and
     timeouts apply up to the point when the new interpreter is ready to accept

docs/changelog.rst

@@ -18,6 +18,16 @@ To avail of fixes in an unreleased version, please download a ZIP file
 `directly from GitHub <https://github.com/mitogen-hq/mitogen/>`_.
 
 
+v0.3.8 (2024-07-30)
+-------------------
+
+* :gh:issue:`952` Fix Ansible `--ask-become-pass`, add test coverage
+* :gh:issue:`957` Fix Ansible exception when executing against 10s of hosts
+  "ValueError: filedescriptor out of range in select()"
+* :gh:issue:`1066` Support Ansible `ansible_host_key_checking` & `ansible_ssh_host_key_checking`
+* :gh:issue:`1090` CI: Migrate macOS integration tests to macOS 12, drop Python 2.7 jobs
+
+
 v0.3.7 (2024-04-08)
 -------------------
 

docs/conf.py

@@ -2,7 +2,7 @@ import sys
 
 sys.path.append('.')
 
-VERSION = '0.3.7'
+VERSION = '0.3.8'
 
 author = u'Network Genomics'
 copyright = u'2021, the Mitogen authors'

docs/contributors.rst

@@ -126,11 +126,13 @@ sponsorship and outstanding future-thinking of its early adopters.
     <li>jgadling</li>
     <li>John F Wall &mdash; <em>Making Ansible Great with Massive Parallelism</em></li>
     <li>KennethC</li>
+    <li><a href="https://github.com/lberruti">Luca Berruti</li>
     <li>Lewis Bellwood &mdash; <em>Happy to be apart of a great project.</em></li>
     <li>luto</li>
     <li><a href="https://mayeu.me/">Mayeu a.k.a Matthieu Maury</a></li>
     <li><a href="https://twitter.com/nathanhruby">@nathanhruby</a></li>
     <li><a href="https://github.com/opoplawski">Orion Poplawski</a></li>
+    <li><a href="https://github.com/philfry">Philippe Kueck</a></li>
     <li><a href="http://pageflows.com/">Ramy</a></li>
     <li>Scott Vokes</li>
     <li><a href="https://twitter.com/sirtux">Tom Eichhorn</a></li>

mitogen/__init__.py

@@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup.
 
 
 #: Library version as a tuple.
-__version__ = (0, 3, 7)
+__version__ = (0, 3, 8)
 
 
 #: This is :data:`False` in slave contexts. Previously it was used to prevent

mitogen/core.py

@@ -2523,8 +2523,7 @@ class Poller(object):
     """
     A poller manages OS file descriptors the user is waiting to become
     available for IO. The :meth:`poll` method blocks the calling thread
-    until one or more become ready. The default implementation is based on
-    :func:`select.poll`.
+    until one or more become ready.
 
     Each descriptor has an associated `data` element, which is unique for each
     readiness type, and defaults to being the same as the file descriptor. The
@@ -2546,19 +2545,13 @@ class Poller(object):
     a resource leak.
 
     Pollers may only be used by one thread at a time.
+
+    This implementation uses :func:`select.select` for wider platform support.
+    That is considered an implementation detail. Previous versions have used
+    :func:`select.poll`. Future versions may decide at runtime.
     """
     SUPPORTED = True
 
-    # This changed from select() to poll() in Mitogen 0.2.4. Since poll() has
-    # no upper FD limit, it is suitable for use with Latch, which must handle
-    # FDs larger than select's limit during many-host runs. We want this
-    # because poll() requires no setup and teardown: just a single system call,
-    # which is important because Latch.get() creates a Poller on each
-    # invocation. In a microbenchmark, poll() vs. epoll_ctl() is 30% faster in
-    # this scenario. If select() must return in future, it is important
-    # Latch.poller_class is set from parent.py to point to the industrial
-    # strength poller for the OS, otherwise Latch will fail randomly.
-
     #: Increments on every poll(). Used to version _rfds and _wfds.
     _generation = 1
 
@@ -2681,11 +2674,10 @@ class Latch(object):
 
     See :ref:`waking-sleeping-threads` for further discussion.
     """
-    #: The :class:`Poller` implementation to use for waiting. Since the poller
-    #: will be very short-lived, we prefer :class:`mitogen.parent.PollPoller`
-    #: if it is available, or :class:`mitogen.core.Poller` otherwise, since
-    #: these implementations require no system calls to create, configure or
-    #: destroy.
+    #: The :class:`Poller` implementation to use. Instances are short lived so
+    #: prefer :class:`mitogen.parent.PollPoller` if it's available, otherwise
+    #: :class:`mitogen.core.Poller`. They don't need syscalls to create,
+    #: configure, or destroy. Replaced during import of :mod:`mitogen.parent`.
     poller_class = Poller
 
     #: If not :data:`None`, a function invoked as `notify(latch)` after a

mitogen/parent.py

@@ -745,8 +745,7 @@ def _upgrade_broker(broker):
     broker.timers = TimerList()
     LOG.debug('upgraded %r with %r (new: %d readers, %d writers; '
               'old: %d readers, %d writers)', old, new,
-              len(new.readers), len(new.writers),
-              len(old.readers), len(old.writers))
+              len(new._rfds), len(new._wfds), len(old._rfds), len(old._wfds))
 
 
 @mitogen.core.takes_econtext
@@ -902,22 +901,18 @@ class CallSpec(object):
 class PollPoller(mitogen.core.Poller):
     """
     Poller based on the POSIX :linux:man2:`poll` interface. Not available on
-    some versions of OS X, otherwise it is the preferred poller for small FD
-    counts, as there is no setup/teardown/configuration system call overhead.
+    some Python/OS X combinations. Otherwise the preferred poller for small
+    FD counts; or if many pollers are created, used once, then closed.
+    There there is no setup/teardown/configuration system call overhead.
     """
     SUPPORTED = hasattr(select, 'poll')
-    _repr = 'PollPoller()'
+    _readmask = SUPPORTED and select.POLLIN | select.POLLHUP
 
     def __init__(self):
         super(PollPoller, self).__init__()
         self._pollobj = select.poll()
 
     # TODO: no proof we dont need writemask too
-    _readmask = (
-        getattr(select, 'POLLIN', 0) |
-        getattr(select, 'POLLHUP', 0)
-    )
-
     def _update(self, fd):
         mask = (((fd in self._rfds) and self._readmask) |
                 ((fd in self._wfds) and select.POLLOUT))
@@ -952,7 +947,6 @@ class KqueuePoller(mitogen.core.Poller):
     Poller based on the FreeBSD/Darwin :freebsd:man2:`kqueue` interface.
     """
     SUPPORTED = hasattr(select, 'kqueue')
-    _repr = 'KqueuePoller()'
 
     def __init__(self):
         super(KqueuePoller, self).__init__()
@@ -1030,7 +1024,7 @@ class EpollPoller(mitogen.core.Poller):
     Poller based on the Linux :linux:man7:`epoll` interface.
     """
     SUPPORTED = hasattr(select, 'epoll')
-    _repr = 'EpollPoller()'
+    _inmask = SUPPORTED and select.EPOLLIN | select.EPOLLHUP
 
     def __init__(self):
         super(EpollPoller, self).__init__()
@@ -1077,9 +1071,6 @@ class EpollPoller(mitogen.core.Poller):
         self._wfds.pop(fd, None)
         self._control(fd)
 
-    _inmask = (getattr(select, 'EPOLLIN', 0) |
-               getattr(select, 'EPOLLHUP', 0))
-
     def _poll(self, timeout):
         the_timeout = -1
         if timeout is not None:
@@ -1100,18 +1091,14 @@ class EpollPoller(mitogen.core.Poller):
                     yield data
 
 
-# 2.4 and 2.5 only had select.select() and select.poll().
-for _klass in mitogen.core.Poller, PollPoller, KqueuePoller, EpollPoller:
-    if _klass.SUPPORTED:
-        PREFERRED_POLLER = _klass
+POLLERS = (EpollPoller, KqueuePoller, PollPoller, mitogen.core.Poller)
+PREFERRED_POLLER = next(cls for cls in POLLERS if cls.SUPPORTED)
+
 
 # For processes that start many threads or connections, it's possible Latch
 # will also get high-numbered FDs, and so select() becomes useless there too.
-# So swap in our favourite poller.
-if PollPoller.SUPPORTED:
-    mitogen.core.Latch.poller_class = PollPoller
-else:
-    mitogen.core.Latch.poller_class = PREFERRED_POLLER
+POLLER_LIGHTWEIGHT = PollPoller.SUPPORTED and PollPoller or PREFERRED_POLLER
+mitogen.core.Latch.poller_class = POLLER_LIGHTWEIGHT
 
 
 class LineLoggingProtocolMixin(object):

tests/ansible/ansible.cfg

@@ -1,9 +1,12 @@
 [defaults]
 any_errors_fatal = true
-# callback_whitelist naming will be deprecated in ansible-core >= 2.15.
-# callbacks_enabled naming was added in ansible-core 2.11
+# callbacks_enabled was added in Ansible 4 (ansible-core 2.11).
 # profile_tasks: Displays timing for each task and summary table of top N tasks
 # timer: Displays "Playbook run took 0 days, 0 hours, ..."
+callbacks_enabled =
+    profile_tasks,
+    timer
+# callback_whitelist was deprecated in Ansible >= 8 (ansible-core >= 2.15).
 callback_whitelist =
     profile_tasks,
     timer
@@ -37,7 +40,9 @@ no_target_syslog = True
 # Required by integration/ssh/timeouts.yml
 timeout = 30
 
-# On Travis, paramiko check fails due to host key checking enabled.
+# Ideally this would be true here and and overridden for hosts/groups. However
+# ansible_host_key_checking don't work on Vanilla Ansible 2.10, even for
+# static inventory hosts (ansible/ansible#49254, ansible/ansible#73708)
 host_key_checking = False
 
 [inventory]

tests/ansible/bench/file_transfer.yml

@@ -1,14 +1,16 @@
-
 - name: bench/file_transfer.yml
   hosts: test-targets
   tasks:
-
   - name: Make 32MiB file
     delegate_to: localhost
+    run_once: true
     shell: openssl rand 33554432 > /tmp/bigfile.in
+    args:
+      creates: /tmp/bigfile.in
 
   - name: Make 320MiB file
     delegate_to: localhost
+    run_once: true
     shell: >
       cat
       /tmp/bigfile.in
@@ -22,6 +24,8 @@
       /tmp/bigfile.in
       /tmp/bigfile.in
       > /tmp/bigbigfile.in
+    args:
+      creates: /tmp/bigbigfile.in
 
   - name: Delete SSH file is present.
     file:
@@ -36,17 +40,20 @@
     copy:
       src: /tmp/bigfile.in
       dest: /tmp/bigfile.out
+      mode: ugo=rw
 
   - name: Copy 320MiB file via SSH
     copy:
       src: /tmp/bigbigfile.in
       dest: /tmp/bigbigfile.out
+      mode: ugo=rw
 
   - name: Delete localhost sudo file if present.
     file:
       path: "{{item}}"
       state: absent
     delegate_to: localhost
+    run_once: true
     become: true
     with_items:
     - /tmp/bigfile.out
@@ -56,21 +63,51 @@
 
   - name: Copy 32MiB file via localhost sudo
     delegate_to: localhost
+    run_once: true
     become: true
     copy:
       src: /tmp/bigfile.in
       dest: /tmp/bigfile.out
+      mode: ugo=rw
     tags:
       - requires_local_sudo
 
   - name: Copy 320MiB file via localhost sudo
     delegate_to: localhost
+    run_once: true
     become: true
     copy:
       src: /tmp/bigbigfile.in
       dest: /tmp/bigbigfile.out
+      mode: ugo=rw
     tags:
       - requires_local_sudo
 
+  - name: Local cleanup
+    file:
+      path: "{{ item.path }}"
+      state: absent
+    loop:
+      - /tmp/bigfile.in
+      - /tmp/bigfile.out
+      - /tmp/bigbigfile.in
+      - /tmp/bigbigfile.out
+    delegate_to: localhost
+    run_once: true
+    tags:
+      - cleanup_local
+      - cleanup
+
+  - name: Target cleanup
+    file:
+      path: "{{ item.path }}"
+      state: absent
+    loop:
+      - /tmp/bigfile.out
+      - /tmp/bigbigfile.out
+    tags:
+      - cleanup_target
+      - cleanup
+
   tags:
   - resource_intensive

tests/ansible/hosts/default.hosts

@@ -10,3 +10,5 @@ target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}"
 
 [test-targets]
 target
+
+[linux_containers]

tests/ansible/hosts/group_vars/all.yml

@@ -1,2 +1,3 @@
 ---
 pkg_mgr_python_interpreter: python
+pkg_repos_overrides: []

tests/ansible/hosts/group_vars/centos8.yml

@@ -1,2 +1,28 @@
 ---
 pkg_mgr_python_interpreter: /usr/libexec/platform-python
+
+pkg_repos_overrides:
+  - dest: /etc/yum.repos.d/CentOS-Linux-AppStream.repo
+    content: |
+      [appstream]
+      name=CentOS Linux $releasever - AppStream
+      baseurl=http://vault.centos.org/$contentdir/$releasever/AppStream/$basearch/os/
+      enabled=1
+      gpgcheck=1
+      gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial
+  - dest: /etc/yum.repos.d/CentOS-Linux-BaseOS.repo
+    content: |
+      [baseos]
+      name=CentOS Linux $releasever - BaseOS
+      baseurl=http://vault.centos.org/$contentdir/$releasever/BaseOS/$basearch/os/
+      enabled=1
+      gpgcheck=1
+      gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial
+  - dest: /etc/yum.repos.d/CentOS-Linux-Extras.repo
+    content: |
+      [extras]
+      name=CentOS Linux $releasever - Extras
+      baseurl=http://vault.centos.org/$contentdir/$releasever/extras/$basearch/os/
+      enabled=1
+      gpgcheck=1
+      gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial

tests/ansible/hosts/group_vars/debian9.yml

@@ -0,0 +1,4 @@
+pkg_repos_overrides:
+  - dest: /etc/apt/sources.list
+    content: |
+      deb http://archive.debian.org/debian stretch main contrib non-free

tests/ansible/hosts/transport_config.hosts

@@ -13,6 +13,7 @@ tc_become
 tc_become_method
 tc_become_pass
 tc_become_user
+tc_host_key_checking
 tc_password
 tc_port
 tc_remote_addr
@@ -74,6 +75,11 @@ tc-become-pass-password ansible_become_password=apassword
 tc-become-pass-pass ansible_become_pass=apass
 tc-become-pass-both ansible_become_pass=bpass ansible_become_password=bpassword
 
+[tc_host_key_checking]
+tc-hkc-unset
+tc-hkc-host-key-checking ansible_host_key_checking=true
+tc-hkc-ssh-host-key-checking ansible_ssh_host_key_checking=true
+
 [tc_port]
 tc-port-unset
 tc-port-explicit-port ansible_port=1234

tests/ansible/integration/action/copy.yml

@@ -9,6 +9,7 @@
         content:
           this is a tiny file.
       delegate_to: localhost
+      run_once: true
 
     - name: Create large file
       copy:
@@ -16,6 +17,7 @@
         # Must be larger than Connection.SMALL_SIZE_LIMIT.
         content: "{% for x in range(200000) %}x{% endfor %}"
       delegate_to: localhost
+      run_once: true
 
     - name: Cleanup copied files
       file:

tests/ansible/integration/action/fixup_perms2__copy.yml

@@ -38,6 +38,7 @@
 
     - name: Create local weird mode file
       delegate_to: localhost
+      run_once: true
       copy:
         content: "weird mode"
         dest: "/tmp/weird-mode"

tests/ansible/integration/action/synchronize.yml

@@ -24,18 +24,21 @@
         path: /tmp/sync-test
         state: absent
       delegate_to: localhost
+      run_once: true
 
     - name: Create sync-test
       file:
         path: /tmp/sync-test
         state: directory
       delegate_to: localhost
+      run_once: true
 
     - name: Create syn-test item
       copy:
         dest: /tmp/sync-test/item
         content: "item!"
       delegate_to: localhost
+      run_once: true
 
     # TODO: https://github.com/dw/mitogen/issues/692
     # - file:

tests/ansible/integration/become/sudo_nonexistent.yml

@@ -21,8 +21,7 @@
           # sudo-1.8.6p3-29.el6_10.3 on RHEL & CentOS 6.10 (final release)
           # removed user/group error messages, as defence against CVE-2019-14287.
           - >-
-            'sudo: unknown user: slartibartfast' in out.module_stdout | default(out.msg)
-            or 'sudo: unknown user: slartibartfast' in out.module_stderr | default(out.msg)
+            (out.module_stderr | default(out.module_stdout, true) | default(out.msg, true)) is search('sudo: unknown user:? slartibartfast')
             or (ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_version == '6.10')
         fail_msg: out={{out}}
       when:

tests/ansible/integration/connection/_cleanup_file.yml

@@ -0,0 +1,17 @@
+- name: Cleanup local file
+  file:
+    path: /tmp/{{ file_name }}
+    state: absent
+  delegate_to: localhost
+  run_once: true
+  tags:
+    - cleanup_local
+    - cleanup
+
+- name: Cleanup target file
+  file:
+    path: /tmp/{{ file_name }}.out
+    state: absent
+  tags:
+    - cleanup_target
+    - cleanup

tests/ansible/integration/connection/_put_file.yml

@@ -1,20 +1,21 @@
----
-
 - name: Create {{ file_name }}
-  shell: dd if=/dev/urandom of=/tmp/{{ file_name }} bs=1024 count={{ file_size }}
-  args:
+  command:
+    cmd: dd if=/dev/urandom of=/tmp/{{ file_name }} bs=1024 count={{ file_size_kib }}
     creates: /tmp/{{file_name}}
   delegate_to: localhost
+  run_once: true
 
 - name: Copy {{ file_name }}
   copy:
     dest: /tmp/{{file_name}}.out
     src: /tmp/{{file_name}}
+    mode: "{{ file_mode }}"
 
 - name: Stat created {{ file_name }}
   stat: path=/tmp/{{ file_name }}
   register: original
   delegate_to: localhost
+  run_once: true
 
 - name: Stat copied {{ file_name }}
   stat: path=/tmp/{{ file_name }}.out

tests/ansible/integration/connection/disconnect_during_module.yml

@@ -10,6 +10,8 @@
 
   - name: Run _disconnect_during_module.yml
     delegate_to: localhost
+    environment:
+      ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
     command: |
       ansible-playbook
       {% for inv in ansible_inventory_sources %}

tests/ansible/integration/connection/put_large_file.yml

@@ -6,9 +6,11 @@
   gather_facts: no
   vars:
     file_name: large-file
-    file_size: 512
+    file_size_kib: 512
+    file_mode: u=rw,go=
   tasks:
     - include_tasks: _put_file.yml
+    - include_tasks: _cleanup_file.yml
   tags:
     - put_file
     - put_large_file

tests/ansible/integration/connection/put_small_file.yml

@@ -6,9 +6,11 @@
   gather_facts: no
   vars:
     file_name: small-file
-    file_size: 123
+    file_size_kib: 123
+    file_mode: u=rw,go=
   tasks:
     - include_tasks: _put_file.yml
+    - include_tasks: _cleanup_file.yml
   tags:
     - put_file
     - put_small_file

tests/ansible/integration/interpreter_discovery/complex_args.yml

@@ -29,20 +29,18 @@
         special_python: "source /tmp/fake || true && python{{ ansible_facts.python.version.major }}"
 
     - name: run get_url with specially-sourced python
-      get_url:
+      uri:
         # Plain http for wider Ansible & Python version compatibility
-        url: http://httpbin.org/get
-        dest: "/tmp/"
-        mode: 0644
+        url: http://www.gstatic.com/generate_204
+        status_code: [204]
       vars:
         ansible_python_interpreter: "{{ special_python }}"
 
     - name: run get_url with specially-sourced python including jinja
-      get_url:
+      uri:
         # Plain http for wider Ansible & Python version compatibility
-        url: http://httpbin.org/get
-        dest: "/tmp/"
-        mode: 0644
+        url: http://www.gstatic.com/generate_204
+        status_code: [204]
       vars:
         ansible_python_interpreter: >
           {% if "1" == "1" %}

tests/ansible/integration/playbook_semantics/delegate_to.yml

@@ -1,23 +1,27 @@
 - name: integration/playbook_semantics/delegate_to.yml
   hosts: test-targets
+  vars:
+    local_path: "/tmp/delegate_to.{{ inventory_hostname }}.txt"
   tasks:
     #
     # delegate_to, no sudo
     #
     - name: "delegate_to, no sudo"
       copy:
-        dest: /tmp/delegate_to.yml.txt
+        dest: "{{ local_path }}"
         content: "Hello, world."
-      register: out
+        mode: u=rw,go=r
       delegate_to: localhost
 
     - name: "delegate_to, no sudo"
       assert:
-        that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'Hello, world.'"
+        that:
+          - lookup('file', local_path) == 'Hello, world.'
+        fail_msg: "{{ lookup('file', local_path) }}"
 
     - name: "delegate_to, no sudo"
       file:
-        path: /tmp/delegate_to.yml.txt
+        path: "{{ local_path }}"
         state: absent
       delegate_to: localhost
 
@@ -27,18 +31,20 @@
     #
     - name: "connection:local, no sudo"
       copy:
-        dest: /tmp/delegate_to.yml.txt
+        dest: "{{ local_path }}"
         content: "Hello, world."
-      register: out
+        mode: u=rw,go=r
       connection: local
 
     - name: "connection:local, no sudo"
       assert:
-        that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'Hello, world.'"
+        that:
+          - lookup('file', local_path) == 'Hello, world.'
+        fail_msg: "{{ lookup('file', local_path) }}"
 
     - name: "connection:local, no sudo"
       file:
-        path: /tmp/delegate_to.yml.txt
+        path: "{{ local_path }}"
         state: absent
       connection: local
 
@@ -47,7 +53,10 @@
     # delegate_to, sudo
     #
     - name: "delegate_to, sudo"
-      shell: whoami > /tmp/delegate_to.yml.txt
+      shell: |
+        whoami > "{{ local_path }}"
+      args:
+        creates: "{{ local_path }}"
       delegate_to: localhost
       become: true
       tags:
@@ -55,13 +64,15 @@
 
     - name: "delegate_to, sudo"
       assert:
-        that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'root'"
+        that:
+          - lookup('file', local_path) == 'root'
+        fail_msg: "{{ lookup('file', local_path) }}"
       tags:
         - requires_local_sudo
 
     - name: "delegate_to, sudo"
       file:
-        path: /tmp/delegate_to.yml.txt
+        path: "{{ local_path }}"
         state: absent
       delegate_to: localhost
       become: true
@@ -73,7 +84,10 @@
     # connection:local, sudo
     #
     - name: "connection:local, sudo"
-      shell: whoami > /tmp/delegate_to.yml.txt
+      shell: |
+        whoami > "{{ local_path }}"
+      args:
+        creates: "{{ local_path }}"
       connection: local
       become: true
       tags:
@@ -81,13 +95,15 @@
 
     - name: "connection:local, sudo"
       assert:
-        that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'root'"
+        that:
+          - lookup('file', local_path) == 'root'
+        fail_msg: "{{ lookup('file', local_path) }}"
       tags:
         - requires_local_sudo
 
     - name: "connection:local, sudo"
       file:
-        path: /tmp/delegate_to.yml.txt
+        path: "{{ local_path }}"
         state: absent
       connection: local
       become: true

tests/ansible/integration/process/unix_socket_cleanup.yml

@@ -10,6 +10,7 @@
   - shell: >
       ANSIBLE_STRATEGY=mitogen_linear
       ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
+      ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
       ansible -m shell -c local -a whoami
       {% for inv in ansible_inventory_sources %}
       -i "{{ inv }}"

tests/ansible/integration/runner/missing_module.yml

@@ -1,12 +1,13 @@
-
 - name: integration/runner/missing_module.yml
   hosts: test-targets[0]
   connection: local
   tasks:
     - name: Run missing_module
       connection: local
+      environment:
+        ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
       command: |
-        ansible -vvv
+        ansible
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
         {% endfor %}
@@ -15,6 +16,8 @@
       args:
         chdir: ../..
       register: out
+      changed_when: false
+      check_mode: false
       ignore_errors: true
 
     - assert:

tests/ansible/integration/ssh/timeouts.yml

@@ -1,7 +1,12 @@
 # Ensure 'ssh' connections time out correctly.
+# mitogen__slow_user performs a long sleep in ~/.profile.
+# Mitogen counts this time towards the connection timeout. Ansible doesn't.
+# ansible_python_interpreter=python3000 is an optimisation, to avoid waiting
+# on the timeout multiple times (e.g. interpreter discovery).
 
 - name: integration/ssh/timeouts.yml
   hosts: test-targets
+  gather_facts: false
   tasks:
     - include_tasks: ../_mitogen_only.yml
 
@@ -9,17 +14,21 @@
       connection: local
       environment:
         ANSIBLE_SSH_TIMEOUT: 10
+        ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
       command: |
-        ansible -vvv
+        ansible
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
         {% endfor %}
-        test-targets
-        -m custom_python_detect_environment
+        "{{ inventory_hostname }}"
+        -m ping
         -e ansible_user=mitogen__slow_user -e ansible_password=slow_user_password
+        -e ansible_python_interpreter=python3000
       args:
         chdir: ../..
       register: out
+      changed_when: false
+      check_mode: false
       ignore_errors: true
 
     - name: Verify connection timeout occurred

tests/ansible/integration/ssh/variables.yml

@@ -20,6 +20,7 @@
         ANSIBLE_ANY_ERRORS_FATAL=false
         ANSIBLE_STRATEGY=mitogen_linear
         ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
+        ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
         ansible -m shell -a whoami
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
@@ -36,6 +37,7 @@
         ANSIBLE_ANY_ERRORS_FATAL=false
         ANSIBLE_STRATEGY=mitogen_linear
         ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
+        ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
         ansible -m shell -a whoami
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
@@ -43,6 +45,7 @@
         test-targets
         -e ansible_ssh_user=mitogen__has_sudo
         -e ansible_ssh_pass=wrong_password
+        -e ansible_python_interpreter=python3000
       args:
         chdir: ../..
       register: out
@@ -59,6 +62,7 @@
         ANSIBLE_ANY_ERRORS_FATAL=false
         ANSIBLE_STRATEGY=mitogen_linear
         ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
+        ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
         ansible -m shell -a whoami
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
@@ -75,6 +79,7 @@
         ANSIBLE_ANY_ERRORS_FATAL=false
         ANSIBLE_STRATEGY=mitogen_linear
         ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
+        ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
         ansible -m shell -a whoami
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
@@ -82,6 +87,7 @@
         test-targets
         -e ansible_user=mitogen__has_sudo
         -e ansible_ssh_pass=wrong_password
+        -e ansible_python_interpreter=python3000
       args:
         chdir: ../..
       register: out
@@ -98,6 +104,7 @@
         ANSIBLE_ANY_ERRORS_FATAL=false
         ANSIBLE_STRATEGY=mitogen_linear
         ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
+        ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
         ansible -m shell -a whoami
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
@@ -114,6 +121,7 @@
         ANSIBLE_ANY_ERRORS_FATAL=false
         ANSIBLE_STRATEGY=mitogen_linear
         ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
+        ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
         ansible -m shell -a whoami
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
@@ -121,6 +129,7 @@
         test-targets
         -e ansible_user=mitogen__has_sudo
         -e ansible_password=wrong_password
+        -e ansible_python_interpreter=python3000
       args:
         chdir: ../..
       register: out
@@ -142,6 +151,7 @@
         ANSIBLE_ANY_ERRORS_FATAL=false
         ANSIBLE_STRATEGY=mitogen_linear
         ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
+        ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
         ansible -m shell -a whoami
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
@@ -158,6 +168,7 @@
         ANSIBLE_ANY_ERRORS_FATAL=false
         ANSIBLE_STRATEGY=mitogen_linear
         ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
+        ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
         ansible -m shell -a whoami
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
@@ -165,6 +176,7 @@
         test-targets
         -e ansible_user=mitogen__has_sudo
         -e ansible_ssh_private_key_file=/dev/null
+        -e ansible_python_interpreter=python3000
       args:
         chdir: ../..
       register: out

tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml

@@ -3,24 +3,26 @@
   hosts: test-targets[0]
   tasks:
     - connection: local
+      environment:
+        ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
       command: |
         ansible-playbook
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
         {% endfor %}
-        -vvv
         integration/strategy/_mixed_mitogen_vanilla.yml
       args:
         chdir: ../..
       register: out
 
     - connection: local
+      environment:
+        ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
       command: |
         ansible-playbook
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
         {% endfor %}
-        -vvv
         integration/strategy/_mixed_vanilla_mitogen.yml
       args:
         chdir: ../..

tests/ansible/integration/stub_connections/setns_lxc.yml

@@ -12,6 +12,8 @@
     - include_tasks: _end_play_if_not_sudo_linux.yml
 
     - name: Run stub-lxc-info.py
+      environment:
+        ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
       command: |
         sudo -nE "{{lookup('env', 'VIRTUAL_ENV')}}/bin/ansible"
         -i localhost,

tests/ansible/integration/stub_connections/setns_lxd.yml

@@ -12,6 +12,8 @@
     - include_tasks: _end_play_if_not_sudo_linux.yml
 
     - name: Run ansible stub-lxc.py
+      environment:
+        ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
       command: |
         sudo -nE "{{lookup('env', 'VIRTUAL_ENV')}}/bin/ansible"
         -i localhost,

tests/ansible/integration/transport_config/all.yml

@@ -2,6 +2,7 @@
 - import_playbook: become_pass.yml
 - import_playbook: become_user.yml
 - import_playbook: become.yml
+- import_playbook: host_key_checking.yml
 - import_playbook: password.yml
 - import_playbook: port.yml
 - import_playbook: python_path.yml

tests/ansible/integration/transport_config/become_pass.yml

@@ -143,8 +143,9 @@
       - out.result|length == 2
       - out.result[0].method == "ssh"
       - out.result[1].method == "sudo"
-      # Ansible >= 2.10 builtin become plugins (e.g. sudo, su) give priority
-      # to ansible_become_pass over ansible_become_password.
+      # Ansible <= 2.9.1 prioritises ansible_become_password.
+      # Ansible >= 2.9.2 prioritises ansible_become_pass.
+      # https://github.com/ansible/ansible/commit/480b106d6535978ae6ecab68b40942ca4fa914a0
       - out.result[1].kwargs.password == "bpass"
       fail_msg: out={{out}}
   tags:

tests/ansible/integration/transport_config/host_key_checking.yml

@@ -0,0 +1,94 @@
+# Each case is followed by mitogen_via= case to test hostvars method.
+
+- name: integration/transport_config/host_key_checking.yml
+  hosts: tc-hkc-unset
+  tasks:
+    - include_tasks: ../_mitogen_only.yml
+    - {mitogen_get_stack: {}, register: out}
+    - assert:
+        that:
+          - out.result | length == 1
+          - out.result[0].method == "ssh"
+          - out.result[0].kwargs.check_host_keys == "ignore"
+        fail_msg: out={{ out }}
+  tags:
+    - mitogen_only
+
+- hosts: tc-hkc-unset
+  vars:
+    mitogen_via: tc-hkc-host-key-checking
+  tasks:
+    - include_tasks: ../_mitogen_only.yml
+    - {mitogen_get_stack: {}, register: out}
+    - assert:
+        that:
+          - out.result | length == 2
+          - out.result[0].method == "ssh"
+          - out.result[0].kwargs.check_host_keys == "enforce"
+          - out.result[1].method == "ssh"
+          - out.result[1].kwargs.check_host_keys == "ignore"
+        fail_msg: out={{ out }}
+  tags:
+    - mitogen_only
+
+
+- hosts: tc-hkc-host-key-checking
+  tasks:
+    - include_tasks: ../_mitogen_only.yml
+    - {mitogen_get_stack: {}, register: out}
+    - assert:
+        that:
+          - out.result | length == 1
+          - out.result[0].method == "ssh"
+          - out.result[0].kwargs.check_host_keys == "enforce"
+        fail_msg: out={{ out }}
+  tags:
+    - mitogen_only
+
+- hosts: tc-hkc-host-key-checking
+  vars:
+    mitogen_via: tc-hkc-unset
+  tasks:
+    - include_tasks: ../_mitogen_only.yml
+    - {mitogen_get_stack: {}, register: out}
+    - assert:
+        that:
+          - out.result | length == 2
+          - out.result[0].method == "ssh"
+          - out.result[0].kwargs.check_host_keys == "ignore"
+          - out.result[1].method == "ssh"
+          - out.result[1].kwargs.check_host_keys == "enforce"
+        fail_msg: out={{ out }}
+  tags:
+    - mitogen_only
+
+
+- hosts: tc-hkc-ssh-host-key-checking
+  tasks:
+    - include_tasks: ../_mitogen_only.yml
+    - {mitogen_get_stack: {}, register: out}
+    - assert:
+        that:
+          - out.result | length == 1
+          - out.result[0].method == "ssh"
+          - out.result[0].kwargs.check_host_keys == "enforce"
+        fail_msg: out={{ out }}
+  tags:
+    - mitogen_only
+
+- hosts: tc-hkc-ssh-host-key-checking
+  vars:
+    mitogen_via: tc-hkc-unset
+  tasks:
+    - include_tasks: ../_mitogen_only.yml
+    - {mitogen_get_stack: {}, register: out}
+    - assert:
+        that:
+          - out.result | length == 2
+          - out.result[0].method == "ssh"
+          - out.result[0].kwargs.check_host_keys == "ignore"
+          - out.result[1].method == "ssh"
+          - out.result[1].kwargs.check_host_keys == "enforce"
+        fail_msg: out={{ out }}
+  tags:
+    - mitogen_only

tests/ansible/lib/modules/custom_python_json_args_module.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # I am an Ansible Python JSONARGS module. I should receive an encoding string.
 
 json_arguments = """<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>"""

tests/ansible/lib/modules/custom_python_os_getcwd.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # #591: call os.getcwd() before AnsibleModule ever gets a chance to fix up the
 # process environment.
 

tests/ansible/lib/modules/custom_python_want_json_module.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # I am an Ansible Python WANT_JSON module. I should receive a JSON-encoded file.
 
 import json

tests/ansible/regression/all.yml

@@ -14,3 +14,5 @@
 - import_playbook: issue_615__streaming_transfer.yml
 - import_playbook: issue_655__wait_for_connection_error.yml
 - import_playbook: issue_776__load_plugins_called_twice.yml
+- import_playbook: issue_952__ask_become_pass.yml
+- import_playbook: issue_1066__add_host__host_key_checking.yml

tests/ansible/regression/become_test.yml

@@ -0,0 +1,9 @@
+- name: regression/become_test.yml
+  hosts: test-targets:&linux_containers
+  become: true
+  become_user: mitogen__pw_required
+  strategy: mitogen_linear
+  tasks:
+    - command: whoami
+      changed_when: false
+      check_mode: false

tests/ansible/regression/issue_1066__add_host__host_key_checking.yml

@@ -0,0 +1,67 @@
+- name: regression/issue_1066__add_host__host_key_checking.yml
+  hosts: test-targets[0]
+  gather_facts: false
+  become: false
+  tasks:
+    - name: Add hosts dynamically
+      add_host:
+        name: "{{ item.name }}"
+        ansible_host_key_checking:        "{{ item.host_key_checking | default(omit) }}"
+        ansible_ssh_host_key_checking:    "{{ item.host_ssh_key_checking | default(omit) }}"
+        ansible_host:                     "{{ hostvars[inventory_hostname].ansible_host | default(omit) }}"
+        ansible_password:                 "{{ hostvars[inventory_hostname].ansible_password | default(omit) }}"
+        ansible_port:                     "{{ hostvars[inventory_hostname].ansible_port | default(omit) }}"
+        ansible_python_interpreter:       "{{ hostvars[inventory_hostname].ansible_python_interpreter | default(omit) }}"
+        ansible_user:                     "{{ hostvars[inventory_hostname].ansible_user | default(omit) }}"
+      loop:
+        - {name: issue-1066-host-hkc-false,   host_key_checking: false}
+        - {name: issue-1066-host-hkc-true,    host_key_checking: true}
+        - {name: issue-1066-host-hskc-false,  host_ssh_key_checking: false}
+        - {name: issue-1066-host-hskc-true,   host_ssh_key_checking: true}
+      delegate_to: localhost
+  tags:
+    - issue_1066
+
+- name: regression/issue_1066__add_host__host_key_checking.yml
+  hosts: issue-1066-host-*
+  gather_facts: false
+  become: false
+  serial: 1
+  tasks:
+    - meta: reset_connection
+
+    # The host key might be in ~/.ssh/known_hosts. If it's removed then no
+    # problem - test-targets hosts have host_key_checking=false.
+    - name: Remove existing host keys
+      known_hosts:
+        name: "{{ ansible_host }}"
+        state: absent
+      delegate_to: localhost
+
+    - name: Ping dynamically added hosts
+      ping:
+      ignore_errors: true
+      ignore_unreachable: true
+      register: issue_1066_ping
+
+    - debug:
+        var: issue_1066_ping
+
+    - name: Confirm dynamically added hosts are/are not reachable
+      vars:
+        expected:
+          issue-1066-host-hkc-false:    {}
+          issue-1066-host-hkc-true:     {unreachable: true}
+          issue-1066-host-hskc-false:   {}
+          issue-1066-host-hskc-true:    {unreachable: true}
+      assert:
+        that:
+          - issue_1066_ping.unreachable is defined == expected[inventory_hostname].unreachable is defined
+          - issue_1066_ping.unreachable | default(42) == expected[inventory_hostname].unreachable | default(42)
+      # ansible_host_key_checking don't work on Vanilla Ansible 2.10, even for
+      # static inventory hosts (ansible/ansible#49254, ansible/ansible#73708).
+      when:
+        - ansible_version.full is version('2.11', '>=', strict=True)
+          or is_mitogen
+  tags:
+    - issue_1066

tests/ansible/regression/issue_140__thread_pileup.yml

@@ -8,6 +8,7 @@
 
   - name: Create file tree
     connection: local
+    run_once: true
     shell: >
       mkdir /tmp/filetree.in;
       seq -f /tmp/filetree.in/%g 1 1000 | xargs touch;
@@ -21,17 +22,30 @@
     file:
       state: directory
       path: /tmp/filetree.out
+      mode: u=rwx,go=rx
 
   - name: Trigger nasty process pileup
     copy:
       src: "{{item.src}}"
       dest: "/tmp/filetree.out/{{item.path}}"
-      mode: 0644
+      mode: u=rw,go=r
     with_filetree: /tmp/filetree.in
     when: item.state == 'file'
     loop_control:
       label: "/tmp/filetree.out/{{ item.path }}"
 
+  - name: Cleanup local file tree
+    connection: local
+    run_once: true
+    file:
+      path: /tmp/filetree.in
+      state: absent
+
+  - name: Cleanup remote file tree
+    file:
+      path: /tmp/filetree.out
+      state: absent
+
   tags:
   - resource_intensive
   - issue_140

tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml

@@ -4,10 +4,9 @@
 # can test for.
 
 - name: regression/issue_152__local_action_wrong_interpreter.yml
-  hosts: test-targets
+  hosts: test-targets[0]
   connection: local
   tasks:
-
     - name: Create /tmp/issue_152_interpreter.sh
       copy:
         dest: /tmp/issue_152_interpreter.sh

tests/ansible/regression/issue_591__setuptools_cwd_crash.yml

@@ -19,9 +19,6 @@
 
   # Will crash if process has a nonexistent CWD.
   - custom_python_os_getcwd:
-      script: |
-        import os
-        self._connection.get_chain().call(os.getcwd)
   tags:
     - issue_591
     - mitogen_only

tests/ansible/regression/issue_615__streaming_transfer.yml

@@ -14,11 +14,14 @@
           shell: |
             dd if=/dev/zero of=/tmp/512mb.zero bs=1048576 count=512;
             chmod go= /tmp/512mb.zero
+          args:
+            creates: /tmp/512mb.zero
 
         - name: Fetch /tmp/512mb.zero
           fetch:
             src: /tmp/512mb.zero
-            dest: /tmp/fetch-out
+            dest: /tmp/fetch-{{ inventory_hostname }}-512mb.zero
+            flat: true
 
         - name: Cleanup /tmp/512mb.zero
           file:
@@ -27,11 +30,10 @@
 
         - name: Cleanup fetched file
           file:
-            path: /tmp/fetch-out
+            path: /tmp/fetch-{{ inventory_hostname }}-512mb.zero
             state: absent
           become: false
           delegate_to: localhost
-          run_once: true
   tags:
     - issue_615
     - mitogen_only

tests/ansible/regression/issue_655__wait_for_connection_error.yml

@@ -4,11 +4,18 @@
 #   since things are ran on localhost; Azure DevOps loses connection and fails
 # TODO: do we want to install docker a different way to be able to do this for other tests too
 ---
-- name: regression/issue_655_wait_for_connection_error.yml
+- name: regression/issue_655__wait_for_connection_error.yml
   hosts: localhost
   gather_facts: yes
   become: no
   tasks:
+    - meta: end_play
+      when:
+        # TODO CI currently runs on macOS 12 & which isn't supported by Podman
+        #      version available in Homebrew.
+        - ansible_facts.system == 'Darwin'
+        - ansible_facts.distribution_version is version('13.0', '<', strict=True)
+
     - name: set up test container and run tests inside it
       block:
         - name: install deps

tests/ansible/regression/issue_776__load_plugins_called_twice.yml

@@ -10,21 +10,14 @@
     ansible_python_interpreter: "{{ pkg_mgr_python_interpreter }}"
     package: rsync  # Chosen to exist in all tested distros/package managers
   tasks:
-    - name: Switch to centos-stream
-      command: dnf --assumeyes --disablerepo="*" --enablerepo=extras swap centos-linux-repos centos-stream-repos
-      when:
-        - ansible_facts.pkg_mgr in ["dnf"]
-
-    - name: Switch to archive.debian.org
-      # Debian 9 has been archived https://lists.debian.org/debian-devel-announce/2023/03/msg00006.html
+    - name: Switch to archived package repositories
       copy:
-        content: |
-          deb http://archive.debian.org/debian stretch main contrib non-free
-        dest: /etc/apt/sources.list
+        dest: "{{ item.dest }}"
+        content: "{{ item.content }}"
         mode: u=rw,go=r
-      when:
-        - ansible_facts.distribution == "Debian"
-        - ansible_facts.distribution_major_version == "9"
+      loop: "{{ pkg_repos_overrides }}"
+      loop_control:
+        label: "{{ item.dest }}"
 
     - name: Add signing keys
       copy:

tests/ansible/regression/issue_952__ask_become_pass.yml

@@ -0,0 +1,23 @@
+- name: regression/issue_952__ask_become_pass.yml
+  hosts: test-targets[0]:&linux_containers
+  gather_facts: false
+  tags:
+    - issue_952
+  tasks:
+    - name: Test --ask-become-pass
+      delegate_to: localhost
+      environment:
+        ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
+      expect:
+        command: >
+          ansible-playbook
+          {% for inv in ansible_inventory_sources %}
+          -i "{{ inv }}"
+          {% endfor %}
+          --ask-become-pass
+          regression/become_test.yml
+        chdir: ../
+        responses:
+          'BECOME password:': pw_required_password
+      changed_when: false
+      check_mode: false

tests/ansible/requirements.txt

@@ -1,4 +1,7 @@
 paramiko==2.3.2  # Last 2.6-compat version.
+# Incompatible with pip >= 72, due to removal of `setup.py test`:
+#   ModuleNotFoundError: No module named 'setuptools.command.test'
+# https://github.com/pypa/setuptools/issues/4519
 hdrhistogram==0.6.1
 PyYAML==3.11; python_version < '2.7'
 PyYAML==5.3.1; python_version >= '2.7'  # Latest release (Jan 2021)

tests/ansible/setup/report_controller.yml

@@ -1,5 +1,5 @@
 - name: Report controller parameters
-  hosts: localhost
+  hosts: test-targets[0]
   gather_facts: false
   tasks:
     - debug:
@@ -9,9 +9,11 @@
           - $(groups):                "{{ lookup('pipe', 'groups') }}"
           - $(pwd):                   "{{ lookup('pipe', 'pwd') }}"
           - $(whoami):                "{{ lookup('pipe', 'whoami') }}"
+          - ansible_inventory_sources:        "{{ ansible_inventory_sources | default('<unset>') }}"
           - ansible_run_tags:         "{{ ansible_run_tags | default('<unset>') }}"
           - ansible_playbook_python:  "{{ ansible_playbook_python | default('<unset>') }}"
           - ansible_skip_tags:        "{{ ansible_skip_tags | default('<unset>') }}"
           - ansible_version.full:     "{{ ansible_version.full | default('<unset>') }}"
           - is_mitogen:               "{{ is_mitogen | default('<unset>') }}"
           - playbook_dir:             "{{ playbook_dir | default('<unset>') }}"
+      delegate_to: localhost

tests/constraints.txt

@@ -0,0 +1,3 @@
+# Setuptools 72 removed `setup.py test`. hdrhistogram 0.6.1 still depends on it.
+# TODO Bump dependencies and unconstrain Pip.
+setuptools<72

tests/requirements.txt

@@ -12,6 +12,8 @@ Django==3.2.20;     python_version >= '3.6'
 mock==3.0.5;                python_version == '2.7'
 mock==5.1.0;                python_version >= '3.6'
 
+pexpect==4.8
+
 psutil==5.9.8
 
 pytest==4.6.11;             python_version == '2.7'

tests/two_three_compat_test.py

@@ -1,3 +1,6 @@
+import os
+import unittest
+
 import mitogen.core
 
 import testlib
@@ -7,6 +10,10 @@ import simple_pkg.ping
 # TODO: this is a joke. 2/3 interop is one of the hardest bits to get right.
 # There should be 100 tests in this file.
 
+@unittest.skipIf(
+    os.uname()[0] == 'Darwin' and int(os.uname()[2].partition('.')[0]) >= 21,
+    "Python 2.x not shipped on macOS 12.3+ (Darwin 21.4+, Monterey)",
+)
 class TwoThreeCompatTest(testlib.RouterMixin, testlib.TestCase):
     if mitogen.core.PY3:
         python_path = 'python2'

tox.ini

@@ -3,7 +3,7 @@
 #
 #   sudo add-apt-repository ppa:deadsnakes/ppa
 #   sudo apt update
-#   sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python2.7 python3.{6..13}{,-venv} python-is-python3 sshpass tox
+#   sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python{2,2.7,3} python3.{6..13}{,-venv} python-is-python3 sshpass tox
 
 # Py   A cntrllr  A target   coverage   Django     Jinja2     pip        psutil     pytest     tox        virtualenv
 # ==== ========== ========== ========== ========== ========== ========== ========== ========== ========== ==========
@@ -109,6 +109,7 @@ setenv =
     ANSIBLE_STRATEGY = mitogen_linear
     NOCOVERAGE_ERASE = 1
     NOCOVERAGE_REPORT = 1
+    PIP_CONSTRAINT={toxinidir}/tests/constraints.txt
     # Only applicable to MODE=mitogen
     distro_centos5: DISTRO=centos5
     distro_centos6: DISTRO=centos6
@@ -142,6 +143,8 @@ setenv =
     distros_ubuntu2004: DISTROS=ubuntu2004
     mode_ansible: MODE=ansible
     mode_ansible: ANSIBLE_SKIP_TAGS=resource_intensive
+    mode_ansible: ANSIBLE_CALLBACK_WHITELIST=profile_tasks
+    mode_ansible: ANSIBLE_CALLBACKS_ENABLED=profile_tasks
     mode_debops_common: MODE=debops_common
     mode_localhost: ANSIBLE_SKIP_TAGS=issue_776,resource_intensive
     mode_mitogen: MODE=mitogen