From 37ebce7e6effbc6cf9dac7fbf58ba31fb006bf17 Mon Sep 17 00:00:00 2001
From: Alex Willmer <alex@moreati.org.uk>
Date: Mon, 8 Apr 2024 18:29:56 +0100
Subject: [PATCH 01/17] Begin 0.3.8dev

---
 docs/changelog.rst  | 5 +++++
 mitogen/__init__.py | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 2f8db62e..0600ca3a 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -18,6 +18,11 @@ To avail of fixes in an unreleased version, please download a ZIP file
 `directly from GitHub <https://github.com/mitogen-hq/mitogen/>`_.
 
 
+Unreleased
+----------
+
+
+
 v0.3.7 (2024-04-08)
 -------------------
 
diff --git a/mitogen/__init__.py b/mitogen/__init__.py
index d72df040..b63cd0cd 100644
--- a/mitogen/__init__.py
+++ b/mitogen/__init__.py
@@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup.
 
 
 #: Library version as a tuple.
-__version__ = (0, 3, 7)
+__version__ = (0, 3, 8, 'dev')
 
 
 #: This is :data:`False` in slave contexts. Previously it was used to prevent

From e1b2f38c8e82363ca1a0b6c92bb85d379b17d63e Mon Sep 17 00:00:00 2001
From: Alex Willmer <alex@moreati.org.uk>
Date: Mon, 8 Apr 2024 20:02:50 +0100
Subject: [PATCH 02/17] tox: Add python2 & python3 to adhoc install hint

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 69639233..fec24df3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -3,7 +3,7 @@
 #
 #   sudo add-apt-repository ppa:deadsnakes/ppa
 #   sudo apt update
-#   sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python2.7 python3.{6..13}{,-venv} python-is-python3 sshpass tox
+#   sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python{2,2.7,3} python3.{6..13}{,-venv} python-is-python3 sshpass tox
 
 # Py   A cntrllr  A target   coverage   Django     Jinja2     pip        psutil     pytest     tox        virtualenv
 # ==== ========== ========== ========== ========== ========== ========== ========== ========== ========== ==========

From c4cf0d5ba209b338dc3d77f57a2884ba2ce670cf Mon Sep 17 00:00:00 2001
From: Alex Willmer <alex@moreati.org.uk>
Date: Mon, 8 Apr 2024 20:03:53 +0100
Subject: [PATCH 03/17] ci: Use profile_tasks callback as rough benchmark of
 Ansible tests

---
 tox.ini | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tox.ini b/tox.ini
index fec24df3..55fd1842 100644
--- a/tox.ini
+++ b/tox.ini
@@ -142,6 +142,8 @@ setenv =
     distros_ubuntu2004: DISTROS=ubuntu2004
     mode_ansible: MODE=ansible
     mode_ansible: ANSIBLE_SKIP_TAGS=resource_intensive
+    mode_ansible: ANSIBLE_CALLBACK_WHITELIST=profile_tasks
+    mode_ansible: ANSIBLE_CALLBACKS_ENABLED=profile_tasks
     mode_debops_common: MODE=debops_common
     mode_localhost: ANSIBLE_SKIP_TAGS=issue_776,resource_intensive
     mode_mitogen: MODE=mitogen

From d5e918628985e497fcffec6568bb0a4a61582830 Mon Sep 17 00:00:00 2001
From: Alex Willmer <alex@moreati.org.uk>
Date: Fri, 7 Jul 2023 21:32:02 +0100
Subject: [PATCH 04/17] ansible_mitogen: Fix --ask-become-pass, add test
 coverage

Previously f1503874de82353cbed8b51408d20fdfa899f8f7 fixed the priority of
ansible_become_pass over ansible_become_password, but broke --ask-become-pass.
Fixes #952.
---
 .ci/ansible_tests.py                          | 11 +++++++++-
 ansible_mitogen/transport_config.py           | 16 ++++++++++----
 docs/changelog.rst                            |  1 +
 tests/ansible/hosts/default.hosts             |  2 ++
 .../transport_config/become_pass.yml          |  5 +++--
 tests/ansible/regression/all.yml              |  1 +
 tests/ansible/regression/become_test.yml      |  9 ++++++++
 .../regression/issue_952__ask_become_pass.yml | 21 +++++++++++++++++++
 tests/requirements.txt                        |  2 ++
 9 files changed, 61 insertions(+), 7 deletions(-)
 create mode 100644 tests/ansible/regression/become_test.yml
 create mode 100644 tests/ansible/regression/issue_952__ask_become_pass.yml

diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py
index 43d4f4cb..0dd978c4 100755
--- a/.ci/ansible_tests.py
+++ b/.ci/ansible_tests.py
@@ -6,6 +6,7 @@ import glob
 import os
 import signal
 import sys
+import textwrap
 
 import ci_lib
 
@@ -74,7 +75,15 @@ with ci_lib.Fold('job_setup'):
             fp.write('\n[%s]\n' % family)
             fp.writelines('%s\n' % name for name in hostnames)
 
-        fp.write('\n[linux:children]\ntest-targets\n')
+        fp.write(textwrap.dedent(
+            '''
+            [linux:children]
+            test-targets
+
+            [linux_containers:children]
+            test-targets
+            '''
+        ))
 
     ci_lib.dump_file(inventory_path)
 
diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py
index 5fc78185..1fc1e80a 100644
--- a/ansible_mitogen/transport_config.py
+++ b/ansible_mitogen/transport_config.py
@@ -79,7 +79,6 @@ try:
 except ImportError:
     from ansible.vars.unsafe_proxy import AnsibleUnsafeText
 
-import ansible_mitogen.loaders
 import mitogen.core
 
 
@@ -436,9 +435,18 @@ class PlayContextSpec(Spec):
         return self._play_context.become_user
 
     def become_pass(self):
-        become_method = self.become_method()
-        become_plugin = ansible_mitogen.loaders.become_loader.get(become_method)
-        become_pass = become_plugin.get_option('become_pass', hostvars=self._task_vars)
+        # become_pass is owned/provided by the active become plugin. However
+        # PlayContext is intertwined with it. Known complications
+        # - ansible_become_password is higher priority than ansible_become_pass,
+        #   `play_context.become_pass` doesn't obey this (atleast with Mitgeon).
+        # - `meta: reset_connection` runs `connection.reset()` but
+        #   `ansible_mitogen.connection.Connection.reset()` recreates the
+        #   connection object, setting `connection.become = None`.
+        become_plugin = self._connection.become
+        try:
+            become_pass = become_plugin.get_option('become_pass', playcontext=self._play_context)
+        except AttributeError:
+            become_pass = self._play_context.become_pass
         return optional_secret(become_pass)
 
     def password(self):
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 0600ca3a..bcf6d01e 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -21,6 +21,7 @@ To avail of fixes in an unreleased version, please download a ZIP file
 Unreleased
 ----------
 
+* :gh:issue:`952` Fix Ansible `--ask-become-pass`, add test coverage
 
 
 v0.3.7 (2024-04-08)
diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts
index 1bec0014..4f5ea4c6 100644
--- a/tests/ansible/hosts/default.hosts
+++ b/tests/ansible/hosts/default.hosts
@@ -10,3 +10,5 @@ target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}"
 
 [test-targets]
 target
+
+[linux_containers]
diff --git a/tests/ansible/integration/transport_config/become_pass.yml b/tests/ansible/integration/transport_config/become_pass.yml
index 8fbe7251..b9bf1a8e 100644
--- a/tests/ansible/integration/transport_config/become_pass.yml
+++ b/tests/ansible/integration/transport_config/become_pass.yml
@@ -143,8 +143,9 @@
       - out.result|length == 2
       - out.result[0].method == "ssh"
       - out.result[1].method == "sudo"
-      # Ansible >= 2.10 builtin become plugins (e.g. sudo, su) give priority
-      # to ansible_become_pass over ansible_become_password.
+      # Ansible <= 2.9.1 prioritises ansible_become_password.
+      # Ansible >= 2.9.2 prioritises ansible_become_pass.
+      # https://github.com/ansible/ansible/commit/480b106d6535978ae6ecab68b40942ca4fa914a0
       - out.result[1].kwargs.password == "bpass"
       fail_msg: out={{out}}
   tags:
diff --git a/tests/ansible/regression/all.yml b/tests/ansible/regression/all.yml
index 0e599476..477caccd 100644
--- a/tests/ansible/regression/all.yml
+++ b/tests/ansible/regression/all.yml
@@ -14,3 +14,4 @@
 - import_playbook: issue_615__streaming_transfer.yml
 - import_playbook: issue_655__wait_for_connection_error.yml
 - import_playbook: issue_776__load_plugins_called_twice.yml
+- import_playbook: issue_952__ask_become_pass.yml
diff --git a/tests/ansible/regression/become_test.yml b/tests/ansible/regression/become_test.yml
new file mode 100644
index 00000000..5af2e123
--- /dev/null
+++ b/tests/ansible/regression/become_test.yml
@@ -0,0 +1,9 @@
+- name: regression/become_test.yml
+  hosts: test-targets:&linux_containers
+  become: true
+  become_user: mitogen__pw_required
+  strategy: mitogen_linear
+  tasks:
+    - command: whoami
+      changed_when: false
+      check_mode: false
diff --git a/tests/ansible/regression/issue_952__ask_become_pass.yml b/tests/ansible/regression/issue_952__ask_become_pass.yml
new file mode 100644
index 00000000..da269e2d
--- /dev/null
+++ b/tests/ansible/regression/issue_952__ask_become_pass.yml
@@ -0,0 +1,21 @@
+- name: regression/issue_952__ask_become_pass.yml
+  hosts: test-targets[0]:&linux_containers
+  gather_facts: false
+  tags:
+    - issue_952
+  tasks:
+    - name: Test --ask-become-pass
+      delegate_to: localhost
+      expect:
+        command: >
+          ansible-playbook
+          {% for inv in ansible_inventory_sources %}
+          -i "{{ inv }}"
+          {% endfor %}
+          --ask-become-pass
+          regression/become_test.yml
+        chdir: ../
+        responses:
+          'BECOME password:': pw_required_password
+      changed_when: false
+      check_mode: false
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 6d87d177..d64d8b87 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -12,6 +12,8 @@ Django==3.2.20;     python_version >= '3.6'
 mock==3.0.5;                python_version == '2.7'
 mock==5.1.0;                python_version >= '3.6'
 
+pexpect==4.8
+
 psutil==5.9.8
 
 pytest==4.6.11;             python_version == '2.7'

From 8c93973f98698128c16f196a968211e86ff5a98b Mon Sep 17 00:00:00 2001
From: Alex Willmer <alex@moreati.org.uk>
Date: Fri, 12 Apr 2024 12:55:28 +0100
Subject: [PATCH 05/17] tests: Use Android portal to check get_url

Should have higher uptime, and make us less of a burden. Refs #1058
---
 .../interpreter_discovery/complex_args.yml         | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/tests/ansible/integration/interpreter_discovery/complex_args.yml b/tests/ansible/integration/interpreter_discovery/complex_args.yml
index 6ffff5f4..a3755ebf 100644
--- a/tests/ansible/integration/interpreter_discovery/complex_args.yml
+++ b/tests/ansible/integration/interpreter_discovery/complex_args.yml
@@ -29,20 +29,18 @@
         special_python: "source /tmp/fake || true && python{{ ansible_facts.python.version.major }}"
 
     - name: run get_url with specially-sourced python
-      get_url:
+      uri:
         # Plain http for wider Ansible & Python version compatibility
-        url: http://httpbin.org/get
-        dest: "/tmp/"
-        mode: 0644
+        url: http://www.gstatic.com/generate_204
+        status_code: [204]
       vars:
         ansible_python_interpreter: "{{ special_python }}"
 
     - name: run get_url with specially-sourced python including jinja
-      get_url:
+      uri:
         # Plain http for wider Ansible & Python version compatibility
-        url: http://httpbin.org/get
-        dest: "/tmp/"
-        mode: 0644
+        url: http://www.gstatic.com/generate_204
+        status_code: [204]
       vars:
         ansible_python_interpreter: >
           {% if "1" == "1" %}

From efdd82d1ab1d7e298de7c19b9a2508ba73d1d539 Mon Sep 17 00:00:00 2001
From: Alex Willmer <alex@moreati.org.uk>
Date: Thu, 18 Apr 2024 14:35:32 +0100
Subject: [PATCH 06/17] mitogen: Streamline Poller classes and
 Latch.poller_class selection

This
- Clarifies and corrects docstrings and comments based on investigation for #957
- Removes unused `Poller*._repr` attributes
- Eliminates some uses of `getattr()`
- Introduces `mitogen.parent.POLLERS` & `mitogen.parent.POLLER_LIGHTWEIGHT`

Preamble size change
```
@@ -1,7 +1,7 @@
 SSH command size: 759
-Bootstrap (mitogen.core) size: 17862 (17.44KiB)
+Bootstrap (mitogen.core) size: 17934 (17.51KiB)

                               Original          Minimized           Compressed
-mitogen.parent            98171 95.9KiB  50569 49.4KiB 51.5%  12742 12.4KiB 13.0%
+mitogen.parent            96979 94.7KiB  49844 48.7KiB 51.4%  12697 12.4KiB 13.1%
 mitogen.fork               8436  8.2KiB   4130  4.0KiB 49.0%   1648  1.6KiB 19.5%
 mitogen.ssh               10892 10.6KiB   6952  6.8KiB 63.8%   2113  2.1KiB 19.4%
```
---
 mitogen/core.py   | 26 +++++++++-----------------
 mitogen/parent.py | 35 +++++++++++------------------------
 2 files changed, 20 insertions(+), 41 deletions(-)

diff --git a/mitogen/core.py b/mitogen/core.py
index cd02012f..cdfbbcde 100644
--- a/mitogen/core.py
+++ b/mitogen/core.py
@@ -2523,8 +2523,7 @@ class Poller(object):
     """
     A poller manages OS file descriptors the user is waiting to become
     available for IO. The :meth:`poll` method blocks the calling thread
-    until one or more become ready. The default implementation is based on
-    :func:`select.poll`.
+    until one or more become ready.
 
     Each descriptor has an associated `data` element, which is unique for each
     readiness type, and defaults to being the same as the file descriptor. The
@@ -2546,19 +2545,13 @@ class Poller(object):
     a resource leak.
 
     Pollers may only be used by one thread at a time.
+
+    This implementation uses :func:`select.select` for wider platform support.
+    That is considered an implementation detail. Previous versions have used
+    :func:`select.poll`. Future versions may decide at runtime.
     """
     SUPPORTED = True
 
-    # This changed from select() to poll() in Mitogen 0.2.4. Since poll() has
-    # no upper FD limit, it is suitable for use with Latch, which must handle
-    # FDs larger than select's limit during many-host runs. We want this
-    # because poll() requires no setup and teardown: just a single system call,
-    # which is important because Latch.get() creates a Poller on each
-    # invocation. In a microbenchmark, poll() vs. epoll_ctl() is 30% faster in
-    # this scenario. If select() must return in future, it is important
-    # Latch.poller_class is set from parent.py to point to the industrial
-    # strength poller for the OS, otherwise Latch will fail randomly.
-
     #: Increments on every poll(). Used to version _rfds and _wfds.
     _generation = 1
 
@@ -2681,11 +2674,10 @@ class Latch(object):
 
     See :ref:`waking-sleeping-threads` for further discussion.
     """
-    #: The :class:`Poller` implementation to use for waiting. Since the poller
-    #: will be very short-lived, we prefer :class:`mitogen.parent.PollPoller`
-    #: if it is available, or :class:`mitogen.core.Poller` otherwise, since
-    #: these implementations require no system calls to create, configure or
-    #: destroy.
+    #: The :class:`Poller` implementation to use. Instances are short lived so
+    #: prefer :class:`mitogen.parent.PollPoller` if it's available, otherwise
+    #: :class:`mitogen.core.Poller`. They don't need syscalls to create,
+    #: configure, or destroy. Replaced during import of :mod:`mitogen.parent`.
     poller_class = Poller
 
     #: If not :data:`None`, a function invoked as `notify(latch)` after a
diff --git a/mitogen/parent.py b/mitogen/parent.py
index 29bcf66d..4b96dcf4 100644
--- a/mitogen/parent.py
+++ b/mitogen/parent.py
@@ -745,8 +745,7 @@ def _upgrade_broker(broker):
     broker.timers = TimerList()
     LOG.debug('upgraded %r with %r (new: %d readers, %d writers; '
               'old: %d readers, %d writers)', old, new,
-              len(new.readers), len(new.writers),
-              len(old.readers), len(old.writers))
+              len(new._rfds), len(new._wfds), len(old._rfds), len(old._wfds))
 
 
 @mitogen.core.takes_econtext
@@ -902,22 +901,18 @@ class CallSpec(object):
 class PollPoller(mitogen.core.Poller):
     """
     Poller based on the POSIX :linux:man2:`poll` interface. Not available on
-    some versions of OS X, otherwise it is the preferred poller for small FD
-    counts, as there is no setup/teardown/configuration system call overhead.
+    some Python/OS X combinations. Otherwise the preferred poller for small
+    FD counts; or if many pollers are created, used once, then closed.
+    There there is no setup/teardown/configuration system call overhead.
     """
     SUPPORTED = hasattr(select, 'poll')
-    _repr = 'PollPoller()'
+    _readmask = SUPPORTED and select.POLLIN | select.POLLHUP
 
     def __init__(self):
         super(PollPoller, self).__init__()
         self._pollobj = select.poll()
 
     # TODO: no proof we dont need writemask too
-    _readmask = (
-        getattr(select, 'POLLIN', 0) |
-        getattr(select, 'POLLHUP', 0)
-    )
-
     def _update(self, fd):
         mask = (((fd in self._rfds) and self._readmask) |
                 ((fd in self._wfds) and select.POLLOUT))
@@ -952,7 +947,6 @@ class KqueuePoller(mitogen.core.Poller):
     Poller based on the FreeBSD/Darwin :freebsd:man2:`kqueue` interface.
     """
     SUPPORTED = hasattr(select, 'kqueue')
-    _repr = 'KqueuePoller()'
 
     def __init__(self):
         super(KqueuePoller, self).__init__()
@@ -1030,7 +1024,7 @@ class EpollPoller(mitogen.core.Poller):
     Poller based on the Linux :linux:man7:`epoll` interface.
     """
     SUPPORTED = hasattr(select, 'epoll')
-    _repr = 'EpollPoller()'
+    _inmask = SUPPORTED and select.EPOLLIN | select.EPOLLHUP
 
     def __init__(self):
         super(EpollPoller, self).__init__()
@@ -1077,9 +1071,6 @@ class EpollPoller(mitogen.core.Poller):
         self._wfds.pop(fd, None)
         self._control(fd)
 
-    _inmask = (getattr(select, 'EPOLLIN', 0) |
-               getattr(select, 'EPOLLHUP', 0))
-
     def _poll(self, timeout):
         the_timeout = -1
         if timeout is not None:
@@ -1100,18 +1091,14 @@ class EpollPoller(mitogen.core.Poller):
                     yield data
 
 
-# 2.4 and 2.5 only had select.select() and select.poll().
-for _klass in mitogen.core.Poller, PollPoller, KqueuePoller, EpollPoller:
-    if _klass.SUPPORTED:
-        PREFERRED_POLLER = _klass
+POLLERS = (EpollPoller, KqueuePoller, PollPoller, mitogen.core.Poller)
+PREFERRED_POLLER = next(cls for cls in POLLERS if cls.SUPPORTED)
+
 
 # For processes that start many threads or connections, it's possible Latch
 # will also get high-numbered FDs, and so select() becomes useless there too.
-# So swap in our favourite poller.
-if PollPoller.SUPPORTED:
-    mitogen.core.Latch.poller_class = PollPoller
-else:
-    mitogen.core.Latch.poller_class = PREFERRED_POLLER
+POLLER_LIGHTWEIGHT = PollPoller.SUPPORTED and PollPoller or PREFERRED_POLLER
+mitogen.core.Latch.poller_class = POLLER_LIGHTWEIGHT
 
 
 class LineLoggingProtocolMixin(object):

From 4996ec2f09ad31a46a2b4e03cf1e765fc0e830d5 Mon Sep 17 00:00:00 2001
From: Alex Willmer <alex@moreati.org.uk>
Date: Fri, 19 Apr 2024 10:39:49 +0100
Subject: [PATCH 07/17] ansible_mitogen: Fix "filedescriptor out of range in
 select()" in WorkerProcess

`mitogen.parent.POLLER_LIGHTWEIGHT` will normally be `PollPoller`, falling
back to `EpollPoller`, `KqueuePoller`, or `Poller`.

Fixes #957

Co-authored-by: Luca Berruti <nadirio@gmail.com>
Co-authored-by: Philippe Kueck <bqobccy6ejnq2bqvmebqiwqha4cs4@protected32.unixadm.org>
---
 ansible_mitogen/process.py | 4 ++--
 docs/changelog.rst         | 2 ++
 docs/contributors.rst      | 2 ++
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py
index 63caa88a..3a41a43d 100644
--- a/ansible_mitogen/process.py
+++ b/ansible_mitogen/process.py
@@ -281,11 +281,11 @@ def get_cpu_count(default=None):
 
 class Broker(mitogen.master.Broker):
     """
-    WorkerProcess maintains at most 2 file descriptors, therefore does not need
+    WorkerProcess maintains fewer file descriptors, therefore does not need
     the exuberant syscall expense of EpollPoller, so override it and restore
     the poll() poller.
     """
-    poller_class = mitogen.core.Poller
+    poller_class = mitogen.parent.POLLER_LIGHTWEIGHT
 
 
 class Binding(object):
diff --git a/docs/changelog.rst b/docs/changelog.rst
index bcf6d01e..407a8c78 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -22,6 +22,8 @@ Unreleased
 ----------
 
 * :gh:issue:`952` Fix Ansible `--ask-become-pass`, add test coverage
+* :gh:issue:`957` Fix Ansible exception when executing against 10s of hosts
+  "ValueError: filedescriptor out of range in select()"
 
 
 v0.3.7 (2024-04-08)
diff --git a/docs/contributors.rst b/docs/contributors.rst
index 207e4d7b..ed7fef11 100644
--- a/docs/contributors.rst
+++ b/docs/contributors.rst
@@ -126,11 +126,13 @@ sponsorship and outstanding future-thinking of its early adopters.
     <li>jgadling</li>
     <li>John F Wall &mdash; <em>Making Ansible Great with Massive Parallelism</em></li>
     <li>KennethC</li>
+    <li><a href="https://github.com/lberruti">Luca Berruti</li>
     <li>Lewis Bellwood &mdash; <em>Happy to be apart of a great project.</em></li>
     <li>luto</li>
     <li><a href="https://mayeu.me/">Mayeu a.k.a Matthieu Maury</a></li>
     <li><a href="https://twitter.com/nathanhruby">@nathanhruby</a></li>
     <li><a href="https://github.com/opoplawski">Orion Poplawski</a></li>
+    <li><a href="https://github.com/philfry">Philippe Kueck</a></li>
     <li><a href="http://pageflows.com/">Ramy</a></li>
     <li>Scott Vokes</li>
     <li><a href="https://twitter.com/sirtux">Tom Eichhorn</a></li>

From ec05e542b46dcfc2a418943c105f03dcfb7c5b69 Mon Sep 17 00:00:00 2001
From: Philippe Kueck <bqobccy6ejnq2bqvmebqiwqha4cs4@protected32.unixadm.org>
Date: Wed, 22 Mar 2023 12:36:14 +0100
Subject: [PATCH 08/17] Fix 'ansible_host_key_checking' and
 'ansible_ssh_host_key_checking' for adding new hosts to the inventory using
 'add_hosts'

Co-authored-by: Alex Willmer <alex@moreati.org.uk>
---
 ansible_mitogen/connection.py       |  2 +-
 ansible_mitogen/transport_config.py | 23 +++++++++++++++++++++++
 docs/changelog.rst                  |  1 +
 3 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py
index dfc3aec4..6bdf11ba 100644
--- a/ansible_mitogen/connection.py
+++ b/ansible_mitogen/connection.py
@@ -119,7 +119,7 @@ def _connect_ssh(spec):
     """
     Return ContextService arguments for an SSH connection.
     """
-    if C.HOST_KEY_CHECKING:
+    if spec.host_key_checking():
         check_host_keys = 'enforce'
     else:
         check_host_keys = 'ignore'
diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py
index 1fc1e80a..3ab623f8 100644
--- a/ansible_mitogen/transport_config.py
+++ b/ansible_mitogen/transport_config.py
@@ -67,6 +67,7 @@ import ansible.utils.shlex
 import ansible.constants as C
 
 from ansible.module_utils.six import with_metaclass
+from ansible.module_utils.parsing.convert_bool import boolean
 
 # this was added in Ansible >= 2.8.0; fallback to the default interpreter if necessary
 try:
@@ -245,6 +246,12 @@ class Spec(with_metaclass(abc.ABCMeta, object)):
         Path to the Python interpreter on the target machine.
         """
 
+    @abc.abstractmethod
+    def host_key_checking(self):
+        """
+        Whether or not to check the keys of the target machine
+        """
+
     @abc.abstractmethod
     def private_key_file(self):
         """
@@ -466,6 +473,14 @@ class PlayContextSpec(Spec):
             action=self._action,
             rediscover_python=rediscover_python)
 
+    def host_key_checking(self):
+        def candidates():
+            yield self._connection.get_task_var('ansible_ssh_host_key_checking')
+            yield self._connection.get_task_var('ansible_host_key_checking')
+            yield C.HOST_KEY_CHECKING
+        val = next((v for v in candidates() if v is not None), True)
+        return boolean(val)
+
     def private_key_file(self):
         return self._play_context.private_key_file
 
@@ -692,6 +707,14 @@ class MitogenViaSpec(Spec):
             action=self._action,
             rediscover_python=rediscover_python)
 
+    def host_key_checking(self):
+        def candidates():
+            yield self._host_vars.get('ansible_ssh_host_key_checking')
+            yield self._host_vars.get('ansible_host_key_checking')
+            yield C.HOST_KEY_CHECKING
+        val = next((v for v in candidates() if v is not None), True)
+        return boolean(val)
+
     def private_key_file(self):
         # TODO: must come from PlayContext too.
         return (
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 407a8c78..5d77910e 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -24,6 +24,7 @@ Unreleased
 * :gh:issue:`952` Fix Ansible `--ask-become-pass`, add test coverage
 * :gh:issue:`957` Fix Ansible exception when executing against 10s of hosts
   "ValueError: filedescriptor out of range in select()"
+* :gh:issue:`1066` Support Ansible `ansible_host_key_checking` & `ansible_ssh_host_key_checking`
 
 
 v0.3.7 (2024-04-08)

From d2eefc06aa45a5f5403a551faf6d743fc6c759dc Mon Sep 17 00:00:00 2001
From: Alex Willmer <alex@moreati.org.uk>
Date: Mon, 6 May 2024 11:44:43 +0100
Subject: [PATCH 09/17] tests: Add regression for add_host with
 host_key_checking

refs #1066
---
 tests/ansible/ansible.cfg                     | 11 ++-
 tests/ansible/hosts/transport_config.hosts    |  6 ++
 .../integration/transport_config/all.yml      |  1 +
 .../transport_config/host_key_checking.yml    | 94 +++++++++++++++++++
 tests/ansible/regression/all.yml              |  1 +
 ...ssue_1066__add_host__host_key_checking.yml | 67 +++++++++++++
 tests/ansible/setup/report_controller.yml     |  4 +-
 7 files changed, 180 insertions(+), 4 deletions(-)
 create mode 100644 tests/ansible/integration/transport_config/host_key_checking.yml
 create mode 100644 tests/ansible/regression/issue_1066__add_host__host_key_checking.yml

diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg
index c34dd219..537b5059 100644
--- a/tests/ansible/ansible.cfg
+++ b/tests/ansible/ansible.cfg
@@ -1,9 +1,12 @@
 [defaults]
 any_errors_fatal = true
-# callback_whitelist naming will be deprecated in ansible-core >= 2.15.
-# callbacks_enabled naming was added in ansible-core 2.11
+# callbacks_enabled was added in Ansible 4 (ansible-core 2.11).
 # profile_tasks: Displays timing for each task and summary table of top N tasks
 # timer: Displays "Playbook run took 0 days, 0 hours, ..."
+callbacks_enabled =
+    profile_tasks,
+    timer
+# callback_whitelist was deprecated in Ansible >= 8 (ansible-core >= 2.15).
 callback_whitelist =
     profile_tasks,
     timer
@@ -37,7 +40,9 @@ no_target_syslog = True
 # Required by integration/ssh/timeouts.yml
 timeout = 30
 
-# On Travis, paramiko check fails due to host key checking enabled.
+# Ideally this would be true here and and overridden for hosts/groups. However
+# ansible_host_key_checking don't work on Vanilla Ansible 2.10, even for
+# static inventory hosts (ansible/ansible#49254, ansible/ansible#73708)
 host_key_checking = False
 
 [inventory]
diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts
index 1c1c2e10..5d5c3834 100644
--- a/tests/ansible/hosts/transport_config.hosts
+++ b/tests/ansible/hosts/transport_config.hosts
@@ -13,6 +13,7 @@ tc_become
 tc_become_method
 tc_become_pass
 tc_become_user
+tc_host_key_checking
 tc_password
 tc_port
 tc_remote_addr
@@ -74,6 +75,11 @@ tc-become-pass-password ansible_become_password=apassword
 tc-become-pass-pass ansible_become_pass=apass
 tc-become-pass-both ansible_become_pass=bpass ansible_become_password=bpassword
 
+[tc_host_key_checking]
+tc-hkc-unset
+tc-hkc-host-key-checking ansible_host_key_checking=true
+tc-hkc-ssh-host-key-checking ansible_ssh_host_key_checking=true
+
 [tc_port]
 tc-port-unset
 tc-port-explicit-port ansible_port=1234
diff --git a/tests/ansible/integration/transport_config/all.yml b/tests/ansible/integration/transport_config/all.yml
index 548e7f7e..b486549b 100644
--- a/tests/ansible/integration/transport_config/all.yml
+++ b/tests/ansible/integration/transport_config/all.yml
@@ -2,6 +2,7 @@
 - import_playbook: become_pass.yml
 - import_playbook: become_user.yml
 - import_playbook: become.yml
+- import_playbook: host_key_checking.yml
 - import_playbook: password.yml
 - import_playbook: port.yml
 - import_playbook: python_path.yml
diff --git a/tests/ansible/integration/transport_config/host_key_checking.yml b/tests/ansible/integration/transport_config/host_key_checking.yml
new file mode 100644
index 00000000..b15e36b1
--- /dev/null
+++ b/tests/ansible/integration/transport_config/host_key_checking.yml
@@ -0,0 +1,94 @@
+# Each case is followed by mitogen_via= case to test hostvars method.
+
+- name: integration/transport_config/host_key_checking.yml
+  hosts: tc-hkc-unset
+  tasks:
+    - include_tasks: ../_mitogen_only.yml
+    - {mitogen_get_stack: {}, register: out}
+    - assert:
+        that:
+          - out.result | length == 1
+          - out.result[0].method == "ssh"
+          - out.result[0].kwargs.check_host_keys == "ignore"
+        fail_msg: out={{ out }}
+  tags:
+    - mitogen_only
+
+- hosts: tc-hkc-unset
+  vars:
+    mitogen_via: tc-hkc-host-key-checking
+  tasks:
+    - include_tasks: ../_mitogen_only.yml
+    - {mitogen_get_stack: {}, register: out}
+    - assert:
+        that:
+          - out.result | length == 2
+          - out.result[0].method == "ssh"
+          - out.result[0].kwargs.check_host_keys == "enforce"
+          - out.result[1].method == "ssh"
+          - out.result[1].kwargs.check_host_keys == "ignore"
+        fail_msg: out={{ out }}
+  tags:
+    - mitogen_only
+
+
+- hosts: tc-hkc-host-key-checking
+  tasks:
+    - include_tasks: ../_mitogen_only.yml
+    - {mitogen_get_stack: {}, register: out}
+    - assert:
+        that:
+          - out.result | length == 1
+          - out.result[0].method == "ssh"
+          - out.result[0].kwargs.check_host_keys == "enforce"
+        fail_msg: out={{ out }}
+  tags:
+    - mitogen_only
+
+- hosts: tc-hkc-host-key-checking
+  vars:
+    mitogen_via: tc-hkc-unset
+  tasks:
+    - include_tasks: ../_mitogen_only.yml
+    - {mitogen_get_stack: {}, register: out}
+    - assert:
+        that:
+          - out.result | length == 2
+          - out.result[0].method == "ssh"
+          - out.result[0].kwargs.check_host_keys == "ignore"
+          - out.result[1].method == "ssh"
+          - out.result[1].kwargs.check_host_keys == "enforce"
+        fail_msg: out={{ out }}
+  tags:
+    - mitogen_only
+
+
+- hosts: tc-hkc-ssh-host-key-checking
+  tasks:
+    - include_tasks: ../_mitogen_only.yml
+    - {mitogen_get_stack: {}, register: out}
+    - assert:
+        that:
+          - out.result | length == 1
+          - out.result[0].method == "ssh"
+          - out.result[0].kwargs.check_host_keys == "enforce"
+        fail_msg: out={{ out }}
+  tags:
+    - mitogen_only
+
+- hosts: tc-hkc-ssh-host-key-checking
+  vars:
+    mitogen_via: tc-hkc-unset
+  tasks:
+    - include_tasks: ../_mitogen_only.yml
+    - {mitogen_get_stack: {}, register: out}
+    - assert:
+        that:
+          - out.result | length == 2
+          - out.result[0].method == "ssh"
+          - out.result[0].kwargs.check_host_keys == "ignore"
+          - out.result[1].method == "ssh"
+          - out.result[1].kwargs.check_host_keys == "enforce"
+        fail_msg: out={{ out }}
+  tags:
+    - mitogen_only
diff --git a/tests/ansible/regression/all.yml b/tests/ansible/regression/all.yml
index 477caccd..31541e9f 100644
--- a/tests/ansible/regression/all.yml
+++ b/tests/ansible/regression/all.yml
@@ -15,3 +15,4 @@
 - import_playbook: issue_655__wait_for_connection_error.yml
 - import_playbook: issue_776__load_plugins_called_twice.yml
 - import_playbook: issue_952__ask_become_pass.yml
+- import_playbook: issue_1066__add_host__host_key_checking.yml
diff --git a/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml b/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml
new file mode 100644
index 00000000..4b4609b6
--- /dev/null
+++ b/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml
@@ -0,0 +1,67 @@
+- name: regression/issue_1066__add_host__host_key_checking.yml
+  hosts: test-targets[0]
+  gather_facts: false
+  become: false
+  tasks:
+    - name: Add hosts dynamically
+      add_host:
+        name: "{{ item.name }}"
+        ansible_host_key_checking:        "{{ item.host_key_checking | default(omit) }}"
+        ansible_ssh_host_key_checking:    "{{ item.host_ssh_key_checking | default(omit) }}"
+        ansible_host:                     "{{ hostvars[inventory_hostname].ansible_host | default(omit) }}"
+        ansible_password:                 "{{ hostvars[inventory_hostname].ansible_password | default(omit) }}"
+        ansible_port:                     "{{ hostvars[inventory_hostname].ansible_port | default(omit) }}"
+        ansible_python_interpreter:       "{{ hostvars[inventory_hostname].ansible_python_interpreter | default(omit) }}"
+        ansible_user:                     "{{ hostvars[inventory_hostname].ansible_user | default(omit) }}"
+      loop:
+        - {name: issue-1066-host-hkc-false,   host_key_checking: false}
+        - {name: issue-1066-host-hkc-true,    host_key_checking: true}
+        - {name: issue-1066-host-hskc-false,  host_ssh_key_checking: false}
+        - {name: issue-1066-host-hskc-true,   host_ssh_key_checking: true}
+      delegate_to: localhost
+  tags:
+    - issue_1066
+
+- name: regression/issue_1066__add_host__host_key_checking.yml
+  hosts: issue-1066-host-*
+  gather_facts: false
+  become: false
+  serial: 1
+  tasks:
+    - meta: reset_connection
+
+    # The host key might be in ~/.ssh/known_hosts. If it's removed then no
+    # problem - test-targets hosts have host_key_checking=false.
+    - name: Remove existing host keys
+      known_hosts:
+        name: "{{ ansible_host }}"
+        state: absent
+      delegate_to: localhost
+
+    - name: Ping dynamically added hosts
+      ping:
+      ignore_errors: true
+      ignore_unreachable: true
+      register: issue_1066_ping
+
+    - debug:
+        var: issue_1066_ping
+
+    - name: Confirm dynamically added hosts are/are not reachable
+      vars:
+        expected:
+          issue-1066-host-hkc-false:    {}
+          issue-1066-host-hkc-true:     {unreachable: true}
+          issue-1066-host-hskc-false:   {}
+          issue-1066-host-hskc-true:    {unreachable: true}
+      assert:
+        that:
+          - issue_1066_ping.unreachable is defined == expected[inventory_hostname].unreachable is defined
+          - issue_1066_ping.unreachable | default(42) == expected[inventory_hostname].unreachable | default(42)
+      # ansible_host_key_checking don't work on Vanilla Ansible 2.10, even for
+      # static inventory hosts (ansible/ansible#49254, ansible/ansible#73708).
+      when:
+        - ansible_version.full is version('2.11', '>=', strict=True)
+          or is_mitogen
+  tags:
+    - issue_1066
diff --git a/tests/ansible/setup/report_controller.yml b/tests/ansible/setup/report_controller.yml
index d0d5cc15..5e34fb17 100644
--- a/tests/ansible/setup/report_controller.yml
+++ b/tests/ansible/setup/report_controller.yml
@@ -1,5 +1,5 @@
 - name: Report controller parameters
-  hosts: localhost
+  hosts: test-targets[0]
   gather_facts: false
   tasks:
     - debug:
@@ -9,9 +9,11 @@
           - $(groups):                "{{ lookup('pipe', 'groups') }}"
           - $(pwd):                   "{{ lookup('pipe', 'pwd') }}"
           - $(whoami):                "{{ lookup('pipe', 'whoami') }}"
+          - ansible_inventory_sources:        "{{ ansible_inventory_sources | default('<unset>') }}"
           - ansible_run_tags:         "{{ ansible_run_tags | default('<unset>') }}"
           - ansible_playbook_python:  "{{ ansible_playbook_python | default('<unset>') }}"
           - ansible_skip_tags:        "{{ ansible_skip_tags | default('<unset>') }}"
           - ansible_version.full:     "{{ ansible_version.full | default('<unset>') }}"
           - is_mitogen:               "{{ is_mitogen | default('<unset>') }}"
           - playbook_dir:             "{{ playbook_dir | default('<unset>') }}"
+      delegate_to: localhost

From 60f868290de1127271e174fc7b6a53c465aabf7d Mon Sep 17 00:00:00 2001
From: Alex Willmer <alex@moreati.org.uk>
Date: Mon, 6 May 2024 13:19:33 +0100
Subject: [PATCH 10/17] tests: Remove --limit when running Ansible localhost CI

Some tests were being incorrectly excluded. Including those that use
`add_host`.
refs #1066, #1069
---
 .ci/localhost_ansible_tests.py                         |  2 +-
 .../issue_655__wait_for_connection_error.yml           | 10 +++++++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py
index c50ef220..c36bad26 100755
--- a/.ci/localhost_ansible_tests.py
+++ b/.ci/localhost_ansible_tests.py
@@ -93,5 +93,5 @@ with ci_lib.Fold('machine_prep'):
 with ci_lib.Fold('ansible'):
     os.chdir(TESTS_DIR)
     playbook = os.environ.get('PLAYBOOK', 'all.yml')
-    ci_lib.run('./run_ansible_playbook.py %s -l target %s',
+    ci_lib.run('./run_ansible_playbook.py %s %s',
         playbook, ' '.join(sys.argv[1:]))
diff --git a/tests/ansible/regression/issue_655__wait_for_connection_error.yml b/tests/ansible/regression/issue_655__wait_for_connection_error.yml
index fbdb9f2b..9ad42a10 100644
--- a/tests/ansible/regression/issue_655__wait_for_connection_error.yml
+++ b/tests/ansible/regression/issue_655__wait_for_connection_error.yml
@@ -4,11 +4,19 @@
 #   since things are ran on localhost; Azure DevOps loses connection and fails
 # TODO: do we want to install docker a different way to be able to do this for other tests too
 ---
-- name: regression/issue_655_wait_for_connection_error.yml
+- name: regression/issue_655__wait_for_connection_error.yml
   hosts: localhost
   gather_facts: yes
   become: no
   tasks:
+    - meta: end_play
+      when:
+        # TODO CI currently runs on macOS 11 images in Azure DevOps. MacOS 11
+        #      is no longer supported by homebrew, so the following install
+        #      task fails.
+        - ansible_facts.system == 'Darwin'
+        - ansible_facts.distribution_major_version == '11'
+
     - name: set up test container and run tests inside it
       block:
         - name: install deps

From 05d98e5b499bed231edda8826d71bd719cc7d831 Mon Sep 17 00:00:00 2001
From: Alex Willmer <alex@moreati.org.uk>
Date: Tue, 14 May 2024 10:53:30 +0100
Subject: [PATCH 11/17] tests: Speed up ssh timeout tests

---
 docs/ansible_detailed.rst                   | 9 +++++++++
 tests/ansible/integration/ssh/timeouts.yml  | 6 ++++++
 tests/ansible/integration/ssh/variables.yml | 4 ++++
 3 files changed, 19 insertions(+)

diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst
index b395bc25..e6f7b42c 100644
--- a/docs/ansible_detailed.rst
+++ b/docs/ansible_detailed.rst
@@ -247,6 +247,15 @@ Noteworthy Differences
   part of the core library, and should therefore be straightforward to fix as
   part of 0.2.x.
 
+* Connection and become timeouts are applied differently. Mitogen may consider
+  a connection to have timed out, when Ansible would have waited longer or
+  indefinately. For example if SSH authentication completes within the
+  timeout, but execution of login scripts exceeds it - then Mitogen will
+  consider the task to have timed out and that host to have failed.
+
+..
+    tests/ansible/integration/ssh/timeouts.yml covers (some of) this behaviour.
+
 ..
     * SSH and ``become`` are treated distinctly when applying timeouts, and
     timeouts apply up to the point when the new interpreter is ready to accept
diff --git a/tests/ansible/integration/ssh/timeouts.yml b/tests/ansible/integration/ssh/timeouts.yml
index a7691bc3..2b14b9f3 100644
--- a/tests/ansible/integration/ssh/timeouts.yml
+++ b/tests/ansible/integration/ssh/timeouts.yml
@@ -1,7 +1,12 @@
 # Ensure 'ssh' connections time out correctly.
+# mitogen__slow_user performs a long sleep in ~/.profile.
+# Mitogen counts this time towards the connection timeout. Ansible doesn't.
+# ansible_python_interpreter=python3000 is an optimisation, to avoid waiting
+# on the timeout multiple times (e.g. interpreter discovery).
 
 - name: integration/ssh/timeouts.yml
   hosts: test-targets
+  gather_facts: false
   tasks:
     - include_tasks: ../_mitogen_only.yml
 
@@ -17,6 +22,7 @@
         test-targets
         -m custom_python_detect_environment
         -e ansible_user=mitogen__slow_user -e ansible_password=slow_user_password
+        -e ansible_python_interpreter=python3000
       args:
         chdir: ../..
       register: out
diff --git a/tests/ansible/integration/ssh/variables.yml b/tests/ansible/integration/ssh/variables.yml
index f51509db..f9c0f911 100644
--- a/tests/ansible/integration/ssh/variables.yml
+++ b/tests/ansible/integration/ssh/variables.yml
@@ -43,6 +43,7 @@
         test-targets
         -e ansible_ssh_user=mitogen__has_sudo
         -e ansible_ssh_pass=wrong_password
+        -e ansible_python_interpreter=python3000
       args:
         chdir: ../..
       register: out
@@ -82,6 +83,7 @@
         test-targets
         -e ansible_user=mitogen__has_sudo
         -e ansible_ssh_pass=wrong_password
+        -e ansible_python_interpreter=python3000
       args:
         chdir: ../..
       register: out
@@ -121,6 +123,7 @@
         test-targets
         -e ansible_user=mitogen__has_sudo
         -e ansible_password=wrong_password
+        -e ansible_python_interpreter=python3000
       args:
         chdir: ../..
       register: out
@@ -165,6 +168,7 @@
         test-targets
         -e ansible_user=mitogen__has_sudo
         -e ansible_ssh_private_key_file=/dev/null
+        -e ansible_python_interpreter=python3000
       args:
         chdir: ../..
       register: out

From 65c8a42c13b32729c2ef52b951e3eb8b7c314f7b Mon Sep 17 00:00:00 2001
From: Alex Willmer <alex@moreati.org.uk>
Date: Tue, 14 May 2024 11:30:23 +0100
Subject: [PATCH 12/17] tests: Use same verbosity when re-executing Ansible
 inside a playbook

---
 .../integration/connection/disconnect_during_module.yml   | 2 ++
 tests/ansible/integration/process/unix_socket_cleanup.yml | 1 +
 tests/ansible/integration/runner/missing_module.yml       | 4 +++-
 tests/ansible/integration/ssh/timeouts.yml                | 3 ++-
 tests/ansible/integration/ssh/variables.yml               | 8 ++++++++
 .../integration/strategy/mixed_vanilla_mitogen.yml        | 6 ++++--
 tests/ansible/integration/stub_connections/setns_lxc.yml  | 2 ++
 tests/ansible/integration/stub_connections/setns_lxd.yml  | 2 ++
 tests/ansible/regression/issue_952__ask_become_pass.yml   | 2 ++
 9 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/tests/ansible/integration/connection/disconnect_during_module.yml b/tests/ansible/integration/connection/disconnect_during_module.yml
index 926291df..5aca75d0 100644
--- a/tests/ansible/integration/connection/disconnect_during_module.yml
+++ b/tests/ansible/integration/connection/disconnect_during_module.yml
@@ -10,6 +10,8 @@
 
   - name: Run _disconnect_during_module.yml
     delegate_to: localhost
+    environment:
+      ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
     command: |
       ansible-playbook
       {% for inv in ansible_inventory_sources %}
diff --git a/tests/ansible/integration/process/unix_socket_cleanup.yml b/tests/ansible/integration/process/unix_socket_cleanup.yml
index b5d40b7d..eb6720d3 100644
--- a/tests/ansible/integration/process/unix_socket_cleanup.yml
+++ b/tests/ansible/integration/process/unix_socket_cleanup.yml
@@ -10,6 +10,7 @@
   - shell: >
       ANSIBLE_STRATEGY=mitogen_linear
       ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
+      ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
       ansible -m shell -c local -a whoami
       {% for inv in ansible_inventory_sources %}
       -i "{{ inv }}"
diff --git a/tests/ansible/integration/runner/missing_module.yml b/tests/ansible/integration/runner/missing_module.yml
index 979cdf21..4aa7b634 100644
--- a/tests/ansible/integration/runner/missing_module.yml
+++ b/tests/ansible/integration/runner/missing_module.yml
@@ -5,8 +5,10 @@
   tasks:
     - name: Run missing_module
       connection: local
+      environment:
+        ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
       command: |
-        ansible -vvv
+        ansible
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
         {% endfor %}
diff --git a/tests/ansible/integration/ssh/timeouts.yml b/tests/ansible/integration/ssh/timeouts.yml
index 2b14b9f3..aed965a8 100644
--- a/tests/ansible/integration/ssh/timeouts.yml
+++ b/tests/ansible/integration/ssh/timeouts.yml
@@ -14,8 +14,9 @@
       connection: local
       environment:
         ANSIBLE_SSH_TIMEOUT: 10
+        ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
       command: |
-        ansible -vvv
+        ansible
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
         {% endfor %}
diff --git a/tests/ansible/integration/ssh/variables.yml b/tests/ansible/integration/ssh/variables.yml
index f9c0f911..d2fa683b 100644
--- a/tests/ansible/integration/ssh/variables.yml
+++ b/tests/ansible/integration/ssh/variables.yml
@@ -20,6 +20,7 @@
         ANSIBLE_ANY_ERRORS_FATAL=false
         ANSIBLE_STRATEGY=mitogen_linear
         ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
+        ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
         ansible -m shell -a whoami
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
@@ -36,6 +37,7 @@
         ANSIBLE_ANY_ERRORS_FATAL=false
         ANSIBLE_STRATEGY=mitogen_linear
         ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
+        ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
         ansible -m shell -a whoami
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
@@ -60,6 +62,7 @@
         ANSIBLE_ANY_ERRORS_FATAL=false
         ANSIBLE_STRATEGY=mitogen_linear
         ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
+        ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
         ansible -m shell -a whoami
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
@@ -76,6 +79,7 @@
         ANSIBLE_ANY_ERRORS_FATAL=false
         ANSIBLE_STRATEGY=mitogen_linear
         ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
+        ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
         ansible -m shell -a whoami
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
@@ -100,6 +104,7 @@
         ANSIBLE_ANY_ERRORS_FATAL=false
         ANSIBLE_STRATEGY=mitogen_linear
         ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
+        ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
         ansible -m shell -a whoami
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
@@ -116,6 +121,7 @@
         ANSIBLE_ANY_ERRORS_FATAL=false
         ANSIBLE_STRATEGY=mitogen_linear
         ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
+        ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
         ansible -m shell -a whoami
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
@@ -145,6 +151,7 @@
         ANSIBLE_ANY_ERRORS_FATAL=false
         ANSIBLE_STRATEGY=mitogen_linear
         ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
+        ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
         ansible -m shell -a whoami
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
@@ -161,6 +168,7 @@
         ANSIBLE_ANY_ERRORS_FATAL=false
         ANSIBLE_STRATEGY=mitogen_linear
         ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
+        ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
         ansible -m shell -a whoami
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
diff --git a/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml b/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml
index d183564f..4220ed4c 100644
--- a/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml
+++ b/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml
@@ -3,24 +3,26 @@
   hosts: test-targets[0]
   tasks:
     - connection: local
+      environment:
+        ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
       command: |
         ansible-playbook
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
         {% endfor %}
-        -vvv
         integration/strategy/_mixed_mitogen_vanilla.yml
       args:
         chdir: ../..
       register: out
 
     - connection: local
+      environment:
+        ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
       command: |
         ansible-playbook
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
         {% endfor %}
-        -vvv
         integration/strategy/_mixed_vanilla_mitogen.yml
       args:
         chdir: ../..
diff --git a/tests/ansible/integration/stub_connections/setns_lxc.yml b/tests/ansible/integration/stub_connections/setns_lxc.yml
index 889a15e9..f8d6bfc5 100644
--- a/tests/ansible/integration/stub_connections/setns_lxc.yml
+++ b/tests/ansible/integration/stub_connections/setns_lxc.yml
@@ -12,6 +12,8 @@
     - include_tasks: _end_play_if_not_sudo_linux.yml
 
     - name: Run stub-lxc-info.py
+      environment:
+        ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
       command: |
         sudo -nE "{{lookup('env', 'VIRTUAL_ENV')}}/bin/ansible"
         -i localhost,
diff --git a/tests/ansible/integration/stub_connections/setns_lxd.yml b/tests/ansible/integration/stub_connections/setns_lxd.yml
index 81a52dbb..5aba96c5 100644
--- a/tests/ansible/integration/stub_connections/setns_lxd.yml
+++ b/tests/ansible/integration/stub_connections/setns_lxd.yml
@@ -12,6 +12,8 @@
     - include_tasks: _end_play_if_not_sudo_linux.yml
 
     - name: Run ansible stub-lxc.py
+      environment:
+        ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
       command: |
         sudo -nE "{{lookup('env', 'VIRTUAL_ENV')}}/bin/ansible"
         -i localhost,
diff --git a/tests/ansible/regression/issue_952__ask_become_pass.yml b/tests/ansible/regression/issue_952__ask_become_pass.yml
index da269e2d..a0b92ff2 100644
--- a/tests/ansible/regression/issue_952__ask_become_pass.yml
+++ b/tests/ansible/regression/issue_952__ask_become_pass.yml
@@ -6,6 +6,8 @@
   tasks:
     - name: Test --ask-become-pass
       delegate_to: localhost
+      environment:
+        ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
       expect:
         command: >
           ansible-playbook

From 7079a07a1527f32ff94cb56ef83c70af4e513bde Mon Sep 17 00:00:00 2001
From: Alex Willmer <alex@moreati.org.uk>
Date: Tue, 14 May 2024 20:08:29 +0100
Subject: [PATCH 13/17] tests: Fix duplicate local task executions

integration/ssh/timeouts.yml is noteworthy. It was an accidental N**2 in time
-  executing num hosts * num hosts tasks.
---
 tests/ansible/bench/file_transfer.yml         | 41 ++++++++++++++++-
 tests/ansible/integration/action/copy.yml     |  2 +
 .../integration/action/fixup_perms2__copy.yml |  1 +
 .../integration/action/synchronize.yml        |  3 ++
 .../integration/connection/_cleanup_file.yml  | 17 +++++++
 .../integration/connection/_put_file.yml      |  9 ++--
 .../integration/connection/put_large_file.yml |  4 +-
 .../integration/connection/put_small_file.yml |  4 +-
 .../playbook_semantics/delegate_to.yml        | 44 +++++++++++++------
 .../integration/runner/missing_module.yml     |  3 +-
 tests/ansible/integration/ssh/timeouts.yml    |  6 ++-
 .../regression/issue_140__thread_pileup.yml   | 16 ++++++-
 ...ue_152__local_action_wrong_interpreter.yml |  3 +-
 .../issue_615__streaming_transfer.yml         |  8 ++--
 14 files changed, 130 insertions(+), 31 deletions(-)
 create mode 100644 tests/ansible/integration/connection/_cleanup_file.yml

diff --git a/tests/ansible/bench/file_transfer.yml b/tests/ansible/bench/file_transfer.yml
index 09534af5..1c8dce79 100644
--- a/tests/ansible/bench/file_transfer.yml
+++ b/tests/ansible/bench/file_transfer.yml
@@ -1,14 +1,16 @@
-
 - name: bench/file_transfer.yml
   hosts: test-targets
   tasks:
-
   - name: Make 32MiB file
     delegate_to: localhost
+    run_once: true
     shell: openssl rand 33554432 > /tmp/bigfile.in
+    args:
+      creates: /tmp/bigfile.in
 
   - name: Make 320MiB file
     delegate_to: localhost
+    run_once: true
     shell: >
       cat
       /tmp/bigfile.in
@@ -22,6 +24,8 @@
       /tmp/bigfile.in
       /tmp/bigfile.in
       > /tmp/bigbigfile.in
+    args:
+      creates: /tmp/bigbigfile.in
 
   - name: Delete SSH file is present.
     file:
@@ -36,17 +40,20 @@
     copy:
       src: /tmp/bigfile.in
       dest: /tmp/bigfile.out
+      mode: ugo=rw
 
   - name: Copy 320MiB file via SSH
     copy:
       src: /tmp/bigbigfile.in
       dest: /tmp/bigbigfile.out
+      mode: ugo=rw
 
   - name: Delete localhost sudo file if present.
     file:
       path: "{{item}}"
       state: absent
     delegate_to: localhost
+    run_once: true
     become: true
     with_items:
     - /tmp/bigfile.out
@@ -56,21 +63,51 @@
 
   - name: Copy 32MiB file via localhost sudo
     delegate_to: localhost
+    run_once: true
     become: true
     copy:
       src: /tmp/bigfile.in
       dest: /tmp/bigfile.out
+      mode: ugo=rw
     tags:
       - requires_local_sudo
 
   - name: Copy 320MiB file via localhost sudo
     delegate_to: localhost
+    run_once: true
     become: true
     copy:
       src: /tmp/bigbigfile.in
       dest: /tmp/bigbigfile.out
+      mode: ugo=rw
     tags:
       - requires_local_sudo
 
+  - name: Local cleanup
+    file:
+      path: "{{ item.path }}"
+      state: absent
+    loop:
+      - /tmp/bigfile.in
+      - /tmp/bigfile.out
+      - /tmp/bigbigfile.in
+      - /tmp/bigbigfile.out
+    delegate_to: localhost
+    run_once: true
+    tags:
+      - cleanup_local
+      - cleanup
+
+  - name: Target cleanup
+    file:
+      path: "{{ item.path }}"
+      state: absent
+    loop:
+      - /tmp/bigfile.out
+      - /tmp/bigbigfile.out
+    tags:
+      - cleanup_target
+      - cleanup
+
   tags:
   - resource_intensive
diff --git a/tests/ansible/integration/action/copy.yml b/tests/ansible/integration/action/copy.yml
index 5dadff9a..73f3bd1e 100644
--- a/tests/ansible/integration/action/copy.yml
+++ b/tests/ansible/integration/action/copy.yml
@@ -9,6 +9,7 @@
         content:
           this is a tiny file.
       delegate_to: localhost
+      run_once: true
 
     - name: Create large file
       copy:
@@ -16,6 +17,7 @@
         # Must be larger than Connection.SMALL_SIZE_LIMIT.
         content: "{% for x in range(200000) %}x{% endfor %}"
       delegate_to: localhost
+      run_once: true
 
     - name: Cleanup copied files
       file:
diff --git a/tests/ansible/integration/action/fixup_perms2__copy.yml b/tests/ansible/integration/action/fixup_perms2__copy.yml
index fa4765f8..662247c6 100644
--- a/tests/ansible/integration/action/fixup_perms2__copy.yml
+++ b/tests/ansible/integration/action/fixup_perms2__copy.yml
@@ -38,6 +38,7 @@
 
     - name: Create local weird mode file
       delegate_to: localhost
+      run_once: true
       copy:
         content: "weird mode"
         dest: "/tmp/weird-mode"
diff --git a/tests/ansible/integration/action/synchronize.yml b/tests/ansible/integration/action/synchronize.yml
index aa87deaf..13c2113a 100644
--- a/tests/ansible/integration/action/synchronize.yml
+++ b/tests/ansible/integration/action/synchronize.yml
@@ -24,18 +24,21 @@
         path: /tmp/sync-test
         state: absent
       delegate_to: localhost
+      run_once: true
 
     - name: Create sync-test
       file:
         path: /tmp/sync-test
         state: directory
       delegate_to: localhost
+      run_once: true
 
     - name: Create syn-test item
       copy:
         dest: /tmp/sync-test/item
         content: "item!"
       delegate_to: localhost
+      run_once: true
 
     # TODO: https://github.com/dw/mitogen/issues/692
     # - file:
diff --git a/tests/ansible/integration/connection/_cleanup_file.yml b/tests/ansible/integration/connection/_cleanup_file.yml
new file mode 100644
index 00000000..c1b86255
--- /dev/null
+++ b/tests/ansible/integration/connection/_cleanup_file.yml
@@ -0,0 +1,17 @@
+- name: Cleanup local file
+  file:
+    path: /tmp/{{ file_name }}
+    state: absent
+  delegate_to: localhost
+  run_once: true
+  tags:
+    - cleanup_local
+    - cleanup
+
+- name: Cleanup target file
+  file:
+    path: /tmp/{{ file_name }}.out
+    state: absent
+  tags:
+    - cleanup_target
+    - cleanup
diff --git a/tests/ansible/integration/connection/_put_file.yml b/tests/ansible/integration/connection/_put_file.yml
index fb5c6014..07837db8 100644
--- a/tests/ansible/integration/connection/_put_file.yml
+++ b/tests/ansible/integration/connection/_put_file.yml
@@ -1,20 +1,21 @@
----
-
 - name: Create {{ file_name }}
-  shell: dd if=/dev/urandom of=/tmp/{{ file_name }} bs=1024 count={{ file_size }}
-  args:
+  command:
+    cmd: dd if=/dev/urandom of=/tmp/{{ file_name }} bs=1024 count={{ file_size_kib }}
     creates: /tmp/{{file_name}}
   delegate_to: localhost
+  run_once: true
 
 - name: Copy {{ file_name }}
   copy:
     dest: /tmp/{{file_name}}.out
     src: /tmp/{{file_name}}
+    mode: "{{ file_mode }}"
 
 - name: Stat created {{ file_name }}
   stat: path=/tmp/{{ file_name }}
   register: original
   delegate_to: localhost
+  run_once: true
 
 - name: Stat copied {{ file_name }}
   stat: path=/tmp/{{ file_name }}.out
diff --git a/tests/ansible/integration/connection/put_large_file.yml b/tests/ansible/integration/connection/put_large_file.yml
index c70e033d..e6f5f645 100644
--- a/tests/ansible/integration/connection/put_large_file.yml
+++ b/tests/ansible/integration/connection/put_large_file.yml
@@ -6,9 +6,11 @@
   gather_facts: no
   vars:
     file_name: large-file
-    file_size: 512
+    file_size_kib: 512
+    file_mode: u=rw,go=
   tasks:
     - include_tasks: _put_file.yml
+    - include_tasks: _cleanup_file.yml
   tags:
     - put_file
     - put_large_file
diff --git a/tests/ansible/integration/connection/put_small_file.yml b/tests/ansible/integration/connection/put_small_file.yml
index 2dc100a1..06c41673 100644
--- a/tests/ansible/integration/connection/put_small_file.yml
+++ b/tests/ansible/integration/connection/put_small_file.yml
@@ -6,9 +6,11 @@
   gather_facts: no
   vars:
     file_name: small-file
-    file_size: 123
+    file_size_kib: 123
+    file_mode: u=rw,go=
   tasks:
     - include_tasks: _put_file.yml
+    - include_tasks: _cleanup_file.yml
   tags:
     - put_file
     - put_small_file
diff --git a/tests/ansible/integration/playbook_semantics/delegate_to.yml b/tests/ansible/integration/playbook_semantics/delegate_to.yml
index c8d9e607..61b4d03d 100644
--- a/tests/ansible/integration/playbook_semantics/delegate_to.yml
+++ b/tests/ansible/integration/playbook_semantics/delegate_to.yml
@@ -1,23 +1,27 @@
 - name: integration/playbook_semantics/delegate_to.yml
   hosts: test-targets
+  vars:
+    local_path: "/tmp/delegate_to.{{ inventory_hostname }}.txt"
   tasks:
     #
     # delegate_to, no sudo
     #
     - name: "delegate_to, no sudo"
       copy:
-        dest: /tmp/delegate_to.yml.txt
+        dest: "{{ local_path }}"
         content: "Hello, world."
-      register: out
+        mode: u=rw,go=r
       delegate_to: localhost
 
     - name: "delegate_to, no sudo"
       assert:
-        that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'Hello, world.'"
+        that:
+          - lookup('file', local_path) == 'Hello, world.'
+        fail_msg: "{{ lookup('file', local_path) }}"
 
     - name: "delegate_to, no sudo"
       file:
-        path: /tmp/delegate_to.yml.txt
+        path: "{{ local_path }}"
         state: absent
       delegate_to: localhost
 
@@ -27,18 +31,20 @@
     #
     - name: "connection:local, no sudo"
       copy:
-        dest: /tmp/delegate_to.yml.txt
+        dest: "{{ local_path }}"
         content: "Hello, world."
-      register: out
+        mode: u=rw,go=r
       connection: local
 
     - name: "connection:local, no sudo"
       assert:
-        that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'Hello, world.'"
+        that:
+          - lookup('file', local_path) == 'Hello, world.'
+        fail_msg: "{{ lookup('file', local_path) }}"
 
     - name: "connection:local, no sudo"
       file:
-        path: /tmp/delegate_to.yml.txt
+        path: "{{ local_path }}"
         state: absent
       connection: local
 
@@ -47,7 +53,10 @@
     # delegate_to, sudo
     #
     - name: "delegate_to, sudo"
-      shell: whoami > /tmp/delegate_to.yml.txt
+      shell: |
+        whoami > "{{ local_path }}"
+      args:
+        creates: "{{ local_path }}"
       delegate_to: localhost
       become: true
       tags:
@@ -55,13 +64,15 @@
 
     - name: "delegate_to, sudo"
       assert:
-        that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'root'"
+        that:
+          - lookup('file', local_path) == 'root'
+        fail_msg: "{{ lookup('file', local_path) }}"
       tags:
         - requires_local_sudo
 
     - name: "delegate_to, sudo"
       file:
-        path: /tmp/delegate_to.yml.txt
+        path: "{{ local_path }}"
         state: absent
       delegate_to: localhost
       become: true
@@ -73,7 +84,10 @@
     # connection:local, sudo
     #
     - name: "connection:local, sudo"
-      shell: whoami > /tmp/delegate_to.yml.txt
+      shell: |
+        whoami > "{{ local_path }}"
+      args:
+        creates: "{{ local_path }}"
       connection: local
       become: true
       tags:
@@ -81,13 +95,15 @@
 
     - name: "connection:local, sudo"
       assert:
-        that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'root'"
+        that:
+          - lookup('file', local_path) == 'root'
+        fail_msg: "{{ lookup('file', local_path) }}"
       tags:
         - requires_local_sudo
 
     - name: "connection:local, sudo"
       file:
-        path: /tmp/delegate_to.yml.txt
+        path: "{{ local_path }}"
         state: absent
       connection: local
       become: true
diff --git a/tests/ansible/integration/runner/missing_module.yml b/tests/ansible/integration/runner/missing_module.yml
index 4aa7b634..d7261463 100644
--- a/tests/ansible/integration/runner/missing_module.yml
+++ b/tests/ansible/integration/runner/missing_module.yml
@@ -1,4 +1,3 @@
-
 - name: integration/runner/missing_module.yml
   hosts: test-targets[0]
   connection: local
@@ -17,6 +16,8 @@
       args:
         chdir: ../..
       register: out
+      changed_when: false
+      check_mode: false
       ignore_errors: true
 
     - assert:
diff --git a/tests/ansible/integration/ssh/timeouts.yml b/tests/ansible/integration/ssh/timeouts.yml
index aed965a8..09ac0006 100644
--- a/tests/ansible/integration/ssh/timeouts.yml
+++ b/tests/ansible/integration/ssh/timeouts.yml
@@ -20,13 +20,15 @@
         {% for inv in ansible_inventory_sources %}
         -i "{{ inv }}"
         {% endfor %}
-        test-targets
-        -m custom_python_detect_environment
+        "{{ inventory_hostname }}"
+        -m ping
         -e ansible_user=mitogen__slow_user -e ansible_password=slow_user_password
         -e ansible_python_interpreter=python3000
       args:
         chdir: ../..
       register: out
+      changed_when: false
+      check_mode: false
       ignore_errors: true
 
     - name: Verify connection timeout occurred
diff --git a/tests/ansible/regression/issue_140__thread_pileup.yml b/tests/ansible/regression/issue_140__thread_pileup.yml
index 1f614634..163440a5 100644
--- a/tests/ansible/regression/issue_140__thread_pileup.yml
+++ b/tests/ansible/regression/issue_140__thread_pileup.yml
@@ -8,6 +8,7 @@
 
   - name: Create file tree
     connection: local
+    run_once: true
     shell: >
       mkdir /tmp/filetree.in;
       seq -f /tmp/filetree.in/%g 1 1000 | xargs touch;
@@ -21,17 +22,30 @@
     file:
       state: directory
       path: /tmp/filetree.out
+      mode: u=rwx,go=rx
 
   - name: Trigger nasty process pileup
     copy:
       src: "{{item.src}}"
       dest: "/tmp/filetree.out/{{item.path}}"
-      mode: 0644
+      mode: u=rw,go=r
     with_filetree: /tmp/filetree.in
     when: item.state == 'file'
     loop_control:
       label: "/tmp/filetree.out/{{ item.path }}"
 
+  - name: Cleanup local file tree
+    connection: local
+    run_once: true
+    file:
+      path: /tmp/filetree.in
+      state: absent
+
+  - name: Cleanup remote file tree
+    file:
+      path: /tmp/filetree.out
+      state: absent
+
   tags:
   - resource_intensive
   - issue_140
diff --git a/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml b/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml
index fe7f5f42..dd86775d 100644
--- a/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml
+++ b/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml
@@ -4,10 +4,9 @@
 # can test for.
 
 - name: regression/issue_152__local_action_wrong_interpreter.yml
-  hosts: test-targets
+  hosts: test-targets[0]
   connection: local
   tasks:
-
     - name: Create /tmp/issue_152_interpreter.sh
       copy:
         dest: /tmp/issue_152_interpreter.sh
diff --git a/tests/ansible/regression/issue_615__streaming_transfer.yml b/tests/ansible/regression/issue_615__streaming_transfer.yml
index 43aa1343..6fe52d55 100644
--- a/tests/ansible/regression/issue_615__streaming_transfer.yml
+++ b/tests/ansible/regression/issue_615__streaming_transfer.yml
@@ -14,11 +14,14 @@
           shell: |
             dd if=/dev/zero of=/tmp/512mb.zero bs=1048576 count=512;
             chmod go= /tmp/512mb.zero
+          args:
+            creates: /tmp/512mb.zero
 
         - name: Fetch /tmp/512mb.zero
           fetch:
             src: /tmp/512mb.zero
-            dest: /tmp/fetch-out
+            dest: /tmp/fetch-{{ inventory_hostname }}-512mb.zero
+            flat: true
 
         - name: Cleanup /tmp/512mb.zero
           file:
@@ -27,11 +30,10 @@
 
         - name: Cleanup fetched file
           file:
-            path: /tmp/fetch-out
+            path: /tmp/fetch-{{ inventory_hostname }}-512mb.zero
             state: absent
           become: false
           delegate_to: localhost
-          run_once: true
   tags:
     - issue_615
     - mitogen_only

From f5a88406685923158cee238ca9a6217fb6b4b630 Mon Sep 17 00:00:00 2001
From: Alex Willmer <alex@moreati.org.uk>
Date: Fri, 26 Jul 2024 14:58:21 +0100
Subject: [PATCH 14/17] CI: Use archived RPMs on CentOS 8

CentOS 8 has reached EOL. Packages are no longer mirrored or maintained. A
historic snapshot of the packages is kept on vault.centos.org.

refs #1088, #1090
---
 tests/ansible/hosts/group_vars/all.yml        |  1 +
 tests/ansible/hosts/group_vars/centos8.yml    | 26 +++++++++++++++++++
 tests/ansible/hosts/group_vars/debian9.yml    |  4 +++
 .../issue_776__load_plugins_called_twice.yml  | 19 +++++---------
 4 files changed, 37 insertions(+), 13 deletions(-)
 create mode 100644 tests/ansible/hosts/group_vars/debian9.yml

diff --git a/tests/ansible/hosts/group_vars/all.yml b/tests/ansible/hosts/group_vars/all.yml
index cea46113..ad4adabb 100644
--- a/tests/ansible/hosts/group_vars/all.yml
+++ b/tests/ansible/hosts/group_vars/all.yml
@@ -1,2 +1,3 @@
 ---
 pkg_mgr_python_interpreter: python
+pkg_repos_overrides: []
diff --git a/tests/ansible/hosts/group_vars/centos8.yml b/tests/ansible/hosts/group_vars/centos8.yml
index 7b9e34f6..c90dd5f4 100644
--- a/tests/ansible/hosts/group_vars/centos8.yml
+++ b/tests/ansible/hosts/group_vars/centos8.yml
@@ -1,2 +1,28 @@
 ---
 pkg_mgr_python_interpreter: /usr/libexec/platform-python
+
+pkg_repos_overrides:
+  - dest: /etc/yum.repos.d/CentOS-Linux-AppStream.repo
+    content: |
+      [appstream]
+      name=CentOS Linux $releasever - AppStream
+      baseurl=http://vault.centos.org/$contentdir/$releasever/AppStream/$basearch/os/
+      enabled=1
+      gpgcheck=1
+      gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial
+  - dest: /etc/yum.repos.d/CentOS-Linux-BaseOS.repo
+    content: |
+      [baseos]
+      name=CentOS Linux $releasever - BaseOS
+      baseurl=http://vault.centos.org/$contentdir/$releasever/BaseOS/$basearch/os/
+      enabled=1
+      gpgcheck=1
+      gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial
+  - dest: /etc/yum.repos.d/CentOS-Linux-Extras.repo
+    content: |
+      [extras]
+      name=CentOS Linux $releasever - Extras
+      baseurl=http://vault.centos.org/$contentdir/$releasever/extras/$basearch/os/
+      enabled=1
+      gpgcheck=1
+      gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial
diff --git a/tests/ansible/hosts/group_vars/debian9.yml b/tests/ansible/hosts/group_vars/debian9.yml
new file mode 100644
index 00000000..4b180b13
--- /dev/null
+++ b/tests/ansible/hosts/group_vars/debian9.yml
@@ -0,0 +1,4 @@
+pkg_repos_overrides:
+  - dest: /etc/apt/sources.list
+    content: |
+      deb http://archive.debian.org/debian stretch main contrib non-free
diff --git a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml
index ad5bbd69..44ff6863 100755
--- a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml
+++ b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml
@@ -10,21 +10,14 @@
     ansible_python_interpreter: "{{ pkg_mgr_python_interpreter }}"
     package: rsync  # Chosen to exist in all tested distros/package managers
   tasks:
-    - name: Switch to centos-stream
-      command: dnf --assumeyes --disablerepo="*" --enablerepo=extras swap centos-linux-repos centos-stream-repos
-      when:
-        - ansible_facts.pkg_mgr in ["dnf"]
-
-    - name: Switch to archive.debian.org
-      # Debian 9 has been archived https://lists.debian.org/debian-devel-announce/2023/03/msg00006.html
+    - name: Switch to archived package repositories
       copy:
-        content: |
-          deb http://archive.debian.org/debian stretch main contrib non-free
-        dest: /etc/apt/sources.list
+        dest: "{{ item.dest }}"
+        content: "{{ item.content }}"
         mode: u=rw,go=r
-      when:
-        - ansible_facts.distribution == "Debian"
-        - ansible_facts.distribution_major_version == "9"
+      loop: "{{ pkg_repos_overrides }}"
+      loop_control:
+        label: "{{ item.dest }}"
 
     - name: Add signing keys
       copy:

From 924dbd6f0c31f1169adc8d969115dbcdfecb3cc9 Mon Sep 17 00:00:00 2001
From: Alex Willmer <alex@moreati.org.uk>
Date: Mon, 29 Jul 2024 11:25:12 +0100
Subject: [PATCH 15/17] CI: Migrate macOS integration tests to macOS 12, drop
 Python 2.7 jobs

macOS 11 is not longer an available runner on Azure Devops. The minimum is now
macOS 12. This runner does not have Python 2.7 installed, so running them
would require a custom install - which I'm declaring too much effort for too
little gain.

refs #1090
---
 .ci/azure-pipelines.yml                         | 17 +++--------------
 docs/changelog.rst                              |  1 +
 .../integration/become/sudo_nonexistent.yml     |  3 +--
 .../modules/custom_python_json_args_module.py   |  2 +-
 .../lib/modules/custom_python_os_getcwd.py      |  2 +-
 .../modules/custom_python_want_json_module.py   |  2 +-
 .../issue_591__setuptools_cwd_crash.yml         |  3 ---
 .../issue_655__wait_for_connection_error.yml    |  7 +++----
 tests/two_three_compat_test.py                  |  7 +++++++
 9 files changed, 18 insertions(+), 26 deletions(-)

diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml
index 94919e35..51908e92 100644
--- a/.ci/azure-pipelines.yml
+++ b/.ci/azure-pipelines.yml
@@ -16,32 +16,21 @@ trigger:
       - docs-master
 
 jobs:
-- job: mac11
+- job: mac12
   # vanilla Ansible is really slow
   timeoutInMinutes: 120
   steps:
   - template: azure-pipelines-steps.yml
   pool:
-    # https://github.com/actions/runner-images/blob/main/images/macos/macos-11-Readme.md
-    vmImage: macOS-11
+    # https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md
+    vmImage: macOS-12
   strategy:
     matrix:
-      Mito_27:
-        tox.env: py27-mode_mitogen
       Mito_312:
-        python.version: '3.12'
         tox.env: py312-mode_mitogen
-
-      Loc_27_210:
-        tox.env: py27-mode_localhost-ansible2.10
       Loc_312_9:
-        python.version: '3.12'
         tox.env: py312-mode_localhost-ansible9
-
-      Van_27_210:
-        tox.env: py27-mode_localhost-ansible2.10-strategy_linear
       Van_312_9:
-        python.version: '3.12'
         tox.env: py312-mode_localhost-ansible9-strategy_linear
 
 - job: Linux
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 5d77910e..ffba69fa 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -25,6 +25,7 @@ Unreleased
 * :gh:issue:`957` Fix Ansible exception when executing against 10s of hosts
   "ValueError: filedescriptor out of range in select()"
 * :gh:issue:`1066` Support Ansible `ansible_host_key_checking` & `ansible_ssh_host_key_checking`
+* :gh:issue:`1090` CI: Migrate macOS integration tests to macOS 12, drop Python 2.7 jobs
 
 
 v0.3.7 (2024-04-08)
diff --git a/tests/ansible/integration/become/sudo_nonexistent.yml b/tests/ansible/integration/become/sudo_nonexistent.yml
index e7a849c2..912d1ba8 100644
--- a/tests/ansible/integration/become/sudo_nonexistent.yml
+++ b/tests/ansible/integration/become/sudo_nonexistent.yml
@@ -21,8 +21,7 @@
           # sudo-1.8.6p3-29.el6_10.3 on RHEL & CentOS 6.10 (final release)
           # removed user/group error messages, as defence against CVE-2019-14287.
           - >-
-            'sudo: unknown user: slartibartfast' in out.module_stdout | default(out.msg)
-            or 'sudo: unknown user: slartibartfast' in out.module_stderr | default(out.msg)
+            (out.module_stderr | default(out.module_stdout, true) | default(out.msg, true)) is search('sudo: unknown user:? slartibartfast')
             or (ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_version == '6.10')
         fail_msg: out={{out}}
       when:
diff --git a/tests/ansible/lib/modules/custom_python_json_args_module.py b/tests/ansible/lib/modules/custom_python_json_args_module.py
index 846037ec..61640579 100755
--- a/tests/ansible/lib/modules/custom_python_json_args_module.py
+++ b/tests/ansible/lib/modules/custom_python_json_args_module.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # I am an Ansible Python JSONARGS module. I should receive an encoding string.
 
 json_arguments = """<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>"""
diff --git a/tests/ansible/lib/modules/custom_python_os_getcwd.py b/tests/ansible/lib/modules/custom_python_os_getcwd.py
index d465ac9e..c5e264ae 100644
--- a/tests/ansible/lib/modules/custom_python_os_getcwd.py
+++ b/tests/ansible/lib/modules/custom_python_os_getcwd.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # #591: call os.getcwd() before AnsibleModule ever gets a chance to fix up the
 # process environment.
 
diff --git a/tests/ansible/lib/modules/custom_python_want_json_module.py b/tests/ansible/lib/modules/custom_python_want_json_module.py
index 23eeeb55..f5e33862 100755
--- a/tests/ansible/lib/modules/custom_python_want_json_module.py
+++ b/tests/ansible/lib/modules/custom_python_want_json_module.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # I am an Ansible Python WANT_JSON module. I should receive a JSON-encoded file.
 
 import json
diff --git a/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml b/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml
index d5f0815b..ff102b13 100644
--- a/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml
+++ b/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml
@@ -19,9 +19,6 @@
 
   # Will crash if process has a nonexistent CWD.
   - custom_python_os_getcwd:
-      script: |
-        import os
-        self._connection.get_chain().call(os.getcwd)
   tags:
     - issue_591
     - mitogen_only
diff --git a/tests/ansible/regression/issue_655__wait_for_connection_error.yml b/tests/ansible/regression/issue_655__wait_for_connection_error.yml
index 9ad42a10..4972d91a 100644
--- a/tests/ansible/regression/issue_655__wait_for_connection_error.yml
+++ b/tests/ansible/regression/issue_655__wait_for_connection_error.yml
@@ -11,11 +11,10 @@
   tasks:
     - meta: end_play
       when:
-        # TODO CI currently runs on macOS 11 images in Azure DevOps. MacOS 11
-        #      is no longer supported by homebrew, so the following install
-        #      task fails.
+        # TODO CI currently runs on macOS 12 & which isn't supported by Podman
+        #      version available in Homebrew.
         - ansible_facts.system == 'Darwin'
-        - ansible_facts.distribution_major_version == '11'
+        - ansible_facts.distribution_version is version('13.0', '<', strict=True)
 
     - name: set up test container and run tests inside it
       block:
diff --git a/tests/two_three_compat_test.py b/tests/two_three_compat_test.py
index 4490e5d2..ab9f4e19 100644
--- a/tests/two_three_compat_test.py
+++ b/tests/two_three_compat_test.py
@@ -1,3 +1,6 @@
+import os
+import unittest
+
 import mitogen.core
 
 import testlib
@@ -7,6 +10,10 @@ import simple_pkg.ping
 # TODO: this is a joke. 2/3 interop is one of the hardest bits to get right.
 # There should be 100 tests in this file.
 
+@unittest.skipIf(
+    os.uname()[0] == 'Darwin' and int(os.uname()[2].partition('.')[0]) >= 21,
+    "Python 2.x not shipped on macOS 12.3+ (Darwin 21.4+, Monterey)",
+)
 class TwoThreeCompatTest(testlib.RouterMixin, testlib.TestCase):
     if mitogen.core.PY3:
         python_path = 'python2'

From fe435bb7d069764b35a8ca77b05cffe19dbd9c1e Mon Sep 17 00:00:00 2001
From: Alex Willmer <alex@moreati.org.uk>
Date: Mon, 29 Jul 2024 11:30:25 +0100
Subject: [PATCH 16/17] CI: Workaround "No module named
 'setuptools.command.test'"

Pip 72 was released yesterday (2024-07-28), dropping `setup.py test` support.
hdrhistogram 0.6.1 requires it to install.

For now constrain Pip to earlier releases, so our tests can be run.

refs #1090
---
 tests/ansible/requirements.txt | 3 +++
 tests/constraints.txt          | 3 +++
 tox.ini                        | 1 +
 3 files changed, 7 insertions(+)
 create mode 100644 tests/constraints.txt

diff --git a/tests/ansible/requirements.txt b/tests/ansible/requirements.txt
index 2c3c87c8..8cfb348a 100644
--- a/tests/ansible/requirements.txt
+++ b/tests/ansible/requirements.txt
@@ -1,4 +1,7 @@
 paramiko==2.3.2  # Last 2.6-compat version.
+# Incompatible with pip >= 72, due to removal of `setup.py test`:
+#   ModuleNotFoundError: No module named 'setuptools.command.test'
+# https://github.com/pypa/setuptools/issues/4519
 hdrhistogram==0.6.1
 PyYAML==3.11; python_version < '2.7'
 PyYAML==5.3.1; python_version >= '2.7'  # Latest release (Jan 2021)
diff --git a/tests/constraints.txt b/tests/constraints.txt
new file mode 100644
index 00000000..6adaa30b
--- /dev/null
+++ b/tests/constraints.txt
@@ -0,0 +1,3 @@
+# Setuptools 72 removed `setup.py test`. hdrhistogram 0.6.1 still depends on it.
+# TODO Bump dependencies and unconstrain Pip.
+setuptools<72
diff --git a/tox.ini b/tox.ini
index 55fd1842..a0b0e1ae 100644
--- a/tox.ini
+++ b/tox.ini
@@ -109,6 +109,7 @@ setenv =
     ANSIBLE_STRATEGY = mitogen_linear
     NOCOVERAGE_ERASE = 1
     NOCOVERAGE_REPORT = 1
+    PIP_CONSTRAINT={toxinidir}/tests/constraints.txt
     # Only applicable to MODE=mitogen
     distro_centos5: DISTRO=centos5
     distro_centos6: DISTRO=centos6

From e334b50d9d5bc5174d1e07c83c6617db6406193e Mon Sep 17 00:00:00 2001
From: Alex Willmer <alex@moreati.org.uk>
Date: Tue, 30 Jul 2024 09:06:57 +0100
Subject: [PATCH 17/17] Prepare v0.3.8

---
 docs/changelog.rst  | 4 ++--
 docs/conf.py        | 2 +-
 mitogen/__init__.py | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index ffba69fa..2273043a 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -18,8 +18,8 @@ To avail of fixes in an unreleased version, please download a ZIP file
 `directly from GitHub <https://github.com/mitogen-hq/mitogen/>`_.
 
 
-Unreleased
-----------
+v0.3.8 (2024-07-30)
+-------------------
 
 * :gh:issue:`952` Fix Ansible `--ask-become-pass`, add test coverage
 * :gh:issue:`957` Fix Ansible exception when executing against 10s of hosts
diff --git a/docs/conf.py b/docs/conf.py
index ad9771b4..fe1a5044 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -2,7 +2,7 @@ import sys
 
 sys.path.append('.')
 
-VERSION = '0.3.7'
+VERSION = '0.3.8'
 
 author = u'Network Genomics'
 copyright = u'2021, the Mitogen authors'
diff --git a/mitogen/__init__.py b/mitogen/__init__.py
index b63cd0cd..8f2741c7 100644
--- a/mitogen/__init__.py
+++ b/mitogen/__init__.py
@@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup.
 
 
 #: Library version as a tuple.
-__version__ = (0, 3, 8, 'dev')
+__version__ = (0, 3, 8)
 
 
 #: This is :data:`False` in slave contexts. Previously it was used to prevent