Merge commit 'f1f8ad1' into release-v0.3.20

.github/workflows/tests.yml

@@ -162,7 +162,7 @@ jobs:
   macos:
     # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md
     runs-on: macos-13
-    timeout-minutes: 120
+    timeout-minutes: 15
 
     strategy:
       fail-fast: false

ansible_mitogen/connection.py

@@ -145,7 +145,7 @@ def _connect_ssh(spec):
             'identity_file': private_key_file,
             'identities_only': False,
             'ssh_path': spec.ssh_executable(),
-            'connect_timeout': spec.ansible_ssh_timeout(),
+            'connect_timeout': spec.timeout(),
             'ssh_args': spec.ssh_args(),
             'ssh_debug_level': spec.mitogen_ssh_debug_level(),
             'remote_name': get_remote_name(spec),
@@ -169,7 +169,7 @@ def _connect_buildah(spec):
             'username': spec.remote_user(),
             'container': spec.remote_addr(),
             'python_path': spec.python_path(),
-            'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(),
+            'connect_timeout': spec.timeout(),
             'remote_name': get_remote_name(spec),
         }
     }
@@ -185,7 +185,7 @@ def _connect_docker(spec):
             'username': spec.remote_user(),
             'container': spec.remote_addr(),
             'python_path': spec.python_path(rediscover_python=True),
-            'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(),
+            'connect_timeout': spec.timeout(),
             'remote_name': get_remote_name(spec),
         }
     }
@@ -200,7 +200,7 @@ def _connect_kubectl(spec):
         'kwargs': {
             'pod': spec.remote_addr(),
             'python_path': spec.python_path(),
-            'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(),
+            'connect_timeout': spec.timeout(),
             'kubectl_path': spec.mitogen_kubectl_path(),
             'kubectl_args': spec.extra_args(),
             'remote_name': get_remote_name(spec),
@@ -218,7 +218,7 @@ def _connect_jail(spec):
             'username': spec.remote_user(),
             'container': spec.remote_addr(),
             'python_path': spec.python_path(),
-            'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(),
+            'connect_timeout': spec.timeout(),
             'remote_name': get_remote_name(spec),
         }
     }
@@ -234,7 +234,7 @@ def _connect_lxc(spec):
             'container': spec.remote_addr(),
             'python_path': spec.python_path(),
             'lxc_attach_path': spec.mitogen_lxc_attach_path(),
-            'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(),
+            'connect_timeout': spec.timeout(),
             'remote_name': get_remote_name(spec),
         }
     }
@@ -250,7 +250,7 @@ def _connect_lxd(spec):
             'container': spec.remote_addr(),
             'python_path': spec.python_path(),
             'lxc_path': spec.mitogen_lxc_path(),
-            'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(),
+            'connect_timeout': spec.timeout(),
             'remote_name': get_remote_name(spec),
         }
     }
@@ -273,7 +273,7 @@ def _connect_podman(spec):
             'username': spec.remote_user(),
             'container': spec.remote_addr(),
             'python_path': spec.python_path(rediscover_python=True),
-            'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(),
+            'connect_timeout': spec.timeout(),
             'remote_name': get_remote_name(spec),
         }
     }
@@ -933,31 +933,39 @@ class Connection(ansible.plugins.connection.ConnectionBase):
                 self.reset_compat_msg
             )
 
-        # Strategy's _execute_meta doesn't have an action obj but we'll need one for
-        # running interpreter_discovery
-        # will create a new temporary action obj for this purpose
-        self._action = ansible_mitogen.mixins.ActionModuleMixin(
-            task=0,
-            connection=self,
-            play_context=self._play_context,
-            loader=0,
-            templar=0,
-            shared_loader_obj=0
-        )
-
-        # Workaround for https://github.com/ansible/ansible/issues/84238
+        # Handle templated connection variables during `meta: reset_connection`.
+        # Many bugs/implementation details of Mitogen & Ansible collide here.
+        # See #1079, #1096, #1132, ansible/ansible#84238, ...
         try:
             task, templar = self._play_context.vars.pop(
                 '_mitogen.smuggled.reset_connection',
             )
         except KeyError:
-            pass
+            self._action_monkey_patched_by_mitogen = False
         else:
+            # LOG.info('%r.reset(): remote_addr=%r', self, self._play_context.remote_addr)
+            # ansible.plugins.strategy.StrategyBase._execute_meta() doesn't
+            # have an action object, which we need for interpreter_discovery.
+            # Create a temporary action object for this purpose.
+            self._action = ansible_mitogen.mixins.ActionModuleMixin(
+                task=task,
+                connection=self,
+                play_context=self._play_context,
+                loader=templar._loader,
+                templar=templar,
+                shared_loader_obj=0,
+            )
+            self._action_monkey_patched_by_mitogen = True
+
+            # Workaround for https://github.com/ansible/ansible/issues/84238
             self.set_options(
                 task_keys=task.dump_attrs(),
                 var_options=self._mitogen_var_options(templar),
             )
 
+            del task
+            del templar
+
         # Clear out state in case we were ever connected.
         self.close()
 
@@ -977,6 +985,11 @@ class Connection(ansible.plugins.connection.ConnectionBase):
         finally:
             binding.close()
 
+        # Cleanup any monkey patching we did for `meta: reset_connection`
+        if self._action_monkey_patched_by_mitogen:
+            del self._action
+        del self._action_monkey_patched_by_mitogen
+
     # Compatibility with Ansible 2.4 wait_for_connection plug-in.
     _reset = reset
 

ansible_mitogen/strategy.py

@@ -328,8 +328,12 @@ class StrategyMixin(object):
         finally:
             ansible_mitogen.process.set_worker_model(None)
 
-    def _smuggle_to_connction_reset(self, task, play_context, iterator, target_host):
-        # Workaround for https://github.com/ansible/ansible/issues/84238
+    def _smuggle_to_connection_reset(self, task, play_context, iterator, target_host):
+        """
+        Create a templar and make it available for use in Connection.reset().
+        This allows templated connection variables to be used when Mitogen
+        reconstructs its connection stack.
+        """
         variables = self._variable_manager.get_vars(
             play=iterator._play, host=target_host, task=task,
             _hosts=self._hosts_cache, _hosts_all=self._hosts_cache_all,
@@ -337,13 +341,29 @@ class StrategyMixin(object):
         templar = ansible.template.Templar(
             loader=self._loader, variables=variables,
         )
+
+        # Required for remote_user option set by variable (e.g. ansible_user).
+        # Without it remote_user in ansible.cfg gets used.
+        play_context = play_context.set_task_and_variable_override(
+            task=task, variables=variables, templar=templar,
+        )
+        play_context.post_validate(templar=templar)
+
+        # Required for timeout option set by variable (e.g. ansible_timeout).
+        # Without it the task timeout keyword (default: 0) gets used.
+        play_context.update_vars(variables)
+
+        # Stash the task and templar somewhere Connection.reset() can find it
         play_context.vars.update({
             '_mitogen.smuggled.reset_connection': (task, templar),
         })
+        return play_context
 
     def _execute_meta(self, task, play_context, iterator, target_host):
         if task.args['_raw_params'] == 'reset_connection':
-            self._smuggle_to_connction_reset(task, play_context, iterator, target_host)
+            play_context = self._smuggle_to_connection_reset(
+                task, play_context, iterator, target_host,
+            )
 
         return super(StrategyMixin, self)._execute_meta(
             task, play_context, iterator, target_host,

ansible_mitogen/transport_config.py

@@ -493,12 +493,24 @@ class PlayContextSpec(Spec):
         return self._connection_option('port')
 
     def python_path(self, rediscover_python=False):
-        s = self._connection.get_task_var('ansible_python_interpreter')
-        # #511, #536: executor/module_common.py::_get_shebang() hard-wires
-        # "/usr/bin/python" as the default interpreter path if no other
-        # interpreter is specified.
+        # See also
+        #   - ansible_mitogen.connecton.Connection.get_task_var()
+        try:
+            delegated_vars = self._task_vars['ansible_delegated_vars']
+            variables = delegated_vars[self._connection.delegate_to_hostname]
+        except KeyError:
+            variables = self._task_vars
+
+        interpreter_python = C.config.get_config_value(
+            'INTERPRETER_PYTHON', variables=variables,
+        )
+
+        if '{{' in interpreter_python or '{%' in interpreter_python:
+            templar = self._connection.templar
+            interpreter_python = templar.template(interpreter_python)
+
         return parse_python_path(
-            s,
+            interpreter_python,
             task_vars=self._task_vars,
             action=self._action,
             rediscover_python=rediscover_python)
@@ -513,14 +525,10 @@ class PlayContextSpec(Spec):
         return self._connection_option('ssh_executable')
 
     def timeout(self):
-        return self._play_context.timeout
+        return self._connection_option('timeout')
 
     def ansible_ssh_timeout(self):
-        return (
-            self._connection.get_task_var('ansible_timeout') or
-            self._connection.get_task_var('ansible_ssh_timeout') or
-            self.timeout()
-        )
+        return self.timeout()
 
     def ssh_args(self):
         return [

docs/changelog.rst

@@ -18,6 +18,19 @@ To avail of fixes in an unreleased version, please download a ZIP file
 `directly from GitHub <https://github.com/mitogen-hq/mitogen/>`_.
 
 
+v0.3.20 (2025-01-07)
+--------------------
+
+* :gh:issue:`1079` :mod:`ansible_mitogen`: Fix :ans:mod:`wait_for_connection`
+  timeout with templated ``ansible_python_interpreter``
+* :gh:issue:`1079` :mod:`ansible_mitogen`: Fix templated python interpreter
+  with `meta: reset_connection`
+* :gh:issue:`1083` :mod:`ansible_mitogen`: Templated connection timeout
+  (e.g. ``ansible_timeout``).
+* :gh:issue:`740` :mod:`ansible_mitogen`: Respect ``interpreter_python``
+  in ``ansible.cfg`` and ``ANSIBLE_PYTHON_INTERPRETER`` environment variable.
+
+
 v0.3.19 (2024-12-02)
 --------------------
 

mitogen/__init__.py

@@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup.
 
 
 #: Library version as a tuple.
-__version__ = (0, 3, 19)
+__version__ = (0, 3, 20)
 
 
 #: This is :data:`False` in slave contexts. Previously it was used to prevent

tests/ansible/hosts/default.hosts

@@ -19,6 +19,9 @@ ssh-common-args ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami')
 ansible_ssh_common_args=-o PermitLocalCommand=yes -o LocalCommand="touch {{ ssh_args_canary_file }}"
 ssh_args_canary_file=/tmp/ssh_args_by_inv_{{ inventory_hostname }}
 
+[issue1079]
+wait-for-connection ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}"
+
 [tt_targets_bare]
 tt-bare
 
@@ -47,5 +50,7 @@ tt-host-key-checking        ansible_host=localhost    ansible_host_key_checking=
 tt-password                 ansible_host=localhost    ansible_password="{{ 'has_sudo_nopw_password' | trim }}"    ansible_user=mitogen__has_sudo_nopw
 tt-port                     ansible_host=localhost    ansible_password=has_sudo_nopw_password    ansible_port="{{ 22 | int }}"    ansible_user=mitogen__has_sudo_nopw
 tt-private-key-file         ansible_host=localhost    ansible_private_key_file="{{ git_basedir }}/tests/data/docker/mitogen__has_sudo_pubkey.key"    ansible_user=mitogen__has_sudo_pubkey
+tt-python-interpreter       ansible_host=localhost    ansible_password=has_sudo_nopw_password    ansible_python_interpreter="{{ ansible_playbook_python | trim }}"    ansible_user=mitogen__has_sudo_nopw
 tt-remote-user              ansible_host=localhost    ansible_password=has_sudo_nopw_password    ansible_user="{{ 'mitogen__has_sudo_nopw' | trim }}"
 tt-ssh-executable           ansible_host=localhost    ansible_password=has_sudo_nopw_password    ansible_ssh_executable="{{ 'ssh' | trim }}"    ansible_user=mitogen__has_sudo_nopw
+tt-timeout                  ansible_host=localhost    ansible_password=has_sudo_nopw_password    ansible_timeout="{{ 5 | int }}"    ansible_user=mitogen__has_sudo_nopw

tests/ansible/integration/ssh/templated_by_play_taskvar.yml

@@ -7,6 +7,7 @@
     ansible_password: "{{ 'has_sudo_nopw_password' | trim }}"
     ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}"
     ansible_ssh_executable: "{{ 'ssh' | trim }}"
+    ansible_timeout: "{{ 5 | int }}"
     ansible_user: "{{ 'mitogen__has_sudo_nopw' | trim }}"
 
   tasks:
@@ -23,6 +24,7 @@
     ansible_private_key_file: "{{ git_basedir }}/tests/data/docker/mitogen__has_sudo_pubkey.key"
     ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}"
     ansible_ssh_executable: "{{ 'ssh' | trim }}"
+    ansible_timeout: "{{ 5 | int }}"
     ansible_user: "{{ 'mitogen__has_sudo_pubkey' | trim }}"
 
   tasks:

tests/ansible/regression/all.yml

@@ -16,4 +16,5 @@
 - import_playbook: issue_776__load_plugins_called_twice.yml
 - import_playbook: issue_952__ask_become_pass.yml
 - import_playbook: issue_1066__add_host__host_key_checking.yml
+- import_playbook: issue_1079__wait_for_connection_timeout.yml
 - import_playbook: issue_1087__template_streamerror.yml

tests/ansible/regression/issue_1079__wait_for_connection_timeout.yml

@@ -0,0 +1,22 @@
+- name: regression/issue_1079__wait_for_connection_timeout.yml
+  hosts: issue1079
+  gather_facts: false
+  tasks:
+    - name: Wait for connection at start of play
+      wait_for_connection:
+        timeout: 5
+  tags:
+    - issue_1079
+    - wait_for_connection
+
+- hosts: issue1079
+  gather_facts: false
+  tasks:
+    - meta: reset_connection
+    - name: Wait for connection after reset_connection
+      wait_for_connection:
+        timeout: 5
+  tags:
+    - issue_1079
+    - reset_connection
+    - wait_for_connection

tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml

@@ -26,6 +26,7 @@
           - env.cwd == ansible_user_dir
           - (not env.mitogen_loaded) or (env.python_path.count("") == 1)
         fail_msg: |
+          ansible_user_dir={{ ansible_user_dir }}
           env={{ env }}
 
     - name: Run some new-style from ansible.module_utils... modules

tests/ansible/templates/test-targets.j2

@@ -40,6 +40,13 @@ ssh_args_canary_file=/tmp/ssh_args_by_inv_{{ '{{' }} inventory_hostname {{ '}}'
 
 {% set tt = containers[0] %}
 
+[issue1079]
+wait-for-connection ansible_host={{ tt.hostname }} ansible_port={{ tt.port }} ansible_python_interpreter="{{ '{{' }} '{{ tt.python_path }}' | trim {{ '}}' }}"
+
+[issue1079:vars]
+ansible_user=mitogen__has_sudo_nopw
+ansible_password=has_sudo_nopw_password
+
 [tt_targets_bare]
 tt-bare
 
@@ -77,5 +84,7 @@ tt-host-key-checking        ansible_host={{ tt.hostname }}    ansible_host_key_c
 tt-password                 ansible_host={{ tt.hostname }}    ansible_password="{{ '{{' }} 'has_sudo_nopw_password' | trim {{ '}}' }}"    ansible_port={{ tt.port }}    ansible_python_interpreter={{ tt.python_path }}    ansible_user=mitogen__has_sudo_nopw
 tt-port                     ansible_host={{ tt.hostname }}    ansible_password=has_sudo_nopw_password    ansible_port="{{ '{{' }} {{ tt.port }} | int {{ '}}' }}"    ansible_python_interpreter={{ tt.python_path }}    ansible_user=mitogen__has_sudo_nopw
 tt-private-key-file         ansible_host={{ tt.hostname }}    ansible_port={{ tt.port }}    ansible_private_key_file="{{ '{{' }} git_basedir {{ '}}' }}/tests/data/docker/mitogen__has_sudo_pubkey.key"    ansible_python_interpreter={{ tt.python_path }}    ansible_user=mitogen__has_sudo_pubkey
+tt-python-interpreter       ansible_host={{ tt.hostname }}    ansible_port={{ tt.port }}    ansible_password=has_sudo_nopw_password    ansible_python_interpreter="{{ '{{' }} '{{ tt.python_path }}' | trim {{ '}}' }}"    ansible_user=mitogen__has_sudo_nopw
 tt-remote-user              ansible_host={{ tt.hostname }}    ansible_password=has_sudo_nopw_password    ansible_port={{ tt.port }}    ansible_python_interpreter={{ tt.python_path }}    ansible_user="{{ '{{' }} 'mitogen__has_sudo_nopw' | trim {{ '}}' }}"
 tt-ssh-executable           ansible_host={{ tt.hostname }}    ansible_password=has_sudo_nopw_password    ansible_port={{ tt.port }}    ansible_python_interpreter={{ tt.python_path }}    ansible_ssh_executable="{{ '{{' }} 'ssh' | trim {{ '}}' }}"    ansible_user=mitogen__has_sudo_nopw
+tt-timeout                  ansible_host={{ tt.hostname }}    ansible_password=has_sudo_nopw_password    ansible_port={{ tt.port }}    ansible_python_interpreter={{ tt.python_path }}    ansible_timeout="{{ '{{' }} 5 | int {{ '}}' }}"    ansible_user=mitogen__has_sudo_nopw