Merge remote-tracking branch 'upstream/master' into docs-master

ansible_mitogen/transport_config.py

@@ -450,19 +450,7 @@ class PlayContextSpec(Spec):
         return self._become_option('become_user')
 
     def become_pass(self):
-        # become_pass is owned/provided by the active become plugin. However
-        # PlayContext is intertwined with it. Known complications
-        # - ansible_become_password is higher priority than ansible_become_pass,
-        #   `play_context.become_pass` doesn't obey this (atleast with Mitgeon).
-        # - `meta: reset_connection` runs `connection.reset()` but
-        #   `ansible_mitogen.connection.Connection.reset()` recreates the
-        #   connection object, setting `connection.become = None`.
-        become_plugin = self._connection.become
-        try:
-            become_pass = become_plugin.get_option('become_pass', playcontext=self._play_context)
-        except AttributeError:
-            become_pass = self._play_context.become_pass
-        return optional_secret(become_pass)
+        return optional_secret(self._become_option('become_pass'))
 
     def password(self):
         return optional_secret(self._connection_option('password'))
@@ -506,13 +494,12 @@ class PlayContextSpec(Spec):
         )
 
     def ssh_args(self):
-        local_vars = self._task_vars.get("hostvars", {}).get(self._inventory_name, {})
         return [
             mitogen.core.to_text(term)
             for s in (
-                C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=local_vars),
-                C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=local_vars),
-                C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=local_vars)
+                self._connection_option('ssh_args'),
+                self._connection_option('ssh_common_args'),
+                self._connection_option('ssh_extra_args'),
             )
             for term in ansible.utils.shlex.shlex_split(s or '')
         ]

docs/changelog.rst

@@ -23,6 +23,16 @@ In progress (unreleased)
 
 
 
+v0.3.15 (2024-10-28)
+--------------------
+
+* :gh:issue:`905` :mod:`ansible_mitogen`: Support templated SSH command
+  arguments (e.g. ``ansible_ssh_args``, ``ansible_ssh_extra_args``).
+* :gh:issue:`692` tests: Fix and re-enable several sudo tests
+* :gh:issue:`1083` :mod:`ansible_mitogen`: Support templated become password
+  (e.g. ``ansible_become_pass``, ``ansible_sudo_pass``)
+
+
 v0.3.14 (2024-10-16)
 --------------------
 

mitogen/__init__.py

@@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup.
 
 
 #: Library version as a tuple.
-__version__ = (0, 3, 15, 'dev')
+__version__ = (0, 3, 16, 'dev')
 
 
 #: This is :data:`False` in slave contexts. Previously it was used to prevent

tests/ansible/hosts/default.hosts

@@ -17,7 +17,7 @@ ssh-common-args ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami')
 
 [issue905:vars]
 ansible_ssh_common_args=-o PermitLocalCommand=yes -o LocalCommand="touch {{ ssh_args_canary_file }}"
-ssh_args_canary_file=/tmp/ssh_args_{{ inventory_hostname }}
+ssh_args_canary_file=/tmp/ssh_args_by_inv_{{ inventory_hostname }}
 
 [tt_targets_bare]
 tt-bare
@@ -33,6 +33,7 @@ ansible_host=localhost
 ansible_user="{{ lookup('pipe', 'whoami') }}"
 
 [tt_become_by_inv]
+tt-become-pass              ansible_become=true ansible_become_pass="{{ 'pw_required_password' | trim }}" ansible_become_user=mitogen__pw_required
 tt-become-user              ansible_become=true ansible_become_user="{{ 'root' | trim }}"
 
 [tt_become_by_inv:vars]

tests/ansible/integration/action/make_tmp_path.yml

@@ -142,17 +142,16 @@
     # readonly homedir
     #
 
-    # TODO: https://github.com/dw/mitogen/issues/692
-    # - name: "Try writing to temp directory for the readonly_homedir user"
-    #   become: true
-    #   become_user: mitogen__readonly_homedir
-    #   custom_python_run_script:
-    #     script: |
-    #       from ansible.module_utils.basic import get_module_path
-    #       path = get_module_path() + '/foo.txt'
-    #       result['path'] = path
-    #       open(path, 'w').write("bar")
-    #   register: tmp_path
+    - name: Try writing to temp directory for the readonly_homedir user
+      become: true
+      become_user: mitogen__readonly_homedir
+      custom_python_run_script:
+        script: |
+          from ansible.module_utils.basic import get_module_path
+          path = get_module_path() + '/foo.txt'
+          result['path'] = path
+          open(path, 'w').write("bar")
+      register: tmp_path
 
     #
     # modules get the same base dir

tests/ansible/integration/action/synchronize.yml

@@ -40,11 +40,11 @@
       delegate_to: localhost
       run_once: true
 
-    # TODO: https://github.com/dw/mitogen/issues/692
-    # - file:
-    #     path: /tmp/sync-test.out
-    #     state: absent
-    #   become: true
+    - name: Ensure clean slate
+      become: true
+      file:
+        path: /tmp/sync-test.out
+        state: absent
 
     # exception:   File "/tmp/venv/lib/python2.7/site-packages/ansible/plugins/action/__init__.py", line 129, in cleanup
     # exception:     self._remove_tmp_path(self._connection._shell.tmpdir)
@@ -70,14 +70,14 @@
               outout={{ outout }}
       when: False
 
-    # TODO: https://github.com/dw/mitogen/issues/692
-    # - file:
-    #     path: "{{item}}"
-    #     state: absent
-    #   become: true
-    #   with_items:
-    #   - /tmp/synchronize-action-key
-    #   - /tmp/sync-test
-    #   - /tmp/sync-test.out
+    - name: Cleanup
+      become: true
+      file:
+        path: "{{ item }}"
+        state: absent
+      with_items:
+        - /tmp/synchronize-action-key
+        - /tmp/sync-test
+        - /tmp/sync-test.out
   tags:
     - synchronize

tests/ansible/integration/become/sudo_password.yml

@@ -5,10 +5,12 @@
   tasks:
 
     - name: Ensure sudo password absent but required.
-      shell: whoami
       become: true
       become_user: mitogen__pw_required
+      command:
+        cmd: whoami
       register: out
+      changed_when: false
       ignore_errors: true
       when:
         # https://github.com/ansible/ansible/pull/70785
@@ -32,10 +34,12 @@
           or is_mitogen
 
     - name: Ensure password sudo incorrect.
-      shell: whoami
       become: true
       become_user: mitogen__pw_required
+      command:
+        cmd: whoami
       register: out
+      changed_when: false
       vars:
         ansible_become_pass: nopes
       ignore_errors: true
@@ -59,18 +63,27 @@
           or ansible_version.full is version("2.11", ">=", strict=True)
           or is_mitogen
 
-    # TODO: https://github.com/dw/mitogen/issues/692
-    # - name: Ensure password sudo succeeds.
-    #   shell: whoami
-    #   become: true
-    #   become_user: mitogen__pw_required
-    #   register: out
-    #   vars:
-    #     ansible_become_pass: pw_required_password
+    - block:
+        - name: Ensure password sudo succeeds
+          become: true
+          become_user: mitogen__pw_required
+          vars:
+            ansible_become_pass: pw_required_password
+          command:
+            cmd: whoami
+          register: sudo_password_success_whoami
+          changed_when: false
 
-    # - assert:
-    #     that:
-    #       - out.stdout == 'mitogen__pw_required'
+        - assert:
+            that:
+              - sudo_password_success_whoami.stdout == 'mitogen__pw_required'
+            fail_msg: |
+              sudo_password_success_whoami={{ sudo_password_success_whoami }}
+      when:
+        # https://github.com/ansible/ansible/pull/70785
+        - ansible_facts.distribution not in ["MacOSX"]
+          or ansible_version.full is version("2.11", ">=", strict=True)
+          or is_mitogen
   tags:
     - sudo
     - sudo_password

tests/ansible/integration/become/sudo_requiretty.yml

@@ -3,34 +3,38 @@
 - name: integration/become/sudo_requiretty.yml
   hosts: test-targets
   tasks:
-    # - include_tasks: ../_mitogen_only.yml
+    # AIUI Vanilla Ansible cannot do sudo when requiretty configured
+    - include_tasks: ../_mitogen_only.yml
 
-    # TODO: https://github.com/dw/mitogen/issues/692
-    # - name: Verify we can login to a non-passworded requiretty account
-    #   shell: whoami
-    #   become: true
-    #   become_user: mitogen__require_tty
-    #   register: out
+    - name: Verify we can login to a non-passworded requiretty account
+      become: true
+      become_user: mitogen__require_tty
+      command:
+        cmd: whoami
+      changed_when: false
+      register: sudo_require_tty_whoami
 
-    # - assert:
-    #     that:
-    #       - out.stdout == 'mitogen__require_tty'
+    - assert:
+        that:
+          - sudo_require_tty_whoami.stdout == 'mitogen__require_tty'
+        fail_msg: |
+          sudo_require_tty_whoami={{ sudo_require_tty_whoami }}
 
+    - name: Verify we can login to a passworded requiretty account
+      become: true
+      become_user: mitogen__require_tty_pw_required
+      vars:
+        ansible_become_pass: require_tty_pw_required_password
+      command:
+        cmd: whoami
+      changed_when: false
+      register: sudo_require_tty_password_whoami
 
-    # ---------------
-
-    # TODO: https://github.com/dw/mitogen/issues/692
-    # - name: Verify we can login to a passworded requiretty account
-    #   shell: whoami
-    #   become: true
-    #   become_user: mitogen__require_tty_pw_required
-    #   vars:
-    #     ansible_become_pass: require_tty_pw_required_password
-    #   register: out
-
-    # - assert:
-    #     that:
-    #       - out.stdout == 'mitogen__require_tty_pw_required'
+    - assert:
+        that:
+          - sudo_require_tty_password_whoami.stdout == 'mitogen__require_tty_pw_required'
+        fail_msg: |
+          sudo_require_tty_password_whoami={{ sudo_require_tty_password_whoami }}
   tags:
     - mitogen_only
     - sudo

tests/ansible/integration/become/templated_by_inv.yml

@@ -2,8 +2,18 @@
   hosts: tt_become_by_inv
   gather_facts: false
   tasks:
+    - name: Gather facts (avoiding any unprivileged become)
+      vars:
+        ansible_become: false
+      setup:
+
     - meta: reset_connection
+
     - name: Templated become in inventory
+      vars:
+        expected_become_users:
+          tt-become-pass: mitogen__pw_required
+          tt-become-user: root
       command:
         cmd: whoami
       changed_when: false
@@ -11,4 +21,10 @@
       register: become_templated_by_inv_whoami
       failed_when:
         - become_templated_by_inv_whoami is failed
-          or become_templated_by_inv_whoami.stdout != 'root'
+          or become_templated_by_inv_whoami.stdout != expected_become_users[inventory_hostname]
+      when:
+        # https://github.com/ansible/ansible/pull/70785
+        - ansible_become_user in ['root']
+          or ansible_facts.distribution not in ["MacOSX"]
+          or ansible_version.full is version("2.11", ">=", strict=True)
+          or is_mitogen

tests/ansible/integration/become/templated_by_play_keywords.yml

@@ -5,7 +5,8 @@
   become_user: "{{ 'root' | trim }}"
   tasks:
     - meta: reset_connection
-    - name: Templated become by play keywords
+
+    - name: Templated become by play keywords, no password
       command:
         cmd: whoami
       changed_when: false
@@ -14,3 +15,33 @@
       failed_when:
         - become_templated_by_play_keywords_whoami is failed
           or become_templated_by_play_keywords_whoami.stdout != 'root'
+
+- name: integration/become/templated_by_play_keywords.yml
+  hosts: tt_become_bare
+  gather_facts: false
+  become: true
+  become_user: "{{ 'mitogen__pw_required' | trim }}"
+  vars:
+    ansible_become_pass: "{{ 'pw_required_password' | trim }}"
+  tasks:
+    - name: Gather facts (avoiding any unprivileged become)
+      vars:
+        ansible_become: false
+      setup:
+
+    - meta: reset_connection
+
+    - name: Templated become by play keywords, password
+      command:
+        cmd: whoami
+      changed_when: false
+      check_mode: false
+      register: become_templated_by_play_keywords_password_whoami
+      failed_when:
+        - become_templated_by_play_keywords_password_whoami is failed
+          or become_templated_by_play_keywords_password_whoami.stdout != 'mitogen__pw_required'
+      when:
+        # https://github.com/ansible/ansible/pull/70785
+        - ansible_facts.distribution not in ["MacOSX"]
+          or ansible_version.full is version("2.11", ">=", strict=True)
+          or is_mitogen

tests/ansible/integration/become/templated_by_play_vars.yml

@@ -5,7 +5,7 @@
     ansible_become: true
     ansible_become_user: "{{ 'root' | trim }}"
   tasks:
-    - name: Templated become by play vars
+    - name: Templated become by play vars, no password
       command:
         cmd: whoami
       changed_when: false
@@ -14,3 +14,33 @@
       failed_when:
         - become_templated_by_play_vars_whoami is failed
           or become_templated_by_play_vars_whoami.stdout != 'root'
+
+- name: integration/become/templated_by_play_vars.yml
+  hosts: tt_become_bare
+  gather_facts: false
+  vars:
+    ansible_become: true
+    ansible_become_pass: "{{ 'pw_required_password' | trim }}"
+    ansible_become_user: "{{ 'mitogen__pw_required' | trim }}"
+  tasks:
+    - name: Gather facts (avoiding any unprivileged become)
+      vars:
+        ansible_become: false
+      setup:
+
+    - meta: reset_connection
+
+    - name: Templated become by play vars, password
+      command:
+        cmd: whoami
+      changed_when: false
+      check_mode: false
+      register: become_templated_by_play_vars_password_whoami
+      failed_when:
+        - become_templated_by_play_vars_password_whoami is failed
+          or become_templated_by_play_vars_password_whoami.stdout != 'mitogen__pw_required'
+      when:
+        # https://github.com/ansible/ansible/pull/70785
+        - ansible_facts.distribution not in ["MacOSX"]
+          or ansible_version.full is version("2.11", ">=", strict=True)
+          or is_mitogen

tests/ansible/integration/become/templated_by_task_keywords.yml

@@ -25,3 +25,47 @@
       failed_when:
         - become_templated_by_task_with_delegate_to_whoami is failed
           or become_templated_by_task_with_delegate_to_whoami.stdout != 'root'
+
+
+- name: integration/become/templated_by_task_keywords.yml
+  hosts: tt_become_bare
+  gather_facts: false
+  # FIXME Resetting the connection shouldn't require credentials
+  #       https://github.com/mitogen-hq/mitogen/issues/1132
+  become: true
+  become_user: "{{ 'mitogen__pw_required' | trim }}"
+  vars:
+    ansible_become_pass: "{{ 'pw_required_password' | trim }}"
+  tasks:
+    - name: Reset connection to target that will be delegate_to
+      meta: reset_connection
+
+- name: Test connection template by task keywords, with delegate_to
+  hosts: test-targets[0]
+  gather_facts: false
+  tasks:
+    - name: Gather facts (avoiding any unprivileged become)
+      delegate_to: "{{ groups.tt_become_bare[0] }}"
+      vars:
+        ansible_become: false
+      setup:
+
+    - name: Templated become by task keywords, with delegate_to
+      become: true
+      become_user: "{{ 'mitogen__pw_required' | trim }}"
+      delegate_to: "{{ groups.tt_become_bare[0] }}"
+      vars:
+        ansible_become_pass: "{{ 'pw_required_password' | trim }}"
+      command:
+        cmd: whoami
+      changed_when: false
+      check_mode: false
+      register: become_templated_by_task_with_delegate_to_password_whoami
+      failed_when:
+        - become_templated_by_task_with_delegate_to_password_whoami is failed
+          or become_templated_by_task_with_delegate_to_password_whoami.stdout != 'mitogen__pw_required'
+      when:
+        # https://github.com/ansible/ansible/pull/70785
+        - ansible_facts.distribution not in ["MacOSX"]
+          or ansible_version.full is version("2.11", ">=", strict=True)
+          or is_mitogen

tests/ansible/integration/playbook_semantics/with_items.yml

@@ -3,30 +3,39 @@
 
 - name: integration/playbook_semantics/with_items.yml
   hosts: test-targets
+  gather_facts: true
   tasks:
+    - block:
+        - name: Spin up a few interpreters
+          become: true
+          vars:
+            ansible_become_user: "mitogen__user{{ item }}"
+          command:
+            cmd: whoami
+          with_sequence: start=1 end=3
+          register: first_run
+          changed_when: false
 
-    # TODO: https://github.com/dw/mitogen/issues/692
-    # - name: Spin up a few interpreters
-    #   shell: whoami
-    #   become: true
-    #   vars:
-    #     ansible_become_user: "mitogen__user{{item}}"
-    #   with_sequence: start=1 end=3
-    #   register: first_run
+        - name: Reuse them
+          become: true
+          vars:
+            ansible_become_user: "mitogen__user{{ item }}"
+          command:
+            cmd: whoami
+          with_sequence: start=1 end=3
+          register: second_run
+          changed_when: false
 
-    # - name: Reuse them
-    #   shell: whoami
-    #   become: true
-    #   vars:
-    #     ansible_become_user: "mitogen__user{{item}}"
-    #   with_sequence: start=1 end=3
-    #   register: second_run
-
-    # - name: Verify first and second run matches expected username.
-    #   assert:
-    #     that:
-    #     - first_run.results[item|int].stdout == ("mitogen__user%d" % (item|int + 1))
-    #     - first_run.results[item|int].stdout == second_run.results[item|int].stdout
-    #   with_sequence: start=0 end=2
-  tags:
-    - custom_python_new_style_module
+        - name: Verify first and second run matches expected username.
+          vars:
+            user_expected: "mitogen__user{{ item | int + 1 }}"
+          assert:
+            that:
+              - first_run.results[item | int].stdout == user_expected
+              - second_run.results[item | int].stdout == user_expected
+          with_sequence: start=0 end=2
+      when:
+        # https://github.com/ansible/ansible/pull/70785
+        - ansible_facts.distribution not in ["MacOSX"]
+          or ansible_version.full is version("2.11", ">=", strict=True)
+          or is_mitogen

tests/ansible/integration/ssh/all.yml

@@ -1,4 +1,5 @@
-- import_playbook: args.yml
+- import_playbook: args_by_inv.yml
+- import_playbook: args_by_play_taskvar.yml
 - import_playbook: config.yml
 - import_playbook: password.yml
 - import_playbook: timeouts.yml

tests/ansible/integration/ssh/args_by_inv.yml

@@ -1,12 +1,9 @@
-- name: integration/ssh/args.yml
+- name: integration/ssh/args_by_inv.yml
   hosts: issue905
   gather_facts: false
   tasks:
     # Test that ansible_ssh_common_args are templated; ansible_ssh_args &
     # ansible_ssh_extra_args aren't directly tested, we assume they're similar.
-    # FIXME This test currently relies on variables set in the host group.
-    #       Ideally they'd be set here, and the host group eliminated, but
-    #       Mitogen currently fails to template when defined in the play.
     # TODO Replace LocalCommand canary with SetEnv canary, to simplify test.
     #      Requires modification of sshd_config files to add AcceptEnv ...
     - name: Test templating of ansible_ssh_common_args et al

tests/ansible/integration/ssh/args_by_play_taskvar.yml

@@ -0,0 +1,52 @@
+- name: integration/ssh/args_by_play_taskvar.yml
+  hosts: tt_targets_bare
+  gather_facts: false
+  vars:
+    ansible_password: "{{ 'has_sudo_nopw_password' | trim }}"
+    ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}"
+    ansible_ssh_common_args: >-
+      -o PermitLocalCommand=yes
+      -o LocalCommand="touch {{ ssh_args_canary_file }}"
+    ansible_user: "{{ 'mitogen__has_sudo_nopw' | trim }}"
+    ssh_args_canary_file: "/tmp/ssh_args_by_play_taskvar_{{ inventory_hostname }}"
+  tasks:
+    # Test that ansible_ssh_common_args are templated; ansible_ssh_args &
+    # ansible_ssh_extra_args aren't directly tested, we assume they're similar.
+    # TODO Replace LocalCommand canary with SetEnv canary, to simplify test.
+    #      Requires modification of sshd_config files to add AcceptEnv ...
+    - name: Test templating of ansible_ssh_common_args et al, by play taskvars
+      block:
+        - name: Ensure no lingering canary files
+          file:
+            path: "{{ ssh_args_canary_file }}"
+            state: absent
+          delegate_to: localhost
+
+        - name: Reset connections to force new ssh execution
+          meta: reset_connection
+
+        - name: Perform SSH connection, to trigger side effect
+          ping:
+
+        - name: Stat for canary file created by side effect
+          stat:
+            path: "{{ ssh_args_canary_file }}"
+          delegate_to: localhost
+          register: ssh_args_by_play_taskvar_canary_stat
+
+        - assert:
+            that:
+              - ssh_args_by_play_taskvar_canary_stat.stat.exists == true
+            quiet: true
+            success_msg: "Canary found: {{ ssh_args_canary_file }}"
+            fail_msg: |
+              ssh_args_canary_file={{ ssh_args_canary_file }}
+              ssh_args_by_play_taskvar_canary_stat={{ ssh_args_by_play_taskvar_canary_stat }}
+      always:
+        - name: Cleanup canary files
+          file:
+            path: "{{ ssh_args_canary_file }}"
+            state: absent
+          delegate_to: localhost
+  tags:
+    - issue_905

tests/ansible/templates/test-targets.j2

@@ -36,7 +36,7 @@ ssh-common-args ansible_host={{ c.hostname }} ansible_port={{ c.port }} ansible_
 ansible_user=mitogen__has_sudo_nopw
 ansible_password=has_sudo_nopw_password
 ansible_ssh_common_args=-o PermitLocalCommand=yes -o LocalCommand="touch {{ '{{' }} ssh_args_canary_file {{ '}}' }}"
-ssh_args_canary_file=/tmp/ssh_args_{{ '{{' }} inventory_hostname {{ '}}' }}
+ssh_args_canary_file=/tmp/ssh_args_by_inv_{{ '{{' }} inventory_hostname {{ '}}' }}
 
 {% set tt = containers[0] %}
 
@@ -45,7 +45,6 @@ tt-bare
 
 [tt_targets_bare:vars]
 ansible_host={{ tt.hostname }}
-ansible_port={{ tt.port }}
 ansible_python_interpreter={{ tt.python_path }}
 
 [tt_become_bare]
@@ -59,6 +58,7 @@ ansible_python_interpreter={{ tt.python_path }}
 ansible_user=mitogen__has_sudo_nopw
 
 [tt_become_by_inv]
+tt-become-pass              ansible_become=true ansible_become_pass="{{ '{{' }} 'pw_required_password' | trim {{ '}}' }}" ansible_become_user=mitogen__pw_required
 tt-become-user              ansible_become=true ansible_become_user="{{ '{{' }} 'root' | trim {{ '}}' }}"
 
 [tt_become_by_inv:vars]

tests/image_prep/_user_accounts.yml

@@ -73,6 +73,7 @@
       - user:
           name: "mitogen__{{item}}"
           shell: /bin/bash
+          group: staff
           groups: |
             {{
                 ['com.apple.access_ssh'] +

tests/image_prep/ansible.cfg

@@ -6,6 +6,7 @@ retry_files_enabled = false
 display_args_to_stdout = True
 no_target_syslog = True
 host_key_checking = False
+stdout_callback = yaml
 
 [inventory]
 unparsed_is_fatal = true