From 06617f8231b4facc0cd7b751b72293a7d8744a5c Mon Sep 17 00:00:00 2001 From: Jonathan Rosser Date: Tue, 16 Jul 2024 15:38:40 +0100 Subject: [PATCH 01/21] ansible_mitogen: Handle unsafe paths in _remote_chmod This is missing from https://github.com/mitogen-hq/mitogen/commit/b822f20007ebe94106b15275962ea4cbbd8a0331 --- ansible_mitogen/mixins.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 0ba41aad..2cd97a3e 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -280,7 +280,9 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): paths, mode, sudoable) return self.fake_shell(lambda: mitogen.select.Select.all( self._connection.get_chain().call_async( - ansible_mitogen.target.set_file_mode, path, mode + ansible_mitogen.target.set_file_mode, + ansible_mitogen.utils.unsafe.cast(path), + mode, ) for path in paths )) From 5ab872f28930a70e03253d0fb97acbbee2a47430 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 29 Aug 2024 11:07:35 +0100 Subject: [PATCH 02/21] ansible_mitogen: Add regression test for ActionModuleMixin._remote_chmod() Adapted from Jonathon's reproducer in #1087. --- docs/changelog.rst | 2 + docs/contributors.rst | 1 + tests/ansible/regression/all.yml | 1 + .../issue_1087__template_streamerror.yml | 43 +++++++++++++++++++ tests/ansible/regression/template_test.yml | 28 ++++++++++++ tests/ansible/regression/templates/foo.bar.j2 | 1 + 6 files changed, 76 insertions(+) create mode 100644 tests/ansible/regression/issue_1087__template_streamerror.yml create mode 100644 tests/ansible/regression/template_test.yml create mode 100644 tests/ansible/regression/templates/foo.bar.j2 diff --git a/docs/changelog.rst b/docs/changelog.rst index 0d1a14a2..63b4da12 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,8 @@ To avail of fixes in an unreleased version, please download a ZIP file Unreleased ---------- +* :gh:issue:`1087` Fix :exc:`mitogen.core.StreamError` when Ansible template + module is called with a ``dest:`` filename that has an extension v0.3.9 (2024-08-13) diff --git a/docs/contributors.rst b/docs/contributors.rst index ed7fef11..1a590869 100644 --- a/docs/contributors.rst +++ b/docs/contributors.rst @@ -125,6 +125,7 @@ sponsorship and outstanding future-thinking of its early adopters.
  • rkrzr
  • jgadling
  • John F Wall — Making Ansible Great with Massive Parallelism
    • +
  • Jonathan Rosser
  • KennethC
  • Luca Berruti
  • Lewis Bellwood — Happy to be apart of a great project.
    • diff --git a/tests/ansible/regression/all.yml b/tests/ansible/regression/all.yml index 31541e9f..a4272805 100644 --- a/tests/ansible/regression/all.yml +++ b/tests/ansible/regression/all.yml @@ -16,3 +16,4 @@ - import_playbook: issue_776__load_plugins_called_twice.yml - import_playbook: issue_952__ask_become_pass.yml - import_playbook: issue_1066__add_host__host_key_checking.yml +- import_playbook: issue_1087__template_streamerror.yml diff --git a/tests/ansible/regression/issue_1087__template_streamerror.yml b/tests/ansible/regression/issue_1087__template_streamerror.yml new file mode 100644 index 00000000..fa950ea4 --- /dev/null +++ b/tests/ansible/regression/issue_1087__template_streamerror.yml @@ -0,0 +1,43 @@ +- name: regression/issue_1087__template_streamerror.yml + # Ansible's template module has been seen to raise mitogen.core.StreamError + # iif there is a with_items loop and the destination path has an extension. + # This printed an error message and left file permissions incorrect, + # but did not cause the task/playbook to fail. + hosts: test-targets + gather_facts: false + become: false + vars: + foos: + - dest: /tmp/foo + - dest: /tmp/foo.txt + foo: Foo + bar: Bar + tasks: + - block: + - name: Test template does not cause StreamError + delegate_to: localhost + run_once: true + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" + command: + cmd: > + ansible-playbook + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} + regression/template_test.yml + chdir: ../ + register: issue_1087_cmd + failed_when: + - issue_1087_cmd is failed + or issue_1087_cmd.stdout is search('ERROR|mitogen\.core\.CallError') + or issue_1087_cmd.stderr is search('ERROR|mitogen\.core\.CallError') + + always: + - name: Cleanup + file: + path: "{{ item.dest }}" + state: absent + with_items: "{{ foos }}" + tags: + - issue_1087 diff --git a/tests/ansible/regression/template_test.yml b/tests/ansible/regression/template_test.yml new file mode 100644 index 00000000..0b7dd36d --- /dev/null +++ b/tests/ansible/regression/template_test.yml @@ -0,0 +1,28 @@ +- name: regression/template_test.yml + # Ansible's template module has been seen to raise mitogen.core.StreamError + # iif there is a with_items loop and the destination path has an extension + hosts: test-targets + gather_facts: false + become: false + vars: + foos: + - dest: /tmp/foo + - dest: /tmp/foo.txt + foo: Foo + bar: Bar + tasks: + - block: + - name: Template files + template: + src: foo.bar.j2 + dest: "{{ item.dest }}" + mode: u=rw,go=r + # This has to be with_items, loop: doesn't trigger the bug + with_items: "{{ foos }}" + + always: + - name: Cleanup + file: + path: "{{ item.dest }}" + state: absent + with_items: "{{ foos }}" diff --git a/tests/ansible/regression/templates/foo.bar.j2 b/tests/ansible/regression/templates/foo.bar.j2 new file mode 100644 index 00000000..ca51a6f4 --- /dev/null +++ b/tests/ansible/regression/templates/foo.bar.j2 @@ -0,0 +1 @@ +A {{ foo }} walks into a {{ bar }}. Ow! From ce1accedbce1d841d2fa88676fabbdaa1487f92f Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 30 Aug 2024 16:00:52 +0100 Subject: [PATCH 03/21] tests: Refactor Ansible copy integration tests to be loop driven This is in anticipation of #1110, which only exhibits inside a with_items: loop. For this refactor `loop:` is used, to confirm the refactored tests are still correct. A subsequent commit will change them to with_items. The content of the files and their SHA1 checksums are unchanged. --- tests/ansible/integration/action/copy.yml | 125 +++++++++++----------- 1 file changed, 62 insertions(+), 63 deletions(-) diff --git a/tests/ansible/integration/action/copy.yml b/tests/ansible/integration/action/copy.yml index 73f3bd1e..cbebe3ab 100644 --- a/tests/ansible/integration/action/copy.yml +++ b/tests/ansible/integration/action/copy.yml @@ -2,91 +2,90 @@ - name: integration/action/copy.yml hosts: test-targets - tasks: - - name: Create tiny file - copy: - dest: /tmp/copy-tiny-file - content: - this is a tiny file. - delegate_to: localhost - run_once: true + vars: + sourced_files: + - src: /tmp/copy-tiny-file + dest: /tmp/copy-tiny-file.out + content: this is a tiny file. + expected_checksum: f29faa9a6f19a700a941bf2aa5b281643c4ec8a0 + - src: /tmp/copy-large-file + dest: /tmp/copy-large-file.out + content: "{{ 'x' * 200000 }}" + expected_checksum: 62951f943c41cdd326e5ce2b53a779e7916a820d + + inline_files: + - dest: /tmp/copy-tiny-inline-file.out + content: tiny inline content + expected_checksum: b26dd6444595e2bdb342aa0a91721b57478b5029 + - dest: /tmp/copy-large-inline-file.out + content: | + {{ 'y' * 200000 }} + expected_checksum: d675f47e467eae19e49032a2cc39118e12a6ee72 - - name: Create large file + files: "{{ sourced_files + inline_files }}" + tasks: + - name: Create sourced files copy: - dest: /tmp/copy-large-file - # Must be larger than Connection.SMALL_SIZE_LIMIT. - content: "{% for x in range(200000) %}x{% endfor %}" + dest: "{{ item.src }}" + content: "{{ item.content }}" + mode: u=rw,go=r + loop: "{{ sourced_files }}" + loop_control: + label: "{{ item.src }}" delegate_to: localhost run_once: true - - name: Cleanup copied files + - name: Cleanup lingering destination files file: + path: "{{ item.dest }}" state: absent - path: "{{item}}" - with_items: - - /tmp/copy-tiny-file.out - - /tmp/copy-large-file.out - - /tmp/copy-tiny-inline-file.out - - /tmp/copy-large-inline-file.out + loop: "{{ files }}" + loop_control: + label: "{{ item.dest }}" - - name: Copy large file + - name: Copy sourced files copy: - dest: /tmp/copy-large-file.out - src: /tmp/copy-large-file - - - name: Copy tiny file - copy: - dest: /tmp/copy-tiny-file.out - src: /tmp/copy-tiny-file + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: u=rw,go=r + loop: "{{ sourced_files }}" + loop_control: + label: "{{ item.dest }}" - - name: Copy tiny inline file + - name: Copy inline files copy: - dest: /tmp/copy-tiny-inline-file.out - content: "tiny inline content" - - - name: Copy large inline file - copy: - dest: /tmp/copy-large-inline-file.out - content: | - {% for x in range(200000) %}y{% endfor %} + dest: "{{ item.dest }}" + content: "{{ item.content }}" + mode: u=rw,go=r + loop: "{{ inline_files }}" + loop_control: + label: "{{ item.dest }}" # stat results - name: Stat copied files stat: - path: "{{item}}" - with_items: - - /tmp/copy-tiny-file.out - - /tmp/copy-large-file.out - - /tmp/copy-tiny-inline-file.out - - /tmp/copy-large-inline-file.out + path: "{{ item.dest }}" + loop: "{{ files }}" + loop_control: + label: "{{ item.dest }}" register: stat - assert: that: - - stat.results[0].stat.checksum == "f29faa9a6f19a700a941bf2aa5b281643c4ec8a0" - - stat.results[1].stat.checksum == "62951f943c41cdd326e5ce2b53a779e7916a820d" - - stat.results[2].stat.checksum == "b26dd6444595e2bdb342aa0a91721b57478b5029" - - stat.results[3].stat.checksum == "d675f47e467eae19e49032a2cc39118e12a6ee72" - fail_msg: stat={{stat}} + - item.stat.checksum == item.item.expected_checksum + quiet: true # Avoid spamming stdout with 400 kB of item.item.content + fail_msg: item={{ item }} + loop: "{{ stat.results }}" + loop_control: + label: "{{ item.stat.path }}" - - name: Cleanup files + - name: Cleanup destination files file: + path: "{{ item.dest }}" state: absent - path: "{{item}}" - with_items: - - /tmp/copy-tiny-file - - /tmp/copy-tiny-file.out - - /tmp/copy-no-mode - - /tmp/copy-no-mode.out - - /tmp/copy-with-mode - - /tmp/copy-with-mode.out - - /tmp/copy-large-file - - /tmp/copy-large-file.out - - /tmp/copy-tiny-inline-file.out - - /tmp/copy-large-inline-file - - /tmp/copy-large-inline-file.out - - # end of cleaning out files (again) + loop: "{{ files }}" + loop_control: + label: "{{ item.dest }}" tags: - copy From 0bd3c6cba50bf9e607aa13432586af8af4fe51ba Mon Sep 17 00:00:00 2001 From: Jonathan Rosser Date: Wed, 28 Aug 2024 08:24:35 +0100 Subject: [PATCH 04/21] Fix AnsibleUnsafeText when copying files larger than SMALL_FILE_LIMIT Small files are carried in-band in the communication between controller and remote, with larger files being copied by falling back to a more traditional ansible put_file mechanism. This large file code path was missed in b822f20. --- ansible_mitogen/connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 6bdf11ba..a3f66eac 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -1129,6 +1129,6 @@ class Connection(ansible.plugins.connection.ConnectionBase): self.get_chain().call( ansible_mitogen.target.transfer_file, context=self.binding.get_child_service_context(), - in_path=in_path, - out_path=out_path + in_path=ansible_mitogen.utils.unsafe.cast(in_path), + out_path=ansible_mitogen.utils.unsafe.cast(out_path) ) From 5af6534a70eb03a71c5a181bd2abcf77e2be903e Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 30 Aug 2024 17:19:20 +0100 Subject: [PATCH 05/21] tests: Test AnsibleUnsafeText when copying files larger SMALL_FILE_LIMIT The bug was fixed in a previous commit by Jonathan Rosser. This adds testing. The bug is only triggered when the copy module is used inside a `with_items:` loop and the destination filename has an extension. A `loop:` loop is not sufficient. refs #1110 --- docs/changelog.rst | 3 +++ tests/ansible/integration/action/copy.yml | 17 ++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 63b4da12..4fc177c1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,6 +23,9 @@ Unreleased * :gh:issue:`1087` Fix :exc:`mitogen.core.StreamError` when Ansible template module is called with a ``dest:`` filename that has an extension +* :gh:issue:`1110` Fix :exc:`mitogen.core.StreamError` when Ansible copy + module is called with a file larger than 124 kibibytes + (:data:`ansible_mitogen.connection.Connection.SMALL_FILE_LIMIT`) v0.3.9 (2024-08-13) diff --git a/tests/ansible/integration/action/copy.yml b/tests/ansible/integration/action/copy.yml index cbebe3ab..edaa3e49 100644 --- a/tests/ansible/integration/action/copy.yml +++ b/tests/ansible/integration/action/copy.yml @@ -1,4 +1,6 @@ # Verify copy module for small and large files, and inline content. +# To exercise https://github.com/mitogen-hq/mitogen/pull/1110 destination +# files must have extensions and loops must use `with_items:`. - name: integration/action/copy.yml hosts: test-targets @@ -29,7 +31,7 @@ dest: "{{ item.src }}" content: "{{ item.content }}" mode: u=rw,go=r - loop: "{{ sourced_files }}" + with_items: "{{ sourced_files }}" loop_control: label: "{{ item.src }}" delegate_to: localhost @@ -39,7 +41,7 @@ file: path: "{{ item.dest }}" state: absent - loop: "{{ files }}" + with_items: "{{ files }}" loop_control: label: "{{ item.dest }}" @@ -48,7 +50,7 @@ src: "{{ item.src }}" dest: "{{ item.dest }}" mode: u=rw,go=r - loop: "{{ sourced_files }}" + with_items: "{{ sourced_files }}" loop_control: label: "{{ item.dest }}" @@ -57,7 +59,7 @@ dest: "{{ item.dest }}" content: "{{ item.content }}" mode: u=rw,go=r - loop: "{{ inline_files }}" + with_items: "{{ inline_files }}" loop_control: label: "{{ item.dest }}" @@ -66,7 +68,7 @@ - name: Stat copied files stat: path: "{{ item.dest }}" - loop: "{{ files }}" + with_items: "{{ files }}" loop_control: label: "{{ item.dest }}" register: stat @@ -76,7 +78,7 @@ - item.stat.checksum == item.item.expected_checksum quiet: true # Avoid spamming stdout with 400 kB of item.item.content fail_msg: item={{ item }} - loop: "{{ stat.results }}" + with_items: "{{ stat.results }}" loop_control: label: "{{ item.stat.path }}" @@ -84,8 +86,9 @@ file: path: "{{ item.dest }}" state: absent - loop: "{{ files }}" + with_items: "{{ files }}" loop_control: label: "{{ item.dest }}" tags: - copy + - issue_1110 From 46c9f772d84e6a42c8774ad38a5be6f804f06f39 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 3 Sep 2024 18:25:56 +0100 Subject: [PATCH 06/21] tests: Simplify Ansible ssh password tests, test priority This - Removes the indirection of calling ansible in a sub-shell - Includes vanilla Ansible, which was previously skipped - Tests whether ansible_ssh_pass overrides ansible_password, as it should As a one off I've the new tests against vanilla Ansible 2.10 through Ansible 10, to confirm the baseline priorities have remained unchanged all releases currently supported by Mitogen 0.3.x. --- tests/ansible/integration/ssh/all.yml | 1 + tests/ansible/integration/ssh/password.yml | 51 ++++++++ tests/ansible/integration/ssh/variables.yml | 128 -------------------- 3 files changed, 52 insertions(+), 128 deletions(-) create mode 100644 tests/ansible/integration/ssh/password.yml diff --git a/tests/ansible/integration/ssh/all.yml b/tests/ansible/integration/ssh/all.yml index a8335ab7..1b0f36e4 100644 --- a/tests/ansible/integration/ssh/all.yml +++ b/tests/ansible/integration/ssh/all.yml @@ -1,3 +1,4 @@ - import_playbook: config.yml +- import_playbook: password.yml - import_playbook: timeouts.yml - import_playbook: variables.yml diff --git a/tests/ansible/integration/ssh/password.yml b/tests/ansible/integration/ssh/password.yml new file mode 100644 index 00000000..cf9396e0 --- /dev/null +++ b/tests/ansible/integration/ssh/password.yml @@ -0,0 +1,51 @@ +- name: integration/ssh/password.yml + hosts: test-targets[0] + gather_facts: false + vars: + ansible_user: mitogen__user1 + tasks: + - meta: reset_connection + - name: ansible_password + vars: + ansible_password: user1_password + ping: + + - meta: reset_connection + - name: ansible_ssh_pass + vars: + ansible_ssh_pass: user1_password + ping: + + - meta: reset_connection + - name: absent password should fail + ping: + ignore_errors: true + ignore_unreachable: true + register: ssh_no_password_result + - assert: + that: + - ssh_no_password_result.unreachable == True + fail_msg: ssh_no_password_result={{ ssh_no_password_result }} + + - meta: reset_connection + - name: ansible_ssh_pass should override ansible_password + ping: + vars: + ansible_password: wrong + ansible_ssh_pass: user1_password + + # Tests that ansible_ssh_pass has priority over ansible_password + # and that a wrong password causes a target to be marked unreachable. + - meta: reset_connection + - name: ansible_password should not override + vars: + ansible_password: user1_password + ansible_ssh_pass: wrong + ping: + ignore_errors: true + ignore_unreachable: true + register: ssh_wrong_password_result + - assert: + that: + - ssh_wrong_password_result.unreachable == True + fail_msg: ssh_wrong_password_result={{ ssh_wrong_password_result }} diff --git a/tests/ansible/integration/ssh/variables.yml b/tests/ansible/integration/ssh/variables.yml index d2fa683b..9f5b16bc 100644 --- a/tests/ansible/integration/ssh/variables.yml +++ b/tests/ansible/integration/ssh/variables.yml @@ -13,134 +13,6 @@ -o "ControlPath /tmp/mitogen-ansible-test-{{18446744073709551615|random}}" tasks: - - include_tasks: ../_mitogen_only.yml - - - name: ansible_ssh_user, ansible_ssh_pass - shell: > - ANSIBLE_ANY_ERRORS_FATAL=false - ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" - ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" - ansible -m shell -a whoami - {% for inv in ansible_inventory_sources %} - -i "{{ inv }}" - {% endfor %} - test-targets - -e ansible_ssh_user=mitogen__has_sudo - -e ansible_ssh_pass=has_sudo_password - args: - chdir: ../.. - register: out - - - name: ansible_ssh_user, wrong ansible_ssh_pass - shell: > - ANSIBLE_ANY_ERRORS_FATAL=false - ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" - ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" - ansible -m shell -a whoami - {% for inv in ansible_inventory_sources %} - -i "{{ inv }}" - {% endfor %} - test-targets - -e ansible_ssh_user=mitogen__has_sudo - -e ansible_ssh_pass=wrong_password - -e ansible_python_interpreter=python3000 - args: - chdir: ../.. - register: out - ignore_errors: true - - - assert: - that: - - out.rc == 4 # ansible.executor.task_queue_manager.TaskQueueManager.RUN_UNREACHABLE_HOSTS - fail_msg: out={{out}} - - - - name: ansible_user, ansible_ssh_pass - shell: > - ANSIBLE_ANY_ERRORS_FATAL=false - ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" - ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" - ansible -m shell -a whoami - {% for inv in ansible_inventory_sources %} - -i "{{ inv }}" - {% endfor %} - test-targets - -e ansible_user=mitogen__has_sudo - -e ansible_ssh_pass=has_sudo_password - args: - chdir: ../.. - register: out - - - name: ansible_user, wrong ansible_ssh_pass - shell: > - ANSIBLE_ANY_ERRORS_FATAL=false - ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" - ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" - ansible -m shell -a whoami - {% for inv in ansible_inventory_sources %} - -i "{{ inv }}" - {% endfor %} - test-targets - -e ansible_user=mitogen__has_sudo - -e ansible_ssh_pass=wrong_password - -e ansible_python_interpreter=python3000 - args: - chdir: ../.. - register: out - ignore_errors: true - - - assert: - that: - - out.rc == 4 # ansible.executor.task_queue_manager.TaskQueueManager.RUN_UNREACHABLE_HOSTS - fail_msg: out={{out}} - - - - name: ansible_user, ansible_password - shell: > - ANSIBLE_ANY_ERRORS_FATAL=false - ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" - ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" - ansible -m shell -a whoami - {% for inv in ansible_inventory_sources %} - -i "{{ inv }}" - {% endfor %} - test-targets - -e ansible_user=mitogen__has_sudo - -e ansible_password=has_sudo_password - args: - chdir: ../.. - register: out - - - name: ansible_user, wrong ansible_password - shell: > - ANSIBLE_ANY_ERRORS_FATAL=false - ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" - ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" - ansible -m shell -a whoami - {% for inv in ansible_inventory_sources %} - -i "{{ inv }}" - {% endfor %} - test-targets - -e ansible_user=mitogen__has_sudo - -e ansible_password=wrong_password - -e ansible_python_interpreter=python3000 - args: - chdir: ../.. - register: out - ignore_errors: true - - - assert: - that: - - out.rc == 4 # ansible.executor.task_queue_manager.TaskQueueManager.RUN_UNREACHABLE_HOSTS - fail_msg: out={{out}} - - - name: setup ansible_ssh_private_key_file shell: chmod 0600 ../data/docker/mitogen__has_sudo_pubkey.key args: From be288ad3985787f4e42132da4ede2f68b3fe05a6 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 22 Aug 2022 13:01:51 +0200 Subject: [PATCH 07/21] patch #509 : ansible_ssh_common_args issues --- ansible_mitogen/transport_config.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index 3ab623f8..144de563 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -498,12 +498,13 @@ class PlayContextSpec(Spec): ) def ssh_args(self): + local_vars = self._task_vars.get("hostvars", {}).get(self._inventory_name, {}) return [ mitogen.core.to_text(term) for s in ( - C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})), - C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})), - C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})) + C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=local_vars), + C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=local_vars), + C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=local_vars) ) for term in ansible.utils.shlex.shlex_split(s or '') ] @@ -738,12 +739,13 @@ class MitogenViaSpec(Spec): ) def ssh_args(self): + local_vars = self._task_vars.get("hostvars", {}).get(self._inventory_name, {}) return [ mitogen.core.to_text(term) for s in ( - C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})), - C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})), - C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})) + C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=local_vars), + C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=local_vars), + C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=local_vars) ) for term in ansible.utils.shlex.shlex_split(s) if s From f3097b5743d543720d9416f490f4ce2b7be952d0 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 5 Sep 2024 17:50:09 +0100 Subject: [PATCH 08/21] ci: Template Ansible test-targets inventory with Jinja2 --- .ci/ansible_tests.py | 44 ++++++++----------------- tests/ansible/templates/test-targets.j2 | 28 ++++++++++++++++ 2 files changed, 42 insertions(+), 30 deletions(-) create mode 100644 tests/ansible/templates/test-targets.j2 diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 0dd978c4..102eda9c 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -6,11 +6,13 @@ import glob import os import signal import sys -import textwrap + +import jinja2 import ci_lib +TEMPLATES_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible/templates') TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible') HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts') @@ -52,37 +54,19 @@ with ci_lib.Fold('job_setup'): distros[container['distro']].append(container['name']) families[container['family']].append(container['name']) + jinja_env = jinja2.Environment( + loader=jinja2.FileSystemLoader(searchpath=TEMPLATES_DIR), + lstrip_blocks=True, # Remove spaces and tabs from before a block + trim_blocks=True, # Remove first newline after a block + ) + inventory_template = jinja_env.get_template('test-targets.j2') inventory_path = os.path.join(HOSTS_DIR, 'target') + with open(inventory_path, 'w') as fp: - fp.write('[test-targets]\n') - fp.writelines( - "%(name)s " - "ansible_host=%(hostname)s " - "ansible_port=%(port)s " - "ansible_python_interpreter=%(python_path)s " - "ansible_user=mitogen__has_sudo_nopw " - "ansible_password=has_sudo_nopw_password" - "\n" - % container - for container in containers - ) - - for distro, hostnames in sorted(distros.items(), key=lambda t: t[0]): - fp.write('\n[%s]\n' % distro) - fp.writelines('%s\n' % name for name in hostnames) - - for family, hostnames in sorted(families.items(), key=lambda t: t[0]): - fp.write('\n[%s]\n' % family) - fp.writelines('%s\n' % name for name in hostnames) - - fp.write(textwrap.dedent( - ''' - [linux:children] - test-targets - - [linux_containers:children] - test-targets - ''' + fp.write(inventory_template.render( + containers=containers, + distros=distros, + families=families, )) ci_lib.dump_file(inventory_path) diff --git a/tests/ansible/templates/test-targets.j2 b/tests/ansible/templates/test-targets.j2 new file mode 100644 index 00000000..f61a3c78 --- /dev/null +++ b/tests/ansible/templates/test-targets.j2 @@ -0,0 +1,28 @@ +[test-targets] +{% for c in containers %} +{{ c.name }} ansible_host={{ c.hostname }} ansible_port={{ c.port }} ansible_python_interpreter={{ c.python_path }} +{% endfor %} + +[test-targets:vars] +ansible_user=mitogen__has_sudo_nopw +ansible_password=has_sudo_nopw_password + +{% for distro, hostnames in distros | dictsort %} +[{{ distro }}] +{% for hostname in hostnames %} +{{ hostname }} +{% endfor %} +{% endfor %} + +{% for family, hostnames in families | dictsort %} +[{{ family }}] +{% for hostname in hostnames %} +{{ hostname }} +{% endfor %} +{% endfor %} + +[linux:children] +test-targets + +[linux_containers:children] +test-targets From 79ed797badd039f1ba719f99a1ac2ebf8940798c Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 6 Sep 2024 11:51:58 +0100 Subject: [PATCH 09/21] tests: Test templating of ansible_ssh_common_args et al refs #905 --- docs/changelog.rst | 3 ++ docs/contributors.rst | 1 + tests/ansible/hosts/default.hosts | 7 ++++ tests/ansible/integration/ssh/all.yml | 1 + tests/ansible/integration/ssh/args.yml | 48 +++++++++++++++++++++++++ tests/ansible/templates/test-targets.j2 | 11 ++++++ 6 files changed, 71 insertions(+) create mode 100644 tests/ansible/integration/ssh/args.yml diff --git a/docs/changelog.rst b/docs/changelog.rst index 4fc177c1..8cc24424 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -26,6 +26,9 @@ Unreleased * :gh:issue:`1110` Fix :exc:`mitogen.core.StreamError` when Ansible copy module is called with a file larger than 124 kibibytes (:data:`ansible_mitogen.connection.Connection.SMALL_FILE_LIMIT`) +* :gh:issue:`905` Initial support for templated ``ansible_ssh_args``, + ``ansible_ssh_common_args``, and ``ansible_ssh_extra_args`` variables. + NB: play or task scoped variables will probably still fail. v0.3.9 (2024-08-13) diff --git a/docs/contributors.rst b/docs/contributors.rst index 1a590869..e40607a0 100644 --- a/docs/contributors.rst +++ b/docs/contributors.rst @@ -116,6 +116,7 @@ sponsorship and outstanding future-thinking of its early adopters.