From ec05e542b46dcfc2a418943c105f03dcfb7c5b69 Mon Sep 17 00:00:00 2001 From: Philippe Kueck Date: Wed, 22 Mar 2023 12:36:14 +0100 Subject: [PATCH 1/6] Fix 'ansible_host_key_checking' and 'ansible_ssh_host_key_checking' for adding new hosts to the inventory using 'add_hosts' Co-authored-by: Alex Willmer --- ansible_mitogen/connection.py | 2 +- ansible_mitogen/transport_config.py | 23 +++++++++++++++++++++++ docs/changelog.rst | 1 + 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index dfc3aec4..6bdf11ba 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -119,7 +119,7 @@ def _connect_ssh(spec): """ Return ContextService arguments for an SSH connection. """ - if C.HOST_KEY_CHECKING: + if spec.host_key_checking(): check_host_keys = 'enforce' else: check_host_keys = 'ignore' diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index 1fc1e80a..3ab623f8 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -67,6 +67,7 @@ import ansible.utils.shlex import ansible.constants as C from ansible.module_utils.six import with_metaclass +from ansible.module_utils.parsing.convert_bool import boolean # this was added in Ansible >= 2.8.0; fallback to the default interpreter if necessary try: @@ -245,6 +246,12 @@ class Spec(with_metaclass(abc.ABCMeta, object)): Path to the Python interpreter on the target machine. """ + @abc.abstractmethod + def host_key_checking(self): + """ + Whether or not to check the keys of the target machine + """ + @abc.abstractmethod def private_key_file(self): """ @@ -466,6 +473,14 @@ class PlayContextSpec(Spec): action=self._action, rediscover_python=rediscover_python) + def host_key_checking(self): + def candidates(): + yield self._connection.get_task_var('ansible_ssh_host_key_checking') + yield self._connection.get_task_var('ansible_host_key_checking') + yield C.HOST_KEY_CHECKING + val = next((v for v in candidates() if v is not None), True) + return boolean(val) + def private_key_file(self): return self._play_context.private_key_file @@ -692,6 +707,14 @@ class MitogenViaSpec(Spec): action=self._action, rediscover_python=rediscover_python) + def host_key_checking(self): + def candidates(): + yield self._host_vars.get('ansible_ssh_host_key_checking') + yield self._host_vars.get('ansible_host_key_checking') + yield C.HOST_KEY_CHECKING + val = next((v for v in candidates() if v is not None), True) + return boolean(val) + def private_key_file(self): # TODO: must come from PlayContext too. return ( diff --git a/docs/changelog.rst b/docs/changelog.rst index 407a8c78..5d77910e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -24,6 +24,7 @@ Unreleased * :gh:issue:`952` Fix Ansible `--ask-become-pass`, add test coverage * :gh:issue:`957` Fix Ansible exception when executing against 10s of hosts "ValueError: filedescriptor out of range in select()" +* :gh:issue:`1066` Support Ansible `ansible_host_key_checking` & `ansible_ssh_host_key_checking` v0.3.7 (2024-04-08) From d2eefc06aa45a5f5403a551faf6d743fc6c759dc Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 6 May 2024 11:44:43 +0100 Subject: [PATCH 2/6] tests: Add regression for add_host with host_key_checking refs #1066 --- tests/ansible/ansible.cfg | 11 ++- tests/ansible/hosts/transport_config.hosts | 6 ++ .../integration/transport_config/all.yml | 1 + .../transport_config/host_key_checking.yml | 94 +++++++++++++++++++ tests/ansible/regression/all.yml | 1 + ...ssue_1066__add_host__host_key_checking.yml | 67 +++++++++++++ tests/ansible/setup/report_controller.yml | 4 +- 7 files changed, 180 insertions(+), 4 deletions(-) create mode 100644 tests/ansible/integration/transport_config/host_key_checking.yml create mode 100644 tests/ansible/regression/issue_1066__add_host__host_key_checking.yml diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index c34dd219..537b5059 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -1,9 +1,12 @@ [defaults] any_errors_fatal = true -# callback_whitelist naming will be deprecated in ansible-core >= 2.15. -# callbacks_enabled naming was added in ansible-core 2.11 +# callbacks_enabled was added in Ansible 4 (ansible-core 2.11). # profile_tasks: Displays timing for each task and summary table of top N tasks # timer: Displays "Playbook run took 0 days, 0 hours, ..." +callbacks_enabled = + profile_tasks, + timer +# callback_whitelist was deprecated in Ansible >= 8 (ansible-core >= 2.15). callback_whitelist = profile_tasks, timer @@ -37,7 +40,9 @@ no_target_syslog = True # Required by integration/ssh/timeouts.yml timeout = 30 -# On Travis, paramiko check fails due to host key checking enabled. +# Ideally this would be true here and and overridden for hosts/groups. However +# ansible_host_key_checking don't work on Vanilla Ansible 2.10, even for +# static inventory hosts (ansible/ansible#49254, ansible/ansible#73708) host_key_checking = False [inventory] diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts index 1c1c2e10..5d5c3834 100644 --- a/tests/ansible/hosts/transport_config.hosts +++ b/tests/ansible/hosts/transport_config.hosts @@ -13,6 +13,7 @@ tc_become tc_become_method tc_become_pass tc_become_user +tc_host_key_checking tc_password tc_port tc_remote_addr @@ -74,6 +75,11 @@ tc-become-pass-password ansible_become_password=apassword tc-become-pass-pass ansible_become_pass=apass tc-become-pass-both ansible_become_pass=bpass ansible_become_password=bpassword +[tc_host_key_checking] +tc-hkc-unset +tc-hkc-host-key-checking ansible_host_key_checking=true +tc-hkc-ssh-host-key-checking ansible_ssh_host_key_checking=true + [tc_port] tc-port-unset tc-port-explicit-port ansible_port=1234 diff --git a/tests/ansible/integration/transport_config/all.yml b/tests/ansible/integration/transport_config/all.yml index 548e7f7e..b486549b 100644 --- a/tests/ansible/integration/transport_config/all.yml +++ b/tests/ansible/integration/transport_config/all.yml @@ -2,6 +2,7 @@ - import_playbook: become_pass.yml - import_playbook: become_user.yml - import_playbook: become.yml +- import_playbook: host_key_checking.yml - import_playbook: password.yml - import_playbook: port.yml - import_playbook: python_path.yml diff --git a/tests/ansible/integration/transport_config/host_key_checking.yml b/tests/ansible/integration/transport_config/host_key_checking.yml new file mode 100644 index 00000000..b15e36b1 --- /dev/null +++ b/tests/ansible/integration/transport_config/host_key_checking.yml @@ -0,0 +1,94 @@ +# Each case is followed by mitogen_via= case to test hostvars method. + +- name: integration/transport_config/host_key_checking.yml + hosts: tc-hkc-unset + tasks: + - include_tasks: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result | length == 1 + - out.result[0].method == "ssh" + - out.result[0].kwargs.check_host_keys == "ignore" + fail_msg: out={{ out }} + tags: + - mitogen_only + +- hosts: tc-hkc-unset + vars: + mitogen_via: tc-hkc-host-key-checking + tasks: + - include_tasks: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result | length == 2 + - out.result[0].method == "ssh" + - out.result[0].kwargs.check_host_keys == "enforce" + - out.result[1].method == "ssh" + - out.result[1].kwargs.check_host_keys == "ignore" + fail_msg: out={{ out }} + tags: + - mitogen_only + + +- hosts: tc-hkc-host-key-checking + tasks: + - include_tasks: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result | length == 1 + - out.result[0].method == "ssh" + - out.result[0].kwargs.check_host_keys == "enforce" + fail_msg: out={{ out }} + tags: + - mitogen_only + +- hosts: tc-hkc-host-key-checking + vars: + mitogen_via: tc-hkc-unset + tasks: + - include_tasks: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result | length == 2 + - out.result[0].method == "ssh" + - out.result[0].kwargs.check_host_keys == "ignore" + - out.result[1].method == "ssh" + - out.result[1].kwargs.check_host_keys == "enforce" + fail_msg: out={{ out }} + tags: + - mitogen_only + + +- hosts: tc-hkc-ssh-host-key-checking + tasks: + - include_tasks: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result | length == 1 + - out.result[0].method == "ssh" + - out.result[0].kwargs.check_host_keys == "enforce" + fail_msg: out={{ out }} + tags: + - mitogen_only + +- hosts: tc-hkc-ssh-host-key-checking + vars: + mitogen_via: tc-hkc-unset + tasks: + - include_tasks: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result | length == 2 + - out.result[0].method == "ssh" + - out.result[0].kwargs.check_host_keys == "ignore" + - out.result[1].method == "ssh" + - out.result[1].kwargs.check_host_keys == "enforce" + fail_msg: out={{ out }} + tags: + - mitogen_only diff --git a/tests/ansible/regression/all.yml b/tests/ansible/regression/all.yml index 477caccd..31541e9f 100644 --- a/tests/ansible/regression/all.yml +++ b/tests/ansible/regression/all.yml @@ -15,3 +15,4 @@ - import_playbook: issue_655__wait_for_connection_error.yml - import_playbook: issue_776__load_plugins_called_twice.yml - import_playbook: issue_952__ask_become_pass.yml +- import_playbook: issue_1066__add_host__host_key_checking.yml diff --git a/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml b/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml new file mode 100644 index 00000000..4b4609b6 --- /dev/null +++ b/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml @@ -0,0 +1,67 @@ +- name: regression/issue_1066__add_host__host_key_checking.yml + hosts: test-targets[0] + gather_facts: false + become: false + tasks: + - name: Add hosts dynamically + add_host: + name: "{{ item.name }}" + ansible_host_key_checking: "{{ item.host_key_checking | default(omit) }}" + ansible_ssh_host_key_checking: "{{ item.host_ssh_key_checking | default(omit) }}" + ansible_host: "{{ hostvars[inventory_hostname].ansible_host | default(omit) }}" + ansible_password: "{{ hostvars[inventory_hostname].ansible_password | default(omit) }}" + ansible_port: "{{ hostvars[inventory_hostname].ansible_port | default(omit) }}" + ansible_python_interpreter: "{{ hostvars[inventory_hostname].ansible_python_interpreter | default(omit) }}" + ansible_user: "{{ hostvars[inventory_hostname].ansible_user | default(omit) }}" + loop: + - {name: issue-1066-host-hkc-false, host_key_checking: false} + - {name: issue-1066-host-hkc-true, host_key_checking: true} + - {name: issue-1066-host-hskc-false, host_ssh_key_checking: false} + - {name: issue-1066-host-hskc-true, host_ssh_key_checking: true} + delegate_to: localhost + tags: + - issue_1066 + +- name: regression/issue_1066__add_host__host_key_checking.yml + hosts: issue-1066-host-* + gather_facts: false + become: false + serial: 1 + tasks: + - meta: reset_connection + + # The host key might be in ~/.ssh/known_hosts. If it's removed then no + # problem - test-targets hosts have host_key_checking=false. + - name: Remove existing host keys + known_hosts: + name: "{{ ansible_host }}" + state: absent + delegate_to: localhost + + - name: Ping dynamically added hosts + ping: + ignore_errors: true + ignore_unreachable: true + register: issue_1066_ping + + - debug: + var: issue_1066_ping + + - name: Confirm dynamically added hosts are/are not reachable + vars: + expected: + issue-1066-host-hkc-false: {} + issue-1066-host-hkc-true: {unreachable: true} + issue-1066-host-hskc-false: {} + issue-1066-host-hskc-true: {unreachable: true} + assert: + that: + - issue_1066_ping.unreachable is defined == expected[inventory_hostname].unreachable is defined + - issue_1066_ping.unreachable | default(42) == expected[inventory_hostname].unreachable | default(42) + # ansible_host_key_checking don't work on Vanilla Ansible 2.10, even for + # static inventory hosts (ansible/ansible#49254, ansible/ansible#73708). + when: + - ansible_version.full is version('2.11', '>=', strict=True) + or is_mitogen + tags: + - issue_1066 diff --git a/tests/ansible/setup/report_controller.yml b/tests/ansible/setup/report_controller.yml index d0d5cc15..5e34fb17 100644 --- a/tests/ansible/setup/report_controller.yml +++ b/tests/ansible/setup/report_controller.yml @@ -1,5 +1,5 @@ - name: Report controller parameters - hosts: localhost + hosts: test-targets[0] gather_facts: false tasks: - debug: @@ -9,9 +9,11 @@ - $(groups): "{{ lookup('pipe', 'groups') }}" - $(pwd): "{{ lookup('pipe', 'pwd') }}" - $(whoami): "{{ lookup('pipe', 'whoami') }}" + - ansible_inventory_sources: "{{ ansible_inventory_sources | default('') }}" - ansible_run_tags: "{{ ansible_run_tags | default('') }}" - ansible_playbook_python: "{{ ansible_playbook_python | default('') }}" - ansible_skip_tags: "{{ ansible_skip_tags | default('') }}" - ansible_version.full: "{{ ansible_version.full | default('') }}" - is_mitogen: "{{ is_mitogen | default('') }}" - playbook_dir: "{{ playbook_dir | default('') }}" + delegate_to: localhost From 60f868290de1127271e174fc7b6a53c465aabf7d Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 6 May 2024 13:19:33 +0100 Subject: [PATCH 3/6] tests: Remove --limit when running Ansible localhost CI Some tests were being incorrectly excluded. Including those that use `add_host`. refs #1066, #1069 --- .ci/localhost_ansible_tests.py | 2 +- .../issue_655__wait_for_connection_error.yml | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index c50ef220..c36bad26 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -93,5 +93,5 @@ with ci_lib.Fold('machine_prep'): with ci_lib.Fold('ansible'): os.chdir(TESTS_DIR) playbook = os.environ.get('PLAYBOOK', 'all.yml') - ci_lib.run('./run_ansible_playbook.py %s -l target %s', + ci_lib.run('./run_ansible_playbook.py %s %s', playbook, ' '.join(sys.argv[1:])) diff --git a/tests/ansible/regression/issue_655__wait_for_connection_error.yml b/tests/ansible/regression/issue_655__wait_for_connection_error.yml index fbdb9f2b..9ad42a10 100644 --- a/tests/ansible/regression/issue_655__wait_for_connection_error.yml +++ b/tests/ansible/regression/issue_655__wait_for_connection_error.yml @@ -4,11 +4,19 @@ # since things are ran on localhost; Azure DevOps loses connection and fails # TODO: do we want to install docker a different way to be able to do this for other tests too --- -- name: regression/issue_655_wait_for_connection_error.yml +- name: regression/issue_655__wait_for_connection_error.yml hosts: localhost gather_facts: yes become: no tasks: + - meta: end_play + when: + # TODO CI currently runs on macOS 11 images in Azure DevOps. MacOS 11 + # is no longer supported by homebrew, so the following install + # task fails. + - ansible_facts.system == 'Darwin' + - ansible_facts.distribution_major_version == '11' + - name: set up test container and run tests inside it block: - name: install deps From 05d98e5b499bed231edda8826d71bd719cc7d831 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 14 May 2024 10:53:30 +0100 Subject: [PATCH 4/6] tests: Speed up ssh timeout tests --- docs/ansible_detailed.rst | 9 +++++++++ tests/ansible/integration/ssh/timeouts.yml | 6 ++++++ tests/ansible/integration/ssh/variables.yml | 4 ++++ 3 files changed, 19 insertions(+) diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index b395bc25..e6f7b42c 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -247,6 +247,15 @@ Noteworthy Differences part of the core library, and should therefore be straightforward to fix as part of 0.2.x. +* Connection and become timeouts are applied differently. Mitogen may consider + a connection to have timed out, when Ansible would have waited longer or + indefinately. For example if SSH authentication completes within the + timeout, but execution of login scripts exceeds it - then Mitogen will + consider the task to have timed out and that host to have failed. + +.. + tests/ansible/integration/ssh/timeouts.yml covers (some of) this behaviour. + .. * SSH and ``become`` are treated distinctly when applying timeouts, and timeouts apply up to the point when the new interpreter is ready to accept diff --git a/tests/ansible/integration/ssh/timeouts.yml b/tests/ansible/integration/ssh/timeouts.yml index a7691bc3..2b14b9f3 100644 --- a/tests/ansible/integration/ssh/timeouts.yml +++ b/tests/ansible/integration/ssh/timeouts.yml @@ -1,7 +1,12 @@ # Ensure 'ssh' connections time out correctly. +# mitogen__slow_user performs a long sleep in ~/.profile. +# Mitogen counts this time towards the connection timeout. Ansible doesn't. +# ansible_python_interpreter=python3000 is an optimisation, to avoid waiting +# on the timeout multiple times (e.g. interpreter discovery). - name: integration/ssh/timeouts.yml hosts: test-targets + gather_facts: false tasks: - include_tasks: ../_mitogen_only.yml @@ -17,6 +22,7 @@ test-targets -m custom_python_detect_environment -e ansible_user=mitogen__slow_user -e ansible_password=slow_user_password + -e ansible_python_interpreter=python3000 args: chdir: ../.. register: out diff --git a/tests/ansible/integration/ssh/variables.yml b/tests/ansible/integration/ssh/variables.yml index f51509db..f9c0f911 100644 --- a/tests/ansible/integration/ssh/variables.yml +++ b/tests/ansible/integration/ssh/variables.yml @@ -43,6 +43,7 @@ test-targets -e ansible_ssh_user=mitogen__has_sudo -e ansible_ssh_pass=wrong_password + -e ansible_python_interpreter=python3000 args: chdir: ../.. register: out @@ -82,6 +83,7 @@ test-targets -e ansible_user=mitogen__has_sudo -e ansible_ssh_pass=wrong_password + -e ansible_python_interpreter=python3000 args: chdir: ../.. register: out @@ -121,6 +123,7 @@ test-targets -e ansible_user=mitogen__has_sudo -e ansible_password=wrong_password + -e ansible_python_interpreter=python3000 args: chdir: ../.. register: out @@ -165,6 +168,7 @@ test-targets -e ansible_user=mitogen__has_sudo -e ansible_ssh_private_key_file=/dev/null + -e ansible_python_interpreter=python3000 args: chdir: ../.. register: out From 65c8a42c13b32729c2ef52b951e3eb8b7c314f7b Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 14 May 2024 11:30:23 +0100 Subject: [PATCH 5/6] tests: Use same verbosity when re-executing Ansible inside a playbook --- .../integration/connection/disconnect_during_module.yml | 2 ++ tests/ansible/integration/process/unix_socket_cleanup.yml | 1 + tests/ansible/integration/runner/missing_module.yml | 4 +++- tests/ansible/integration/ssh/timeouts.yml | 3 ++- tests/ansible/integration/ssh/variables.yml | 8 ++++++++ .../integration/strategy/mixed_vanilla_mitogen.yml | 6 ++++-- tests/ansible/integration/stub_connections/setns_lxc.yml | 2 ++ tests/ansible/integration/stub_connections/setns_lxd.yml | 2 ++ tests/ansible/regression/issue_952__ask_become_pass.yml | 2 ++ 9 files changed, 26 insertions(+), 4 deletions(-) diff --git a/tests/ansible/integration/connection/disconnect_during_module.yml b/tests/ansible/integration/connection/disconnect_during_module.yml index 926291df..5aca75d0 100644 --- a/tests/ansible/integration/connection/disconnect_during_module.yml +++ b/tests/ansible/integration/connection/disconnect_during_module.yml @@ -10,6 +10,8 @@ - name: Run _disconnect_during_module.yml delegate_to: localhost + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: | ansible-playbook {% for inv in ansible_inventory_sources %} diff --git a/tests/ansible/integration/process/unix_socket_cleanup.yml b/tests/ansible/integration/process/unix_socket_cleanup.yml index b5d40b7d..eb6720d3 100644 --- a/tests/ansible/integration/process/unix_socket_cleanup.yml +++ b/tests/ansible/integration/process/unix_socket_cleanup.yml @@ -10,6 +10,7 @@ - shell: > ANSIBLE_STRATEGY=mitogen_linear ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" ansible -m shell -c local -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" diff --git a/tests/ansible/integration/runner/missing_module.yml b/tests/ansible/integration/runner/missing_module.yml index 979cdf21..4aa7b634 100644 --- a/tests/ansible/integration/runner/missing_module.yml +++ b/tests/ansible/integration/runner/missing_module.yml @@ -5,8 +5,10 @@ tasks: - name: Run missing_module connection: local + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: | - ansible -vvv + ansible {% for inv in ansible_inventory_sources %} -i "{{ inv }}" {% endfor %} diff --git a/tests/ansible/integration/ssh/timeouts.yml b/tests/ansible/integration/ssh/timeouts.yml index 2b14b9f3..aed965a8 100644 --- a/tests/ansible/integration/ssh/timeouts.yml +++ b/tests/ansible/integration/ssh/timeouts.yml @@ -14,8 +14,9 @@ connection: local environment: ANSIBLE_SSH_TIMEOUT: 10 + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: | - ansible -vvv + ansible {% for inv in ansible_inventory_sources %} -i "{{ inv }}" {% endfor %} diff --git a/tests/ansible/integration/ssh/variables.yml b/tests/ansible/integration/ssh/variables.yml index f9c0f911..d2fa683b 100644 --- a/tests/ansible/integration/ssh/variables.yml +++ b/tests/ansible/integration/ssh/variables.yml @@ -20,6 +20,7 @@ ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -36,6 +37,7 @@ ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -60,6 +62,7 @@ ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -76,6 +79,7 @@ ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -100,6 +104,7 @@ ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -116,6 +121,7 @@ ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -145,6 +151,7 @@ ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -161,6 +168,7 @@ ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" diff --git a/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml b/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml index d183564f..4220ed4c 100644 --- a/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml +++ b/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml @@ -3,24 +3,26 @@ hosts: test-targets[0] tasks: - connection: local + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: | ansible-playbook {% for inv in ansible_inventory_sources %} -i "{{ inv }}" {% endfor %} - -vvv integration/strategy/_mixed_mitogen_vanilla.yml args: chdir: ../.. register: out - connection: local + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: | ansible-playbook {% for inv in ansible_inventory_sources %} -i "{{ inv }}" {% endfor %} - -vvv integration/strategy/_mixed_vanilla_mitogen.yml args: chdir: ../.. diff --git a/tests/ansible/integration/stub_connections/setns_lxc.yml b/tests/ansible/integration/stub_connections/setns_lxc.yml index 889a15e9..f8d6bfc5 100644 --- a/tests/ansible/integration/stub_connections/setns_lxc.yml +++ b/tests/ansible/integration/stub_connections/setns_lxc.yml @@ -12,6 +12,8 @@ - include_tasks: _end_play_if_not_sudo_linux.yml - name: Run stub-lxc-info.py + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: | sudo -nE "{{lookup('env', 'VIRTUAL_ENV')}}/bin/ansible" -i localhost, diff --git a/tests/ansible/integration/stub_connections/setns_lxd.yml b/tests/ansible/integration/stub_connections/setns_lxd.yml index 81a52dbb..5aba96c5 100644 --- a/tests/ansible/integration/stub_connections/setns_lxd.yml +++ b/tests/ansible/integration/stub_connections/setns_lxd.yml @@ -12,6 +12,8 @@ - include_tasks: _end_play_if_not_sudo_linux.yml - name: Run ansible stub-lxc.py + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: | sudo -nE "{{lookup('env', 'VIRTUAL_ENV')}}/bin/ansible" -i localhost, diff --git a/tests/ansible/regression/issue_952__ask_become_pass.yml b/tests/ansible/regression/issue_952__ask_become_pass.yml index da269e2d..a0b92ff2 100644 --- a/tests/ansible/regression/issue_952__ask_become_pass.yml +++ b/tests/ansible/regression/issue_952__ask_become_pass.yml @@ -6,6 +6,8 @@ tasks: - name: Test --ask-become-pass delegate_to: localhost + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" expect: command: > ansible-playbook From 7079a07a1527f32ff94cb56ef83c70af4e513bde Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 14 May 2024 20:08:29 +0100 Subject: [PATCH 6/6] tests: Fix duplicate local task executions integration/ssh/timeouts.yml is noteworthy. It was an accidental N**2 in time - executing num hosts * num hosts tasks. --- tests/ansible/bench/file_transfer.yml | 41 ++++++++++++++++- tests/ansible/integration/action/copy.yml | 2 + .../integration/action/fixup_perms2__copy.yml | 1 + .../integration/action/synchronize.yml | 3 ++ .../integration/connection/_cleanup_file.yml | 17 +++++++ .../integration/connection/_put_file.yml | 9 ++-- .../integration/connection/put_large_file.yml | 4 +- .../integration/connection/put_small_file.yml | 4 +- .../playbook_semantics/delegate_to.yml | 44 +++++++++++++------ .../integration/runner/missing_module.yml | 3 +- tests/ansible/integration/ssh/timeouts.yml | 6 ++- .../regression/issue_140__thread_pileup.yml | 16 ++++++- ...ue_152__local_action_wrong_interpreter.yml | 3 +- .../issue_615__streaming_transfer.yml | 8 ++-- 14 files changed, 130 insertions(+), 31 deletions(-) create mode 100644 tests/ansible/integration/connection/_cleanup_file.yml diff --git a/tests/ansible/bench/file_transfer.yml b/tests/ansible/bench/file_transfer.yml index 09534af5..1c8dce79 100644 --- a/tests/ansible/bench/file_transfer.yml +++ b/tests/ansible/bench/file_transfer.yml @@ -1,14 +1,16 @@ - - name: bench/file_transfer.yml hosts: test-targets tasks: - - name: Make 32MiB file delegate_to: localhost + run_once: true shell: openssl rand 33554432 > /tmp/bigfile.in + args: + creates: /tmp/bigfile.in - name: Make 320MiB file delegate_to: localhost + run_once: true shell: > cat /tmp/bigfile.in @@ -22,6 +24,8 @@ /tmp/bigfile.in /tmp/bigfile.in > /tmp/bigbigfile.in + args: + creates: /tmp/bigbigfile.in - name: Delete SSH file is present. file: @@ -36,17 +40,20 @@ copy: src: /tmp/bigfile.in dest: /tmp/bigfile.out + mode: ugo=rw - name: Copy 320MiB file via SSH copy: src: /tmp/bigbigfile.in dest: /tmp/bigbigfile.out + mode: ugo=rw - name: Delete localhost sudo file if present. file: path: "{{item}}" state: absent delegate_to: localhost + run_once: true become: true with_items: - /tmp/bigfile.out @@ -56,21 +63,51 @@ - name: Copy 32MiB file via localhost sudo delegate_to: localhost + run_once: true become: true copy: src: /tmp/bigfile.in dest: /tmp/bigfile.out + mode: ugo=rw tags: - requires_local_sudo - name: Copy 320MiB file via localhost sudo delegate_to: localhost + run_once: true become: true copy: src: /tmp/bigbigfile.in dest: /tmp/bigbigfile.out + mode: ugo=rw tags: - requires_local_sudo + - name: Local cleanup + file: + path: "{{ item.path }}" + state: absent + loop: + - /tmp/bigfile.in + - /tmp/bigfile.out + - /tmp/bigbigfile.in + - /tmp/bigbigfile.out + delegate_to: localhost + run_once: true + tags: + - cleanup_local + - cleanup + + - name: Target cleanup + file: + path: "{{ item.path }}" + state: absent + loop: + - /tmp/bigfile.out + - /tmp/bigbigfile.out + tags: + - cleanup_target + - cleanup + tags: - resource_intensive diff --git a/tests/ansible/integration/action/copy.yml b/tests/ansible/integration/action/copy.yml index 5dadff9a..73f3bd1e 100644 --- a/tests/ansible/integration/action/copy.yml +++ b/tests/ansible/integration/action/copy.yml @@ -9,6 +9,7 @@ content: this is a tiny file. delegate_to: localhost + run_once: true - name: Create large file copy: @@ -16,6 +17,7 @@ # Must be larger than Connection.SMALL_SIZE_LIMIT. content: "{% for x in range(200000) %}x{% endfor %}" delegate_to: localhost + run_once: true - name: Cleanup copied files file: diff --git a/tests/ansible/integration/action/fixup_perms2__copy.yml b/tests/ansible/integration/action/fixup_perms2__copy.yml index fa4765f8..662247c6 100644 --- a/tests/ansible/integration/action/fixup_perms2__copy.yml +++ b/tests/ansible/integration/action/fixup_perms2__copy.yml @@ -38,6 +38,7 @@ - name: Create local weird mode file delegate_to: localhost + run_once: true copy: content: "weird mode" dest: "/tmp/weird-mode" diff --git a/tests/ansible/integration/action/synchronize.yml b/tests/ansible/integration/action/synchronize.yml index aa87deaf..13c2113a 100644 --- a/tests/ansible/integration/action/synchronize.yml +++ b/tests/ansible/integration/action/synchronize.yml @@ -24,18 +24,21 @@ path: /tmp/sync-test state: absent delegate_to: localhost + run_once: true - name: Create sync-test file: path: /tmp/sync-test state: directory delegate_to: localhost + run_once: true - name: Create syn-test item copy: dest: /tmp/sync-test/item content: "item!" delegate_to: localhost + run_once: true # TODO: https://github.com/dw/mitogen/issues/692 # - file: diff --git a/tests/ansible/integration/connection/_cleanup_file.yml b/tests/ansible/integration/connection/_cleanup_file.yml new file mode 100644 index 00000000..c1b86255 --- /dev/null +++ b/tests/ansible/integration/connection/_cleanup_file.yml @@ -0,0 +1,17 @@ +- name: Cleanup local file + file: + path: /tmp/{{ file_name }} + state: absent + delegate_to: localhost + run_once: true + tags: + - cleanup_local + - cleanup + +- name: Cleanup target file + file: + path: /tmp/{{ file_name }}.out + state: absent + tags: + - cleanup_target + - cleanup diff --git a/tests/ansible/integration/connection/_put_file.yml b/tests/ansible/integration/connection/_put_file.yml index fb5c6014..07837db8 100644 --- a/tests/ansible/integration/connection/_put_file.yml +++ b/tests/ansible/integration/connection/_put_file.yml @@ -1,20 +1,21 @@ ---- - - name: Create {{ file_name }} - shell: dd if=/dev/urandom of=/tmp/{{ file_name }} bs=1024 count={{ file_size }} - args: + command: + cmd: dd if=/dev/urandom of=/tmp/{{ file_name }} bs=1024 count={{ file_size_kib }} creates: /tmp/{{file_name}} delegate_to: localhost + run_once: true - name: Copy {{ file_name }} copy: dest: /tmp/{{file_name}}.out src: /tmp/{{file_name}} + mode: "{{ file_mode }}" - name: Stat created {{ file_name }} stat: path=/tmp/{{ file_name }} register: original delegate_to: localhost + run_once: true - name: Stat copied {{ file_name }} stat: path=/tmp/{{ file_name }}.out diff --git a/tests/ansible/integration/connection/put_large_file.yml b/tests/ansible/integration/connection/put_large_file.yml index c70e033d..e6f5f645 100644 --- a/tests/ansible/integration/connection/put_large_file.yml +++ b/tests/ansible/integration/connection/put_large_file.yml @@ -6,9 +6,11 @@ gather_facts: no vars: file_name: large-file - file_size: 512 + file_size_kib: 512 + file_mode: u=rw,go= tasks: - include_tasks: _put_file.yml + - include_tasks: _cleanup_file.yml tags: - put_file - put_large_file diff --git a/tests/ansible/integration/connection/put_small_file.yml b/tests/ansible/integration/connection/put_small_file.yml index 2dc100a1..06c41673 100644 --- a/tests/ansible/integration/connection/put_small_file.yml +++ b/tests/ansible/integration/connection/put_small_file.yml @@ -6,9 +6,11 @@ gather_facts: no vars: file_name: small-file - file_size: 123 + file_size_kib: 123 + file_mode: u=rw,go= tasks: - include_tasks: _put_file.yml + - include_tasks: _cleanup_file.yml tags: - put_file - put_small_file diff --git a/tests/ansible/integration/playbook_semantics/delegate_to.yml b/tests/ansible/integration/playbook_semantics/delegate_to.yml index c8d9e607..61b4d03d 100644 --- a/tests/ansible/integration/playbook_semantics/delegate_to.yml +++ b/tests/ansible/integration/playbook_semantics/delegate_to.yml @@ -1,23 +1,27 @@ - name: integration/playbook_semantics/delegate_to.yml hosts: test-targets + vars: + local_path: "/tmp/delegate_to.{{ inventory_hostname }}.txt" tasks: # # delegate_to, no sudo # - name: "delegate_to, no sudo" copy: - dest: /tmp/delegate_to.yml.txt + dest: "{{ local_path }}" content: "Hello, world." - register: out + mode: u=rw,go=r delegate_to: localhost - name: "delegate_to, no sudo" assert: - that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'Hello, world.'" + that: + - lookup('file', local_path) == 'Hello, world.' + fail_msg: "{{ lookup('file', local_path) }}" - name: "delegate_to, no sudo" file: - path: /tmp/delegate_to.yml.txt + path: "{{ local_path }}" state: absent delegate_to: localhost @@ -27,18 +31,20 @@ # - name: "connection:local, no sudo" copy: - dest: /tmp/delegate_to.yml.txt + dest: "{{ local_path }}" content: "Hello, world." - register: out + mode: u=rw,go=r connection: local - name: "connection:local, no sudo" assert: - that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'Hello, world.'" + that: + - lookup('file', local_path) == 'Hello, world.' + fail_msg: "{{ lookup('file', local_path) }}" - name: "connection:local, no sudo" file: - path: /tmp/delegate_to.yml.txt + path: "{{ local_path }}" state: absent connection: local @@ -47,7 +53,10 @@ # delegate_to, sudo # - name: "delegate_to, sudo" - shell: whoami > /tmp/delegate_to.yml.txt + shell: | + whoami > "{{ local_path }}" + args: + creates: "{{ local_path }}" delegate_to: localhost become: true tags: @@ -55,13 +64,15 @@ - name: "delegate_to, sudo" assert: - that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'root'" + that: + - lookup('file', local_path) == 'root' + fail_msg: "{{ lookup('file', local_path) }}" tags: - requires_local_sudo - name: "delegate_to, sudo" file: - path: /tmp/delegate_to.yml.txt + path: "{{ local_path }}" state: absent delegate_to: localhost become: true @@ -73,7 +84,10 @@ # connection:local, sudo # - name: "connection:local, sudo" - shell: whoami > /tmp/delegate_to.yml.txt + shell: | + whoami > "{{ local_path }}" + args: + creates: "{{ local_path }}" connection: local become: true tags: @@ -81,13 +95,15 @@ - name: "connection:local, sudo" assert: - that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'root'" + that: + - lookup('file', local_path) == 'root' + fail_msg: "{{ lookup('file', local_path) }}" tags: - requires_local_sudo - name: "connection:local, sudo" file: - path: /tmp/delegate_to.yml.txt + path: "{{ local_path }}" state: absent connection: local become: true diff --git a/tests/ansible/integration/runner/missing_module.yml b/tests/ansible/integration/runner/missing_module.yml index 4aa7b634..d7261463 100644 --- a/tests/ansible/integration/runner/missing_module.yml +++ b/tests/ansible/integration/runner/missing_module.yml @@ -1,4 +1,3 @@ - - name: integration/runner/missing_module.yml hosts: test-targets[0] connection: local @@ -17,6 +16,8 @@ args: chdir: ../.. register: out + changed_when: false + check_mode: false ignore_errors: true - assert: diff --git a/tests/ansible/integration/ssh/timeouts.yml b/tests/ansible/integration/ssh/timeouts.yml index aed965a8..09ac0006 100644 --- a/tests/ansible/integration/ssh/timeouts.yml +++ b/tests/ansible/integration/ssh/timeouts.yml @@ -20,13 +20,15 @@ {% for inv in ansible_inventory_sources %} -i "{{ inv }}" {% endfor %} - test-targets - -m custom_python_detect_environment + "{{ inventory_hostname }}" + -m ping -e ansible_user=mitogen__slow_user -e ansible_password=slow_user_password -e ansible_python_interpreter=python3000 args: chdir: ../.. register: out + changed_when: false + check_mode: false ignore_errors: true - name: Verify connection timeout occurred diff --git a/tests/ansible/regression/issue_140__thread_pileup.yml b/tests/ansible/regression/issue_140__thread_pileup.yml index 1f614634..163440a5 100644 --- a/tests/ansible/regression/issue_140__thread_pileup.yml +++ b/tests/ansible/regression/issue_140__thread_pileup.yml @@ -8,6 +8,7 @@ - name: Create file tree connection: local + run_once: true shell: > mkdir /tmp/filetree.in; seq -f /tmp/filetree.in/%g 1 1000 | xargs touch; @@ -21,17 +22,30 @@ file: state: directory path: /tmp/filetree.out + mode: u=rwx,go=rx - name: Trigger nasty process pileup copy: src: "{{item.src}}" dest: "/tmp/filetree.out/{{item.path}}" - mode: 0644 + mode: u=rw,go=r with_filetree: /tmp/filetree.in when: item.state == 'file' loop_control: label: "/tmp/filetree.out/{{ item.path }}" + - name: Cleanup local file tree + connection: local + run_once: true + file: + path: /tmp/filetree.in + state: absent + + - name: Cleanup remote file tree + file: + path: /tmp/filetree.out + state: absent + tags: - resource_intensive - issue_140 diff --git a/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml b/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml index fe7f5f42..dd86775d 100644 --- a/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml +++ b/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml @@ -4,10 +4,9 @@ # can test for. - name: regression/issue_152__local_action_wrong_interpreter.yml - hosts: test-targets + hosts: test-targets[0] connection: local tasks: - - name: Create /tmp/issue_152_interpreter.sh copy: dest: /tmp/issue_152_interpreter.sh diff --git a/tests/ansible/regression/issue_615__streaming_transfer.yml b/tests/ansible/regression/issue_615__streaming_transfer.yml index 43aa1343..6fe52d55 100644 --- a/tests/ansible/regression/issue_615__streaming_transfer.yml +++ b/tests/ansible/regression/issue_615__streaming_transfer.yml @@ -14,11 +14,14 @@ shell: | dd if=/dev/zero of=/tmp/512mb.zero bs=1048576 count=512; chmod go= /tmp/512mb.zero + args: + creates: /tmp/512mb.zero - name: Fetch /tmp/512mb.zero fetch: src: /tmp/512mb.zero - dest: /tmp/fetch-out + dest: /tmp/fetch-{{ inventory_hostname }}-512mb.zero + flat: true - name: Cleanup /tmp/512mb.zero file: @@ -27,11 +30,10 @@ - name: Cleanup fetched file file: - path: /tmp/fetch-out + path: /tmp/fetch-{{ inventory_hostname }}-512mb.zero state: absent become: false delegate_to: localhost - run_once: true tags: - issue_615 - mitogen_only