diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 43d4f4cb..0dd978c4 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -6,6 +6,7 @@ import glob import os import signal import sys +import textwrap import ci_lib @@ -74,7 +75,15 @@ with ci_lib.Fold('job_setup'): fp.write('\n[%s]\n' % family) fp.writelines('%s\n' % name for name in hostnames) - fp.write('\n[linux:children]\ntest-targets\n') + fp.write(textwrap.dedent( + ''' + [linux:children] + test-targets + + [linux_containers:children] + test-targets + ''' + )) ci_lib.dump_file(inventory_path) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 94919e35..51908e92 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -16,32 +16,21 @@ trigger: - docs-master jobs: -- job: mac11 +- job: mac12 # vanilla Ansible is really slow timeoutInMinutes: 120 steps: - template: azure-pipelines-steps.yml pool: - # https://github.com/actions/runner-images/blob/main/images/macos/macos-11-Readme.md - vmImage: macOS-11 + # https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md + vmImage: macOS-12 strategy: matrix: - Mito_27: - tox.env: py27-mode_mitogen Mito_312: - python.version: '3.12' tox.env: py312-mode_mitogen - - Loc_27_210: - tox.env: py27-mode_localhost-ansible2.10 Loc_312_9: - python.version: '3.12' tox.env: py312-mode_localhost-ansible9 - - Van_27_210: - tox.env: py27-mode_localhost-ansible2.10-strategy_linear Van_312_9: - python.version: '3.12' tox.env: py312-mode_localhost-ansible9-strategy_linear - job: Linux diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index c50ef220..c36bad26 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -93,5 +93,5 @@ with ci_lib.Fold('machine_prep'): with ci_lib.Fold('ansible'): os.chdir(TESTS_DIR) playbook = os.environ.get('PLAYBOOK', 'all.yml') - ci_lib.run('./run_ansible_playbook.py %s -l target %s', + ci_lib.run('./run_ansible_playbook.py %s %s', playbook, ' '.join(sys.argv[1:])) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index dfc3aec4..6bdf11ba 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -119,7 +119,7 @@ def _connect_ssh(spec): """ Return ContextService arguments for an SSH connection. """ - if C.HOST_KEY_CHECKING: + if spec.host_key_checking(): check_host_keys = 'enforce' else: check_host_keys = 'ignore' diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index 63caa88a..3a41a43d 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -281,11 +281,11 @@ def get_cpu_count(default=None): class Broker(mitogen.master.Broker): """ - WorkerProcess maintains at most 2 file descriptors, therefore does not need + WorkerProcess maintains fewer file descriptors, therefore does not need the exuberant syscall expense of EpollPoller, so override it and restore the poll() poller. """ - poller_class = mitogen.core.Poller + poller_class = mitogen.parent.POLLER_LIGHTWEIGHT class Binding(object): diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index 5fc78185..3ab623f8 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -67,6 +67,7 @@ import ansible.utils.shlex import ansible.constants as C from ansible.module_utils.six import with_metaclass +from ansible.module_utils.parsing.convert_bool import boolean # this was added in Ansible >= 2.8.0; fallback to the default interpreter if necessary try: @@ -79,7 +80,6 @@ try: except ImportError: from ansible.vars.unsafe_proxy import AnsibleUnsafeText -import ansible_mitogen.loaders import mitogen.core @@ -246,6 +246,12 @@ class Spec(with_metaclass(abc.ABCMeta, object)): Path to the Python interpreter on the target machine. """ + @abc.abstractmethod + def host_key_checking(self): + """ + Whether or not to check the keys of the target machine + """ + @abc.abstractmethod def private_key_file(self): """ @@ -436,9 +442,18 @@ class PlayContextSpec(Spec): return self._play_context.become_user def become_pass(self): - become_method = self.become_method() - become_plugin = ansible_mitogen.loaders.become_loader.get(become_method) - become_pass = become_plugin.get_option('become_pass', hostvars=self._task_vars) + # become_pass is owned/provided by the active become plugin. However + # PlayContext is intertwined with it. Known complications + # - ansible_become_password is higher priority than ansible_become_pass, + # `play_context.become_pass` doesn't obey this (atleast with Mitgeon). + # - `meta: reset_connection` runs `connection.reset()` but + # `ansible_mitogen.connection.Connection.reset()` recreates the + # connection object, setting `connection.become = None`. + become_plugin = self._connection.become + try: + become_pass = become_plugin.get_option('become_pass', playcontext=self._play_context) + except AttributeError: + become_pass = self._play_context.become_pass return optional_secret(become_pass) def password(self): @@ -458,6 +473,14 @@ class PlayContextSpec(Spec): action=self._action, rediscover_python=rediscover_python) + def host_key_checking(self): + def candidates(): + yield self._connection.get_task_var('ansible_ssh_host_key_checking') + yield self._connection.get_task_var('ansible_host_key_checking') + yield C.HOST_KEY_CHECKING + val = next((v for v in candidates() if v is not None), True) + return boolean(val) + def private_key_file(self): return self._play_context.private_key_file @@ -684,6 +707,14 @@ class MitogenViaSpec(Spec): action=self._action, rediscover_python=rediscover_python) + def host_key_checking(self): + def candidates(): + yield self._host_vars.get('ansible_ssh_host_key_checking') + yield self._host_vars.get('ansible_host_key_checking') + yield C.HOST_KEY_CHECKING + val = next((v for v in candidates() if v is not None), True) + return boolean(val) + def private_key_file(self): # TODO: must come from PlayContext too. return ( diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index b395bc25..e6f7b42c 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -247,6 +247,15 @@ Noteworthy Differences part of the core library, and should therefore be straightforward to fix as part of 0.2.x. +* Connection and become timeouts are applied differently. Mitogen may consider + a connection to have timed out, when Ansible would have waited longer or + indefinately. For example if SSH authentication completes within the + timeout, but execution of login scripts exceeds it - then Mitogen will + consider the task to have timed out and that host to have failed. + +.. + tests/ansible/integration/ssh/timeouts.yml covers (some of) this behaviour. + .. * SSH and ``become`` are treated distinctly when applying timeouts, and timeouts apply up to the point when the new interpreter is ready to accept diff --git a/docs/changelog.rst b/docs/changelog.rst index 0b8eeb20..eaffc46c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,16 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. +v0.3.8 (2024-07-30) +------------------- + +* :gh:issue:`952` Fix Ansible `--ask-become-pass`, add test coverage +* :gh:issue:`957` Fix Ansible exception when executing against 10s of hosts + "ValueError: filedescriptor out of range in select()" +* :gh:issue:`1066` Support Ansible `ansible_host_key_checking` & `ansible_ssh_host_key_checking` +* :gh:issue:`1090` CI: Migrate macOS integration tests to macOS 12, drop Python 2.7 jobs + + v0.3.7 (2024-04-08) ------------------- diff --git a/docs/conf.py b/docs/conf.py index ad9771b4..fe1a5044 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -2,7 +2,7 @@ import sys sys.path.append('.') -VERSION = '0.3.7' +VERSION = '0.3.8' author = u'Network Genomics' copyright = u'2021, the Mitogen authors' diff --git a/docs/contributors.rst b/docs/contributors.rst index 207e4d7b..ed7fef11 100644 --- a/docs/contributors.rst +++ b/docs/contributors.rst @@ -126,11 +126,13 @@ sponsorship and outstanding future-thinking of its early adopters.
  • jgadling
  • John F Wall — Making Ansible Great with Massive Parallelism
  • KennethC
  • +
  • Luca Berruti
  • Lewis Bellwood — Happy to be apart of a great project.
  • luto
  • Mayeu a.k.a Matthieu Maury
  • @nathanhruby
  • Orion Poplawski
  • +
  • Philippe Kueck
  • Ramy
  • Scott Vokes
  • Tom Eichhorn
  • diff --git a/mitogen/__init__.py b/mitogen/__init__.py index d72df040..8f2741c7 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 7) +__version__ = (0, 3, 8) #: This is :data:`False` in slave contexts. Previously it was used to prevent diff --git a/mitogen/core.py b/mitogen/core.py index cd02012f..cdfbbcde 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -2523,8 +2523,7 @@ class Poller(object): """ A poller manages OS file descriptors the user is waiting to become available for IO. The :meth:`poll` method blocks the calling thread - until one or more become ready. The default implementation is based on - :func:`select.poll`. + until one or more become ready. Each descriptor has an associated `data` element, which is unique for each readiness type, and defaults to being the same as the file descriptor. The @@ -2546,19 +2545,13 @@ class Poller(object): a resource leak. Pollers may only be used by one thread at a time. + + This implementation uses :func:`select.select` for wider platform support. + That is considered an implementation detail. Previous versions have used + :func:`select.poll`. Future versions may decide at runtime. """ SUPPORTED = True - # This changed from select() to poll() in Mitogen 0.2.4. Since poll() has - # no upper FD limit, it is suitable for use with Latch, which must handle - # FDs larger than select's limit during many-host runs. We want this - # because poll() requires no setup and teardown: just a single system call, - # which is important because Latch.get() creates a Poller on each - # invocation. In a microbenchmark, poll() vs. epoll_ctl() is 30% faster in - # this scenario. If select() must return in future, it is important - # Latch.poller_class is set from parent.py to point to the industrial - # strength poller for the OS, otherwise Latch will fail randomly. - #: Increments on every poll(). Used to version _rfds and _wfds. _generation = 1 @@ -2681,11 +2674,10 @@ class Latch(object): See :ref:`waking-sleeping-threads` for further discussion. """ - #: The :class:`Poller` implementation to use for waiting. Since the poller - #: will be very short-lived, we prefer :class:`mitogen.parent.PollPoller` - #: if it is available, or :class:`mitogen.core.Poller` otherwise, since - #: these implementations require no system calls to create, configure or - #: destroy. + #: The :class:`Poller` implementation to use. Instances are short lived so + #: prefer :class:`mitogen.parent.PollPoller` if it's available, otherwise + #: :class:`mitogen.core.Poller`. They don't need syscalls to create, + #: configure, or destroy. Replaced during import of :mod:`mitogen.parent`. poller_class = Poller #: If not :data:`None`, a function invoked as `notify(latch)` after a diff --git a/mitogen/parent.py b/mitogen/parent.py index 29bcf66d..4b96dcf4 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -745,8 +745,7 @@ def _upgrade_broker(broker): broker.timers = TimerList() LOG.debug('upgraded %r with %r (new: %d readers, %d writers; ' 'old: %d readers, %d writers)', old, new, - len(new.readers), len(new.writers), - len(old.readers), len(old.writers)) + len(new._rfds), len(new._wfds), len(old._rfds), len(old._wfds)) @mitogen.core.takes_econtext @@ -902,22 +901,18 @@ class CallSpec(object): class PollPoller(mitogen.core.Poller): """ Poller based on the POSIX :linux:man2:`poll` interface. Not available on - some versions of OS X, otherwise it is the preferred poller for small FD - counts, as there is no setup/teardown/configuration system call overhead. + some Python/OS X combinations. Otherwise the preferred poller for small + FD counts; or if many pollers are created, used once, then closed. + There there is no setup/teardown/configuration system call overhead. """ SUPPORTED = hasattr(select, 'poll') - _repr = 'PollPoller()' + _readmask = SUPPORTED and select.POLLIN | select.POLLHUP def __init__(self): super(PollPoller, self).__init__() self._pollobj = select.poll() # TODO: no proof we dont need writemask too - _readmask = ( - getattr(select, 'POLLIN', 0) | - getattr(select, 'POLLHUP', 0) - ) - def _update(self, fd): mask = (((fd in self._rfds) and self._readmask) | ((fd in self._wfds) and select.POLLOUT)) @@ -952,7 +947,6 @@ class KqueuePoller(mitogen.core.Poller): Poller based on the FreeBSD/Darwin :freebsd:man2:`kqueue` interface. """ SUPPORTED = hasattr(select, 'kqueue') - _repr = 'KqueuePoller()' def __init__(self): super(KqueuePoller, self).__init__() @@ -1030,7 +1024,7 @@ class EpollPoller(mitogen.core.Poller): Poller based on the Linux :linux:man7:`epoll` interface. """ SUPPORTED = hasattr(select, 'epoll') - _repr = 'EpollPoller()' + _inmask = SUPPORTED and select.EPOLLIN | select.EPOLLHUP def __init__(self): super(EpollPoller, self).__init__() @@ -1077,9 +1071,6 @@ class EpollPoller(mitogen.core.Poller): self._wfds.pop(fd, None) self._control(fd) - _inmask = (getattr(select, 'EPOLLIN', 0) | - getattr(select, 'EPOLLHUP', 0)) - def _poll(self, timeout): the_timeout = -1 if timeout is not None: @@ -1100,18 +1091,14 @@ class EpollPoller(mitogen.core.Poller): yield data -# 2.4 and 2.5 only had select.select() and select.poll(). -for _klass in mitogen.core.Poller, PollPoller, KqueuePoller, EpollPoller: - if _klass.SUPPORTED: - PREFERRED_POLLER = _klass +POLLERS = (EpollPoller, KqueuePoller, PollPoller, mitogen.core.Poller) +PREFERRED_POLLER = next(cls for cls in POLLERS if cls.SUPPORTED) + # For processes that start many threads or connections, it's possible Latch # will also get high-numbered FDs, and so select() becomes useless there too. -# So swap in our favourite poller. -if PollPoller.SUPPORTED: - mitogen.core.Latch.poller_class = PollPoller -else: - mitogen.core.Latch.poller_class = PREFERRED_POLLER +POLLER_LIGHTWEIGHT = PollPoller.SUPPORTED and PollPoller or PREFERRED_POLLER +mitogen.core.Latch.poller_class = POLLER_LIGHTWEIGHT class LineLoggingProtocolMixin(object): diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index c34dd219..537b5059 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -1,9 +1,12 @@ [defaults] any_errors_fatal = true -# callback_whitelist naming will be deprecated in ansible-core >= 2.15. -# callbacks_enabled naming was added in ansible-core 2.11 +# callbacks_enabled was added in Ansible 4 (ansible-core 2.11). # profile_tasks: Displays timing for each task and summary table of top N tasks # timer: Displays "Playbook run took 0 days, 0 hours, ..." +callbacks_enabled = + profile_tasks, + timer +# callback_whitelist was deprecated in Ansible >= 8 (ansible-core >= 2.15). callback_whitelist = profile_tasks, timer @@ -37,7 +40,9 @@ no_target_syslog = True # Required by integration/ssh/timeouts.yml timeout = 30 -# On Travis, paramiko check fails due to host key checking enabled. +# Ideally this would be true here and and overridden for hosts/groups. However +# ansible_host_key_checking don't work on Vanilla Ansible 2.10, even for +# static inventory hosts (ansible/ansible#49254, ansible/ansible#73708) host_key_checking = False [inventory] diff --git a/tests/ansible/bench/file_transfer.yml b/tests/ansible/bench/file_transfer.yml index 09534af5..1c8dce79 100644 --- a/tests/ansible/bench/file_transfer.yml +++ b/tests/ansible/bench/file_transfer.yml @@ -1,14 +1,16 @@ - - name: bench/file_transfer.yml hosts: test-targets tasks: - - name: Make 32MiB file delegate_to: localhost + run_once: true shell: openssl rand 33554432 > /tmp/bigfile.in + args: + creates: /tmp/bigfile.in - name: Make 320MiB file delegate_to: localhost + run_once: true shell: > cat /tmp/bigfile.in @@ -22,6 +24,8 @@ /tmp/bigfile.in /tmp/bigfile.in > /tmp/bigbigfile.in + args: + creates: /tmp/bigbigfile.in - name: Delete SSH file is present. file: @@ -36,17 +40,20 @@ copy: src: /tmp/bigfile.in dest: /tmp/bigfile.out + mode: ugo=rw - name: Copy 320MiB file via SSH copy: src: /tmp/bigbigfile.in dest: /tmp/bigbigfile.out + mode: ugo=rw - name: Delete localhost sudo file if present. file: path: "{{item}}" state: absent delegate_to: localhost + run_once: true become: true with_items: - /tmp/bigfile.out @@ -56,21 +63,51 @@ - name: Copy 32MiB file via localhost sudo delegate_to: localhost + run_once: true become: true copy: src: /tmp/bigfile.in dest: /tmp/bigfile.out + mode: ugo=rw tags: - requires_local_sudo - name: Copy 320MiB file via localhost sudo delegate_to: localhost + run_once: true become: true copy: src: /tmp/bigbigfile.in dest: /tmp/bigbigfile.out + mode: ugo=rw tags: - requires_local_sudo + - name: Local cleanup + file: + path: "{{ item.path }}" + state: absent + loop: + - /tmp/bigfile.in + - /tmp/bigfile.out + - /tmp/bigbigfile.in + - /tmp/bigbigfile.out + delegate_to: localhost + run_once: true + tags: + - cleanup_local + - cleanup + + - name: Target cleanup + file: + path: "{{ item.path }}" + state: absent + loop: + - /tmp/bigfile.out + - /tmp/bigbigfile.out + tags: + - cleanup_target + - cleanup + tags: - resource_intensive diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts index 1bec0014..4f5ea4c6 100644 --- a/tests/ansible/hosts/default.hosts +++ b/tests/ansible/hosts/default.hosts @@ -10,3 +10,5 @@ target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}" [test-targets] target + +[linux_containers] diff --git a/tests/ansible/hosts/group_vars/all.yml b/tests/ansible/hosts/group_vars/all.yml index cea46113..ad4adabb 100644 --- a/tests/ansible/hosts/group_vars/all.yml +++ b/tests/ansible/hosts/group_vars/all.yml @@ -1,2 +1,3 @@ --- pkg_mgr_python_interpreter: python +pkg_repos_overrides: [] diff --git a/tests/ansible/hosts/group_vars/centos8.yml b/tests/ansible/hosts/group_vars/centos8.yml index 7b9e34f6..c90dd5f4 100644 --- a/tests/ansible/hosts/group_vars/centos8.yml +++ b/tests/ansible/hosts/group_vars/centos8.yml @@ -1,2 +1,28 @@ --- pkg_mgr_python_interpreter: /usr/libexec/platform-python + +pkg_repos_overrides: + - dest: /etc/yum.repos.d/CentOS-Linux-AppStream.repo + content: | + [appstream] + name=CentOS Linux $releasever - AppStream + baseurl=http://vault.centos.org/$contentdir/$releasever/AppStream/$basearch/os/ + enabled=1 + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial + - dest: /etc/yum.repos.d/CentOS-Linux-BaseOS.repo + content: | + [baseos] + name=CentOS Linux $releasever - BaseOS + baseurl=http://vault.centos.org/$contentdir/$releasever/BaseOS/$basearch/os/ + enabled=1 + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial + - dest: /etc/yum.repos.d/CentOS-Linux-Extras.repo + content: | + [extras] + name=CentOS Linux $releasever - Extras + baseurl=http://vault.centos.org/$contentdir/$releasever/extras/$basearch/os/ + enabled=1 + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial diff --git a/tests/ansible/hosts/group_vars/debian9.yml b/tests/ansible/hosts/group_vars/debian9.yml new file mode 100644 index 00000000..4b180b13 --- /dev/null +++ b/tests/ansible/hosts/group_vars/debian9.yml @@ -0,0 +1,4 @@ +pkg_repos_overrides: + - dest: /etc/apt/sources.list + content: | + deb http://archive.debian.org/debian stretch main contrib non-free diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts index 1c1c2e10..5d5c3834 100644 --- a/tests/ansible/hosts/transport_config.hosts +++ b/tests/ansible/hosts/transport_config.hosts @@ -13,6 +13,7 @@ tc_become tc_become_method tc_become_pass tc_become_user +tc_host_key_checking tc_password tc_port tc_remote_addr @@ -74,6 +75,11 @@ tc-become-pass-password ansible_become_password=apassword tc-become-pass-pass ansible_become_pass=apass tc-become-pass-both ansible_become_pass=bpass ansible_become_password=bpassword +[tc_host_key_checking] +tc-hkc-unset +tc-hkc-host-key-checking ansible_host_key_checking=true +tc-hkc-ssh-host-key-checking ansible_ssh_host_key_checking=true + [tc_port] tc-port-unset tc-port-explicit-port ansible_port=1234 diff --git a/tests/ansible/integration/action/copy.yml b/tests/ansible/integration/action/copy.yml index 5dadff9a..73f3bd1e 100644 --- a/tests/ansible/integration/action/copy.yml +++ b/tests/ansible/integration/action/copy.yml @@ -9,6 +9,7 @@ content: this is a tiny file. delegate_to: localhost + run_once: true - name: Create large file copy: @@ -16,6 +17,7 @@ # Must be larger than Connection.SMALL_SIZE_LIMIT. content: "{% for x in range(200000) %}x{% endfor %}" delegate_to: localhost + run_once: true - name: Cleanup copied files file: diff --git a/tests/ansible/integration/action/fixup_perms2__copy.yml b/tests/ansible/integration/action/fixup_perms2__copy.yml index fa4765f8..662247c6 100644 --- a/tests/ansible/integration/action/fixup_perms2__copy.yml +++ b/tests/ansible/integration/action/fixup_perms2__copy.yml @@ -38,6 +38,7 @@ - name: Create local weird mode file delegate_to: localhost + run_once: true copy: content: "weird mode" dest: "/tmp/weird-mode" diff --git a/tests/ansible/integration/action/synchronize.yml b/tests/ansible/integration/action/synchronize.yml index aa87deaf..13c2113a 100644 --- a/tests/ansible/integration/action/synchronize.yml +++ b/tests/ansible/integration/action/synchronize.yml @@ -24,18 +24,21 @@ path: /tmp/sync-test state: absent delegate_to: localhost + run_once: true - name: Create sync-test file: path: /tmp/sync-test state: directory delegate_to: localhost + run_once: true - name: Create syn-test item copy: dest: /tmp/sync-test/item content: "item!" delegate_to: localhost + run_once: true # TODO: https://github.com/dw/mitogen/issues/692 # - file: diff --git a/tests/ansible/integration/become/sudo_nonexistent.yml b/tests/ansible/integration/become/sudo_nonexistent.yml index e7a849c2..912d1ba8 100644 --- a/tests/ansible/integration/become/sudo_nonexistent.yml +++ b/tests/ansible/integration/become/sudo_nonexistent.yml @@ -21,8 +21,7 @@ # sudo-1.8.6p3-29.el6_10.3 on RHEL & CentOS 6.10 (final release) # removed user/group error messages, as defence against CVE-2019-14287. - >- - 'sudo: unknown user: slartibartfast' in out.module_stdout | default(out.msg) - or 'sudo: unknown user: slartibartfast' in out.module_stderr | default(out.msg) + (out.module_stderr | default(out.module_stdout, true) | default(out.msg, true)) is search('sudo: unknown user:? slartibartfast') or (ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_version == '6.10') fail_msg: out={{out}} when: diff --git a/tests/ansible/integration/connection/_cleanup_file.yml b/tests/ansible/integration/connection/_cleanup_file.yml new file mode 100644 index 00000000..c1b86255 --- /dev/null +++ b/tests/ansible/integration/connection/_cleanup_file.yml @@ -0,0 +1,17 @@ +- name: Cleanup local file + file: + path: /tmp/{{ file_name }} + state: absent + delegate_to: localhost + run_once: true + tags: + - cleanup_local + - cleanup + +- name: Cleanup target file + file: + path: /tmp/{{ file_name }}.out + state: absent + tags: + - cleanup_target + - cleanup diff --git a/tests/ansible/integration/connection/_put_file.yml b/tests/ansible/integration/connection/_put_file.yml index fb5c6014..07837db8 100644 --- a/tests/ansible/integration/connection/_put_file.yml +++ b/tests/ansible/integration/connection/_put_file.yml @@ -1,20 +1,21 @@ ---- - - name: Create {{ file_name }} - shell: dd if=/dev/urandom of=/tmp/{{ file_name }} bs=1024 count={{ file_size }} - args: + command: + cmd: dd if=/dev/urandom of=/tmp/{{ file_name }} bs=1024 count={{ file_size_kib }} creates: /tmp/{{file_name}} delegate_to: localhost + run_once: true - name: Copy {{ file_name }} copy: dest: /tmp/{{file_name}}.out src: /tmp/{{file_name}} + mode: "{{ file_mode }}" - name: Stat created {{ file_name }} stat: path=/tmp/{{ file_name }} register: original delegate_to: localhost + run_once: true - name: Stat copied {{ file_name }} stat: path=/tmp/{{ file_name }}.out diff --git a/tests/ansible/integration/connection/disconnect_during_module.yml b/tests/ansible/integration/connection/disconnect_during_module.yml index 926291df..5aca75d0 100644 --- a/tests/ansible/integration/connection/disconnect_during_module.yml +++ b/tests/ansible/integration/connection/disconnect_during_module.yml @@ -10,6 +10,8 @@ - name: Run _disconnect_during_module.yml delegate_to: localhost + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: | ansible-playbook {% for inv in ansible_inventory_sources %} diff --git a/tests/ansible/integration/connection/put_large_file.yml b/tests/ansible/integration/connection/put_large_file.yml index c70e033d..e6f5f645 100644 --- a/tests/ansible/integration/connection/put_large_file.yml +++ b/tests/ansible/integration/connection/put_large_file.yml @@ -6,9 +6,11 @@ gather_facts: no vars: file_name: large-file - file_size: 512 + file_size_kib: 512 + file_mode: u=rw,go= tasks: - include_tasks: _put_file.yml + - include_tasks: _cleanup_file.yml tags: - put_file - put_large_file diff --git a/tests/ansible/integration/connection/put_small_file.yml b/tests/ansible/integration/connection/put_small_file.yml index 2dc100a1..06c41673 100644 --- a/tests/ansible/integration/connection/put_small_file.yml +++ b/tests/ansible/integration/connection/put_small_file.yml @@ -6,9 +6,11 @@ gather_facts: no vars: file_name: small-file - file_size: 123 + file_size_kib: 123 + file_mode: u=rw,go= tasks: - include_tasks: _put_file.yml + - include_tasks: _cleanup_file.yml tags: - put_file - put_small_file diff --git a/tests/ansible/integration/interpreter_discovery/complex_args.yml b/tests/ansible/integration/interpreter_discovery/complex_args.yml index 6ffff5f4..a3755ebf 100644 --- a/tests/ansible/integration/interpreter_discovery/complex_args.yml +++ b/tests/ansible/integration/interpreter_discovery/complex_args.yml @@ -29,20 +29,18 @@ special_python: "source /tmp/fake || true && python{{ ansible_facts.python.version.major }}" - name: run get_url with specially-sourced python - get_url: + uri: # Plain http for wider Ansible & Python version compatibility - url: http://httpbin.org/get - dest: "/tmp/" - mode: 0644 + url: http://www.gstatic.com/generate_204 + status_code: [204] vars: ansible_python_interpreter: "{{ special_python }}" - name: run get_url with specially-sourced python including jinja - get_url: + uri: # Plain http for wider Ansible & Python version compatibility - url: http://httpbin.org/get - dest: "/tmp/" - mode: 0644 + url: http://www.gstatic.com/generate_204 + status_code: [204] vars: ansible_python_interpreter: > {% if "1" == "1" %} diff --git a/tests/ansible/integration/playbook_semantics/delegate_to.yml b/tests/ansible/integration/playbook_semantics/delegate_to.yml index c8d9e607..61b4d03d 100644 --- a/tests/ansible/integration/playbook_semantics/delegate_to.yml +++ b/tests/ansible/integration/playbook_semantics/delegate_to.yml @@ -1,23 +1,27 @@ - name: integration/playbook_semantics/delegate_to.yml hosts: test-targets + vars: + local_path: "/tmp/delegate_to.{{ inventory_hostname }}.txt" tasks: # # delegate_to, no sudo # - name: "delegate_to, no sudo" copy: - dest: /tmp/delegate_to.yml.txt + dest: "{{ local_path }}" content: "Hello, world." - register: out + mode: u=rw,go=r delegate_to: localhost - name: "delegate_to, no sudo" assert: - that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'Hello, world.'" + that: + - lookup('file', local_path) == 'Hello, world.' + fail_msg: "{{ lookup('file', local_path) }}" - name: "delegate_to, no sudo" file: - path: /tmp/delegate_to.yml.txt + path: "{{ local_path }}" state: absent delegate_to: localhost @@ -27,18 +31,20 @@ # - name: "connection:local, no sudo" copy: - dest: /tmp/delegate_to.yml.txt + dest: "{{ local_path }}" content: "Hello, world." - register: out + mode: u=rw,go=r connection: local - name: "connection:local, no sudo" assert: - that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'Hello, world.'" + that: + - lookup('file', local_path) == 'Hello, world.' + fail_msg: "{{ lookup('file', local_path) }}" - name: "connection:local, no sudo" file: - path: /tmp/delegate_to.yml.txt + path: "{{ local_path }}" state: absent connection: local @@ -47,7 +53,10 @@ # delegate_to, sudo # - name: "delegate_to, sudo" - shell: whoami > /tmp/delegate_to.yml.txt + shell: | + whoami > "{{ local_path }}" + args: + creates: "{{ local_path }}" delegate_to: localhost become: true tags: @@ -55,13 +64,15 @@ - name: "delegate_to, sudo" assert: - that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'root'" + that: + - lookup('file', local_path) == 'root' + fail_msg: "{{ lookup('file', local_path) }}" tags: - requires_local_sudo - name: "delegate_to, sudo" file: - path: /tmp/delegate_to.yml.txt + path: "{{ local_path }}" state: absent delegate_to: localhost become: true @@ -73,7 +84,10 @@ # connection:local, sudo # - name: "connection:local, sudo" - shell: whoami > /tmp/delegate_to.yml.txt + shell: | + whoami > "{{ local_path }}" + args: + creates: "{{ local_path }}" connection: local become: true tags: @@ -81,13 +95,15 @@ - name: "connection:local, sudo" assert: - that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'root'" + that: + - lookup('file', local_path) == 'root' + fail_msg: "{{ lookup('file', local_path) }}" tags: - requires_local_sudo - name: "connection:local, sudo" file: - path: /tmp/delegate_to.yml.txt + path: "{{ local_path }}" state: absent connection: local become: true diff --git a/tests/ansible/integration/process/unix_socket_cleanup.yml b/tests/ansible/integration/process/unix_socket_cleanup.yml index b5d40b7d..eb6720d3 100644 --- a/tests/ansible/integration/process/unix_socket_cleanup.yml +++ b/tests/ansible/integration/process/unix_socket_cleanup.yml @@ -10,6 +10,7 @@ - shell: > ANSIBLE_STRATEGY=mitogen_linear ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" ansible -m shell -c local -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" diff --git a/tests/ansible/integration/runner/missing_module.yml b/tests/ansible/integration/runner/missing_module.yml index 979cdf21..d7261463 100644 --- a/tests/ansible/integration/runner/missing_module.yml +++ b/tests/ansible/integration/runner/missing_module.yml @@ -1,12 +1,13 @@ - - name: integration/runner/missing_module.yml hosts: test-targets[0] connection: local tasks: - name: Run missing_module connection: local + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: | - ansible -vvv + ansible {% for inv in ansible_inventory_sources %} -i "{{ inv }}" {% endfor %} @@ -15,6 +16,8 @@ args: chdir: ../.. register: out + changed_when: false + check_mode: false ignore_errors: true - assert: diff --git a/tests/ansible/integration/ssh/timeouts.yml b/tests/ansible/integration/ssh/timeouts.yml index a7691bc3..09ac0006 100644 --- a/tests/ansible/integration/ssh/timeouts.yml +++ b/tests/ansible/integration/ssh/timeouts.yml @@ -1,7 +1,12 @@ # Ensure 'ssh' connections time out correctly. +# mitogen__slow_user performs a long sleep in ~/.profile. +# Mitogen counts this time towards the connection timeout. Ansible doesn't. +# ansible_python_interpreter=python3000 is an optimisation, to avoid waiting +# on the timeout multiple times (e.g. interpreter discovery). - name: integration/ssh/timeouts.yml hosts: test-targets + gather_facts: false tasks: - include_tasks: ../_mitogen_only.yml @@ -9,17 +14,21 @@ connection: local environment: ANSIBLE_SSH_TIMEOUT: 10 + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: | - ansible -vvv + ansible {% for inv in ansible_inventory_sources %} -i "{{ inv }}" {% endfor %} - test-targets - -m custom_python_detect_environment + "{{ inventory_hostname }}" + -m ping -e ansible_user=mitogen__slow_user -e ansible_password=slow_user_password + -e ansible_python_interpreter=python3000 args: chdir: ../.. register: out + changed_when: false + check_mode: false ignore_errors: true - name: Verify connection timeout occurred diff --git a/tests/ansible/integration/ssh/variables.yml b/tests/ansible/integration/ssh/variables.yml index f51509db..d2fa683b 100644 --- a/tests/ansible/integration/ssh/variables.yml +++ b/tests/ansible/integration/ssh/variables.yml @@ -20,6 +20,7 @@ ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -36,6 +37,7 @@ ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -43,6 +45,7 @@ test-targets -e ansible_ssh_user=mitogen__has_sudo -e ansible_ssh_pass=wrong_password + -e ansible_python_interpreter=python3000 args: chdir: ../.. register: out @@ -59,6 +62,7 @@ ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -75,6 +79,7 @@ ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -82,6 +87,7 @@ test-targets -e ansible_user=mitogen__has_sudo -e ansible_ssh_pass=wrong_password + -e ansible_python_interpreter=python3000 args: chdir: ../.. register: out @@ -98,6 +104,7 @@ ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -114,6 +121,7 @@ ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -121,6 +129,7 @@ test-targets -e ansible_user=mitogen__has_sudo -e ansible_password=wrong_password + -e ansible_python_interpreter=python3000 args: chdir: ../.. register: out @@ -142,6 +151,7 @@ ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -158,6 +168,7 @@ ANSIBLE_ANY_ERRORS_FATAL=false ANSIBLE_STRATEGY=mitogen_linear ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} -i "{{ inv }}" @@ -165,6 +176,7 @@ test-targets -e ansible_user=mitogen__has_sudo -e ansible_ssh_private_key_file=/dev/null + -e ansible_python_interpreter=python3000 args: chdir: ../.. register: out diff --git a/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml b/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml index d183564f..4220ed4c 100644 --- a/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml +++ b/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml @@ -3,24 +3,26 @@ hosts: test-targets[0] tasks: - connection: local + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: | ansible-playbook {% for inv in ansible_inventory_sources %} -i "{{ inv }}" {% endfor %} - -vvv integration/strategy/_mixed_mitogen_vanilla.yml args: chdir: ../.. register: out - connection: local + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: | ansible-playbook {% for inv in ansible_inventory_sources %} -i "{{ inv }}" {% endfor %} - -vvv integration/strategy/_mixed_vanilla_mitogen.yml args: chdir: ../.. diff --git a/tests/ansible/integration/stub_connections/setns_lxc.yml b/tests/ansible/integration/stub_connections/setns_lxc.yml index 889a15e9..f8d6bfc5 100644 --- a/tests/ansible/integration/stub_connections/setns_lxc.yml +++ b/tests/ansible/integration/stub_connections/setns_lxc.yml @@ -12,6 +12,8 @@ - include_tasks: _end_play_if_not_sudo_linux.yml - name: Run stub-lxc-info.py + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: | sudo -nE "{{lookup('env', 'VIRTUAL_ENV')}}/bin/ansible" -i localhost, diff --git a/tests/ansible/integration/stub_connections/setns_lxd.yml b/tests/ansible/integration/stub_connections/setns_lxd.yml index 81a52dbb..5aba96c5 100644 --- a/tests/ansible/integration/stub_connections/setns_lxd.yml +++ b/tests/ansible/integration/stub_connections/setns_lxd.yml @@ -12,6 +12,8 @@ - include_tasks: _end_play_if_not_sudo_linux.yml - name: Run ansible stub-lxc.py + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: | sudo -nE "{{lookup('env', 'VIRTUAL_ENV')}}/bin/ansible" -i localhost, diff --git a/tests/ansible/integration/transport_config/all.yml b/tests/ansible/integration/transport_config/all.yml index 548e7f7e..b486549b 100644 --- a/tests/ansible/integration/transport_config/all.yml +++ b/tests/ansible/integration/transport_config/all.yml @@ -2,6 +2,7 @@ - import_playbook: become_pass.yml - import_playbook: become_user.yml - import_playbook: become.yml +- import_playbook: host_key_checking.yml - import_playbook: password.yml - import_playbook: port.yml - import_playbook: python_path.yml diff --git a/tests/ansible/integration/transport_config/become_pass.yml b/tests/ansible/integration/transport_config/become_pass.yml index 8fbe7251..b9bf1a8e 100644 --- a/tests/ansible/integration/transport_config/become_pass.yml +++ b/tests/ansible/integration/transport_config/become_pass.yml @@ -143,8 +143,9 @@ - out.result|length == 2 - out.result[0].method == "ssh" - out.result[1].method == "sudo" - # Ansible >= 2.10 builtin become plugins (e.g. sudo, su) give priority - # to ansible_become_pass over ansible_become_password. + # Ansible <= 2.9.1 prioritises ansible_become_password. + # Ansible >= 2.9.2 prioritises ansible_become_pass. + # https://github.com/ansible/ansible/commit/480b106d6535978ae6ecab68b40942ca4fa914a0 - out.result[1].kwargs.password == "bpass" fail_msg: out={{out}} tags: diff --git a/tests/ansible/integration/transport_config/host_key_checking.yml b/tests/ansible/integration/transport_config/host_key_checking.yml new file mode 100644 index 00000000..b15e36b1 --- /dev/null +++ b/tests/ansible/integration/transport_config/host_key_checking.yml @@ -0,0 +1,94 @@ +# Each case is followed by mitogen_via= case to test hostvars method. + +- name: integration/transport_config/host_key_checking.yml + hosts: tc-hkc-unset + tasks: + - include_tasks: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result | length == 1 + - out.result[0].method == "ssh" + - out.result[0].kwargs.check_host_keys == "ignore" + fail_msg: out={{ out }} + tags: + - mitogen_only + +- hosts: tc-hkc-unset + vars: + mitogen_via: tc-hkc-host-key-checking + tasks: + - include_tasks: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result | length == 2 + - out.result[0].method == "ssh" + - out.result[0].kwargs.check_host_keys == "enforce" + - out.result[1].method == "ssh" + - out.result[1].kwargs.check_host_keys == "ignore" + fail_msg: out={{ out }} + tags: + - mitogen_only + + +- hosts: tc-hkc-host-key-checking + tasks: + - include_tasks: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result | length == 1 + - out.result[0].method == "ssh" + - out.result[0].kwargs.check_host_keys == "enforce" + fail_msg: out={{ out }} + tags: + - mitogen_only + +- hosts: tc-hkc-host-key-checking + vars: + mitogen_via: tc-hkc-unset + tasks: + - include_tasks: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result | length == 2 + - out.result[0].method == "ssh" + - out.result[0].kwargs.check_host_keys == "ignore" + - out.result[1].method == "ssh" + - out.result[1].kwargs.check_host_keys == "enforce" + fail_msg: out={{ out }} + tags: + - mitogen_only + + +- hosts: tc-hkc-ssh-host-key-checking + tasks: + - include_tasks: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result | length == 1 + - out.result[0].method == "ssh" + - out.result[0].kwargs.check_host_keys == "enforce" + fail_msg: out={{ out }} + tags: + - mitogen_only + +- hosts: tc-hkc-ssh-host-key-checking + vars: + mitogen_via: tc-hkc-unset + tasks: + - include_tasks: ../_mitogen_only.yml + - {mitogen_get_stack: {}, register: out} + - assert: + that: + - out.result | length == 2 + - out.result[0].method == "ssh" + - out.result[0].kwargs.check_host_keys == "ignore" + - out.result[1].method == "ssh" + - out.result[1].kwargs.check_host_keys == "enforce" + fail_msg: out={{ out }} + tags: + - mitogen_only diff --git a/tests/ansible/lib/modules/custom_python_json_args_module.py b/tests/ansible/lib/modules/custom_python_json_args_module.py index 846037ec..61640579 100755 --- a/tests/ansible/lib/modules/custom_python_json_args_module.py +++ b/tests/ansible/lib/modules/custom_python_json_args_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible Python JSONARGS module. I should receive an encoding string. json_arguments = """<>""" diff --git a/tests/ansible/lib/modules/custom_python_os_getcwd.py b/tests/ansible/lib/modules/custom_python_os_getcwd.py index d465ac9e..c5e264ae 100644 --- a/tests/ansible/lib/modules/custom_python_os_getcwd.py +++ b/tests/ansible/lib/modules/custom_python_os_getcwd.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # #591: call os.getcwd() before AnsibleModule ever gets a chance to fix up the # process environment. diff --git a/tests/ansible/lib/modules/custom_python_want_json_module.py b/tests/ansible/lib/modules/custom_python_want_json_module.py index 23eeeb55..f5e33862 100755 --- a/tests/ansible/lib/modules/custom_python_want_json_module.py +++ b/tests/ansible/lib/modules/custom_python_want_json_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible Python WANT_JSON module. I should receive a JSON-encoded file. import json diff --git a/tests/ansible/regression/all.yml b/tests/ansible/regression/all.yml index 0e599476..31541e9f 100644 --- a/tests/ansible/regression/all.yml +++ b/tests/ansible/regression/all.yml @@ -14,3 +14,5 @@ - import_playbook: issue_615__streaming_transfer.yml - import_playbook: issue_655__wait_for_connection_error.yml - import_playbook: issue_776__load_plugins_called_twice.yml +- import_playbook: issue_952__ask_become_pass.yml +- import_playbook: issue_1066__add_host__host_key_checking.yml diff --git a/tests/ansible/regression/become_test.yml b/tests/ansible/regression/become_test.yml new file mode 100644 index 00000000..5af2e123 --- /dev/null +++ b/tests/ansible/regression/become_test.yml @@ -0,0 +1,9 @@ +- name: regression/become_test.yml + hosts: test-targets:&linux_containers + become: true + become_user: mitogen__pw_required + strategy: mitogen_linear + tasks: + - command: whoami + changed_when: false + check_mode: false diff --git a/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml b/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml new file mode 100644 index 00000000..4b4609b6 --- /dev/null +++ b/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml @@ -0,0 +1,67 @@ +- name: regression/issue_1066__add_host__host_key_checking.yml + hosts: test-targets[0] + gather_facts: false + become: false + tasks: + - name: Add hosts dynamically + add_host: + name: "{{ item.name }}" + ansible_host_key_checking: "{{ item.host_key_checking | default(omit) }}" + ansible_ssh_host_key_checking: "{{ item.host_ssh_key_checking | default(omit) }}" + ansible_host: "{{ hostvars[inventory_hostname].ansible_host | default(omit) }}" + ansible_password: "{{ hostvars[inventory_hostname].ansible_password | default(omit) }}" + ansible_port: "{{ hostvars[inventory_hostname].ansible_port | default(omit) }}" + ansible_python_interpreter: "{{ hostvars[inventory_hostname].ansible_python_interpreter | default(omit) }}" + ansible_user: "{{ hostvars[inventory_hostname].ansible_user | default(omit) }}" + loop: + - {name: issue-1066-host-hkc-false, host_key_checking: false} + - {name: issue-1066-host-hkc-true, host_key_checking: true} + - {name: issue-1066-host-hskc-false, host_ssh_key_checking: false} + - {name: issue-1066-host-hskc-true, host_ssh_key_checking: true} + delegate_to: localhost + tags: + - issue_1066 + +- name: regression/issue_1066__add_host__host_key_checking.yml + hosts: issue-1066-host-* + gather_facts: false + become: false + serial: 1 + tasks: + - meta: reset_connection + + # The host key might be in ~/.ssh/known_hosts. If it's removed then no + # problem - test-targets hosts have host_key_checking=false. + - name: Remove existing host keys + known_hosts: + name: "{{ ansible_host }}" + state: absent + delegate_to: localhost + + - name: Ping dynamically added hosts + ping: + ignore_errors: true + ignore_unreachable: true + register: issue_1066_ping + + - debug: + var: issue_1066_ping + + - name: Confirm dynamically added hosts are/are not reachable + vars: + expected: + issue-1066-host-hkc-false: {} + issue-1066-host-hkc-true: {unreachable: true} + issue-1066-host-hskc-false: {} + issue-1066-host-hskc-true: {unreachable: true} + assert: + that: + - issue_1066_ping.unreachable is defined == expected[inventory_hostname].unreachable is defined + - issue_1066_ping.unreachable | default(42) == expected[inventory_hostname].unreachable | default(42) + # ansible_host_key_checking don't work on Vanilla Ansible 2.10, even for + # static inventory hosts (ansible/ansible#49254, ansible/ansible#73708). + when: + - ansible_version.full is version('2.11', '>=', strict=True) + or is_mitogen + tags: + - issue_1066 diff --git a/tests/ansible/regression/issue_140__thread_pileup.yml b/tests/ansible/regression/issue_140__thread_pileup.yml index 1f614634..163440a5 100644 --- a/tests/ansible/regression/issue_140__thread_pileup.yml +++ b/tests/ansible/regression/issue_140__thread_pileup.yml @@ -8,6 +8,7 @@ - name: Create file tree connection: local + run_once: true shell: > mkdir /tmp/filetree.in; seq -f /tmp/filetree.in/%g 1 1000 | xargs touch; @@ -21,17 +22,30 @@ file: state: directory path: /tmp/filetree.out + mode: u=rwx,go=rx - name: Trigger nasty process pileup copy: src: "{{item.src}}" dest: "/tmp/filetree.out/{{item.path}}" - mode: 0644 + mode: u=rw,go=r with_filetree: /tmp/filetree.in when: item.state == 'file' loop_control: label: "/tmp/filetree.out/{{ item.path }}" + - name: Cleanup local file tree + connection: local + run_once: true + file: + path: /tmp/filetree.in + state: absent + + - name: Cleanup remote file tree + file: + path: /tmp/filetree.out + state: absent + tags: - resource_intensive - issue_140 diff --git a/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml b/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml index fe7f5f42..dd86775d 100644 --- a/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml +++ b/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml @@ -4,10 +4,9 @@ # can test for. - name: regression/issue_152__local_action_wrong_interpreter.yml - hosts: test-targets + hosts: test-targets[0] connection: local tasks: - - name: Create /tmp/issue_152_interpreter.sh copy: dest: /tmp/issue_152_interpreter.sh diff --git a/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml b/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml index d5f0815b..ff102b13 100644 --- a/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml +++ b/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml @@ -19,9 +19,6 @@ # Will crash if process has a nonexistent CWD. - custom_python_os_getcwd: - script: | - import os - self._connection.get_chain().call(os.getcwd) tags: - issue_591 - mitogen_only diff --git a/tests/ansible/regression/issue_615__streaming_transfer.yml b/tests/ansible/regression/issue_615__streaming_transfer.yml index 43aa1343..6fe52d55 100644 --- a/tests/ansible/regression/issue_615__streaming_transfer.yml +++ b/tests/ansible/regression/issue_615__streaming_transfer.yml @@ -14,11 +14,14 @@ shell: | dd if=/dev/zero of=/tmp/512mb.zero bs=1048576 count=512; chmod go= /tmp/512mb.zero + args: + creates: /tmp/512mb.zero - name: Fetch /tmp/512mb.zero fetch: src: /tmp/512mb.zero - dest: /tmp/fetch-out + dest: /tmp/fetch-{{ inventory_hostname }}-512mb.zero + flat: true - name: Cleanup /tmp/512mb.zero file: @@ -27,11 +30,10 @@ - name: Cleanup fetched file file: - path: /tmp/fetch-out + path: /tmp/fetch-{{ inventory_hostname }}-512mb.zero state: absent become: false delegate_to: localhost - run_once: true tags: - issue_615 - mitogen_only diff --git a/tests/ansible/regression/issue_655__wait_for_connection_error.yml b/tests/ansible/regression/issue_655__wait_for_connection_error.yml index fbdb9f2b..4972d91a 100644 --- a/tests/ansible/regression/issue_655__wait_for_connection_error.yml +++ b/tests/ansible/regression/issue_655__wait_for_connection_error.yml @@ -4,11 +4,18 @@ # since things are ran on localhost; Azure DevOps loses connection and fails # TODO: do we want to install docker a different way to be able to do this for other tests too --- -- name: regression/issue_655_wait_for_connection_error.yml +- name: regression/issue_655__wait_for_connection_error.yml hosts: localhost gather_facts: yes become: no tasks: + - meta: end_play + when: + # TODO CI currently runs on macOS 12 & which isn't supported by Podman + # version available in Homebrew. + - ansible_facts.system == 'Darwin' + - ansible_facts.distribution_version is version('13.0', '<', strict=True) + - name: set up test container and run tests inside it block: - name: install deps diff --git a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml index ad5bbd69..44ff6863 100755 --- a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml +++ b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml @@ -10,21 +10,14 @@ ansible_python_interpreter: "{{ pkg_mgr_python_interpreter }}" package: rsync # Chosen to exist in all tested distros/package managers tasks: - - name: Switch to centos-stream - command: dnf --assumeyes --disablerepo="*" --enablerepo=extras swap centos-linux-repos centos-stream-repos - when: - - ansible_facts.pkg_mgr in ["dnf"] - - - name: Switch to archive.debian.org - # Debian 9 has been archived https://lists.debian.org/debian-devel-announce/2023/03/msg00006.html + - name: Switch to archived package repositories copy: - content: | - deb http://archive.debian.org/debian stretch main contrib non-free - dest: /etc/apt/sources.list + dest: "{{ item.dest }}" + content: "{{ item.content }}" mode: u=rw,go=r - when: - - ansible_facts.distribution == "Debian" - - ansible_facts.distribution_major_version == "9" + loop: "{{ pkg_repos_overrides }}" + loop_control: + label: "{{ item.dest }}" - name: Add signing keys copy: diff --git a/tests/ansible/regression/issue_952__ask_become_pass.yml b/tests/ansible/regression/issue_952__ask_become_pass.yml new file mode 100644 index 00000000..a0b92ff2 --- /dev/null +++ b/tests/ansible/regression/issue_952__ask_become_pass.yml @@ -0,0 +1,23 @@ +- name: regression/issue_952__ask_become_pass.yml + hosts: test-targets[0]:&linux_containers + gather_facts: false + tags: + - issue_952 + tasks: + - name: Test --ask-become-pass + delegate_to: localhost + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" + expect: + command: > + ansible-playbook + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} + --ask-become-pass + regression/become_test.yml + chdir: ../ + responses: + 'BECOME password:': pw_required_password + changed_when: false + check_mode: false diff --git a/tests/ansible/requirements.txt b/tests/ansible/requirements.txt index 2c3c87c8..8cfb348a 100644 --- a/tests/ansible/requirements.txt +++ b/tests/ansible/requirements.txt @@ -1,4 +1,7 @@ paramiko==2.3.2 # Last 2.6-compat version. +# Incompatible with pip >= 72, due to removal of `setup.py test`: +# ModuleNotFoundError: No module named 'setuptools.command.test' +# https://github.com/pypa/setuptools/issues/4519 hdrhistogram==0.6.1 PyYAML==3.11; python_version < '2.7' PyYAML==5.3.1; python_version >= '2.7' # Latest release (Jan 2021) diff --git a/tests/ansible/setup/report_controller.yml b/tests/ansible/setup/report_controller.yml index d0d5cc15..5e34fb17 100644 --- a/tests/ansible/setup/report_controller.yml +++ b/tests/ansible/setup/report_controller.yml @@ -1,5 +1,5 @@ - name: Report controller parameters - hosts: localhost + hosts: test-targets[0] gather_facts: false tasks: - debug: @@ -9,9 +9,11 @@ - $(groups): "{{ lookup('pipe', 'groups') }}" - $(pwd): "{{ lookup('pipe', 'pwd') }}" - $(whoami): "{{ lookup('pipe', 'whoami') }}" + - ansible_inventory_sources: "{{ ansible_inventory_sources | default('') }}" - ansible_run_tags: "{{ ansible_run_tags | default('') }}" - ansible_playbook_python: "{{ ansible_playbook_python | default('') }}" - ansible_skip_tags: "{{ ansible_skip_tags | default('') }}" - ansible_version.full: "{{ ansible_version.full | default('') }}" - is_mitogen: "{{ is_mitogen | default('') }}" - playbook_dir: "{{ playbook_dir | default('') }}" + delegate_to: localhost diff --git a/tests/constraints.txt b/tests/constraints.txt new file mode 100644 index 00000000..6adaa30b --- /dev/null +++ b/tests/constraints.txt @@ -0,0 +1,3 @@ +# Setuptools 72 removed `setup.py test`. hdrhistogram 0.6.1 still depends on it. +# TODO Bump dependencies and unconstrain Pip. +setuptools<72 diff --git a/tests/requirements.txt b/tests/requirements.txt index 6d87d177..d64d8b87 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -12,6 +12,8 @@ Django==3.2.20; python_version >= '3.6' mock==3.0.5; python_version == '2.7' mock==5.1.0; python_version >= '3.6' +pexpect==4.8 + psutil==5.9.8 pytest==4.6.11; python_version == '2.7' diff --git a/tests/two_three_compat_test.py b/tests/two_three_compat_test.py index 4490e5d2..ab9f4e19 100644 --- a/tests/two_three_compat_test.py +++ b/tests/two_three_compat_test.py @@ -1,3 +1,6 @@ +import os +import unittest + import mitogen.core import testlib @@ -7,6 +10,10 @@ import simple_pkg.ping # TODO: this is a joke. 2/3 interop is one of the hardest bits to get right. # There should be 100 tests in this file. +@unittest.skipIf( + os.uname()[0] == 'Darwin' and int(os.uname()[2].partition('.')[0]) >= 21, + "Python 2.x not shipped on macOS 12.3+ (Darwin 21.4+, Monterey)", +) class TwoThreeCompatTest(testlib.RouterMixin, testlib.TestCase): if mitogen.core.PY3: python_path = 'python2' diff --git a/tox.ini b/tox.ini index 69639233..a0b0e1ae 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ # # sudo add-apt-repository ppa:deadsnakes/ppa # sudo apt update -# sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python2.7 python3.{6..13}{,-venv} python-is-python3 sshpass tox +# sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python{2,2.7,3} python3.{6..13}{,-venv} python-is-python3 sshpass tox # Py A cntrllr A target coverage Django Jinja2 pip psutil pytest tox virtualenv # ==== ========== ========== ========== ========== ========== ========== ========== ========== ========== ========== @@ -109,6 +109,7 @@ setenv = ANSIBLE_STRATEGY = mitogen_linear NOCOVERAGE_ERASE = 1 NOCOVERAGE_REPORT = 1 + PIP_CONSTRAINT={toxinidir}/tests/constraints.txt # Only applicable to MODE=mitogen distro_centos5: DISTRO=centos5 distro_centos6: DISTRO=centos6 @@ -142,6 +143,8 @@ setenv = distros_ubuntu2004: DISTROS=ubuntu2004 mode_ansible: MODE=ansible mode_ansible: ANSIBLE_SKIP_TAGS=resource_intensive + mode_ansible: ANSIBLE_CALLBACK_WHITELIST=profile_tasks + mode_ansible: ANSIBLE_CALLBACKS_ENABLED=profile_tasks mode_debops_common: MODE=debops_common mode_localhost: ANSIBLE_SKIP_TAGS=issue_776,resource_intensive mode_mitogen: MODE=mitogen