diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index c50c76da..9901c742 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -34,12 +34,12 @@ ANSIBLE_TESTS_HOSTS_DIR = os.path.join(GIT_ROOT, 'tests/ansible/hosts') ANSIBLE_TESTS_TEMPLATES_DIR = os.path.join(GIT_ROOT, 'tests/ansible/templates') DISTRO_SPECS = os.environ.get( 'MITOGEN_TEST_DISTRO_SPECS', - 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004', + 'alma9-py3 centos5 centos8-py3 debian9 debian12-py3 ubuntu1604 ubuntu2404-py3', ) IMAGE_PREP_DIR = os.path.join(GIT_ROOT, 'tests/image_prep') IMAGE_TEMPLATE = os.environ.get( 'MITOGEN_TEST_IMAGE_TEMPLATE', - 'ghcr.io/mitogen-hq/%(distro)s-test:2021', + 'ghcr.io/mitogen-hq/%(distro)s-test:2025.02', ) SUDOERS_DEFAULTS_SRC = './tests/image_prep/files/sudoers_defaults' SUDOERS_DEFAULTS_DEST = '/etc/sudoers.d/mitogen_test_defaults' @@ -156,7 +156,15 @@ def run_batches(batches): subprocess.Popen(combine(batch), shell=True) for batch in batches ] - assert [proc.wait() for proc in procs] == [0] * len(procs) + for proc in procs: + proc.wait() + if proc.returncode: + print( + 'proc: pid=%i rc=%i args=%r' + % (proc.pid, proc.returncode, proc.args), + file=sys.stderr, flush=True, + ) + assert [proc.returncode for proc in procs] == [0] * len(procs) def get_output(s, *args, **kwargs): @@ -231,7 +239,7 @@ def container_specs( [{'distro': 'debian11', 'family': 'debian', 'hostname': 'localhost', - 'image': 'ghcr.io/mitogen-hq/debian11-test:2021', + 'image': 'ghcr.io/mitogen-hq/debian11-test:2025.02', 'index': 1, 'name': 'target-debian11-1', 'port': 2201, @@ -239,7 +247,7 @@ def container_specs( {'distro': 'centos6', 'family': 'centos', 'hostname': 'localhost', - 'image': 'ghcr.io/mitogen-hq/centos6-test:2021', + 'image': 'ghcr.io/mitogen-hq/centos6-test:2025.02', 'index': 2, 'name': 'target-centos6-2', 'port': 2202, diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ebfad8fd..7d91954a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -153,8 +153,8 @@ jobs: macos: name: macos ${{ matrix.tox_env }} - # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md - runs-on: macos-13 + # https://github.com/actions/runner-images/blob/main/images/macos/macos-15-Readme.md + runs-on: macos-15 timeout-minutes: 15 strategy: diff --git a/.gitignore b/.gitignore index 43e46a19..b3710ea2 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ build/ dist/ extra/ tests/ansible/.*.pid +tests/image_prep/logs docs/_build/ htmlcov/ *.egg-info diff --git a/docs/changelog.rst b/docs/changelog.rst index 9eabf7a7..a2fcba66 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,21 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. +v0.3.34 (2025-11-27) +-------------------- + +* :gh:issue:`1118` CI: Use 2025.02 test images, keeping same OS releases +* :gh:issue:`1358` CI: Bump deprecated macOS 13 runner to macOS 15 +* :gh:issue:`1118` CI: Add OS release coverage: AlmaLinux 9 +* :gh:issue:`1118` CI: Add OS release coverage: CentOS 5 +* :gh:issue:`1118` CI: Add OS release coverage: Debian 12 +* :gh:issue:`1118` CI: Add OS release coverage: Ubuntu 22.04, Ubuntu 24.04 +* :gh:issue:`1124` :mod:`mitogen`: Log why a module is sent or not sent by + :class:`mitogen.master.ModuleResponder` +* :gh:issue:`1124` :mod:`ansible_mitogen`: Speedup startup by not sending + ``__main__`` as a related module + + v0.3.33 (2025-11-22) -------------------- diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 2d0af387..740515a7 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 33) +__version__ = (0, 3, 34) #: This is :data:`False` in slave contexts. Previously it was used to prevent diff --git a/mitogen/master.py b/mitogen/master.py index 3b896504..e495197d 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -843,6 +843,12 @@ class ModuleFinder(object): related modules likely needed by a child context requesting the original module. """ + + # Fullnames of modules that should not be sent as a related module + _related_modules_denylist = frozenset({ + '__main__', + }) + def __init__(self): #: Import machinery is expensive, keep :py:meth`:get_module_source` #: results around. @@ -931,6 +937,34 @@ class ModuleFinder(object): fullname, _, _ = str_rpartition(to_text(fullname), u'.') yield fullname + def _reject_related_module(self, requested_fullname, related_fullname): + def _log_reject(reason): + LOG.debug( + '%r: Rejected related module %s of requested module %s: %s', + self, related_fullname, requested_fullname, reason, + ) + return reason + + try: + related_module = sys.modules[related_fullname] + except KeyError: + return _log_reject('sys.modules entry absent') + + # Python 2.x "indirection entry" + if related_module is None: + return _log_reject('sys.modules entry is None') + + if is_stdlib_name(related_fullname): + return _log_reject('stdlib module') + + if 'six.moves' in related_fullname: + return _log_reject('six.moves avoidence') + + if related_fullname in self._related_modules_denylist: + return _log_reject('on denylist') + + return False + def find_related_imports(self, fullname): """ Return a list of non-stdlib modules that are directly imported by @@ -973,9 +1007,7 @@ class ModuleFinder(object): set( mitogen.core.to_text(name) for name in maybe_names - if sys.modules.get(name) is not None - and not is_stdlib_name(name) - and u'six.moves' not in name # TODO: crap + if not self._reject_related_module(fullname, name) ) )) @@ -1138,7 +1170,7 @@ class ModuleResponder(object): self._cache[fullname] = tup return tup - def _send_load_module(self, stream, fullname): + def _send_load_module(self, stream, fullname, reason): if fullname not in stream.protocol.sent_modules: tup = self._build_tuple(fullname) msg = mitogen.core.Message.pickled( @@ -1146,8 +1178,10 @@ class ModuleResponder(object): dst_id=stream.protocol.remote_id, handle=mitogen.core.LOAD_MODULE, ) - self._log.debug('sending %s (%.2f KiB) to %s', - fullname, len(msg.data) / 1024.0, stream.name) + self._log.debug( + 'sending %s %s (%.2f KiB) to %s', + reason, fullname, len(msg.data) / 1024.0, stream.name, + ) self._router._async_route(msg) stream.protocol.sent_modules.add(fullname) if tup[2] is not None: @@ -1178,8 +1212,8 @@ class ModuleResponder(object): # Parent hasn't been sent, so don't load submodule yet. continue - self._send_load_module(stream, name) - self._send_load_module(stream, fullname) + self._send_load_module(stream, name, 'related') + self._send_load_module(stream, fullname, 'requested') except Exception: LOG.debug('While importing %r', fullname, exc_info=True) self._send_module_load_failed(stream, fullname) diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index 4060d9ea..533c58a9 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -76,6 +76,7 @@ ssh_args = -o ControlPersist=60s -o ForwardAgent=yes -o HostKeyAlgorithms=+ssh-rsa + -o KexAlgorithms=+diffie-hellman-group1-sha1 -o PubkeyAcceptedKeyTypes=+ssh-rsa -o UserKnownHostsFile=/dev/null pipelining = True diff --git a/tests/ansible/hosts/group_vars/all.yml b/tests/ansible/hosts/group_vars/all.yml index 7ec10d6a..ca96f25f 100644 --- a/tests/ansible/hosts/group_vars/all.yml +++ b/tests/ansible/hosts/group_vars/all.yml @@ -6,6 +6,7 @@ ansible_version_major_minor: "{{ ansible_version.major }}.{{ ansible_version.minor }}" ansible_version_major_minor_patch: "{{ ansible_version.major }}.{{ ansible_version.minor }}.{{ ansible_version.revision | regex_search('^[0-9]+') }}" +become_doas_available: false become_unpriv_available: >- {# Vanilla Ansible >= 4 (ansible-core >= 2.11) can use `setfacl` for @@ -34,3 +35,11 @@ become_unpriv_available: >- -}} pkg_mgr_python_interpreter: python + +virtualenv_create_argv: + - virtualenv + - -p + - "{{ virtualenv_python }}" + - "{{ virtualenv_path }}" +virtualenv_path: /path/intentionally/left/invalid +virtualenv_python: /path/intentionally/left/invalid diff --git a/tests/ansible/hosts/group_vars/alma9.yml b/tests/ansible/hosts/group_vars/alma9.yml new file mode 100644 index 00000000..25fd10bd --- /dev/null +++ b/tests/ansible/hosts/group_vars/alma9.yml @@ -0,0 +1,8 @@ +pkg_mgr_python_interpreter: python3 + +# Alma Linux 9, RHEL 9, etc. lack a virtualenv package +virtualenv_create_argv: + - "{{ virtualenv_python }}" + - -m + - venv + - "{{ virtualenv_path }}" diff --git a/tests/ansible/hosts/group_vars/debian11.yml b/tests/ansible/hosts/group_vars/debian11.yml index 30c98341..cecc3c50 100644 --- a/tests/ansible/hosts/group_vars/debian11.yml +++ b/tests/ansible/hosts/group_vars/debian11.yml @@ -1,3 +1,4 @@ +become_doas_available: true package_manager_keys: - src: debian-archive-bullseye-automatic.gpg # Debian 11 dest: /etc/apt/trusted.gpg.d/debian-archive-bullseye-automatic.gpg diff --git a/tests/ansible/hosts/group_vars/debian12.yml b/tests/ansible/hosts/group_vars/debian12.yml new file mode 100644 index 00000000..f2f5fd56 --- /dev/null +++ b/tests/ansible/hosts/group_vars/debian12.yml @@ -0,0 +1,2 @@ +become_doas_available: true +pkg_mgr_python_interpreter: python3 diff --git a/tests/ansible/hosts/group_vars/debian9.yml b/tests/ansible/hosts/group_vars/debian9.yml index e08b1ed2..5be6ee80 100644 --- a/tests/ansible/hosts/group_vars/debian9.yml +++ b/tests/ansible/hosts/group_vars/debian9.yml @@ -1,4 +1,6 @@ package_manager_repos: - dest: /etc/apt/sources.list content: | - deb http://archive.debian.org/debian stretch main contrib non-free + deb http://archive.debian.org/debian/ stretch main contrib non-free + deb http://archive.debian.org/debian/ stretch-proposed-updates main contrib non-free + deb http://archive.debian.org/debian-security stretch/updates main contrib non-free diff --git a/tests/ansible/hosts/group_vars/ubuntu2204.yml b/tests/ansible/hosts/group_vars/ubuntu2204.yml new file mode 100644 index 00000000..f2f5fd56 --- /dev/null +++ b/tests/ansible/hosts/group_vars/ubuntu2204.yml @@ -0,0 +1,2 @@ +become_doas_available: true +pkg_mgr_python_interpreter: python3 diff --git a/tests/ansible/hosts/group_vars/ubuntu2404.yml b/tests/ansible/hosts/group_vars/ubuntu2404.yml new file mode 100644 index 00000000..f2f5fd56 --- /dev/null +++ b/tests/ansible/hosts/group_vars/ubuntu2404.yml @@ -0,0 +1,2 @@ +become_doas_available: true +pkg_mgr_python_interpreter: python3 diff --git a/tests/ansible/integration/become/doas.yml b/tests/ansible/integration/become/doas.yml index 31858168..296b3084 100644 --- a/tests/ansible/integration/become/doas.yml +++ b/tests/ansible/integration/become/doas.yml @@ -15,12 +15,16 @@ changed_when: false check_mode: false register: doas_default_user + when: + - become_doas_available - assert: that: - doas_default_user.stdout == 'root' fail_msg: doas_default_user={{ doas_default_user }} + when: + - become_doas_available - name: Test doas -> mitogen__user1 become: true @@ -30,6 +34,7 @@ check_mode: false register: doas_mitogen__user1 when: + - become_doas_available - become_unpriv_available - assert: @@ -38,6 +43,7 @@ fail_msg: doas_mitogen__user1={{ doas_mitogen__user1 }} when: + - become_doas_available - become_unpriv_available tags: - doas @@ -61,12 +67,16 @@ changed_when: false check_mode: false register: fq_doas_default_user + when: + - become_doas_available - assert: that: - fq_doas_default_user.stdout == 'root' fail_msg: fq_doas_default_user={{ fq_doas_default_user }} + when: + - become_doas_available - name: Test community.general.doas -> mitogen__user1 become: true @@ -76,6 +86,7 @@ check_mode: false register: fq_doas_mitogen__user1 when: + - become_doas_available - become_unpriv_available - assert: @@ -84,6 +95,7 @@ fail_msg: fq_doas_mitogen__user1={{ fq_doas_mitogen__user1 }} when: + - become_doas_available - become_unpriv_available tags: - doas diff --git a/tests/ansible/integration/connection_delegation/delegate_to_template.yml b/tests/ansible/integration/connection_delegation/delegate_to_template.yml index 8cd50f98..d46103d6 100644 --- a/tests/ansible/integration/connection_delegation/delegate_to_template.yml +++ b/tests/ansible/integration/connection_delegation/delegate_to_template.yml @@ -47,6 +47,7 @@ -o, ControlPersist=60s, -o, ForwardAgent=yes, -o, HostKeyAlgorithms=+ssh-rsa, + -o, KexAlgorithms=+diffie-hellman-group1-sha1, -o, PubkeyAcceptedKeyTypes=+ssh-rsa, -o, UserKnownHostsFile=/dev/null, ], @@ -75,6 +76,7 @@ -o, ControlPersist=60s, -o, ForwardAgent=yes, -o, HostKeyAlgorithms=+ssh-rsa, + -o, KexAlgorithms=+diffie-hellman-group1-sha1, -o, PubkeyAcceptedKeyTypes=+ssh-rsa, -o, UserKnownHostsFile=/dev/null, ], diff --git a/tests/ansible/integration/connection_delegation/stack_construction.yml b/tests/ansible/integration/connection_delegation/stack_construction.yml index 58abac7b..b38d835b 100644 --- a/tests/ansible/integration/connection_delegation/stack_construction.yml +++ b/tests/ansible/integration/connection_delegation/stack_construction.yml @@ -84,6 +84,7 @@ -o, ControlPersist=60s, -o, ForwardAgent=yes, -o, HostKeyAlgorithms=+ssh-rsa, + -o, KexAlgorithms=+diffie-hellman-group1-sha1, -o, PubkeyAcceptedKeyTypes=+ssh-rsa, -o, UserKnownHostsFile=/dev/null, ], @@ -128,6 +129,7 @@ -o, ControlPersist=60s, -o, ForwardAgent=yes, -o, HostKeyAlgorithms=+ssh-rsa, + -o, KexAlgorithms=+diffie-hellman-group1-sha1, -o, PubkeyAcceptedKeyTypes=+ssh-rsa, -o, UserKnownHostsFile=/dev/null, ], @@ -183,6 +185,7 @@ -o, ControlPersist=60s, -o, ForwardAgent=yes, -o, HostKeyAlgorithms=+ssh-rsa, + -o, KexAlgorithms=+diffie-hellman-group1-sha1, -o, PubkeyAcceptedKeyTypes=+ssh-rsa, -o, UserKnownHostsFile=/dev/null, ], @@ -227,6 +230,7 @@ -o, ControlPersist=60s, -o, ForwardAgent=yes, -o, HostKeyAlgorithms=+ssh-rsa, + -o, KexAlgorithms=+diffie-hellman-group1-sha1, -o, PubkeyAcceptedKeyTypes=+ssh-rsa, -o, UserKnownHostsFile=/dev/null, ], @@ -255,6 +259,7 @@ -o, ControlPersist=60s, -o, ForwardAgent=yes, -o, HostKeyAlgorithms=+ssh-rsa, + -o, KexAlgorithms=+diffie-hellman-group1-sha1, -o, PubkeyAcceptedKeyTypes=+ssh-rsa, -o, UserKnownHostsFile=/dev/null, ], @@ -309,6 +314,7 @@ -o, ControlPersist=60s, -o, ForwardAgent=yes, -o, HostKeyAlgorithms=+ssh-rsa, + -o, KexAlgorithms=+diffie-hellman-group1-sha1, -o, PubkeyAcceptedKeyTypes=+ssh-rsa, -o, UserKnownHostsFile=/dev/null, ], @@ -354,6 +360,7 @@ -o, ControlPersist=60s, -o, ForwardAgent=yes, -o, HostKeyAlgorithms=+ssh-rsa, + -o, KexAlgorithms=+diffie-hellman-group1-sha1, -o, PubkeyAcceptedKeyTypes=+ssh-rsa, -o, UserKnownHostsFile=/dev/null, ], diff --git a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml index 451b4dc3..fd82d2ea 100644 --- a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml +++ b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml @@ -1,58 +1,20 @@ # ripped and ported from https://github.com/ansible/ansible/pull/50163/files, when interpreter discovery was added to ansible --- +- name: integration/interpreter_discovery/ansible_2_8_tests.yml, baseline + hosts: test-targets + strategy: linear + tasks: + - meta: clear_facts + - name: Discover interpreter, linear, auto + vars: + ansible_python_interpreter: auto + ping: + register: linear_auto_result + - name: integration/interpreter_discovery/ansible_2_8_tests.yml hosts: test-targets gather_facts: true - vars: - DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_lt_2_12: - centos: - '6': /usr/bin/python - '7': /usr/bin/python - '8': /usr/libexec/platform-python - debian: - '9': /usr/bin/python - '10': /usr/bin/python3 - '11': /usr/bin/python - 'NA': /usr/bin/python # Debian 11, Ansible <= 7 (ansible-core <= 2.14) - 'bullseye/sid': /usr/bin/python # Debian 11, Ansible 8 - 9 (ansible-core 2.15 - 2.16) - ubuntu: - '16': /usr/bin/python3 - '18': /usr/bin/python3 - '20': /usr/bin/python3 - - DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_2_12_to_2_16: - centos: - '6': /usr/bin/python - '7': /usr/bin/python - '8': /usr/libexec/platform-python - debian: - '9': /usr/bin/python - '10': /usr/bin/python3 - '11': /usr/bin/python3.9 - 'NA': /usr/bin/python3.9 # Debian 11, Ansible <= 7 (ansible-core <= 2.14) - 'bullseye/sid': /usr/bin/python3.9 # Debian 11, Ansible 8 - 9 (ansible-core 2.15 - 2.16) - ubuntu: - '16': /usr/bin/python3 - '18': /usr/bin/python3 - '20': /usr/bin/python3 - - DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_ge_2_17: - debian: - '10': /usr/bin/python3.7 - '11': /usr/bin/python3.9 - 'bullseye/sid': /usr/bin/python3.9 - ubuntu: - '20': /usr/bin/python3.8 - - discovered_interpreter_expected: >- - {%- if ansible_version_major_minor is version('2.12', '<', strict=True) -%} - {{ DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_lt_2_12[distro][distro_major] }} - {%- elif ansible_version_major_minor is version('2.17', '<', strict=True) -%} - {{ DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_2_12_to_2_16[distro][distro_major] }} - {%- else -%} - {{ DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_ge_2_17[distro][distro_major] }} - {%- endif -%} tasks: - name: can only run these tests on ansible >= 2.8.0 block: @@ -65,12 +27,6 @@ fail_msg: "'ansible_python_interpreter' appears to be set at a high precedence to {{ ansible_python_interpreter }}, which breaks this test." - - name: snag some facts to validate for later - set_fact: - distro: '{{ ansible_facts.distribution | lower }}' - distro_major: '{{ ansible_facts.distribution_major_version }}' - system: '{{ ansible_facts.system }}' - - name: test that python discovery is working and that fact persistence makes it only run once block: - name: clear facts to force interpreter discovery to run @@ -215,16 +171,10 @@ - name: Check discovered interpreter matches expected assert: that: - - auto_out.ansible_facts.discovered_interpreter_python == discovered_interpreter_expected + - auto_out.ansible_facts.discovered_interpreter_python == linear_auto_result.ansible_facts.discovered_interpreter_python fail_msg: | - distro={{ distro }} - distro_major= {{ distro_major }} - system={{ system }} auto_out={{ auto_out }} - discovered_interpreter_expected={{ discovered_interpreter_expected }} - ansible_version.full={{ ansible_version.full }} - when: - - system in ['Linux'] + linear_auto_result={{ linear_auto_result }} always: - meta: clear_facts diff --git a/tests/ansible/integration/process/unix_socket_cleanup.yml b/tests/ansible/integration/process/unix_socket_cleanup.yml index 4466aa2e..4dce1d41 100644 --- a/tests/ansible/integration/process/unix_socket_cleanup.yml +++ b/tests/ansible/integration/process/unix_socket_cleanup.yml @@ -14,7 +14,7 @@ ANSIBLE_CALLBACK_RESULT_FORMAT=json ANSIBLE_LOAD_CALLBACK_PLUGINS=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o KexAlgorithms=+diffie-hellman-group1-sha1 -o PubkeyAcceptedKeyTypes=+ssh-rsa" ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" ansible -m shell -c local -a whoami {% for inv in ansible_inventory_sources %} diff --git a/tests/ansible/integration/ssh/password.yml b/tests/ansible/integration/ssh/password.yml index ca08fa5b..5301c084 100644 --- a/tests/ansible/integration/ssh/password.yml +++ b/tests/ansible/integration/ssh/password.yml @@ -35,7 +35,7 @@ ssh_no_password_result.msg is search('SSH password was requested, but none specified') or ssh_no_password_result.msg is search('SSH password is incorrect') or ssh_no_password_result.msg is search('Invalid/incorrect password') - or ssh_no_password_result.msg is search('Permission denied \(publickey,password(,keyboard-interactive)?\)') + or ssh_no_password_result.msg is search('Permission denied \(publickey(,gssapi-keyex)?(,gssapi-with-mic)?,password(,keyboard-interactive)?\)') fail_msg: | ssh_no_password_result={{ ssh_no_password_result }} @@ -72,6 +72,6 @@ - >- ssh_wrong_password_result.msg is search('SSH password is incorrect') or ssh_wrong_password_result.msg is search('Invalid/incorrect password') - or ssh_wrong_password_result.msg is search('Permission denied \(publickey,password(,keyboard-interactive)?\)') + or ssh_no_password_result.msg is search('Permission denied \(publickey(,gssapi-keyex)?(,gssapi-with-mic)?,password(,keyboard-interactive)?\)') fail_msg: | ssh_wrong_password_result={{ ssh_wrong_password_result }} diff --git a/tests/ansible/integration/ssh/variables.yml b/tests/ansible/integration/ssh/variables.yml index 5eb54dde..17d53350 100644 --- a/tests/ansible/integration/ssh/variables.yml +++ b/tests/ansible/integration/ssh/variables.yml @@ -22,7 +22,7 @@ ANSIBLE_CALLBACK_RESULT_FORMAT=json ANSIBLE_LOAD_CALLBACK_PLUGINS=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o KexAlgorithms=+diffie-hellman-group1-sha1 -o PubkeyAcceptedKeyTypes=+ssh-rsa" ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} @@ -42,7 +42,7 @@ ANSIBLE_CALLBACK_RESULT_FORMAT=json ANSIBLE_LOAD_CALLBACK_PLUGINS=false ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" + ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o KexAlgorithms=+diffie-hellman-group1-sha1 -o PubkeyAcceptedKeyTypes=+ssh-rsa" ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" ansible -m shell -a whoami {% for inv in ansible_inventory_sources %} diff --git a/tests/ansible/regression/issue_122__environment_difference.yml b/tests/ansible/regression/issue_122__environment_difference.yml index 273a49ae..510dbf86 100644 --- a/tests/ansible/regression/issue_122__environment_difference.yml +++ b/tests/ansible/regression/issue_122__environment_difference.yml @@ -8,9 +8,12 @@ - name: regression/issue_122__environment_difference.yml hosts: test-targets tasks: - - - script: scripts/print_env.py - register: env - - debug: msg={{env}} + - name: Run print_env.py + script: + cmd: scripts/print_env.py + executable: "{{ ansible_python_interpreter | default(ansible_facts.discovered_interpreter_python) }}" + register: print_env_result + - debug: + var: print_env_result tags: - issue_122 diff --git a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml index 43b00de5..c90f63d2 100644 --- a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml +++ b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml @@ -1,26 +1,29 @@ - name: regression/issue_152__virtualenv_python_fails.yml gather_facts: true hosts: test-targets + vars: + virtualenv_path: /tmp/issue_152_virtualenv + virtualenv_python: "{{ ansible_facts.python.executable }}" tasks: - custom_python_detect_environment: register: lout # Can't use pip module because it can't create virtualenvs, must call it # directly. - - name: Create /tmp/issue_152_virtualenv + - name: Create temporary virtualenv environment: https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}" no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}" PATH: "{{ lookup('env', 'PATH') }}" command: - cmd: virtualenv -p "{{ ansible_facts.python.executable }}" /tmp/issue_152_virtualenv - creates: /tmp/issue_152_virtualenv + argv: "{{ virtualenv_create_argv }}" + creates: "{{ virtualenv_path }}" when: - lout.python.version.full is version('2.7', '>=', strict=True) - custom_python_detect_environment: vars: - ansible_python_interpreter: /tmp/issue_152_virtualenv/bin/python + ansible_python_interpreter: "{{ virtualenv_path }}/bin/python" register: out when: - lout.python.version.full is version('2.7', '>=', strict=True) @@ -28,7 +31,7 @@ - name: Check virtualenv was used # On macOS runners a symlink /tmp -> /private/tmp has been seen vars: - requested_executable: /tmp/issue_152_virtualenv/bin/python + requested_executable: "{{ virtualenv_path }}/bin/python" expected_executables: - "{{ requested_executable }}" - "{{ requested_executable.replace('/tmp', out.fs['/tmp'].resolved) }}" @@ -40,9 +43,9 @@ when: - lout.python.version.full is version('2.7', '>=', strict=True) - - name: Cleanup /tmp/issue_152_virtualenv + - name: Cleanup temporary virtualenv file: - path: /tmp/issue_152_virtualenv + path: "{{ virtualenv_path }}" state: absent when: - lout.python.version.full is version('2.7', '>=', strict=True) diff --git a/tests/data/docker/doas-debian.tar.gz b/tests/data/docker/doas-debian.tar.gz deleted file mode 100644 index 2deb72ff..00000000 Binary files a/tests/data/docker/doas-debian.tar.gz and /dev/null differ diff --git a/tests/data/plain_old_module.py b/tests/data/plain_old_module.py index 7239f76a..2c14e0ff 100755 --- a/tests/data/plain_old_module.py +++ b/tests/data/plain_old_module.py @@ -12,8 +12,12 @@ class MyError(Exception): def get_sentinel_value(): # Some proof we're even talking to the mitogen-test Docker image - with open('/etc/sentinel', 'rb') as f: - return f.read().decode() + f = open('/etc/sentinel', 'rb') + try: + value = f.read().decode() + finally: + f.close() + return value def add(x, y): diff --git a/tests/image_prep/_container_create.yml b/tests/image_prep/_container_create.yml index b07c46eb..ab52b73b 100644 --- a/tests/image_prep/_container_create.yml +++ b/tests/image_prep/_container_create.yml @@ -3,18 +3,30 @@ strategy: mitogen_free gather_facts: false tasks: - - name: Fetch container images - docker_image: - name: "{{ docker_base }}" - delegate_to: localhost - - name: Start containers docker_container: name: "{{ inventory_hostname }}" image: "{{ docker_base }}" command: /bin/bash hostname: "mitogen-{{ inventory_hostname }}" + etc_hosts: + centos-vault-proxy: host-gateway detach: true interactive: true tty: true delegate_to: localhost + + - name: Wait for containers + # Can't use wait_for_connection yet, not all base images have a python + command: >- + docker inspect + --format "{% raw %}{{.State.Running}}{% endraw %}" + "{{ inventory_hostname }}" + register: container_inspect_result + retries: 5 + delay: 10 + until: + - container_inspect_result is succeeded + - container_inspect_result.stdout == "true" + changed_when: false + delegate_to: localhost diff --git a/tests/image_prep/_container_finalize.yml b/tests/image_prep/_container_finalize.yml index 5329fefa..7e090870 100644 --- a/tests/image_prep/_container_finalize.yml +++ b/tests/image_prep/_container_finalize.yml @@ -1,7 +1,7 @@ - name: Prepare images hosts: all strategy: mitogen_free - gather_facts: true + gather_facts: false tasks: - name: Commit containers command: > @@ -10,9 +10,11 @@ --change 'CMD ["/usr/sbin/sshd", "-D"]' {{ inventory_hostname }} {{ container_image_name }} + changed_when: true delegate_to: localhost - name: Stop containers command: > docker rm -f {{ inventory_hostname }} + changed_when: true delegate_to: localhost diff --git a/tests/image_prep/_container_host.yml b/tests/image_prep/_container_host.yml new file mode 100644 index 00000000..d3c2aca8 --- /dev/null +++ b/tests/image_prep/_container_host.yml @@ -0,0 +1,5 @@ +- name: Setup container host + hosts: localhost + become: true + roles: + - role: container_host diff --git a/tests/image_prep/_container_setup.yml b/tests/image_prep/_container_setup.yml index b95d67a9..276004f1 100644 --- a/tests/image_prep/_container_setup.yml +++ b/tests/image_prep/_container_setup.yml @@ -2,113 +2,44 @@ hosts: all strategy: linear gather_facts: false - tasks: - - name: Install bootstrap packages - raw: | - set -o errexit - set -o nounset - if type -p yum; then - yum -y install {{ bootstrap_packages | join(' ') }} - else - apt-get -y update - apt-get -y --no-install-recommends install {{ bootstrap_packages | join(' ') }} - fi - when: bootstrap_packages | length + roles: + - role: bootstrap - name: Setup containers hosts: all strategy: mitogen_free - # Resource limitation, my laptop freezes doing every container concurrently - serial: 4 # Can't gather facts before here. gather_facts: true vars: distro: "{{ansible_distribution}}" - pre_tasks: - - meta: end_play - when: - - ansible_facts.virtualization_type != "docker" - roles: - role: package_manager + - role: packages - role: sshd - role: sshd_container tasks: - - name: Ensure requisite apt packages are installed - apt: - name: "{{ common_packages + packages }}" - state: present - install_recommends: false - update_cache: true - when: ansible_pkg_mgr == 'apt' - - - name: Ensure requisite yum packages are installed - yum: - name: "{{ common_packages + packages }}" - state: present - update_cache: true - when: ansible_pkg_mgr == 'yum' - - - name: Ensure requisite dnf packages are installed - dnf: - name: "{{ common_packages + packages }}" - state: present - update_cache: true - when: ansible_pkg_mgr == 'dnf' - - - name: Clean up package cache - vars: - clean_command: - apt: apt-get clean - yum: yum clean all - dnf: dnf clean all - command: "{{ clean_command[ansible_pkg_mgr] }}" - args: - warn: "{{ False if ansible_version_major_minor is version('2.10', '<=', strict=True) else omit }}" - - - name: Clean up apt package lists - shell: rm -rf {{item}}/* - with_items: - - /var/cache/apt - - /var/lib/apt/lists - when: ansible_pkg_mgr == 'apt' - - - name: Configure /usr/bin/python - command: alternatives --set python /usr/bin/python3.8 - args: - creates: /usr/bin/python - when: inventory_hostname in ["centos8"] - - name: Enable UTF-8 locale on Debian copy: dest: /etc/locale.gen content: | en_US.UTF-8 UTF-8 fr_FR.UTF-8 UTF-8 + mode: u=rw,go=r when: ansible_pkg_mgr == 'apt' - name: Generate UTF-8 locale on Debian - shell: locale-gen + command: + cmd: locale-gen + changed_when: true when: ansible_pkg_mgr == 'apt' - name: Write Unicode into /etc/environment copy: dest: /etc/environment content: "UNICODE_SNOWMAN=\u2603\n" - - - name: Install prebuilt 'doas' binary - unarchive: - dest: / - src: ../data/docker/doas-debian.tar.gz - - - name: Make prebuilt 'doas' binary executable - file: - path: /usr/local/bin/doas - mode: 'u=rwxs,go=rx' - owner: root - group: root + mode: u=rw,go=r - name: Install doas.conf copy: @@ -116,6 +47,7 @@ content: | permit :mitogen__group permit :root + mode: u=rw,go= - name: Set root user password and shell user: @@ -127,6 +59,7 @@ file: path: /var/run/sshd state: directory + mode: u=rwx,go=rx - name: Generate SSH host key command: ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key @@ -142,6 +75,7 @@ dest: /etc/sentinel content: | i-am-mitogen-test-docker-image + mode: u=rw,go=r - name: Ensure /etc/sudoers.d exists file: diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index a1701e55..0f167e7a 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -181,6 +181,6 @@ {% endfor %} validate: '/usr/sbin/visudo -cf %s' when: - - ansible_virtualization_type != "docker" + - ansible_connection == "local" roles: - role: user_policies diff --git a/tests/image_prep/ansible.cfg b/tests/image_prep/ansible.cfg index f698ed89..cc63c3a3 100644 --- a/tests/image_prep/ansible.cfg +++ b/tests/image_prep/ansible.cfg @@ -1,13 +1,17 @@ [defaults] +any_errors_fatal = true # Ansible >= 6 (ansible-core >= 2.13) callback_result_format = yaml deprecation_warnings = false +duplicate_dict_key = error +inventory = hosts.ini strategy_plugins = ../../ansible_mitogen/plugins/strategy retry_files_enabled = false -display_args_to_stdout = True no_target_syslog = True host_key_checking = False [inventory] -unparsed_is_fatal = true +any_unparsed_is_failed = true +host_pattern_mismatch = error +unparsed_is_failed = true diff --git a/tests/image_prep/apache_proxy.conf b/tests/image_prep/apache_proxy.conf new file mode 100644 index 00000000..79022df7 --- /dev/null +++ b/tests/image_prep/apache_proxy.conf @@ -0,0 +1,33 @@ +DefaultRuntimeDir ${XDG_RUNTIME_DIR} +PidFile ${XDG_RUNTIME_DIR}/apache2.pid + +LoadModule alias_module /usr/lib/apache2/modules/mod_alias.so +LoadModule authz_core_module /usr/lib/apache2/modules/mod_authz_core.so +LoadModule mpm_event_module /usr/lib/apache2/modules/mod_mpm_event.so +LoadModule proxy_module /usr/lib/apache2/modules/mod_proxy.so +LoadModule proxy_http_module /usr/lib/apache2/modules/mod_proxy_http.so +LoadModule rewrite_module /usr/lib/apache2/modules/mod_rewrite.so +LoadModule ssl_module /usr/lib/apache2/modules/mod_ssl.so + +LogFormat "%v:%p %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" vhost_combined + +KeepAlive On +Listen 8090 + + + Require all denied + AllowOverride None + + + + ServerName centos-vault-proxy + SSLProxyEngine On + CustomLog logs/access.log vhost_combined + ProxyPass "/" "https://vault.centos.org/" + ProxyPassReverse "https://vault.centos.org/" "/" + RedirectMatch "^/(.*)" "http://centos-vault-proxy:8090/$1" + + +# /usr/sbin/apache2 -d . -f apache_proxy.conf -D FOREGROUND + +# vim: syntax=apache diff --git a/tests/image_prep/group_vars/all.yml b/tests/image_prep/group_vars/all.yml index 6545e432..fc9ffddf 100644 --- a/tests/image_prep/group_vars/all.yml +++ b/tests/image_prep/group_vars/all.yml @@ -1,16 +1,18 @@ ansible_version_major_minor: "{{ ansible_version.major }}.{{ ansible_version.minor }}" common_packages: + - acl - openssh-server - rsync - strace - sudo container_image_name: "{{ container_registry }}/{{ inventory_hostname }}-test" -container_registry: public.ecr.aws/n5z0e8q9 +container_registry: ghcr.io/mitogen-hq sudo_group: MacOSX: admin Debian: sudo Ubuntu: sudo CentOS: wheel + AlmaLinux: wheel diff --git a/tests/image_prep/host_vars/alma9.yml b/tests/image_prep/host_vars/alma9.yml new file mode 100644 index 00000000..430d6bf2 --- /dev/null +++ b/tests/image_prep/host_vars/alma9.yml @@ -0,0 +1,6 @@ +bootstrap_packages: [python3] +docker_base: almalinux:9 + +packages: + - perl-JSON + - procps-ng diff --git a/tests/image_prep/host_vars/centos5.yml b/tests/image_prep/host_vars/centos5.yml index 1828c29e..19397096 100644 --- a/tests/image_prep/host_vars/centos5.yml +++ b/tests/image_prep/host_vars/centos5.yml @@ -1,6 +1,36 @@ bootstrap_packages: [python-simplejson] -docker_base: astj/centos5-vault +docker_base: centos:5 packages: - perl +package_manager_repos: + - dest: /etc/yum.repos.d/CentOS-Base.repo + content: | + [base] + name=CentOS-$releasever - Base + baseurl=http://centos-vault-proxy:8090/5.11/os/$basearch/ + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-5 + + [updates] + name=CentOS-$releasever - Updates + baseurl=http://centos-vault-proxy:8090/5.11/updates/$basearch/ + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-5 + + [extras] + name=CentOS-$releasever - Extras + baseurl=http://centos-vault-proxy:8090/5.11/extras/$basearch/ + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-5 + + - dest: /etc/yum.repos.d/libselinux.repo + content: | + [libselinux] + name=CentOS-$releasever - libselinux + baseurl=http://centos-vault-proxy:8090/5.11/centosplus/$basearch/ + gpgcheck=1 + enabled=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-5 + includepkgs=libselinux* diff --git a/tests/image_prep/host_vars/centos6.yml b/tests/image_prep/host_vars/centos6.yml index aae7965f..2eb20f48 100644 --- a/tests/image_prep/host_vars/centos6.yml +++ b/tests/image_prep/host_vars/centos6.yml @@ -1,6 +1,27 @@ bootstrap_packages: [python] -docker_base: moreati/centos6-vault +docker_base: centos:6 packages: - perl-JSON + +package_manager_repos: + - dest: /etc/yum.repos.d/CentOS-Base.repo + content: | + [base] + name=CentOS-$releasever - Base + baseurl=http://vault.centos.org/6.10/os/$basearch/ + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6 + + [updates] + name=CentOS-$releasever - Updates + baseurl=http://vault.centos.org/6.10/updates/$basearch/ + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6 + + [extras] + name=CentOS-$releasever - Extras + baseurl=http://vault.centos.org/6.10/extras/$basearch/ + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6 diff --git a/tests/image_prep/host_vars/centos7.yml b/tests/image_prep/host_vars/centos7.yml index fec83471..513e4bb9 100644 --- a/tests/image_prep/host_vars/centos7.yml +++ b/tests/image_prep/host_vars/centos7.yml @@ -6,3 +6,24 @@ packages: - perl-JSON - python-virtualenv - python3 + +package_manager_repos: + - dest: /etc/yum.repos.d/CentOS-Base.repo + content: | + [base] + name=CentOS-$releasever - Base + baseurl=http://vault.centos.org/$contentdir/$releasever/os/$basearch/ + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + + [updates] + name=CentOS-$releasever - Updates + baseurl=http://vault.centos.org/$contentdir/$releasever/updates/$basearch/ + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + + [extras] + name=CentOS-$releasever - Extras + baseurl=http://vault.centos.org/$contentdir/$releasever/extras/$basearch/ + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 diff --git a/tests/image_prep/host_vars/centos8.yml b/tests/image_prep/host_vars/centos8.yml index 17eccd01..baccafea 100644 --- a/tests/image_prep/host_vars/centos8.yml +++ b/tests/image_prep/host_vars/centos8.yml @@ -6,5 +6,29 @@ packages: - perl-JSON - python2-virtualenv - python3-virtualenv - - python36 - - python38 + +package_manager_repos: + - dest: /etc/yum.repos.d/CentOS-Linux-AppStream.repo + content: | + [appstream] + name=CentOS Linux $releasever - AppStream + baseurl=http://vault.centos.org/$contentdir/$releasever/AppStream/$basearch/os/ + enabled=1 + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial + - dest: /etc/yum.repos.d/CentOS-Linux-BaseOS.repo + content: | + [baseos] + name=CentOS Linux $releasever - BaseOS + baseurl=http://vault.centos.org/$contentdir/$releasever/BaseOS/$basearch/os/ + enabled=1 + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial + - dest: /etc/yum.repos.d/CentOS-Linux-Extras.repo + content: | + [extras] + name=CentOS Linux $releasever - Extras + baseurl=http://vault.centos.org/$contentdir/$releasever/extras/$basearch/os/ + enabled=1 + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial diff --git a/tests/image_prep/host_vars/debian10.yml b/tests/image_prep/host_vars/debian10.yml index 1b03d6a2..3e3c000f 100644 --- a/tests/image_prep/host_vars/debian10.yml +++ b/tests/image_prep/host_vars/debian10.yml @@ -1,4 +1,4 @@ -bootstrap_packages: [python] +bootstrap_packages: [python, python-apt] docker_base: debian:10 @@ -9,3 +9,11 @@ packages: - python3 - python3-virtualenv - virtualenv + +package_manager_repos: + - dest: /etc/apt/sources.list + content: | + deb http://archive.debian.org/debian/ buster main non-free contrib + deb http://archive.debian.org/debian/ buster-updates main non-free contrib + deb http://archive.debian.org/debian/ buster-proposed-updates main non-free contrib + deb http://security.debian.org/ buster/updates main non-free contrib diff --git a/tests/image_prep/host_vars/debian11.yml b/tests/image_prep/host_vars/debian11.yml index 5ab2d761..55d34ae4 100644 --- a/tests/image_prep/host_vars/debian11.yml +++ b/tests/image_prep/host_vars/debian11.yml @@ -1,11 +1,18 @@ bootstrap_packages: [python3, python3-apt] -docker_base: debian:bullseye +docker_base: debian:11 packages: + - doas - libjson-perl - locales - python-is-python3 - python2 - python3-virtualenv - virtualenv + +package_manager_keys: + - src: debian-archive-bullseye-automatic.gpg # Debian 11 + dest: /etc/apt/trusted.gpg.d/ + - src: debian-archive-bookworm-automatic.gpg # Debian 12 + dest: /etc/apt/trusted.gpg.d/ diff --git a/tests/image_prep/host_vars/debian12.yml b/tests/image_prep/host_vars/debian12.yml new file mode 100644 index 00000000..8c4afc66 --- /dev/null +++ b/tests/image_prep/host_vars/debian12.yml @@ -0,0 +1,8 @@ +bootstrap_packages: [python3, python3-apt] +docker_base: debian:12 + +packages: + - libjson-perl + - locales + - opendoas + - virtualenv diff --git a/tests/image_prep/host_vars/debian9.yml b/tests/image_prep/host_vars/debian9.yml index cbd22e0f..6676d7ce 100644 --- a/tests/image_prep/host_vars/debian9.yml +++ b/tests/image_prep/host_vars/debian9.yml @@ -1,4 +1,4 @@ -bootstrap_packages: [python] +bootstrap_packages: [python, python-apt] docker_base: debian:9 @@ -9,3 +9,10 @@ packages: - python3 - python3-virtualenv - virtualenv + +package_manager_repos: + - dest: /etc/apt/sources.list + content: | + deb http://archive.debian.org/debian/ stretch main contrib non-free + deb http://archive.debian.org/debian/ stretch-proposed-updates main contrib non-free + deb http://archive.debian.org/debian-security stretch/updates main contrib non-free diff --git a/tests/image_prep/host_vars/ubuntu1604.yml b/tests/image_prep/host_vars/ubuntu1604.yml index 461e522d..6afe326f 100644 --- a/tests/image_prep/host_vars/ubuntu1604.yml +++ b/tests/image_prep/host_vars/ubuntu1604.yml @@ -1,4 +1,4 @@ -bootstrap_packages: [python] +bootstrap_packages: [python, python-apt] docker_base: ubuntu:16.04 diff --git a/tests/image_prep/host_vars/ubuntu1804.yml b/tests/image_prep/host_vars/ubuntu1804.yml index 4c913e2d..e174cbb8 100644 --- a/tests/image_prep/host_vars/ubuntu1804.yml +++ b/tests/image_prep/host_vars/ubuntu1804.yml @@ -1,4 +1,4 @@ -bootstrap_packages: [python] +bootstrap_packages: [python, python-apt] docker_base: ubuntu:18.04 diff --git a/tests/image_prep/host_vars/ubuntu2004.yml b/tests/image_prep/host_vars/ubuntu2004.yml index 4ee5b331..f8179f1d 100644 --- a/tests/image_prep/host_vars/ubuntu2004.yml +++ b/tests/image_prep/host_vars/ubuntu2004.yml @@ -1,4 +1,4 @@ -bootstrap_packages: [python3] +bootstrap_packages: [python3, python3-apt] docker_base: ubuntu:20.04 diff --git a/tests/image_prep/host_vars/ubuntu2204.yml b/tests/image_prep/host_vars/ubuntu2204.yml new file mode 100644 index 00000000..e8a21c65 --- /dev/null +++ b/tests/image_prep/host_vars/ubuntu2204.yml @@ -0,0 +1,10 @@ +bootstrap_packages: [python3, python3-apt] +docker_base: ubuntu:22.04 + +packages: + - doas + - libjson-perl + - locales + - python2 + - python3-virtualenv + - virtualenv diff --git a/tests/image_prep/host_vars/ubuntu2404.yml b/tests/image_prep/host_vars/ubuntu2404.yml new file mode 100644 index 00000000..76eefc5f --- /dev/null +++ b/tests/image_prep/host_vars/ubuntu2404.yml @@ -0,0 +1,9 @@ +bootstrap_packages: [python3, python3-apt] +docker_base: ubuntu:24.04 + +packages: + - libjson-perl + - locales + - opendoas + - python3-virtualenv + - virtualenv diff --git a/tests/image_prep/hosts.ini b/tests/image_prep/hosts.ini index 68f8be62..254643a7 100644 --- a/tests/image_prep/hosts.ini +++ b/tests/image_prep/hosts.ini @@ -1,4 +1,5 @@ [all:children] +alma centos debian ubuntu @@ -6,6 +7,9 @@ ubuntu [all:vars] ansible_connection = docker +[alma] +alma9 + [centos] centos5 centos6 @@ -16,8 +20,36 @@ centos8 debian9 debian10 debian11 +debian12 [ubuntu] ubuntu1604 ubuntu1804 ubuntu2004 +ubuntu2204 +ubuntu2404 + +[ansible_2_3] +# Python 2.4 on targets +centos5 + +[ansible_5] +# Python 2.6 on targets +centos6 + +[ansible_9] +# Python 2.7 and/or 3.6 on targets +centos7 +centos8 +debian9 +debian10 +ubuntu1804 + +[ansible_11] +# Python >= 3.8 on targets +alma9 +debian11 +debian12 +ubuntu2004 +ubuntu2204 +ubuntu2404 diff --git a/tests/image_prep/roles/bootstrap/defaults/main.yml b/tests/image_prep/roles/bootstrap/defaults/main.yml new file mode 100644 index 00000000..198a6135 --- /dev/null +++ b/tests/image_prep/roles/bootstrap/defaults/main.yml @@ -0,0 +1,2 @@ +bootstrap_packages: [] +package_manager_repos: [] diff --git a/tests/image_prep/roles/bootstrap/tasks/main.yml b/tests/image_prep/roles/bootstrap/tasks/main.yml new file mode 100644 index 00000000..5ecaa49d --- /dev/null +++ b/tests/image_prep/roles/bootstrap/tasks/main.yml @@ -0,0 +1,3 @@ +- name: Bootstrap + raw: "{{ lookup('template', 'bootstrap.sh.j2') }}" + changed_when: true diff --git a/tests/image_prep/roles/bootstrap/templates/bootstrap.sh.j2 b/tests/image_prep/roles/bootstrap/templates/bootstrap.sh.j2 new file mode 100644 index 00000000..0d25a95b --- /dev/null +++ b/tests/image_prep/roles/bootstrap/templates/bootstrap.sh.j2 @@ -0,0 +1,21 @@ +set -o errexit +set -o nounset + +{% for item in package_manager_repos %} +cat << "EOF" > "{{ item.dest }}" +{{ item.content }} +EOF +{% endfor %} + +{% if bootstrap_packages %} +if command -v apt-get; then + apt-get -y update + apt-get -y --no-install-recommends install {{ bootstrap_packages | join(' ') }} +elif command -v dnf; then + dnf -y install {{ bootstrap_packages | join(' ') }} +elif command -v yum; then + yum -y install {{ bootstrap_packages | join(' ') }} +else + exit 42 +fi +{% endif %} diff --git a/tests/image_prep/roles/container_host/handlers/main.yml b/tests/image_prep/roles/container_host/handlers/main.yml new file mode 100644 index 00000000..60c76cee --- /dev/null +++ b/tests/image_prep/roles/container_host/handlers/main.yml @@ -0,0 +1,6 @@ +- name: Update GRUB + command: update-grub + changed_when: true + +- name: Reboot + reboot: diff --git a/tests/image_prep/roles/container_host/tasks/main.yml b/tests/image_prep/roles/container_host/tasks/main.yml new file mode 100644 index 00000000..c7736790 --- /dev/null +++ b/tests/image_prep/roles/container_host/tasks/main.yml @@ -0,0 +1,27 @@ +# > If running `docker run --rm -it centos:centos6.7 bash` immediately exits +# > with status code 139, check to see if your system has disabled vsyscall: +# > ... +# > If you do not see a vsyscall mapping, and you need to run a CentOS 6 +# > container, try adding vsyscall=emulated to the kernel options. +# > -- https://hub.docker.com/_/centos + +- name: Check vsyscall enabled + command: + cmd: grep -c vsyscall /proc/self/maps + register: grep_self_maps_result + changed_when: false + check_mode: false + failed_when: + # 0 -> match, 1 -> no match, 2 -> error + - grep_self_maps_result.rc not in [0, 1] + +- name: Enable vsyscall + lineinfile: + path: /etc/default/grub + regexp: '^GRUB_CMDLINE_LINUX_DEFAULT.+' + line: GRUB_CMDLINE_LINUX_DEFAULT="quiet vsyscall=emulate" + when: + - grep_self_maps_result.rc != 0 + notify: + - Update GRUB + - Reboot diff --git a/tests/image_prep/roles/packages/defaults/main.yml b/tests/image_prep/roles/packages/defaults/main.yml new file mode 100644 index 00000000..c6ee8cc4 --- /dev/null +++ b/tests/image_prep/roles/packages/defaults/main.yml @@ -0,0 +1,14 @@ +common_packages: [] +packages: [] + +packages_clean_command: + apt: apt-get clean + dnf: dnf clean all + yum: yum clean all + +packages_cleanup_directories: + apt: + - /var/cache/apt + - /var/lib/apt/lists + dnf: [] + yum: [] diff --git a/tests/image_prep/roles/packages/tasks/main.yml b/tests/image_prep/roles/packages/tasks/main.yml new file mode 100644 index 00000000..30076690 --- /dev/null +++ b/tests/image_prep/roles/packages/tasks/main.yml @@ -0,0 +1,35 @@ +- name: Ensure requisite apt packages are installed + apt: + name: "{{ common_packages + packages }}" + state: present + install_recommends: false + update_cache: true + when: + - ansible_pkg_mgr == 'apt' + +- name: Ensure requisite yum packages are installed + yum: + name: "{{ common_packages + packages }}" + state: present + update_cache: true + when: + - ansible_pkg_mgr == 'yum' + +- name: Ensure requisite dnf packages are installed + dnf: + name: "{{ common_packages + packages }}" + state: present + update_cache: true + when: + - ansible_pkg_mgr == 'dnf' + +- name: Clean up package cache + command: + cmd: "{{ packages_clean_command[ansible_pkg_mgr] }}" + changed_when: true + +- name: Clean up package directories + shell: + rm -rf {{ item }}/* + with_items: "{{ packages_cleanup_directories }}" + changed_when: true diff --git a/tests/image_prep/roles/sshd_container/handlers/main.yml b/tests/image_prep/roles/sshd_container/handlers/main.yml index cc7b9166..79902ed6 100644 --- a/tests/image_prep/roles/sshd_container/handlers/main.yml +++ b/tests/image_prep/roles/sshd_container/handlers/main.yml @@ -1,2 +1,4 @@ - name: Restart sshd - meta: noop + command: "true" + changed_when: false + check_mode: false diff --git a/tests/image_prep/setup.yml b/tests/image_prep/setup.yml index b820e1bc..afe189a5 100755 --- a/tests/image_prep/setup.yml +++ b/tests/image_prep/setup.yml @@ -1,6 +1,17 @@ #!/usr/bin/env ansible-playbook -- include_playbook: _container_create.yml -- include_playbook: _container_setup.yml -- include_playbook: _user_accounts.yml -- include_playbook: _container_finalize.yml +- name: Get base images + hosts: all + # strategy: mitogen_free + gather_facts: false + tasks: + - name: Fetch container base images + docker_image: + name: "{{ docker_base }}" + source: pull # Added in Ansible 2.8, required circa 2.12 + delegate_to: localhost + +- import_playbook: _container_create.yml +- import_playbook: _container_setup.yml +- import_playbook: _user_accounts.yml +- import_playbook: _container_finalize.yml diff --git a/tests/image_prep/setup_ansible2.3.yml b/tests/image_prep/setup_ansible2.3.yml new file mode 100644 index 00000000..cc7025c8 --- /dev/null +++ b/tests/image_prep/setup_ansible2.3.yml @@ -0,0 +1,15 @@ +#!/usr/bin/env ansible-playbook + +- name: Get base images + hosts: all + gather_facts: false + tasks: + - name: Fetch container base images + docker_image: + name: "{{ docker_base }}" + delegate_to: localhost + +- include: _container_create.yml +- include: _container_setup.yml +- include: _user_accounts.yml +- include: _container_finalize.yml diff --git a/tests/image_prep/tox.ini b/tests/image_prep/tox.ini index d28d1512..2e32cdce 100644 --- a/tests/image_prep/tox.ini +++ b/tests/image_prep/tox.ini @@ -1,7 +1,9 @@ [tox] envlist = ansible2.3, - ansible2.10, + ansible5 + ansible9 + ansible11 skipsdist = true [testenv] @@ -13,17 +15,47 @@ basepython = python2 deps = ansible>=2.3,<2.4 docker-py>=1.7.0 - mitogen>=0.2.10rc1,<0.3 + mitogen~=0.2.0 install_command = python -m pip --no-python-version-warning install {opts} {packages} commands = - ./setup.yml -i hosts.ini -l 'localhost,centos5' {posargs} + ansible-playbook -l 'localhost,ansible_2_3' setup_ansible2.3.yml -[testenv:ansible2.10] +[testenv:ansible5] basepython = python3 deps = - ansible>=2.10,<2.11 + ansible~=5.0 docker>=1.8.0 - mitogen>=0.3.0rc1,<0.4 + mitogen~=0.3.0 + passlib +setenv = + ANSIBLE_PYTHON_INTERPRETER=auto_silent + ANSIBLE_STDOUT_CALLBACK=yaml +commands = + ansible-playbook -l 'localhost,ansible_5' setup.yml + +[testenv:ansible9] +basepython = python3 +deps = + ansible~=9.0 + docker>=1.8.0 + mitogen~=0.3.0 + passlib +setenv = + ANSIBLE_PYTHON_INTERPRETER=auto_silent + ANSIBLE_STDOUT_CALLBACK=yaml +commands = + ansible-playbook -l 'localhost,ansible_9' setup.yml + +[testenv:ansible11] +basepython = python3 +deps = + ansible~=11.0 + docker>=1.8.0 + mitogen~=0.3.0 + passlib +setenv = + ANSIBLE_PYTHON_INTERPRETER=auto_silent + ANSIBLE_STDOUT_CALLBACK=yaml commands = - ./setup.yml -i hosts.ini -l '!centos5' {posargs} + ansible-playbook -l 'localhost,ansible_11' setup.yml diff --git a/tests/testlib.py b/tests/testlib.py index 3b5dc8b6..15016964 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -53,11 +53,11 @@ LOG = logging.getLogger(__name__) DISTRO_SPECS = os.environ.get( 'MITOGEN_TEST_DISTRO_SPECS', - 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004', + 'alma9-py3 centos5 centos8-py3 debian9 debian12-py3 ubuntu1604 ubuntu2404-py3', ) IMAGE_TEMPLATE = os.environ.get( 'MITOGEN_TEST_IMAGE_TEMPLATE', - 'ghcr.io/mitogen-hq/%(distro)s-test:2021', + 'ghcr.io/mitogen-hq/%(distro)s-test:2025.02', ) TESTS_DIR = os.path.join(os.path.dirname(__file__)) @@ -725,6 +725,7 @@ class DockerMixin(RouterMixin): # - tests/testlib.py 'ssh_args': [ '-o', 'HostKeyAlgorithms +ssh-rsa', + '-o', 'KexAlgorithms +diffie-hellman-group1-sha1', '-o', 'PubkeyAcceptedKeyTypes +ssh-rsa', ], 'python_path': self.dockerized_ssh.python_path, diff --git a/tox.ini b/tox.ini index 08b2eb98..28c97496 100644 --- a/tox.ini +++ b/tox.ini @@ -6,8 +6,8 @@ # Py A cntrllr A target coverage Django Jinja2 pip psutil pytest tox virtualenv # ==== ========== ========== ========== ========== ========== ========== ========== ========== ========== ========== -# 2.4 2.3? <= 3.7.1 <= 1.3.7 <= 1.1 <= 2.1.3 <= 1.4 <= 1.8 -# 2.5 <= 3.7.1 <= 1.4.22 <= 1.3.1 <= 2.1.3 <= 2.8.7 <= 1.6.1 <= 1.9.1 +# 2.4 <= 2.3³ <= 3.7.1 <= 1.3.7 <= 1.1 <= 2.1.3 <= 1.4 <= 1.8 +# 2.5 <= 2.3³ <= 3.7.1 <= 1.4.22 <= 1.3.1 <= 2.1.3 <= 2.8.7 <= 1.6.1 <= 1.9.1 # 2.6 <= 2.6.20 <= 2.12 <= 4.5.4 <= 1.6.11 <= 2.10.3 <= 9.0.3 <= 5.9.0 <= 3.2.5 <= 2.9.1 <= 15.2.0 # 2.7 <= 2.11 <= 2.16 <= 5.5 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.15² # 3.5 <= 2.11 <= 2.15 <= 5.5 <= 2.2.28 <= 2.11.3 <= 20 <= 5.9.5 <= 6.1.0 <= 3.28 <= 20.15² @@ -35,6 +35,8 @@ # Virtualenv <= 20.21.1 supports creating virtualenvs with any *target* Python. # Virtualenv >= 20.22 supports creating virtualenvs with target Python >= 3.7. # https://virtualenv.pypa.io/en/latest/#compatibility +# +# 3. https://docs.ansible.com/ansible/2.7/dev_guide/developing_python_3.html#minimum-version-of-python-3-x-and-python-2-x # Ansible Dependency # ================== ====================== @@ -121,30 +123,18 @@ setenv = ans{2.10,3,4,5}: ANSIBLE_STDOUT_CALLBACK=yaml # Print warning on the first occurence at each module:linenno in Mitogen. Available Python 2.7, 3.2+. PYTHONWARNINGS=default:::ansible_mitogen,default:::mitogen + ans{2.10,3,4,5}: MITOGEN_TEST_DISTRO_SPECS=alma9-py3 centos6 centos8-py3 debian9 debian12-py3 ubuntu1604 ubuntu2204-py3 # Ansible 6 - 8 (ansible-core 2.13 - 2.15) require Python 2.7 or >= 3.5 on targets - ans{6,7,8}: MITOGEN_TEST_DISTRO_SPECS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 + ans{6,7,8}: MITOGEN_TEST_DISTRO_SPECS=alma9-py3 centos7 centos8-py3 debian9 debian10 debian12-py3 ubuntu1604 ubuntu1804 ubuntu2404-py3 # Ansible 9 (ansible-core 2.16) requires Python 2.7 or >= 3.6 on targets - ans9: MITOGEN_TEST_DISTRO_SPECS=centos7 centos8 debian9 debian10 debian11 ubuntu1804 ubuntu2004 + ans9: MITOGEN_TEST_DISTRO_SPECS=alma9-py3 centos7 centos8-py3 debian9 debian10 debian12-py3 ubuntu1804 ubuntu2404-py3 # Ansible 10 (ansible-core 2.17) requires Python >= 3.7 on targets - ans10: MITOGEN_TEST_DISTRO_SPECS=debian10-py3 debian11-py3 ubuntu2004-py3 + ans10: MITOGEN_TEST_DISTRO_SPECS=alma9-py3 debian10-py3 debian12-py3 ubuntu2404-py3 # Ansible 11 (ansible-core 2.18) requires Python >= 3.8 on targets - ans11: MITOGEN_TEST_DISTRO_SPECS=debian11-py3 ubuntu2004-py3 - ans12: MITOGEN_TEST_DISTRO_SPECS=debian11-py3 ubuntu2004-py3 + ans11: MITOGEN_TEST_DISTRO_SPECS=alma9-py3 debian12-py3 ubuntu2004-py3 + ans12: MITOGEN_TEST_DISTRO_SPECS=alma9-py3 debian12-py3 ubuntu2404-py3 # Ansible 13 (ansible-core 2.20) requires Python >= 3.9 on targets - ans13: MITOGEN_TEST_DISTRO_SPECS=debian11-py3 - distros_centos: MITOGEN_TEST_DISTRO_SPECS=centos6 centos7 centos8 - distros_centos5: MITOGEN_TEST_DISTRO_SPECS=centos5 - distros_centos6: MITOGEN_TEST_DISTRO_SPECS=centos6 - distros_centos7: MITOGEN_TEST_DISTRO_SPECS=centos7 - distros_centos8: MITOGEN_TEST_DISTRO_SPECS=centos8 - distros_debian: MITOGEN_TEST_DISTRO_SPECS=debian9 debian10 debian11 - distros_debian9: MITOGEN_TEST_DISTRO_SPECS=debian9 - distros_debian10: MITOGEN_TEST_DISTRO_SPECS=debian10 - distros_debian11: MITOGEN_TEST_DISTRO_SPECS=debian11 - distros_ubuntu: MITOGEN_TEST_DISTRO_SPECS=ubuntu1604 ubuntu1804 ubuntu2004 - distros_ubuntu1604: MITOGEN_TEST_DISTRO_SPECS=ubuntu1604 - distros_ubuntu1804: MITOGEN_TEST_DISTRO_SPECS=ubuntu1804 - distros_ubuntu2004: MITOGEN_TEST_DISTRO_SPECS=ubuntu2004 + ans13: MITOGEN_TEST_DISTRO_SPECS=alma9-py3 debian12-py3 ubuntu2404-py3 m_ans: MODE=ansible m_ans: ANSIBLE_SKIP_TAGS=resource_intensive m_ans: ANSIBLE_CALLBACK_WHITELIST=profile_tasks