From f5a88406685923158cee238ca9a6217fb6b4b630 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 26 Jul 2024 14:58:21 +0100 Subject: [PATCH 01/46] CI: Use archived RPMs on CentOS 8 CentOS 8 has reached EOL. Packages are no longer mirrored or maintained. A historic snapshot of the packages is kept on vault.centos.org. refs #1088, #1090 --- tests/ansible/hosts/group_vars/all.yml | 1 + tests/ansible/hosts/group_vars/centos8.yml | 26 +++++++++++++++++++ tests/ansible/hosts/group_vars/debian9.yml | 4 +++ .../issue_776__load_plugins_called_twice.yml | 19 +++++--------- 4 files changed, 37 insertions(+), 13 deletions(-) create mode 100644 tests/ansible/hosts/group_vars/debian9.yml diff --git a/tests/ansible/hosts/group_vars/all.yml b/tests/ansible/hosts/group_vars/all.yml index cea46113..ad4adabb 100644 --- a/tests/ansible/hosts/group_vars/all.yml +++ b/tests/ansible/hosts/group_vars/all.yml @@ -1,2 +1,3 @@ --- pkg_mgr_python_interpreter: python +pkg_repos_overrides: [] diff --git a/tests/ansible/hosts/group_vars/centos8.yml b/tests/ansible/hosts/group_vars/centos8.yml index 7b9e34f6..c90dd5f4 100644 --- a/tests/ansible/hosts/group_vars/centos8.yml +++ b/tests/ansible/hosts/group_vars/centos8.yml @@ -1,2 +1,28 @@ --- pkg_mgr_python_interpreter: /usr/libexec/platform-python + +pkg_repos_overrides: + - dest: /etc/yum.repos.d/CentOS-Linux-AppStream.repo + content: | + [appstream] + name=CentOS Linux $releasever - AppStream + baseurl=http://vault.centos.org/$contentdir/$releasever/AppStream/$basearch/os/ + enabled=1 + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial + - dest: /etc/yum.repos.d/CentOS-Linux-BaseOS.repo + content: | + [baseos] + name=CentOS Linux $releasever - BaseOS + baseurl=http://vault.centos.org/$contentdir/$releasever/BaseOS/$basearch/os/ + enabled=1 + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial + - dest: /etc/yum.repos.d/CentOS-Linux-Extras.repo + content: | + [extras] + name=CentOS Linux $releasever - Extras + baseurl=http://vault.centos.org/$contentdir/$releasever/extras/$basearch/os/ + enabled=1 + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial diff --git a/tests/ansible/hosts/group_vars/debian9.yml b/tests/ansible/hosts/group_vars/debian9.yml new file mode 100644 index 00000000..4b180b13 --- /dev/null +++ b/tests/ansible/hosts/group_vars/debian9.yml @@ -0,0 +1,4 @@ +pkg_repos_overrides: + - dest: /etc/apt/sources.list + content: | + deb http://archive.debian.org/debian stretch main contrib non-free diff --git a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml index ad5bbd69..44ff6863 100755 --- a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml +++ b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml @@ -10,21 +10,14 @@ ansible_python_interpreter: "{{ pkg_mgr_python_interpreter }}" package: rsync # Chosen to exist in all tested distros/package managers tasks: - - name: Switch to centos-stream - command: dnf --assumeyes --disablerepo="*" --enablerepo=extras swap centos-linux-repos centos-stream-repos - when: - - ansible_facts.pkg_mgr in ["dnf"] - - - name: Switch to archive.debian.org - # Debian 9 has been archived https://lists.debian.org/debian-devel-announce/2023/03/msg00006.html + - name: Switch to archived package repositories copy: - content: | - deb http://archive.debian.org/debian stretch main contrib non-free - dest: /etc/apt/sources.list + dest: "{{ item.dest }}" + content: "{{ item.content }}" mode: u=rw,go=r - when: - - ansible_facts.distribution == "Debian" - - ansible_facts.distribution_major_version == "9" + loop: "{{ pkg_repos_overrides }}" + loop_control: + label: "{{ item.dest }}" - name: Add signing keys copy: From 924dbd6f0c31f1169adc8d969115dbcdfecb3cc9 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 29 Jul 2024 11:25:12 +0100 Subject: [PATCH 02/46] CI: Migrate macOS integration tests to macOS 12, drop Python 2.7 jobs macOS 11 is not longer an available runner on Azure Devops. The minimum is now macOS 12. This runner does not have Python 2.7 installed, so running them would require a custom install - which I'm declaring too much effort for too little gain. refs #1090 --- .ci/azure-pipelines.yml | 17 +++-------------- docs/changelog.rst | 1 + .../integration/become/sudo_nonexistent.yml | 3 +-- .../modules/custom_python_json_args_module.py | 2 +- .../lib/modules/custom_python_os_getcwd.py | 2 +- .../modules/custom_python_want_json_module.py | 2 +- .../issue_591__setuptools_cwd_crash.yml | 3 --- .../issue_655__wait_for_connection_error.yml | 7 +++---- tests/two_three_compat_test.py | 7 +++++++ 9 files changed, 18 insertions(+), 26 deletions(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 94919e35..51908e92 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -16,32 +16,21 @@ trigger: - docs-master jobs: -- job: mac11 +- job: mac12 # vanilla Ansible is really slow timeoutInMinutes: 120 steps: - template: azure-pipelines-steps.yml pool: - # https://github.com/actions/runner-images/blob/main/images/macos/macos-11-Readme.md - vmImage: macOS-11 + # https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md + vmImage: macOS-12 strategy: matrix: - Mito_27: - tox.env: py27-mode_mitogen Mito_312: - python.version: '3.12' tox.env: py312-mode_mitogen - - Loc_27_210: - tox.env: py27-mode_localhost-ansible2.10 Loc_312_9: - python.version: '3.12' tox.env: py312-mode_localhost-ansible9 - - Van_27_210: - tox.env: py27-mode_localhost-ansible2.10-strategy_linear Van_312_9: - python.version: '3.12' tox.env: py312-mode_localhost-ansible9-strategy_linear - job: Linux diff --git a/docs/changelog.rst b/docs/changelog.rst index 5d77910e..ffba69fa 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,6 +25,7 @@ Unreleased * :gh:issue:`957` Fix Ansible exception when executing against 10s of hosts "ValueError: filedescriptor out of range in select()" * :gh:issue:`1066` Support Ansible `ansible_host_key_checking` & `ansible_ssh_host_key_checking` +* :gh:issue:`1090` CI: Migrate macOS integration tests to macOS 12, drop Python 2.7 jobs v0.3.7 (2024-04-08) diff --git a/tests/ansible/integration/become/sudo_nonexistent.yml b/tests/ansible/integration/become/sudo_nonexistent.yml index e7a849c2..912d1ba8 100644 --- a/tests/ansible/integration/become/sudo_nonexistent.yml +++ b/tests/ansible/integration/become/sudo_nonexistent.yml @@ -21,8 +21,7 @@ # sudo-1.8.6p3-29.el6_10.3 on RHEL & CentOS 6.10 (final release) # removed user/group error messages, as defence against CVE-2019-14287. - >- - 'sudo: unknown user: slartibartfast' in out.module_stdout | default(out.msg) - or 'sudo: unknown user: slartibartfast' in out.module_stderr | default(out.msg) + (out.module_stderr | default(out.module_stdout, true) | default(out.msg, true)) is search('sudo: unknown user:? slartibartfast') or (ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_version == '6.10') fail_msg: out={{out}} when: diff --git a/tests/ansible/lib/modules/custom_python_json_args_module.py b/tests/ansible/lib/modules/custom_python_json_args_module.py index 846037ec..61640579 100755 --- a/tests/ansible/lib/modules/custom_python_json_args_module.py +++ b/tests/ansible/lib/modules/custom_python_json_args_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible Python JSONARGS module. I should receive an encoding string. json_arguments = """<>""" diff --git a/tests/ansible/lib/modules/custom_python_os_getcwd.py b/tests/ansible/lib/modules/custom_python_os_getcwd.py index d465ac9e..c5e264ae 100644 --- a/tests/ansible/lib/modules/custom_python_os_getcwd.py +++ b/tests/ansible/lib/modules/custom_python_os_getcwd.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # #591: call os.getcwd() before AnsibleModule ever gets a chance to fix up the # process environment. diff --git a/tests/ansible/lib/modules/custom_python_want_json_module.py b/tests/ansible/lib/modules/custom_python_want_json_module.py index 23eeeb55..f5e33862 100755 --- a/tests/ansible/lib/modules/custom_python_want_json_module.py +++ b/tests/ansible/lib/modules/custom_python_want_json_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible Python WANT_JSON module. I should receive a JSON-encoded file. import json diff --git a/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml b/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml index d5f0815b..ff102b13 100644 --- a/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml +++ b/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml @@ -19,9 +19,6 @@ # Will crash if process has a nonexistent CWD. - custom_python_os_getcwd: - script: | - import os - self._connection.get_chain().call(os.getcwd) tags: - issue_591 - mitogen_only diff --git a/tests/ansible/regression/issue_655__wait_for_connection_error.yml b/tests/ansible/regression/issue_655__wait_for_connection_error.yml index 9ad42a10..4972d91a 100644 --- a/tests/ansible/regression/issue_655__wait_for_connection_error.yml +++ b/tests/ansible/regression/issue_655__wait_for_connection_error.yml @@ -11,11 +11,10 @@ tasks: - meta: end_play when: - # TODO CI currently runs on macOS 11 images in Azure DevOps. MacOS 11 - # is no longer supported by homebrew, so the following install - # task fails. + # TODO CI currently runs on macOS 12 & which isn't supported by Podman + # version available in Homebrew. - ansible_facts.system == 'Darwin' - - ansible_facts.distribution_major_version == '11' + - ansible_facts.distribution_version is version('13.0', '<', strict=True) - name: set up test container and run tests inside it block: diff --git a/tests/two_three_compat_test.py b/tests/two_three_compat_test.py index 4490e5d2..ab9f4e19 100644 --- a/tests/two_three_compat_test.py +++ b/tests/two_three_compat_test.py @@ -1,3 +1,6 @@ +import os +import unittest + import mitogen.core import testlib @@ -7,6 +10,10 @@ import simple_pkg.ping # TODO: this is a joke. 2/3 interop is one of the hardest bits to get right. # There should be 100 tests in this file. +@unittest.skipIf( + os.uname()[0] == 'Darwin' and int(os.uname()[2].partition('.')[0]) >= 21, + "Python 2.x not shipped on macOS 12.3+ (Darwin 21.4+, Monterey)", +) class TwoThreeCompatTest(testlib.RouterMixin, testlib.TestCase): if mitogen.core.PY3: python_path = 'python2' From fe435bb7d069764b35a8ca77b05cffe19dbd9c1e Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 29 Jul 2024 11:30:25 +0100 Subject: [PATCH 03/46] CI: Workaround "No module named 'setuptools.command.test'" Pip 72 was released yesterday (2024-07-28), dropping `setup.py test` support. hdrhistogram 0.6.1 requires it to install. For now constrain Pip to earlier releases, so our tests can be run. refs #1090 --- tests/ansible/requirements.txt | 3 +++ tests/constraints.txt | 3 +++ tox.ini | 1 + 3 files changed, 7 insertions(+) create mode 100644 tests/constraints.txt diff --git a/tests/ansible/requirements.txt b/tests/ansible/requirements.txt index 2c3c87c8..8cfb348a 100644 --- a/tests/ansible/requirements.txt +++ b/tests/ansible/requirements.txt @@ -1,4 +1,7 @@ paramiko==2.3.2 # Last 2.6-compat version. +# Incompatible with pip >= 72, due to removal of `setup.py test`: +# ModuleNotFoundError: No module named 'setuptools.command.test' +# https://github.com/pypa/setuptools/issues/4519 hdrhistogram==0.6.1 PyYAML==3.11; python_version < '2.7' PyYAML==5.3.1; python_version >= '2.7' # Latest release (Jan 2021) diff --git a/tests/constraints.txt b/tests/constraints.txt new file mode 100644 index 00000000..6adaa30b --- /dev/null +++ b/tests/constraints.txt @@ -0,0 +1,3 @@ +# Setuptools 72 removed `setup.py test`. hdrhistogram 0.6.1 still depends on it. +# TODO Bump dependencies and unconstrain Pip. +setuptools<72 diff --git a/tox.ini b/tox.ini index 55fd1842..a0b0e1ae 100644 --- a/tox.ini +++ b/tox.ini @@ -109,6 +109,7 @@ setenv = ANSIBLE_STRATEGY = mitogen_linear NOCOVERAGE_ERASE = 1 NOCOVERAGE_REPORT = 1 + PIP_CONSTRAINT={toxinidir}/tests/constraints.txt # Only applicable to MODE=mitogen distro_centos5: DISTRO=centos5 distro_centos6: DISTRO=centos6 From e334b50d9d5bc5174d1e07c83c6617db6406193e Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 30 Jul 2024 09:06:57 +0100 Subject: [PATCH 04/46] Prepare v0.3.8 --- docs/changelog.rst | 4 ++-- docs/conf.py | 2 +- mitogen/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ffba69fa..2273043a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,8 +18,8 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. -Unreleased ----------- +v0.3.8 (2024-07-30) +------------------- * :gh:issue:`952` Fix Ansible `--ask-become-pass`, add test coverage * :gh:issue:`957` Fix Ansible exception when executing against 10s of hosts diff --git a/docs/conf.py b/docs/conf.py index ad9771b4..fe1a5044 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -2,7 +2,7 @@ import sys sys.path.append('.') -VERSION = '0.3.7' +VERSION = '0.3.8' author = u'Network Genomics' copyright = u'2021, the Mitogen authors' diff --git a/mitogen/__init__.py b/mitogen/__init__.py index b63cd0cd..8f2741c7 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 8, 'dev') +__version__ = (0, 3, 8) #: This is :data:`False` in slave contexts. Previously it was used to prevent From 62cde1715090c3db92fe8e6ee6732d4b71d869fb Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 30 Jul 2024 09:09:32 +0100 Subject: [PATCH 05/46] Start v0.3.9 development --- docs/changelog.rst | 5 +++++ mitogen/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2273043a..ec122abf 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,11 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. +Unreleased +---------- + + + v0.3.8 (2024-07-30) ------------------- diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 8f2741c7..ce6c2de0 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 8) +__version__ = (0, 3, 9, 'dev0') #: This is :data:`False` in slave contexts. Previously it was used to prevent From 8613f685abdfe5cbe192221eb14ec26a33d3683b Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sat, 10 Aug 2024 19:37:25 +0100 Subject: [PATCH 06/46] tests: Skip AWS ECR login outside of CI jobs To avoid rate limiting errors, CI (currently Azure Devops) logs into the container registry (currently AWS ECR). Outside CI this is unnnecessary and makes it harder to run the tests, because very few people have access to a suitable AWS secret token. Following this change `aws ecr-public get-login-password` will only be run if the environment variable $TF_BUILD==True. This is set by Azure Pipelines jobs. If the CI platform is changed then another indicator should be used. https://adamj.eu/tech/2020/03/09/detect-if-your-tests-are-running-on-ci/ --- .ci/ansible_install.py | 2 +- .ci/debops_common_install.py | 2 +- .ci/mitogen_install.py | 2 +- .ci/mitogen_py24_install.py | 2 +- tox.ini | 3 +++ 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.ci/ansible_install.py b/.ci/ansible_install.py index 900303f6..3b217ff2 100755 --- a/.ci/ansible_install.py +++ b/.ci/ansible_install.py @@ -4,7 +4,7 @@ import ci_lib batches = [ [ - 'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws', + 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', ] ] diff --git a/.ci/debops_common_install.py b/.ci/debops_common_install.py index afafe39a..565f9488 100755 --- a/.ci/debops_common_install.py +++ b/.ci/debops_common_install.py @@ -10,7 +10,7 @@ ci_lib.run_batches([ 'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"', ], [ - 'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws', + 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', ], ]) diff --git a/.ci/mitogen_install.py b/.ci/mitogen_install.py index b0b1eb14..23ff384b 100755 --- a/.ci/mitogen_install.py +++ b/.ci/mitogen_install.py @@ -7,7 +7,7 @@ batches = [ if ci_lib.have_docker(): batches.append([ - 'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws', + 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', ]) diff --git a/.ci/mitogen_py24_install.py b/.ci/mitogen_py24_install.py index bd6ecb24..8af90405 100755 --- a/.ci/mitogen_py24_install.py +++ b/.ci/mitogen_py24_install.py @@ -4,7 +4,7 @@ import ci_lib batches = [ [ - 'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws', + 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', ], [ 'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv', diff --git a/tox.ini b/tox.ini index a0b0e1ae..5dcb1bf2 100644 --- a/tox.ini +++ b/tox.ini @@ -104,6 +104,9 @@ passenv = AWS_DEFAULT_REGION AWS_SECRET_ACCESS_KEY HOME + # Azure DevOps, TF_BUILD is set to 'True' when running in a build task + # https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables + TF_BUILD setenv = # See also azure-pipelines.yml ANSIBLE_STRATEGY = mitogen_linear From 9185805bf2b6467cb22622e4428531faa1fae1fa Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sat, 10 Aug 2024 21:38:51 +0100 Subject: [PATCH 07/46] ansible_mitogen: cast ansible_python_interpreter value This was the last remaining use of `mitogen.utils.cast()`. I missed it in #1046. --- ansible_mitogen/planner.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 0b1b7aab..b557272f 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -54,6 +54,7 @@ import mitogen.select import ansible_mitogen.loaders import ansible_mitogen.parsing import ansible_mitogen.target +import ansible_mitogen.utils.unsafe LOG = logging.getLogger(__name__) @@ -220,7 +221,7 @@ class ScriptPlanner(BinaryPlanner): variable, render it and return it. :param str path: - Absolute UNIX path to original interpreter. + Absolute path to original interpreter (e.g. '/usr/bin/python'). :returns: Shell fragment prefix used to execute the script via "/bin/sh -c". @@ -232,9 +233,12 @@ class ScriptPlanner(BinaryPlanner): try: template = self._inv.task_vars[key] except KeyError: - return path + pass + else: + configured_interpreter = self._inv.templar.template(template) + return ansible_mitogen.utils.unsafe.cast(configured_interpreter) - return mitogen.utils.cast(self._inv.templar.template(template)) + return path def _get_interpreter(self): path, arg = ansible_mitogen.parsing.parse_hashbang( From 40695f413b0a278b7d689462cf6e9ac8a218d243 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sat, 10 Aug 2024 22:05:22 +0100 Subject: [PATCH 08/46] ansible_mitogen: Respect ansible_facts.discovered_interpreter_python more fixes #1097 --- ansible_mitogen/planner.py | 23 +++++++++++++++---- docs/changelog.rst | 2 ++ .../module_utils/adjacent_to_role.yml | 2 +- .../roles/modrole/library/uses_external3.py | 2 +- .../library/uses_custom_known_hosts.py | 2 +- .../modules/custom_python_json_args_module.py | 2 +- .../lib/modules/custom_python_os_getcwd.py | 2 +- .../modules/custom_python_want_json_module.py | 2 +- 8 files changed, 26 insertions(+), 11 deletions(-) diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index b557272f..0a91039a 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -216,9 +216,12 @@ class ScriptPlanner(BinaryPlanner): """ def _rewrite_interpreter(self, path): """ - Given the original interpreter binary extracted from the script's - interpreter line, look up the associated `ansible_*_interpreter` - variable, render it and return it. + Given the interpreter path (from the script's hashbang line), return + the desired interpreter path. This tries, in order + + 1. Look up & render the `ansible_*_interpreter` variable, if set + 2. Look up the `discovered_interpreter_*` fact, if present + 3. The unmodified path from the hashbang line. :param str path: Absolute path to original interpreter (e.g. '/usr/bin/python'). @@ -229,7 +232,8 @@ class ScriptPlanner(BinaryPlanner): involved here, the vanilla implementation uses it and that use is exploited in common playbooks. """ - key = u'ansible_%s_interpreter' % os.path.basename(path).strip() + interpreter_name = os.path.basename(path).strip() + key = u'ansible_%s_interpreter' % interpreter_name try: template = self._inv.task_vars[key] except KeyError: @@ -238,6 +242,14 @@ class ScriptPlanner(BinaryPlanner): configured_interpreter = self._inv.templar.template(template) return ansible_mitogen.utils.unsafe.cast(configured_interpreter) + key = u'discovered_interpreter_%s' % interpreter_name + try: + discovered_interpreter = self._inv.task_vars['ansible_facts'][key] + except KeyError: + pass + else: + return ansible_mitogen.utils.unsafe.cast(discovered_interpreter) + return path def _get_interpreter(self): @@ -253,7 +265,8 @@ class ScriptPlanner(BinaryPlanner): if arg: fragment += ' ' + arg - return fragment, path.startswith('python') + is_python = path.startswith('python') + return fragment, is_python def get_kwargs(self, **kwargs): interpreter_fragment, is_python = self._get_interpreter() diff --git a/docs/changelog.rst b/docs/changelog.rst index ec122abf..c1754b9e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,8 @@ To avail of fixes in an unreleased version, please download a ZIP file Unreleased ---------- +* :gh:issue:`1097` Respect `ansible_facts.discovered_interpreter_python` when + executing non new-style modules (e.g. JSONARGS style, WANT_JSON style). v0.3.8 (2024-07-30) diff --git a/tests/ansible/integration/module_utils/adjacent_to_role.yml b/tests/ansible/integration/module_utils/adjacent_to_role.yml index c4fb813f..93499790 100644 --- a/tests/ansible/integration/module_utils/adjacent_to_role.yml +++ b/tests/ansible/integration/module_utils/adjacent_to_role.yml @@ -1,7 +1,7 @@ # external2 is loaded from config path. # external1 is loaded from integration/module_utils/roles/modrole/module_utils/.. -- name: integration/module_utils/adjacent_to_playbook.yml +- name: integration/module_utils/adjacent_to_role.yml hosts: test-targets roles: - modrole diff --git a/tests/ansible/integration/module_utils/roles/modrole/library/uses_external3.py b/tests/ansible/integration/module_utils/roles/modrole/library/uses_external3.py index faf8a9a5..78f2d71d 100644 --- a/tests/ansible/integration/module_utils/roles/modrole/library/uses_external3.py +++ b/tests/ansible/integration/module_utils/roles/modrole/library/uses_external3.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python from ansible.module_utils.basic import AnsibleModule from ansible.module_utils import external3 diff --git a/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py b/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py index 4fc4e04c..688fc45a 100644 --- a/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py +++ b/tests/ansible/integration/module_utils/roles/overrides_modrole/library/uses_custom_known_hosts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python import json import ansible.module_utils.basic diff --git a/tests/ansible/lib/modules/custom_python_json_args_module.py b/tests/ansible/lib/modules/custom_python_json_args_module.py index 61640579..846037ec 100755 --- a/tests/ansible/lib/modules/custom_python_json_args_module.py +++ b/tests/ansible/lib/modules/custom_python_json_args_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible Python JSONARGS module. I should receive an encoding string. json_arguments = """<>""" diff --git a/tests/ansible/lib/modules/custom_python_os_getcwd.py b/tests/ansible/lib/modules/custom_python_os_getcwd.py index c5e264ae..d465ac9e 100644 --- a/tests/ansible/lib/modules/custom_python_os_getcwd.py +++ b/tests/ansible/lib/modules/custom_python_os_getcwd.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # #591: call os.getcwd() before AnsibleModule ever gets a chance to fix up the # process environment. diff --git a/tests/ansible/lib/modules/custom_python_want_json_module.py b/tests/ansible/lib/modules/custom_python_want_json_module.py index f5e33862..23eeeb55 100755 --- a/tests/ansible/lib/modules/custom_python_want_json_module.py +++ b/tests/ansible/lib/modules/custom_python_want_json_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible Python WANT_JSON module. I should receive a JSON-encoded file. import json From 863f923f1410c7fa34dc8a7cd0f8e0f695a7b5a4 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 11 Aug 2024 01:05:10 +0100 Subject: [PATCH 09/46] tests: Bypass interpreter discovery on non-existant connection delegation targets By setting ansible_python_interpreter for these fictious hosts we avoid Ansible trying and failing to connect to them in a attempt to populate ansible_facts.discovered_interpreter_python. This speeds up these tests by avoiding a timeout. It is also a necessary pre-requisite for Ansible 10 (ansible-core 2.17). In that release no hardcoded fallback is used, failure to determine a valid Python interpreter is a fatal error. refs #1074 --- tests/ansible/hosts/become_same_user.hosts | 3 +++ .../ansible/hosts/connection_delegation.hosts | 4 +++- .../delegate_to_template.yml | 4 ++-- .../osa_container_standalone.yml | 2 +- .../osa_delegate_to_self.yml | 2 +- .../stack_construction.yml | 24 +++++++++---------- 6 files changed, 22 insertions(+), 17 deletions(-) diff --git a/tests/ansible/hosts/become_same_user.hosts b/tests/ansible/hosts/become_same_user.hosts index ac744ed7..249246cb 100644 --- a/tests/ansible/hosts/become_same_user.hosts +++ b/tests/ansible/hosts/become_same_user.hosts @@ -1,6 +1,9 @@ # code: language=ini # vim: syntax=dosini +[become_same_user] # become_same_user.yml bsu-joe ansible_user=joe +[become_same_user:vars] +ansible_python_interpreter=python3000 diff --git a/tests/ansible/hosts/connection_delegation.hosts b/tests/ansible/hosts/connection_delegation.hosts index 4ae861b0..7de798a1 100644 --- a/tests/ansible/hosts/connection_delegation.hosts +++ b/tests/ansible/hosts/connection_delegation.hosts @@ -4,7 +4,7 @@ # Connection delegation scenarios. It's impossible to connect to them, but their would-be # config can be inspected using "mitogen_get_stack" action. - +[cd] # Normal inventory host, no aliasing. cd-normal ansible_connection=mitogen_doas ansible_user=normal-user @@ -23,6 +23,8 @@ cd-newuser-normal-normal mitogen_via=cd-normal ansible_user=newuser-normal-norma # doas:newuser via host. cd-newuser-doas-normal mitogen_via=cd-normal ansible_connection=mitogen_doas ansible_user=newuser-doas-normal-user +[cd:vars] +ansible_python_interpreter = python3000 [connection-delegation-test] cd-bastion diff --git a/tests/ansible/integration/connection_delegation/delegate_to_template.yml b/tests/ansible/integration/connection_delegation/delegate_to_template.yml index 0c3f35c2..f9ad9e0b 100644 --- a/tests/ansible/integration/connection_delegation/delegate_to_template.yml +++ b/tests/ansible/integration/connection_delegation/delegate_to_template.yml @@ -40,7 +40,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - 'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"], + 'python_path': ['python3000'], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, @@ -68,7 +68,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - 'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"], + 'python_path': ['python3000'], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, diff --git a/tests/ansible/integration/connection_delegation/osa_container_standalone.yml b/tests/ansible/integration/connection_delegation/osa_container_standalone.yml index 98788d39..df295fd0 100644 --- a/tests/ansible/integration/connection_delegation/osa_container_standalone.yml +++ b/tests/ansible/integration/connection_delegation/osa_container_standalone.yml @@ -19,7 +19,7 @@ 'kind': 'lxc', 'lxc_info_path': null, 'machinectl_path': null, - 'python_path': ['/usr/bin/python'], + 'python_path': ['python3000'], 'remote_name': null, 'username': null, }, diff --git a/tests/ansible/integration/connection_delegation/osa_delegate_to_self.yml b/tests/ansible/integration/connection_delegation/osa_delegate_to_self.yml index f97d1c01..53addbfe 100644 --- a/tests/ansible/integration/connection_delegation/osa_delegate_to_self.yml +++ b/tests/ansible/integration/connection_delegation/osa_delegate_to_self.yml @@ -23,7 +23,7 @@ 'lxc_info_path': null, 'lxc_path': null, 'machinectl_path': null, - 'python_path': ["/usr/bin/python"], + 'python_path': ["python3000"], 'username': 'ansible-cfg-remote-user', }, 'method': 'setns', diff --git a/tests/ansible/integration/connection_delegation/stack_construction.yml b/tests/ansible/integration/connection_delegation/stack_construction.yml index 784d2af2..8cf064bb 100644 --- a/tests/ansible/integration/connection_delegation/stack_construction.yml +++ b/tests/ansible/integration/connection_delegation/stack_construction.yml @@ -40,7 +40,7 @@ "connect_timeout": 30, "doas_path": null, "password": null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, "username": "normal-user", }, @@ -73,7 +73,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, @@ -115,7 +115,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, @@ -150,7 +150,7 @@ 'connect_timeout': 30, 'doas_path': null, 'password': null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, 'username': 'normal-user', }, @@ -168,7 +168,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, @@ -210,7 +210,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, @@ -238,7 +238,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, @@ -272,7 +272,7 @@ 'connect_timeout': 30, 'doas_path': null, 'password': null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, 'username': 'normal-user', }, @@ -290,7 +290,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, @@ -333,7 +333,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - "python_path": ["/usr/bin/python"], + "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, @@ -388,7 +388,7 @@ 'connect_timeout': 30, 'doas_path': null, 'password': null, - 'python_path': ["/usr/bin/python"], + 'python_path': ["python3000"], 'remote_name': null, 'username': 'normal-user', }, @@ -399,7 +399,7 @@ 'connect_timeout': 30, 'doas_path': null, 'password': null, - 'python_path': ["/usr/bin/python"], + 'python_path': ["python3000"], 'remote_name': null, 'username': 'newuser-doas-normal-user', }, From 85b1b4070a6521b48265d4ef2d2d6d1cb9d061d1 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 11 Aug 2024 02:12:16 +0100 Subject: [PATCH 10/46] tests: Respect configured or detected Python more often Relying on the virtualenv default or hardcoding "python" results in a Python 2.x virtualenv on some targets (e.g. debian10-test). This caused a failure when testing with Ansible >= 10 (ansible-core >= 2.17), which have dropped Python 2.x support. refs #1074 --- .../integration/interpreter_discovery/complex_args.yml | 3 +-- tests/ansible/integration/stub_connections/kubectl.yml | 1 - tests/ansible/integration/stub_connections/lxc.yml | 1 - tests/ansible/integration/stub_connections/lxd.yml | 1 - tests/ansible/integration/stub_connections/mitogen_doas.yml | 1 - tests/ansible/integration/stub_connections/mitogen_sudo.yml | 1 - .../ansible/regression/issue_152__virtualenv_python_fails.yml | 4 +++- 7 files changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/ansible/integration/interpreter_discovery/complex_args.yml b/tests/ansible/integration/interpreter_discovery/complex_args.yml index a3755ebf..ea7e2d63 100644 --- a/tests/ansible/integration/interpreter_discovery/complex_args.yml +++ b/tests/ansible/integration/interpreter_discovery/complex_args.yml @@ -25,8 +25,7 @@ # special_python: source /tmp/fake && python - name: set python using sourced file set_fact: - # Avoid 2.x vs 3.x cross-compatiblity issues (that I can't remember the exact details of). - special_python: "source /tmp/fake || true && python{{ ansible_facts.python.version.major }}" + special_python: "source /tmp/fake || true && {{ ansible_facts.python.executable }}" - name: run get_url with specially-sourced python uri: diff --git a/tests/ansible/integration/stub_connections/kubectl.yml b/tests/ansible/integration/stub_connections/kubectl.yml index e3e927c0..13e522d2 100644 --- a/tests/ansible/integration/stub_connections/kubectl.yml +++ b/tests/ansible/integration/stub_connections/kubectl.yml @@ -12,7 +12,6 @@ - custom_python_detect_environment: vars: ansible_connection: kubectl - ansible_python_interpreter: python # avoid Travis virtualenv breakage mitogen_kubectl_path: stub-kubectl.py register: out diff --git a/tests/ansible/integration/stub_connections/lxc.yml b/tests/ansible/integration/stub_connections/lxc.yml index ae71a603..3f1a2282 100644 --- a/tests/ansible/integration/stub_connections/lxc.yml +++ b/tests/ansible/integration/stub_connections/lxc.yml @@ -8,7 +8,6 @@ - custom_python_detect_environment: vars: ansible_connection: lxc - ansible_python_interpreter: python # avoid Travis virtualenv breakage mitogen_lxc_attach_path: stub-lxc-attach.py register: out diff --git a/tests/ansible/integration/stub_connections/lxd.yml b/tests/ansible/integration/stub_connections/lxd.yml index a2f8f80a..fd811fb7 100644 --- a/tests/ansible/integration/stub_connections/lxd.yml +++ b/tests/ansible/integration/stub_connections/lxd.yml @@ -8,7 +8,6 @@ - custom_python_detect_environment: vars: ansible_connection: lxd - ansible_python_interpreter: python # avoid Travis virtualenv breakage mitogen_lxc_path: stub-lxc.py register: out diff --git a/tests/ansible/integration/stub_connections/mitogen_doas.yml b/tests/ansible/integration/stub_connections/mitogen_doas.yml index 338bc09b..a8b77edd 100644 --- a/tests/ansible/integration/stub_connections/mitogen_doas.yml +++ b/tests/ansible/integration/stub_connections/mitogen_doas.yml @@ -8,7 +8,6 @@ - custom_python_detect_environment: vars: ansible_connection: mitogen_doas - ansible_python_interpreter: python # avoid Travis virtualenv breakage ansible_doas_exe: stub-doas.py ansible_user: someuser register: out diff --git a/tests/ansible/integration/stub_connections/mitogen_sudo.yml b/tests/ansible/integration/stub_connections/mitogen_sudo.yml index 0ab18f3b..bcf0f22f 100644 --- a/tests/ansible/integration/stub_connections/mitogen_sudo.yml +++ b/tests/ansible/integration/stub_connections/mitogen_sudo.yml @@ -8,7 +8,6 @@ - custom_python_detect_environment: vars: ansible_connection: mitogen_sudo - ansible_python_interpreter: python # avoid Travis virtualenv breakage ansible_user: root ansible_become_exe: stub-sudo.py ansible_become_flags: -H --type=sometype --role=somerole diff --git a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml index 610eaf33..e0439fdd 100644 --- a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml +++ b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml @@ -12,7 +12,9 @@ https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}" no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}" PATH: "{{ lookup('env', 'PATH') }}" - shell: virtualenv /tmp/issue_152_virtualenv + command: + cmd: virtualenv -p "{{ ansible_facts.python.executable }}" /tmp/issue_152_virtualenv + creates: /tmp/issue_152_virtualenv when: - lout.python.version.full is version('2.7', '>=', strict=True) From 357fe38766e0db4c3d75b1eaf946e32b10597db6 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 11 Aug 2024 09:27:15 +0100 Subject: [PATCH 11/46] Ansible 10 (ansible-core 2.17) support Notably - Python 2.7 and 3.6 are no longer supported by Ansible on targets - The yum module has been removed, and redirected to dnf - _INTERPRETER_PYTHON_DISTRO_MAP has been neutered. Interpreter discovery always favours specific `python3.` interpreters in decending version order, then generic `python3` or `python`. - Add the ability for an action plugin to call self._execute_module(*, ignore_unknown_opts=True) to execute a module with options that may not be supported for the version being called. https://docs.ansible.com/ansible/devel/porting_guides/porting_guide_10.html https://github.com/ansible-community/ansible-build-data/blob/main/10/CHANGELOG-v10.md https://github.com/ansible/ansible/blob/stable-2.17/changelogs/CHANGELOG-v2.17.rst fixes #1074, refs #1082 Co-authored-by: Claude Becker --- .ci/azure-pipelines.yml | 9 +- ansible_mitogen/loaders.py | 2 +- ansible_mitogen/mixins.py | 12 +- docs/ansible_detailed.rst | 21 ++-- docs/changelog.rst | 1 + .../ansible_2_8_tests.yml | 106 ++++++++++-------- .../interpreter_discovery/complex_args.yml | 7 ++ .../runner/custom_bash_hashbang_argument.yml | 6 + .../transport_config/python_path.yml | 18 ++- ...ssue_1066__add_host__host_key_checking.yml | 4 + .../issue_776__load_plugins_called_twice.yml | 14 ++- tox.ini | 17 ++- 12 files changed, 147 insertions(+), 70 deletions(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 51908e92..a3ffa9b4 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -32,10 +32,14 @@ jobs: tox.env: py312-mode_localhost-ansible9 Van_312_9: tox.env: py312-mode_localhost-ansible9-strategy_linear + Loc_312_10: + tox.env: py312-mode_localhost-ansible10 + Van_312_10: + tox.env: py312-mode_localhost-ansible10-strategy_linear - job: Linux pool: - # https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2004-Readme.md + # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md vmImage: ubuntu-20.04 steps: - template: azure-pipelines-steps.yml @@ -152,3 +156,6 @@ jobs: Ans_312_9: python.version: '3.12' tox.env: py312-mode_ansible-ansible9 + Ans_312_10: + python.version: '3.12' + tox.env: py312-mode_ansible-ansible10 diff --git a/ansible_mitogen/loaders.py b/ansible_mitogen/loaders.py index 9729b8a1..9d9876ac 100644 --- a/ansible_mitogen/loaders.py +++ b/ansible_mitogen/loaders.py @@ -49,7 +49,7 @@ __all__ = [ ANSIBLE_VERSION_MIN = (2, 10) -ANSIBLE_VERSION_MAX = (2, 16) +ANSIBLE_VERSION_MAX = (2, 17) NEW_VERSION_MSG = ( "Your Ansible version (%s) is too recent. The most recent version\n" diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 9cc97a48..0ba41aad 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -357,7 +357,9 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): def _execute_module(self, module_name=None, module_args=None, tmp=None, task_vars=None, persist_files=False, - delete_remote_tmp=True, wrap_async=False): + delete_remote_tmp=True, wrap_async=False, + ignore_unknown_opts=False, + ): """ Collect up a module's execution environment then use it to invoke target.run_module() or helpers.run_module_async() in the target @@ -370,7 +372,13 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): if task_vars is None: task_vars = {} - self._update_module_args(module_name, module_args, task_vars) + if ansible_mitogen.utils.ansible_version[:2] >= (2, 17): + self._update_module_args( + module_name, module_args, task_vars, + ignore_unknown_opts=ignore_unknown_opts, + ) + else: + self._update_module_args(module_name, module_args, task_vars) env = {} self._compute_environment_string(env) self._set_temp_file_args(module_args, wrap_async) diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index e6f7b42c..7d960ed9 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -165,7 +165,9 @@ Noteworthy Differences +-----------------+-----------------+ | 8 | 3.9 - 3.12 | +-----------------+-----------------+ - | 9 | 3.10 - 3.12 | + | 9 | | + +-----------------+ 3.10 - 3.12 | + | 10 | | +-----------------+-----------------+ Verify your installation is running one of these versions by checking @@ -271,15 +273,14 @@ Noteworthy Differences * "Module Replacer" style modules are not supported. These rarely appear in practice, and light web searches failed to reveal many examples of them. -.. - * The ``ansible_python_interpreter`` variable is parsed using a restrictive - :mod:`shell-like ` syntax, permitting values such as ``/usr/bin/env - FOO=bar python`` or ``source /opt/rh/rh-python36/enable && python``, which - occur in practice. Jinja2 templating is also supported for complex task-level - interpreter settings. Ansible `documents this - `_ - as an absolute path, however the implementation passes it unquoted through - the shell, permitting arbitrary code to be injected. +* The ``ansible_python_interpreter`` variable is parsed using a restrictive + :mod:`shell-like ` syntax, permitting values such as ``/usr/bin/env + FOO=bar python`` or ``source /opt/rh/rh-python36/enable && python``. + Jinja2 templating is also supported for complex task-level + interpreter settings. Ansible documents `ansible_python_interpreter + `_ + as an absolute path and releases since June 2024 (e.g. Ansible 10.1) + reflect this. Older Ansible releases passed it to the shell unquoted. .. * Configurations will break that rely on the `hashbang argument splitting diff --git a/docs/changelog.rst b/docs/changelog.rst index c1754b9e..33ff9f9b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,6 +23,7 @@ Unreleased * :gh:issue:`1097` Respect `ansible_facts.discovered_interpreter_python` when executing non new-style modules (e.g. JSONARGS style, WANT_JSON style). +* :gh:issue:`1074` Support Ansible 10 (ansible-core 2.17) v0.3.8 (2024-07-30) diff --git a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml index 6ac9bada..0c7d30c9 100644 --- a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml +++ b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml @@ -4,6 +4,55 @@ - name: integration/interpreter_discovery/ansible_2_8_tests.yml hosts: test-targets gather_facts: true + vars: + DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_lt_2_12: + centos: + '6': /usr/bin/python + '7': /usr/bin/python + '8': /usr/libexec/platform-python + debian: + '9': /usr/bin/python + '10': /usr/bin/python3 + '11': /usr/bin/python + 'NA': /usr/bin/python # Debian 11, Ansible <= 7 (ansible-core <= 2.14) + 'bullseye/sid': /usr/bin/python # Debian 11, Ansible 8 - 9 (ansible-core 2.15 - 2.16) + ubuntu: + '16': /usr/bin/python3 + '18': /usr/bin/python3 + '20': /usr/bin/python3 + + DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_2_12_to_2_16: + centos: + '6': /usr/bin/python + '7': /usr/bin/python + '8': /usr/libexec/platform-python + debian: + '9': /usr/bin/python + '10': /usr/bin/python3 + '11': /usr/bin/python3.9 + 'NA': /usr/bin/python3.9 # Debian 11, Ansible <= 7 (ansible-core <= 2.14) + 'bullseye/sid': /usr/bin/python3.9 # Debian 11, Ansible 8 - 9 (ansible-core 2.15 - 2.16) + ubuntu: + '16': /usr/bin/python3 + '18': /usr/bin/python3 + '20': /usr/bin/python3 + + DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_ge_2_17: + debian: + '10': /usr/bin/python3.7 + '11': /usr/bin/python3.9 + 'bullseye/sid': /usr/bin/python3.9 + ubuntu: + '20': /usr/bin/python3.8 + + discovered_interpreter_expected: >- + {%- if ansible_version.full is version('2.12', '<', strict=True) -%} + {{ DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_lt_2_12[distro][distro_major] }} + {%- elif ansible_version.full is version('2.17', '<', strict=True) -%} + {{ DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_2_12_to_2_16[distro][distro_major] }} + {%- else -%} + {{ DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_ge_2_17[distro][distro_major] }} + {%- endif -%} tasks: - name: can only run these tests on ansible >= 2.8.0 block: @@ -18,9 +67,9 @@ - name: snag some facts to validate for later set_fact: - distro: '{{ ansible_distribution | default("unknown") | lower }}' - distro_version: '{{ ansible_distribution_version | default("unknown") }}' - os_family: '{{ ansible_os_family | default("unknown") }}' + distro: '{{ ansible_facts.distribution | lower }}' + distro_major: '{{ ansible_facts.distribution_major_version }}' + system: '{{ ansible_facts.system }}' - name: test that python discovery is working and that fact persistence makes it only run once block: @@ -159,50 +208,19 @@ - ansible_facts['ansible_bogus_interpreter'] | default('nope') == 'nope' - ansible_facts['discovered_interpreter_bogus'] | default('nope') == 'nope' - - name: fedora assertions + - name: Check discovered interpreter matches expected assert: that: - - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python3' - fail_msg: auto_out={{auto_out}} + - auto_out.ansible_facts.discovered_interpreter_python == discovered_interpreter_expected + fail_msg: >- + distro={{ distro }} + distro_major= {{ distro_major }} + system={{ system }} + auto_out={{ auto_out }} + discovered_interpreter_expected={{ discovered_interpreter_expected }} + ansible_version.full={{ ansible_version.full }} when: - - distro == 'fedora' - - distro_version is version('23.0', '>=', strict=True) - - - name: rhel < 8 assertions - assert: - that: - - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python' - fail_msg: auto_out={{auto_out}} - when: - - distro in ('redhat', 'centos') - - distro_version is version('8.0', '<', strict=true) - - - name: rhel 8+ assertions - assert: - that: - - auto_out.ansible_facts.discovered_interpreter_python == '/usr/libexec/platform-python' - fail_msg: auto_out={{auto_out}} - when: - - distro in ('redhat', 'centos') - - distro_version is version('8.0', '>=', strict=true) - - - name: ubuntu < 16.04 assertions - assert: - that: - - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python' - fail_msg: auto_out={{auto_out}} - when: - - distro == 'ubuntu' - - distro_version is version('16.04', '<', strict=true) - - - name: ubuntu 16.04+ assertions - assert: - that: - - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python3' - fail_msg: auto_out={{auto_out}} - when: - - distro == 'ubuntu' - - distro_version is version('16.04', '>=', strict=True) + - system in ['Linux'] always: - meta: clear_facts diff --git a/tests/ansible/integration/interpreter_discovery/complex_args.yml b/tests/ansible/integration/interpreter_discovery/complex_args.yml index ea7e2d63..af2b5e46 100644 --- a/tests/ansible/integration/interpreter_discovery/complex_args.yml +++ b/tests/ansible/integration/interpreter_discovery/complex_args.yml @@ -9,6 +9,13 @@ https_proxy: "{{ lookup('env', 'https_proxy') | default(omit) }}" no_proxy: "{{ lookup('env', 'no_proxy') | default(omit) }}" tasks: + # Ansible releases after June 2024 quote ansible_python_interpreter + # https://github.com/ansible/ansible/pull/83365 + - meta: end_play + when: + - not is_mitogen + - ansible_version.full is version('2.17.1', '>=', strict=True) + - name: create temp file to source file: path: /tmp/fake diff --git a/tests/ansible/integration/runner/custom_bash_hashbang_argument.yml b/tests/ansible/integration/runner/custom_bash_hashbang_argument.yml index ebb7966e..82535d56 100644 --- a/tests/ansible/integration/runner/custom_bash_hashbang_argument.yml +++ b/tests/ansible/integration/runner/custom_bash_hashbang_argument.yml @@ -2,6 +2,12 @@ - name: integration/runner/custom_bash_hashbang_argument.yml hosts: test-targets tasks: + # Ansible releases after June 2024 quote ansible_python_interpreter + # https://github.com/ansible/ansible/pull/83365 + - meta: end_play + when: + - not is_mitogen + - ansible_version.full is version('2.17.1', '>=', strict=True) - custom_bash_old_style_module: foo: true diff --git a/tests/ansible/integration/transport_config/python_path.yml b/tests/ansible/integration/transport_config/python_path.yml index 0f04ed59..ecc24374 100644 --- a/tests/ansible/integration/transport_config/python_path.yml +++ b/tests/ansible/integration/transport_config/python_path.yml @@ -2,12 +2,17 @@ # Each case is followed by mitogen_via= case to test hostvars method. -# When no ansible_python_interpreter is set, ansible 2.8+ automatically -# tries to detect the desired interpreter, falling back to "/usr/bin/python" if necessary +# If ansible_python_interpreter isn't set, Ansible 2.8+ tries to connect and +# detect the interpreter. If that fails (e.g. connection timeout) +# - Ansible 2.8 - 9 (ansible-core 2.8 - 2.16) assumes "/usr/bin/python" +# - Ansible 10+ (ansible-core 2.17+) marks the target unreachable - name: integration/transport_config/python_path.yml hosts: tc-python-path-unset tasks: - include_tasks: ../_mitogen_only.yml + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path @@ -19,6 +24,9 @@ vars: {mitogen_via: tc-python-path-unset} tasks: - include_tasks: ../_mitogen_only.yml + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path @@ -45,6 +53,9 @@ vars: {mitogen_via: tc-python-path-hostvar} tasks: - include_tasks: ../_mitogen_only.yml + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path @@ -71,6 +82,9 @@ vars: {mitogen_via: localhost} tasks: - include_tasks: ../_mitogen_only.yml + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) - {mitogen_get_stack: {}, register: out} - assert_equal: left: out.result[0].kwargs.python_path diff --git a/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml b/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml index 4b4609b6..dd754f84 100644 --- a/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml +++ b/tests/ansible/regression/issue_1066__add_host__host_key_checking.yml @@ -28,6 +28,10 @@ become: false serial: 1 tasks: + # FIXME https://github.com/mitogen-hq/mitogen/issues/1096 + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) - meta: reset_connection # The host key might be in ~/.ssh/known_hosts. If it's removed then no diff --git a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml index 44ff6863..ef573276 100755 --- a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml +++ b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml @@ -3,13 +3,25 @@ - name: regression/issue_776__load_plugins_called_twice.yml hosts: test-targets become: "{{ groups.linux is defined and inventory_hostname in groups.linux }}" - gather_facts: yes + # Delayed until after the end_play, due to Python version requirements + gather_facts: false tags: - issue_776 vars: ansible_python_interpreter: "{{ pkg_mgr_python_interpreter }}" package: rsync # Chosen to exist in all tested distros/package managers tasks: + # The package management modules require using the same Python version + # as the target's package manager libraries. This is sometimes in conflict + # with Ansible requirements, e.g. Ansible 10 (ansible-core 2.17) does not + # support Python 2.x on targets. + - meta: end_play + when: + - ansible_version.full is version('2.17', '>=', strict=True) + + - name: Gather facts manually + setup: + - name: Switch to archived package repositories copy: dest: "{{ item.dest }}" diff --git a/tox.ini b/tox.ini index 5dcb1bf2..08f4c371 100644 --- a/tox.ini +++ b/tox.ini @@ -10,9 +10,9 @@ # 2.4 2.3? <= 3.7.1 <= 1.3.7 <= 1.1 <= 2.1.3 <= 1.4 <= 1.8 # 2.5 <= 3.7.1 <= 1.4.22 <= 1.3.1 <= 2.1.3 <= 2.8.7 <= 1.6.1 <= 1.9.1 # 2.6 <= 2.6.20 <= 2.12 <= 4.5.4 <= 1.6.11 <= 2.10.3 <= 9.0.3 <= 5.9.0 <= 3.2.5 <= 2.9.1 <= 15.2.0 -# 2.7 <= 2.11 <= 5.5 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.15² +# 2.7 <= 2.11 <= 2.16 <= 5.5 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.15² # 3.5 <= 2.11 <= 2.15 <= 5.5 <= 2.2.28 <= 2.11.3 <= 20 <= 5.9.5 <= 6.1.0 <= 3.28 <= 20.15² -# 3.6 <= 2.11 <= 6.2 <= 3.2.20 <= 3.0.3 <= 21 <= 7.0.1 <= 3.28 <= 20.17² +# 3.6 <= 2.11 <= 2.16 <= 6.2 <= 3.2.20 <= 3.0.3 <= 21 <= 7.0.1 <= 3.28 <= 20.17² # 3.7 <= 2.12 <= 7.2.7 <= 3.2.20 <= 7.4.4 <= 4.8.0 # 3.8 <= 2.12 # 3.9 <= 2.15 @@ -46,18 +46,14 @@ # ansible == 7.x ansible-core ~= 2.14.0 # ansible == 8.x ansible-core ~= 2.15.0 # ansible == 9.x ansible-core ~= 2.16.0 - -# pip --no-python-version-warning -# pip --disable-pip-version-check - -# TODO distros=-py3 +# ansible == 10.x ansible-core ~= 2.17.0 [tox] envlist = init, py{27,36}-mode_ansible-ansible{2.10,3,4}, py{311}-mode_ansible-ansible{2.10,3,4,5}, - py{312}-mode_ansible-ansible{6,7,8,9}, + py{312}-mode_ansible-ansible{6,7,8,9,10}, py{27,36,312}-mode_mitogen-distro_centos{6,7,8}, py{27,36,312}-mode_mitogen-distro_debian{9,10,11}, py{27,36,312}-mode_mitogen-distro_ubuntu{1604,1804,2004}, @@ -86,6 +82,7 @@ deps = ansible7: ansible~=7.0 ansible8: ansible~=8.0 ansible9: ansible~=9.0 + ansible10: ansible~=10.0 install_command = python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages} commands_pre = @@ -129,8 +126,10 @@ setenv = ansible6: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 ansible7: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 ansible8: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004 - # Ansible >= 9 (ansible-core >= 2.16) require Python 2.7 or >= 3.6 on targets + # Ansible 9 (ansible-core 2.16) requires Python 2.7 or >= 3.6 on targets ansible9: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1804 ubuntu2004 + # Ansible 10 (ansible-core 2.17) requires Python >= 3.7 on targets + ansible10: DISTROS=debian10-py3 debian11-py3 ubuntu2004-py3 distros_centos: DISTROS=centos6 centos7 centos8 distros_centos5: DISTROS=centos5 distros_centos6: DISTROS=centos6 From 2edcb31996b894e1683c5c23114f944572d97690 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 31 Jul 2024 08:30:38 +0100 Subject: [PATCH 12/46] docs: Remove piwik analytics hooks from website https://networkgenomics.com is no longer served, so the javascript and other attempts to beacon or redirect result in HTTP errors. --- docs/_templates/ansible.html | 14 -------------- docs/_templates/layout.html | 18 ------------------ docs/_templates/piwik-config.js | 5 ----- docs/conf.py | 1 - 4 files changed, 38 deletions(-) delete mode 100644 docs/_templates/ansible.html delete mode 100644 docs/_templates/piwik-config.js diff --git a/docs/_templates/ansible.html b/docs/_templates/ansible.html deleted file mode 100644 index 770e0a45..00000000 --- a/docs/_templates/ansible.html +++ /dev/null @@ -1,14 +0,0 @@ - -Mitogen for Ansible (Redirect) - - - - diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index 97771afa..a01b497f 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -14,23 +14,5 @@ {% block footer %} {{ super() }} - - - - - {% endblock %} diff --git a/docs/_templates/piwik-config.js b/docs/_templates/piwik-config.js deleted file mode 100644 index ad24c9f6..00000000 --- a/docs/_templates/piwik-config.js +++ /dev/null @@ -1,5 +0,0 @@ -window._paq = []; -window._paq.push(['trackPageView']); -window._paq.push(['enableLinkTracking']); -window._paq.push(['enableHeartBeatTimer', 30]); -window._paq.push(['setSiteId', 6]); diff --git a/docs/conf.py b/docs/conf.py index fe1a5044..93469733 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,7 +16,6 @@ html_show_copyright = False html_show_sourcelink = False html_show_sphinx = False html_sidebars = {'**': ['globaltoc.html', 'github.html']} -html_additional_pages = {'ansible': 'ansible.html'} html_static_path = ['_static'] html_theme = 'alabaster' html_theme_options = { From c948e6668a8dfd9533a9941fa736d04cc2f18c5f Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 31 Jul 2024 08:39:36 +0100 Subject: [PATCH 13/46] docs: Remove email form from website https://networkgenomics.com is no longer served, so the form submission would fail. --- docs/ansible_detailed.rst | 45 --------------------------------------- 1 file changed, 45 deletions(-) diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index 7d960ed9..fed347c8 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -75,34 +75,6 @@ Installation ``mitogen_host_pinned`` strategies exists to mimic the ``free`` and ``host_pinned`` strategies. -4. - - .. raw:: html - -
- - - Get notified of new releases and important fixes. - -

-
- - - ↻ - - - -

- - - -

-

- Demo ~~~~ @@ -1419,20 +1391,3 @@ Despite the small margin for optimization, Mitogen still manages **6.2x less bandwidth and 1.8x less time**. .. image:: images/ansible/pcaps/costapp-uk-india.svg - - -.. raw:: html - - - From 6fbad3ae7dd03d6f41d3b0a4ceb7d0d4ed3832b8 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 13 Aug 2024 10:32:31 +0100 Subject: [PATCH 14/46] Prepare v0.3.9 --- docs/changelog.rst | 4 ++-- docs/conf.py | 2 +- mitogen/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 33ff9f9b..7c6fc96d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,8 +18,8 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. -Unreleased ----------- +v0.3.9 (2024-08-13) +------------------- * :gh:issue:`1097` Respect `ansible_facts.discovered_interpreter_python` when executing non new-style modules (e.g. JSONARGS style, WANT_JSON style). diff --git a/docs/conf.py b/docs/conf.py index 93469733..b7dd1525 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -2,7 +2,7 @@ import sys sys.path.append('.') -VERSION = '0.3.8' +VERSION = '0.3.9' author = u'Network Genomics' copyright = u'2021, the Mitogen authors' diff --git a/mitogen/__init__.py b/mitogen/__init__.py index ce6c2de0..b0c66793 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 9, 'dev0') +__version__ = (0, 3, 9) #: This is :data:`False` in slave contexts. Previously it was used to prevent From d15051b187b1c613a526c39c468a96d4449c0890 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 13 Aug 2024 10:35:50 +0100 Subject: [PATCH 15/46] Begin v0.3.10dev --- docs/changelog.rst | 5 +++++ mitogen/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7c6fc96d..0d1a14a2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,11 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. +Unreleased +---------- + + + v0.3.9 (2024-08-13) ------------------- diff --git a/mitogen/__init__.py b/mitogen/__init__.py index b0c66793..1d483ead 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 9) +__version__ = (0, 3, 10, 'dev') #: This is :data:`False` in slave contexts. Previously it was used to prevent From 06617f8231b4facc0cd7b751b72293a7d8744a5c Mon Sep 17 00:00:00 2001 From: Jonathan Rosser Date: Tue, 16 Jul 2024 15:38:40 +0100 Subject: [PATCH 16/46] ansible_mitogen: Handle unsafe paths in _remote_chmod This is missing from https://github.com/mitogen-hq/mitogen/commit/b822f20007ebe94106b15275962ea4cbbd8a0331 --- ansible_mitogen/mixins.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 0ba41aad..2cd97a3e 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -280,7 +280,9 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): paths, mode, sudoable) return self.fake_shell(lambda: mitogen.select.Select.all( self._connection.get_chain().call_async( - ansible_mitogen.target.set_file_mode, path, mode + ansible_mitogen.target.set_file_mode, + ansible_mitogen.utils.unsafe.cast(path), + mode, ) for path in paths )) From 5ab872f28930a70e03253d0fb97acbbee2a47430 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 29 Aug 2024 11:07:35 +0100 Subject: [PATCH 17/46] ansible_mitogen: Add regression test for ActionModuleMixin._remote_chmod() Adapted from Jonathon's reproducer in #1087. --- docs/changelog.rst | 2 + docs/contributors.rst | 1 + tests/ansible/regression/all.yml | 1 + .../issue_1087__template_streamerror.yml | 43 +++++++++++++++++++ tests/ansible/regression/template_test.yml | 28 ++++++++++++ tests/ansible/regression/templates/foo.bar.j2 | 1 + 6 files changed, 76 insertions(+) create mode 100644 tests/ansible/regression/issue_1087__template_streamerror.yml create mode 100644 tests/ansible/regression/template_test.yml create mode 100644 tests/ansible/regression/templates/foo.bar.j2 diff --git a/docs/changelog.rst b/docs/changelog.rst index 0d1a14a2..63b4da12 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,8 @@ To avail of fixes in an unreleased version, please download a ZIP file Unreleased ---------- +* :gh:issue:`1087` Fix :exc:`mitogen.core.StreamError` when Ansible template + module is called with a ``dest:`` filename that has an extension v0.3.9 (2024-08-13) diff --git a/docs/contributors.rst b/docs/contributors.rst index ed7fef11..1a590869 100644 --- a/docs/contributors.rst +++ b/docs/contributors.rst @@ -125,6 +125,7 @@ sponsorship and outstanding future-thinking of its early adopters.
  • rkrzr
  • jgadling
  • John F Wall — Making Ansible Great with Massive Parallelism
    • +
  • Jonathan Rosser
  • KennethC
  • Luca Berruti
  • Lewis Bellwood — Happy to be apart of a great project.
    • diff --git a/tests/ansible/regression/all.yml b/tests/ansible/regression/all.yml index 31541e9f..a4272805 100644 --- a/tests/ansible/regression/all.yml +++ b/tests/ansible/regression/all.yml @@ -16,3 +16,4 @@ - import_playbook: issue_776__load_plugins_called_twice.yml - import_playbook: issue_952__ask_become_pass.yml - import_playbook: issue_1066__add_host__host_key_checking.yml +- import_playbook: issue_1087__template_streamerror.yml diff --git a/tests/ansible/regression/issue_1087__template_streamerror.yml b/tests/ansible/regression/issue_1087__template_streamerror.yml new file mode 100644 index 00000000..fa950ea4 --- /dev/null +++ b/tests/ansible/regression/issue_1087__template_streamerror.yml @@ -0,0 +1,43 @@ +- name: regression/issue_1087__template_streamerror.yml + # Ansible's template module has been seen to raise mitogen.core.StreamError + # iif there is a with_items loop and the destination path has an extension. + # This printed an error message and left file permissions incorrect, + # but did not cause the task/playbook to fail. + hosts: test-targets + gather_facts: false + become: false + vars: + foos: + - dest: /tmp/foo + - dest: /tmp/foo.txt + foo: Foo + bar: Bar + tasks: + - block: + - name: Test template does not cause StreamError + delegate_to: localhost + run_once: true + environment: + ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" + command: + cmd: > + ansible-playbook + {% for inv in ansible_inventory_sources %} + -i "{{ inv }}" + {% endfor %} + regression/template_test.yml + chdir: ../ + register: issue_1087_cmd + failed_when: + - issue_1087_cmd is failed + or issue_1087_cmd.stdout is search('ERROR|mitogen\.core\.CallError') + or issue_1087_cmd.stderr is search('ERROR|mitogen\.core\.CallError') + + always: + - name: Cleanup + file: + path: "{{ item.dest }}" + state: absent + with_items: "{{ foos }}" + tags: + - issue_1087 diff --git a/tests/ansible/regression/template_test.yml b/tests/ansible/regression/template_test.yml new file mode 100644 index 00000000..0b7dd36d --- /dev/null +++ b/tests/ansible/regression/template_test.yml @@ -0,0 +1,28 @@ +- name: regression/template_test.yml + # Ansible's template module has been seen to raise mitogen.core.StreamError + # iif there is a with_items loop and the destination path has an extension + hosts: test-targets + gather_facts: false + become: false + vars: + foos: + - dest: /tmp/foo + - dest: /tmp/foo.txt + foo: Foo + bar: Bar + tasks: + - block: + - name: Template files + template: + src: foo.bar.j2 + dest: "{{ item.dest }}" + mode: u=rw,go=r + # This has to be with_items, loop: doesn't trigger the bug + with_items: "{{ foos }}" + + always: + - name: Cleanup + file: + path: "{{ item.dest }}" + state: absent + with_items: "{{ foos }}" diff --git a/tests/ansible/regression/templates/foo.bar.j2 b/tests/ansible/regression/templates/foo.bar.j2 new file mode 100644 index 00000000..ca51a6f4 --- /dev/null +++ b/tests/ansible/regression/templates/foo.bar.j2 @@ -0,0 +1 @@ +A {{ foo }} walks into a {{ bar }}. Ow! From ce1accedbce1d841d2fa88676fabbdaa1487f92f Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 30 Aug 2024 16:00:52 +0100 Subject: [PATCH 18/46] tests: Refactor Ansible copy integration tests to be loop driven This is in anticipation of #1110, which only exhibits inside a with_items: loop. For this refactor `loop:` is used, to confirm the refactored tests are still correct. A subsequent commit will change them to with_items. The content of the files and their SHA1 checksums are unchanged. --- tests/ansible/integration/action/copy.yml | 125 +++++++++++----------- 1 file changed, 62 insertions(+), 63 deletions(-) diff --git a/tests/ansible/integration/action/copy.yml b/tests/ansible/integration/action/copy.yml index 73f3bd1e..cbebe3ab 100644 --- a/tests/ansible/integration/action/copy.yml +++ b/tests/ansible/integration/action/copy.yml @@ -2,91 +2,90 @@ - name: integration/action/copy.yml hosts: test-targets - tasks: - - name: Create tiny file - copy: - dest: /tmp/copy-tiny-file - content: - this is a tiny file. - delegate_to: localhost - run_once: true + vars: + sourced_files: + - src: /tmp/copy-tiny-file + dest: /tmp/copy-tiny-file.out + content: this is a tiny file. + expected_checksum: f29faa9a6f19a700a941bf2aa5b281643c4ec8a0 + - src: /tmp/copy-large-file + dest: /tmp/copy-large-file.out + content: "{{ 'x' * 200000 }}" + expected_checksum: 62951f943c41cdd326e5ce2b53a779e7916a820d + + inline_files: + - dest: /tmp/copy-tiny-inline-file.out + content: tiny inline content + expected_checksum: b26dd6444595e2bdb342aa0a91721b57478b5029 + - dest: /tmp/copy-large-inline-file.out + content: | + {{ 'y' * 200000 }} + expected_checksum: d675f47e467eae19e49032a2cc39118e12a6ee72 - - name: Create large file + files: "{{ sourced_files + inline_files }}" + tasks: + - name: Create sourced files copy: - dest: /tmp/copy-large-file - # Must be larger than Connection.SMALL_SIZE_LIMIT. - content: "{% for x in range(200000) %}x{% endfor %}" + dest: "{{ item.src }}" + content: "{{ item.content }}" + mode: u=rw,go=r + loop: "{{ sourced_files }}" + loop_control: + label: "{{ item.src }}" delegate_to: localhost run_once: true - - name: Cleanup copied files + - name: Cleanup lingering destination files file: + path: "{{ item.dest }}" state: absent - path: "{{item}}" - with_items: - - /tmp/copy-tiny-file.out - - /tmp/copy-large-file.out - - /tmp/copy-tiny-inline-file.out - - /tmp/copy-large-inline-file.out + loop: "{{ files }}" + loop_control: + label: "{{ item.dest }}" - - name: Copy large file + - name: Copy sourced files copy: - dest: /tmp/copy-large-file.out - src: /tmp/copy-large-file - - - name: Copy tiny file - copy: - dest: /tmp/copy-tiny-file.out - src: /tmp/copy-tiny-file + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: u=rw,go=r + loop: "{{ sourced_files }}" + loop_control: + label: "{{ item.dest }}" - - name: Copy tiny inline file + - name: Copy inline files copy: - dest: /tmp/copy-tiny-inline-file.out - content: "tiny inline content" - - - name: Copy large inline file - copy: - dest: /tmp/copy-large-inline-file.out - content: | - {% for x in range(200000) %}y{% endfor %} + dest: "{{ item.dest }}" + content: "{{ item.content }}" + mode: u=rw,go=r + loop: "{{ inline_files }}" + loop_control: + label: "{{ item.dest }}" # stat results - name: Stat copied files stat: - path: "{{item}}" - with_items: - - /tmp/copy-tiny-file.out - - /tmp/copy-large-file.out - - /tmp/copy-tiny-inline-file.out - - /tmp/copy-large-inline-file.out + path: "{{ item.dest }}" + loop: "{{ files }}" + loop_control: + label: "{{ item.dest }}" register: stat - assert: that: - - stat.results[0].stat.checksum == "f29faa9a6f19a700a941bf2aa5b281643c4ec8a0" - - stat.results[1].stat.checksum == "62951f943c41cdd326e5ce2b53a779e7916a820d" - - stat.results[2].stat.checksum == "b26dd6444595e2bdb342aa0a91721b57478b5029" - - stat.results[3].stat.checksum == "d675f47e467eae19e49032a2cc39118e12a6ee72" - fail_msg: stat={{stat}} + - item.stat.checksum == item.item.expected_checksum + quiet: true # Avoid spamming stdout with 400 kB of item.item.content + fail_msg: item={{ item }} + loop: "{{ stat.results }}" + loop_control: + label: "{{ item.stat.path }}" - - name: Cleanup files + - name: Cleanup destination files file: + path: "{{ item.dest }}" state: absent - path: "{{item}}" - with_items: - - /tmp/copy-tiny-file - - /tmp/copy-tiny-file.out - - /tmp/copy-no-mode - - /tmp/copy-no-mode.out - - /tmp/copy-with-mode - - /tmp/copy-with-mode.out - - /tmp/copy-large-file - - /tmp/copy-large-file.out - - /tmp/copy-tiny-inline-file.out - - /tmp/copy-large-inline-file - - /tmp/copy-large-inline-file.out - - # end of cleaning out files (again) + loop: "{{ files }}" + loop_control: + label: "{{ item.dest }}" tags: - copy From 0bd3c6cba50bf9e607aa13432586af8af4fe51ba Mon Sep 17 00:00:00 2001 From: Jonathan Rosser Date: Wed, 28 Aug 2024 08:24:35 +0100 Subject: [PATCH 19/46] Fix AnsibleUnsafeText when copying files larger than SMALL_FILE_LIMIT Small files are carried in-band in the communication between controller and remote, with larger files being copied by falling back to a more traditional ansible put_file mechanism. This large file code path was missed in b822f20. --- ansible_mitogen/connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 6bdf11ba..a3f66eac 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -1129,6 +1129,6 @@ class Connection(ansible.plugins.connection.ConnectionBase): self.get_chain().call( ansible_mitogen.target.transfer_file, context=self.binding.get_child_service_context(), - in_path=in_path, - out_path=out_path + in_path=ansible_mitogen.utils.unsafe.cast(in_path), + out_path=ansible_mitogen.utils.unsafe.cast(out_path) ) From 5af6534a70eb03a71c5a181bd2abcf77e2be903e Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 30 Aug 2024 17:19:20 +0100 Subject: [PATCH 20/46] tests: Test AnsibleUnsafeText when copying files larger SMALL_FILE_LIMIT The bug was fixed in a previous commit by Jonathan Rosser. This adds testing. The bug is only triggered when the copy module is used inside a `with_items:` loop and the destination filename has an extension. A `loop:` loop is not sufficient. refs #1110 --- docs/changelog.rst | 3 +++ tests/ansible/integration/action/copy.yml | 17 ++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 63b4da12..4fc177c1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,6 +23,9 @@ Unreleased * :gh:issue:`1087` Fix :exc:`mitogen.core.StreamError` when Ansible template module is called with a ``dest:`` filename that has an extension +* :gh:issue:`1110` Fix :exc:`mitogen.core.StreamError` when Ansible copy + module is called with a file larger than 124 kibibytes + (:data:`ansible_mitogen.connection.Connection.SMALL_FILE_LIMIT`) v0.3.9 (2024-08-13) diff --git a/tests/ansible/integration/action/copy.yml b/tests/ansible/integration/action/copy.yml index cbebe3ab..edaa3e49 100644 --- a/tests/ansible/integration/action/copy.yml +++ b/tests/ansible/integration/action/copy.yml @@ -1,4 +1,6 @@ # Verify copy module for small and large files, and inline content. +# To exercise https://github.com/mitogen-hq/mitogen/pull/1110 destination +# files must have extensions and loops must use `with_items:`. - name: integration/action/copy.yml hosts: test-targets @@ -29,7 +31,7 @@ dest: "{{ item.src }}" content: "{{ item.content }}" mode: u=rw,go=r - loop: "{{ sourced_files }}" + with_items: "{{ sourced_files }}" loop_control: label: "{{ item.src }}" delegate_to: localhost @@ -39,7 +41,7 @@ file: path: "{{ item.dest }}" state: absent - loop: "{{ files }}" + with_items: "{{ files }}" loop_control: label: "{{ item.dest }}" @@ -48,7 +50,7 @@ src: "{{ item.src }}" dest: "{{ item.dest }}" mode: u=rw,go=r - loop: "{{ sourced_files }}" + with_items: "{{ sourced_files }}" loop_control: label: "{{ item.dest }}" @@ -57,7 +59,7 @@ dest: "{{ item.dest }}" content: "{{ item.content }}" mode: u=rw,go=r - loop: "{{ inline_files }}" + with_items: "{{ inline_files }}" loop_control: label: "{{ item.dest }}" @@ -66,7 +68,7 @@ - name: Stat copied files stat: path: "{{ item.dest }}" - loop: "{{ files }}" + with_items: "{{ files }}" loop_control: label: "{{ item.dest }}" register: stat @@ -76,7 +78,7 @@ - item.stat.checksum == item.item.expected_checksum quiet: true # Avoid spamming stdout with 400 kB of item.item.content fail_msg: item={{ item }} - loop: "{{ stat.results }}" + with_items: "{{ stat.results }}" loop_control: label: "{{ item.stat.path }}" @@ -84,8 +86,9 @@ file: path: "{{ item.dest }}" state: absent - loop: "{{ files }}" + with_items: "{{ files }}" loop_control: label: "{{ item.dest }}" tags: - copy + - issue_1110 From 46c9f772d84e6a42c8774ad38a5be6f804f06f39 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 3 Sep 2024 18:25:56 +0100 Subject: [PATCH 21/46] tests: Simplify Ansible ssh password tests, test priority This - Removes the indirection of calling ansible in a sub-shell - Includes vanilla Ansible, which was previously skipped - Tests whether ansible_ssh_pass overrides ansible_password, as it should As a one off I've the new tests against vanilla Ansible 2.10 through Ansible 10, to confirm the baseline priorities have remained unchanged all releases currently supported by Mitogen 0.3.x. --- tests/ansible/integration/ssh/all.yml | 1 + tests/ansible/integration/ssh/password.yml | 51 ++++++++ tests/ansible/integration/ssh/variables.yml | 128 -------------------- 3 files changed, 52 insertions(+), 128 deletions(-) create mode 100644 tests/ansible/integration/ssh/password.yml diff --git a/tests/ansible/integration/ssh/all.yml b/tests/ansible/integration/ssh/all.yml index a8335ab7..1b0f36e4 100644 --- a/tests/ansible/integration/ssh/all.yml +++ b/tests/ansible/integration/ssh/all.yml @@ -1,3 +1,4 @@ - import_playbook: config.yml +- import_playbook: password.yml - import_playbook: timeouts.yml - import_playbook: variables.yml diff --git a/tests/ansible/integration/ssh/password.yml b/tests/ansible/integration/ssh/password.yml new file mode 100644 index 00000000..cf9396e0 --- /dev/null +++ b/tests/ansible/integration/ssh/password.yml @@ -0,0 +1,51 @@ +- name: integration/ssh/password.yml + hosts: test-targets[0] + gather_facts: false + vars: + ansible_user: mitogen__user1 + tasks: + - meta: reset_connection + - name: ansible_password + vars: + ansible_password: user1_password + ping: + + - meta: reset_connection + - name: ansible_ssh_pass + vars: + ansible_ssh_pass: user1_password + ping: + + - meta: reset_connection + - name: absent password should fail + ping: + ignore_errors: true + ignore_unreachable: true + register: ssh_no_password_result + - assert: + that: + - ssh_no_password_result.unreachable == True + fail_msg: ssh_no_password_result={{ ssh_no_password_result }} + + - meta: reset_connection + - name: ansible_ssh_pass should override ansible_password + ping: + vars: + ansible_password: wrong + ansible_ssh_pass: user1_password + + # Tests that ansible_ssh_pass has priority over ansible_password + # and that a wrong password causes a target to be marked unreachable. + - meta: reset_connection + - name: ansible_password should not override + vars: + ansible_password: user1_password + ansible_ssh_pass: wrong + ping: + ignore_errors: true + ignore_unreachable: true + register: ssh_wrong_password_result + - assert: + that: + - ssh_wrong_password_result.unreachable == True + fail_msg: ssh_wrong_password_result={{ ssh_wrong_password_result }} diff --git a/tests/ansible/integration/ssh/variables.yml b/tests/ansible/integration/ssh/variables.yml index d2fa683b..9f5b16bc 100644 --- a/tests/ansible/integration/ssh/variables.yml +++ b/tests/ansible/integration/ssh/variables.yml @@ -13,134 +13,6 @@ -o "ControlPath /tmp/mitogen-ansible-test-{{18446744073709551615|random}}" tasks: - - include_tasks: ../_mitogen_only.yml - - - name: ansible_ssh_user, ansible_ssh_pass - shell: > - ANSIBLE_ANY_ERRORS_FATAL=false - ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" - ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" - ansible -m shell -a whoami - {% for inv in ansible_inventory_sources %} - -i "{{ inv }}" - {% endfor %} - test-targets - -e ansible_ssh_user=mitogen__has_sudo - -e ansible_ssh_pass=has_sudo_password - args: - chdir: ../.. - register: out - - - name: ansible_ssh_user, wrong ansible_ssh_pass - shell: > - ANSIBLE_ANY_ERRORS_FATAL=false - ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" - ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" - ansible -m shell -a whoami - {% for inv in ansible_inventory_sources %} - -i "{{ inv }}" - {% endfor %} - test-targets - -e ansible_ssh_user=mitogen__has_sudo - -e ansible_ssh_pass=wrong_password - -e ansible_python_interpreter=python3000 - args: - chdir: ../.. - register: out - ignore_errors: true - - - assert: - that: - - out.rc == 4 # ansible.executor.task_queue_manager.TaskQueueManager.RUN_UNREACHABLE_HOSTS - fail_msg: out={{out}} - - - - name: ansible_user, ansible_ssh_pass - shell: > - ANSIBLE_ANY_ERRORS_FATAL=false - ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" - ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" - ansible -m shell -a whoami - {% for inv in ansible_inventory_sources %} - -i "{{ inv }}" - {% endfor %} - test-targets - -e ansible_user=mitogen__has_sudo - -e ansible_ssh_pass=has_sudo_password - args: - chdir: ../.. - register: out - - - name: ansible_user, wrong ansible_ssh_pass - shell: > - ANSIBLE_ANY_ERRORS_FATAL=false - ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" - ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" - ansible -m shell -a whoami - {% for inv in ansible_inventory_sources %} - -i "{{ inv }}" - {% endfor %} - test-targets - -e ansible_user=mitogen__has_sudo - -e ansible_ssh_pass=wrong_password - -e ansible_python_interpreter=python3000 - args: - chdir: ../.. - register: out - ignore_errors: true - - - assert: - that: - - out.rc == 4 # ansible.executor.task_queue_manager.TaskQueueManager.RUN_UNREACHABLE_HOSTS - fail_msg: out={{out}} - - - - name: ansible_user, ansible_password - shell: > - ANSIBLE_ANY_ERRORS_FATAL=false - ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" - ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" - ansible -m shell -a whoami - {% for inv in ansible_inventory_sources %} - -i "{{ inv }}" - {% endfor %} - test-targets - -e ansible_user=mitogen__has_sudo - -e ansible_password=has_sudo_password - args: - chdir: ../.. - register: out - - - name: ansible_user, wrong ansible_password - shell: > - ANSIBLE_ANY_ERRORS_FATAL=false - ANSIBLE_STRATEGY=mitogen_linear - ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" - ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" - ansible -m shell -a whoami - {% for inv in ansible_inventory_sources %} - -i "{{ inv }}" - {% endfor %} - test-targets - -e ansible_user=mitogen__has_sudo - -e ansible_password=wrong_password - -e ansible_python_interpreter=python3000 - args: - chdir: ../.. - register: out - ignore_errors: true - - - assert: - that: - - out.rc == 4 # ansible.executor.task_queue_manager.TaskQueueManager.RUN_UNREACHABLE_HOSTS - fail_msg: out={{out}} - - - name: setup ansible_ssh_private_key_file shell: chmod 0600 ../data/docker/mitogen__has_sudo_pubkey.key args: From be288ad3985787f4e42132da4ede2f68b3fe05a6 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 22 Aug 2022 13:01:51 +0200 Subject: [PATCH 22/46] patch #509 : ansible_ssh_common_args issues --- ansible_mitogen/transport_config.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index 3ab623f8..144de563 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -498,12 +498,13 @@ class PlayContextSpec(Spec): ) def ssh_args(self): + local_vars = self._task_vars.get("hostvars", {}).get(self._inventory_name, {}) return [ mitogen.core.to_text(term) for s in ( - C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})), - C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})), - C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})) + C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=local_vars), + C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=local_vars), + C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=local_vars) ) for term in ansible.utils.shlex.shlex_split(s or '') ] @@ -738,12 +739,13 @@ class MitogenViaSpec(Spec): ) def ssh_args(self): + local_vars = self._task_vars.get("hostvars", {}).get(self._inventory_name, {}) return [ mitogen.core.to_text(term) for s in ( - C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})), - C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})), - C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})) + C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=local_vars), + C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=local_vars), + C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=local_vars) ) for term in ansible.utils.shlex.shlex_split(s) if s From f3097b5743d543720d9416f490f4ce2b7be952d0 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 5 Sep 2024 17:50:09 +0100 Subject: [PATCH 23/46] ci: Template Ansible test-targets inventory with Jinja2 --- .ci/ansible_tests.py | 44 ++++++++----------------- tests/ansible/templates/test-targets.j2 | 28 ++++++++++++++++ 2 files changed, 42 insertions(+), 30 deletions(-) create mode 100644 tests/ansible/templates/test-targets.j2 diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 0dd978c4..102eda9c 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -6,11 +6,13 @@ import glob import os import signal import sys -import textwrap + +import jinja2 import ci_lib +TEMPLATES_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible/templates') TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible') HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts') @@ -52,37 +54,19 @@ with ci_lib.Fold('job_setup'): distros[container['distro']].append(container['name']) families[container['family']].append(container['name']) + jinja_env = jinja2.Environment( + loader=jinja2.FileSystemLoader(searchpath=TEMPLATES_DIR), + lstrip_blocks=True, # Remove spaces and tabs from before a block + trim_blocks=True, # Remove first newline after a block + ) + inventory_template = jinja_env.get_template('test-targets.j2') inventory_path = os.path.join(HOSTS_DIR, 'target') + with open(inventory_path, 'w') as fp: - fp.write('[test-targets]\n') - fp.writelines( - "%(name)s " - "ansible_host=%(hostname)s " - "ansible_port=%(port)s " - "ansible_python_interpreter=%(python_path)s " - "ansible_user=mitogen__has_sudo_nopw " - "ansible_password=has_sudo_nopw_password" - "\n" - % container - for container in containers - ) - - for distro, hostnames in sorted(distros.items(), key=lambda t: t[0]): - fp.write('\n[%s]\n' % distro) - fp.writelines('%s\n' % name for name in hostnames) - - for family, hostnames in sorted(families.items(), key=lambda t: t[0]): - fp.write('\n[%s]\n' % family) - fp.writelines('%s\n' % name for name in hostnames) - - fp.write(textwrap.dedent( - ''' - [linux:children] - test-targets - - [linux_containers:children] - test-targets - ''' + fp.write(inventory_template.render( + containers=containers, + distros=distros, + families=families, )) ci_lib.dump_file(inventory_path) diff --git a/tests/ansible/templates/test-targets.j2 b/tests/ansible/templates/test-targets.j2 new file mode 100644 index 00000000..f61a3c78 --- /dev/null +++ b/tests/ansible/templates/test-targets.j2 @@ -0,0 +1,28 @@ +[test-targets] +{% for c in containers %} +{{ c.name }} ansible_host={{ c.hostname }} ansible_port={{ c.port }} ansible_python_interpreter={{ c.python_path }} +{% endfor %} + +[test-targets:vars] +ansible_user=mitogen__has_sudo_nopw +ansible_password=has_sudo_nopw_password + +{% for distro, hostnames in distros | dictsort %} +[{{ distro }}] +{% for hostname in hostnames %} +{{ hostname }} +{% endfor %} +{% endfor %} + +{% for family, hostnames in families | dictsort %} +[{{ family }}] +{% for hostname in hostnames %} +{{ hostname }} +{% endfor %} +{% endfor %} + +[linux:children] +test-targets + +[linux_containers:children] +test-targets From 79ed797badd039f1ba719f99a1ac2ebf8940798c Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 6 Sep 2024 11:51:58 +0100 Subject: [PATCH 24/46] tests: Test templating of ansible_ssh_common_args et al refs #905 --- docs/changelog.rst | 3 ++ docs/contributors.rst | 1 + tests/ansible/hosts/default.hosts | 7 ++++ tests/ansible/integration/ssh/all.yml | 1 + tests/ansible/integration/ssh/args.yml | 48 +++++++++++++++++++++++++ tests/ansible/templates/test-targets.j2 | 11 ++++++ 6 files changed, 71 insertions(+) create mode 100644 tests/ansible/integration/ssh/args.yml diff --git a/docs/changelog.rst b/docs/changelog.rst index 4fc177c1..8cc24424 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -26,6 +26,9 @@ Unreleased * :gh:issue:`1110` Fix :exc:`mitogen.core.StreamError` when Ansible copy module is called with a file larger than 124 kibibytes (:data:`ansible_mitogen.connection.Connection.SMALL_FILE_LIMIT`) +* :gh:issue:`905` Initial support for templated ``ansible_ssh_args``, + ``ansible_ssh_common_args``, and ``ansible_ssh_extra_args`` variables. + NB: play or task scoped variables will probably still fail. v0.3.9 (2024-08-13) diff --git a/docs/contributors.rst b/docs/contributors.rst index 1a590869..e40607a0 100644 --- a/docs/contributors.rst +++ b/docs/contributors.rst @@ -116,6 +116,7 @@ sponsorship and outstanding future-thinking of its early adopters.
    • Alex Willmer
      • +
    • Christian Bourgeois
    • Dan Dorman — - When I truly understand my enemy … then in that very moment I also love him.
    • Daniel Foerster
    • Deps — Private Maven Repository Hosting for Java, Scala, Groovy, Clojure
      • diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts index 4f5ea4c6..adc271e2 100644 --- a/tests/ansible/hosts/default.hosts +++ b/tests/ansible/hosts/default.hosts @@ -12,3 +12,10 @@ target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}" target [linux_containers] + +[issue905] +ssh-common-args ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}" + +[issue905:vars] +ansible_ssh_common_args=-o PermitLocalCommand=yes -o LocalCommand="touch {{ ssh_args_canary_file }}" +ssh_args_canary_file=/tmp/ssh_args_{{ inventory_hostname }} diff --git a/tests/ansible/integration/ssh/all.yml b/tests/ansible/integration/ssh/all.yml index 1b0f36e4..5c16f187 100644 --- a/tests/ansible/integration/ssh/all.yml +++ b/tests/ansible/integration/ssh/all.yml @@ -1,3 +1,4 @@ +- import_playbook: args.yml - import_playbook: config.yml - import_playbook: password.yml - import_playbook: timeouts.yml diff --git a/tests/ansible/integration/ssh/args.yml b/tests/ansible/integration/ssh/args.yml new file mode 100644 index 00000000..5892b5fe --- /dev/null +++ b/tests/ansible/integration/ssh/args.yml @@ -0,0 +1,48 @@ +- name: integration/ssh/args.yml + hosts: issue905 + gather_facts: false + tasks: + # Test that ansible_ssh_common_args are templated; ansible_ssh_args & + # ansible_ssh_extra_args aren't directly tested, we assume they're similar. + # FIXME This test currently relies on variables set in the host group. + # Ideally they'd be set here, and the host group eliminated, but + # Mitogen currently fails to template when defined in the play. + # TODO Replace LocalCommand canary with SetEnv canary, to simplify test. + # Requires modification of sshd_config files to add AcceptEnv ... + - name: Test templating of ansible_ssh_common_args et al + block: + - name: Ensure no lingering canary files + file: + path: "{{ ssh_args_canary_file }}" + state: absent + delegate_to: localhost + + - name: Reset connections to force new ssh execution + meta: reset_connection + + - name: Perform SSH connection, to trigger side effect + ping: + + # LocalCommand="touch {{ ssh_args_canary_file }}" in ssh_*_args + - name: Stat for canary file created by side effect + stat: + path: "{{ ssh_args_canary_file }}" + delegate_to: localhost + register: ssh_args_canary_stat + + - assert: + that: + - ssh_args_canary_stat.stat.exists == true + quiet: true + success_msg: "Canary found: {{ ssh_args_canary_file }}" + fail_msg: | + ssh_args_canary_file={{ ssh_args_canary_file }} + ssh_args_canary_stat={{ ssh_args_canary_stat }} + always: + - name: Cleanup canary files + file: + path: "{{ ssh_args_canary_file }}" + state: absent + delegate_to: localhost + tags: + - issue_905 diff --git a/tests/ansible/templates/test-targets.j2 b/tests/ansible/templates/test-targets.j2 index f61a3c78..e2708192 100644 --- a/tests/ansible/templates/test-targets.j2 +++ b/tests/ansible/templates/test-targets.j2 @@ -26,3 +26,14 @@ test-targets [linux_containers:children] test-targets + +[issue905] +{% for c in containers[:1] %} +ssh-common-args ansible_host={{ c.hostname }} ansible_port={{ c.port }} ansible_python_interpreter={{ c.python_path }} +{% endfor %} + +[issue905:vars] +ansible_user=mitogen__has_sudo_nopw +ansible_password=has_sudo_nopw_password +ansible_ssh_common_args=-o PermitLocalCommand=yes -o LocalCommand="touch {{ '{{' }} ssh_args_canary_file {{ '}}' }}" +ssh_args_canary_file=/tmp/ssh_args_{{ '{{' }} inventory_hostname {{ '}}' }} From f76ccbf8ed0d5468837574ebc23f0f39d58ea915 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 3 May 2024 22:08:14 +0100 Subject: [PATCH 25/46] tests: Unpin versions of Ansible 2.10, 3, & 4 --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 08f4c371..870a6345 100644 --- a/tox.ini +++ b/tox.ini @@ -74,9 +74,9 @@ basepython = deps = -r{toxinidir}/tests/requirements.txt mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt - ansible2.10: ansible==2.10.7 - ansible3: ansible==3.4.0 - ansible4: ansible==4.10.0 + ansible2.10: ansible~=2.10.0 + ansible3: ansible~=3.0 + ansible4: ansible~=4.0 ansible5: ansible~=5.0 ansible6: ansible~=6.0 ansible7: ansible~=7.0 From 4b4bfdc0f3a677d3523211b14e669ed539a7f423 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 6 Sep 2024 21:43:43 +0100 Subject: [PATCH 26/46] ci: Drop macOS Python 3.12 + Ansible 9 tests They were meant to be replaced by Python 3.12 + ANsible 10, not supplemented. --- .ci/azure-pipelines.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index a3ffa9b4..0bf4556b 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -28,10 +28,6 @@ jobs: matrix: Mito_312: tox.env: py312-mode_mitogen - Loc_312_9: - tox.env: py312-mode_localhost-ansible9 - Van_312_9: - tox.env: py312-mode_localhost-ansible9-strategy_linear Loc_312_10: tox.env: py312-mode_localhost-ansible10 Van_312_10: From 7d9eebdb9a548fbaff9163a6afce250f8777c2a3 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 10 Sep 2024 15:44:17 +0100 Subject: [PATCH 27/46] tests: Close file object in six_brokenpkg --- tests/data/importer/six_brokenpkg/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/importer/six_brokenpkg/__init__.py b/tests/data/importer/six_brokenpkg/__init__.py index e5944b83..32356972 100644 --- a/tests/data/importer/six_brokenpkg/__init__.py +++ b/tests/data/importer/six_brokenpkg/__init__.py @@ -53,4 +53,4 @@ if _system_six: else: from . import _six as six six_py_file = '{0}.py'.format(os.path.splitext(six.__file__)[0]) -exec(open(six_py_file, 'rb').read()) +with open(six_py_file, 'rb') as f: exec(f.read()) From 7c92b8ef2bfa418ef7501297826d641c4d8d2d04 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 10 Sep 2024 15:47:01 +0100 Subject: [PATCH 28/46] tests: Shutdown contexts on completion This should terminate any child processes and connections. --- tests/id_allocation_test.py | 3 +++ tests/ssh_test.py | 1 + 2 files changed, 4 insertions(+) diff --git a/tests/id_allocation_test.py b/tests/id_allocation_test.py index 850a68a5..91ff3d4b 100644 --- a/tests/id_allocation_test.py +++ b/tests/id_allocation_test.py @@ -27,3 +27,6 @@ class SlaveTest(testlib.RouterMixin, testlib.TestCase): # Subsequent master allocation does not collide c2 = self.router.local() self.assertEqual(1002, c2.context_id) + + context.shutdown() + c2.shutdown() diff --git a/tests/ssh_test.py b/tests/ssh_test.py index 3149fcbc..ce7dce96 100644 --- a/tests/ssh_test.py +++ b/tests/ssh_test.py @@ -190,6 +190,7 @@ class BannerTest(testlib.DockerMixin, testlib.TestCase): self.dockerized_ssh.port, ) self.assertEqual(name, context.name) + context.shutdown(wait=True) class StubPermissionDeniedTest(StubSshMixin, testlib.TestCase): From 72384033920d541199a882624d110d7977c43eb7 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 10 Sep 2024 15:47:55 +0100 Subject: [PATCH 29/46] tests: Add missing logging import --- tests/connection_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/connection_test.py b/tests/connection_test.py index 5c3e678d..6cfa384c 100644 --- a/tests/connection_test.py +++ b/tests/connection_test.py @@ -1,3 +1,4 @@ +import logging import os import signal import sys From 598de81143a629dffab45b1602b8dcbb06646f5b Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 10 Sep 2024 16:02:49 +0100 Subject: [PATCH 30/46] mitogen: Fix subprocess ResourceWarning Python 3.x emits `ResourceWarning`s if certains resources aren't correctly closed. Due to the way Mitogen has been terminating child processes this has been occurring. ``` test_dev_tty_open_succeeds (create_child_test.TtyCreateChildTest.test_dev_tty_open_succeeds) ... /opt/hostedtoolcache/Python/3.12.5/x64/lib/python3.12/subprocess.py:1127: ResourceWarning: subprocess 3313 is still running _warn("subprocess %s is still running" % self.pid, ResourceWarning: Enable tracemalloc to get the object allocation traceback ok ``` During garbage collection subprocess.Popen() objects emit ResourceWarning("subprocess 123 is still running") if proc.returncode hasn't been set. Typically calling proc.wait() does so, once the sub-process has exited. Calling os.waitpid(proc.pid, 0) also waits for the sub-process to exit, but it doesn't update proc.returncode, so the ResourceWarning is still emitted. This change exposes `subprocess.Popen` methods on `mitogen.parent.PopenProcess`, so that the returncode can be set. See https://gist.github.com/moreati/b8d157ff82cb15234bece4033accc5e5 --- mitogen/parent.py | 16 +++++++++++++++- tests/connection_test.py | 8 +++++--- tests/create_child_test.py | 1 + tests/reaper_test.py | 20 +++++++++----------- 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index 4b96dcf4..2a43cad2 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -2542,7 +2542,7 @@ class Reaper(object): # because it is setuid, so this is best-effort only. LOG.debug('%r: sending %s', self.proc, SIGNAL_BY_NUM[signum]) try: - os.kill(self.proc.pid, signum) + self.proc.send_signal(signum) except OSError: e = sys.exc_info()[1] if e.args[0] != errno.EPERM: @@ -2662,6 +2662,17 @@ class Process(object): """ raise NotImplementedError() + def send_signal(self, sig): + os.kill(self.pid, sig) + + def terminate(self): + "Ask the process to gracefully shutdown." + self.send_signal(signal.SIGTERM) + + def kill(self): + "Ask the operating system to forcefully destroy the process." + self.send_signal(signal.SIGKILL) + class PopenProcess(Process): """ @@ -2678,6 +2689,9 @@ class PopenProcess(Process): def poll(self): return self.proc.poll() + def send_signal(self, sig): + self.proc.send_signal(sig) + class ModuleForwarder(object): """ diff --git a/tests/connection_test.py b/tests/connection_test.py index 6cfa384c..c3146954 100644 --- a/tests/connection_test.py +++ b/tests/connection_test.py @@ -55,7 +55,9 @@ def do_detach(econtext): class DetachReapTest(testlib.RouterMixin, testlib.TestCase): def test_subprocess_preserved_on_shutdown(self): c1 = self.router.local() + c1_stream = self.router.stream_by_id(c1.context_id) pid = c1.call(os.getpid) + self.assertEqual(pid, c1_stream.conn.proc.pid) l = mitogen.core.Latch() mitogen.core.listen(c1, 'disconnect', l.put) @@ -65,8 +67,8 @@ class DetachReapTest(testlib.RouterMixin, testlib.TestCase): self.broker.shutdown() self.broker.join() - os.kill(pid, 0) # succeeds if process still alive + self.assertIsNone(os.kill(pid, 0)) # succeeds if process still alive # now clean up - os.kill(pid, signal.SIGTERM) - os.waitpid(pid, 0) + c1_stream.conn.proc.terminate() + c1_stream.conn.proc.proc.wait() diff --git a/tests/create_child_test.py b/tests/create_child_test.py index acf3ea66..57b04b3f 100644 --- a/tests/create_child_test.py +++ b/tests/create_child_test.py @@ -76,6 +76,7 @@ def close_proc(proc): proc.stdout.close() if proc.stderr: proc.stderr.close() + proc.proc.wait() def wait_read(fp, n): diff --git a/tests/reaper_test.py b/tests/reaper_test.py index 8588a1bc..560d48ff 100644 --- a/tests/reaper_test.py +++ b/tests/reaper_test.py @@ -10,8 +10,7 @@ import mitogen.parent class ReaperTest(testlib.TestCase): - @mock.patch('os.kill') - def test_calc_delay(self, kill): + def test_calc_delay(self): broker = mock.Mock() proc = mock.Mock() proc.poll.return_value = None @@ -24,8 +23,7 @@ class ReaperTest(testlib.TestCase): self.assertEqual(752, int(1000 * reaper._calc_delay(5))) self.assertEqual(1294, int(1000 * reaper._calc_delay(6))) - @mock.patch('os.kill') - def test_reap_calls(self, kill): + def test_reap_calls(self): broker = mock.Mock() proc = mock.Mock() proc.poll.return_value = None @@ -33,20 +31,20 @@ class ReaperTest(testlib.TestCase): reaper = mitogen.parent.Reaper(broker, proc, True, True) reaper.reap() - self.assertEqual(0, kill.call_count) + self.assertEqual(0, proc.send_signal.call_count) reaper.reap() - self.assertEqual(1, kill.call_count) + self.assertEqual(1, proc.send_signal.call_count) reaper.reap() reaper.reap() reaper.reap() - self.assertEqual(1, kill.call_count) + self.assertEqual(1, proc.send_signal.call_count) reaper.reap() - self.assertEqual(2, kill.call_count) + self.assertEqual(2, proc.send_signal.call_count) - self.assertEqual(kill.mock_calls, [ - mock.call(proc.pid, signal.SIGTERM), - mock.call(proc.pid, signal.SIGKILL), + self.assertEqual(proc.send_signal.mock_calls, [ + mock.call(signal.SIGTERM), + mock.call(signal.SIGKILL), ]) From a3192d2bebc0e38a3f48c13a14cdd4d052a72b3d Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 10 Sep 2024 16:05:28 +0100 Subject: [PATCH 31/46] mitogen: close mitogen.unix.Listener socket in error conditions To avoid ResourceWarning --- mitogen/unix.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mitogen/unix.py b/mitogen/unix.py index 1af1c0ec..b241a403 100644 --- a/mitogen/unix.py +++ b/mitogen/unix.py @@ -143,19 +143,23 @@ class Listener(mitogen.core.Protocol): def on_accept_client(self, sock): sock.setblocking(True) try: - pid, = struct.unpack('>L', sock.recv(4)) + data = sock.recv(4) + pid, = struct.unpack('>L', data) except (struct.error, socket.error): - LOG.error('listener: failed to read remote identity: %s', - sys.exc_info()[1]) + LOG.error('listener: failed to read remote identity, got %d bytes: %s', + len(data), sys.exc_info()[1]) + sock.close() return context_id = self._router.id_allocator.allocate() try: + # FIXME #1109 send() returns number of bytes sent, check it sock.send(struct.pack('>LLL', context_id, mitogen.context_id, os.getpid())) except socket.error: LOG.error('listener: failed to assign identity to PID %d: %s', pid, sys.exc_info()[1]) + sock.close() return context = mitogen.parent.Context(self._router, context_id) From 315204271e3fd58a6db8981e48629986cd451f4a Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 10 Sep 2024 16:07:53 +0100 Subject: [PATCH 32/46] tests: Don't suppress output while testing unix Listener It's not noisy, and it has been hiding an error I wasn't aware of. --- tests/unix_test.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/tests/unix_test.py b/tests/unix_test.py index 14fc54ae..e251a7ad 100644 --- a/tests/unix_test.py +++ b/tests/unix_test.py @@ -65,17 +65,13 @@ class ListenerTest(testlib.RouterMixin, testlib.TestCase): def test_constructor_basic(self): listener = self.klass.build_stream(router=self.router) - capture = testlib.LogCapturer() - capture.start() - try: - self.assertFalse(mitogen.unix.is_path_dead(listener.protocol.path)) - os.unlink(listener.protocol.path) - # ensure we catch 0 byte read error log message - self.broker.shutdown() - self.broker.join() - self.broker_shutdown = True - finally: - capture.stop() + self.assertFalse(mitogen.unix.is_path_dead(listener.protocol.path)) + os.unlink(listener.protocol.path) + + # ensure we catch 0 byte read error log message + self.broker.shutdown() + self.broker.join() + self.broker_shutdown = True class ClientTest(testlib.TestCase): From d032c591c2f861a66430ac0a2741be1b3b118198 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 10 Sep 2024 16:12:47 +0100 Subject: [PATCH 33/46] tests: Retry container process check during teardown I'm about 75% sure the check is an unavoidable race condition, see https://github.com/mitogen-hq/mitogen/issues/694#issuecomment-2338001694. If it occurs again, then reopen the issue. Fixes #694 --- docs/changelog.rst | 2 ++ tests/testlib.py | 34 ++++++++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8cc24424..14f86e77 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -29,6 +29,8 @@ Unreleased * :gh:issue:`905` Initial support for templated ``ansible_ssh_args``, ``ansible_ssh_common_args``, and ``ansible_ssh_extra_args`` variables. NB: play or task scoped variables will probably still fail. +* :gh:issue:`694` CI: Fixed a race condition and some resource leaks causing + some of intermittent failures when running the test suite. v0.3.9 (2024-08-13) diff --git a/tests/testlib.py b/tests/testlib.py index 8c40e7ff..a52292ce 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -146,6 +146,17 @@ def data_path(suffix): return path +def retry(fn, on, max_attempts, delay): + for i in range(max_attempts): + try: + return fn() + except on: + if i >= max_attempts - 1: + raise + else: + time.sleep(delay) + + def threading__thread_is_alive(thread): """Return whether the thread is alive (Python version compatibility shim). @@ -562,18 +573,24 @@ class DockerizedSshDaemon(object): wait_for_port(self.get_host(), self.port, pattern='OpenSSH') def check_processes(self): - args = ['docker', 'exec', self.container_name, 'ps', '-o', 'comm='] + # Get Accounting name (ucomm) & command line (args) of each process + # in the container. No truncation (-ww). No column headers (foo=). + ps_output = subprocess.check_output([ + 'docker', 'exec', self.container_name, + 'ps', '-w', '-w', '-o', 'ucomm=', '-o', 'args=', + ]) + ps_lines = ps_output.decode().splitlines() + processes = [tuple(line.split(None, 1)) for line in ps_lines] counts = {} - for comm in subprocess.check_output(args).decode().splitlines(): - comm = comm.strip() - counts[comm] = counts.get(comm, 0) + 1 + for ucomm, _ in processes: + counts[ucomm] = counts.get(ucomm, 0) + 1 if counts != {'ps': 1, 'sshd': 1}: assert 0, ( 'Docker container %r contained extra running processes ' 'after test completed: %r' % ( self.container_name, - counts + processes, ) ) @@ -630,7 +647,12 @@ class DockerMixin(RouterMixin): @classmethod def tearDownClass(cls): - cls.dockerized_ssh.check_processes() + retry( + cls.dockerized_ssh.check_processes, + on=AssertionError, + max_attempts=5, + delay=0.1, + ) cls.dockerized_ssh.close() super(DockerMixin, cls).tearDownClass() From 2ba1b2b3f86814498bdc48cdfb77872a55c15124 Mon Sep 17 00:00:00 2001 From: Gaige B Paulsen Date: Fri, 20 Sep 2024 15:13:33 -0400 Subject: [PATCH 34/46] Fix: termios.error: (22, 'Invalid argument') during `become` on Solaris/Illumos/SmartOS (#1089) This fixes compatibility with Solaris/Illumos/SmartOS, addressing an issue that shows up most frequently with become. The issue was mostly due to differences in how the TTY driver is handled and the pty driver not supporting echo on both sides of the pipe (as designed, from a Solaris point of view). Fixes #950 Co-authored-by: Alex Willmer --- docs/changelog.rst | 1 + mitogen/parent.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 14f86e77..1d8deb92 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,7 @@ To avail of fixes in an unreleased version, please download a ZIP file Unreleased ---------- +* :gh:issue:`950` Fix Solaris/Illumos/SmartOS compatibility with become * :gh:issue:`1087` Fix :exc:`mitogen.core.StreamError` when Ansible template module is called with a ``dest:`` filename that has an extension * :gh:issue:`1110` Fix :exc:`mitogen.core.StreamError` when Ansible copy diff --git a/mitogen/parent.py b/mitogen/parent.py index 2a43cad2..2ed7e8ba 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -147,6 +147,8 @@ LINUX_TIOCGPTN = _ioctl_cast(2147767344) LINUX_TIOCSPTLCK = _ioctl_cast(1074025521) IS_LINUX = os.uname()[0] == 'Linux' +IS_SOLARIS = os.uname()[0] == 'SunOS' + SIGNAL_BY_NUM = dict( (getattr(signal, name), name) @@ -411,7 +413,7 @@ def _acquire_controlling_tty(): # On Linux, the controlling tty becomes the first tty opened by a # process lacking any prior tty. os.close(os.open(os.ttyname(2), os.O_RDWR)) - if hasattr(termios, 'TIOCSCTTY') and not mitogen.core.IS_WSL: + if hasattr(termios, 'TIOCSCTTY') and not mitogen.core.IS_WSL and not IS_SOLARIS: # #550: prehistoric WSL does not like TIOCSCTTY. # On BSD an explicit ioctl is required. For some inexplicable reason, # Python 2.6 on Travis also requires it. @@ -479,7 +481,8 @@ def openpty(): master_fp = os.fdopen(master_fd, 'r+b', 0) slave_fp = os.fdopen(slave_fd, 'r+b', 0) - disable_echo(master_fd) + if not IS_SOLARIS: + disable_echo(master_fd) disable_echo(slave_fd) mitogen.core.set_block(slave_fd) return master_fp, slave_fp From cea2e7b98dc9e255b87f64471ed808a8e004afc1 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 20 Sep 2024 20:24:31 +0100 Subject: [PATCH 35/46] Prepare v0.3.10 --- docs/changelog.rst | 4 ++-- mitogen/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1d8deb92..ee4faafc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,8 +18,8 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. -Unreleased ----------- +v0.3.10 (2024-09-20) +-------------------- * :gh:issue:`950` Fix Solaris/Illumos/SmartOS compatibility with become * :gh:issue:`1087` Fix :exc:`mitogen.core.StreamError` when Ansible template diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 1d483ead..9c1ddc6c 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 10, 'dev') +__version__ = (0, 3, 10) #: This is :data:`False` in slave contexts. Previously it was used to prevent From ce6297b0e95d4e155a265cb5365d230f7bab53aa Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 20 Sep 2024 20:25:51 +0100 Subject: [PATCH 36/46] Begin v0.3.11 --- docs/changelog.rst | 5 +++++ mitogen/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ee4faafc..0825f1f7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,11 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. +Unreleased +---------- + + + v0.3.10 (2024-09-20) -------------------- diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 9c1ddc6c..6ceb60db 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 10) +__version__ = (0, 3, 11, 'dev') #: This is :data:`False` in slave contexts. Previously it was used to prevent From c6cf08ab39738a7317ba4379088c6357642b1c12 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 24 Sep 2024 16:33:14 +0100 Subject: [PATCH 37/46] mitogen: Consolidate back compatibility fallbacks and polyfills in mitogen.core This saves some bytes on the wire ad simplifies reasoning about the code. --- ansible_mitogen/target.py | 4 +--- docs/changelog.rst | 2 ++ mitogen/core.py | 48 ++++++++++++++++++--------------------- mitogen/master.py | 13 ++--------- mitogen/parent.py | 17 +++----------- mitogen/service.py | 10 +------- mitogen/ssh.py | 5 ---- mitogen/su.py | 5 ---- mitogen/utils.py | 8 +------ tests/iter_split_test.py | 6 +---- tests/lxc_test.py | 5 +--- tests/poller_test.py | 8 ++----- 12 files changed, 36 insertions(+), 95 deletions(-) diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index 7d907d62..21eae594 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -746,9 +746,7 @@ def set_file_mode(path, spec, fd=None): """ Update the permissions of a file using the same syntax as chmod(1). """ - if isinstance(spec, int): - new_mode = spec - elif not mitogen.core.PY3 and isinstance(spec, long): + if isinstance(spec, mitogen.core.integer_types): new_mode = spec elif spec.isdigit(): new_mode = int(spec, 8) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0825f1f7..c1017601 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,8 @@ To avail of fixes in an unreleased version, please download a ZIP file Unreleased ---------- +* :gh:issue:`1127` :mod:`mitogen`: Consolidate mitogen backward compatibility + fallbacks and polyfills into :mod:`mitogen.core` v0.3.10 (2024-09-20) diff --git a/mitogen/core.py b/mitogen/core.py index cdfbbcde..9b225ed7 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -102,21 +102,6 @@ try: except ImportError: cProfile = None -try: - import thread -except ImportError: - import threading as thread - -try: - import cPickle as pickle -except ImportError: - import pickle - -try: - from cStringIO import StringIO as BytesIO -except ImportError: - from io import BytesIO - try: BaseException except NameError: @@ -169,31 +154,35 @@ STUB_CALL_SERVICE = 111 #: :meth:`mitogen.core.Router.add_handler` callbacks to clean up. IS_DEAD = 999 -try: - BaseException -except NameError: - BaseException = Exception - PY24 = sys.version_info < (2, 5) PY3 = sys.version_info > (3,) if PY3: + import pickle + import _thread as thread + from io import BytesIO b = str.encode BytesType = bytes UnicodeType = str FsPathTypes = (str,) BufferType = lambda buf, start: memoryview(buf)[start:] - long = int + integer_types = (int,) + iteritems, iterkeys, itervalues = dict.items, dict.keys, dict.values else: + import cPickle as pickle + import thread + from cStringIO import StringIO as BytesIO b = str BytesType = str FsPathTypes = (str, unicode) BufferType = buffer UnicodeType = unicode + integer_types = (int, long) + iteritems, iterkeys, itervalues = dict.iteritems, dict.iterkeys, dict.itervalues AnyTextType = (BytesType, UnicodeType) try: - next + next = next except NameError: next = lambda it: it.next() @@ -400,12 +389,19 @@ now = getattr(time, 'monotonic', time.time) # Python 2.4 try: - any + all, any = all, any except NameError: + def all(it): + for elem in it: + if not elem: + return False + return True + def any(it): for elem in it: if elem: return True + return False def _partition(s, sep, find): @@ -1065,8 +1061,8 @@ class Sender(object): def _unpickle_sender(router, context_id, dst_handle): if not (isinstance(router, Router) and - isinstance(context_id, (int, long)) and context_id >= 0 and - isinstance(dst_handle, (int, long)) and dst_handle > 0): + isinstance(context_id, integer_types) and context_id >= 0 and + isinstance(dst_handle, integer_types) and dst_handle > 0): raise TypeError('cannot unpickle Sender: bad input or missing router') return Sender(Context(router, context_id), dst_handle) @@ -2508,7 +2504,7 @@ class Context(object): def _unpickle_context(context_id, name, router=None): - if not (isinstance(context_id, (int, long)) and context_id >= 0 and ( + if not (isinstance(context_id, integer_types) and context_id >= 0 and ( (name is None) or (isinstance(name, UnicodeType) and len(name) < 100)) ): diff --git a/mitogen/master.py b/mitogen/master.py index b1e0a1de..51b29b82 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -74,9 +74,11 @@ import mitogen.core import mitogen.minify import mitogen.parent +from mitogen.core import any from mitogen.core import b from mitogen.core import IOLOG from mitogen.core import LOG +from mitogen.core import next from mitogen.core import str_partition from mitogen.core import str_rpartition from mitogen.core import to_text @@ -84,17 +86,6 @@ from mitogen.core import to_text imap = getattr(itertools, 'imap', map) izip = getattr(itertools, 'izip', zip) -try: - any -except NameError: - from mitogen.core import any - -try: - next -except NameError: - from mitogen.core import next - - RLOG = logging.getLogger('mitogen.ctx') diff --git a/mitogen/parent.py b/mitogen/parent.py index 2ed7e8ba..dd51b697 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -56,15 +56,13 @@ import zlib # Absolute imports for <2.5. select = __import__('select') -try: - import thread -except ImportError: - import threading as thread - import mitogen.core from mitogen.core import b from mitogen.core import bytes_partition from mitogen.core import IOLOG +from mitogen.core import itervalues +from mitogen.core import next +from mitogen.core import thread LOG = logging.getLogger(__name__) @@ -80,15 +78,6 @@ except IOError: SELINUX_ENABLED = False -try: - next -except NameError: - # Python 2.4/2.5 - from mitogen.core import next - - -itervalues = getattr(dict, 'itervalues', dict.values) - if mitogen.core.PY3: xrange = range closure_attr = '__closure__' diff --git a/mitogen/service.py b/mitogen/service.py index 7fde9013..ffc1085b 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -39,18 +39,10 @@ import threading import mitogen.core import mitogen.select +from mitogen.core import all from mitogen.core import b from mitogen.core import str_rpartition -try: - all -except NameError: - def all(it): - for elem in it: - if not elem: - return False - return True - LOG = logging.getLogger(__name__) diff --git a/mitogen/ssh.py b/mitogen/ssh.py index 656dc72c..f32d2cab 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -43,11 +43,6 @@ except ImportError: import mitogen.parent from mitogen.core import b -try: - any -except NameError: - from mitogen.core import any - LOG = logging.getLogger(__name__) diff --git a/mitogen/su.py b/mitogen/su.py index 080c9782..9b908460 100644 --- a/mitogen/su.py +++ b/mitogen/su.py @@ -34,11 +34,6 @@ import re import mitogen.core import mitogen.parent -try: - any -except NameError: - from mitogen.core import any - LOG = logging.getLogger(__name__) diff --git a/mitogen/utils.py b/mitogen/utils.py index 1fbf71fe..9d1c1bc9 100644 --- a/mitogen/utils.py +++ b/mitogen/utils.py @@ -37,13 +37,7 @@ import sys import mitogen.core import mitogen.master - -iteritems = getattr(dict, 'iteritems', dict.items) - -if mitogen.core.PY3: - iteritems = dict.items -else: - iteritems = dict.iteritems +from mitogen.core import iteritems def setup_gil(): diff --git a/tests/iter_split_test.py b/tests/iter_split_test.py index 74c46c0a..39c11a47 100644 --- a/tests/iter_split_test.py +++ b/tests/iter_split_test.py @@ -2,11 +2,7 @@ import unittest import mitogen.core -try: - next -except NameError: - def next(it): - return it.next() +from mitogen.core import next class IterSplitTest(unittest.TestCase): diff --git a/tests/lxc_test.py b/tests/lxc_test.py index b9ebfa53..a613cefa 100644 --- a/tests/lxc_test.py +++ b/tests/lxc_test.py @@ -3,10 +3,7 @@ import os import mitogen.lxc import mitogen.parent -try: - any -except NameError: - from mitogen.core import any +from mitogen.core import any import testlib diff --git a/tests/poller_test.py b/tests/poller_test.py index f915df0a..0abc836d 100644 --- a/tests/poller_test.py +++ b/tests/poller_test.py @@ -8,13 +8,9 @@ import unittest import mitogen.core import mitogen.parent -import testlib +from mitogen.core import next -try: - next -except NameError: - # Python 2.4 - from mitogen.core import next +import testlib class SockMixin(object): From b1fd6038bfa10decae38488a8ac2a65678fa7bbe Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 24 Sep 2024 16:50:37 +0100 Subject: [PATCH 38/46] ansible_mitogen: Remove Python 2.4 and 2.5 backward compatibility fallbacks Because ansible_mitogen >= 0.3 supports Ansible >= 2.10 and Ansible 2.10 requires supports Python >= 2.7 on controllers and Python >= 2.6 on targets these are dead weight. See - https://docs.ansible.com/ansible/latest/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix - tox.ini --- ansible_mitogen/affinity.py | 3 +- ansible_mitogen/planner.py | 2 +- ansible_mitogen/process.py | 5 +- ansible_mitogen/runner.py | 54 ++++++++----------- ansible_mitogen/target.py | 16 ++---- docs/changelog.rst | 2 + .../custom_python_detect_environment.py | 11 ---- tox.ini | 3 ++ 8 files changed, 33 insertions(+), 63 deletions(-) diff --git a/ansible_mitogen/affinity.py b/ansible_mitogen/affinity.py index 635ee7b9..223794ab 100644 --- a/ansible_mitogen/affinity.py +++ b/ansible_mitogen/affinity.py @@ -83,7 +83,6 @@ import multiprocessing import os import struct -import mitogen.core import mitogen.parent @@ -265,7 +264,7 @@ class LinuxPolicy(FixedPolicy): for x in range(16): chunks.append(struct.pack('>= 64 - return mitogen.core.b('').join(chunks) + return b''.join(chunks) def _get_thread_ids(self): try: diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 0a91039a..4cdc0f20 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -477,7 +477,7 @@ def read_file(path): finally: os.close(fd) - return mitogen.core.b('').join(bits) + return b''.join(bits) def _propagate_deps(invocation, planner, context): diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index 3a41a43d..7ec70f2a 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -61,10 +61,9 @@ import mitogen.utils import ansible import ansible.constants as C import ansible.errors + import ansible_mitogen.logging import ansible_mitogen.services - -from mitogen.core import b import ansible_mitogen.affinity @@ -639,7 +638,7 @@ class MuxProcess(object): try: # Let the parent know our listening socket is ready. - mitogen.core.io_op(self.model.child_sock.send, b('1')) + mitogen.core.io_op(self.model.child_sock.send, b'1') # Block until the socket is closed, which happens on parent exit. mitogen.core.io_op(self.model.child_sock.recv, 1) finally: diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 8da1b670..16e43059 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -40,7 +40,9 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type import atexit +import ctypes import json +import logging import os import re import shlex @@ -52,17 +54,8 @@ import types import mitogen.core import ansible_mitogen.target # TODO: circular import -from mitogen.core import b -from mitogen.core import bytes_partition -from mitogen.core import str_rpartition from mitogen.core import to_text -try: - import ctypes -except ImportError: - # Python 2.4 - ctypes = None - try: # Python >= 3.4, PEP 451 ModuleSpec API import importlib.machinery @@ -82,9 +75,6 @@ try: except ImportError: from pipes import quote as shlex_quote -# Absolute imports for <2.5. -logging = __import__('logging') - # Prevent accidental import of an Ansible module from hanging on stdin read. import ansible.module_utils.basic @@ -95,13 +85,12 @@ ansible.module_utils.basic._ANSIBLE_ARGS = '{}' # explicit call to res_init() on each task invocation. BSD-alikes export it # directly, Linux #defines it as "__res_init". libc__res_init = None -if ctypes: - libc = ctypes.CDLL(None) - for symbol in 'res_init', '__res_init': - try: - libc__res_init = getattr(libc, symbol) - except AttributeError: - pass +libc = ctypes.CDLL(None) +for symbol in 'res_init', '__res_init': + try: + libc__res_init = getattr(libc, symbol) + except AttributeError: + pass iteritems = getattr(dict, 'iteritems', dict.items) LOG = logging.getLogger(__name__) @@ -217,13 +206,13 @@ class EnvironmentFileWatcher(object): for line in fp: # ' #export foo=some var ' -> ['#export', 'foo=some var '] bits = shlex_split_b(line) - if (not bits) or bits[0].startswith(b('#')): + if (not bits) or bits[0].startswith(b'#'): continue - if bits[0] == b('export'): + if bits[0] == b'export': bits.pop(0) - key, sep, value = bytes_partition(b(' ').join(bits), b('=')) + key, sep, value = b' '.join(bits).partition(b'=') if key and sep: yield key, value @@ -596,7 +585,7 @@ class ModuleUtilsImporter(object): mod.__path__ = [] mod.__package__ = str(fullname) else: - mod.__package__ = str(str_rpartition(to_text(fullname), '.')[0]) + mod.__package__ = str(to_text(fullname).rpartition('.')[0]) exec(code, mod.__dict__) self._loaded.add(fullname) return mod @@ -819,7 +808,7 @@ class ScriptRunner(ProgramRunner): self.interpreter_fragment = interpreter_fragment self.is_python = is_python - b_ENCODING_STRING = b('# -*- coding: utf-8 -*-') + b_ENCODING_STRING = b'# -*- coding: utf-8 -*-' def _get_program(self): return self._rewrite_source( @@ -852,13 +841,13 @@ class ScriptRunner(ProgramRunner): # While Ansible rewrites the #! using ansible_*_interpreter, it is # never actually used to execute the script, instead it is a shell # fragment consumed by shell/__init__.py::build_module_command(). - new = [b('#!') + utf8(self.interpreter_fragment)] + new = [b'#!' + utf8(self.interpreter_fragment)] if self.is_python: new.append(self.b_ENCODING_STRING) - _, _, rest = bytes_partition(s, b('\n')) + _, _, rest = s.partition(b'\n') new.append(rest) - return b('\n').join(new) + return b'\n'.join(new) class NewStyleRunner(ScriptRunner): @@ -971,8 +960,7 @@ class NewStyleRunner(ScriptRunner): # change the default encoding. This hack was removed from Ansible long ago, # but not before permeating into many third party modules. PREHISTORIC_HACK_RE = re.compile( - b(r'reload\s*\(\s*sys\s*\)\s*' - r'sys\s*\.\s*setdefaultencoding\([^)]+\)') + br'reload\s*\(\s*sys\s*\)\s*sys\s*\.\s*setdefaultencoding\([^)]+\)', ) def _setup_program(self): @@ -980,7 +968,7 @@ class NewStyleRunner(ScriptRunner): context=self.service_context, path=self.path, ) - self.source = self.PREHISTORIC_HACK_RE.sub(b(''), source) + self.source = self.PREHISTORIC_HACK_RE.sub(b'', source) def _get_code(self): try: @@ -998,7 +986,7 @@ class NewStyleRunner(ScriptRunner): if mitogen.core.PY3: main_module_name = '__main__' else: - main_module_name = b('__main__') + main_module_name = b'__main__' def _handle_magic_exception(self, mod, exc): """ @@ -1030,7 +1018,7 @@ class NewStyleRunner(ScriptRunner): approximation of the original package hierarchy, so that relative imports function correctly. """ - pkg, sep, modname = str_rpartition(self.py_module_name, '.') + pkg, sep, _ = self.py_module_name.rpartition('.') if not sep: return None if mitogen.core.PY3: @@ -1073,7 +1061,7 @@ class NewStyleRunner(ScriptRunner): class JsonArgsRunner(ScriptRunner): - JSON_ARGS = b('<>') + JSON_ARGS = b'<>' def _get_args_contents(self): return json.dumps(self.args).encode() diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index 21eae594..b79dc492 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -39,6 +39,7 @@ __metaclass__ = type import errno import grp import json +import logging import operator import os import pwd @@ -51,26 +52,15 @@ import tempfile import traceback import types -# Absolute imports for <2.5. -logging = __import__('logging') - import mitogen.core import mitogen.parent import mitogen.service -from mitogen.core import b - try: reduce except NameError: # Python 3.x. from functools import reduce -try: - BaseException -except NameError: - # Python 2.4 - BaseException = Exception - # Ansible since PR #41749 inserts "import __main__" into # ansible.module_utils.basic. Mitogen's importer will refuse such an import, so @@ -615,8 +605,8 @@ def exec_args(args, in_data='', chdir=None, shell=None, emulate_tty=False): stdout, stderr = proc.communicate(in_data) if emulate_tty: - stdout = stdout.replace(b('\n'), b('\r\n')) - return proc.returncode, stdout, stderr or b('') + stdout = stdout.replace(b'\n', b'\r\n') + return proc.returncode, stdout, stderr or b'' def exec_command(cmd, in_data='', chdir=None, shell=None, emulate_tty=False): diff --git a/docs/changelog.rst b/docs/changelog.rst index c1017601..bf9c12e1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,6 +23,8 @@ Unreleased * :gh:issue:`1127` :mod:`mitogen`: Consolidate mitogen backward compatibility fallbacks and polyfills into :mod:`mitogen.core` +* :gh:issue:`1127` :mod:`ansible_mitogen`: Remove backward compatibility + fallbacks for Python 2.4 & 2.5. v0.3.10 (2024-09-20) diff --git a/tests/ansible/lib/modules/custom_python_detect_environment.py b/tests/ansible/lib/modules/custom_python_detect_environment.py index d2ceaf0a..4879ac33 100644 --- a/tests/ansible/lib/modules/custom_python_detect_environment.py +++ b/tests/ansible/lib/modules/custom_python_detect_environment.py @@ -11,17 +11,6 @@ import socket import sys -try: - all -except NameError: - # Python 2.4 - def all(it): - for elem in it: - if not elem: - return False - return True - - def main(): module = AnsibleModule(argument_spec={}) module.exit_json( diff --git a/tox.ini b/tox.ini index 870a6345..9bb82def 100644 --- a/tox.ini +++ b/tox.ini @@ -48,6 +48,9 @@ # ansible == 9.x ansible-core ~= 2.16.0 # ansible == 10.x ansible-core ~= 2.17.0 +# See also +# - https://docs.ansible.com/ansible/latest/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix + [tox] envlist = init, From 0a908d76dad880e78336c228d37f5b6626cf8290 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 24 Sep 2024 17:01:03 +0100 Subject: [PATCH 39/46] ansible_mitogen: Remove fallback imports for Ansible < 2.10 --- ansible_mitogen/logging.py | 8 +++---- ansible_mitogen/mixins.py | 33 ++++++----------------------- ansible_mitogen/transport_config.py | 20 ++++++----------- docs/changelog.rst | 2 ++ 4 files changed, 18 insertions(+), 45 deletions(-) diff --git a/ansible_mitogen/logging.py b/ansible_mitogen/logging.py index 40b2b339..4d5647a4 100644 --- a/ansible_mitogen/logging.py +++ b/ansible_mitogen/logging.py @@ -32,15 +32,13 @@ __metaclass__ = type import logging import os +import ansible.utils.display + import mitogen.core import mitogen.utils -try: - from __main__ import display -except ImportError: - import ansible.utils.display - display = ansible.utils.display.Display() +display = ansible.utils.display.Display() #: The process name set via :func:`set_process_name`. _process_name = None diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 2cd97a3e..d67174bd 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -40,13 +40,15 @@ try: except ImportError: from pipes import quote as shlex_quote -from ansible.module_utils._text import to_bytes -from ansible.parsing.utils.jsonify import jsonify - import ansible import ansible.constants import ansible.plugins import ansible.plugins.action +import ansible.utils.unsafe_proxy +import ansible.vars.clean + +from ansible.module_utils.common.text.converters import to_bytes, to_text +from ansible.parsing.utils.jsonify import jsonify import mitogen.core import mitogen.select @@ -57,24 +59,6 @@ import ansible_mitogen.target import ansible_mitogen.utils import ansible_mitogen.utils.unsafe -from ansible.module_utils._text import to_text - -try: - from ansible.utils.unsafe_proxy import wrap_var -except ImportError: - from ansible.vars.unsafe_proxy import wrap_var - -try: - # ansible 2.8 moved remove_internal_keys to the clean module - from ansible.vars.clean import remove_internal_keys -except ImportError: - try: - from ansible.vars.manager import remove_internal_keys - except ImportError: - # ansible 2.3.3 has remove_internal_keys as a protected func on the action class - # we'll fallback to calling self._remove_internal_keys in this case - remove_internal_keys = lambda a: "Not found" - LOG = logging.getLogger(__name__) @@ -413,10 +397,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): self._remove_tmp_path(tmp) # prevents things like discovered_interpreter_* or ansible_discovered_interpreter_* from being set - # handle ansible 2.3.3 that has remove_internal_keys in a different place - check = remove_internal_keys(result) - if check == 'Not found': - self._remove_internal_keys(result) + ansible.vars.clean.remove_internal_keys(result) # taken from _execute_module of ansible 2.8.6 # propagate interpreter discovery results back to the controller @@ -440,7 +421,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): result['deprecations'] = [] result['deprecations'].extend(self._discovery_deprecation_warnings) - return wrap_var(result) + return ansible.utils.unsafe_proxy.wrap_var(result) def _postprocess_response(self, result): """ diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index 144de563..39a4b604 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -65,21 +65,12 @@ import abc import os import ansible.utils.shlex import ansible.constants as C +import ansible.executor.interpreter_discovery +import ansible.utils.unsafe_proxy from ansible.module_utils.six import with_metaclass from ansible.module_utils.parsing.convert_bool import boolean -# this was added in Ansible >= 2.8.0; fallback to the default interpreter if necessary -try: - from ansible.executor.interpreter_discovery import discover_interpreter -except ImportError: - discover_interpreter = lambda action,interpreter_name,discovery_mode,task_vars: '/usr/bin/python' - -try: - from ansible.utils.unsafe_proxy import AnsibleUnsafeText -except ImportError: - from ansible.vars.unsafe_proxy import AnsibleUnsafeText - import mitogen.core @@ -115,12 +106,13 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth action._finding_python_interpreter = True # fake pipelining so discover_interpreter can be happy action._connection.has_pipelining = True - s = AnsibleUnsafeText(discover_interpreter( + s = ansible.executor.interpreter_discovery.discover_interpreter( action=action, interpreter_name=interpreter_name, discovery_mode=s, - task_vars=task_vars)) - + task_vars=task_vars, + ) + s = ansible.utils.unsafe_proxy.AnsibleUnsafeText(s) # cache discovered interpreter task_vars['ansible_facts'][discovered_interpreter_config] = s action._connection.has_pipelining = False diff --git a/docs/changelog.rst b/docs/changelog.rst index bf9c12e1..31aa51ad 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,6 +25,8 @@ Unreleased fallbacks and polyfills into :mod:`mitogen.core` * :gh:issue:`1127` :mod:`ansible_mitogen`: Remove backward compatibility fallbacks for Python 2.4 & 2.5. +* :gh:issue:`1127` :mod:`ansible_mitogen`: Remove fallback imports for Ansible + releases before 2.10 v0.3.10 (2024-09-20) From 34088a8b7f89f1eac158fa395c410395d677bb0f Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 24 Sep 2024 17:06:17 +0100 Subject: [PATCH 40/46] ansible_mitogen: Consolidate Python 2 & 3 compatibility Rough guidelines, in decending preference: - Use mitogen.core if possible - Use ansible.module_utils.six if possible - Embed a getattr() or try/except viewkeys() et al can't be brought into mitogen.core because that package still targets Python 2.4. dict.viewkeys() were introduced in Python 2.7. --- ansible_mitogen/mixins.py | 6 +----- .../plugins/connection/mitogen_local.py | 8 +------- ansible_mitogen/runner.py | 11 +++-------- ansible_mitogen/services.py | 16 ++-------------- ansible_mitogen/target.py | 9 +++------ docs/changelog.rst | 2 ++ 6 files changed, 12 insertions(+), 40 deletions(-) diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index d67174bd..38f351ed 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -35,11 +35,6 @@ import pwd import random import traceback -try: - from shlex import quote as shlex_quote -except ImportError: - from pipes import quote as shlex_quote - import ansible import ansible.constants import ansible.plugins @@ -48,6 +43,7 @@ import ansible.utils.unsafe_proxy import ansible.vars.clean from ansible.module_utils.common.text.converters import to_bytes, to_text +from ansible.module_utils.six.moves import shlex_quote from ansible.parsing.utils.jsonify import jsonify import mitogen.core diff --git a/ansible_mitogen/plugins/connection/mitogen_local.py b/ansible_mitogen/plugins/connection/mitogen_local.py index 6ff86733..2d1e7052 100644 --- a/ansible_mitogen/plugins/connection/mitogen_local.py +++ b/ansible_mitogen/plugins/connection/mitogen_local.py @@ -42,13 +42,7 @@ except ImportError: import ansible_mitogen.connection import ansible_mitogen.process - -if sys.version_info > (3,): - viewkeys = dict.keys -elif sys.version_info > (2, 7): - viewkeys = dict.viewkeys -else: - viewkeys = lambda dct: set(dct) +viewkeys = getattr(dict, 'viewkeys', dict.keys) def dict_diff(old, new): diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 16e43059..b60e537c 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -52,6 +52,8 @@ import tempfile import traceback import types +from ansible.module_utils.six.moves import shlex_quote + import mitogen.core import ansible_mitogen.target # TODO: circular import from mitogen.core import to_text @@ -70,12 +72,6 @@ try: except ImportError: from io import StringIO -try: - from shlex import quote as shlex_quote -except ImportError: - from pipes import quote as shlex_quote - - # Prevent accidental import of an Ansible module from hanging on stdin read. import ansible.module_utils.basic ansible.module_utils.basic._ANSIBLE_ARGS = '{}' @@ -92,7 +88,6 @@ for symbol in 'res_init', '__res_init': except AttributeError: pass -iteritems = getattr(dict, 'iteritems', dict.items) LOG = logging.getLogger(__name__) @@ -600,7 +595,7 @@ class TemporaryEnvironment(object): def __init__(self, env=None): self.original = dict(os.environ) self.env = env or {} - for key, value in iteritems(self.env): + for key, value in mitogen.core.iteritems(self.env): key = mitogen.core.to_text(key) value = mitogen.core.to_text(value) if value is None: diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index 3e9de652..abc0e379 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -50,6 +50,8 @@ import threading import ansible.constants +from ansible.module_utils.six import reraise + import mitogen.core import mitogen.service import ansible_mitogen.loaders @@ -66,20 +68,6 @@ LOG = logging.getLogger(__name__) ansible_mitogen.loaders.shell_loader.get('sh') -if sys.version_info[0] == 3: - def reraise(tp, value, tb): - if value is None: - value = tp() - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value -else: - exec( - "def reraise(tp, value, tb=None):\n" - " raise tp, value, tb\n" - ) - - def _get_candidate_temp_dirs(): try: # >=2.5 diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index b79dc492..ee4cb398 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -55,12 +55,6 @@ import types import mitogen.core import mitogen.parent import mitogen.service -try: - reduce -except NameError: - # Python 3.x. - from functools import reduce - # Ansible since PR #41749 inserts "import __main__" into # ansible.module_utils.basic. Mitogen's importer will refuse such an import, so @@ -70,6 +64,9 @@ if not sys.modules.get(str('__main__')): sys.modules[str('__main__')] = types.ModuleType(str('__main__')) import ansible.module_utils.json_utils + +from ansible.module_utils.six.moves import reduce + import ansible_mitogen.runner diff --git a/docs/changelog.rst b/docs/changelog.rst index 31aa51ad..66a8077e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -27,6 +27,8 @@ Unreleased fallbacks for Python 2.4 & 2.5. * :gh:issue:`1127` :mod:`ansible_mitogen`: Remove fallback imports for Ansible releases before 2.10 +* :gh:issue:`1127` :mod:`ansible_mitogen`: Consolidate Python 2 & 3 + compatibility v0.3.10 (2024-09-20) From 0e7eefbc708c7eae060e166c7e158b620f2d7f58 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 24 Sep 2024 17:06:36 +0100 Subject: [PATCH 41/46] tests: Remove unused import --- tests/connection_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/connection_test.py b/tests/connection_test.py index c3146954..79e27aee 100644 --- a/tests/connection_test.py +++ b/tests/connection_test.py @@ -1,6 +1,5 @@ import logging import os -import signal import sys import tempfile import threading From b926795973906d77918843bf5f750f5c92315204 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 24 Sep 2024 18:04:18 +0100 Subject: [PATCH 42/46] ci: Move container registry authentication to an Azure Devops step This aims to - Reduce duplication - Seperate CI specific setup from test setup - Prepare for migration from Azure DevOps to GitHub Actions --- .ci/ansible_install.py | 11 ----------- .ci/azure-pipelines-steps.yml | 17 +++++++++++++---- .ci/debops_common_install.py | 3 --- .ci/localhost_ansible_install.py | 8 -------- .ci/mitogen_install.py | 14 -------------- .ci/mitogen_py24_install.py | 3 --- tox.ini | 6 ------ 7 files changed, 13 insertions(+), 49 deletions(-) delete mode 100755 .ci/ansible_install.py delete mode 100755 .ci/localhost_ansible_install.py delete mode 100755 .ci/mitogen_install.py diff --git a/.ci/ansible_install.py b/.ci/ansible_install.py deleted file mode 100755 index 3b217ff2..00000000 --- a/.ci/ansible_install.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python - -import ci_lib - -batches = [ - [ - 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', - ] -] - -ci_lib.run_batches(batches) diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml index 919b992b..791372af 100644 --- a/.ci/azure-pipelines-steps.yml +++ b/.ci/azure-pipelines-steps.yml @@ -14,6 +14,19 @@ steps: versionSpec: '$(python.version)' condition: ne(variables['python.version'], '') +- script: | + set -o errexit + set -o nounset + set -o pipefail + + aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws + displayName: Authenticate to container registry + condition: eq(variables['Agent.OS'], 'Linux') + env: + AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) + AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) + AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION) + - script: | set -o errexit set -o nounset @@ -90,7 +103,3 @@ steps: "$PYTHON" -m tox -e "$(tox.env)" displayName: "Run tests" - env: - AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) - AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) - AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION) diff --git a/.ci/debops_common_install.py b/.ci/debops_common_install.py index 565f9488..13217133 100755 --- a/.ci/debops_common_install.py +++ b/.ci/debops_common_install.py @@ -9,9 +9,6 @@ ci_lib.run_batches([ [ 'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"', ], - [ - 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', - ], ]) ci_lib.run('ansible-galaxy collection install debops.debops:==2.1.2') diff --git a/.ci/localhost_ansible_install.py b/.ci/localhost_ansible_install.py deleted file mode 100755 index d08ddafc..00000000 --- a/.ci/localhost_ansible_install.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python - -import ci_lib - -batches = [ -] - -ci_lib.run_batches(batches) diff --git a/.ci/mitogen_install.py b/.ci/mitogen_install.py deleted file mode 100755 index 23ff384b..00000000 --- a/.ci/mitogen_install.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python - -import ci_lib - -batches = [ -] - -if ci_lib.have_docker(): - batches.append([ - 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', - ]) - - -ci_lib.run_batches(batches) diff --git a/.ci/mitogen_py24_install.py b/.ci/mitogen_py24_install.py index 8af90405..85ea013c 100755 --- a/.ci/mitogen_py24_install.py +++ b/.ci/mitogen_py24_install.py @@ -3,9 +3,6 @@ import ci_lib batches = [ - [ - 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', - ], [ 'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv', ] diff --git a/tox.ini b/tox.ini index 9bb82def..9820a79f 100644 --- a/tox.ini +++ b/tox.ini @@ -89,10 +89,7 @@ deps = install_command = python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages} commands_pre = - mode_ansible: {toxinidir}/.ci/ansible_install.py mode_debops_common: {toxinidir}/.ci/debops_common_install.py - mode_localhost: {toxinidir}/.ci/localhost_ansible_install.py - mode_mitogen: {toxinidir}/.ci/mitogen_install.py commands = mode_ansible: {toxinidir}/.ci/ansible_tests.py mode_debops_common: {toxinidir}/.ci/debops_common_tests.py @@ -100,9 +97,6 @@ commands = mode_mitogen: {toxinidir}/.ci/mitogen_tests.py passenv = ANSIBLE_* - AWS_ACCESS_KEY_ID - AWS_DEFAULT_REGION - AWS_SECRET_ACCESS_KEY HOME # Azure DevOps, TF_BUILD is set to 'True' when running in a build task # https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables From 8b92e09655e31f0404b302e001c1f4f024616c57 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 25 Sep 2024 12:19:39 +0100 Subject: [PATCH 43/46] ci: Extract container registry location into variables Preperation for migrating from Azure DevOps with Amazon Elastic Container Registry (AWS ECR), to GitHub Actions with GitHub Container Registry (GHCR). DebOps tests are not currently being run, the updates to .ci/debops*.py are best effort only. --- .ci/README.md | 1 - .ci/ansible_tests.py | 2 +- .ci/ci_lib.py | 48 ++++++++++++++---------- .ci/debops_common_install.py | 3 -- .ci/debops_common_tests.py | 9 +++-- tests/image_prep/_container_finalize.yml | 2 +- tests/image_prep/group_vars/all.yml | 3 ++ tests/testlib.py | 30 +++++++++++---- tox.ini | 1 + 9 files changed, 62 insertions(+), 37 deletions(-) diff --git a/.ci/README.md b/.ci/README.md index 67a3805b..9248ac58 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -28,7 +28,6 @@ for doing `setup.py install` while pulling a Docker container, for example. ### Environment Variables -* `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2. * `DISTRO`: the `mitogen_` tests need a target Docker container distro. This name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test` * `DISTROS`: the `ansible_` tests can run against multiple targets diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 102eda9c..3ec48dfd 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -35,7 +35,7 @@ ci_lib.check_stray_processes(interesting) with ci_lib.Fold('docker_setup'): - containers = ci_lib.make_containers() + containers = ci_lib.container_specs(ci_lib.DISTROS) ci_lib.start_containers(containers) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 3e716385..dfe49b97 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -27,6 +27,13 @@ os.chdir( ) ) + +IMAGE_TEMPLATE = os.environ.get( + 'MITOGEN_TEST_IMAGE_TEMPLATE', + 'public.ecr.aws/n5z0e8q9/%(distro)s-test', +) + + _print = print def print(*args, **kwargs): file = kwargs.get('file', sys.stdout) @@ -193,8 +200,6 @@ GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) DISTRO = os.environ.get('DISTRO', 'debian9') # Used only when MODE=ansible DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split() -TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2')) -BASE_PORT = 2200 TMP = TempDir().path @@ -217,6 +222,7 @@ os.environ['PYTHONPATH'] = '%s:%s' % ( def get_docker_hostname(): """Return the hostname where the docker daemon is running. """ + # Duplicated in testlib url = os.environ.get('DOCKER_HOST') if url in (None, 'http+docker://localunixsocket'): return 'localhost' @@ -225,27 +231,34 @@ def get_docker_hostname(): return parsed.netloc.partition(':')[0] -def make_containers(name_prefix='', port_offset=0): +def container_specs( + distros, + base_port=2200, + image_template=IMAGE_TEMPLATE, + name_template='target-%(distro)s-%(index)d', +): """ >>> import pprint - >>> BASE_PORT=2200; DISTROS=['debian11', 'centos6'] - >>> pprint.pprint(make_containers()) + >>> pprint.pprint(container_specs(['debian11-py3', 'centos6'])) [{'distro': 'debian11', 'family': 'debian', 'hostname': 'localhost', 'image': 'public.ecr.aws/n5z0e8q9/debian11-test', + 'index': 1, 'name': 'target-debian11-1', 'port': 2201, - 'python_path': '/usr/bin/python'}, + 'python_path': '/usr/bin/python3'}, {'distro': 'centos6', 'family': 'centos', 'hostname': 'localhost', 'image': 'public.ecr.aws/n5z0e8q9/centos6-test', + 'index': 2, 'name': 'target-centos6-2', 'port': 2202, 'python_path': '/usr/bin/python'}] """ docker_hostname = get_docker_hostname() + # Code duplicated in testlib.py, both should be updated together distro_pattern = re.compile(r''' (?P(?P[a-z]+)[0-9]+) (?:-(?Ppy3))? @@ -256,30 +269,27 @@ def make_containers(name_prefix='', port_offset=0): i = 1 lst = [] - for distro in DISTROS: + for distro in distros: + # Code duplicated in testlib.py, both should be updated together d = distro_pattern.match(distro).groupdict(default=None) - distro = d['distro'] - family = d['family'] - image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,) - if d['py'] == 'py3': + if d.pop('py') == 'py3': python_path = '/usr/bin/python3' else: python_path = '/usr/bin/python' - if d['count']: - count = int(count) - else: - count = 1 + count = int(d.pop('count') or '1', 10) for x in range(count): - lst.append({ - "distro": distro, "family": family, "image": image, - "name": name_prefix + ("target-%s-%s" % (distro, i)), + d['index'] = i + d.update({ + 'image': image_template % d, + 'name': name_template % d, "hostname": docker_hostname, - "port": BASE_PORT + i + port_offset, + 'port': base_port + i, "python_path": python_path, }) + lst.append(d) i += 1 return lst diff --git a/.ci/debops_common_install.py b/.ci/debops_common_install.py index 13217133..825126c7 100755 --- a/.ci/debops_common_install.py +++ b/.ci/debops_common_install.py @@ -2,9 +2,6 @@ import ci_lib -# Naturally DebOps only supports Debian. -ci_lib.DISTROS = ['debian'] - ci_lib.run_batches([ [ 'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"', diff --git a/.ci/debops_common_tests.py b/.ci/debops_common_tests.py index 7db8a797..b065486f 100755 --- a/.ci/debops_common_tests.py +++ b/.ci/debops_common_tests.py @@ -6,9 +6,6 @@ import sys import ci_lib -# DebOps only supports Debian. -ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT - project_dir = os.path.join(ci_lib.TMP, 'project') vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml' inventory_path = 'ansible/inventory/hosts' @@ -16,7 +13,11 @@ docker_hostname = ci_lib.get_docker_hostname() with ci_lib.Fold('docker_setup'): - containers = ci_lib.make_containers(port_offset=500, name_prefix='debops-') + containers = ci_lib.container_specs( + ['debian*2'], + base_port=2700, + name_template='debops-target-%(distro)s-%(index)d', + ) ci_lib.start_containers(containers) diff --git a/tests/image_prep/_container_finalize.yml b/tests/image_prep/_container_finalize.yml index d61d9b3b..5329fefa 100644 --- a/tests/image_prep/_container_finalize.yml +++ b/tests/image_prep/_container_finalize.yml @@ -9,7 +9,7 @@ --change 'EXPOSE 22' --change 'CMD ["/usr/sbin/sshd", "-D"]' {{ inventory_hostname }} - public.ecr.aws/n5z0e8q9/{{ inventory_hostname }}-test + {{ container_image_name }} delegate_to: localhost - name: Stop containers diff --git a/tests/image_prep/group_vars/all.yml b/tests/image_prep/group_vars/all.yml index 5f182f86..91ff934d 100644 --- a/tests/image_prep/group_vars/all.yml +++ b/tests/image_prep/group_vars/all.yml @@ -4,6 +4,9 @@ common_packages: - strace - sudo +container_image_name: "{{ container_registry }}/{{ inventory_hostname }}-test" +container_registry: public.ecr.aws/n5z0e8q9 + sudo_group: MacOSX: admin Debian: sudo diff --git a/tests/testlib.py b/tests/testlib.py index a52292ce..76743e82 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -51,6 +51,12 @@ except NameError: LOG = logging.getLogger(__name__) +DISTRO = os.environ.get('MITOGEN_TEST_DISTRO', 'debian9') +IMAGE_TEMPLATE = os.environ.get( + 'MITOGEN_TEST_IMAGE_TEMPLATE', + 'public.ecr.aws/n5z0e8q9/%(distro)s-test', +) + TESTS_DIR = os.path.join(os.path.dirname(__file__)) ANSIBLE_LIB_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib') ANSIBLE_MODULE_UTILS_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils') @@ -509,6 +515,7 @@ class TestCase(unittest.TestCase): def get_docker_host(): + # Duplicated in ci_lib url = os.environ.get('DOCKER_HOST') if url in (None, 'http+docker://localunixsocket'): return 'localhost' @@ -549,19 +556,23 @@ class DockerizedSshDaemon(object): ] subprocess.check_output(args) - def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')): - if '-' in mitogen_test_distro: - distro, _py3 = mitogen_test_distro.split('-') - else: - distro = mitogen_test_distro - _py3 = None + def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE): + # Code duplicated in ci_lib.py, both should be updated together + distro_pattern = re.compile(r''' + (?P(?P[a-z]+)[0-9]+) + (?:-(?Ppy3))? + (?:\*(?P[0-9]+))? + ''', + re.VERBOSE, + ) + d = distro_pattern.match(distro).groupdict(default=None) - if _py3 == 'py3': + if d.pop('py') == 'py3': self.python_path = '/usr/bin/python3' else: self.python_path = '/usr/bin/python' - self.image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,) + self.image = image_template % d self.start_container() self.host = self.get_host() self.port = self.get_port(self.container_name) @@ -601,6 +612,9 @@ class DockerizedSshDaemon(object): class BrokerMixin(object): broker_class = mitogen.master.Broker + + # Flag for tests that shutdown the broker themself + # e.g. unix_test.ListenerTest broker_shutdown = False def setUp(self): diff --git a/tox.ini b/tox.ini index 9820a79f..9fb31bdc 100644 --- a/tox.ini +++ b/tox.ini @@ -98,6 +98,7 @@ commands = passenv = ANSIBLE_* HOME + MITOGEN_* # Azure DevOps, TF_BUILD is set to 'True' when running in a build task # https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables TF_BUILD From c6c8bfb690b8423b140917c9f5e9dcca72f17d30 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 25 Sep 2024 13:30:22 +0100 Subject: [PATCH 44/46] tests: Skip vanilla Ansible on Linux unpriviliged -> unprivileged become CI containers lack the necessary `setfacl` command. This has not previously been noticed because no vanilla Ansible jobs were being run on Linux, only on macOS. refs #1118 --- .../integration/become/su_password.yml | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/tests/ansible/integration/become/su_password.yml b/tests/ansible/integration/become/su_password.yml index 52d420db..207980c4 100644 --- a/tests/ansible/integration/become/su_password.yml +++ b/tests/ansible/integration/become/su_password.yml @@ -53,20 +53,22 @@ vars: ansible_become_pass: user1_password when: - # https://github.com/ansible/ansible/pull/70785 - - ansible_facts.distribution not in ["MacOSX"] - or ansible_version.full is version("2.11", ">=", strict=True) - or is_mitogen + # CI containers lack `setfacl` for unpriv -> unpriv + # https://github.com/mitogen-hq/mitogen/issues/1118 + - is_mitogen + or (ansible_facts.distribution in ["MacOSX"] + and ansible_version.full is version("2.11", ">=", strict=True)) - assert: that: - out.stdout == 'mitogen__user1' fail_msg: out={{out}} when: - # https://github.com/ansible/ansible/pull/70785 - - ansible_facts.distribution not in ["MacOSX"] - or ansible_version.full is version("2.11", ">=", strict=True) - or is_mitogen + # CI containers lack `setfacl` for unpriv -> unpriv + # https://github.com/mitogen-hq/mitogen/issues/1118 + - is_mitogen + or (ansible_facts.distribution in ["MacOSX"] + and ansible_version.full is version("2.11", ">=", strict=True)) - name: Ensure password su without chdir succeeds shell: whoami @@ -76,20 +78,22 @@ vars: ansible_become_pass: user1_password when: - # https://github.com/ansible/ansible/pull/70785 - - ansible_facts.distribution not in ["MacOSX"] - or ansible_version.full is version("2.11", ">=", strict=True) - or is_mitogen + # CI containers lack `setfacl` for unpriv -> unpriv + # https://github.com/mitogen-hq/mitogen/issues/1118 + - is_mitogen + or (ansible_facts.distribution in ["MacOSX"] + and ansible_version.full is version("2.11", ">=", strict=True)) - assert: that: - out.stdout == 'mitogen__user1' fail_msg: out={{out}} when: - # https://github.com/ansible/ansible/pull/70785 - - ansible_facts.distribution not in ["MacOSX"] - or ansible_version.full is version("2.11", ">=", strict=True) - or is_mitogen + # CI containers lack `setfacl` for unpriv -> unpriv + # https://github.com/mitogen-hq/mitogen/issues/1118 + - is_mitogen + or (ansible_facts.distribution in ["MacOSX"] + and ansible_version.full is version("2.11", ">=", strict=True)) tags: - su From 27214517a724431fb5dc73255b608bc4fe078479 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 26 Sep 2024 02:08:17 +0100 Subject: [PATCH 45/46] tests: Use a subprocess to check discovered python == running This replaces the use of `os.path.realpath()` which gave incorrect results on macOS - depending on the exact Python build, Python version, macOS version, installation method, and phase of the moon. realpath information kept around to aid debugging. --- .../ansible_2_8_tests.yml | 2 +- tests/ansible/lib/modules/test_echo_module.py | 112 +++++++++++++++++- 2 files changed, 111 insertions(+), 3 deletions(-) diff --git a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml index 0c7d30c9..fc0d3cf7 100644 --- a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml +++ b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml @@ -99,7 +99,7 @@ that: - auto_out.ansible_facts.discovered_interpreter_python is defined - auto_out.ansible_facts.discovered_interpreter_python == echoout.discovered_python.as_seen - - echoout.discovered_python.resolved == echoout.running_python.sys.executable.resolved + - echoout.discovered_python.sys.executable.as_seen == echoout.running_python.sys.executable.as_seen fail_msg: - "auto_out: {{ auto_out }}" - "echoout: {{ echoout }}" diff --git a/tests/ansible/lib/modules/test_echo_module.py b/tests/ansible/lib/modules/test_echo_module.py index d6a5fb9e..fe8ed69a 100644 --- a/tests/ansible/lib/modules/test_echo_module.py +++ b/tests/ansible/lib/modules/test_echo_module.py @@ -10,11 +10,97 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type import os +import stat import platform +import subprocess import sys + from ansible.module_utils.basic import AnsibleModule +# trace_realpath() and _join_tracepath() adapated from stdlib posixpath.py +# https://github.com/python/cpython/blob/v3.12.6/Lib/posixpath.py#L423-L492 +# Copyright (c) 2001 - 2023 Python Software Foundation +# Copyright (c) 2024 Alex Willmer +# License: Python Software Foundation License Version 2 + +def trace_realpath(filename, strict=False): + """ + Return the canonical path of the specified filename, and a trace of + the route taken, eliminating any symbolic links encountered in the path. + """ + path, trace, ok = _join_tracepath(filename[:0], filename, strict, seen={}, trace=[]) + return os.path.abspath(path), trace + + +def _join_tracepath(path, rest, strict, seen, trace): + """ + Join two paths, normalizing and eliminating any symbolic links encountered + in the second path. + """ + trace.append(rest) + if isinstance(path, bytes): + sep = b'/' + curdir = b'.' + pardir = b'..' + else: + sep = '/' + curdir = '.' + pardir = '..' + + if os.path.isabs(rest): + rest = rest[1:] + path = sep + + while rest: + name, _, rest = rest.partition(sep) + if not name or name == curdir: + # current dir + continue + if name == pardir: + # parent dir + if path: + path, name = os.path.split(path) + if name == pardir: + path = os.path.join(path, pardir, pardir) + else: + path = pardir + continue + newpath = os.path.join(path, name) + try: + st = os.lstat(newpath) + except OSError: + if strict: + raise + is_link = False + else: + is_link = stat.S_ISLNK(st.st_mode) + if not is_link: + path = newpath + continue + # Resolve the symbolic link + if newpath in seen: + # Already seen this path + path = seen[newpath] + if path is not None: + # use cached value + continue + # The symlink is not resolved, so we must have a symlink loop. + if strict: + # Raise OSError(errno.ELOOP) + os.stat(newpath) + else: + # Return already resolved part + rest of the path unchanged. + return os.path.join(newpath, rest), trace, False + seen[newpath] = None # not resolved symlink + path, trace, ok = _join_tracepath(path, os.readlink(newpath), strict, seen, trace) + if not ok: + return os.path.join(path, rest), False + seen[newpath] = path # resolved symlink + + return path, trace, True + + def main(): module = AnsibleModule(argument_spec=dict( facts_copy=dict(type=dict, default={}), @@ -33,7 +119,18 @@ def main(): sys.executable = "/usr/bin/python" facts_copy = module.params['facts_copy'] + discovered_interpreter_python = facts_copy['discovered_interpreter_python'] + d_i_p_realpath, d_i_p_trace = trace_realpath(discovered_interpreter_python) + d_i_p_proc = subprocess.Popen( + [discovered_interpreter_python, '-c', 'import sys; print(sys.executable)'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + + ) + d_i_p_stdout, d_i_p_stderr = d_i_p_proc.communicate() + + sys_exec_realpath, sys_exec_trace = trace_realpath(sys.executable) + result = { 'changed': False, 'ansible_facts': module.params['facts_to_override'], @@ -43,7 +140,17 @@ def main(): ), 'discovered_python': { 'as_seen': discovered_interpreter_python, - 'resolved': os.path.realpath(discovered_interpreter_python), + 'resolved': d_i_p_realpath, + 'trace': [os.path.abspath(p) for p in d_i_p_trace], + 'sys': { + 'executable': { + 'as_seen': d_i_p_stdout.decode('ascii').rstrip('\n'), + 'proc': { + 'stderr': d_i_p_stderr.decode('ascii'), + 'returncode': d_i_p_proc.returncode, + }, + }, + }, }, 'running_python': { 'platform': { @@ -54,7 +161,8 @@ def main(): 'sys': { 'executable': { 'as_seen': sys.executable, - 'resolved': os.path.realpath(sys.executable), + 'resolved': sys_exec_realpath, + 'trace': [os.path.abspath(p) for p in sys_exec_trace], }, 'platform': sys.platform, 'version_info': { From 4f60d01f09f2ae31c8f58cddafb17766e1054fa4 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 26 Sep 2024 02:15:39 +0100 Subject: [PATCH 46/46] CI: Enable GitHub Actions testing workflow This replicate the existing Azure DevOps workflow, and adds a couple of new jobs (Python 2.7 on macOS, Python + vanilla Ansible on Linux). The GitHub Actions use container images hosted on GitHub Container Registry (GHCR, ghcr.io/mitogen-hq). These images have been copied straight from the existing Amazon Elastic Cloud Registry (AWS ECR, public.ecr.aws/n5z0e8q9). A short period of parallel running is planned. Then a second PR will remove the Azure DevOps workflow. --- .github/workflows/tests.yml | 326 ++++++++++++++++++++++++++++++++++++ docs/changelog.rst | 1 + 2 files changed, 327 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..4520c3cf --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,326 @@ +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions + +name: Tests + +on: + pull_request: + push: + branches-ignore: + - docs-master + +env: + #ANSIBLE_VERBOSITY: 3 + #MITOGEN_LOG_LEVEL: DEBUG + MITOGEN_TEST_IMAGE_TEMPLATE: "ghcr.io/mitogen-hq/%(distro)s-test" + +# https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners +# https://github.com/actions/runner-images/blob/main/README.md#software-and-image-support +jobs: + linux: + # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md + runs-on: ubuntu-20.04 + + strategy: + fail-fast: false + matrix: + include: + - name: Ans_27_210 + tox_env: py27-mode_ansible-ansible2.10 + - name: Ans_27_4 + tox_env: py27-mode_ansible-ansible4 + + - name: Ans_36_210 + python_version: '3.6' + tox_env: py36-mode_ansible-ansible2.10 + - name: Ans_36_4 + python_version: '3.6' + tox_env: py36-mode_ansible-ansible4 + + - name: Ans_311_210 + python_version: '3.11' + tox_env: py311-mode_ansible-ansible2.10 + - name: Ans_311_3 + python_version: '3.11' + tox_env: py311-mode_ansible-ansible3 + - name: Ans_311_4 + python_version: '3.11' + tox_env: py311-mode_ansible-ansible4 + - name: Ans_311_5 + python_version: '3.11' + tox_env: py311-mode_ansible-ansible5 + - name: Ans_312_6 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible6 + - name: Ans_312_7 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible7 + - name: Ans_312_8 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible8 + - name: Ans_312_9 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible9 + - name: Ans_312_10 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible10 + - name: Van_312_10 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible10-strategy_linear + + - name: Mito_27_centos6 + tox_env: py27-mode_mitogen-distro_centos6 + - name: Mito_27_centos7 + tox_env: py27-mode_mitogen-distro_centos7 + - name: Mito_27_centos8 + tox_env: py27-mode_mitogen-distro_centos8 + - name: Mito_27_debian9 + tox_env: py27-mode_mitogen-distro_debian9 + - name: Mito_27_debian10 + tox_env: py27-mode_mitogen-distro_debian10 + - name: Mito_27_debian11 + tox_env: py27-mode_mitogen-distro_debian11 + - name: Mito_27_ubuntu1604 + tox_env: py27-mode_mitogen-distro_ubuntu1604 + - name: Mito_27_ubuntu1804 + tox_env: py27-mode_mitogen-distro_ubuntu1804 + - name: Mito_27_ubuntu2004 + tox_env: py27-mode_mitogen-distro_ubuntu2004 + + - name: Mito_36_centos6 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_centos6 + - name: Mito_36_centos7 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_centos7 + - name: Mito_36_centos8 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_centos8 + - name: Mito_36_debian9 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_debian9 + - name: Mito_36_debian10 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_debian10 + - name: Mito_36_debian11 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_debian11 + - name: Mito_36_ubuntu1604 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_ubuntu1604 + - name: Mito_36_ubuntu1804 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_ubuntu1804 + - name: Mito_36_ubuntu2004 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_ubuntu2004 + + - name: Mito_312_centos6 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_centos6 + - name: Mito_312_centos7 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_centos7 + - name: Mito_312_centos8 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_centos8 + - name: Mito_312_debian9 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_debian9 + - name: Mito_312_debian10 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_debian10 + - name: Mito_312_debian11 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_debian11 + - name: Mito_312_ubuntu1604 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_ubuntu1604 + - name: Mito_312_ubuntu1804 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_ubuntu1804 + - name: Mito_312_ubuntu2004 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_ubuntu2004 + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python_version }} + if: ${{ matrix.python_version }} + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Install build deps + run: | + set -o errexit -o nounset -o pipefail + + sudo apt-get update + sudo apt-get install -y python2-dev python3-pip virtualenv + - name: Show Python versions + run: | + set -o errexit -o nounset -o pipefail + + # macOS builders lack a realpath command + type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version + type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version + type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version + echo + + if [ -e /usr/bin/python ]; then + echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2 ]; then + echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2.7 ]; then + echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')" + fi + - name: Install tooling + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then + "$PYTHON" -m ensurepip --user --altinstall --no-default-pip + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + elif [[ $PYTHON == "python2.7" ]]; then + curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py" + "$PYTHON" get-pip.py --user --no-python-version-warning + # Avoid Python 2.x pip masking system pip + rm -f ~/.local/bin/{easy_install,pip,wheel} + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + else + "$PYTHON" -m pip install -r "tests/requirements-tox.txt" + fi + - name: Run tests + env: + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + "$PYTHON" -m tox -e "${{ matrix.tox_env }}" + + macos: + # https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md + runs-on: macos-12 + timeout-minutes: 120 + + strategy: + fail-fast: false + matrix: + include: + - name: Mito_27 + tox_env: py27-mode_mitogen + - name: Mito_312 + tox_env: py312-mode_mitogen + + - name: Loc_27_210 + tox_env: py27-mode_localhost-ansible2.10 + - name: Loc_312_10 + tox_env: py312-mode_localhost-ansible10 + + - name: Van_27_210 + tox_env: py27-mode_localhost-ansible2.10-strategy_linear + - name: Van_312_10 + tox_env: py312-mode_localhost-ansible10-strategy_linear + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python_version }} + if: ${{ matrix.python_version }} + - name: Show Python versions + run: | + set -o errexit -o nounset -o pipefail + + # macOS builders lack a realpath command + type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version + type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version + type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version + echo + + if [ -e /usr/bin/python ]; then + echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2 ]; then + echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2.7 ]; then + echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')" + fi + + if [ -e /Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7 ]; then + # GitHub macOS 12 images: python2.7 is installed, but not on $PATH + echo "/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7: sys.executable: $(/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7 -c 'import sys; print(sys.executable)')" + fi + - name: Install tooling + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then + # GitHub macOS 12 images: python2.7 is installed, but not on $PATH + PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7" + "$PYTHON" -m ensurepip --user --altinstall --no-default-pip + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + elif [[ $PYTHON == "python2.7" ]]; then + curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py" + "$PYTHON" get-pip.py --user --no-python-version-warning + # Avoid Python 2.x pip masking system pip + rm -f ~/.local/bin/{easy_install,pip,wheel} + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + else + "$PYTHON" -m pip install -r "tests/requirements-tox.txt" + fi + - name: Run tests + env: + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then + # GitHub macOS 12 images: python2.7 is installed, but not on $PATH + PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7" + fi + + "$PYTHON" -m tox -e "${{ matrix.tox_env }}" diff --git a/docs/changelog.rst b/docs/changelog.rst index 66a8077e..3adaec58 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -29,6 +29,7 @@ Unreleased releases before 2.10 * :gh:issue:`1127` :mod:`ansible_mitogen`: Consolidate Python 2 & 3 compatibility +* :gh:issue:`1128` CI: Start migration from Azure DevOps to GitHub Actions v0.3.10 (2024-09-20)