From f5a88406685923158cee238ca9a6217fb6b4b630 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 26 Jul 2024 14:58:21 +0100 Subject: [PATCH 1/3] CI: Use archived RPMs on CentOS 8 CentOS 8 has reached EOL. Packages are no longer mirrored or maintained. A historic snapshot of the packages is kept on vault.centos.org. refs #1088, #1090 --- tests/ansible/hosts/group_vars/all.yml | 1 + tests/ansible/hosts/group_vars/centos8.yml | 26 +++++++++++++++++++ tests/ansible/hosts/group_vars/debian9.yml | 4 +++ .../issue_776__load_plugins_called_twice.yml | 19 +++++--------- 4 files changed, 37 insertions(+), 13 deletions(-) create mode 100644 tests/ansible/hosts/group_vars/debian9.yml diff --git a/tests/ansible/hosts/group_vars/all.yml b/tests/ansible/hosts/group_vars/all.yml index cea46113..ad4adabb 100644 --- a/tests/ansible/hosts/group_vars/all.yml +++ b/tests/ansible/hosts/group_vars/all.yml @@ -1,2 +1,3 @@ --- pkg_mgr_python_interpreter: python +pkg_repos_overrides: [] diff --git a/tests/ansible/hosts/group_vars/centos8.yml b/tests/ansible/hosts/group_vars/centos8.yml index 7b9e34f6..c90dd5f4 100644 --- a/tests/ansible/hosts/group_vars/centos8.yml +++ b/tests/ansible/hosts/group_vars/centos8.yml @@ -1,2 +1,28 @@ --- pkg_mgr_python_interpreter: /usr/libexec/platform-python + +pkg_repos_overrides: + - dest: /etc/yum.repos.d/CentOS-Linux-AppStream.repo + content: | + [appstream] + name=CentOS Linux $releasever - AppStream + baseurl=http://vault.centos.org/$contentdir/$releasever/AppStream/$basearch/os/ + enabled=1 + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial + - dest: /etc/yum.repos.d/CentOS-Linux-BaseOS.repo + content: | + [baseos] + name=CentOS Linux $releasever - BaseOS + baseurl=http://vault.centos.org/$contentdir/$releasever/BaseOS/$basearch/os/ + enabled=1 + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial + - dest: /etc/yum.repos.d/CentOS-Linux-Extras.repo + content: | + [extras] + name=CentOS Linux $releasever - Extras + baseurl=http://vault.centos.org/$contentdir/$releasever/extras/$basearch/os/ + enabled=1 + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial diff --git a/tests/ansible/hosts/group_vars/debian9.yml b/tests/ansible/hosts/group_vars/debian9.yml new file mode 100644 index 00000000..4b180b13 --- /dev/null +++ b/tests/ansible/hosts/group_vars/debian9.yml @@ -0,0 +1,4 @@ +pkg_repos_overrides: + - dest: /etc/apt/sources.list + content: | + deb http://archive.debian.org/debian stretch main contrib non-free diff --git a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml index ad5bbd69..44ff6863 100755 --- a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml +++ b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml @@ -10,21 +10,14 @@ ansible_python_interpreter: "{{ pkg_mgr_python_interpreter }}" package: rsync # Chosen to exist in all tested distros/package managers tasks: - - name: Switch to centos-stream - command: dnf --assumeyes --disablerepo="*" --enablerepo=extras swap centos-linux-repos centos-stream-repos - when: - - ansible_facts.pkg_mgr in ["dnf"] - - - name: Switch to archive.debian.org - # Debian 9 has been archived https://lists.debian.org/debian-devel-announce/2023/03/msg00006.html + - name: Switch to archived package repositories copy: - content: | - deb http://archive.debian.org/debian stretch main contrib non-free - dest: /etc/apt/sources.list + dest: "{{ item.dest }}" + content: "{{ item.content }}" mode: u=rw,go=r - when: - - ansible_facts.distribution == "Debian" - - ansible_facts.distribution_major_version == "9" + loop: "{{ pkg_repos_overrides }}" + loop_control: + label: "{{ item.dest }}" - name: Add signing keys copy: From 924dbd6f0c31f1169adc8d969115dbcdfecb3cc9 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 29 Jul 2024 11:25:12 +0100 Subject: [PATCH 2/3] CI: Migrate macOS integration tests to macOS 12, drop Python 2.7 jobs macOS 11 is not longer an available runner on Azure Devops. The minimum is now macOS 12. This runner does not have Python 2.7 installed, so running them would require a custom install - which I'm declaring too much effort for too little gain. refs #1090 --- .ci/azure-pipelines.yml | 17 +++-------------- docs/changelog.rst | 1 + .../integration/become/sudo_nonexistent.yml | 3 +-- .../modules/custom_python_json_args_module.py | 2 +- .../lib/modules/custom_python_os_getcwd.py | 2 +- .../modules/custom_python_want_json_module.py | 2 +- .../issue_591__setuptools_cwd_crash.yml | 3 --- .../issue_655__wait_for_connection_error.yml | 7 +++---- tests/two_three_compat_test.py | 7 +++++++ 9 files changed, 18 insertions(+), 26 deletions(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 94919e35..51908e92 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -16,32 +16,21 @@ trigger: - docs-master jobs: -- job: mac11 +- job: mac12 # vanilla Ansible is really slow timeoutInMinutes: 120 steps: - template: azure-pipelines-steps.yml pool: - # https://github.com/actions/runner-images/blob/main/images/macos/macos-11-Readme.md - vmImage: macOS-11 + # https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md + vmImage: macOS-12 strategy: matrix: - Mito_27: - tox.env: py27-mode_mitogen Mito_312: - python.version: '3.12' tox.env: py312-mode_mitogen - - Loc_27_210: - tox.env: py27-mode_localhost-ansible2.10 Loc_312_9: - python.version: '3.12' tox.env: py312-mode_localhost-ansible9 - - Van_27_210: - tox.env: py27-mode_localhost-ansible2.10-strategy_linear Van_312_9: - python.version: '3.12' tox.env: py312-mode_localhost-ansible9-strategy_linear - job: Linux diff --git a/docs/changelog.rst b/docs/changelog.rst index 5d77910e..ffba69fa 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,6 +25,7 @@ Unreleased * :gh:issue:`957` Fix Ansible exception when executing against 10s of hosts "ValueError: filedescriptor out of range in select()" * :gh:issue:`1066` Support Ansible `ansible_host_key_checking` & `ansible_ssh_host_key_checking` +* :gh:issue:`1090` CI: Migrate macOS integration tests to macOS 12, drop Python 2.7 jobs v0.3.7 (2024-04-08) diff --git a/tests/ansible/integration/become/sudo_nonexistent.yml b/tests/ansible/integration/become/sudo_nonexistent.yml index e7a849c2..912d1ba8 100644 --- a/tests/ansible/integration/become/sudo_nonexistent.yml +++ b/tests/ansible/integration/become/sudo_nonexistent.yml @@ -21,8 +21,7 @@ # sudo-1.8.6p3-29.el6_10.3 on RHEL & CentOS 6.10 (final release) # removed user/group error messages, as defence against CVE-2019-14287. - >- - 'sudo: unknown user: slartibartfast' in out.module_stdout | default(out.msg) - or 'sudo: unknown user: slartibartfast' in out.module_stderr | default(out.msg) + (out.module_stderr | default(out.module_stdout, true) | default(out.msg, true)) is search('sudo: unknown user:? slartibartfast') or (ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_version == '6.10') fail_msg: out={{out}} when: diff --git a/tests/ansible/lib/modules/custom_python_json_args_module.py b/tests/ansible/lib/modules/custom_python_json_args_module.py index 846037ec..61640579 100755 --- a/tests/ansible/lib/modules/custom_python_json_args_module.py +++ b/tests/ansible/lib/modules/custom_python_json_args_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible Python JSONARGS module. I should receive an encoding string. json_arguments = """<>""" diff --git a/tests/ansible/lib/modules/custom_python_os_getcwd.py b/tests/ansible/lib/modules/custom_python_os_getcwd.py index d465ac9e..c5e264ae 100644 --- a/tests/ansible/lib/modules/custom_python_os_getcwd.py +++ b/tests/ansible/lib/modules/custom_python_os_getcwd.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # #591: call os.getcwd() before AnsibleModule ever gets a chance to fix up the # process environment. diff --git a/tests/ansible/lib/modules/custom_python_want_json_module.py b/tests/ansible/lib/modules/custom_python_want_json_module.py index 23eeeb55..f5e33862 100755 --- a/tests/ansible/lib/modules/custom_python_want_json_module.py +++ b/tests/ansible/lib/modules/custom_python_want_json_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # I am an Ansible Python WANT_JSON module. I should receive a JSON-encoded file. import json diff --git a/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml b/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml index d5f0815b..ff102b13 100644 --- a/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml +++ b/tests/ansible/regression/issue_591__setuptools_cwd_crash.yml @@ -19,9 +19,6 @@ # Will crash if process has a nonexistent CWD. - custom_python_os_getcwd: - script: | - import os - self._connection.get_chain().call(os.getcwd) tags: - issue_591 - mitogen_only diff --git a/tests/ansible/regression/issue_655__wait_for_connection_error.yml b/tests/ansible/regression/issue_655__wait_for_connection_error.yml index 9ad42a10..4972d91a 100644 --- a/tests/ansible/regression/issue_655__wait_for_connection_error.yml +++ b/tests/ansible/regression/issue_655__wait_for_connection_error.yml @@ -11,11 +11,10 @@ tasks: - meta: end_play when: - # TODO CI currently runs on macOS 11 images in Azure DevOps. MacOS 11 - # is no longer supported by homebrew, so the following install - # task fails. + # TODO CI currently runs on macOS 12 & which isn't supported by Podman + # version available in Homebrew. - ansible_facts.system == 'Darwin' - - ansible_facts.distribution_major_version == '11' + - ansible_facts.distribution_version is version('13.0', '<', strict=True) - name: set up test container and run tests inside it block: diff --git a/tests/two_three_compat_test.py b/tests/two_three_compat_test.py index 4490e5d2..ab9f4e19 100644 --- a/tests/two_three_compat_test.py +++ b/tests/two_three_compat_test.py @@ -1,3 +1,6 @@ +import os +import unittest + import mitogen.core import testlib @@ -7,6 +10,10 @@ import simple_pkg.ping # TODO: this is a joke. 2/3 interop is one of the hardest bits to get right. # There should be 100 tests in this file. +@unittest.skipIf( + os.uname()[0] == 'Darwin' and int(os.uname()[2].partition('.')[0]) >= 21, + "Python 2.x not shipped on macOS 12.3+ (Darwin 21.4+, Monterey)", +) class TwoThreeCompatTest(testlib.RouterMixin, testlib.TestCase): if mitogen.core.PY3: python_path = 'python2' From fe435bb7d069764b35a8ca77b05cffe19dbd9c1e Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 29 Jul 2024 11:30:25 +0100 Subject: [PATCH 3/3] CI: Workaround "No module named 'setuptools.command.test'" Pip 72 was released yesterday (2024-07-28), dropping `setup.py test` support. hdrhistogram 0.6.1 requires it to install. For now constrain Pip to earlier releases, so our tests can be run. refs #1090 --- tests/ansible/requirements.txt | 3 +++ tests/constraints.txt | 3 +++ tox.ini | 1 + 3 files changed, 7 insertions(+) create mode 100644 tests/constraints.txt diff --git a/tests/ansible/requirements.txt b/tests/ansible/requirements.txt index 2c3c87c8..8cfb348a 100644 --- a/tests/ansible/requirements.txt +++ b/tests/ansible/requirements.txt @@ -1,4 +1,7 @@ paramiko==2.3.2 # Last 2.6-compat version. +# Incompatible with pip >= 72, due to removal of `setup.py test`: +# ModuleNotFoundError: No module named 'setuptools.command.test' +# https://github.com/pypa/setuptools/issues/4519 hdrhistogram==0.6.1 PyYAML==3.11; python_version < '2.7' PyYAML==5.3.1; python_version >= '2.7' # Latest release (Jan 2021) diff --git a/tests/constraints.txt b/tests/constraints.txt new file mode 100644 index 00000000..6adaa30b --- /dev/null +++ b/tests/constraints.txt @@ -0,0 +1,3 @@ +# Setuptools 72 removed `setup.py test`. hdrhistogram 0.6.1 still depends on it. +# TODO Bump dependencies and unconstrain Pip. +setuptools<72 diff --git a/tox.ini b/tox.ini index 55fd1842..a0b0e1ae 100644 --- a/tox.ini +++ b/tox.ini @@ -109,6 +109,7 @@ setenv = ANSIBLE_STRATEGY = mitogen_linear NOCOVERAGE_ERASE = 1 NOCOVERAGE_REPORT = 1 + PIP_CONSTRAINT={toxinidir}/tests/constraints.txt # Only applicable to MODE=mitogen distro_centos5: DISTRO=centos5 distro_centos6: DISTRO=centos6