From 657e40b982b481f86763e9c14e7cc9bc47ab2906 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 4 Feb 2025 10:41:46 +0000 Subject: [PATCH 01/25] Begin 0.3.23dev --- docs/changelog.rst | 5 +++++ mitogen/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 80fe7387..dfbf1643 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,11 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. +In progress (unreleased) +------------------------ + + + v0.3.22 (2025-02-04) -------------------- diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 61fce888..125ab052 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 22) +__version__ = (0, 3, 23, 'dev') #: This is :data:`False` in slave contexts. Previously it was used to prevent From 5f42da36f3632782c71746c477f2eaa1c3602c60 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 6 Feb 2025 16:41:36 +0000 Subject: [PATCH 02/25] mitogen: Deduplicate cfmakeraw() flags refs #1121 --- mitogen/parent.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index f301a42c..3c5e48ba 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -234,12 +234,9 @@ def cfmakeraw(tflags): modified in a manner similar to the `cfmakeraw()` C library function, but additionally disabling local echo. """ - # BSD: github.com/freebsd/freebsd/blob/master/lib/libc/gen/termios.c#L162 - # Linux: github.com/lattera/glibc/blob/master/termios/cfmakeraw.c#L20 iflag, oflag, cflag, lflag, ispeed, ospeed, cc = tflags iflag &= ~flags('IMAXBEL IXOFF INPCK BRKINT PARMRK ' - 'ISTRIP INLCR ICRNL IXON IGNPAR') - iflag &= ~flags('IGNBRK BRKINT PARMRK') + 'ISTRIP INLCR ICRNL IXON IGNPAR IGNBRK') oflag &= ~flags('OPOST') lflag &= ~flags('ECHO ECHOE ECHOK ECHONL ICANON ISIG ' 'IEXTEN NOFLSH TOSTOP PENDIN') From 927fb172d8eb2af5d0635d9e4099b471309432c1 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 6 Feb 2025 16:45:59 +0000 Subject: [PATCH 03/25] mitogen: Log skipped termios attributes refs #1121 --- docs/changelog.rst | 1 + mitogen/parent.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index dfbf1643..92cd8966 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,7 @@ To avail of fixes in an unreleased version, please download a ZIP file In progress (unreleased) ------------------------ +* :gh:issue:`1121` :mod:`mitogen`: Log skipped :py:mod:`termios` attributes v0.3.22 (2025-02-04) diff --git a/mitogen/parent.py b/mitogen/parent.py index 3c5e48ba..5b397d9f 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -224,8 +224,16 @@ def flags(names): Return the result of ORing a set of (space separated) :py:mod:`termios` module constants together. """ - return sum(getattr(termios, name, 0) - for name in names.split()) + i = 0 + skipped = [] + for name in names.split(): + try: + i |= getattr(termios, name) + except AttributeError: + skipped.append(name) + if skipped: + LOG.debug('Skipped termios attributes: %s', ', '.join(skipped)) + return i def cfmakeraw(tflags): From a0d3858ff9307fcd523d775f36bc853b128413fd Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 6 Feb 2025 17:38:57 +0000 Subject: [PATCH 04/25] packaging: Avoid ast module, requires Python = 2.6 fixes #1238, refs #1236 --- docs/changelog.rst | 1 + setup.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 92cd8966..e8c7e956 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -22,6 +22,7 @@ In progress (unreleased) ------------------------ * :gh:issue:`1121` :mod:`mitogen`: Log skipped :py:mod:`termios` attributes +* :gh:issue:`1238` packaging: Avoid :py:mod:`ast`, requires Python = 2.6 v0.3.22 (2025-02-04) diff --git a/setup.py b/setup.py index 9cb30427..375a96ff 100644 --- a/setup.py +++ b/setup.py @@ -26,20 +26,22 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import ast import os +import re from setuptools import find_packages, setup def grep_version(): path = os.path.join(os.path.dirname(__file__), 'mitogen/__init__.py') + version_pattern = re.compile( + r"__version__ = \((\d+), (\d+), (\d+)(?:, '(dev)')?\)", + ) with open(path) as fp: - for line in fp: - if line.startswith('__version__'): - _, _, s = line.partition('=') - parts = ast.literal_eval(s.strip()) - return '.'.join(str(part) for part in parts) + match = version_pattern.search(fp.read()) + if match is None: + raise ValueError('Could not find __version__ string in %s', path) + return '.'.join(str(part) for part in match.groups()) def long_description(): From 5283e6756bce78bb39cbda372545ab3869c82095 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 6 Feb 2025 18:55:57 +0000 Subject: [PATCH 05/25] CI: Statically specify test usernames and group names This makes it easier to grep for a username and to discover how the user was create. Hence it should be easier to understand/debug tests. --- docs/changelog.rst | 1 + tests/image_prep/_user_accounts.yml | 83 ++++++++++++++++------------- 2 files changed, 46 insertions(+), 38 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e8c7e956..7ae6e5da 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,6 +23,7 @@ In progress (unreleased) * :gh:issue:`1121` :mod:`mitogen`: Log skipped :py:mod:`termios` attributes * :gh:issue:`1238` packaging: Avoid :py:mod:`ast`, requires Python = 2.6 +* :gh:issue:`1118` CI: Statically specify test usernames and group names v0.3.22 (2025-02-04) diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index ad5a4ef5..bb8067f9 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -13,38 +13,45 @@ vars: distro: "{{ansible_distribution}}" special_users: - - has_sudo - - has_sudo_nopw - - has_sudo_pubkey - - pw_required - - readonly_homedir - - require_tty - - require_tty_pw_required - - permdenied - - slow_user - - webapp - - sudo1 - - sudo2 - - sudo3 - - sudo4 + - name: mitogen__has_sudo + - name: mitogen__has_sudo_nopw + - name: mitogen__has_sudo_pubkey + - name: mitogen__pw_required + - name: mitogen__readonly_homedir + - name: mitogen__require_tty + - name: mitogen__require_tty_pw_required + - name: mitogen__permdenied + - name: mitogen__slow_user + - name: mitogen__webapp + - name: mitogen__sudo1 + - name: mitogen__sudo2 + - name: mitogen__sudo3 + - name: mitogen__sudo4 user_groups: - has_sudo: ['mitogen__group', '{{sudo_group[distro]}}'] - has_sudo_pubkey: ['mitogen__group', '{{sudo_group[distro]}}'] - has_sudo_nopw: ['mitogen__group', 'mitogen__sudo_nopw'] - sudo1: ['mitogen__group', 'mitogen__sudo_nopw'] - sudo2: ['mitogen__group', '{{sudo_group[distro]}}'] - sudo3: ['mitogen__group', '{{sudo_group[distro]}}'] - sudo4: ['mitogen__group', '{{sudo_group[distro]}}'] - - normal_users: "{{ - lookup('sequence', 'start=1 end=5 format=user%d', wantlist=True) - }}" + mitogen__has_sudo: ['mitogen__group', '{{ sudo_group[distro] }}'] + mitogen__has_sudo_pubkey: ['mitogen__group', '{{ sudo_group[distro] }}'] + mitogen__has_sudo_nopw: ['mitogen__group', 'mitogen__sudo_nopw'] + mitogen__sudo1: ['mitogen__group', 'mitogen__sudo_nopw'] + mitogen__sudo2: ['mitogen__group', '{{ sudo_group[distro] }}'] + mitogen__sudo3: ['mitogen__group', '{{ sudo_group[distro] }}'] + mitogen__sudo4: ['mitogen__group', '{{ sudo_group[distro] }}'] + + normal_users: + - name: mitogen__user1 + - name: mitogen__user2 + - name: mitogen__user3 + - name: mitogen__user4 + - name: mitogen__user5 all_users: "{{ special_users + normal_users }}" + + mitogen_test_groups: + - name: mitogen__group + - name: mitogen__sudo_nopw tasks: - name: Disable non-localhost SSH for Mitogen users when: false @@ -56,30 +63,30 @@ - name: Create Mitogen test groups group: - name: "mitogen__{{item}}" - with_items: - - group - - sudo_nopw + name: "{{ item.name }}" + loop: "{{ mitogen_test_groups }}" - name: Create user accounts + vars: + password: "{{ item.name | replace('mitogen__', '') }}_password" block: - user: - name: "mitogen__{{item}}" + name: "{{ item.name }}" shell: /bin/bash - groups: "{{user_groups[item]|default(['mitogen__group'])}}" - password: "{{ (item + '_password') | password_hash('sha256') }}" + groups: "{{ user_groups[item.name] | default(['mitogen__group']) }}" + password: "{{ password | password_hash('sha256') }}" with_items: "{{all_users}}" when: ansible_system != 'Darwin' - user: - name: "mitogen__{{item}}" + name: "{{ item.name }}" shell: /bin/bash group: staff groups: | {{ ['com.apple.access_ssh'] + - (user_groups[item] | default(['mitogen__group'])) + (user_groups[item.name] | default(['mitogen__group'])) }} - password: "{{item}}_password" + password: "{{ password }}" with_items: "{{all_users}}" when: ansible_system == 'Darwin' @@ -91,7 +98,7 @@ domain: /Library/Preferences/com.apple.loginwindow type: array key: HiddenUsersList - value: ['mitogen_{{item}}'] + value: ['{{ item.name }}'] - name: Check if AccountsService is used stat: @@ -102,7 +109,7 @@ when: ansible_system == 'Linux' and out.stat.exists with_items: "{{all_users}}" copy: - dest: /var/lib/AccountsService/users/mitogen__{{item}} + dest: /var/lib/AccountsService/users/{{ item.name }} mode: u=rw,go= content: | [User] @@ -188,7 +195,7 @@ - name: Allow passwordless for many accounts lineinfile: path: /etc/sudoers - line: "{{lookup('pipe', 'whoami')}} ALL = (mitogen__{{item}}:ALL) NOPASSWD:ALL" + line: "{{ lookup('pipe', 'whoami') }} ALL = ({{ item.name }}:ALL) NOPASSWD:ALL" validate: '/usr/sbin/visudo -cf %s' with_items: "{{normal_users}}" when: From 11d2d70fd8a0d4b79da6fc6b0f2c792a4497085d Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 7 Feb 2025 00:16:57 +0000 Subject: [PATCH 06/25] CI: Use native Ansible support to hide macOS users --- tests/image_prep/_user_accounts.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index bb8067f9..c47b1dce 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -86,20 +86,11 @@ ['com.apple.access_ssh'] + (user_groups[item.name] | default(['mitogen__group'])) }} + hidden: true password: "{{ password }}" with_items: "{{all_users}}" when: ansible_system == 'Darwin' - - name: Hide users from login window (Darwin). - when: ansible_system == 'Darwin' - with_items: "{{all_users}}" - osx_defaults: - array_add: true - domain: /Library/Preferences/com.apple.loginwindow - type: array - key: HiddenUsersList - value: ['{{ item.name }}'] - - name: Check if AccountsService is used stat: path: /var/lib/AccountsService/users From c92df356e645f7d318f82648e0de7e6d1dd9f67b Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 7 Feb 2025 00:23:18 +0000 Subject: [PATCH 07/25] CI: Consolidate sudoers config tasks --- tests/image_prep/_user_accounts.yml | 54 +++++++++++------------------ 1 file changed, 21 insertions(+), 33 deletions(-) diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index c47b1dce..d099091a 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -152,42 +152,30 @@ owner: mitogen__has_sudo_pubkey group: mitogen__group - - name: Require a TTY for two accounts - lineinfile: - path: /etc/sudoers - line: "{{item}}" - with_items: - - Defaults>mitogen__pw_required targetpw - - Defaults>mitogen__require_tty requiretty - - Defaults>mitogen__require_tty_pw_required requiretty,targetpw - - - name: Require password for two accounts - lineinfile: - path: /etc/sudoers - line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}:ALL) ALL" - validate: '/usr/sbin/visudo -cf %s' - with_items: - - mitogen__pw_required - - mitogen__require_tty_pw_required - when: - - ansible_virtualization_type != "docker" - - - name: Allow passwordless sudo for require_tty/readonly_homedir - lineinfile: + - name: Configure sudoers defaults + blockinfile: path: /etc/sudoers - line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}:ALL) NOPASSWD:ALL" - validate: '/usr/sbin/visudo -cf %s' - with_items: - - mitogen__require_tty - - mitogen__readonly_homedir - when: - - ansible_virtualization_type != "docker" + marker: "# {mark} Mitogen test defaults" + block: | + Defaults>mitogen__pw_required targetpw + Defaults>mitogen__require_tty requiretty + Defaults>mitogen__require_tty_pw_required requiretty,targetpw + prepend_newline: true - - name: Allow passwordless for many accounts - lineinfile: + - name: Configure sudoers users + blockinfile: path: /etc/sudoers - line: "{{ lookup('pipe', 'whoami') }} ALL = ({{ item.name }}:ALL) NOPASSWD:ALL" + marker: "# {mark} Mitogen test users" + block: | + # User Host(s) = (runas user:runas group) Command(s) + {{ lookup('pipe', 'whoami') }} ALL = (mitogen__pw_required:ALL) ALL + {{ lookup('pipe', 'whoami') }} ALL = (mitogen__require_tty_pw_required:ALL) ALL + {{ lookup('pipe', 'whoami') }} ALL = (mitogen__require_tty:ALL) NOPASSWD:ALL + {{ lookup('pipe', 'whoami') }} ALL = (mitogen__readonly_homedir:ALL) NOPASSWD:ALL + {% for runas_user in normal_users %} + {{ lookup('pipe', 'whoami') }} ALL = ({{ runas_user.name }}:ALL) NOPASSWD:ALL + {% endfor %} + prepend_newline: true validate: '/usr/sbin/visudo -cf %s' - with_items: "{{normal_users}}" when: - ansible_virtualization_type != "docker" From 78b440104e51c132ca33a6bbc73d70ac61a60333 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 7 Feb 2025 00:26:32 +0000 Subject: [PATCH 08/25] CI: Validate sudoers file --- tests/image_prep/_user_accounts.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index d099091a..10834efc 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -161,6 +161,7 @@ Defaults>mitogen__require_tty requiretty Defaults>mitogen__require_tty_pw_required requiretty,targetpw prepend_newline: true + validate: '/usr/sbin/visudo -cf %s' - name: Configure sudoers users blockinfile: From 0b216c815abb14bc16d48fe9b0dd74f78ad4aad2 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 7 Feb 2025 10:52:55 +0000 Subject: [PATCH 09/25] CI: Consolidate `apt-get install`s --- .ci/ansible_tests.py | 5 ----- .github/workflows/tests.yml | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 4a7bedae..6c78bb0c 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -71,11 +71,6 @@ with ci_lib.Fold('job_setup'): ci_lib.dump_file(inventory_path) - if not ci_lib.exists_in_path('sshpass'): - ci_lib.run("sudo apt-get update") - ci_lib.run("sudo apt-get install -y sshpass") - - with ci_lib.Fold('ansible'): playbook = os.environ.get('PLAYBOOK', 'all.yml') try: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9a275a00..bbbc33e8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -96,7 +96,7 @@ jobs: set -o errexit -o nounset -o pipefail sudo apt-get update - sudo apt-get install -y python2-dev python3-pip virtualenv + sudo apt-get install -y python2-dev python3-pip sshpass virtualenv - name: Show Python versions run: | set -o errexit -o nounset -o pipefail From 2c7eda1dc101cc6346b8120c5ce8fb2cac37fe75 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 7 Feb 2025 10:53:45 +0000 Subject: [PATCH 10/25] CI: Fix NameError in ci_lib._have_cmd() --- .ci/ci_lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index afb62e02..25648bbe 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -56,7 +56,7 @@ def _have_cmd(args): if exc.errno == errno.ENOENT: return False raise - except subprocess.CallProcessError: + except subprocess.CalledProcessError: return False return True From 2095342245a2af83926bf09eb726555e31cbfd49 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 7 Feb 2025 10:55:07 +0000 Subject: [PATCH 11/25] CI: Remove unused ci_lib.have_*() functions --- .ci/ci_lib.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 25648bbe..c078791f 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -61,14 +61,6 @@ def _have_cmd(args): return True -def have_apt(): - return _have_cmd(['apt', '--help']) - - -def have_brew(): - return _have_cmd(['brew', 'help']) - - def have_docker(): return _have_cmd(['docker', 'info']) From 620bc3a944a788a4f645edbcbbc940eee3e9e764 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 7 Feb 2025 11:44:00 +0000 Subject: [PATCH 12/25] CI: Don't copy SSH private key to temporary dir Running tests aren't using the copy & it wasn't being cleaned up. --- .ci/ansible_tests.py | 2 +- .ci/ci_lib.py | 13 ++----------- .ci/debops_common_tests.py | 3 ++- .ci/localhost_ansible_tests.py | 3 +-- docs/changelog.rst | 1 + 5 files changed, 7 insertions(+), 15 deletions(-) diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 6c78bb0c..01a012f5 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -40,8 +40,8 @@ with ci_lib.Fold('docker_setup'): with ci_lib.Fold('job_setup'): + os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8)) os.chdir(TESTS_DIR) - os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 8)) ci_lib.run("mkdir %s", HOSTS_DIR) for path in glob.glob(TESTS_DIR + '/hosts/*'): diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index c078791f..b0c9c04a 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -28,6 +28,7 @@ os.chdir( ) +GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) DISTRO_SPECS = os.environ.get( 'MITOGEN_TEST_DISTRO_SPECS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004', @@ -36,6 +37,7 @@ IMAGE_TEMPLATE = os.environ.get( 'MITOGEN_TEST_IMAGE_TEMPLATE', 'public.ecr.aws/n5z0e8q9/%(distro)s-test', ) +TESTS_SSH_PRIVATE_KEY_FILE = os.path.join(GIT_ROOT, 'tests/data/docker/mitogen__has_sudo_pubkey.key') _print = print @@ -191,20 +193,9 @@ class Fold(object): def __exit__(self, _1, _2, _3): pass -GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) TMP = TempDir().path -# We copy this out of the way to avoid random stuff modifying perms in the Git -# tree (like git pull). -src_key_file = os.path.join(GIT_ROOT, - 'tests/data/docker/mitogen__has_sudo_pubkey.key') -key_file = os.path.join(TMP, - 'mitogen__has_sudo_pubkey.key') -shutil.copyfile(src_key_file, key_file) -os.chmod(key_file, int('0600', 8)) - - os.environ['PYTHONDONTWRITEBYTECODE'] = 'x' os.environ['PYTHONPATH'] = '%s:%s' % ( os.environ.get('PYTHONPATH', ''), diff --git a/.ci/debops_common_tests.py b/.ci/debops_common_tests.py index b065486f..b9b0030c 100755 --- a/.ci/debops_common_tests.py +++ b/.ci/debops_common_tests.py @@ -22,6 +22,7 @@ with ci_lib.Fold('docker_setup'): with ci_lib.Fold('job_setup'): + os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8)) ci_lib.run('debops-init %s', project_dir) os.chdir(project_dir) @@ -45,7 +46,7 @@ with ci_lib.Fold('job_setup'): "\n" # Speed up slow DH generation. "dhparam__bits: ['128', '64']\n" - % (ci_lib.key_file,) + % (ci_lib.TESTS_SSH_PRIVATE_KEY_FILE,) ) with open(inventory_path, 'a') as fp: diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index e4b8329b..29022bf7 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -13,7 +13,6 @@ import ci_lib TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible') IMAGE_PREP_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/image_prep') HOSTS_DIR = os.path.join(TESTS_DIR, 'hosts') -KEY_PATH = os.path.join(TESTS_DIR, '../data/docker/mitogen__has_sudo_pubkey.key') with ci_lib.Fold('unit_tests'): @@ -22,7 +21,7 @@ with ci_lib.Fold('unit_tests'): with ci_lib.Fold('job_setup'): - os.chmod(KEY_PATH, int('0600', 8)) + os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8)) # NOTE: sshpass v1.06 causes errors so pegging to 1.05 -> "msg": "Error when changing password","out": "passwd: DS error: eDSAuthFailed\n", # there's a checksum error with "brew install http://git.io/sshpass.rb" though, so installing manually if not ci_lib.exists_in_path('sshpass'): diff --git a/docs/changelog.rst b/docs/changelog.rst index 7ae6e5da..1fbee883 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -24,6 +24,7 @@ In progress (unreleased) * :gh:issue:`1121` :mod:`mitogen`: Log skipped :py:mod:`termios` attributes * :gh:issue:`1238` packaging: Avoid :py:mod:`ast`, requires Python = 2.6 * :gh:issue:`1118` CI: Statically specify test usernames and group names +* :gh:issue:`1118` CI: Don't copy SSH private key to temporary dir v0.3.22 (2025-02-04) From f659213159db7fd7d556a5ef0038f775d5e89af3 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 7 Feb 2025 12:19:16 +0000 Subject: [PATCH 13/25] CI: Don't share temporary directory between test groupings Each grouping gets an independant dir, e.g. - ansible -> /tmp/mitogen_ci_ansible - debops -> /tmp/mitogen_ci_debops Importing ci_lib no longer creates a temporary directory as a side effect. --- .ci/ansible_tests.py | 12 +++++++----- .ci/ci_lib.py | 7 ++----- .ci/debops_common_tests.py | 3 ++- docs/changelog.rst | 1 + 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 01a012f5..0d67c3b2 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -14,7 +14,8 @@ import ci_lib TEMPLATES_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible/templates') TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible') -HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts') +TMP = ci_lib.TempDir(prefix='mitogen_ci_ansible') +TMP_HOSTS_DIR = os.path.join(TMP.path, 'hosts') def pause_if_interactive(): @@ -43,10 +44,10 @@ with ci_lib.Fold('job_setup'): os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8)) os.chdir(TESTS_DIR) - ci_lib.run("mkdir %s", HOSTS_DIR) + os.mkdir(TMP_HOSTS_DIR) for path in glob.glob(TESTS_DIR + '/hosts/*'): if not path.endswith('default.hosts'): - ci_lib.run("ln -s %s %s", path, HOSTS_DIR) + os.symlink(path, os.path.join(TMP_HOSTS_DIR, os.path.basename(path))) distros = collections.defaultdict(list) families = collections.defaultdict(list) @@ -60,7 +61,7 @@ with ci_lib.Fold('job_setup'): trim_blocks=True, # Remove first newline after a block ) inventory_template = jinja_env.get_template('test-targets.j2') - inventory_path = os.path.join(HOSTS_DIR, 'target') + inventory_path = os.path.join(TMP_HOSTS_DIR, 'test-targets.ini') with open(inventory_path, 'w') as fp: fp.write(inventory_template.render( @@ -75,7 +76,8 @@ with ci_lib.Fold('ansible'): playbook = os.environ.get('PLAYBOOK', 'all.yml') try: ci_lib.run('./run_ansible_playbook.py %s -i "%s" %s', - playbook, HOSTS_DIR, ' '.join(sys.argv[1:])) + playbook, TMP_HOSTS_DIR, ' '.join(sys.argv[1:]), + ) except: pause_if_interactive() raise diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index b0c9c04a..b37472a2 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -179,8 +179,8 @@ def exists_in_path(progname): class TempDir(object): - def __init__(self): - self.path = tempfile.mkdtemp(prefix='mitogen_ci_lib') + def __init__(self, prefix='mitogen_ci_lib'): + self.path = tempfile.mkdtemp(prefix=prefix) atexit.register(self.destroy) def destroy(self, rmtree=shutil.rmtree): @@ -193,9 +193,6 @@ class Fold(object): def __exit__(self, _1, _2, _3): pass -TMP = TempDir().path - - os.environ['PYTHONDONTWRITEBYTECODE'] = 'x' os.environ['PYTHONPATH'] = '%s:%s' % ( os.environ.get('PYTHONPATH', ''), diff --git a/.ci/debops_common_tests.py b/.ci/debops_common_tests.py index b9b0030c..6fd50124 100755 --- a/.ci/debops_common_tests.py +++ b/.ci/debops_common_tests.py @@ -6,7 +6,8 @@ import sys import ci_lib -project_dir = os.path.join(ci_lib.TMP, 'project') +TMP = ci_lib.TempDir(prefix='mitogen_ci_debops') +project_dir = os.path.join(TMP.path, 'project') vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml' inventory_path = 'ansible/inventory/hosts' docker_hostname = ci_lib.get_docker_hostname() diff --git a/docs/changelog.rst b/docs/changelog.rst index 1fbee883..ba4a608b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,6 +25,7 @@ In progress (unreleased) * :gh:issue:`1238` packaging: Avoid :py:mod:`ast`, requires Python = 2.6 * :gh:issue:`1118` CI: Statically specify test usernames and group names * :gh:issue:`1118` CI: Don't copy SSH private key to temporary dir +* :gh:issue:`1118` CI: Don't share temporary directory between test groupings v0.3.22 (2025-02-04) From a376daa04db22cb04e455298692ceb5dc6c5b7de Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 7 Feb 2025 12:27:11 +0000 Subject: [PATCH 14/25] CI: Consolidate directory path constants --- .ci/ansible_tests.py | 10 +++++----- .ci/ci_lib.py | 4 ++++ .ci/localhost_ansible_tests.py | 11 +++-------- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 0d67c3b2..4bf86bca 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -12,8 +12,6 @@ import jinja2 import ci_lib -TEMPLATES_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible/templates') -TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible') TMP = ci_lib.TempDir(prefix='mitogen_ci_ansible') TMP_HOSTS_DIR = os.path.join(TMP.path, 'hosts') @@ -42,10 +40,10 @@ with ci_lib.Fold('docker_setup'): with ci_lib.Fold('job_setup'): os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8)) - os.chdir(TESTS_DIR) + os.chdir(ci_lib.ANSIBLE_TESTS_DIR) os.mkdir(TMP_HOSTS_DIR) - for path in glob.glob(TESTS_DIR + '/hosts/*'): + for path in glob.glob(os.path.join(ci_lib.ANSIBLE_TESTS_HOSTS_DIR, '*')): if not path.endswith('default.hosts'): os.symlink(path, os.path.join(TMP_HOSTS_DIR, os.path.basename(path))) @@ -56,7 +54,9 @@ with ci_lib.Fold('job_setup'): families[container['family']].append(container['name']) jinja_env = jinja2.Environment( - loader=jinja2.FileSystemLoader(searchpath=TEMPLATES_DIR), + loader=jinja2.FileSystemLoader( + searchpath=ci_lib.ANSIBLE_TESTS_TEMPLATES_DIR, + ), lstrip_blocks=True, # Remove spaces and tabs from before a block trim_blocks=True, # Remove first newline after a block ) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index b37472a2..9f333dde 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -29,10 +29,14 @@ os.chdir( GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +ANSIBLE_TESTS_DIR = os.path.join(GIT_ROOT, 'tests/ansible') +ANSIBLE_TESTS_HOSTS_DIR = os.path.join(GIT_ROOT, 'tests/ansible/hosts') +ANSIBLE_TESTS_TEMPLATES_DIR = os.path.join(GIT_ROOT, 'tests/ansible/templates') DISTRO_SPECS = os.environ.get( 'MITOGEN_TEST_DISTRO_SPECS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004', ) +IMAGE_PREP_DIR = os.path.join(GIT_ROOT, 'tests/image_prep') IMAGE_TEMPLATE = os.environ.get( 'MITOGEN_TEST_IMAGE_TEMPLATE', 'public.ecr.aws/n5z0e8q9/%(distro)s-test', diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index 29022bf7..359dc195 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -10,11 +10,6 @@ import sys import ci_lib -TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible') -IMAGE_PREP_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/image_prep') -HOSTS_DIR = os.path.join(TESTS_DIR, 'hosts') - - with ci_lib.Fold('unit_tests'): os.environ['SKIP_MITOGEN'] = '1' ci_lib.run('./run_tests -v') @@ -50,11 +45,11 @@ with ci_lib.Fold('machine_prep'): subprocess.check_call('sudo chmod 700 ~root/.ssh', shell=True) subprocess.check_call('sudo chmod 600 ~root/.ssh/authorized_keys', shell=True) - os.chdir(IMAGE_PREP_DIR) + os.chdir(ci_lib.IMAGE_PREP_DIR) ci_lib.run("ansible-playbook -c local -i localhost, macos_localhost.yml") if os.path.expanduser('~mitogen__user1') == '~mitogen__user1': - os.chdir(IMAGE_PREP_DIR) + os.chdir(ci_lib.IMAGE_PREP_DIR) ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml") cmd = ';'.join([ @@ -79,7 +74,7 @@ with ci_lib.Fold('machine_prep'): with ci_lib.Fold('ansible'): - os.chdir(TESTS_DIR) + os.chdir(ci_lib.ANSIBLE_TESTS_DIR) playbook = os.environ.get('PLAYBOOK', 'all.yml') ci_lib.run('./run_ansible_playbook.py %s %s', playbook, ' '.join(sys.argv[1:])) From 67ececc804bcd207d9429fe9cc0e32ba74bf6ecc Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 7 Feb 2025 15:42:11 +0000 Subject: [PATCH 15/25] ci: Use GitHub Container Registery images tagged 2021 Previously (and implicitly) used "latest". The tag 2021 is new today, the image contents have not changed since they were generated in 2021. They have moved container registry twice since 2021 - #791 Docker -> Amazon Elastic Container Registry (public.ecr.aws/n5z0e8q) - #1128 Amazon ECR -> GitHub Container Registry (ghcr.io/mitogen-hq) This commit also removes the last references to ECR. --- .ci/ci_lib.py | 6 +++--- .github/workflows/tests.yml | 5 ----- tests/testlib.py | 2 +- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 9f333dde..96c85811 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -39,7 +39,7 @@ DISTRO_SPECS = os.environ.get( IMAGE_PREP_DIR = os.path.join(GIT_ROOT, 'tests/image_prep') IMAGE_TEMPLATE = os.environ.get( 'MITOGEN_TEST_IMAGE_TEMPLATE', - 'public.ecr.aws/n5z0e8q9/%(distro)s-test', + 'ghcr.io/mitogen-hq/%(distro)s-test:2021', ) TESTS_SSH_PRIVATE_KEY_FILE = os.path.join(GIT_ROOT, 'tests/data/docker/mitogen__has_sudo_pubkey.key') @@ -227,7 +227,7 @@ def container_specs( [{'distro': 'debian11', 'family': 'debian', 'hostname': 'localhost', - 'image': 'public.ecr.aws/n5z0e8q9/debian11-test', + 'image': 'ghcr.io/mitogen-hq/debian11-test:2021', 'index': 1, 'name': 'target-debian11-1', 'port': 2201, @@ -235,7 +235,7 @@ def container_specs( {'distro': 'centos6', 'family': 'centos', 'hostname': 'localhost', - 'image': 'public.ecr.aws/n5z0e8q9/centos6-test', + 'image': 'ghcr.io/mitogen-hq/centos6-test:2021', 'index': 2, 'name': 'target-centos6-2', 'port': 2202, diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bbbc33e8..393a52c7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,11 +8,6 @@ on: branches-ignore: - docs-master -env: - #ANSIBLE_VERBOSITY: 3 - #MITOGEN_LOG_LEVEL: DEBUG - MITOGEN_TEST_IMAGE_TEMPLATE: "ghcr.io/mitogen-hq/%(distro)s-test" - # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners # https://github.com/actions/runner-images/blob/main/README.md#software-and-image-support jobs: diff --git a/tests/testlib.py b/tests/testlib.py index 05779dc0..e4fa25b8 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -57,7 +57,7 @@ DISTRO_SPECS = os.environ.get( ) IMAGE_TEMPLATE = os.environ.get( 'MITOGEN_TEST_IMAGE_TEMPLATE', - 'public.ecr.aws/n5z0e8q9/%(distro)s-test', + 'ghcr.io/mitogen-hq/%(distro)s-test:2021', ) TESTS_DIR = os.path.join(os.path.dirname(__file__)) From 8e58c4a7592c640fb5e5e80c5878c1a79eb89f55 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 26 Feb 2025 11:05:54 +0000 Subject: [PATCH 16/25] ci: Decouple image prep from Ansible controller reporting This will allow image preparation using Ansible versions that predate import_playbook. --- tests/image_prep/_user_accounts.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 10834efc..01747876 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -3,9 +3,6 @@ # # WARNING: this creates non-privilged accounts with pre-set passwords! # - -- import_playbook: ../ansible/setup/report_controller.yml - - hosts: all gather_facts: true strategy: mitogen_free From 20e23b5bd9621f7c5d45e60c38c91dff2bd54150 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 26 Feb 2025 11:11:08 +0000 Subject: [PATCH 17/25] ci: Name all image prep plays --- tests/image_prep/_container_setup.yml | 7 ++++--- tests/image_prep/_user_accounts.yml | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/image_prep/_container_setup.yml b/tests/image_prep/_container_setup.yml index 2972adda..7b466fc0 100644 --- a/tests/image_prep/_container_setup.yml +++ b/tests/image_prep/_container_setup.yml @@ -1,5 +1,5 @@ - -- hosts: all +- name: Bootstrap containers + hosts: all strategy: linear gather_facts: false tasks: @@ -15,7 +15,8 @@ fi when: bootstrap_packages | length -- hosts: all +- name: Setup containers + hosts: all strategy: mitogen_free # Resource limitation, my laptop freezes doing every container concurrently serial: 4 diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 01747876..14ac87f0 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -3,7 +3,8 @@ # # WARNING: this creates non-privilged accounts with pre-set passwords! # -- hosts: all +- name: Mitogen test users and groups + hosts: all gather_facts: true strategy: mitogen_free become: true From f2e0e552accafd924f71c95005f666e37a018386 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 26 Feb 2025 11:20:35 +0000 Subject: [PATCH 18/25] ci: Fix sshd configuration during image prep This will allow image preparation using Ansible versions that - predate ansible_facts.* - predate loop keyword - predate collections --- tests/image_prep/_container_setup.yml | 1 + tests/image_prep/_user_accounts.yml | 4 +--- tests/image_prep/macos_localhost.yml | 1 + tests/image_prep/roles/sshd/tasks/main.yml | 17 +++-------------- .../roles/sshd_container/handlers/main.yml | 2 ++ .../roles/sshd_macos/handlers/main.yml | 6 ++++++ 6 files changed, 14 insertions(+), 17 deletions(-) create mode 100644 tests/image_prep/roles/sshd_container/handlers/main.yml create mode 100644 tests/image_prep/roles/sshd_macos/handlers/main.yml diff --git a/tests/image_prep/_container_setup.yml b/tests/image_prep/_container_setup.yml index 7b466fc0..06cfc967 100644 --- a/tests/image_prep/_container_setup.yml +++ b/tests/image_prep/_container_setup.yml @@ -32,6 +32,7 @@ roles: - role: sshd + - role: sshd_container tasks: - name: Ensure requisite apt packages are installed diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 14ac87f0..3af21db7 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -62,7 +62,7 @@ - name: Create Mitogen test groups group: name: "{{ item.name }}" - loop: "{{ mitogen_test_groups }}" + with_items: "{{ mitogen_test_groups }}" - name: Create user accounts vars: @@ -158,7 +158,6 @@ Defaults>mitogen__pw_required targetpw Defaults>mitogen__require_tty requiretty Defaults>mitogen__require_tty_pw_required requiretty,targetpw - prepend_newline: true validate: '/usr/sbin/visudo -cf %s' - name: Configure sudoers users @@ -174,7 +173,6 @@ {% for runas_user in normal_users %} {{ lookup('pipe', 'whoami') }} ALL = ({{ runas_user.name }}:ALL) NOPASSWD:ALL {% endfor %} - prepend_newline: true validate: '/usr/sbin/visudo -cf %s' when: - ansible_virtualization_type != "docker" diff --git a/tests/image_prep/macos_localhost.yml b/tests/image_prep/macos_localhost.yml index c046a2bc..b93bc780 100644 --- a/tests/image_prep/macos_localhost.yml +++ b/tests/image_prep/macos_localhost.yml @@ -5,3 +5,4 @@ become: true roles: - role: sshd + - role: sshd_macos diff --git a/tests/image_prep/roles/sshd/tasks/main.yml b/tests/image_prep/roles/sshd/tasks/main.yml index 837c7d15..8f3e7e03 100644 --- a/tests/image_prep/roles/sshd/tasks/main.yml +++ b/tests/image_prep/roles/sshd/tasks/main.yml @@ -9,23 +9,12 @@ path: "{{ sshd_config_file }}" line: "{{ item.line }}" regexp: "{{ item.regexp }}" - loop: + with_items: - line: Banner /etc/ssh/banner.txt regexp: '^#? *Banner.*' - line: MaxAuthTries {{ sshd_config__max_auth_tries }} regexp: '^#? *MaxAuthTries.*' - line: PermitRootLogin yes regexp: '.*PermitRootLogin.*' - loop_control: - label: "{{ item.line }}" - register: configure_sshd_result - -- name: Restart sshd - shell: | - launchctl unload /System/Library/LaunchDaemons/ssh.plist - wait 5 - launchctl load -w /System/Library/LaunchDaemons/ssh.plist - changed_when: true - when: - - ansible_facts.distribution == "MacOSX" - - configure_sshd_result is changed + notify: + - Restart sshd # Handler in platform specific role diff --git a/tests/image_prep/roles/sshd_container/handlers/main.yml b/tests/image_prep/roles/sshd_container/handlers/main.yml new file mode 100644 index 00000000..cc7b9166 --- /dev/null +++ b/tests/image_prep/roles/sshd_container/handlers/main.yml @@ -0,0 +1,2 @@ +- name: Restart sshd + meta: noop diff --git a/tests/image_prep/roles/sshd_macos/handlers/main.yml b/tests/image_prep/roles/sshd_macos/handlers/main.yml new file mode 100644 index 00000000..6d4a7e21 --- /dev/null +++ b/tests/image_prep/roles/sshd_macos/handlers/main.yml @@ -0,0 +1,6 @@ +- name: Restart sshd + shell: | + launchctl unload /System/Library/LaunchDaemons/ssh.plist + wait 5 + launchctl load -w /System/Library/LaunchDaemons/ssh.plist + changed_when: true From 1e12edbf9517402fef40c315a4d641b1603828a8 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 26 Feb 2025 11:24:40 +0000 Subject: [PATCH 19/25] ci: Use file module to set mitogen__readonly_homedir permissions --- tests/image_prep/_user_accounts.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 3af21db7..bee89084 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -111,7 +111,11 @@ state: restarted - name: Readonly homedir for one account - shell: "chown -R root: ~mitogen__readonly_homedir" + file: + path: ~mitogen__readonly_homedir + owner: root + recurse: true + state: directory - name: Slow bash profile for one account copy: From fdbd1a8c9b73ada07f90cbb4d0637a35cf7134c2 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 26 Feb 2025 12:13:11 +0000 Subject: [PATCH 20/25] ci: Configure package managers using a role This allows code sharing between integration tests and test image prep. --- tests/ansible/hosts/group_vars/all.yml | 1 - tests/ansible/hosts/group_vars/centos8.yml | 2 +- tests/ansible/hosts/group_vars/debian11.yml | 5 ++++ tests/ansible/hosts/group_vars/debian9.yml | 2 +- .../issue_776__load_plugins_called_twice.yml | 28 ++++-------------- tests/image_prep/_container_setup.yml | 1 + .../roles/package_manager/defaults/main.yml | 2 ++ .../debian-archive-bookworm-automatic.gpg | Bin .../debian-archive-bullseye-automatic.gpg | Bin .../roles/package_manager/tasks/main.yml | 13 ++++++++ tox.ini | 1 + 11 files changed, 29 insertions(+), 26 deletions(-) create mode 100644 tests/ansible/hosts/group_vars/debian11.yml create mode 100644 tests/image_prep/roles/package_manager/defaults/main.yml rename tests/{ansible/regression => image_prep/roles/package_manager}/files/debian-archive-bookworm-automatic.gpg (100%) rename tests/{ansible/regression => image_prep/roles/package_manager}/files/debian-archive-bullseye-automatic.gpg (100%) create mode 100644 tests/image_prep/roles/package_manager/tasks/main.yml diff --git a/tests/ansible/hosts/group_vars/all.yml b/tests/ansible/hosts/group_vars/all.yml index 44e660fc..fa70b89d 100644 --- a/tests/ansible/hosts/group_vars/all.yml +++ b/tests/ansible/hosts/group_vars/all.yml @@ -27,4 +27,3 @@ become_unpriv_available: >- -}} pkg_mgr_python_interpreter: python -pkg_repos_overrides: [] diff --git a/tests/ansible/hosts/group_vars/centos8.yml b/tests/ansible/hosts/group_vars/centos8.yml index c90dd5f4..5ab4577d 100644 --- a/tests/ansible/hosts/group_vars/centos8.yml +++ b/tests/ansible/hosts/group_vars/centos8.yml @@ -1,7 +1,7 @@ --- pkg_mgr_python_interpreter: /usr/libexec/platform-python -pkg_repos_overrides: +package_manager_repos: - dest: /etc/yum.repos.d/CentOS-Linux-AppStream.repo content: | [appstream] diff --git a/tests/ansible/hosts/group_vars/debian11.yml b/tests/ansible/hosts/group_vars/debian11.yml new file mode 100644 index 00000000..9f62f43c --- /dev/null +++ b/tests/ansible/hosts/group_vars/debian11.yml @@ -0,0 +1,5 @@ +package_manager_keys: + - src: debian-archive-bullseye-automatic.gpg # Debian 11 + dest: /etc/apt/trusted.gpg.d/debian-archive-bullseye-automatic.gpg + - src: debian-archive-bookworm-automatic.gpg # Debian 12 + dest: /etc/apt/trusted.gpg.d/debian-archive-bookworm-automatic.gpg diff --git a/tests/ansible/hosts/group_vars/debian9.yml b/tests/ansible/hosts/group_vars/debian9.yml index 4b180b13..e08b1ed2 100644 --- a/tests/ansible/hosts/group_vars/debian9.yml +++ b/tests/ansible/hosts/group_vars/debian9.yml @@ -1,4 +1,4 @@ -pkg_repos_overrides: +package_manager_repos: - dest: /etc/apt/sources.list content: | deb http://archive.debian.org/debian stretch main contrib non-free diff --git a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml index ef573276..d482c41c 100755 --- a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml +++ b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml @@ -10,7 +10,7 @@ vars: ansible_python_interpreter: "{{ pkg_mgr_python_interpreter }}" package: rsync # Chosen to exist in all tested distros/package managers - tasks: + pre_tasks: # The package management modules require using the same Python version # as the target's package manager libraries. This is sometimes in conflict # with Ansible requirements, e.g. Ansible 10 (ansible-core 2.17) does not @@ -19,31 +19,13 @@ when: - ansible_version.full is version('2.17', '>=', strict=True) + roles: + - role: package_manager + + tasks: - name: Gather facts manually setup: - - name: Switch to archived package repositories - copy: - dest: "{{ item.dest }}" - content: "{{ item.content }}" - mode: u=rw,go=r - loop: "{{ pkg_repos_overrides }}" - loop_control: - label: "{{ item.dest }}" - - - name: Add signing keys - copy: - src: "{{ item.src }}" - dest: "/etc/apt/trusted.gpg.d/{{ item.src | basename }}" - mode: u=rw,go=r - loop: - - src: debian-archive-bullseye-automatic.gpg # Debian 11 - - src: debian-archive-bookworm-automatic.gpg # Debian 12 - when: - # Ideally this would check for Debian 11, but distribution_major_version - # is unpopulated sometimes. - - ansible_facts.distribution == "Debian" - - name: Update package index apt: update_cache: true diff --git a/tests/image_prep/_container_setup.yml b/tests/image_prep/_container_setup.yml index 06cfc967..4aa3b46d 100644 --- a/tests/image_prep/_container_setup.yml +++ b/tests/image_prep/_container_setup.yml @@ -31,6 +31,7 @@ - ansible_facts.virtualization_type != "docker" roles: + - role: package_manager - role: sshd - role: sshd_container diff --git a/tests/image_prep/roles/package_manager/defaults/main.yml b/tests/image_prep/roles/package_manager/defaults/main.yml new file mode 100644 index 00000000..c384b19a --- /dev/null +++ b/tests/image_prep/roles/package_manager/defaults/main.yml @@ -0,0 +1,2 @@ +package_manager_keys: [] +package_manager_repos: [] diff --git a/tests/ansible/regression/files/debian-archive-bookworm-automatic.gpg b/tests/image_prep/roles/package_manager/files/debian-archive-bookworm-automatic.gpg similarity index 100% rename from tests/ansible/regression/files/debian-archive-bookworm-automatic.gpg rename to tests/image_prep/roles/package_manager/files/debian-archive-bookworm-automatic.gpg diff --git a/tests/ansible/regression/files/debian-archive-bullseye-automatic.gpg b/tests/image_prep/roles/package_manager/files/debian-archive-bullseye-automatic.gpg similarity index 100% rename from tests/ansible/regression/files/debian-archive-bullseye-automatic.gpg rename to tests/image_prep/roles/package_manager/files/debian-archive-bullseye-automatic.gpg diff --git a/tests/image_prep/roles/package_manager/tasks/main.yml b/tests/image_prep/roles/package_manager/tasks/main.yml new file mode 100644 index 00000000..f0bc8ad8 --- /dev/null +++ b/tests/image_prep/roles/package_manager/tasks/main.yml @@ -0,0 +1,13 @@ +- name: Add signing keys + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: u=rw,go=r + with_items: "{{ package_manager_keys }}" + +- name: Configure package repositories + copy: + dest: "{{ item.dest }}" + content: "{{ item.content }}" + mode: u=rw,go=r + with_items: "{{ package_manager_repos }}" diff --git a/tox.ini b/tox.ini index bd2d65e9..ddb8c88d 100644 --- a/tox.ini +++ b/tox.ini @@ -100,6 +100,7 @@ passenv = HOME MITOGEN_* setenv = + ANSIBLE_ROLES_PATH = {toxinidir}/tests/image_prep/roles # See also azure-pipelines.yml ANSIBLE_STRATEGY = mitogen_linear NOCOVERAGE_ERASE = 1 From 9b17f2839cdd89a1789278164e339fd198cffeb9 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 15 Apr 2025 08:53:37 +0100 Subject: [PATCH 21/25] ci: Label Github jobs by OS/distribution & Tox environment --- .github/workflows/tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 393a52c7..280211c4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,7 +11,8 @@ on: # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners # https://github.com/actions/runner-images/blob/main/README.md#software-and-image-support jobs: - linux: + u2004: + name: u2004 ${{ matrix.tox_env }} # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md runs-on: ubuntu-20.04 @@ -155,6 +156,7 @@ jobs: "$PYTHON" -m tox -e "${{ matrix.tox_env }}" macos: + name: macos ${{ matrix.tox_env }} # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md runs-on: macos-13 timeout-minutes: 15 @@ -256,7 +258,7 @@ jobs: check: if: always() needs: - - linux + - u2004 - macos runs-on: ubuntu-latest steps: From 1f737568b220d91b4c22b7a65e475dba833154c3 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 28 Apr 2025 09:48:46 +0100 Subject: [PATCH 22/25] ci: Remove vestigial Python 2.7 support in macOS Github jobs --- .github/workflows/tests.yml | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 280211c4..fb963a9b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -218,20 +218,7 @@ jobs: exit 1 fi - if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then - # GitHub macOS 12 images: python2.7 is installed, but not on $PATH - PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7" - "$PYTHON" -m ensurepip --user --altinstall --no-default-pip - "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" - elif [[ $PYTHON == "python2.7" ]]; then - curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py" - "$PYTHON" get-pip.py --user --no-python-version-warning - # Avoid Python 2.x pip masking system pip - rm -f ~/.local/bin/{easy_install,pip,wheel} - "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" - else - "$PYTHON" -m pip install -r "tests/requirements-tox.txt" - fi + "$PYTHON" -m pip install -r "tests/requirements-tox.txt" - name: Run tests env: GITHUB_ACTOR: ${{ github.actor }} @@ -247,11 +234,6 @@ jobs: exit 1 fi - if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then - # GitHub macOS 12 images: python2.7 is installed, but not on $PATH - PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7" - fi - "$PYTHON" -m tox -e "${{ matrix.tox_env }}" # https://github.com/marketplace/actions/alls-green From 27b4b77bbacd4e7985d257f961629ad41352baa5 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 28 Apr 2025 10:05:57 +0100 Subject: [PATCH 23/25] CI: Upgrade Github jobs from Ubuntu 20.04 to 22.04 & 24.04 Python 2.7 (distro package) and 3.6 (pyenv managed) jobs run on Ubuntu 22.04. More recent Pythons (distro or Github provided) run on 24.04. fixes #1256 Ansible tasks that run locally (e.g. `connection: local`, `delegate_to: localhost`) must now specify their `ansible_python_interpreter`, typically as `{{ ansible_playbook_python }}`; otherwise the system Python on the controller (e.g. `/usr/bin/python`) is likely to be used and this is often outside the version range supported by the Ansible verison under test. If this occurs then the symptom is often a failure to import a builtin from `ansible.module_utils.six.moves`, e.g. ``` fatal: [target-centos6-1]: FAILED! => changed=true cmd: - ansible - -m - shell - -c - local - -a - whoami - -i - /tmp/mitogen_ci_ansibled3llejls/hosts - test-targets delta: '0:00:02.076385' end: '2025-04-17 17:27:02.561500' msg: non-zero return code rc: 8 start: '2025-04-17 17:27:00.485115' stderr: |- stderr_lines: stdout: |- An exception occurred during task execution. To see the full traceback, use -vvv. The error was: from ansible.module_utils.six.moves import map, reduce, shlex_quote ``` --- .ci/ci_lib.py | 1 + .github/workflows/tests.yml | 144 +++++++++++++++--- docs/changelog.rst | 1 + tests/ansible/bench/loop-100-copies.yml | 2 + .../connection/disconnect_during_module.yml | 2 + .../context_service/reconnection.yml | 2 + .../integration/local/cwd_preserved.yml | 4 + .../playbook_semantics/delegate_to.yml | 8 + .../process/unix_socket_cleanup.yml | 6 +- .../integration/runner/missing_module.yml | 2 + tests/ansible/integration/ssh/timeouts.yml | 2 + tests/ansible/integration/ssh/variables.yml | 1 + .../strategy/mixed_vanilla_mitogen.yml | 5 + .../integration/stub_connections/kubectl.yml | 1 + .../integration/stub_connections/lxc.yml | 1 + .../integration/stub_connections/lxd.yml | 1 + .../stub_connections/mitogen_doas.yml | 1 + .../stub_connections/mitogen_sudo.yml | 1 + .../stub_connections/setns_lxc.yml | 4 +- .../stub_connections/setns_lxd.yml | 4 +- .../issue_1087__template_streamerror.yml | 2 + .../regression/issue_140__thread_pileup.yml | 4 + ...ue_152__local_action_wrong_interpreter.yml | 4 + .../regression/issue_952__ask_become_pass.yml | 2 + tests/testlib.py | 23 +++ tests/two_three_compat_test.py | 5 +- 26 files changed, 204 insertions(+), 29 deletions(-) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 96c85811..8193a34c 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -54,6 +54,7 @@ def print(*args, **kwargs): def _have_cmd(args): + # Code duplicated in testlib.py try: subprocess.run( args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fb963a9b..2524c2e7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,6 +2,10 @@ name: Tests +# env: +# ANSIBLE_VERBOSITY: 3 +# MITOGEN_LOG_LEVEL: DEBUG + on: pull_request: push: @@ -11,10 +15,10 @@ on: # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners # https://github.com/actions/runner-images/blob/main/README.md#software-and-image-support jobs: - u2004: - name: u2004 ${{ matrix.tox_env }} - # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md - runs-on: ubuntu-20.04 + u2204: + name: u2204 ${{ matrix.tox_env }} + # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md + runs-on: ubuntu-22.04 strategy: fail-fast: false @@ -32,6 +36,115 @@ jobs: python_version: '3.6' tox_env: py36-mode_ansible-ansible4 + - name: Mito_27 + tox_env: py27-mode_mitogen + - name: Mito_36 + python_version: '3.6' + tox_env: py36-mode_mitogen + + steps: + - uses: actions/checkout@v4 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Install build deps + run: | + set -o errexit -o nounset -o pipefail + + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + sudo apt-get update + + if [[ $PYTHON == "python2.7" ]]; then + sudo apt install -y python2-dev sshpass virtualenv + elif [[ $PYTHON == "python3.6" ]]; then + sudo apt install -y gcc-10 make libbz2-dev liblzma-dev libreadline-dev libsqlite3-dev libssl-dev sshpass virtualenv zlib1g-dev + curl --fail --silent --show-error --location https://pyenv.run | bash + CC=gcc-10 ~/.pyenv/bin/pyenv install --force 3.6 + else + echo 1>&2 "Python interpreter $PYTHON not available" + exit 1 + fi + - name: Show Python versions + run: | + set -o errexit -o nounset -o pipefail + + # macOS builders lack a realpath command + type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version + type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version + type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version + echo + + if [ -e /usr/bin/python ]; then + echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2 ]; then + echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2.7 ]; then + echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')" + fi + - name: Install tooling + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python2.7" ]]; then + curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py" + "$PYTHON" get-pip.py --user --no-python-version-warning + # Avoid Python 2.x pip masking system pip + rm -f ~/.local/bin/{easy_install,pip,wheel} + elif [[ $PYTHON == "python3.6" ]]; then + PYTHON="$HOME/.pyenv/versions/3.6.15/bin/python3.6" + fi + + "$PYTHON" -m pip install -r "tests/requirements-tox.txt" + - name: Run tests + env: + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python3.6" ]]; then + PYTHON="$HOME/.pyenv/versions/3.6.15/bin/python3.6" + fi + + "$PYTHON" -m tox -e "${{ matrix.tox_env }}" + + u2404: + name: u2404 ${{ matrix.tox_env }} + # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md + runs-on: ubuntu-24.04 + + strategy: + fail-fast: false + matrix: + include: - name: Ans_311_210 python_version: '3.11' tox_env: py311-mode_ansible-ansible2.10 @@ -67,11 +180,6 @@ jobs: python_version: '3.13' tox_env: py313-mode_ansible-ansible11-strategy_linear - - name: Mito_27 - tox_env: py27-mode_mitogen - - name: Mito_36 - python_version: '3.6' - tox_env: py36-mode_mitogen - name: Mito_313 python_version: '3.13' tox_env: py313-mode_mitogen @@ -92,7 +200,7 @@ jobs: set -o errexit -o nounset -o pipefail sudo apt-get update - sudo apt-get install -y python2-dev python3-pip sshpass virtualenv + sudo apt-get install -y sshpass virtualenv - name: Show Python versions run: | set -o errexit -o nounset -o pipefail @@ -126,18 +234,7 @@ jobs: exit 1 fi - if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then - "$PYTHON" -m ensurepip --user --altinstall --no-default-pip - "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" - elif [[ $PYTHON == "python2.7" ]]; then - curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py" - "$PYTHON" get-pip.py --user --no-python-version-warning - # Avoid Python 2.x pip masking system pip - rm -f ~/.local/bin/{easy_install,pip,wheel} - "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" - else - "$PYTHON" -m pip install -r "tests/requirements-tox.txt" - fi + "$PYTHON" -m pip install -r "tests/requirements-tox.txt" - name: Run tests env: GITHUB_ACTOR: ${{ github.actor }} @@ -240,7 +337,8 @@ jobs: check: if: always() needs: - - u2004 + - u2204 + - u2404 - macos runs-on: ubuntu-latest steps: diff --git a/docs/changelog.rst b/docs/changelog.rst index ba4a608b..904c4dd7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -26,6 +26,7 @@ In progress (unreleased) * :gh:issue:`1118` CI: Statically specify test usernames and group names * :gh:issue:`1118` CI: Don't copy SSH private key to temporary dir * :gh:issue:`1118` CI: Don't share temporary directory between test groupings +* :gh:issue:`1256` CI: Upgrade Github jobs from Ubuntu 20.04 to 22.04 & 24.04 v0.3.22 (2025-02-04) diff --git a/tests/ansible/bench/loop-100-copies.yml b/tests/ansible/bench/loop-100-copies.yml index 2a8e7d6f..4fe5276e 100644 --- a/tests/ansible/bench/loop-100-copies.yml +++ b/tests/ansible/bench/loop-100-copies.yml @@ -4,6 +4,8 @@ - name: Create file tree connection: local + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" shell: > mkdir -p /tmp/filetree.in; for i in `seq -f /tmp/filetree.in/%g 1 100`; do echo $RANDOM > $i; done; diff --git a/tests/ansible/integration/connection/disconnect_during_module.yml b/tests/ansible/integration/connection/disconnect_during_module.yml index 3846d037..d72062e9 100644 --- a/tests/ansible/integration/connection/disconnect_during_module.yml +++ b/tests/ansible/integration/connection/disconnect_during_module.yml @@ -10,6 +10,8 @@ - name: Run _disconnect_during_module.yml delegate_to: localhost + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" environment: ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: | diff --git a/tests/ansible/integration/context_service/reconnection.yml b/tests/ansible/integration/context_service/reconnection.yml index 6fb605a2..b0b4a735 100644 --- a/tests/ansible/integration/context_service/reconnection.yml +++ b/tests/ansible/integration/context_service/reconnection.yml @@ -19,6 +19,8 @@ bash -c "( sleep 3; kill -9 {{ssh_account_env.pid}}; ) & disown" - connection: local + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" shell: sleep 3 - wait_for_connection: diff --git a/tests/ansible/integration/local/cwd_preserved.yml b/tests/ansible/integration/local/cwd_preserved.yml index 0b1ddef4..f64ebd79 100644 --- a/tests/ansible/integration/local/cwd_preserved.yml +++ b/tests/ansible/integration/local/cwd_preserved.yml @@ -8,11 +8,15 @@ hosts: test-targets tasks: - name: Get local cwd + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" connection: local command: pwd register: pwd - connection: local + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" stat: path: "{{pwd.stdout}}/cwd_preserved.yml" register: stat diff --git a/tests/ansible/integration/playbook_semantics/delegate_to.yml b/tests/ansible/integration/playbook_semantics/delegate_to.yml index 61b4d03d..88415f68 100644 --- a/tests/ansible/integration/playbook_semantics/delegate_to.yml +++ b/tests/ansible/integration/playbook_semantics/delegate_to.yml @@ -30,6 +30,8 @@ # connection:local, no sudo # - name: "connection:local, no sudo" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" copy: dest: "{{ local_path }}" content: "Hello, world." @@ -43,6 +45,8 @@ fail_msg: "{{ lookup('file', local_path) }}" - name: "connection:local, no sudo" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" file: path: "{{ local_path }}" state: absent @@ -84,6 +88,8 @@ # connection:local, sudo # - name: "connection:local, sudo" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" shell: | whoami > "{{ local_path }}" args: @@ -102,6 +108,8 @@ - requires_local_sudo - name: "connection:local, sudo" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" file: path: "{{ local_path }}" state: absent diff --git a/tests/ansible/integration/process/unix_socket_cleanup.yml b/tests/ansible/integration/process/unix_socket_cleanup.yml index eb6720d3..21747494 100644 --- a/tests/ansible/integration/process/unix_socket_cleanup.yml +++ b/tests/ansible/integration/process/unix_socket_cleanup.yml @@ -7,7 +7,10 @@ result['sockets'] = glob.glob('/tmp/mitogen_unix*.sock') register: socks - - shell: > + - name: Run whoami locally in an ansible subprocess + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" + shell: >- ANSIBLE_STRATEGY=mitogen_linear ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa" ANSIBLE_VERBOSITY="{{ ansible_verbosity }}" @@ -15,6 +18,7 @@ {% for inv in ansible_inventory_sources %} -i "{{ inv }}" {% endfor %} + -e ansible_python_interpreter="{{ ansible_playbook_python }}" test-targets args: chdir: ../.. diff --git a/tests/ansible/integration/runner/missing_module.yml b/tests/ansible/integration/runner/missing_module.yml index e074c524..8e732477 100644 --- a/tests/ansible/integration/runner/missing_module.yml +++ b/tests/ansible/integration/runner/missing_module.yml @@ -6,6 +6,8 @@ connection: local environment: ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" command: | ansible {% for inv in ansible_inventory_sources %} diff --git a/tests/ansible/integration/ssh/timeouts.yml b/tests/ansible/integration/ssh/timeouts.yml index ec5aed05..fc581a3b 100644 --- a/tests/ansible/integration/ssh/timeouts.yml +++ b/tests/ansible/integration/ssh/timeouts.yml @@ -15,6 +15,8 @@ environment: ANSIBLE_SSH_TIMEOUT: 10 ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" command: | ansible {% for inv in ansible_inventory_sources %} diff --git a/tests/ansible/integration/ssh/variables.yml b/tests/ansible/integration/ssh/variables.yml index 51783881..5cb5b489 100644 --- a/tests/ansible/integration/ssh/variables.yml +++ b/tests/ansible/integration/ssh/variables.yml @@ -6,6 +6,7 @@ hosts: test-targets[0] connection: local vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" # ControlMaster has the effect of caching the previous auth to the same # account, so disable it. Can't disable with ControlMaster no since that # already appears on command line, so override ControlPath with junk. diff --git a/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml b/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml index 4220ed4c..7f8a3d2d 100644 --- a/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml +++ b/tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml @@ -4,7 +4,10 @@ tasks: - connection: local environment: + ANSIBLE_PYTHON_INTERPRETER: "{{ ansible_playbook_python }}" ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" command: | ansible-playbook {% for inv in ansible_inventory_sources %} @@ -18,6 +21,8 @@ - connection: local environment: ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" command: | ansible-playbook {% for inv in ansible_inventory_sources %} diff --git a/tests/ansible/integration/stub_connections/kubectl.yml b/tests/ansible/integration/stub_connections/kubectl.yml index 4ee00e39..8fe061d1 100644 --- a/tests/ansible/integration/stub_connections/kubectl.yml +++ b/tests/ansible/integration/stub_connections/kubectl.yml @@ -12,6 +12,7 @@ - custom_python_detect_environment: vars: ansible_connection: kubectl + ansible_python_interpreter: "{{ ansible_playbook_python }}" mitogen_kubectl_path: stub-kubectl.py register: out diff --git a/tests/ansible/integration/stub_connections/lxc.yml b/tests/ansible/integration/stub_connections/lxc.yml index eba808fc..97d8e598 100644 --- a/tests/ansible/integration/stub_connections/lxc.yml +++ b/tests/ansible/integration/stub_connections/lxc.yml @@ -8,6 +8,7 @@ - custom_python_detect_environment: vars: ansible_connection: lxc + ansible_python_interpreter: "{{ ansible_playbook_python }}" mitogen_lxc_attach_path: stub-lxc-attach.py register: out diff --git a/tests/ansible/integration/stub_connections/lxd.yml b/tests/ansible/integration/stub_connections/lxd.yml index c1825d73..71108df9 100644 --- a/tests/ansible/integration/stub_connections/lxd.yml +++ b/tests/ansible/integration/stub_connections/lxd.yml @@ -8,6 +8,7 @@ - custom_python_detect_environment: vars: ansible_connection: lxd + ansible_python_interpreter: "{{ ansible_playbook_python }}" mitogen_lxc_path: stub-lxc.py register: out diff --git a/tests/ansible/integration/stub_connections/mitogen_doas.yml b/tests/ansible/integration/stub_connections/mitogen_doas.yml index c1bf991a..52c0ee3d 100644 --- a/tests/ansible/integration/stub_connections/mitogen_doas.yml +++ b/tests/ansible/integration/stub_connections/mitogen_doas.yml @@ -9,6 +9,7 @@ vars: ansible_connection: mitogen_doas ansible_doas_exe: stub-doas.py + ansible_python_interpreter: "{{ ansible_playbook_python }}" ansible_user: someuser register: out diff --git a/tests/ansible/integration/stub_connections/mitogen_sudo.yml b/tests/ansible/integration/stub_connections/mitogen_sudo.yml index 3634dd21..624cb538 100644 --- a/tests/ansible/integration/stub_connections/mitogen_sudo.yml +++ b/tests/ansible/integration/stub_connections/mitogen_sudo.yml @@ -8,6 +8,7 @@ - custom_python_detect_environment: vars: ansible_connection: mitogen_sudo + ansible_python_interpreter: "{{ ansible_playbook_python }}" ansible_user: root ansible_become_exe: stub-sudo.py ansible_become_flags: -H --type=sometype --role=somerole diff --git a/tests/ansible/integration/stub_connections/setns_lxc.yml b/tests/ansible/integration/stub_connections/setns_lxc.yml index 58744a52..709f385c 100644 --- a/tests/ansible/integration/stub_connections/setns_lxc.yml +++ b/tests/ansible/integration/stub_connections/setns_lxc.yml @@ -7,6 +7,8 @@ gather_facts: false any_errors_fatal: false connection: local + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" tasks: - include_tasks: ../_mitogen_only.yml - include_tasks: _end_play_if_not_sudo_linux.yml @@ -19,7 +21,7 @@ -i localhost, -c setns -e mitogen_kind=lxc - -e ansible_python_interpreter=python + -e ansible_python_interpreter="{{ ansible_playbook_python }}" -e mitogen_lxc_info_path={{git_basedir}}/tests/data/stubs/stub-lxc-info.py -m shell -a "echo hi" diff --git a/tests/ansible/integration/stub_connections/setns_lxd.yml b/tests/ansible/integration/stub_connections/setns_lxd.yml index dd9fbe63..627b29f9 100644 --- a/tests/ansible/integration/stub_connections/setns_lxd.yml +++ b/tests/ansible/integration/stub_connections/setns_lxd.yml @@ -7,6 +7,8 @@ gather_facts: false any_errors_fatal: false connection: local + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" tasks: - include_tasks: ../_mitogen_only.yml - include_tasks: _end_play_if_not_sudo_linux.yml @@ -19,7 +21,7 @@ -i localhost, -c setns -e mitogen_kind=lxd - -e ansible_python_interpreter=python + -e ansible_python_interpreter="{{ ansible_playbook_python }}" -e mitogen_lxc_path={{git_basedir}}/tests/data/stubs/stub-lxc.py -m shell -a "echo hi" diff --git a/tests/ansible/regression/issue_1087__template_streamerror.yml b/tests/ansible/regression/issue_1087__template_streamerror.yml index fa950ea4..ffb565ce 100644 --- a/tests/ansible/regression/issue_1087__template_streamerror.yml +++ b/tests/ansible/regression/issue_1087__template_streamerror.yml @@ -17,6 +17,8 @@ - name: Test template does not cause StreamError delegate_to: localhost run_once: true + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" environment: ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" command: diff --git a/tests/ansible/regression/issue_140__thread_pileup.yml b/tests/ansible/regression/issue_140__thread_pileup.yml index 163440a5..78864545 100644 --- a/tests/ansible/regression/issue_140__thread_pileup.yml +++ b/tests/ansible/regression/issue_140__thread_pileup.yml @@ -9,6 +9,8 @@ - name: Create file tree connection: local run_once: true + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" shell: > mkdir /tmp/filetree.in; seq -f /tmp/filetree.in/%g 1 1000 | xargs touch; @@ -37,6 +39,8 @@ - name: Cleanup local file tree connection: local run_once: true + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" file: path: /tmp/filetree.in state: absent diff --git a/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml b/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml index aafcfad4..a4d4b7f1 100644 --- a/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml +++ b/tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml @@ -8,6 +8,8 @@ connection: local tasks: - name: Create /tmp/issue_152_interpreter.sh + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" copy: dest: /tmp/issue_152_interpreter.sh mode: u+x @@ -28,6 +30,8 @@ out={{ out }} - name: Cleanup /tmp/issue_152_interpreter.sh + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" file: path: /tmp/issue_152_interpreter.sh state: absent diff --git a/tests/ansible/regression/issue_952__ask_become_pass.yml b/tests/ansible/regression/issue_952__ask_become_pass.yml index a0b92ff2..17163a94 100644 --- a/tests/ansible/regression/issue_952__ask_become_pass.yml +++ b/tests/ansible/regression/issue_952__ask_become_pass.yml @@ -6,6 +6,8 @@ tasks: - name: Test --ask-become-pass delegate_to: localhost + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" environment: ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}" expect: diff --git a/tests/testlib.py b/tests/testlib.py index e4fa25b8..a0e27d95 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -155,6 +155,29 @@ def data_path(suffix): return path +def _have_cmd(args): + # Code duplicated in ci_lib.py + try: + subprocess.run( + args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + except OSError as exc: + if exc.errno == errno.ENOENT: + return False + raise + except subprocess.CalledProcessError: + return False + return True + + +def have_python2(): + return _have_cmd(['python2']) + + +def have_python3(): + return _have_cmd(['python3']) + + def retry(fn, on, max_attempts, delay): for i in range(max_attempts): try: diff --git a/tests/two_three_compat_test.py b/tests/two_three_compat_test.py index ab9f4e19..4a529dc7 100644 --- a/tests/two_three_compat_test.py +++ b/tests/two_three_compat_test.py @@ -1,4 +1,3 @@ -import os import unittest import mitogen.core @@ -11,8 +10,8 @@ import simple_pkg.ping # There should be 100 tests in this file. @unittest.skipIf( - os.uname()[0] == 'Darwin' and int(os.uname()[2].partition('.')[0]) >= 21, - "Python 2.x not shipped on macOS 12.3+ (Darwin 21.4+, Monterey)", + not testlib.have_python2() or not testlib.have_python3(), + "Python 2/3 compatibility tests require both versions on the controller", ) class TwoThreeCompatTest(testlib.RouterMixin, testlib.TestCase): if mitogen.core.PY3: From 4a756487743cad10a0fd7d2a816c3c27304ea7d4 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 28 Apr 2025 13:10:56 +0100 Subject: [PATCH 24/25] packaging: Fix InvalidVersion in release versions fixes #1263 --- docs/changelog.rst | 1 + setup.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 904c4dd7..ecc037db 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -27,6 +27,7 @@ In progress (unreleased) * :gh:issue:`1118` CI: Don't copy SSH private key to temporary dir * :gh:issue:`1118` CI: Don't share temporary directory between test groupings * :gh:issue:`1256` CI: Upgrade Github jobs from Ubuntu 20.04 to 22.04 & 24.04 +* :gh:issue:`1263` packaging: Fix InvalidVersion in release versions v0.3.22 (2025-02-04) diff --git a/setup.py b/setup.py index 375a96ff..1639d383 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,8 @@ def grep_version(): match = version_pattern.search(fp.read()) if match is None: raise ValueError('Could not find __version__ string in %s', path) - return '.'.join(str(part) for part in match.groups()) + # E.g. '0.1.2', '0.1.3dev' + return '.'.join(str(part) for part in match.groups() if part) def long_description(): From 8e25944c9486d854d749b727deb0ca2ef21a9e46 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Mon, 28 Apr 2025 11:13:31 +0100 Subject: [PATCH 25/25] Prepare v0.3.23 --- docs/changelog.rst | 4 ++-- mitogen/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ecc037db..acf64e53 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,8 +18,8 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. -In progress (unreleased) ------------------------- +v0.3.23 (2025-04-28) +-------------------- * :gh:issue:`1121` :mod:`mitogen`: Log skipped :py:mod:`termios` attributes * :gh:issue:`1238` packaging: Avoid :py:mod:`ast`, requires Python = 2.6 diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 125ab052..88994957 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 23, 'dev') +__version__ = (0, 3, 23) #: This is :data:`False` in slave contexts. Previously it was used to prevent