CI: Upgrade Github jobs from Ubuntu 20.04 to 22.04 & 24.04

Python 2.7 (distro package) and 3.6 (pyenv managed) jobs run on Ubuntu 22.04.
More recent Pythons (distro or Github provided) run on 24.04.

fixes #1256

Ansible tasks that run locally (e.g. `connection: local`, `delegate_to:
localhost`) must now specify their `ansible_python_interpreter`, typically as
`{{ ansible_playbook_python }}`; otherwise the system Python on the controller
(e.g. `/usr/bin/python`) is likely to be used and this is often outside the
version range supported by the Ansible verison under test. If this occurs then
the symptom is often a failure to import a builtin from
`ansible.module_utils.six.moves`, e.g.

```
fatal: [target-centos6-1]: FAILED! => changed=true
  cmd:
  - ansible
  - -m
  - shell
  - -c
  - local
  - -a
  - whoami
  - -i
  - /tmp/mitogen_ci_ansibled3llejls/hosts
  - test-targets
  delta: '0:00:02.076385'
  end: '2025-04-17 17:27:02.561500'
  msg: non-zero return code
  rc: 8
  start: '2025-04-17 17:27:00.485115'
  stderr: |-
  stderr_lines: <omitted>
  stdout: |-
    An exception occurred during task execution. To see the full traceback,
    use -vvv. The error was:     from ansible.module_utils.six.moves import
    map, reduce, shlex_quote
```

.ci/ci_lib.py

@@ -54,6 +54,7 @@ def print(*args, **kwargs):
 
 
 def _have_cmd(args):
+    # Code duplicated in testlib.py
     try:
         subprocess.run(
             args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,

.github/workflows/tests.yml

@@ -2,6 +2,10 @@
 
 name: Tests
 
+# env:
+#  ANSIBLE_VERBOSITY: 3
+#   MITOGEN_LOG_LEVEL: DEBUG
+
 on:
   pull_request:
   push:
@@ -11,10 +15,10 @@ on:
 # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners
 # https://github.com/actions/runner-images/blob/main/README.md#software-and-image-support
 jobs:
-  u2004:
-    name: u2004 ${{ matrix.tox_env }}
-    # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md
-    runs-on: ubuntu-20.04
+  u2204:
+    name: u2204 ${{ matrix.tox_env }}
+    # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md
+    runs-on: ubuntu-22.04
 
     strategy:
       fail-fast: false
@@ -32,6 +36,115 @@ jobs:
             python_version: '3.6'
             tox_env: py36-mode_ansible-ansible4
 
+          - name: Mito_27
+            tox_env: py27-mode_mitogen
+          - name: Mito_36
+            python_version: '3.6'
+            tox_env: py36-mode_mitogen
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install build deps
+        run: |
+          set -o errexit -o nounset -o pipefail
+
+          PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))')
+
+          if [[ -z $PYTHON ]]; then
+              echo 1>&2 "Python interpreter could not be determined"
+              exit 1
+          fi
+
+          sudo apt-get update
+
+          if [[ $PYTHON == "python2.7" ]]; then
+              sudo apt install -y python2-dev sshpass virtualenv
+          elif [[ $PYTHON == "python3.6" ]]; then
+              sudo apt install -y gcc-10 make libbz2-dev liblzma-dev libreadline-dev libsqlite3-dev libssl-dev sshpass virtualenv zlib1g-dev
+              curl --fail --silent --show-error --location https://pyenv.run | bash
+              CC=gcc-10 ~/.pyenv/bin/pyenv install --force 3.6
+          else
+              echo 1>&2 "Python interpreter $PYTHON not available"
+              exit 1
+          fi
+      - name: Show Python versions
+        run: |
+          set -o errexit -o nounset -o pipefail
+
+          # macOS builders lack a realpath command
+          type python  && python  -c"import os.path;print(os.path.realpath('$(type -p python)'))" &&  python --version
+          type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version
+          type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version
+          echo
+
+          if [ -e /usr/bin/python ]; then
+              echo "/usr/bin/python:    sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')"
+          fi
+
+          if [ -e /usr/bin/python2 ]; then
+              echo "/usr/bin/python2:   sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')"
+          fi
+
+          if [ -e /usr/bin/python2.7 ]; then
+              echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')"
+          fi
+      - name: Install tooling
+        run: |
+          set -o errexit -o nounset -o pipefail
+
+          # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
+          PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))')
+
+          if [[ -z $PYTHON ]]; then
+              echo 1>&2 "Python interpreter could not be determined"
+              exit 1
+          fi
+
+          if [[ $PYTHON == "python2.7" ]]; then
+              curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py"
+              "$PYTHON" get-pip.py --user --no-python-version-warning
+              # Avoid Python 2.x pip masking system pip
+              rm -f ~/.local/bin/{easy_install,pip,wheel}
+          elif [[ $PYTHON == "python3.6" ]]; then
+              PYTHON="$HOME/.pyenv/versions/3.6.15/bin/python3.6"
+          fi
+
+          "$PYTHON" -m pip install -r "tests/requirements-tox.txt"
+      - name: Run tests
+        env:
+          GITHUB_ACTOR: ${{ github.actor }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -o errexit -o nounset -o pipefail
+
+          # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
+          PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))')
+
+          if [[ -z $PYTHON ]]; then
+              echo 1>&2 "Python interpreter could not be determined"
+              exit 1
+          fi
+
+          if [[ $PYTHON == "python3.6" ]]; then
+              PYTHON="$HOME/.pyenv/versions/3.6.15/bin/python3.6"
+          fi
+
+          "$PYTHON" -m tox -e "${{ matrix.tox_env }}"
+
+  u2404:
+    name: u2404 ${{ matrix.tox_env }}
+    # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md
+    runs-on: ubuntu-24.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
           - name: Ans_311_210
             python_version: '3.11'
             tox_env: py311-mode_ansible-ansible2.10
@@ -67,11 +180,6 @@ jobs:
             python_version: '3.13'
             tox_env: py313-mode_ansible-ansible11-strategy_linear
 
-          - name: Mito_27
-            tox_env: py27-mode_mitogen
-          - name: Mito_36
-            python_version: '3.6'
-            tox_env: py36-mode_mitogen
           - name: Mito_313
             python_version: '3.13'
             tox_env: py313-mode_mitogen
@@ -92,7 +200,7 @@ jobs:
           set -o errexit -o nounset -o pipefail
 
           sudo apt-get update
-          sudo apt-get install -y python2-dev python3-pip sshpass virtualenv
+          sudo apt-get install -y sshpass virtualenv
       - name: Show Python versions
         run: |
           set -o errexit -o nounset -o pipefail
@@ -126,18 +234,7 @@ jobs:
               exit 1
           fi
 
-          if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then
-              "$PYTHON" -m ensurepip --user --altinstall --no-default-pip
-              "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
-          elif [[ $PYTHON == "python2.7" ]]; then
-              curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py"
-              "$PYTHON" get-pip.py --user --no-python-version-warning
-              # Avoid Python 2.x pip masking system pip
-              rm -f ~/.local/bin/{easy_install,pip,wheel}
-              "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
-          else
-              "$PYTHON" -m pip install -r "tests/requirements-tox.txt"
-          fi
+          "$PYTHON" -m pip install -r "tests/requirements-tox.txt"
       - name: Run tests
         env:
           GITHUB_ACTOR: ${{ github.actor }}
@@ -240,7 +337,8 @@ jobs:
   check:
     if: always()
     needs:
-      - u2004
+      - u2204
+      - u2404
       - macos
     runs-on: ubuntu-latest
     steps:

docs/changelog.rst

@@ -26,6 +26,7 @@ In progress (unreleased)
 * :gh:issue:`1118` CI: Statically specify test usernames and group names
 * :gh:issue:`1118` CI: Don't copy SSH private key to temporary dir
 * :gh:issue:`1118` CI: Don't share temporary directory between test groupings
+* :gh:issue:`1256` CI: Upgrade Github jobs from Ubuntu 20.04 to 22.04 & 24.04
 
 
 v0.3.22 (2025-02-04)

tests/ansible/bench/loop-100-copies.yml

@@ -4,6 +4,8 @@
 
   - name: Create file tree
     connection: local
+    vars:
+      ansible_python_interpreter: "{{ ansible_playbook_python }}"
     shell: >
       mkdir -p /tmp/filetree.in;
       for i in `seq -f /tmp/filetree.in/%g 1 100`; do echo $RANDOM > $i; done;

tests/ansible/integration/connection/disconnect_during_module.yml

@@ -10,6 +10,8 @@
 
   - name: Run _disconnect_during_module.yml
     delegate_to: localhost
+    vars:
+      ansible_python_interpreter: "{{ ansible_playbook_python }}"
     environment:
       ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
     command: |

tests/ansible/integration/context_service/reconnection.yml

@@ -19,6 +19,8 @@
         bash -c "( sleep 3; kill -9 {{ssh_account_env.pid}}; ) & disown"
 
     - connection: local
+      vars:
+        ansible_python_interpreter: "{{ ansible_playbook_python }}"
       shell: sleep 3
 
     - wait_for_connection:

tests/ansible/integration/local/cwd_preserved.yml

@@ -8,11 +8,15 @@
   hosts: test-targets
   tasks:
   - name: Get local cwd
+    vars:
+      ansible_python_interpreter: "{{ ansible_playbook_python }}"
     connection: local
     command: pwd
     register: pwd
 
   - connection: local
+    vars:
+      ansible_python_interpreter: "{{ ansible_playbook_python }}"
     stat:
       path: "{{pwd.stdout}}/cwd_preserved.yml"
     register: stat

tests/ansible/integration/playbook_semantics/delegate_to.yml

@@ -30,6 +30,8 @@
     # connection:local, no sudo
     #
     - name: "connection:local, no sudo"
+      vars:
+        ansible_python_interpreter: "{{ ansible_playbook_python }}"
       copy:
         dest: "{{ local_path }}"
         content: "Hello, world."
@@ -43,6 +45,8 @@
         fail_msg: "{{ lookup('file', local_path) }}"
 
     - name: "connection:local, no sudo"
+      vars:
+        ansible_python_interpreter: "{{ ansible_playbook_python }}"
       file:
         path: "{{ local_path }}"
         state: absent
@@ -84,6 +88,8 @@
     # connection:local, sudo
     #
     - name: "connection:local, sudo"
+      vars:
+        ansible_python_interpreter: "{{ ansible_playbook_python }}"
       shell: |
         whoami > "{{ local_path }}"
       args:
@@ -102,6 +108,8 @@
         - requires_local_sudo
 
     - name: "connection:local, sudo"
+      vars:
+        ansible_python_interpreter: "{{ ansible_playbook_python }}"
       file:
         path: "{{ local_path }}"
         state: absent

tests/ansible/integration/process/unix_socket_cleanup.yml

@@ -7,7 +7,10 @@
         result['sockets'] = glob.glob('/tmp/mitogen_unix*.sock')
     register: socks
 
-  - shell: >
+  - name: Run whoami locally in an ansible subprocess
+    vars:
+      ansible_python_interpreter: "{{ ansible_playbook_python }}"
+    shell: >-
       ANSIBLE_STRATEGY=mitogen_linear
       ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
       ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
@@ -15,6 +18,7 @@
       {% for inv in ansible_inventory_sources %}
       -i "{{ inv }}"
       {% endfor %}
+      -e ansible_python_interpreter="{{ ansible_playbook_python }}"
       test-targets
     args:
       chdir: ../..

tests/ansible/integration/runner/missing_module.yml

@@ -6,6 +6,8 @@
       connection: local
       environment:
         ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
+      vars:
+        ansible_python_interpreter: "{{ ansible_playbook_python }}"
       command: |
         ansible
         {% for inv in ansible_inventory_sources %}

tests/ansible/integration/ssh/timeouts.yml

@@ -15,6 +15,8 @@
       environment:
         ANSIBLE_SSH_TIMEOUT: 10
         ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
+      vars:
+        ansible_python_interpreter: "{{ ansible_playbook_python }}"
       command: |
         ansible
         {% for inv in ansible_inventory_sources %}

tests/ansible/integration/ssh/variables.yml

@@ -6,6 +6,7 @@
   hosts: test-targets[0]
   connection: local
   vars:
+    ansible_python_interpreter: "{{ ansible_playbook_python }}"
     # ControlMaster has the effect of caching the previous auth to the same
     # account, so disable it. Can't disable with ControlMaster no since that
     # already appears on command line, so override ControlPath with junk.

tests/ansible/integration/strategy/mixed_vanilla_mitogen.yml

@@ -4,7 +4,10 @@
   tasks:
     - connection: local
       environment:
+        ANSIBLE_PYTHON_INTERPRETER: "{{ ansible_playbook_python }}"
         ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
+      vars:
+        ansible_python_interpreter: "{{ ansible_playbook_python }}"
       command: |
         ansible-playbook
         {% for inv in ansible_inventory_sources %}
@@ -18,6 +21,8 @@
     - connection: local
       environment:
         ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
+      vars:
+        ansible_python_interpreter: "{{ ansible_playbook_python }}"
       command: |
         ansible-playbook
         {% for inv in ansible_inventory_sources %}

tests/ansible/integration/stub_connections/kubectl.yml

@@ -12,6 +12,7 @@
   - custom_python_detect_environment:
     vars:
       ansible_connection: kubectl
+      ansible_python_interpreter: "{{ ansible_playbook_python }}"
       mitogen_kubectl_path: stub-kubectl.py
     register: out
 

tests/ansible/integration/stub_connections/lxc.yml

@@ -8,6 +8,7 @@
   - custom_python_detect_environment:
     vars:
       ansible_connection: lxc
+      ansible_python_interpreter: "{{ ansible_playbook_python }}"
       mitogen_lxc_attach_path: stub-lxc-attach.py
     register: out
 

tests/ansible/integration/stub_connections/lxd.yml

@@ -8,6 +8,7 @@
   - custom_python_detect_environment:
     vars:
       ansible_connection: lxd
+      ansible_python_interpreter: "{{ ansible_playbook_python }}"
       mitogen_lxc_path: stub-lxc.py
     register: out
 

tests/ansible/integration/stub_connections/mitogen_doas.yml

@@ -9,6 +9,7 @@
     vars:
       ansible_connection: mitogen_doas
       ansible_doas_exe: stub-doas.py
+      ansible_python_interpreter: "{{ ansible_playbook_python }}"
       ansible_user: someuser
     register: out
 

tests/ansible/integration/stub_connections/mitogen_sudo.yml

@@ -8,6 +8,7 @@
   - custom_python_detect_environment:
     vars:
       ansible_connection: mitogen_sudo
+      ansible_python_interpreter: "{{ ansible_playbook_python }}"
       ansible_user: root
       ansible_become_exe: stub-sudo.py
       ansible_become_flags: -H --type=sometype --role=somerole

tests/ansible/integration/stub_connections/setns_lxc.yml

@@ -7,6 +7,8 @@
   gather_facts: false
   any_errors_fatal: false
   connection: local
+  vars:
+    ansible_python_interpreter: "{{ ansible_playbook_python }}"
   tasks:
     - include_tasks: ../_mitogen_only.yml
     - include_tasks: _end_play_if_not_sudo_linux.yml
@@ -19,7 +21,7 @@
         -i localhost,
         -c setns
         -e mitogen_kind=lxc
-        -e ansible_python_interpreter=python
+        -e ansible_python_interpreter="{{ ansible_playbook_python }}"
         -e mitogen_lxc_info_path={{git_basedir}}/tests/data/stubs/stub-lxc-info.py
         -m shell
         -a "echo hi"

tests/ansible/integration/stub_connections/setns_lxd.yml

@@ -7,6 +7,8 @@
   gather_facts: false
   any_errors_fatal: false
   connection: local
+  vars:
+    ansible_python_interpreter: "{{ ansible_playbook_python }}"
   tasks:
     - include_tasks: ../_mitogen_only.yml
     - include_tasks: _end_play_if_not_sudo_linux.yml
@@ -19,7 +21,7 @@
         -i localhost,
         -c setns
         -e mitogen_kind=lxd
-        -e ansible_python_interpreter=python
+        -e ansible_python_interpreter="{{ ansible_playbook_python }}"
         -e mitogen_lxc_path={{git_basedir}}/tests/data/stubs/stub-lxc.py
         -m shell
         -a "echo hi"

tests/ansible/regression/issue_1087__template_streamerror.yml

@@ -17,6 +17,8 @@
         - name: Test template does not cause StreamError
           delegate_to: localhost
           run_once: true
+          vars:
+            ansible_python_interpreter: "{{ ansible_playbook_python }}"
           environment:
             ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
           command:

tests/ansible/regression/issue_140__thread_pileup.yml

@@ -9,6 +9,8 @@
   - name: Create file tree
     connection: local
     run_once: true
+    vars:
+      ansible_python_interpreter: "{{ ansible_playbook_python }}"
     shell: >
       mkdir /tmp/filetree.in;
       seq -f /tmp/filetree.in/%g 1 1000 | xargs touch;
@@ -37,6 +39,8 @@
   - name: Cleanup local file tree
     connection: local
     run_once: true
+    vars:
+      ansible_python_interpreter: "{{ ansible_playbook_python }}"
     file:
       path: /tmp/filetree.in
       state: absent

tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml

@@ -8,6 +8,8 @@
   connection: local
   tasks:
     - name: Create /tmp/issue_152_interpreter.sh
+      vars:
+        ansible_python_interpreter: "{{ ansible_playbook_python }}"
       copy:
         dest: /tmp/issue_152_interpreter.sh
         mode: u+x
@@ -28,6 +30,8 @@
           out={{ out }}
 
     - name: Cleanup /tmp/issue_152_interpreter.sh
+      vars:
+        ansible_python_interpreter: "{{ ansible_playbook_python }}"
       file:
         path: /tmp/issue_152_interpreter.sh
         state: absent

tests/ansible/regression/issue_952__ask_become_pass.yml

@@ -6,6 +6,8 @@
   tasks:
     - name: Test --ask-become-pass
       delegate_to: localhost
+      vars:
+        ansible_python_interpreter: "{{ ansible_playbook_python }}"
       environment:
         ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
       expect:

tests/testlib.py

@@ -155,6 +155,29 @@ def data_path(suffix):
     return path
 
 
+def _have_cmd(args):
+    # Code duplicated in ci_lib.py
+    try:
+        subprocess.run(
+            args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+        )
+    except OSError as exc:
+        if exc.errno == errno.ENOENT:
+            return False
+        raise
+    except subprocess.CalledProcessError:
+        return False
+    return True
+
+
+def have_python2():
+    return _have_cmd(['python2'])
+
+
+def have_python3():
+    return _have_cmd(['python3'])
+
+
 def retry(fn, on, max_attempts, delay):
     for i in range(max_attempts):
         try:

tests/two_three_compat_test.py

@@ -1,4 +1,3 @@
-import os
 import unittest
 
 import mitogen.core
@@ -11,8 +10,8 @@ import simple_pkg.ping
 # There should be 100 tests in this file.
 
 @unittest.skipIf(
-    os.uname()[0] == 'Darwin' and int(os.uname()[2].partition('.')[0]) >= 21,
-    "Python 2.x not shipped on macOS 12.3+ (Darwin 21.4+, Monterey)",
+    not testlib.have_python2() or not testlib.have_python3(),
+    "Python 2/3 compatibility tests require both versions on the controller",
 )
 class TwoThreeCompatTest(testlib.RouterMixin, testlib.TestCase):
     if mitogen.core.PY3: