Merge pull request #1433 from moreati/prepare-0.3.39

Prepare 0.3.39

.ci/README.md

@@ -2,7 +2,7 @@
 # `.ci`
 
 This directory contains scripts for Continuous Integration platforms. Currently
-Azure Pipelines, but they will also happily run on any Debian-like machine.
+GitHub Actions, but ideally they will also run on any Debian-like machine.
 
 The scripts are usually split into `_install` and `_test` steps. The `_install`
 step will damage your machine, the `_test` step will just run the tests the way
@@ -28,15 +28,15 @@ for doing `setup.py install` while pulling a Docker container, for example.
 
 ### Environment Variables
 
-* `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2.
+* `MITOGEN_TEST_DISTRO_SPECS`: a space delimited list of distro specs to run
-* `DISTRO`: the `mitogen_` tests need a target Docker container distro. This
+  the tests against. (e.g. `centos6 ubuntu2004-py3*4`). Each spec determines
-  name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test`
+  the Linux distribution, target Python interepreter & number of instances.
-* `DISTROS`: the `ansible_` tests can run against multiple targets
+  Only distributions with a pre-built Linux container image can be used.
-  simultaneously, which speeds things up. This is a space-separated list of
-  DISTRO names, but additionally, supports:
     * `debian-py3`: when generating Ansible inventory file, set
       `ansible_python_interpreter` to `python3`, i.e. run a test where the
       target interpreter is Python 3.
     * `debian*16`: generate 16 Docker containers running Debian. Also works
       with -py3.
 
+* `MITOGEN_TEST_IMAGE_TEMPLATE`: specifies the Linux container image name,
+  and hence the container registry used for test targets.

.ci/ansible_install.py

@@ -1,11 +0,0 @@
-#!/usr/bin/env python
-
-import ci_lib
-
-batches = [
-    [
-        'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
-    ]
-]
-
-ci_lib.run_batches(batches)

.ci/ansible_tests.py

@@ -1,17 +1,19 @@
 #!/usr/bin/env python
 # Run tests/ansible/all.yml under Ansible and Ansible-Mitogen
 
+import collections
 import glob
 import os
 import signal
 import sys
 
+import jinja2
+
 import ci_lib
-from ci_lib import run
 
 
-TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible')
+TMP = ci_lib.TempDir(prefix='mitogen_ci_ansible')
-HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts')
+TMP_HOSTS_DIR = os.path.join(TMP.path, 'hosts')
 
 
 def pause_if_interactive():
@@ -32,46 +34,50 @@ ci_lib.check_stray_processes(interesting)
 
 
 with ci_lib.Fold('docker_setup'):
-    containers = ci_lib.make_containers()
+    containers = ci_lib.container_specs(ci_lib.DISTRO_SPECS.split())
     ci_lib.start_containers(containers)
 
 
 with ci_lib.Fold('job_setup'):
-    os.chdir(TESTS_DIR)
+    os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8))
-    os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 7))
+    os.chdir(ci_lib.ANSIBLE_TESTS_DIR)
 
-    run("mkdir %s", HOSTS_DIR)
+    os.mkdir(TMP_HOSTS_DIR)
-    for path in glob.glob(TESTS_DIR + '/hosts/*'):
+    for path in glob.glob(os.path.join(ci_lib.ANSIBLE_TESTS_HOSTS_DIR, '*')):
         if not path.endswith('default.hosts'):
-            run("ln -s %s %s", path, HOSTS_DIR)
+            os.symlink(path, os.path.join(TMP_HOSTS_DIR, os.path.basename(path)))
+
+    distros = collections.defaultdict(list)
+    families = collections.defaultdict(list)
+    for container in containers:
+        distros[container['distro']].append(container['name'])
+        families[container['family']].append(container['name'])
+
+    jinja_env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader(
+            searchpath=ci_lib.ANSIBLE_TESTS_TEMPLATES_DIR,
+        ),
+        lstrip_blocks=True,  # Remove spaces and tabs from before a block
+        trim_blocks=True,  # Remove first newline after a block
+    )
+    inventory_template = jinja_env.get_template('test-targets.j2')
+    inventory_path = os.path.join(TMP_HOSTS_DIR, 'test-targets.ini')
 
-    inventory_path = os.path.join(HOSTS_DIR, 'target')
     with open(inventory_path, 'w') as fp:
-        fp.write('[test-targets]\n')
+        fp.write(inventory_template.render(
-        fp.writelines(
+            containers=containers,
-            "%(name)s "
+            distros=distros,
-            "ansible_host=%(hostname)s "
+            families=families,
-            "ansible_port=%(port)s "
+        ))
-            "ansible_python_interpreter=%(python_path)s "
-            "ansible_user=mitogen__has_sudo_nopw "
-            "ansible_password=has_sudo_nopw_password"
-            "\n"
-            % container
-            for container in containers
-        )
 
     ci_lib.dump_file(inventory_path)
 
-    if not ci_lib.exists_in_path('sshpass'):
-        run("sudo apt-get update")
-        run("sudo apt-get install -y sshpass")
-
-
 with ci_lib.Fold('ansible'):
     playbook = os.environ.get('PLAYBOOK', 'all.yml')
     try:
-        run('./run_ansible_playbook.py %s -i "%s" %s',
+        ci_lib.run('./run_ansible_playbook.py %s -i "%s" %s',
-            playbook, HOSTS_DIR, ' '.join(sys.argv[1:]))
+            playbook, TMP_HOSTS_DIR, ' '.join(sys.argv[1:]),
+        )
     except:
         pause_if_interactive()
         raise

.ci/azure-pipelines-steps.yml

@@ -1,22 +0,0 @@
-
-parameters:
-  name: ''
-  pool: ''
-  sign: false
-
-steps:
-- task: UsePythonVersion@0
-  displayName: Install python
-  inputs:
-    versionSpec: '$(python.version)'
-  condition: ne(variables['python.version'], '')
-
-- script: python -mpip install tox
-  displayName: Install tooling
-
-- script: python -mtox -e "$(tox.env)"
-  displayName: "Run tests"
-  env:
-    AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
-    AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
-    AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)

.ci/azure-pipelines.yml

@@ -1,261 +0,0 @@
-# Python package
-# Create and test a Python package on multiple Python versions.
-# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
-# https://docs.microsoft.com/azure/devops/pipelines/languages/python
-
-# User defined variables are also injected as environment variables
-# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables#environment-variables
-#variables:
-  #ANSIBLE_VERBOSITY: 3
-
-jobs:
-- job: Mac1015
-  # vanilla Ansible is really slow
-  timeoutInMinutes: 120
-  steps:
-  - template: azure-pipelines-steps.yml
-  pool:
-    # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md
-    vmImage: macOS-10.15
-  strategy:
-    matrix:
-      Mito_27:
-        python.version: '2.7'
-        tox.env: py27-mode_mitogen
-      Mito_36:
-        python.version: '3.6'
-        tox.env: py36-mode_mitogen
-      Mito_39:
-        python.version: '3.9'
-        tox.env: py39-mode_mitogen
-
-      # TODO: test python3, python3 tests are broken
-      Loc_27_210:
-        python.version: '2.7'
-        tox.env: py27-mode_localhost-ansible2.10
-      Loc_27_3:
-        python.version: '2.7'
-        tox.env: py27-mode_localhost-ansible3
-      Loc_27_4:
-        python.version: '2.7'
-        tox.env: py27-mode_localhost-ansible4
-
-      # NOTE: this hangs when ran in Ubuntu 18.04
-      Van_27_210:
-        python.version: '2.7'
-        tox.env: py27-mode_localhost-ansible2.10
-        STRATEGY: linear
-        ANSIBLE_SKIP_TAGS: resource_intensive
-      Van_27_3:
-        python.version: '2.7'
-        tox.env: py27-mode_localhost-ansible3
-        STRATEGY: linear
-        ANSIBLE_SKIP_TAGS: resource_intensive
-      Van_27_4:
-        python.version: '2.7'
-        tox.env: py27-mode_localhost-ansible4
-        STRATEGY: linear
-        ANSIBLE_SKIP_TAGS: resource_intensive
-
-- job: Mac11
-  # vanilla Ansible is really slow
-  timeoutInMinutes: 120
-  steps:
-  - template: azure-pipelines-steps.yml
-  pool:
-    # https://github.com/actions/virtual-environments/blob/main/images/macos/
-    vmImage: macOS-11
-  strategy:
-    matrix:
-      Mito_27:
-        tox.env: py27-mode_mitogen
-      Mito_37:
-        python.version: '3.7'
-        tox.env: py37-mode_mitogen
-      Mito_39:
-        python.version: '3.9'
-        tox.env: py39-mode_mitogen
-
-      # TODO: test python3, python3 tests are broken
-      Loc_27_210:
-        tox.env: py27-mode_localhost-ansible2.10
-      Loc_27_3:
-        tox.env: py27-mode_localhost-ansible3
-      Loc_27_4:
-        tox.env: py27-mode_localhost-ansible4
-
-      # NOTE: this hangs when ran in Ubuntu 18.04
-      Van_27_210:
-        tox.env: py27-mode_localhost-ansible2.10
-        STRATEGY: linear
-        ANSIBLE_SKIP_TAGS: resource_intensive
-      Van_27_3:
-        tox.env: py27-mode_localhost-ansible3
-        STRATEGY: linear
-        ANSIBLE_SKIP_TAGS: resource_intensive
-      Van_27_4:
-        tox.env: py27-mode_localhost-ansible4
-        STRATEGY: linear
-        ANSIBLE_SKIP_TAGS: resource_intensive
-
-- job: Linux
-  pool:
-    # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-README.md
-    vmImage: "Ubuntu 18.04"
-  steps:
-  - template: azure-pipelines-steps.yml
-  strategy:
-    matrix:
-      Mito_27_centos6:
-        python.version: '2.7'
-        tox.env: py27-mode_mitogen-distro_centos6
-      Mito_27_centos7:
-        python.version: '2.7'
-        tox.env: py27-mode_mitogen-distro_centos7
-      Mito_27_centos8:
-        python.version: '2.7'
-        tox.env: py27-mode_mitogen-distro_centos8
-      Mito_27_debian9:
-        python.version: '2.7'
-        tox.env: py27-mode_mitogen-distro_debian9
-      Mito_27_debian10:
-        python.version: '2.7'
-        tox.env: py27-mode_mitogen-distro_debian10
-      Mito_27_debian11:
-        python.version: '2.7'
-        tox.env: py27-mode_mitogen-distro_debian11
-      Mito_27_ubuntu1604:
-        python.version: '2.7'
-        tox.env: py27-mode_mitogen-distro_ubuntu1604
-      Mito_27_ubuntu1804:
-        python.version: '2.7'
-        tox.env: py27-mode_mitogen-distro_ubuntu1804
-      Mito_27_ubuntu2004:
-        python.version: '2.7'
-        tox.env: py27-mode_mitogen-distro_ubuntu2004
-
-      Mito_36_centos6:
-        python.version: '3.6'
-        tox.env: py36-mode_mitogen-distro_centos6
-      Mito_36_centos7:
-        python.version: '3.6'
-        tox.env: py36-mode_mitogen-distro_centos7
-      Mito_36_centos8:
-        python.version: '3.6'
-        tox.env: py36-mode_mitogen-distro_centos8
-      Mito_36_debian9:
-        python.version: '3.6'
-        tox.env: py36-mode_mitogen-distro_debian9
-      Mito_36_debian10:
-        python.version: '3.6'
-        tox.env: py36-mode_mitogen-distro_debian10
-      Mito_36_debian11:
-        python.version: '3.6'
-        tox.env: py36-mode_mitogen-distro_debian11
-      Mito_36_ubuntu1604:
-        python.version: '3.6'
-        tox.env: py36-mode_mitogen-distro_ubuntu1604
-      Mito_36_ubuntu1804:
-        python.version: '3.6'
-        tox.env: py36-mode_mitogen-distro_ubuntu1804
-      Mito_36_ubuntu2004:
-        python.version: '3.6'
-        tox.env: py36-mode_mitogen-distro_ubuntu2004
-
-      Mito_39_centos6:
-        python.version: '3.9'
-        tox.env: py39-mode_mitogen-distro_centos6
-      Mito_39_centos7:
-        python.version: '3.9'
-        tox.env: py39-mode_mitogen-distro_centos7
-      Mito_39_centos8:
-        python.version: '3.9'
-        tox.env: py39-mode_mitogen-distro_centos8
-      Mito_39_debian9:
-        python.version: '3.9'
-        tox.env: py39-mode_mitogen-distro_debian9
-      Mito_39_debian10:
-        python.version: '3.9'
-        tox.env: py39-mode_mitogen-distro_debian10
-      Mito_39_debian11:
-        python.version: '3.9'
-        tox.env: py39-mode_mitogen-distro_debian11
-      Mito_39_ubuntu1604:
-        python.version: '3.9'
-        tox.env: py39-mode_mitogen-distro_ubuntu1604
-      Mito_39_ubuntu1804:
-        python.version: '3.9'
-        tox.env: py39-mode_mitogen-distro_ubuntu1804
-      Mito_39_ubuntu2004:
-        python.version: '3.9'
-        tox.env: py39-mode_mitogen-distro_ubuntu2004
-
-      #DebOps_2460_27_27:
-        #python.version: '2.7'
-        #MODE: debops_common
-        #VER: 2.4.6.0
-
-      #DebOps_262_36_27:
-        #python.version: '3.6'
-        #MODE: debops_common
-        #VER: 2.6.2
-
-      #Ansible_2460_26:
-        #python.version: '2.7'
-        #MODE: ansible
-        #VER: 2.4.6.0
-
-      #Ansible_262_26:
-        #python.version: '2.7'
-        #MODE: ansible
-        #VER: 2.6.2
-
-      #Ansible_2460_36:
-        #python.version: '3.6'
-        #MODE: ansible
-        #VER: 2.4.6.0
-
-      #Ansible_262_36:
-        #python.version: '3.6'
-        #MODE: ansible
-        #VER: 2.6.2
-
-      #Vanilla_262_27:
-        #python.version: '2.7'
-        #MODE: ansible
-        #VER: 2.6.2
-        #DISTROS: debian
-        #STRATEGY: linear
-
-      Ans_27_210:
-        python.version: '2.7'
-        tox.env: py27-mode_ansible-ansible2.10
-      Ans_27_3:
-        python.version: '2.7'
-        tox.env: py27-mode_ansible-ansible3
-      Ans_27_4:
-        python.version: '2.7'
-        tox.env: py27-mode_ansible-ansible4
-
-      Ans_36_210:
-        python.version: '3.6'
-        tox.env: py36-mode_ansible-ansible2.10
-      Ans_36_3:
-        python.version: '3.6'
-        tox.env: py36-mode_ansible-ansible3
-      Ans_36_4:
-        python.version: '3.6'
-        tox.env: py36-mode_ansible-ansible4
-
-      Ans_39_210:
-        python.version: '3.9'
-        tox.env: py39-mode_ansible-ansible2.10
-      Ans_39_3:
-        python.version: '3.9'
-        tox.env: py39-mode_ansible-ansible3
-      Ans_39_4:
-        python.version: '3.9'
-        tox.env: py39-mode_ansible-ansible4
-      Ans_39_5:
-        python.version: '3.9'
-        tox.env: py39-mode_ansible-ansible5

.ci/bash_functions

@@ -0,0 +1,13 @@
+# shellcheck shell=bash
+
+# Tox environment name -> Python executable name (e.g. py312-m_mtg -> python3.12)
+toxenv-python() {
+    local pattern='^py([23])([0-9]{1,2}).*'
+    if [[ $1 =~ $pattern ]]; then
+        echo "python${BASH_REMATCH[1]}.${BASH_REMATCH[2]}"
+        return
+    else
+        echo "${FUNCNAME[0]}: $1: environment name not recognised" >&2
+        return 1
+    fi
+}

.ci/ci_lib.py

@@ -1,15 +1,20 @@
-
 from __future__ import absolute_import
 from __future__ import print_function
 
 import atexit
+import errno
 import os
+import re
 import shlex
 import shutil
-import subprocess
 import sys
 import tempfile
 
+if sys.version_info < (3, 0):
+    import subprocess32 as subprocess
+else:
+    import subprocess
+
 try:
     import urlparse
 except ImportError:
@@ -22,6 +27,25 @@ os.chdir(
     )
 )
 
+
+GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+ANSIBLE_TESTS_DIR = os.path.join(GIT_ROOT, 'tests/ansible')
+ANSIBLE_TESTS_HOSTS_DIR = os.path.join(GIT_ROOT, 'tests/ansible/hosts')
+ANSIBLE_TESTS_TEMPLATES_DIR = os.path.join(GIT_ROOT, 'tests/ansible/templates')
+DISTRO_SPECS = os.environ.get(
+    'MITOGEN_TEST_DISTRO_SPECS',
+    'alma9-py3 centos5 centos8-py3 debian9 debian12-py3 ubuntu1604 ubuntu2404-py3',
+)
+IMAGE_PREP_DIR = os.path.join(GIT_ROOT, 'tests/image_prep')
+IMAGE_TEMPLATE = os.environ.get(
+    'MITOGEN_TEST_IMAGE_TEMPLATE',
+    'ghcr.io/mitogen-hq/%(distro)s-test:2025.02',
+)
+SUDOERS_DEFAULTS_SRC = './tests/image_prep/files/sudoers_defaults'
+SUDOERS_DEFAULTS_DEST = '/etc/sudoers.d/mitogen_test_defaults'
+TESTS_SSH_PRIVATE_KEY_FILE = os.path.join(GIT_ROOT, 'tests/data/docker/mitogen__has_sudo_pubkey.key')
+
+
 _print = print
 def print(*args, **kwargs):
     file = kwargs.get('file', sys.stdout)
@@ -31,40 +55,24 @@ def print(*args, **kwargs):
         file.flush()
 
 
-#
+def _have_cmd(args):
-# check_output() monkeypatch cutpasted from testlib.py
+    # Code duplicated in testlib.py
-#
+    try:
-
+        subprocess.run(
-def subprocess__check_output(*popenargs, **kwargs):
+            args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
-    # Missing from 2.6.
+            check=True,
-    process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
+        )
-    output, _ = process.communicate()
+    except OSError as exc:
-    retcode = process.poll()
+        if exc.errno == errno.ENOENT:
-    if retcode:
+            return False
-        cmd = kwargs.get("args")
+        raise
-        if cmd is None:
+    except subprocess.CalledProcessError:
-            cmd = popenargs[0]
+        return False
-        raise subprocess.CalledProcessError(retcode, cmd)
+    return True
-    return output
-
-if not hasattr(subprocess, 'check_output'):
-    subprocess.check_output = subprocess__check_output
-
-
-# ------------------
-
-def have_apt():
-    proc = subprocess.Popen('apt --help >/dev/null 2>/dev/null', shell=True)
-    return proc.wait() == 0
-
-def have_brew():
-    proc = subprocess.Popen('brew help >/dev/null 2>/dev/null', shell=True)
-    return proc.wait() == 0
 
 
 def have_docker():
-    proc = subprocess.Popen('docker info >/dev/null 2>/dev/null', shell=True)
+    return _have_cmd(['docker', 'info'])
-    return proc.wait() == 0
 
 
 def _argv(s, *args):
@@ -148,7 +156,15 @@ def run_batches(batches):
         subprocess.Popen(combine(batch), shell=True)
         for batch in batches
     ]
-    assert [proc.wait() for proc in procs] == [0] * len(procs)
+    for proc in procs:
+        proc.wait()
+        if proc.returncode:
+            print(
+                'proc: pid=%i rc=%i args=%r'
+                % (proc.pid, proc.returncode, proc.args),
+                file=sys.stderr, flush=True,
+            )
+    assert [proc.returncode for proc in procs] == [0] * len(procs)
 
 
 def get_output(s, *args, **kwargs):
@@ -179,8 +195,8 @@ def exists_in_path(progname):
 
 
 class TempDir(object):
-    def __init__(self):
+    def __init__(self, prefix='mitogen_ci_lib'):
-        self.path = tempfile.mkdtemp(prefix='mitogen_ci_lib')
+        self.path = tempfile.mkdtemp(prefix=prefix)
         atexit.register(self.destroy)
 
     def destroy(self, rmtree=shutil.rmtree):
@@ -193,28 +209,6 @@ class Fold(object):
     def __exit__(self, _1, _2, _3): pass
 
 
-os.environ.setdefault('ANSIBLE_STRATEGY',
-    os.environ.get('STRATEGY', 'mitogen_linear'))
-GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
-# Used only when MODE=mitogen
-DISTRO = os.environ.get('DISTRO', 'debian9')
-# Used only when MODE=ansible
-DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split()
-TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2'))
-BASE_PORT = 2200
-TMP = TempDir().path
-
-
-# We copy this out of the way to avoid random stuff modifying perms in the Git
-# tree (like git pull).
-src_key_file = os.path.join(GIT_ROOT,
-    'tests/data/docker/mitogen__has_sudo_pubkey.key')
-key_file = os.path.join(TMP,
-    'mitogen__has_sudo_pubkey.key')
-shutil.copyfile(src_key_file, key_file)
-os.chmod(key_file, int('0600', 8))
-
-
 os.environ['PYTHONDONTWRITEBYTECODE'] = 'x'
 os.environ['PYTHONPATH'] = '%s:%s' % (
     os.environ.get('PYTHONPATH', ''),
@@ -224,6 +218,7 @@ os.environ['PYTHONPATH'] = '%s:%s' % (
 def get_docker_hostname():
     """Return the hostname where the docker daemon is running.
     """
+    # Duplicated in testlib
     url = os.environ.get('DOCKER_HOST')
     if url in (None, 'http+docker://localunixsocket'):
         return 'localhost'
@@ -232,64 +227,65 @@ def get_docker_hostname():
     return parsed.netloc.partition(':')[0]
 
 
-def image_for_distro(distro):
+def container_specs(
-    """Return the container image name or path for a test distro name.
+        distros,
-
+        base_port=2200,
-    The returned value is suitable for use with `docker pull`.
+        image_template=IMAGE_TEMPLATE,
-
+        name_template='target-%(distro)s-%(index)d',
-    >>> image_for_distro('centos5')
+):
-    'public.ecr.aws/n5z0e8q9/centos5-test'
-    >>> image_for_distro('centos5-something_custom')
-    'public.ecr.aws/n5z0e8q9/centos5-test'
-    """
-    return 'public.ecr.aws/n5z0e8q9/%s-test' % (distro.partition('-')[0],)
-
-
-def make_containers(name_prefix='', port_offset=0):
     """
     >>> import pprint
-    >>> BASE_PORT=2200; DISTROS=['debian', 'centos6']
+    >>> pprint.pprint(container_specs(['debian11-py3', 'centos6']))
-    >>> pprint.pprint(make_containers())
+    [{'distro': 'debian11',
-    [{'distro': 'debian',
+      'family': 'debian',
       'hostname': 'localhost',
-      'image': 'public.ecr.aws/n5z0e8q9/debian-test',
+      'image': 'ghcr.io/mitogen-hq/debian11-test:2025.02',
-      'name': 'target-debian-1',
+      'index': 1,
+      'name': 'target-debian11-1',
       'port': 2201,
-      'python_path': '/usr/bin/python'},
+      'python_path': '/usr/bin/python3'},
      {'distro': 'centos6',
+      'family': 'centos',
       'hostname': 'localhost',
-      'image': 'public.ecr.aws/n5z0e8q9/centos6-test',
+      'image': 'ghcr.io/mitogen-hq/centos6-test:2025.02',
+      'index': 2,
       'name': 'target-centos6-2',
       'port': 2202,
       'python_path': '/usr/bin/python'}]
     """
     docker_hostname = get_docker_hostname()
-    firstbit = lambda s: (s+'-').split('-')[0]
+    # Code duplicated in testlib.py, both should be updated together
-    secondbit = lambda s: (s+'-').split('-')[1]
+    distro_pattern = re.compile(r'''
-
+        (?P<distro>(?P<family>[a-z]+)[0-9]+)
+        (?:-(?P<py>py3))?
+        (?:\*(?P<count>[0-9]+))?
+        ''',
+        re.VERBOSE,
+    )
     i = 1
     lst = []
 
-    for distro in DISTROS:
+    for distro in distros:
-        distro, star, count = distro.partition('*')
+        # Code duplicated in testlib.py, both should be updated together
-        if star:
+        d = distro_pattern.match(distro).groupdict(default=None)
-            count = int(count)
+
+        if d.pop('py') == 'py3':
+            python_path = '/usr/bin/python3'
         else:
-            count = 1
+            python_path = '/usr/bin/python'
+
+        count = int(d.pop('count') or '1', 10)
 
         for x in range(count):
-            lst.append({
+            d['index'] = i
-                "distro": firstbit(distro),
+            d.update({
-                "image": image_for_distro(distro),
+                'image': image_template % d,
-                "name": name_prefix + ("target-%s-%s" % (distro, i)),
+                'name': name_template % d,
                 "hostname": docker_hostname,
-                "port": BASE_PORT + i + port_offset,
+                'port': base_port + i,
-                "python_path": (
+                "python_path": python_path,
-                    '/usr/bin/python3'
-                    if secondbit(distro) == 'py3'
-                    else '/usr/bin/python'
-                )
             })
+            lst.append(d)
             i += 1
 
     return lst
@@ -313,18 +309,24 @@ def proc_is_docker(pid):
 
 
 def get_interesting_procs(container_name=None):
+    """
+    Return a list of (pid, line) tuples for processes considered interesting.
+    """
     args = ['ps', 'ax', '-oppid=', '-opid=', '-ocomm=', '-ocommand=']
     if container_name is not None:
         args = ['docker', 'exec', container_name] + args
 
     out = []
-    for line in subprocess__check_output(args).decode().splitlines():
+    for line in subprocess.check_output(args).decode().splitlines():
         ppid, pid, comm, rest = line.split(None, 3)
         if (
             (
                 any(comm.startswith(s) for s in INTERESTING_COMMS) or
                 'mitogen:' in rest
             ) and
+            (
+                'WALinuxAgent' not in rest
+            ) and
             (
                 container_name is not None or
                 (not proc_is_docker(pid))

.ci/debops_common_install.py

@@ -2,15 +2,9 @@
 
 import ci_lib
 
-# Naturally DebOps only supports Debian.
-ci_lib.DISTROS = ['debian']
-
 ci_lib.run_batches([
     [
-        'pip install -qqq "debops[ansible]==2.1.2"',
+        'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"',
-    ],
-    [
-        'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
     ],
 ])
 

.ci/debops_common_tests.py

@@ -6,21 +6,24 @@ import sys
 import ci_lib
 
 
-# DebOps only supports Debian.
+TMP = ci_lib.TempDir(prefix='mitogen_ci_debops')
-ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT
+project_dir = os.path.join(TMP.path, 'project')
-
-project_dir = os.path.join(ci_lib.TMP, 'project')
 vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml'
 inventory_path = 'ansible/inventory/hosts'
 docker_hostname = ci_lib.get_docker_hostname()
 
 
 with ci_lib.Fold('docker_setup'):
-    containers = ci_lib.make_containers(port_offset=500, name_prefix='debops-')
+    containers = ci_lib.container_specs(
+        ['debian*2'],
+        base_port=2700,
+        name_template='debops-target-%(distro)s-%(index)d',
+    )
     ci_lib.start_containers(containers)
 
 
 with ci_lib.Fold('job_setup'):
+    os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8))
     ci_lib.run('debops-init %s', project_dir)
     os.chdir(project_dir)
 
@@ -44,7 +47,7 @@ with ci_lib.Fold('job_setup'):
             "\n"
             # Speed up slow DH generation.
             "dhparam__bits: ['128', '64']\n"
-            % (ci_lib.key_file,)
+            % (ci_lib.TESTS_SSH_PRIVATE_KEY_FILE,)
         )
 
     with open(inventory_path, 'a') as fp:

.ci/install_sshpass

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+VERSION="$1"
+
+curl \
+    --fail \
+    --location \
+    --no-progress-meter \
+    --remote-name \
+    "https://downloads.sourceforge.net/project/sshpass/sshpass/${VERSION}/sshpass-${VERSION}.tar.gz"
+tar xvf "sshpass-${VERSION}.tar.gz"
+cd "sshpass-${VERSION}"
+./configure
+sudo make install

.ci/localhost_ansible_install.py

@@ -1,8 +0,0 @@
-#!/usr/bin/env python
-
-import ci_lib
-
-batches = [
-]
-
-ci_lib.run_batches(batches)

.ci/localhost_ansible_tests.py

@@ -1,17 +1,13 @@
 #!/usr/bin/env python
 # Run tests/ansible/all.yml under Ansible and Ansible-Mitogen
 
+from __future__ import print_function
+
 import os
+import subprocess
 import sys
 
 import ci_lib
-from ci_lib import run
-
-
-TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible')
-IMAGE_PREP_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/image_prep')
-HOSTS_DIR = os.path.join(TESTS_DIR, 'hosts')
-KEY_PATH = os.path.join(TESTS_DIR, '../data/docker/mitogen__has_sudo_pubkey.key')
 
 
 with ci_lib.Fold('unit_tests'):
@@ -20,41 +16,54 @@ with ci_lib.Fold('unit_tests'):
 
 
 with ci_lib.Fold('job_setup'):
-    os.chmod(KEY_PATH, int('0600', 8))
+    os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8))
-    # NOTE: sshpass v1.06 causes errors so pegging to 1.05 -> "msg": "Error when changing password","out": "passwd: DS error: eDSAuthFailed\n", 
-    # there's a checksum error with "brew install http://git.io/sshpass.rb" though, so installing manually
-    if not ci_lib.exists_in_path('sshpass'):
-        os.system("curl -O -L  https://sourceforge.net/projects/sshpass/files/sshpass/1.05/sshpass-1.05.tar.gz && \
-                tar xvf sshpass-1.05.tar.gz && \
-                cd sshpass-1.05 && \
-                ./configure && \
-                sudo make install")
 
 
 with ci_lib.Fold('machine_prep'):
     # generate a new ssh key for localhost ssh
     if not os.path.exists(os.path.expanduser("~/.ssh/id_rsa")):
-        os.system("ssh-keygen -P '' -m pem -f ~/.ssh/id_rsa")
+        subprocess.check_call("ssh-keygen -P '' -m pem -f ~/.ssh/id_rsa", shell=True)
-        os.system("cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys")
+        subprocess.check_call("cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys", shell=True)
+        os.chmod(os.path.expanduser('~/.ssh'), int('0700', 8))
+        os.chmod(os.path.expanduser('~/.ssh/authorized_keys'), int('0600', 8))
 
     # also generate it for the sudo user
-    if os.system("sudo [ -f /var/root/.ssh/id_rsa ]") != 0:
+    if os.system("sudo [ -f ~root/.ssh/id_rsa ]") != 0:
-        os.system("sudo ssh-keygen -P '' -m pem -f /var/root/.ssh/id_rsa")
+        subprocess.check_call("sudo ssh-keygen -P '' -m pem -f ~root/.ssh/id_rsa", shell=True)
-        os.system("sudo cat /var/root/.ssh/id_rsa.pub | sudo tee -a /var/root/.ssh/authorized_keys")
+        subprocess.check_call("sudo cat ~root/.ssh/id_rsa.pub | sudo tee -a ~root/.ssh/authorized_keys", shell=True)
+        subprocess.check_call('sudo chmod 700 ~root/.ssh', shell=True)
+        subprocess.check_call('sudo chmod 600 ~root/.ssh/authorized_keys', shell=True)
 
-    os.chmod(os.path.expanduser('~/.ssh'), int('0700', 8))
+    os.chdir(ci_lib.IMAGE_PREP_DIR)
-    os.chmod(os.path.expanduser('~/.ssh/authorized_keys'), int('0600', 8))
+    ci_lib.run("ansible-playbook -c local -i localhost, macos_localhost.yml")
-    # run chmod through sudo since it's owned by root
-    os.system('sudo chmod 700 /var/root/.ssh')
-    os.system('sudo chmod 600 /var/root/.ssh/authorized_keys')
 
     if os.path.expanduser('~mitogen__user1') == '~mitogen__user1':
-        os.chdir(IMAGE_PREP_DIR)
+        os.chdir(ci_lib.IMAGE_PREP_DIR)
-        run("ansible-playbook -c local -i localhost, _user_accounts.yml")
+        ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml")
+
+    cmd = ';'.join([
+        'from __future__ import print_function',
+        'import os, sys',
+        'print(sys.executable, os.path.realpath(sys.executable))',
+    ])
+    for interpreter in ['/usr/bin/python', '/usr/bin/python2', '/usr/bin/python2.7']:
+        print(interpreter)
+        try:
+            subprocess.call([interpreter, '-c', cmd])
+        except OSError as exc:
+            print(exc)
+
+        print(interpreter, 'with PYTHON_LAUNCHED_FROM_WRAPPER=1')
+        environ = os.environ.copy()
+        environ['PYTHON_LAUNCHED_FROM_WRAPPER'] = '1'
+        try:
+            subprocess.call([interpreter, '-c', cmd], env=environ)
+        except OSError as exc:
+            print(exc)
 
 
 with ci_lib.Fold('ansible'):
-    os.chdir(TESTS_DIR)
+    os.chdir(ci_lib.ANSIBLE_TESTS_DIR)
     playbook = os.environ.get('PLAYBOOK', 'all.yml')
-    run('./run_ansible_playbook.py %s -l target %s',
+    ci_lib.run('./run_ansible_playbook.py %s %s',
         playbook, ' '.join(sys.argv[1:]))

.ci/mitogen_install.py

@@ -1,14 +0,0 @@
-#!/usr/bin/env python
-
-import ci_lib
-
-batches = [
-]
-
-if ci_lib.have_docker():
-    batches.append([
-        'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
-    ])
-
-
-ci_lib.run_batches(batches)

.ci/mitogen_py24_install.py

@@ -3,9 +3,6 @@
 import ci_lib
 
 batches = [
-    [
-        'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
-    ],
     [
         'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv',
     ]

.ci/mitogen_py24_tests.py

@@ -8,8 +8,6 @@ import ci_lib
 os.environ.update({
     'NOCOVERAGE': '1',
     'UNIT2': '/usr/local/python2.4.6/bin/unit2',
-
-    'MITOGEN_TEST_DISTRO': ci_lib.DISTRO,
     'MITOGEN_LOG_LEVEL': 'debug',
     'SKIP_ANSIBLE': '1',
 })

.ci/mitogen_tests.py

@@ -2,11 +2,11 @@
 # Run the Mitogen tests.
 
 import os
+import subprocess
 
 import ci_lib
 
 os.environ.update({
-    'MITOGEN_TEST_DISTRO': ci_lib.DISTRO,
     'MITOGEN_LOG_LEVEL': 'debug',
     'SKIP_ANSIBLE': '1',
 })
@@ -14,6 +14,14 @@ os.environ.update({
 if not ci_lib.have_docker():
     os.environ['SKIP_DOCKER_TESTS'] = '1'
 
+subprocess.check_call(
+    "umask 0022; sudo cp '%s' '%s'"
+    % (ci_lib.SUDOERS_DEFAULTS_SRC, ci_lib.SUDOERS_DEFAULTS_DEST),
+    shell=True,
+)
+subprocess.check_call(['sudo', 'visudo', '-cf', ci_lib.SUDOERS_DEFAULTS_DEST])
+subprocess.check_call(['sudo', '-l'])
+
 interesting = ci_lib.get_interesting_procs()
 ci_lib.run('./run_tests -v')
 ci_lib.check_stray_processes(interesting)

.ci/show_python_versions

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -o errexit -o nounset -o pipefail
+
+INDENT="    "
+POSSIBLE_PYTHONS=(
+    python
+    python2
+    python3
+    /usr/bin/python
+    /usr/bin/python2
+    /usr/bin/python3
+    # GitHub macOS 12 images: python2.7 is installed, but not on $PATH
+    /Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7
+)
+
+for p in "${POSSIBLE_PYTHONS[@]}"; do
+    echo "$p"
+    if [[ ${p:0:1} == "/" && -e $p ]]; then
+        :
+    elif type "$p" > /dev/null 2>&1; then
+        type "$p" 2>&1 | sed -e "s/^/${INDENT}type: /"
+    else
+        echo "${INDENT}Not present"
+        echo
+        continue
+    fi
+
+    $p -c "import sys;          print('${INDENT}version:        %d.%d.%d' % sys.version_info[:3])"
+    # macOS builders lack a realpath command
+    $p -c "import os.path;      print('${INDENT}realpath:       %s' % os.path.realpath('$(type -p "$p")'))"
+    $p -c "import sys;          print('${INDENT}sys.executable: %s' % sys.executable)"
+    echo
+done

.github/ISSUE_TEMPLATE/bug-0.2.md

@@ -1,33 +0,0 @@
----
-name: Mitogen 0.2.x bug report
-about: Report a bug in Mitogen 0.2.x (for Ansible 2.5, 2.6, 2.7, 2.8, or 2.9)
-title: ''
-labels: affects-0.2, bug
-assignees: ''
-
----
-
-Please drag-drop large logs as text file attachments.
-
-Feel free to write an issue in your preferred format, however if in doubt, use
-the following checklist as a guide for what to include.
-
-* Which version of Ansible are you running?
-* Is your version of Ansible patched in any way?
-* Are you running with any custom modules, or `module_utils` loaded?
-
-* Have you tried the latest master version from Git?
-* Do you have some idea of what the underlying problem may be?
-  https://mitogen.networkgenomics.com/ansible_detailed.html#common-problems has
-  instructions to help figure out the likely cause and how to gather relevant
-  logs.
-* Mention your host and target OS and versions
-* Mention your host and target Python versions
-* If reporting a performance issue, mention the number of targets and a rough
-  description of your workload (lots of copies, lots of tiny file edits, etc.)
-* If reporting a crash or hang in Ansible, please rerun with -vvv and include
-  200 lines of output around the point of the error, along with a full copy of
-  any traceback or error text in the log. Beware "-vvv" may include secret
-  data! Edit as necessary before posting.
-* If reporting any kind of problem with Ansible, please include the Ansible
-  version along with output of "ansible-config dump --only-changed".

.github/ISSUE_TEMPLATE/bug-0.3.md

@@ -1,33 +0,0 @@
----
-name: Mitogen 0.3.x bug report
-about: Report a bug in Mitogen 0.3.x (for Ansible 2.10.x)
-title: ''
-labels: affects-0.3, bug
-assignees: ''
-
----
-
-Please drag-drop large logs as text file attachments.
-
-Feel free to write an issue in your preferred format, however if in doubt, use
-the following checklist as a guide for what to include.
-
-* Which version of Ansible are you running?
-* Is your version of Ansible patched in any way?
-* Are you running with any custom modules, or `module_utils` loaded?
-
-* Have you tried the latest master version from Git?
-* Do you have some idea of what the underlying problem may be?
-  https://mitogen.networkgenomics.com/ansible_detailed.html#common-problems has
-  instructions to help figure out the likely cause and how to gather relevant
-  logs.
-* Mention your host and target OS and versions
-* Mention your host and target Python versions
-* If reporting a performance issue, mention the number of targets and a rough
-  description of your workload (lots of copies, lots of tiny file edits, etc.)
-* If reporting a crash or hang in Ansible, please rerun with -vvv and include
-  200 lines of output around the point of the error, along with a full copy of
-  any traceback or error text in the log. Beware "-vvv" may include secret
-  data! Edit as necessary before posting.
-* If reporting any kind of problem with Ansible, please include the Ansible
-  version along with output of "ansible-config dump --only-changed".

.github/ISSUE_TEMPLATE/bug-0.3.yml

@@ -0,0 +1,62 @@
+name: Bug report
+description: Report a bug in Mitogen 0.3.x (for Ansible 2.10 and above)
+labels:
+  - affects-0.3
+type: bug
+
+body:
+  - type: textarea
+    attributes:
+      label: Description
+      description: >
+        When does the problem occur?
+        What happens after?
+        How is this different?
+        Did it previously behave as expected?
+      placeholder: |
+        When I do X, Y happens, but I was expecting Z because ...
+        Before version 1.2.3 it worked as expected.
+    validations:
+      required: true
+
+  - type: input
+    attributes:
+      label: Mitogen version
+      placeholder: 0.3.31, 0.3.3-9+deb12u1
+    validations:
+      required: true
+
+  - type: input
+    attributes:
+      label: Ansible version (if applicable)
+      placeholder: 2.18.11
+
+  - type: textarea
+    attributes:
+      label: OS and environment
+      description: >
+        What operating system version(s), Python version(s), etc. are you using?
+      placeholder: |
+        Controller (master): Debian 13, Python 3.14
+        Targets (slaves): Ubuntu 20.04/Python 2.7, RHEL 10, ...
+
+  - type: textarea
+    attributes:
+      label: Steps to reproduce
+      description: >
+        Instructions, code, or playbook(s) recreate the beahviour
+      value: |
+        Steps:
+        1. Set config `foo = 42` in somefile.cfg
+        2. Run the following Python or Playbook with `cmd --option bar ...`
+
+        ```
+        Code or playbook here
+        ```
+
+  - type: textarea
+    attributes:
+      label: Anything else
+      description: >
+        Include any other details you think might be relevant or helpful.
+        Examples might include logs, unusual settings, environment variables, ...

.github/workflows/tests.yml

@@ -0,0 +1,211 @@
+# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
+
+name: Tests
+
+# env:
+#  ANSIBLE_VERBOSITY: 3
+#  MITOGEN_LOG_LEVEL: DEBUG
+
+on:
+  pull_request:
+  push:
+    branches-ignore:
+      - docs-master
+
+# https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners
+# https://github.com/actions/runner-images/blob/main/README.md#software-and-image-support
+jobs:
+  u2204:
+    name: u2204 ${{ matrix.tox_env }}
+    # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md
+    runs-on: ubuntu-22.04
+    timeout-minutes: 25
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - tox_env: py27-m_ans-ans2.10
+          - tox_env: py27-m_ans-ans4
+
+          - tox_env: py36-m_ans-ans2.10
+          - tox_env: py36-m_ans-ans4
+
+          - tox_env: py27-m_mtg
+          - tox_env: py36-m_mtg
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - run: .ci/show_python_versions
+      - name: Install deps
+        id: install-deps
+        run: |
+          set -o errexit -o nounset -o pipefail
+          source .ci/bash_functions
+          PYTHON="$(toxenv-python '${{ matrix.tox_env }}')"
+
+          sudo apt-get update
+
+          if [[ $PYTHON == "python2.7" ]]; then
+              sudo apt install -y python2-dev sshpass virtualenv
+              curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py"
+              "$PYTHON" get-pip.py --user --no-python-version-warning
+              # Avoid Python 2.x pip masking system pip
+              rm -f ~/.local/bin/{easy_install,pip,wheel}
+          elif [[ $PYTHON == "python3.6" ]]; then
+              sudo apt install -y gcc-10 make libbz2-dev liblzma-dev libreadline-dev libsqlite3-dev libssl-dev sshpass virtualenv zlib1g-dev
+              curl --fail --silent --show-error --location https://pyenv.run | bash
+              CC=gcc-10 ~/.pyenv/bin/pyenv install --force 3.6
+              PYTHON="$HOME/.pyenv/versions/3.6.15/bin/python3.6"
+          fi
+
+          "$PYTHON" -m pip install -r "tests/requirements-tox.txt"
+          echo "python=$PYTHON" >> $GITHUB_OUTPUT
+      - name: Run tests
+        env:
+          GITHUB_ACTOR: ${{ github.actor }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -o errexit -o nounset -o pipefail
+          PYTHON="${{ steps.install-deps.outputs.python }}"
+
+          "$PYTHON" -m tox -e "${{ matrix.tox_env }}"
+
+  u2404:
+    name: u2404 ${{ matrix.tox_env }}
+    # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md
+    runs-on: ubuntu-24.04
+    timeout-minutes: 25
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - tox_env: py311-m_ans-ans2.10
+            python_version: '3.11'
+          - tox_env: py311-m_ans-ans3
+            python_version: '3.11'
+          - tox_env: py311-m_ans-ans4
+            python_version: '3.11'
+          - tox_env: py311-m_ans-ans5
+            python_version: '3.11'
+          - tox_env: py313-m_ans-ans6
+            python_version: '3.13'
+          - tox_env: py313-m_ans-ans7
+            python_version: '3.13'
+          - tox_env: py313-m_ans-ans8
+            python_version: '3.13'
+          - tox_env: py314-m_ans-ans9
+            python_version: '3.14'
+          - tox_env: py314-m_ans-ans10
+            python_version: '3.14'
+          - tox_env: py314-m_ans-ans11
+            python_version: '3.14'
+          - tox_env: py314-m_ans-ans12
+            python_version: '3.14'
+          - tox_env: py314-m_ans-ans13
+            python_version: '3.14'
+
+          - tox_env: py314-m_ans-ans13-s_lin
+            python_version: '3.14'
+
+          - tox_env: py314-m_mtg
+            python_version: '3.14'
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python_version }}
+        if: ${{ matrix.python_version }}
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - run: .ci/show_python_versions
+      - name: Install deps
+        id: install-deps
+        run: |
+          set -o errexit -o nounset -o pipefail
+          source .ci/bash_functions
+          PYTHON="$(toxenv-python '${{ matrix.tox_env }}')"
+
+          sudo apt-get update
+          sudo apt-get install -y sshpass virtualenv
+
+          "$PYTHON" -m pip install -r "tests/requirements-tox.txt"
+          echo "python=$PYTHON" >> $GITHUB_OUTPUT
+      - name: Run tests
+        env:
+          GITHUB_ACTOR: ${{ github.actor }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -o errexit -o nounset -o pipefail
+          PYTHON="${{ steps.install-deps.outputs.python }}"
+
+          "$PYTHON" -m tox -e "${{ matrix.tox_env }}"
+
+  macos:
+    name: macos ${{ matrix.tox_env }}
+    # https://github.com/actions/runner-images/blob/main/images/macos/macos-15-Readme.md
+    runs-on: macos-15
+    timeout-minutes: 15
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - tox_env: py314-m_lcl-ans13
+            python_version: '3.14'
+          - tox_env: py314-m_lcl-ans13-s_lin
+            python_version: '3.14'
+
+          - tox_env: py314-m_mtg
+            python_version: '3.14'
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python_version }}
+        if: ${{ matrix.python_version }}
+      - run: .ci/show_python_versions
+      - run: .ci/install_sshpass ${{ matrix.sshpass_version }}
+        if: ${{ matrix.sshpass_version }}
+      - name: Install deps
+        id: install-deps
+        run: |
+          set -o errexit -o nounset -o pipefail
+          source .ci/bash_functions
+          PYTHON="$(toxenv-python '${{ matrix.tox_env }}')"
+
+          "$PYTHON" -m pip install -r "tests/requirements-tox.txt"
+          echo "python=$PYTHON" >> $GITHUB_OUTPUT
+      - name: Run tests
+        env:
+          GITHUB_ACTOR: ${{ github.actor }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -o errexit -o nounset -o pipefail
+          PYTHON="${{ steps.install-deps.outputs.python }}"
+
+          "$PYTHON" -m tox -e "${{ matrix.tox_env }}"
+
+  # https://github.com/marketplace/actions/alls-green
+  check:
+    if: always()
+    needs:
+      - u2204
+      - u2404
+      - macos
+    runs-on: ubuntu-latest
+    steps:
+      - uses: re-actors/alls-green@release/v1
+        with:
+          jobs: ${{ toJSON(needs) }}

.gitignore

@@ -1,3 +1,4 @@
+.ansible/
 .coverage
 .tox
 .venv
@@ -6,11 +7,13 @@ venvs/**
 *.pyc
 *.pyd
 *.pyo
+*.retry
 MANIFEST
 build/
 dist/
 extra/
 tests/ansible/.*.pid
+tests/image_prep/logs
 docs/_build/
 htmlcov/
 *.egg-info

.lgtm.yml

@@ -1,10 +0,0 @@
-path_classifiers:
-  library:
-    - "mitogen/compat"
-    - "ansible_mitogen/compat"
-queries:
-  # Mitogen 2.4 compatibility trips this query everywhere, so just disable it
-  - exclude: py/unreachable-statement
-  - exclude: py/should-use-with
-  # mitogen.core.b() trips this query everywhere, so just disable it
-  - exclude: py/import-and-import-from

README.md

@@ -1,9 +1,9 @@
 # Mitogen
 
+[![PyPI - Version](https://img.shields.io/pypi/v/mitogen)](https://pypi.org/project/mitogen/)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/mitogen)](https://pypi.org/project/mitogen/)
+[![Build Status](https://img.shields.io/github/actions/workflow/status/mitogen-hq/mitogen/tests.yml?branch=master)](https://github.com/mitogen-hq/mitogen/actions?query=branch%3Amaster)
+
 <a href="https://mitogen.networkgenomics.com/">Please see the documentation</a>.
 
 ![](https://i.imgur.com/eBM6LhJ.gif)
-
-[![Total alerts](https://img.shields.io/lgtm/alerts/g/mitogen-hq/mitogen.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/mitogen-hq/mitogen/alerts/)
-
-[![Build Status](https://dev.azure.com/mitogen-hq/mitogen/_apis/build/status/mitogen-hq.mitogen?branchName=master)](https://dev.azure.com/mitogen-hq/mitogen/_build/latest?definitionId=1&branchName=master)

ansible_mitogen/affinity.py

@@ -73,7 +73,9 @@ necessarily involves preventing the scheduler from making load balancing
 decisions.
 """
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
 import ctypes
 import logging
 import mmap
@@ -81,7 +83,6 @@ import multiprocessing
 import os
 import struct
 
-import mitogen.core
 import mitogen.parent
 
 
@@ -263,7 +264,7 @@ class LinuxPolicy(FixedPolicy):
         for x in range(16):
             chunks.append(struct.pack('<Q', mask & shiftmask))
             mask >>= 64
-        return mitogen.core.b('').join(chunks)
+        return b''.join(chunks)
 
     def _get_thread_ids(self):
         try:

ansible_mitogen/compat/simplejson/__init__.py

@@ -1,318 +0,0 @@
-r"""JSON (JavaScript Object Notation) <http://json.org> is a subset of
-JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data
-interchange format.
-
-:mod:`simplejson` exposes an API familiar to users of the standard library
-:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained
-version of the :mod:`json` library contained in Python 2.6, but maintains
-compatibility with Python 2.4 and Python 2.5 and (currently) has
-significant performance advantages, even without using the optional C
-extension for speedups.
-
-Encoding basic Python object hierarchies::
-
-    >>> import simplejson as json
-    >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}])
-    '["foo", {"bar": ["baz", null, 1.0, 2]}]'
-    >>> print json.dumps("\"foo\bar")
-    "\"foo\bar"
-    >>> print json.dumps(u'\u1234')
-    "\u1234"
-    >>> print json.dumps('\\')
-    "\\"
-    >>> print json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True)
-    {"a": 0, "b": 0, "c": 0}
-    >>> from StringIO import StringIO
-    >>> io = StringIO()
-    >>> json.dump(['streaming API'], io)
-    >>> io.getvalue()
-    '["streaming API"]'
-
-Compact encoding::
-
-    >>> import simplejson as json
-    >>> json.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':'))
-    '[1,2,3,{"4":5,"6":7}]'
-
-Pretty printing::
-
-    >>> import simplejson as json
-    >>> s = json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4)
-    >>> print '\n'.join([l.rstrip() for l in  s.splitlines()])
-    {
-        "4": 5,
-        "6": 7
-    }
-
-Decoding JSON::
-
-    >>> import simplejson as json
-    >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}]
-    >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj
-    True
-    >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar'
-    True
-    >>> from StringIO import StringIO
-    >>> io = StringIO('["streaming API"]')
-    >>> json.load(io)[0] == 'streaming API'
-    True
-
-Specializing JSON object decoding::
-
-    >>> import simplejson as json
-    >>> def as_complex(dct):
-    ...     if '__complex__' in dct:
-    ...         return complex(dct['real'], dct['imag'])
-    ...     return dct
-    ...
-    >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}',
-    ...     object_hook=as_complex)
-    (1+2j)
-    >>> import decimal
-    >>> json.loads('1.1', parse_float=decimal.Decimal) == decimal.Decimal('1.1')
-    True
-
-Specializing JSON object encoding::
-
-    >>> import simplejson as json
-    >>> def encode_complex(obj):
-    ...     if isinstance(obj, complex):
-    ...         return [obj.real, obj.imag]
-    ...     raise TypeError(repr(o) + " is not JSON serializable")
-    ...
-    >>> json.dumps(2 + 1j, default=encode_complex)
-    '[2.0, 1.0]'
-    >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j)
-    '[2.0, 1.0]'
-    >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j))
-    '[2.0, 1.0]'
-
-
-Using simplejson.tool from the shell to validate and pretty-print::
-
-    $ echo '{"json":"obj"}' | python -m simplejson.tool
-    {
-        "json": "obj"
-    }
-    $ echo '{ 1.2:3.4}' | python -m simplejson.tool
-    Expecting property name: line 1 column 2 (char 2)
-"""
-__version__ = '2.0.9'
-__all__ = [
-    'dump', 'dumps', 'load', 'loads',
-    'JSONDecoder', 'JSONEncoder',
-]
-
-__author__ = 'Bob Ippolito <bob@redivi.com>'
-
-from decoder import JSONDecoder
-from encoder import JSONEncoder
-
-_default_encoder = JSONEncoder(
-    skipkeys=False,
-    ensure_ascii=True,
-    check_circular=True,
-    allow_nan=True,
-    indent=None,
-    separators=None,
-    encoding='utf-8',
-    default=None,
-)
-
-def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
-        allow_nan=True, cls=None, indent=None, separators=None,
-        encoding='utf-8', default=None, **kw):
-    """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a
-    ``.write()``-supporting file-like object).
-
-    If ``skipkeys`` is true then ``dict`` keys that are not basic types
-    (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``)
-    will be skipped instead of raising a ``TypeError``.
-
-    If ``ensure_ascii`` is false, then the some chunks written to ``fp``
-    may be ``unicode`` instances, subject to normal Python ``str`` to
-    ``unicode`` coercion rules. Unless ``fp.write()`` explicitly
-    understands ``unicode`` (as in ``codecs.getwriter()``) this is likely
-    to cause an error.
-
-    If ``check_circular`` is false, then the circular reference check
-    for container types will be skipped and a circular reference will
-    result in an ``OverflowError`` (or worse).
-
-    If ``allow_nan`` is false, then it will be a ``ValueError`` to
-    serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``)
-    in strict compliance of the JSON specification, instead of using the
-    JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
-
-    If ``indent`` is a non-negative integer, then JSON array elements and object
-    members will be pretty-printed with that indent level. An indent level
-    of 0 will only insert newlines. ``None`` is the most compact representation.
-
-    If ``separators`` is an ``(item_separator, dict_separator)`` tuple
-    then it will be used instead of the default ``(', ', ': ')`` separators.
-    ``(',', ':')`` is the most compact JSON representation.
-
-    ``encoding`` is the character encoding for str instances, default is UTF-8.
-
-    ``default(obj)`` is a function that should return a serializable version
-    of obj or raise TypeError. The default simply raises TypeError.
-
-    To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
-    ``.default()`` method to serialize additional types), specify it with
-    the ``cls`` kwarg.
-
-    """
-    # cached encoder
-    if (not skipkeys and ensure_ascii and
-        check_circular and allow_nan and
-        cls is None and indent is None and separators is None and
-        encoding == 'utf-8' and default is None and not kw):
-        iterable = _default_encoder.iterencode(obj)
-    else:
-        if cls is None:
-            cls = JSONEncoder
-        iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii,
-            check_circular=check_circular, allow_nan=allow_nan, indent=indent,
-            separators=separators, encoding=encoding,
-            default=default, **kw).iterencode(obj)
-    # could accelerate with writelines in some versions of Python, at
-    # a debuggability cost
-    for chunk in iterable:
-        fp.write(chunk)
-
-
-def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
-        allow_nan=True, cls=None, indent=None, separators=None,
-        encoding='utf-8', default=None, **kw):
-    """Serialize ``obj`` to a JSON formatted ``str``.
-
-    If ``skipkeys`` is false then ``dict`` keys that are not basic types
-    (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``)
-    will be skipped instead of raising a ``TypeError``.
-
-    If ``ensure_ascii`` is false, then the return value will be a
-    ``unicode`` instance subject to normal Python ``str`` to ``unicode``
-    coercion rules instead of being escaped to an ASCII ``str``.
-
-    If ``check_circular`` is false, then the circular reference check
-    for container types will be skipped and a circular reference will
-    result in an ``OverflowError`` (or worse).
-
-    If ``allow_nan`` is false, then it will be a ``ValueError`` to
-    serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in
-    strict compliance of the JSON specification, instead of using the
-    JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
-
-    If ``indent`` is a non-negative integer, then JSON array elements and
-    object members will be pretty-printed with that indent level. An indent
-    level of 0 will only insert newlines. ``None`` is the most compact
-    representation.
-
-    If ``separators`` is an ``(item_separator, dict_separator)`` tuple
-    then it will be used instead of the default ``(', ', ': ')`` separators.
-    ``(',', ':')`` is the most compact JSON representation.
-
-    ``encoding`` is the character encoding for str instances, default is UTF-8.
-
-    ``default(obj)`` is a function that should return a serializable version
-    of obj or raise TypeError. The default simply raises TypeError.
-
-    To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
-    ``.default()`` method to serialize additional types), specify it with
-    the ``cls`` kwarg.
-
-    """
-    # cached encoder
-    if (not skipkeys and ensure_ascii and
-        check_circular and allow_nan and
-        cls is None and indent is None and separators is None and
-        encoding == 'utf-8' and default is None and not kw):
-        return _default_encoder.encode(obj)
-    if cls is None:
-        cls = JSONEncoder
-    return cls(
-        skipkeys=skipkeys, ensure_ascii=ensure_ascii,
-        check_circular=check_circular, allow_nan=allow_nan, indent=indent,
-        separators=separators, encoding=encoding, default=default,
-        **kw).encode(obj)
-
-
-_default_decoder = JSONDecoder(encoding=None, object_hook=None)
-
-
-def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None,
-        parse_int=None, parse_constant=None, **kw):
-    """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing
-    a JSON document) to a Python object.
-
-    If the contents of ``fp`` is encoded with an ASCII based encoding other
-    than utf-8 (e.g. latin-1), then an appropriate ``encoding`` name must
-    be specified. Encodings that are not ASCII based (such as UCS-2) are
-    not allowed, and should be wrapped with
-    ``codecs.getreader(fp)(encoding)``, or simply decoded to a ``unicode``
-    object and passed to ``loads()``
-
-    ``object_hook`` is an optional function that will be called with the
-    result of any object literal decode (a ``dict``). The return value of
-    ``object_hook`` will be used instead of the ``dict``. This feature
-    can be used to implement custom decoders (e.g. JSON-RPC class hinting).
-
-    To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
-    kwarg.
-
-    """
-    return loads(fp.read(),
-        encoding=encoding, cls=cls, object_hook=object_hook,
-        parse_float=parse_float, parse_int=parse_int,
-        parse_constant=parse_constant, **kw)
-
-
-def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None,
-        parse_int=None, parse_constant=None, **kw):
-    """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON
-    document) to a Python object.
-
-    If ``s`` is a ``str`` instance and is encoded with an ASCII based encoding
-    other than utf-8 (e.g. latin-1) then an appropriate ``encoding`` name
-    must be specified. Encodings that are not ASCII based (such as UCS-2)
-    are not allowed and should be decoded to ``unicode`` first.
-
-    ``object_hook`` is an optional function that will be called with the
-    result of any object literal decode (a ``dict``). The return value of
-    ``object_hook`` will be used instead of the ``dict``. This feature
-    can be used to implement custom decoders (e.g. JSON-RPC class hinting).
-
-    ``parse_float``, if specified, will be called with the string
-    of every JSON float to be decoded. By default this is equivalent to
-    float(num_str). This can be used to use another datatype or parser
-    for JSON floats (e.g. decimal.Decimal).
-
-    ``parse_int``, if specified, will be called with the string
-    of every JSON int to be decoded. By default this is equivalent to
-    int(num_str). This can be used to use another datatype or parser
-    for JSON integers (e.g. float).
-
-    ``parse_constant``, if specified, will be called with one of the
-    following strings: -Infinity, Infinity, NaN, null, true, false.
-    This can be used to raise an exception if invalid JSON numbers
-    are encountered.
-
-    To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
-    kwarg.
-
-    """
-    if (cls is None and encoding is None and object_hook is None and
-            parse_int is None and parse_float is None and
-            parse_constant is None and not kw):
-        return _default_decoder.decode(s)
-    if cls is None:
-        cls = JSONDecoder
-    if object_hook is not None:
-        kw['object_hook'] = object_hook
-    if parse_float is not None:
-        kw['parse_float'] = parse_float
-    if parse_int is not None:
-        kw['parse_int'] = parse_int
-    if parse_constant is not None:
-        kw['parse_constant'] = parse_constant
-    return cls(encoding=encoding, **kw).decode(s)

ansible_mitogen/compat/simplejson/decoder.py

@@ -1,354 +0,0 @@
-"""Implementation of JSONDecoder
-"""
-import re
-import sys
-import struct
-
-from simplejson.scanner import make_scanner
-try:
-    from simplejson._speedups import scanstring as c_scanstring
-except ImportError:
-    c_scanstring = None
-
-__all__ = ['JSONDecoder']
-
-FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
-
-def _floatconstants():
-    _BYTES = '7FF80000000000007FF0000000000000'.decode('hex')
-    if sys.byteorder != 'big':
-        _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1]
-    nan, inf = struct.unpack('dd', _BYTES)
-    return nan, inf, -inf
-
-NaN, PosInf, NegInf = _floatconstants()
-
-
-def linecol(doc, pos):
-    lineno = doc.count('\n', 0, pos) + 1
-    if lineno == 1:
-        colno = pos
-    else:
-        colno = pos - doc.rindex('\n', 0, pos)
-    return lineno, colno
-
-
-def errmsg(msg, doc, pos, end=None):
-    # Note that this function is called from _speedups
-    lineno, colno = linecol(doc, pos)
-    if end is None:
-        #fmt = '{0}: line {1} column {2} (char {3})'
-        #return fmt.format(msg, lineno, colno, pos)
-        fmt = '%s: line %d column %d (char %d)'
-        return fmt % (msg, lineno, colno, pos)
-    endlineno, endcolno = linecol(doc, end)
-    #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})'
-    #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end)
-    fmt = '%s: line %d column %d - line %d column %d (char %d - %d)'
-    return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end)
-
-
-_CONSTANTS = {
-    '-Infinity': NegInf,
-    'Infinity': PosInf,
-    'NaN': NaN,
-}
-
-STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
-BACKSLASH = {
-    '"': u'"', '\\': u'\\', '/': u'/',
-    'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t',
-}
-
-DEFAULT_ENCODING = "utf-8"
-
-def py_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match):
-    """Scan the string s for a JSON string. End is the index of the
-    character in s after the quote that started the JSON string.
-    Unescapes all valid JSON string escape sequences and raises ValueError
-    on attempt to decode an invalid string. If strict is False then literal
-    control characters are allowed in the string.
-    
-    Returns a tuple of the decoded string and the index of the character in s
-    after the end quote."""
-    if encoding is None:
-        encoding = DEFAULT_ENCODING
-    chunks = []
-    _append = chunks.append
-    begin = end - 1
-    while 1:
-        chunk = _m(s, end)
-        if chunk is None:
-            raise ValueError(
-                errmsg("Unterminated string starting at", s, begin))
-        end = chunk.end()
-        content, terminator = chunk.groups()
-        # Content is contains zero or more unescaped string characters
-        if content:
-            if not isinstance(content, unicode):
-                content = unicode(content, encoding)
-            _append(content)
-        # Terminator is the end of string, a literal control character,
-        # or a backslash denoting that an escape sequence follows
-        if terminator == '"':
-            break
-        elif terminator != '\\':
-            if strict:
-                msg = "Invalid control character %r at" % (terminator,)
-                #msg = "Invalid control character {0!r} at".format(terminator)
-                raise ValueError(errmsg(msg, s, end))
-            else:
-                _append(terminator)
-                continue
-        try:
-            esc = s[end]
-        except IndexError:
-            raise ValueError(
-                errmsg("Unterminated string starting at", s, begin))
-        # If not a unicode escape sequence, must be in the lookup table
-        if esc != 'u':
-            try:
-                char = _b[esc]
-            except KeyError:
-                msg = "Invalid \\escape: " + repr(esc)
-                raise ValueError(errmsg(msg, s, end))
-            end += 1
-        else:
-            # Unicode escape sequence
-            esc = s[end + 1:end + 5]
-            next_end = end + 5
-            if len(esc) != 4:
-                msg = "Invalid \\uXXXX escape"
-                raise ValueError(errmsg(msg, s, end))
-            uni = int(esc, 16)
-            # Check for surrogate pair on UCS-4 systems
-            if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535:
-                msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
-                if not s[end + 5:end + 7] == '\\u':
-                    raise ValueError(errmsg(msg, s, end))
-                esc2 = s[end + 7:end + 11]
-                if len(esc2) != 4:
-                    raise ValueError(errmsg(msg, s, end))
-                uni2 = int(esc2, 16)
-                uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
-                next_end += 6
-            char = unichr(uni)
-            end = next_end
-        # Append the unescaped character
-        _append(char)
-    return u''.join(chunks), end
-
-
-# Use speedup if available
-scanstring = c_scanstring or py_scanstring
-
-WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS)
-WHITESPACE_STR = ' \t\n\r'
-
-def JSONObject((s, end), encoding, strict, scan_once, object_hook, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
-    pairs = {}
-    # Use a slice to prevent IndexError from being raised, the following
-    # check will raise a more specific ValueError if the string is empty
-    nextchar = s[end:end + 1]
-    # Normally we expect nextchar == '"'
-    if nextchar != '"':
-        if nextchar in _ws:
-            end = _w(s, end).end()
-            nextchar = s[end:end + 1]
-        # Trivial empty object
-        if nextchar == '}':
-            return pairs, end + 1
-        elif nextchar != '"':
-            raise ValueError(errmsg("Expecting property name", s, end))
-    end += 1
-    while True:
-        key, end = scanstring(s, end, encoding, strict)
-
-        # To skip some function call overhead we optimize the fast paths where
-        # the JSON key separator is ": " or just ":".
-        if s[end:end + 1] != ':':
-            end = _w(s, end).end()
-            if s[end:end + 1] != ':':
-                raise ValueError(errmsg("Expecting : delimiter", s, end))
-
-        end += 1
-
-        try:
-            if s[end] in _ws:
-                end += 1
-                if s[end] in _ws:
-                    end = _w(s, end + 1).end()
-        except IndexError:
-            pass
-
-        try:
-            value, end = scan_once(s, end)
-        except StopIteration:
-            raise ValueError(errmsg("Expecting object", s, end))
-        pairs[key] = value
-
-        try:
-            nextchar = s[end]
-            if nextchar in _ws:
-                end = _w(s, end + 1).end()
-                nextchar = s[end]
-        except IndexError:
-            nextchar = ''
-        end += 1
-
-        if nextchar == '}':
-            break
-        elif nextchar != ',':
-            raise ValueError(errmsg("Expecting , delimiter", s, end - 1))
-
-        try:
-            nextchar = s[end]
-            if nextchar in _ws:
-                end += 1
-                nextchar = s[end]
-                if nextchar in _ws:
-                    end = _w(s, end + 1).end()
-                    nextchar = s[end]
-        except IndexError:
-            nextchar = ''
-
-        end += 1
-        if nextchar != '"':
-            raise ValueError(errmsg("Expecting property name", s, end - 1))
-
-    if object_hook is not None:
-        pairs = object_hook(pairs)
-    return pairs, end
-
-def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
-    values = []
-    nextchar = s[end:end + 1]
-    if nextchar in _ws:
-        end = _w(s, end + 1).end()
-        nextchar = s[end:end + 1]
-    # Look-ahead for trivial empty array
-    if nextchar == ']':
-        return values, end + 1
-    _append = values.append
-    while True:
-        try:
-            value, end = scan_once(s, end)
-        except StopIteration:
-            raise ValueError(errmsg("Expecting object", s, end))
-        _append(value)
-        nextchar = s[end:end + 1]
-        if nextchar in _ws:
-            end = _w(s, end + 1).end()
-            nextchar = s[end:end + 1]
-        end += 1
-        if nextchar == ']':
-            break
-        elif nextchar != ',':
-            raise ValueError(errmsg("Expecting , delimiter", s, end))
-
-        try:
-            if s[end] in _ws:
-                end += 1
-                if s[end] in _ws:
-                    end = _w(s, end + 1).end()
-        except IndexError:
-            pass
-
-    return values, end
-
-class JSONDecoder(object):
-    """Simple JSON <http://json.org> decoder
-
-    Performs the following translations in decoding by default:
-
-    +---------------+-------------------+
-    | JSON          | Python            |
-    +===============+===================+
-    | object        | dict              |
-    +---------------+-------------------+
-    | array         | list              |
-    +---------------+-------------------+
-    | string        | unicode           |
-    +---------------+-------------------+
-    | number (int)  | int, long         |
-    +---------------+-------------------+
-    | number (real) | float             |
-    +---------------+-------------------+
-    | true          | True              |
-    +---------------+-------------------+
-    | false         | False             |
-    +---------------+-------------------+
-    | null          | None              |
-    +---------------+-------------------+
-
-    It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
-    their corresponding ``float`` values, which is outside the JSON spec.
-
-    """
-
-    def __init__(self, encoding=None, object_hook=None, parse_float=None,
-            parse_int=None, parse_constant=None, strict=True):
-        """``encoding`` determines the encoding used to interpret any ``str``
-        objects decoded by this instance (utf-8 by default).  It has no
-        effect when decoding ``unicode`` objects.
-
-        Note that currently only encodings that are a superset of ASCII work,
-        strings of other encodings should be passed in as ``unicode``.
-
-        ``object_hook``, if specified, will be called with the result
-        of every JSON object decoded and its return value will be used in
-        place of the given ``dict``.  This can be used to provide custom
-        deserializations (e.g. to support JSON-RPC class hinting).
-
-        ``parse_float``, if specified, will be called with the string
-        of every JSON float to be decoded. By default this is equivalent to
-        float(num_str). This can be used to use another datatype or parser
-        for JSON floats (e.g. decimal.Decimal).
-
-        ``parse_int``, if specified, will be called with the string
-        of every JSON int to be decoded. By default this is equivalent to
-        int(num_str). This can be used to use another datatype or parser
-        for JSON integers (e.g. float).
-
-        ``parse_constant``, if specified, will be called with one of the
-        following strings: -Infinity, Infinity, NaN.
-        This can be used to raise an exception if invalid JSON numbers
-        are encountered.
-
-        """
-        self.encoding = encoding
-        self.object_hook = object_hook
-        self.parse_float = parse_float or float
-        self.parse_int = parse_int or int
-        self.parse_constant = parse_constant or _CONSTANTS.__getitem__
-        self.strict = strict
-        self.parse_object = JSONObject
-        self.parse_array = JSONArray
-        self.parse_string = scanstring
-        self.scan_once = make_scanner(self)
-
-    def decode(self, s, _w=WHITESPACE.match):
-        """Return the Python representation of ``s`` (a ``str`` or ``unicode``
-        instance containing a JSON document)
-
-        """
-        obj, end = self.raw_decode(s, idx=_w(s, 0).end())
-        end = _w(s, end).end()
-        if end != len(s):
-            raise ValueError(errmsg("Extra data", s, end, len(s)))
-        return obj
-
-    def raw_decode(self, s, idx=0):
-        """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning
-        with a JSON document) and return a 2-tuple of the Python
-        representation and the index in ``s`` where the document ended.
-
-        This can be used to decode a JSON document from a string that may
-        have extraneous data at the end.
-
-        """
-        try:
-            obj, end = self.scan_once(s, idx)
-        except StopIteration:
-            raise ValueError("No JSON object could be decoded")
-        return obj, end

ansible_mitogen/compat/simplejson/encoder.py

@@ -1,440 +0,0 @@
-"""Implementation of JSONEncoder
-"""
-import re
-
-try:
-    from simplejson._speedups import encode_basestring_ascii as c_encode_basestring_ascii
-except ImportError:
-    c_encode_basestring_ascii = None
-try:
-    from simplejson._speedups import make_encoder as c_make_encoder
-except ImportError:
-    c_make_encoder = None
-
-ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
-ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
-HAS_UTF8 = re.compile(r'[\x80-\xff]')
-ESCAPE_DCT = {
-    '\\': '\\\\',
-    '"': '\\"',
-    '\b': '\\b',
-    '\f': '\\f',
-    '\n': '\\n',
-    '\r': '\\r',
-    '\t': '\\t',
-}
-for i in range(0x20):
-    #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
-    ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
-
-# Assume this produces an infinity on all machines (probably not guaranteed)
-INFINITY = float('1e66666')
-FLOAT_REPR = repr
-
-def encode_basestring(s):
-    """Return a JSON representation of a Python string
-
-    """
-    def replace(match):
-        return ESCAPE_DCT[match.group(0)]
-    return '"' + ESCAPE.sub(replace, s) + '"'
-
-
-def py_encode_basestring_ascii(s):
-    """Return an ASCII-only JSON representation of a Python string
-
-    """
-    if isinstance(s, str) and HAS_UTF8.search(s) is not None:
-        s = s.decode('utf-8')
-    def replace(match):
-        s = match.group(0)
-        try:
-            return ESCAPE_DCT[s]
-        except KeyError:
-            n = ord(s)
-            if n < 0x10000:
-                #return '\\u{0:04x}'.format(n)
-                return '\\u%04x' % (n,)
-            else:
-                # surrogate pair
-                n -= 0x10000
-                s1 = 0xd800 | ((n >> 10) & 0x3ff)
-                s2 = 0xdc00 | (n & 0x3ff)
-                #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
-                return '\\u%04x\\u%04x' % (s1, s2)
-    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
-
-
-encode_basestring_ascii = c_encode_basestring_ascii or py_encode_basestring_ascii
-
-class JSONEncoder(object):
-    """Extensible JSON <http://json.org> encoder for Python data structures.
-
-    Supports the following objects and types by default:
-
-    +-------------------+---------------+
-    | Python            | JSON          |
-    +===================+===============+
-    | dict              | object        |
-    +-------------------+---------------+
-    | list, tuple       | array         |
-    +-------------------+---------------+
-    | str, unicode      | string        |
-    +-------------------+---------------+
-    | int, long, float  | number        |
-    +-------------------+---------------+
-    | True              | true          |
-    +-------------------+---------------+
-    | False             | false         |
-    +-------------------+---------------+
-    | None              | null          |
-    +-------------------+---------------+
-
-    To extend this to recognize other objects, subclass and implement a
-    ``.default()`` method with another method that returns a serializable
-    object for ``o`` if possible, otherwise it should call the superclass
-    implementation (to raise ``TypeError``).
-
-    """
-    item_separator = ', '
-    key_separator = ': '
-    def __init__(self, skipkeys=False, ensure_ascii=True,
-            check_circular=True, allow_nan=True, sort_keys=False,
-            indent=None, separators=None, encoding='utf-8', default=None):
-        """Constructor for JSONEncoder, with sensible defaults.
-
-        If skipkeys is false, then it is a TypeError to attempt
-        encoding of keys that are not str, int, long, float or None.  If
-        skipkeys is True, such items are simply skipped.
-
-        If ensure_ascii is true, the output is guaranteed to be str
-        objects with all incoming unicode characters escaped.  If
-        ensure_ascii is false, the output will be unicode object.
-
-        If check_circular is true, then lists, dicts, and custom encoded
-        objects will be checked for circular references during encoding to
-        prevent an infinite recursion (which would cause an OverflowError).
-        Otherwise, no such check takes place.
-
-        If allow_nan is true, then NaN, Infinity, and -Infinity will be
-        encoded as such.  This behavior is not JSON specification compliant,
-        but is consistent with most JavaScript based encoders and decoders.
-        Otherwise, it will be a ValueError to encode such floats.
-
-        If sort_keys is true, then the output of dictionaries will be
-        sorted by key; this is useful for regression tests to ensure
-        that JSON serializations can be compared on a day-to-day basis.
-
-        If indent is a non-negative integer, then JSON array
-        elements and object members will be pretty-printed with that
-        indent level.  An indent level of 0 will only insert newlines.
-        None is the most compact representation.
-
-        If specified, separators should be a (item_separator, key_separator)
-        tuple.  The default is (', ', ': ').  To get the most compact JSON
-        representation you should specify (',', ':') to eliminate whitespace.
-
-        If specified, default is a function that gets called for objects
-        that can't otherwise be serialized.  It should return a JSON encodable
-        version of the object or raise a ``TypeError``.
-
-        If encoding is not None, then all input strings will be
-        transformed into unicode using that encoding prior to JSON-encoding.
-        The default is UTF-8.
-
-        """
-
-        self.skipkeys = skipkeys
-        self.ensure_ascii = ensure_ascii
-        self.check_circular = check_circular
-        self.allow_nan = allow_nan
-        self.sort_keys = sort_keys
-        self.indent = indent
-        if separators is not None:
-            self.item_separator, self.key_separator = separators
-        if default is not None:
-            self.default = default
-        self.encoding = encoding
-
-    def default(self, o):
-        """Implement this method in a subclass such that it returns
-        a serializable object for ``o``, or calls the base implementation
-        (to raise a ``TypeError``).
-
-        For example, to support arbitrary iterators, you could
-        implement default like this::
-
-            def default(self, o):
-                try:
-                    iterable = iter(o)
-                except TypeError:
-                    pass
-                else:
-                    return list(iterable)
-                return JSONEncoder.default(self, o)
-
-        """
-        raise TypeError(repr(o) + " is not JSON serializable")
-
-    def encode(self, o):
-        """Return a JSON string representation of a Python data structure.
-
-        >>> JSONEncoder().encode({"foo": ["bar", "baz"]})
-        '{"foo": ["bar", "baz"]}'
-
-        """
-        # This is for extremely simple cases and benchmarks.
-        if isinstance(o, basestring):
-            if isinstance(o, str):
-                _encoding = self.encoding
-                if (_encoding is not None
-                        and not (_encoding == 'utf-8')):
-                    o = o.decode(_encoding)
-            if self.ensure_ascii:
-                return encode_basestring_ascii(o)
-            else:
-                return encode_basestring(o)
-        # This doesn't pass the iterator directly to ''.join() because the
-        # exceptions aren't as detailed.  The list call should be roughly
-        # equivalent to the PySequence_Fast that ''.join() would do.
-        chunks = self.iterencode(o, _one_shot=True)
-        if not isinstance(chunks, (list, tuple)):
-            chunks = list(chunks)
-        return ''.join(chunks)
-
-    def iterencode(self, o, _one_shot=False):
-        """Encode the given object and yield each string
-        representation as available.
-
-        For example::
-
-            for chunk in JSONEncoder().iterencode(bigobject):
-                mysocket.write(chunk)
-
-        """
-        if self.check_circular:
-            markers = {}
-        else:
-            markers = None
-        if self.ensure_ascii:
-            _encoder = encode_basestring_ascii
-        else:
-            _encoder = encode_basestring
-        if self.encoding != 'utf-8':
-            def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
-                if isinstance(o, str):
-                    o = o.decode(_encoding)
-                return _orig_encoder(o)
-
-        def floatstr(o, allow_nan=self.allow_nan, _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
-            # Check for specials.  Note that this type of test is processor- and/or
-            # platform-specific, so do tests which don't depend on the internals.
-
-            if o != o:
-                text = 'NaN'
-            elif o == _inf:
-                text = 'Infinity'
-            elif o == _neginf:
-                text = '-Infinity'
-            else:
-                return _repr(o)
-
-            if not allow_nan:
-                raise ValueError(
-                    "Out of range float values are not JSON compliant: " +
-                    repr(o))
-
-            return text
-
-
-        if _one_shot and c_make_encoder is not None and not self.indent and not self.sort_keys:
-            _iterencode = c_make_encoder(
-                markers, self.default, _encoder, self.indent,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, self.allow_nan)
-        else:
-            _iterencode = _make_iterencode(
-                markers, self.default, _encoder, self.indent, floatstr,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, _one_shot)
-        return _iterencode(o, 0)
-
-def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
-        ## HACK: hand-optimized bytecode; turn globals into locals
-        False=False,
-        True=True,
-        ValueError=ValueError,
-        basestring=basestring,
-        dict=dict,
-        float=float,
-        id=id,
-        int=int,
-        isinstance=isinstance,
-        list=list,
-        long=long,
-        str=str,
-        tuple=tuple,
-    ):
-
-    def _iterencode_list(lst, _current_indent_level):
-        if not lst:
-            yield '[]'
-            return
-        if markers is not None:
-            markerid = id(lst)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = lst
-        buf = '['
-        if _indent is not None:
-            _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            separator = _item_separator + newline_indent
-            buf += newline_indent
-        else:
-            newline_indent = None
-            separator = _item_separator
-        first = True
-        for value in lst:
-            if first:
-                first = False
-            else:
-                buf = separator
-            if isinstance(value, basestring):
-                yield buf + _encoder(value)
-            elif value is None:
-                yield buf + 'null'
-            elif value is True:
-                yield buf + 'true'
-            elif value is False:
-                yield buf + 'false'
-            elif isinstance(value, (int, long)):
-                yield buf + str(value)
-            elif isinstance(value, float):
-                yield buf + _floatstr(value)
-            else:
-                yield buf
-                if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
-                elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
-                else:
-                    chunks = _iterencode(value, _current_indent_level)
-                for chunk in chunks:
-                    yield chunk
-        if newline_indent is not None:
-            _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
-        yield ']'
-        if markers is not None:
-            del markers[markerid]
-
-    def _iterencode_dict(dct, _current_indent_level):
-        if not dct:
-            yield '{}'
-            return
-        if markers is not None:
-            markerid = id(dct)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = dct
-        yield '{'
-        if _indent is not None:
-            _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            item_separator = _item_separator + newline_indent
-            yield newline_indent
-        else:
-            newline_indent = None
-            item_separator = _item_separator
-        first = True
-        if _sort_keys:
-            items = dct.items()
-            items.sort(key=lambda kv: kv[0])
-        else:
-            items = dct.iteritems()
-        for key, value in items:
-            if isinstance(key, basestring):
-                pass
-            # JavaScript is weakly typed for these, so it makes sense to
-            # also allow them.  Many encoders seem to do something like this.
-            elif isinstance(key, float):
-                key = _floatstr(key)
-            elif key is True:
-                key = 'true'
-            elif key is False:
-                key = 'false'
-            elif key is None:
-                key = 'null'
-            elif isinstance(key, (int, long)):
-                key = str(key)
-            elif _skipkeys:
-                continue
-            else:
-                raise TypeError("key " + repr(key) + " is not a string")
-            if first:
-                first = False
-            else:
-                yield item_separator
-            yield _encoder(key)
-            yield _key_separator
-            if isinstance(value, basestring):
-                yield _encoder(value)
-            elif value is None:
-                yield 'null'
-            elif value is True:
-                yield 'true'
-            elif value is False:
-                yield 'false'
-            elif isinstance(value, (int, long)):
-                yield str(value)
-            elif isinstance(value, float):
-                yield _floatstr(value)
-            else:
-                if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
-                elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
-                else:
-                    chunks = _iterencode(value, _current_indent_level)
-                for chunk in chunks:
-                    yield chunk
-        if newline_indent is not None:
-            _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
-        yield '}'
-        if markers is not None:
-            del markers[markerid]
-
-    def _iterencode(o, _current_indent_level):
-        if isinstance(o, basestring):
-            yield _encoder(o)
-        elif o is None:
-            yield 'null'
-        elif o is True:
-            yield 'true'
-        elif o is False:
-            yield 'false'
-        elif isinstance(o, (int, long)):
-            yield str(o)
-        elif isinstance(o, float):
-            yield _floatstr(o)
-        elif isinstance(o, (list, tuple)):
-            for chunk in _iterencode_list(o, _current_indent_level):
-                yield chunk
-        elif isinstance(o, dict):
-            for chunk in _iterencode_dict(o, _current_indent_level):
-                yield chunk
-        else:
-            if markers is not None:
-                markerid = id(o)
-                if markerid in markers:
-                    raise ValueError("Circular reference detected")
-                markers[markerid] = o
-            o = _default(o)
-            for chunk in _iterencode(o, _current_indent_level):
-                yield chunk
-            if markers is not None:
-                del markers[markerid]
-
-    return _iterencode

ansible_mitogen/compat/simplejson/scanner.py

@@ -1,65 +0,0 @@
-"""JSON token scanner
-"""
-import re
-try:
-    from simplejson._speedups import make_scanner as c_make_scanner
-except ImportError:
-    c_make_scanner = None
-
-__all__ = ['make_scanner']
-
-NUMBER_RE = re.compile(
-    r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?',
-    (re.VERBOSE | re.MULTILINE | re.DOTALL))
-
-def py_make_scanner(context):
-    parse_object = context.parse_object
-    parse_array = context.parse_array
-    parse_string = context.parse_string
-    match_number = NUMBER_RE.match
-    encoding = context.encoding
-    strict = context.strict
-    parse_float = context.parse_float
-    parse_int = context.parse_int
-    parse_constant = context.parse_constant
-    object_hook = context.object_hook
-
-    def _scan_once(string, idx):
-        try:
-            nextchar = string[idx]
-        except IndexError:
-            raise StopIteration
-
-        if nextchar == '"':
-            return parse_string(string, idx + 1, encoding, strict)
-        elif nextchar == '{':
-            return parse_object((string, idx + 1), encoding, strict, _scan_once, object_hook)
-        elif nextchar == '[':
-            return parse_array((string, idx + 1), _scan_once)
-        elif nextchar == 'n' and string[idx:idx + 4] == 'null':
-            return None, idx + 4
-        elif nextchar == 't' and string[idx:idx + 4] == 'true':
-            return True, idx + 4
-        elif nextchar == 'f' and string[idx:idx + 5] == 'false':
-            return False, idx + 5
-
-        m = match_number(string, idx)
-        if m is not None:
-            integer, frac, exp = m.groups()
-            if frac or exp:
-                res = parse_float(integer + (frac or '') + (exp or ''))
-            else:
-                res = parse_int(integer)
-            return res, m.end()
-        elif nextchar == 'N' and string[idx:idx + 3] == 'NaN':
-            return parse_constant('NaN'), idx + 3
-        elif nextchar == 'I' and string[idx:idx + 8] == 'Infinity':
-            return parse_constant('Infinity'), idx + 8
-        elif nextchar == '-' and string[idx:idx + 9] == '-Infinity':
-            return parse_constant('-Infinity'), idx + 9
-        else:
-            raise StopIteration
-
-    return _scan_once
-
-make_scanner = c_make_scanner or py_make_scanner

ansible_mitogen/connection.py

@@ -26,8 +26,9 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals
+__metaclass__ = type
 
 import errno
 import logging
@@ -40,18 +41,16 @@ import time
 import ansible.constants as C
 import ansible.errors
 import ansible.plugins.connection
-import ansible.utils.shlex
 
 import mitogen.core
-import mitogen.fork
+import mitogen.parent
-import mitogen.utils
+import mitogen.service
 
 import ansible_mitogen.mixins
-import ansible_mitogen.parsing
 import ansible_mitogen.process
-import ansible_mitogen.services
 import ansible_mitogen.target
 import ansible_mitogen.transport_config
+import ansible_mitogen.utils.unsafe
 
 
 LOG = logging.getLogger(__name__)
@@ -120,7 +119,7 @@ def _connect_ssh(spec):
     """
     Return ContextService arguments for an SSH connection.
     """
-    if C.HOST_KEY_CHECKING:
+    if spec.host_key_checking():
         check_host_keys = 'enforce'
     else:
         check_host_keys = 'ignore'
@@ -146,9 +145,9 @@ def _connect_ssh(spec):
             'identity_file': private_key_file,
             'identities_only': False,
             'ssh_path': spec.ssh_executable(),
-            'connect_timeout': spec.ansible_ssh_timeout(),
+            'connect_timeout': spec.timeout(),
             'ssh_args': spec.ssh_args(),
-            'ssh_debug_level': spec.mitogen_ssh_debug_level(),
+            'ssh_debug_level': spec.verbosity(),
             'remote_name': get_remote_name(spec),
             'keepalive_count': (
                 spec.mitogen_ssh_keepalive_count() or 10
@@ -159,6 +158,7 @@ def _connect_ssh(spec):
         }
     }
 
+
 def _connect_buildah(spec):
     """
     Return ContextService arguments for a Buildah connection.
@@ -169,11 +169,12 @@ def _connect_buildah(spec):
             'username': spec.remote_user(),
             'container': spec.remote_addr(),
             'python_path': spec.python_path(),
-            'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(),
+            'connect_timeout': spec.timeout(),
             'remote_name': get_remote_name(spec),
         }
     }
 
+
 def _connect_docker(spec):
     """
     Return ContextService arguments for a Docker connection.
@@ -184,7 +185,7 @@ def _connect_docker(spec):
             'username': spec.remote_user(),
             'container': spec.remote_addr(),
             'python_path': spec.python_path(rediscover_python=True),
-            'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(),
+            'connect_timeout': spec.timeout(),
             'remote_name': get_remote_name(spec),
         }
     }
@@ -199,7 +200,7 @@ def _connect_kubectl(spec):
         'kwargs': {
             'pod': spec.remote_addr(),
             'python_path': spec.python_path(),
-            'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(),
+            'connect_timeout': spec.timeout(),
             'kubectl_path': spec.mitogen_kubectl_path(),
             'kubectl_args': spec.extra_args(),
             'remote_name': get_remote_name(spec),
@@ -217,7 +218,7 @@ def _connect_jail(spec):
             'username': spec.remote_user(),
             'container': spec.remote_addr(),
             'python_path': spec.python_path(),
-            'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(),
+            'connect_timeout': spec.timeout(),
             'remote_name': get_remote_name(spec),
         }
     }
@@ -233,7 +234,7 @@ def _connect_lxc(spec):
             'container': spec.remote_addr(),
             'python_path': spec.python_path(),
             'lxc_attach_path': spec.mitogen_lxc_attach_path(),
-            'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(),
+            'connect_timeout': spec.timeout(),
             'remote_name': get_remote_name(spec),
         }
     }
@@ -249,7 +250,7 @@ def _connect_lxd(spec):
             'container': spec.remote_addr(),
             'python_path': spec.python_path(),
             'lxc_path': spec.mitogen_lxc_path(),
-            'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(),
+            'connect_timeout': spec.timeout(),
             'remote_name': get_remote_name(spec),
         }
     }
@@ -262,6 +263,22 @@ def _connect_machinectl(spec):
     return _connect_setns(spec, kind='machinectl')
 
 
+def _connect_podman(spec):
+    """
+    Return ContextService arguments for a Docker connection.
+    """
+    return {
+        'method': 'podman',
+        'kwargs': {
+            'username': spec.remote_user(),
+            'container': spec.remote_addr(),
+            'python_path': spec.python_path(rediscover_python=True),
+            'connect_timeout': spec.timeout(),
+            'remote_name': get_remote_name(spec),
+        }
+    }
+
+
 def _connect_setns(spec, kind=None):
     """
     Return ContextService arguments for a mitogen_setns connection.
@@ -392,6 +409,7 @@ def _connect_mitogen_doas(spec):
 #: generating ContextService keyword arguments matching a connection
 #: specification.
 CONNECTION_METHOD = {
+    # Ansible connection plugins
     'buildah': _connect_buildah,
     'docker': _connect_docker,
     'kubectl': _connect_kubectl,
@@ -400,12 +418,18 @@ CONNECTION_METHOD = {
     'lxc': _connect_lxc,
     'lxd': _connect_lxd,
     'machinectl': _connect_machinectl,
+    'podman': _connect_podman,
     'setns': _connect_setns,
     'ssh': _connect_ssh,
     'smart': _connect_ssh,  # issue #548.
+
+    # Ansible become plugins
+    'community.general.doas': _connect_doas,
     'su': _connect_su,
     'sudo': _connect_sudo,
     'doas': _connect_doas,
+
+    # Mitogen specific methods
     'mitogen_su': _connect_mitogen_su,
     'mitogen_sudo': _connect_mitogen_sudo,
     'mitogen_doas': _connect_mitogen_doas,
@@ -469,6 +493,7 @@ class Connection(ansible.plugins.connection.ConnectionBase):
     login_context = None
 
     #: Only sudo, su, and doas are supported for now.
+    # Ansible ConnectionBase attribute, removed in Ansible >= 2.8
     become_methods = ['sudo', 'su', 'doas']
 
     #: Dict containing init_child() return value as recorded at startup by
@@ -506,15 +531,6 @@ class Connection(ansible.plugins.connection.ConnectionBase):
     # set by `_get_task_vars()` for interpreter discovery
     _action = None
 
-    def __del__(self):
-        """
-        Ansible cannot be trusted to always call close() e.g. the synchronize
-        action constructs a local connection like this. So provide a destructor
-        in the hopes of catching these cases.
-        """
-        # https://github.com/dw/mitogen/issues/140
-        self.close()
-
     def on_action_run(self, task_vars, delegate_to_hostname, loader_basedir):
         """
         Invoked by ActionModuleMixin to indicate a new task is about to start
@@ -669,6 +685,9 @@ class Connection(ansible.plugins.connection.ConnectionBase):
 
     @property
     def connected(self):
+        """
+        Ansible connection plugin property. Used by ansible-connection command.
+        """
         return self.context is not None
 
     def _spec_from_via(self, proxied_inventory_name, via_spec):
@@ -754,7 +773,7 @@ class Connection(ansible.plugins.connection.ConnectionBase):
                               C.BECOME_ALLOW_SAME_USER):
             stack += (CONNECTION_METHOD[spec.become_method()](spec),)
 
-        return stack
+        return ansible_mitogen.utils.unsafe.cast(stack)
 
     def _build_stack(self):
         """
@@ -787,7 +806,7 @@ class Connection(ansible.plugins.connection.ConnectionBase):
                 call_context=self.binding.get_service_context(),
                 service_name='ansible_mitogen.services.ContextService',
                 method_name='get',
-                stack=mitogen.utils.cast(list(stack)),
+                stack=ansible_mitogen.utils.unsafe.cast(list(stack)),
             )
         except mitogen.core.CallError:
             LOG.warning('Connection failed; stack configuration was:\n%s',
@@ -801,7 +820,7 @@ class Connection(ansible.plugins.connection.ConnectionBase):
 
         self.context = dct['context']
         self.chain = CallChain(self, self.context, pipelined=True)
-        if self._play_context.become:
+        if self.become:
             self.login_context = dct['via']
         else:
             self.login_context = self.context
@@ -827,14 +846,18 @@ class Connection(ansible.plugins.connection.ConnectionBase):
         the _connect_*() service calls defined above to cause the master
         process to establish the real connection on our behalf, or return a
         reference to the existing one.
+
+        Ansible connection plugin method.
         """
+        # In some Ansible connection plugins this method returns self.
+        # However nothing I've found uses it, it's not even assigned.
         if self.connected:
             return
 
         inventory_name, stack = self._build_stack()
         worker_model = ansible_mitogen.process.get_worker_model()
         self.binding = worker_model.get_binding(
-            mitogen.utils.cast(inventory_name)
+            ansible_mitogen.utils.unsafe.cast(inventory_name)
         )
         self._connect_stack(stack)
 
@@ -865,12 +888,37 @@ class Connection(ansible.plugins.connection.ConnectionBase):
         Arrange for the mitogen.master.Router running in the worker to
         gracefully shut down, and wait for shutdown to complete. Safe to call
         multiple times.
+
+        Ansible connection plugin method.
         """
         self._put_connection()
         if self.binding:
             self.binding.close()
             self.binding = None
 
+    def _mitogen_var_options(self, templar):
+        # Workaround for https://github.com/ansible/ansible/issues/84238
+        var_names = C.config.get_plugin_vars('connection', self._load_name)
+        variables = templar.available_variables
+        var_options = {
+            var_name: templar.template(variables[var_name])
+            for var_name in var_names
+            if var_name in variables
+        }
+
+        if self.allow_extras:
+            extras_var_prefix = 'ansible_%s_' % self.extras_prefix
+            var_options['_extras'] = {
+                var_name: templar.template(variables[var_name])
+                for var_name in variables
+                if var_name not in var_options
+                and var_name.startswith(extras_var_prefix)
+            }
+        else:
+            var_options['_extras'] = {}
+
+        return var_options
+
     reset_compat_msg = (
         'Mitogen only supports "reset_connection" on Ansible 2.5.6 or later'
     )
@@ -881,6 +929,8 @@ class Connection(ansible.plugins.connection.ConnectionBase):
         any local state we hold for the connection, returns the Connection to
         the 'disconnected' state, and informs ContextService the connection is
         bad somehow, and should be shut down and discarded.
+
+        Ansible connection plugin method.
         """
         if self._play_context.remote_addr is None:
             # <2.5.6 incorrectly populate PlayContext for reset_connection
@@ -889,23 +939,44 @@ class Connection(ansible.plugins.connection.ConnectionBase):
                 self.reset_compat_msg
             )
 
-        # Strategy's _execute_meta doesn't have an action obj but we'll need one for
+        # Handle templated connection variables during `meta: reset_connection`.
-        # running interpreter_discovery
+        # Many bugs/implementation details of Mitogen & Ansible collide here.
-        # will create a new temporary action obj for this purpose
+        # See #1079, #1096, #1132, ansible/ansible#84238, ...
-        self._action = ansible_mitogen.mixins.ActionModuleMixin(
+        try:
-            task=0,
+            task, templar = self._play_context.vars.pop(
-            connection=self,
+                '_mitogen.smuggled.reset_connection',
-            play_context=self._play_context,
+            )
-            loader=0,
+        except KeyError:
-            templar=0,
+            self._action_monkey_patched_by_mitogen = False
-            shared_loader_obj=0
+        else:
-        )
+            # LOG.info('%r.reset(): remote_addr=%r', self, self._play_context.remote_addr)
+            # ansible.plugins.strategy.StrategyBase._execute_meta() doesn't
+            # have an action object, which we need for interpreter_discovery.
+            # Create a temporary action object for this purpose.
+            self._action = ansible_mitogen.mixins.ActionModuleMixin(
+                task=task,
+                connection=self,
+                play_context=self._play_context,
+                loader=templar._loader,
+                templar=templar,
+                shared_loader_obj=0,
+            )
+            self._action_monkey_patched_by_mitogen = True
+
+            # Workaround for https://github.com/ansible/ansible/issues/84238
+            self.set_options(
+                task_keys=task.dump_attrs(),
+                var_options=self._mitogen_var_options(templar),
+            )
+
+            del task
+            del templar
 
         # Clear out state in case we were ever connected.
         self.close()
 
         inventory_name, stack = self._build_stack()
-        if self._play_context.become:
+        if self.become:
             stack = stack[:-1]
 
         worker_model = ansible_mitogen.process.get_worker_model()
@@ -915,11 +986,16 @@ class Connection(ansible.plugins.connection.ConnectionBase):
                 call_context=binding.get_service_context(),
                 service_name='ansible_mitogen.services.ContextService',
                 method_name='reset',
-                stack=mitogen.utils.cast(list(stack)),
+                stack=ansible_mitogen.utils.unsafe.cast(list(stack)),
             )
         finally:
             binding.close()
 
+        # Cleanup any monkey patching we did for `meta: reset_connection`
+        if self._action_monkey_patched_by_mitogen:
+            del self._action
+        del self._action_monkey_patched_by_mitogen
+
     # Compatibility with Ansible 2.4 wait_for_connection plug-in.
     _reset = reset
 
@@ -987,12 +1063,14 @@ class Connection(ansible.plugins.connection.ConnectionBase):
             Data to supply on ``stdin`` of the process.
         :returns:
             (return code, stdout bytes, stderr bytes)
+
+        Ansible connection plugin method.
         """
         emulate_tty = (not in_data and sudoable)
         rc, stdout, stderr = self.get_chain().call(
             ansible_mitogen.target.exec_command,
-            cmd=mitogen.utils.cast(cmd),
+            cmd=ansible_mitogen.utils.unsafe.cast(cmd),
-            in_data=mitogen.utils.cast(in_data),
+            in_data=ansible_mitogen.utils.unsafe.cast(in_data),
             chdir=mitogen_chdir or self.get_default_cwd(),
             emulate_tty=emulate_tty,
         )
@@ -1012,12 +1090,14 @@ class Connection(ansible.plugins.connection.ConnectionBase):
             Remote filesystem path to read.
         :param str out_path:
             Local filesystem path to write.
+
+        Ansible connection plugin method.
         """
         self._connect()
         ansible_mitogen.target.transfer_file(
             context=self.context,
             # in_path may be AnsibleUnicode
-            in_path=mitogen.utils.cast(in_path),
+            in_path=ansible_mitogen.utils.unsafe.cast(in_path),
             out_path=out_path
         )
 
@@ -1035,7 +1115,7 @@ class Connection(ansible.plugins.connection.ConnectionBase):
         """
         self.get_chain().call_no_reply(
             ansible_mitogen.target.write_path,
-            mitogen.utils.cast(out_path),
+            ansible_mitogen.utils.unsafe.cast(out_path),
             mitogen.core.Blob(data),
             mode=mode,
             utimes=utimes,
@@ -1061,6 +1141,8 @@ class Connection(ansible.plugins.connection.ConnectionBase):
             Local filesystem path to read.
         :param str out_path:
             Remote filesystem path to write.
+
+        Ansible connection plugin method.
         """
         try:
             st = os.stat(in_path)
@@ -1081,7 +1163,7 @@ class Connection(ansible.plugins.connection.ConnectionBase):
                     s = fp.read(self.SMALL_FILE_LIMIT + 1)
                 finally:
                     fp.close()
-            except OSError:
+            except OSError as e:
                 self._throw_io_error(e, in_path)
                 raise
 
@@ -1095,7 +1177,7 @@ class Connection(ansible.plugins.connection.ConnectionBase):
             call_context=self.binding.get_service_context(),
             service_name='mitogen.service.FileService',
             method_name='register',
-            path=mitogen.utils.cast(in_path)
+            path=ansible_mitogen.utils.unsafe.cast(in_path)
         )
 
         # For now this must remain synchronous, as the action plug-in may have
@@ -1105,6 +1187,6 @@ class Connection(ansible.plugins.connection.ConnectionBase):
         self.get_chain().call(
             ansible_mitogen.target.transfer_file,
             context=self.binding.get_child_service_context(),
-            in_path=in_path,
+            in_path=ansible_mitogen.utils.unsafe.cast(in_path),
-            out_path=out_path
+            out_path=ansible_mitogen.utils.unsafe.cast(out_path)
         )

ansible_mitogen/loaders.py

@@ -30,12 +30,16 @@
 Stable names for PluginLoader instances across Ansible versions.
 """
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import ansible.errors
 
 import ansible_mitogen.utils
 
 __all__ = [
     'action_loader',
+    'become_loader',
     'connection_loader',
     'module_loader',
     'module_utils_loader',
@@ -45,18 +49,6 @@ __all__ = [
 
 
 ANSIBLE_VERSION_MIN = (2, 10)
-ANSIBLE_VERSION_MAX = (2, 12)
-
-NEW_VERSION_MSG = (
-    "Your Ansible version (%s) is too recent. The most recent version\n"
-    "supported by Mitogen for Ansible is %s.x. Please check the Mitogen\n"
-    "release notes to see if a new version is available, otherwise\n"
-    "subscribe to the corresponding GitHub issue to be notified when\n"
-    "support becomes available.\n"
-    "\n"
-    "    https://mitogen.rtfd.io/en/latest/changelog.html\n"
-    "    https://github.com/mitogen-hq/mitogen/issues/\n"
-)
 OLD_VERSION_MSG = (
     "Your version of Ansible (%s) is too old. The oldest version supported by "
     "Mitogen for Ansible is %s."
@@ -74,11 +66,6 @@ def assert_supported_release():
             OLD_VERSION_MSG % (v, ANSIBLE_VERSION_MIN)
         )
 
-    if v[:2] > ANSIBLE_VERSION_MAX:
-        raise ansible.errors.AnsibleError(
-            NEW_VERSION_MSG % (v, ANSIBLE_VERSION_MAX)
-        )
-
 
 # this is the first file our strategy plugins import, so we need to check this here
 # in prior Ansible versions, connection_loader.get_with_context didn't exist, so if a user
@@ -87,6 +74,7 @@ assert_supported_release()
 
 
 from ansible.plugins.loader import action_loader
+from ansible.plugins.loader import become_loader
 from ansible.plugins.loader import connection_loader
 from ansible.plugins.loader import module_loader
 from ansible.plugins.loader import module_utils_loader
@@ -94,5 +82,5 @@ from ansible.plugins.loader import shell_loader
 from ansible.plugins.loader import strategy_loader
 
 # These are original, unwrapped implementations
-action_loader__get = action_loader.get
+action_loader__get_with_context = action_loader.get_with_context
-connection_loader__get = connection_loader.get_with_context
+connection_loader__get_with_context = connection_loader.get_with_context

ansible_mitogen/logging.py

@@ -26,19 +26,18 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
 import logging
 import os
 
-import mitogen.core
+import ansible.utils.display
+
 import mitogen.utils
 
-try:
-    from __main__ import display
-except ImportError:
-    from ansible.utils.display import Display
-    display = Display()
 
+display = ansible.utils.display.Display()
 
 #: The process name set via :func:`set_process_name`.
 _process_name = None

ansible_mitogen/mixins.py

@@ -26,52 +26,32 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import json
 import logging
 import os
 import pwd
 import random
 import traceback
 
-try:
-    from shlex import quote as shlex_quote
-except ImportError:
-    from pipes import quote as shlex_quote
-
-from ansible.module_utils._text import to_bytes
-from ansible.parsing.utils.jsonify import jsonify
-
 import ansible
-import ansible.constants
-import ansible.plugins
 import ansible.plugins.action
+import ansible.utils.unsafe_proxy
+import ansible.vars.clean
+
+from ansible.module_utils.common.text.converters import to_bytes, to_text
+from ansible.module_utils.six.moves import shlex_quote
 
 import mitogen.core
 import mitogen.select
-import mitogen.utils
 
 import ansible_mitogen.connection
 import ansible_mitogen.planner
 import ansible_mitogen.target
 import ansible_mitogen.utils
-
+import ansible_mitogen.utils.unsafe
-from ansible.module_utils._text import to_text
-
-try:
-    from ansible.utils.unsafe_proxy import wrap_var
-except ImportError:
-    from ansible.vars.unsafe_proxy import wrap_var
-
-try:
-    # ansible 2.8 moved remove_internal_keys to the clean module
-    from ansible.vars.clean import remove_internal_keys
-except ImportError:
-    try:
-        from ansible.vars.manager import remove_internal_keys
-    except ImportError:
-        # ansible 2.3.3 has remove_internal_keys as a protected func on the action class
-        # we'll fallback to calling self._remove_internal_keys in this case
-        remove_internal_keys = lambda a: "Not found"
 
 
 LOG = logging.getLogger(__name__)
@@ -123,13 +103,10 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
 
         # required for python interpreter discovery
         connection.templar = self._templar
-        self._finding_python_interpreter = False
+
-        self._rediscovered_python = False
+        self._mitogen_discovering_interpreter = False
-        # redeclaring interpreter discovery vars here in case running ansible < 2.8.0
+        self._mitogen_interpreter_candidate = None
-        self._discovered_interpreter_key = None
+        self._mitogen_rediscovered_interpreter = False
-        self._discovered_interpreter = False
-        self._discovery_deprecation_warnings = []
-        self._discovery_warnings = []
 
     def run(self, tmp=None, task_vars=None):
         """
@@ -185,7 +162,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
         LOG.debug('_remote_file_exists(%r)', path)
         return self._connection.get_chain().call(
             ansible_mitogen.target.file_exists,
-            mitogen.utils.cast(path)
+            ansible_mitogen.utils.unsafe.cast(path)
         )
 
     def _configure_module(self, module_name, module_args, task_vars=None):
@@ -242,8 +219,13 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
         Used by the base _execute_module(), and in <2.4 also by the template
         action module, and probably others.
         """
+        if data is None and ansible_mitogen.utils.ansible_version[:2] <= (2, 18):
+            data = '{}'
         if isinstance(data, dict):
-            data = jsonify(data)
+            try:
+                data = json.dumps(data, ensure_ascii=False)
+            except UnicodeDecodeError:
+                data = json.dumps(data)
         if not isinstance(data, bytes):
             data = to_bytes(data, errors='surrogate_or_strict')
 
@@ -278,7 +260,9 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
                   paths, mode, sudoable)
         return self.fake_shell(lambda: mitogen.select.Select.all(
             self._connection.get_chain().call_async(
-                ansible_mitogen.target.set_file_mode, path, mode
+                ansible_mitogen.target.set_file_mode,
+                ansible_mitogen.utils.unsafe.cast(path),
+                mode,
             )
             for path in paths
         ))
@@ -312,7 +296,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
         if not path.startswith('~'):
             # /home/foo -> /home/foo
             return path
-        if sudoable or not self._play_context.become:
+        if sudoable or not self._connection.become:
             if path == '~':
                 # ~ -> /home/dmw
                 return self._connection.homedir
@@ -322,7 +306,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
         # ~root/.ansible -> /root/.ansible
         return self._connection.get_chain(use_login=(not sudoable)).call(
             os.path.expanduser,
-            mitogen.utils.cast(path),
+            ansible_mitogen.utils.unsafe.cast(path),
         )
 
     def get_task_timeout_secs(self):
@@ -355,7 +339,9 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
 
     def _execute_module(self, module_name=None, module_args=None, tmp=None,
                         task_vars=None, persist_files=False,
-                        delete_remote_tmp=True, wrap_async=False):
+                        delete_remote_tmp=True, wrap_async=False,
+                        ignore_unknown_opts=False,
+                        ):
         """
         Collect up a module's execution environment then use it to invoke
         target.run_module() or helpers.run_module_async() in the target
@@ -368,7 +354,13 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
         if task_vars is None:
             task_vars = {}
 
-        self._update_module_args(module_name, module_args, task_vars)
+        if ansible_mitogen.utils.ansible_version[:2] >= (2, 17):
+            self._update_module_args(
+                module_name, module_args, task_vars,
+                ignore_unknown_opts=ignore_unknown_opts,
+            )
+        else:
+            self._update_module_args(module_name, module_args, task_vars)
         env = {}
         self._compute_environment_string(env)
         self._set_temp_file_args(module_args, wrap_async)
@@ -385,11 +377,11 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
             ansible_mitogen.planner.Invocation(
                 action=self,
                 connection=self._connection,
-                module_name=mitogen.core.to_text(module_name),
+                module_name=ansible_mitogen.utils.unsafe.cast(mitogen.core.to_text(module_name)),
-                module_args=mitogen.utils.cast(module_args),
+                module_args=ansible_mitogen.utils.unsafe.cast(module_args),
                 task_vars=task_vars,
                 templar=self._templar,
-                env=mitogen.utils.cast(env),
+                env=ansible_mitogen.utils.unsafe.cast(env),
                 wrap_async=wrap_async,
                 timeout_secs=self.get_task_timeout_secs(),
             )
@@ -401,10 +393,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
             self._remove_tmp_path(tmp)
 
         # prevents things like discovered_interpreter_* or ansible_discovered_interpreter_* from being set
-        # handle ansible 2.3.3 that has remove_internal_keys in a different place
+        ansible.vars.clean.remove_internal_keys(result)
-        check = remove_internal_keys(result)
-        if check == 'Not found':
-            self._remove_internal_keys(result)
 
         # taken from _execute_module of ansible 2.8.6
         # propagate interpreter discovery results back to the controller
@@ -415,20 +404,22 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
             # only cache discovered_interpreter if we're not running a rediscovery
             # rediscovery happens in places like docker connections that could have different
             # python interpreters than the main host
-            if not self._rediscovered_python:
+            if not self._mitogen_rediscovered_interpreter:
                 result['ansible_facts'][self._discovered_interpreter_key] = self._discovered_interpreter
 
-        if self._discovery_warnings:
+        discovery_warnings = getattr(self, '_discovery_warnings', [])
+        if discovery_warnings:
             if result.get('warnings') is None:
                 result['warnings'] = []
-            result['warnings'].extend(self._discovery_warnings)
+            result['warnings'].extend(discovery_warnings)
 
-        if self._discovery_deprecation_warnings:
+        discovery_deprecation_warnings = getattr(self, '_discovery_deprecation_warnings', [])
+        if discovery_deprecation_warnings:
             if result.get('deprecations') is None:
                 result['deprecations'] = []
-            result['deprecations'].extend(self._discovery_deprecation_warnings)
+            result['deprecations'].extend(discovery_deprecation_warnings)
 
-        return wrap_var(result)
+        return ansible.utils.unsafe_proxy.wrap_var(result)
 
     def _postprocess_response(self, result):
         """
@@ -445,7 +436,10 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
                     "stderr": "stderr data"
                 }
         """
-        data = self._parse_returned_data(result)
+        if ansible_mitogen.utils.ansible_version[:2] >= (2, 19):
+            data = self._parse_returned_data(result, profile='legacy')
+        else:
+            data = self._parse_returned_data(result)
 
         # Cutpasted from the base implementation.
         if 'stdout' in data and 'stdout_lines' not in data:
@@ -475,49 +469,36 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
         # calling exec_command until we run into the right python we'll use
         # chicken-and-egg issue, mitogen needs a python to run low_level_execute_command
         # which is required by Ansible's discover_interpreter function
-        if self._finding_python_interpreter:
+        if self._mitogen_discovering_interpreter:
-            possible_pythons = [
+            possible_pythons = self._mitogen_interpreter_candidates
-                '/usr/bin/python',
-                'python3',
-                'python3.7',
-                'python3.6',
-                'python3.5',
-                'python2.7',
-                'python2.6',
-                '/usr/libexec/platform-python',
-                '/usr/bin/python3',
-                'python'
-            ]
         else:
             # not used, just adding a filler value
             possible_pythons = ['python']
 
-        def _run_cmd():
-            return self._connection.exec_command(
-                cmd=cmd,
-                in_data=in_data,
-                sudoable=sudoable,
-                mitogen_chdir=chdir,
-            )
-
         for possible_python in possible_pythons:
             try:
-                self._possible_python_interpreter = possible_python
+                self._mitogen_interpreter_candidate = possible_python
-                rc, stdout, stderr = _run_cmd()
+                rc, stdout, stderr = self._connection.exec_command(
-            # TODO: what exception is thrown?
+                    cmd, in_data, sudoable, mitogen_chdir=chdir,
-            except:
+                )
+            except BaseException as exc:
                 # we've reached the last python attempted and failed
-                # TODO: could use enumerate(), need to check which version of python first had it though
+                if possible_python == possible_pythons[-1]:
-                if possible_python == 'python':
                     raise
                 else:
+                    LOG.debug(
+                        '%r._low_level_execute_command: candidate=%r ignored: %s, %r',
+                        self, possible_python, type(exc), exc,
+                    )
                     continue
 
         stdout_text = to_text(stdout, errors=encoding_errors)
+        stderr_text = to_text(stderr, errors=encoding_errors)
 
         return {
             'rc': rc,
             'stdout': stdout_text,
             'stdout_lines': stdout_text.splitlines(),
-            'stderr': stderr,
+            'stderr': stderr_text,
+            'stderr_lines': stderr_text.splitlines(),
         }

ansible_mitogen/module_finder.py

@@ -26,19 +26,34 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals
+__metaclass__ = type
 
 import collections
-import imp
+import logging
 import os
+import re
+import sys
 
-import mitogen.master
+if sys.version_info >= (3, 4):
+    import importlib.machinery
+    import importlib.util
+else:
+    import imp
 
+import mitogen.imports
 
+
+LOG = logging.getLogger(__name__)
 PREFIX = 'ansible.module_utils.'
 
 
+# Analog of `importlib.machinery.ModuleSpec` or `pkgutil.ModuleInfo`.
+#   name    Unqualified name of the module.
+#   path    Filesystem path of the module.
+#   kind    One of the constants in `imp`, as returned in `imp.find_module()`
+#   parent  `ansible_mitogen.module_finder.Module` of parent package (if any).
 Module = collections.namedtuple('Module', 'name path kind parent')
 
 
@@ -118,14 +133,121 @@ def find_relative(parent, name, path=()):
 
 
 def scan_fromlist(code):
-    for level, modname_s, fromlist in mitogen.master.scan_code_imports(code):
+    """Return an iterator of (level, name) for explicit imports in a code
+    object.
+
+    Not all names identify a module. `from os import name, path` generates
+    `(0, 'os.name'), (0, 'os.path')`, but `os.name` is usually a string.
+
+    >>> src = 'import a; import b.c; from d.e import f; from g import h, i\\n'
+    >>> code = compile(src, '<str>', 'exec')
+    >>> list(scan_fromlist(code))
+    [(0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')]
+    """
+    for level, modname_s, fromlist in mitogen.imports.codeobj_imports(code):
         for name in fromlist:
-            yield level, '%s.%s' % (modname_s, name)
+            yield level, str('%s.%s' % (modname_s, name))
         if not fromlist:
             yield level, modname_s
 
 
+def walk_imports(code, prefix=None):
+    """Return an iterator of names for implicit parent imports & explicit
+    imports in a code object.
+
+    If a prefix is provided, then only children of that prefix are included.
+    Not all names identify a module. `from os import name, path` generates
+    `'os', 'os.name', 'os.path'`, but `os.name` is usually a string.
+
+    >>> source = 'import a; import b; import b.c; from b.d import e, f\\n'
+    >>> code = compile(source, '<str>', 'exec')
+    >>> list(walk_imports(code))
+    ['a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f']
+    >>> list(walk_imports(code, prefix='b'))
+    ['b.c', 'b.d', 'b.d.e', 'b.d.f']
+    """
+    if prefix is None:
+        prefix = ''
+    pattern = re.compile(r'(^|\.)(\w+)')
+    start = len(prefix)
+    for _, name, fromlist in mitogen.imports.codeobj_imports(code):
+        if not name.startswith(prefix):
+            continue
+        for match in pattern.finditer(name, start):
+            yield name[:match.end()]
+        for leaf in fromlist:
+            yield str('%s.%s' % (name, leaf))
+
+
 def scan(module_name, module_path, search_path):
+    # type: (str, str, list[str]) -> list[(str, str, bool)]
+    """Return a list of (name, path, is_package) for ansible.module_utils
+    imports used by an Ansible module.
+    """
+    log = LOG.getChild('scan')
+    log.debug('%r, %r, %r', module_name, module_path, search_path)
+
+    if sys.version_info >= (3, 4):
+        result = _scan_importlib_find_spec(
+            module_name, module_path, search_path,
+        )
+        log.debug('_scan_importlib_find_spec %r', result)
+    else:
+        result = _scan_imp_find_module(module_name, module_path, search_path)
+        log.debug('_scan_imp_find_module %r', result)
+    return result
+
+
+def _scan_importlib_find_spec(module_name, module_path, search_path):
+    # type: (str, str, list[str]) -> list[(str, str, bool)]
+    module = importlib.machinery.ModuleSpec(
+        module_name, loader=None, origin=module_path,
+    )
+    prefix = importlib.machinery.ModuleSpec(
+        PREFIX.rstrip('.'), loader=None,
+    )
+    prefix.submodule_search_locations = search_path
+    queue = collections.deque([module])
+    specs = {prefix.name: prefix}
+    while queue:
+        spec = queue.popleft()
+        if spec.origin is None:
+            continue
+        try:
+            with open(spec.origin, 'rb') as f:
+                code = compile(f.read(), spec.name, 'exec')
+        except Exception as exc:
+            raise ValueError((exc, module, spec, specs))
+
+        for name in walk_imports(code, prefix.name):
+            if name in specs:
+                continue
+
+            parent_name = name.rpartition('.')[0]
+            parent = specs[parent_name]
+            if parent is None or not parent.submodule_search_locations:
+                specs[name] = None
+                continue
+
+            child = importlib.util._find_spec(
+                name, parent.submodule_search_locations,
+            )
+            if child is None or child.origin is None:
+                specs[name] = None
+                continue
+
+            specs[name] = child
+            queue.append(child)
+
+    del specs[prefix.name]
+    return sorted(
+        (spec.name, spec.origin, spec.submodule_search_locations is not None)
+        for spec in specs.values() if spec is not None
+    )
+
+
+def _scan_imp_find_module(module_name, module_path, search_path):
+    # type: (str, str, list[str]) -> list[(str, str, bool)]
     module = Module(module_name, module_path, imp.PY_SOURCE, None)
     stack = [module]
     seen = set()

ansible_mitogen/parsing.py

@@ -26,8 +26,9 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals
+__metaclass__ = type
 
 import mitogen.core
 

ansible_mitogen/planner.py

@@ -34,8 +34,9 @@ files/modules known missing.
 [0] "Ansible Module Architecture", developing_program_flow_modules.html
 """
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals
+__metaclass__ = type
 
 import json
 import logging
@@ -43,17 +44,18 @@ import os
 import random
 import re
 
-from ansible.executor import module_common
+import ansible.collections.list
-from ansible.collections.list import list_collection_dirs
 import ansible.errors
-import ansible.module_utils
+import ansible.executor.module_common
-import ansible.release
+
 import mitogen.core
 import mitogen.select
+import mitogen.service
 
 import ansible_mitogen.loaders
 import ansible_mitogen.parsing
 import ansible_mitogen.target
+import ansible_mitogen.utils.unsafe
 
 
 LOG = logging.getLogger(__name__)
@@ -169,6 +171,7 @@ class Planner(object):
         """
         binding = self._inv.connection.get_binding()
 
+        kwargs = ansible_mitogen.utils.unsafe.cast(kwargs)
         new = dict((mitogen.core.UnicodeType(k), kwargs[k])
                    for k in kwargs)
         new.setdefault('good_temp_dir',
@@ -192,7 +195,7 @@ class BinaryPlanner(Planner):
 
     @classmethod
     def detect(cls, path, source):
-        return module_common._is_binary(source)
+        return ansible.executor.module_common._is_binary(source)
 
     def get_push_files(self):
         return [mitogen.core.to_text(self._inv.module_path)]
@@ -203,7 +206,7 @@ class BinaryPlanner(Planner):
             module=self._inv.module_name,
             path=self._inv.module_path,
             json_args=json.dumps(self._inv.module_args),
-            env=self._inv.env,
+            env=ansible_mitogen.utils.unsafe.cast(self._inv.env),
             **kwargs
         )
 
@@ -215,12 +218,15 @@ class ScriptPlanner(BinaryPlanner):
     """
     def _rewrite_interpreter(self, path):
         """
-        Given the original interpreter binary extracted from the script's
+        Given the interpreter path (from the script's hashbang line), return
-        interpreter line, look up the associated `ansible_*_interpreter`
+        the desired interpreter path. This tries, in order
-        variable, render it and return it.
+
+        1. Look up & render the `ansible_*_interpreter` variable, if set
+        2. Look up the `discovered_interpreter_*` fact, if present
+        3. The unmodified path from the hashbang line.
 
         :param str path:
-            Absolute UNIX path to original interpreter.
+            Absolute path to original interpreter (e.g. '/usr/bin/python').
 
         :returns:
             Shell fragment prefix used to execute the script via "/bin/sh -c".
@@ -228,13 +234,25 @@ class ScriptPlanner(BinaryPlanner):
             involved here, the vanilla implementation uses it and that use is
             exploited in common playbooks.
         """
-        key = u'ansible_%s_interpreter' % os.path.basename(path).strip()
+        interpreter_name = os.path.basename(path).strip()
+        key = u'ansible_%s_interpreter' % interpreter_name
         try:
             template = self._inv.task_vars[key]
         except KeyError:
-            return path
+            pass
+        else:
+            configured_interpreter = self._inv.templar.template(template)
+            return ansible_mitogen.utils.unsafe.cast(configured_interpreter)
 
-        return mitogen.utils.cast(self._inv.templar.template(template))
+        key = u'discovered_interpreter_%s' % interpreter_name
+        try:
+            discovered_interpreter = self._inv.task_vars['ansible_facts'][key]
+        except KeyError:
+            pass
+        else:
+            return ansible_mitogen.utils.unsafe.cast(discovered_interpreter)
+
+        return path
 
     def _get_interpreter(self):
         path, arg = ansible_mitogen.parsing.parse_hashbang(
@@ -249,7 +267,8 @@ class ScriptPlanner(BinaryPlanner):
         if arg:
             fragment += ' ' + arg
 
-        return fragment, path.startswith('python')
+        is_python = path.startswith('python')
+        return fragment, is_python
 
     def get_kwargs(self, **kwargs):
         interpreter_fragment, is_python = self._get_interpreter()
@@ -269,7 +288,7 @@ class JsonArgsPlanner(ScriptPlanner):
 
     @classmethod
     def detect(cls, path, source):
-        return module_common.REPLACER_JSONARGS in source
+        return ansible.executor.module_common.REPLACER_JSONARGS in source
 
 
 class WantJsonPlanner(ScriptPlanner):
@@ -298,11 +317,11 @@ class NewStylePlanner(ScriptPlanner):
     preprocessing the module.
     """
     runner_name = 'NewStyleRunner'
-    MARKER = re.compile(b'from ansible(?:_collections|\.module_utils)\.')
+    MARKER = re.compile(br'from ansible(?:_collections|\.module_utils)\.')
 
     @classmethod
     def detect(cls, path, source):
-        return cls.MARKER.search(source) != None
+        return cls.MARKER.search(source) is not None
 
     def _get_interpreter(self):
         return None, None
@@ -323,6 +342,67 @@ class NewStylePlanner(ScriptPlanner):
         'dnf',  # issue #280; py-dnf/hawkey need therapy
         'firewalld',  # issue #570: ansible module_utils caches dbus conn
         'ansible.legacy.dnf',  # issue #776
+        'ansible.builtin.dnf', # issue #832
+        'freeipa.ansible_freeipa.ipaautomember', # issue #1216
+        'freeipa.ansible_freeipa.ipaautomountkey',
+        'freeipa.ansible_freeipa.ipaautomountlocation',
+        'freeipa.ansible_freeipa.ipaautomountmap',
+        'freeipa.ansible_freeipa.ipacert',
+        'freeipa.ansible_freeipa.ipaclient_api',
+        'freeipa.ansible_freeipa.ipaclient_fix_ca',
+        'freeipa.ansible_freeipa.ipaclient_fstore',
+        'freeipa.ansible_freeipa.ipaclient_get_otp',
+        'freeipa.ansible_freeipa.ipaclient_ipa_conf',
+        'freeipa.ansible_freeipa.ipaclient_join',
+        'freeipa.ansible_freeipa.ipaclient_set_hostname',
+        'freeipa.ansible_freeipa.ipaclient_setup_automount',
+        'freeipa.ansible_freeipa.ipaclient_setup_certmonger',
+        'freeipa.ansible_freeipa.ipaclient_setup_firefox',
+        'freeipa.ansible_freeipa.ipaclient_setup_krb5',
+        'freeipa.ansible_freeipa.ipaclient_setup_nis',
+        'freeipa.ansible_freeipa.ipaclient_setup_nss',
+        'freeipa.ansible_freeipa.ipaclient_setup_ntp',
+        'freeipa.ansible_freeipa.ipaclient_setup_ssh',
+        'freeipa.ansible_freeipa.ipaclient_setup_sshd',
+        'freeipa.ansible_freeipa.ipaclient_temp_krb5',
+        'freeipa.ansible_freeipa.ipaclient_test',
+        'freeipa.ansible_freeipa.ipaclient_test_keytab',
+        'freeipa.ansible_freeipa.ipaconfig',
+        'freeipa.ansible_freeipa.ipadelegation',
+        'freeipa.ansible_freeipa.ipadnsconfig',
+        'freeipa.ansible_freeipa.ipadnsforwardzone',
+        'freeipa.ansible_freeipa.ipadnsrecord',
+        'freeipa.ansible_freeipa.ipadnszone',
+        'freeipa.ansible_freeipa.ipagroup',
+        'freeipa.ansible_freeipa.ipahbacrule',
+        'freeipa.ansible_freeipa.ipahbacsvc',
+        'freeipa.ansible_freeipa.ipahbacsvcgroup',
+        'freeipa.ansible_freeipa.ipahost',
+        'freeipa.ansible_freeipa.ipahostgroup',
+        'freeipa.ansible_freeipa.idoverridegroup',
+        'freeipa.ansible_freeipa.idoverrideuser',
+        'freeipa.ansible_freeipa.idp',
+        'freeipa.ansible_freeipa.idrange',
+        'freeipa.ansible_freeipa.idview',
+        'freeipa.ansible_freeipa.ipalocation',
+        'freeipa.ansible_freeipa.ipanetgroup',
+        'freeipa.ansible_freeipa.ipapermission',
+        'freeipa.ansible_freeipa.ipaprivilege',
+        'freeipa.ansible_freeipa.ipapwpolicy',
+        'freeipa.ansible_freeipa.iparole',
+        'freeipa.ansible_freeipa.ipaselfservice',
+        'freeipa.ansible_freeipa.ipaserver',
+        'freeipa.ansible_freeipa.ipaservice',
+        'freeipa.ansible_freeipa.ipaservicedelegationrule',
+        'freeipa.ansible_freeipa.ipaservicedelegationtarget',
+        'freeipa.ansible_freeipa.ipasudocmd',
+        'freeipa.ansible_freeipa.ipasudocmdgroup',
+        'freeipa.ansible_freeipa.ipasudorule',
+        'freeipa.ansible_freeipa.ipatopologysegment',
+        'freeipa.ansible_freeipa.ipatopologysuffix',
+        'freeipa.ansible_freeipa.ipatrust',
+        'freeipa.ansible_freeipa.ipauser',
+        'freeipa.ansible_freeipa.ipavault',
     ])
 
     def should_fork(self):
@@ -362,7 +442,7 @@ class NewStylePlanner(ScriptPlanner):
                 module_name='ansible_module_%s' % (self._inv.module_name,),
                 module_path=self._inv.module_path,
                 search_path=self.get_search_path(),
-                builtin_path=module_common._MODULE_UTILS_PATH,
+                builtin_path=ansible.executor.module_common._MODULE_UTILS_PATH,
                 context=self._inv.connection.context,
             )
         return self._module_map
@@ -405,7 +485,7 @@ class ReplacerPlanner(NewStylePlanner):
 
     @classmethod
     def detect(cls, path, source):
-        return module_common.REPLACER in source
+        return ansible.executor.module_common.REPLACER in source
 
 
 class OldStylePlanner(ScriptPlanner):
@@ -427,12 +507,6 @@ _planners = [
 ]
 
 
-try:
-    _get_ansible_module_fqn = module_common._get_ansible_module_fqn
-except AttributeError:
-    _get_ansible_module_fqn = None
-
-
 def py_modname_from_path(name, path):
     """
     Fetch the logical name of a new-style module as it might appear in
@@ -442,11 +516,12 @@ def py_modname_from_path(name, path):
       package hierarchy approximated on the target, enabling relative imports
       to function correctly. For example, "ansible.modules.system.setup".
     """
-    if _get_ansible_module_fqn:
+    try:
-        try:
+        return ansible.executor.module_common._get_ansible_module_fqn(path)
-            return _get_ansible_module_fqn(path)
+    except AttributeError:
-        except ValueError:
+        pass
-            pass
+    except ValueError:
+        pass
 
     return 'ansible.modules.' + name
 
@@ -464,7 +539,7 @@ def read_file(path):
     finally:
         os.close(fd)
 
-    return mitogen.core.b('').join(bits)
+    return b''.join(bits)
 
 
 def _propagate_deps(invocation, planner, context):
@@ -492,7 +567,7 @@ def _invoke_async_task(invocation, planner):
         call_recv = context.call_async(
             ansible_mitogen.target.run_module_async,
             job_id=job_id,
-            timeout_secs=invocation.timeout_secs,
+            timeout_secs=ansible_mitogen.utils.unsafe.cast(invocation.timeout_secs),
             started_sender=started_recv.to_sender(),
             kwargs=planner.get_kwargs(),
         )
@@ -528,12 +603,15 @@ def _invoke_isolated_task(invocation, planner):
         context.shutdown()
 
 
-def _get_planner(name, path, source):
+def _get_planner(invocation, source):
     for klass in _planners:
-        if klass.detect(path, source):
+        if klass.detect(invocation.module_path, source):
-            LOG.debug('%r accepted %r (filename %r)', klass, name, path)
+            LOG.debug(
+                '%r accepted %r (filename %r)',
+                klass, invocation.module_name, invocation.module_path,
+            )
             return klass
-        LOG.debug('%r rejected %r', klass, name)
+        LOG.debug('%r rejected %r', klass, invocation.module_name)
     raise ansible.errors.AnsibleError(NO_METHOD_MSG + repr(invocation))
 
 
@@ -558,13 +636,30 @@ def _fix_py35(invocation, module_source):
         invocation._overridden_sources[invocation.module_path] = module_source
 
 
+def _fix_dnf(invocation, module_source):
+    """
+    Handles edge case where dnf ansible module showed failure due to a missing import in the dnf module.
+    Specifically addresses errors like "Failed loading plugin 'debuginfo-install': module 'dnf' has no attribute 'cli'".
+    https://github.com/mitogen-hq/mitogen/issues/1143
+    This issue is resolved by adding 'dnf.cli' to the import statement in the module source.
+    This works in vanilla Ansible but not in Mitogen otherwise.
+    """
+    if invocation.module_name in {'ansible.builtin.dnf', 'ansible.legacy.dnf', 'dnf'} and \
+            invocation.module_path not in invocation._overridden_sources:
+        module_source = module_source.replace(
+            b"import dnf\n",
+            b"import dnf, dnf.cli\n"
+        )
+        invocation._overridden_sources[invocation.module_path] = module_source
+
+
 def _load_collections(invocation):
     """
     Special loader that ensures that `ansible_collections` exist as a module path for import
     Goes through all collection path possibilities and stores paths to installed collections
     Stores them on the current invocation to later be passed to the master service
     """
-    for collection_path in list_collection_dirs():
+    for collection_path in ansible.collections.list.list_collection_dirs():
         invocation._extra_sys_paths.add(collection_path.decode('utf-8'))
 
 
@@ -595,9 +690,9 @@ def invoke(invocation):
 
         module_source = invocation.get_module_source()
         _fix_py35(invocation, module_source)
+        _fix_dnf(invocation, module_source)
         _planner_by_path[invocation.module_path] = _get_planner(
-            invocation.module_name,
+            invocation,
-            invocation.module_path,
             module_source
         )
 

ansible_mitogen/plugins/action/mitogen_fetch.py

@@ -18,23 +18,17 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import os
-
+import base64
-from ansible.module_utils._text import to_bytes
+from ansible.errors import AnsibleError, AnsibleActionFail, AnsibleActionSkip
+from ansible.module_utils.common.text.converters import to_bytes, to_text
 from ansible.module_utils.six import string_types
 from ansible.module_utils.parsing.convert_bool import boolean
 from ansible.plugins.action import ActionBase
-from ansible.utils.hashing import checksum, md5, secure_hash
+from ansible.utils.display import Display
-from ansible.utils.path import makedirs_safe
+from ansible.utils.hashing import checksum, checksum_s, md5, secure_hash
-
+from ansible.utils.path import makedirs_safe, is_subpath
 
-REMOTE_CHECKSUM_ERRORS = {
+display = Display()
-    '0': "unable to calculate the checksum of the remote file",
-    '1': "the remote file does not exist",
-    '2': "no read permission on remote file",
-    '3': "remote file is a directory, fetch cannot work on directories",
-    '4': "python isn't present on the system.  Unable to compute checksum",
-    '5': "stdlib json was not found on the remote machine. Only the raw module can work without those installed",
-}
 
 
 class ActionModule(ActionBase):
@@ -45,36 +39,94 @@ class ActionModule(ActionBase):
             task_vars = dict()
 
         result = super(ActionModule, self).run(tmp, task_vars)
+        del tmp  # tmp no longer has any effect
+
         try:
             if self._play_context.check_mode:
-                result['skipped'] = True
+                raise AnsibleActionSkip('check mode not (yet) supported for this module')
-                result['msg'] = 'check mode not (yet) supported for this module'
-                return result
 
+            source = self._task.args.get('src', None)
+            original_dest = dest = self._task.args.get('dest', None)
             flat = boolean(self._task.args.get('flat'), strict=False)
             fail_on_missing = boolean(self._task.args.get('fail_on_missing', True), strict=False)
             validate_checksum = boolean(self._task.args.get('validate_checksum', True), strict=False)
 
+            msg = ''
             # validate source and dest are strings FIXME: use basic.py and module specs
-            source = self._task.args.get('src')
             if not isinstance(source, string_types):
-                result['msg'] = "Invalid type supplied for source option, it must be a string"
+                msg = "Invalid type supplied for source option, it must be a string"
 
-            dest = self._task.args.get('dest')
             if not isinstance(dest, string_types):
-                result['msg'] = "Invalid type supplied for dest option, it must be a string"
+                msg = "Invalid type supplied for dest option, it must be a string"
+
+            if source is None or dest is None:
+                msg = "src and dest are required"
 
-            if result.get('msg'):
+            if msg:
-                result['failed'] = True
+                raise AnsibleActionFail(msg)
-                return result
 
             source = self._connection._shell.join_path(source)
             source = self._remote_expand_user(source)
 
-            # calculate checksum for the remote file, don't bother if using
+            remote_stat = {}
-            # become as slurp will be used Force remote_checksum to follow
+            remote_checksum = None
-            # symlinks because fetch always follows symlinks
+            if True:
-            remote_checksum = self._remote_checksum(source, all_vars=task_vars, follow=True)
+                # Get checksum for the remote file even using become. Mitogen doesn't need slurp.
+                # Follow symlinks because fetch always follows symlinks
+                try:
+                    remote_stat = self._execute_remote_stat(source, all_vars=task_vars, follow=True)
+                except AnsibleError as ae:
+                    result['changed'] = False
+                    result['file'] = source
+                    if fail_on_missing:
+                        result['failed'] = True
+                        result['msg'] = to_text(ae)
+                    else:
+                        result['msg'] = "%s, ignored" % to_text(ae, errors='surrogate_or_replace')
+
+                    return result
+
+                remote_checksum = remote_stat.get('checksum')
+                if remote_stat.get('exists'):
+                    if remote_stat.get('isdir'):
+                        result['failed'] = True
+                        result['changed'] = False
+                        result['msg'] = "remote file is a directory, fetch cannot work on directories"
+
+                        # Historically, these don't fail because you may want to transfer
+                        # a log file that possibly MAY exist but keep going to fetch other
+                        # log files. Today, this is better achieved by adding
+                        # ignore_errors or failed_when to the task.  Control the behaviour
+                        # via fail_when_missing
+                        if not fail_on_missing:
+                            result['msg'] += ", not transferring, ignored"
+                            del result['changed']
+                            del result['failed']
+
+                        return result
+
+            # use slurp if permissions are lacking or privilege escalation is needed
+            remote_data = None
+            if remote_checksum in (None, '1', ''):
+                slurpres = self._execute_module(module_name='ansible.legacy.slurp', module_args=dict(src=source), task_vars=task_vars)
+                if slurpres.get('failed'):
+                    if not fail_on_missing:
+                        result['file'] = source
+                        result['changed'] = False
+                    else:
+                        result.update(slurpres)
+
+                    if 'not found' in slurpres.get('msg', ''):
+                        result['msg'] = "the remote file does not exist, not transferring, ignored"
+                    elif slurpres.get('msg', '').startswith('source is a directory'):
+                        result['msg'] = "remote file is a directory, fetch cannot work on directories"
+
+                    return result
+                else:
+                    if slurpres['encoding'] == 'base64':
+                        remote_data = base64.b64decode(slurpres['content'])
+                    if remote_data is not None:
+                        remote_checksum = checksum_s(remote_data)
 
             # calculate the destination name
             if os.path.sep not in self._connection._shell.join_path('a', ''):
@@ -83,13 +135,14 @@ class ActionModule(ActionBase):
             else:
                 source_local = source
 
-            dest = os.path.expanduser(dest)
+            # ensure we only use file name, avoid relative paths
+            if not is_subpath(dest, original_dest):
+                # TODO: ? dest = os.path.expanduser(dest.replace(('../','')))
+                raise AnsibleActionFail("Detected directory traversal, expected to be contained in '%s' but got '%s'" % (original_dest, dest))
+
             if flat:
                 if os.path.isdir(to_bytes(dest, errors='surrogate_or_strict')) and not dest.endswith(os.sep):
-                    result['msg'] = "dest is an existing directory, use a trailing slash if you want to fetch src into that directory"
+                    raise AnsibleActionFail("dest is an existing directory, use a trailing slash if you want to fetch src into that directory")
-                    result['file'] = dest
-                    result['failed'] = True
-                    return result
                 if dest.endswith(os.sep):
                     # if the path ends with "/", we'll use the source filename as the
                     # destination filename
@@ -106,23 +159,7 @@ class ActionModule(ActionBase):
                     target_name = self._play_context.remote_addr
                 dest = "%s/%s/%s" % (self._loader.path_dwim(dest), target_name, source_local)
 
-            dest = dest.replace("//", "/")
+            dest = os.path.normpath(dest)
-
-            if remote_checksum in REMOTE_CHECKSUM_ERRORS:
-                result['changed'] = False
-                result['file'] = source
-                result['msg'] = REMOTE_CHECKSUM_ERRORS[remote_checksum]
-                # Historically, these don't fail because you may want to transfer
-                # a log file that possibly MAY exist but keep going to fetch other
-                # log files. Today, this is better achieved by adding
-                # ignore_errors or failed_when to the task.  Control the behaviour
-                # via fail_when_missing
-                if fail_on_missing:
-                    result['failed'] = True
-                    del result['changed']
-                else:
-                    result['msg'] += ", not transferring, ignored"
-                return result
 
             # calculate checksum for the local file
             local_checksum = checksum(dest)
@@ -132,7 +169,15 @@ class ActionModule(ActionBase):
                 makedirs_safe(os.path.dirname(dest))
 
                 # fetch the file and check for changes
-                self._connection.fetch_file(source, dest)
+                if remote_data is None:
+                    self._connection.fetch_file(source, dest)
+                else:
+                    try:
+                        f = open(to_bytes(dest, errors='surrogate_or_strict'), 'wb')
+                        f.write(remote_data)
+                        f.close()
+                    except (IOError, OSError) as e:
+                        raise AnsibleActionFail("Failed to fetch the file: %s" % e)
                 new_checksum = secure_hash(dest)
                 # For backwards compatibility. We'll return None on FIPS enabled systems
                 try:
@@ -157,10 +202,6 @@ class ActionModule(ActionBase):
                 result.update(dict(changed=False, md5sum=local_md5, file=source, dest=dest, checksum=local_checksum))
 
         finally:
-            try:
+            self._remove_tmp_path(self._connection._shell.tmpdir)
-                self._remove_tmp_path(self._connection._shell.tmpdir)
-            except AttributeError:
-                # .tmpdir was added to ShellModule in v2.6.0, so old versions don't have it
-                pass
 
         return result

ansible_mitogen/plugins/action/mitogen_get_stack.py

@@ -26,14 +26,15 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
 """
 Fetch the connection configuration stack that would be used to connect to a
 target, without actually connecting to it.
 """
 
+from __future__ import absolute_import, division, print_function
+from __future__ import unicode_literals
+__metaclass__ = type
+
 import ansible_mitogen.connection
 
 from ansible.plugins.action import ActionBase

ansible_mitogen/plugins/connection/mitogen_buildah.py

@@ -26,16 +26,16 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
-import os.path
+__metaclass__ = type
+
+import os
 import sys
 
 try:
     import ansible_mitogen
 except ImportError:
-    base_dir = os.path.dirname(__file__)
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
-    sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
-    del base_dir
 
 import ansible_mitogen.connection
 

ansible_mitogen/plugins/connection/mitogen_doas.py

@@ -26,16 +26,16 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
-import os.path
+__metaclass__ = type
+
+import os
 import sys
 
 try:
-    import ansible_mitogen.connection
+    import ansible_mitogen
 except ImportError:
-    base_dir = os.path.dirname(__file__)
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
-    sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
-    del base_dir
 
 import ansible_mitogen.connection
 

ansible_mitogen/plugins/connection/mitogen_docker.py

@@ -26,16 +26,16 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
-import os.path
+__metaclass__ = type
+
+import os
 import sys
 
 try:
     import ansible_mitogen
 except ImportError:
-    base_dir = os.path.dirname(__file__)
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
-    sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
-    del base_dir
 
 import ansible_mitogen.connection
 

ansible_mitogen/plugins/connection/mitogen_jail.py

@@ -26,16 +26,16 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
-import os.path
+__metaclass__ = type
+
+import os
 import sys
 
 try:
     import ansible_mitogen
 except ImportError:
-    base_dir = os.path.dirname(__file__)
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
-    sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
-    del base_dir
 
 import ansible_mitogen.connection
 

ansible_mitogen/plugins/connection/mitogen_kubectl.py

@@ -27,32 +27,29 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
-import os.path
+__metaclass__ = type
+
+import os
 import sys
 
-from ansible.errors import AnsibleConnectionFailure
+import ansible.errors
-from ansible.module_utils.six import iteritems
 
 try:
     import ansible_mitogen
 except ImportError:
-    base_dir = os.path.dirname(__file__)
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
-    sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
-    del base_dir
 
 import ansible_mitogen.connection
 import ansible_mitogen.loaders
 
 
-_get_result = ansible_mitogen.loaders.connection_loader__get(
-    'kubectl',
-    class_only=True,
-)
-
-
 class Connection(ansible_mitogen.connection.Connection):
     transport = 'kubectl'
+    (vanilla_class, load_context) = ansible_mitogen.loaders.connection_loader__get_with_context(
+        'kubectl',
+        class_only=True,
+    )
 
     not_supported_msg = (
         'The "mitogen_kubectl" plug-in requires a version of Ansible '
@@ -60,20 +57,17 @@ class Connection(ansible_mitogen.connection.Connection):
     )
 
     def __init__(self, *args, **kwargs):
-        if not _get_result:
+        if not Connection.vanilla_class:
-            raise AnsibleConnectionFailure(self.not_supported_msg)
+            raise ansible.errors.AnsibleConnectionFailure(self.not_supported_msg)
         super(Connection, self).__init__(*args, **kwargs)
 
     def get_extra_args(self):
-        try:
+        connection_options = Connection.vanilla_class.connection_options
-            # Ansible < 2.10, _get_result is the connection class
-            connection_options = _get_result.connection_options
-        except AttributeError:
-            # Ansible >= 2.10, _get_result is a get_with_context_result
-            connection_options = _get_result.object.connection_options
         parameters = []
-        for key, option in iteritems(connection_options):
+        for key in connection_options:
-            if self.get_task_var('ansible_' + key) is not None:
+            task_var_name = 'ansible_%s' % key
-                parameters += [ option, self.get_task_var('ansible_' + key) ]
+            task_var = self.get_task_var(task_var_name)
+            if task_var is not None:
+                parameters += [connection_options[key], task_var]
 
         return parameters

ansible_mitogen/plugins/connection/mitogen_local.py

@@ -26,27 +26,21 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
-import os.path
+__metaclass__ = type
+
+import os
 import sys
 
 try:
-    import ansible_mitogen.connection
+    import ansible_mitogen
 except ImportError:
-    base_dir = os.path.dirname(__file__)
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
-    sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
-    del base_dir
 
 import ansible_mitogen.connection
 import ansible_mitogen.process
 
-
+viewkeys = getattr(dict, 'viewkeys', dict.keys)
-if sys.version_info > (3,):
-    viewkeys = dict.keys
-elif sys.version_info > (2, 7):
-    viewkeys = dict.viewkeys
-else:
-    viewkeys = lambda dct: set(dct)
 
 
 def dict_diff(old, new):

ansible_mitogen/plugins/connection/mitogen_lxc.py

@@ -26,16 +26,16 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
-import os.path
+__metaclass__ = type
+
+import os
 import sys
 
 try:
     import ansible_mitogen
 except ImportError:
-    base_dir = os.path.dirname(__file__)
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
-    sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
-    del base_dir
 
 import ansible_mitogen.connection
 

ansible_mitogen/plugins/connection/mitogen_lxd.py

@@ -26,16 +26,16 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
-import os.path
+__metaclass__ = type
+
+import os
 import sys
 
 try:
     import ansible_mitogen
 except ImportError:
-    base_dir = os.path.dirname(__file__)
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
-    sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
-    del base_dir
 
 import ansible_mitogen.connection
 

ansible_mitogen/plugins/connection/mitogen_machinectl.py

@@ -26,16 +26,16 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
-import os.path
+__metaclass__ = type
+
+import os
 import sys
 
 try:
-    import ansible_mitogen.connection
+    import ansible_mitogen
 except ImportError:
-    base_dir = os.path.dirname(__file__)
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
-    sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
-    del base_dir
 
 import ansible_mitogen.connection
 

ansible_mitogen/plugins/connection/mitogen_podman.py

@@ -0,0 +1,44 @@
+# Copyright 2022, Mitogen contributers
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+try:
+    import ansible_mitogen
+except ImportError:
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
+
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'podman'

ansible_mitogen/plugins/connection/mitogen_setns.py

@@ -26,16 +26,16 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
-import os.path
+__metaclass__ = type
+
+import os
 import sys
 
 try:
-    import ansible_mitogen.connection
+    import ansible_mitogen
 except ImportError:
-    base_dir = os.path.dirname(__file__)
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
-    sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
-    del base_dir
 
 import ansible_mitogen.connection
 

ansible_mitogen/plugins/connection/mitogen_ssh.py

@@ -26,28 +26,31 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
-import os.path
+__metaclass__ = type
+
+import os
 import sys
 
+from ansible.plugins.connection.ssh import (
+    DOCUMENTATION as _ansible_ssh_DOCUMENTATION,
+)
+
 DOCUMENTATION = """
+    name: mitogen_ssh
     author: David Wilson <dw@botanicus.net>
-    connection: mitogen_ssh
     short_description: Connect over SSH via Mitogen
     description:
         - This connects using an OpenSSH client controlled by the Mitogen for
           Ansible extension. It accepts every option the vanilla ssh plugin
           accepts.
-    version_added: "2.5"
     options:
-"""
+""" + _ansible_ssh_DOCUMENTATION.partition('options:\n')[2]
 
 try:
     import ansible_mitogen
 except ImportError:
-    base_dir = os.path.dirname(__file__)
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
-    sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
-    del base_dir
 
 import ansible_mitogen.connection
 import ansible_mitogen.loaders
@@ -55,7 +58,7 @@ import ansible_mitogen.loaders
 
 class Connection(ansible_mitogen.connection.Connection):
     transport = 'ssh'
-    vanilla_class = ansible_mitogen.loaders.connection_loader__get(
+    (vanilla_class, load_context) = ansible_mitogen.loaders.connection_loader__get_with_context(
         'ssh',
         class_only=True,
     )

ansible_mitogen/plugins/connection/mitogen_su.py

@@ -26,16 +26,16 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
-import os.path
+__metaclass__ = type
+
+import os
 import sys
 
 try:
-    import ansible_mitogen.connection
+    import ansible_mitogen
 except ImportError:
-    base_dir = os.path.dirname(__file__)
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
-    sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
-    del base_dir
 
 import ansible_mitogen.connection
 

ansible_mitogen/plugins/connection/mitogen_sudo.py

@@ -26,16 +26,16 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
-import os.path
+__metaclass__ = type
+
+import os
 import sys
 
 try:
-    import ansible_mitogen.connection
+    import ansible_mitogen
 except ImportError:
-    base_dir = os.path.dirname(__file__)
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
-    sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
-    del base_dir
 
 import ansible_mitogen.connection
 

ansible_mitogen/plugins/strategy/mitogen.py

@@ -26,8 +26,10 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
-import os.path
+__metaclass__ = type
+
+import os
 import sys
 
 #
@@ -45,12 +47,10 @@ import sys
 # debuggers and isinstance() work predictably.
 #
 
-BASE_DIR = os.path.abspath(
+try:
-    os.path.join(os.path.dirname(__file__), '../../..')
+    import ansible_mitogen
-)
+except ImportError:
-
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
-if BASE_DIR not in sys.path:
-    sys.path.insert(0, BASE_DIR)
 
 import ansible_mitogen.strategy
 import ansible.plugins.strategy.linear

ansible_mitogen/plugins/strategy/mitogen_free.py

@@ -26,8 +26,10 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
-import os.path
+__metaclass__ = type
+
+import os
 import sys
 
 #
@@ -45,12 +47,10 @@ import sys
 # debuggers and isinstance() work predictably.
 #
 
-BASE_DIR = os.path.abspath(
+try:
-    os.path.join(os.path.dirname(__file__), '../../..')
+    import ansible_mitogen
-)
+except ImportError:
-
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
-if BASE_DIR not in sys.path:
-    sys.path.insert(0, BASE_DIR)
 
 import ansible_mitogen.loaders
 import ansible_mitogen.strategy

ansible_mitogen/plugins/strategy/mitogen_host_pinned.py

@@ -26,8 +26,10 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
-import os.path
+__metaclass__ = type
+
+import os
 import sys
 
 #
@@ -45,12 +47,10 @@ import sys
 # debuggers and isinstance() work predictably.
 #
 
-BASE_DIR = os.path.abspath(
+try:
-    os.path.join(os.path.dirname(__file__), '../../..')
+    import ansible_mitogen
-)
+except ImportError:
-
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
-if BASE_DIR not in sys.path:
-    sys.path.insert(0, BASE_DIR)
 
 import ansible_mitogen.loaders
 import ansible_mitogen.strategy

ansible_mitogen/plugins/strategy/mitogen_linear.py

@@ -26,8 +26,10 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
-import os.path
+__metaclass__ = type
+
+import os
 import sys
 
 #
@@ -45,12 +47,10 @@ import sys
 # debuggers and isinstance() work predictably.
 #
 
-BASE_DIR = os.path.abspath(
+try:
-    os.path.join(os.path.dirname(__file__), '../../..')
+    import ansible_mitogen
-)
+except ImportError:
-
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
-if BASE_DIR not in sys.path:
-    sys.path.insert(0, BASE_DIR)
 
 import ansible_mitogen.loaders
 import ansible_mitogen.strategy

ansible_mitogen/process.py

@@ -26,7 +26,9 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
 import atexit
 import logging
 import multiprocessing
@@ -36,9 +38,9 @@ import socket
 import signal
 import sys
 
-try:
+if sys.version_info >= (3, 3):
     import faulthandler
-except ImportError:
+else:
     faulthandler = None
 
 try:
@@ -59,10 +61,9 @@ import mitogen.utils
 import ansible
 import ansible.constants as C
 import ansible.errors
+
 import ansible_mitogen.logging
 import ansible_mitogen.services
-
-from mitogen.core import b
 import ansible_mitogen.affinity
 
 
@@ -178,42 +179,6 @@ def setup_pool(pool):
     LOG.debug('Service pool configured: size=%d', pool.size)
 
 
-def _setup_simplejson(responder):
-    """
-    We support serving simplejson for Python 2.4 targets on Ansible 2.3, at
-    least so the package's own CI Docker scripts can run without external
-    help, however newer versions of simplejson no longer support Python
-    2.4. Therefore override any installed/loaded version with a
-    2.4-compatible version we ship in the compat/ directory.
-    """
-    responder.whitelist_prefix('simplejson')
-
-    # issue #536: must be at end of sys.path, in case existing newer
-    # version is already loaded.
-    compat_path = os.path.join(os.path.dirname(__file__), 'compat')
-    sys.path.append(compat_path)
-
-    for fullname, is_pkg, suffix in (
-        (u'simplejson', True, '__init__.py'),
-        (u'simplejson.decoder', False, 'decoder.py'),
-        (u'simplejson.encoder', False, 'encoder.py'),
-        (u'simplejson.scanner', False, 'scanner.py'),
-    ):
-        path = os.path.join(compat_path, 'simplejson', suffix)
-        fp = open(path, 'rb')
-        try:
-            source = fp.read()
-        finally:
-            fp.close()
-
-        responder.add_source_override(
-            fullname=fullname,
-            path=path,
-            source=source,
-            is_pkg=is_pkg,
-        )
-
-
 def _setup_responder(responder):
     """
     Configure :class:`mitogen.master.ModuleResponder` to only permit
@@ -221,7 +186,6 @@ def _setup_responder(responder):
     """
     responder.whitelist_prefix('ansible')
     responder.whitelist_prefix('ansible_mitogen')
-    _setup_simplejson(responder)
 
     # Ansible 2.3 is compatible with Python 2.4 targets, however
     # ansible/__init__.py is not. Instead, executor/module_common.py writes
@@ -316,11 +280,11 @@ def get_cpu_count(default=None):
 
 class Broker(mitogen.master.Broker):
     """
-    WorkerProcess maintains at most 2 file descriptors, therefore does not need
+    WorkerProcess maintains fewer file descriptors, therefore does not need
     the exuberant syscall expense of EpollPoller, so override it and restore
     the poll() poller.
     """
-    poller_class = mitogen.core.Poller
+    poller_class = mitogen.parent.POLLER_LIGHTWEIGHT
 
 
 class Binding(object):
@@ -462,7 +426,7 @@ class ClassicWorkerModel(WorkerModel):
 
         common_setup(_init_logging=_init_logging)
 
-        self.parent_sock, self.child_sock = socket.socketpair()
+        self.parent_sock, self.child_sock = mitogen.core.socketpair()
         mitogen.core.set_cloexec(self.parent_sock.fileno())
         mitogen.core.set_cloexec(self.child_sock.fileno())
 
@@ -674,7 +638,7 @@ class MuxProcess(object):
 
         try:
             # Let the parent know our listening socket is ready.
-            mitogen.core.io_op(self.model.child_sock.send, b('1'))
+            mitogen.core.io_op(self.model.child_sock.send, b'1')
             # Block until the socket is closed, which happens on parent exit.
             mitogen.core.io_op(self.model.child_sock.recv, 1)
         finally:

ansible_mitogen/runner.py

@@ -36,8 +36,13 @@ Each class in here has a corresponding Planner class in planners.py that knows
 how to build arguments for it, preseed related data, etc.
 """
 
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
 import atexit
-import imp
+import ctypes
+import json
+import logging
 import os
 import re
 import shlex
@@ -47,41 +52,25 @@ import tempfile
 import traceback
 import types
 
+if sys.version_info >= (3, 4):
+    import importlib.machinery
+else:
+    import imp
+
+from ansible.module_utils.six.moves import shlex_quote
+
 import mitogen.core
 import ansible_mitogen.target  # TODO: circular import
-from mitogen.core import b
-from mitogen.core import bytes_partition
-from mitogen.core import str_rpartition
 from mitogen.core import to_text
 
-try:
-    import ctypes
-except ImportError:
-    # Python 2.4
-    ctypes = None
-
-try:
-    import json
-except ImportError:
-    # Python 2.4
-    import simplejson as json
-
 try:
     # Cannot use cStringIO as it does not support Unicode.
     from StringIO import StringIO
 except ImportError:
     from io import StringIO
 
-try:
-    from shlex import quote as shlex_quote
-except ImportError:
-    from pipes import quote as shlex_quote
-
-# Absolute imports for <2.5.
-logging = __import__('logging')
-
-
 # Prevent accidental import of an Ansible module from hanging on stdin read.
+# FIXME Should probably be b'{}' or None. Ansible 2.19 has bytes | None = None.
 import ansible.module_utils.basic
 ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
 
@@ -90,15 +79,13 @@ ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
 # explicit call to res_init() on each task invocation. BSD-alikes export it
 # directly, Linux #defines it as "__res_init".
 libc__res_init = None
-if ctypes:
+libc = ctypes.CDLL(None)
-    libc = ctypes.CDLL(None)
+for symbol in 'res_init', '__res_init':
-    for symbol in 'res_init', '__res_init':
+    try:
-        try:
+        libc__res_init = getattr(libc, symbol)
-            libc__res_init = getattr(libc, symbol)
+    except AttributeError:
-        except AttributeError:
+        pass
-            pass
 
-iteritems = getattr(dict, 'iteritems', dict.items)
 LOG = logging.getLogger(__name__)
 
 
@@ -109,7 +96,7 @@ def shlex_split_b(s):
     bytes.
     """
     assert isinstance(s, mitogen.core.BytesType)
-    if mitogen.core.PY3:
+    if sys.version_info >= (3, 0):
         return [
             t.encode('latin1')
             for t in shlex.split(s.decode('latin1'), comments=True)
@@ -212,13 +199,13 @@ class EnvironmentFileWatcher(object):
         for line in fp:
             # '   #export foo=some var  ' -> ['#export', 'foo=some var  ']
             bits = shlex_split_b(line)
-            if (not bits) or bits[0].startswith(b('#')):
+            if (not bits) or bits[0].startswith(b'#'):
                 continue
 
-            if bits[0] == b('export'):
+            if bits[0] == b'export':
                 bits.pop(0)
 
-            key, sep, value = bytes_partition(b(' ').join(bits), b('='))
+            key, sep, value = b' '.join(bits).partition(b'=')
             if key and sep:
                 yield key, value
 
@@ -516,10 +503,71 @@ class ModuleUtilsImporter(object):
             sys.modules.pop(fullname, None)
 
     def find_module(self, fullname, path=None):
+        """
+        Return a loader for the module with fullname, if we will load it.
+
+        Implements importlib.abc.MetaPathFinder.find_module().
+        Deprecrated in Python 3.4+, replaced by find_spec().
+        Raises ImportWarning in Python 3.10+. Removed in Python 3.12.
+        """
         if fullname in self._by_fullname:
             return self
 
+    def find_spec(self, fullname, path, target=None):
+        """
+        Return a `ModuleSpec` for module with `fullname` if we will load it.
+        Otherwise return `None`.
+
+        Implements importlib.abc.MetaPathFinder.find_spec(). Python 3.4+.
+        """
+        if fullname.endswith('.'):
+            return None
+
+        try:
+            module_path, is_package = self._by_fullname[fullname]
+        except KeyError:
+            LOG.debug('Skipping %s: not present', fullname)
+            return None
+
+        LOG.debug('Handling %s', fullname)
+        origin = 'master:%s' % (module_path,)
+        return importlib.machinery.ModuleSpec(
+            fullname, loader=self, origin=origin, is_package=is_package,
+        )
+
+    def create_module(self, spec):
+        """
+        Return a module object for the given ModuleSpec.
+
+        Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4.
+        Unlike Loader.load_module() this shouldn't populate sys.modules or
+        set module attributes. Both are done by Python.
+        """
+        module = types.ModuleType(spec.name)
+        # FIXME create_module() shouldn't initialise module attributes
+        module.__file__ = spec.origin
+        return module
+
+    def exec_module(self, module):
+        """
+        Execute the module to initialise it. Don't return anything.
+
+        Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4.
+        """
+        spec = module.__spec__
+        path, _ = self._by_fullname[spec.name]
+        source = ansible_mitogen.target.get_small_file(self._context, path)
+        code = compile(source, path, 'exec', 0, 1)
+        exec(code, module.__dict__)
+        self._loaded.add(spec.name)
+
     def load_module(self, fullname):
+        """
+        Return the loaded module specified by fullname.
+
+        Implements PEP 302 importlib.abc.Loader.load_module().
+        Deprecated in Python 3.4+, replaced by create_module() & exec_module().
+        """
         path, is_pkg = self._by_fullname[fullname]
         source = ansible_mitogen.target.get_small_file(self._context, path)
         code = compile(source, path, 'exec', 0, 1)
@@ -530,7 +578,7 @@ class ModuleUtilsImporter(object):
             mod.__path__ = []
             mod.__package__ = str(fullname)
         else:
-            mod.__package__ = str(str_rpartition(to_text(fullname), '.')[0])
+            mod.__package__ = str(to_text(fullname).rpartition('.')[0])
         exec(code, mod.__dict__)
         self._loaded.add(fullname)
         return mod
@@ -545,7 +593,7 @@ class TemporaryEnvironment(object):
     def __init__(self, env=None):
         self.original = dict(os.environ)
         self.env = env or {}
-        for key, value in iteritems(self.env):
+        for key, value in mitogen.core.iteritems(self.env):
             key = mitogen.core.to_text(key)
             value = mitogen.core.to_text(value)
             if value is None:
@@ -585,6 +633,7 @@ class NewStyleStdio(object):
         sys.stderr = StringIO()
         encoded = json.dumps({'ANSIBLE_MODULE_ARGS': args})
         ansible.module_utils.basic._ANSIBLE_ARGS = utf8(encoded)
+        ansible.module_utils.basic._ANSIBLE_PROFILE = 'legacy'
         sys.stdin = StringIO(mitogen.core.to_text(encoded))
 
         self.original_get_path = getattr(ansible.module_utils.basic,
@@ -599,7 +648,9 @@ class NewStyleStdio(object):
         sys.stdout = self.original_stdout
         sys.stderr = self.original_stderr
         sys.stdin = self.original_stdin
+        # FIXME Should probably be b'{}' or None. Ansible 2.19 has bytes | None = None.
         ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
+        ansible.module_utils.basic._ANSIBLE_PROFILE = None
 
 
 class ProgramRunner(Runner):
@@ -753,7 +804,7 @@ class ScriptRunner(ProgramRunner):
         self.interpreter_fragment = interpreter_fragment
         self.is_python = is_python
 
-    b_ENCODING_STRING = b('# -*- coding: utf-8 -*-')
+    b_ENCODING_STRING = b'# -*- coding: utf-8 -*-'
 
     def _get_program(self):
         return self._rewrite_source(
@@ -786,13 +837,13 @@ class ScriptRunner(ProgramRunner):
         # While Ansible rewrites the #! using ansible_*_interpreter, it is
         # never actually used to execute the script, instead it is a shell
         # fragment consumed by shell/__init__.py::build_module_command().
-        new = [b('#!') + utf8(self.interpreter_fragment)]
+        new = [b'#!' + utf8(self.interpreter_fragment)]
         if self.is_python:
             new.append(self.b_ENCODING_STRING)
 
-        _, _, rest = bytes_partition(s, b('\n'))
+        _, _, rest = s.partition(b'\n')
         new.append(rest)
-        return b('\n').join(new)
+        return b'\n'.join(new)
 
 
 class NewStyleRunner(ScriptRunner):
@@ -820,12 +871,17 @@ class NewStyleRunner(ScriptRunner):
         synchronization mechanism by importing everything the module will need
         prior to detaching.
         """
+        # I think "custom" means "found in custom module_utils search path",
+        # e.g. playbook relative dir, ~/.ansible/..., Ansible collection.
         for fullname, _, _ in self.module_map['custom']:
             mitogen.core.import_module(fullname)
+
+        # I think "builtin" means "part of ansible/ansible-base/ansible-core",
+        # as opposed to Python builtin modules such as sys.
         for fullname in self.module_map['builtin']:
             try:
                 mitogen.core.import_module(fullname)
-            except ImportError:
+            except ImportError as exc:
                 # #590: Ansible 2.8 module_utils.distro is a package that
                 # replaces itself in sys.modules with a non-package during
                 # import. Prior to replacement, it is a real package containing
@@ -836,8 +892,18 @@ class NewStyleRunner(ScriptRunner):
                 # loop progresses to the next entry and attempts to preload
                 # 'distro._distro', the import mechanism will fail. So here we
                 # silently ignore any failure for it.
-                if fullname != 'ansible.module_utils.distro._distro':
+                if fullname == 'ansible.module_utils.distro._distro':
-                    raise
+                    continue
+
+                # ansible.module_utils.compat.selinux raises ImportError if it
+                # can't load libselinux.so. The importer would usually catch
+                # this & skip selinux operations. We don't care about selinux,
+                # we're using import to get a copy of the module.
+                if (fullname == 'ansible.module_utils.compat.selinux'
+                    and exc.msg == 'unable to load libselinux.so'):
+                    continue
+
+                raise
 
     def _setup_excepthook(self):
         """
@@ -890,8 +956,7 @@ class NewStyleRunner(ScriptRunner):
     # change the default encoding. This hack was removed from Ansible long ago,
     # but not before permeating into many third party modules.
     PREHISTORIC_HACK_RE = re.compile(
-        b(r'reload\s*\(\s*sys\s*\)\s*'
+        br'reload\s*\(\s*sys\s*\)\s*sys\s*\.\s*setdefaultencoding\([^)]+\)',
-          r'sys\s*\.\s*setdefaultencoding\([^)]+\)')
     )
 
     def _setup_program(self):
@@ -899,7 +964,7 @@ class NewStyleRunner(ScriptRunner):
             context=self.service_context,
             path=self.path,
         )
-        self.source = self.PREHISTORIC_HACK_RE.sub(b(''), source)
+        self.source = self.PREHISTORIC_HACK_RE.sub(b'', source)
 
     def _get_code(self):
         try:
@@ -914,10 +979,10 @@ class NewStyleRunner(ScriptRunner):
                 True,                   # dont_inherit
             ))
 
-    if mitogen.core.PY3:
+    if sys.version_info >= (3, 0):
         main_module_name = '__main__'
     else:
-        main_module_name = b('__main__')
+        main_module_name = b'__main__'
 
     def _handle_magic_exception(self, mod, exc):
         """
@@ -935,7 +1000,7 @@ class NewStyleRunner(ScriptRunner):
 
     def _run_code(self, code, mod):
         try:
-            if mitogen.core.PY3:
+            if sys.version_info >= (3, 0):
                 exec(code, vars(mod))
             else:
                 exec('exec code in vars(mod)')
@@ -949,10 +1014,10 @@ class NewStyleRunner(ScriptRunner):
         approximation of the original package hierarchy, so that relative
         imports function correctly.
         """
-        pkg, sep, modname = str_rpartition(self.py_module_name, '.')
+        pkg, sep, _ = self.py_module_name.rpartition('.')
         if not sep:
             return None
-        if mitogen.core.PY3:
+        if sys.version_info >= (3, 0):
             return pkg
         return pkg.encode()
 
@@ -992,7 +1057,7 @@ class NewStyleRunner(ScriptRunner):
 
 
 class JsonArgsRunner(ScriptRunner):
-    JSON_ARGS = b('<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>')
+    JSON_ARGS = b'<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>'
 
     def _get_args_contents(self):
         return json.dumps(self.args).encode()

ansible_mitogen/services.py

@@ -39,23 +39,26 @@ connections, grant access to files by children, and register for notification
 when a child has completed a job.
 """
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
 from __future__ import unicode_literals
+__metaclass__ = type
 
 import logging
 import os
-import os.path
 import sys
 import threading
 
 import ansible.constants
 
-import mitogen
+from ansible.module_utils.six import reraise
+
+import mitogen.core
 import mitogen.service
-import mitogen.utils
 import ansible_mitogen.loaders
 import ansible_mitogen.module_finder
 import ansible_mitogen.target
+import ansible_mitogen.utils
+import ansible_mitogen.utils.unsafe
 
 
 LOG = logging.getLogger(__name__)
@@ -66,20 +69,6 @@ LOG = logging.getLogger(__name__)
 ansible_mitogen.loaders.shell_loader.get('sh')
 
 
-if sys.version_info[0] == 3:
-    def reraise(tp, value, tb):
-        if value is None:
-            value = tp()
-        if value.__traceback__ is not tb:
-            raise value.with_traceback(tb)
-        raise value
-else:
-    exec(
-        "def reraise(tp, value, tb=None):\n"
-        "    raise tp, value, tb\n"
-     )
-
-
 def _get_candidate_temp_dirs():
     try:
         # >=2.5
@@ -91,7 +80,7 @@ def _get_candidate_temp_dirs():
         remote_tmp = ansible.constants.DEFAULT_REMOTE_TMP
         system_tmpdirs = ('/var/tmp', '/tmp')
 
-    return mitogen.utils.cast([remote_tmp] + list(system_tmpdirs))
+    return ansible_mitogen.utils.unsafe.cast([remote_tmp] + list(system_tmpdirs))
 
 
 def key_from_dict(**kwargs):
@@ -350,7 +339,12 @@ class ContextService(mitogen.service.Service):
         'ansible_mitogen.target',
         'mitogen.fork',
         'mitogen.service',
-    )
+    ) + ((
+        'ansible.module_utils._internal._json._profiles._module_legacy_c2m',
+        'ansible.module_utils._internal._json._profiles._module_legacy_m2c',
+        'ansible.module_utils._internal._json._profiles._module_modern_c2m',
+        'ansible.module_utils._internal._json._profiles._module_legacy_m2c',
+    ) if ansible_mitogen.utils.ansible_version[:2] >= (2, 19) else ())
 
     def _send_module_forwards(self, context):
         if hasattr(self.router.responder, 'forward_modules'):

ansible_mitogen/strategy.py

@@ -26,7 +26,9 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
 import os
 import signal
 import threading
@@ -39,11 +41,15 @@ except ImportError:
 import mitogen.core
 import ansible_mitogen.affinity
 import ansible_mitogen.loaders
+import ansible_mitogen.logging
 import ansible_mitogen.mixins
 import ansible_mitogen.process
 
 import ansible.executor.process.worker
-from ansible.utils.sentinel import Sentinel
+import ansible.template
+import ansible.utils.sentinel
+import ansible.playbook.play_context
+import ansible.plugins.loader
 
 
 def _patch_awx_callback():
@@ -54,12 +60,11 @@ def _patch_awx_callback():
     # AWX uses sitecustomize.py to force-load this package. If it exists, we're
     # running under AWX.
     try:
-        from awx_display_callback.events import EventContext
+        import awx_display_callback.events
-        from awx_display_callback.events import event_context
     except ImportError:
         return
 
-    if hasattr(EventContext(), '_local'):
+    if hasattr(awx_display_callback.events.EventContext(), '_local'):
         # Patched version.
         return
 
@@ -68,18 +73,18 @@ def _patch_awx_callback():
         ctx = tls.setdefault('_ctx', {})
         ctx.update(kwargs)
 
-    EventContext._local = threading.local()
+    awx_display_callback.events.EventContext._local = threading.local()
-    EventContext.add_local = patch_add_local
+    awx_display_callback.events.EventContext.add_local = patch_add_local
 
 _patch_awx_callback()
 
 
-def wrap_action_loader__get(name, *args, **kwargs):
+def wrap_action_loader__get_with_context(name, *args, **kwargs):
     """
-    While the mitogen strategy is active, trap action_loader.get() calls,
+    While the mitogen strategy is active, trap action_loader.get_with_context()
-    augmenting any fetched class with ActionModuleMixin, which replaces various
+    calls, augmenting any fetched class with ActionModuleMixin, which replaces
-    helper methods inherited from ActionBase with implementations that avoid
+    various helper methods inherited from ActionBase with implementations that
-    the use of shell fragments wherever possible.
+    avoid the use of shell fragments wherever possible.
 
     This is used instead of static subclassing as it generalizes to third party
     action plugins outside the Ansible tree.
@@ -89,13 +94,26 @@ def wrap_action_loader__get(name, *args, **kwargs):
         name = 'mitogen_' + name
     get_kwargs['collection_list'] = kwargs.pop('collection_list', None)
 
-    klass = ansible_mitogen.loaders.action_loader__get(name, **get_kwargs)
+    (klass, context) = ansible_mitogen.loaders.action_loader__get_with_context(
+        name,
+        **get_kwargs
+    )
+
     if klass:
         bases = (ansible_mitogen.mixins.ActionModuleMixin, klass)
         adorned_klass = type(str(name), bases, {})
         if kwargs.get('class_only'):
-            return adorned_klass
+            return ansible.plugins.loader.get_with_context_result(
-        return adorned_klass(*args, **kwargs)
+                adorned_klass,
+                context
+            )
+
+        return ansible.plugins.loader.get_with_context_result(
+            adorned_klass(*args, **kwargs),
+            context
+        )
+
+    return ansible.plugins.loader.get_with_context_result(None, context)
 
 
 REDIRECTED_CONNECTION_PLUGINS = (
@@ -107,20 +125,32 @@ REDIRECTED_CONNECTION_PLUGINS = (
     'lxc',
     'lxd',
     'machinectl',
+    'podman',
     'setns',
     'ssh',
 )
 
 
-def wrap_connection_loader__get(name, *args, **kwargs):
+def wrap_connection_loader__get_with_context(name, *args, **kwargs):
     """
-    While a Mitogen strategy is active, rewrite connection_loader.get() calls
+    While a Mitogen strategy is active, rewrite
-    for some transports into requests for a compatible Mitogen transport.
+    connection_loader.get_with_context() calls for some transports into
+    requests for a compatible Mitogen transport.
     """
-    if name in REDIRECTED_CONNECTION_PLUGINS:
+    is_play_using_mitogen_connection = None
+    if len(args) > 0 and isinstance(args[0], ansible.playbook.play_context.PlayContext):
+        play_context = args[0]
+        is_play_using_mitogen_connection = play_context.connection in REDIRECTED_CONNECTION_PLUGINS
+
+    # assume true if we're not in a play context since we're using a Mitogen strategy
+    if is_play_using_mitogen_connection is None:
+        is_play_using_mitogen_connection = True
+
+    redirect_connection = name in REDIRECTED_CONNECTION_PLUGINS and is_play_using_mitogen_connection
+    if redirect_connection:
         name = 'mitogen_' + name
 
-    return ansible_mitogen.loaders.connection_loader__get(name, *args, **kwargs)
+    return ansible_mitogen.loaders.connection_loader__get_with_context(name, *args, **kwargs)
 
 
 def wrap_worker__run(self):
@@ -170,8 +200,8 @@ class AnsibleWrappers(object):
         Install our PluginLoader monkey patches and update global variables
         with references to the real functions.
         """
-        ansible_mitogen.loaders.action_loader.get = wrap_action_loader__get
+        ansible_mitogen.loaders.action_loader.get_with_context = wrap_action_loader__get_with_context
-        ansible_mitogen.loaders.connection_loader.get_with_context = wrap_connection_loader__get
+        ansible_mitogen.loaders.connection_loader.get_with_context = wrap_connection_loader__get_with_context
 
         global worker__run
         worker__run = ansible.executor.process.worker.WorkerProcess.run
@@ -181,11 +211,11 @@ class AnsibleWrappers(object):
         """
         Uninstall the PluginLoader monkey patches.
         """
-        ansible_mitogen.loaders.action_loader.get = (
+        ansible_mitogen.loaders.action_loader.get_with_context = (
-            ansible_mitogen.loaders.action_loader__get
+            ansible_mitogen.loaders.action_loader__get_with_context
         )
         ansible_mitogen.loaders.connection_loader.get_with_context = (
-            ansible_mitogen.loaders.connection_loader__get
+            ansible_mitogen.loaders.connection_loader__get_with_context
         )
         ansible.executor.process.worker.WorkerProcess.run = worker__run
 
@@ -278,7 +308,7 @@ class StrategyMixin(object):
             name=task.action,
             class_only=True,
         )
-        if play_context.connection is not Sentinel:
+        if play_context.connection is not ansible.utils.sentinel.Sentinel:
             # 2.8 appears to defer computing this until inside the worker.
             # TODO: figure out where it has moved.
             ansible_mitogen.loaders.connection_loader.get(
@@ -324,3 +354,44 @@ class StrategyMixin(object):
                 self._worker_model.on_strategy_complete()
         finally:
             ansible_mitogen.process.set_worker_model(None)
+
+    def _smuggle_to_connection_reset(self, task, play_context, iterator, target_host):
+        """
+        Create a templar and make it available for use in Connection.reset().
+        This allows templated connection variables to be used when Mitogen
+        reconstructs its connection stack.
+        """
+        variables = self._variable_manager.get_vars(
+            play=iterator._play, host=target_host, task=task,
+            _hosts=self._hosts_cache, _hosts_all=self._hosts_cache_all,
+        )
+        templar = ansible.template.Templar(
+            loader=self._loader, variables=variables,
+        )
+
+        # Required for remote_user option set by variable (e.g. ansible_user).
+        # Without it remote_user in ansible.cfg gets used.
+        play_context = play_context.set_task_and_variable_override(
+            task=task, variables=variables, templar=templar,
+        )
+        play_context.post_validate(templar=templar)
+
+        # Required for timeout option set by variable (e.g. ansible_timeout).
+        # Without it the task timeout keyword (default: 0) gets used.
+        play_context.update_vars(variables)
+
+        # Stash the task and templar somewhere Connection.reset() can find it
+        play_context.vars.update({
+            '_mitogen.smuggled.reset_connection': (task, templar),
+        })
+        return play_context
+
+    def _execute_meta(self, task, play_context, iterator, target_host):
+        if task.args['_raw_params'] == 'reset_connection':
+            play_context = self._smuggle_to_connection_reset(
+                task, play_context, iterator, target_host,
+            )
+
+        return super(StrategyMixin, self)._execute_meta(
+            task, play_context, iterator, target_host,
+        )

ansible_mitogen/target.py

@@ -33,10 +33,15 @@ Helper functions intended to be executed on the target. These are entrypoints
 for file transfer, module execution and sundry bits like changing file modes.
 """
 
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
 import errno
 import grp
-import operator
+import json
+import logging
 import os
+import pty
 import pwd
 import re
 import signal
@@ -47,32 +52,9 @@ import tempfile
 import traceback
 import types
 
-# Absolute imports for <2.5.
-logging = __import__('logging')
-
 import mitogen.core
-import mitogen.fork
 import mitogen.parent
 import mitogen.service
-from mitogen.core import b
-
-try:
-    import json
-except ImportError:
-    import simplejson as json
-
-try:
-    reduce
-except NameError:
-    # Python 3.x.
-    from functools import reduce
-
-try:
-    BaseException
-except NameError:
-    # Python 2.4
-    BaseException = Exception
-
 
 # Ansible since PR #41749 inserts "import __main__" into
 # ansible.module_utils.basic. Mitogen's importer will refuse such an import, so
@@ -82,6 +64,7 @@ if not sys.modules.get(str('__main__')):
     sys.modules[str('__main__')] = types.ModuleType(str('__main__'))
 
 import ansible.module_utils.json_utils
+
 import ansible_mitogen.runner
 
 
@@ -135,7 +118,7 @@ def subprocess__Popen__close_fds(self, but):
             continue
 
         fd = int(name, 10)
-        if fd > 2 and fd != but:
+        if fd > pty.STDERR_FILENO and fd != but:
             try:
                 os.close(fd)
             except OSError:
@@ -369,11 +352,6 @@ def init_child(econtext, log_level, candidate_temp_dirs):
     LOG.setLevel(log_level)
     logging.getLogger('ansible_mitogen').setLevel(log_level)
 
-    # issue #536: if the json module is available, remove simplejson from the
-    # importer whitelist to avoid confusing certain Ansible modules.
-    if json.__name__ == 'json':
-        econtext.importer.whitelist.remove('simplejson')
-
     global _fork_parent
     if FORK_SUPPORTED:
         mitogen.parent.upgrade_router(econtext)
@@ -622,8 +600,8 @@ def exec_args(args, in_data='', chdir=None, shell=None, emulate_tty=False):
     stdout, stderr = proc.communicate(in_data)
 
     if emulate_tty:
-        stdout = stdout.replace(b('\n'), b('\r\n'))
+        stdout = stdout.replace(b'\n', b'\r\n')
-    return proc.returncode, stdout, stderr or b('')
+    return proc.returncode, stdout, stderr or b''
 
 
 def exec_command(cmd, in_data='', chdir=None, shell=None, emulate_tty=False):
@@ -652,7 +630,8 @@ def read_path(path):
     """
     Fetch the contents of a filesystem `path` as bytes.
     """
-    return open(path, 'rb').read()
+    with open(path, 'rb') as f:
+        return f.read()
 
 
 def set_file_owner(path, owner, group=None, fd=None):
@@ -666,11 +645,10 @@ def set_file_owner(path, owner, group=None, fd=None):
     else:
         gid = os.getegid()
 
-    if fd is not None and hasattr(os, 'fchown'):
+    if fd is not None:
-        os.fchown(fd, (uid, gid))
+        os.fchown(fd, uid, gid)
     else:
-        # Python<2.6
+        os.chown(path, uid, gid)
-        os.chown(path, (uid, gid))
 
 
 def write_path(path, s, owner=None, group=None, mode=None,
@@ -737,7 +715,9 @@ def apply_mode_spec(spec, mode):
             mask = CHMOD_MASKS[ch]
             bits = CHMOD_BITS[ch]
             cur_perm_bits = mode & mask
-            new_perm_bits = reduce(operator.or_, (bits[p] for p in perms), 0)
+            new_perm_bits = 0
+            for perm in perms:
+                new_perm_bits |= bits[perm]
             mode &= ~mask
             if op == '=':
                 mode |= new_perm_bits
@@ -752,9 +732,7 @@ def set_file_mode(path, spec, fd=None):
     """
     Update the permissions of a file using the same syntax as chmod(1).
     """
-    if isinstance(spec, int):
+    if isinstance(spec, mitogen.core.integer_types):
-        new_mode = spec
-    elif not mitogen.core.PY3 and isinstance(spec, long):
         new_mode = spec
     elif spec.isdigit():
         new_mode = int(spec, 8)
@@ -762,7 +740,7 @@ def set_file_mode(path, spec, fd=None):
         mode = os.stat(path).st_mode
         new_mode = apply_mode_spec(spec, mode)
 
-    if fd is not None and hasattr(os, 'fchmod'):
+    if fd is not None:
         os.fchmod(fd, new_mode)
     else:
         os.chmod(path, new_mode)

ansible_mitogen/transport_config.py

@@ -26,9 +26,6 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
 """
 Mitogen extends Ansible's target configuration mechanism in several ways that
 require some care:
@@ -60,28 +57,37 @@ information from PlayContext, and another that takes (almost) all information
 from HostVars.
 """
 
+from __future__ import absolute_import, division, print_function
+from __future__ import unicode_literals
+__metaclass__ = type
+
 import abc
+import logging
 import os
+
 import ansible.utils.shlex
 import ansible.constants as C
+import ansible.executor.interpreter_discovery
+import ansible.utils.unsafe_proxy
 
 from ansible.module_utils.six import with_metaclass
+from ansible.module_utils.parsing.convert_bool import boolean
+
+import ansible_mitogen.utils
+import mitogen.core
 
-# this was added in Ansible >= 2.8.0; fallback to the default interpreter if necessary
-try:
-    from ansible.executor.interpreter_discovery import discover_interpreter
-except ImportError:
-    discover_interpreter = lambda action,interpreter_name,discovery_mode,task_vars: '/usr/bin/python'
 
-try:
+LOG = logging.getLogger(__name__)
-    from ansible.utils.unsafe_proxy import AnsibleUnsafeText
-except ImportError:
-    from ansible.vars.unsafe_proxy import AnsibleUnsafeText
 
-import mitogen.core
+if ansible_mitogen.utils.ansible_version[:2] >= (2, 19):
+    _FALLBACK_INTERPRETER = ansible.executor.interpreter_discovery._FALLBACK_INTERPRETER
+elif ansible_mitogen.utils.ansible_version[:2] >= (2, 17):
+    _FALLBACK_INTERPRETER = u'/usr/bin/python3'
+else:
+    _FALLBACK_INTERPRETER = u'/usr/bin/python'
 
 
-def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_python):
+def run_interpreter_discovery_if_necessary(s, candidates, task_vars, action, rediscover_python):
     """
     Triggers ansible python interpreter discovery if requested.
     Caches this value the same way Ansible does it.
@@ -89,14 +95,14 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth
     it could be different than what's ran on the host
     """
     # keep trying different interpreters until we don't error
-    if action._finding_python_interpreter:
+    if action._mitogen_discovering_interpreter:
-        return action._possible_python_interpreter
+        return action._mitogen_interpreter_candidate
-    
+
     if s in ['auto', 'auto_legacy', 'auto_silent', 'auto_legacy_silent']:
         # python is the only supported interpreter_name as of Ansible 2.8.8
         interpreter_name = 'python'
         discovered_interpreter_config = u'discovered_interpreter_%s' % interpreter_name
-        
+
         if task_vars.get('ansible_facts') is None:
            task_vars['ansible_facts'] = {}
 
@@ -104,39 +110,42 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth
             # if we're rediscovering python then chances are we're running something like a docker connection
             # this will handle scenarios like running a playbook that does stuff + then dynamically creates a docker container,
             # then runs the rest of the playbook inside that container, and then rerunning the playbook again
-            action._rediscovered_python = True
+            action._mitogen_rediscovered_interpreter = True
 
             # blow away the discovered_interpreter_config cache and rediscover
             del task_vars['ansible_facts'][discovered_interpreter_config]
 
-        if discovered_interpreter_config not in task_vars['ansible_facts']:
+        try:
-            action._finding_python_interpreter = True
+            s = task_vars[u'ansible_facts'][discovered_interpreter_config]
+        except KeyError:
+            action._mitogen_discovering_interpreter = True
+            action._mitogen_interpreter_candidates = candidates
             # fake pipelining so discover_interpreter can be happy
             action._connection.has_pipelining = True
-            s = AnsibleUnsafeText(discover_interpreter(
+            s = ansible.executor.interpreter_discovery.discover_interpreter(
                 action=action,
                 interpreter_name=interpreter_name,
                 discovery_mode=s,
-                task_vars=task_vars))
+                task_vars=task_vars,
-
+            )
+            s = ansible.utils.unsafe_proxy.AnsibleUnsafeText(s)
             # cache discovered interpreter
             task_vars['ansible_facts'][discovered_interpreter_config] = s
             action._connection.has_pipelining = False
-        else:
-            s = task_vars['ansible_facts'][discovered_interpreter_config]
 
         # propagate discovered interpreter as fact
         action._discovered_interpreter_key = discovered_interpreter_config
         action._discovered_interpreter = s
 
-    action._finding_python_interpreter = False
+    action._mitogen_discovering_interpreter = False
+    action._mitogen_interpreter_candidates = None
     return s
 
 
-def parse_python_path(s, task_vars, action, rediscover_python):
+def parse_python_path(s, candidates, task_vars, action, rediscover_python):
     """
     Given the string set for ansible_python_interpeter, parse it using shell
-    syntax and return an appropriate argument vector. If the value detected is 
+    syntax and return an appropriate argument vector. If the value detected is
     one of interpreter discovery then run that first. Caches python interpreter
     discovery value in `facts_from_task_vars` like how Ansible handles this.
     """
@@ -144,10 +153,9 @@ def parse_python_path(s, task_vars, action, rediscover_python):
         # if python_path doesn't exist, default to `auto` and attempt to discover it
         s = 'auto'
 
-    s = run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_python)
+    s = run_interpreter_discovery_if_necessary(s, candidates, task_vars, action, rediscover_python)
-    # if unable to determine python_path, fallback to '/usr/bin/python'
     if not s:
-        s = '/usr/bin/python'
+        s = _FALLBACK_INTERPRETER
 
     return ansible.utils.shlex.shlex_split(s)
 
@@ -214,6 +222,12 @@ class Spec(with_metaclass(abc.ABCMeta, object)):
         :data:`True` if privilege escalation should be active.
         """
 
+    @abc.abstractmethod
+    def become_flags(self):
+        """
+        The command line arguments passed to the become executable.
+        """
+
     @abc.abstractmethod
     def become_method(self):
         """
@@ -244,6 +258,12 @@ class Spec(with_metaclass(abc.ABCMeta, object)):
         Path to the Python interpreter on the target machine.
         """
 
+    @abc.abstractmethod
+    def host_key_checking(self):
+        """
+        Whether or not to check the keys of the target machine
+        """
+
     @abc.abstractmethod
     def private_key_file(self):
         """
@@ -285,10 +305,9 @@ class Spec(with_metaclass(abc.ABCMeta, object)):
     @abc.abstractmethod
     def sudo_args(self):
         """
-        The list of additional arguments that should be included in a become
+        The list of additional arguments that should be included in a sudo
         invocation.
         """
-        # TODO: split out into sudo_args/become_args.
 
     @abc.abstractmethod
     def mitogen_via(self):
@@ -354,6 +373,12 @@ class Spec(with_metaclass(abc.ABCMeta, object)):
         The path to the "machinectl" program for the 'setns' transport.
         """
 
+    @abc.abstractmethod
+    def mitogen_podman_path(self):
+        """
+        The path to the "podman" program for the 'podman' transport.
+        """
+
     @abc.abstractmethod
     def mitogen_ssh_keepalive_interval(self):
         """
@@ -390,6 +415,12 @@ class Spec(with_metaclass(abc.ABCMeta, object)):
         Value of "ansible_doas_exe" variable.
         """
 
+    @abc.abstractmethod
+    def verbosity(self):
+        """
+        How verbose to make logging or diagnostics output.
+        """
+
 
 class PlayContextSpec(Spec):
     """
@@ -406,6 +437,43 @@ class PlayContextSpec(Spec):
         # used to run interpreter discovery
         self._action = connection._action
 
+    def _become_option(self, name):
+        plugin = self._connection.become
+        try:
+            return plugin.get_option(name, self._task_vars, self._play_context)
+        except AttributeError:
+            # A few ansible_mitogen connection plugins look more like become
+            # plugins. They don't quite fit Ansible's plugin.get_option() API.
+            # https://github.com/mitogen-hq/mitogen/issues/1173
+            fallback_plugins = {'mitogen_doas', 'mitogen_sudo', 'mitogen_su'}
+            if self._connection.transport not in fallback_plugins:
+                raise
+
+            fallback_options = {
+                'become_exe',
+                'become_flags',
+            }
+            if name not in fallback_options:
+                raise
+
+            LOG.info(
+                'Used fallback=PlayContext.%s for plugin=%r, option=%r',
+                name, self._connection, name,
+            )
+            return getattr(self._play_context, name)
+
+    def _connection_option(self, name, fallback_attr=None):
+        try:
+            return self._connection.get_option(name, hostvars=self._task_vars)
+        except KeyError:
+            if fallback_attr is None:
+                fallback_attr = name
+            LOG.info(
+                'Used fallback=PlayContext.%s for plugin=%r, option=%r',
+                fallback_attr, self._connection, name,
+            )
+            return getattr(self._play_context, fallback_attr)
+
     def transport(self):
         return self._transport
 
@@ -413,92 +481,90 @@ class PlayContextSpec(Spec):
         return self._inventory_name
 
     def remote_addr(self):
-        return self._play_context.remote_addr
+        return self._connection_option('host', fallback_attr='remote_addr')
 
     def remote_user(self):
-        return self._play_context.remote_user
+        return self._connection_option('remote_user')
 
     def become(self):
-        return self._play_context.become
+        return self._connection.become
+
+    def become_flags(self):
+        return self._become_option('become_flags')
 
     def become_method(self):
-        return self._play_context.become_method
+        return self._connection.become.name
 
     def become_user(self):
-        return self._play_context.become_user
+        return self._become_option('become_user')
 
     def become_pass(self):
-        return optional_secret(self._play_context.become_pass)
+        return optional_secret(self._become_option('become_pass'))
 
     def password(self):
-        return optional_secret(self._play_context.password)
+        return optional_secret(self._connection_option('password'))
 
     def port(self):
-        return self._play_context.port
+        return self._connection_option('port')
 
     def python_path(self, rediscover_python=False):
-        s = self._connection.get_task_var('ansible_python_interpreter')
+        # See also
-        # #511, #536: executor/module_common.py::_get_shebang() hard-wires
+        #   - ansible_mitogen.connecton.Connection.get_task_var()
-        # "/usr/bin/python" as the default interpreter path if no other
+        try:
-        # interpreter is specified.
+            delegated_vars = self._task_vars['ansible_delegated_vars']
+            variables = delegated_vars[self._connection.delegate_to_hostname]
+        except KeyError:
+            variables = self._task_vars
+
+        interpreter_python = C.config.get_config_value(
+            'INTERPRETER_PYTHON', variables=variables,
+        )
+        interpreter_python_fallback = C.config.get_config_value(
+            'INTERPRETER_PYTHON_FALLBACK', variables=variables,
+        )
+
+        if '{{' in interpreter_python or '{%' in interpreter_python:
+            templar = self._connection.templar
+            interpreter_python = templar.template(interpreter_python)
+
         return parse_python_path(
-            s,
+            interpreter_python,
+            candidates=interpreter_python_fallback,
             task_vars=self._task_vars,
             action=self._action,
             rediscover_python=rediscover_python)
 
+    def host_key_checking(self):
+        return self._connection_option('host_key_checking')
+
     def private_key_file(self):
-        return self._play_context.private_key_file
+        return self._connection_option('private_key_file')
 
     def ssh_executable(self):
-        return C.config.get_config_value("ssh_executable", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
+        return self._connection_option('ssh_executable')
 
     def timeout(self):
-        return self._play_context.timeout
+        return self._connection_option('timeout')
 
     def ansible_ssh_timeout(self):
-        return (
+        return self.timeout()
-            self._connection.get_task_var('ansible_timeout') or
-            self._connection.get_task_var('ansible_ssh_timeout') or
-            self.timeout()
-        )
 
     def ssh_args(self):
         return [
             mitogen.core.to_text(term)
             for s in (
-                C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
+                self._connection_option('ssh_args'),
-                C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
+                self._connection_option('ssh_common_args'),
-                C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
+                self._connection_option('ssh_extra_args'),
             )
             for term in ansible.utils.shlex.shlex_split(s or '')
         ]
 
     def become_exe(self):
-        # In Ansible 2.8, PlayContext.become_exe always has a default value due
+        return self._become_option('become_exe')
-        # to the new options mechanism. Previously it was only set if a value
-        # ("somewhere") had been specified for the task.
-        # For consistency in the tests, here we make older Ansibles behave like
-        # newer Ansibles.
-        exe = self._play_context.become_exe
-        if exe is None and self._play_context.become_method == 'sudo':
-            exe = 'sudo'
-        return exe
 
     def sudo_args(self):
-        return [
+        return ansible.utils.shlex.shlex_split(self.become_flags() or '')
-            mitogen.core.to_text(term)
-            for term in ansible.utils.shlex.shlex_split(
-                first_true((
-                    self._play_context.become_flags,
-                    # Ansible <=2.7.
-                    getattr(self._play_context, 'sudo_flags', ''),
-                    # Ansible <=2.3.
-                    getattr(C, 'DEFAULT_BECOME_FLAGS', ''),
-                    getattr(C, 'DEFAULT_SUDO_FLAGS', '')
-                ), default='')
-            )
-        ]
 
     def mitogen_via(self):
         return self._connection.get_task_var('mitogen_via')
@@ -527,6 +593,9 @@ class PlayContextSpec(Spec):
     def mitogen_lxc_info_path(self):
         return self._connection.get_task_var('mitogen_lxc_info_path')
 
+    def mitogen_podman_path(self):
+        return self._connection.get_task_var('mitogen_podman_path')
+
     def mitogen_ssh_keepalive_interval(self):
         return self._connection.get_task_var('mitogen_ssh_keepalive_interval')
 
@@ -551,6 +620,17 @@ class PlayContextSpec(Spec):
             os.environ.get('ANSIBLE_DOAS_EXE')
         )
 
+    def verbosity(self):
+        try:
+            verbosity = self._connection.get_option('verbosity', hostvars=self._task_vars)
+        except KeyError:
+            verbosity = self.mitogen_ssh_debug_level()
+
+        if verbosity:
+            return int(verbosity)
+
+        return 0
+
 
 class MitogenViaSpec(Spec):
     """
@@ -630,6 +710,9 @@ class MitogenViaSpec(Spec):
     def become(self):
         return bool(self._become_user)
 
+    def become_flags(self):
+        return self._host_vars.get('ansible_become_flags')
+
     def become_method(self):
         return (
             self._become_method or
@@ -642,12 +725,13 @@ class MitogenViaSpec(Spec):
 
     def become_pass(self):
         return optional_secret(
-            self._host_vars.get('ansible_become_password') or
+            self._host_vars.get('ansible_become_pass') or
-            self._host_vars.get('ansible_become_pass')
+            self._host_vars.get('ansible_become_password')
         )
 
     def password(self):
         return optional_secret(
+            self._host_vars.get('ansible_ssh_password') or
             self._host_vars.get('ansible_ssh_pass') or
             self._host_vars.get('ansible_password')
         )
@@ -661,15 +745,24 @@ class MitogenViaSpec(Spec):
 
     def python_path(self, rediscover_python=False):
         s = self._host_vars.get('ansible_python_interpreter')
-        # #511, #536: executor/module_common.py::_get_shebang() hard-wires
+        interpreter_python_fallback = self._host_vars.get(
-        # "/usr/bin/python" as the default interpreter path if no other
+            'ansible_interpreter_python_fallback', [],
-        # interpreter is specified.
+        )
         return parse_python_path(
             s,
+            candidates=interpreter_python_fallback,
             task_vars=self._task_vars,
             action=self._action,
             rediscover_python=rediscover_python)
 
+    def host_key_checking(self):
+        def candidates():
+            yield self._host_vars.get('ansible_ssh_host_key_checking')
+            yield self._host_vars.get('ansible_host_key_checking')
+            yield C.HOST_KEY_CHECKING
+        val = next((v for v in candidates() if v is not None), True)
+        return boolean(val)
+
     def private_key_file(self):
         # TODO: must come from PlayContext too.
         return (
@@ -693,12 +786,13 @@ class MitogenViaSpec(Spec):
         )
 
     def ssh_args(self):
+        local_vars = self._task_vars.get("hostvars", {}).get(self._inventory_name, {})
         return [
             mitogen.core.to_text(term)
             for s in (
-                C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
+                C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=local_vars),
-                C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
+                C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=local_vars),
-                C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
+                C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=local_vars)
             )
             for term in ansible.utils.shlex.shlex_split(s)
             if s
@@ -715,7 +809,7 @@ class MitogenViaSpec(Spec):
             mitogen.core.to_text(term)
             for s in (
                 self._host_vars.get('ansible_sudo_flags') or '',
-                self._host_vars.get('ansible_become_flags') or '',
+                self.become_flags() or '',
             )
             for term in ansible.utils.shlex.shlex_split(s)
         ]
@@ -739,7 +833,7 @@ class MitogenViaSpec(Spec):
         return self._host_vars.get('mitogen_kubectl_path')
 
     def mitogen_lxc_path(self):
-        return self.host_vars.get('mitogen_lxc_path')
+        return self._host_vars.get('mitogen_lxc_path')
 
     def mitogen_lxc_attach_path(self):
         return self._host_vars.get('mitogen_lxc_attach_path')
@@ -747,6 +841,9 @@ class MitogenViaSpec(Spec):
     def mitogen_lxc_info_path(self):
         return self._host_vars.get('mitogen_lxc_info_path')
 
+    def mitogen_podman_path(self):
+        return self._host_vars.get('mitogen_podman_path')
+
     def mitogen_ssh_keepalive_interval(self):
         return self._host_vars.get('mitogen_ssh_keepalive_interval')
 
@@ -770,3 +867,13 @@ class MitogenViaSpec(Spec):
             self._host_vars.get('ansible_doas_exe') or
             os.environ.get('ANSIBLE_DOAS_EXE')
         )
+
+    def verbosity(self):
+        verbosity = self._host_vars.get('ansible_ssh_verbosity')
+        if verbosity is None:
+            verbosity = self.mitogen_ssh_debug_level()
+
+        if verbosity:
+            return int(verbosity)
+
+        return 0

ansible_mitogen/utils.py

@@ -1,13 +0,0 @@
-from __future__ import absolute_import
-
-import distutils.version
-
-import ansible
-
-__all__ = [
-    'ansible_version',
-]
-
-ansible_version = tuple(distutils.version.LooseVersion(ansible.__version__).version)
-del distutils
-del ansible

ansible_mitogen/utils/__init__.py

@@ -0,0 +1,29 @@
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import re
+
+import ansible
+
+__all__ = [
+    'ansible_version',
+]
+
+
+def _parse(v_string):
+    # Adapted from distutils.version.LooseVersion.parse()
+    component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE)
+    for component in component_re.split(v_string):
+        if not component or component == '.':
+            continue
+        try:
+            yield int(component)
+        except ValueError:
+            yield component
+
+
+ansible_version = tuple(_parse(ansible.__version__))
+
+del _parse
+del re
+del ansible

ansible_mitogen/utils/unsafe.py

@@ -0,0 +1,123 @@
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import ansible
+import ansible.utils.unsafe_proxy
+
+import ansible_mitogen.utils
+
+import mitogen
+import mitogen.core
+import mitogen.utils
+
+__all__ = [
+    'cast',
+]
+
+def _cast_to_dict(obj): return {cast(k): cast(v) for k, v in obj.items()}
+def _cast_to_list(obj): return [cast(v) for v in obj]
+def _cast_to_set(obj): return set(cast(v) for v in obj)
+def _cast_to_tuple(obj): return tuple(cast(v) for v in obj)
+def _cast_unsafe(obj): return obj._strip_unsafe()
+def _passthrough(obj): return obj
+def _untag(obj): return obj._native_copy()
+
+
+# A dispatch table to cast objects based on their exact type.
+# This is an optimisation, reliable fallbacks are required (e.g. isinstance())
+_CAST_DISPATCH = {
+    bytes: bytes,
+    dict: _cast_to_dict,
+    list: _cast_to_list,
+    mitogen.core.UnicodeType: mitogen.core.UnicodeType,
+}
+_CAST_DISPATCH.update({t: _passthrough for t in mitogen.utils.PASSTHROUGH})
+
+_CAST_SUBTYPES = [
+    dict,
+    list,
+]
+
+if hasattr(ansible.utils.unsafe_proxy, 'TrustedAsTemplate'):
+    import datetime
+    import ansible.module_utils._internal._datatag
+    _CAST_DISPATCH.update({
+        set: _cast_to_set,
+        tuple: _cast_to_tuple,
+        ansible.module_utils._internal._datatag._AnsibleTaggedBytes: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedDate: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedDateTime: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedDict: _cast_to_dict,
+        ansible.module_utils._internal._datatag._AnsibleTaggedFloat: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedInt: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedList: _cast_to_list,
+        ansible.module_utils._internal._datatag._AnsibleTaggedSet: _cast_to_set,
+        ansible.module_utils._internal._datatag._AnsibleTaggedStr: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedTime: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedTuple: _cast_to_tuple,
+        ansible.utils.unsafe_proxy.AnsibleUnsafeBytes: bytes,
+        ansible.utils.unsafe_proxy.AnsibleUnsafeText: mitogen.core.UnicodeType,
+        datetime.date: _passthrough,
+        datetime.datetime: _passthrough,
+        datetime.time: _passthrough,
+    })
+    _CAST_SUBTYPES.extend([
+        set,
+        tuple,
+    ])
+elif hasattr(ansible.utils.unsafe_proxy.AnsibleUnsafeText, '_strip_unsafe'):
+    _CAST_DISPATCH.update({
+        tuple: _cast_to_list,
+        ansible.utils.unsafe_proxy.AnsibleUnsafeBytes: _cast_unsafe,
+        ansible.utils.unsafe_proxy.AnsibleUnsafeText: _cast_unsafe,
+        ansible.utils.unsafe_proxy.NativeJinjaUnsafeText: _cast_unsafe,
+    })
+    _CAST_SUBTYPES.extend([
+        tuple,
+    ])
+elif ansible_mitogen.utils.ansible_version[:2] <= (2, 16):
+    _CAST_DISPATCH.update({
+        tuple: _cast_to_list,
+        ansible.utils.unsafe_proxy.AnsibleUnsafeBytes: bytes,
+        ansible.utils.unsafe_proxy.AnsibleUnsafeText: mitogen.core.UnicodeType,
+    })
+    _CAST_SUBTYPES.extend([
+        tuple,
+    ])
+else:
+    mitogen_ver = '.'.join(str(v) for v in mitogen.__version__)
+    raise ImportError("Mitogen %s can't cast Ansible %s objects"
+                      % (mitogen_ver, ansible.__version__))
+
+
+def cast(obj):
+    """
+    Return obj (or a copy) with subtypes of builtins cast to their supertype.
+
+    This is an enhanced version of :func:`mitogen.utils.cast`. In addition it
+    handles ``ansible.utils.unsafe_proxy.AnsibleUnsafeText`` and variants.
+
+    There are types handled by :func:`ansible.utils.unsafe_proxy.wrap_var()`
+    that this function currently does not handle (e.g. `set()`), or preserve
+    preserve (e.g. `tuple()`). Future enhancements may change this.
+
+    :param obj:
+        Object to undecorate.
+    :returns:
+        Undecorated object.
+    """
+    # Fast path: obj is a known type, dispatch directly
+    try:
+        unwrapper = _CAST_DISPATCH[type(obj)]
+    except KeyError:
+        pass
+    else:
+        return unwrapper(obj)
+
+    # Slow path: obj is some unknown subclass
+    for typ_ in _CAST_SUBTYPES:
+        if isinstance(obj, typ_):
+            unwrapper = _CAST_DISPATCH[typ_]
+            return unwrapper(obj)
+
+    return mitogen.utils.cast(obj)

docs/_templates/ansible.html

@@ -1,14 +0,0 @@
-<!doctype html>
-<title>Mitogen for Ansible (Redirect)</title>
-<script>
-    {% include "piwik-config.js" %}
-    var u="https://networkgenomics.com/p/tr/";
-    _paq.push(['setTrackerUrl', u+'ep']);
-</script>
-<script src="https://networkgenomics.com/p/tr/js"></script>
-<script>
-    setTimeout(function() {
-        window.location = 'https://networkgenomics.com/ansible/';
-    }, 0);
-</script>
-<meta http-equiv="Refresh" content="0; url=https://networkgenomics.com/ansible/">

docs/_templates/github.html

@@ -1,4 +1,4 @@
 <p>
 <br>
-<a class="github-button" href="https://github.com/dw/mitogen/" data-size="large" data-show-count="true" aria-label="Star dw/mitogen on GitHub">Star</a>
+<a class="github-button" href="https://github.com/mitogen-hq/mitogen/" data-size="large" data-show-count="true" aria-label="Star mitogen on GitHub">Star</a>
 </p>

docs/_templates/layout.html

@@ -14,23 +14,5 @@
 
 {% block footer %}
     {{ super() }}
-
-    <script>
-      (function() {
-        {% include "piwik-config.js" %}
-        var u="https://networkgenomics.com/p/tr/";
-        _paq.push(['setTrackerUrl', u+'ep']);
-        var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; g.type='text/javascript';
-        g.defer=true; g.async=true; g.src=u+'js'; s.parentNode.insertBefore(g,s);
-      })();
-    </script>
-
-    <noscript>
-        <p>
-        {% set fulltitle = (title|striptags|e) + titlesuffix -%}
-        <img src="https://networkgenomics.com/p/tr/ep?idsite=6&action_name={{fulltitle}}" style="border:0" alt="">
-        </p>
-    </noscript>
-
     <script async defer src="https://buttons.github.io/buttons.js"></script>
 {% endblock %}

docs/_templates/piwik-config.js

@@ -1,5 +0,0 @@
-window._paq = [];
-window._paq.push(['trackPageView']);
-window._paq.push(['enableLinkTracking']);
-window._paq.push(['enableHeartBeatTimer', 30]);
-window._paq.push(['setSiteId', 6]);

docs/ansible_detailed.rst

@@ -18,7 +18,7 @@ The extension is considered stable and real-world use is encouraged.
 
 .. _Ansible: https://www.ansible.com/
 
-.. _Bug reports: https://goo.gl/yLKZiJ
+.. _Bug reports: https://github.com/mitogen-hq/mitogen/issues/new/choose
 
 
 Overview
@@ -75,34 +75,6 @@ Installation
    ``mitogen_host_pinned`` strategies exists to mimic the ``free`` and
    ``host_pinned`` strategies.
 
-4.
-
-   .. raw:: html
-
-    <form action="https://networkgenomics.com/save-email/" method="post" id="emailform">
-        <input type=hidden name="list_name" value="mitogen-announce">
-
-        Get notified of new releases and important fixes.
-
-        <p>
-        <input type="email" placeholder="E-mail Address" name="email" style="font-size: 105%;"><br>
-        <input name="captcha_1" placeholder="Captcha" style="width: 10ex;">
-        <img class="captcha-image">
-        <a class="captcha-refresh" href="#">&#x21bb</a>
-
-        <button type="submit" style="font-size: 105%;">
-            Subscribe
-        </button>
-
-        </p>
-
-        <div id="emailthanks" style="display:none">
-            Thanks!
-        </div>
-
-        <p>
-    </form>
-
 
 Demo
 ~~~~
@@ -147,8 +119,35 @@ Noteworthy Differences
 
 * Mitogen 0.2.x supports Ansible 2.3-2.9; with Python 2.6, 2.7, or 3.6.
   Mitogen 0.3.1+ supports
-    - Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.9
+
-    - Ansible 5; with Python 3.8-3.9
+  +-----------------+-----------------+
+  | Ansible version | Python versions |
+  +=================+=================+
+  | 2.10            |                 |
+  +-----------------+                 |
+  | 3               | 2.7, 3.6 - 3.11 |
+  +-----------------+                 |
+  | 4               |                 |
+  +-----------------+-----------------+
+  | 5               | 3.8 - 3.11      |
+  +-----------------+-----------------+
+  | 6               |                 |
+  +-----------------+ 3.8 - 3.13      |
+  | 7               |                 |
+  +-----------------+-----------------+
+  | 8               | 3.9 - 3.13      |
+  +-----------------+-----------------+
+  | 9               |                 |
+  +-----------------+ 3.10 - 3.14     |
+  | 10              |                 |
+  +-----------------+-----------------+
+  | 11              |                 |
+  +-----------------+ 3.11 - 3.14     |
+  | 12              |                 |
+  +-----------------+-----------------+
+  | 13              | 3.12 - 3.14     |
+  +-----------------+-----------------+
+
   Verify your installation is running one of these versions by checking
   ``ansible --version`` output.
 
@@ -188,9 +187,9 @@ Noteworthy Differences
        your_ssh_username = (ALL) NOPASSWD:/usr/bin/python -c*
 
 * The :ans:conn:`~buildah`, :ans:conn:`~docker`, :ans:conn:`~jail`,
-  :ans:conn:`~kubectl`, :ans:conn:`~local`, :ans:conn:`~lxd`, and
+  :ans:conn:`~kubectl`, :ans:conn:`~local`, :ans:conn:`~lxd`,
-  :ans:conn:`~ssh` built-in connection types are supported, along with
+  :ans:conn:`~podman`, & :ans:conn:`~ssh` connection types are supported; also
-  Mitogen-specific :ref:`machinectl <machinectl>`, :ref:`mitogen_doas <doas>`,
+  Mitogen-specific :ref:`mitogen_doas <doas>`, :ref:`machinectl <machinectl>`,
   :ref:`mitogen_su <su>`, :ref:`mitogen_sudo <sudo>`, and :ref:`setns <setns>`
   types. File bugs to register interest in others.
 
@@ -228,6 +227,15 @@ Noteworthy Differences
   part of the core library, and should therefore be straightforward to fix as
   part of 0.2.x.
 
+* Connection and become timeouts are applied differently. Mitogen may consider
+  a connection to have timed out, when Ansible would have waited longer or
+  indefinately. For example if SSH authentication completes within the
+  timeout, but execution of login scripts exceeds it - then Mitogen will
+  consider the task to have timed out and that host to have failed.
+
+..
+    tests/ansible/integration/ssh/timeouts.yml covers (some of) this behaviour.
+
 ..
     * SSH and ``become`` are treated distinctly when applying timeouts, and
     timeouts apply up to the point when the new interpreter is ready to accept
@@ -243,15 +251,14 @@ Noteworthy Differences
     * "Module Replacer" style modules are not supported. These rarely appear in
     practice, and light web searches failed to reveal many examples of them.
 
-..
+* The ``ansible_python_interpreter`` variable is parsed using a restrictive
-    * The ``ansible_python_interpreter`` variable is parsed using a restrictive
+  :mod:`shell-like <shlex>` syntax, permitting values such as ``/usr/bin/env
-      :mod:`shell-like <shlex>` syntax, permitting values such as ``/usr/bin/env
+  FOO=bar python`` or ``source /opt/rh/rh-python36/enable && python``.
-      FOO=bar python`` or ``source /opt/rh/rh-python36/enable && python``, which
+  Jinja2 templating is also supported for complex task-level
-      occur in practice. Jinja2 templating is also supported for complex task-level
+  interpreter settings. Ansible documents `ansible_python_interpreter
-      interpreter settings. Ansible `documents this
+  <https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html#ansible-python-interpreter>`_
-      <https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html#ansible-python-interpreter>`_
+  as an absolute path and releases since June 2024 (e.g. Ansible 10.1)
-      as an absolute path, however the implementation passes it unquoted through
+  reflect this. Older Ansible releases passed it to the shell unquoted.
-      the shell, permitting arbitrary code to be injected.
 
 ..
     * Configurations will break that rely on the `hashbang argument splitting
@@ -305,7 +312,8 @@ container.
     * Intermediary machines cannot use login and become passwords that were
       supplied to Ansible interactively. If an intermediary requires a
       password, it must be supplied via ``ansible_ssh_pass``,
-      ``ansible_password``, or ``ansible_become_pass`` inventory variables.
+      ``ansible_ssh_password``, ``ansible_password``, or
+      ``ansible_become_pass`` inventory variables.
 
     * Automatic tunnelling of SSH-dependent actions, such as the
       ``synchronize`` module, is not yet supported. This will be addressed in a
@@ -819,6 +827,20 @@ Like the :ans:conn:`local` except connection delegation is supported.
 * ``ansible_python_interpreter``
 
 
+Podman
+~~~~~~
+
+Like :ans:conn:`podman` except connection delegation is supported.
+
+* ``ansible_host``: Name of container (default: inventory hostname).
+* ``ansible_user``: Name of user within the container to execute as.
+* ``mitogen_mask_remote_name``: if :data:`True`, mask the identity of the
+  Ansible controller process on remote machines. To simplify diagnostics,
+  Mitogen produces remote processes named like
+  `"mitogen:user@controller.name:1234"`, however this may be a privacy issue in
+  some circumstances.
+
+
 Process Model
 ^^^^^^^^^^^^^
 
@@ -996,7 +1018,10 @@ Like the :ans:conn:`ssh` except connection delegation is supported.
 * ``ansible_port``, ``ssh_port``
 * ``ansible_ssh_executable``, ``ssh_executable``
 * ``ansible_ssh_private_key_file``
-* ``ansible_ssh_pass``, ``ansible_password`` (default: assume passwordless)
+* ``ansible_ssh_pass``, ``ansible_ssh_password``, ``ansible_password``
+  (default: assume passwordless)
+* ``ansible_ssh_host_key_checking``, ``ansible_host_key_checking`` (default: 
+  :data:`True`)
 * ``ssh_args``, ``ssh_common_args``, ``ssh_extra_args``
 * ``mitogen_mask_remote_name``: if :data:`True`, mask the identity of the
   Ansible controller process on remote machines. To simplify diagnostics,
@@ -1234,18 +1259,17 @@ with ``-vvv``.
 
 However, certain controller hangs may render ``MITOGEN_DUMP_THREAD_STACKS``
 ineffective, or occur too infrequently for interactive reproduction. In these
-cases `faulthandler <https://faulthandler.readthedocs.io/>`_ may be used:
+cases :py:mod:`faulthandler` may be used with Python >= 3.3:
 
-1. For Python 2, ``pip install faulthandler``. This is unnecessary on Python 3.
+1. Once the hang occurs, observe the process tree using ``pstree`` or ``ps
-2. Once the hang occurs, observe the process tree using ``pstree`` or ``ps
    --forest``.
-3. The most likely process to be hung is the connection multiplexer, which can
+2. The most likely process to be hung is the connection multiplexer, which can
    easily be identified as the parent of all SSH client processes.
-4. Send ``kill -SEGV <pid>`` to the multiplexer PID, causing it to print all
+3. Send ``kill -SEGV <pid>`` to the multiplexer PID, causing it to print all
    thread stacks.
-5. `File a bug <https://github.com/dw/mitogen/issues/new/>`_ including a copy
+4. `File a bug <https://github.com/mitogen-hq/mitogen/issues/new/>`_
-   of the stacks, along with a description of the last task executing prior to
+   including a copy of the stacks and a description of the last task executing
-   the hang.
+   before the hang
 
 It is possible the hang occurred in a process on a target. If ``strace`` is
 available, look for the host name not listed in Ansible output as reporting a
@@ -1259,7 +1283,7 @@ on each process whose name begins with ``mitogen:``::
     [pid 29858] futex(0x55ea9be52f60, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, NULL, 0xffffffff
     ^C
 
-    $ 
+    $
 
 This shows one thread waiting on IO (``poll``) and two more waiting on the same
 lock. It is taken from a real example of a deadlock due to a forking bug.
@@ -1278,7 +1302,7 @@ Sample Profiles
 ---------------
 
 The summaries below may be reproduced using data and scripts maintained in the
-`pcaps branch <https://github.com/dw/mitogen/tree/pcaps/>`_. Traces were
+`pcaps branch <https://github.com/mitogen-hq/mitogen/tree/pcaps/>`_. Traces were
 recorded using Ansible 2.5.14.
 
 
@@ -1287,7 +1311,7 @@ Trivial Loop: Local Host
 
 This demonstrates Mitogen vs. SSH pipelining to the local machine running
 `bench/loop-100-items.yml
-<https://github.com/dw/mitogen/blob/master/tests/ansible/bench/loop-100-items.yml>`_,
+<https://github.com/mitogen-hq/mitogen/blob/master/tests/ansible/bench/loop-100-items.yml>`_,
 executing a simple command 100 times. Most Ansible controller overhead is
 isolated, characterizing just module executor and connection layer performance.
 Mitogen requires **63x less bandwidth and 5.9x less time**.
@@ -1315,7 +1339,7 @@ File Transfer: UK to France
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 `This playbook
-<https://github.com/dw/mitogen/blob/master/tests/ansible/regression/issue_140__thread_pileup.yml>`_
+<https://github.com/mitogen-hq/mitogen/blob/master/tests/ansible/regression/issue_140__thread_pileup.yml>`_
 was used to compare file transfer performance over a ~26 ms link. It uses the
 ``with_filetree`` loop syntax to copy a directory of 1,000 0-byte files to the
 target.
@@ -1377,20 +1401,3 @@ Despite the small margin for optimization, Mitogen still manages **6.2x less
 bandwidth and 1.8x less time**.
 
 .. image:: images/ansible/pcaps/costapp-uk-india.svg
-
-
-.. raw:: html
-
-    <script src="https://networkgenomics.com/static/js/public_all.js?92d49a3a"></script>
-    <script>
-        NetGen = {
-            public: {
-                page_id: "operon",
-                urls: {
-                    save_email: "https://networkgenomics.com/save-email/",
-                    save_email_captcha: "https://networkgenomics.com/save-email/captcha/",
-                }
-            }
-        };
-        setupEmailForm();
-    </script>

docs/changelog.rst

@@ -15,12 +15,473 @@ Release Notes
     </style>
 
 To avail of fixes in an unreleased version, please download a ZIP file
-`directly from GitHub <https://github.com/dw/mitogen/>`_.
+`directly from GitHub <https://github.com/mitogen-hq/mitogen/>`_.
 
 
-v0.3.1.dev0 (unreleased)
+In progress (unreleased)
 ------------------------
 
+
+v0.3.39 (2026-01-27)
+--------------------
+
+* :gh:issue:`1430` :mod:`mitogen`: Pickle :data:`mitogen.core.GET_RESOURCE`
+  parameters directly as textual strings (rather than ASCII in byte strings)
+* :gh:issue:`1430` :mod:`mitogen`: Explicitly mark messages known to carry
+  pickled data, using :data:`mitogen.core.Message.ENC_PKL`. This repurposes
+  the magic field as a content encoding enumeration.
+* :gh:issue:`1430` :mod:`mitogen`: Add explicit binary Message encoding,
+  marked using :data:`mitogen.core.Message.ENC_BIN`.
+* :gh:issue:`1430` :mod:`mitogen`: Send :class:`mitogen.service.FileService`
+  content raw, without pickle encoding
+
+
+v0.3.38 (2026-01-23)
+--------------------
+
+* :gh:issue:`1418` :mod:`mitogen`: Format :class:`mitogen.core.Message` source
+  and destination as ``<context>:<handle>``, for clarity
+* :gh:issue:`1415` :mod:`mitogen`: Put fallbacks & polyfills into
+  ``if sys.version_info`` blocks
+* :gh:issue:`1423` tests: Group and unify naming of connection benchmarks
+* :gh:issue:`1424` tests: Parameterize connection benchmarks
+* :gh:issue:`1424` tests: Standardise output of connection benchmarks
+* :gh:issue:`1424` tests: Parameterize throughput benchmark
+* :gh:issue:`1424` tests: Parameterize large message benchmark
+* :gh:issue:`1424` :mod:`mitogen`: Consolidate all ``range`` and ``xrange``
+  polyfills into :attr:`mitogen.core.range`
+
+
+v0.3.37 (2026-01-08)
+--------------------
+
+* :gh:issue:`1398` :mod:`mitogen`: Fix :exc:`FileNotFoundError` during
+  ``import requests`` in a Mitogen child
+* :gh:issue:`1403` :mod:`mitogen`: Add initial support for
+  :py:class:`importlib.resource.abc.ResourceReader` protocol
+* :gh:issue:`1407` :mod:`mitogen`: Fix :exc:`AttributeError` in
+  :mod:`mitogen.profiler`
+
+
+v0.3.36 (2025-12-01)
+--------------------
+
+* :gh:issue:`1237` :mod:`mitogen`: Re-declare Python 2.4 compatibility
+* :gh:issue:`1385` :mod:`ansible_mitogen`: Remove a use of
+  ``ansible.module_utils.six``
+* :gh:issue:`1354` docs: Document Ansible 13 (ansible-core 2.20) support
+* :gh:issue:`1354` :mod:`mitogen`: Clarify error message when a module
+  request would be refused by allow or deny listing
+* :gh:issue:`1348` :mod:`mitogen`: Fix hanging process with 100% CPU usage
+
+
+v0.3.35 (2025-12-01)
+--------------------
+
+* :gh:issue:`1132` :mod:`ansible_mitogen` During intrepreter discovery use
+  Ansible ``INTERPRETER_PYTHON_FALLBACK`` config as list of candidates
+
+
+v0.3.34 (2025-11-27)
+--------------------
+
+* :gh:issue:`1118` CI: Use 2025.02 test images, keeping same OS releases
+* :gh:issue:`1358` CI: Bump deprecated macOS 13 runner to macOS 15
+* :gh:issue:`1118` CI: Add OS release coverage: AlmaLinux 9
+* :gh:issue:`1118` CI: Add OS release coverage: CentOS 5
+* :gh:issue:`1118` CI: Add OS release coverage: Debian 12
+* :gh:issue:`1118` CI: Add OS release coverage: Ubuntu 22.04, Ubuntu 24.04
+* :gh:issue:`1124` :mod:`mitogen`: Log why a module is sent or not sent by
+  :class:`mitogen.master.ModuleResponder`
+* :gh:issue:`1124` :mod:`ansible_mitogen`: Speedup startup by not sending
+  ``__main__`` as a related module
+
+
+v0.3.33 (2025-11-22)
+--------------------
+
+* :gh:issue:`1354` :mod:`ansible_mitogen`: ansible_mitogen: Ansible 13
+  (ansible-core 2.20) support
+
+
+v0.3.32 (2025-11-21)
+--------------------
+
+* :gh:issue:`1243` :mod:`mitogen`: Pass first stage, context name, & preamble
+  size as seperate **argv** arguments
+* :gh:issue:`1218` :mod:`ansible_mitogen`: Remove maximum Ansible version check
+* :gh:issue:`1260` CI: Remove integration of retired lgtm.com
+
+
+v0.3.31 (2025-11-05)
+--------------------
+
+* :gh:issue:`1350` :mod:`ansible_mitogen`: Fix regression when loading plugins
+  from ``/custom/path/to/mitogen``
+
+
+v0.3.30 (2025-10-30)
+--------------------
+
+* :gh:issue:`1266` Import cleanups
+* :gh:issue:`1266` :mod:`ansible_mitogen`: De-duplicate sys.path manipulations
+* :gh:issue:`1344` Correct SPDX license declarations
+* :gh:issue:`1344` Declare BSD-3-Clause SPDX license in package metadata
+* :gh:issue:`1344` :mod:`mitogen`: Use :py:func:`logging.makeLogRecord`
+
+
+v0.3.29 (2025-09-18)
+--------------------
+
+* :gh:issue:`1287` Python 3.14 support
+* :gh:issue:`1287` tests: Bump dependencies
+
+
+v0.3.28 (2025-09-17)
+--------------------
+
+* :gh:issue:`1306` :mod:`ansible_mitogen`: Fix non-blocking IO errors in
+  first stage of bootstrap
+* :gh:issue:`1306` CI: Report sudo version on Ansible targets
+* :gh:issue:`1306` CI: Move sudo test users defaults into ``/etc/sudoers.d``
+* :gh:issue:`1306` preamble_size: Fix variability of measured command size
+* :gh:issue:`1306` tests: Count bytes written in ``stdio_test.StdIOTest``
+* :gh:issue:`1306` tests: Check stdio is blocking in sudo contexts
+* :gh:issue:`1327` :mod:`ansible_mitogen`: Add FreeIPA client modules to the
+  always-fork list
+
+
+v0.3.27 (2025-08-20)
+--------------------
+
+* :gh:issue:`1325` :mod:`mitogen`: Refactor
+  ``mitogen.master.scan_code_imports()`` as
+  :func:`mitogen.import.codeobj_imports` and speed-up by 1.5 - 2.5 x
+* :gh:issue:`1329` CI: Refactor and de-duplicate Github Actions workflow
+* :gh:issue:`1315` CI: macOS: Increase failed logins limit of test users
+* :gh:issue:`1325` tests: Improve ``master_test.ScanCodeImportsTest`` coverage
+
+
+v0.3.26 (2025-08-04)
+--------------------
+
+* :gh:issue:`1318` CI: Abbreviate Github Actions job names
+* :gh:issue:`1309` :mod:`ansible_mitogen`: Fix ``become_method: doas``
+* :gh:issue:`712` :mod:`mitogen`: Fix :exc:`BlockingIOError` & ``EAGAIN``
+  errors in subprocesses that write to stdio
+
+
+v0.3.25 (2025-07-29)
+--------------------
+
+Ansible 12 has deprecated third-party strategy plugins. This is currently
+how Mitogen integrates with Ansible (e.g. `ANSIBLE_STRATEGY=mitogen_linear`).
+Running Ansible 12 + Mitogen will currently print a deprecation warning
+
+    [DEPRECATION WARNING]: Use of strategy plugins not included in
+    ansible.builtin are deprecated [...]. This feature will be removed from
+    ansible-core in a future release.
+
+Ansible + Mitogen will still work for now. Mitogen is considering alternatives
+to strategy plugins under :gh:issue:`1278`.
+
+* :gh:issue:`1258` Ansible 12 (ansible-core 2.19) support
+
+
+v0.3.25b1 (2025-07-21)
+----------------------
+
+* :gh:issue:`1303` CI: Switch to archived Debian 10 (buster) apt repository
+
+
+v0.3.25a3 (2025-07-02)
+----------------------
+
+* :gh:issue:`1285` CI: use `result_format = yaml` for Ansible test output,
+  instead of deprecated `stdout_callback = yaml`
+* :gh:issue:`1293` CI: Fix ``ansible_version`` comparisons when an Ansible
+  release candidate is under test
+* :gh:issue:`1275` CI: Test ``ansible_ssh_password`` behaviour without
+  ``sshpass`` installed
+* :gh:issue:`1282` :mod:`ansible_mitogen`: Support ``ANSIBLE_SSH_VERBOSITY``
+  with Ansible 12
+
+
+v0.3.25a2 (2025-06-21)
+----------------------
+
+* :gh:issue:`1274` :mod:`ansible_mitogen`: Replace use of `jsonify()`, which
+  is deprecated form Ansible 12 (ansible-core 2.19)
+
+
+v0.3.25a1 (2025-06-05)
+----------------------
+
+* :gh:issue:`1258` Initial Ansible 12 (ansible-core 2.19) support
+* :gh:issue:`1258` :mod:`ansible_mitogen`: Initial Ansible datatag support
+  (:gh:anspull:`84621`)
+* :gh:issue:`1258` :mod:`ansible_mitogen`: Ansible 12 (ansible-core 2.19) test
+  jobs
+
+
+v0.3.24 (2025-05-29)
+--------------------
+
+* :gh:issue:`1268` :mod:`mitogen` Only close stdin, stdout, and stderr file
+  descriptors (0, 1, and 2) if they were open at process startup.
+
+
+v0.3.23 (2025-04-28)
+--------------------
+
+* :gh:issue:`1121` :mod:`mitogen`: Log skipped :py:mod:`termios` attributes
+* :gh:issue:`1238` packaging: Avoid :py:mod:`ast`, requires Python = 2.6
+* :gh:issue:`1118` CI: Statically specify test usernames and group names
+* :gh:issue:`1118` CI: Don't copy SSH private key to temporary dir
+* :gh:issue:`1118` CI: Don't share temporary directory between test groupings
+* :gh:issue:`1256` CI: Upgrade Github jobs from Ubuntu 20.04 to 22.04 & 24.04
+* :gh:issue:`1263` packaging: Fix InvalidVersion in release versions
+
+
+v0.3.22 (2025-02-04)
+--------------------
+
+* :gh:issue:`1213` tests: Enable default Python warnings
+* :gh:issue:`1111` :mod:`mitogen`: Replace uses of deprecated
+  :py:func:`pkgutil.find_loader`
+* :gh:issue:`1213` :mod:`mitogen`: Fix unclosed file in first stage
+* :gh:issue:`1213` tests: Fix unclosed file in fd_check script
+* :gh:issue:`1213` :mod:`ansible_mitogen`: Don't redeclare Ansible interpreter
+  discovery attributes
+* :gh:issue:`1213` :mod:`ansible_mitogen`: Rename Mitogen interpreter discovery
+  attributes
+* :gh:issue:`1213` :mod:`ansible_mitogen`: Decouple possible_pythons order &
+  error handling
+* :gh:issue:`1213` :mod:`ansible_mitogen`: Return ``stderr_lines`` from
+  ``_low_level_execute_command()``
+* :gh:issue:`1227` tests: Name transport_config tests that use ``mitogen_via``
+* :gh:issue:`1143` :mod:`ansible_mitogen`: Fix dnf module include for dnf.cli
+* :gh:issue:`1234` :mod:`ansible_mitogen`: Fix :exc:`TypeError` in
+  :func:`ansible_mitogen.target.set_file_owner`
+
+
+v0.3.21 (2025-01-20)
+--------------------
+
+* :gh:issue:`1209` docs: Fix Netlify build of website
+* :gh:issue:`1216` :mod:`ansible_mitogen`: Add all ansible_freeipa modules to
+  the always-fork list.
+* :gh:issue:`766` :mod:`ansible_mitogen`: Fix ""could not recover task_vars"
+  and "get_with_context_result object has no attribute _create_control_path"
+  when using ``kubectl``, ``netconf``, or ``network_cli`` connection plugins.
+
+
+v0.3.20 (2025-01-07)
+--------------------
+
+* :gh:issue:`1079` :mod:`ansible_mitogen`: Fix :ans:mod:`wait_for_connection`
+  timeout with templated ``ansible_python_interpreter``
+* :gh:issue:`1079` :mod:`ansible_mitogen`: Fix templated python interpreter
+  with `meta: reset_connection`
+* :gh:issue:`1083` :mod:`ansible_mitogen`: Templated connection timeout
+  (e.g. ``ansible_timeout``).
+* :gh:issue:`740` :mod:`ansible_mitogen`: Respect ``interpreter_python``
+  in ``ansible.cfg`` and ``ANSIBLE_PYTHON_INTERPRETER`` environment variable.
+
+
+v0.3.19 (2024-12-02)
+--------------------
+
+* :gh:issue:`1129` :mod:`ansible_mitogen`: Ansible 11 support
+
+
+v0.3.18 (2024-11-07)
+--------------------
+
+* :gh:issue:`1083` :mod:`ansible_mitogen`: Templated become method
+  (e.g. ``ansible_become_method``).
+* :gh:issue:`1083` :mod:`ansible_mitogen`: Templated become flag
+  (e.g. ``ansible_become_method``, ``become`` keyword).
+
+
+v0.3.17 (2024-11-07)
+--------------------
+
+* :gh:issue:`1182` CI: Fix incorrect world readable/writable file permissions
+  on SSH key ``mitogen__has_sudo_pubkey.key`` during Ansible tests.
+* :gh:issue:`1083` :mod:`ansible_mitogen`: Templated SSH private key file
+  (e.g. ``ansible_private_key_file``).
+* :gh:issue:`1083` :mod:`ansible_mitogen`: Templated SSH host key checking
+  (e.g. ``ansible_host_key_checking``, ``ansible_ssh_host_key_checking``).
+* :gh:issue:`1083` :mod:`ansible_mitogen`: Templated host address
+  (e.g. ``ansible_host``, ``ansible_ssh_host``)
+* :gh:issue:`1184` Test templated SSH host key checking in task vars
+
+
+v0.3.16 (2024-11-05)
+--------------------
+
+* :gh:issue:`1083` :mod:`ansible_mitogen`: Templated become executable
+  (e.g. ``become_exe``).
+* :gh:issue:`1083` :mod:`ansible_mitogen`: Templated become executable
+  arguments (e.g. ``become_flags``).
+* :gh:issue:`1083` :mod:`ansible_mitogen`: Templated ssh executable
+  (``ansible_ssh_executable``).
+* :gh:issue:`1083` :mod:`ansible_mitogen`: Fixed templated connection options
+  during a ``meta: reset_connection`` task.
+* :gh:issue:`1129` CI: Migrated macOS 12 runners to macOS 13, due to EOL.
+
+
+v0.3.15 (2024-10-28)
+--------------------
+
+* :gh:issue:`905` :mod:`ansible_mitogen`: Support templated SSH command
+  arguments (e.g. ``ansible_ssh_args``, ``ansible_ssh_extra_args``).
+* :gh:issue:`692` tests: Fix and re-enable several sudo tests
+* :gh:issue:`1083` :mod:`ansible_mitogen`: Support templated become password
+  (e.g. ``ansible_become_pass``, ``ansible_sudo_pass``)
+
+
+v0.3.14 (2024-10-16)
+--------------------
+
+* :gh:issue:`1159` CI: Reduce number of Jobs by parameterizing Mitogen Docker
+  SSH tests
+* :gh:issue:`1083` :mod:`ansible_mitogen`: Support templated become username.
+
+
+v0.3.13 (2024-10-09)
+--------------------
+
+* :gh:issue:`1138` CI: Complete migration from Azure DevOps Pipelines to
+  GitHub Actions
+* :gh:issue:`1116` :mod:`ansible_mitogen`: Support for templated variable
+  `ansible_ssh_user`.
+* :gh:issue:`978` :mod:`ansible_mitogen`: Support templated Ansible SSH port.
+* :gh:issue:`1073` Python 3.13 support
+
+
+v0.3.12 (2024-10-07)
+--------------------
+
+* :gh:issue:`1106` :mod:`ansible_mitogen`: Support for `ansible_ssh_password`
+  connection variable, and templated SSH connection password.
+* :gh:issue:`1136` tests: Improve Ansible fail_msg formatting.
+* :gh:issue:`1137` tests: Ignore inventory files of inactive tests & benchmarks
+* :gh:issue:`1138` CI: Add re-actors/alls-green GitHub Actions job to simplify
+  branch protections configuration.
+
+
+v0.3.11 (2024-09-30)
+--------------------
+
+* :gh:issue:`1127` :mod:`mitogen`: Consolidate mitogen backward compatibility
+  fallbacks and polyfills into :mod:`mitogen.core`
+* :gh:issue:`1127` :mod:`ansible_mitogen`: Remove backward compatibility
+  fallbacks for Python 2.4 & 2.5.
+* :gh:issue:`1127` :mod:`ansible_mitogen`: Remove fallback imports for Ansible
+  releases before 2.10
+* :gh:issue:`1127` :mod:`ansible_mitogen`: Consolidate Python 2 & 3
+  compatibility
+* :gh:issue:`1128` CI: Start migration from Azure DevOps to GitHub Actions
+
+
+v0.3.10 (2024-09-20)
+--------------------
+
+* :gh:issue:`950` Fix Solaris/Illumos/SmartOS compatibility with become
+* :gh:issue:`1087` Fix :exc:`mitogen.core.StreamError` when Ansible template
+  module is called with a ``dest:`` filename that has an extension
+* :gh:issue:`1110` Fix :exc:`mitogen.core.StreamError` when Ansible copy
+  module is called with a file larger than 124 kibibytes
+  (:data:`ansible_mitogen.connection.Connection.SMALL_FILE_LIMIT`)
+* :gh:issue:`905` Initial support for templated ``ansible_ssh_args``,
+  ``ansible_ssh_common_args``, and ``ansible_ssh_extra_args`` variables.
+  NB: play or task scoped variables will probably still fail.
+* :gh:issue:`694` CI: Fixed a race condition and some resource leaks causing
+  some of intermittent failures when running the test suite.
+
+
+v0.3.9 (2024-08-13)
+-------------------
+
+* :gh:issue:`1097` Respect `ansible_facts.discovered_interpreter_python` when
+  executing non new-style modules (e.g. JSONARGS style, WANT_JSON style).
+* :gh:issue:`1074` Support Ansible 10 (ansible-core 2.17)
+
+
+v0.3.8 (2024-07-30)
+-------------------
+
+* :gh:issue:`952` Fix Ansible `--ask-become-pass`, add test coverage
+* :gh:issue:`957` Fix Ansible exception when executing against 10s of hosts
+  "ValueError: filedescriptor out of range in select()"
+* :gh:issue:`1066` Support Ansible `ansible_host_key_checking` & `ansible_ssh_host_key_checking`
+* :gh:issue:`1090` CI: Migrate macOS integration tests to macOS 12, drop Python 2.7 jobs
+
+
+v0.3.7 (2024-04-08)
+-------------------
+
+* :gh:issue:`1021` Support for Ansible 8 (ansible-core 2.15)
+* tests: Replace uses of ``include:`` & ``import:``, unsupported in Ansible 9
+* :gh:issue:`1053` Support for Ansible 9 (ansible-core 2.16)
+
+
+v0.3.6 (2024-04-04)
+-------------------
+
+* :gh:issue:`974` Support Ansible 7
+* :gh:issue:`1046` Raise :py:exc:`TypeError` in :func:`<mitogen.util.cast()>`
+  when casting a string subtype to `bytes()` or `str()` fails. This is
+  potentially an API breaking change. Failures previously passed silently.
+* :gh:issue:`1046` Add :func:`<ansible_mitogen.util.cast()>`, to cast
+  :class:`ansible.utils.unsafe_proxy.AnsibleUnsafe` objects in Ansible 7+.
+
+
+v0.3.5 (2024-03-17)
+-------------------
+
+* :gh:issue:`987` Support Python 3.11
+* :gh:issue:`885` Fix :py:exc:`PermissionError` in :py:mod:`importlib` when
+  becoming an unprivileged user with Python 3.x
+* :gh:issue:`1033` Support `PEP 451 <https://peps.python.org/pep-0451/>`_,
+  required by Python 3.12
+* :gh:issue:`1033` Support Python 3.12
+
+
+v0.3.4 (2023-07-02)
+-------------------
+
+* :gh:issue:`929` Support Ansible 6 and ansible-core 2.13
+* :gh:issue:`832` Fix runtime error when using the ansible.builtin.dnf module multiple times
+* :gh:issue:`925` :class:`ansible_mitogen.connection.Connection` no longer tries to close the
+  connection on destruction. This is expected to reduce cases of `mitogen.core.Error: An attempt
+  was made to enqueue a message with a Broker that has already exitted`. However it may result in
+  resource leaks.
+* :gh:issue:`659` Removed :mod:`mitogen.compat.simplejson`, not needed with Python 2.7+, contained Python 3.x syntax errors
+* :gh:issue:`983` CI: Removed PyPI faulthandler requirement from tests
+* :gh:issue:`1001` CI: Fixed Debian 9 & 11 tests
+
+v0.3.3 (2022-06-03)
+-------------------
+
+* :gh:issue:`906` Support packages dynamically inserted into sys.modules, e.g. `distro` >= 1.7.0 as `ansible.module_utils.distro`.
+* :gh:issue:`918` Support Python 3.10
+* :gh:issue:`920` Support Ansible :ans:conn:`~podman` connection plugin
+* :gh:issue:`836` :func:`mitogen.utils.with_router` decorator preserves the docstring in addition to the name.
+* :gh:issue:`936` :ans:mod:`fetch` no longer emits `[DEPRECATION WARNING]: The '_remote_checksum()' method is deprecated.`
+
+
+v0.3.2 (2022-01-12)
+-------------------
+
+* :gh:issue:`891` Correct `Framework :: Ansible` Trove classifier
+
+
+v0.3.1 (unreleased)
+-------------------
+
 * :gh:issue:`874` Support for Ansible 5 (ansible-core 2.12)
 * :gh:issue:`774` Fix bootstrap failures on macOS 11.x and 12.x, involving Python 2.7 wrapper
 * :gh:issue:`834` Support for Ansible 3 and 4 (ansible-core 2.11)
@@ -37,7 +498,7 @@ v0.3.0 (2021-11-24)
 -------------------
 
 This release separates itself from the v0.2.X releases. Ansible's API changed too much to support backwards compatibility so from now on, v0.2.X releases will be for Ansible < 2.10 and v0.3.X will be for Ansible 2.10+.
-`See here for details <https://github.com/dw/mitogen pull/715#issuecomment-750697248>`_.
+`See here for details <https://github.com/mitogen-hq/mitogen/pull/715#issuecomment-750697248>`_.
 
 * :gh:issue:`827` NewStylePlanner: detect `ansible_collections` imports
 * :gh:issue:`770` better check for supported Ansible version
@@ -58,7 +519,7 @@ v0.2.10 (2021-11-24)
 * :gh:issue:`756` ssh connections with `check_host_keys='accept'` would
   timeout, when using recent OpenSSH client versions.
 * :gh:issue:`758` fix initilialisation of callback plugins in test suite, to address a `KeyError` in
-  :method:`ansible.plugins.callback.CallbackBase.v2_runner_on_start`
+  :py:meth:`ansible.plugins.callback.CallbackBase.v2_runner_on_start`
 * :gh:issue:`775` Test with Python 3.9
 * :gh:issue:`775` Add msvcrt to the default module deny list
 
@@ -144,7 +605,7 @@ Mitogen for Ansible
   :linux:man7:`unix` sockets across privilege domains.
 
 * :gh:issue:`467`: an incompatibility running Mitogen under `Molecule
-  <https://molecule.readthedocs.io/en/stable/>`_ was resolved.
+  <https://ansible.readthedocs.io/projects/molecule/>`_ was resolved.
 
 * :gh:issue:`547`, :gh:issue:`598`: fix a deadlock during initialization of
   connections, ``async`` tasks, tasks using custom :mod:`module_utils`,
@@ -1196,9 +1657,8 @@ Core Library
   parameter may specify an argument vector prefix rather than a string program
   path.
 
-* :gh:issue:`300`: the broker could crash on
+* :gh:issue:`300`: the broker could crash on OS X during shutdown due to
-  OS X during shutdown due to scheduled `kqueue
+  scheduled :freebsd:man2:`kqueue` filter changes for
-  <https://www.freebsd.org/cgi/man.cgi?query=kqueue>`_ filter changes for
   descriptors that were closed before the IO loop resumes. As a temporary
   workaround, kqueue's bulk change feature is not used.
 

docs/conf.py

@@ -1,10 +1,24 @@
-import os
+import re
 import sys
 
-sys.path.append('..')
 sys.path.append('.')
-import mitogen
+
-VERSION = '%s.%s.%s' % mitogen.__version__
+
+def changelog_version(path, encoding='utf-8'):
+    "Return the 1st *stable* (not pre, dev) version in the changelog"
+    # See also grep_version() in setup.py
+    # e.g. "0.1.2, (1999-12-31)\n"
+    version_pattern = re.compile(
+        r'^v(?P<version>\d+\.\d+\.\d+) \((?P<date>\d\d\d\d-\d\d-\d\d)\)$',
+        re.MULTILINE,
+    )
+
+    with open(path, encoding=encoding) as f:
+        match = version_pattern.search(f.read())
+        return match.group('version')
+
+
+VERSION = changelog_version('changelog.rst')
 
 author = u'Network Genomics'
 copyright = u'2021, the Mitogen authors'
@@ -18,7 +32,6 @@ html_show_copyright = False
 html_show_sourcelink = False
 html_show_sphinx = False
 html_sidebars = {'**': ['globaltoc.html', 'github.html']}
-html_additional_pages = {'ansible': 'ansible.html'}
 html_static_path = ['_static']
 html_theme = 'alabaster'
 html_theme_options = {
@@ -44,16 +57,25 @@ version = VERSION
 domainrefs = {
     'gh:commit': {
         'text': '%s',
-        'url': 'https://github.com/dw/mitogen/commit/%s',
+        'url': 'https://github.com/mitogen-hq/mitogen/commit/%s',
     },
     'gh:issue': {
         'text': '#%s',
-        'url': 'https://github.com/dw/mitogen/issues/%s',
+        'url': 'https://github.com/mitogen-hq/mitogen/issues/%s',
     },
     'gh:pull': {
         'text': '#%s',
-        'url': 'https://github.com/dw/mitogen/pull/%s',
+        'url': 'https://github.com/mitogen-hq/mitogen/pull/%s',
+    },
+    'gh:ansissue': {
+        'text': 'Ansible #%s',
+        'url': 'https://github.com/ansible/ansible/issues/%s',
     },
+    'gh:anspull': {
+        'text': 'Ansible #%s',
+        'url': 'https://github.com/ansible/ansible/pull/%s',
+    },
+
     'ans:mod': {
         'text': '%s module',
         'url': 'https://docs.ansible.com/ansible/latest/modules/%s_module.html',
@@ -64,30 +86,36 @@ domainrefs = {
     },
     'freebsd:man2': {
         'text': '%s(2)',
-        'url': 'https://www.freebsd.org/cgi/man.cgi?query=%s',
+        'url': 'https://man.freebsd.org/cgi/man.cgi?query=%s',
     },
     'linux:man1': {
         'text': '%s(1)',
-        'url': 'http://man7.org/linux/man-pages/man1/%s.1.html',
+        'url': 'https://man7.org/linux/man-pages/man1/%s.1.html',
     },
     'linux:man2': {
         'text': '%s(2)',
-        'url': 'http://man7.org/linux/man-pages/man2/%s.2.html',
+        'url': 'https://man7.org/linux/man-pages/man2/%s.2.html',
     },
     'linux:man3': {
         'text': '%s(3)',
-        'url': 'http://man7.org/linux/man-pages/man3/%s.3.html',
+        'url': 'https://man7.org/linux/man-pages/man3/%s.3.html',
     },
     'linux:man7': {
         'text': '%s(7)',
-        'url': 'http://man7.org/linux/man-pages/man7/%s.7.html',
+        'url': 'https://man7.org/linux/man-pages/man7/%s.7.html',
     },
 }
 
+# > ## Official guidance
+# > Query PyPI’s JSON API to determine where to download files from.
+# > ## Predictable URLs
+# > You can use our conveyor service to fetch this file, which exists for
+# > cases where using the API is impractical or impossible.
+# > -- https://warehouse.pypa.io/api-reference/integration-guide.html#predictable-urls
 rst_epilog = """
 
 .. |mitogen_version| replace:: %(VERSION)s
 
-.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz <https://networkgenomics.com/try/mitogen-%(VERSION)s.tar.gz>`__
+.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz <https://files.pythonhosted.org/packages/source/m/mitogen/mitogen-%(VERSION)s.tar.gz>`__
 
 """ % locals()

docs/contributors.rst

@@ -116,6 +116,7 @@ sponsorship and outstanding future-thinking of its early adopters.
 
     <ul>
     <li>Alex Willmer</li>
+    <li><a href="https://github.com/momiji">Christian Bourgeois </a></li>
     <li><a href="https://underwhelm.net/">Dan Dorman</a> &mdash; - <em>When I truly understand my enemy … then in that very moment I also love him.</em></li>
     <li>Daniel Foerster</li>
     <li><a href="https://www.deps.co/">Deps</a> &mdash; <em>Private Maven Repository Hosting for Java, Scala, Groovy, Clojure</em></li>
@@ -123,13 +124,22 @@ sponsorship and outstanding future-thinking of its early adopters.
     <li><a href="https://www.epartment.nl/">Epartment</a></li>
     <li><a href="http://andrianaivo.org/">Fidy Andrianaivo</a> &mdash; <em>never let a human do an ansible job ;)</em></li>
     <li><a href="https://www.channable.com">rkrzr</a></li>
+    <li><a href="https://github.com/Nihlus">Jarl Gullberg</a></li>
     <li>jgadling</li>
     <li>John F Wall &mdash; <em>Making Ansible Great with Massive Parallelism</em></li>
+    <li><a href="https://github.com/jrosser">Jonathan Rosser</a></li>
+    <li><a href="https://github.com/jmkeyes">Joshua M. Keyes</a></li>
     <li>KennethC</li>
+    <li><a href="https://github.com/lberruti">Luca Berruti</li>
     <li>Lewis Bellwood &mdash; <em>Happy to be apart of a great project.</em></li>
     <li>luto</li>
+    <li><a href="https://github.com/markafarrell">@markafarrell</a></li>
     <li><a href="https://mayeu.me/">Mayeu a.k.a Matthieu Maury</a></li>
+    <li><a href="https://github.com/madsi1m">Michael D'Silva</a></li>
+    <li><a href="https://github.com/mordekasg">mordek</a></li>
     <li><a href="https://twitter.com/nathanhruby">@nathanhruby</a></li>
+    <li><a href="https://github.com/opoplawski">Orion Poplawski</a></li>
+    <li><a href="https://github.com/philfry">Philippe Kueck</a></li>
     <li><a href="http://pageflows.com/">Ramy</a></li>
     <li>Scott Vokes</li>
     <li><a href="https://twitter.com/sirtux">Tom Eichhorn</a></li>
@@ -138,4 +148,5 @@ sponsorship and outstanding future-thinking of its early adopters.
     <li>randy &mdash; <em>desperate for automation</em></li>
     <li>Michael & Vicky Twomey-Lee</li>
     <li><a href="http://www.wezm.net/">Wesley Moore</a></li>
+    <li><a href="https://github.com/baryluk">Witold Baryluk</a></li>
     </ul>

docs/examples.rst

@@ -201,7 +201,7 @@ nested.py:
             print('Connect local%d via %s' % (x, context))
             context = router.local(via=context, name='local%d' % x)
 
-        context.call(os.system, 'pstree -s python -s mitogen')
+        context.call(subprocess.check_call, ['pstree', '-s', 'python', '-s', 'mitogen'])
 
 
 Output:

docs/howitworks.rst

@@ -27,14 +27,13 @@ Python Command Line
 ###################
 
 The Python command line sent to the host is a :mod:`zlib`-compressed [#f2]_ and
-base64-encoded copy of the :py:meth:`mitogen.master.Stream._first_stage`
+base64-encoded copy of :py:meth:`mitogen.parent.Connection._first_stage`,
-function, which has been carefully optimized to reduce its size. Prior to
+which is carefully written to maximize it compatibility and minimize its size.
-compression and encoding, ``CONTEXT_NAME`` is replaced with the desired context
+A simplified illustration of the bootstrap command is
-name in the function's source code.
 
 .. code::
 
-    python -c 'exec "xxx".decode("base64").decode("zlib")'
+    python -c 'exec(sys.argv[1].decode("base64").decode("zlib"))' <base64> ...
 
 The command-line arranges for the Python interpreter to decode the base64'd
 component, decompress it and execute it as Python code. Base64 is used since
@@ -71,8 +70,8 @@ of the large base64-encoded first stage parameter, and to replace **argv[0]**
 with something descriptive.
 
 After configuring its ``stdin`` to point to the read end of the pipe, the
-parent half of the fork re-executes Python, with **argv[0]** taken from the
+fork parent re-executes Python with **argv[0]** composed of the Python
-``CONTEXT_NAME`` variable earlier substituted into its source code. As no
+interpreter path and a remote name supplied by the Mitogen parent.  As no
 arguments are provided to this new execution of Python, and since ``stdin`` is
 connected to a pipe (whose write end is connected to the first stage), the
 Python interpreter begins reading source code to execute from the pipe
@@ -1038,7 +1037,7 @@ receive items in the order they are requested, as they become available.
    Mitogen enables SSH compression by default, there are circumstances where
    disabling SSH compression is desirable, and many scenarios for future
    connection methods where transport-layer compression is not supported at
-   all. 
+   all.
 
 .. [#f2] Compression may seem redundant, however it is basically free and reducing IO
    is always a good idea. The 33% / 200 byte saving may mean the presence or

docs/index.rst

@@ -26,7 +26,7 @@ and efficient low-level API on which tools like `Salt`_, `Ansible`_, or
 `Fabric`_ can be built, and while the API is quite friendly and comparable to
 `Fabric`_, ultimately it is not intended for direct use by consumer software.
 
-.. _Salt: https://docs.saltstack.com/en/latest/
+.. _Salt: https://docs.saltproject.io/en/latest/
 .. _Ansible: https://docs.ansible.com/
 .. _Fabric: https://www.fabfile.org/
 
@@ -101,7 +101,7 @@ to your network topology**.
         container='billing0',
     )
 
-    internal_box.call(os.system, './run-nightly-billing.py')
+    internal_box.call(subprocess.check_call, ['./run-nightly-billing.py'])
 
 The multiplexer also ensures the remote process is terminated if your Python
 program crashes, communication is lost, or the application code running in the
@@ -250,7 +250,7 @@ After:
         """
         Install our application.
         """
-        os.system('tar zxvf app.tar.gz')
+        subprocess.check_call(['tar', 'zxvf', 'app.tar.gz'])
 
     context.call(install_app)
 
@@ -258,7 +258,7 @@ Or even:
 
 .. code-block:: python
 
-    context.call(os.system, 'tar zxvf app.tar.gz')
+    context.call(subprocess.check_call, ['tar', 'zxvf', 'app.tar.gz'])
 
 Exceptions raised by function calls are propagated back to the parent program,
 and timeouts can be configured to ensure failed calls do not block progress of
@@ -332,12 +332,16 @@ a large fleet of machines, or to alert the parent of unexpected state changes.
 Compatibility
 #############
 
-Mitogen is compatible with **Python 2.4** released November 2004, making it
+``mitogen.*`` is compatible with Python 2.4 - 2.7 and 3.6 onward; making it
 suitable for managing a fleet of potentially ancient corporate hardware, such
 as Red Hat Enterprise Linux 5, released in 2007.
 
-Every combination of Python 3.x/2.x parent and child should be possible,
+Every combination of Python 3.x/2.x parent and child should be possible.
-however at present only Python 2.4, 2.6, 2.7 and 3.6 are tested automatically.
+Automated testing cannot cover every combination, automated testing tries to
+cover the extemities (e.g. Python 3.14 parent -> Python 2.4 child).
+
+``ansible_mitogen.*`` is compatible with Python 2.7 and 3.6 onward; making it
+suitable for Ansible 2.10 onward.
 
 
 Zero Dependencies

docs/netlify.toml

@@ -0,0 +1,2 @@
+[build.environment]
+PYTHON_VERSION = "3.8"

docs/requirements.txt

@@ -1,3 +1,6 @@
+docutils<0.18
+Jinja2<3
+MarkupSafe<2.1
 Sphinx==2.1.2; python_version > '3.0'
 sphinxcontrib-programoutput==0.14; python_version > '3.0'
 alabaster==0.7.10; python_version > '3.0'

examples/install_app.py

@@ -8,14 +8,14 @@ Usage:
 Where:
     <hostname>  Hostname to install to.
 """
-import os
+import subprocess
 import sys
 
 import mitogen
 
 
 def install_app():
-    os.system('tar zxvf my_app.tar.gz')
+    subprocess.check_call(['tar', 'zxvf', 'my_app.tar.gz'])
 
 
 @mitogen.main()

examples/mitogen-fuse.py

@@ -119,7 +119,7 @@ def _chroot(path):
     os.chroot(path)
 
 
-class Operations(fuse.Operations):  # fuse.LoggingMixIn, 
+class Operations(fuse.Operations):  # fuse.LoggingMixIn,
     def __init__(self, host, path='.'):
         self.host = host
         self.root = path

examples/mitop.py

@@ -61,7 +61,7 @@ def child_main(sender, delay):
     Executed on the main thread of the Python interpreter running on each
     target machine, Context.call() from the master. It simply sends the output
     of the UNIX 'ps' command at regular intervals toward a Receiver on master.
-    
+
     :param mitogen.core.Sender sender:
         The Sender to use for delivering our result. This could target
         anywhere, but the sender supplied by the master simply causes results

examples/the_basics.py

@@ -10,7 +10,6 @@ from __future__ import print_function
 import hashlib
 import io
 import os
-import spwd
 
 import mitogen.core
 import mitogen.master
@@ -57,21 +56,6 @@ def streamy_download_file(context, path):
     }
 
 
-def get_password_hash(username):
-    """
-    Fetch a user's password hash.
-    """
-    try:
-        h = spwd.getspnam(username)
-    except KeyError:
-        return None
-
-    # mitogen.core.Secret() is a Unicode subclass with a repr() that hides the
-    # secret data. This keeps secret stuff out of logs. Like blobs, secrets can
-    # also be serialized.
-    return mitogen.core.Secret(h)
-
-
 def md5sum(path):
     """
     Return the MD5 checksum for a file.

mitogen/__init__.py

@@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup.
 
 
 #: Library version as a tuple.
-__version__ = (0, 3, 1, 'dev0')
+__version__ = (0, 3, 40, 'dev')
 
 
 #: This is :data:`False` in slave contexts. Previously it was used to prevent
@@ -106,7 +106,8 @@ def main(log_level='INFO', profiling=_default_profiling):
     def wrapper(func):
         if func.__module__ != '__main__':
             return func
-        import mitogen.parent
+        import mitogen.core
+        import mitogen.master
         import mitogen.utils
         if profiling:
             mitogen.core.enable_profiling()

mitogen/buildah.py

@@ -30,7 +30,6 @@
 
 import logging
 
-import mitogen.core
 import mitogen.parent
 
 

mitogen/fakessh.py

@@ -95,15 +95,14 @@ Sequence:
 import getopt
 import inspect
 import os
+import pty
 import shutil
-import socket
 import subprocess
 import sys
 import tempfile
 import threading
 
 import mitogen.core
-import mitogen.master
 import mitogen.parent
 
 from mitogen.core import LOG, IOLOG
@@ -179,6 +178,9 @@ class Process(object):
         self.control_handle = router.add_handler(self._on_control)
         self.stdin_handle = router.add_handler(self._on_stdin)
         self.pump = IoPump.build_stream(router.broker)
+        for fp in stdin, stdout:
+            fd = fp.fileno()
+            mitogen.core.set_blocking(fd, False)
         self.pump.accept(stdin, stdout)
         self.stdin = None
         self.control = None
@@ -200,7 +202,7 @@ class Process(object):
 
     def _on_stdin(self, msg):
         if msg.is_dead:
-            IOLOG.debug('%r._on_stdin() -> %r', self, data)
+            IOLOG.debug('%r._on_stdin() -> %r', self, msg)
             self.pump.protocol.close()
             return
 
@@ -355,8 +357,9 @@ def _fakessh_main(dest_context_id, econtext):
               control_handle, stdin_handle)
 
     process = Process(econtext.router,
-                      stdin=os.fdopen(1, 'w+b', 0),
+        stdin=os.fdopen(pty.STDOUT_FILENO, 'w+b', 0),
-                      stdout=os.fdopen(0, 'r+b', 0))
+        stdout=os.fdopen(pty.STDIN_FILENO, 'r+b', 0),
+    )
     process.start_master(
         stdin=mitogen.core.Sender(dest, stdin_handle),
         control=mitogen.core.Sender(dest, control_handle),
@@ -418,10 +421,11 @@ def run(dest, router, args, deadline=None, econtext=None):
     fakessh = mitogen.parent.Context(router, context_id)
     fakessh.name = u'fakessh.%d' % (context_id,)
 
-    sock1, sock2 = socket.socketpair()
+    sock1, sock2 = mitogen.core.socketpair()
 
     stream = mitogen.core.Stream(router, context_id)
     stream.name = u'fakessh'
+    mitogen.core.set_blocking(sock1.fileno(), False)
     stream.accept(sock1, sock1)
     router.register(fakessh, stream)
 
@@ -437,7 +441,7 @@ def run(dest, router, args, deadline=None, econtext=None):
             fp.write(inspect.getsource(mitogen.core))
             fp.write('\n')
             fp.write('ExternalContext(%r).main()\n' % (
-                _get_econtext_config(context, sock2),
+                _get_econtext_config(econtext, sock2),
             ))
         finally:
             fp.close()

mitogen/fork.py

@@ -211,7 +211,7 @@ class Connection(mitogen.parent.Connection):
         on_fork()
         if self.options.on_fork:
             self.options.on_fork()
-        mitogen.core.set_block(childfp.fileno())
+        mitogen.core.set_blocking(childfp.fileno(), True)
 
         childfp.send(b('MITO002\n'))
 

mitogen/imports/__init__.py

@@ -0,0 +1,38 @@
+# SPDX-FileCopyrightText: 2025 Mitogen authors <https://github.com/mitogen-hq>
+# SPDX-License-Identifier: BSD-3-Clause
+# !mitogen: minify_safe
+
+import sys
+
+if sys.version_info >= (3, 14):
+    from mitogen.imports._py314 import _code_imports
+elif sys.version_info >= (3, 6):
+    from mitogen.imports._py36 import _code_imports
+elif sys.version_info >= (2, 5):
+    from mitogen.imports._py2 import _code_imports_py25 as _code_imports
+else:
+    from mitogen.imports._py2 import _code_imports_py24 as _code_imports
+
+
+def codeobj_imports(co):
+    """
+    Yield (level, modname, names) tuples by scanning the code object `co`.
+
+    Top level `import mod` & `from mod import foo` statements are matched.
+    Those inside a `class ...` or `def ...` block are currently skipped.
+
+    >>> co = compile('import a, b; from c import d, e as f', '<str>', 'exec')
+    >>> list(codeobj_imports(co))  # doctest: +ELLIPSIS
+    [(..., 'a', ()), (..., 'b', ()), (..., 'c', ('d', 'e'))]
+
+    :return:
+        Generator producing `(level, modname, names)` tuples, where:
+
+        * `level`:
+            -1 implicit relative (Python 2.x default)
+            0  absolute (Python 3.x, `from __future__ import absolute_import`)
+            >0 explicit relative (`from . import a`, `from ..b, import c`)
+        * `modname`: Name of module to import, or to import `names` from.
+        * `names`: tuple of names in `from mod import ..`.
+    """
+    return _code_imports(co.co_code, co.co_consts, co.co_names)

mitogen/imports/_py2.py

@@ -0,0 +1,54 @@
+# SPDX-FileCopyrightText: 2025 Mitogen authors <https://github.com/mitogen-hq>
+# SPDX-License-Identifier: BSD-3-Clause
+# !mitogen: minify_safe
+
+import array
+import itertools
+import opcode
+
+
+IMPORT_NAME = opcode.opmap['IMPORT_NAME']
+LOAD_CONST = opcode.opmap['LOAD_CONST']
+
+
+def _opargs(code, _have_arg=opcode.HAVE_ARGUMENT):
+    it = iter(array.array('B', code))
+    nexti = it.next
+    for i in it:
+        if i >= _have_arg:
+            yield (i, nexti() | (nexti() << 8))
+        else:
+            yield (i, None)
+
+
+def _code_imports_py25(code, consts, names):
+    it1, it2, it3 = itertools.tee(_opargs(code), 3)
+    try:
+        next(it2)
+        next(it3)
+        next(it3)
+    except StopIteration:
+        return
+    for oparg1, oparg2, (op3, arg3) in itertools.izip(it1, it2, it3):
+        if op3 != IMPORT_NAME:
+            continue
+        op1, arg1 = oparg1
+        op2, arg2 = oparg2
+        if op1 != LOAD_CONST or op2 != LOAD_CONST:
+            continue
+        yield (consts[arg1], names[arg3], consts[arg2] or ())
+
+
+def _code_imports_py24(code, consts, names):
+    it1, it2 = itertools.tee(_opargs(code), 2)
+    try:
+        next(it2)
+    except StopIteration:
+        return
+    for oparg1, (op2, arg2) in itertools.izip(it1, it2):
+        if op2 != IMPORT_NAME:
+            continue
+        op1, arg1 = oparg1
+        if op1 != LOAD_CONST:
+            continue
+        yield (-1, names[arg2], consts[arg1] or ())

mitogen/imports/_py314.py

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: 2025 Mitogen authors <https://github.com/mitogen-hq>
+# SPDX-License-Identifier: BSD-3-Clause
+# !mitogen: minify_safe
+
+import opcode
+
+IMPORT_NAME = opcode.opmap['IMPORT_NAME']
+LOAD_CONST = opcode.opmap['LOAD_CONST']
+LOAD_SMALL_INT = opcode.opmap['LOAD_SMALL_INT']
+
+
+def _code_imports(code, consts, names):
+    start = 4
+    while True:
+        op3_idx = code.find(IMPORT_NAME, start, -1)
+        if op3_idx < 0:
+            return
+        if op3_idx % 2:
+            start = op3_idx + 1
+            continue
+        if code[op3_idx-4] != LOAD_SMALL_INT or code[op3_idx-2] != LOAD_CONST:
+            start = op3_idx + 2
+            continue
+        start = op3_idx + 6
+        arg1, arg2, arg3 = code[op3_idx-3], code[op3_idx-1], code[op3_idx+1]
+        yield (arg1, names[arg3], consts[arg2] or ())

mitogen/imports/_py36.py

@@ -0,0 +1,25 @@
+# SPDX-FileCopyrightText: 2025 Mitogen authors <https://github.com/mitogen-hq>
+# SPDX-License-Identifier: BSD-3-Clause
+# !mitogen: minify_safe
+
+import opcode
+
+IMPORT_NAME = opcode.opmap['IMPORT_NAME']
+LOAD_CONST = opcode.opmap['LOAD_CONST']
+
+
+def _code_imports(code, consts, names):
+    start = 4
+    while True:
+        op3_idx = code.find(IMPORT_NAME, start, -1)
+        if op3_idx < 0:
+            return
+        if op3_idx % 2:
+            start = op3_idx + 1
+            continue
+        if code[op3_idx-4] != LOAD_CONST or code[op3_idx-2] != LOAD_CONST:
+            start = op3_idx + 2
+            continue
+        start = op3_idx + 6
+        arg1, arg2, arg3 = code[op3_idx-3], code[op3_idx-1], code[op3_idx+1]
+        yield (consts[arg1], names[arg3], consts[arg2] or ())

mitogen/kubectl.py

@@ -28,7 +28,6 @@
 
 # !mitogen: minify_safe
 
-import mitogen.core
 import mitogen.parent
 
 

mitogen/lxc.py

@@ -28,7 +28,6 @@
 
 # !mitogen: minify_safe
 
-import mitogen.core
 import mitogen.parent
 
 

mitogen/lxd.py

@@ -28,7 +28,6 @@
 
 # !mitogen: minify_safe
 
-import mitogen.core
 import mitogen.parent
 
 

mitogen/master.py

@@ -35,11 +35,8 @@ be sent to any context that will be used to establish additional child
 contexts.
 """
 
-import dis
 import errno
-import imp
 import inspect
-import itertools
 import logging
 import os
 import pkgutil
@@ -50,21 +47,44 @@ import threading
 import types
 import zlib
 
-try:
+if sys.version_info >= (3, 7):
+    import importlib.resources
+
+if sys.version_info >= (3, 4):
+    import importlib.util
+    from _imp import is_builtin as _is_builtin
+
+    def _find_loader(fullname):
+        try:
+            maybe_spec = importlib.util.find_spec(fullname)
+        except (ImportError, AttributeError, TypeError, ValueError):
+            exc = sys.exc_info()[1]
+            raise ImportError(*exc.args)
+        try:
+            return maybe_spec.loader
+        except AttributeError:
+            return None
+else:
+    import imp
+    from imp import is_builtin as _is_builtin
+
+    if sys.version_info >= (2, 5):
+        from pkgutil import find_loader as _find_loader
+    else:
+        from mitogen.compat.pkgutil import find_loader as _find_loader
+
+if sys.version_info >= (2, 7):
     import sysconfig
-except ImportError:
+else:
     sysconfig = None
 
-if not hasattr(pkgutil, 'find_loader'):
-    # find_loader() was new in >=2.5, but the modern pkgutil.py syntax has
-    # been kept intentionally 2.3 compatible so we can reuse it.
-    from mitogen.compat import pkgutil
-
 import mitogen
 import mitogen.core
+import mitogen.imports
 import mitogen.minify
 import mitogen.parent
 
+from mitogen.core import any
 from mitogen.core import b
 from mitogen.core import IOLOG
 from mitogen.core import LOG
@@ -72,20 +92,6 @@ from mitogen.core import str_partition
 from mitogen.core import str_rpartition
 from mitogen.core import to_text
 
-imap = getattr(itertools, 'imap', map)
-izip = getattr(itertools, 'izip', zip)
-
-try:
-    any
-except NameError:
-    from mitogen.core import any
-
-try:
-    next
-except NameError:
-    from mitogen.core import next
-
-
 RLOG = logging.getLogger('mitogen.ctx')
 
 
@@ -122,7 +128,16 @@ def is_stdlib_name(modname):
     """
     Return :data:`True` if `modname` appears to come from the standard library.
     """
-    if imp.is_builtin(modname) != 0:
+    # `(_imp|imp).is_builtin()` isn't a documented part of Python's stdlib.
+    # Returns 1 if modname names a module that is "builtin" to the the Python
+    # interpreter (e.g. '_sre'). Otherwise 0 (e.g. 're', 'netifaces').
+    #
+    # """
+    # Main is a little special - imp.is_builtin("__main__") will return False,
+    # but BuiltinImporter is still the most appropriate initial setting for
+    # its __loader__ attribute.
+    # """ -- comment in CPython pylifecycle.c:add_main_module()
+    if _is_builtin(modname) != 0:
         return True
 
     module = sys.modules.get(modname)
@@ -166,7 +181,7 @@ def get_child_modules(path, fullname):
         return [to_text(name) for _, name, _ in pkgutil.iter_modules([mod_path])]
     else:
         # we loaded some weird package in memory, so we'll see if it has a custom loader we can use
-        loader = pkgutil.find_loader(fullname)
+        loader = _find_loader(fullname)
         return [to_text(name) for name, _ in loader.iter_modules(None)] if loader else []
 
 
@@ -232,80 +247,6 @@ if mitogen.is_master:
     mitogen.parent._get_core_source = _get_core_source
 
 
-LOAD_CONST = dis.opname.index('LOAD_CONST')
-IMPORT_NAME = dis.opname.index('IMPORT_NAME')
-
-
-def _getarg(nextb, c):
-    if c >= dis.HAVE_ARGUMENT:
-        return nextb() | (nextb() << 8)
-
-
-if sys.version_info < (3, 0):
-    def iter_opcodes(co):
-        # Yield `(op, oparg)` tuples from the code object `co`.
-        ordit = imap(ord, co.co_code)
-        nextb = ordit.next
-        return ((c, _getarg(nextb, c)) for c in ordit)
-elif sys.version_info < (3, 6):
-    def iter_opcodes(co):
-        # Yield `(op, oparg)` tuples from the code object `co`.
-        ordit = iter(co.co_code)
-        nextb = ordit.__next__
-        return ((c, _getarg(nextb, c)) for c in ordit)
-else:
-    def iter_opcodes(co):
-        # Yield `(op, oparg)` tuples from the code object `co`.
-        ordit = iter(co.co_code)
-        nextb = ordit.__next__
-        # https://github.com/abarnert/cpython/blob/c095a32f/Python/wordcode.md
-        return ((c, nextb()) for c in ordit)
-
-
-def scan_code_imports(co):
-    """
-    Given a code object `co`, scan its bytecode yielding any ``IMPORT_NAME``
-    and associated prior ``LOAD_CONST`` instructions representing an `Import`
-    statement or `ImportFrom` statement.
-
-    :return:
-        Generator producing `(level, modname, namelist)` tuples, where:
-
-        * `level`: -1 for normal import, 0, for absolute import, and >0 for
-          relative import.
-        * `modname`: Name of module to import, or from where `namelist` names
-          are imported.
-        * `namelist`: for `ImportFrom`, the list of names to be imported from
-          `modname`.
-    """
-    opit = iter_opcodes(co)
-    opit, opit2, opit3 = itertools.tee(opit, 3)
-
-    try:
-        next(opit2)
-        next(opit3)
-        next(opit3)
-    except StopIteration:
-        return
-
-    if sys.version_info >= (2, 5):
-        for oparg1, oparg2, (op3, arg3) in izip(opit, opit2, opit3):
-            if op3 == IMPORT_NAME:
-                op2, arg2 = oparg2
-                op1, arg1 = oparg1
-                if op1 == op2 == LOAD_CONST:
-                    yield (co.co_consts[arg1],
-                           co.co_names[arg3],
-                           co.co_consts[arg2] or ())
-    else:
-        # Python 2.4 did not yet have 'level', so stack format differs.
-        for oparg1, (op2, arg2) in izip(opit, opit2):
-            if op2 == IMPORT_NAME:
-                op1, arg1 = oparg1
-                if op1 == LOAD_CONST:
-                    yield (-1, co.co_names[arg2], co.co_consts[arg1] or ())
-
-
 class ThreadWatcher(object):
     """
     Manage threads that wait for another thread to shut down, before invoking
@@ -428,15 +369,19 @@ class LogForwarder(object):
         if logger is None:
             self._cache[logger_name] = logger = logging.getLogger(logger_name)
 
+        levelno = int(level_s)
         # See logging.Handler.makeRecord()
-        record = logging.LogRecord(
+        record = logging.makeLogRecord(
-            name=logger.name,
+            {
-            level=int(level_s),
+                "name": logger.name,
-            pathname='(unknown file)',
+                "levelname": logging.getLevelName(levelno),
-            lineno=0,
+                "levelno": levelno,
-            msg=s,
+                "pathname": "(unknown file)",
-            args=(),
+                "lineno": 0,
-            exc_info=None,
+                "msg": s,
+                "args": (),
+                "exc_info": None,
+            }
         )
         record.mitogen_message = s
         record.mitogen_context = self._router.context_by_id(msg.src_id)
@@ -453,6 +398,9 @@ class FinderMethod(object):
     name according to the running Python interpreter. You'd think this was a
     simple task, right? Naive young fellow, welcome to the real world.
     """
+    def __init__(self):
+        self.log = LOG.getChild(self.__class__.__name__)
+
     def __repr__(self):
         return '%s()' % (type(self).__name__,)
 
@@ -512,42 +460,57 @@ class PkgutilMethod(FinderMethod):
         Find `fullname` using :func:`pkgutil.find_loader`.
         """
         try:
+            # If fullname refers to a submodule that's not already imported
+            # then the containing package is imported.
             # Pre-'import spec' this returned None, in Python3.6 it raises
             # ImportError.
-            loader = pkgutil.find_loader(fullname)
+            loader = _find_loader(fullname)
         except ImportError:
             e = sys.exc_info()[1]
-            LOG.debug('%r._get_module_via_pkgutil(%r): %s',
+            LOG.debug('%r: find_loader(%r) failed: %s', self, fullname, e)
-                      self, fullname, e)
             return None
 
-        IOLOG.debug('%r._get_module_via_pkgutil(%r) -> %r',
-                    self, fullname, loader)
         if not loader:
+            LOG.debug('%r: find_loader(%r) returned %r, aborting',
+                      self, fullname, loader)
             return
 
         try:
-            path, is_special = _py_filename(loader.get_filename(fullname))
+            path = loader.get_filename(fullname)
-            source = loader.get_source(fullname)
+        except (AttributeError, ImportError, ValueError):
-            is_pkg = loader.is_package(fullname)
-
-            # workaround for special python modules that might only exist in memory
-            if is_special and is_pkg and not source:
-                source = '\n'
-        except (AttributeError, ImportError):
-            # - Per PEP-302, get_source() and is_package() are optional,
-            #   calling them may throw AttributeError.
             # - get_filename() may throw ImportError if pkgutil.find_loader()
             #   picks a "parent" package's loader for some crap that's been
             #   stuffed in sys.modules, for example in the case of urllib3:
             #       "loader for urllib3.contrib.pyopenssl cannot handle
             #        requests.packages.urllib3.contrib.pyopenssl"
             e = sys.exc_info()[1]
-            LOG.debug('%r: loading %r using %r failed: %s',
+            LOG.debug('%r: %r.get_file_name(%r) failed: %r', self, loader, fullname, e)
-                      self, fullname, loader, e)
+            return
+
+        path, is_special = _py_filename(path)
+
+        try:
+            source = loader.get_source(fullname)
+        except AttributeError:
+            # Per PEP-302, get_source() is optional,
+            e = sys.exc_info()[1]
+            LOG.debug('%r: %r.get_source() failed: %r', self, loader, fullname, e)
+            return
+
+        try:
+            is_pkg = loader.is_package(fullname)
+        except AttributeError:
+            # Per PEP-302, is_package() is optional,
+            e = sys.exc_info()[1]
+            LOG.debug('%r: %r.is_package(%r) failed: %r', self, loader, fullname, e)
             return
 
+        # workaround for special python modules that might only exist in memory
+        if is_special and is_pkg and not source:
+            source = '\n'
+
         if path is None or source is None:
+            LOG.debug('%r: path=%r, source=%r, aborting', self, path, source)
             return
 
         if isinstance(source, mitogen.core.UnicodeType):
@@ -567,23 +530,37 @@ class SysModulesMethod(FinderMethod):
         """
         Find `fullname` using its :data:`__file__` attribute.
         """
-        module = sys.modules.get(fullname)
+        try:
+            module = sys.modules[fullname]
+        except KeyError:
+            LOG.debug('%r: sys.modules[%r] absent, aborting', self, fullname)
+            return
+
         if not isinstance(module, types.ModuleType):
-            LOG.debug('%r: sys.modules[%r] absent or not a regular module',
+            LOG.debug('%r: sys.modules[%r] is %r, aborting',
-                      self, fullname)
+                      self, fullname, module)
+            return
+
+        try:
+            resolved_name = module.__name__
+        except AttributeError:
+            LOG.debug('%r: %r has no __name__, aborting', self, module)
             return
 
-        LOG.debug('_get_module_via_sys_modules(%r) -> %r', fullname, module)
+        if resolved_name != fullname:
-        alleged_name = getattr(module, '__name__', None)
+            LOG.debug('%r: %r.__name__ is %r, aborting',
-        if alleged_name != fullname:
+                      self, module, resolved_name)
-            LOG.debug('sys.modules[%r].__name__ is incorrect, assuming '
+            return
-                      'this is a hacky module alias and ignoring it. '
+
-                      'Got %r, module object: %r',
+        try:
-                      fullname, alleged_name, module)
+            path = module.__file__
+        except AttributeError:
+            LOG.debug('%r: %r has no __file__, aborting', self, module)
             return
 
-        path, _ = _py_filename(getattr(module, '__file__', ''))
+        path, _ = _py_filename(path)
         if not path:
+            LOG.debug('%r: %r.__file__ is %r, aborting', self, module, path)
             return
 
         LOG.debug('%r: sys.modules[%r]: found %s', self, fullname, path)
@@ -605,13 +582,13 @@ class SysModulesMethod(FinderMethod):
         return path, source, is_pkg
 
 
-class ParentEnumerationMethod(FinderMethod):
+class ParentImpEnumerationMethod(FinderMethod):
     """
     Attempt to fetch source code by examining the module's (hopefully less
     insane) parent package, and if no insane parents exist, simply use
     :mod:`sys.path` to search for it from scratch on the filesystem using the
     normal Python lookup mechanism.
-    
+
     This is required for older versions of :mod:`ansible.compat.six`,
     :mod:`plumbum.colors`, Ansible 2.8 :mod:`ansible.module_utils.distro` and
     its submodule :mod:`ansible.module_utils.distro._distro`.
@@ -628,10 +605,24 @@ class ParentEnumerationMethod(FinderMethod):
     module object or any parent package's :data:`__path__`, since they have all
     been overwritten. Some men just want to watch the world burn.
     """
+
+    @staticmethod
+    def _iter_parents(fullname):
+        """
+        >>> list(ParentEnumerationMethod._iter_parents('a'))
+        [('', 'a')]
+        >>> list(ParentEnumerationMethod._iter_parents('a.b.c'))
+        [('a.b', 'c'), ('a', 'b'), ('', 'a')]
+        """
+        while fullname:
+            fullname, _, modname = str_rpartition(fullname, u'.')
+            yield fullname, modname
+
     def _find_sane_parent(self, fullname):
         """
         Iteratively search :data:`sys.modules` for the least indirect parent of
-        `fullname` that is loaded and contains a :data:`__path__` attribute.
+        `fullname` that's from the same package and has a :data:`__path__`
+        attribute.
 
         :return:
             `(parent_name, path, modpath)` tuple, where:
@@ -644,21 +635,40 @@ class ParentEnumerationMethod(FinderMethod):
                 * `modpath`: list of module name components leading from `path`
                    to the target module.
         """
-        path = None
         modpath = []
-        while True:
+        for pkgname, modname in self._iter_parents(fullname):
-            pkgname, _, modname = str_rpartition(to_text(fullname), u'.')
             modpath.insert(0, modname)
             if not pkgname:
                 return [], None, modpath
 
-            pkg = sys.modules.get(pkgname)
+            try:
-            path = getattr(pkg, '__path__', None)
+                pkg = sys.modules[pkgname]
-            if pkg and path:
+            except KeyError:
-                return pkgname.split('.'), path, modpath
+                LOG.debug('%r: sys.modules[%r] absent, skipping', self, pkgname)
+                continue
+
+            try:
+                resolved_pkgname = pkg.__name__
+            except AttributeError:
+                LOG.debug('%r: %r has no __name__, skipping', self, pkg)
+                continue
+
+            if resolved_pkgname != pkgname:
+                LOG.debug('%r: %r.__name__ is %r, skipping',
+                          self, pkg, resolved_pkgname)
+                continue
+
+            try:
+                path = pkg.__path__
+            except AttributeError:
+                LOG.debug('%r: %r has no __path__, skipping', self, pkg)
+                continue
 
-            LOG.debug('%r: %r lacks __path__ attribute', self, pkgname)
+            if not path:
-            fullname = pkgname
+                LOG.debug('%r: %r.__path__ is %r, skipping', self, pkg, path)
+                continue
+
+            return pkgname.split('.'), path, modpath
 
     def _found_package(self, fullname, path):
         path = os.path.join(path, '__init__.py')
@@ -690,6 +700,7 @@ class ParentEnumerationMethod(FinderMethod):
     def _find_one_component(self, modname, search_path):
         try:
             #fp, path, (suffix, _, kind) = imp.find_module(modname, search_path)
+            # FIXME The imp module was removed in Python 3.12.
             return imp.find_module(modname, search_path)
         except ImportError:
             e = sys.exc_info()[1]
@@ -701,6 +712,9 @@ class ParentEnumerationMethod(FinderMethod):
         """
         See implementation for a description of how this works.
         """
+        if sys.version_info >= (3, 4):
+            return None
+
         #if fullname not in sys.modules:
             # Don't attempt this unless a module really exists in sys.modules,
             # else we could return junk.
@@ -729,12 +743,111 @@ class ParentEnumerationMethod(FinderMethod):
                 return self._found_module(fullname, path, fp)
 
 
+class ParentSpecEnumerationMethod(ParentImpEnumerationMethod):
+    def _find_parent_spec(self, fullname):
+        #history = []
+        debug = self.log.debug
+        children = []
+        for parent_name, child_name in self._iter_parents(fullname):
+            children.insert(0, child_name)
+            if not parent_name:
+                debug('abandoning %r, reached top-level', fullname)
+                return None, children
+
+            try:
+                parent = sys.modules[parent_name]
+            except KeyError:
+                debug('skipping %r, not in sys.modules', parent_name)
+                continue
+
+            try:
+                spec = parent.__spec__
+            except AttributeError:
+                debug('skipping %r: %r.__spec__ is absent',
+                      parent_name, parent)
+                continue
+
+            if not spec:
+                debug('skipping %r: %r.__spec__=%r',
+                      parent_name, parent, spec)
+                continue
+
+            if spec.name != parent_name:
+                debug('skipping %r: %r.__spec__.name=%r does not match',
+                      parent_name, parent, spec.name)
+                continue
+
+            if not spec.submodule_search_locations:
+                debug('skipping %r: %r.__spec__.submodule_search_locations=%r',
+                      parent_name, parent, spec.submodule_search_locations)
+                continue
+
+            return spec, children
+
+        raise ValueError('%s._find_parent_spec(%r) unexpectedly reached bottom'
+                         % (self.__class__.__name__, fullname))
+
+    def find(self, fullname):
+        # Returns absolute path, ParentImpEnumerationMethod returns relative
+        # >>> spec_pem.find('six_brokenpkg._six')[::2]
+        # ('/Users/alex/src/mitogen/tests/data/importer/six_brokenpkg/_six.py', False)
+
+        if sys.version_info < (3, 4):
+            return None
+
+        fullname = to_text(fullname)
+        spec, children = self._find_parent_spec(fullname)
+        for child_name in children:
+            if spec:
+                name = '%s.%s' % (spec.name, child_name)
+                submodule_search_locations = spec.submodule_search_locations
+            else:
+                name = child_name
+                submodule_search_locations = None
+            spec = importlib.util._find_spec(name, submodule_search_locations)
+            if spec is None:
+                self.log.debug('%r spec unavailable from %s', fullname, spec)
+                return None
+
+            is_package = spec.submodule_search_locations is not None
+            if name != fullname:
+                if not is_package:
+                    self.log.debug('%r appears to be child of non-package %r',
+                                   fullname, spec)
+                    return None
+                continue
+
+            if not spec.has_location:
+                self.log.debug('%r.origin cannot be read as a file', spec)
+                return None
+
+            if os.path.splitext(spec.origin)[1] != '.py':
+                self.log.debug('%r.origin does not contain Python source code',
+                               spec)
+                return None
+
+            # FIXME This should use loader.get_source()
+            with open(spec.origin, 'rb') as f:
+                source = f.read()
+
+            return spec.origin, source, is_package
+
+        raise ValueError('%s.find(%r) unexpectedly reached bottom'
+                         % (self.__class__.__name__, fullname))
+
+
 class ModuleFinder(object):
     """
     Given the name of a loaded module, make a best-effort attempt at finding
     related modules likely needed by a child context requesting the original
     module.
     """
+
+    # Fullnames of modules that should not be sent as a related module
+    _related_modules_denylist = frozenset({
+        '__main__',
+    })
+
     def __init__(self):
         #: Import machinery is expensive, keep :py:meth`:get_module_source`
         #: results around.
@@ -769,7 +882,8 @@ class ModuleFinder(object):
         DefectivePython3xMainMethod(),
         PkgutilMethod(),
         SysModulesMethod(),
-        ParentEnumerationMethod(),
+        ParentSpecEnumerationMethod(),
+        ParentImpEnumerationMethod(),
     ]
 
     def get_module_source(self, fullname):
@@ -822,6 +936,34 @@ class ModuleFinder(object):
             fullname, _, _ = str_rpartition(to_text(fullname), u'.')
             yield fullname
 
+    def _reject_related_module(self, requested_fullname, related_fullname):
+        def _log_reject(reason):
+            LOG.debug(
+                '%r: Rejected related module %s of requested module %s: %s',
+                self, related_fullname, requested_fullname, reason,
+            )
+            return reason
+
+        try:
+            related_module = sys.modules[related_fullname]
+        except KeyError:
+            return _log_reject('sys.modules entry absent')
+
+        # Python 2.x "indirection entry"
+        if related_module is None:
+            return _log_reject('sys.modules entry is None')
+
+        if is_stdlib_name(related_fullname):
+            return _log_reject('stdlib module')
+
+        if 'six.moves' in related_fullname:
+            return _log_reject('six.moves avoidence')
+
+        if related_fullname in self._related_modules_denylist:
+            return _log_reject('on denylist')
+
+        return False
+
     def find_related_imports(self, fullname):
         """
         Return a list of non-stdlib modules that are directly imported by
@@ -845,7 +987,7 @@ class ModuleFinder(object):
         maybe_names = list(self.generate_parent_names(fullname))
 
         co = compile(src, modpath, 'exec')
-        for level, modname, namelist in scan_code_imports(co):
+        for level, modname, namelist in mitogen.imports.codeobj_imports(co):
             if level == -1:
                 modnames = [modname, '%s.%s' % (fullname, modname)]
             else:
@@ -864,9 +1006,7 @@ class ModuleFinder(object):
             set(
                 mitogen.core.to_text(name)
                 for name in maybe_names
-                if sys.modules.get(name) is not None
+                if not self._reject_related_module(fullname, name)
-                and not is_stdlib_name(name)
-                and u'six.moves' not in name  # TODO: crap
             )
         ))
 
@@ -1029,7 +1169,7 @@ class ModuleResponder(object):
         self._cache[fullname] = tup
         return tup
 
-    def _send_load_module(self, stream, fullname):
+    def _send_load_module(self, stream, fullname, reason):
         if fullname not in stream.protocol.sent_modules:
             tup = self._build_tuple(fullname)
             msg = mitogen.core.Message.pickled(
@@ -1037,8 +1177,10 @@ class ModuleResponder(object):
                 dst_id=stream.protocol.remote_id,
                 handle=mitogen.core.LOAD_MODULE,
             )
-            self._log.debug('sending %s (%.2f KiB) to %s',
+            self._log.debug(
-                            fullname, len(msg.data) / 1024.0, stream.name)
+                'sending %s %s (%.2f KiB) to %s',
+                reason, fullname, len(msg.data) / 1024.0, stream.name,
+            )
             self._router._async_route(msg)
             stream.protocol.sent_modules.add(fullname)
             if tup[2] is not None:
@@ -1069,8 +1211,8 @@ class ModuleResponder(object):
                     # Parent hasn't been sent, so don't load submodule yet.
                     continue
 
-                self._send_load_module(stream, name)
+                self._send_load_module(stream, name, 'related')
-            self._send_load_module(stream, fullname)
+            self._send_load_module(stream, fullname, 'requested')
         except Exception:
             LOG.debug('While importing %r', fullname, exc_info=True)
             self._send_module_load_failed(stream, fullname)
@@ -1139,6 +1281,47 @@ class ModuleResponder(object):
         self._router.broker.defer(self._forward_modules, context, fullnames)
 
 
+class ResourceResponder(object):
+    def __init__(self, router):
+        self._router = router
+        self._router.add_handler(
+            self._on_get_resource,
+            mitogen.core.GET_RESOURCE,
+        )
+
+    def _on_get_resource(self, msg):
+        if msg.is_dead:
+            return
+        stream = self._router.stream_by_id(msg.src_id)
+        if stream is None:
+            return
+        fullname, resource = msg.unpickle()
+        try:
+            content = importlib.resources.read_binary(fullname, resource)
+        except (FileNotFoundError, IsADirectoryError):
+            content = None
+        if content is not None:
+            self._send_resource(stream, fullname, resource, content)
+        else:
+            self._send_not_found(stream, fullname, resource)
+
+    def _send_resource(self, stream, fullname, resource, content):
+        msg = mitogen.core.Message.pickled(
+            (fullname, resource, content),
+            dst_id=stream.protocol.remote_id,
+            handle=mitogen.core.LOAD_RESOURCE,
+        )
+        self._router._async_route(msg)
+
+    def _send_not_found(self, stream, fullname, resource):
+        msg = mitogen.core.Message.pickled(
+            (fullname, resource, None),
+            dst_id=stream.protocol.remote_id,
+            handle=mitogen.core.LOAD_RESOURCE,
+        )
+        stream.protocol.send(msg)
+
+
 class Broker(mitogen.core.Broker):
     """
     .. note::
@@ -1167,7 +1350,7 @@ class Broker(mitogen.core.Broker):
     def __init__(self, install_watcher=True):
         if install_watcher:
             self._watcher = ThreadWatcher.watch(
-                target=threading.currentThread(),
+                target=mitogen.core.threading__current_thread(),
                 on_join=self.shutdown,
             )
         super(Broker, self).__init__()
@@ -1229,6 +1412,7 @@ class Router(mitogen.parent.Router):
     def upgrade(self):
         self.id_allocator = IdAllocator(self)
         self.responder = ModuleResponder(self)
+        self.resource_responder = ResourceResponder(self)
         self.log_forwarder = LogForwarder(self)
         self.route_monitor = mitogen.parent.RouteMonitor(router=self)
         self.add_handler(  # TODO: cutpaste.

mitogen/minify.py

@@ -104,7 +104,7 @@ def strip_docstrings(tokens):
             elif typ == tokenize.NEWLINE:
                 stack.append(t)
                 start_line, end_line = stack[0][2][0], stack[-1][3][0]+1
-                for i in range(start_line, end_line):
+                for i in mitogen.core.range(start_line, end_line):
                     yield tokenize.NL, '\n', (i, 0), (i,1), '\n'
                 for t in stack:
                     if t[0] in (tokenize.DEDENT, tokenize.INDENT):

mitogen/os_fork.py

@@ -35,10 +35,10 @@ Support for operating in a mixed threading/forking environment.
 import os
 import socket
 import sys
-import threading
 import weakref
 
 import mitogen.core
+import mitogen.parent
 
 
 # List of weakrefs. On Python 2.4, mitogen.core registers its Broker on this
@@ -132,9 +132,9 @@ class Corker(object):
         `obj` to be written to by one of its threads.
         """
         rsock, wsock = mitogen.parent.create_socketpair(size=4096)
+        mitogen.core.set_blocking(wsock.fileno(), True)  # gevent
         mitogen.core.set_cloexec(rsock.fileno())
         mitogen.core.set_cloexec(wsock.fileno())
-        mitogen.core.set_block(wsock)  # gevent
         self._rsocks.append(rsock)
         obj.defer(self._do_cork, s, wsock)
 
@@ -158,7 +158,7 @@ class Corker(object):
         held. This will not return until each thread acknowledges it has ceased
         execution.
         """
-        current = threading.currentThread()
+        current = mitogen.core.threading__current_thread()
         s = mitogen.core.b('CORK') * ((128 // 4) * 1024)
         self._rsocks = []
 

mitogen/parent.py

@@ -34,7 +34,7 @@ sent to any child context that is due to become a parent, due to recursive
 connection.
 """
 
-import codecs
+import binascii
 import errno
 import fcntl
 import getpass
@@ -43,6 +43,7 @@ import inspect
 import logging
 import os
 import re
+import pty
 import signal
 import socket
 import struct
@@ -56,15 +57,13 @@ import zlib
 # Absolute imports for <2.5.
 select = __import__('select')
 
-try:
-    import thread
-except ImportError:
-    import threading as thread
-
 import mitogen.core
 from mitogen.core import b
 from mitogen.core import bytes_partition
 from mitogen.core import IOLOG
+from mitogen.core import itervalues
+from mitogen.core import next
+from mitogen.core import thread
 
 
 LOG = logging.getLogger(__name__)
@@ -80,17 +79,7 @@ except IOError:
     SELINUX_ENABLED = False
 
 
-try:
+if sys.version_info >= (3, 0):
-    next
-except NameError:
-    # Python 2.4/2.5
-    from mitogen.core import next
-
-
-itervalues = getattr(dict, 'itervalues', dict.values)
-
-if mitogen.core.PY3:
-    xrange = range
     closure_attr = '__closure__'
     IM_SELF_ATTR = '__self__'
 else:
@@ -147,6 +136,8 @@ LINUX_TIOCGPTN = _ioctl_cast(2147767344)
 LINUX_TIOCSPTLCK = _ioctl_cast(1074025521)
 
 IS_LINUX = os.uname()[0] == 'Linux'
+IS_SOLARIS = os.uname()[0] == 'SunOS'
+
 
 SIGNAL_BY_NUM = dict(
     (getattr(signal, name), name)
@@ -233,8 +224,16 @@ def flags(names):
     Return the result of ORing a set of (space separated) :py:mod:`termios`
     module constants together.
     """
-    return sum(getattr(termios, name, 0)
+    i = 0
-               for name in names.split())
+    skipped = []
+    for name in names.split():
+        try:
+            i |= getattr(termios, name)
+        except AttributeError:
+            skipped.append(name)
+    if skipped:
+        LOG.debug('Skipped termios attributes: %s', ', '.join(skipped))
+    return i
 
 
 def cfmakeraw(tflags):
@@ -243,12 +242,9 @@ def cfmakeraw(tflags):
     modified in a manner similar to the `cfmakeraw()` C library function, but
     additionally disabling local echo.
     """
-    # BSD: github.com/freebsd/freebsd/blob/master/lib/libc/gen/termios.c#L162
-    # Linux: github.com/lattera/glibc/blob/master/termios/cfmakeraw.c#L20
     iflag, oflag, cflag, lflag, ispeed, ospeed, cc = tflags
     iflag &= ~flags('IMAXBEL IXOFF INPCK BRKINT PARMRK '
-                    'ISTRIP INLCR ICRNL IXON IGNPAR')
+                    'ISTRIP INLCR ICRNL IXON IGNPAR IGNBRK')
-    iflag &= ~flags('IGNBRK BRKINT PARMRK')
     oflag &= ~flags('OPOST')
     lflag &= ~flags('ECHO ECHOE ECHOK ECHONL ICANON ISIG '
                     'IEXTEN NOFLSH TOSTOP PENDIN')
@@ -268,7 +264,7 @@ def disable_echo(fd):
     termios.tcsetattr(fd, flags, new)
 
 
-def create_socketpair(size=None):
+def create_socketpair(size=None, blocking=None):
     """
     Create a :func:`socket.socketpair` for use as a child's UNIX stdio
     channels. As socketpairs are bidirectional, they are economical on file
@@ -279,14 +275,14 @@ def create_socketpair(size=None):
     if size is None:
         size = mitogen.core.CHUNK_SIZE
 
-    parentfp, childfp = socket.socketpair()
+    parentfp, childfp = mitogen.core.socketpair(blocking)
     for fp in parentfp, childfp:
         fp.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, size)
 
     return parentfp, childfp
 
 
-def create_best_pipe(escalates_privilege=False):
+def create_best_pipe(escalates_privilege=False, blocking=None):
     """
     By default we prefer to communicate with children over a UNIX socket, as a
     single file descriptor can represent bidirectional communication, and a
@@ -304,16 +300,19 @@ def create_best_pipe(escalates_privilege=False):
     :param bool escalates_privilege:
         If :data:`True`, the target program may escalate privileges, causing
         SELinux to disconnect AF_UNIX sockets, so avoid those.
+    :param None|bool blocking:
+        If :data:`False` or :data:`True`, set non-blocking or blocking mode.
+        If :data:`None` (default), use default.
     :returns:
         `(parent_rfp, child_wfp, child_rfp, parent_wfp)`
     """
     if (not escalates_privilege) or (not SELINUX_ENABLED):
-        parentfp, childfp = create_socketpair()
+        parentfp, childfp = create_socketpair(blocking=blocking)
         return parentfp, childfp, childfp, parentfp
 
-    parent_rfp, child_wfp = mitogen.core.pipe()
+    parent_rfp, child_wfp = mitogen.core.pipe(blocking)
     try:
-        child_rfp, parent_wfp = mitogen.core.pipe()
+        child_rfp, parent_wfp = mitogen.core.pipe(blocking)
         return parent_rfp, child_wfp, child_rfp, parent_wfp
     except:
         parent_rfp.close()
@@ -365,13 +364,13 @@ def create_child(args, merge_stdio=False, stderr_pipe=False,
         escalates_privilege=escalates_privilege
     )
 
-    stderr = None
-    stderr_r = None
     if merge_stdio:
-        stderr = child_wfp
+        stderr_r, stderr = None, child_wfp
     elif stderr_pipe:
         stderr_r, stderr = mitogen.core.pipe()
         mitogen.core.set_cloexec(stderr_r.fileno())
+    else:
+        stderr_r, stderr = None, None
 
     try:
         proc = popen(
@@ -410,12 +409,14 @@ def _acquire_controlling_tty():
     if sys.platform in ('linux', 'linux2'):
         # On Linux, the controlling tty becomes the first tty opened by a
         # process lacking any prior tty.
-        os.close(os.open(os.ttyname(2), os.O_RDWR))
+        tty_path = os.ttyname(pty.STDERR_FILENO)
-    if hasattr(termios, 'TIOCSCTTY') and not mitogen.core.IS_WSL:
+        tty_fd = os.open(tty_path, os.O_RDWR)
+        os.close(tty_fd)
+    if hasattr(termios, 'TIOCSCTTY') and not mitogen.core.IS_WSL and not IS_SOLARIS:
         # #550: prehistoric WSL does not like TIOCSCTTY.
         # On BSD an explicit ioctl is required. For some inexplicable reason,
         # Python 2.6 on Travis also requires it.
-        fcntl.ioctl(2, termios.TIOCSCTTY)
+        fcntl.ioctl(pty.STDERR_FILENO, termios.TIOCSCTTY)
 
 
 def _linux_broken_devpts_openpty():
@@ -479,9 +480,10 @@ def openpty():
 
     master_fp = os.fdopen(master_fd, 'r+b', 0)
     slave_fp = os.fdopen(slave_fd, 'r+b', 0)
-    disable_echo(master_fd)
+    if not IS_SOLARIS:
+        disable_echo(master_fd)
     disable_echo(slave_fd)
-    mitogen.core.set_block(slave_fd)
+    mitogen.core.set_blocking(slave_fd, True)
     return master_fp, slave_fp
 
 
@@ -547,8 +549,8 @@ def hybrid_tty_create_child(args, escalates_privilege=False):
             escalates_privilege=escalates_privilege,
         )
         try:
-            mitogen.core.set_block(child_rfp)
+            mitogen.core.set_blocking(child_rfp.fileno(), True)
-            mitogen.core.set_block(child_wfp)
+            mitogen.core.set_blocking(child_wfp.fileno(), True)
             proc = popen(
                 args=args,
                 stdin=child_rfp,
@@ -639,7 +641,7 @@ class TimerList(object):
     def get_timeout(self):
         """
         Return the floating point seconds until the next event is due.
-        
+
         :returns:
             Floating point delay, or 0.0, or :data:`None` if no events are
             scheduled.
@@ -745,8 +747,7 @@ def _upgrade_broker(broker):
     broker.timers = TimerList()
     LOG.debug('upgraded %r with %r (new: %d readers, %d writers; '
               'old: %d readers, %d writers)', old, new,
-              len(new.readers), len(new.writers),
+              len(new._rfds), len(new._wfds), len(old._rfds), len(old._wfds))
-              len(old.readers), len(old.writers))
 
 
 @mitogen.core.takes_econtext
@@ -902,22 +903,18 @@ class CallSpec(object):
 class PollPoller(mitogen.core.Poller):
     """
     Poller based on the POSIX :linux:man2:`poll` interface. Not available on
-    some versions of OS X, otherwise it is the preferred poller for small FD
+    some Python/OS X combinations. Otherwise the preferred poller for small
-    counts, as there is no setup/teardown/configuration system call overhead.
+    FD counts; or if many pollers are created, used once, then closed.
+    There there is no setup/teardown/configuration system call overhead.
     """
     SUPPORTED = hasattr(select, 'poll')
-    _repr = 'PollPoller()'
+    _readmask = SUPPORTED and select.POLLIN | select.POLLHUP
 
     def __init__(self):
         super(PollPoller, self).__init__()
         self._pollobj = select.poll()
 
     # TODO: no proof we dont need writemask too
-    _readmask = (
-        getattr(select, 'POLLIN', 0) |
-        getattr(select, 'POLLHUP', 0)
-    )
-
     def _update(self, fd):
         mask = (((fd in self._rfds) and self._readmask) |
                 ((fd in self._wfds) and select.POLLOUT))
@@ -952,7 +949,6 @@ class KqueuePoller(mitogen.core.Poller):
     Poller based on the FreeBSD/Darwin :freebsd:man2:`kqueue` interface.
     """
     SUPPORTED = hasattr(select, 'kqueue')
-    _repr = 'KqueuePoller()'
 
     def __init__(self):
         super(KqueuePoller, self).__init__()
@@ -1027,10 +1023,10 @@ class KqueuePoller(mitogen.core.Poller):
 
 class EpollPoller(mitogen.core.Poller):
     """
-    Poller based on the Linux :linux:man2:`epoll` interface.
+    Poller based on the Linux :linux:man7:`epoll` interface.
     """
     SUPPORTED = hasattr(select, 'epoll')
-    _repr = 'EpollPoller()'
+    _inmask = SUPPORTED and select.EPOLLIN | select.EPOLLHUP
 
     def __init__(self):
         super(EpollPoller, self).__init__()
@@ -1077,9 +1073,6 @@ class EpollPoller(mitogen.core.Poller):
         self._wfds.pop(fd, None)
         self._control(fd)
 
-    _inmask = (getattr(select, 'EPOLLIN', 0) |
-               getattr(select, 'EPOLLHUP', 0))
-
     def _poll(self, timeout):
         the_timeout = -1
         if timeout is not None:
@@ -1100,18 +1093,14 @@ class EpollPoller(mitogen.core.Poller):
                     yield data
 
 
-# 2.4 and 2.5 only had select.select() and select.poll().
+POLLERS = (EpollPoller, KqueuePoller, PollPoller, mitogen.core.Poller)
-for _klass in mitogen.core.Poller, PollPoller, KqueuePoller, EpollPoller:
+PREFERRED_POLLER = next(cls for cls in POLLERS if cls.SUPPORTED)
-    if _klass.SUPPORTED:
+
-        PREFERRED_POLLER = _klass
 
 # For processes that start many threads or connections, it's possible Latch
 # will also get high-numbered FDs, and so select() becomes useless there too.
-# So swap in our favourite poller.
+POLLER_LIGHTWEIGHT = PollPoller.SUPPORTED and PollPoller or PREFERRED_POLLER
-if PollPoller.SUPPORTED:
+mitogen.core.Latch.poller_class = POLLER_LIGHTWEIGHT
-    mitogen.core.Latch.poller_class = PollPoller
-else:
-    mitogen.core.Latch.poller_class = PREFERRED_POLLER
 
 
 class LineLoggingProtocolMixin(object):
@@ -1405,10 +1394,10 @@ class Connection(object):
     # file descriptor 0 as 100, creates a pipe, then execs a new interpreter
     # with a custom argv.
     #   * Optimized for minimum byte count after minification & compression.
-    #   * 'CONTEXT_NAME' and 'PREAMBLE_COMPRESSED_LEN' are substituted with
+    #     The script preamble_size.py measures this.
-    #     their respective values.
+    #
-    #   * CONTEXT_NAME must be prefixed with the name of the Python binary in
+    # macOS tweaks for Python 2.7 must be kept in sync with the the Ansible
-    #     order to allow virtualenvs to detect their install prefix.
+    # module test_echo_module, used by the integration tests.
     #   * macOS <= 10.14 (Darwin <= 18) install an unreliable Python version
     #     switcher as /usr/bin/python, which introspects argv0. To workaround
     #     it we redirect attempts to call /usr/bin/python with an explicit
@@ -1417,16 +1406,16 @@ class Connection(object):
     #     do something slightly different. The Python executable is patched to
     #     perform an extra execvp(). I don't fully understand the details, but
     #     setting PYTHON_LAUNCHED_FROM_WRAPPER=1 avoids it.
-    #   * macOS 13.x (Darwin 22?) may remove python 2.x entirely.
+    #   * macOS 12.3+ (Darwin 21.4+, Monterey) doesn't ship Python.
+    #     https://developer.apple.com/documentation/macos-release-notes/macos-12_3-release-notes#Python
     #
     # Locals:
     #   R: read side of interpreter stdin.
     #   W: write side of interpreter stdin.
     #   r: read side of core_src FD.
     #   w: write side of core_src FD.
-    #   C: the decompressed core source.
 
-    # Final os.close(2) to avoid --py-debug build from corrupting stream with
+    # Final os.close(STDERR_FILENO) to avoid --py-debug build corrupting stream with
     # "[1234 refs]" during exit.
     @staticmethod
     def _first_stage():
@@ -1440,18 +1429,30 @@ class Connection(object):
             os.close(r)
             os.close(W)
             os.close(w)
-            if os.uname()[0]=='Darwin'and os.uname()[2][:2]<'19'and sys.executable=='/usr/bin/python':sys.executable='/usr/bin/python2.7'
+            if os.uname()[0]+os.uname()[2][:2]+sys.executable=='Darwin19/usr/bin/python':sys.executable+='2.7'
-            if os.uname()[0]=='Darwin'and os.uname()[2][:2]in'2021'and sys.version[:3]=='2.7':os.environ['PYTHON_LAUNCHED_FROM_WRAPPER']='1'
+            if os.uname()[0]+os.uname()[2][:2]+sys.version[:3]=='Darwin202.7':os.environ['PYTHON_LAUNCHED_FROM_WRAPPER']='1'
+            if os.uname()[0]+os.uname()[2][:2]+sys.version[:3]=='Darwin212.7':os.environ['PYTHON_LAUNCHED_FROM_WRAPPER']='1'
             os.environ['ARGV0']=sys.executable
-            os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)')
+            os.execl(sys.executable,sys.executable+'(mitogen:%s)'%sys.argv[2])
         os.write(1,'MITO000\n'.encode())
-        C=_(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN),'zip')
+        # Size of the compressed core source to be read
-        fp=os.fdopen(W,'wb',0)
+        n=int(sys.argv[3])
-        fp.write(C)
+        # Read `len(compressed preamble)` bytes sent by our Mitogen parent.
-        fp.close()
+        # `select()` handles non-blocking stdin (e.g. sudo + log_output).
-        fp=os.fdopen(w,'wb',0)
+        # `C` accumulates compressed bytes.
-        fp.write(C)
+        C=''.encode()
-        fp.close()
+        # data chunk
+        V='V'
+        # Stop looping if no more data is needed or EOF is detected (empty bytes).
+        while n-len(C) and V:select.select([0],[],[]);V=os.read(0,n-len(C));C+=V
+        # Raises `zlib.error` if compressed preamble is truncated or invalid
+        C=zlib.decompress(C)
+        f=os.fdopen(W,'wb',0)
+        f.write(C)
+        f.close()
+        f=os.fdopen(w,'wb',0)
+        f.write(C)
+        f.close()
         os.write(1,'MITO001\n'.encode())
         os.close(2)
 
@@ -1469,24 +1470,28 @@ class Connection(object):
         return [self.options.python_path]
 
     def get_boot_command(self):
-        source = inspect.getsource(self._first_stage)
+        lines = inspect.getsourcelines(self._first_stage)[0][2:]
-        source = textwrap.dedent('\n'.join(source.strip().split('\n')[2:]))
+        # Remove line comments, leading indentation, trailing newline
+        source = textwrap.dedent(''.join(s for s in lines if '#' not in s))[:-1]
         source = source.replace('    ', ' ')
-        source = source.replace('CONTEXT_NAME', self.options.remote_name)
+        compressor = zlib.compressobj(
-        preamble_compressed = self.get_preamble()
+            zlib.Z_BEST_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS,
-        source = source.replace('PREAMBLE_COMPRESSED_LEN',
+        )
-                                str(len(preamble_compressed)))
+        compressed = compressor.compress(source.encode()) + compressor.flush()
-        compressed = zlib.compress(source.encode(), 9)
+        encoded = binascii.b2a_base64(compressed).replace(b('\n'), b(''))
-        encoded = codecs.encode(compressed, 'base64').replace(b('\n'), b(''))
+
-        # We can't use bytes.decode() in 3.x since it was restricted to always
+        # Just enough to decode, decompress, and exec the first stage.
-        # return unicode, so codecs.decode() is used instead. In 3.x
+        # Priorities: wider compatibility, faster startup, shorter length.
-        # codecs.decode() requires a bytes object. Since we must be compatible
+        # `sys.path=...` for https://github.com/python/cpython/issues/115911.
-        # with 2.4 (no bytes literal), an extra .encode() either returns the
+        # `import os,select` here (not stage 1) to save a few bytes overall.
-        # same str (2.x) or an equivalent bytes (3.x).
         return self.get_python_argv() + [
             '-c',
-            'import codecs,os,sys;_=codecs.decode;'
+            'import sys;sys.path=[p for p in sys.path if p];'
-            'exec(_(_("%s".encode(),"base64"),"zip"))' % (encoded.decode(),)
+            'import binascii,os,select,zlib;'
+            'exec(zlib.decompress(binascii.a2b_base64(sys.argv[1]),-15))',
+            encoded.decode(),
+            self.options.remote_name,
+            str(len(self.get_preamble())),
         ]
 
     def get_econtext_config(self):
@@ -1508,7 +1513,7 @@ class Connection(object):
 
     def get_preamble(self):
         suffix = (
-            '\nExternalContext(%r).main()\n' %\
+            '\nExternalContext(%r).main()\n' %
             (self.get_econtext_config(),)
         )
         partial = get_core_source_partial()
@@ -1649,6 +1654,9 @@ class Connection(object):
         stream = self.stream_factory()
         stream.conn = self
         stream.name = self.options.name or self._get_name()
+        for fp in self.proc.stdout, self.proc.stdin:
+            fd = fp.fileno()
+            mitogen.core.set_blocking(fd, False)
         stream.accept(self.proc.stdout, self.proc.stdin)
 
         mitogen.core.listen(stream, 'disconnect', self.on_stdio_disconnect)
@@ -1659,6 +1667,8 @@ class Connection(object):
         stream = self.stderr_stream_factory()
         stream.conn = self
         stream.name = self.options.name or self._get_name()
+        fd = self.proc.stderr.fileno()
+        mitogen.core.set_blocking(fd, False)
         stream.accept(self.proc.stderr, self.proc.stderr)
 
         mitogen.core.listen(stream, 'disconnect', self.on_stderr_disconnect)
@@ -1692,9 +1702,7 @@ class Connection(object):
 
         LOG.debug('child for %r started: pid:%r stdin:%r stdout:%r stderr:%r',
                   self, self.proc.pid,
-                  self.proc.stdin.fileno(),
+                  self.proc.stdin, self.proc.stdout, self.proc.stderr)
-                  self.proc.stdout.fileno(),
-                  self.proc.stderr and self.proc.stderr.fileno())
 
         self.stdio_stream = self._setup_stdio_stream()
         if self.context.name is None:
@@ -1720,7 +1728,7 @@ class ChildIdAllocator(object):
     def __init__(self, router):
         self.router = router
         self.lock = threading.Lock()
-        self.it = iter(xrange(0))
+        self.it = iter(mitogen.core.range(0))
 
     def allocate(self):
         """
@@ -1744,7 +1752,7 @@ class ChildIdAllocator(object):
             start, end = master.send_await(
                 mitogen.core.Message(dst_id=0, handle=mitogen.core.ALLOCATE_ID)
             )
-            self.it = iter(xrange(start, end))
+            self.it = iter(mitogen.core.range(start, end))
         finally:
             self.lock.release()
 
@@ -2305,6 +2313,11 @@ class Router(mitogen.core.Router):
             parent_context=parent,
             importer=importer,
         )
+        self.resource_responder = ResourceForwarder(
+            self,
+            parent,
+            importer._resource_requester,
+        )
         self.route_monitor = RouteMonitor(self, parent)
         self.add_handler(
             fn=self._on_detaching,
@@ -2550,7 +2563,7 @@ class Reaper(object):
         # because it is setuid, so this is best-effort only.
         LOG.debug('%r: sending %s', self.proc, SIGNAL_BY_NUM[signum])
         try:
-            os.kill(self.proc.pid, signum)
+            self.proc.send_signal(signum)
         except OSError:
             e = sys.exc_info()[1]
             if e.args[0] != errno.EPERM:
@@ -2563,9 +2576,8 @@ class Reaper(object):
         relatively conservative retries.
         """
         delay = 0.05
-        for _ in xrange(count):
+        factor = 1.72
-            delay *= 1.72
+        return delay * factor ** count
-        return delay
 
     def _on_broker_shutdown(self):
         """
@@ -2670,6 +2682,17 @@ class Process(object):
         """
         raise NotImplementedError()
 
+    def send_signal(self, sig):
+        os.kill(self.pid, sig)
+
+    def terminate(self):
+        "Ask the process to gracefully shutdown."
+        self.send_signal(signal.SIGTERM)
+
+    def kill(self):
+        "Ask the operating system to forcefully destroy the process."
+        self.send_signal(signal.SIGKILL)
+
 
 class PopenProcess(Process):
     """
@@ -2686,6 +2709,9 @@ class PopenProcess(Process):
     def poll(self):
         return self.proc.poll()
 
+    def send_signal(self, sig):
+        self.proc.send_signal(sig)
+
 
 class ModuleForwarder(object):
     """
@@ -2777,3 +2803,43 @@ class ModuleForwarder(object):
                     handle=mitogen.core.LOAD_MODULE,
                 )
             )
+
+
+class ResourceForwarder(object):
+    """
+    Handle :data:`mitogen.core.GET_RESOURCE` requests from children by
+    forwarding the request to our parent, or satisfying the request from
+    our local :class:`mitogen.core.ResourceRequester` cache.
+    """
+    def __init__(self, router, parent_context, requester):
+        self.router = router
+        self.parent_context = parent_context
+        self.requester = requester
+        router.add_handler(
+            fn=self._on_get_resource,
+            handle=mitogen.core.GET_RESOURCE,
+            persist=True,
+            policy=is_immediate_child,
+        )
+
+    def _on_get_resource(self, msg):
+        if msg.is_dead:
+            return
+
+        fullname, resource = msg.unpickle()
+        callback = lambda: self._on_cache_callback(msg, fullname, resource)
+        self.requester._request_resource(fullname, resource, callback)
+
+    def _on_cache_callback(self, msg, fullname, resource):
+        stream = self.router.stream_by_id(msg.src_id)
+        self._send_resource(stream, fullname, resource)
+
+    def _send_resource(self, stream, fullname, resource):
+        content = self.requester._cache[(fullname, resource)]
+
+        msg = mitogen.core.Message.pickled(
+            (fullname, resource, content),
+            dst_id=stream.protocol.remote_id,
+            handle=mitogen.core.LOAD_RESOURCE,
+        )
+        self.router._async_route(msg)

mitogen/podman.py

@@ -31,7 +31,6 @@
 
 import logging
 
-import mitogen.core
 import mitogen.parent
 
 

mitogen/profiler.py

@@ -90,7 +90,7 @@ def merge_stats(outpath, inpaths):
                     break
                 time.sleep(0.2)
 
-    pstats.dump_stats(outpath)
+    stats.dump_stats(outpath)
 
 
 def generate_stats(outpath, tmpdir):