.ci/README.md

@@ -0,0 +1,42 @@
+
+# `.ci`
+
+This directory contains scripts for Continuous Integration platforms. Currently
+GitHub Actions, but ideally they will also run on any Debian-like machine.
+
+The scripts are usually split into `_install` and `_test` steps. The `_install`
+step will damage your machine, the `_test` step will just run the tests the way
+CI runs them.
+
+There is a common library, `ci_lib.py`, which just centralized a bunch of
+random macros and also environment parsing.
+
+Some of the scripts allow you to pass extra flags through to the component
+under test, e.g. `../../.ci/ansible_tests.py -vvv` will run with verbose.
+
+Hack these scripts until your heart is content. There is no pride to be found
+here, just necessity.
+
+
+### `ci_lib.run_batches()`
+
+There are some weird looking functions to extract more paralellism from the
+build. The above function takes lists of strings, arranging for the strings in
+each list to run in order, but for the lists to run in parallel. That's great
+for doing `setup.py install` while pulling a Docker container, for example.
+
+
+### Environment Variables
+
+* `MITOGEN_TEST_DISTRO_SPECS`: a space delimited list of distro specs to run
+  the tests against. (e.g. `centos6 ubuntu2004-py3*4`). Each spec determines
+  the Linux distribution, target Python interepreter & number of instances.
+  Only distributions with a pre-built Linux container image can be used.
+    * `debian-py3`: when generating Ansible inventory file, set
+      `ansible_python_interpreter` to `python3`, i.e. run a test where the
+      target interpreter is Python 3.
+    * `debian*16`: generate 16 Docker containers running Debian. Also works
+      with -py3.
+
+* `MITOGEN_TEST_IMAGE_TEMPLATE`: specifies the Linux container image name,
+  and hence the container registry used for test targets.

.ci/ansible_tests.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+# Run tests/ansible/all.yml under Ansible and Ansible-Mitogen
+
+import collections
+import glob
+import os
+import signal
+import sys
+
+import jinja2
+
+import ci_lib
+
+
+TMP = ci_lib.TempDir(prefix='mitogen_ci_ansible')
+TMP_HOSTS_DIR = os.path.join(TMP.path, 'hosts')
+
+
+def pause_if_interactive():
+    if os.path.exists('/tmp/interactive'):
+        while True:
+            signal.pause()
+
+
+interesting = ci_lib.get_interesting_procs()
+
+
+with ci_lib.Fold('unit_tests'):
+    os.environ['SKIP_MITOGEN'] = '1'
+    ci_lib.run('./run_tests -v')
+
+
+ci_lib.check_stray_processes(interesting)
+
+
+with ci_lib.Fold('docker_setup'):
+    containers = ci_lib.container_specs(ci_lib.DISTRO_SPECS.split())
+    ci_lib.start_containers(containers)
+
+
+with ci_lib.Fold('job_setup'):
+    os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8))
+    os.chdir(ci_lib.ANSIBLE_TESTS_DIR)
+
+    os.mkdir(TMP_HOSTS_DIR)
+    for path in glob.glob(os.path.join(ci_lib.ANSIBLE_TESTS_HOSTS_DIR, '*')):
+        if not path.endswith('default.hosts'):
+            os.symlink(path, os.path.join(TMP_HOSTS_DIR, os.path.basename(path)))
+
+    distros = collections.defaultdict(list)
+    families = collections.defaultdict(list)
+    for container in containers:
+        distros[container['distro']].append(container['name'])
+        families[container['family']].append(container['name'])
+
+    jinja_env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader(
+            searchpath=ci_lib.ANSIBLE_TESTS_TEMPLATES_DIR,
+        ),
+        lstrip_blocks=True,  # Remove spaces and tabs from before a block
+        trim_blocks=True,  # Remove first newline after a block
+    )
+    inventory_template = jinja_env.get_template('test-targets.j2')
+    inventory_path = os.path.join(TMP_HOSTS_DIR, 'test-targets.ini')
+
+    with open(inventory_path, 'w') as fp:
+        fp.write(inventory_template.render(
+            containers=containers,
+            distros=distros,
+            families=families,
+        ))
+
+    ci_lib.dump_file(inventory_path)
+
+with ci_lib.Fold('ansible'):
+    playbook = os.environ.get('PLAYBOOK', 'all.yml')
+    try:
+        ci_lib.run('./run_ansible_playbook.py %s -i "%s" %s',
+            playbook, TMP_HOSTS_DIR, ' '.join(sys.argv[1:]),
+        )
+    except:
+        pause_if_interactive()
+        raise
+
+
+ci_lib.check_stray_processes(interesting, containers)
+
+pause_if_interactive()

.ci/bash_functions

@@ -0,0 +1,13 @@
+# shellcheck shell=bash
+
+# Tox environment name -> Python executable name (e.g. py312-m_mtg -> python3.12)
+toxenv-python() {
+    local pattern='^py([23])([0-9]{1,2}).*'
+    if [[ $1 =~ $pattern ]]; then
+        echo "python${BASH_REMATCH[1]}.${BASH_REMATCH[2]}"
+        return
+    else
+        echo "${FUNCNAME[0]}: $1: environment name not recognised" >&2
+        return 1
+    fi
+}

.ci/ci_lib.py

@@ -0,0 +1,415 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+import atexit
+import errno
+import os
+import re
+import shlex
+import shutil
+import sys
+import tempfile
+
+if sys.version_info < (3, 0):
+    import subprocess32 as subprocess
+else:
+    import subprocess
+
+try:
+    import urlparse
+except ImportError:
+    import urllib.parse as urlparse
+
+os.chdir(
+    os.path.join(
+        os.path.dirname(__file__),
+        '..'
+    )
+)
+
+
+GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+ANSIBLE_TESTS_DIR = os.path.join(GIT_ROOT, 'tests/ansible')
+ANSIBLE_TESTS_HOSTS_DIR = os.path.join(GIT_ROOT, 'tests/ansible/hosts')
+ANSIBLE_TESTS_TEMPLATES_DIR = os.path.join(GIT_ROOT, 'tests/ansible/templates')
+DISTRO_SPECS = os.environ.get(
+    'MITOGEN_TEST_DISTRO_SPECS',
+    'alma9-py3 centos5 centos8-py3 debian9 debian12-py3 ubuntu1604 ubuntu2404-py3',
+)
+IMAGE_PREP_DIR = os.path.join(GIT_ROOT, 'tests/image_prep')
+IMAGE_TEMPLATE = os.environ.get(
+    'MITOGEN_TEST_IMAGE_TEMPLATE',
+    'ghcr.io/mitogen-hq/%(distro)s-test:2025.02',
+)
+SUDOERS_DEFAULTS_SRC = './tests/image_prep/files/sudoers_defaults'
+SUDOERS_DEFAULTS_DEST = '/etc/sudoers.d/mitogen_test_defaults'
+TESTS_SSH_PRIVATE_KEY_FILE = os.path.join(GIT_ROOT, 'tests/data/docker/mitogen__has_sudo_pubkey.key')
+
+
+_print = print
+def print(*args, **kwargs):
+    file = kwargs.get('file', sys.stdout)
+    flush = kwargs.pop('flush', False)
+    _print(*args, **kwargs)
+    if flush:
+        file.flush()
+
+
+def _have_cmd(args):
+    # Code duplicated in testlib.py
+    try:
+        subprocess.run(
+            args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+            check=True,
+        )
+    except OSError as exc:
+        if exc.errno == errno.ENOENT:
+            return False
+        raise
+    except subprocess.CalledProcessError:
+        return False
+    return True
+
+
+def have_docker():
+    return _have_cmd(['docker', 'info'])
+
+
+def _argv(s, *args):
+    """Interpolate a command line using *args, return an argv style list.
+
+    >>> _argv('git commit -m "Use frobnicate 2.0 (fixes #%d)"', 1234)
+    ['git', commit', '-m', 'Use frobnicate 2.0 (fixes #1234)']
+    """
+    if args:
+        s %= args
+    return shlex.split(s)
+
+
+def run(s, *args, **kwargs):
+    """ Run a command, with arguments
+
+    >>> rc = run('echo "%s %s"', 'foo', 'bar')
+    Running: ['echo', 'foo bar']
+    foo bar
+    Finished running: ['echo', 'foo bar']
+    >>> rc
+    0
+    """
+    argv = _argv(s, *args)
+    print('Running: %s' % (argv,), flush=True)
+    try:
+        ret = subprocess.check_call(argv, **kwargs)
+        print('Finished running: %s' % (argv,), flush=True)
+    except Exception:
+        print('Exception occurred while running: %s' % (argv,), file=sys.stderr, flush=True)
+        raise
+
+    return ret
+
+
+def combine(batch):
+    """
+    >>> combine(['ls -l', 'echo foo'])
+    'set -x; ( ls -l; ) && ( echo foo; )'
+    """
+    return 'set -x; ' + (' && '.join(
+        '( %s; )' % (cmd,)
+        for cmd in batch
+    ))
+
+
+def throttle(batch, pause=1):
+    """
+    Add pauses between commands in a batch
+
+    >>> throttle(['echo foo', 'echo bar', 'echo baz'])
+    ['echo foo', 'sleep 1', 'echo bar', 'sleep 1', 'echo baz']
+    """
+    def _with_pause(batch, pause):
+        for cmd in batch:
+            yield cmd
+            yield 'sleep %i' % (pause,)
+    return list(_with_pause(batch, pause))[:-1]
+
+
+def run_batches(batches):
+    """ Run shell commands grouped into batches, showing an execution trace.
+
+    Raise AssertionError if any command has exits with a non-zero status.
+
+    >>> run_batches([['echo foo', 'true']])
+    + echo foo
+    foo
+    + true
+    >>> run_batches([['true', 'echo foo'], ['false']])
+    + true
+    + echo foo
+    foo
+    + false
+    Traceback (most recent call last):
+    File "...", line ..., in <module>
+    File "...", line ..., in run_batches
+    AssertionError
+    """
+    procs = [
+        subprocess.Popen(combine(batch), shell=True)
+        for batch in batches
+    ]
+    for proc in procs:
+        proc.wait()
+        if proc.returncode:
+            print(
+                'proc: pid=%i rc=%i args=%r'
+                % (proc.pid, proc.returncode, proc.args),
+                file=sys.stderr, flush=True,
+            )
+    assert [proc.returncode for proc in procs] == [0] * len(procs)
+
+
+def get_output(s, *args, **kwargs):
+    """
+    Print and run command line s, %-interopolated using *args. Return stdout.
+
+    >>> s = get_output('echo "%s %s"', 'foo', 'bar')
+    Running: ['echo', 'foo bar']
+    >>> s
+    'foo bar\n'
+    """
+    argv = _argv(s, *args)
+    print('Running: %s' % (argv,), flush=True)
+    return subprocess.check_output(argv, **kwargs)
+
+
+def exists_in_path(progname):
+    """
+    Return True if progname exists in $PATH.
+
+    >>> exists_in_path('echo')
+    True
+    >>> exists_in_path('kwyjibo') # Only found in North American cartoons
+    False
+    """
+    return any(os.path.exists(os.path.join(dirname, progname))
+               for dirname in os.environ['PATH'].split(os.pathsep))
+
+
+class TempDir(object):
+    def __init__(self, prefix='mitogen_ci_lib'):
+        self.path = tempfile.mkdtemp(prefix=prefix)
+        atexit.register(self.destroy)
+
+    def destroy(self, rmtree=shutil.rmtree):
+        rmtree(self.path)
+
+
+class Fold(object):
+    def __init__(self, name): pass
+    def __enter__(self): pass
+    def __exit__(self, _1, _2, _3): pass
+
+
+os.environ['PYTHONDONTWRITEBYTECODE'] = 'x'
+os.environ['PYTHONPATH'] = '%s:%s' % (
+    os.environ.get('PYTHONPATH', ''),
+    GIT_ROOT
+)
+
+def get_docker_hostname():
+    """Return the hostname where the docker daemon is running.
+    """
+    # Duplicated in testlib
+    url = os.environ.get('DOCKER_HOST')
+    if url in (None, 'http+docker://localunixsocket'):
+        return 'localhost'
+
+    parsed = urlparse.urlparse(url)
+    return parsed.netloc.partition(':')[0]
+
+
+def container_specs(
+        distros,
+        base_port=2200,
+        image_template=IMAGE_TEMPLATE,
+        name_template='target-%(distro)s-%(index)d',
+):
+    """
+    >>> import pprint
+    >>> pprint.pprint(container_specs(['debian11-py3', 'centos6']))
+    [{'distro': 'debian11',
+      'family': 'debian',
+      'hostname': 'localhost',
+      'image': 'ghcr.io/mitogen-hq/debian11-test:2025.02',
+      'index': 1,
+      'name': 'target-debian11-1',
+      'port': 2201,
+      'python_path': '/usr/bin/python3'},
+     {'distro': 'centos6',
+      'family': 'centos',
+      'hostname': 'localhost',
+      'image': 'ghcr.io/mitogen-hq/centos6-test:2025.02',
+      'index': 2,
+      'name': 'target-centos6-2',
+      'port': 2202,
+      'python_path': '/usr/bin/python'}]
+    """
+    docker_hostname = get_docker_hostname()
+    # Code duplicated in testlib.py, both should be updated together
+    distro_pattern = re.compile(r'''
+        (?P<distro>(?P<family>[a-z]+)[0-9]+)
+        (?:-(?P<py>py3))?
+        (?:\*(?P<count>[0-9]+))?
+        ''',
+        re.VERBOSE,
+    )
+    i = 1
+    lst = []
+
+    for distro in distros:
+        # Code duplicated in testlib.py, both should be updated together
+        d = distro_pattern.match(distro).groupdict(default=None)
+
+        if d.pop('py') == 'py3':
+            python_path = '/usr/bin/python3'
+        else:
+            python_path = '/usr/bin/python'
+
+        count = int(d.pop('count') or '1', 10)
+
+        for x in range(count):
+            d['index'] = i
+            d.update({
+                'image': image_template % d,
+                'name': name_template % d,
+                "hostname": docker_hostname,
+                'port': base_port + i,
+                "python_path": python_path,
+            })
+            lst.append(d)
+            i += 1
+
+    return lst
+
+
+# ssh removed from here because 'linear' strategy relies on processes that hang
+# around after the Ansible run completes
+INTERESTING_COMMS = ('python', 'sudo', 'su', 'doas')
+
+
+def proc_is_docker(pid):
+    try:
+        fp = open('/proc/%s/cgroup' % (pid,), 'r')
+    except IOError:
+        return False
+
+    try:
+        return 'docker' in fp.read()
+    finally:
+        fp.close()
+
+
+def get_interesting_procs(container_name=None):
+    """
+    Return a list of (pid, line) tuples for processes considered interesting.
+    """
+    args = ['ps', 'ax', '-oppid=', '-opid=', '-ocomm=', '-ocommand=']
+    if container_name is not None:
+        args = ['docker', 'exec', container_name] + args
+
+    out = []
+    for line in subprocess.check_output(args).decode().splitlines():
+        ppid, pid, comm, rest = line.split(None, 3)
+        if (
+            (
+                any(comm.startswith(s) for s in INTERESTING_COMMS) or
+                'mitogen:' in rest
+            ) and
+            (
+                'WALinuxAgent' not in rest
+            ) and
+            (
+                container_name is not None or
+                (not proc_is_docker(pid))
+            )
+        ):
+            out.append((int(pid), line))
+
+    return sorted(out)
+
+
+def start_containers(containers):
+    """Run docker containers in the background, with sshd on specified ports.
+
+    >>> containers = start_containers([
+    ...     {'distro': 'debian', 'hostname': 'localhost',
+    ...      'name': 'target-debian-1', 'port': 2201,
+    ...      'python_path': '/usr/bin/python'},
+    ... ])
+    """
+    if os.environ.get('KEEP'):
+        return
+
+    run_batches([
+        [
+            "docker rm -f %(name)s || true" % container,
+            "docker run "
+                "--rm "
+                # "--cpuset-cpus 0,1 "
+                "--detach "
+                "--privileged "
+                "--cap-add=SYS_PTRACE "
+                "--publish 0.0.0.0:%(port)s:22/tcp "
+                "--hostname=%(name)s "
+                "--name=%(name)s "
+                "%(image)s"
+            % container
+        ]
+        for container in containers
+    ])
+
+    for container in containers:
+        container['interesting'] = get_interesting_procs(container['name'])
+
+    return containers
+
+
+def verify_procs(hostname, old, new):
+    oldpids = set(pid for pid, _ in old)
+    if any(pid not in oldpids for pid, _ in new):
+        print('%r had stray processes running:' % (hostname,), file=sys.stderr, flush=True)
+        for pid, line in new:
+            if pid not in oldpids:
+                print('New process:', line, flush=True)
+        return False
+
+    return True
+
+
+def check_stray_processes(old, containers=None):
+    ok = True
+
+    new = get_interesting_procs()
+    if old is not None:
+        ok &= verify_procs('test host machine', old, new)
+
+    for container in containers or ():
+        ok &= verify_procs(
+            container['name'],
+            container['interesting'],
+            get_interesting_procs(container['name'])
+        )
+
+    assert ok, 'stray processes were found'
+
+
+def dump_file(path):
+    print('--- %s ---' % (path,), flush=True)
+    with open(path, 'r') as fp:
+        print(fp.read().rstrip(), flush=True)
+    print('---', flush=True)
+
+
+# SSH passes these through to the container when run interactively, causing
+# stdout to get messed up with libc warnings.
+os.environ.pop('LANG', None)
+os.environ.pop('LC_ALL', None)

.ci/debops_common_install.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+import ci_lib
+
+ci_lib.run_batches([
+    [
+        'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"',
+    ],
+])
+
+ci_lib.run('ansible-galaxy collection install debops.debops:==2.1.2')

.ci/debops_common_tests.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+
+import os
+import sys
+
+import ci_lib
+
+
+TMP = ci_lib.TempDir(prefix='mitogen_ci_debops')
+project_dir = os.path.join(TMP.path, 'project')
+vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml'
+inventory_path = 'ansible/inventory/hosts'
+docker_hostname = ci_lib.get_docker_hostname()
+
+
+with ci_lib.Fold('docker_setup'):
+    containers = ci_lib.container_specs(
+        ['debian*2'],
+        base_port=2700,
+        name_template='debops-target-%(distro)s-%(index)d',
+    )
+    ci_lib.start_containers(containers)
+
+
+with ci_lib.Fold('job_setup'):
+    os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8))
+    ci_lib.run('debops-init %s', project_dir)
+    os.chdir(project_dir)
+
+    ansible_strategy_plugin = "{}/ansible_mitogen/plugins/strategy".format(ci_lib.GIT_ROOT)
+
+    with open('.debops.cfg', 'w') as fp:
+        fp.write(
+            "[ansible defaults]\n"
+            "strategy_plugins = {}\n"
+            "strategy = mitogen_linear\n"
+            .format(ansible_strategy_plugin)
+        )
+
+    with open(vars_path, 'w') as fp:
+        fp.write(
+            "ansible_python_interpreter: /usr/bin/python2.7\n"
+            "\n"
+            "ansible_user: mitogen__has_sudo_pubkey\n"
+            "ansible_become_pass: has_sudo_pubkey_password\n"
+            "ansible_ssh_private_key_file: %s\n"
+            "\n"
+            # Speed up slow DH generation.
+            "dhparam__bits: ['128', '64']\n"
+            % (ci_lib.TESTS_SSH_PRIVATE_KEY_FILE,)
+        )
+
+    with open(inventory_path, 'a') as fp:
+        fp.writelines(
+            '%(name)s '
+            'ansible_host=%(hostname)s '
+            'ansible_port=%(port)d '
+            'ansible_python_interpreter=%(python_path)s '
+            '\n'
+            % container
+            for container in containers
+        )
+
+    ci_lib.dump_file('ansible/inventory/hosts')
+
+    # Now we have real host key checking, we need to turn it off
+    os.environ['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
+
+
+interesting = ci_lib.get_interesting_procs()
+
+with ci_lib.Fold('first_run'):
+    ci_lib.run('debops common %s', ' '.join(sys.argv[1:]))
+
+ci_lib.check_stray_processes(interesting, containers)
+
+
+with ci_lib.Fold('second_run'):
+    ci_lib.run('debops common %s', ' '.join(sys.argv[1:]))
+
+ci_lib.check_stray_processes(interesting, containers)

.ci/install_sshpass

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+VERSION="$1"
+
+curl \
+    --fail \
+    --location \
+    --no-progress-meter \
+    --remote-name \
+    "https://downloads.sourceforge.net/project/sshpass/sshpass/${VERSION}/sshpass-${VERSION}.tar.gz"
+tar xvf "sshpass-${VERSION}.tar.gz"
+cd "sshpass-${VERSION}"
+./configure
+sudo make install

.ci/localhost_ansible_tests.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+# Run tests/ansible/all.yml under Ansible and Ansible-Mitogen
+
+from __future__ import print_function
+
+import os
+import subprocess
+import sys
+
+import ci_lib
+
+
+with ci_lib.Fold('unit_tests'):
+    os.environ['SKIP_MITOGEN'] = '1'
+    ci_lib.run('./run_tests -v')
+
+
+with ci_lib.Fold('job_setup'):
+    os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8))
+
+
+with ci_lib.Fold('machine_prep'):
+    # generate a new ssh key for localhost ssh
+    if not os.path.exists(os.path.expanduser("~/.ssh/id_rsa")):
+        subprocess.check_call("ssh-keygen -P '' -m pem -f ~/.ssh/id_rsa", shell=True)
+        subprocess.check_call("cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys", shell=True)
+        os.chmod(os.path.expanduser('~/.ssh'), int('0700', 8))
+        os.chmod(os.path.expanduser('~/.ssh/authorized_keys'), int('0600', 8))
+
+    # also generate it for the sudo user
+    if os.system("sudo [ -f ~root/.ssh/id_rsa ]") != 0:
+        subprocess.check_call("sudo ssh-keygen -P '' -m pem -f ~root/.ssh/id_rsa", shell=True)
+        subprocess.check_call("sudo cat ~root/.ssh/id_rsa.pub | sudo tee -a ~root/.ssh/authorized_keys", shell=True)
+        subprocess.check_call('sudo chmod 700 ~root/.ssh', shell=True)
+        subprocess.check_call('sudo chmod 600 ~root/.ssh/authorized_keys', shell=True)
+
+    os.chdir(ci_lib.IMAGE_PREP_DIR)
+    ci_lib.run("ansible-playbook -c local -i localhost, macos_localhost.yml")
+
+    if os.path.expanduser('~mitogen__user1') == '~mitogen__user1':
+        os.chdir(ci_lib.IMAGE_PREP_DIR)
+        ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml")
+
+    cmd = ';'.join([
+        'from __future__ import print_function',
+        'import os, sys',
+        'print(sys.executable, os.path.realpath(sys.executable))',
+    ])
+    for interpreter in ['/usr/bin/python', '/usr/bin/python2', '/usr/bin/python2.7']:
+        print(interpreter)
+        try:
+            subprocess.call([interpreter, '-c', cmd])
+        except OSError as exc:
+            print(exc)
+
+        print(interpreter, 'with PYTHON_LAUNCHED_FROM_WRAPPER=1')
+        environ = os.environ.copy()
+        environ['PYTHON_LAUNCHED_FROM_WRAPPER'] = '1'
+        try:
+            subprocess.call([interpreter, '-c', cmd], env=environ)
+        except OSError as exc:
+            print(exc)
+
+
+with ci_lib.Fold('ansible'):
+    os.chdir(ci_lib.ANSIBLE_TESTS_DIR)
+    playbook = os.environ.get('PLAYBOOK', 'all.yml')
+    ci_lib.run('./run_ansible_playbook.py %s %s',
+        playbook, ' '.join(sys.argv[1:]))

.ci/mitogen_py24_install.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+import ci_lib
+
+batches = [
+    [
+        'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv',
+    ]
+]
+
+ci_lib.run_batches(batches)

.ci/mitogen_py24_tests.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+# Mitogen tests for Python 2.4.
+
+import os
+
+import ci_lib
+
+os.environ.update({
+    'NOCOVERAGE': '1',
+    'UNIT2': '/usr/local/python2.4.6/bin/unit2',
+    'MITOGEN_LOG_LEVEL': 'debug',
+    'SKIP_ANSIBLE': '1',
+})
+
+ci_lib.run('./run_tests -v')

.ci/mitogen_tests.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+# Run the Mitogen tests.
+
+import os
+import subprocess
+
+import ci_lib
+
+os.environ.update({
+    'MITOGEN_LOG_LEVEL': 'debug',
+    'SKIP_ANSIBLE': '1',
+})
+
+if not ci_lib.have_docker():
+    os.environ['SKIP_DOCKER_TESTS'] = '1'
+
+subprocess.check_call(
+    "umask 0022; sudo cp '%s' '%s'"
+    % (ci_lib.SUDOERS_DEFAULTS_SRC, ci_lib.SUDOERS_DEFAULTS_DEST),
+    shell=True,
+)
+subprocess.check_call(['sudo', 'visudo', '-cf', ci_lib.SUDOERS_DEFAULTS_DEST])
+subprocess.check_call(['sudo', '-l'])
+
+interesting = ci_lib.get_interesting_procs()
+ci_lib.run('./run_tests -v')
+ci_lib.check_stray_processes(interesting)

.ci/show_python_versions

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -o errexit -o nounset -o pipefail
+
+INDENT="    "
+POSSIBLE_PYTHONS=(
+    python
+    python2
+    python3
+    /usr/bin/python
+    /usr/bin/python2
+    /usr/bin/python3
+    # GitHub macOS 12 images: python2.7 is installed, but not on $PATH
+    /Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7
+)
+
+for p in "${POSSIBLE_PYTHONS[@]}"; do
+    echo "$p"
+    if [[ ${p:0:1} == "/" && -e $p ]]; then
+        :
+    elif type "$p" > /dev/null 2>&1; then
+        type "$p" 2>&1 | sed -e "s/^/${INDENT}type: /"
+    else
+        echo "${INDENT}Not present"
+        echo
+        continue
+    fi
+
+    $p -c "import sys;          print('${INDENT}version:        %d.%d.%d' % sys.version_info[:3])"
+    # macOS builders lack a realpath command
+    $p -c "import os.path;      print('${INDENT}realpath:       %s' % os.path.realpath('$(type -p "$p")'))"
+    $p -c "import sys;          print('${INDENT}sys.executable: %s' % sys.executable)"
+    echo
+done

.ci/soak/debops_common.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+export NOCOVERAGE=1
+
+# Make Docker containers once.
+/usr/bin/time -v ./.ci/debops_common_tests.py "$@" || break
+export KEEP=1
+
+i=0
+while :
+do
+    i=$((i + 1))
+    /usr/bin/time -v ./.ci/debops_common_tests.py "$@" || break
+done
+
+echo $i

.ci/soak/mitogen.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+export NOCOVERAGE=1
+export DISTROS="debian*4"
+
+# Make Docker containers once.
+/usr/bin/time -v ./.ci/ansible_tests.py "$@"
+export KEEP=1
+
+i=0
+while :
+do
+    i=$((i + 1))
+    /usr/bin/time -v ./.ci/ansible_tests.py "$@" || break
+done
+
+echo $i

.ci/soak/mitogen_py24.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+export NOCOVERAGE=1
+
+i=0
+while :
+do
+    i=$((i + 1))
+    /usr/bin/time -v ./.ci/mitogen_py24_tests.py "$@" || break
+done
+
+echo $i

.github/ISSUE_TEMPLATE/bug-0.3.yml

@@ -0,0 +1,62 @@
+name: Bug report
+description: Report a bug in Mitogen 0.3.x (for Ansible 2.10 and above)
+labels:
+  - affects-0.3
+type: bug
+
+body:
+  - type: textarea
+    attributes:
+      label: Description
+      description: >
+        When does the problem occur?
+        What happens after?
+        How is this different?
+        Did it previously behave as expected?
+      placeholder: |
+        When I do X, Y happens, but I was expecting Z because ...
+        Before version 1.2.3 it worked as expected.
+    validations:
+      required: true
+
+  - type: input
+    attributes:
+      label: Mitogen version
+      placeholder: 0.3.31, 0.3.3-9+deb12u1
+    validations:
+      required: true
+
+  - type: input
+    attributes:
+      label: Ansible version (if applicable)
+      placeholder: 2.18.11
+
+  - type: textarea
+    attributes:
+      label: OS and environment
+      description: >
+        What operating system version(s), Python version(s), etc. are you using?
+      placeholder: |
+        Controller (master): Debian 13, Python 3.14
+        Targets (slaves): Ubuntu 20.04/Python 2.7, RHEL 10, ...
+
+  - type: textarea
+    attributes:
+      label: Steps to reproduce
+      description: >
+        Instructions, code, or playbook(s) recreate the beahviour
+      value: |
+        Steps:
+        1. Set config `foo = 42` in somefile.cfg
+        2. Run the following Python or Playbook with `cmd --option bar ...`
+
+        ```
+        Code or playbook here
+        ```
+
+  - type: textarea
+    attributes:
+      label: Anything else
+      description: >
+        Include any other details you think might be relevant or helpful.
+        Examples might include logs, unusual settings, environment variables, ...

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,16 @@
+
+Thanks for creating a PR! Here's a quick checklist to pay attention to:
+
+* Please add an entry to docs/changelog.rst as appropriate.
+
+* Has some new parameter been added or semantics modified somehow? Please
+  ensure relevant documentation is updated in docs/ansible.rst and
+  docs/api.rst.
+
+* If it's for new functionality, is there at least a basic test in either
+  tests/ or tests/ansible/ covering it?
+
+* If it's for a new connection method, please try to stub out the
+  implementation as in tests/data/stubs/, so that construction can be tested
+  without having a working configuration.
+

.github/workflows/tests.yml

@@ -0,0 +1,211 @@
+# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
+
+name: Tests
+
+# env:
+#  ANSIBLE_VERBOSITY: 3
+#  MITOGEN_LOG_LEVEL: DEBUG
+
+on:
+  pull_request:
+  push:
+    branches-ignore:
+      - docs-master
+
+# https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners
+# https://github.com/actions/runner-images/blob/main/README.md#software-and-image-support
+jobs:
+  u2204:
+    name: u2204 ${{ matrix.tox_env }}
+    # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md
+    runs-on: ubuntu-22.04
+    timeout-minutes: 25
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - tox_env: py27-m_ans-ans2.10
+          - tox_env: py27-m_ans-ans4
+
+          - tox_env: py36-m_ans-ans2.10
+          - tox_env: py36-m_ans-ans4
+
+          - tox_env: py27-m_mtg
+          - tox_env: py36-m_mtg
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - run: .ci/show_python_versions
+      - name: Install deps
+        id: install-deps
+        run: |
+          set -o errexit -o nounset -o pipefail
+          source .ci/bash_functions
+          PYTHON="$(toxenv-python '${{ matrix.tox_env }}')"
+
+          sudo apt-get update
+
+          if [[ $PYTHON == "python2.7" ]]; then
+              sudo apt install -y python2-dev sshpass virtualenv
+              curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py"
+              "$PYTHON" get-pip.py --user --no-python-version-warning
+              # Avoid Python 2.x pip masking system pip
+              rm -f ~/.local/bin/{easy_install,pip,wheel}
+          elif [[ $PYTHON == "python3.6" ]]; then
+              sudo apt install -y gcc-10 make libbz2-dev liblzma-dev libreadline-dev libsqlite3-dev libssl-dev sshpass virtualenv zlib1g-dev
+              curl --fail --silent --show-error --location https://pyenv.run | bash
+              CC=gcc-10 ~/.pyenv/bin/pyenv install --force 3.6
+              PYTHON="$HOME/.pyenv/versions/3.6.15/bin/python3.6"
+          fi
+
+          "$PYTHON" -m pip install -r "tests/requirements-tox.txt"
+          echo "python=$PYTHON" >> $GITHUB_OUTPUT
+      - name: Run tests
+        env:
+          GITHUB_ACTOR: ${{ github.actor }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -o errexit -o nounset -o pipefail
+          PYTHON="${{ steps.install-deps.outputs.python }}"
+
+          "$PYTHON" -m tox -e "${{ matrix.tox_env }}"
+
+  u2404:
+    name: u2404 ${{ matrix.tox_env }}
+    # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md
+    runs-on: ubuntu-24.04
+    timeout-minutes: 25
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - tox_env: py311-m_ans-ans2.10
+            python_version: '3.11'
+          - tox_env: py311-m_ans-ans3
+            python_version: '3.11'
+          - tox_env: py311-m_ans-ans4
+            python_version: '3.11'
+          - tox_env: py311-m_ans-ans5
+            python_version: '3.11'
+          - tox_env: py313-m_ans-ans6
+            python_version: '3.13'
+          - tox_env: py313-m_ans-ans7
+            python_version: '3.13'
+          - tox_env: py313-m_ans-ans8
+            python_version: '3.13'
+          - tox_env: py314-m_ans-ans9
+            python_version: '3.14'
+          - tox_env: py314-m_ans-ans10
+            python_version: '3.14'
+          - tox_env: py314-m_ans-ans11
+            python_version: '3.14'
+          - tox_env: py314-m_ans-ans12
+            python_version: '3.14'
+          - tox_env: py314-m_ans-ans13
+            python_version: '3.14'
+
+          - tox_env: py314-m_ans-ans13-s_lin
+            python_version: '3.14'
+
+          - tox_env: py314-m_mtg
+            python_version: '3.14'
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python_version }}
+        if: ${{ matrix.python_version }}
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - run: .ci/show_python_versions
+      - name: Install deps
+        id: install-deps
+        run: |
+          set -o errexit -o nounset -o pipefail
+          source .ci/bash_functions
+          PYTHON="$(toxenv-python '${{ matrix.tox_env }}')"
+
+          sudo apt-get update
+          sudo apt-get install -y sshpass virtualenv
+
+          "$PYTHON" -m pip install -r "tests/requirements-tox.txt"
+          echo "python=$PYTHON" >> $GITHUB_OUTPUT
+      - name: Run tests
+        env:
+          GITHUB_ACTOR: ${{ github.actor }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -o errexit -o nounset -o pipefail
+          PYTHON="${{ steps.install-deps.outputs.python }}"
+
+          "$PYTHON" -m tox -e "${{ matrix.tox_env }}"
+
+  macos:
+    name: macos ${{ matrix.tox_env }}
+    # https://github.com/actions/runner-images/blob/main/images/macos/macos-15-Readme.md
+    runs-on: macos-15
+    timeout-minutes: 15
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - tox_env: py314-m_lcl-ans13
+            python_version: '3.14'
+          - tox_env: py314-m_lcl-ans13-s_lin
+            python_version: '3.14'
+
+          - tox_env: py314-m_mtg
+            python_version: '3.14'
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python_version }}
+        if: ${{ matrix.python_version }}
+      - run: .ci/show_python_versions
+      - run: .ci/install_sshpass ${{ matrix.sshpass_version }}
+        if: ${{ matrix.sshpass_version }}
+      - name: Install deps
+        id: install-deps
+        run: |
+          set -o errexit -o nounset -o pipefail
+          source .ci/bash_functions
+          PYTHON="$(toxenv-python '${{ matrix.tox_env }}')"
+
+          "$PYTHON" -m pip install -r "tests/requirements-tox.txt"
+          echo "python=$PYTHON" >> $GITHUB_OUTPUT
+      - name: Run tests
+        env:
+          GITHUB_ACTOR: ${{ github.actor }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -o errexit -o nounset -o pipefail
+          PYTHON="${{ steps.install-deps.outputs.python }}"
+
+          "$PYTHON" -m tox -e "${{ matrix.tox_env }}"
+
+  # https://github.com/marketplace/actions/alls-green
+  check:
+    if: always()
+    needs:
+      - u2204
+      - u2404
+      - macos
+    runs-on: ubuntu-latest
+    steps:
+      - uses: re-actors/alls-green@release/v1
+        with:
+          jobs: ${{ toJSON(needs) }}

.gitignore

@@ -1,2 +1,22 @@
-docs/_build
+.ansible/
+.coverage
+.tox
+.venv
+venvs/**
+**/.DS_Store
+*.pyc
+*.pyd
+*.pyo
+*.retry
+MANIFEST
+build/
+dist/
+extra/
+tests/ansible/.*.pid
+tests/image_prep/logs
+docs/_build/
+htmlcov/
 *.egg-info
+__pycache__/
+extra
+**/.*.pid

LICENSE

@@ -0,0 +1,26 @@
+Copyright 2021, the Mitogen authors
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+may be used to endorse or promote products derived from this software without
+specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

MANIFEST.in

@@ -0,0 +1 @@
+include LICENSE

README.md

@@ -0,0 +1,9 @@
+# Mitogen
+
+[![PyPI - Version](https://img.shields.io/pypi/v/mitogen)](https://pypi.org/project/mitogen/)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/mitogen)](https://pypi.org/project/mitogen/)
+[![Build Status](https://img.shields.io/github/actions/workflow/status/mitogen-hq/mitogen/tests.yml?branch=master)](https://github.com/mitogen-hq/mitogen/actions?query=branch%3Amaster)
+
+<a href="https://mitogen.networkgenomics.com/">Please see the documentation</a>.
+
+![](https://i.imgur.com/eBM6LhJ.gif)

ansible_mitogen/affinity.py

@@ -0,0 +1,287 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+"""
+As Mitogen separates asynchronous IO out to a broker thread, communication
+necessarily involves context switching and waking that thread. When application
+threads and the broker share a CPU, this can be almost invisibly fast - around
+25 microseconds for a full A->B->A round-trip.
+
+However when threads are scheduled on different CPUs, round-trip delays
+regularly vary wildly, and easily into milliseconds. Many contributing factors
+exist, not least scenarios like:
+
+1. A is preempted immediately after waking B, but before releasing the GIL.
+2. B wakes from IO wait only to immediately enter futex wait.
+3. A may wait 10ms or more for another timeslice, as the scheduler on its CPU
+   runs threads unrelated to its transaction (i.e. not B), wake only to release
+   its GIL, before entering IO sleep waiting for a reply from B, which cannot
+   exist yet.
+4. B wakes, acquires GIL, performs work, and sends reply to A, causing it to
+   wake. B is preempted before releasing GIL.
+5. A wakes from IO wait only to immediately enter futex wait.
+6. B may wait 10ms or more for another timeslice, wake only to release its GIL,
+   before sleeping again.
+7. A wakes, acquires GIL, finally receives reply.
+
+Per above if we are unlucky, on an even moderately busy machine it is possible
+to lose milliseconds just in scheduling delay, and the effect is compounded
+when pairs of threads in process A are communicating with pairs of threads in
+process B using the same scheme, such as when Ansible WorkerProcess is
+communicating with ContextService in the connection multiplexer. In the worst
+case it could involve 4 threads working in lockstep spread across 4 busy CPUs.
+
+Since multithreading in Python is essentially useless except for waiting on IO
+due to the presence of the GIL, at least in Ansible there is no good reason for
+threads in the same process to run on distinct CPUs - they always operate in
+lockstep due to the GIL, and are thus vulnerable to issues like above.
+
+Linux lacks any natural API to describe what we want, it only permits
+individual threads to be constrained to run on specific CPUs, and for that
+constraint to be inherited by new threads and forks of the constrained thread.
+
+This module therefore implements a CPU pinning policy for Ansible processes,
+providing methods that should be called early in any new process, either to
+rebalance which CPU it is pinned to, or in the case of subprocesses, to remove
+the pinning entirely. It is likely to require ongoing tweaking, since pinning
+necessarily involves preventing the scheduler from making load balancing
+decisions.
+"""
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import ctypes
+import logging
+import mmap
+import multiprocessing
+import os
+import struct
+
+import mitogen.parent
+
+
+LOG = logging.getLogger(__name__)
+
+
+try:
+    _libc = ctypes.CDLL(None, use_errno=True)
+    _strerror = _libc.strerror
+    _strerror.restype = ctypes.c_char_p
+    _sem_init = _libc.sem_init
+    _sem_wait = _libc.sem_wait
+    _sem_post = _libc.sem_post
+    _sched_setaffinity = _libc.sched_setaffinity
+except (OSError, AttributeError):
+    _libc = None
+    _strerror = None
+    _sem_init = None
+    _sem_wait = None
+    _sem_post = None
+    _sched_setaffinity = None
+
+
+class sem_t(ctypes.Structure):
+    """
+    Wrap sem_t to allow storing a lock in shared memory.
+    """
+    _fields_ = [
+        ('data', ctypes.c_uint8 * 128),
+    ]
+
+    def init(self):
+        if _sem_init(self.data, 1, 1):
+            raise Exception(_strerror(ctypes.get_errno()))
+
+    def acquire(self):
+        if _sem_wait(self.data):
+            raise Exception(_strerror(ctypes.get_errno()))
+
+    def release(self):
+        if _sem_post(self.data):
+            raise Exception(_strerror(ctypes.get_errno()))
+
+
+class State(ctypes.Structure):
+    """
+    Contents of shared memory segment. This allows :meth:`Manager.assign` to be
+    called from any child, since affinity assignment must happen from within
+    the context of the new child process.
+    """
+    _fields_ = [
+        ('lock', sem_t),
+        ('counter', ctypes.c_uint8),
+    ]
+
+
+class Policy(object):
+    """
+    Process affinity policy.
+    """
+    def assign_controller(self):
+        """
+        Assign the Ansible top-level policy to this process.
+        """
+
+    def assign_muxprocess(self, index):
+        """
+        Assign the MuxProcess policy to this process.
+        """
+
+    def assign_worker(self):
+        """
+        Assign the WorkerProcess policy to this process.
+        """
+
+    def assign_subprocess(self):
+        """
+        Assign the helper subprocess policy to this process.
+        """
+
+class FixedPolicy(Policy):
+    """
+    :class:`Policy` for machines where the only control method available is
+    fixed CPU placement. The scheme here was tested on an otherwise idle 16
+    thread machine.
+
+    - The connection multiplexer is pinned to CPU 0.
+    - The Ansible top-level (strategy) is pinned to CPU 1.
+    - WorkerProcesses are pinned sequentually to 2..N, wrapping around when no
+      more CPUs exist.
+    - Children such as SSH may be scheduled on any CPU except 0/1.
+
+    If the machine has less than 4 cores available, the top-level and workers
+    are pinned between CPU 2..N, i.e. no CPU is reserved for the top-level
+    process.
+
+    This could at least be improved by having workers pinned to independent
+    cores, before reusing the second hyperthread of an existing core.
+
+    A hook is installed that causes :meth:`reset` to run in the child of any
+    process created with :func:`mitogen.parent.popen`, ensuring CPU-intensive
+    children like SSH are not forced to share the same core as the (otherwise
+    potentially very busy) parent.
+    """
+    def __init__(self, cpu_count=None):
+        #: For tests.
+        self.cpu_count = cpu_count or multiprocessing.cpu_count()
+        self.mem = mmap.mmap(-1, 4096)
+        self.state = State.from_buffer(self.mem)
+        self.state.lock.init()
+
+        if self.cpu_count < 2:
+            # uniprocessor
+            self._reserve_mux = False
+            self._reserve_controller = False
+            self._reserve_mask = 0
+            self._reserve_shift = 0
+        elif self.cpu_count < 4:
+            # small SMP
+            self._reserve_mux = True
+            self._reserve_controller = False
+            self._reserve_mask = 1
+            self._reserve_shift = 1
+        else:
+            # big SMP
+            self._reserve_mux = True
+            self._reserve_controller = True
+            self._reserve_mask = 3
+            self._reserve_shift = 2
+
+    def _set_affinity(self, descr, mask):
+        if descr:
+            LOG.debug('CPU mask for %s: %#08x', descr, mask)
+        mitogen.parent._preexec_hook = self._clear
+        self._set_cpu_mask(mask)
+
+    def _balance(self, descr):
+        self.state.lock.acquire()
+        try:
+            n = self.state.counter
+            self.state.counter += 1
+        finally:
+            self.state.lock.release()
+
+        self._set_cpu(descr, self._reserve_shift + (
+            (n % (self.cpu_count - self._reserve_shift))
+        ))
+
+    def _set_cpu(self, descr, cpu):
+        self._set_affinity(descr, 1 << (cpu % self.cpu_count))
+
+    def _clear(self):
+        all_cpus = (1 << self.cpu_count) - 1
+        self._set_affinity(None, all_cpus & ~self._reserve_mask)
+
+    def assign_controller(self):
+        if self._reserve_controller:
+            self._set_cpu('Ansible top-level process', 1)
+        else:
+            self._balance('Ansible top-level process')
+
+    def assign_muxprocess(self, index):
+        self._set_cpu('MuxProcess %d' % (index,), index)
+
+    def assign_worker(self):
+        self._balance('WorkerProcess')
+
+    def assign_subprocess(self):
+        self._clear()
+
+
+class LinuxPolicy(FixedPolicy):
+    def _mask_to_bytes(self, mask):
+        """
+        Convert the (type long) mask to a cpu_set_t.
+        """
+        chunks = []
+        shiftmask = (2 ** 64) - 1
+        for x in range(16):
+            chunks.append(struct.pack('<Q', mask & shiftmask))
+            mask >>= 64
+        return b''.join(chunks)
+
+    def _get_thread_ids(self):
+        try:
+            ents = os.listdir('/proc/self/task')
+        except OSError:
+            LOG.debug('cannot fetch thread IDs for current process')
+            return [os.getpid()]
+
+        return [int(s) for s in ents if s.isdigit()]
+
+    def _set_cpu_mask(self, mask):
+        s = self._mask_to_bytes(mask)
+        for tid in self._get_thread_ids():
+            _sched_setaffinity(tid, len(s), s)
+
+
+if _sched_setaffinity is not None:
+    policy = LinuxPolicy()
+else:
+    policy = Policy()

ansible_mitogen/loaders.py

@@ -0,0 +1,86 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+"""
+Stable names for PluginLoader instances across Ansible versions.
+"""
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import ansible.errors
+
+import ansible_mitogen.utils
+
+__all__ = [
+    'action_loader',
+    'become_loader',
+    'connection_loader',
+    'module_loader',
+    'module_utils_loader',
+    'shell_loader',
+    'strategy_loader',
+]
+
+
+ANSIBLE_VERSION_MIN = (2, 10)
+OLD_VERSION_MSG = (
+    "Your version of Ansible (%s) is too old. The oldest version supported by "
+    "Mitogen for Ansible is %s."
+)
+
+
+def assert_supported_release():
+    """
+    Throw AnsibleError with a descriptive message in case of being loaded into
+    an unsupported Ansible release.
+    """
+    v = ansible_mitogen.utils.ansible_version
+    if v[:2] < ANSIBLE_VERSION_MIN:
+        raise ansible.errors.AnsibleError(
+            OLD_VERSION_MSG % (v, ANSIBLE_VERSION_MIN)
+        )
+
+
+# this is the first file our strategy plugins import, so we need to check this here
+# in prior Ansible versions, connection_loader.get_with_context didn't exist, so if a user
+# is trying to load an old Ansible version, we'll fail and error gracefully
+assert_supported_release()
+
+
+from ansible.plugins.loader import action_loader
+from ansible.plugins.loader import become_loader
+from ansible.plugins.loader import connection_loader
+from ansible.plugins.loader import module_loader
+from ansible.plugins.loader import module_utils_loader
+from ansible.plugins.loader import shell_loader
+from ansible.plugins.loader import strategy_loader
+
+# These are original, unwrapped implementations
+action_loader__get_with_context = action_loader.get_with_context
+connection_loader__get_with_context = connection_loader.get_with_context

ansible_mitogen/logging.py

@@ -0,0 +1,127 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import logging
+import os
+
+import ansible.utils.display
+
+import mitogen.utils
+
+
+display = ansible.utils.display.Display()
+
+#: The process name set via :func:`set_process_name`.
+_process_name = None
+
+#: The PID of the process that last called :func:`set_process_name`, so its
+#: value can be ignored in unknown fork children.
+_process_pid = None
+
+
+def set_process_name(name):
+    """
+    Set a name to adorn log messages with.
+    """
+    global _process_name
+    _process_name = name
+
+    global _process_pid
+    _process_pid = os.getpid()
+
+
+class Handler(logging.Handler):
+    """
+    Use Mitogen's log format, but send the result to a Display method.
+    """
+    def __init__(self, normal_method):
+        logging.Handler.__init__(self)
+        self.formatter = mitogen.utils.log_get_formatter()
+        self.normal_method = normal_method
+
+    #: Set of target loggers that produce warnings and errors that spam the
+    #: console needlessly. Their log level is forced to INFO. A better strategy
+    #: may simply be to bury all target logs in DEBUG output, but not by
+    #: overriding their log level as done here.
+    NOISY_LOGGERS = frozenset([
+        'dnf',   # issue #272; warns when a package is already installed.
+        'boto',  # issue #541; normal boto retry logic can cause ERROR logs.
+    ])
+
+    def emit(self, record):
+        mitogen_name = getattr(record, 'mitogen_name', '')
+        if mitogen_name == 'stderr':
+            record.levelno = logging.ERROR
+        if mitogen_name in self.NOISY_LOGGERS and record.levelno >= logging.WARNING:
+            record.levelno = logging.DEBUG
+
+        if _process_pid == os.getpid():
+            process_name = _process_name
+        else:
+            process_name = '?'
+
+        s = '[%-4s %d] %s' % (process_name, os.getpid(), self.format(record))
+        if record.levelno >= logging.ERROR:
+            display.error(s, wrap_text=False)
+        elif record.levelno >= logging.WARNING:
+            display.warning(s, formatted=True)
+        else:
+            self.normal_method(s)
+
+
+def setup():
+    """
+    Install handlers for Mitogen loggers to redirect them into the Ansible
+    display framework. Ansible installs its own logging framework handlers when
+    C.DEFAULT_LOG_PATH is set, therefore disable propagation for our handlers.
+    """
+    l_mitogen = logging.getLogger('mitogen')
+    l_mitogen_io = logging.getLogger('mitogen.io')
+    l_ansible_mitogen = logging.getLogger('ansible_mitogen')
+    l_operon = logging.getLogger('operon')
+
+    for logger in l_mitogen, l_mitogen_io, l_ansible_mitogen, l_operon:
+        logger.handlers = [Handler(display.vvv)]
+        logger.propagate = False
+
+    if display.verbosity > 2:
+        l_ansible_mitogen.setLevel(logging.DEBUG)
+        l_mitogen.setLevel(logging.DEBUG)
+    else:
+        # Mitogen copies the active log level into new children, allowing them
+        # to filter tiny messages before they hit the network, and therefore
+        # before they wake the IO loop. Explicitly setting INFO saves ~4%
+        # running against just the local machine.
+        l_mitogen.setLevel(logging.ERROR)
+        l_ansible_mitogen.setLevel(logging.ERROR)
+
+    if display.verbosity > 3:
+        l_mitogen_io.setLevel(logging.DEBUG)

ansible_mitogen/mixins.py

@@ -0,0 +1,504 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import json
+import logging
+import os
+import pwd
+import random
+import traceback
+
+import ansible
+import ansible.plugins.action
+import ansible.utils.unsafe_proxy
+import ansible.vars.clean
+
+from ansible.module_utils.common.text.converters import to_bytes, to_text
+from ansible.module_utils.six.moves import shlex_quote
+
+import mitogen.core
+import mitogen.select
+
+import ansible_mitogen.connection
+import ansible_mitogen.planner
+import ansible_mitogen.target
+import ansible_mitogen.utils
+import ansible_mitogen.utils.unsafe
+
+
+LOG = logging.getLogger(__name__)
+
+
+class ActionModuleMixin(ansible.plugins.action.ActionBase):
+    """
+    The Mitogen-patched PluginLoader dynamically mixes this into every action
+    class that Ansible attempts to load. It exists to override all the
+    assumptions built into the base action class that should really belong in
+    some middle layer, or at least in the connection layer.
+
+    Functionality is defined here for:
+
+    * Capturing the final set of task variables and giving Connection a chance
+      to update its idea of the correct execution environment, before any
+      attempt is made to call a Connection method. While it's not expected for
+      the interpreter to change on a per-task basis, Ansible permits this, and
+      so it must be supported.
+
+    * Overriding lots of methods that try to call out to shell for mundane
+      reasons, such as copying files around, changing file permissions,
+      creating temporary directories and suchlike.
+
+    * Short-circuiting any use of Ansiballz or related code for executing a
+      module remotely using shell commands and SSH.
+
+    * Short-circuiting most of the logic in dealing with the fact that Ansible
+      always runs become: tasks across at least the SSH user account and the
+      destination user account, and handling the security permission issues
+      that crop up due to this. Mitogen always runs a task completely within
+      the target user account, so it's not a problem for us.
+    """
+    def __init__(self, task, connection, *args, **kwargs):
+        """
+        Verify the received connection is really a Mitogen connection. If not,
+        transmute this instance back into the original unadorned base class.
+
+        This allows running the Mitogen strategy in mixed-target playbooks,
+        where some targets use SSH while others use WinRM or some fancier UNIX
+        connection plug-in. That's because when the Mitogen strategy is active,
+        ActionModuleMixin is unconditionally mixed into any action module that
+        is instantiated, and there is no direct way for the monkey-patch to
+        know what kind of connection will be used upfront.
+        """
+        super(ActionModuleMixin, self).__init__(task, connection, *args, **kwargs)
+        if not isinstance(connection, ansible_mitogen.connection.Connection):
+            _, self.__class__ = type(self).__bases__
+
+        # required for python interpreter discovery
+        connection.templar = self._templar
+
+        self._mitogen_discovering_interpreter = False
+        self._mitogen_interpreter_candidate = None
+        self._mitogen_rediscovered_interpreter = False
+
+    def run(self, tmp=None, task_vars=None):
+        """
+        Override run() to notify Connection of task-specific data, so it has a
+        chance to know e.g. the Python interpreter in use.
+        """
+        self._connection.on_action_run(
+            task_vars=task_vars,
+            delegate_to_hostname=self._task.delegate_to,
+            loader_basedir=self._loader.get_basedir(),
+        )
+        return super(ActionModuleMixin, self).run(tmp, task_vars)
+
+    COMMAND_RESULT = {
+        'rc': 0,
+        'stdout': '',
+        'stdout_lines': [],
+        'stderr': ''
+    }
+
+    def fake_shell(self, func, stdout=False):
+        """
+        Execute a function and decorate its return value in the style of
+        _low_level_execute_command(). This produces a return value that looks
+        like some shell command was run, when really func() was implemented
+        entirely in Python.
+
+        If the function raises :py:class:`mitogen.core.CallError`, this will be
+        translated into a failed shell command with a non-zero exit status.
+
+        :param func:
+            Function invoked as `func()`.
+        :returns:
+            See :py:attr:`COMMAND_RESULT`.
+        """
+        dct = self.COMMAND_RESULT.copy()
+        try:
+            rc = func()
+            if stdout:
+                dct['stdout'] = repr(rc)
+        except mitogen.core.CallError:
+            LOG.exception('While emulating a shell command')
+            dct['rc'] = 1
+            dct['stderr'] = traceback.format_exc()
+
+        return dct
+
+    def _remote_file_exists(self, path):
+        """
+        Determine if `path` exists by directly invoking os.path.exists() in the
+        target user account.
+        """
+        LOG.debug('_remote_file_exists(%r)', path)
+        return self._connection.get_chain().call(
+            ansible_mitogen.target.file_exists,
+            ansible_mitogen.utils.unsafe.cast(path)
+        )
+
+    def _configure_module(self, module_name, module_args, task_vars=None):
+        """
+        Mitogen does not use the Ansiballz framework. This call should never
+        happen when ActionMixin is active, so crash if it does.
+        """
+        assert False, "_configure_module() should never be called."
+
+    def _is_pipelining_enabled(self, module_style, wrap_async=False):
+        """
+        Mitogen does not use SSH pipelining. This call should never happen when
+        ActionMixin is active, so crash if it does.
+        """
+        assert False, "_is_pipelining_enabled() should never be called."
+
+    def _generate_tmp_path(self):
+        return os.path.join(
+            self._connection.get_good_temp_dir(),
+            'ansible_mitogen_action_%016x' % (
+                random.getrandbits(8*8),
+            )
+        )
+
+    def _make_tmp_path(self, remote_user=None):
+        """
+        Create a temporary subdirectory as a child of the temporary directory
+        managed by the remote interpreter.
+        """
+        LOG.debug('_make_tmp_path(remote_user=%r)', remote_user)
+        path = self._generate_tmp_path()
+        LOG.debug('Temporary directory: %r', path)
+        self._connection.get_chain().call_no_reply(os.mkdir, path)
+        self._connection._shell.tmpdir = path
+        return path
+
+    def _remove_tmp_path(self, tmp_path):
+        """
+        Replace the base implementation's invocation of rm -rf, replacing it
+        with a pipelined call to :func:`ansible_mitogen.target.prune_tree`.
+        """
+        LOG.debug('_remove_tmp_path(%r)', tmp_path)
+        if tmp_path is None and ansible_mitogen.utils.ansible_version[:2] >= (2, 6):
+            tmp_path = self._connection._shell.tmpdir  # 06f73ad578d
+        if tmp_path is not None:
+            self._connection.get_chain().call_no_reply(
+                ansible_mitogen.target.prune_tree,
+                tmp_path,
+            )
+        self._connection._shell.tmpdir = None
+
+    def _transfer_data(self, remote_path, data):
+        """
+        Used by the base _execute_module(), and in <2.4 also by the template
+        action module, and probably others.
+        """
+        if data is None and ansible_mitogen.utils.ansible_version[:2] <= (2, 18):
+            data = '{}'
+        if isinstance(data, dict):
+            try:
+                data = json.dumps(data, ensure_ascii=False)
+            except UnicodeDecodeError:
+                data = json.dumps(data)
+        if not isinstance(data, bytes):
+            data = to_bytes(data, errors='surrogate_or_strict')
+
+        LOG.debug('_transfer_data(%r, %s ..%d bytes)',
+                  remote_path, type(data), len(data))
+        self._connection.put_data(remote_path, data)
+        return remote_path
+
+    #: Actions listed here cause :func:`_fixup_perms2` to avoid a needless
+    #: roundtrip, as they modify file modes separately afterwards. This is due
+    #: to the method prototype having a default of `execute=True`.
+    FIXUP_PERMS_RED_HERRING = set(['copy'])
+
+    def _fixup_perms2(self, remote_paths, remote_user=None, execute=True):
+        """
+        Mitogen always executes ActionBase helper methods in the context of the
+        target user account, so it is never necessary to modify permissions
+        except to ensure the execute bit is set if requested.
+        """
+        LOG.debug('_fixup_perms2(%r, remote_user=%r, execute=%r)',
+                  remote_paths, remote_user, execute)
+        if execute and self._task.action not in self.FIXUP_PERMS_RED_HERRING:
+            return self._remote_chmod(remote_paths, mode='u+x')
+        return self.COMMAND_RESULT.copy()
+
+    def _remote_chmod(self, paths, mode, sudoable=False):
+        """
+        Issue an asynchronous set_file_mode() call for every path in `paths`,
+        then format the resulting return value list with fake_shell().
+        """
+        LOG.debug('_remote_chmod(%r, mode=%r, sudoable=%r)',
+                  paths, mode, sudoable)
+        return self.fake_shell(lambda: mitogen.select.Select.all(
+            self._connection.get_chain().call_async(
+                ansible_mitogen.target.set_file_mode,
+                ansible_mitogen.utils.unsafe.cast(path),
+                mode,
+            )
+            for path in paths
+        ))
+
+    def _remote_chown(self, paths, user, sudoable=False):
+        """
+        Issue an asynchronous os.chown() call for every path in `paths`, then
+        format the resulting return value list with fake_shell().
+        """
+        LOG.debug('_remote_chown(%r, user=%r, sudoable=%r)',
+                  paths, user, sudoable)
+        ent = self._connection.get_chain().call(pwd.getpwnam, user)
+        return self.fake_shell(lambda: mitogen.select.Select.all(
+            self._connection.get_chain().call_async(
+                os.chown, path, ent.pw_uid, ent.pw_gid
+            )
+            for path in paths
+        ))
+
+    def _remote_expand_user(self, path, sudoable=True):
+        """
+        Replace the base implementation's attempt to emulate
+        os.path.expanduser() with an actual call to os.path.expanduser().
+
+        :param bool sudoable:
+            If :data:`True`, indicate unqualified tilde ("~" with no username)
+            should be evaluated in the context of the login account, not any
+            become_user.
+        """
+        LOG.debug('_remote_expand_user(%r, sudoable=%r)', path, sudoable)
+        if not path.startswith('~'):
+            # /home/foo -> /home/foo
+            return path
+        if sudoable or not self._connection.become:
+            if path == '~':
+                # ~ -> /home/dmw
+                return self._connection.homedir
+            if path.startswith('~/'):
+                # ~/.ansible -> /home/dmw/.ansible
+                return os.path.join(self._connection.homedir, path[2:])
+        # ~root/.ansible -> /root/.ansible
+        return self._connection.get_chain(use_login=(not sudoable)).call(
+            os.path.expanduser,
+            ansible_mitogen.utils.unsafe.cast(path),
+        )
+
+    def get_task_timeout_secs(self):
+        """
+        Return the task "async:" value, portable across 2.4-2.5.
+        """
+        try:
+            return self._task.async_val
+        except AttributeError:
+            return getattr(self._task, 'async')
+
+    def _set_temp_file_args(self, module_args, wrap_async):
+        # Ansible>2.5 module_utils reuses the action's temporary directory if
+        # one exists. Older versions error if this key is present.
+        if ansible_mitogen.utils.ansible_version[:2] >= (2, 5):
+            if wrap_async:
+                # Sharing is not possible with async tasks, as in that case,
+                # the directory must outlive the action plug-in.
+                module_args['_ansible_tmpdir'] = None
+            else:
+                module_args['_ansible_tmpdir'] = self._connection._shell.tmpdir
+
+        # If _ansible_tmpdir is unset, Ansible>2.6 module_utils will use
+        # _ansible_remote_tmp as the location to create the module's temporary
+        # directory. Older versions error if this key is present.
+        if ansible_mitogen.utils.ansible_version[:2] >= (2, 6):
+            module_args['_ansible_remote_tmp'] = (
+                self._connection.get_good_temp_dir()
+            )
+
+    def _execute_module(self, module_name=None, module_args=None, tmp=None,
+                        task_vars=None, persist_files=False,
+                        delete_remote_tmp=True, wrap_async=False,
+                        ignore_unknown_opts=False,
+                        ):
+        """
+        Collect up a module's execution environment then use it to invoke
+        target.run_module() or helpers.run_module_async() in the target
+        context.
+        """
+        if module_name is None:
+            module_name = self._task.action
+        if module_args is None:
+            module_args = self._task.args
+        if task_vars is None:
+            task_vars = {}
+
+        if ansible_mitogen.utils.ansible_version[:2] >= (2, 17):
+            self._update_module_args(
+                module_name, module_args, task_vars,
+                ignore_unknown_opts=ignore_unknown_opts,
+            )
+        else:
+            self._update_module_args(module_name, module_args, task_vars)
+        env = {}
+        self._compute_environment_string(env)
+        self._set_temp_file_args(module_args, wrap_async)
+
+        # there's a case where if a task shuts down the node and then immediately calls
+        # wait_for_connection, the `ping` test from Ansible won't pass because we lost connection
+        # clearing out context forces a reconnect
+        # see https://github.com/dw/mitogen/issues/655 and Ansible's `wait_for_connection` module for more info
+        if module_name == 'ansible.legacy.ping' and type(self).__name__ == 'wait_for_connection':
+            self._connection.context = None
+
+        self._connection._connect()
+        result = ansible_mitogen.planner.invoke(
+            ansible_mitogen.planner.Invocation(
+                action=self,
+                connection=self._connection,
+                module_name=ansible_mitogen.utils.unsafe.cast(mitogen.core.to_text(module_name)),
+                module_args=ansible_mitogen.utils.unsafe.cast(module_args),
+                task_vars=task_vars,
+                templar=self._templar,
+                env=ansible_mitogen.utils.unsafe.cast(env),
+                wrap_async=wrap_async,
+                timeout_secs=self.get_task_timeout_secs(),
+            )
+        )
+
+        if tmp and delete_remote_tmp and ansible_mitogen.utils.ansible_version[:2] < (2, 5):
+            # Built-in actions expected tmpdir to be cleaned up automatically
+            # on _execute_module().
+            self._remove_tmp_path(tmp)
+
+        # prevents things like discovered_interpreter_* or ansible_discovered_interpreter_* from being set
+        ansible.vars.clean.remove_internal_keys(result)
+
+        # taken from _execute_module of ansible 2.8.6
+        # propagate interpreter discovery results back to the controller
+        if self._discovered_interpreter_key:
+            if result.get('ansible_facts') is None:
+                result['ansible_facts'] = {}
+
+            # only cache discovered_interpreter if we're not running a rediscovery
+            # rediscovery happens in places like docker connections that could have different
+            # python interpreters than the main host
+            if not self._mitogen_rediscovered_interpreter:
+                result['ansible_facts'][self._discovered_interpreter_key] = self._discovered_interpreter
+
+        discovery_warnings = getattr(self, '_discovery_warnings', [])
+        if discovery_warnings:
+            if result.get('warnings') is None:
+                result['warnings'] = []
+            result['warnings'].extend(discovery_warnings)
+
+        discovery_deprecation_warnings = getattr(self, '_discovery_deprecation_warnings', [])
+        if discovery_deprecation_warnings:
+            if result.get('deprecations') is None:
+                result['deprecations'] = []
+            result['deprecations'].extend(discovery_deprecation_warnings)
+
+        return ansible.utils.unsafe_proxy.wrap_var(result)
+
+    def _postprocess_response(self, result):
+        """
+        Apply fixups mimicking ActionBase._execute_module(); this is copied
+        verbatim from action/__init__.py, the guts of _parse_returned_data are
+        garbage and should be removed or reimplemented once tests exist.
+
+        :param dict result:
+            Dictionary with format::
+
+                {
+                    "rc": int,
+                    "stdout": "stdout data",
+                    "stderr": "stderr data"
+                }
+        """
+        if ansible_mitogen.utils.ansible_version[:2] >= (2, 19):
+            data = self._parse_returned_data(result, profile='legacy')
+        else:
+            data = self._parse_returned_data(result)
+
+        # Cutpasted from the base implementation.
+        if 'stdout' in data and 'stdout_lines' not in data:
+            data['stdout_lines'] = (data['stdout'] or u'').splitlines()
+        if 'stderr' in data and 'stderr_lines' not in data:
+            data['stderr_lines'] = (data['stderr'] or u'').splitlines()
+
+        return data
+
+    def _low_level_execute_command(self, cmd, sudoable=True, in_data=None,
+                                   executable=None,
+                                   encoding_errors='surrogate_then_replace',
+                                   chdir=None):
+        """
+        Override the base implementation by simply calling
+        target.exec_command() in the target context.
+        """
+        LOG.debug('_low_level_execute_command(%r, in_data=%r, exe=%r, dir=%r)',
+                  cmd, type(in_data), executable, chdir)
+
+        if executable is None:  # executable defaults to False
+            executable = self._play_context.executable
+        if executable:
+            cmd = executable + ' -c ' + shlex_quote(cmd)
+
+        # TODO: HACK: if finding python interpreter then we need to keep
+        # calling exec_command until we run into the right python we'll use
+        # chicken-and-egg issue, mitogen needs a python to run low_level_execute_command
+        # which is required by Ansible's discover_interpreter function
+        if self._mitogen_discovering_interpreter:
+            possible_pythons = self._mitogen_interpreter_candidates
+        else:
+            # not used, just adding a filler value
+            possible_pythons = ['python']
+
+        for possible_python in possible_pythons:
+            try:
+                self._mitogen_interpreter_candidate = possible_python
+                rc, stdout, stderr = self._connection.exec_command(
+                    cmd, in_data, sudoable, mitogen_chdir=chdir,
+                )
+            except BaseException as exc:
+                # we've reached the last python attempted and failed
+                if possible_python == possible_pythons[-1]:
+                    raise
+                else:
+                    LOG.debug(
+                        '%r._low_level_execute_command: candidate=%r ignored: %s, %r',
+                        self, possible_python, type(exc), exc,
+                    )
+                    continue
+
+        stdout_text = to_text(stdout, errors=encoding_errors)
+        stderr_text = to_text(stderr, errors=encoding_errors)
+
+        return {
+            'rc': rc,
+            'stdout': stdout_text,
+            'stdout_lines': stdout_text.splitlines(),
+            'stderr': stderr_text,
+            'stderr_lines': stderr_text.splitlines(),
+        }

ansible_mitogen/module_finder.py

@@ -0,0 +1,281 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+from __future__ import unicode_literals
+__metaclass__ = type
+
+import collections
+import logging
+import os
+import re
+import sys
+
+try:
+    # Python >= 3.4, PEP 451 ModuleSpec API
+    import importlib.machinery
+    import importlib.util
+except ImportError:
+    # Python < 3.4, PEP 302 Import Hooks
+    import imp
+
+import mitogen.imports
+
+
+LOG = logging.getLogger(__name__)
+PREFIX = 'ansible.module_utils.'
+
+
+# Analog of `importlib.machinery.ModuleSpec` or `pkgutil.ModuleInfo`.
+#   name    Unqualified name of the module.
+#   path    Filesystem path of the module.
+#   kind    One of the constants in `imp`, as returned in `imp.find_module()`
+#   parent  `ansible_mitogen.module_finder.Module` of parent package (if any).
+Module = collections.namedtuple('Module', 'name path kind parent')
+
+
+def get_fullname(module):
+    """
+    Reconstruct a Module's canonical path by recursing through its parents.
+    """
+    bits = [str(module.name)]
+    while module.parent:
+        bits.append(str(module.parent.name))
+        module = module.parent
+    return '.'.join(reversed(bits))
+
+
+def get_code(module):
+    """
+    Compile and return a Module's code object.
+    """
+    fp = open(module.path, 'rb')
+    try:
+        return compile(fp.read(), str(module.name), 'exec')
+    finally:
+        fp.close()
+
+
+def is_pkg(module):
+    """
+    Return :data:`True` if a Module represents a package.
+    """
+    return module.kind == imp.PKG_DIRECTORY
+
+
+def find(name, path=(), parent=None):
+    """
+    Return a Module instance describing the first matching module found on the
+    search path.
+
+    :param str name:
+        Module name.
+    :param list path:
+        List of directory names to search for the module.
+    :param Module parent:
+        Optional module parent.
+    """
+    assert isinstance(path, tuple)
+    head, _, tail = name.partition('.')
+    try:
+        tup = imp.find_module(head, list(path))
+    except ImportError:
+        return parent
+
+    fp, modpath, (suffix, mode, kind) = tup
+    if fp:
+        fp.close()
+
+    if parent and modpath == parent.path:
+        # 'from timeout import timeout', where 'timeout' is a function but also
+        # the name of the module being imported.
+        return None
+
+    if kind == imp.PKG_DIRECTORY:
+        modpath = os.path.join(modpath, '__init__.py')
+
+    module = Module(head, modpath, kind, parent)
+    # TODO: this code is entirely wrong on Python 3.x, but works well enough
+    # for Ansible. We need a new find_child() that only looks in the package
+    # directory, never falling back to the parent search path.
+    if tail and kind == imp.PKG_DIRECTORY:
+        return find_relative(module, tail, path)
+    return module
+
+
+def find_relative(parent, name, path=()):
+    if parent.kind == imp.PKG_DIRECTORY:
+        path = (os.path.dirname(parent.path),) + path
+    return find(name, path, parent=parent)
+
+
+def scan_fromlist(code):
+    """Return an iterator of (level, name) for explicit imports in a code
+    object.
+
+    Not all names identify a module. `from os import name, path` generates
+    `(0, 'os.name'), (0, 'os.path')`, but `os.name` is usually a string.
+
+    >>> src = 'import a; import b.c; from d.e import f; from g import h, i\\n'
+    >>> code = compile(src, '<str>', 'exec')
+    >>> list(scan_fromlist(code))
+    [(0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')]
+    """
+    for level, modname_s, fromlist in mitogen.imports.codeobj_imports(code):
+        for name in fromlist:
+            yield level, str('%s.%s' % (modname_s, name))
+        if not fromlist:
+            yield level, modname_s
+
+
+def walk_imports(code, prefix=None):
+    """Return an iterator of names for implicit parent imports & explicit
+    imports in a code object.
+
+    If a prefix is provided, then only children of that prefix are included.
+    Not all names identify a module. `from os import name, path` generates
+    `'os', 'os.name', 'os.path'`, but `os.name` is usually a string.
+
+    >>> source = 'import a; import b; import b.c; from b.d import e, f\\n'
+    >>> code = compile(source, '<str>', 'exec')
+    >>> list(walk_imports(code))
+    ['a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f']
+    >>> list(walk_imports(code, prefix='b'))
+    ['b.c', 'b.d', 'b.d.e', 'b.d.f']
+    """
+    if prefix is None:
+        prefix = ''
+    pattern = re.compile(r'(^|\.)(\w+)')
+    start = len(prefix)
+    for _, name, fromlist in mitogen.imports.codeobj_imports(code):
+        if not name.startswith(prefix):
+            continue
+        for match in pattern.finditer(name, start):
+            yield name[:match.end()]
+        for leaf in fromlist:
+            yield str('%s.%s' % (name, leaf))
+
+
+def scan(module_name, module_path, search_path):
+    # type: (str, str, list[str]) -> list[(str, str, bool)]
+    """Return a list of (name, path, is_package) for ansible.module_utils
+    imports used by an Ansible module.
+    """
+    log = LOG.getChild('scan')
+    log.debug('%r, %r, %r', module_name, module_path, search_path)
+
+    if sys.version_info >= (3, 4):
+        result = _scan_importlib_find_spec(
+            module_name, module_path, search_path,
+        )
+        log.debug('_scan_importlib_find_spec %r', result)
+    else:
+        result = _scan_imp_find_module(module_name, module_path, search_path)
+        log.debug('_scan_imp_find_module %r', result)
+    return result
+
+
+def _scan_importlib_find_spec(module_name, module_path, search_path):
+    # type: (str, str, list[str]) -> list[(str, str, bool)]
+    module = importlib.machinery.ModuleSpec(
+        module_name, loader=None, origin=module_path,
+    )
+    prefix = importlib.machinery.ModuleSpec(
+        PREFIX.rstrip('.'), loader=None,
+    )
+    prefix.submodule_search_locations = search_path
+    queue = collections.deque([module])
+    specs = {prefix.name: prefix}
+    while queue:
+        spec = queue.popleft()
+        if spec.origin is None:
+            continue
+        try:
+            with open(spec.origin, 'rb') as f:
+                code = compile(f.read(), spec.name, 'exec')
+        except Exception as exc:
+            raise ValueError((exc, module, spec, specs))
+
+        for name in walk_imports(code, prefix.name):
+            if name in specs:
+                continue
+
+            parent_name = name.rpartition('.')[0]
+            parent = specs[parent_name]
+            if parent is None or not parent.submodule_search_locations:
+                specs[name] = None
+                continue
+
+            child = importlib.util._find_spec(
+                name, parent.submodule_search_locations,
+            )
+            if child is None or child.origin is None:
+                specs[name] = None
+                continue
+
+            specs[name] = child
+            queue.append(child)
+
+    del specs[prefix.name]
+    return sorted(
+        (spec.name, spec.origin, spec.submodule_search_locations is not None)
+        for spec in specs.values() if spec is not None
+    )
+
+
+def _scan_imp_find_module(module_name, module_path, search_path):
+    # type: (str, str, list[str]) -> list[(str, str, bool)]
+    module = Module(module_name, module_path, imp.PY_SOURCE, None)
+    stack = [module]
+    seen = set()
+
+    while stack:
+        module = stack.pop(0)
+        for level, fromname in scan_fromlist(get_code(module)):
+            if not fromname.startswith(PREFIX):
+                continue
+
+            imported = find(fromname[len(PREFIX):], search_path)
+            if imported is None or imported in seen:
+                continue
+
+            seen.add(imported)
+            stack.append(imported)
+            parent = imported.parent
+            while parent:
+                fullname = get_fullname(parent)
+                module = Module(fullname, parent.path, parent.kind, None)
+                if module not in seen:
+                    seen.add(module)
+                    stack.append(module)
+                parent = parent.parent
+
+    return sorted(
+        (PREFIX + get_fullname(module), module.path, is_pkg(module))
+        for module in seen
+    )

ansible_mitogen/parsing.py

@@ -0,0 +1,77 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+from __future__ import unicode_literals
+__metaclass__ = type
+
+import mitogen.core
+
+
+def parse_script_interpreter(source):
+    """
+    Parse the script interpreter portion of a UNIX hashbang using the rules
+    Linux uses.
+
+    :param str source: String like "/usr/bin/env python".
+
+    :returns:
+        Tuple of `(interpreter, arg)`, where `intepreter` is the script
+        interpreter and `arg` is its sole argument if present, otherwise
+        :py:data:`None`.
+    """
+    # Find terminating newline. Assume last byte of binprm_buf if absent.
+    nl = source.find(b'\n', 0, 128)
+    if nl == -1:
+        nl = min(128, len(source))
+
+    # Split once on the first run of whitespace. If no whitespace exists,
+    # bits just contains the interpreter filename.
+    bits = source[0:nl].strip().split(None, 1)
+    if len(bits) == 1:
+        return mitogen.core.to_text(bits[0]), None
+    return mitogen.core.to_text(bits[0]), mitogen.core.to_text(bits[1])
+
+
+def parse_hashbang(source):
+    """
+    Parse a UNIX "hashbang line" using the syntax supported by Linux.
+
+    :param str source: String like "#!/usr/bin/env python".
+
+    :returns:
+        Tuple of `(interpreter, arg)`, where `intepreter` is the script
+        interpreter and `arg` is its sole argument if present, otherwise
+        :py:data:`None`.
+    """
+    # Linux requires first 2 bytes with no whitespace, pretty sure it's the
+    # same everywhere. See binfmt_script.c.
+    if not source.startswith(b'#!'):
+        return None, None
+
+    return parse_script_interpreter(source[2:])

ansible_mitogen/planner.py

@@ -0,0 +1,711 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+"""
+Classes to detect each case from [0] and prepare arguments necessary for the
+corresponding Runner class within the target, including preloading requisite
+files/modules known missing.
+
+[0] "Ansible Module Architecture", developing_program_flow_modules.html
+"""
+
+from __future__ import absolute_import, division, print_function
+from __future__ import unicode_literals
+__metaclass__ = type
+
+import json
+import logging
+import os
+import random
+import re
+
+import ansible.collections.list
+import ansible.errors
+import ansible.executor.module_common
+
+import mitogen.core
+import mitogen.select
+import mitogen.service
+
+import ansible_mitogen.loaders
+import ansible_mitogen.parsing
+import ansible_mitogen.target
+import ansible_mitogen.utils.unsafe
+
+
+LOG = logging.getLogger(__name__)
+NO_METHOD_MSG = 'Mitogen: no invocation method found for: '
+NO_INTERPRETER_MSG = 'module (%s) is missing interpreter line'
+# NOTE: Ansible 2.10 no longer has a `.` at the end of NO_MODULE_MSG error
+NO_MODULE_MSG = 'The module %s was not found in configured module paths'
+
+_planner_by_path = {}
+
+
+class Invocation(object):
+    """
+    Collect up a module's execution environment then use it to invoke
+    target.run_module() or helpers.run_module_async() in the target context.
+    """
+    def __init__(self, action, connection, module_name, module_args,
+                 task_vars, templar, env, wrap_async, timeout_secs):
+        #: ActionBase instance invoking the module. Required to access some
+        #: output postprocessing methods that don't belong in ActionBase at
+        #: all.
+        self.action = action
+        #: Ansible connection to use to contact the target. Must be an
+        #: ansible_mitogen connection.
+        self.connection = connection
+        #: Name of the module ('command', 'shell', etc.) to execute.
+        self.module_name = module_name
+        #: Final module arguments.
+        self.module_args = module_args
+        #: Task variables, needed to extract ansible_*_interpreter.
+        self.task_vars = task_vars
+        #: Templar, needed to extract ansible_*_interpreter.
+        self.templar = templar
+        #: Final module environment.
+        self.env = env
+        #: Boolean, if :py:data:`True`, launch the module asynchronously.
+        self.wrap_async = wrap_async
+        #: Integer, if >0, limit the time an asynchronous job may run for.
+        self.timeout_secs = timeout_secs
+        #: Initially ``None``, but set by :func:`invoke`. The path on the
+        #: master to the module's implementation file.
+        self.module_path = None
+        #: Initially ``None``, but set by :func:`invoke`. The raw source or
+        #: binary contents of the module.
+        self._module_source = None
+        #: Initially ``{}``, but set by :func:`invoke`. Optional source to send
+        #: to :func:`propagate_paths_and_modules` to fix Python3.5 relative import errors
+        self._overridden_sources = {}
+        #: Initially ``set()``, but set by :func:`invoke`. Optional source paths to send
+        #: to :func:`propagate_paths_and_modules` to handle loading source dependencies from
+        #: places outside of the main source path, such as collections
+        self._extra_sys_paths = set()
+
+    def get_module_source(self):
+        if self._module_source is None:
+            self._module_source = read_file(self.module_path)
+        return self._module_source
+
+    def __repr__(self):
+        return 'Invocation(module_name=%s)' % (self.module_name,)
+
+
+class Planner(object):
+    """
+    A Planner receives a module name and the contents of its implementation
+    file, indicates whether or not it understands how to run the module, and
+    exports a method to run the module.
+    """
+    def __init__(self, invocation):
+        self._inv = invocation
+
+    @classmethod
+    def detect(cls, path, source):
+        """
+        Return true if the supplied `invocation` matches the module type
+        implemented by this planner.
+        """
+        raise NotImplementedError()
+
+    def should_fork(self):
+        """
+        Asynchronous tasks must always be forked.
+        """
+        return self._inv.wrap_async
+
+    def get_push_files(self):
+        """
+        Return a list of files that should be propagated to the target context
+        using PushFileService. The default implementation pushes nothing.
+        """
+        return []
+
+    def get_module_deps(self):
+        """
+        Return a list of the Python module names imported by the module.
+        """
+        return []
+
+    def get_kwargs(self, **kwargs):
+        """
+        If :meth:`detect` returned :data:`True`, plan for the module's
+        execution, including granting access to or delivering any files to it
+        that are known to be absent, and finally return a dict::
+
+            {
+                # Name of the class from runners.py that implements the
+                # target-side execution of this module type.
+                "runner_name": "...",
+
+                # Remaining keys are passed to the constructor of the class
+                # named by `runner_name`.
+            }
+        """
+        binding = self._inv.connection.get_binding()
+
+        kwargs = ansible_mitogen.utils.unsafe.cast(kwargs)
+        new = dict((mitogen.core.UnicodeType(k), kwargs[k])
+                   for k in kwargs)
+        new.setdefault('good_temp_dir',
+            self._inv.connection.get_good_temp_dir())
+        new.setdefault('cwd', self._inv.connection.get_default_cwd())
+        new.setdefault('extra_env', self._inv.connection.get_default_env())
+        new.setdefault('emulate_tty', True)
+        new.setdefault('service_context', binding.get_child_service_context())
+        return new
+
+    def __repr__(self):
+        return '%s()' % (type(self).__name__,)
+
+
+class BinaryPlanner(Planner):
+    """
+    Binary modules take their arguments and will return data to Ansible in the
+    same way as want JSON modules.
+    """
+    runner_name = 'BinaryRunner'
+
+    @classmethod
+    def detect(cls, path, source):
+        return ansible.executor.module_common._is_binary(source)
+
+    def get_push_files(self):
+        return [mitogen.core.to_text(self._inv.module_path)]
+
+    def get_kwargs(self, **kwargs):
+        return super(BinaryPlanner, self).get_kwargs(
+            runner_name=self.runner_name,
+            module=self._inv.module_name,
+            path=self._inv.module_path,
+            json_args=json.dumps(self._inv.module_args),
+            env=ansible_mitogen.utils.unsafe.cast(self._inv.env),
+            **kwargs
+        )
+
+
+class ScriptPlanner(BinaryPlanner):
+    """
+    Common functionality for script module planners -- handle interpreter
+    detection and rewrite.
+    """
+    def _rewrite_interpreter(self, path):
+        """
+        Given the interpreter path (from the script's hashbang line), return
+        the desired interpreter path. This tries, in order
+
+        1. Look up & render the `ansible_*_interpreter` variable, if set
+        2. Look up the `discovered_interpreter_*` fact, if present
+        3. The unmodified path from the hashbang line.
+
+        :param str path:
+            Absolute path to original interpreter (e.g. '/usr/bin/python').
+
+        :returns:
+            Shell fragment prefix used to execute the script via "/bin/sh -c".
+            While `ansible_*_interpreter` documentation suggests shell isn't
+            involved here, the vanilla implementation uses it and that use is
+            exploited in common playbooks.
+        """
+        interpreter_name = os.path.basename(path).strip()
+        key = u'ansible_%s_interpreter' % interpreter_name
+        try:
+            template = self._inv.task_vars[key]
+        except KeyError:
+            pass
+        else:
+            configured_interpreter = self._inv.templar.template(template)
+            return ansible_mitogen.utils.unsafe.cast(configured_interpreter)
+
+        key = u'discovered_interpreter_%s' % interpreter_name
+        try:
+            discovered_interpreter = self._inv.task_vars['ansible_facts'][key]
+        except KeyError:
+            pass
+        else:
+            return ansible_mitogen.utils.unsafe.cast(discovered_interpreter)
+
+        return path
+
+    def _get_interpreter(self):
+        path, arg = ansible_mitogen.parsing.parse_hashbang(
+            self._inv.get_module_source()
+        )
+        if path is None:
+            raise ansible.errors.AnsibleError(NO_INTERPRETER_MSG % (
+                self._inv.module_name,
+            ))
+
+        fragment = self._rewrite_interpreter(path)
+        if arg:
+            fragment += ' ' + arg
+
+        is_python = path.startswith('python')
+        return fragment, is_python
+
+    def get_kwargs(self, **kwargs):
+        interpreter_fragment, is_python = self._get_interpreter()
+        return super(ScriptPlanner, self).get_kwargs(
+            interpreter_fragment=interpreter_fragment,
+            is_python=is_python,
+            **kwargs
+        )
+
+
+class JsonArgsPlanner(ScriptPlanner):
+    """
+    Script that has its interpreter directive and the task arguments
+    substituted into its source as a JSON string.
+    """
+    runner_name = 'JsonArgsRunner'
+
+    @classmethod
+    def detect(cls, path, source):
+        return ansible.executor.module_common.REPLACER_JSONARGS in source
+
+
+class WantJsonPlanner(ScriptPlanner):
+    """
+    If a module has the string WANT_JSON in it anywhere, Ansible treats it as a
+    non-native module that accepts a filename as its only command line
+    parameter. The filename is for a temporary file containing a JSON string
+    containing the module's parameters. The module needs to open the file, read
+    and parse the parameters, operate on the data, and print its return data as
+    a JSON encoded dictionary to stdout before exiting.
+
+    These types of modules are self-contained entities. As of Ansible 2.1,
+    Ansible only modifies them to change a shebang line if present.
+    """
+    runner_name = 'WantJsonRunner'
+
+    @classmethod
+    def detect(cls, path, source):
+        return b'WANT_JSON' in source
+
+
+class NewStylePlanner(ScriptPlanner):
+    """
+    The Ansiballz framework differs from module replacer in that it uses real
+    Python imports of things in ansible/module_utils instead of merely
+    preprocessing the module.
+    """
+    runner_name = 'NewStyleRunner'
+    MARKER = re.compile(br'from ansible(?:_collections|\.module_utils)\.')
+
+    @classmethod
+    def detect(cls, path, source):
+        return cls.MARKER.search(source) is not None
+
+    def _get_interpreter(self):
+        return None, None
+
+    def get_push_files(self):
+        return super(NewStylePlanner, self).get_push_files() + [
+            mitogen.core.to_text(path)
+            for fullname, path, is_pkg in self.get_module_map()['custom']
+        ]
+
+    def get_module_deps(self):
+        return self.get_module_map()['builtin']
+
+    #: Module names appearing in this set always require forking, usually due
+    #: to some terminal leakage that cannot be worked around in any sane
+    #: manner.
+    ALWAYS_FORK_MODULES = frozenset([
+        'dnf',  # issue #280; py-dnf/hawkey need therapy
+        'firewalld',  # issue #570: ansible module_utils caches dbus conn
+        'ansible.legacy.dnf',  # issue #776
+        'ansible.builtin.dnf', # issue #832
+        'freeipa.ansible_freeipa.ipaautomember', # issue #1216
+        'freeipa.ansible_freeipa.ipaautomountkey',
+        'freeipa.ansible_freeipa.ipaautomountlocation',
+        'freeipa.ansible_freeipa.ipaautomountmap',
+        'freeipa.ansible_freeipa.ipacert',
+        'freeipa.ansible_freeipa.ipaclient_api',
+        'freeipa.ansible_freeipa.ipaclient_fix_ca',
+        'freeipa.ansible_freeipa.ipaclient_fstore',
+        'freeipa.ansible_freeipa.ipaclient_get_otp',
+        'freeipa.ansible_freeipa.ipaclient_ipa_conf',
+        'freeipa.ansible_freeipa.ipaclient_join',
+        'freeipa.ansible_freeipa.ipaclient_set_hostname',
+        'freeipa.ansible_freeipa.ipaclient_setup_automount',
+        'freeipa.ansible_freeipa.ipaclient_setup_certmonger',
+        'freeipa.ansible_freeipa.ipaclient_setup_firefox',
+        'freeipa.ansible_freeipa.ipaclient_setup_krb5',
+        'freeipa.ansible_freeipa.ipaclient_setup_nis',
+        'freeipa.ansible_freeipa.ipaclient_setup_nss',
+        'freeipa.ansible_freeipa.ipaclient_setup_ntp',
+        'freeipa.ansible_freeipa.ipaclient_setup_ssh',
+        'freeipa.ansible_freeipa.ipaclient_setup_sshd',
+        'freeipa.ansible_freeipa.ipaclient_temp_krb5',
+        'freeipa.ansible_freeipa.ipaclient_test',
+        'freeipa.ansible_freeipa.ipaclient_test_keytab',
+        'freeipa.ansible_freeipa.ipaconfig',
+        'freeipa.ansible_freeipa.ipadelegation',
+        'freeipa.ansible_freeipa.ipadnsconfig',
+        'freeipa.ansible_freeipa.ipadnsforwardzone',
+        'freeipa.ansible_freeipa.ipadnsrecord',
+        'freeipa.ansible_freeipa.ipadnszone',
+        'freeipa.ansible_freeipa.ipagroup',
+        'freeipa.ansible_freeipa.ipahbacrule',
+        'freeipa.ansible_freeipa.ipahbacsvc',
+        'freeipa.ansible_freeipa.ipahbacsvcgroup',
+        'freeipa.ansible_freeipa.ipahost',
+        'freeipa.ansible_freeipa.ipahostgroup',
+        'freeipa.ansible_freeipa.idoverridegroup',
+        'freeipa.ansible_freeipa.idoverrideuser',
+        'freeipa.ansible_freeipa.idp',
+        'freeipa.ansible_freeipa.idrange',
+        'freeipa.ansible_freeipa.idview',
+        'freeipa.ansible_freeipa.ipalocation',
+        'freeipa.ansible_freeipa.ipanetgroup',
+        'freeipa.ansible_freeipa.ipapermission',
+        'freeipa.ansible_freeipa.ipaprivilege',
+        'freeipa.ansible_freeipa.ipapwpolicy',
+        'freeipa.ansible_freeipa.iparole',
+        'freeipa.ansible_freeipa.ipaselfservice',
+        'freeipa.ansible_freeipa.ipaserver',
+        'freeipa.ansible_freeipa.ipaservice',
+        'freeipa.ansible_freeipa.ipaservicedelegationrule',
+        'freeipa.ansible_freeipa.ipaservicedelegationtarget',
+        'freeipa.ansible_freeipa.ipasudocmd',
+        'freeipa.ansible_freeipa.ipasudocmdgroup',
+        'freeipa.ansible_freeipa.ipasudorule',
+        'freeipa.ansible_freeipa.ipatopologysegment',
+        'freeipa.ansible_freeipa.ipatopologysuffix',
+        'freeipa.ansible_freeipa.ipatrust',
+        'freeipa.ansible_freeipa.ipauser',
+        'freeipa.ansible_freeipa.ipavault',
+    ])
+
+    def should_fork(self):
+        """
+        In addition to asynchronous tasks, new-style modules should be forked
+        if:
+
+        * the user specifies mitogen_task_isolation=fork, or
+        * the new-style module has a custom module search path, or
+        * the module is known to leak like a sieve.
+        """
+        return (
+            super(NewStylePlanner, self).should_fork() or
+            (self._inv.task_vars.get('mitogen_task_isolation') == 'fork') or
+            (self._inv.module_name in self.ALWAYS_FORK_MODULES) or
+            (len(self.get_module_map()['custom']) > 0)
+        )
+
+    def get_search_path(self):
+        return tuple(
+            path
+            for path in ansible_mitogen.loaders.module_utils_loader._get_paths(
+                subdirs=False
+            )
+        )
+
+    _module_map = None
+
+    def get_module_map(self):
+        if self._module_map is None:
+            binding = self._inv.connection.get_binding()
+            self._module_map = mitogen.service.call(
+                call_context=binding.get_service_context(),
+                service_name='ansible_mitogen.services.ModuleDepService',
+                method_name='scan',
+
+                module_name='ansible_module_%s' % (self._inv.module_name,),
+                module_path=self._inv.module_path,
+                search_path=self.get_search_path(),
+                builtin_path=ansible.executor.module_common._MODULE_UTILS_PATH,
+                context=self._inv.connection.context,
+            )
+        return self._module_map
+
+    def get_kwargs(self):
+        return super(NewStylePlanner, self).get_kwargs(
+            module_map=self.get_module_map(),
+            py_module_name=py_modname_from_path(
+                self._inv.module_name,
+                self._inv.module_path,
+            ),
+        )
+
+
+class ReplacerPlanner(NewStylePlanner):
+    """
+    The Module Replacer framework is the original framework implementing
+    new-style modules. It is essentially a preprocessor (like the C
+    Preprocessor for those familiar with that programming language). It does
+    straight substitutions of specific substring patterns in the module file.
+    There are two types of substitutions.
+
+    * Replacements that only happen in the module file. These are public
+      replacement strings that modules can utilize to get helpful boilerplate
+      or access to arguments.
+
+      "from ansible.module_utils.MOD_LIB_NAME import *" is replaced with the
+      contents of the ansible/module_utils/MOD_LIB_NAME.py. These should only
+      be used with new-style Python modules.
+
+      "#<<INCLUDE_ANSIBLE_MODULE_COMMON>>" is equivalent to
+      "from ansible.module_utils.basic import *" and should also only apply to
+      new-style Python modules.
+
+      "# POWERSHELL_COMMON" substitutes the contents of
+      "ansible/module_utils/powershell.ps1". It should only be used with
+      new-style Powershell modules.
+    """
+    runner_name = 'ReplacerRunner'
+
+    @classmethod
+    def detect(cls, path, source):
+        return ansible.executor.module_common.REPLACER in source
+
+
+class OldStylePlanner(ScriptPlanner):
+    runner_name = 'OldStyleRunner'
+
+    @classmethod
+    def detect(cls, path, source):
+        # Everything else.
+        return True
+
+
+_planners = [
+    BinaryPlanner,
+    # ReplacerPlanner,
+    NewStylePlanner,
+    JsonArgsPlanner,
+    WantJsonPlanner,
+    OldStylePlanner,
+]
+
+
+def py_modname_from_path(name, path):
+    """
+    Fetch the logical name of a new-style module as it might appear in
+    :data:`sys.modules` of the target's Python interpreter.
+
+    * Since Ansible 2.9, modules appearing within a package have the original
+      package hierarchy approximated on the target, enabling relative imports
+      to function correctly. For example, "ansible.modules.system.setup".
+    """
+    try:
+        return ansible.executor.module_common._get_ansible_module_fqn(path)
+    except AttributeError:
+        pass
+    except ValueError:
+        pass
+
+    return 'ansible.modules.' + name
+
+
+def read_file(path):
+    fd = os.open(path, os.O_RDONLY)
+    try:
+        bits = []
+        chunk = True
+        while True:
+            chunk = os.read(fd, 65536)
+            if not chunk:
+                break
+            bits.append(chunk)
+    finally:
+        os.close(fd)
+
+    return b''.join(bits)
+
+
+def _propagate_deps(invocation, planner, context):
+    binding = invocation.connection.get_binding()
+    mitogen.service.call(
+        call_context=binding.get_service_context(),
+        service_name='mitogen.service.PushFileService',
+        method_name='propagate_paths_and_modules',
+
+        context=context,
+        paths=planner.get_push_files(),
+        # modules=planner.get_module_deps(), TODO
+        overridden_sources=invocation._overridden_sources,
+        # needs to be a list because can't unpickle() a set()
+        extra_sys_paths=list(invocation._extra_sys_paths),
+    )
+
+
+def _invoke_async_task(invocation, planner):
+    job_id = '%016x' % random.randint(0, 2**64)
+    context = invocation.connection.spawn_isolated_child()
+    _propagate_deps(invocation, planner, context)
+
+    with mitogen.core.Receiver(context.router) as started_recv:
+        call_recv = context.call_async(
+            ansible_mitogen.target.run_module_async,
+            job_id=job_id,
+            timeout_secs=ansible_mitogen.utils.unsafe.cast(invocation.timeout_secs),
+            started_sender=started_recv.to_sender(),
+            kwargs=planner.get_kwargs(),
+        )
+
+        # Wait for run_module_async() to crash, or for AsyncRunner to indicate
+        # the job file has been written.
+        for msg in mitogen.select.Select([started_recv, call_recv]):
+            if msg.receiver is call_recv:
+                # It can only be an exception.
+                raise msg.unpickle()
+            break
+
+        return {
+            'stdout': json.dumps({
+                # modules/utilities/logic/async_wrapper.py::_run_module().
+                'changed': True,
+                'started': 1,
+                'finished': 0,
+                'ansible_job_id': job_id,
+            })
+        }
+
+
+def _invoke_isolated_task(invocation, planner):
+    context = invocation.connection.spawn_isolated_child()
+    _propagate_deps(invocation, planner, context)
+    try:
+        return context.call(
+            ansible_mitogen.target.run_module,
+            kwargs=planner.get_kwargs(),
+        )
+    finally:
+        context.shutdown()
+
+
+def _get_planner(invocation, source):
+    for klass in _planners:
+        if klass.detect(invocation.module_path, source):
+            LOG.debug(
+                '%r accepted %r (filename %r)',
+                klass, invocation.module_name, invocation.module_path,
+            )
+            return klass
+        LOG.debug('%r rejected %r', klass, invocation.module_name)
+    raise ansible.errors.AnsibleError(NO_METHOD_MSG + repr(invocation))
+
+
+def _fix_py35(invocation, module_source):
+    """
+    super edge case with a relative import error in Python 3.5.1-3.5.3
+    in Ansible's setup module when using Mitogen
+    https://github.com/dw/mitogen/issues/672#issuecomment-636408833
+    We replace a relative import in the setup module with the actual full file path
+    This works in vanilla Ansible but not in Mitogen otherwise
+    """
+    if invocation.module_name in {'ansible.builtin.setup', 'ansible.legacy.setup', 'setup'} and \
+            invocation.module_path not in invocation._overridden_sources:
+        # in-memory replacement of setup module's relative import
+        # would check for just python3.5 and run this then but we don't know the
+        # target python at this time yet
+        # NOTE: another ansible 2.10-specific fix: `from ..module_utils` used to be `from ...module_utils`
+        module_source = module_source.replace(
+            b"from ..module_utils.basic import AnsibleModule",
+            b"from ansible.module_utils.basic import AnsibleModule"
+        )
+        invocation._overridden_sources[invocation.module_path] = module_source
+
+
+def _fix_dnf(invocation, module_source):
+    """
+    Handles edge case where dnf ansible module showed failure due to a missing import in the dnf module.
+    Specifically addresses errors like "Failed loading plugin 'debuginfo-install': module 'dnf' has no attribute 'cli'".
+    https://github.com/mitogen-hq/mitogen/issues/1143
+    This issue is resolved by adding 'dnf.cli' to the import statement in the module source.
+    This works in vanilla Ansible but not in Mitogen otherwise.
+    """
+    if invocation.module_name in {'ansible.builtin.dnf', 'ansible.legacy.dnf', 'dnf'} and \
+            invocation.module_path not in invocation._overridden_sources:
+        module_source = module_source.replace(
+            b"import dnf\n",
+            b"import dnf, dnf.cli\n"
+        )
+        invocation._overridden_sources[invocation.module_path] = module_source
+
+
+def _load_collections(invocation):
+    """
+    Special loader that ensures that `ansible_collections` exist as a module path for import
+    Goes through all collection path possibilities and stores paths to installed collections
+    Stores them on the current invocation to later be passed to the master service
+    """
+    for collection_path in ansible.collections.list.list_collection_dirs():
+        invocation._extra_sys_paths.add(collection_path.decode('utf-8'))
+
+
+def invoke(invocation):
+    """
+    Find a Planner subclass corresponding to `invocation` and use it to invoke
+    the module.
+
+    :param Invocation invocation:
+    :returns:
+        Module return dict.
+    :raises ansible.errors.AnsibleError:
+        Unrecognized/unsupported module type.
+    """
+    path = ansible_mitogen.loaders.module_loader.find_plugin(
+        invocation.module_name,
+        '',
+    )
+    if path is None:
+        raise ansible.errors.AnsibleError(NO_MODULE_MSG % (
+            invocation.module_name,
+        ))
+
+    invocation.module_path = mitogen.core.to_text(path)
+    if invocation.module_path not in _planner_by_path:
+        if 'ansible_collections' in invocation.module_path:
+            _load_collections(invocation)
+
+        module_source = invocation.get_module_source()
+        _fix_py35(invocation, module_source)
+        _fix_dnf(invocation, module_source)
+        _planner_by_path[invocation.module_path] = _get_planner(
+            invocation,
+            module_source
+        )
+
+    planner = _planner_by_path[invocation.module_path](invocation)
+    if invocation.wrap_async:
+        response = _invoke_async_task(invocation, planner)
+    elif planner.should_fork():
+        response = _invoke_isolated_task(invocation, planner)
+    else:
+        _propagate_deps(invocation, planner, invocation.connection.context)
+        response = invocation.connection.get_chain().call(
+            ansible_mitogen.target.run_module,
+            kwargs=planner.get_kwargs(),
+        )
+
+    return invocation.action._postprocess_response(response)

ansible_mitogen/plugins/action/mitogen_fetch.py

@@ -0,0 +1,207 @@
+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+import base64
+from ansible.errors import AnsibleError, AnsibleActionFail, AnsibleActionSkip
+from ansible.module_utils.common.text.converters import to_bytes, to_text
+from ansible.module_utils.six import string_types
+from ansible.module_utils.parsing.convert_bool import boolean
+from ansible.plugins.action import ActionBase
+from ansible.utils.display import Display
+from ansible.utils.hashing import checksum, checksum_s, md5, secure_hash
+from ansible.utils.path import makedirs_safe, is_subpath
+
+display = Display()
+
+
+class ActionModule(ActionBase):
+
+    def run(self, tmp=None, task_vars=None):
+        ''' handler for fetch operations '''
+        if task_vars is None:
+            task_vars = dict()
+
+        result = super(ActionModule, self).run(tmp, task_vars)
+        del tmp  # tmp no longer has any effect
+
+        try:
+            if self._play_context.check_mode:
+                raise AnsibleActionSkip('check mode not (yet) supported for this module')
+
+            source = self._task.args.get('src', None)
+            original_dest = dest = self._task.args.get('dest', None)
+            flat = boolean(self._task.args.get('flat'), strict=False)
+            fail_on_missing = boolean(self._task.args.get('fail_on_missing', True), strict=False)
+            validate_checksum = boolean(self._task.args.get('validate_checksum', True), strict=False)
+
+            msg = ''
+            # validate source and dest are strings FIXME: use basic.py and module specs
+            if not isinstance(source, string_types):
+                msg = "Invalid type supplied for source option, it must be a string"
+
+            if not isinstance(dest, string_types):
+                msg = "Invalid type supplied for dest option, it must be a string"
+
+            if source is None or dest is None:
+                msg = "src and dest are required"
+
+            if msg:
+                raise AnsibleActionFail(msg)
+
+            source = self._connection._shell.join_path(source)
+            source = self._remote_expand_user(source)
+
+            remote_stat = {}
+            remote_checksum = None
+            if True:
+                # Get checksum for the remote file even using become. Mitogen doesn't need slurp.
+                # Follow symlinks because fetch always follows symlinks
+                try:
+                    remote_stat = self._execute_remote_stat(source, all_vars=task_vars, follow=True)
+                except AnsibleError as ae:
+                    result['changed'] = False
+                    result['file'] = source
+                    if fail_on_missing:
+                        result['failed'] = True
+                        result['msg'] = to_text(ae)
+                    else:
+                        result['msg'] = "%s, ignored" % to_text(ae, errors='surrogate_or_replace')
+
+                    return result
+
+                remote_checksum = remote_stat.get('checksum')
+                if remote_stat.get('exists'):
+                    if remote_stat.get('isdir'):
+                        result['failed'] = True
+                        result['changed'] = False
+                        result['msg'] = "remote file is a directory, fetch cannot work on directories"
+
+                        # Historically, these don't fail because you may want to transfer
+                        # a log file that possibly MAY exist but keep going to fetch other
+                        # log files. Today, this is better achieved by adding
+                        # ignore_errors or failed_when to the task.  Control the behaviour
+                        # via fail_when_missing
+                        if not fail_on_missing:
+                            result['msg'] += ", not transferring, ignored"
+                            del result['changed']
+                            del result['failed']
+
+                        return result
+
+            # use slurp if permissions are lacking or privilege escalation is needed
+            remote_data = None
+            if remote_checksum in (None, '1', ''):
+                slurpres = self._execute_module(module_name='ansible.legacy.slurp', module_args=dict(src=source), task_vars=task_vars)
+                if slurpres.get('failed'):
+                    if not fail_on_missing:
+                        result['file'] = source
+                        result['changed'] = False
+                    else:
+                        result.update(slurpres)
+
+                    if 'not found' in slurpres.get('msg', ''):
+                        result['msg'] = "the remote file does not exist, not transferring, ignored"
+                    elif slurpres.get('msg', '').startswith('source is a directory'):
+                        result['msg'] = "remote file is a directory, fetch cannot work on directories"
+
+                    return result
+                else:
+                    if slurpres['encoding'] == 'base64':
+                        remote_data = base64.b64decode(slurpres['content'])
+                    if remote_data is not None:
+                        remote_checksum = checksum_s(remote_data)
+
+            # calculate the destination name
+            if os.path.sep not in self._connection._shell.join_path('a', ''):
+                source = self._connection._shell._unquote(source)
+                source_local = source.replace('\\', '/')
+            else:
+                source_local = source
+
+            # ensure we only use file name, avoid relative paths
+            if not is_subpath(dest, original_dest):
+                # TODO: ? dest = os.path.expanduser(dest.replace(('../','')))
+                raise AnsibleActionFail("Detected directory traversal, expected to be contained in '%s' but got '%s'" % (original_dest, dest))
+
+            if flat:
+                if os.path.isdir(to_bytes(dest, errors='surrogate_or_strict')) and not dest.endswith(os.sep):
+                    raise AnsibleActionFail("dest is an existing directory, use a trailing slash if you want to fetch src into that directory")
+                if dest.endswith(os.sep):
+                    # if the path ends with "/", we'll use the source filename as the
+                    # destination filename
+                    base = os.path.basename(source_local)
+                    dest = os.path.join(dest, base)
+                if not dest.startswith("/"):
+                    # if dest does not start with "/", we'll assume a relative path
+                    dest = self._loader.path_dwim(dest)
+            else:
+                # files are saved in dest dir, with a subdir for each host, then the filename
+                if 'inventory_hostname' in task_vars:
+                    target_name = task_vars['inventory_hostname']
+                else:
+                    target_name = self._play_context.remote_addr
+                dest = "%s/%s/%s" % (self._loader.path_dwim(dest), target_name, source_local)
+
+            dest = os.path.normpath(dest)
+
+            # calculate checksum for the local file
+            local_checksum = checksum(dest)
+
+            if remote_checksum != local_checksum:
+                # create the containing directories, if needed
+                makedirs_safe(os.path.dirname(dest))
+
+                # fetch the file and check for changes
+                if remote_data is None:
+                    self._connection.fetch_file(source, dest)
+                else:
+                    try:
+                        f = open(to_bytes(dest, errors='surrogate_or_strict'), 'wb')
+                        f.write(remote_data)
+                        f.close()
+                    except (IOError, OSError) as e:
+                        raise AnsibleActionFail("Failed to fetch the file: %s" % e)
+                new_checksum = secure_hash(dest)
+                # For backwards compatibility. We'll return None on FIPS enabled systems
+                try:
+                    new_md5 = md5(dest)
+                except ValueError:
+                    new_md5 = None
+
+                if validate_checksum and new_checksum != remote_checksum:
+                    result.update(dict(failed=True, md5sum=new_md5,
+                                       msg="checksum mismatch", file=source, dest=dest, remote_md5sum=None,
+                                       checksum=new_checksum, remote_checksum=remote_checksum))
+                else:
+                    result.update({'changed': True, 'md5sum': new_md5, 'dest': dest,
+                                   'remote_md5sum': None, 'checksum': new_checksum,
+                                   'remote_checksum': remote_checksum})
+            else:
+                # For backwards compatibility. We'll return None on FIPS enabled systems
+                try:
+                    local_md5 = md5(dest)
+                except ValueError:
+                    local_md5 = None
+                result.update(dict(changed=False, md5sum=local_md5, file=source, dest=dest, checksum=local_checksum))
+
+        finally:
+            self._remove_tmp_path(self._connection._shell.tmpdir)
+
+        return result

ansible_mitogen/plugins/action/mitogen_get_stack.py

@@ -0,0 +1,58 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+"""
+Fetch the connection configuration stack that would be used to connect to a
+target, without actually connecting to it.
+"""
+
+from __future__ import absolute_import, division, print_function
+from __future__ import unicode_literals
+__metaclass__ = type
+
+import ansible_mitogen.connection
+
+from ansible.plugins.action import ActionBase
+
+
+class ActionModule(ActionBase):
+    def run(self, tmp=None, task_vars=None):
+        if not isinstance(self._connection,
+                          ansible_mitogen.connection.Connection):
+            return {
+                'skipped': True,
+            }
+
+        _, stack = self._connection._build_stack()
+        return {
+            'changed': True,
+            'result': stack,
+            '_ansible_verbose_always': True,
+            # for ansible < 2.8, we'll default to /usr/bin/python like before
+            'discovered_interpreter': self._connection._action._discovered_interpreter
+        }

ansible_mitogen/plugins/connection/mitogen_buildah.py

@@ -0,0 +1,44 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+try:
+    import ansible_mitogen
+except ImportError:
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
+
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'buildah'

ansible_mitogen/plugins/connection/mitogen_doas.py

@@ -0,0 +1,44 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+try:
+    import ansible_mitogen
+except ImportError:
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
+
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'mitogen_doas'

ansible_mitogen/plugins/connection/mitogen_docker.py

@@ -0,0 +1,51 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+try:
+    import ansible_mitogen
+except ImportError:
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
+
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'docker'
+
+    @property
+    def docker_cmd(self):
+        """
+        Ansible 2.3 synchronize module wants to know how we run Docker.
+        """
+        return 'docker'

ansible_mitogen/plugins/connection/mitogen_jail.py

@@ -0,0 +1,44 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+try:
+    import ansible_mitogen
+except ImportError:
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
+
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'jail'

ansible_mitogen/plugins/connection/mitogen_kubectl.py

@@ -0,0 +1,73 @@
+# coding: utf-8
+# Copyright 2018, Yannig Perré
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+import ansible.errors
+
+try:
+    import ansible_mitogen
+except ImportError:
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
+
+import ansible_mitogen.connection
+import ansible_mitogen.loaders
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'kubectl'
+    (vanilla_class, load_context) = ansible_mitogen.loaders.connection_loader__get_with_context(
+        'kubectl',
+        class_only=True,
+    )
+
+    not_supported_msg = (
+        'The "mitogen_kubectl" plug-in requires a version of Ansible '
+        'that ships with the "kubectl" connection plug-in.'
+    )
+
+    def __init__(self, *args, **kwargs):
+        if not Connection.vanilla_class:
+            raise ansible.errors.AnsibleConnectionFailure(self.not_supported_msg)
+        super(Connection, self).__init__(*args, **kwargs)
+
+    def get_extra_args(self):
+        connection_options = Connection.vanilla_class.connection_options
+        parameters = []
+        for key in connection_options:
+            task_var_name = 'ansible_%s' % key
+            task_var = self.get_task_var(task_var_name)
+            if task_var is not None:
+                parameters += [connection_options[key], task_var]
+
+        return parameters

ansible_mitogen/plugins/connection/mitogen_local.py

@@ -0,0 +1,80 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+try:
+    import ansible_mitogen
+except ImportError:
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
+
+import ansible_mitogen.connection
+import ansible_mitogen.process
+
+viewkeys = getattr(dict, 'viewkeys', dict.keys)
+
+
+def dict_diff(old, new):
+    """
+    Return a dict representing the differences between the dicts `old` and
+    `new`. Deleted keys appear as a key with the value :data:`None`, added and
+    changed keys appear as a key with the new value.
+    """
+    old_keys = viewkeys(old)
+    new_keys = viewkeys(dict(new))
+    out = {}
+    for key in new_keys - old_keys:
+        out[key] = new[key]
+    for key in old_keys - new_keys:
+        out[key] = None
+    for key in old_keys & new_keys:
+        if old[key] != new[key]:
+            out[key] = new[key]
+    return out
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'local'
+
+    def get_default_cwd(self):
+        # https://github.com/ansible/ansible/issues/14489
+        return self.loader_basedir
+
+    def get_default_env(self):
+        """
+        Vanilla Ansible local commands execute with an environment inherited
+        from WorkerProcess, we must emulate that.
+        """
+        return dict_diff(
+            old=ansible_mitogen.process.MuxProcess.cls_original_env,
+            new=os.environ,
+        )

ansible_mitogen/plugins/connection/mitogen_lxc.py

@@ -0,0 +1,44 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+try:
+    import ansible_mitogen
+except ImportError:
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
+
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'lxc'

ansible_mitogen/plugins/connection/mitogen_lxd.py

@@ -0,0 +1,44 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+try:
+    import ansible_mitogen
+except ImportError:
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
+
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'lxd'

ansible_mitogen/plugins/connection/mitogen_machinectl.py

@@ -0,0 +1,44 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+try:
+    import ansible_mitogen
+except ImportError:
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
+
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'machinectl'

ansible_mitogen/plugins/connection/mitogen_podman.py

@@ -0,0 +1,44 @@
+# Copyright 2022, Mitogen contributers
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+try:
+    import ansible_mitogen
+except ImportError:
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
+
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'podman'

ansible_mitogen/plugins/connection/mitogen_setns.py

@@ -0,0 +1,44 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+try:
+    import ansible_mitogen
+except ImportError:
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
+
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'setns'

ansible_mitogen/plugins/connection/mitogen_ssh.py

@@ -0,0 +1,70 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+from ansible.plugins.connection.ssh import (
+    DOCUMENTATION as _ansible_ssh_DOCUMENTATION,
+)
+
+DOCUMENTATION = """
+    name: mitogen_ssh
+    author: David Wilson <dw@botanicus.net>
+    short_description: Connect over SSH via Mitogen
+    description:
+        - This connects using an OpenSSH client controlled by the Mitogen for
+          Ansible extension. It accepts every option the vanilla ssh plugin
+          accepts.
+    options:
+""" + _ansible_ssh_DOCUMENTATION.partition('options:\n')[2]
+
+try:
+    import ansible_mitogen
+except ImportError:
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
+
+import ansible_mitogen.connection
+import ansible_mitogen.loaders
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'ssh'
+    (vanilla_class, load_context) = ansible_mitogen.loaders.connection_loader__get_with_context(
+        'ssh',
+        class_only=True,
+    )
+
+    @staticmethod
+    def _create_control_path(*args, **kwargs):
+        """Forward _create_control_path() to the implementation in ssh.py."""
+        # https://github.com/dw/mitogen/issues/342
+        return Connection.vanilla_class._create_control_path(*args, **kwargs)

ansible_mitogen/plugins/connection/mitogen_su.py

@@ -0,0 +1,44 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+try:
+    import ansible_mitogen
+except ImportError:
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
+
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'mitogen_su'

ansible_mitogen/plugins/connection/mitogen_sudo.py

@@ -0,0 +1,44 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+try:
+    import ansible_mitogen
+except ImportError:
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
+
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'mitogen_sudo'

ansible_mitogen/plugins/strategy/mitogen.py

@@ -0,0 +1,61 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+#
+# This is not the real Strategy implementation module, it simply exists as a
+# proxy to the real module, which is loaded using Python's regular import
+# mechanism, to prevent Ansible's PluginLoader from making up a fake name that
+# results in ansible_mitogen plugin modules being loaded twice: once by
+# PluginLoader with a name like "ansible.plugins.strategy.mitogen", which is
+# stuffed into sys.modules even though attempting to import it will trigger an
+# ImportError, and once under its canonical name, "ansible_mitogen.strategy".
+#
+# Therefore we have a proxy module that imports it under the real name, and
+# sets up the duff PluginLoader-imported module to just contain objects from
+# the real module, so duplicate types don't exist in memory, and things like
+# debuggers and isinstance() work predictably.
+#
+
+try:
+    import ansible_mitogen
+except ImportError:
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
+
+import ansible_mitogen.strategy
+import ansible.plugins.strategy.linear
+
+
+class StrategyModule(ansible_mitogen.strategy.StrategyMixin,
+                     ansible.plugins.strategy.linear.StrategyModule):
+    pass

ansible_mitogen/plugins/strategy/mitogen_free.py

@@ -0,0 +1,62 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+#
+# This is not the real Strategy implementation module, it simply exists as a
+# proxy to the real module, which is loaded using Python's regular import
+# mechanism, to prevent Ansible's PluginLoader from making up a fake name that
+# results in ansible_mitogen plugin modules being loaded twice: once by
+# PluginLoader with a name like "ansible.plugins.strategy.mitogen", which is
+# stuffed into sys.modules even though attempting to import it will trigger an
+# ImportError, and once under its canonical name, "ansible_mitogen.strategy".
+#
+# Therefore we have a proxy module that imports it under the real name, and
+# sets up the duff PluginLoader-imported module to just contain objects from
+# the real module, so duplicate types don't exist in memory, and things like
+# debuggers and isinstance() work predictably.
+#
+
+try:
+    import ansible_mitogen
+except ImportError:
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
+
+import ansible_mitogen.loaders
+import ansible_mitogen.strategy
+
+
+Base = ansible_mitogen.loaders.strategy_loader.get('free', class_only=True)
+
+class StrategyModule(ansible_mitogen.strategy.StrategyMixin, Base):
+    pass

ansible_mitogen/plugins/strategy/mitogen_host_pinned.py

@@ -0,0 +1,67 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+#
+# This is not the real Strategy implementation module, it simply exists as a
+# proxy to the real module, which is loaded using Python's regular import
+# mechanism, to prevent Ansible's PluginLoader from making up a fake name that
+# results in ansible_mitogen plugin modules being loaded twice: once by
+# PluginLoader with a name like "ansible.plugins.strategy.mitogen", which is
+# stuffed into sys.modules even though attempting to import it will trigger an
+# ImportError, and once under its canonical name, "ansible_mitogen.strategy".
+#
+# Therefore we have a proxy module that imports it under the real name, and
+# sets up the duff PluginLoader-imported module to just contain objects from
+# the real module, so duplicate types don't exist in memory, and things like
+# debuggers and isinstance() work predictably.
+#
+
+try:
+    import ansible_mitogen
+except ImportError:
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
+
+import ansible_mitogen.loaders
+import ansible_mitogen.strategy
+
+
+Base = ansible_mitogen.loaders.strategy_loader.get('host_pinned', class_only=True)
+
+if Base is None:
+    raise ImportError(
+        'The host_pinned strategy is only available in Ansible 2.7 or newer.'
+    )
+
+class StrategyModule(ansible_mitogen.strategy.StrategyMixin, Base):
+    pass

ansible_mitogen/plugins/strategy/mitogen_linear.py

@@ -0,0 +1,62 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+#
+# This is not the real Strategy implementation module, it simply exists as a
+# proxy to the real module, which is loaded using Python's regular import
+# mechanism, to prevent Ansible's PluginLoader from making up a fake name that
+# results in ansible_mitogen plugin modules being loaded twice: once by
+# PluginLoader with a name like "ansible.plugins.strategy.mitogen", which is
+# stuffed into sys.modules even though attempting to import it will trigger an
+# ImportError, and once under its canonical name, "ansible_mitogen.strategy".
+#
+# Therefore we have a proxy module that imports it under the real name, and
+# sets up the duff PluginLoader-imported module to just contain objects from
+# the real module, so duplicate types don't exist in memory, and things like
+# debuggers and isinstance() work predictably.
+#
+
+try:
+    import ansible_mitogen
+except ImportError:
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
+
+import ansible_mitogen.loaders
+import ansible_mitogen.strategy
+
+
+Base = ansible_mitogen.loaders.strategy_loader.get('linear', class_only=True)
+
+class StrategyModule(ansible_mitogen.strategy.StrategyMixin, Base):
+    pass

ansible_mitogen/process.py

@@ -0,0 +1,709 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import atexit
+import logging
+import multiprocessing
+import os
+import resource
+import socket
+import signal
+import sys
+
+try:
+    import faulthandler
+except ImportError:
+    faulthandler = None
+
+try:
+    import setproctitle
+except ImportError:
+    setproctitle = None
+
+import mitogen
+import mitogen.core
+import mitogen.debug
+import mitogen.fork
+import mitogen.master
+import mitogen.parent
+import mitogen.service
+import mitogen.unix
+import mitogen.utils
+
+import ansible
+import ansible.constants as C
+import ansible.errors
+
+import ansible_mitogen.logging
+import ansible_mitogen.services
+import ansible_mitogen.affinity
+
+
+LOG = logging.getLogger(__name__)
+
+ANSIBLE_PKG_OVERRIDE = (
+    u"__version__ = %r\n"
+    u"__author__ = %r\n"
+)
+
+MAX_MESSAGE_SIZE = 4096 * 1048576
+
+worker_model_msg = (
+    'Mitogen connection types may only be instantiated when one of the '
+    '"mitogen_*" or "operon_*" strategies are active.'
+)
+
+shutting_down_msg = (
+    'The task worker cannot connect. Ansible may be shutting down, or '
+    'the maximum open files limit may have been exceeded. If this occurs '
+    'midway through a run, please retry after increasing the open file '
+    'limit (ulimit -n). Original error: %s'
+)
+
+
+#: The worker model as configured by the currently running strategy. This is
+#: managed via :func:`get_worker_model` / :func:`set_worker_model` functions by
+#: :class:`StrategyMixin`.
+_worker_model = None
+
+
+#: A copy of the sole :class:`ClassicWorkerModel` that ever exists during a
+#: classic run, as return by :func:`get_classic_worker_model`.
+_classic_worker_model = None
+
+
+def set_worker_model(model):
+    """
+    To remove process model-wiring from
+    :class:`ansible_mitogen.connection.Connection`, it is necessary to track
+    some idea of the configured execution environment outside the connection
+    plug-in.
+
+    That is what :func:`set_worker_model` and :func:`get_worker_model` are for.
+    """
+    global _worker_model
+    assert model is None or _worker_model is None
+    _worker_model = model
+
+
+def get_worker_model():
+    """
+    Return the :class:`WorkerModel` currently configured by the running
+    strategy.
+    """
+    if _worker_model is None:
+        raise ansible.errors.AnsibleConnectionFailure(worker_model_msg)
+    return _worker_model
+
+
+def get_classic_worker_model(**kwargs):
+    """
+    Return the single :class:`ClassicWorkerModel` instance, constructing it if
+    necessary.
+    """
+    global _classic_worker_model
+    assert _classic_worker_model is None or (not kwargs), \
+        "ClassicWorkerModel kwargs supplied but model already constructed"
+
+    if _classic_worker_model is None:
+        _classic_worker_model = ClassicWorkerModel(**kwargs)
+    return _classic_worker_model
+
+
+def getenv_int(key, default=0):
+    """
+    Get an integer-valued environment variable `key`, if it exists and parses
+    as an integer, otherwise return `default`.
+    """
+    try:
+        return int(os.environ.get(key, str(default)))
+    except ValueError:
+        return default
+
+
+def save_pid(name):
+    """
+    When debugging and profiling, it is very annoying to poke through the
+    process list to discover the currently running Ansible and MuxProcess IDs,
+    especially when trying to catch an issue during early startup. So here, if
+    a magic environment variable set, stash them in hidden files in the CWD::
+
+        alias muxpid="cat .ansible-mux.pid"
+        alias anspid="cat .ansible-controller.pid"
+
+        gdb -p $(muxpid)
+        perf top -p $(anspid)
+    """
+    if os.environ.get('MITOGEN_SAVE_PIDS'):
+        with open('.ansible-%s.pid' % (name,), 'w') as fp:
+            fp.write(str(os.getpid()))
+
+
+def setup_pool(pool):
+    """
+    Configure a connection multiplexer's :class:`mitogen.service.Pool` with
+    services accessed by clients and WorkerProcesses.
+    """
+    pool.add(mitogen.service.FileService(router=pool.router))
+    pool.add(mitogen.service.PushFileService(router=pool.router))
+    pool.add(ansible_mitogen.services.ContextService(router=pool.router))
+    pool.add(ansible_mitogen.services.ModuleDepService(pool.router))
+    LOG.debug('Service pool configured: size=%d', pool.size)
+
+
+def _setup_responder(responder):
+    """
+    Configure :class:`mitogen.master.ModuleResponder` to only permit
+    certain packages, and to generate custom responses for certain modules.
+    """
+    responder.whitelist_prefix('ansible')
+    responder.whitelist_prefix('ansible_mitogen')
+
+    # Ansible 2.3 is compatible with Python 2.4 targets, however
+    # ansible/__init__.py is not. Instead, executor/module_common.py writes
+    # out a 2.4-compatible namespace package for unknown reasons. So we
+    # copy it here.
+    responder.add_source_override(
+        fullname='ansible',
+        path=ansible.__file__,
+        source=(ANSIBLE_PKG_OVERRIDE % (
+            ansible.__version__,
+            ansible.__author__,
+        )).encode(),
+        is_pkg=True,
+    )
+
+
+def increase_open_file_limit():
+    """
+    #549: in order to reduce the possibility of hitting an open files limit,
+    increase :data:`resource.RLIMIT_NOFILE` from its soft limit to its hard
+    limit, if they differ.
+
+    It is common that a low soft limit is configured by default, where the hard
+    limit is much higher.
+    """
+    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    if hard == resource.RLIM_INFINITY:
+        hard_s = '(infinity)'
+        # cap in case of O(RLIMIT_NOFILE) algorithm in some subprocess.
+        hard = 524288
+    else:
+        hard_s = str(hard)
+
+    LOG.debug('inherited open file limits: soft=%d hard=%s', soft, hard_s)
+    if soft >= hard:
+        LOG.debug('max open files already set to hard limit: %d', hard)
+        return
+
+    # OS X is limited by kern.maxfilesperproc sysctl, rather than the
+    # advertised unlimited hard RLIMIT_NOFILE. Just hard-wire known defaults
+    # for that sysctl, to avoid the mess of querying it.
+    for value in (hard, 10240):
+        try:
+            resource.setrlimit(resource.RLIMIT_NOFILE, (value, hard))
+            LOG.debug('raised soft open file limit from %d to %d', soft, value)
+            break
+        except ValueError as e:
+            LOG.debug('could not raise soft open file limit from %d to %d: %s',
+                      soft, value, e)
+
+
+def common_setup(enable_affinity=True, _init_logging=True):
+    save_pid('controller')
+    ansible_mitogen.logging.set_process_name('top')
+
+    if _init_logging:
+        ansible_mitogen.logging.setup()
+
+    if enable_affinity:
+        ansible_mitogen.affinity.policy.assign_controller()
+
+    mitogen.utils.setup_gil()
+    if faulthandler is not None:
+        faulthandler.enable()
+
+    MuxProcess.profiling = getenv_int('MITOGEN_PROFILING') > 0
+    if MuxProcess.profiling:
+        mitogen.core.enable_profiling()
+
+    MuxProcess.cls_original_env = dict(os.environ)
+    increase_open_file_limit()
+
+
+def get_cpu_count(default=None):
+    """
+    Get the multiplexer CPU count from the MITOGEN_CPU_COUNT environment
+    variable, returning `default` if one isn't set, or is out of range.
+
+    :param int default:
+        Default CPU, or :data:`None` to use all available CPUs.
+    """
+    max_cpus = multiprocessing.cpu_count()
+    if default is None:
+        default = max_cpus
+
+    cpu_count = getenv_int('MITOGEN_CPU_COUNT', default=default)
+    if cpu_count < 1 or cpu_count > max_cpus:
+        cpu_count = default
+
+    return cpu_count
+
+
+class Broker(mitogen.master.Broker):
+    """
+    WorkerProcess maintains fewer file descriptors, therefore does not need
+    the exuberant syscall expense of EpollPoller, so override it and restore
+    the poll() poller.
+    """
+    poller_class = mitogen.parent.POLLER_LIGHTWEIGHT
+
+
+class Binding(object):
+    """
+    Represent a bound connection for a particular inventory hostname. When
+    operating in sharded mode, the actual MuxProcess implementing a connection
+    varies according to the target machine. Depending on the particular
+    implementation, this class represents a binding to the correct MuxProcess.
+    """
+    def get_child_service_context(self):
+        """
+        Return the :class:`mitogen.core.Context` to which children should
+        direct requests for services such as FileService, or :data:`None` for
+        the local process.
+
+        This can be different from :meth:`get_service_context` where MuxProcess
+        and WorkerProcess are combined, and it is discovered a task is
+        delegated after being assigned to its initial worker for the original
+        un-delegated hostname. In that case, connection management and
+        expensive services like file transfer must be implemented by the
+        MuxProcess connected to the target, rather than routed to the
+        MuxProcess responsible for executing the task.
+        """
+        raise NotImplementedError()
+
+    def get_service_context(self):
+        """
+        Return the :class:`mitogen.core.Context` to which this process should
+        direct ContextService requests, or :data:`None` for the local process.
+        """
+        raise NotImplementedError()
+
+    def close(self):
+        """
+        Finalize any associated resources.
+        """
+        raise NotImplementedError()
+
+
+class WorkerModel(object):
+    """
+    Interface used by StrategyMixin to manage various Mitogen services, by
+    default running in one or more connection multiplexer subprocesses spawned
+    off the top-level Ansible process.
+    """
+    def on_strategy_start(self):
+        """
+        Called prior to strategy start in the top-level process. Responsible
+        for preparing any worker/connection multiplexer state.
+        """
+        raise NotImplementedError()
+
+    def on_strategy_complete(self):
+        """
+        Called after strategy completion in the top-level process. Must place
+        Ansible back in a "compatible" state where any other strategy plug-in
+        may execute.
+        """
+        raise NotImplementedError()
+
+    def get_binding(self, inventory_name):
+        """
+        Return a :class:`Binding` to access Mitogen services for
+        `inventory_name`. Usually called from worker processes, but may also be
+        called from top-level process to handle "meta: reset_connection".
+        """
+        raise NotImplementedError()
+
+
+class ClassicBinding(Binding):
+    """
+    Only one connection may be active at a time in a classic worker, so its
+    binding just provides forwarders back to :class:`ClassicWorkerModel`.
+    """
+    def __init__(self, model):
+        self.model = model
+
+    def get_service_context(self):
+        """
+        See Binding.get_service_context().
+        """
+        return self.model.parent
+
+    def get_child_service_context(self):
+        """
+        See Binding.get_child_service_context().
+        """
+        return self.model.parent
+
+    def close(self):
+        """
+        See Binding.close().
+        """
+        self.model.on_binding_close()
+
+
+class ClassicWorkerModel(WorkerModel):
+    #: In the top-level process, this references one end of a socketpair(),
+    #: whose other end child MuxProcesses block reading from to determine when
+    #: the master process dies. When the top-level exits abnormally, or
+    #: normally but where :func:`_on_process_exit` has been called, this socket
+    #: will be closed, causing all the children to wake.
+    parent_sock = None
+
+    #: In the mux process, this is the other end of :attr:`cls_parent_sock`.
+    #: The main thread blocks on a read from it until :attr:`cls_parent_sock`
+    #: is closed.
+    child_sock = None
+
+    #: mitogen.master.Router for this worker.
+    router = None
+
+    #: mitogen.master.Broker for this worker.
+    broker = None
+
+    #: Name of multiplexer process socket we are currently connected to.
+    listener_path = None
+
+    #: mitogen.parent.Context representing the parent Context, which is the
+    #: connection multiplexer process when running in classic mode, or the
+    #: top-level process when running a new-style mode.
+    parent = None
+
+    def __init__(self, _init_logging=True):
+        """
+        Arrange for classic model multiplexers to be started. The parent choses
+        UNIX socket paths each child will use prior to fork, creates a
+        socketpair used essentially as a semaphore, then blocks waiting for the
+        child to indicate the UNIX socket is ready for use.
+
+        :param bool _init_logging:
+            For testing, if :data:`False`, don't initialize logging.
+        """
+        # #573: The process ID that installed the :mod:`atexit` handler. If
+        # some unknown Ansible plug-in forks the Ansible top-level process and
+        # later performs a graceful Python exit, it may try to wait for child
+        # PIDs it never owned, causing a crash. We want to avoid that.
+        self._pid = os.getpid()
+
+        common_setup(_init_logging=_init_logging)
+
+        self.parent_sock, self.child_sock = mitogen.core.socketpair()
+        mitogen.core.set_cloexec(self.parent_sock.fileno())
+        mitogen.core.set_cloexec(self.child_sock.fileno())
+
+        self._muxes = [
+            MuxProcess(self, index)
+            for index in range(get_cpu_count(default=1))
+        ]
+        for mux in self._muxes:
+            mux.start()
+
+        atexit.register(self._on_process_exit)
+        self.child_sock.close()
+        self.child_sock = None
+
+    def _listener_for_name(self, name):
+        """
+        Given an inventory hostname, return the UNIX listener that should
+        communicate with it. This is a simple hash of the inventory name.
+        """
+        mux = self._muxes[abs(hash(name)) % len(self._muxes)]
+        LOG.debug('will use multiplexer %d (%s) to connect to "%s"',
+                  mux.index, mux.path, name)
+        return mux.path
+
+    def _reconnect(self, path):
+        if self.router is not None:
+            # Router can just be overwritten, but the previous parent
+            # connection must explicitly be removed from the broker first.
+            self.router.disconnect(self.parent)
+            self.parent = None
+            self.router = None
+
+        try:
+            self.router, self.parent = mitogen.unix.connect(
+                path=path,
+                broker=self.broker,
+            )
+        except mitogen.unix.ConnectError as e:
+            # This is not AnsibleConnectionFailure since we want to break
+            # with_items loops.
+            raise ansible.errors.AnsibleError(shutting_down_msg % (e,))
+
+        self.router.max_message_size = MAX_MESSAGE_SIZE
+        self.listener_path = path
+
+    def _on_process_exit(self):
+        """
+        This is an :mod:`atexit` handler installed in the top-level process.
+
+        Shut the write end of `sock`, causing the receive side of the socket in
+        every :class:`MuxProcess` to return 0-byte reads, and causing their
+        main threads to wake and initiate shutdown. After shutting the socket
+        down, wait on each child to finish exiting.
+
+        This is done using :mod:`atexit` since Ansible lacks any better hook to
+        run code during exit, and unless some synchronization exists with
+        MuxProcess, debug logs may appear on the user's terminal *after* the
+        prompt has been printed.
+        """
+        if self._pid != os.getpid():
+            return
+
+        try:
+            self.parent_sock.shutdown(socket.SHUT_WR)
+        except socket.error:
+            # Already closed. This is possible when tests are running.
+            LOG.debug('_on_process_exit: ignoring duplicate call')
+            return
+
+        mitogen.core.io_op(self.parent_sock.recv, 1)
+        self.parent_sock.close()
+
+        for mux in self._muxes:
+            _, status = os.waitpid(mux.pid, 0)
+            status = mitogen.fork._convert_exit_status(status)
+            LOG.debug('multiplexer %d PID %d %s', mux.index, mux.pid,
+                      mitogen.parent.returncode_to_str(status))
+
+    def _test_reset(self):
+        """
+        Used to clean up in unit tests.
+        """
+        self.on_binding_close()
+        self._on_process_exit()
+        set_worker_model(None)
+
+        global _classic_worker_model
+        _classic_worker_model = None
+
+    def on_strategy_start(self):
+        """
+        See WorkerModel.on_strategy_start().
+        """
+
+    def on_strategy_complete(self):
+        """
+        See WorkerModel.on_strategy_complete().
+        """
+
+    def get_binding(self, inventory_name):
+        """
+        See WorkerModel.get_binding().
+        """
+        if self.broker is None:
+            self.broker = Broker()
+
+        path = self._listener_for_name(inventory_name)
+        if path != self.listener_path:
+            self._reconnect(path)
+
+        return ClassicBinding(self)
+
+    def on_binding_close(self):
+        if not self.broker:
+            return
+
+        self.broker.shutdown()
+        self.broker.join()
+        self.router = None
+        self.broker = None
+        self.parent = None
+        self.listener_path = None
+
+        # #420: Ansible executes "meta" actions in the top-level process,
+        # meaning "reset_connection" will cause :class:`mitogen.core.Latch` FDs
+        # to be cached and erroneously shared by children on subsequent
+        # WorkerProcess forks. To handle that, call on_fork() to ensure any
+        # shared state is discarded.
+        # #490: only attempt to clean up when it's known that some resources
+        # exist to cleanup, otherwise later __del__ double-call to close() due
+        # to GC at random moment may obliterate an unrelated Connection's
+        # related resources.
+        mitogen.fork.on_fork()
+
+
+class MuxProcess(object):
+    """
+    Implement a subprocess forked from the Ansible top-level, as a safe place
+    to contain the Mitogen IO multiplexer thread, keeping its use of the
+    logging package (and the logging package's heavy use of locks) far away
+    from os.fork(), which is used continuously by the multiprocessing package
+    in the top-level process.
+
+    The problem with running the multiplexer in that process is that should the
+    multiplexer thread be in the process of emitting a log entry (and holding
+    its lock) at the point of fork, in the child, the first attempt to log any
+    log entry using the same handler will deadlock the child, as in the memory
+    image the child received, the lock will always be marked held.
+
+    See https://bugs.python.org/issue6721 for a thorough description of the
+    class of problems this worker is intended to avoid.
+    """
+    #: A copy of :data:`os.environ` at the time the multiplexer process was
+    #: started. It's used by mitogen_local.py to find changes made to the
+    #: top-level environment (e.g. vars plugins -- issue #297) that must be
+    #: applied to locally executed commands and modules.
+    cls_original_env = None
+
+    def __init__(self, model, index):
+        #: :class:`ClassicWorkerModel` instance we were created by.
+        self.model = model
+        #: MuxProcess CPU index.
+        self.index = index
+        #: Individual path of this process.
+        self.path = mitogen.unix.make_socket_path()
+
+    def start(self):
+        self.pid = os.fork()
+        if self.pid:
+            # Wait for child to boot before continuing.
+            mitogen.core.io_op(self.model.parent_sock.recv, 1)
+            return
+
+        ansible_mitogen.logging.set_process_name('mux:' + str(self.index))
+        if setproctitle:
+            setproctitle.setproctitle('mitogen mux:%s (%s)' % (
+                self.index,
+                os.path.basename(self.path),
+            ))
+
+        self.model.parent_sock.close()
+        self.model.parent_sock = None
+        try:
+            try:
+                self.worker_main()
+            except Exception:
+                LOG.exception('worker_main() crashed')
+        finally:
+            sys.exit()
+
+    def worker_main(self):
+        """
+        The main function of the mux process: setup the Mitogen broker thread
+        and ansible_mitogen services, then sleep waiting for the socket
+        connected to the parent to be closed (indicating the parent has died).
+        """
+        save_pid('mux')
+
+        # #623: MuxProcess ignores SIGINT because it wants to live until every
+        # Ansible worker process has been cleaned up by
+        # TaskQueueManager.cleanup(), otherwise harmles yet scary warnings
+        # about being unable connect to MuxProess could be printed.
+        signal.signal(signal.SIGINT, signal.SIG_IGN)
+        ansible_mitogen.logging.set_process_name('mux')
+        ansible_mitogen.affinity.policy.assign_muxprocess(self.index)
+
+        self._setup_master()
+        self._setup_services()
+
+        try:
+            # Let the parent know our listening socket is ready.
+            mitogen.core.io_op(self.model.child_sock.send, b'1')
+            # Block until the socket is closed, which happens on parent exit.
+            mitogen.core.io_op(self.model.child_sock.recv, 1)
+        finally:
+            self.broker.shutdown()
+            self.broker.join()
+
+            # Test frameworks living somewhere higher on the stack of the
+            # original parent process may try to catch sys.exit(), so do a C
+            # level exit instead.
+            os._exit(0)
+
+    def _enable_router_debug(self):
+        if 'MITOGEN_ROUTER_DEBUG' in os.environ:
+            self.router.enable_debug()
+
+    def _enable_stack_dumps(self):
+        secs = getenv_int('MITOGEN_DUMP_THREAD_STACKS', default=0)
+        if secs:
+            mitogen.debug.dump_to_logger(secs=secs)
+
+    def _setup_master(self):
+        """
+        Construct a Router, Broker, and mitogen.unix listener
+        """
+        self.broker = mitogen.master.Broker(install_watcher=False)
+        self.router = mitogen.master.Router(
+            broker=self.broker,
+            max_message_size=MAX_MESSAGE_SIZE,
+        )
+        _setup_responder(self.router.responder)
+        mitogen.core.listen(self.broker, 'shutdown', self._on_broker_shutdown)
+        mitogen.core.listen(self.broker, 'exit', self._on_broker_exit)
+        self.listener = mitogen.unix.Listener.build_stream(
+            router=self.router,
+            path=self.path,
+            backlog=C.DEFAULT_FORKS,
+        )
+        self._enable_router_debug()
+        self._enable_stack_dumps()
+
+    def _setup_services(self):
+        """
+        Construct a ContextService and a thread to service requests for it
+        arriving from worker processes.
+        """
+        self.pool = mitogen.service.Pool(
+            router=self.router,
+            size=getenv_int('MITOGEN_POOL_SIZE', default=32),
+        )
+        setup_pool(self.pool)
+
+    def _on_broker_shutdown(self):
+        """
+        Respond to broker shutdown by shutting down the pool. Do not join on it
+        yet, since that would block the broker thread which then cannot clean
+        up pending handlers and connections, which is required for the threads
+        to exit gracefully.
+        """
+        self.pool.stop(join=False)
+
+    def _on_broker_exit(self):
+        """
+        Respond to the broker thread about to exit by finally joining on the
+        pool. This is safe since pools only block in connection attempts, and
+        connection attempts fail with CancelledError when broker shutdown
+        begins.
+        """
+        self.pool.join()

ansible_mitogen/services.py

@@ -0,0 +1,559 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+"""
+Classes in this file define Mitogen 'services' that run (initially) within the
+connection multiplexer process that is forked off the top-level controller
+process.
+
+Once a worker process connects to a multiplexer process
+(Connection._connect()), it communicates with these services to establish new
+connections, grant access to files by children, and register for notification
+when a child has completed a job.
+"""
+
+from __future__ import absolute_import, division, print_function
+from __future__ import unicode_literals
+__metaclass__ = type
+
+import logging
+import os
+import sys
+import threading
+
+import ansible.constants
+
+from ansible.module_utils.six import reraise
+
+import mitogen.core
+import mitogen.service
+import ansible_mitogen.loaders
+import ansible_mitogen.module_finder
+import ansible_mitogen.target
+import ansible_mitogen.utils
+import ansible_mitogen.utils.unsafe
+
+
+LOG = logging.getLogger(__name__)
+
+# Force load of plugin to ensure ConfigManager has definitions loaded. Done
+# during module import to ensure a single-threaded environment; PluginLoader
+# is not thread-safe.
+ansible_mitogen.loaders.shell_loader.get('sh')
+
+
+def _get_candidate_temp_dirs():
+    try:
+        # >=2.5
+        options = ansible.constants.config.get_plugin_options('shell', 'sh')
+        remote_tmp = options.get('remote_tmp') or ansible.constants.DEFAULT_REMOTE_TMP
+        system_tmpdirs = options.get('system_tmpdirs', ('/var/tmp', '/tmp'))
+    except AttributeError:
+        # 2.3
+        remote_tmp = ansible.constants.DEFAULT_REMOTE_TMP
+        system_tmpdirs = ('/var/tmp', '/tmp')
+
+    return ansible_mitogen.utils.unsafe.cast([remote_tmp] + list(system_tmpdirs))
+
+
+def key_from_dict(**kwargs):
+    """
+    Return a unique string representation of a dict as quickly as possible.
+    Used to generated deduplication keys from a request.
+    """
+    out = []
+    stack = [kwargs]
+    while stack:
+        obj = stack.pop()
+        if isinstance(obj, dict):
+            stack.extend(sorted(obj.items()))
+        elif isinstance(obj, (list, tuple)):
+            stack.extend(obj)
+        else:
+            out.append(str(obj))
+    return ''.join(out)
+
+
+class Error(Exception):
+    pass
+
+
+class ContextService(mitogen.service.Service):
+    """
+    Used by workers to fetch the single Context instance corresponding to a
+    connection configuration, creating the matching connection if it does not
+    exist.
+
+    For connection methods and their parameters, see:
+        https://mitogen.readthedocs.io/en/latest/api.html#context-factories
+
+    This concentrates connections in the top-level process, which may become a
+    bottleneck. The bottleneck can be removed using per-CPU connection
+    processes and arranging for the worker to select one according to a hash of
+    the connection parameters (sharding).
+    """
+    max_interpreters = int(os.getenv('MITOGEN_MAX_INTERPRETERS', '20'))
+
+    def __init__(self, *args, **kwargs):
+        super(ContextService, self).__init__(*args, **kwargs)
+        self._lock = threading.Lock()
+        #: Records the :meth:`get` result dict for successful calls, returned
+        #: for identical subsequent calls. Keyed by :meth:`key_from_dict`.
+        self._response_by_key = {}
+        #: List of :class:`mitogen.core.Latch` awaiting the result for a
+        #: particular key.
+        self._latches_by_key = {}
+        #: Mapping of :class:`mitogen.core.Context` -> reference count. Each
+        #: call to :meth:`get` increases this by one. Calls to :meth:`put`
+        #: decrease it by one.
+        self._refs_by_context = {}
+        #: List of contexts in creation order by via= parameter. When
+        #: :attr:`max_interpreters` is reached, the most recently used context
+        #: is destroyed to make room for any additional context.
+        self._lru_by_via = {}
+        #: :func:`key_from_dict` result by Context.
+        self._key_by_context = {}
+        #: Mapping of Context -> parent Context
+        self._via_by_context = {}
+
+    @mitogen.service.expose(mitogen.service.AllowParents())
+    @mitogen.service.arg_spec({
+        'stack': list,
+    })
+    def reset(self, stack):
+        """
+        Return a reference, forcing close and discard of the underlying
+        connection. Used for 'meta: reset_connection' or when some other error
+        is detected.
+
+        :returns:
+            :data:`True` if a connection was found to discard, otherwise
+            :data:`False`.
+        """
+        LOG.debug('%r.reset(%r)', self, stack)
+
+        # this could happen if we have a `shutdown -r` shell command
+        # and then a `wait_for_connection` right afterwards
+        # in this case, we have no stack to disconnect from
+        if not stack:
+            return False
+
+        l = mitogen.core.Latch()
+        context = None
+        with self._lock:
+            for i, spec in enumerate(stack):
+                key = key_from_dict(via=context, **spec)
+                response = self._response_by_key.get(key)
+                if response is None:
+                    LOG.debug('%r: could not find connection to shut down; '
+                              'failed at hop %d', self, i)
+                    return False
+
+                context = response['context']
+
+            mitogen.core.listen(context, 'disconnect', l.put)
+            self._shutdown_unlocked(context)
+
+        # The timeout below is to turn a hang into a crash in case there is any
+        # possible race between 'disconnect' signal subscription, and the child
+        # abruptly disconnecting.
+        l.get(timeout=30.0)
+        return True
+
+    @mitogen.service.expose(mitogen.service.AllowParents())
+    @mitogen.service.arg_spec({
+        'context': mitogen.core.Context
+    })
+    def put(self, context):
+        """
+        Return a reference, making it eligable for recycling once its reference
+        count reaches zero.
+        """
+        LOG.debug('decrementing reference count for %r', context)
+        self._lock.acquire()
+        try:
+            if self._refs_by_context.get(context, 0) == 0:
+                LOG.warning('%r.put(%r): refcount was 0. shutdown_all called?',
+                            self, context)
+                return
+            self._refs_by_context[context] -= 1
+        finally:
+            self._lock.release()
+
+    def _produce_response(self, key, response):
+        """
+        Reply to every waiting request matching a configuration key with a
+        response dictionary, deleting the list of waiters when done.
+
+        :param str key:
+            Result of :meth:`key_from_dict`
+        :param dict response:
+            Response dictionary
+        :returns:
+            Number of waiters that were replied to.
+        """
+        self._lock.acquire()
+        try:
+            latches = self._latches_by_key.pop(key)
+            count = len(latches)
+            for latch in latches:
+                latch.put(response)
+        finally:
+            self._lock.release()
+        return count
+
+    def _forget_context_unlocked(self, context):
+        key = self._key_by_context.get(context)
+        if key is None:
+            LOG.debug('%r: attempt to forget unknown %r', self, context)
+            return
+
+        self._response_by_key.pop(key, None)
+        self._latches_by_key.pop(key, None)
+        self._key_by_context.pop(context, None)
+        self._refs_by_context.pop(context, None)
+        self._via_by_context.pop(context, None)
+        self._lru_by_via.pop(context, None)
+
+    def _shutdown_unlocked(self, context, lru=None, new_context=None):
+        """
+        Arrange for `context` to be shut down, and optionally add `new_context`
+        to the LRU list while holding the lock.
+        """
+        LOG.info('%r._shutdown_unlocked(): shutting down %r', self, context)
+        context.shutdown()
+        via = self._via_by_context.get(context)
+        if via:
+            lru = self._lru_by_via.get(via)
+            if lru:
+                if context in lru:
+                    lru.remove(context)
+                if new_context:
+                    lru.append(new_context)
+        self._forget_context_unlocked(context)
+
+    def _update_lru_unlocked(self, new_context, spec, via):
+        """
+        Update the LRU ("MRU"?) list associated with the connection described
+        by `kwargs`, destroying the most recently created context if the list
+        is full. Finally add `new_context` to the list.
+        """
+        self._via_by_context[new_context] = via
+
+        lru = self._lru_by_via.setdefault(via, [])
+        if len(lru) < self.max_interpreters:
+            lru.append(new_context)
+            return
+
+        for context in reversed(lru):
+            if self._refs_by_context[context] == 0:
+                break
+        else:
+            LOG.warning('via=%r reached maximum number of interpreters, '
+                        'but they are all marked as in-use.', via)
+            return
+
+        self._shutdown_unlocked(context, lru=lru, new_context=new_context)
+
+    def _update_lru(self, new_context, spec, via):
+        self._lock.acquire()
+        try:
+            self._update_lru_unlocked(new_context, spec, via)
+        finally:
+            self._lock.release()
+
+    @mitogen.service.expose(mitogen.service.AllowParents())
+    def dump(self):
+        """
+        For testing, return a list of dicts describing every currently
+        connected context.
+        """
+        return [
+            {
+                'context_name': context.name,
+                'via': getattr(self._via_by_context.get(context),
+                               'name', None),
+                'refs': self._refs_by_context.get(context),
+            }
+            for context, key in sorted(self._key_by_context.items(),
+                                       key=lambda c_k: c_k[0].context_id)
+        ]
+
+    @mitogen.service.expose(mitogen.service.AllowParents())
+    def shutdown_all(self):
+        """
+        For testing use, arrange for all connections to be shut down.
+        """
+        self._lock.acquire()
+        try:
+            for context in list(self._key_by_context):
+                self._shutdown_unlocked(context)
+        finally:
+            self._lock.release()
+
+    def _on_context_disconnect(self, context):
+        """
+        Respond to Context disconnect event by deleting any record of the no
+        longer reachable context.  This method runs in the Broker thread and
+        must not to block.
+        """
+        self._lock.acquire()
+        try:
+            LOG.info('%r: Forgetting %r due to stream disconnect', self, context)
+            self._forget_context_unlocked(context)
+        finally:
+            self._lock.release()
+
+    ALWAYS_PRELOAD = (
+        'ansible.module_utils.basic',
+        'ansible.module_utils.json_utils',
+        'ansible.release',
+        'ansible_mitogen.runner',
+        'ansible_mitogen.target',
+        'mitogen.fork',
+        'mitogen.service',
+    ) + ((
+        'ansible.module_utils._internal._json._profiles._module_legacy_c2m',
+        'ansible.module_utils._internal._json._profiles._module_legacy_m2c',
+        'ansible.module_utils._internal._json._profiles._module_modern_c2m',
+        'ansible.module_utils._internal._json._profiles._module_legacy_m2c',
+    ) if ansible_mitogen.utils.ansible_version[:2] >= (2, 19) else ())
+
+    def _send_module_forwards(self, context):
+        if hasattr(self.router.responder, 'forward_modules'):
+            self.router.responder.forward_modules(context, self.ALWAYS_PRELOAD)
+
+    _candidate_temp_dirs = None
+
+    def _get_candidate_temp_dirs(self):
+        """
+        Return a list of locations to try to create the single temporary
+        directory used by the run. This simply caches the (expensive) plugin
+        load of :func:`_get_candidate_temp_dirs`.
+        """
+        if self._candidate_temp_dirs is None:
+            self._candidate_temp_dirs = _get_candidate_temp_dirs()
+        return self._candidate_temp_dirs
+
+    def _connect(self, key, spec, via=None):
+        """
+        Actual connect implementation. Arranges for the Mitogen connection to
+        be created and enqueues an asynchronous call to start the forked task
+        parent in the remote context.
+
+        :param key:
+            Deduplication key representing the connection configuration.
+        :param spec:
+            Connection specification.
+        :returns:
+            Dict like::
+
+                {
+                    'context': mitogen.core.Context or None,
+                    'via': mitogen.core.Context or None,
+                    'init_child_result': {
+                        'fork_context': mitogen.core.Context,
+                        'home_dir': str or None,
+                    },
+                    'msg': str or None
+                }
+
+            Where `context` is a reference to the newly constructed context,
+            `init_child_result` is the result of executing
+            :func:`ansible_mitogen.target.init_child` in that context, `msg` is
+            an error message and the remaining fields are :data:`None`, or
+            `msg` is :data:`None` and the remaining fields are set.
+        """
+        try:
+            method = getattr(self.router, spec['method'])
+        except AttributeError:
+            raise Error('unsupported method: %(method)s' % spec)
+
+        context = method(via=via, unidirectional=True, **spec['kwargs'])
+        if via and spec.get('enable_lru'):
+            self._update_lru(context, spec, via)
+
+        # Forget the context when its disconnect event fires.
+        mitogen.core.listen(context, 'disconnect',
+            lambda: self._on_context_disconnect(context))
+
+        self._send_module_forwards(context)
+        init_child_result = context.call(
+            ansible_mitogen.target.init_child,
+            log_level=LOG.getEffectiveLevel(),
+            candidate_temp_dirs=self._get_candidate_temp_dirs(),
+        )
+
+        if os.environ.get('MITOGEN_DUMP_THREAD_STACKS'):
+            from mitogen import debug
+            context.call(debug.dump_to_logger)
+
+        self._key_by_context[context] = key
+        self._refs_by_context[context] = 0
+        return {
+            'context': context,
+            'via': via,
+            'init_child_result': init_child_result,
+            'msg': None,
+        }
+
+    def _wait_or_start(self, spec, via=None):
+        latch = mitogen.core.Latch()
+        key = key_from_dict(via=via, **spec)
+        self._lock.acquire()
+        try:
+            response = self._response_by_key.get(key)
+            if response is not None:
+                self._refs_by_context[response['context']] += 1
+                latch.put(response)
+                return latch
+
+            latches = self._latches_by_key.setdefault(key, [])
+            first = len(latches) == 0
+            latches.append(latch)
+        finally:
+            self._lock.release()
+
+        if first:
+            # I'm the first requestee, so I will create the connection.
+            try:
+                response = self._connect(key, spec, via=via)
+                count = self._produce_response(key, response)
+                # Only record the response for non-error results.
+                self._response_by_key[key] = response
+                # Set the reference count to the number of waiters.
+                self._refs_by_context[response['context']] += count
+            except Exception:
+                self._produce_response(key, sys.exc_info())
+
+        return latch
+
+    disconnect_msg = (
+        'Channel was disconnected while connection attempt was in progress; '
+        'this may be caused by an abnormal Ansible exit, or due to an '
+        'unreliable target.'
+    )
+
+    @mitogen.service.expose(mitogen.service.AllowParents())
+    @mitogen.service.arg_spec({
+        'stack': list
+    })
+    def get(self, stack):
+        """
+        Return a Context referring to an established connection with the given
+        configuration, establishing new connections as necessary.
+
+        :param list stack:
+            Connection descriptions. Each element is a dict containing 'method'
+            and 'kwargs' keys describing the Router method and arguments.
+            Subsequent elements are proxied via the previous.
+
+        :returns dict:
+            * context: mitogen.parent.Context or None.
+            * init_child_result: Result of :func:`init_child`.
+            * msg: StreamError exception text or None.
+            * method_name: string failing method name.
+        """
+        via = None
+        for spec in stack:
+            try:
+                result = self._wait_or_start(spec, via=via).get()
+                if isinstance(result, tuple):  # exc_info()
+                    reraise(*result)
+                via = result['context']
+            except mitogen.core.ChannelError:
+                return {
+                    'context': None,
+                    'init_child_result': None,
+                    'method_name': spec['method'],
+                    'msg': self.disconnect_msg,
+                }
+            except mitogen.core.StreamError as e:
+                return {
+                    'context': None,
+                    'init_child_result': None,
+                    'method_name': spec['method'],
+                    'msg': str(e),
+                }
+
+        return result
+
+
+class ModuleDepService(mitogen.service.Service):
+    """
+    Scan a new-style module and produce a cached mapping of module_utils names
+    to their resolved filesystem paths.
+    """
+    invoker_class = mitogen.service.SerializedInvoker
+
+    def __init__(self, *args, **kwargs):
+        super(ModuleDepService, self).__init__(*args, **kwargs)
+        self._cache = {}
+
+    def _get_builtin_names(self, builtin_path, resolved):
+        return [
+            mitogen.core.to_text(fullname)
+            for fullname, path, is_pkg in resolved
+            if os.path.abspath(path).startswith(builtin_path)
+        ]
+
+    def _get_custom_tups(self, builtin_path, resolved):
+        return [
+            (mitogen.core.to_text(fullname),
+             mitogen.core.to_text(path),
+             is_pkg)
+            for fullname, path, is_pkg in resolved
+            if not os.path.abspath(path).startswith(builtin_path)
+        ]
+
+    @mitogen.service.expose(policy=mitogen.service.AllowParents())
+    @mitogen.service.arg_spec({
+        'module_name': mitogen.core.UnicodeType,
+        'module_path': mitogen.core.FsPathTypes,
+        'search_path': tuple,
+        'builtin_path': mitogen.core.FsPathTypes,
+        'context': mitogen.core.Context,
+    })
+    def scan(self, module_name, module_path, search_path, builtin_path, context):
+        key = (module_name, search_path)
+        if key not in self._cache:
+            resolved = ansible_mitogen.module_finder.scan(
+                module_name=module_name,
+                module_path=module_path,
+                search_path=tuple(search_path) + (builtin_path,),
+            )
+            builtin_path = os.path.abspath(builtin_path)
+            builtin = self._get_builtin_names(builtin_path, resolved)
+            custom = self._get_custom_tups(builtin_path, resolved)
+            self._cache[key] = {
+                'builtin': builtin,
+                'custom': custom,
+            }
+        return self._cache[key]

ansible_mitogen/strategy.py

@@ -0,0 +1,397 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import signal
+import threading
+
+try:
+    import setproctitle
+except ImportError:
+    setproctitle = None
+
+import mitogen.core
+import ansible_mitogen.affinity
+import ansible_mitogen.loaders
+import ansible_mitogen.logging
+import ansible_mitogen.mixins
+import ansible_mitogen.process
+
+import ansible.executor.process.worker
+import ansible.template
+import ansible.utils.sentinel
+import ansible.playbook.play_context
+import ansible.plugins.loader
+
+
+def _patch_awx_callback():
+    """
+    issue #400: AWX loads a display callback that suffers from thread-safety
+    issues. Detect the presence of older AWX versions and patch the bug.
+    """
+    # AWX uses sitecustomize.py to force-load this package. If it exists, we're
+    # running under AWX.
+    try:
+        import awx_display_callback.events
+    except ImportError:
+        return
+
+    if hasattr(awx_display_callback.events.EventContext(), '_local'):
+        # Patched version.
+        return
+
+    def patch_add_local(self, **kwargs):
+        tls = vars(self._local)
+        ctx = tls.setdefault('_ctx', {})
+        ctx.update(kwargs)
+
+    awx_display_callback.events.EventContext._local = threading.local()
+    awx_display_callback.events.EventContext.add_local = patch_add_local
+
+_patch_awx_callback()
+
+
+def wrap_action_loader__get_with_context(name, *args, **kwargs):
+    """
+    While the mitogen strategy is active, trap action_loader.get_with_context()
+    calls, augmenting any fetched class with ActionModuleMixin, which replaces
+    various helper methods inherited from ActionBase with implementations that
+    avoid the use of shell fragments wherever possible.
+
+    This is used instead of static subclassing as it generalizes to third party
+    action plugins outside the Ansible tree.
+    """
+    get_kwargs = {'class_only': True}
+    if name in ('fetch',):
+        name = 'mitogen_' + name
+    get_kwargs['collection_list'] = kwargs.pop('collection_list', None)
+
+    (klass, context) = ansible_mitogen.loaders.action_loader__get_with_context(
+        name,
+        **get_kwargs
+    )
+
+    if klass:
+        bases = (ansible_mitogen.mixins.ActionModuleMixin, klass)
+        adorned_klass = type(str(name), bases, {})
+        if kwargs.get('class_only'):
+            return ansible.plugins.loader.get_with_context_result(
+                adorned_klass,
+                context
+            )
+
+        return ansible.plugins.loader.get_with_context_result(
+            adorned_klass(*args, **kwargs),
+            context
+        )
+
+    return ansible.plugins.loader.get_with_context_result(None, context)
+
+
+REDIRECTED_CONNECTION_PLUGINS = (
+    'buildah',
+    'docker',
+    'kubectl',
+    'jail',
+    'local',
+    'lxc',
+    'lxd',
+    'machinectl',
+    'podman',
+    'setns',
+    'ssh',
+)
+
+
+def wrap_connection_loader__get_with_context(name, *args, **kwargs):
+    """
+    While a Mitogen strategy is active, rewrite
+    connection_loader.get_with_context() calls for some transports into
+    requests for a compatible Mitogen transport.
+    """
+    is_play_using_mitogen_connection = None
+    if len(args) > 0 and isinstance(args[0], ansible.playbook.play_context.PlayContext):
+        play_context = args[0]
+        is_play_using_mitogen_connection = play_context.connection in REDIRECTED_CONNECTION_PLUGINS
+
+    # assume true if we're not in a play context since we're using a Mitogen strategy
+    if is_play_using_mitogen_connection is None:
+        is_play_using_mitogen_connection = True
+
+    redirect_connection = name in REDIRECTED_CONNECTION_PLUGINS and is_play_using_mitogen_connection
+    if redirect_connection:
+        name = 'mitogen_' + name
+
+    return ansible_mitogen.loaders.connection_loader__get_with_context(name, *args, **kwargs)
+
+
+def wrap_worker__run(self):
+    """
+    While a Mitogen strategy is active, trap WorkerProcess.run() calls and use
+    the opportunity to set the worker's name in the process list and log
+    output, activate profiling if requested, and bind the worker to a specific
+    CPU.
+    """
+    if setproctitle:
+        setproctitle.setproctitle('worker:%s task:%s' % (
+            self._host.name,
+            self._task.action,
+        ))
+
+    # Ignore parent's attempts to murder us when we still need to write
+    # profiling output.
+    if mitogen.core._profile_hook.__name__ != '_profile_hook':
+        signal.signal(signal.SIGTERM, signal.SIG_IGN)
+
+    ansible_mitogen.logging.set_process_name('task')
+    ansible_mitogen.affinity.policy.assign_worker()
+    return mitogen.core._profile_hook('WorkerProcess',
+        lambda: worker__run(self)
+    )
+
+
+class AnsibleWrappers(object):
+    """
+    Manage add/removal of various Ansible runtime hooks.
+    """
+    def _add_plugin_paths(self):
+        """
+        Add the Mitogen plug-in directories to the ModuleLoader path, avoiding
+        the need for manual configuration.
+        """
+        base_dir = os.path.join(os.path.dirname(__file__), 'plugins')
+        ansible_mitogen.loaders.connection_loader.add_directory(
+            os.path.join(base_dir, 'connection')
+        )
+        ansible_mitogen.loaders.action_loader.add_directory(
+            os.path.join(base_dir, 'action')
+        )
+
+    def _install_wrappers(self):
+        """
+        Install our PluginLoader monkey patches and update global variables
+        with references to the real functions.
+        """
+        ansible_mitogen.loaders.action_loader.get_with_context = wrap_action_loader__get_with_context
+        ansible_mitogen.loaders.connection_loader.get_with_context = wrap_connection_loader__get_with_context
+
+        global worker__run
+        worker__run = ansible.executor.process.worker.WorkerProcess.run
+        ansible.executor.process.worker.WorkerProcess.run = wrap_worker__run
+
+    def _remove_wrappers(self):
+        """
+        Uninstall the PluginLoader monkey patches.
+        """
+        ansible_mitogen.loaders.action_loader.get_with_context = (
+            ansible_mitogen.loaders.action_loader__get_with_context
+        )
+        ansible_mitogen.loaders.connection_loader.get_with_context = (
+            ansible_mitogen.loaders.connection_loader__get_with_context
+        )
+        ansible.executor.process.worker.WorkerProcess.run = worker__run
+
+    def install(self):
+        self._add_plugin_paths()
+        self._install_wrappers()
+
+    def remove(self):
+        self._remove_wrappers()
+
+
+class StrategyMixin(object):
+    """
+    This mix-in enhances any built-in strategy by arranging for an appropriate
+    WorkerModel instance to be constructed as necessary, or for the existing
+    one to be reused.
+
+    The WorkerModel in turn arranges for a connection multiplexer to be started
+    somewhere (by default in an external process), and for WorkerProcesses to
+    grow support for using those top-level services to communicate with remote
+    hosts.
+
+    Mitogen:
+
+        A private Broker IO multiplexer thread is created to dispatch IO
+        between the local Router and any connected streams, including streams
+        connected to Ansible WorkerProcesses, and SSH commands implementing
+        connections to remote machines.
+
+        A Router is created that implements message dispatch to any locally
+        registered handlers, and message routing for remote streams. Router is
+        the junction point through which WorkerProceses and remote SSH contexts
+        can communicate.
+
+        Router additionally adds message handlers for a variety of base
+        services, review the Standard Handles section of the How It Works guide
+        in the documentation.
+
+        A ContextService is installed as a message handler in the connection
+        mutliplexer subprocess and run on a private thread. It is responsible
+        for accepting requests to establish new SSH connections from worker
+        processes, and ensuring precisely one connection exists and is reused
+        for subsequent playbook steps. The service presently runs in a single
+        thread, so to begin with, new SSH connections are serialized.
+
+        Finally a mitogen.unix listener is created through which WorkerProcess
+        can establish a connection back into the connection multiplexer, in
+        order to avail of ContextService. A UNIX listener socket is necessary
+        as there is no more sane mechanism to arrange for IPC between the
+        Router in the connection multiplexer, and the corresponding Router in
+        the worker process.
+
+    Ansible:
+
+        PluginLoader monkey patches are installed to catch attempts to create
+        connection and action plug-ins.
+
+        For connection plug-ins, if the desired method is "local" or "ssh", it
+        is redirected to one of the "mitogen_*" connection plug-ins. That
+        plug-in implements communication via a UNIX socket connection to the
+        connection multiplexer process, and uses ContextService running there
+        to establish a persistent connection to the target.
+
+        For action plug-ins, the original class is looked up as usual, but a
+        new subclass is created dynamically in order to mix-in
+        ansible_mitogen.target.ActionModuleMixin, which overrides many of the
+        methods usually inherited from ActionBase in order to replace them with
+        pure-Python equivalents that avoid the use of shell.
+
+        In particular, _execute_module() is overridden with an implementation
+        that uses ansible_mitogen.target.run_module() executed in the target
+        Context. run_module() implements module execution by importing the
+        module as if it were a normal Python module, and capturing its output
+        in the remote process. Since the Mitogen module loader is active in the
+        remote process, all the heavy lifting of transferring the action module
+        and its dependencies are automatically handled by Mitogen.
+    """
+
+    def _queue_task(self, host, task, task_vars, play_context):
+        """
+        Many PluginLoader caches are defective as they are only populated in
+        the ephemeral WorkerProcess. Touch each plug-in path before forking to
+        ensure all workers receive a hot cache.
+        """
+        ansible_mitogen.loaders.module_loader.find_plugin(
+            name=task.action,
+            mod_type='',
+        )
+        ansible_mitogen.loaders.action_loader.get(
+            name=task.action,
+            class_only=True,
+        )
+        if play_context.connection is not ansible.utils.sentinel.Sentinel:
+            # 2.8 appears to defer computing this until inside the worker.
+            # TODO: figure out where it has moved.
+            ansible_mitogen.loaders.connection_loader.get(
+                name=play_context.connection,
+                class_only=True,
+            )
+
+        return super(StrategyMixin, self)._queue_task(
+            host=host,
+            task=task,
+            task_vars=task_vars,
+            play_context=play_context,
+        )
+
+    def _get_worker_model(self):
+        """
+        In classic mode a single :class:`WorkerModel` exists, which manages
+        references and configuration of the associated connection multiplexer
+        process.
+        """
+        return ansible_mitogen.process.get_classic_worker_model()
+
+    def run(self, iterator, play_context, result=0):
+        """
+        Wrap :meth:`run` to ensure requisite infrastructure and modifications
+        are configured for the duration of the call.
+        """
+        wrappers = AnsibleWrappers()
+        self._worker_model = self._get_worker_model()
+        ansible_mitogen.process.set_worker_model(self._worker_model)
+        try:
+            self._worker_model.on_strategy_start()
+            try:
+                wrappers.install()
+                try:
+                    run = super(StrategyMixin, self).run
+                    return mitogen.core._profile_hook('Strategy',
+                        lambda: run(iterator, play_context)
+                    )
+                finally:
+                    wrappers.remove()
+            finally:
+                self._worker_model.on_strategy_complete()
+        finally:
+            ansible_mitogen.process.set_worker_model(None)
+
+    def _smuggle_to_connection_reset(self, task, play_context, iterator, target_host):
+        """
+        Create a templar and make it available for use in Connection.reset().
+        This allows templated connection variables to be used when Mitogen
+        reconstructs its connection stack.
+        """
+        variables = self._variable_manager.get_vars(
+            play=iterator._play, host=target_host, task=task,
+            _hosts=self._hosts_cache, _hosts_all=self._hosts_cache_all,
+        )
+        templar = ansible.template.Templar(
+            loader=self._loader, variables=variables,
+        )
+
+        # Required for remote_user option set by variable (e.g. ansible_user).
+        # Without it remote_user in ansible.cfg gets used.
+        play_context = play_context.set_task_and_variable_override(
+            task=task, variables=variables, templar=templar,
+        )
+        play_context.post_validate(templar=templar)
+
+        # Required for timeout option set by variable (e.g. ansible_timeout).
+        # Without it the task timeout keyword (default: 0) gets used.
+        play_context.update_vars(variables)
+
+        # Stash the task and templar somewhere Connection.reset() can find it
+        play_context.vars.update({
+            '_mitogen.smuggled.reset_connection': (task, templar),
+        })
+        return play_context
+
+    def _execute_meta(self, task, play_context, iterator, target_host):
+        if task.args['_raw_params'] == 'reset_connection':
+            play_context = self._smuggle_to_connection_reset(
+                task, play_context, iterator, target_host,
+            )
+
+        return super(StrategyMixin, self)._execute_meta(
+            task, play_context, iterator, target_host,
+        )

ansible_mitogen/target.py

@@ -0,0 +1,755 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+"""
+Helper functions intended to be executed on the target. These are entrypoints
+for file transfer, module execution and sundry bits like changing file modes.
+"""
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import errno
+import grp
+import json
+import logging
+import os
+import pty
+import pwd
+import re
+import signal
+import stat
+import subprocess
+import sys
+import tempfile
+import traceback
+import types
+
+import mitogen.core
+import mitogen.parent
+import mitogen.service
+
+# Ansible since PR #41749 inserts "import __main__" into
+# ansible.module_utils.basic. Mitogen's importer will refuse such an import, so
+# we must setup a fake "__main__" before that module is ever imported. The
+# str() is to cast Unicode to bytes on Python 2.6.
+if not sys.modules.get(str('__main__')):
+    sys.modules[str('__main__')] = types.ModuleType(str('__main__'))
+
+import ansible.module_utils.json_utils
+
+import ansible_mitogen.runner
+
+
+LOG = logging.getLogger(__name__)
+
+MAKE_TEMP_FAILED_MSG = (
+    u"Unable to find a useable temporary directory. This likely means no\n"
+    u"system-supplied TMP directory can be written to, or all directories\n"
+    u"were mounted on 'noexec' filesystems.\n"
+    u"\n"
+    u"The following paths were tried:\n"
+    u"    %(paths)s\n"
+    u"\n"
+    u"Please check '-vvv' output for a log of individual path errors."
+)
+
+# Python 2.4/2.5 cannot support fork+threads whatsoever, it doesn't even fix up
+# interpreter state. So 2.4/2.5 interpreters start .local() contexts for
+# isolation instead. Since we don't have any crazy memory sharing problems to
+# avoid, there is no virginal fork parent either. The child is started directly
+# from the login/become process. In future this will be default everywhere,
+# fork is brainwrong from the stone age.
+FORK_SUPPORTED = sys.version_info >= (2, 6)
+
+#: Initialized to an econtext.parent.Context pointing at a pristine fork of
+#: the target Python interpreter before it executes any code or imports.
+_fork_parent = None
+
+#: Set by :func:`init_child` to the name of a writeable and executable
+#: temporary directory accessible by the active user account.
+good_temp_dir = None
+
+
+def subprocess__Popen__close_fds(self, but):
+    """
+    issue #362, #435: subprocess.Popen(close_fds=True) aka.
+    AnsibleModule.run_command() loops the entire FD space on Python<3.2.
+    CentOS>5 ships with 1,048,576 FDs by default, resulting in huge (>500ms)
+    latency starting children. Therefore replace Popen._close_fds on Linux with
+    a version that is O(fds) rather than O(_SC_OPEN_MAX).
+    """
+    try:
+        names = os.listdir(u'/proc/self/fd')
+    except OSError:
+        # May fail if acting on a container that does not have /proc mounted.
+        self._original_close_fds(but)
+        return
+
+    for name in names:
+        if not name.isdigit():
+            continue
+
+        fd = int(name, 10)
+        if fd > pty.STDERR_FILENO and fd != but:
+            try:
+                os.close(fd)
+            except OSError:
+                pass
+
+
+if (
+    sys.platform.startswith(u'linux') and
+    sys.version_info < (3,) and
+    hasattr(subprocess.Popen, u'_close_fds') and
+    not mitogen.is_master
+):
+    subprocess.Popen._original_close_fds = subprocess.Popen._close_fds
+    subprocess.Popen._close_fds = subprocess__Popen__close_fds
+
+
+def get_small_file(context, path):
+    """
+    Basic in-memory caching module fetcher. This generates one roundtrip for
+    every previously unseen file, so it is only a temporary solution.
+
+    :param context:
+        Context we should direct FileService requests to. For now (and probably
+        forever) this is just the top-level Mitogen connection manager process.
+    :param path:
+        Path to fetch from FileService, must previously have been registered by
+        a privileged context using the `register` command.
+    :returns:
+        Bytestring file data.
+    """
+    pool = mitogen.service.get_or_create_pool(router=context.router)
+    service = pool.get_service(u'mitogen.service.PushFileService')
+    return service.get(path)
+
+
+def transfer_file(context, in_path, out_path, sync=False, set_owner=False):
+    """
+    Streamily download a file from the connection multiplexer process in the
+    controller.
+
+    :param mitogen.core.Context context:
+        Reference to the context hosting the FileService that will transmit the
+        file.
+    :param bytes in_path:
+        FileService registered name of the input file.
+    :param bytes out_path:
+        Name of the output path on the local disk.
+    :param bool sync:
+        If :data:`True`, ensure the file content and metadat are fully on disk
+        before renaming the temporary file over the existing file. This should
+        ensure in the case of system crash, either the entire old or new file
+        are visible post-reboot.
+    :param bool set_owner:
+        If :data:`True`, look up the metadata username and group on the local
+        system and file the file owner using :func:`os.fchmod`.
+    """
+    out_path = os.path.abspath(out_path)
+    fd, tmp_path = tempfile.mkstemp(suffix='.tmp',
+                                    prefix='.ansible_mitogen_transfer-',
+                                    dir=os.path.dirname(out_path))
+    fp = os.fdopen(fd, 'wb', mitogen.core.CHUNK_SIZE)
+    LOG.debug('transfer_file(%r) temporary file: %s', out_path, tmp_path)
+
+    try:
+        try:
+            ok, metadata = mitogen.service.FileService.get(
+                context=context,
+                path=in_path,
+                out_fp=fp,
+            )
+            if not ok:
+                raise IOError('transfer of %r was interrupted.' % (in_path,))
+
+            set_file_mode(tmp_path, metadata['mode'], fd=fp.fileno())
+            if set_owner:
+                set_file_owner(tmp_path, metadata['owner'], metadata['group'],
+                               fd=fp.fileno())
+        finally:
+            fp.close()
+
+        if sync:
+            os.fsync(fp.fileno())
+        os.rename(tmp_path, out_path)
+    except BaseException:
+        os.unlink(tmp_path)
+        raise
+
+    os.utime(out_path, (metadata['atime'], metadata['mtime']))
+
+
+def prune_tree(path):
+    """
+    Like shutil.rmtree(), but log errors rather than discard them, and do not
+    waste multiple os.stat() calls discovering whether the object can be
+    deleted, just try deleting it instead.
+    """
+    try:
+        os.unlink(path)
+        return
+    except OSError:
+        e = sys.exc_info()[1]
+        if not (os.path.isdir(path) and
+                e.args[0] in (errno.EPERM, errno.EISDIR)):
+            LOG.error('prune_tree(%r): %s', path, e)
+            return
+
+    try:
+        # Ensure write access for readonly directories. Ignore error in case
+        # path is on a weird filesystem (e.g. vfat).
+        os.chmod(path, int('0700', 8))
+    except OSError:
+        e = sys.exc_info()[1]
+        LOG.warning('prune_tree(%r): %s', path, e)
+
+    try:
+        for name in os.listdir(path):
+            if name not in ('.', '..'):
+                prune_tree(os.path.join(path, name))
+        os.rmdir(path)
+    except OSError:
+        e = sys.exc_info()[1]
+        LOG.error('prune_tree(%r): %s', path, e)
+
+
+def is_good_temp_dir(path):
+    """
+    Return :data:`True` if `path` can be used as a temporary directory, logging
+    any failures that may cause it to be unsuitable. If the directory doesn't
+    exist, we attempt to create it using :func:`os.makedirs`.
+    """
+    if not os.path.exists(path):
+        try:
+            os.makedirs(path, mode=int('0700', 8))
+        except OSError:
+            e = sys.exc_info()[1]
+            LOG.debug('temp dir %r unusable: did not exist and attempting '
+                      'to create it failed: %s', path, e)
+            return False
+
+    try:
+        tmp = tempfile.NamedTemporaryFile(
+            prefix='ansible_mitogen_is_good_temp_dir',
+            dir=path,
+        )
+    except (OSError, IOError):
+        e = sys.exc_info()[1]
+        LOG.debug('temp dir %r unusable: %s', path, e)
+        return False
+
+    try:
+        try:
+            os.chmod(tmp.name, int('0700', 8))
+        except OSError:
+            e = sys.exc_info()[1]
+            LOG.debug('temp dir %r unusable: chmod failed: %s', path, e)
+            return False
+
+        try:
+            # access(.., X_OK) is sufficient to detect noexec.
+            if not os.access(tmp.name, os.X_OK):
+                raise OSError('filesystem appears to be mounted noexec')
+        except OSError:
+            e = sys.exc_info()[1]
+            LOG.debug('temp dir %r unusable: %s', path, e)
+            return False
+    finally:
+        tmp.close()
+
+    return True
+
+
+def find_good_temp_dir(candidate_temp_dirs):
+    """
+    Given a list of candidate temp directories extracted from ``ansible.cfg``,
+    combine it with the Python-builtin list of candidate directories used by
+    :mod:`tempfile`, then iteratively try each until one is found that is both
+    writeable and executable.
+
+    :param list candidate_temp_dirs:
+        List of candidate $variable-expanded and tilde-expanded directory paths
+        that may be usable as a temporary directory.
+    """
+    paths = [os.path.expandvars(os.path.expanduser(p))
+             for p in candidate_temp_dirs]
+    paths.extend(tempfile._candidate_tempdir_list())
+
+    for path in paths:
+        if is_good_temp_dir(path):
+            LOG.debug('Selected temp directory: %r (from %r)', path, paths)
+            return path
+
+    raise IOError(MAKE_TEMP_FAILED_MSG % {
+        'paths': '\n    '.join(paths),
+    })
+
+
+@mitogen.core.takes_econtext
+def init_child(econtext, log_level, candidate_temp_dirs):
+    """
+    Called by ContextService immediately after connection; arranges for the
+    (presently) spotless Python interpreter to be forked, where the newly
+    forked interpreter becomes the parent of any newly forked future
+    interpreters.
+
+    This is necessary to prevent modules that are executed in-process from
+    polluting the global interpreter state in a way that effects explicitly
+    isolated modules.
+
+    :param int log_level:
+        Logging package level active in the master.
+    :param list[str] candidate_temp_dirs:
+        List of $variable-expanded and tilde-expanded directory names to add to
+        candidate list of temporary directories.
+
+    :returns:
+        Dict like::
+
+            {
+                'fork_context': mitogen.core.Context or None,
+                'good_temp_dir': ...
+                'home_dir': str
+            }
+
+        Where `fork_context` refers to the newly forked 'fork parent' context
+        the controller will use to start forked jobs, and `home_dir` is the
+        home directory for the active user account.
+    """
+    # Copying the master's log level causes log messages to be filtered before
+    # they reach LogForwarder, thus reducing an influx of tiny messges waking
+    # the connection multiplexer process in the master.
+    LOG.setLevel(log_level)
+    logging.getLogger('ansible_mitogen').setLevel(log_level)
+
+    global _fork_parent
+    if FORK_SUPPORTED:
+        mitogen.parent.upgrade_router(econtext)
+        _fork_parent = econtext.router.fork()
+
+    global good_temp_dir
+    good_temp_dir = find_good_temp_dir(candidate_temp_dirs)
+
+    return {
+        u'fork_context': _fork_parent,
+        u'home_dir': mitogen.core.to_text(os.path.expanduser('~')),
+        u'good_temp_dir': good_temp_dir,
+    }
+
+
+@mitogen.core.takes_econtext
+def spawn_isolated_child(econtext):
+    """
+    For helper functions executed in the fork parent context, arrange for
+    the context's router to be upgraded as necessary and for a new child to be
+    prepared.
+
+    The actual fork occurs from the 'virginal fork parent', which does not have
+    any Ansible modules loaded prior to fork, to avoid conflicts resulting from
+    custom module_utils paths.
+    """
+    mitogen.parent.upgrade_router(econtext)
+    if FORK_SUPPORTED:
+        context = econtext.router.fork()
+    else:
+        context = econtext.router.local()
+    LOG.debug('create_fork_child() -> %r', context)
+    return context
+
+
+def run_module(kwargs):
+    """
+    Set up the process environment in preparation for running an Ansible
+    module. This monkey-patches the Ansible libraries in various places to
+    prevent it from trying to kill the process on completion, and to prevent it
+    from reading sys.stdin.
+    """
+    runner_name = kwargs.pop('runner_name')
+    klass = getattr(ansible_mitogen.runner, runner_name)
+    impl = klass(**mitogen.core.Kwargs(kwargs))
+    return impl.run()
+
+
+def _get_async_dir():
+    return os.path.expanduser(
+        os.environ.get('ANSIBLE_ASYNC_DIR', '~/.ansible_async')
+    )
+
+
+class AsyncRunner(object):
+    def __init__(self, job_id, timeout_secs, started_sender, econtext, kwargs):
+        self.job_id = job_id
+        self.timeout_secs = timeout_secs
+        self.started_sender = started_sender
+        self.econtext = econtext
+        self.kwargs = kwargs
+        self._timed_out = False
+        self._init_path()
+
+    def _init_path(self):
+        async_dir = _get_async_dir()
+        if not os.path.exists(async_dir):
+            os.makedirs(async_dir)
+        self.path = os.path.join(async_dir, self.job_id)
+
+    def _update(self, dct):
+        """
+        Update an async job status file.
+        """
+        LOG.info('%r._update(%r, %r)', self, self.job_id, dct)
+        dct.setdefault('ansible_job_id', self.job_id)
+        dct.setdefault('data', '')
+
+        fp = open(self.path + '.tmp', 'w')
+        try:
+            fp.write(json.dumps(dct))
+        finally:
+            fp.close()
+        os.rename(self.path + '.tmp', self.path)
+
+    def _on_sigalrm(self, signum, frame):
+        """
+        Respond to SIGALRM (job timeout) by updating the job file and killing
+        the process.
+        """
+        msg = "Job reached maximum time limit of %d seconds." % (
+            self.timeout_secs,
+        )
+        self._update({
+            "failed": 1,
+            "finished": 1,
+            "msg": msg,
+        })
+        self._timed_out = True
+        self.econtext.broker.shutdown()
+
+    def _install_alarm(self):
+        signal.signal(signal.SIGALRM, self._on_sigalrm)
+        signal.alarm(self.timeout_secs)
+
+    def _run_module(self):
+        kwargs = dict(self.kwargs, **{
+            'detach': True,
+            'econtext': self.econtext,
+            'emulate_tty': False,
+        })
+        return run_module(kwargs)
+
+    def _parse_result(self, dct):
+        filtered, warnings = (
+            ansible.module_utils.json_utils.
+            _filter_non_json_lines(dct['stdout'])
+        )
+        result = json.loads(filtered)
+        result.setdefault('warnings', []).extend(warnings)
+        result['stderr'] = dct['stderr'] or result.get('stderr', '')
+        self._update(result)
+
+    def _run(self):
+        """
+        1. Immediately updates the status file to mark the job as started.
+        2. Installs a timer/signal handler to implement the time limit.
+        3. Runs as with run_module(), writing the result to the status file.
+
+        :param dict kwargs:
+            Runner keyword arguments.
+        :param str job_id:
+            String job ID.
+        :param int timeout_secs:
+            If >0, limit the task's maximum run time.
+        """
+        self._update({
+            'started': 1,
+            'finished': 0,
+            'pid': os.getpid()
+        })
+        self.started_sender.send(True)
+
+        if self.timeout_secs > 0:
+            self._install_alarm()
+
+        dct = self._run_module()
+        if not self._timed_out:
+            # After SIGALRM fires, there is a window between broker responding
+            # to shutdown() by killing the process, and work continuing on the
+            # main thread. If main thread was asleep in at least
+            # basic.py/select.select(), an EINTR will be raised. We want to
+            # discard that exception.
+            try:
+                self._parse_result(dct)
+            except Exception:
+                self._update({
+                    "failed": 1,
+                    "msg": traceback.format_exc(),
+                    "data": dct['stdout'],  # temporary notice only
+                    "stderr": dct['stderr']
+                })
+
+    def run(self):
+        try:
+            try:
+                self._run()
+            except Exception:
+                self._update({
+                    "failed": 1,
+                    "msg": traceback.format_exc(),
+                })
+        finally:
+            self.econtext.broker.shutdown()
+
+
+@mitogen.core.takes_econtext
+def run_module_async(kwargs, job_id, timeout_secs, started_sender, econtext):
+    """
+    Execute a module with its run status and result written to a file,
+    terminating on the process on completion. This function must run in a child
+    forked using :func:`create_fork_child`.
+
+    @param mitogen.core.Sender started_sender:
+        A sender that will receive :data:`True` once the job has reached a
+        point where its initial job file has been written. This is required to
+        avoid a race where an overly eager controller can check for a task
+        before it has reached that point in execution, which is possible at
+        least on Python 2.4, where forking is not available for async tasks.
+    """
+    arunner = AsyncRunner(
+        job_id,
+        timeout_secs,
+        started_sender,
+        econtext,
+        kwargs
+    )
+    arunner.run()
+
+
+def get_user_shell():
+    """
+    For commands executed directly via an SSH command-line, SSH looks up the
+    user's shell via getpwuid() and only defaults to /bin/sh if that field is
+    missing or empty.
+    """
+    try:
+        pw_shell = pwd.getpwuid(os.geteuid()).pw_shell
+    except KeyError:
+        pw_shell = None
+
+    return pw_shell or '/bin/sh'
+
+
+def exec_args(args, in_data='', chdir=None, shell=None, emulate_tty=False):
+    """
+    Run a command in a subprocess, emulating the argument handling behaviour of
+    SSH.
+
+    :param list[str]:
+        Argument vector.
+    :param bytes in_data:
+        Optional standard input for the command.
+    :param bool emulate_tty:
+        If :data:`True`, arrange for stdout and stderr to be merged into the
+        stdout pipe and for LF to be translated into CRLF, emulating the
+        behaviour of a TTY.
+    :return:
+        (return code, stdout bytes, stderr bytes)
+    """
+    LOG.debug('exec_args(%r, ..., chdir=%r)', args, chdir)
+    assert isinstance(args, list)
+
+    if emulate_tty:
+        stderr = subprocess.STDOUT
+    else:
+        stderr = subprocess.PIPE
+
+    proc = subprocess.Popen(
+        args=args,
+        stdout=subprocess.PIPE,
+        stderr=stderr,
+        stdin=subprocess.PIPE,
+        cwd=chdir,
+    )
+    stdout, stderr = proc.communicate(in_data)
+
+    if emulate_tty:
+        stdout = stdout.replace(b'\n', b'\r\n')
+    return proc.returncode, stdout, stderr or b''
+
+
+def exec_command(cmd, in_data='', chdir=None, shell=None, emulate_tty=False):
+    """
+    Run a command in a subprocess, emulating the argument handling behaviour of
+    SSH.
+
+    :param bytes cmd:
+        String command line, passed to user's shell.
+    :param bytes in_data:
+        Optional standard input for the command.
+    :return:
+        (return code, stdout bytes, stderr bytes)
+    """
+    assert isinstance(cmd, mitogen.core.UnicodeType)
+    return exec_args(
+        args=[get_user_shell(), '-c', cmd],
+        in_data=in_data,
+        chdir=chdir,
+        shell=shell,
+        emulate_tty=emulate_tty,
+    )
+
+
+def read_path(path):
+    """
+    Fetch the contents of a filesystem `path` as bytes.
+    """
+    with open(path, 'rb') as f:
+        return f.read()
+
+
+def set_file_owner(path, owner, group=None, fd=None):
+    if owner:
+        uid = pwd.getpwnam(owner).pw_uid
+    else:
+        uid = os.geteuid()
+
+    if group:
+        gid = grp.getgrnam(group).gr_gid
+    else:
+        gid = os.getegid()
+
+    if fd is not None:
+        os.fchown(fd, uid, gid)
+    else:
+        os.chown(path, uid, gid)
+
+
+def write_path(path, s, owner=None, group=None, mode=None,
+               utimes=None, sync=False):
+    """
+    Writes bytes `s` to a filesystem `path`.
+    """
+    path = os.path.abspath(path)
+    fd, tmp_path = tempfile.mkstemp(suffix='.tmp',
+                                    prefix='.ansible_mitogen_transfer-',
+                                    dir=os.path.dirname(path))
+    fp = os.fdopen(fd, 'wb', mitogen.core.CHUNK_SIZE)
+    LOG.debug('write_path(path=%r) temporary file: %s', path, tmp_path)
+
+    try:
+        try:
+            if mode:
+                set_file_mode(tmp_path, mode, fd=fp.fileno())
+            if owner or group:
+                set_file_owner(tmp_path, owner, group, fd=fp.fileno())
+            fp.write(s)
+        finally:
+            fp.close()
+
+        if sync:
+            os.fsync(fp.fileno())
+        os.rename(tmp_path, path)
+    except BaseException:
+        os.unlink(tmp_path)
+        raise
+
+    if utimes:
+        os.utime(path, utimes)
+
+
+CHMOD_CLAUSE_PAT = re.compile(r'([uoga]*)([+\-=])([ugo]|[rwx]*)')
+CHMOD_MASKS = {
+    'u': stat.S_IRWXU,
+    'g': stat.S_IRWXG,
+    'o': stat.S_IRWXO,
+    'a': (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO),
+}
+CHMOD_BITS = {
+    'u': {'r': stat.S_IRUSR, 'w': stat.S_IWUSR, 'x': stat.S_IXUSR},
+    'g': {'r': stat.S_IRGRP, 'w': stat.S_IWGRP, 'x': stat.S_IXGRP},
+    'o': {'r': stat.S_IROTH, 'w': stat.S_IWOTH, 'x': stat.S_IXOTH},
+    'a': {
+        'r': (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH),
+        'w': (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH),
+        'x': (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+    }
+}
+
+
+def apply_mode_spec(spec, mode):
+    """
+    Given a symbolic file mode change specification in the style of chmod(1)
+    `spec`, apply changes in the specification to the numeric file mode `mode`.
+    """
+    for clause in mitogen.core.to_text(spec).split(','):
+        match = CHMOD_CLAUSE_PAT.match(clause)
+        who, op, perms = match.groups()
+        for ch in who or 'a':
+            mask = CHMOD_MASKS[ch]
+            bits = CHMOD_BITS[ch]
+            cur_perm_bits = mode & mask
+            new_perm_bits = 0
+            for perm in perms:
+                new_perm_bits |= bits[perm]
+            mode &= ~mask
+            if op == '=':
+                mode |= new_perm_bits
+            elif op == '+':
+                mode |= new_perm_bits | cur_perm_bits
+            else:
+                mode |= cur_perm_bits & ~new_perm_bits
+    return mode
+
+
+def set_file_mode(path, spec, fd=None):
+    """
+    Update the permissions of a file using the same syntax as chmod(1).
+    """
+    if isinstance(spec, mitogen.core.integer_types):
+        new_mode = spec
+    elif spec.isdigit():
+        new_mode = int(spec, 8)
+    else:
+        mode = os.stat(path).st_mode
+        new_mode = apply_mode_spec(spec, mode)
+
+    if fd is not None:
+        os.fchmod(fd, new_mode)
+    else:
+        os.chmod(path, new_mode)
+
+
+def file_exists(path):
+    """
+    Return :data:`True` if `path` exists. This is a wrapper function over
+    :func:`os.path.exists`, since its implementation module varies across
+    Python versions.
+    """
+    return os.path.exists(path)

ansible_mitogen/transport_config.py

@@ -0,0 +1,879 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+"""
+Mitogen extends Ansible's target configuration mechanism in several ways that
+require some care:
+
+* Per-task configurables in Ansible like ansible_python_interpreter are
+  connection-layer configurables in Mitogen. They must be extracted during each
+  task execution to form the complete connection-layer configuration.
+
+* Mitogen has extra configurables not supported by Ansible at all, such as
+  mitogen_ssh_debug_level. These are extracted the same way as
+  ansible_python_interpreter.
+
+* Mitogen allows connections to be delegated to other machines. Ansible has no
+  internal framework for this, and so Mitogen must figure out a delegated
+  connection configuration all on its own. It cannot reuse much of the Ansible
+  machinery for building a connection configuration, as that machinery is
+  deeply spread out and hard-wired to expect Ansible's usual mode of operation.
+
+For normal and delegate_to connections, Ansible's PlayContext is reused where
+possible to maximize compatibility, but for proxy hops, configurations are
+built up using the HostVars magic class to call VariableManager.get_vars()
+behind the scenes on our behalf. Where Ansible has multiple sources of a
+configuration item, for example, ansible_ssh_extra_args, Mitogen must (ideally
+perfectly) reproduce how Ansible arrives at its value, without using mechanisms
+that are hard-wired or change across Ansible versions.
+
+That is what this file is for. It exports two spec classes, one that takes all
+information from PlayContext, and another that takes (almost) all information
+from HostVars.
+"""
+
+from __future__ import absolute_import, division, print_function
+from __future__ import unicode_literals
+__metaclass__ = type
+
+import abc
+import logging
+import os
+
+import ansible.utils.shlex
+import ansible.constants as C
+import ansible.executor.interpreter_discovery
+import ansible.utils.unsafe_proxy
+
+from ansible.module_utils.six import with_metaclass
+from ansible.module_utils.parsing.convert_bool import boolean
+
+import ansible_mitogen.utils
+import mitogen.core
+
+
+LOG = logging.getLogger(__name__)
+
+if ansible_mitogen.utils.ansible_version[:2] >= (2, 19):
+    _FALLBACK_INTERPRETER = ansible.executor.interpreter_discovery._FALLBACK_INTERPRETER
+elif ansible_mitogen.utils.ansible_version[:2] >= (2, 17):
+    _FALLBACK_INTERPRETER = u'/usr/bin/python3'
+else:
+    _FALLBACK_INTERPRETER = u'/usr/bin/python'
+
+
+def run_interpreter_discovery_if_necessary(s, candidates, task_vars, action, rediscover_python):
+    """
+    Triggers ansible python interpreter discovery if requested.
+    Caches this value the same way Ansible does it.
+    For connections like `docker`, we want to rediscover the python interpreter because
+    it could be different than what's ran on the host
+    """
+    # keep trying different interpreters until we don't error
+    if action._mitogen_discovering_interpreter:
+        return action._mitogen_interpreter_candidate
+
+    if s in ['auto', 'auto_legacy', 'auto_silent', 'auto_legacy_silent']:
+        # python is the only supported interpreter_name as of Ansible 2.8.8
+        interpreter_name = 'python'
+        discovered_interpreter_config = u'discovered_interpreter_%s' % interpreter_name
+
+        if task_vars.get('ansible_facts') is None:
+           task_vars['ansible_facts'] = {}
+
+        if rediscover_python and task_vars.get('ansible_facts', {}).get(discovered_interpreter_config):
+            # if we're rediscovering python then chances are we're running something like a docker connection
+            # this will handle scenarios like running a playbook that does stuff + then dynamically creates a docker container,
+            # then runs the rest of the playbook inside that container, and then rerunning the playbook again
+            action._mitogen_rediscovered_interpreter = True
+
+            # blow away the discovered_interpreter_config cache and rediscover
+            del task_vars['ansible_facts'][discovered_interpreter_config]
+
+        try:
+            s = task_vars[u'ansible_facts'][discovered_interpreter_config]
+        except KeyError:
+            action._mitogen_discovering_interpreter = True
+            action._mitogen_interpreter_candidates = candidates
+            # fake pipelining so discover_interpreter can be happy
+            action._connection.has_pipelining = True
+            s = ansible.executor.interpreter_discovery.discover_interpreter(
+                action=action,
+                interpreter_name=interpreter_name,
+                discovery_mode=s,
+                task_vars=task_vars,
+            )
+            s = ansible.utils.unsafe_proxy.AnsibleUnsafeText(s)
+            # cache discovered interpreter
+            task_vars['ansible_facts'][discovered_interpreter_config] = s
+            action._connection.has_pipelining = False
+
+        # propagate discovered interpreter as fact
+        action._discovered_interpreter_key = discovered_interpreter_config
+        action._discovered_interpreter = s
+
+    action._mitogen_discovering_interpreter = False
+    action._mitogen_interpreter_candidates = None
+    return s
+
+
+def parse_python_path(s, candidates, task_vars, action, rediscover_python):
+    """
+    Given the string set for ansible_python_interpeter, parse it using shell
+    syntax and return an appropriate argument vector. If the value detected is
+    one of interpreter discovery then run that first. Caches python interpreter
+    discovery value in `facts_from_task_vars` like how Ansible handles this.
+    """
+    if not s:
+        # if python_path doesn't exist, default to `auto` and attempt to discover it
+        s = 'auto'
+
+    s = run_interpreter_discovery_if_necessary(s, candidates, task_vars, action, rediscover_python)
+    if not s:
+        s = _FALLBACK_INTERPRETER
+
+    return ansible.utils.shlex.shlex_split(s)
+
+
+def optional_secret(value):
+    """
+    Wrap `value` in :class:`mitogen.core.Secret` if it is not :data:`None`,
+    otherwise return :data:`None`.
+    """
+    if value is not None:
+        return mitogen.core.Secret(value)
+
+
+def first_true(it, default=None):
+    """
+    Return the first truthy element from `it`.
+    """
+    for elem in it:
+        if elem:
+            return elem
+    return default
+
+
+class Spec(with_metaclass(abc.ABCMeta, object)):
+    """
+    A source for variables that comprise a connection configuration.
+    """
+
+    @abc.abstractmethod
+    def transport(self):
+        """
+        The name of the Ansible plug-in implementing the connection.
+        """
+
+    @abc.abstractmethod
+    def inventory_name(self):
+        """
+        The name of the target being connected to as it appears in Ansible's
+        inventory.
+        """
+
+    @abc.abstractmethod
+    def remote_addr(self):
+        """
+        The network address of the target, or for container and other special
+        targets, some other unique identifier.
+        """
+
+    @abc.abstractmethod
+    def remote_user(self):
+        """
+        The username of the login account on the target.
+        """
+
+    @abc.abstractmethod
+    def password(self):
+        """
+        The password of the login account on the target.
+        """
+
+    @abc.abstractmethod
+    def become(self):
+        """
+        :data:`True` if privilege escalation should be active.
+        """
+
+    @abc.abstractmethod
+    def become_flags(self):
+        """
+        The command line arguments passed to the become executable.
+        """
+
+    @abc.abstractmethod
+    def become_method(self):
+        """
+        The name of the Ansible become method to use.
+        """
+
+    @abc.abstractmethod
+    def become_user(self):
+        """
+        The username of the target account for become.
+        """
+
+    @abc.abstractmethod
+    def become_pass(self):
+        """
+        The password of the target account for become.
+        """
+
+    @abc.abstractmethod
+    def port(self):
+        """
+        The port of the login service on the target machine.
+        """
+
+    @abc.abstractmethod
+    def python_path(self):
+        """
+        Path to the Python interpreter on the target machine.
+        """
+
+    @abc.abstractmethod
+    def host_key_checking(self):
+        """
+        Whether or not to check the keys of the target machine
+        """
+
+    @abc.abstractmethod
+    def private_key_file(self):
+        """
+        Path to the SSH private key file to use to login.
+        """
+
+    @abc.abstractmethod
+    def ssh_executable(self):
+        """
+        Path to the SSH executable.
+        """
+
+    @abc.abstractmethod
+    def timeout(self):
+        """
+        The generic timeout for all connections.
+        """
+
+    @abc.abstractmethod
+    def ansible_ssh_timeout(self):
+        """
+        The SSH-specific timeout for a connection.
+        """
+
+    @abc.abstractmethod
+    def ssh_args(self):
+        """
+        The list of additional arguments that should be included in an SSH
+        invocation.
+        """
+
+    @abc.abstractmethod
+    def become_exe(self):
+        """
+        The path to the executable implementing the become method on the remote
+        machine.
+        """
+
+    @abc.abstractmethod
+    def sudo_args(self):
+        """
+        The list of additional arguments that should be included in a sudo
+        invocation.
+        """
+
+    @abc.abstractmethod
+    def mitogen_via(self):
+        """
+        The value of the mitogen_via= variable for this connection. Indicates
+        the connection should be established via an intermediary.
+        """
+
+    @abc.abstractmethod
+    def mitogen_kind(self):
+        """
+        The type of container to use with the "setns" transport.
+        """
+
+    @abc.abstractmethod
+    def mitogen_mask_remote_name(self):
+        """
+        Specifies whether to set a fixed "remote_name" field. The remote_name
+        is the suffix of `argv[0]` for remote interpreters. By default it
+        includes identifying information from the local process, which may be
+        undesirable in some circumstances.
+        """
+
+    @abc.abstractmethod
+    def mitogen_buildah_path(self):
+        """
+        The path to the "buildah" program for the 'buildah' transport.
+        """
+
+    @abc.abstractmethod
+    def mitogen_docker_path(self):
+        """
+        The path to the "docker" program for the 'docker' transport.
+        """
+
+    @abc.abstractmethod
+    def mitogen_kubectl_path(self):
+        """
+        The path to the "kubectl" program for the 'docker' transport.
+        """
+
+    @abc.abstractmethod
+    def mitogen_lxc_path(self):
+        """
+        The path to the "lxc" program for the 'lxd' transport.
+        """
+
+    @abc.abstractmethod
+    def mitogen_lxc_attach_path(self):
+        """
+        The path to the "lxc-attach" program for the 'lxc' transport.
+        """
+
+    @abc.abstractmethod
+    def mitogen_lxc_info_path(self):
+        """
+        The path to the "lxc-info" program for the 'lxc' transport.
+        """
+
+    @abc.abstractmethod
+    def mitogen_machinectl_path(self):
+        """
+        The path to the "machinectl" program for the 'setns' transport.
+        """
+
+    @abc.abstractmethod
+    def mitogen_podman_path(self):
+        """
+        The path to the "podman" program for the 'podman' transport.
+        """
+
+    @abc.abstractmethod
+    def mitogen_ssh_keepalive_interval(self):
+        """
+        The SSH ServerAliveInterval.
+        """
+
+    @abc.abstractmethod
+    def mitogen_ssh_keepalive_count(self):
+        """
+        The SSH ServerAliveCount.
+        """
+
+    @abc.abstractmethod
+    def mitogen_ssh_debug_level(self):
+        """
+        The SSH debug level.
+        """
+
+    @abc.abstractmethod
+    def mitogen_ssh_compression(self):
+        """
+        Whether SSH compression is enabled.
+        """
+
+    @abc.abstractmethod
+    def extra_args(self):
+        """
+        Connection-specific arguments.
+        """
+
+    @abc.abstractmethod
+    def ansible_doas_exe(self):
+        """
+        Value of "ansible_doas_exe" variable.
+        """
+
+    @abc.abstractmethod
+    def verbosity(self):
+        """
+        How verbose to make logging or diagnostics output.
+        """
+
+
+class PlayContextSpec(Spec):
+    """
+    PlayContextSpec takes almost all its information as-is from Ansible's
+    PlayContext. It is used for normal connections and delegate_to connections,
+    and should always be accurate.
+    """
+    def __init__(self, connection, play_context, transport, inventory_name):
+        self._connection = connection
+        self._play_context = play_context
+        self._transport = transport
+        self._inventory_name = inventory_name
+        self._task_vars = self._connection._get_task_vars()
+        # used to run interpreter discovery
+        self._action = connection._action
+
+    def _become_option(self, name):
+        plugin = self._connection.become
+        try:
+            return plugin.get_option(name, self._task_vars, self._play_context)
+        except AttributeError:
+            # A few ansible_mitogen connection plugins look more like become
+            # plugins. They don't quite fit Ansible's plugin.get_option() API.
+            # https://github.com/mitogen-hq/mitogen/issues/1173
+            fallback_plugins = {'mitogen_doas', 'mitogen_sudo', 'mitogen_su'}
+            if self._connection.transport not in fallback_plugins:
+                raise
+
+            fallback_options = {
+                'become_exe',
+                'become_flags',
+            }
+            if name not in fallback_options:
+                raise
+
+            LOG.info(
+                'Used fallback=PlayContext.%s for plugin=%r, option=%r',
+                name, self._connection, name,
+            )
+            return getattr(self._play_context, name)
+
+    def _connection_option(self, name, fallback_attr=None):
+        try:
+            return self._connection.get_option(name, hostvars=self._task_vars)
+        except KeyError:
+            if fallback_attr is None:
+                fallback_attr = name
+            LOG.info(
+                'Used fallback=PlayContext.%s for plugin=%r, option=%r',
+                fallback_attr, self._connection, name,
+            )
+            return getattr(self._play_context, fallback_attr)
+
+    def transport(self):
+        return self._transport
+
+    def inventory_name(self):
+        return self._inventory_name
+
+    def remote_addr(self):
+        return self._connection_option('host', fallback_attr='remote_addr')
+
+    def remote_user(self):
+        return self._connection_option('remote_user')
+
+    def become(self):
+        return self._connection.become
+
+    def become_flags(self):
+        return self._become_option('become_flags')
+
+    def become_method(self):
+        return self._connection.become.name
+
+    def become_user(self):
+        return self._become_option('become_user')
+
+    def become_pass(self):
+        return optional_secret(self._become_option('become_pass'))
+
+    def password(self):
+        return optional_secret(self._connection_option('password'))
+
+    def port(self):
+        return self._connection_option('port')
+
+    def python_path(self, rediscover_python=False):
+        # See also
+        #   - ansible_mitogen.connecton.Connection.get_task_var()
+        try:
+            delegated_vars = self._task_vars['ansible_delegated_vars']
+            variables = delegated_vars[self._connection.delegate_to_hostname]
+        except KeyError:
+            variables = self._task_vars
+
+        interpreter_python = C.config.get_config_value(
+            'INTERPRETER_PYTHON', variables=variables,
+        )
+        interpreter_python_fallback = C.config.get_config_value(
+            'INTERPRETER_PYTHON_FALLBACK', variables=variables,
+        )
+
+        if '{{' in interpreter_python or '{%' in interpreter_python:
+            templar = self._connection.templar
+            interpreter_python = templar.template(interpreter_python)
+
+        return parse_python_path(
+            interpreter_python,
+            candidates=interpreter_python_fallback,
+            task_vars=self._task_vars,
+            action=self._action,
+            rediscover_python=rediscover_python)
+
+    def host_key_checking(self):
+        return self._connection_option('host_key_checking')
+
+    def private_key_file(self):
+        return self._connection_option('private_key_file')
+
+    def ssh_executable(self):
+        return self._connection_option('ssh_executable')
+
+    def timeout(self):
+        return self._connection_option('timeout')
+
+    def ansible_ssh_timeout(self):
+        return self.timeout()
+
+    def ssh_args(self):
+        return [
+            mitogen.core.to_text(term)
+            for s in (
+                self._connection_option('ssh_args'),
+                self._connection_option('ssh_common_args'),
+                self._connection_option('ssh_extra_args'),
+            )
+            for term in ansible.utils.shlex.shlex_split(s or '')
+        ]
+
+    def become_exe(self):
+        return self._become_option('become_exe')
+
+    def sudo_args(self):
+        return ansible.utils.shlex.shlex_split(self.become_flags() or '')
+
+    def mitogen_via(self):
+        return self._connection.get_task_var('mitogen_via')
+
+    def mitogen_kind(self):
+        return self._connection.get_task_var('mitogen_kind')
+
+    def mitogen_mask_remote_name(self):
+        return self._connection.get_task_var('mitogen_mask_remote_name')
+
+    def mitogen_buildah_path(self):
+        return self._connection.get_task_var('mitogen_buildah_path')
+
+    def mitogen_docker_path(self):
+        return self._connection.get_task_var('mitogen_docker_path')
+
+    def mitogen_kubectl_path(self):
+        return self._connection.get_task_var('mitogen_kubectl_path')
+
+    def mitogen_lxc_path(self):
+        return self._connection.get_task_var('mitogen_lxc_path')
+
+    def mitogen_lxc_attach_path(self):
+        return self._connection.get_task_var('mitogen_lxc_attach_path')
+
+    def mitogen_lxc_info_path(self):
+        return self._connection.get_task_var('mitogen_lxc_info_path')
+
+    def mitogen_podman_path(self):
+        return self._connection.get_task_var('mitogen_podman_path')
+
+    def mitogen_ssh_keepalive_interval(self):
+        return self._connection.get_task_var('mitogen_ssh_keepalive_interval')
+
+    def mitogen_ssh_keepalive_count(self):
+        return self._connection.get_task_var('mitogen_ssh_keepalive_count')
+
+    def mitogen_machinectl_path(self):
+        return self._connection.get_task_var('mitogen_machinectl_path')
+
+    def mitogen_ssh_debug_level(self):
+        return self._connection.get_task_var('mitogen_ssh_debug_level')
+
+    def mitogen_ssh_compression(self):
+        return self._connection.get_task_var('mitogen_ssh_compression')
+
+    def extra_args(self):
+        return self._connection.get_extra_args()
+
+    def ansible_doas_exe(self):
+        return (
+            self._connection.get_task_var('ansible_doas_exe') or
+            os.environ.get('ANSIBLE_DOAS_EXE')
+        )
+
+    def verbosity(self):
+        try:
+            verbosity = self._connection.get_option('verbosity', hostvars=self._task_vars)
+        except KeyError:
+            verbosity = self.mitogen_ssh_debug_level()
+
+        if verbosity:
+            return int(verbosity)
+
+        return 0
+
+
+class MitogenViaSpec(Spec):
+    """
+    MitogenViaSpec takes most of its information from the HostVars of the
+    running task. HostVars is a lightweight wrapper around VariableManager, so
+    it is better to say that VariableManager.get_vars() is the ultimate source
+    of MitogenViaSpec's information.
+
+    Due to this, mitogen_via= hosts must have all their configuration
+    information represented as host and group variables. We cannot use any
+    per-task configuration, as all that data belongs to the real target host.
+
+    Ansible uses all kinds of strange historical logic for calculating
+    variables, including making their precedence configurable. MitogenViaSpec
+    must ultimately reimplement all of that logic. It is likely that if you are
+    having a configruation problem with connection delegation, the answer to
+    your problem lies in the method implementations below!
+    """
+    def __init__(self, inventory_name, host_vars, task_vars, become_method, become_user,
+                 play_context, action):
+        """
+        :param str inventory_name:
+            The inventory name of the intermediary machine, i.e. not the target
+            machine.
+        :param dict host_vars:
+            The HostVars magic dictionary provided by Ansible in task_vars.
+        :param dict task_vars:
+            Task vars provided by Ansible.
+        :param str become_method:
+            If the mitogen_via= spec included a become method, the method it
+            specifies.
+        :param str become_user:
+            If the mitogen_via= spec included a become user, the user it
+            specifies.
+        :param PlayContext play_context:
+            For some global values **only**, the PlayContext used to describe
+            the real target machine. Values from this object are **strictly
+            restricted** to values that are Ansible-global, e.g. the passwords
+            specified interactively.
+        :param ActionModuleMixin action:
+            Backref to the ActionModuleMixin required for ansible interpreter discovery
+        """
+        self._inventory_name = inventory_name
+        self._host_vars = host_vars
+        self._task_vars = task_vars
+        self._become_method = become_method
+        self._become_user = become_user
+        # Dangerous! You may find a variable you want in this object, but it's
+        # almost certainly for the wrong machine!
+        self._dangerous_play_context = play_context
+        self._action = action
+
+    def transport(self):
+        return (
+            self._host_vars.get('ansible_connection') or
+            C.DEFAULT_TRANSPORT
+        )
+
+    def inventory_name(self):
+        return self._inventory_name
+
+    def remote_addr(self):
+        # play_context.py::MAGIC_VARIABLE_MAPPING
+        return (
+            self._host_vars.get('ansible_ssh_host') or
+            self._host_vars.get('ansible_host') or
+            self._inventory_name
+        )
+
+    def remote_user(self):
+        return (
+            self._host_vars.get('ansible_ssh_user') or
+            self._host_vars.get('ansible_user') or
+            C.DEFAULT_REMOTE_USER
+        )
+
+    def become(self):
+        return bool(self._become_user)
+
+    def become_flags(self):
+        return self._host_vars.get('ansible_become_flags')
+
+    def become_method(self):
+        return (
+            self._become_method or
+            self._host_vars.get('ansible_become_method') or
+            C.DEFAULT_BECOME_METHOD
+        )
+
+    def become_user(self):
+        return self._become_user
+
+    def become_pass(self):
+        return optional_secret(
+            self._host_vars.get('ansible_become_pass') or
+            self._host_vars.get('ansible_become_password')
+        )
+
+    def password(self):
+        return optional_secret(
+            self._host_vars.get('ansible_ssh_password') or
+            self._host_vars.get('ansible_ssh_pass') or
+            self._host_vars.get('ansible_password')
+        )
+
+    def port(self):
+        return (
+            self._host_vars.get('ansible_ssh_port') or
+            self._host_vars.get('ansible_port') or
+            C.DEFAULT_REMOTE_PORT
+        )
+
+    def python_path(self, rediscover_python=False):
+        s = self._host_vars.get('ansible_python_interpreter')
+        interpreter_python_fallback = self._host_vars.get(
+            'ansible_interpreter_python_fallback', [],
+        )
+        return parse_python_path(
+            s,
+            candidates=interpreter_python_fallback,
+            task_vars=self._task_vars,
+            action=self._action,
+            rediscover_python=rediscover_python)
+
+    def host_key_checking(self):
+        def candidates():
+            yield self._host_vars.get('ansible_ssh_host_key_checking')
+            yield self._host_vars.get('ansible_host_key_checking')
+            yield C.HOST_KEY_CHECKING
+        val = next((v for v in candidates() if v is not None), True)
+        return boolean(val)
+
+    def private_key_file(self):
+        # TODO: must come from PlayContext too.
+        return (
+            self._host_vars.get('ansible_ssh_private_key_file') or
+            self._host_vars.get('ansible_private_key_file') or
+            C.DEFAULT_PRIVATE_KEY_FILE
+        )
+
+    def ssh_executable(self):
+        return C.config.get_config_value("ssh_executable", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
+
+    def timeout(self):
+        # TODO: must come from PlayContext too.
+        return C.DEFAULT_TIMEOUT
+
+    def ansible_ssh_timeout(self):
+        return (
+            self._host_vars.get('ansible_timeout') or
+            self._host_vars.get('ansible_ssh_timeout') or
+            self.timeout()
+        )
+
+    def ssh_args(self):
+        local_vars = self._task_vars.get("hostvars", {}).get(self._inventory_name, {})
+        return [
+            mitogen.core.to_text(term)
+            for s in (
+                C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=local_vars),
+                C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=local_vars),
+                C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=local_vars)
+            )
+            for term in ansible.utils.shlex.shlex_split(s)
+            if s
+        ]
+
+    def become_exe(self):
+        return (
+            self._host_vars.get('ansible_become_exe') or
+            C.DEFAULT_BECOME_EXE
+        )
+
+    def sudo_args(self):
+        return [
+            mitogen.core.to_text(term)
+            for s in (
+                self._host_vars.get('ansible_sudo_flags') or '',
+                self.become_flags() or '',
+            )
+            for term in ansible.utils.shlex.shlex_split(s)
+        ]
+
+    def mitogen_via(self):
+        return self._host_vars.get('mitogen_via')
+
+    def mitogen_kind(self):
+        return self._host_vars.get('mitogen_kind')
+
+    def mitogen_mask_remote_name(self):
+        return self._host_vars.get('mitogen_mask_remote_name')
+
+    def mitogen_buildah_path(self):
+        return self._host_vars.get('mitogen_buildah_path')
+
+    def mitogen_docker_path(self):
+        return self._host_vars.get('mitogen_docker_path')
+
+    def mitogen_kubectl_path(self):
+        return self._host_vars.get('mitogen_kubectl_path')
+
+    def mitogen_lxc_path(self):
+        return self._host_vars.get('mitogen_lxc_path')
+
+    def mitogen_lxc_attach_path(self):
+        return self._host_vars.get('mitogen_lxc_attach_path')
+
+    def mitogen_lxc_info_path(self):
+        return self._host_vars.get('mitogen_lxc_info_path')
+
+    def mitogen_podman_path(self):
+        return self._host_vars.get('mitogen_podman_path')
+
+    def mitogen_ssh_keepalive_interval(self):
+        return self._host_vars.get('mitogen_ssh_keepalive_interval')
+
+    def mitogen_ssh_keepalive_count(self):
+        return self._host_vars.get('mitogen_ssh_keepalive_count')
+
+    def mitogen_machinectl_path(self):
+        return self._host_vars.get('mitogen_machinectl_path')
+
+    def mitogen_ssh_debug_level(self):
+        return self._host_vars.get('mitogen_ssh_debug_level')
+
+    def mitogen_ssh_compression(self):
+        return self._host_vars.get('mitogen_ssh_compression')
+
+    def extra_args(self):
+        return []  # TODO
+
+    def ansible_doas_exe(self):
+        return (
+            self._host_vars.get('ansible_doas_exe') or
+            os.environ.get('ANSIBLE_DOAS_EXE')
+        )
+
+    def verbosity(self):
+        verbosity = self._host_vars.get('ansible_ssh_verbosity')
+        if verbosity is None:
+            verbosity = self.mitogen_ssh_debug_level()
+
+        if verbosity:
+            return int(verbosity)
+
+        return 0

ansible_mitogen/utils/__init__.py

@@ -0,0 +1,29 @@
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import re
+
+import ansible
+
+__all__ = [
+    'ansible_version',
+]
+
+
+def _parse(v_string):
+    # Adapted from distutils.version.LooseVersion.parse()
+    component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE)
+    for component in component_re.split(v_string):
+        if not component or component == '.':
+            continue
+        try:
+            yield int(component)
+        except ValueError:
+            yield component
+
+
+ansible_version = tuple(_parse(ansible.__version__))
+
+del _parse
+del re
+del ansible

ansible_mitogen/utils/unsafe.py

@@ -0,0 +1,123 @@
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import ansible
+import ansible.utils.unsafe_proxy
+
+import ansible_mitogen.utils
+
+import mitogen
+import mitogen.core
+import mitogen.utils
+
+__all__ = [
+    'cast',
+]
+
+def _cast_to_dict(obj): return {cast(k): cast(v) for k, v in obj.items()}
+def _cast_to_list(obj): return [cast(v) for v in obj]
+def _cast_to_set(obj): return set(cast(v) for v in obj)
+def _cast_to_tuple(obj): return tuple(cast(v) for v in obj)
+def _cast_unsafe(obj): return obj._strip_unsafe()
+def _passthrough(obj): return obj
+def _untag(obj): return obj._native_copy()
+
+
+# A dispatch table to cast objects based on their exact type.
+# This is an optimisation, reliable fallbacks are required (e.g. isinstance())
+_CAST_DISPATCH = {
+    bytes: bytes,
+    dict: _cast_to_dict,
+    list: _cast_to_list,
+    mitogen.core.UnicodeType: mitogen.core.UnicodeType,
+}
+_CAST_DISPATCH.update({t: _passthrough for t in mitogen.utils.PASSTHROUGH})
+
+_CAST_SUBTYPES = [
+    dict,
+    list,
+]
+
+if hasattr(ansible.utils.unsafe_proxy, 'TrustedAsTemplate'):
+    import datetime
+    import ansible.module_utils._internal._datatag
+    _CAST_DISPATCH.update({
+        set: _cast_to_set,
+        tuple: _cast_to_tuple,
+        ansible.module_utils._internal._datatag._AnsibleTaggedBytes: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedDate: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedDateTime: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedDict: _cast_to_dict,
+        ansible.module_utils._internal._datatag._AnsibleTaggedFloat: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedInt: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedList: _cast_to_list,
+        ansible.module_utils._internal._datatag._AnsibleTaggedSet: _cast_to_set,
+        ansible.module_utils._internal._datatag._AnsibleTaggedStr: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedTime: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedTuple: _cast_to_tuple,
+        ansible.utils.unsafe_proxy.AnsibleUnsafeBytes: bytes,
+        ansible.utils.unsafe_proxy.AnsibleUnsafeText: mitogen.core.UnicodeType,
+        datetime.date: _passthrough,
+        datetime.datetime: _passthrough,
+        datetime.time: _passthrough,
+    })
+    _CAST_SUBTYPES.extend([
+        set,
+        tuple,
+    ])
+elif hasattr(ansible.utils.unsafe_proxy.AnsibleUnsafeText, '_strip_unsafe'):
+    _CAST_DISPATCH.update({
+        tuple: _cast_to_list,
+        ansible.utils.unsafe_proxy.AnsibleUnsafeBytes: _cast_unsafe,
+        ansible.utils.unsafe_proxy.AnsibleUnsafeText: _cast_unsafe,
+        ansible.utils.unsafe_proxy.NativeJinjaUnsafeText: _cast_unsafe,
+    })
+    _CAST_SUBTYPES.extend([
+        tuple,
+    ])
+elif ansible_mitogen.utils.ansible_version[:2] <= (2, 16):
+    _CAST_DISPATCH.update({
+        tuple: _cast_to_list,
+        ansible.utils.unsafe_proxy.AnsibleUnsafeBytes: bytes,
+        ansible.utils.unsafe_proxy.AnsibleUnsafeText: mitogen.core.UnicodeType,
+    })
+    _CAST_SUBTYPES.extend([
+        tuple,
+    ])
+else:
+    mitogen_ver = '.'.join(str(v) for v in mitogen.__version__)
+    raise ImportError("Mitogen %s can't cast Ansible %s objects"
+                      % (mitogen_ver, ansible.__version__))
+
+
+def cast(obj):
+    """
+    Return obj (or a copy) with subtypes of builtins cast to their supertype.
+
+    This is an enhanced version of :func:`mitogen.utils.cast`. In addition it
+    handles ``ansible.utils.unsafe_proxy.AnsibleUnsafeText`` and variants.
+
+    There are types handled by :func:`ansible.utils.unsafe_proxy.wrap_var()`
+    that this function currently does not handle (e.g. `set()`), or preserve
+    preserve (e.g. `tuple()`). Future enhancements may change this.
+
+    :param obj:
+        Object to undecorate.
+    :returns:
+        Undecorated object.
+    """
+    # Fast path: obj is a known type, dispatch directly
+    try:
+        unwrapper = _CAST_DISPATCH[type(obj)]
+    except KeyError:
+        pass
+    else:
+        return unwrapper(obj)
+
+    # Slow path: obj is some unknown subclass
+    for typ_ in _CAST_SUBTYPES:
+        if isinstance(obj, typ_):
+            unwrapper = _CAST_DISPATCH[typ_]
+            return unwrapper(obj)
+
+    return mitogen.utils.cast(obj)

dev_requirements.txt

@@ -0,0 +1,11 @@
+# This file is no longer used by CI jobs, it's mostly for interactive use.
+# Instead CI jobs grab the relevant sub-requirement.
+
+# mitogen_tests
+-r tests/requirements.txt
+
+# ansible_tests
+-r tests/ansible/requirements.txt
+
+# readthedocs
+-r docs/requirements.txt

docs/.gitignore

@@ -0,0 +1 @@
+build

docs/Makefile

@@ -1,15 +1,18 @@
 # Makefile for Sphinx documentation
 #
 
+default:
+	sphinx-build . build/html/
+
 # You can set these variables from the command line.
 SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
 PAPER         =
-BUILDDIR      = _build
+BUILDDIR      = build
 
 # User-friendly check for sphinx-build
 ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
-$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
+$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from https://sphinx-doc.org/)
 endif
 
 # Internal variables.

docs/_static/style.css

@@ -1,4 +1,142 @@
 
+body {
+    font-size: 100%;
+}
+
+.sphinxsidebarwrapper {
+    padding-top: 0 !important;
+}
+
+.sphinxsidebar {
+    font-size: 80% !important;
+}
+
+.sphinxsidebar h3 {
+    font-size: 130% !important;
+}
+
+img + p,
+h1 + p,
+h2 + p,
+h3 + p,
+h4 + p,
+h5 + p
+{
+    margin-top: 0;
+}
+
+.section > h3:first-child {
+    margin-top: 15px !important;
+}
+
+.body h1 { font-size: 200% !important; }
+.body h2 { font-size: 165% !important; }
+.body h3 { font-size: 125% !important; }
+.body h4 { font-size: 110% !important; font-weight: bold; }
+.body h5 { font-size: 100% !important; font-weight: bold; }
+
+.body h1,
+.body h2,
+.body h3,
+.body h4,
+.body h5 {
+    margin-top: 30px !important;
+    color: #7f0000;
+}
+
+.body h1 {
+    margin-top: 0 !important;
+}
+
+body,
+.sphinxsidebar,
+.sphinxsidebar h1,
+.sphinxsidebar h2,
+.sphinxsidebar h3,
+.sphinxsidebar h4,
+.sphinxsidebar h5,
+.body h1,
+.body h2,
+.body h3,
+.body h4,
+.body h5 {
+    /*font-family: sans-serif !important;*/
+    font-family: -apple-system,BlinkMacSystemFont,Segoe UI,Helvetica,Arial,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol !important;
+}
+
+
+.document {
+    width: 1000px !important;
+}
+
 div.figure {
     padding: 0;
 }
+
+div.body li {
+    margin-bottom: 0.5em;
+}
+
+
+/*
+ * Undo the hyphens: auto in Sphinx basic.css.
+ */
+div.body p, div.body dd, div.body li, div.body blockquote {
+    -moz-hyphens: inherit;
+    -ms-hyphens: inherit;
+    -webkit-hyphens: inherit;
+    hyphens: inherit;
+}
+
+
+
+/*
+ * Setting :width; on an image causes Sphinx to turn the image into a link, so
+ * set :Class: instead.
+ */
+.mitogen-full-width {
+    width: 100%;
+}
+
+.mitogen-right-150 {
+    float: right;
+    padding-left: 8px;
+    width: 150px;
+}
+
+.mitogen-right-180 {
+    float: right;
+    padding-left: 8px;
+    width: 180px;
+}
+
+.mitogen-right-225 {
+    float: right;
+    padding-left: 8px;
+    width: 225px;
+}
+
+.mitogen-right-275 {
+    float: right;
+    padding-left: 8px;
+    width: 275px;
+}
+
+.mitogen-right-300 {
+    float: right;
+    padding-left: 8px;
+    width: 300px;
+}
+
+.mitogen-right-350 {
+    float: right;
+    padding-left: 8px;
+    width: 350px;
+}
+
+.mitogen-logo-wrap {
+    shape-margin: 8px;
+    shape-outside: polygon(
+        100% 0, 50% 10%, 24% 24%, 0% 50%, 24% 75%, 50% 90%, 100% 100%
+    );
+}

docs/_templates/github.html

@@ -1,4 +1,4 @@
 <p>
 <br>
-<a href="https://github.com/dw/econtext/">GitHub Repository</a>
+<a class="github-button" href="https://github.com/mitogen-hq/mitogen/" data-size="large" data-show-count="true" aria-label="Star mitogen on GitHub">Star</a>
 </p>

docs/_templates/globaltoc.html

@@ -0,0 +1 @@
+{{ toctree() }}

docs/_templates/layout.html

@@ -1,2 +1,18 @@
 {% extends "!layout.html" %}
 {% set css_files = css_files + ['_static/style.css'] %}
+
+{# We don't support Sphinx search, so don't let its JS either. #}
+{% block scripts %}
+{% endblock %}
+
+{# Alabaster ships a completely useless custom.css, suppress it. #}
+{%- block extrahead %}
+    <meta name="referrer" content="strict-origin">
+    <meta name="google-site-verification" content="oq5hNxRYo25tcfjfs3l6pPxfNgY3JzDYSpskc9q4TYI" />
+    <meta name="viewport" content="width=device-width, initial-scale=0.9, maximum-scale=0.9" />
+{% endblock %}
+
+{% block footer %}
+    {{ super() }}
+    <script async defer src="https://buttons.github.io/buttons.js"></script>
+{% endblock %}

docs/api.rst

@@ -3,117 +3,682 @@ API Reference
 *************
 
 
-econtext Package
-================
+Package Layout
+==============
 
-.. automodule:: econtext
 
-.. autodata:: econtext.slave
+mitogen Package
+---------------
 
+.. automodule:: mitogen
 
-econtext.core
-=============
+.. autodata:: mitogen.__version__
+.. autodata:: mitogen.is_master
+.. autodata:: mitogen.context_id
+.. autodata:: mitogen.parent_id
+.. autodata:: mitogen.parent_ids
+.. autofunction:: mitogen.main
 
-.. automodule:: econtext.core
 
+mitogen.core
+------------
 
-Exceptions
-----------
+.. automodule:: mitogen.core
+
+.. currentmodule:: mitogen.core
+.. autodecorator:: takes_econtext
 
-.. autoclass:: econtext.core.Error
-.. autoclass:: econtext.core.CallError
-.. autoclass:: econtext.core.ChannelError
-.. autoclass:: econtext.core.StreamError
-.. autoclass:: econtext.core.TimeoutError
+.. currentmodule:: mitogen.core
+.. autodecorator:: takes_router
 
 
-Stream Classes
+mitogen.master
 --------------
 
-.. autoclass:: econtext.core.Stream
-   :members:
+.. automodule:: mitogen.master
 
 
-Broker Class
-------------
+mitogen.parent
+--------------
 
-.. autoclass:: econtext.core.Broker
-   :members:
+.. automodule:: mitogen.parent
 
 
-Context Class
--------------
+mitogen.fakessh
+---------------
 
-.. autoclass:: econtext.core.Context
-   :members:
+.. image:: images/fakessh.svg
+    :class: mitogen-right-300
 
+.. automodule:: mitogen.fakessh
+.. currentmodule:: mitogen.fakessh
+.. autofunction:: run (dest, router, args, daedline=None, econtext=None)
 
-Channel Class
--------------
 
-.. autoclass:: econtext.core.Channel
+
+Message Class
+=============
+
+.. currentmodule:: mitogen.core
+.. autoclass:: Message
+    :members:
+
+
+Router Class
+============
+
+.. currentmodule:: mitogen.core
+.. autoclass:: Router
+    :members:
+
+
+.. currentmodule:: mitogen.parent
+.. autoclass:: Router
    :members:
 
 
-ExternalContext Class
----------------------
+.. currentmodule:: mitogen.master
+.. autoclass:: Router (broker=None)
+   :members:
+
+
+.. _context-factories:
+
+Connection Methods
+==================
+
+.. currentmodule:: mitogen.parent
+.. method:: Router.buildah (container=None, buildah_path=None, username=None, \**kwargs)
+
+    Construct a context on the local machine over a ``buildah`` invocation.
+    Accepts all parameters accepted by :meth:`local`, in addition to:
+
+    :param str container:
+        The name of the Buildah container to connect to.
+    :param str buildah_path:
+        Filename or complete path to the ``buildah`` binary. ``PATH`` will be
+        searched if given as a filename. Defaults to ``buildah``.
+    :param str username:
+        Username to use, defaults to unset.
+
+.. currentmodule:: mitogen.parent
+.. method:: Router.fork (on_fork=None, on_start=None, debug=False, profiling=False, via=None)
+
+    Construct a context on the local machine by forking the current
+    process. The forked child receives a new identity, sets up a new broker
+    and router, and responds to function calls identically to children
+    created using other methods.
 
-.. class:: econtext.core.ExternalContext
+    The use of this method is strongly discouraged. It requires Python 2.6 or
+    newer, as older Pythons made no effort to reset threading state upon fork.
+
+    For long-lived processes, :meth:`local` is always better as it
+    guarantees a pristine interpreter state that inherited little from the
+    parent. Forking should only be used in performance-sensitive scenarios
+    where short-lived children must be spawned to isolate potentially buggy
+    code, and only after accounting for all the bad things possible as a
+    result of, at a minimum:
 
-    External context implementation.
+    * Files open in the parent remaining open in the child,
+      causing the lifetime of the underlying object to be extended
+      indefinitely.
 
-    .. attribute:: broker
+      * From the perspective of external components, this is observable
+        in the form of pipes and sockets that are never closed, which may
+        break anything relying on closure to signal protocol termination.
 
-        The :py:class:`econtext.core.Broker` instance.
+      * Descriptors that reference temporary files will not have their disk
+        space reclaimed until the child exits.
 
-    .. attribute:: context
+    * Third party package state, such as urllib3's HTTP connection pool,
+      attempting to write to file descriptors shared with the parent,
+      causing random failures in both parent and child.
 
-            The :py:class:`econtext.core.Context` instance.
+    * UNIX signal handlers installed in the parent process remaining active
+      in the child, despite associated resources, such as service threads,
+      child processes, resource usage counters or process timers becoming
+      absent or reset in the child.
+
+    * Library code that makes assumptions about the process ID remaining
+      unchanged, for example to implement inter-process locking, or to
+      generate file names.
+
+    * Anonymous ``MAP_PRIVATE`` memory mappings whose storage requirement
+      doubles as either parent or child dirties their pages.
+
+    * File-backed memory mappings that cannot have their space freed on
+      disk due to the mapping living on in the child.
+
+    * Difficult to diagnose memory usage and latency spikes due to object
+      graphs becoming unreferenced in either parent or child, causing
+      immediate copy-on-write to large portions of the process heap.
+
+    * Locks held in the parent causing random deadlocks in the child, such
+      as when another thread emits a log entry via the :mod:`logging`
+      package concurrent to another thread calling :meth:`fork`, or when a C
+      extension module calls the C library allocator, or when a thread is using
+      the C library DNS resolver, for example via :func:`socket.gethostbyname`.
+
+    * Objects existing in Thread-Local Storage of every non-:meth:`fork`
+      thread becoming permanently inaccessible, and never having their
+      object destructors called, including TLS usage by native extension
+      code, triggering many new variants of all the issues above.
+
+    * Pseudo-Random Number Generator state that is easily observable by
+      network peers to be duplicate, violating requirements of
+      cryptographic protocols through one-time state reuse. In the worst
+      case, children continually reuse the same state due to repeatedly
+      forking from a static parent.
 
-    .. attribute:: channel
+    :meth:`fork` cleans up Mitogen-internal objects, in addition to
+    locks held by the :mod:`logging` package, reseeds
+    :func:`random.random`, and the OpenSSL PRNG via
+    :func:`ssl.RAND_add`, but only if the :mod:`ssl` module is
+    already loaded. You must arrange for your program's state, including
+    any third party packages in use, to be cleaned up by specifying an
+    `on_fork` function.
+
+    The associated stream implementation is
+    :class:`mitogen.fork.Stream`.
+
+    :param function on_fork:
+        Function invoked as `on_fork()` from within the child process. This
+        permits supplying a program-specific cleanup function to break
+        locks and close file descriptors belonging to the parent from
+        within the child.
+
+    :param function on_start:
+        Invoked as `on_start(econtext)` from within the child process after
+        it has been set up, but before the function dispatch loop starts.
+        This permits supplying a custom child main function that inherits
+        rich data structures that cannot normally be passed via a
+        serialization.
+
+    :param mitogen.core.Context via:
+        Same as the `via` parameter for :meth:`local`.
+
+    :param bool debug:
+        Same as the `debug` parameter for :meth:`local`.
+
+    :param bool profiling:
+        Same as the `profiling` parameter for :meth:`local`.
+
+.. method:: Router.local (remote_name=None, python_path=None, debug=False, connect_timeout=None, profiling=False, via=None)
+
+    Construct a context on the local machine as a subprocess of the current
+    process. The associated stream implementation is
+    :class:`mitogen.master.Stream`.
+
+    :param str remote_name:
+        The ``argv[0]`` suffix for the new process. If `remote_name` is
+        ``test``, the new process ``argv[0]`` will be ``mitogen:test``.
+
+        If unspecified, defaults to ``<username>@<hostname>:<pid>``.
+
+        This variable cannot contain slash characters, as the resulting
+        ``argv[0]`` must be presented in such a way as to allow Python to
+        determine its installation prefix. This is required to support
+        virtualenv.
+
+    :param str|list python_path:
+        String or list path to the Python interpreter to use for bootstrap.
+        Defaults to :data:`sys.executable` for local connections, and
+        ``python`` for remote connections.
+
+        It is possible to pass a list to invoke Python wrapped using
+        another tool, such as ``["/usr/bin/env", "python"]``.
+
+    :param bool debug:
+        If :data:`True`, arrange for debug logging (:meth:`enable_debug`) to
+        be enabled in the new context. Automatically :data:`True` when
+        :meth:`enable_debug` has been called, but may be used
+        selectively otherwise.
+
+    :param bool unidirectional:
+        If :data:`True`, arrange for the child's router to be constructed
+        with :attr:`unidirectional routing
+        <mitogen.core.Router.unidirectional>` enabled. Automatically
+        :data:`True` when it was enabled for this router, but may still be
+        explicitly set to :data:`False`.
+
+    :param float connect_timeout:
+        Fractional seconds to wait for the subprocess to indicate it is
+        healthy. Defaults to 30 seconds.
+
+    :param bool profiling:
+        If :data:`True`, arrange for profiling (:data:`profiling`) to be
+        enabled in the new context. Automatically :data:`True` when
+        :data:`profiling` is :data:`True`, but may be used selectively
+        otherwise.
+
+    :param mitogen.core.Context via:
+        If not :data:`None`, arrange for construction to occur via RPCs
+        made to the context `via`, and for :data:`ADD_ROUTE
+        <mitogen.core.ADD_ROUTE>` messages to be generated as appropriate.
+
+        .. code-block:: python
+
+            # SSH to the remote machine.
+            remote_machine = router.ssh(hostname='mybox.com')
+
+            # Use the SSH connection to create a sudo connection.
+            remote_root = router.sudo(username='root', via=remote_machine)
+
+.. method:: Router.doas (username=None, password=None, doas_path=None, password_prompt=None, incorrect_prompts=None, \**kwargs)
+
+    Construct a context on the local machine over a ``doas`` invocation.
+    The ``doas`` process is started in a newly allocated pseudo-terminal,
+    and supports typing interactive passwords.
+
+    Accepts all parameters accepted by :meth:`local`, in addition to:
+
+    :param str username:
+        Username to use, defaults to ``root``.
+    :param str password:
+        The account password to use if requested.
+    :param str doas_path:
+        Filename or complete path to the ``doas`` binary. ``PATH`` will be
+        searched if given as a filename. Defaults to ``doas``.
+    :param bytes password_prompt:
+        A string that indicates ``doas`` is requesting a password. Defaults
+        to ``Password:``.
+    :param list incorrect_prompts:
+        List of bytestrings indicating the password is incorrect. Defaults
+        to `(b"doas: authentication failed")`.
+    :raises mitogen.doas.PasswordError:
+        A password was requested but none was provided, the supplied
+        password was incorrect, or the target account did not exist.
+
+.. method:: Router.docker (container=None, image=None, docker_path=None, \**kwargs)
+
+    Construct a context on the local machine within an existing or
+    temporary new Docker container using the ``docker`` program. One of
+    `container` or `image` must be specified.
+
+    Accepts all parameters accepted by :meth:`local`, in addition to:
 
-            The :py:class:`econtext.core.Channel` over which
-            :py:data:`CALL_FUNCTION` requests are received.
+    :param str container:
+        Existing container to connect to. Defaults to :data:`None`.
+    :param str username:
+        Username within the container to :func:`setuid` to. Defaults to
+        :data:`None`, which Docker interprets as ``root``.
+    :param str image:
+        Image tag to use to construct a temporary container. Defaults to
+        :data:`None`.
+    :param str docker_path:
+        Filename or complete path to the Docker binary. ``PATH`` will be
+        searched if given as a filename. Defaults to ``docker``.
+
+.. method:: Router.jail (container, jexec_path=None, \**kwargs)
+
+    Construct a context on the local machine within a FreeBSD jail using
+    the ``jexec`` program.
+
+    Accepts all parameters accepted by :meth:`local`, in addition to:
+
+    :param str container:
+        Existing container to connect to. Defaults to :data:`None`.
+    :param str username:
+        Username within the container to :func:`setuid` to. Defaults to
+        :data:`None`, which ``jexec`` interprets as ``root``.
+    :param str jexec_path:
+        Filename or complete path to the ``jexec`` binary. ``PATH`` will be
+        searched if given as a filename. Defaults to ``/usr/sbin/jexec``.
+
+.. method:: Router.kubectl (pod, kubectl_path=None, kubectl_args=None, \**kwargs)
+
+    Construct a context in a container via the Kubernetes ``kubectl``
+    program.
+
+    Accepts all parameters accepted by :meth:`local`, in addition to:
+
+    :param str pod:
+        Kubernetes pod to connect to.
+    :param str kubectl_path:
+        Filename or complete path to the ``kubectl`` binary. ``PATH`` will
+        be searched if given as a filename. Defaults to ``kubectl``.
+    :param list kubectl_args:
+        Additional arguments to pass to the ``kubectl`` command.
+
+.. method:: Router.lxc (container, lxc_attach_path=None, \**kwargs)
+
+    Construct a context on the local machine within an LXC classic
+    container using the ``lxc-attach`` program.
+
+    Accepts all parameters accepted by :meth:`local`, in addition to:
+
+    :param str container:
+        Existing container to connect to. Defaults to :data:`None`.
+    :param str lxc_attach_path:
+        Filename or complete path to the ``lxc-attach`` binary. ``PATH``
+        will be searched if given as a filename. Defaults to
+        ``lxc-attach``.
+
+.. method:: Router.lxd (container, lxc_path=None, \**kwargs)
+
+    Construct a context on the local machine within a LXD container using
+    the ``lxc`` program.
+
+    Accepts all parameters accepted by :meth:`local`, in addition to:
+
+    :param str container:
+        Existing container to connect to. Defaults to :data:`None`.
+    :param str lxc_path:
+        Filename or complete path to the ``lxc`` binary. ``PATH`` will be
+        searched if given as a filename. Defaults to ``lxc``.
+
+.. currentmodule:: mitogen.parent
+.. method:: Router.podman (container=None, podman_path=None, username=None, \**kwargs)
+
+    Construct a context on the local machine over a ``podman`` invocation.
+    Accepts all parameters accepted by :meth:`local`, in addition to:
+
+    :param str container:
+        The name of the Podman container to connect to.
+    :param str podman_path:
+        Filename or complete path to the ``podman`` binary. ``PATH`` will be
+        searched if given as a filename. Defaults to ``podman``.
+    :param str username:
+        Username to use, defaults to unset.
+
+.. method:: Router.setns (container, kind, username=None, docker_path=None, lxc_info_path=None, machinectl_path=None, \**kwargs)
+
+    Construct a context in the style of :meth:`local`, but change the
+    active Linux process namespaces via calls to `setns(2)` before
+    executing Python.
+
+    The namespaces to use, and the active root file system are taken from
+    the root PID of a running Docker, LXC, LXD, or systemd-nspawn
+    container.
+
+    The setns method depends on the built-in :mod:`ctypes` module, and thus
+    does not support Python 2.4.
+
+    A program is required only to find the root PID, after which management
+    of the child Python interpreter is handled directly.
+
+    :param str container:
+        Container to connect to.
+    :param str kind:
+        One of ``docker``, ``lxc``, ``lxd`` or ``machinectl``.
+    :param str username:
+        Username within the container to :func:`setuid` to. Defaults to
+        ``root``.
+    :param str docker_path:
+        Filename or complete path to the Docker binary. ``PATH`` will be
+        searched if given as a filename. Defaults to ``docker``.
+    :param str lxc_path:
+        Filename or complete path to the LXD ``lxc`` binary. ``PATH`` will
+        be searched if given as a filename. Defaults to ``lxc``.
+    :param str lxc_info_path:
+        Filename or complete path to the LXC ``lxc-info`` binary. ``PATH``
+        will be searched if given as a filename. Defaults to ``lxc-info``.
+    :param str machinectl_path:
+        Filename or complete path to the ``machinectl`` binary. ``PATH``
+        will be searched if given as a filename. Defaults to
+        ``machinectl``.
+
+.. method:: Router.su (username=None, password=None, su_path=None, password_prompt=None, incorrect_prompts=None, \**kwargs)
+
+    Construct a context on the local machine over a ``su`` invocation. The
+    ``su`` process is started in a newly allocated pseudo-terminal, and
+    supports typing interactive passwords.
+
+    Accepts all parameters accepted by :meth:`local`, in addition to:
+
+    :param str username:
+        Username to pass to ``su``, defaults to ``root``.
+    :param str password:
+        The account password to use if requested.
+    :param str su_path:
+        Filename or complete path to the ``su`` binary. ``PATH`` will be
+        searched if given as a filename. Defaults to ``su``.
+    :param bytes password_prompt:
+        The string that indicates ``su`` is requesting a password. Defaults
+        to ``Password:``.
+    :param str incorrect_prompts:
+        Strings that signal the password is incorrect. Defaults to `("su:
+        sorry", "su: authentication failure")`.
+
+    :raises mitogen.su.PasswordError:
+        A password was requested but none was provided, the supplied
+        password was incorrect, or (on BSD) the target account did not
+        exist.
+
+.. method:: Router.sudo (username=None, sudo_path=None, password=None, \**kwargs)
+
+    Construct a context on the local machine over a ``sudo`` invocation.
+    The ``sudo`` process is started in a newly allocated pseudo-terminal,
+    and supports typing interactive passwords.
+
+    Accepts all parameters accepted by :meth:`local`, in addition to:
+
+    :param str username:
+        Username to pass to sudo as the ``-u`` parameter, defaults to
+        ``root``.
+    :param str sudo_path:
+        Filename or complete path to the sudo binary. ``PATH`` will be
+        searched if given as a filename. Defaults to ``sudo``.
+    :param str password:
+        The password to use if/when sudo requests it. Depending on the sudo
+        configuration, this is either the current account password or the
+        target account password. :class:`mitogen.sudo.PasswordError`
+        will be raised if sudo requests a password but none is provided.
+    :param bool set_home:
+        If :data:`True`, request ``sudo`` set the ``HOME`` environment
+        variable to match the target UNIX account.
+    :param bool preserve_env:
+        If :data:`True`, request ``sudo`` to preserve the environment of
+        the parent process.
+    :param str selinux_type:
+        If not :data:`None`, the SELinux security context to use.
+    :param str selinux_role:
+        If not :data:`None`, the SELinux role to use.
+    :param list sudo_args:
+        Arguments in the style of :data:`sys.argv` that would normally
+        be passed to ``sudo``. The arguments are parsed in-process to set
+        equivalent parameters. Re-parsing ensures unsupported options cause
+        :class:`mitogen.core.StreamError` to be raised, and that
+        attributes of the stream match the actual behaviour of ``sudo``.
+
+.. method:: Router.ssh (hostname, username=None, ssh_path=None, ssh_args=None, port=None, check_host_keys='enforce', password=None, identity_file=None, identities_only=True, compression=True, \**kwargs)
+
+    Construct a remote context over an OpenSSH ``ssh`` invocation.
+
+    The ``ssh`` process is started in a newly allocated pseudo-terminal to
+    support typing interactive passwords and responding to prompts, if a
+    password is specified, or `check_host_keys=accept`. In other scenarios,
+    ``BatchMode`` is enabled and no PTY is allocated. For many-target
+    configurations, both options should be avoided as most systems have a
+    conservative limit on the number of pseudo-terminals that may exist.
+
+    Accepts all parameters accepted by :meth:`local`, in addition to:
+
+    :param str username:
+        The SSH username; default is unspecified, which causes SSH to pick
+        the username to use.
+    :param str ssh_path:
+        Absolute or relative path to ``ssh``. Defaults to ``ssh``.
+    :param list ssh_args:
+        Additional arguments to pass to the SSH command.
+    :param int port:
+        Port number to connect to; default is unspecified, which causes SSH
+        to pick the port number.
+    :param str check_host_keys:
+        Specifies the SSH host key checking mode. Defaults to ``enforce``.
+
+        * ``ignore``: no host key checking is performed. Connections never
+          fail due to an unknown or changed host key.
+        * ``accept``: known hosts keys are checked to ensure they match,
+          new host keys are automatically accepted and verified in future
+          connections.
+        * ``enforce``: known host keys are checked to ensure they match,
+          unknown hosts cause a connection failure.
+    :param str password:
+        Password to type if/when ``ssh`` requests it. If not specified and
+        a password is requested, :class:`mitogen.ssh.PasswordError` is
+        raised.
+    :param str identity_file:
+        Path to an SSH private key file to use for authentication. Default
+        is unspecified, which causes SSH to pick the identity file.
+
+        When this option is specified, only `identity_file` will be used by
+        the SSH client to perform authenticaion; agent authentication is
+        automatically disabled, as is reading the default private key from
+        ``~/.ssh/id_rsa``, or ``~/.ssh/id_dsa``.
+    :param bool identities_only:
+        If :data:`True` and a password or explicit identity file is
+        specified, instruct the SSH client to disable any authentication
+        identities inherited from the surrounding environment, such as
+        those loaded in any running ``ssh-agent``, or default key files
+        present in ``~/.ssh``. This ensures authentication attempts only
+        occur using the supplied password or SSH key.
+    :param bool compression:
+        If :data:`True`, enable ``ssh`` compression support. Compression
+        has a minimal effect on the size of modules transmitted, as they
+        are already compressed, however it has a large effect on every
+        remaining message in the otherwise uncompressed stream protocol,
+        such as function call arguments and return values.
+    :param int ssh_debug_level:
+        Optional integer `0..3` indicating the SSH client debug level.
+    :raises mitogen.ssh.PasswordError:
+        A password was requested but none was specified, or the specified
+        password was incorrect.
+
+    :raises mitogen.ssh.HostKeyError:
+        When `check_host_keys` is set to either ``accept``, indicates a
+        previously recorded key no longer matches the remote machine. When
+        set to ``enforce``, as above, but additionally indicates no
+        previously recorded key exists for the remote machine.
 
-    .. attribute:: stdout_log
 
-        The :py:class:`econtext.core.IoLogger` connected to ``stdout``.
+Context Class
+=============
+
+.. currentmodule:: mitogen.core
+.. autoclass:: Context
+    :members:
+
 
-    .. attribute:: importer
+.. currentmodule:: mitogen.parent
+.. autoclass:: Context
+    :members:
+
+.. currentmodule:: mitogen.parent
+.. autoclass:: CallChain
+    :members:
+
+
+Receiver Class
+==============
 
-        The :py:class:`econtext.core.Importer` instance.
+.. currentmodule:: mitogen.core
+.. autoclass:: Receiver
+    :members:
 
-    .. attribute:: stdout_log
 
-        The :py:class:`IoLogger` connected to ``stdout``.
+Sender Class
+============
 
-    .. attribute:: stderr_log
+.. currentmodule:: mitogen.core
+.. autoclass:: Sender
+    :members:
 
-        The :py:class:`IoLogger` connected to ``stderr``.
 
+Select Class
+============
 
-econtext.master
-===============
+.. module:: mitogen.select
+.. currentmodule:: mitogen.select
+
+.. autoclass:: Event
+    :members:
+
+.. autoclass:: Select
+    :members:
+
+
+Channel Class
+=============
+
+.. currentmodule:: mitogen.core
+.. autoclass:: Channel
+   :members:
 
-.. automodule:: econtext.master
 
 
 Broker Class
-------------
+============
 
-.. autoclass:: econtext.master.Broker
+.. currentmodule:: mitogen.core
+.. autoclass:: Broker
    :members:
 
 
-Context Class
--------------
+.. currentmodule:: mitogen.master
+.. autoclass:: Broker
+   :members:
+
 
-.. autoclass:: econtext.master.Context
+Fork Safety
+===========
+
+.. currentmodule:: mitogen.os_fork
+.. autoclass:: Corker
    :members:
 
 
-econtext.utils
-==============
+Utility Functions
+=================
+
+.. currentmodule:: mitogen.core
+.. function:: now
+
+   A reference to :func:`time.time` on Python 2, or :func:`time.monotonic` on
+   Python >3.3. We prefer :func:`time.monotonic` when available to ensure
+   timers are not impacted by system clock changes.
+
+
+.. module:: mitogen.utils
+
+A random assortment of utility functions useful on masters and children.
+
+.. currentmodule:: mitogen.utils
+.. autofunction:: cast
+
+
+.. currentmodule:: mitogen.utils
+.. autofunction:: setup_gil
+.. autofunction:: disable_site_packages
+.. autofunction:: log_to_file
+.. autofunction:: run_with_router(func, \*args, \**kwargs)
+
+.. currentmodule:: mitogen.utils
+.. decorator:: with_router
+
+    Decorator version of :func:`run_with_router`. Example:
+
+    .. code-block:: python
+
+        @with_router
+        def do_stuff(router, arg):
+            pass
+
+        do_stuff(blah, 123)
+
+
+Exceptions
+==========
+
+.. currentmodule:: mitogen.core
+
+.. autoclass:: Error
+.. autoclass:: CallError
+.. autoclass:: ChannelError
+.. autoclass:: LatchError
+.. autoclass:: StreamError
+.. autoclass:: TimeoutError
 
-.. automodule:: econtext.utils
-  :members:
+.. currentmodule:: mitogen.parent
+.. autoclass:: EofError
+.. autoclass:: CancelledError

docs/conf.py

@@ -1,23 +1,121 @@
+import re
 import sys
-sys.path.append('..')
 
-author = u'David Wilson'
-copyright = u'2016, David Wilson'
-exclude_patterns = ['_build']
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx']
+sys.path.append('.')
+
+
+def changelog_version(path, encoding='utf-8'):
+    "Return the 1st *stable* (not pre, dev) version in the changelog"
+    # See also grep_version() in setup.py
+    # e.g. "0.1.2, (1999-12-31)\n"
+    version_pattern = re.compile(
+        r'^v(?P<version>\d+\.\d+\.\d+) \((?P<date>\d\d\d\d-\d\d-\d\d)\)$',
+        re.MULTILINE,
+    )
+
+    with open(path, encoding=encoding) as f:
+        match = version_pattern.search(f.read())
+        return match.group('version')
+
+
+VERSION = changelog_version('changelog.rst')
+
+author = u'Network Genomics'
+copyright = u'2021, the Mitogen authors'
+exclude_patterns = ['_build', '.venv']
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinxcontrib.programoutput', 'domainrefs']
+
+# get rid of version from <title>, it messes with piwik
+html_title = 'Mitogen Documentation'
+
+html_show_copyright = False
 html_show_sourcelink = False
 html_show_sphinx = False
 html_sidebars = {'**': ['globaltoc.html', 'github.html']}
 html_static_path = ['_static']
 html_theme = 'alabaster'
-htmlhelp_basename = 'econtextdoc'
-intersphinx_mapping = {'python': ('https://docs.python.org/2', None)}
+html_theme_options = {
+    'font_family': "Georgia, serif",
+    'head_font_family': "Georgia, serif",
+    'fixed_sidebar': True,
+    'show_powered_by': False,
+    'pink_2': 'fffafaf',
+    'pink_1': '#fff0f0',
+}
+htmlhelp_basename = 'mitogendoc'
+intersphinx_mapping = {'python': ('https://docs.python.org/3', None)}
 language = None
-master_doc = 'index'
-project = u'econtext'
+master_doc = 'toc'
+project = u'Mitogen'
 pygments_style = 'sphinx'
-release = u'master'
+release = VERSION
 source_suffix = '.rst'
 templates_path = ['_templates']
 todo_include_todos = False
-version = u'master'
+version = VERSION
+
+domainrefs = {
+    'gh:commit': {
+        'text': '%s',
+        'url': 'https://github.com/mitogen-hq/mitogen/commit/%s',
+    },
+    'gh:issue': {
+        'text': '#%s',
+        'url': 'https://github.com/mitogen-hq/mitogen/issues/%s',
+    },
+    'gh:pull': {
+        'text': '#%s',
+        'url': 'https://github.com/mitogen-hq/mitogen/pull/%s',
+    },
+    'gh:ansissue': {
+        'text': 'Ansible #%s',
+        'url': 'https://github.com/ansible/ansible/issues/%s',
+    },
+    'gh:anspull': {
+        'text': 'Ansible #%s',
+        'url': 'https://github.com/ansible/ansible/pull/%s',
+    },
+
+    'ans:mod': {
+        'text': '%s module',
+        'url': 'https://docs.ansible.com/ansible/latest/modules/%s_module.html',
+    },
+    'ans:conn': {
+        'text': '%s connection plug-in',
+        'url': 'https://docs.ansible.com/ansible/latest/plugins/connection/%s.html',
+    },
+    'freebsd:man2': {
+        'text': '%s(2)',
+        'url': 'https://man.freebsd.org/cgi/man.cgi?query=%s',
+    },
+    'linux:man1': {
+        'text': '%s(1)',
+        'url': 'https://man7.org/linux/man-pages/man1/%s.1.html',
+    },
+    'linux:man2': {
+        'text': '%s(2)',
+        'url': 'https://man7.org/linux/man-pages/man2/%s.2.html',
+    },
+    'linux:man3': {
+        'text': '%s(3)',
+        'url': 'https://man7.org/linux/man-pages/man3/%s.3.html',
+    },
+    'linux:man7': {
+        'text': '%s(7)',
+        'url': 'https://man7.org/linux/man-pages/man7/%s.7.html',
+    },
+}
+
+# > ## Official guidance
+# > Query PyPI’s JSON API to determine where to download files from.
+# > ## Predictable URLs
+# > You can use our conveyor service to fetch this file, which exists for
+# > cases where using the API is impractical or impossible.
+# > -- https://warehouse.pypa.io/api-reference/integration-guide.html#predictable-urls
+rst_epilog = """
+
+.. |mitogen_version| replace:: %(VERSION)s
+
+.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz <https://files.pythonhosted.org/packages/source/m/mitogen/mitogen-%(VERSION)s.tar.gz>`__
+
+""" % locals()

docs/contributors.rst

@@ -0,0 +1,152 @@
+
+Contributors
+============
+
+Mitogen is production ready exclusively thanks to the careful testing, gracious
+sponsorship and outstanding future-thinking of its early adopters.
+
+.. raw:: html
+
+    <!--
+
+.. image:: images/sponsors/cgi.svg
+.. image:: images/sponsors/grx.svg
+.. image:: images/sponsors/securelink.svg
+.. image:: images/sponsors/seantis.svg
+.. raw:: html
+
+    -->
+
+    <div style="background: #efefef; padding: 16px; margin: 1.5em 0;">
+        <div style="float: left; padding: 8px 32px 16px 8px;">
+            <img src="_images/cgi.svg" height=110 width=238>
+        </div>
+        <div>
+            <p>
+            Founded in 1976, CGI is one of the world’s largest IT and business
+            consulting services firms, helping clients achieve their goals,
+            including becoming customer-centric digital organizations.
+            </p>
+
+            <p>
+            <br clear="all">
+            For career opportunities, please visit <a
+            href="http://cgi-group.co.uk/defence-and-intelligence-opportunities">cgi-group.co.uk/defence-and-intelligence-opportunities</a>.
+            </p>
+
+            <p style="margin-bottom: 0px;">
+            To <a
+            href="https://cgi.njoyn.com/CGI/xweb/XWeb.asp?page=jobdetails&CLID=21001&SBDID=21814&jobid=J0118-0787">directly
+            apply</a> to a UK team currently using Mitogen, contact us
+            regarding <a
+            href="https://cgi.njoyn.com/CGI/xweb/XWeb.asp?page=jobdetails&CLID=21001&SBDID=21814&jobid=J0118-0787">Open
+            Source Developer/DevOps</a> opportunities.
+            </p>
+        </div>
+    </div>
+
+.. raw:: html
+
+    <table border="0" width="100%">
+    <tr>
+        <td width="160" style="padding: 12px; text-align: center;">
+            <a href="https://www.goodrx.com/"
+                ><img src="_images/grx.svg" width="160"></a>
+        </td>
+        <td style="padding: 12px;" valign=middle>
+            GoodRx is America’s #1 prescription price transparency platform.
+            Join GoodRx and help consumers save up to 80% on their
+            medications. Apply today at <a
+            href="https://www.goodrx.com/jobs">www.goodrx.com/jobs</a>
+        </td>
+    </tr>
+    <tr>
+        <td width="160" style="padding: 12px; text-align: center;">
+            <a href="https://www.seantis.ch/"
+                ><img src="_images/seantis.svg" width="160"></a>
+        </td>
+        <td style="padding: 12px;" valign=middle>
+            Seantis GmbH<br>
+            <a href="https://www.seantis.ch">www.seantis.ch</a>
+        </td>
+    </tr>
+    <tr>
+        <td width="160" style="padding: 12px; text-align: center;">
+            <a href="https://www.securelink.com/"
+                ><img src="_images/securelink.svg" width="160"></a>
+        </td>
+        <td style="padding: 12px;">
+            Secure Third-Party Remote Access for Highly Regulated Industries<br>
+            <a href="https://www.securelink.com/">www.securelink.com</a>
+        </td>
+    </tr>
+    </table>
+
+
+.. raw:: html
+
+    <h3>Private Sponsors</h3>
+
+    <ul style="line-height: 120% !important;">
+    <li><a href="https://skunkwerks.at/">SkunkWerks</a> &mdash;
+        <em>Mitogen on FreeBSD runs like a kid in a candy store: fast &amp;
+        sweet.</em></li>
+    <li>Donald Clark Jackson &mdash;
+        <em>Mitogen is an exciting project, and I am happy to support its
+        development.</em></li>
+    <li><a href="https://nuvini.com">Niels Hendriks</a></li>
+    <li><a href="https://uberspace.de/">Uberspace</a> &mdash;
+        <em>Shared hosting for command-line lovers</em></li>
+    </ul>
+
+
+    <h3>Defenders of Time</h3>
+
+    <ul>
+    <li>Icil &mdash; <em>Time saving, money saving...phenomenal! Keep going and
+        give us more. We await with anticipation.</em></li>
+    <li><a href="https://www.systemli.org/">systemli tech collective</a> &mdash;
+        <em>D.I.Y.</em></li>
+    </ul>
+
+
+.. raw:: html
+
+    <h3>Productivity Lovers</h3>
+
+    <ul>
+    <li>Alex Willmer</li>
+    <li><a href="https://github.com/momiji">Christian Bourgeois </a></li>
+    <li><a href="https://underwhelm.net/">Dan Dorman</a> &mdash; - <em>When I truly understand my enemy … then in that very moment I also love him.</em></li>
+    <li>Daniel Foerster</li>
+    <li><a href="https://www.deps.co/">Deps</a> &mdash; <em>Private Maven Repository Hosting for Java, Scala, Groovy, Clojure</em></li>
+    <li><a href="https://www.edport.co.uk/">Edward Wilson</a> &mdash; <em>To efficiency and beyond! I wish Mitogen and all who sail in her the best of luck.</em></li>
+    <li><a href="https://www.epartment.nl/">Epartment</a></li>
+    <li><a href="http://andrianaivo.org/">Fidy Andrianaivo</a> &mdash; <em>never let a human do an ansible job ;)</em></li>
+    <li><a href="https://www.channable.com">rkrzr</a></li>
+    <li><a href="https://github.com/Nihlus">Jarl Gullberg</a></li>
+    <li>jgadling</li>
+    <li>John F Wall &mdash; <em>Making Ansible Great with Massive Parallelism</em></li>
+    <li><a href="https://github.com/jrosser">Jonathan Rosser</a></li>
+    <li><a href="https://github.com/jmkeyes">Joshua M. Keyes</a></li>
+    <li>KennethC</li>
+    <li><a href="https://github.com/lberruti">Luca Berruti</li>
+    <li>Lewis Bellwood &mdash; <em>Happy to be apart of a great project.</em></li>
+    <li>luto</li>
+    <li><a href="https://github.com/markafarrell">@markafarrell</a></li>
+    <li><a href="https://mayeu.me/">Mayeu a.k.a Matthieu Maury</a></li>
+    <li><a href="https://github.com/madsi1m">Michael D'Silva</a></li>
+    <li><a href="https://github.com/mordekasg">mordek</a></li>
+    <li><a href="https://twitter.com/nathanhruby">@nathanhruby</a></li>
+    <li><a href="https://github.com/opoplawski">Orion Poplawski</a></li>
+    <li><a href="https://github.com/philfry">Philippe Kueck</a></li>
+    <li><a href="http://pageflows.com/">Ramy</a></li>
+    <li>Scott Vokes</li>
+    <li><a href="https://twitter.com/sirtux">Tom Eichhorn</a></li>
+    <li><a href="https://dotat.at/">Tony Finch</a></li>
+    <li>Tony Million &mdash; Never wear socks and sandles.</li>
+    <li>randy &mdash; <em>desperate for automation</em></li>
+    <li>Michael & Vicky Twomey-Lee</li>
+    <li><a href="http://www.wezm.net/">Wesley Moore</a></li>
+    <li><a href="https://github.com/baryluk">Witold Baryluk</a></li>
+    </ul>

docs/domainrefs.py

@@ -0,0 +1,41 @@
+
+import functools
+import re
+
+import docutils.nodes
+import docutils.utils
+
+
+CUSTOM_RE = re.compile('(.*) <(.*)>')
+
+
+def role(config, role, rawtext, text, lineno, inliner, options={}, content=[]):
+    template = 'https://docs.ansible.com/ansible/latest/modules/%s_module.html'
+
+    match = CUSTOM_RE.match(text)
+    if match:  # "custom text <real link>"
+        title = match.group(1)
+        text = match.group(2)
+    elif text.startswith('~'):  # brief
+        text = text[1:]
+        title = config.get('brief', '%s') % (
+            docutils.utils.unescape(text),
+        )
+    else:
+        title = config.get('text', '%s') % (
+            docutils.utils.unescape(text),
+        )
+
+    node = docutils.nodes.reference(
+        rawsource=rawtext,
+        text=title,
+        refuri=config['url'] % (text,),
+        **options
+    )
+
+    return [node], []
+
+
+def setup(app):
+    for name, info in app.config._raw_config['domainrefs'].items():
+        app.add_role(name, functools.partial(role, info))

docs/examples.rst

@@ -0,0 +1,238 @@
+
+Examples
+========
+
+
+Fixing Bugs By Replacing Shell
+------------------------------
+
+Have you ever encountered shell like this? It arranges to conditionally execute
+an ``if`` statement as root on a file server behind a bastion host:
+
+.. code-block:: bash
+
+    ssh bastion "
+        if [ \"$PROD\" ];
+        then
+            ssh fileserver sudo su -c \"
+                if grep -qs /dev/sdb1 /proc/mounts;
+                then
+                    echo \\\"sdb1 already mounted!\\\";
+                    umount /dev/sdb1
+                fi;
+                rm -rf \\\"/media/Main Backup Volume\\\"/*;
+                mount /dev/sdb1 \\\"/media/Main Backup Volume\\\"
+            \";
+        fi;
+        sudo touch /var/run/start_backup;
+    "
+
+Chances are high this is familiar territory, we've all seen it, and those
+working in infrastructure have almost certainly written it. At first glance,
+ignoring that annoying quoting, it looks perfectly fine: well structured,
+neatly indented, and the purpose of the snippet seems clear.
+
+1. At first glance, is ``"/media/Main Backup Volume"`` quoted correctly?
+2. How will the ``if`` statement behave if there is a problem with the machine,
+   and, say, the ``/bin/grep`` binary is absent?
+3. Ignoring quoting, are there any other syntax problems?
+4. If this snippet is pasted from its original script into an interactive
+   shell, will it behave the same as before?
+5. Can you think offhand of differences in how the arguments to ``sudo
+   ...`` and ``ssh fileserver ...`` are parsed?
+6. In which context will the ``*`` glob be expanded, if it is expanded at all?
+7. What will the exit status of ``ssh bastion`` be if ``ssh fileserver`` fails?
+
+
+Innocent But Deadly
+~~~~~~~~~~~~~~~~~~~
+
+1. The quoting used is nonsense! At best, ``mount`` will receive 3 arguments.
+   At worst, the snippet will not parse at all.
+2. The ``if`` statement will treat a missing ``grep`` binary (exit status 127)
+   the same as if ``/dev/sdb1`` was not mounted at all (exit status 1). Unless
+   the program executing this script is parsing ``stderr`` output, the failure
+   won't be noticed. Consequently, since the volume was still mounted when
+   ``rm`` was executed, it got wiped.
+3. There is at least one more syntax error present: a semicolon missing after
+   the ``umount`` command.
+4. If you paste the snippet into an interactive shell, the apparently quoted
+   "!" character in the ``echo`` command will be interpreted as a history
+   expansion.
+5. ``sudo`` preserves the remainder of the argument vector as-is, while
+   ``ssh`` **concatenates** each part into a single string that is passed to
+   the login shell. While quotes appearing within arguments are preserved by
+   ``sudo``, without additional effort, pairs of quotes are effectively
+   stripped by ``ssh``.
+6. As for where the glob is expanded, the answer is I have absolutely no idea
+   without running the code, which might wipe out the backups!
+7. If the ``ssh fileserver`` command fails, the exit status of ``ssh bastion``
+   will continue to indicate success.
+8. Depending in which environment the ``PROD`` variable is set, either it will
+   always evaluate to false, because it was set by the bastion host, or it
+   will do the right thing, because it was set by the script host.
+
+Golly, we've managed to hit at least 8 potentially mission-critical gotchas in
+only 14 lines of code, and they are just those I can count! Welcome to the
+reality of "programming" in shell.
+
+In the end, superficial legibility counted for nothing, it's 4AM, you've been
+paged, the network is down and your boss is angry.
+
+
+Shell Quoting Madness
+~~~~~~~~~~~~~~~~~~~~~
+
+Let's assume on first approach that we really want to handle those quoting
+issues. I wrote a little Python script based around the :py:func:`shlex.quote`
+function to construct, to the best of my knowledge, the quoting required for
+each stage:
+
+.. code-block:: bash
+
+    ssh bastion '
+        if [ "$PROD" ];
+        then
+            ssh fileserver sudo su -c '"'"'
+                if grep -qs /dev/sdb1 /proc/mounts;
+                then
+                    echo "sdb1 already mounted!";
+                    umount /dev/sdb1
+                fi;
+                rm -rf "/media/Main Backup Volume"/*;
+                mount /dev/sdb1 "/media/Main Backup Volume"
+            '"'"';
+        fi;
+        sudo touch /var/run/start_backup
+    '
+
+Even with Python handling the heavy lifting of quoting each shell layer, and
+even if the aforementioned minor disk-wiping issue was fixed, it is still not
+100% clear that argument handling rules for all of ``su``, ``sudo``, ``ssh``,
+and ``bash`` are correctly respected.
+
+Finally, if any login shell involved is not ``bash``, we must introduce
+additional quoting in order to explicitly invoke ``bash`` at each stage,
+causing an explosion in quoting:
+
+.. code-block:: bash
+
+    ssh bastion 'bash -c '"'"'if [ "$PROD" ]; then ssh fileserver bash -c '"'"'
+    "'"'"'"'"'"'sudo su -c '"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"
+    'bash -c '"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"
+    '"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'
+    "'"'"'"'"'"'"'"'"'"'if grep -qs /dev/sdb1 /proc/mounts; then echo "sdb1 alr
+    eady mounted!"; umount /dev/sdb1 fi; rm -rf "/media/Main Backup Volume"/*;
+    mount /dev/sdb1 "/media/Main Backup Volume"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"
+    '"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'
+    "'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"''"'"'"'"'"'"'"'"'"'"'
+    "'"'"'"'"'"'"'"'"'"'"'"'"'"'"'"''"'"'"'"'"'"'"'"'; fi; sudo touch /var/run/
+    start_backup'"'"''
+
+
+There Is Hope
+~~~~~~~~~~~~~
+
+We could instead express the above using Mitogen:
+
+::
+
+    import shutil, os, subprocess
+    import mitogen
+
+    def run(*args):
+        return subprocess.check_call(args)
+
+    def file_contains(s, path):
+        with open(path, 'rb') as fp:
+            return s in fp.read()
+
+    @mitogen.main()
+    def main(router):
+        device = '/dev/sdb1'
+        mount_point = '/media/Media Volume'
+
+        bastion = router.ssh(hostname='bastion')
+        bastion_sudo = router.sudo(via=bastion)
+
+        if PROD:
+            fileserver = router.ssh(hostname='fileserver', via=bastion)
+            if fileserver.call(file_contains, device, '/proc/mounts'):
+                print('{} already mounted!'.format(device))
+                fileserver.call(run, 'umount', device)
+            fileserver.call(shutil.rmtree, mount_point)
+            fileserver.call(os.mkdir, mount_point, 0777)
+            fileserver.call(run, 'mount', device, mount_point)
+
+        bastion_sudo.call(run, 'touch', '/var/run/start_backup')
+
+* In which context must the ``PROD`` variable be defined?
+* On which machine is each step executed?
+* Are there any escaping issues?
+* What will happen if the ``grep`` binary is missing?
+* What will happen if any step fails?
+* What will happen if any login shell is not ``bash``?
+
+
+Recursively Nested Bootstrap
+----------------------------
+
+This demonstrates the library's ability to use slave contexts to recursively
+proxy connections to additional slave contexts, with a uniform API to any
+slave, and all features (function calls, import forwarding, stdio forwarding,
+log forwarding) functioning transparently.
+
+This example uses a chain of local contexts for clarity, however SSH and sudo
+contexts work identically.
+
+nested.py:
+
+.. code-block:: python
+
+    import os
+    import mitogen
+
+    @mitogen.main()
+    def main(router):
+        mitogen.utils.log_to_file()
+
+        context = None
+        for x in range(1, 11):
+            print('Connect local%d via %s' % (x, context))
+            context = router.local(via=context, name='local%d' % x)
+
+        context.call(subprocess.check_call, ['pstree', '-s', 'python', '-s', 'mitogen'])
+
+
+Output:
+
+.. code-block:: shell
+
+    $ python nested.py
+    Connect local1 via None
+    Connect local2 via Context(1, 'local1')
+    Connect local3 via Context(2, 'local2')
+    Connect local4 via Context(3, 'local3')
+    Connect local5 via Context(4, 'local4')
+    Connect local6 via Context(5, 'local5')
+    Connect local7 via Context(6, 'local6')
+    Connect local8 via Context(7, 'local7')
+    Connect local9 via Context(8, 'local8')
+    Connect local10 via Context(9, 'local9')
+    18:14:07 I ctx.local10: stdout: -+= 00001 root /sbin/launchd
+    18:14:07 I ctx.local10: stdout:  \-+= 08126 dmw /Applications/iTerm.app/Contents/MacOS/iTerm2
+    18:14:07 I ctx.local10: stdout:    \-+= 10638 dmw /Applications/iTerm.app/Contents/MacOS/iTerm2 --server bash --login
+    18:14:07 I ctx.local10: stdout:      \-+= 10639 dmw bash --login
+    18:14:07 I ctx.local10: stdout:        \-+= 13632 dmw python nested.py
+    18:14:07 I ctx.local10: stdout:          \-+- 13633 dmw mitogen:dmw@Eldil.local:13632
+    18:14:07 I ctx.local10: stdout:            \-+- 13635 dmw mitogen:dmw@Eldil.local:13633
+    18:14:07 I ctx.local10: stdout:              \-+- 13637 dmw mitogen:dmw@Eldil.local:13635
+    18:14:07 I ctx.local10: stdout:                \-+- 13639 dmw mitogen:dmw@Eldil.local:13637
+    18:14:07 I ctx.local10: stdout:                  \-+- 13641 dmw mitogen:dmw@Eldil.local:13639
+    18:14:07 I ctx.local10: stdout:                    \-+- 13643 dmw mitogen:dmw@Eldil.local:13641
+    18:14:07 I ctx.local10: stdout:                      \-+- 13645 dmw mitogen:dmw@Eldil.local:13643
+    18:14:07 I ctx.local10: stdout:                        \-+- 13647 dmw mitogen:dmw@Eldil.local:13645
+    18:14:07 I ctx.local10: stdout:                          \-+- 13649 dmw mitogen:dmw@Eldil.local:13647
+    18:14:07 I ctx.local10: stdout:                            \-+- 13651 dmw mitogen:dmw@Eldil.local:13649
+    18:14:07 I ctx.local10: stdout:                              \-+- 13653 dmw pstree -s python -s mitogen
+    18:14:07 I ctx.local10: stdout:                                \--- 13654 root ps -axwwo user,pid,ppid,pgid,command

docs/getting_started.rst

@@ -2,4 +2,387 @@
 Getting Started
 ===============
 
-xxx
+.. warning::
+
+    This section is incomplete.
+
+
+Liability Waiver
+----------------
+
+Before proceeding, it is critical you understand what you're involving yourself
+and possibly your team and its successors with:
+
+.. image:: images/pandora.svg
+    :class: mitogen-right-350
+
+* Constructing the most fundamental class, :py:class:`Broker
+  <mitogen.master.Broker>`, causes a new thread to be spawned, exposing a huge
+  class of difficult to analyse behaviours that Python software generally does
+  not suffer from.
+
+  While every effort is made to hide this complexity, you should expect
+  threading-related encounters during development, and crucially, years after
+  your program reached production. See :ref:`troubleshooting` for more
+  information.
+
+* While high-level abstractions are provided, they are only a convenience, you
+  must still understand :ref:`how Mitogen works <howitworks>` before depending
+  on it. Mitogen interacts with many aspects of the operating system,
+  threading, SSH, sudo, sockets, TTYs, shell, Python runtime, and timing and
+  ordering uncertainty introduced through interaction with the network, GIL and
+  OS scheduling.
+
+  Knowledge of this domain is typically attained through painful years of
+  failed attempts hacking system-level programs, and learning through continual
+  suffering how to debug the atrocities left behind. If you feel you lack
+  resources or willpower to diagnose problems independently, Mitogen is not
+  appropriate, prefer a higher level solution instead.
+
+
+First Principles
+----------------
+
+Before starting, take a moment to reflect on writing a program that will
+operate across machines and privilege domains:
+
+* As with multithreaded programming, writing a program that spans multiple
+  hosts is exposed to many asynchrony issues. Unlike multithreaded programming,
+  the margin for unexpected failures is much higher, even between only two
+  peers, as communication may be fail at any moment, since that communication
+  depends on reliability of an external network.
+
+* Since a multi-host program always spans trust and privilege domains, trust
+  must be taken into consideration in your design from the outset. Mitogen
+  attempts to protect the consuming application by default where possible,
+  however it is paramount that trust considerations are always in mind when
+  exposing any privileged functionality to a potentially untrusted network of
+  peers.
+
+  A parent must always assume data received from a child is suspect, and must
+  not base privileged control decisions on that data. As a small example, a
+  parent should not form a command to execute in a subprocess using strings
+  received from a child.
+
+* As the program spans multiple hosts, its design will benefit from a strict
+  separation of program and data. This entails avoiding some common Python
+  idioms that rely on its ability to manipulate functions and closures as if
+  they were data, such as passing a lambda closed over some program state as a
+  callback parameter.
+
+  In the general case this is both difficult and unsafe to support in a
+  distributed program, and so (for now at least) it should be assumed this
+  functionality is unlikely to appear in future.
+
+
+Broker And Router
+-----------------
+
+.. image:: images/layout.svg
+    :class: mitogen-full-width
+
+.. currentmodule:: mitogen.core
+
+Execution starts when your program constructs a :py:class:`Broker` and
+associated :py:class:`Router`. The broker is responsible for multiplexing IO to
+children from a private thread, while in children, it is additionally
+responsible for ensuring robust destruction if communication with the master
+is lost.
+
+:py:class:`Router` is responsible for receiving messages and dispatching them
+to a callback from the broker thread (registered by :py:meth:`add_handler()
+<mitogen.core.Router.add_handler>`), or forwarding them to a :py:class:`Stream
+<mitogen.core.Stream>`. See :ref:`routing` for an in-depth description.
+:py:class:`Router` also doubles as the entry point to Mitogen's public API::
+
+    >>> import mitogen.master
+
+    >>> broker = mitogen.master.Broker()
+    >>> router = mitogen.master.Router(broker)
+
+    >>> try:
+    ...     # Your code here.
+    ...     pass
+    ... finally:
+    ...     broker.shutdown()
+
+As Python will not stop if threads still exist after the main thread exits,
+:py:meth:`Broker.shutdown` must be called reliably at exit. Helpers are
+provided by :py:mod:`mitogen.utils` to ensure :py:class:`Broker` is reliably
+destroyed::
+
+    def do_mitogen_stuff(router):
+        # Your code here.
+
+    mitogen.utils.run_with_router(do_mitogen_stuff)
+
+If your program cannot live beneath :py:func:`mitogen.utils.run_with_router` on
+the stack, you must arrange for :py:meth:`Broker.shutdown` to be called
+anywhere the main thread may exit.
+
+
+Enable Logging
+--------------
+
+Mitogen makes heavy use of the :py:mod:`logging` package, both for child
+``stdio`` redirection, and soft errors and warnings that may be generated.
+
+You should always configure the :py:mod:`logging` package in any program that
+integrates Mitogen. If your program does not otherwise use the
+:py:mod:`logging` package, a basic configuration can be performed by calling
+:py:func:`mitogen.utils.log_to_file`::
+
+    >>> import mitogen.utils
+
+    # Errors, warnings, and child stdio will be written to stderr.
+    >>> mitogen.utils.log_to_file()
+
+Additionally, if your program has :py:const:`logging.DEBUG` as the default
+logging level, you may wish to update its configuration to restrict the
+``mitogen`` logger to :py:const:`logging.INFO`, otherwise vast amounts of
+output will be generated by default.
+
+
+.. _logging-env-vars:
+
+Logging Environment Variables
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``MITOGEN_LOG_LEVEL``
+    Overrides the :py:mod:`logging` package log level set by any call to
+    :py:func:`mitogen.utils.log_to_file`. Defaults to ``INFO``.
+
+    If set to ``IO``, equivalent to ``DEBUG`` but additionally enabled IO
+    logging for any call to :py:func:`mitogen.utils.log_to_file`. IO logging
+    produces verbose records of any IO interaction, which is useful for
+    debugging hangs and deadlocks.
+
+
+
+Logging Records
+~~~~~~~~~~~~~~~
+
+Messages received from a child context via :class:`mitogen.master.LogForwarder`
+receive extra attributes:
+
+* `mitogen_context`: :class:`mitogen.parent.Context` referring to the message
+  source.
+* `mitogen_name`: original logger name in the source context.
+* `mitogen_msg`: original message in the source context.
+
+
+Creating A Context
+------------------
+
+Contexts are simply external Python programs over which your program has
+control, and can execute code within. They can be created as subprocesses on
+the local machine, in another user account via `sudo`, on a remote machine via
+`ssh`, or any recursive combination of the above.
+
+Now a :py:class:`Router` exists, our first :py:class:`contexts <Context>` can
+be created. To demonstrate basic functionality, we will start with some
+:py:meth:`local() <Router.local>` contexts created as subprocesses::
+
+    >>> local = router.local()
+    >>> local_with_name = router.local(remote_name='i-have-a-name')
+
+Examination of the system process list with the ``pstree`` utility reveals the
+resulting process hierarchy::
+
+    | |   \-+= 27660 dmw python
+    | |     |--- 27661 dmw mitogen:dmw@Eldil.local:27660
+    | |     \--- 27663 dmw mitogen:i-have-a-name
+
+Both contexts are visible as subprocesses of the interactive Python
+interpreter, with their ``argv[0]`` including a description of their identity.
+To aid systems administrators in identifying errant software running on their
+machines, the default `remote_name` includes the location of the program that
+started the context, however as shown, this can be overridden.
+
+.. note::
+
+    Presently contexts are constructed in a blocking manner on the thread that
+    invoked the :ref:`context factory <context-factories>`. In a future
+    release, the factory will instead return immediately, and construction will
+    happen asynchronously on the broker thread.
+
+
+Calling A Function
+------------------
+
+.. currentmodule:: mitogen.parent
+
+Now that some contexts exist, it is time to execute code in them. Any regular
+function, static method, or class method reachable directly from module scope
+may be used, including built-in functions such as :func:`time.time`.
+
+The :py:meth:`Context.call` method is used to execute a function and block the
+caller until the return value is available or an exception is raised::
+
+    >>> import time
+    >>> import os
+
+    >>> # Returns the current time.
+    >>> print('Time in remote context:', local.call(time.time))
+
+    >>> try:
+    ...     # Raises OSError.
+    ...     local.call(os.chdir, '/nonexistent')
+    ... except mitogen.core.CallError, e:
+    ...     print('Call failed:', str(e))
+
+It is a simple wrapper around the more flexible :meth:`Context.call_async`,
+which immediately returns a :class:`Receiver <mitogen.core.Receiver>` wired up
+to receive the return value instead. A receiver may simply be discarded, kept
+around indefinitely without ever reading its result, or used to wait on the
+results from several calls. Here :meth:`get() <mitogen.core.Receiver.get>`
+is called to block the thread until the result arrives::
+
+    >>> call = local.call_async(time.time)
+    >>> msg = call.get()
+    >>> print(msg.unpickle())
+    1507292737.75547
+
+
+Running User Functions
+----------------------
+
+So far we have used the interactive interpreter to call some standard library
+functions, but since the source code typed at the interpreter cannot be
+recovered, Mitogen is unable to execute functions defined in this way.
+
+We must therefore continue by writing our code as a script::
+
+    # first-script.py
+    import mitogen.utils
+
+    def my_first_function():
+        print('Hello from remote context!')
+        return 123
+
+    def main(router):
+        local = router.local()
+        print(local.call(my_first_function))
+
+    if __name__ == '__main__':
+        mitogen.utils.log_to_file("mitogen.log")
+        mitogen.utils.run_with_router(main)
+
+Let's try running it:
+
+.. code-block:: bash
+
+    $ python first-script.py
+    19:11:32 I mitogen.ctx.local.32466: stdout: Hello from remote context!
+    123
+
+
+Waiting On Multiple Calls
+-------------------------
+
+Using :meth:`Context.call_async` it is possible to start multiple function
+calls then sleep waiting for responses as they are available. This makes it
+trivial to run tasks in parallel across processes (including remote processes)
+without the need for writing asynchronous code::
+
+    hostnames = ['host1', 'host2', 'host3', 'host4']
+    contexts = [router.ssh(hostname=hn) for hn in hostnames]
+    calls = [context.call(my_func) for context in contexts]
+
+    for msg in mitogen.select.Select(calls):
+        print('Reply from %s: %s' % (recv.context, data))
+
+
+Running Code That May Hang
+--------------------------
+
+When executing code that may hang due to, for example, talking to network peers
+that may become unavailable, it is desirable to be able to recover control in
+the case a remote call has hung.
+
+By specifying the `timeout` parameter to :meth:`Receiver.get` on the receiver
+returned by `Context.call_async`, it becomes possible to wait for a function to
+complete, but time out if its result does not become available.
+
+When a context has become hung like this, it is still possible to gracefully
+terminate it using the :meth:`Context.shutdown` method. This method sends a
+shutdown message to the target process, where its IO multiplexer thread can
+still process it independently of the hung function running on on the target's
+main thread.
+
+
+
+Recovering Mitogen Object References In Children
+------------------------------------------------
+
+::
+
+    @mitogen.core.takes_econtext
+    def func1(a, b, econtext):
+        ...
+
+    @mitogen.core.takes_router
+    def func2(a, b, router):
+        ...
+
+
+Recursion
+---------
+
+Let's try something a little more complex:
+
+
+.. _serialization-rules:
+
+RPC Serialization Rules
+-----------------------
+
+The following built-in types may be used as parameters or return values in
+remote procedure calls:
+
+* :class:`bool`
+* :func:`bytes` (:class:`str` on Python 2.x)
+* :class:`dict`
+* :class:`int`
+* :func:`list`
+* :class:`long`
+* :func:`tuple`
+* :func:`unicode` (:class:`str` on Python 3.x)
+
+User-defined types may not be used, except for:
+
+* :py:class:`mitogen.core.Blob`
+* :py:class:`mitogen.core.Secret`
+* :py:class:`mitogen.core.CallError`
+* :py:class:`mitogen.core.Context`
+* :py:class:`mitogen.core.Sender`
+
+Subclasses of built-in types must be undecorated using
+:py:func:`mitogen.utils.cast`.
+
+
+Test Your Design
+----------------
+
+``tc qdisc add dev eth0 root netem delay 250ms``
+
+
+.. _troubleshooting:
+
+Troubleshooting
+---------------
+
+.. warning::
+
+    This section is incomplete.
+
+A typical example is a hang due to your application's main thread exitting
+perhaps due to an unhandled exception, without first arranging for any
+:py:class:`Broker <mitogen.master.Broker>` to be shut down gracefully.
+
+Another example would be your main thread hanging indefinitely because a bug
+in Mitogen fails to notice an event (such as RPC completion) your thread is
+waiting for will never complete. Solving this kind of hang is a work in
+progress.
+
+router.enable_debug()

docs/history.rst

@@ -1,65 +0,0 @@
-
-History And Future
-==================
-
-
-History
-#######
-
-The first version of econtext was written in late 2006 for use in an
-infrastructure management program, however at the time I lacked the pragmatism
-necessary for pushing my little design from concept to finished implementation.
-I tired of it when no way could be found to unify every communication style
-(*blocking execute function, asynchronous execute function, proxy
-slave-of-slave context*) into one neat abstraction. That unification never
-happened, but I'm no longer worried by it.
-
-Every few years I would pick through the source code, especially after periods
-of commercial work involving some contemporary infrastructure management
-systems, none of which had nearly as neat an approach to running Python code
-remotely, and suffered from shockingly beginner-level bugs such as failing to
-report SSH diagnostic messages.
-
-And every few years I'd put that code down again, especially since moving to an
-OS X laptop where :py:func:`select.poll` was not available, the struggle to get
-back on top seemed more hassle than it was worth.
-
-That changed in 2016 during a quiet evening at home with a clear head and
-nothing better to do, after a full day of exposure to Ansible's intensely
-unbearable tendency to make running a 50 line Python script across a 1Gbit/sec
-LAN feel like I were configuring a host on Mars. Poking through Ansible, I was
-shocked to discover it writing temporary files everywhere, and uploading a
-56KiB zip file apparently for every playbook step.
-
-.. figure:: _static/wtf.gif
-
-    All contemporary Devops tooling
-
-Searching around for something to play with, I came across my forgotten
-``src/econtext`` directory and somehow in a few hours managed to squash most of
-the race conditions and logic bugs that were preventing reliable operation,
-write the IO and log forwarders, rewrite the module importer, move from
-:py:func:`select.poll` to :py:func:`select.select`, and even refactor the
-special cases out of the main loop.
-
-So there you have it. As of writing :py:mod:`econtext.core` consists of 528
-source lines, and those 528 lines have taken me almost a decade to write. I
-have long had a preference for avoiding infrastructure work commercially, not
-least for the inescapable depression induced by considering the wasted effort
-across the world caused by universally horrific tooling. This is my small
-contribution to a solution, I hope you find it useful.
-
-
-Future
-######
-
-* Connect back using TCP and SSL.
-* Python 3 support.
-* Windows support via psexec or similar.
-* Investigate cPickle safety and potentially replace it, or implement a strict
-  format validator for messages received by master.
-* Predictive import: reduce roundtrips by pipelining modules observed to
-  probably be requested in future.
-* Provide a means for waiting on multiple
-  :py:class:`Channels <econtext.core.Channel>`.
-* Comprehensive integration tests.

docs/images/ansible/pcaps/.gitattributes

@@ -0,0 +1,3 @@
+**pcap** filter=lfs diff=lfs merge=lfs -text
+run_hostname_100_times_mito.pcap.gz filter=lfs diff=lfs merge=lfs -text
+run_hostname_100_times_vanilla.pcap.gz filter=lfs diff=lfs merge=lfs -text

docs/images/ansible/pcaps/stroke-width.py

@@ -0,0 +1,16 @@
+
+import sys
+# Add viewBox attr to SVGs lacking it, so IE scales properly.
+
+import lxml.etree
+import glob
+
+
+for name in sys.argv[1:]:  # glob.glob('*/*.svg'): #+ glob.glob('images/ansible/*.svg'):
+    doc = lxml.etree.parse(open(name))
+    svg = doc.getroot()
+    for elem in svg.cssselect('[stroke-width]'):
+        if elem.attrib['stroke-width'] < '2':
+            elem.attrib['stroke-width'] = '2'
+
+    open(name, 'w').write(lxml.etree.tostring(svg, xml_declaration=True, encoding='UTF-8'))

docs/images/ansible/pcaps/svg-boxify.py

@@ -0,0 +1,13 @@
+
+# Add viewBox attr to SVGs lacking it, so IE scales properly.
+
+import lxml.etree
+import glob
+
+
+for name in glob.glob('images/*.svg') + glob.glob('images/ansible/*.svg'):
+    doc = lxml.etree.parse(open(name))
+    svg = doc.getroot()
+    if 'viewBox' not in svg.attrib:
+        svg.attrib['viewBox'] = '0 0 %(width)s %(height)s' % svg.attrib
+        open(name, 'w').write(lxml.etree.tostring(svg, xml_declaration=True, encoding='UTF-8'))

docs/images/billing.graphml

@@ -0,0 +1,174 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<graphml xmlns="http://graphml.graphdrawing.org/xmlns" xmlns:java="http://www.yworks.com/xml/yfiles-common/1.0/java" xmlns:sys="http://www.yworks.com/xml/yfiles-common/markup/primitives/2.0" xmlns:x="http://www.yworks.com/xml/yfiles-common/markup/2.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:y="http://www.yworks.com/xml/graphml" xmlns:yed="http://www.yworks.com/xml/yed/3" xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://www.yworks.com/xml/schema/graphml/1.1/ygraphml.xsd">
+  <!--Created by yEd 3.14.4-->
+  <key attr.name="Description" attr.type="string" for="graph" id="d0"/>
+  <key for="port" id="d1" yfiles.type="portgraphics"/>
+  <key for="port" id="d2" yfiles.type="portgeometry"/>
+  <key for="port" id="d3" yfiles.type="portuserdata"/>
+  <key attr.name="url" attr.type="string" for="node" id="d4"/>
+  <key attr.name="description" attr.type="string" for="node" id="d5"/>
+  <key for="node" id="d6" yfiles.type="nodegraphics"/>
+  <key for="graphml" id="d7" yfiles.type="resources"/>
+  <key attr.name="url" attr.type="string" for="edge" id="d8"/>
+  <key attr.name="description" attr.type="string" for="edge" id="d9"/>
+  <key for="edge" id="d10" yfiles.type="edgegraphics"/>
+  <graph edgedefault="directed" id="G">
+    <data key="d0"/>
+    <node id="n0">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="140.0" x="319.0" y="189.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="44.03125" x="47.984375" y="5.93359375">master<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n1">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="140.0" x="319.0" y="239.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="70.55078125" x="34.724609375" y="5.93359375">ssh:bastion<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n2">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="140.0" x="319.0" y="342.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="60.42578125" x="39.787109375" y="5.93359375">sudo:root<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n3">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="140.0" x="319.0" y="392.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="91.421875" x="24.2890625" y="5.93359375">docker:billing0<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n4">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="140.0" x="319.0" y="442.0"/>
+          <y:Fill color="#99CC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="131.740234375" x="4.1298828125" y="5.93359375">run-nightly-billing.py<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n5">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="140.0" x="319.0" y="290.5"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="80.728515625" x="29.6357421875" y="5.93359375">ssh:docker-a<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <edge id="e0" source="n0" target="n1">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e1" source="n1" target="n5">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e2" source="n2" target="n3">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e3" source="n3" target="n4">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e4" source="n5" target="n2">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+  </graph>
+  <data key="d7">
+    <y:Resources/>
+  </data>
+</graphml>

docs/images/context-tree.graphml

@@ -0,0 +1,497 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<graphml xmlns="http://graphml.graphdrawing.org/xmlns" xmlns:java="http://www.yworks.com/xml/yfiles-common/1.0/java" xmlns:sys="http://www.yworks.com/xml/yfiles-common/markup/primitives/2.0" xmlns:x="http://www.yworks.com/xml/yfiles-common/markup/2.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:y="http://www.yworks.com/xml/graphml" xmlns:yed="http://www.yworks.com/xml/yed/3" xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://www.yworks.com/xml/schema/graphml/1.1/ygraphml.xsd">
+  <!--Created by yEd 3.14.4-->
+  <key attr.name="Description" attr.type="string" for="graph" id="d0"/>
+  <key for="port" id="d1" yfiles.type="portgraphics"/>
+  <key for="port" id="d2" yfiles.type="portgeometry"/>
+  <key for="port" id="d3" yfiles.type="portuserdata"/>
+  <key attr.name="url" attr.type="string" for="node" id="d4"/>
+  <key attr.name="description" attr.type="string" for="node" id="d5"/>
+  <key for="node" id="d6" yfiles.type="nodegraphics"/>
+  <key for="graphml" id="d7" yfiles.type="resources"/>
+  <key attr.name="url" attr.type="string" for="edge" id="d8"/>
+  <key attr.name="description" attr.type="string" for="edge" id="d9"/>
+  <key for="edge" id="d10" yfiles.type="edgegraphics"/>
+  <graph edgedefault="directed" id="G">
+    <data key="d0"/>
+    <node id="n0">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="303.75" y="0.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="44.03125" x="13.484375" y="5.93359375">master<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n1">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="303.75" y="50.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="47.072265625" x="11.9638671875" y="5.93359375">bastion<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n2">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="121.5" y="100.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="25.287109375" x="22.8564453125" y="5.93359375">dc1<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n3">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="486.0" y="100.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="25.287109375" x="22.8564453125" y="5.93359375">dc2<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n4">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="40.5" y="150.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="43.873046875" x="13.5634765625" y="5.93359375">rack11<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n5">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="202.5" y="150.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="43.873046875" x="13.5634765625" y="5.93359375">rack12<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n6">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="81.0" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="54.859375" x="8.0703125" y="5.93359375">node11a<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n7">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="0.0" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="55.78515625" x="7.607421875" y="5.93359375">node11b<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n8">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="162.0" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="54.859375" x="8.0703125" y="5.93359375">node12a<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n9">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="243.0" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="55.78515625" x="7.607421875" y="5.93359375">node12b<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n10">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="526.5" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="54.859375" x="8.0703125" y="5.93359375">node21a<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n11">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="607.5" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="55.78515625" x="7.607421875" y="5.93359375">node21b<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n12">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="445.5" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="54.859375" x="8.0703125" y="5.93359375">node22a<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n13">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="364.5" y="200.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="55.78515625" x="7.607421875" y="5.93359375">node22b<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n14">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="567.0" y="150.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="43.873046875" x="13.5634765625" y="5.93359375">rack21<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n15">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="405.0" y="150.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="43.873046875" x="13.5634765625" y="5.93359375">rack22<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n16">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="152.0" x="202.5" y="250.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="137.083984375" x="7.4580078125" y="5.93359375">sudo:node12b:webapp<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n17">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="152.0" x="405.0" y="250.0"/>
+          <y:Fill color="#FFFF99" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="136.158203125" x="7.9208984375" y="5.93359375">sudo:node22a:webapp<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <edge id="e0" source="n0" target="n1">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e1" source="n1" target="n3">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e2" source="n1" target="n2">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e3" source="n2" target="n4">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e4" source="n4" target="n7">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e5" source="n4" target="n6">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e6" source="n5" target="n9">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e7" source="n5" target="n8">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e8" source="n2" target="n5">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e9" source="n15" target="n13">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e10" source="n15" target="n12">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e11" source="n3" target="n14">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e12" source="n14" target="n10">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e13" source="n14" target="n11">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e14" source="n9" target="n16">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e15" source="n3" target="n15">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e16" source="n12" target="n17">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+  </graph>
+  <data key="d7">
+    <y:Resources/>
+  </data>
+</graphml>

docs/images/detached-subtree.graphml

@@ -0,0 +1,541 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<graphml xmlns="http://graphml.graphdrawing.org/xmlns" xmlns:java="http://www.yworks.com/xml/yfiles-common/1.0/java" xmlns:sys="http://www.yworks.com/xml/yfiles-common/markup/primitives/2.0" xmlns:x="http://www.yworks.com/xml/yfiles-common/markup/2.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:y="http://www.yworks.com/xml/graphml" xmlns:yed="http://www.yworks.com/xml/yed/3" xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://www.yworks.com/xml/schema/graphml/1.1/ygraphml.xsd">
+  <!--Created by yEd 3.14.4-->
+  <key attr.name="Description" attr.type="string" for="graph" id="d0"/>
+  <key for="port" id="d1" yfiles.type="portgraphics"/>
+  <key for="port" id="d2" yfiles.type="portgeometry"/>
+  <key for="port" id="d3" yfiles.type="portuserdata"/>
+  <key attr.name="url" attr.type="string" for="node" id="d4"/>
+  <key attr.name="description" attr.type="string" for="node" id="d5"/>
+  <key for="node" id="d6" yfiles.type="nodegraphics"/>
+  <key for="graphml" id="d7" yfiles.type="resources"/>
+  <key attr.name="url" attr.type="string" for="edge" id="d8"/>
+  <key attr.name="description" attr.type="string" for="edge" id="d9"/>
+  <key for="edge" id="d10" yfiles.type="edgegraphics"/>
+  <graph edgedefault="directed" id="G">
+    <data key="d0"/>
+    <node id="n0">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="303.75" y="0.0"/>
+          <y:Fill color="#C0C0C0" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="44.03125" x="13.484375" y="5.93359375">master<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n1">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="71.0" x="303.75" y="50.0"/>
+          <y:Fill color="#C0C0C0" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="47.072265625" x="11.9638671875" y="5.93359375">bastion<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n2" yfiles.foldertype="group">
+      <data key="d4"/>
+      <data key="d5"/>
+      <data key="d6">
+        <y:ProxyAutoBoundsNode>
+          <y:Realizers active="0">
+            <y:GroupNode>
+              <y:Geometry height="231.666015625" width="384.5" x="-56.65467625899282" y="110.0"/>
+              <y:Fill color="#F5F5F5" transparent="false"/>
+              <y:BorderStyle color="#000000" type="dashed" width="1.0"/>
+              <y:NodeLabel alignment="right" autoSizePolicy="node_width" backgroundColor="#EBEBEB" borderDistance="0.0" fontFamily="Dialog" fontSize="15" fontStyle="plain" hasLineColor="false" height="21.666015625" modelName="internal" modelPosition="t" textColor="#000000" visible="true" width="384.5" x="0.0" y="0.0">Subtree 1</y:NodeLabel>
+              <y:Shape type="roundrectangle"/>
+              <y:State closed="false" closedHeight="50.0" closedWidth="50.0" innerGraphDisplayEnabled="false"/>
+              <y:Insets bottom="15" bottomF="15.0" left="15" leftF="15.0" right="15" rightF="15.0" top="15" topF="15.0"/>
+              <y:BorderInsets bottom="0" bottomF="0.0" left="0" leftF="0.0" right="0" rightF="0.0" top="0" topF="0.0"/>
+            </y:GroupNode>
+            <y:GroupNode>
+              <y:Geometry height="50.0" width="50.0" x="-15.0" y="63.333984375"/>
+              <y:Fill color="#F5F5F5" transparent="false"/>
+              <y:BorderStyle color="#000000" type="dashed" width="1.0"/>
+              <y:NodeLabel alignment="right" autoSizePolicy="node_width" backgroundColor="#EBEBEB" borderDistance="0.0" fontFamily="Dialog" fontSize="15" fontStyle="plain" hasLineColor="false" height="21.666015625" modelName="internal" modelPosition="t" textColor="#000000" visible="true" width="63.75830078125" x="-6.879150390625" y="0.0">Folder 1</y:NodeLabel>
+              <y:Shape type="roundrectangle"/>
+              <y:State closed="true" closedHeight="50.0" closedWidth="50.0" innerGraphDisplayEnabled="false"/>
+              <y:Insets bottom="5" bottomF="5.0" left="5" leftF="5.0" right="5" rightF="5.0" top="5" topF="5.0"/>
+              <y:BorderInsets bottom="0" bottomF="0.0" left="0" leftF="0.0" right="0" rightF="0.0" top="0" topF="0.0"/>
+            </y:GroupNode>
+          </y:Realizers>
+        </y:ProxyAutoBoundsNode>
+      </data>
+      <graph edgedefault="directed" id="n2:">
+        <node id="n2::n0">
+          <data key="d6">
+            <y:ShapeNode>
+              <y:Geometry height="30.0" width="71.0" x="79.84532374100718" y="146.666015625"/>
+              <y:Fill color="#FFFF99" transparent="false"/>
+              <y:BorderStyle color="#000000" type="line" width="1.0"/>
+              <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="25.287109375" x="22.8564453125" y="5.93359375">dc1<y:LabelModel>
+                  <y:SmartNodeLabelModel distance="4.0"/>
+                </y:LabelModel>
+                <y:ModelParameter>
+                  <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+                </y:ModelParameter>
+              </y:NodeLabel>
+              <y:Shape type="rectangle"/>
+            </y:ShapeNode>
+          </data>
+        </node>
+        <node id="n2::n1">
+          <data key="d6">
+            <y:ShapeNode>
+              <y:Geometry height="30.0" width="71.0" x="-1.1546762589928221" y="196.666015625"/>
+              <y:Fill color="#FFFF99" transparent="false"/>
+              <y:BorderStyle color="#000000" type="line" width="1.0"/>
+              <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="43.873046875" x="13.5634765625" y="5.93359375">rack11<y:LabelModel>
+                  <y:SmartNodeLabelModel distance="4.0"/>
+                </y:LabelModel>
+                <y:ModelParameter>
+                  <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+                </y:ModelParameter>
+              </y:NodeLabel>
+              <y:Shape type="rectangle"/>
+            </y:ShapeNode>
+          </data>
+        </node>
+        <node id="n2::n2">
+          <data key="d6">
+            <y:ShapeNode>
+              <y:Geometry height="30.0" width="71.0" x="160.84532374100718" y="196.666015625"/>
+              <y:Fill color="#FFFF99" transparent="false"/>
+              <y:BorderStyle color="#000000" type="line" width="1.0"/>
+              <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="43.873046875" x="13.5634765625" y="5.93359375">rack12<y:LabelModel>
+                  <y:SmartNodeLabelModel distance="4.0"/>
+                </y:LabelModel>
+                <y:ModelParameter>
+                  <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+                </y:ModelParameter>
+              </y:NodeLabel>
+              <y:Shape type="rectangle"/>
+            </y:ShapeNode>
+          </data>
+        </node>
+        <node id="n2::n3">
+          <data key="d6">
+            <y:ShapeNode>
+              <y:Geometry height="30.0" width="71.0" x="39.34532374100718" y="246.666015625"/>
+              <y:Fill color="#FFFF99" transparent="false"/>
+              <y:BorderStyle color="#000000" type="line" width="1.0"/>
+              <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="54.859375" x="8.0703125" y="5.93359375">node11a<y:LabelModel>
+                  <y:SmartNodeLabelModel distance="4.0"/>
+                </y:LabelModel>
+                <y:ModelParameter>
+                  <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+                </y:ModelParameter>
+              </y:NodeLabel>
+              <y:Shape type="rectangle"/>
+            </y:ShapeNode>
+          </data>
+        </node>
+        <node id="n2::n4">
+          <data key="d6">
+            <y:ShapeNode>
+              <y:Geometry height="30.0" width="71.0" x="-41.65467625899282" y="246.666015625"/>
+              <y:Fill color="#FFFF99" transparent="false"/>
+              <y:BorderStyle color="#000000" type="line" width="1.0"/>
+              <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="55.78515625" x="7.607421875" y="5.93359375">node11b<y:LabelModel>
+                  <y:SmartNodeLabelModel distance="4.0"/>
+                </y:LabelModel>
+                <y:ModelParameter>
+                  <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+                </y:ModelParameter>
+              </y:NodeLabel>
+              <y:Shape type="rectangle"/>
+            </y:ShapeNode>
+          </data>
+        </node>
+        <node id="n2::n5">
+          <data key="d6">
+            <y:ShapeNode>
+              <y:Geometry height="30.0" width="71.0" x="120.34532374100718" y="246.666015625"/>
+              <y:Fill color="#FFFF99" transparent="false"/>
+              <y:BorderStyle color="#000000" type="line" width="1.0"/>
+              <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="54.859375" x="8.0703125" y="5.93359375">node12a<y:LabelModel>
+                  <y:SmartNodeLabelModel distance="4.0"/>
+                </y:LabelModel>
+                <y:ModelParameter>
+                  <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+                </y:ModelParameter>
+              </y:NodeLabel>
+              <y:Shape type="rectangle"/>
+            </y:ShapeNode>
+          </data>
+        </node>
+        <node id="n2::n6">
+          <data key="d6">
+            <y:ShapeNode>
+              <y:Geometry height="30.0" width="71.0" x="201.34532374100718" y="246.666015625"/>
+              <y:Fill color="#FFFF99" transparent="false"/>
+              <y:BorderStyle color="#000000" type="line" width="1.0"/>
+              <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="55.78515625" x="7.607421875" y="5.93359375">node12b<y:LabelModel>
+                  <y:SmartNodeLabelModel distance="4.0"/>
+                </y:LabelModel>
+                <y:ModelParameter>
+                  <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+                </y:ModelParameter>
+              </y:NodeLabel>
+              <y:Shape type="rectangle"/>
+            </y:ShapeNode>
+          </data>
+        </node>
+        <node id="n2::n7">
+          <data key="d6">
+            <y:ShapeNode>
+              <y:Geometry height="30.0" width="152.0" x="160.84532374100718" y="296.666015625"/>
+              <y:Fill color="#FFFF99" transparent="false"/>
+              <y:BorderStyle color="#000000" type="line" width="1.0"/>
+              <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="137.083984375" x="7.4580078125" y="5.93359375">sudo:node12b:webapp<y:LabelModel>
+                  <y:SmartNodeLabelModel distance="4.0"/>
+                </y:LabelModel>
+                <y:ModelParameter>
+                  <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+                </y:ModelParameter>
+              </y:NodeLabel>
+              <y:Shape type="rectangle"/>
+            </y:ShapeNode>
+          </data>
+        </node>
+      </graph>
+    </node>
+    <node id="n3" yfiles.foldertype="group">
+      <data key="d4"/>
+      <data key="d5"/>
+      <data key="d6">
+        <y:ProxyAutoBoundsNode>
+          <y:Realizers active="0">
+            <y:GroupNode>
+              <y:Geometry height="231.666015625" width="344.0" x="342.8453237410072" y="110.0"/>
+              <y:Fill color="#F5F5F5" transparent="false"/>
+              <y:BorderStyle color="#000000" type="dashed" width="1.0"/>
+              <y:NodeLabel alignment="right" autoSizePolicy="node_width" backgroundColor="#EBEBEB" borderDistance="0.0" fontFamily="Dialog" fontSize="15" fontStyle="plain" hasLineColor="false" height="21.666015625" modelName="internal" modelPosition="t" textColor="#000000" visible="true" width="344.0" x="0.0" y="0.0">Subtree 2</y:NodeLabel>
+              <y:Shape type="roundrectangle"/>
+              <y:State closed="false" closedHeight="50.0" closedWidth="50.0" innerGraphDisplayEnabled="false"/>
+              <y:Insets bottom="15" bottomF="15.0" left="15" leftF="15.0" right="15" rightF="15.0" top="15" topF="15.0"/>
+              <y:BorderInsets bottom="0" bottomF="0.0" left="0" leftF="0.0" right="0" rightF="0.0" top="0" topF="0.0"/>
+            </y:GroupNode>
+            <y:GroupNode>
+              <y:Geometry height="50.0" width="50.0" x="0.0" y="60.0"/>
+              <y:Fill color="#F5F5F5" transparent="false"/>
+              <y:BorderStyle color="#000000" type="dashed" width="1.0"/>
+              <y:NodeLabel alignment="right" autoSizePolicy="node_width" backgroundColor="#EBEBEB" borderDistance="0.0" fontFamily="Dialog" fontSize="15" fontStyle="plain" hasLineColor="false" height="21.666015625" modelName="internal" modelPosition="t" textColor="#000000" visible="true" width="63.75830078125" x="-6.879150390625" y="0.0">Folder 2</y:NodeLabel>
+              <y:Shape type="roundrectangle"/>
+              <y:State closed="true" closedHeight="50.0" closedWidth="50.0" innerGraphDisplayEnabled="false"/>
+              <y:Insets bottom="5" bottomF="5.0" left="5" leftF="5.0" right="5" rightF="5.0" top="5" topF="5.0"/>
+              <y:BorderInsets bottom="0" bottomF="0.0" left="0" leftF="0.0" right="0" rightF="0.0" top="0" topF="0.0"/>
+            </y:GroupNode>
+          </y:Realizers>
+        </y:ProxyAutoBoundsNode>
+      </data>
+      <graph edgedefault="directed" id="n3:">
+        <node id="n3::n0">
+          <data key="d6">
+            <y:ShapeNode>
+              <y:Geometry height="30.0" width="71.0" x="479.3453237410072" y="146.666015625"/>
+              <y:Fill color="#FFFF99" transparent="false"/>
+              <y:BorderStyle color="#000000" type="line" width="1.0"/>
+              <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="25.287109375" x="22.8564453125" y="5.93359375">dc2<y:LabelModel>
+                  <y:SmartNodeLabelModel distance="4.0"/>
+                </y:LabelModel>
+                <y:ModelParameter>
+                  <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+                </y:ModelParameter>
+              </y:NodeLabel>
+              <y:Shape type="rectangle"/>
+            </y:ShapeNode>
+          </data>
+        </node>
+        <node id="n3::n1">
+          <data key="d6">
+            <y:ShapeNode>
+              <y:Geometry height="30.0" width="71.0" x="519.8453237410072" y="246.666015625"/>
+              <y:Fill color="#FFFF99" transparent="false"/>
+              <y:BorderStyle color="#000000" type="line" width="1.0"/>
+              <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="54.859375" x="8.0703125" y="5.93359375">node21a<y:LabelModel>
+                  <y:SmartNodeLabelModel distance="4.0"/>
+                </y:LabelModel>
+                <y:ModelParameter>
+                  <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+                </y:ModelParameter>
+              </y:NodeLabel>
+              <y:Shape type="rectangle"/>
+            </y:ShapeNode>
+          </data>
+        </node>
+        <node id="n3::n2">
+          <data key="d6">
+            <y:ShapeNode>
+              <y:Geometry height="30.0" width="71.0" x="600.8453237410072" y="246.666015625"/>
+              <y:Fill color="#FFFF99" transparent="false"/>
+              <y:BorderStyle color="#000000" type="line" width="1.0"/>
+              <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="55.78515625" x="7.607421875" y="5.93359375">node21b<y:LabelModel>
+                  <y:SmartNodeLabelModel distance="4.0"/>
+                </y:LabelModel>
+                <y:ModelParameter>
+                  <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+                </y:ModelParameter>
+              </y:NodeLabel>
+              <y:Shape type="rectangle"/>
+            </y:ShapeNode>
+          </data>
+        </node>
+        <node id="n3::n3">
+          <data key="d6">
+            <y:ShapeNode>
+              <y:Geometry height="30.0" width="71.0" x="438.8453237410072" y="246.666015625"/>
+              <y:Fill color="#FFFF99" transparent="false"/>
+              <y:BorderStyle color="#000000" type="line" width="1.0"/>
+              <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="54.859375" x="8.0703125" y="5.93359375">node22a<y:LabelModel>
+                  <y:SmartNodeLabelModel distance="4.0"/>
+                </y:LabelModel>
+                <y:ModelParameter>
+                  <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+                </y:ModelParameter>
+              </y:NodeLabel>
+              <y:Shape type="rectangle"/>
+            </y:ShapeNode>
+          </data>
+        </node>
+        <node id="n3::n4">
+          <data key="d6">
+            <y:ShapeNode>
+              <y:Geometry height="30.0" width="71.0" x="357.8453237410072" y="246.666015625"/>
+              <y:Fill color="#FFFF99" transparent="false"/>
+              <y:BorderStyle color="#000000" type="line" width="1.0"/>
+              <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="55.78515625" x="7.607421875" y="5.93359375">node22b<y:LabelModel>
+                  <y:SmartNodeLabelModel distance="4.0"/>
+                </y:LabelModel>
+                <y:ModelParameter>
+                  <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+                </y:ModelParameter>
+              </y:NodeLabel>
+              <y:Shape type="rectangle"/>
+            </y:ShapeNode>
+          </data>
+        </node>
+        <node id="n3::n5">
+          <data key="d6">
+            <y:ShapeNode>
+              <y:Geometry height="30.0" width="71.0" x="560.3453237410072" y="196.666015625"/>
+              <y:Fill color="#FFFF99" transparent="false"/>
+              <y:BorderStyle color="#000000" type="line" width="1.0"/>
+              <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="43.873046875" x="13.5634765625" y="5.93359375">rack21<y:LabelModel>
+                  <y:SmartNodeLabelModel distance="4.0"/>
+                </y:LabelModel>
+                <y:ModelParameter>
+                  <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+                </y:ModelParameter>
+              </y:NodeLabel>
+              <y:Shape type="rectangle"/>
+            </y:ShapeNode>
+          </data>
+        </node>
+        <node id="n3::n6">
+          <data key="d6">
+            <y:ShapeNode>
+              <y:Geometry height="30.0" width="71.0" x="398.3453237410072" y="196.666015625"/>
+              <y:Fill color="#FFFF99" transparent="false"/>
+              <y:BorderStyle color="#000000" type="line" width="1.0"/>
+              <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="43.873046875" x="13.5634765625" y="5.93359375">rack22<y:LabelModel>
+                  <y:SmartNodeLabelModel distance="4.0"/>
+                </y:LabelModel>
+                <y:ModelParameter>
+                  <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+                </y:ModelParameter>
+              </y:NodeLabel>
+              <y:Shape type="rectangle"/>
+            </y:ShapeNode>
+          </data>
+        </node>
+        <node id="n3::n7">
+          <data key="d6">
+            <y:ShapeNode>
+              <y:Geometry height="30.0" width="152.0" x="398.3453237410072" y="296.666015625"/>
+              <y:Fill color="#FFFF99" transparent="false"/>
+              <y:BorderStyle color="#000000" type="line" width="1.0"/>
+              <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="136.158203125" x="7.9208984375" y="5.93359375">sudo:node22a:webapp<y:LabelModel>
+                  <y:SmartNodeLabelModel distance="4.0"/>
+                </y:LabelModel>
+                <y:ModelParameter>
+                  <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+                </y:ModelParameter>
+              </y:NodeLabel>
+              <y:Shape type="rectangle"/>
+            </y:ShapeNode>
+          </data>
+        </node>
+      </graph>
+    </node>
+    <edge id="e0" source="n0" target="n1">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="n3::e0" source="n3::n6" target="n3::n4">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="n3::e1" source="n3::n6" target="n3::n3">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="n3::e2" source="n3::n0" target="n3::n5">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="n3::e3" source="n3::n5" target="n3::n1">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="n3::e4" source="n3::n5" target="n3::n2">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="n3::e5" source="n3::n0" target="n3::n6">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="n3::e6" source="n3::n3" target="n3::n7">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="n2::e0" source="n2::n0" target="n2::n1">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="n2::e1" source="n2::n0" target="n2::n2">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="n2::e2" source="n2::n1" target="n2::n4">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="n2::e3" source="n2::n1" target="n2::n3">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="n2::e4" source="n2::n2" target="n2::n6">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="n2::e5" source="n2::n2" target="n2::n5">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="n2::e6" source="n2::n6" target="n2::n7">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+  </graph>
+  <data key="d7">
+    <y:Resources/>
+  </data>
+</graphml>

docs/images/distribute.graphml

@@ -0,0 +1,827 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<graphml xmlns="http://graphml.graphdrawing.org/xmlns" xmlns:java="http://www.yworks.com/xml/yfiles-common/1.0/java" xmlns:sys="http://www.yworks.com/xml/yfiles-common/markup/primitives/2.0" xmlns:x="http://www.yworks.com/xml/yfiles-common/markup/2.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:y="http://www.yworks.com/xml/graphml" xmlns:yed="http://www.yworks.com/xml/yed/3" xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://www.yworks.com/xml/schema/graphml/1.1/ygraphml.xsd">
+  <!--Created by yEd 3.14.4-->
+  <key attr.name="Description" attr.type="string" for="graph" id="d0"/>
+  <key for="port" id="d1" yfiles.type="portgraphics"/>
+  <key for="port" id="d2" yfiles.type="portgeometry"/>
+  <key for="port" id="d3" yfiles.type="portuserdata"/>
+  <key attr.name="url" attr.type="string" for="node" id="d4"/>
+  <key attr.name="description" attr.type="string" for="node" id="d5"/>
+  <key for="node" id="d6" yfiles.type="nodegraphics"/>
+  <key for="graphml" id="d7" yfiles.type="resources"/>
+  <key attr.name="url" attr.type="string" for="edge" id="d8"/>
+  <key attr.name="description" attr.type="string" for="edge" id="d9"/>
+  <key for="edge" id="d10" yfiles.type="edgegraphics"/>
+  <graph edgedefault="directed" id="G">
+    <data key="d0"/>
+    <node id="n0">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="88.0" x="225.0" y="137.0078125"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="50.01953125" x="18.990234375" y="5.93359375">rack12c<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n1">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="88.0" x="165.0" y="-38.9921875"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="47.072265625" x="20.4638671875" y="5.93359375">bastion<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n2">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="88.0" x="225.0" y="32.0078125"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="50.5" x="18.75" y="5.93359375">rack12a<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n3">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="88.0" x="165.0" y="-110.9921875"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="41.060546875" x="23.4697265625" y="5.93359375">laptop<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n4">
+      <data key="d6">
+        <y:GenericNode configuration="com.yworks.flowchart.dataBase">
+          <y:Geometry height="30.0" width="26.0" x="401.0" y="-110.9921875"/>
+          <y:Fill color="#E8EEF7" color2="#B7C9E3" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" hasText="false" height="4.0" modelName="custom" textColor="#000000" visible="true" width="4.0" x="11.0" y="13.0">
+            <y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+        </y:GenericNode>
+      </data>
+    </node>
+    <node id="n5">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="88.0" x="283.0" y="-110.9921875"/>
+          <y:Fill color="#99CC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="60.1796875" x="13.91015625" y="5.93359375">distribute<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n6">
+      <data key="d6">
+        <y:GenericNode configuration="com.yworks.flowchart.dataBase">
+          <y:Geometry height="30.0" width="26.0" x="401.0" y="-38.9921875"/>
+          <y:Fill color="#E8EEF7" color2="#B7C9E3" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" hasText="false" height="4.0" modelName="custom" textColor="#000000" visible="true" width="4.0" x="11.0" y="13.0">
+            <y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+        </y:GenericNode>
+      </data>
+    </node>
+    <node id="n7">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="88.0" x="283.0" y="-38.9921875"/>
+          <y:Fill color="#99CC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="60.1796875" x="13.91015625" y="5.93359375">distribute<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n8">
+      <data key="d6">
+        <y:GenericNode configuration="com.yworks.flowchart.dataBase">
+          <y:Geometry height="30.0" width="26.0" x="461.0" y="32.0078125"/>
+          <y:Fill color="#E8EEF7" color2="#B7C9E3" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" hasText="false" height="4.0" modelName="custom" textColor="#000000" visible="true" width="4.0" x="11.0" y="13.0">
+            <y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+        </y:GenericNode>
+      </data>
+    </node>
+    <node id="n9">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="88.0" x="343.0" y="33.0078125"/>
+          <y:Fill color="#99CC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="60.1796875" x="13.91015625" y="5.93359375">distribute<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n10">
+      <data key="d6">
+        <y:GenericNode configuration="com.yworks.flowchart.dataBase">
+          <y:Geometry height="30.0" width="26.0" x="-67.0" y="32.0078125"/>
+          <y:Fill color="#E8EEF7" color2="#B7C9E3" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" hasText="false" height="4.0" modelName="custom" textColor="#000000" visible="true" width="4.0" x="11.0" y="13.0">
+            <y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+        </y:GenericNode>
+      </data>
+    </node>
+    <node id="n11">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="88.0" x="-11.0" y="32.0078125"/>
+          <y:Fill color="#99CC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="60.1796875" x="13.91015625" y="5.93359375">distribute<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n12">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="88.0" x="107.0" y="32.0078125"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="50.5" x="18.75" y="5.93359375">rack11a<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n13">
+      <data key="d6">
+        <y:GenericNode configuration="com.yworks.flowchart.dataBase">
+          <y:Geometry height="30.0" width="26.0" x="461.0" y="85.0078125"/>
+          <y:Fill color="#E8EEF7" color2="#B7C9E3" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" hasText="false" height="4.0" modelName="custom" textColor="#000000" visible="true" width="4.0" x="11.0" y="13.0">
+            <y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+        </y:GenericNode>
+      </data>
+    </node>
+    <node id="n14">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="88.0" x="343.0" y="85.0078125"/>
+          <y:Fill color="#99CC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="60.1796875" x="13.91015625" y="5.93359375">distribute<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n15">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="88.0" x="107.0" y="85.0078125"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="51.42578125" x="18.287109375" y="5.93359375">rack11b<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n16">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="88.0" x="-11.0" y="85.0078125"/>
+          <y:Fill color="#99CC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="60.1796875" x="13.91015625" y="5.93359375">distribute<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n17">
+      <data key="d6">
+        <y:GenericNode configuration="com.yworks.flowchart.dataBase">
+          <y:Geometry height="30.0" width="26.0" x="-67.0" y="85.0078125"/>
+          <y:Fill color="#E8EEF7" color2="#B7C9E3" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" hasText="false" height="4.0" modelName="custom" textColor="#000000" visible="true" width="4.0" x="11.0" y="13.0">
+            <y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+        </y:GenericNode>
+      </data>
+    </node>
+    <node id="n18">
+      <data key="d6">
+        <y:GenericNode configuration="com.yworks.flowchart.dataBase">
+          <y:Geometry height="30.0" width="26.0" x="461.0" y="137.0078125"/>
+          <y:Fill color="#E8EEF7" color2="#B7C9E3" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" hasText="false" height="4.0" modelName="custom" textColor="#000000" visible="true" width="4.0" x="11.0" y="13.0">
+            <y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+        </y:GenericNode>
+      </data>
+    </node>
+    <node id="n19">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="88.0" x="343.0" y="137.0078125"/>
+          <y:Fill color="#99CC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="60.1796875" x="13.91015625" y="5.93359375">distribute<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n20">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="88.0" x="107.0" y="137.0078125"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="50.01953125" x="18.990234375" y="5.93359375">rack11c<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n21">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="88.0" x="-11.0" y="137.0078125"/>
+          <y:Fill color="#99CC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="60.1796875" x="13.91015625" y="5.93359375">distribute<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n22">
+      <data key="d6">
+        <y:GenericNode configuration="com.yworks.flowchart.dataBase">
+          <y:Geometry height="30.0" width="26.0" x="-67.0" y="137.0078125"/>
+          <y:Fill color="#E8EEF7" color2="#B7C9E3" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" hasText="false" height="4.0" modelName="custom" textColor="#000000" visible="true" width="4.0" x="11.0" y="13.0">
+            <y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+        </y:GenericNode>
+      </data>
+    </node>
+    <node id="n23">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="88.0" x="225.0" y="85.0078125"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="51.42578125" x="18.287109375" y="5.93359375">rack12b<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n24">
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="11.0" x="482.0" y="58.5078125"/>
+          <y:Fill hasColor="false" transparent="false"/>
+          <y:BorderStyle hasColor="false" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" hasText="false" height="4.0" modelName="custom" textColor="#000000" visible="true" width="4.0" x="3.5" y="13.0">
+            <y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n25">
+      <data key="d5"/>
+      <data key="d6">
+        <y:SVGNode>
+          <y:Geometry height="35.484542934485205" width="32.2024545674844" x="192.89877271625778" y="-161.4767304344852"/>
+          <y:Fill color="#CCCCFF" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" hasText="false" height="4.0" modelName="custom" textColor="#000000" visible="true" width="4.0" x="14.10122728374219" y="39.484542934485205">
+            <y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="-0.5" nodeRatioX="0.0" nodeRatioY="0.5" offsetX="0.0" offsetY="4.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:SVGNodeProperties usingVisualBounds="true"/>
+          <y:SVGModel svgBoundsPolicy="0">
+            <y:SVGContent refid="1"/>
+          </y:SVGModel>
+        </y:SVGNode>
+      </data>
+    </node>
+    <edge id="e0" source="n1" target="n1">
+      <data key="d10">
+        <y:ArcEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0">
+            <y:Point x="209.0" y="-23.9921875"/>
+          </y:Path>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:Arc height="0.0" ratio="1.0" type="fixedRatio"/>
+        </y:ArcEdge>
+      </data>
+    </edge>
+    <edge id="e1" source="n3" target="n3">
+      <data key="d10">
+        <y:ArcEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0">
+            <y:Point x="209.0" y="-95.9921875"/>
+          </y:Path>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:Arc height="0.0" ratio="1.0" type="fixedRatio"/>
+        </y:ArcEdge>
+      </data>
+    </edge>
+    <edge id="e2" source="n7" target="n6">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="44.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="standard" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e3" source="n4" target="n5">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e4" source="n5" target="n3">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e5" source="n3" target="n1">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="standard" target="standard"/>
+          <y:EdgeLabel alignment="center" backgroundColor="#FFFFFF" configuration="AutoFlippingLabel" distance="2.0" fontFamily="Dialog" fontSize="10" fontStyle="bold" hasLineColor="false" height="15.77734375" modelName="custom" preferredPlacement="anywhere" ratio="0.5" textColor="#000000" visible="true" width="46.5537109375" x="-23.27685546875" y="13.111328124999986">(300ms)<y:LabelModel>
+              <y:SmartEdgeLabelModel autoRotationEnabled="false" defaultAngle="0.0" defaultDistance="10.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartEdgeLabelModelParameter angle="0.0" distance="30.0" distanceToCenter="true" position="center" ratio="0.5" segment="0"/>
+            </y:ModelParameter>
+            <y:PreferredPlacementDescriptor angle="0.0" angleOffsetOnRightSide="0" angleReference="absolute" angleRotationOnRightSide="co" distance="-1.0" frozen="true" placement="anywhere" side="anywhere" sideReference="relative_to_edge_flow"/>
+          </y:EdgeLabel>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e6" source="n1" target="n7">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="standard" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e7" source="n1" target="n2">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="standard" target="standard"/>
+          <y:EdgeLabel alignment="center" backgroundColor="#FFFFFF" configuration="AutoFlippingLabel" distance="2.0" fontFamily="Dialog" fontSize="10" fontStyle="bold" hasLineColor="false" height="15.77734375" modelName="custom" preferredPlacement="anywhere" ratio="0.5" textColor="#000000" visible="true" width="43.78515625" x="-4.568634033203125" y="12.611328124999993">(250μs)<y:LabelModel>
+              <y:SmartEdgeLabelModel autoRotationEnabled="false" defaultAngle="0.0" defaultDistance="10.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartEdgeLabelModelParameter angle="0.0" distance="30.0" distanceToCenter="true" position="center" ratio="0.5" segment="0"/>
+            </y:ModelParameter>
+            <y:PreferredPlacementDescriptor angle="0.0" angleOffsetOnRightSide="0" angleReference="absolute" angleRotationOnRightSide="co" distance="-1.0" frozen="true" placement="anywhere" side="anywhere" sideReference="relative_to_edge_flow"/>
+          </y:EdgeLabel>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e8" source="n8" target="n9">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="44.0" ty="0.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="standard" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e9" source="n11" target="n10">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="-44.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="standard" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e10" source="n2" target="n23">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e11" source="n1" target="n12">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="standard" target="standard"/>
+          <y:EdgeLabel alignment="center" backgroundColor="#FFFFFF" configuration="AutoFlippingLabel" distance="2.0" fontFamily="Dialog" fontSize="10" fontStyle="bold" hasLineColor="false" height="15.77734375" modelName="custom" preferredPlacement="anywhere" ratio="0.5" textColor="#000000" visible="true" width="43.78515625" x="-38.63905334472656" y="12.611328124999993">(250μs)<y:LabelModel>
+              <y:SmartEdgeLabelModel autoRotationEnabled="false" defaultAngle="0.0" defaultDistance="10.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartEdgeLabelModelParameter angle="0.0" distance="30.0" distanceToCenter="true" position="center" ratio="0.5" segment="0"/>
+            </y:ModelParameter>
+            <y:PreferredPlacementDescriptor angle="0.0" angleOffsetOnRightSide="0" angleReference="absolute" angleRotationOnRightSide="co" distance="-1.0" frozen="true" placement="anywhere" side="anywhere" sideReference="relative_to_edge_flow"/>
+          </y:EdgeLabel>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e12" source="n9" target="n2">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="-44.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="standard" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e13" source="n14" target="n13">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e14" source="n23" target="n14">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e15" source="n12" target="n15">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e16" source="n12" target="n11">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="44.0" ty="0.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="standard" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e17" source="n15" target="n16">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e18" source="n16" target="n17">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e19" source="n19" target="n18">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e20" source="n0" target="n19">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e21" source="n20" target="n21">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e22" source="n21" target="n22">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e23" source="n2" target="n0">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="-43.863669958563634" sy="14.9921875" tx="-43.863669958563634" ty="-15.020771131556558">
+            <y:Point x="212.0" y="73.0"/>
+            <y:Point x="212.0" y="119.5078125"/>
+          </y:Path>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="true"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e24" source="n12" target="n20">
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="42.85528831480789" sy="15.017222708103716" tx="42.85528831480789" ty="-15.0390625">
+            <y:Point x="205.0" y="74.0"/>
+            <y:Point x="205.0" y="120.5078125"/>
+          </y:Path>
+          <y:LineStyle color="#3366FF" type="dotted" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="true"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+  </graph>
+  <data key="d7">
+    <y:Resources>
+      <y:Resource id="1">&lt;?xml version="1.0" encoding="utf-8"?&gt;
+&lt;svg version="1.1" id="Ebene_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 width="57px" height="63px" viewBox="0 0 57 63" enable-background="new 0 0 57 63" xml:space="preserve"&gt;
+&lt;g&gt;
+	
+		&lt;linearGradient id="SVGID_1_" gradientUnits="userSpaceOnUse" x1="26.5" y1="1570.3457" x2="27.741" y2="1600.1431" gradientTransform="matrix(1 0 0 1 0.1602 -1546.3828)"&gt;
+		&lt;stop  offset="0.2711" style="stop-color:#FFAB4F"/&gt;
+		&lt;stop  offset="1" style="stop-color:#FFD28F"/&gt;
+	&lt;/linearGradient&gt;
+	&lt;path fill="url(#SVGID_1_)" stroke="#ED9135" stroke-miterlimit="10" d="M49.529,51.225c-4.396-4.396-10.951-5.884-12.063-6.109
+		V37.8H19.278c0,0,0.038,6.903,0,6.868c0,0-6.874,0.997-12.308,6.432C1.378,56.691,0.5,62.77,0.5,62.77
+		c0,1.938,1.575,3.492,3.523,3.492h48.51c1.947,0,3.521-1.558,3.521-3.492C56.055,62.768,54.211,55.906,49.529,51.225z"/&gt;
+	
+		&lt;radialGradient id="face_x5F_white_1_" cx="27.7427" cy="1572.1094" r="23.4243" fx="23.1732" fy="1569.6195" gradientTransform="matrix(1 0 0 1 0.1602 -1546.3828)" gradientUnits="userSpaceOnUse"&gt;
+		&lt;stop  offset="0" style="stop-color:#FFD28F"/&gt;
+		&lt;stop  offset="1" style="stop-color:#FFAB4F"/&gt;
+	&lt;/radialGradient&gt;
+	&lt;path id="face_x5F_white_3_" fill="url(#face_x5F_white_1_)" stroke="#ED9135" stroke-miterlimit="10" d="M43.676,23.357
+		c0.086,10.2-6.738,18.52-15.247,18.586c-8.502,0.068-15.466-8.146-15.552-18.344C12.794,13.4,19.618,5.079,28.123,5.012
+		C36.627,4.945,43.59,13.158,43.676,23.357z"/&gt;
+	
+		&lt;linearGradient id="face_highlight_1_" gradientUnits="userSpaceOnUse" x1="3646.5117" y1="-6644.2471" x2="3670.1414" y2="-6737.6978" gradientTransform="matrix(0.275 0 0 -0.2733 -977.2951 -1807.6279)"&gt;
+		&lt;stop  offset="0" style="stop-color:#FFFFFF;stop-opacity:0.24"/&gt;
+		&lt;stop  offset="1" style="stop-color:#FFFFFF;stop-opacity:0.16"/&gt;
+	&lt;/linearGradient&gt;
+	&lt;path id="face_highlight_3_" fill="url(#face_highlight_1_)" d="M27.958,6.333c-6.035,0.047-10.747,4.493-12.787,10.386
+		c-0.664,1.919-0.294,4.043,0.98,5.629c2.73,3.398,5.729,6.283,9.461,8.088c3.137,1.518,7.535,2.384,11.893,1.247
+		c2.274-0.592,3.988-2.459,4.375-4.766c0.183-1.094,0.293-2.289,0.283-3.553C42.083,13.952,36.271,6.268,27.958,6.333z"/&gt;
+	&lt;path fill="#656565" stroke="#4B4B4B" stroke-linejoin="round" stroke-miterlimit="10" d="M15.038,26.653
+		c0.145,2.05,3.468,2.593,6.477,2.56c2.298-0.026,3.25-0.889,4.746-2.685c2.539-3.05-0.767-3.715-4.817-3.67
+		C15.984,22.919,14.777,22.933,15.038,26.653z"/&gt;
+	&lt;path fill="#656565" stroke="#4B4B4B" stroke-linejoin="round" stroke-miterlimit="10" d="M41.116,26.653
+		c-0.146,2.05-3.47,2.593-6.478,2.56c-2.299-0.026-3.252-0.889-4.746-2.685c-2.538-3.05,0.769-3.715,4.816-3.67
+		C40.17,22.919,41.377,22.933,41.116,26.653z"/&gt;
+	&lt;path fill="none" stroke="#4B4B4B" stroke-linecap="round" stroke-linejoin="round" stroke-miterlimit="10" d="M27.453,24.375
+		c0,0,0.604-0.469,1.305,0"/&gt;
+	
+		&lt;line fill="none" stroke="#4B4B4B" stroke-linecap="round" stroke-miterlimit="10" x1="41.727" y1="24.592" x2="41.844" y2="25.375"/&gt;
+	
+		&lt;line fill="none" stroke="#4B4B4B" stroke-linecap="round" stroke-miterlimit="10" x1="42.165" y1="24.938" x2="44.027" y2="24.938"/&gt;
+	
+		&lt;line fill="none" stroke="#4B4B4B" stroke-linecap="round" stroke-miterlimit="10" x1="14.374" y1="24.592" x2="14.257" y2="25.375"/&gt;
+	
+		&lt;line fill="none" stroke="#4B4B4B" stroke-linecap="round" stroke-miterlimit="10" x1="13.937" y1="24.938" x2="12.073" y2="24.938"/&gt;
+	&lt;path id="body_9_" fill="#9B9B9B" stroke="#4B4B4B" stroke-linecap="round" stroke-linejoin="round" stroke-miterlimit="10" d="
+		M0.5,62.768c0,1.938,1.575,3.494,3.523,3.494h48.51c1.947,0,3.521-1.559,3.521-3.494c0,0-1.844-6.861-6.525-11.543
+		c-4.815-4.813-11.244-6.146-11.244-6.146c-1.771,1.655-5.61,2.802-10.063,2.802c-4.453,0-8.292-1.146-10.063-2.802
+		c0,0-5.755,0.586-11.189,6.021C1.378,56.689,0.5,62.768,0.5,62.768z"/&gt;
+	
+		&lt;path id="turtleneck_6_" fill="#9B9B9B" stroke="#4B4B4B" stroke-linecap="round" stroke-linejoin="round" stroke-miterlimit="10" d="
+		M39.715,44.786l-1.557-3.405c0,0-0.574,2.369-3.012,4.441c-2.109,1.795-6.785,2.072-6.785,2.072s-4.753-0.356-6.722-2.031
+		c-2.436-2.072-3.012-4.441-3.012-4.441l-1.555,3.404c0,0-0.552,1.404,1.37,3.479c1.025,1.105,5.203,3.611,9.682,3.582
+		c4.479-0.029,9.264-2.594,10.218-3.623C40.266,46.191,39.715,44.786,39.715,44.786z"/&gt;
+	&lt;path fill="#656565" stroke="#4B4B4B" stroke-linecap="round" stroke-linejoin="round" stroke-miterlimit="10" d="M49.529,51.225
+		c-1.094-1.094-2.319-2.006-3.563-2.766c0.193,0.346,0.401,0.68,0.574,1.041c-4.906,6.014-15.921,9.289-21.743,16.709
+		c1.969-7.594-11.166-13.127-14.493-16.926c-0.158-0.182-0.258-0.422-0.332-0.686c-1.015,0.707-2.031,1.525-3.001,2.5
+		c-5.592,5.592-6.47,11.67-6.47,11.67c0,1.936,1.575,3.489,3.523,3.489h48.51c1.948,0,3.521-1.558,3.521-3.489
+		C56.055,62.768,54.211,55.906,49.529,51.225z"/&gt;
+	&lt;path fill="#656565" stroke="#4B4B4B" stroke-linejoin="round" stroke-miterlimit="10" d="M3.007,32.205
+		c1.521,2.295,10.771,12.17,10.771,12.17s-5.137,3.012-3.474,4.908c3.327,3.799,10.533,14.018,14.865,16.467
+		c2.499-4.6-3.906-23.327-5.724-25.833c-1.296-1.786-3.22-3.269-4.598-5.417C14.846,34.5,9.195,34.5,3.007,32.205z"/&gt;
+	&lt;path fill="#656565" stroke="#4B4B4B" stroke-linecap="round" stroke-linejoin="round" stroke-miterlimit="10" d="M52.277,32.205
+		c-4.791,3.299-10.368,10.391-11.074,11.066c2.313,1.744,4.9,3.799,6.146,6.406c-4.906,6.014-14.766,9.277-21.747,16.069
+		c2.015-7.771,5.157-20.46,12.517-27.083c1.667-1.5,2.713-2.833,4.043-5.391C42.165,33.275,45.637,33.25,52.277,32.205z"/&gt;
+	&lt;path id="wh2_1_" fill="#9B9B9B" stroke="#4B4B4B" stroke-linecap="round" stroke-linejoin="round" stroke-miterlimit="10" d="
+		M28.276,15.5c5.635,0,10.826,1.416,14.979,3.794c-1.614-8.228-7.794-14.34-15.132-14.282c-7.272,0.057-13.299,6.155-14.846,14.294
+		C17.434,16.921,22.632,15.5,28.276,15.5z"/&gt;
+	&lt;path id="wh1_1_" fill="#9B9B9B" stroke="#4B4B4B" stroke-linecap="round" stroke-linejoin="round" stroke-miterlimit="10" d="
+		M28.278,20.808c5.662,0,11.937,0.811,16.391,2.207c-0.11-2.059-0.274-2.826-0.413-3.72c-4.154-2.379-10.344-3.795-15.98-3.795
+		c-5.644,0-11.842,1.421-16,3.807c-0.228,1.197-0.362,2.436-0.388,3.707C16.343,21.618,22.618,20.808,28.278,20.808z"/&gt;
+&lt;/g&gt;
+&lt;/svg&gt;
+</y:Resource>
+    </y:Resources>
+  </data>
+</graphml>