Merge pull request #1032 from moreati/docs-download-url

Python 3.12 support

.ci/azure-pipelines-steps.yml

@@ -2,6 +2,7 @@
 # https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/?view=azure-pipelines&viewFallbackFrom=azure-devops#tool
 
 # `{script: ...}` is shorthand for `{task: CmdLine@<mumble>, inputs: {script: ...}}`.
+# The shell is bash.
 # https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/steps-script?view=azure-pipelines
 # https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/cmd-line-v2?view=azure-pipelines
 
@@ -14,20 +15,80 @@ steps:
   condition: ne(variables['python.version'], '')
 
 - script: |
-    type python
+    set -o errexit
-    python --version
+    set -o nounset
-  displayName: Show python version
+    set -o pipefail
 
-- script: |
     sudo apt-get update
     sudo apt-get install -y python2-dev python3-pip virtualenv
   displayName: Install build deps
   condition: and(eq(variables['python.version'], ''), eq(variables['Agent.OS'], 'Linux'))
 
-- script: python -mpip install "tox<4.0"
+- script: |
+    set -o errexit
+    set -o nounset
+    set -o pipefail
+
+    # macOS builders lack a realpath command
+    type python  && python  -c"import os.path;print(os.path.realpath('$(type -p python)'))" &&  python --version
+    type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version
+    type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version
+    echo
+
+    if [ -e /usr/bin/python ]; then
+        echo "/usr/bin/python:    sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')"
+    fi
+
+    if [ -e /usr/bin/python2 ]; then
+        echo "/usr/bin/python2:   sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')"
+    fi
+
+    if [ -e /usr/bin/python2.7 ]; then
+        echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')"
+    fi
+  displayName: Show python versions
+
+- script: |
+    set -o errexit
+    set -o nounset
+    set -o pipefail
+
+    # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
+    PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "$(tox.env)"))')
+
+    if [[ -z $PYTHON ]]; then
+        echo 1>&2 "Python interpreter could not be determined"
+        exit 1
+    fi
+
+    if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then
+        "$PYTHON" -m ensurepip --user --altinstall --no-default-pip
+        "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
+    elif [[ $PYTHON == "python2.7" ]]; then
+        curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py"
+        "$PYTHON" get-pip.py --user --no-python-version-warning
+        # Avoid Python 2.x pip masking system pip
+        rm -f ~/.local/bin/{easy_install,pip,wheel}
+        "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
+    else
+        "$PYTHON" -m pip install -r "tests/requirements-tox.txt"
+    fi
   displayName: Install tooling
 
-- script: python -mtox -e "$(tox.env)"
+- script: |
+    set -o errexit
+    set -o nounset
+    set -o pipefail
+
+    # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
+    PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "$(tox.env)"))')
+
+    if [[ -z $PYTHON ]]; then
+        echo 1>&2 "Python interpreter could not be determined"
+        exit 1
+    fi
+
+    "$PYTHON" -m tox -e "$(tox.env)"
   displayName: "Run tests"
   env:
     AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)

.ci/azure-pipelines.yml

@@ -21,25 +21,21 @@ jobs:
     matrix:
       Mito_27:
         tox.env: py27-mode_mitogen
-      Mito_311:
+      Mito_312:
-        python.version: '3.11'
+        python.version: '3.12'
-        tox.env: py311-mode_mitogen
+        tox.env: py312-mode_mitogen
 
-      # TODO: test python3, python3 tests are broken
       Loc_27_210:
         tox.env: py27-mode_localhost-ansible2.10
-      Loc_27_4:
+      Loc_312_6:
-        tox.env: py27-mode_localhost-ansible4
+        python.version: '3.12'
+        tox.env: py312-mode_localhost-ansible6
 
-      # NOTE: this hangs when ran in Ubuntu 18.04
       Van_27_210:
-        tox.env: py27-mode_localhost-ansible2.10
+        tox.env: py27-mode_localhost-ansible2.10-strategy_linear
-        STRATEGY: linear
+      Van_312_6:
-        ANSIBLE_SKIP_TAGS: resource_intensive
+        python.version: '3.12'
-      Van_27_4:
+        tox.env: py312-mode_localhost-ansible6-strategy_linear
-        tox.env: py27-mode_localhost-ansible4
-        STRATEGY: linear
-        ANSIBLE_SKIP_TAGS: resource_intensive
 
 - job: Linux
   pool:
@@ -96,33 +92,33 @@ jobs:
         python.version: '3.6'
         tox.env: py36-mode_mitogen-distro_ubuntu2004
 
-      Mito_311_centos6:
+      Mito_312_centos6:
-        python.version: '3.11'
+        python.version: '3.12'
-        tox.env: py311-mode_mitogen-distro_centos6
+        tox.env: py312-mode_mitogen-distro_centos6
-      Mito_311_centos7:
+      Mito_312_centos7:
-        python.version: '3.11'
+        python.version: '3.12'
-        tox.env: py311-mode_mitogen-distro_centos7
+        tox.env: py312-mode_mitogen-distro_centos7
-      Mito_311_centos8:
+      Mito_312_centos8:
-        python.version: '3.11'
+        python.version: '3.12'
-        tox.env: py311-mode_mitogen-distro_centos8
+        tox.env: py312-mode_mitogen-distro_centos8
-      Mito_311_debian9:
+      Mito_312_debian9:
-        python.version: '3.11'
+        python.version: '3.12'
-        tox.env: py311-mode_mitogen-distro_debian9
+        tox.env: py312-mode_mitogen-distro_debian9
-      Mito_311_debian10:
+      Mito_312_debian10:
-        python.version: '3.11'
+        python.version: '3.12'
-        tox.env: py311-mode_mitogen-distro_debian10
+        tox.env: py312-mode_mitogen-distro_debian10
-      Mito_311_debian11:
+      Mito_312_debian11:
-        python.version: '3.11'
+        python.version: '3.12'
-        tox.env: py311-mode_mitogen-distro_debian11
+        tox.env: py312-mode_mitogen-distro_debian11
-      Mito_311_ubuntu1604:
+      Mito_312_ubuntu1604:
-        python.version: '3.11'
+        python.version: '3.12'
-        tox.env: py311-mode_mitogen-distro_ubuntu1604
+        tox.env: py312-mode_mitogen-distro_ubuntu1604
-      Mito_311_ubuntu1804:
+      Mito_312_ubuntu1804:
-        python.version: '3.11'
+        python.version: '3.12'
-        tox.env: py311-mode_mitogen-distro_ubuntu1804
+        tox.env: py312-mode_mitogen-distro_ubuntu1804
-      Mito_311_ubuntu2004:
+      Mito_312_ubuntu2004:
-        python.version: '3.11'
+        python.version: '3.12'
-        tox.env: py311-mode_mitogen-distro_ubuntu2004
+        tox.env: py312-mode_mitogen-distro_ubuntu2004
 
       Ans_27_210:
         tox.env: py27-mode_ansible-ansible2.10
@@ -148,6 +144,6 @@ jobs:
       Ans_311_5:
         python.version: '3.11'
         tox.env: py311-mode_ansible-ansible5
-      Ans_311_6:
+      Ans_312_6:
-        python.version: '3.11'
+        python.version: '3.12'
-        tox.env: py311-mode_ansible-ansible6
+        tox.env: py312-mode_ansible-ansible6

.ci/localhost_ansible_tests.py

@@ -1,6 +1,10 @@
 #!/usr/bin/env python
 # Run tests/ansible/all.yml under Ansible and Ansible-Mitogen
 
+from __future__ import print_function
+
+import getpass
+import io
 import os
 import subprocess
 import sys
@@ -53,6 +57,38 @@ with ci_lib.Fold('machine_prep'):
         os.chdir(IMAGE_PREP_DIR)
         ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml")
 
+    # FIXME Don't hardcode https://github.com/mitogen-hq/mitogen/issues/1022
+    #       and os.environ['USER'] is not populated on Azure macOS runners.
+    os.chdir(HOSTS_DIR)
+    with io.open('default.hosts', 'r+', encoding='utf-8') as f:
+        user = getpass.getuser()
+        content = f.read()
+        content = content.replace("{{ lookup('pipe', 'whoami') }}", user)
+        f.seek(0)
+        f.write(content)
+        f.truncate()
+    ci_lib.dump_file('default.hosts')
+
+    cmd = ';'.join([
+        'from __future__ import print_function',
+        'import os, sys',
+        'print(sys.executable, os.path.realpath(sys.executable))',
+    ])
+    for interpreter in ['/usr/bin/python', '/usr/bin/python2', '/usr/bin/python2.7']:
+        print(interpreter)
+        try:
+            subprocess.call([interpreter, '-c', cmd])
+        except OSError as exc:
+            print(exc)
+
+        print(interpreter, 'with PYTHON_LAUNCHED_FROM_WRAPPER=1')
+        environ = os.environ.copy()
+        environ['PYTHON_LAUNCHED_FROM_WRAPPER'] = '1'
+        try:
+            subprocess.call([interpreter, '-c', cmd], env=environ)
+        except OSError as exc:
+            print(exc)
+
 
 with ci_lib.Fold('ansible'):
     os.chdir(TESTS_DIR)

.gitignore

@@ -6,6 +6,7 @@ venvs/**
 *.pyc
 *.pyd
 *.pyo
+*.retry
 MANIFEST
 build/
 dist/

ansible_mitogen/module_finder.py

@@ -31,15 +31,31 @@ from __future__ import unicode_literals
 __metaclass__ = type
 
 import collections
-import imp
+import logging
 import os
+import re
+import sys
+
+try:
+    # Python >= 3.4, PEP 451 ModuleSpec API
+    import importlib.machinery
+    import importlib.util
+except ImportError:
+    # Python < 3.4, PEP 302 Import Hooks
+    import imp
 
 import mitogen.master
 
 
+LOG = logging.getLogger(__name__)
 PREFIX = 'ansible.module_utils.'
 
 
+# Analog of `importlib.machinery.ModuleSpec` or `pkgutil.ModuleInfo`.
+#   name    Unqualified name of the module.
+#   path    Filesystem path of the module.
+#   kind    One of the constants in `imp`, as returned in `imp.find_module()`
+#   parent  `ansible_mitogen.module_finder.Module` of parent package (if any).
 Module = collections.namedtuple('Module', 'name path kind parent')
 
 
@@ -119,14 +135,121 @@ def find_relative(parent, name, path=()):
 
 
 def scan_fromlist(code):
+    """Return an iterator of (level, name) for explicit imports in a code
+    object.
+
+    Not all names identify a module. `from os import name, path` generates
+    `(0, 'os.name'), (0, 'os.path')`, but `os.name` is usually a string.
+
+    >>> src = 'import a; import b.c; from d.e import f; from g import h, i\\n'
+    >>> code = compile(src, '<str>', 'exec')
+    >>> list(scan_fromlist(code))
+    [(0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')]
+    """
     for level, modname_s, fromlist in mitogen.master.scan_code_imports(code):
         for name in fromlist:
-            yield level, '%s.%s' % (modname_s, name)
+            yield level, str('%s.%s' % (modname_s, name))
         if not fromlist:
             yield level, modname_s
 
 
+def walk_imports(code, prefix=None):
+    """Return an iterator of names for implicit parent imports & explicit
+    imports in a code object.
+
+    If a prefix is provided, then only children of that prefix are included.
+    Not all names identify a module. `from os import name, path` generates
+    `'os', 'os.name', 'os.path'`, but `os.name` is usually a string.
+
+    >>> source = 'import a; import b; import b.c; from b.d import e, f\\n'
+    >>> code = compile(source, '<str>', 'exec')
+    >>> list(walk_imports(code))
+    ['a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f']
+    >>> list(walk_imports(code, prefix='b'))
+    ['b.c', 'b.d', 'b.d.e', 'b.d.f']
+    """
+    if prefix is None:
+        prefix = ''
+    pattern = re.compile(r'(^|\.)(\w+)')
+    start = len(prefix)
+    for _, name, fromlist in mitogen.master.scan_code_imports(code):
+        if not name.startswith(prefix):
+            continue
+        for match in pattern.finditer(name, start):
+            yield name[:match.end()]
+        for leaf in fromlist:
+            yield str('%s.%s' % (name, leaf))
+
+
 def scan(module_name, module_path, search_path):
+    # type: (str, str, list[str]) -> list[(str, str, bool)]
+    """Return a list of (name, path, is_package) for ansible.module_utils
+    imports used by an Ansible module.
+    """
+    log = LOG.getChild('scan')
+    log.debug('%r, %r, %r', module_name, module_path, search_path)
+
+    if sys.version_info >= (3, 4):
+        result = _scan_importlib_find_spec(
+            module_name, module_path, search_path,
+        )
+        log.debug('_scan_importlib_find_spec %r', result)
+    else:
+        result = _scan_imp_find_module(module_name, module_path, search_path)
+        log.debug('_scan_imp_find_module %r', result)
+    return result
+
+
+def _scan_importlib_find_spec(module_name, module_path, search_path):
+    # type: (str, str, list[str]) -> list[(str, str, bool)]
+    module = importlib.machinery.ModuleSpec(
+        module_name, loader=None, origin=module_path,
+    )
+    prefix = importlib.machinery.ModuleSpec(
+        PREFIX.rstrip('.'), loader=None,
+    )
+    prefix.submodule_search_locations = search_path
+    queue = collections.deque([module])
+    specs = {prefix.name: prefix}
+    while queue:
+        spec = queue.popleft()
+        if spec.origin is None:
+            continue
+        try:
+            with open(spec.origin, 'rb') as f:
+                code = compile(f.read(), spec.name, 'exec')
+        except Exception as exc:
+            raise ValueError((exc, module, spec, specs))
+
+        for name in walk_imports(code, prefix.name):
+            if name in specs:
+                continue
+
+            parent_name = name.rpartition('.')[0]
+            parent = specs[parent_name]
+            if parent is None or not parent.submodule_search_locations:
+                specs[name] = None
+                continue
+
+            child = importlib.util._find_spec(
+                name, parent.submodule_search_locations,
+            )
+            if child is None or child.origin is None:
+                specs[name] = None
+                continue
+
+            specs[name] = child
+            queue.append(child)
+
+    del specs[prefix.name]
+    return sorted(
+        (spec.name, spec.origin, spec.submodule_search_locations is not None)
+        for spec in specs.values() if spec is not None
+    )
+
+
+def _scan_imp_find_module(module_name, module_path, search_path):
+    # type: (str, str, list[str]) -> list[(str, str, bool)]
     module = Module(module_name, module_path, imp.PY_SOURCE, None)
     stack = [module]
     seen = set()

ansible_mitogen/runner.py

@@ -40,7 +40,6 @@ from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
 import atexit
-import imp
 import json
 import os
 import re
@@ -64,6 +63,14 @@ except ImportError:
     # Python 2.4
     ctypes = None
 
+try:
+    # Python >= 3.4, PEP 451 ModuleSpec API
+    import importlib.machinery
+    import importlib.util
+except ImportError:
+    # Python < 3.4, PEP 302 Import Hooks
+    import imp
+
 try:
     # Cannot use cStringIO as it does not support Unicode.
     from StringIO import StringIO
@@ -514,10 +521,71 @@ class ModuleUtilsImporter(object):
             sys.modules.pop(fullname, None)
 
     def find_module(self, fullname, path=None):
+        """
+        Return a loader for the module with fullname, if we will load it.
+
+        Implements importlib.abc.MetaPathFinder.find_module().
+        Deprecrated in Python 3.4+, replaced by find_spec().
+        Raises ImportWarning in Python 3.10+. Removed in Python 3.12.
+        """
         if fullname in self._by_fullname:
             return self
 
+    def find_spec(self, fullname, path, target=None):
+        """
+        Return a `ModuleSpec` for module with `fullname` if we will load it.
+        Otherwise return `None`.
+
+        Implements importlib.abc.MetaPathFinder.find_spec(). Python 3.4+.
+        """
+        if fullname.endswith('.'):
+            return None
+
+        try:
+            module_path, is_package = self._by_fullname[fullname]
+        except KeyError:
+            LOG.debug('Skipping %s: not present', fullname)
+            return None
+
+        LOG.debug('Handling %s', fullname)
+        origin = 'master:%s' % (module_path,)
+        return importlib.machinery.ModuleSpec(
+            fullname, loader=self, origin=origin, is_package=is_package,
+        )
+
+    def create_module(self, spec):
+        """
+        Return a module object for the given ModuleSpec.
+
+        Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4.
+        Unlike Loader.load_module() this shouldn't populate sys.modules or
+        set module attributes. Both are done by Python.
+        """
+        module = types.ModuleType(spec.name)
+        # FIXME create_module() shouldn't initialise module attributes
+        module.__file__ = spec.origin
+        return module
+
+    def exec_module(self, module):
+        """
+        Execute the module to initialise it. Don't return anything.
+
+        Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4.
+        """
+        spec = module.__spec__
+        path, _ = self._by_fullname[spec.name]
+        source = ansible_mitogen.target.get_small_file(self._context, path)
+        code = compile(source, path, 'exec', 0, 1)
+        exec(code, module.__dict__)
+        self._loaded.add(spec.name)
+
     def load_module(self, fullname):
+        """
+        Return the loaded module specified by fullname.
+
+        Implements PEP 302 importlib.abc.Loader.load_module().
+        Deprecated in Python 3.4+, replaced by create_module() & exec_module().
+        """
         path, is_pkg = self._by_fullname[fullname]
         source = ansible_mitogen.target.get_small_file(self._context, path)
         code = compile(source, path, 'exec', 0, 1)
@@ -818,12 +886,17 @@ class NewStyleRunner(ScriptRunner):
         synchronization mechanism by importing everything the module will need
         prior to detaching.
         """
+        # I think "custom" means "found in custom module_utils search path",
+        # e.g. playbook relative dir, ~/.ansible/..., Ansible collection.
         for fullname, _, _ in self.module_map['custom']:
             mitogen.core.import_module(fullname)
+
+        # I think "builtin" means "part of ansible/ansible-base/ansible-core",
+        # as opposed to Python builtin modules such as sys.
         for fullname in self.module_map['builtin']:
             try:
                 mitogen.core.import_module(fullname)
-            except ImportError:
+            except ImportError as exc:
                 # #590: Ansible 2.8 module_utils.distro is a package that
                 # replaces itself in sys.modules with a non-package during
                 # import. Prior to replacement, it is a real package containing
@@ -834,8 +907,18 @@ class NewStyleRunner(ScriptRunner):
                 # loop progresses to the next entry and attempts to preload
                 # 'distro._distro', the import mechanism will fail. So here we
                 # silently ignore any failure for it.
-                if fullname != 'ansible.module_utils.distro._distro':
+                if fullname == 'ansible.module_utils.distro._distro':
-                    raise
+                    continue
+
+                # ansible.module_utils.compat.selinux raises ImportError if it
+                # can't load libselinux.so. The importer would usually catch
+                # this & skip selinux operations. We don't care about selinux,
+                # we're using import to get a copy of the module.
+                if (fullname == 'ansible.module_utils.compat.selinux'
+                    and exc.msg == 'unable to load libselinux.so'):
+                    continue
+
+                raise
 
     def _setup_excepthook(self):
         """

ansible_mitogen/utils.py

@@ -1,7 +1,7 @@
 from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
-import distutils.version
+import re
 
 import ansible
 
@@ -9,6 +9,21 @@ __all__ = [
     'ansible_version',
 ]
 
-ansible_version = tuple(distutils.version.LooseVersion(ansible.__version__).version)
+
-del distutils
+def _parse(v_string):
+    # Adapted from distutils.version.LooseVersion.parse()
+    component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE)
+    for component in component_re.split(v_string):
+        if not component or component == '.':
+            continue
+        try:
+            yield int(component)
+        except ValueError:
+            yield component
+
+
+ansible_version = tuple(_parse(ansible.__version__))
+
+del _parse
+del re
 del ansible

docs/ansible_detailed.rst

@@ -149,7 +149,8 @@ Noteworthy Differences
   Mitogen 0.3.1+ supports
 
     - Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.11
-    - Ansible 5 and 6; with Python 3.8-3.11
+    - Ansible 5; with Python 3.8-3.11
+    - Ansible 6; with Python 3.8-3.12
 
   Verify your installation is running one of these versions by checking
   ``ansible --version`` output.

docs/changelog.rst

@@ -21,6 +21,11 @@ Unreleased
 ----------
 
 * :gh:issue:`987` Support Python 3.11
+* :gh:issue:`885` Fix :py:exc:`PermissionError` in :py:mod:`importlib` when
+  becoming an unprivileged user with Python 3.x
+* :gh:issue:`1033` Support `PEP 451 <https://peps.python.org/pep-0451/>,
+  required by Python 3.12
+* :gh:issue:`1033` Support Python 3.12
 
 
 v0.3.4 (2023-07-02)

docs/conf.py

@@ -1,9 +1,8 @@
 import sys
 
-sys.path.append('..')
 sys.path.append('.')
-import mitogen
+
-VERSION = '.'.join(str(part) for part in mitogen.__version__)
+VERSION = '0.3.4'
 
 author = u'Network Genomics'
 copyright = u'2021, the Mitogen authors'
@@ -83,10 +82,16 @@ domainrefs = {
     },
 }
 
+# > ## Official guidance
+# > Query PyPI’s JSON API to determine where to download files from.
+# > ## Predictable URLs
+# > You can use our conveyor service to fetch this file, which exists for
+# > cases where using the API is impractical or impossible.
+# > -- https://warehouse.pypa.io/api-reference/integration-guide.html#predictable-urls
 rst_epilog = """
 
 .. |mitogen_version| replace:: %(VERSION)s
 
-.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz <https://networkgenomics.com/try/mitogen-%(VERSION)s.tar.gz>`__
+.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz <https://files.pythonhosted.org/packages/source/m/mitogen/mitogen-%(VERSION)s.tar.gz>`__
 
 """ % locals()

docs/contributors.rst

@@ -138,4 +138,5 @@ sponsorship and outstanding future-thinking of its early adopters.
     <li>randy &mdash; <em>desperate for automation</em></li>
     <li>Michael & Vicky Twomey-Lee</li>
     <li><a href="http://www.wezm.net/">Wesley Moore</a></li>
+    <li><a href="https://github.com/baryluk">Witold Baryluk</a></li>
     </ul>

mitogen/core.py

@@ -34,6 +34,34 @@ non-essential code in order to reduce its size, since it is also serves as the
 bootstrap implementation sent to every new slave context.
 """
 
+import sys
+try:
+    import _frozen_importlib_external
+except ImportError:
+    pass
+else:
+    class MonkeyPatchedPathFinder(_frozen_importlib_external.PathFinder):
+        """
+        Meta path finder for sys.path and package __path__ attributes.
+
+        Patched for https://github.com/python/cpython/issues/115911.
+        """
+        @classmethod
+        def _path_importer_cache(cls, path):
+            if path == '':
+                try:
+                    path = _frozen_importlib_external._os.getcwd()
+                except (FileNotFoundError, PermissionError):
+                    return None
+            return super()._path_importer_cache(path)
+
+    if sys.version_info[:2] <= (3, 12):
+        for i, mpf in enumerate(sys.meta_path):
+            if mpf is _frozen_importlib_external.PathFinder:
+                sys.meta_path[i] = MonkeyPatchedPathFinder
+        del i, mpf
+
+
 import binascii
 import collections
 import encodings.latin_1
@@ -49,18 +77,22 @@ import pstats
 import signal
 import socket
 import struct
-import sys
 import syslog
 import threading
 import time
 import traceback
+import types
 import warnings
 import weakref
 import zlib
 
-# Python >3.7 deprecated the imp module.
+try:
-warnings.filterwarnings('ignore', message='the imp module is deprecated')
+    # Python >= 3.4, PEP 451 ModuleSpec API
-import imp
+    import importlib.machinery
+    import importlib.util
+except ImportError:
+    # Python < 3.4, PEP 302 Import Hooks
+    import imp
 
 # Absolute imports for <2.5.
 select = __import__('select')
@@ -1353,6 +1385,19 @@ class Importer(object):
     def __repr__(self):
         return 'Importer'
 
+    @staticmethod
+    def _loader_from_module(module, default=None):
+        """Return the loader for a module object."""
+        try:
+            return module.__spec__.loader
+        except AttributeError:
+            pass
+        try:
+            return module.__loader__
+        except AttributeError:
+            pass
+        return default
+
     def builtin_find_module(self, fullname):
         # imp.find_module() will always succeed for __main__, because it is a
         # built-in module. That means it exists on a special linked list deep
@@ -1360,12 +1405,19 @@ class Importer(object):
         if fullname == '__main__':
             raise ModuleNotFoundError()
 
+        # For a module inside a package (e.g. pkg_a.mod_b) use the search path
+        # of that package (e.g. ['/usr/lib/python3.11/site-packages/pkg_a']).
         parent, _, modname = str_rpartition(fullname, '.')
         if parent:
             path = sys.modules[parent].__path__
         else:
             path = None
 
+        # For a top-level module search builtin modules, frozen modules,
+        # system specific locations (e.g. Windows registry, site-packages).
+        # Otherwise use search path of the parent package.
+        # Works for both stdlib modules & third-party modules.
+        # If the search is unsuccessful then raises ImportError.
         fp, pathname, description = imp.find_module(modname, path)
         if fp:
             fp.close()
@@ -1377,8 +1429,9 @@ class Importer(object):
         Implements importlib.abc.MetaPathFinder.find_module().
         Deprecrated in Python 3.4+, replaced by find_spec().
         Raises ImportWarning in Python 3.10+.
+        Removed in Python 3.12.
 
-        fullname    A (fully qualified?) module name, e.g. "os.path".
+        fullname    Fully qualified module name, e.g. "os.path".
         path        __path__ of parent packge. None for a top level module.
         """
         if hasattr(_tls, 'running'):
@@ -1388,14 +1441,13 @@ class Importer(object):
         try:
             #_v and self._log.debug('Python requested %r', fullname)
             fullname = to_text(fullname)
-            pkgname, dot, _ = str_rpartition(fullname, '.')
+            pkgname, _, suffix = str_rpartition(fullname, '.')
             pkg = sys.modules.get(pkgname)
             if pkgname and getattr(pkg, '__loader__', None) is not self:
                 self._log.debug('%s is submodule of a locally loaded package',
                                 fullname)
                 return None
 
-            suffix = fullname[len(pkgname+dot):]
             if pkgname and suffix not in self._present.get(pkgname, ()):
                 self._log.debug('%s has no submodule %s', pkgname, suffix)
                 return None
@@ -1415,6 +1467,66 @@ class Importer(object):
         finally:
             del _tls.running
 
+    def find_spec(self, fullname, path, target=None):
+        """
+        Return a `ModuleSpec` for module with `fullname` if we will load it.
+        Otherwise return `None`, allowing other finders to try.
+
+        fullname    Fully qualified name of the module (e.g. foo.bar.baz)
+        path        Path entries to search. None for a top-level module.
+        target      Existing module to be reloaded (if any).
+
+        Implements importlib.abc.MetaPathFinder.find_spec()
+        Python 3.4+.
+        """
+        # Presence of _tls.running indicates we've re-invoked importlib.
+        # Abort early to prevent infinite recursion. See below.
+        if hasattr(_tls, 'running'):
+            return None
+
+        log = self._log.getChild('find_spec')
+
+        if fullname.endswith('.'):
+            return None
+
+        pkgname, _, modname = fullname.rpartition('.')
+        if pkgname and modname not in self._present.get(pkgname, ()):
+            log.debug('Skipping %s. Parent %s has no submodule %s',
+                      fullname, pkgname, modname)
+            return None
+
+        pkg = sys.modules.get(pkgname)
+        pkg_loader = self._loader_from_module(pkg)
+        if pkgname and pkg_loader is not self:
+            log.debug('Skipping %s. Parent %s was loaded by %r',
+                      fullname, pkgname, pkg_loader)
+            return None
+
+        # #114: whitelisted prefixes override any system-installed package.
+        if self.whitelist != ['']:
+            if any(s and fullname.startswith(s) for s in self.whitelist):
+                log.debug('Handling %s. It is whitelisted', fullname)
+                return importlib.machinery.ModuleSpec(fullname, loader=self)
+
+        if fullname == '__main__':
+            log.debug('Handling %s. A special case', fullname)
+            return importlib.machinery.ModuleSpec(fullname, loader=self)
+
+        # Re-invoke the import machinery to allow other finders to try.
+        # Set a guard, so we don't infinitely recurse. See top of this method.
+        _tls.running = True
+        try:
+            spec = importlib.util._find_spec(fullname, path, target)
+        finally:
+            del _tls.running
+
+        if spec:
+            log.debug('Skipping %s. Available as %r', fullname, spec)
+            return spec
+
+        log.debug('Handling %s. Unavailable locally', fullname)
+        return importlib.machinery.ModuleSpec(fullname, loader=self)
+
     blacklisted_msg = (
         '%r is present in the Mitogen importer blacklist, therefore this '
         'context will not attempt to request it from the master, as the '
@@ -1501,6 +1613,64 @@ class Importer(object):
         if present:
             callback()
 
+    def create_module(self, spec):
+        """
+        Return a module object for the given ModuleSpec.
+
+        Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4.
+        Unlike Loader.load_module() this shouldn't populate sys.modules or
+        set module attributes. Both are done by Python.
+        """
+        self._log.debug('Creating module for %r', spec)
+
+        # FIXME Should this be done in find_spec()? Can it?
+        self._refuse_imports(spec.name)
+
+        # FIXME "create_module() should properly handle the case where it is
+        #       called more than once for the same spec/module." -- PEP-451
+        event = threading.Event()
+        self._request_module(spec.name, callback=event.set)
+        event.wait()
+
+        # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+        _, pkg_present, path, _, _ = self._cache[spec.name]
+
+        if path is None:
+            raise ImportError(self.absent_msg % (spec.name))
+
+        spec.origin = self.get_filename(spec.name)
+        if pkg_present is not None:
+            # TODO Namespace packages
+            spec.submodule_search_locations = []
+            self._present[spec.name] = pkg_present
+
+        module = types.ModuleType(spec.name)
+        # FIXME create_module() shouldn't initialise module attributes
+        module.__file__ = spec.origin
+        return module
+
+    def exec_module(self, module):
+        """
+        Execute the module to initialise it. Don't return anything.
+
+        Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4.
+        """
+        name = module.__spec__.name
+        origin = module.__spec__.origin
+        self._log.debug('Executing %s from %s', name, origin)
+        source = self.get_source(name)
+        try:
+            # Compile the source into a code object. Don't add any __future__
+            # flags and don't inherit any from this module.
+            # FIXME Should probably be exposed as get_code()
+            code = compile(source, origin, 'exec', flags=0, dont_inherit=True)
+        except SyntaxError:
+            # FIXME Why is this LOG, rather than self._log?
+            LOG.exception('while importing %r', name)
+            raise
+
+        exec(code, module.__dict__)
+
     def load_module(self, fullname):
         """
         Return the loaded module specified by fullname.
@@ -1516,11 +1686,11 @@ class Importer(object):
         self._request_module(fullname, event.set)
         event.wait()
 
-        ret = self._cache[fullname]
+        # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
-        if ret[2] is None:
+        _, pkg_present, path, _, _ = self._cache[fullname]
+        if path is None:
             raise ModuleNotFoundError(self.absent_msg % (fullname,))
 
-        pkg_present = ret[1]
         mod = sys.modules.setdefault(fullname, imp.new_module(fullname))
         mod.__file__ = self.get_filename(fullname)
         mod.__loader__ = self
@@ -3921,7 +4091,7 @@ class ExternalContext(object):
 
     def _setup_package(self):
         global mitogen
-        mitogen = imp.new_module('mitogen')
+        mitogen = types.ModuleType('mitogen')
         mitogen.__package__ = 'mitogen'
         mitogen.__path__ = []
         mitogen.__loader__ = self.importer

mitogen/master.py

@@ -37,7 +37,6 @@ contexts.
 
 import dis
 import errno
-import imp
 import inspect
 import itertools
 import logging
@@ -50,6 +49,16 @@ import threading
 import types
 import zlib
 
+try:
+    # Python >= 3.4, PEP 451 ModuleSpec API
+    import importlib.machinery
+    import importlib.util
+    from _imp import is_builtin as _is_builtin
+except ImportError:
+    # Python < 3.4, PEP 302 Import Hooks
+    import imp
+    from imp import is_builtin as _is_builtin
+
 try:
     import sysconfig
 except ImportError:
@@ -122,14 +131,16 @@ def is_stdlib_name(modname):
     """
     Return :data:`True` if `modname` appears to come from the standard library.
     """
-    # `imp.is_builtin()` isn't a documented as part of Python's stdlib API.
+    # `(_imp|imp).is_builtin()` isn't a documented part of Python's stdlib.
+    # Returns 1 if modname names a module that is "builtin" to the the Python
+    # interpreter (e.g. '_sre'). Otherwise 0 (e.g. 're', 'netifaces').
     #
     # """
     # Main is a little special - imp.is_builtin("__main__") will return False,
     # but BuiltinImporter is still the most appropriate initial setting for
     # its __loader__ attribute.
     # """ -- comment in CPython pylifecycle.c:add_main_module()
-    if imp.is_builtin(modname) != 0:
+    if _is_builtin(modname) != 0:
         return True
 
     module = sys.modules.get(modname)
@@ -460,6 +471,9 @@ class FinderMethod(object):
     name according to the running Python interpreter. You'd think this was a
     simple task, right? Naive young fellow, welcome to the real world.
     """
+    def __init__(self):
+        self.log = LOG.getChild(self.__class__.__name__)
+
     def __repr__(self):
         return '%s()' % (type(self).__name__,)
 
@@ -641,7 +655,7 @@ class SysModulesMethod(FinderMethod):
         return path, source, is_pkg
 
 
-class ParentEnumerationMethod(FinderMethod):
+class ParentImpEnumerationMethod(FinderMethod):
     """
     Attempt to fetch source code by examining the module's (hopefully less
     insane) parent package, and if no insane parents exist, simply use
@@ -759,6 +773,7 @@ class ParentEnumerationMethod(FinderMethod):
     def _find_one_component(self, modname, search_path):
         try:
             #fp, path, (suffix, _, kind) = imp.find_module(modname, search_path)
+            # FIXME The imp module was removed in Python 3.12.
             return imp.find_module(modname, search_path)
         except ImportError:
             e = sys.exc_info()[1]
@@ -770,6 +785,9 @@ class ParentEnumerationMethod(FinderMethod):
         """
         See implementation for a description of how this works.
         """
+        if sys.version_info >= (3, 4):
+            return None
+
         #if fullname not in sys.modules:
             # Don't attempt this unless a module really exists in sys.modules,
             # else we could return junk.
@@ -798,6 +816,99 @@ class ParentEnumerationMethod(FinderMethod):
                 return self._found_module(fullname, path, fp)
 
 
+class ParentSpecEnumerationMethod(ParentImpEnumerationMethod):
+    def _find_parent_spec(self, fullname):
+        #history = []
+        debug = self.log.debug
+        children = []
+        for parent_name, child_name in self._iter_parents(fullname):
+            children.insert(0, child_name)
+            if not parent_name:
+                debug('abandoning %r, reached top-level', fullname)
+                return None, children
+
+            try:
+                parent = sys.modules[parent_name]
+            except KeyError:
+                debug('skipping %r, not in sys.modules', parent_name)
+                continue
+
+            try:
+                spec = parent.__spec__
+            except AttributeError:
+                debug('skipping %r: %r.__spec__ is absent',
+                      parent_name, parent)
+                continue
+
+            if not spec:
+                debug('skipping %r: %r.__spec__=%r',
+                      parent_name, parent, spec)
+                continue
+
+            if spec.name != parent_name:
+                debug('skipping %r: %r.__spec__.name=%r does not match',
+                      parent_name, parent, spec.name)
+                continue
+
+            if not spec.submodule_search_locations:
+                debug('skipping %r: %r.__spec__.submodule_search_locations=%r',
+                      parent_name, parent, spec.submodule_search_locations)
+                continue
+
+            return spec, children
+
+        raise ValueError('%s._find_parent_spec(%r) unexpectedly reached bottom'
+                         % (self.__class__.__name__, fullname))
+
+    def find(self, fullname):
+        # Returns absolute path, ParentImpEnumerationMethod returns relative
+        # >>> spec_pem.find('six_brokenpkg._six')[::2]
+        # ('/Users/alex/src/mitogen/tests/data/importer/six_brokenpkg/_six.py', False)
+
+        if sys.version_info < (3, 4):
+            return None
+
+        fullname = to_text(fullname)
+        spec, children = self._find_parent_spec(fullname)
+        for child_name in children:
+            if spec:
+                name = '%s.%s' % (spec.name, child_name)
+                submodule_search_locations = spec.submodule_search_locations
+            else:
+                name = child_name
+                submodule_search_locations = None
+            spec = importlib.util._find_spec(name, submodule_search_locations)
+            if spec is None:
+                self.log.debug('%r spec unavailable from %s', fullname, spec)
+                return None
+
+            is_package = spec.submodule_search_locations is not None
+            if name != fullname:
+                if not is_package:
+                    self.log.debug('%r appears to be child of non-package %r',
+                                   fullname, spec)
+                    return None
+                continue
+
+            if not spec.has_location:
+                self.log.debug('%r.origin cannot be read as a file', spec)
+                return None
+
+            if os.path.splitext(spec.origin)[1] != '.py':
+                self.log.debug('%r.origin does not contain Python source code',
+                               spec)
+                return None
+
+            # FIXME This should use loader.get_source()
+            with open(spec.origin, 'rb') as f:
+                source = f.read()
+
+            return spec.origin, source, is_package
+
+        raise ValueError('%s.find(%r) unexpectedly reached bottom'
+                         % (self.__class__.__name__, fullname))
+
+
 class ModuleFinder(object):
     """
     Given the name of a loaded module, make a best-effort attempt at finding
@@ -838,7 +949,8 @@ class ModuleFinder(object):
         DefectivePython3xMainMethod(),
         PkgutilMethod(),
         SysModulesMethod(),
-        ParentEnumerationMethod(),
+        ParentSpecEnumerationMethod(),
+        ParentImpEnumerationMethod(),
     ]
 
     def get_module_source(self, fullname):

mitogen/parent.py

@@ -34,7 +34,7 @@ sent to any child context that is due to become a parent, due to recursive
 connection.
 """
 
-import codecs
+import binascii
 import errno
 import fcntl
 import getpass
@@ -1405,10 +1405,14 @@ class Connection(object):
     # file descriptor 0 as 100, creates a pipe, then execs a new interpreter
     # with a custom argv.
     #   * Optimized for minimum byte count after minification & compression.
+    #     The script preamble_size.py measures this.
     #   * 'CONTEXT_NAME' and 'PREAMBLE_COMPRESSED_LEN' are substituted with
     #     their respective values.
     #   * CONTEXT_NAME must be prefixed with the name of the Python binary in
     #     order to allow virtualenvs to detect their install prefix.
+    #
+    # macOS tweaks for Python 2.7 must be kept in sync with the the Ansible
+    # module test_echo_module, used by the integration tests.
     #   * macOS <= 10.14 (Darwin <= 18) install an unreliable Python version
     #     switcher as /usr/bin/python, which introspects argv0. To workaround
     #     it we redirect attempts to call /usr/bin/python with an explicit
@@ -1417,7 +1421,8 @@ class Connection(object):
     #     do something slightly different. The Python executable is patched to
     #     perform an extra execvp(). I don't fully understand the details, but
     #     setting PYTHON_LAUNCHED_FROM_WRAPPER=1 avoids it.
-    #   * macOS 13.x (Darwin 22?) may remove python 2.x entirely.
+    #   * macOS 12.3+ (Darwin 21.4+, Monterey) doesn't ship Python.
+    #     https://developer.apple.com/documentation/macos-release-notes/macos-12_3-release-notes#Python
     #
     # Locals:
     #   R: read side of interpreter stdin.
@@ -1445,7 +1450,7 @@ class Connection(object):
             os.environ['ARGV0']=sys.executable
             os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)')
         os.write(1,'MITO000\n'.encode())
-        C=_(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN),'zip')
+        C=zlib.decompress(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN))
         fp=os.fdopen(W,'wb',0)
         fp.write(C)
         fp.close()
@@ -1477,16 +1482,16 @@ class Connection(object):
         source = source.replace('PREAMBLE_COMPRESSED_LEN',
                                 str(len(preamble_compressed)))
         compressed = zlib.compress(source.encode(), 9)
-        encoded = codecs.encode(compressed, 'base64').replace(b('\n'), b(''))
+        encoded = binascii.b2a_base64(compressed).replace(b('\n'), b(''))
-        # We can't use bytes.decode() in 3.x since it was restricted to always
+
-        # return unicode, so codecs.decode() is used instead. In 3.x
+        # Just enough to decode, decompress, and exec the first stage.
-        # codecs.decode() requires a bytes object. Since we must be compatible
+        # Priorities: wider compatibility, faster startup, shorter length.
-        # with 2.4 (no bytes literal), an extra .encode() either returns the
+        # `import os` here, instead of stage 1, to save a few bytes.
-        # same str (2.x) or an equivalent bytes (3.x).
+        # `sys.path=...` for https://github.com/python/cpython/issues/115911.
         return self.get_python_argv() + [
             '-c',
-            'import codecs,os,sys;_=codecs.decode;'
+            'import sys;sys.path=[p for p in sys.path if p];import binascii,os,zlib;'
-            'exec(_(_("%s".encode(),"base64"),"zip"))' % (encoded.decode(),)
+            'exec(zlib.decompress(binascii.a2b_base64("%s")))' % (encoded.decode(),),
         ]
 
     def get_econtext_config(self):

setup.py

@@ -78,6 +78,7 @@ setup(
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
         'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
         'Programming Language :: Python :: Implementation :: CPython',
         'Topic :: System :: Distributed Computing',
         'Topic :: System :: Systems Administration',

tests/ansible/files/cwd_show

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# Show permissions and identities that impact the current working directory.
+# On macOS libc cwd() can return EACCES after su or sudo.
+# See also
+#   - https://github.com/ansible/ansible/pull/7078
+#   - https://github.com/python/cpython/issues/115911
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+whoami
+groups
+pwd
+
+d=$(pwd)
+while [[ "$d" != "/" && -n "$d" ]]; do
+    ls -ld "$d"
+    d=$(dirname "$d")
+done
+ls -ld /

tests/ansible/hosts/become_same_user.hosts

@@ -1,3 +1,5 @@
+# code: language=ini
+# vim: syntax=dosini
 
 # become_same_user.yml
 bsu-joe ansible_user=joe

tests/ansible/hosts/connection_delegation.hosts

@@ -1,3 +1,4 @@
+# code: language=ini
 # vim: syntax=dosini
 
 # Connection delegation scenarios. It's impossible to connect to them, but their would-be

tests/ansible/hosts/default.hosts

@@ -1,9 +1,12 @@
+# code: language=ini
 # vim: syntax=dosini
 
 # When running the tests outside CI, make a single 'target' host which is the
 # local machine. The ansible_user override is necessary since some tests want a
 # fixed ansible.cfg remote_user setting to test against.
-target ansible_host=localhost ansible_user="{{lookup('env', 'USER')}}"
+# FIXME Hardcoded by replacement in some CI runs https://github.com/mitogen-hq/mitogen/issues/1022
+#       and os.environ['USER'] is not populated on Azure macOS runners.
+target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}"
 
 [test-targets]
 target

tests/ansible/hosts/k3.hosts

@@ -1,3 +1,4 @@
+# code: language=ini
 # vim: syntax=dosini
 
 # Used for manual testing.

tests/ansible/hosts/localhost.hosts

@@ -1,3 +1,4 @@
+# code: language=ini
 # vim: syntax=dosini
 
 # issue #511, #536: we must not define an explicit localhost, as some

tests/ansible/hosts/transport_config.hosts

@@ -1,3 +1,6 @@
+# code: language=ini
+# vim: syntax=dosini
+
 # integration/transport_config
 # Hosts with twiddled configs that need to be checked somehow.
 
@@ -17,11 +20,12 @@ tc_remote_user
 tc_transport
 
 [transport_config_undiscover:vars]
-# If python interpreter path is unset, Ansible tries to connect & discover it.
+# If ansible_*_interpreter isn't set Ansible tries to connect & discover it.
-# That causes approx 10 seconds timeout per task - there's no host to connect to.
+# If that target doesn't exist we must wait $timeout seconds for each attempt.
+# Setting a known (invalid) interpreter skips discovery & the many timeouts.
 # This optimisation should not be relied in any test.
 # Note: tc-python-path-* are intentionally not included.
-ansible_python_interpreter = python3000  # Not expected to exist
+ansible_python_interpreter = python3000
 
 [tc_transport]
 tc-transport-unset

tests/ansible/integration/become/su_password.yml

@@ -1,5 +1,4 @@
 # Verify passwordful su behaviour
-  # Ansible can't handle this on OS X. I don't care why.
 
 - name: integration/become/su_password.yml
   hosts: test-targets
@@ -44,20 +43,54 @@
         fail_msg: out={{out}}
       when: is_mitogen
 
-    - name: Ensure password su succeeds.
+    - name: Ensure password su with chdir succeeds
       shell: whoami
+      args:
+        chdir: ~mitogen__user1
       become: true
       become_user: mitogen__user1
       register: out
       vars:
         ansible_become_pass: user1_password
-      when: is_mitogen
+      when:
+        # https://github.com/ansible/ansible/pull/70785
+        - ansible_facts.distribution not in ["MacOSX"]
+          or ansible_version.full is version("2.11", ">=", strict=True)
+          or is_mitogen
 
     - assert:
         that:
           - out.stdout == 'mitogen__user1'
         fail_msg: out={{out}}
-      when: is_mitogen
+      when:
+        # https://github.com/ansible/ansible/pull/70785
+        - ansible_facts.distribution not in ["MacOSX"]
+          or ansible_version.full is version("2.11", ">=", strict=True)
+          or is_mitogen
+
+    - name: Ensure password su without chdir succeeds
+      shell: whoami
+      become: true
+      become_user: mitogen__user1
+      register: out
+      vars:
+        ansible_become_pass: user1_password
+      when:
+        # https://github.com/ansible/ansible/pull/70785
+        - ansible_facts.distribution not in ["MacOSX"]
+          or ansible_version.full is version("2.11", ">=", strict=True)
+          or is_mitogen
+
+    - assert:
+        that:
+          - out.stdout == 'mitogen__user1'
+        fail_msg: out={{out}}
+      when:
+        # https://github.com/ansible/ansible/pull/70785
+        - ansible_facts.distribution not in ["MacOSX"]
+          or ansible_version.full is version("2.11", ">=", strict=True)
+          or is_mitogen
+
   tags:
     - su
     - su_password

tests/ansible/integration/connection_delegation/delegate_to_template.yml

@@ -41,7 +41,7 @@
               'keepalive_count': 10,
               'password': null,
               'port': null,
-              'python_path': ["/usr/bin/python"],
+              'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"],
               'remote_name': null,
               'ssh_args': [
                 -o, ControlMaster=auto,
@@ -69,7 +69,7 @@
               'keepalive_count': 10,
               'password': null,
               'port': null,
-              'python_path': ["/usr/bin/python"],
+              'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"],
               'remote_name': null,
               'ssh_args': [
                 -o, ControlMaster=auto,

tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml

@@ -37,6 +37,7 @@
             vars:
               ansible_python_interpreter: auto
             test_echo_module:
+              facts_copy: "{{ ansible_facts }}"
             register: echoout
 
           # can't test this assertion:
@@ -44,11 +45,24 @@
           # because Mitogen's ansible_python_interpreter is a connection-layer configurable that
           # "must be extracted during each task execution to form the complete connection-layer configuration".
           # Discovery won't be reran though; the ansible_python_interpreter is read from the cache if already discovered
-          - assert:
+          - name: assert discovered python matches invoked python
+            assert:
               that:
               - auto_out.ansible_facts.discovered_interpreter_python is defined
-              - echoout.running_python_interpreter == auto_out.ansible_facts.discovered_interpreter_python
+              - auto_out.ansible_facts.discovered_interpreter_python == echoout.discovered_python.as_seen
-              fail_msg: auto_out={{auto_out}} echoout={{echoout}}
+              - echoout.discovered_python.resolved == echoout.running_python.sys.executable.resolved
+              fail_msg:
+                - "auto_out: {{ auto_out }}"
+                - "echoout: {{ echoout }}"
+            when:
+              # On macOS 11 (Darwin 20) CI runners the Python 2.7 binary always
+              # reports the same path. I can't reach via symlinks.
+              # >>> sys.executable
+              # /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python
+              - is_mitogen
+                or echoout.running_python.sys.version_info.major != 2
+                or not (echoout.running_python.sys.platform == "darwin"
+                        and echoout.running_python.platform.release.major == 20)
 
 
         - name: test that auto_legacy gives a dep warning when /usr/bin/python present but != auto result
@@ -128,7 +142,8 @@
         - name: ensure modules can't set discovered_interpreter_X or ansible_X_interpreter
           block:
           - test_echo_module:
-              facts:
+              facts_copy: "{{ ansible_facts }}"
+              facts_to_override:
                 ansible_discovered_interpreter_bogus: from module
                 discovered_interpreter_bogus: from_module
                 ansible_bogus_interpreter: from_module
@@ -189,13 +204,6 @@
               - distro == 'ubuntu'
               - distro_version is version('16.04', '>=', strict=True)
 
-          - name: mac assertions
-            assert:
-              that:
-              - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python'
-              fail_msg: auto_out={{auto_out}}
-            when: os_family == 'Darwin'
-
           always:
           - meta: clear_facts
       when:

tests/ansible/integration/interpreter_discovery/complex_args.yml

@@ -4,6 +4,10 @@
 - name: integration/interpreter_discovery/complex_args.yml
   hosts: test-targets
   gather_facts: true
+  environment:
+      http_proxy: "{{ lookup('env', 'http_proxy') | default(omit) }}"
+      https_proxy: "{{ lookup('env', 'https_proxy') | default(omit) }}"
+      no_proxy: "{{ lookup('env', 'no_proxy') | default(omit) }}"
   tasks:
     - name: create temp file to source
       file:
@@ -21,28 +25,24 @@
     #     special_python: source /tmp/fake && python
     - name: set python using sourced file
       set_fact:
-        special_python: source /tmp/fake || true && python
+        # Avoid 2.x vs 3.x cross-compatiblity issues (that I can't remember the exact details of).
+        special_python: "source /tmp/fake || true && python{{ ansible_facts.python.version.major }}"
 
     - name: run get_url with specially-sourced python
       get_url:
-        url: https://google.com
+        # Plain http for wider Ansible & Python version compatibility
+        url: http://httpbin.org/get
         dest: "/tmp/"
         mode: 0644
-        # this url is the build pic from mitogen's github site; some python versions require ssl stuff installed so will disable need to validate certs
-        validate_certs: no
       vars:
         ansible_python_interpreter: "{{ special_python }}"
-      environment:
-        https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}"
-        no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}"
 
     - name: run get_url with specially-sourced python including jinja
       get_url:
-        url: https://google.com
+        # Plain http for wider Ansible & Python version compatibility
+        url: http://httpbin.org/get
         dest: "/tmp/"
         mode: 0644
-        # this url is the build pic from mitogen's github site; some python versions require ssl stuff installed so will disable need to validate certs
-        validate_certs: no
       vars:
         ansible_python_interpreter: >
           {% if "1" == "1" %}
@@ -50,8 +50,5 @@
           {% else %}
             python 
           {% endif %}
-      environment:
-        https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}"
-        no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}"
   tags:
     - complex_args

tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml

@@ -2,6 +2,11 @@
 - name: integration/runner/custom_python_new_style_module.yml
   hosts: test-targets
   tasks:
+    # FIXME Without Mitogen Ansible often reads stdin before the module.
+    #       Either don't read directly from stdin, or figure out the cause.
+    - meta: end_play
+      when: not is_mitogen
+
     - custom_python_new_style_missing_interpreter:
         foo: true
       with_sequence: start=0 end={{end|default(1)}}

tests/ansible/integration/runner/custom_python_new_style_module.yml

@@ -1,7 +1,8 @@
 - name: integration/runner/custom_python_new_style_module.yml
   hosts: test-targets
   tasks:
-    # without Mitogen Ansible 2.10 hangs on this play
+    # FIXME Without Mitogen Ansible often reads stdin before the module.
+    #       Either don't read directly from stdin, or figure out the cause.
     - meta: end_play
       when: not is_mitogen
 

tests/ansible/integration/runner/custom_python_prehistoric_module.yml

@@ -1,3 +1,7 @@
+# Test functionality of ansible_mitogen.runner.PREHISTORIC_HACK_RE, which
+# removes `reload(sys); sys.setdefaultencoding(...)` from an Ansible module
+# as it is sent to a target. There are probably very few modules in the wild
+# that still do this, if any - reload() is a Python 2.x builtin function.
 # issue #555
 
 - name: integration/runner/custom_python_prehistoric_module.yml
@@ -5,9 +9,11 @@
   tasks:
     - custom_python_prehistoric_module:
       register: out
+      when: is_mitogen
 
     - assert:
         that: out.ok
         fail_msg: out={{out}}
+      when: is_mitogen
   tags:
     - custom_python_prehistoric_module

tests/ansible/lib/modules/custom_python_detect_environment.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
 # I am an Ansible new-style Python module. I return details about the Python
 # interpreter I run within.
 
@@ -25,6 +25,11 @@ except NameError:
 def main():
     module = AnsibleModule(argument_spec={})
     module.exit_json(
+        fs={
+            '/tmp': {
+                'resolved': os.path.realpath('/tmp'),
+            },
+        },
         python={
             'version': {
                 'full': '%i.%i.%i' % sys.version_info[:3],

tests/ansible/lib/modules/custom_python_external_module.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
 # I expect the quote from modules2/module_utils/joker.py.
 
 from ansible.module_utils.basic import AnsibleModule

tests/ansible/lib/modules/custom_python_external_pkg.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.externalpkg import extmod

tests/ansible/lib/modules/custom_python_json_args_module.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
 # I am an Ansible Python JSONARGS module. I should receive an encoding string.
 
 json_arguments = """<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>"""

tests/ansible/lib/modules/custom_python_leaky_class_vars.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
 # I am an Ansible new-style Python module. I leak state from each invocation
 # into a class variable and a global variable.
 

tests/ansible/lib/modules/custom_python_modify_environ.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
 # I am an Ansible new-style Python module. I modify the process environment and
 # don't clean up after myself.
 

tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py

@@ -1,6 +1,20 @@
 # I am an Ansible new-style Python module, but I lack an interpreter.
+# See also custom_python_new_style_module, we should be updated in tandem.
 
+import io
+import json
+import select
+import signal
 import sys
+import warnings
+
+# Ansible 2.7 changed how new style modules are invoked. It seems that module
+# parameters are *sometimes* read before the module runs. Modules that try
+# to read directly from stdin, such as this, are unable to. However it doesn't
+# always fail, influences seem to include Ansible & Python version. As noted
+# in ansible.module_utils.basic._load_params() we should probably use that.
+# I think (medium confidence) I narrowed the inflection (with git bisect) to
+# https://github.com/ansible/ansible/commit/52449cc01a71778ef94ea0237eed0284f5d75582
 
 # As of Ansible 2.10, Ansible changed new-style detection: # https://github.com/ansible/ansible/pull/61196/files#diff-5675e463b6ce1fbe274e5e7453f83cd71e61091ea211513c93e7c0b4d527d637L828-R980
 # NOTE: this import works for Mitogen, and the import below matches new-style Ansible 2.10
@@ -8,11 +22,46 @@ import sys
 # from ansible.module_utils.
 # import ansible.module_utils.
 
+# These timeouts should prevent hard-to-attribute, 2+ hour CI job timeouts.
+# Previously this module has waited on stdin forever (timeoutInMinutes=120).
+SELECT_TIMEOUT = 5.0    # seconds
+SIGNAL_TIMEOUT = 10     # seconds
+
+
+def fail_json(msg, **kwargs):
+    kwargs.update(failed=True, msg=msg)
+    print(json.dumps(kwargs, sys.stdout, indent=2, sort_keys=True))
+    sys.exit(1)
+
+
+def sigalrm_handler(signum, frame):
+    fail_json("Still executing after SIGNAL_TIMEOUT=%ds" % (SIGNAL_TIMEOUT,))
+
 
 def usage():
     sys.stderr.write('Usage: %s <input.json>\n' % (sys.argv[0],))
     sys.exit(1)
 
+
+# Wait SIGNAL_TIMEOUT seconds, exit with failure if still running.
+signal.signal(signal.SIGALRM, sigalrm_handler)
+signal.alarm(SIGNAL_TIMEOUT)
+
+# Wait SELECT_TIMEOUT seconds, exit with failure if no data appears on stdin.
+# TODO Combine select() & read() in a loop, to handle slow trickle of data.
+#      Consider buffering, line buffering, `f.read()` vs `f.read1()`.
+# TODO Document that sys.stdin may be a StringIO under Ansible + Mitogen.
+try:
+    inputs_ready, _, _ = select.select([sys.stdin], [], [], SELECT_TIMEOUT)
+except (AttributeError, TypeError, io.UnsupportedOperation) as exc:
+    # sys.stdin.fileno() doesn't exist or can't return a real file descriptor.
+    warnings.warn("Could not wait on sys.stdin=%r: %r" % (sys.stdin, exc))
+else:
+    if not inputs_ready:
+        fail_json("Gave up waiting on sys.stdin after SELECT_TIMEOUT=%ds"
+                  % (SELECT_TIMEOUT,))
+
+# Read all data on stdin. May block forever, if EOF is not reached.
 input_json = sys.stdin.read()
 
 print("{")

tests/ansible/lib/modules/custom_python_new_style_module.py

@@ -1,16 +1,65 @@
-#!/usr/bin/env python
+#!/usr/bin/python
 # I am an Ansible new-style Python module. I should receive an encoding string.
+# See also custom_python_new_style_module, we should be updated in tandem.
 
+import io
+import json
+import select
+import signal
 import sys
+import warnings
+
+# Ansible 2.7 changed how new style modules are invoked. It seems that module
+# parameters are *sometimes* read before the module runs. Modules that try
+# to read directly from stdin, such as this, are unable to. However it doesn't
+# always fail, influences seem to include Ansible & Python version. As noted
+# in ansible.module_utils.basic._load_params() we should probably use that.
+# I think (medium confidence) I narrowed the inflection (with git bisect) to
+# https://github.com/ansible/ansible/commit/52449cc01a71778ef94ea0237eed0284f5d75582
 
 # This is the magic marker Ansible looks for:
 # from ansible.module_utils.
 
+# These timeouts should prevent hard-to-attribute, 2+ hour CI job timeouts.
+# Previously this module has waited on stdin forever (timeoutInMinutes=120).
+SELECT_TIMEOUT = 5.0    # seconds
+SIGNAL_TIMEOUT = 10     # seconds
+
+
+def fail_json(msg, **kwargs):
+    kwargs.update(failed=True, msg=msg)
+    print(json.dumps(kwargs, sys.stdout, indent=2, sort_keys=True))
+    sys.exit(1)
+
+
+def sigalrm_handler(signum, frame):
+    fail_json("Still executing after SIGNAL_TIMEOUT=%ds" % (SIGNAL_TIMEOUT,))
+
 
 def usage():
     sys.stderr.write('Usage: %s <input.json>\n' % (sys.argv[0],))
     sys.exit(1)
 
+
+# Wait SIGNAL_TIMEOUT seconds, exit with failure if still running.
+signal.signal(signal.SIGALRM, sigalrm_handler)
+signal.alarm(SIGNAL_TIMEOUT)
+
+# Wait SELECT_TIMEOUT seconds, exit with failure if no data appears on stdin.
+# TODO Combine select() & read() in a loop, to handle slow trickle of data.
+#      Consider buffering, line buffering, `f.read()` vs `f.read1()`.
+# TODO Document that sys.stdin may be a StringIO under Ansible + Mitogen.
+try:
+    inputs_ready, _, _ = select.select([sys.stdin], [], [], SELECT_TIMEOUT)
+except (AttributeError, TypeError, io.UnsupportedOperation) as exc:
+    # sys.stdin.fileno() doesn't exist or can't return a real file descriptor.
+    warnings.warn("Could not wait on sys.stdin=%r: %r" % (sys.stdin, exc))
+else:
+    if not inputs_ready:
+        fail_json("Gave up waiting on sys.stdin after SELECT_TIMEOUT=%ds"
+                  % (SELECT_TIMEOUT,))
+
+# Read all data on stdin. May block forever, if EOF is not reached.
 input_json = sys.stdin.read()
 
 print("{")

tests/ansible/lib/modules/custom_python_os_getcwd.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
 # #591: call os.getcwd() before AnsibleModule ever gets a chance to fix up the
 # process environment.
 

tests/ansible/lib/modules/custom_python_prehistoric_module.py

@@ -1,4 +1,9 @@
-#!/usr/bin/env python
+#!/usr/bin/python
+
+# Test functionality of ansible_mitogen.runner.PREHISTORIC_HACK_RE, which
+# removes `reload(sys); sys.setdefaultencoding(...)` from an Ansible module
+# as it is sent to a target. There are probably very few modules in the wild
+# that still do this, reload() is a Python 2.x builtin function.
 # issue #555: I'm a module that cutpastes an old hack.
 
 from ansible.module_utils.basic import AnsibleModule

tests/ansible/lib/modules/custom_python_run_script.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
 # I am an Ansible new-style Python module. I run the script provided in the
 # parameter.
 

tests/ansible/lib/modules/custom_python_uses_distro.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
 # issue #590: I am an Ansible new-style Python module that tries to use
 # ansible.module_utils.distro.
 

tests/ansible/lib/modules/custom_python_want_json_module.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
 # I am an Ansible Python WANT_JSON module. I should receive a JSON-encoded file.
 
 import json

tests/ansible/lib/modules/mitogen_test_gethostbyname.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
 
 # I am a module that indirectly depends on glibc cached /etc/resolv.conf state.
 

tests/ansible/lib/modules/module_finder_test.py

@@ -0,0 +1,12 @@
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+import ansible.module_utils.external1
+
+from ansible.module_utils.externalpkg.extmod import path as epem_path
+
+def main():
+    pass

tests/ansible/lib/modules/test_echo_module.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
 # -*- coding: utf-8 -*-
 
 # (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
@@ -9,28 +9,61 @@
 from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
+import os
 import platform
 import sys
 from ansible.module_utils.basic import AnsibleModule
 
 
 def main():
-    result = dict(changed=False)
-
     module = AnsibleModule(argument_spec=dict(
-        facts=dict(type=dict, default={})
+        facts_copy=dict(type=dict, default={}),
+        facts_to_override=dict(type=dict, default={})
     ))
 
-    result['ansible_facts'] = module.params['facts']
     # revert the Mitogen OSX tweak since discover_interpreter() doesn't return this info
-    if sys.platform == 'darwin' and sys.executable != '/usr/bin/python':
+    # NB This must be synced with mitogen.parent.Connection.get_boot_command()
-        if int(platform.release()[:2]) < 19:
+    platform_release_major = int(platform.release().partition('.')[0])
-            sys.executable = sys.executable[:-3]
+    if sys.modules.get('mitogen') and sys.platform == 'darwin':
-        else:
+        if platform_release_major < 19 and sys.executable == '/usr/bin/python2.7':
+            sys.executable = '/usr/bin/python'
+        if platform_release_major in (20, 21) and sys.version_info[:2] == (2, 7):
             # only for tests to check version of running interpreter -- Mac 10.15+ changed python2
             # so it looks like it's /usr/bin/python but actually it's /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python
             sys.executable = "/usr/bin/python"
-    result['running_python_interpreter'] = sys.executable
+
+    facts_copy = module.params['facts_copy']
+    discovered_interpreter_python = facts_copy['discovered_interpreter_python']
+    result = {
+        'changed': False,
+        'ansible_facts': module.params['facts_to_override'],
+        'discovered_and_running_samefile': os.path.samefile(
+            os.path.realpath(discovered_interpreter_python),
+            os.path.realpath(sys.executable),
+        ),
+        'discovered_python': {
+            'as_seen': discovered_interpreter_python,
+            'resolved': os.path.realpath(discovered_interpreter_python),
+        },
+        'running_python': {
+            'platform': {
+                'release': {
+                    'major': platform_release_major,
+                },
+            },
+            'sys': {
+                'executable': {
+                    'as_seen': sys.executable,
+                    'resolved': os.path.realpath(sys.executable),
+                },
+                'platform': sys.platform,
+                'version_info': {
+                    'major': sys.version_info[0],
+                    'minor': sys.version_info[1],
+                },
+            },
+        },
+    }
 
     module.exit_json(**result)
 

tests/ansible/regression/issue_152__virtualenv_python_fails.yml

@@ -23,9 +23,16 @@
       when:
         - lout.python.version.full is version('2.7', '>=', strict=True)
 
-    - assert:
+    - name: Check virtualenv was used
+      # On macOS runners a symlink /tmp -> /private/tmp has been seen
+      vars:
+        requested_executable: /tmp/issue_152_virtualenv/bin/python
+        expected_executables:
+          - "{{ requested_executable }}"
+          - "{{ requested_executable.replace('/tmp', out.fs['/tmp'].resolved) }}"
+      assert:
         that:
-          - out.sys_executable == "/tmp/issue_152_virtualenv/bin/python"
+          - out.sys_executable in expected_executables
         fail_msg: out={{out}}
       when:
         - lout.python.version.full is version('2.7', '>=', strict=True)

tests/ansible/setup/all.yml

@@ -1 +1,2 @@
-- import_playbook: report.yml
+- import_playbook: report_controller.yml
+- import_playbook: report_targets.yml

tests/ansible/setup/report_controller.yml

@@ -0,0 +1,17 @@
+- name: Report controller parameters
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - debug:
+        msg:
+          - ${ANSIBLE_STRATEGY}:      "{{ lookup('env', 'ANSIBLE_STRATEGY') | default('<unset>') }}"
+          - ${USER}:                  "{{ lookup('env', 'USER') | default('<unset>') }}"
+          - $(groups):                "{{ lookup('pipe', 'groups') }}"
+          - $(pwd):                   "{{ lookup('pipe', 'pwd') }}"
+          - $(whoami):                "{{ lookup('pipe', 'whoami') }}"
+          - ansible_run_tags:         "{{ ansible_run_tags | default('<unset>') }}"
+          - ansible_playbook_python:  "{{ ansible_playbook_python | default('<unset>') }}"
+          - ansible_skip_tags:        "{{ ansible_skip_tags | default('<unset>') }}"
+          - ansible_version.full:     "{{ ansible_version.full | default('<unset>') }}"
+          - is_mitogen:               "{{ is_mitogen | default('<unset>') }}"
+          - playbook_dir:             "{{ playbook_dir | default('<unset>') }}"

tests/ansible/setup/report_targets.yml

@@ -1,4 +1,4 @@
-- name: Report runtime settings
+- name: Report target facts
   hosts: localhost:test-targets
   gather_facts: true
   tasks:
@@ -13,8 +13,3 @@
     - debug: {var: ansible_facts.osversion}
     - debug: {var: ansible_facts.python}
     - debug: {var: ansible_facts.system}
-    - debug: {var: ansible_forks}
-    - debug: {var: ansible_run_tags}
-    - debug: {var: ansible_skip_tags}
-    - debug: {var: ansible_version.full}
-    - debug: {var: is_mitogen}

tests/ansible/tests/module_finder_test.py

@@ -0,0 +1,80 @@
+import os.path
+import sys
+import textwrap
+import unittest
+
+import ansible_mitogen.module_finder
+
+import testlib
+
+
+class ScanFromListTest(testlib.TestCase):
+    def test_absolute_imports(self):
+        source = textwrap.dedent('''\
+            from __future__ import absolute_import
+            import a; import b.c; from d.e import f; from g import h, i
+        ''')
+        code = compile(source, '<str>', 'exec')
+        self.assertEqual(
+            list(ansible_mitogen.module_finder.scan_fromlist(code)),
+            [(0, '__future__.absolute_import'), (0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')],
+        )
+
+
+class WalkImportsTest(testlib.TestCase):
+    def test_absolute_imports(self):
+        source = textwrap.dedent('''\
+            from __future__ import absolute_import
+            import a; import b; import b.c; from b.d import e, f
+        ''')
+        code = compile(source, '<str>', 'exec')
+
+        self.assertEqual(
+            list(ansible_mitogen.module_finder.walk_imports(code)),
+            ['__future__', '__future__.absolute_import', 'a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f'],
+        )
+        self.assertEqual(
+            list(ansible_mitogen.module_finder.walk_imports(code, prefix='b')),
+            ['b.c', 'b.d', 'b.d.e', 'b.d.f'],
+        )
+
+
+class ScanTest(testlib.TestCase):
+    module_name = 'ansible_module_module_finder_test__this_should_not_matter'
+    module_path = os.path.join(testlib.ANSIBLE_MODULES_DIR, 'module_finder_test.py')
+    search_path = (
+        'does_not_exist/module_utils',
+        testlib.ANSIBLE_MODULE_UTILS_DIR,
+    )
+
+    @staticmethod
+    def relpath(path):
+        return os.path.relpath(path, testlib.ANSIBLE_MODULE_UTILS_DIR)
+
+    @unittest.skipIf(sys.version_info < (3, 4), 'find spec() unavailable')
+    def test_importlib_find_spec(self):
+        scan = ansible_mitogen.module_finder._scan_importlib_find_spec
+        actual = scan(self.module_name, self.module_path, self.search_path)
+        self.assertEqual(
+            [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual],
+            [
+                ('ansible.module_utils.external1', 'external1.py', False),
+                ('ansible.module_utils.external2', 'external2.py', False),
+                ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True),
+                ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False),
+            ],
+        )
+
+    @unittest.skipIf(sys.version_info >= (3, 4), 'find spec() preferred')
+    def test_imp_find_module(self):
+        scan = ansible_mitogen.module_finder._scan_imp_find_module
+        actual = scan(self.module_name, self.module_path, self.search_path)
+        self.assertEqual(
+            [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual],
+            [
+                ('ansible.module_utils.external1', 'external1.py', False),
+                ('ansible.module_utils.external2', 'external2.py', False),
+                ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True),
+                ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False),
+            ],
+        )

tests/image_prep/_user_accounts.yml

@@ -4,6 +4,8 @@
 # WARNING: this creates non-privilged accounts with pre-set passwords!
 #
 
+- import_playbook: ../ansible/setup/report_controller.yml
+
 - hosts: all
   gather_facts: true
   strategy: mitogen_free
@@ -37,12 +39,12 @@
 
     normal_users: "{{
       lookup('sequence', 'start=1 end=5 format=user%d', wantlist=True)
-    }}"
+      }}"
 
     all_users: "{{
       special_users +
       normal_users
-    }}"
+      }}"
   tasks:
     - name: Disable non-localhost SSH for Mitogen users
       when: false
@@ -100,6 +102,7 @@
       with_items: "{{all_users}}"
       copy:
         dest: /var/lib/AccountsService/users/mitogen__{{item}}
+        mode: u=rw,go=
         content: |
           [User]
           SystemAccount=true
@@ -108,7 +111,7 @@
       when: ansible_system == 'Linux' and out.stat.exists
       service:
         name: accounts-daemon
-        restarted: true
+        state: restarted
 
     - name: Readonly homedir for one account
       shell: "chown -R root: ~mitogen__readonly_homedir"
@@ -117,6 +120,9 @@
       copy:
         dest: ~mitogen__slow_user/.{{item}}
         src: ../data/docker/mitogen__slow_user.profile
+        owner: mitogen__slow_user
+        group: mitogen__group
+        mode: u=rw,go=r
       with_items:
       - bashrc
       - profile
@@ -125,6 +131,9 @@
       copy:
         dest: ~mitogen__permdenied/.{{item}}
         src: ../data/docker/mitogen__permdenied.profile
+        owner: mitogen__permdenied
+        group: mitogen__group
+        mode: u=rw,go=r
       with_items:
       - bashrc
       - profile
@@ -136,20 +145,13 @@
             state: directory
             mode: go=
             owner: mitogen__has_sudo_pubkey
+            group: mitogen__group
         - copy:
             dest: ~mitogen__has_sudo_pubkey/.ssh/authorized_keys
             src: ../data/docker/mitogen__has_sudo_pubkey.key.pub
             mode: go=
             owner: mitogen__has_sudo_pubkey
-
+            group: mitogen__group
-    - name: Install slow profile for one account
-      block:
-        - copy:
-            dest: ~mitogen__slow_user/.profile
-            src: ../data/docker/mitogen__slow_user.profile
-        - copy:
-            dest: ~mitogen__slow_user/.bashrc
-            src: ../data/docker/mitogen__slow_user.profile
 
     - name: Require a TTY for two accounts
       lineinfile:

tests/importer_test.py

@@ -2,6 +2,7 @@ import sys
 import threading
 import types
 import zlib
+import unittest
 
 import mock
 
@@ -42,6 +43,49 @@ class ImporterMixin(testlib.RouterMixin):
         super(ImporterMixin, self).tearDown()
 
 
+class InvalidNameTest(ImporterMixin, testlib.TestCase):
+    modname = 'trailingdot.'
+    # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+    response = (modname, None, None, None, None)
+
+    @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+    def test_find_spec_invalid(self):
+        self.set_get_module_response(self.response)
+        self.assertEqual(self.importer.find_spec(self.modname, path=None), None)
+
+
+class MissingModuleTest(ImporterMixin, testlib.TestCase):
+    modname = 'missing'
+    # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+    response = (modname, None, None, None, None)
+
+    @unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+')
+    def test_load_module_missing(self):
+        self.set_get_module_response(self.response)
+        self.assertRaises(ImportError, self.importer.load_module, self.modname)
+
+    @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+    def test_find_spec_missing(self):
+        """
+        Importer should optimistically offer itself as a module loader
+        when there are no disqualifying criteria.
+        """
+        import importlib.machinery
+        self.set_get_module_response(self.response)
+        spec = self.importer.find_spec(self.modname, path=None)
+        self.assertIsInstance(spec, importlib.machinery.ModuleSpec)
+        self.assertEqual(spec.name, self.modname)
+        self.assertEqual(spec.loader, self.importer)
+
+    @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+    def test_create_module_missing(self):
+        import importlib.machinery
+        self.set_get_module_response(self.response)
+        spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+        self.assertRaises(ImportError, self.importer.create_module, spec)
+
+
+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+')
 class LoadModuleTest(ImporterMixin, testlib.TestCase):
     data = zlib.compress(b("data = 1\n\n"))
     path = 'fake_module.py'
@@ -50,14 +94,6 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase):
     # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
     response = (modname, None, path, data, [])
 
-    def test_no_such_module(self):
-        self.set_get_module_response(
-            # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
-            (self.modname, None, None, None, None)
-        )
-        self.assertRaises(ImportError,
-            lambda: self.importer.load_module(self.modname))
-
     def test_module_added_to_sys_modules(self):
         self.set_get_module_response(self.response)
         mod = self.importer.load_module(self.modname)
@@ -80,6 +116,26 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase):
         self.assertIsNone(mod.__package__)
 
 
+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+class ModuleSpecTest(ImporterMixin, testlib.TestCase):
+    data = zlib.compress(b("data = 1\n\n"))
+    path = 'fake_module.py'
+    modname = 'fake_module'
+
+    # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+    response = (modname, None, path, data, [])
+
+    def test_module_attributes(self):
+        import importlib.machinery
+        self.set_get_module_response(self.response)
+        spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+        mod = self.importer.create_module(spec)
+        self.assertIsInstance(mod, types.ModuleType)
+        self.assertEqual(mod.__name__, 'fake_module')
+        #self.assertFalse(hasattr(mod, '__file__'))
+
+
+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+')
 class LoadSubmoduleTest(ImporterMixin, testlib.TestCase):
     data = zlib.compress(b("data = 1\n\n"))
     path = 'fake_module.py'
@@ -93,6 +149,25 @@ class LoadSubmoduleTest(ImporterMixin, testlib.TestCase):
         self.assertEqual(mod.__package__, 'mypkg')
 
 
+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+class SubmoduleSpecTest(ImporterMixin, testlib.TestCase):
+    data = zlib.compress(b("data = 1\n\n"))
+    path = 'fake_module.py'
+    modname = 'mypkg.fake_module'
+    # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+    response = (modname, None, path, data, [])
+
+    def test_module_attributes(self):
+        import importlib.machinery
+        self.set_get_module_response(self.response)
+        spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+        mod = self.importer.create_module(spec)
+        self.assertIsInstance(mod, types.ModuleType)
+        self.assertEqual(mod.__name__, 'mypkg.fake_module')
+        #self.assertFalse(hasattr(mod, '__file__'))
+
+
+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+')
 class LoadModulePackageTest(ImporterMixin, testlib.TestCase):
     data = zlib.compress(b("func = lambda: 1\n\n"))
     path = 'fake_pkg/__init__.py'
@@ -140,6 +215,41 @@ class LoadModulePackageTest(ImporterMixin, testlib.TestCase):
         self.assertEqual(mod.func.__module__, self.modname)
 
 
+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+class PackageSpecTest(ImporterMixin, testlib.TestCase):
+    data = zlib.compress(b("func = lambda: 1\n\n"))
+    path = 'fake_pkg/__init__.py'
+    modname = 'fake_pkg'
+    # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+    response = (modname, [], path, data, [])
+
+    def test_module_attributes(self):
+        import importlib.machinery
+        self.set_get_module_response(self.response)
+        spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+        mod = self.importer.create_module(spec)
+        self.assertIsInstance(mod, types.ModuleType)
+        self.assertEqual(mod.__name__, 'fake_pkg')
+        #self.assertFalse(hasattr(mod, '__file__'))
+
+    def test_get_filename(self):
+        import importlib.machinery
+        self.set_get_module_response(self.response)
+        spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+        _ = self.importer.create_module(spec)
+        filename = self.importer.get_filename(self.modname)
+        self.assertEqual('master:fake_pkg/__init__.py', filename)
+
+    def test_get_source(self):
+        import importlib.machinery
+        self.set_get_module_response(self.response)
+        spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+        _ = self.importer.create_module(spec)
+        source = self.importer.get_source(self.modname)
+        self.assertEqual(source,
+            mitogen.core.to_text(zlib.decompress(self.data)))
+
+
 class EmailParseAddrSysTest(testlib.RouterMixin, testlib.TestCase):
     def initdir(self, caplog):
         self.caplog = caplog

tests/module_finder_test.py

@@ -140,9 +140,7 @@ class SysModulesMethodTest(testlib.TestCase):
         self.assertIsNone(tup)
 
 
-class GetModuleViaParentEnumerationTest(testlib.TestCase):
+class ParentEnumerationMixin(object):
-    klass = mitogen.master.ParentEnumerationMethod
-
     def call(self, fullname):
         return self.klass().find(fullname)
 
@@ -232,6 +230,16 @@ class GetModuleViaParentEnumerationTest(testlib.TestCase):
         self.assertEqual(is_pkg, False)
 
 
+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python >= 3.4')
+class ParentImpEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase):
+    klass = mitogen.master.ParentImpEnumerationMethod
+
+
+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+class ParentSpecEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase):
+    klass = mitogen.master.ParentSpecEnumerationMethod
+
+
 class ResolveRelPathTest(testlib.TestCase):
     klass = mitogen.master.ModuleFinder
 

tests/requirements-tox.txt

@@ -0,0 +1,4 @@
+tox==3.28;                  python_version == '2.7'
+tox==3.28;                  python_version == '3.6'
+tox==4.8.0;                 python_version == '3.7'
+tox>=4.13.0,~=4.0;          python_version >= '3.8'

tests/requirements.txt

@@ -1,12 +1,24 @@
-cffi==1.15.1
+cffi==1.15.1;               python_version < '3.8'
-coverage==5.5;      python_version < '3.7'
+cffi==1.16;                 python_version >= '3.8'
-coverage==6.4.4;    python_version >= '3.7'
+
+coverage==5.5;              python_version == '2.7'
+coverage==6.2;              python_version == '3.6'
+coverage==7.2.7;            python_version == '3.7'
+coverage==7.4.3;            python_version >= '3.8'
+
 Django==1.11.29;    python_version < '3.0'
 Django==3.2.20;     python_version >= '3.6'
-mock==2.0.0
+
-psutil==5.9.5
+mock==3.0.5;                python_version == '2.7'
-pytest-catchlog==1.2.2
+mock==5.1.0;                python_version >= '3.6'
-pytest==3.1.2
+
+psutil==5.9.8
+
+pytest==4.6.11;             python_version == '2.7'
+pytest==7.0.1;              python_version == '3.6'
+pytest==7.4.4;              python_version == '3.7'
+pytest==8.0.2;              python_version >= '3.8'
+
 subprocess32==3.5.4; python_version < '3.0'
 timeoutcontext==1.2.0
 # Fix InsecurePlatformWarning while creating py26 tox environment
@@ -15,4 +27,7 @@ urllib3[secure]==1.23; python_version < '2.7'
 urllib3[secure]==1.26; python_version > '2.6' and python_version < '2.7.9'
 # Last idna compatible with Python 2.6 was idna 2.7.
 idna==2.7; python_version < '2.7'
-virtualenv==20.10.0
+
+virtualenv==20.15.1;        python_version == '2.7'
+virtualenv==20.17.1;        python_version == '3.6'
+virtualenv==20.25.1;        python_version >= '3.7'

tests/testlib.py

@@ -50,8 +50,13 @@ except NameError:
 
 
 LOG = logging.getLogger(__name__)
-DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
+
-MODS_DIR = os.path.join(DATA_DIR, 'importer')
+TESTS_DIR =                     os.path.join(os.path.dirname(__file__))
+ANSIBLE_LIB_DIR =               os.path.join(TESTS_DIR, 'ansible', 'lib')
+ANSIBLE_MODULE_UTILS_DIR =      os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils')
+ANSIBLE_MODULES_DIR =           os.path.join(TESTS_DIR, 'ansible', 'lib', 'modules')
+DATA_DIR =                      os.path.join(TESTS_DIR, 'data')
+MODS_DIR =                      os.path.join(TESTS_DIR, 'data', 'importer')
 
 sys.path.append(DATA_DIR)
 sys.path.append(MODS_DIR)

tox.ini

@@ -3,18 +3,37 @@
 #
 #   sudo add-apt-repository ppa:deadsnakes/ppa
 #   sudo apt update
-#   sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python2.7 python3.{6..11} python-is-python3 sshpass tox
+#   sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python2.7 python3.{6..13}{,-venv} python-is-python3 sshpass tox
 
 # Py   A cntrllr  A target   coverage   Django     Jinja2     pip        psutil     pytest     tox        virtualenv
 # ==== ========== ========== ========== ========== ========== ========== ========== ========== ========== ==========
 # 2.4             2.3?       <= 3.7.1   <= 1.3.7              <= 1.1     <= 2.1.3              <= 1.4     <= 1.8
 # 2.5                        <= 3.7.1   <= 1.4.22             <= 1.3.1   <= 2.1.3   <= 2.8.7   <= 1.6.1   <= 1.9.1
-# 2.6  <= 2.6.20  <= 2.13    <= 4.5.4   <= 1.6.11  <= 2.10.3  <= 9.0.3   <= 5.9.0   <= 3.2.5   <= 2.9.1   <= 15.2.0
+# 2.6  <= 2.6.20  <= 2.12    <= 4.5.4   <= 1.6.11  <= 2.10.3  <= 9.0.3   <= 5.9.0   <= 3.2.5   <= 2.9.1   <= 15.2.0
-# 2.7  <= 2.11               <= 5.6     <= 1.11.29 <= 2.11.3  <= 20                 <= 4.6.11  <= 3.28    <= 20.3?
+# 2.7  <= 2.11               <= 5.5     <= 1.11.29 <= 2.11.3  <= 20                 <= 4.6.11  <= 3.28    <= 20.15²
-# 3.5  <= 2.11    <= 2.13    <= 5.6     <= 2.2.28  <= 2.11.3  <= 20      <= 5.9.5   <= 6.1.0   <= 3.28    <= 20.15
+# 3.5  <= 2.11    <= 2.13    <= 5.5     <= 2.2.28  <= 2.11.3  <= 20      <= 5.9.5   <= 6.1.0   <= 3.28    <= 20.15²
-# 3.6  <= 2.11               <= 6.2     <= 3.2.20  <= 3.0.3   <= 21      <= 5.9.5   <= 7.0.1   <= 3.28    <= 20.16
+# 3.6  <= 2.11               <= 6.2     <= 3.2.20  <= 3.0.3   <= 21                 <= 7.0.1   <= 3.28    <= 20.17²
-# 3.7  <= 2.12                          <= 3.2.20
+# 3.7  <= 2.12               <= 7.2.7   <= 3.2.20                                   <= 7.4.4   <= 4.8.0
 # 3.8  <= 2.12
+# 3.9  <= 2.15
+# 3.10
+# 3.11
+# 3.12            >= 2.13¹
+#
+# Notes
+# 1. Python 3.12 on a target requires Ansible >= 6 (ansible-core >= 2.13).
+#    Python 3.12 removed support for find_module(), replaced by find_spec().
+#    In Ansible <= 4.x ansible.module_utils.six lacks find_spec().
+#    https://github.com/ansible/ansible/commit/d6e28e68599e703c153914610152cf4492851eb3
+#    In Ansible <= 5.x ansible.utils.collection_loader._AnsibleCollectionFinder
+#    lacks find_spec(). https://github.com/ansible/ansible/pull/76225
+#
+#    Python 3.12 + get_uri requires Ansible >= 8 (ansible-core >= 2.15).
+#    Python 3.12 removed deprecated httplib.HTTPSConnection() arguments.
+#    https://github.com/ansible/ansible/pull/80751
+#
+# 2. Higher virtualenv versions cannot run under this Python version. They can
+#    still generate virtual environments for it.
 
 # Ansible            Dependency
 # ================== ======================
@@ -24,6 +43,9 @@
 # ansible == 4.*     ansible-core ~= 2.11.0
 # ansible == 5.*     ansible-core ~= 2.12.0
 # ansible == 6.*     ansible-core ~= 2.13.0
+# ansible == 7.x     ansible-core ~= 2.14.0
+# ansible == 8.x     ansible-core ~= 2.15.0
+# ansible == 9.x     ansible-core ~= 2.16.0
 
 # pip --no-python-version-warning
 # pip --disable-pip-version-check
@@ -34,10 +56,11 @@
 envlist =
     init,
     py{27,36}-mode_ansible-ansible{2.10,3,4},
-    py{311}-mode_ansible-ansible{2.10,3,4,5,6},
+    py{311}-mode_ansible-ansible{2.10,3,4,5},
-    py{27,36,311}-mode_mitogen-distro_centos{6,7,8},
+    py{312}-mode_ansible-ansible{6},
-    py{27,36,311}-mode_mitogen-distro_debian{9,10,11},
+    py{27,36,312}-mode_mitogen-distro_centos{6,7,8},
-    py{27,36,311}-mode_mitogen-distro_ubuntu{1604,1804,2004},
+    py{27,36,312}-mode_mitogen-distro_debian{9,10,11},
+    py{27,36,312}-mode_mitogen-distro_ubuntu{1604,1804,2004},
     report,
 
 [testenv]
@@ -51,14 +74,15 @@ basepython =
     py39: python3.9
     py310: python3.10
     py311: python3.11
+    py312: python3.12
 deps =
     -r{toxinidir}/tests/requirements.txt
     mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt
     ansible2.10: ansible==2.10.7
     ansible3: ansible==3.4.0
     ansible4: ansible==4.10.0
-    ansible5: ansible==5.8.0
+    ansible5: ansible~=5.0
-    ansible6: ansible==6.0.0
+    ansible6: ansible~=6.0
 install_command =
     python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages}
 commands_pre =
@@ -79,7 +103,6 @@ passenv =
     HOME
 setenv =
     # See also azure-pipelines.yml
-    ANSIBLE_SKIP_TAGS = requires_local_sudo,resource_intensive
     ANSIBLE_STRATEGY = mitogen_linear
     NOCOVERAGE_ERASE = 1
     NOCOVERAGE_REPORT = 1
@@ -111,10 +134,26 @@ setenv =
     distros_ubuntu1804: DISTROS=ubuntu1804
     distros_ubuntu2004: DISTROS=ubuntu2004
     mode_ansible: MODE=ansible
+    mode_ansible: ANSIBLE_SKIP_TAGS=resource_intensive
     mode_debops_common: MODE=debops_common
+    mode_localhost: ANSIBLE_SKIP_TAGS=issue_776,resource_intensive
     mode_mitogen: MODE=mitogen
     strategy_linear: ANSIBLE_STRATEGY=linear
+allowlist_externals =
+    # Added: Tox 3.18: Tox 4.0+
+    *_install.py
+    *_tests.py
+    aws
+    docker
+    docker-credential-secretservice
+    echo
+    gpg2
+    pass
 whitelist_externals =
+    # Deprecated: Tox 3.18+; Removed: Tox 4.0
+    *_install.py
+    *_tests.py
+    aws
     docker
     docker-credential-secretservice
     echo