mitogen: Support PEP 451 ModuleSpec API, required for Python 3.12

importlib.machinery.ModuleSpec and find_spec() were introduced in Python 3.4
under PEP 451. They replace the find_module() API of PEP 302, which was
deprecated from Python 3.4. They were removed in Python 3.12 along with the
imp module.

This change adds support for the PEP 451 APIs. Mitogen should no longer import
imp on Python versions that support ModuleSpec. Tests have been added to cover
the new APIs.

CI jobs have been added to cover Python 3.x on macOS.

Refs #1033
Co-authored-by: Witold Baryluk <witold.baryluk@gmail.com>

.ci/azure-pipelines.yml

@@ -25,21 +25,17 @@ jobs:
         python.version: '3.11'
         tox.env: py311-mode_mitogen
 
-      # TODO: test python3, python3 tests are broken
       Loc_27_210:
         tox.env: py27-mode_localhost-ansible2.10
-      Loc_27_4:
-        tox.env: py27-mode_localhost-ansible4
+      Loc_311_6:
+        python.version: '3.11'
+        tox.env: py311-mode_localhost-ansible6
 
-      # NOTE: this hangs when ran in Ubuntu 18.04
       Van_27_210:
-        tox.env: py27-mode_localhost-ansible2.10
-        STRATEGY: linear
-        ANSIBLE_SKIP_TAGS: resource_intensive
-      Van_27_4:
-        tox.env: py27-mode_localhost-ansible4
-        STRATEGY: linear
-        ANSIBLE_SKIP_TAGS: resource_intensive
+        tox.env: py27-mode_localhost-ansible2.10-strategy_linear
+      Van_311_6:
+        python.version: '3.11'
+        tox.env: py311-mode_localhost-ansible6-strategy_linear
 
 - job: Linux
   pool:

.ci/localhost_ansible_tests.py

@@ -1,6 +1,10 @@
 #!/usr/bin/env python
 # Run tests/ansible/all.yml under Ansible and Ansible-Mitogen
 
+from __future__ import print_function
+
+import getpass
+import io
 import os
 import subprocess
 import sys
@@ -53,6 +57,38 @@ with ci_lib.Fold('machine_prep'):
         os.chdir(IMAGE_PREP_DIR)
         ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml")
 
+    # FIXME Don't hardcode https://github.com/mitogen-hq/mitogen/issues/1022
+    #       and os.environ['USER'] is not populated on Azure macOS runners.
+    os.chdir(HOSTS_DIR)
+    with io.open('default.hosts', 'r+', encoding='utf-8') as f:
+        user = getpass.getuser()
+        content = f.read()
+        content = content.replace("{{ lookup('pipe', 'whoami') }}", user)
+        f.seek(0)
+        f.write(content)
+        f.truncate()
+    ci_lib.dump_file('default.hosts')
+
+    cmd = ';'.join([
+        'from __future__ import print_function',
+        'import os, sys',
+        'print(sys.executable, os.path.realpath(sys.executable))',
+    ])
+    for interpreter in ['/usr/bin/python', '/usr/bin/python2', '/usr/bin/python2.7']:
+        print(interpreter)
+        try:
+            subprocess.call([interpreter, '-c', cmd])
+        except OSError as exc:
+            print(exc)
+
+        print(interpreter, 'with PYTHON_LAUNCHED_FROM_WRAPPER=1')
+        environ = os.environ.copy()
+        environ['PYTHON_LAUNCHED_FROM_WRAPPER'] = '1'
+        try:
+            subprocess.call([interpreter, '-c', cmd], env=environ)
+        except OSError as exc:
+            print(exc)
+
 
 with ci_lib.Fold('ansible'):
     os.chdir(TESTS_DIR)

ansible_mitogen/module_finder.py

@@ -31,12 +31,23 @@ from __future__ import unicode_literals
 __metaclass__ = type
 
 import collections
-import imp
+import logging
 import os
+import re
+import sys
+
+try:
+    # Python >= 3.4, PEP 451 ModuleSpec API
+    import importlib.machinery
+    import importlib.util
+except ImportError:
+    # Python < 3.4, PEP 302 Import Hooks
+    import imp
 
 import mitogen.master
 
 
+LOG = logging.getLogger(__name__)
 PREFIX = 'ansible.module_utils.'
 
 
@@ -45,8 +56,6 @@ PREFIX = 'ansible.module_utils.'
 #   path    Filesystem path of the module.
 #   kind    One of the constants in `imp`, as returned in `imp.find_module()`
 #   parent  `ansible_mitogen.module_finder.Module` of parent package (if any).
-#
-# FIXME Python 3.12 removed `imp`, leaving no constants for `Module.kind`.
 Module = collections.namedtuple('Module', 'name path kind parent')
 
 
@@ -126,14 +135,121 @@ def find_relative(parent, name, path=()):
 
 
 def scan_fromlist(code):
+    """Return an iterator of (level, name) for explicit imports in a code
+    object.
+
+    Not all names identify a module. `from os import name, path` generates
+    `(0, 'os.name'), (0, 'os.path')`, but `os.name` is usually a string.
+
+    >>> src = 'import a; import b.c; from d.e import f; from g import h, i\\n'
+    >>> code = compile(src, '<str>', 'exec')
+    >>> list(scan_fromlist(code))
+    [(0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')]
+    """
     for level, modname_s, fromlist in mitogen.master.scan_code_imports(code):
         for name in fromlist:
-            yield level, '%s.%s' % (modname_s, name)
+            yield level, str('%s.%s' % (modname_s, name))
         if not fromlist:
             yield level, modname_s
 
 
+def walk_imports(code, prefix=None):
+    """Return an iterator of names for implicit parent imports & explicit
+    imports in a code object.
+
+    If a prefix is provided, then only children of that prefix are included.
+    Not all names identify a module. `from os import name, path` generates
+    `'os', 'os.name', 'os.path'`, but `os.name` is usually a string.
+
+    >>> source = 'import a; import b; import b.c; from b.d import e, f\\n'
+    >>> code = compile(source, '<str>', 'exec')
+    >>> list(walk_imports(code))
+    ['a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f']
+    >>> list(walk_imports(code, prefix='b'))
+    ['b.c', 'b.d', 'b.d.e', 'b.d.f']
+    """
+    if prefix is None:
+        prefix = ''
+    pattern = re.compile(r'(^|\.)(\w+)')
+    start = len(prefix)
+    for _, name, fromlist in mitogen.master.scan_code_imports(code):
+        if not name.startswith(prefix):
+            continue
+        for match in pattern.finditer(name, start):
+            yield name[:match.end()]
+        for leaf in fromlist:
+            yield str('%s.%s' % (name, leaf))
+
+
 def scan(module_name, module_path, search_path):
+    # type: (str, str, list[str]) -> list[(str, str, bool)]
+    """Return a list of (name, path, is_package) for ansible.module_utils
+    imports used by an Ansible module.
+    """
+    log = LOG.getChild('scan')
+    log.debug('%r, %r, %r', module_name, module_path, search_path)
+
+    if sys.version_info >= (3, 4):
+        result = _scan_importlib_find_spec(
+            module_name, module_path, search_path,
+        )
+        log.debug('_scan_importlib_find_spec %r', result)
+    else:
+        result = _scan_imp_find_module(module_name, module_path, search_path)
+        log.debug('_scan_imp_find_module %r', result)
+    return result
+
+
+def _scan_importlib_find_spec(module_name, module_path, search_path):
+    # type: (str, str, list[str]) -> list[(str, str, bool)]
+    module = importlib.machinery.ModuleSpec(
+        module_name, loader=None, origin=module_path,
+    )
+    prefix = importlib.machinery.ModuleSpec(
+        PREFIX.rstrip('.'), loader=None,
+    )
+    prefix.submodule_search_locations = search_path
+    queue = collections.deque([module])
+    specs = {prefix.name: prefix}
+    while queue:
+        spec = queue.popleft()
+        if spec.origin is None:
+            continue
+        try:
+            with open(spec.origin, 'rb') as f:
+                code = compile(f.read(), spec.name, 'exec')
+        except Exception as exc:
+            raise ValueError((exc, module, spec, specs))
+
+        for name in walk_imports(code, prefix.name):
+            if name in specs:
+                continue
+
+            parent_name = name.rpartition('.')[0]
+            parent = specs[parent_name]
+            if parent is None or not parent.submodule_search_locations:
+                specs[name] = None
+                continue
+
+            child = importlib.util._find_spec(
+                name, parent.submodule_search_locations,
+            )
+            if child is None or child.origin is None:
+                specs[name] = None
+                continue
+
+            specs[name] = child
+            queue.append(child)
+
+    del specs[prefix.name]
+    return sorted(
+        (spec.name, spec.origin, spec.submodule_search_locations is not None)
+        for spec in specs.values() if spec is not None
+    )
+
+
+def _scan_imp_find_module(module_name, module_path, search_path):
+    # type: (str, str, list[str]) -> list[(str, str, bool)]
     module = Module(module_name, module_path, imp.PY_SOURCE, None)
     stack = [module]
     seen = set()

ansible_mitogen/runner.py

@@ -40,7 +40,6 @@ from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
 import atexit
-import imp
 import json
 import os
 import re
@@ -64,6 +63,14 @@ except ImportError:
     # Python 2.4
     ctypes = None
 
+try:
+    # Python >= 3.4, PEP 451 ModuleSpec API
+    import importlib.machinery
+    import importlib.util
+except ImportError:
+    # Python < 3.4, PEP 302 Import Hooks
+    import imp
+
 try:
     # Cannot use cStringIO as it does not support Unicode.
     from StringIO import StringIO
@@ -514,14 +521,74 @@ class ModuleUtilsImporter(object):
             sys.modules.pop(fullname, None)
 
     def find_module(self, fullname, path=None):
+        """
+        Return a loader for the module with fullname, if we will load it.
+
+        Implements importlib.abc.MetaPathFinder.find_module().
+        Deprecrated in Python 3.4+, replaced by find_spec().
+        Raises ImportWarning in Python 3.10+. Removed in Python 3.12.
+        """
         if fullname in self._by_fullname:
             return self
 
+    def find_spec(self, fullname, path, target=None):
+        """
+        Return a `ModuleSpec` for module with `fullname` if we will load it.
+        Otherwise return `None`.
+
+        Implements importlib.abc.MetaPathFinder.find_spec(). Python 3.4+.
+        """
+        if fullname.endswith('.'):
+            return None
+
+        try:
+            module_path, is_package = self._by_fullname[fullname]
+        except KeyError:
+            LOG.debug('Skipping %s: not present', fullname)
+            return None
+
+        LOG.debug('Handling %s', fullname)
+        origin = 'master:%s' % (module_path,)
+        return importlib.machinery.ModuleSpec(
+            fullname, loader=self, origin=origin, is_package=is_package,
+        )
+
+    def create_module(self, spec):
+        """
+        Return a module object for the given ModuleSpec.
+
+        Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4.
+        Unlike Loader.load_module() this shouldn't populate sys.modules or
+        set module attributes. Both are done by Python.
+        """
+        module = types.ModuleType(spec.name)
+        # FIXME create_module() shouldn't initialise module attributes
+        module.__file__ = spec.origin
+        return module
+
+    def exec_module(self, module):
+        """
+        Execute the module to initialise it. Don't return anything.
+
+        Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4.
+        """
+        spec = module.__spec__
+        path, _ = self._by_fullname[spec.name]
+        source = ansible_mitogen.target.get_small_file(self._context, path)
+        code = compile(source, path, 'exec', 0, 1)
+        exec(code, module.__dict__)
+        self._loaded.add(spec.name)
+
     def load_module(self, fullname):
+        """
+        Return the loaded module specified by fullname.
+
+        Implements PEP 302 importlib.abc.Loader.load_module().
+        Deprecated in Python 3.4+, replaced by create_module() & exec_module().
+        """
         path, is_pkg = self._by_fullname[fullname]
         source = ansible_mitogen.target.get_small_file(self._context, path)
         code = compile(source, path, 'exec', 0, 1)
-        # FIXME Python 3.12 removed `imp`
         mod = sys.modules.setdefault(fullname, imp.new_module(fullname))
         mod.__file__ = "master:%s" % (path,)
         mod.__loader__ = self
@@ -819,12 +886,17 @@ class NewStyleRunner(ScriptRunner):
         synchronization mechanism by importing everything the module will need
         prior to detaching.
         """
+        # I think "custom" means "found in custom module_utils search path",
+        # e.g. playbook relative dir, ~/.ansible/..., Ansible collection.
         for fullname, _, _ in self.module_map['custom']:
             mitogen.core.import_module(fullname)
+
+        # I think "builtin" means "part of ansible/ansible-base/ansible-core",
+        # as opposed to Python builtin modules such as sys.
         for fullname in self.module_map['builtin']:
             try:
                 mitogen.core.import_module(fullname)
-            except ImportError:
+            except ImportError as exc:
                 # #590: Ansible 2.8 module_utils.distro is a package that
                 # replaces itself in sys.modules with a non-package during
                 # import. Prior to replacement, it is a real package containing
@@ -835,8 +907,18 @@ class NewStyleRunner(ScriptRunner):
                 # loop progresses to the next entry and attempts to preload
                 # 'distro._distro', the import mechanism will fail. So here we
                 # silently ignore any failure for it.
-                if fullname != 'ansible.module_utils.distro._distro':
-                    raise
+                if fullname == 'ansible.module_utils.distro._distro':
+                    continue
+
+                # ansible.module_utils.compat.selinux raises ImportError if it
+                # can't load libselinux.so. The importer would usually catch
+                # this & skip selinux operations. We don't care about selinux,
+                # we're using import to get a copy of the module.
+                if (fullname == 'ansible.module_utils.compat.selinux'
+                    and exc.msg == 'unable to load libselinux.so'):
+                    continue
+
+                raise
 
     def _setup_excepthook(self):
         """

docs/changelog.rst

@@ -23,6 +23,8 @@ Unreleased
 * :gh:issue:`987` Support Python 3.11
 * :gh:issue:`885` Fix :py:exc:`PermissionError` in :py:mod:`importlib` when
   becoming an unprivileged user with Python 3.x
+* :gh:issue:`1033` Support `PEP 451 <https://peps.python.org/pep-0451/>,
+  required by Python 3.12
 
 
 v0.3.4 (2023-07-02)

docs/contributors.rst

@@ -138,4 +138,5 @@ sponsorship and outstanding future-thinking of its early adopters.
     <li>randy &mdash; <em>desperate for automation</em></li>
     <li>Michael & Vicky Twomey-Lee</li>
     <li><a href="http://www.wezm.net/">Wesley Moore</a></li>
+    <li><a href="https://github.com/baryluk">Witold Baryluk</a></li>
     </ul>

mitogen/core.py

@@ -81,13 +81,18 @@ import syslog
 import threading
 import time
 import traceback
+import types
 import warnings
 import weakref
 import zlib
 
-# Python >3.7 deprecated the imp module.
-warnings.filterwarnings('ignore', message='the imp module is deprecated')
-import imp
+try:
+    # Python >= 3.4, PEP 451 ModuleSpec API
+    import importlib.machinery
+    import importlib.util
+except ImportError:
+    # Python < 3.4, PEP 302 Import Hooks
+    import imp
 
 # Absolute imports for <2.5.
 select = __import__('select')
@@ -1380,6 +1385,19 @@ class Importer(object):
     def __repr__(self):
         return 'Importer'
 
+    @staticmethod
+    def _loader_from_module(module, default=None):
+        """Return the loader for a module object."""
+        try:
+            return module.__spec__.loader
+        except AttributeError:
+            pass
+        try:
+            return module.__loader__
+        except AttributeError:
+            pass
+        return default
+
     def builtin_find_module(self, fullname):
         # imp.find_module() will always succeed for __main__, because it is a
         # built-in module. That means it exists on a special linked list deep
@@ -1400,7 +1418,6 @@ class Importer(object):
         # Otherwise use search path of the parent package.
         # Works for both stdlib modules & third-party modules.
         # If the search is unsuccessful then raises ImportError.
-        # FIXME Python 3.12 removed `imp`.
         fp, pathname, description = imp.find_module(modname, path)
         if fp:
             fp.close()
@@ -1424,14 +1441,13 @@ class Importer(object):
         try:
             #_v and self._log.debug('Python requested %r', fullname)
             fullname = to_text(fullname)
-            pkgname, dot, _ = str_rpartition(fullname, '.')
+            pkgname, _, suffix = str_rpartition(fullname, '.')
             pkg = sys.modules.get(pkgname)
             if pkgname and getattr(pkg, '__loader__', None) is not self:
                 self._log.debug('%s is submodule of a locally loaded package',
                                 fullname)
                 return None
 
-            suffix = fullname[len(pkgname+dot):]
             if pkgname and suffix not in self._present.get(pkgname, ()):
                 self._log.debug('%s has no submodule %s', pkgname, suffix)
                 return None
@@ -1451,6 +1467,66 @@ class Importer(object):
         finally:
             del _tls.running
 
+    def find_spec(self, fullname, path, target=None):
+        """
+        Return a `ModuleSpec` for module with `fullname` if we will load it.
+        Otherwise return `None`, allowing other finders to try.
+
+        fullname    Fully qualified name of the module (e.g. foo.bar.baz)
+        path        Path entries to search. None for a top-level module.
+        target      Existing module to be reloaded (if any).
+
+        Implements importlib.abc.MetaPathFinder.find_spec()
+        Python 3.4+.
+        """
+        # Presence of _tls.running indicates we've re-invoked importlib.
+        # Abort early to prevent infinite recursion. See below.
+        if hasattr(_tls, 'running'):
+            return None
+
+        log = self._log.getChild('find_spec')
+
+        if fullname.endswith('.'):
+            return None
+
+        pkgname, _, modname = fullname.rpartition('.')
+        if pkgname and modname not in self._present.get(pkgname, ()):
+            log.debug('Skipping %s. Parent %s has no submodule %s',
+                      fullname, pkgname, modname)
+            return None
+
+        pkg = sys.modules.get(pkgname)
+        pkg_loader = self._loader_from_module(pkg)
+        if pkgname and pkg_loader is not self:
+            log.debug('Skipping %s. Parent %s was loaded by %r',
+                      fullname, pkgname, pkg_loader)
+            return None
+
+        # #114: whitelisted prefixes override any system-installed package.
+        if self.whitelist != ['']:
+            if any(s and fullname.startswith(s) for s in self.whitelist):
+                log.debug('Handling %s. It is whitelisted', fullname)
+                return importlib.machinery.ModuleSpec(fullname, loader=self)
+
+        if fullname == '__main__':
+            log.debug('Handling %s. A special case', fullname)
+            return importlib.machinery.ModuleSpec(fullname, loader=self)
+
+        # Re-invoke the import machinery to allow other finders to try.
+        # Set a guard, so we don't infinitely recurse. See top of this method.
+        _tls.running = True
+        try:
+            spec = importlib.util._find_spec(fullname, path, target)
+        finally:
+            del _tls.running
+
+        if spec:
+            log.debug('Skipping %s. Available as %r', fullname, spec)
+            return spec
+
+        log.debug('Handling %s. Unavailable locally', fullname)
+        return importlib.machinery.ModuleSpec(fullname, loader=self)
+
     blacklisted_msg = (
         '%r is present in the Mitogen importer blacklist, therefore this '
         'context will not attempt to request it from the master, as the '
@@ -1537,6 +1613,64 @@ class Importer(object):
         if present:
             callback()
 
+    def create_module(self, spec):
+        """
+        Return a module object for the given ModuleSpec.
+
+        Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4.
+        Unlike Loader.load_module() this shouldn't populate sys.modules or
+        set module attributes. Both are done by Python.
+        """
+        self._log.debug('Creating module for %r', spec)
+
+        # FIXME Should this be done in find_spec()? Can it?
+        self._refuse_imports(spec.name)
+
+        # FIXME "create_module() should properly handle the case where it is
+        #       called more than once for the same spec/module." -- PEP-451
+        event = threading.Event()
+        self._request_module(spec.name, callback=event.set)
+        event.wait()
+
+        # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+        _, pkg_present, path, _, _ = self._cache[spec.name]
+
+        if path is None:
+            raise ImportError(self.absent_msg % (spec.name))
+
+        spec.origin = self.get_filename(spec.name)
+        if pkg_present is not None:
+            # TODO Namespace packages
+            spec.submodule_search_locations = []
+            self._present[spec.name] = pkg_present
+
+        module = types.ModuleType(spec.name)
+        # FIXME create_module() shouldn't initialise module attributes
+        module.__file__ = spec.origin
+        return module
+
+    def exec_module(self, module):
+        """
+        Execute the module to initialise it. Don't return anything.
+
+        Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4.
+        """
+        name = module.__spec__.name
+        origin = module.__spec__.origin
+        self._log.debug('Executing %s from %s', name, origin)
+        source = self.get_source(name)
+        try:
+            # Compile the source into a code object. Don't add any __future__
+            # flags and don't inherit any from this module.
+            # FIXME Should probably be exposed as get_code()
+            code = compile(source, origin, 'exec', flags=0, dont_inherit=True)
+        except SyntaxError:
+            # FIXME Why is this LOG, rather than self._log?
+            LOG.exception('while importing %r', name)
+            raise
+
+        exec(code, module.__dict__)
+
     def load_module(self, fullname):
         """
         Return the loaded module specified by fullname.
@@ -1552,12 +1686,11 @@ class Importer(object):
         self._request_module(fullname, event.set)
         event.wait()
 
-        ret = self._cache[fullname]
-        if ret[2] is None:
+        # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+        _, pkg_present, path, _, _ = self._cache[fullname]
+        if path is None:
             raise ModuleNotFoundError(self.absent_msg % (fullname,))
 
-        pkg_present = ret[1]
-        # FIXME Python 3.12 removed `imp`
         mod = sys.modules.setdefault(fullname, imp.new_module(fullname))
         mod.__file__ = self.get_filename(fullname)
         mod.__loader__ = self
@@ -3958,8 +4091,7 @@ class ExternalContext(object):
 
     def _setup_package(self):
         global mitogen
-        # FIXME Python 3.12 removed `imp`
-        mitogen = imp.new_module('mitogen')
+        mitogen = types.ModuleType('mitogen')
         mitogen.__package__ = 'mitogen'
         mitogen.__path__ = []
         mitogen.__loader__ = self.importer

mitogen/master.py

@@ -37,7 +37,6 @@ contexts.
 
 import dis
 import errno
-import imp
 import inspect
 import itertools
 import logging
@@ -50,6 +49,16 @@ import threading
 import types
 import zlib
 
+try:
+    # Python >= 3.4, PEP 451 ModuleSpec API
+    import importlib.machinery
+    import importlib.util
+    from _imp import is_builtin as _is_builtin
+except ImportError:
+    # Python < 3.4, PEP 302 Import Hooks
+    import imp
+    from imp import is_builtin as _is_builtin
+
 try:
     import sysconfig
 except ImportError:
@@ -122,18 +131,16 @@ def is_stdlib_name(modname):
     """
     Return :data:`True` if `modname` appears to come from the standard library.
     """
-    # `imp.is_builtin()` isn't a documented as part of Python's stdlib API.
+    # `(_imp|imp).is_builtin()` isn't a documented part of Python's stdlib.
     # Returns 1 if modname names a module that is "builtin" to the the Python
     # interpreter (e.g. '_sre'). Otherwise 0 (e.g. 're', 'netifaces').
-    # FIXME Python 3.12 removed `imp`, but `_imp.is_builtin()` remains.
-    #       `sys.builtin_module_names` (Python 2.2+) may be an alternative.
     #
     # """
     # Main is a little special - imp.is_builtin("__main__") will return False,
     # but BuiltinImporter is still the most appropriate initial setting for
     # its __loader__ attribute.
     # """ -- comment in CPython pylifecycle.c:add_main_module()
-    if imp.is_builtin(modname) != 0:
+    if _is_builtin(modname) != 0:
         return True
 
     module = sys.modules.get(modname)
@@ -464,6 +471,9 @@ class FinderMethod(object):
     name according to the running Python interpreter. You'd think this was a
     simple task, right? Naive young fellow, welcome to the real world.
     """
+    def __init__(self):
+        self.log = LOG.getChild(self.__class__.__name__)
+
     def __repr__(self):
         return '%s()' % (type(self).__name__,)
 
@@ -645,7 +655,7 @@ class SysModulesMethod(FinderMethod):
         return path, source, is_pkg
 
 
-class ParentEnumerationMethod(FinderMethod):
+class ParentImpEnumerationMethod(FinderMethod):
     """
     Attempt to fetch source code by examining the module's (hopefully less
     insane) parent package, and if no insane parents exist, simply use
@@ -775,6 +785,9 @@ class ParentEnumerationMethod(FinderMethod):
         """
         See implementation for a description of how this works.
         """
+        if sys.version_info >= (3, 4):
+            return None
+
         #if fullname not in sys.modules:
             # Don't attempt this unless a module really exists in sys.modules,
             # else we could return junk.
@@ -792,19 +805,110 @@ class ParentEnumerationMethod(FinderMethod):
                 # Still more components to descent. Result must be a package
                 if fp:
                     fp.close()
-                # FIXME The imp module was removed in Python 3.12.
                 if kind != imp.PKG_DIRECTORY:
                     LOG.debug('%r: %r appears to be child of non-package %r',
                               self, fullname, path)
                     return None
                 search_path = [path]
-            # FIXME The imp module was removed in Python 3.12.
             elif kind == imp.PKG_DIRECTORY:
                 return self._found_package(fullname, path)
             else:
                 return self._found_module(fullname, path, fp)
 
 
+class ParentSpecEnumerationMethod(ParentImpEnumerationMethod):
+    def _find_parent_spec(self, fullname):
+        #history = []
+        debug = self.log.debug
+        children = []
+        for parent_name, child_name in self._iter_parents(fullname):
+            children.insert(0, child_name)
+            if not parent_name:
+                debug('abandoning %r, reached top-level', fullname)
+                return None, children
+
+            try:
+                parent = sys.modules[parent_name]
+            except KeyError:
+                debug('skipping %r, not in sys.modules', parent_name)
+                continue
+
+            try:
+                spec = parent.__spec__
+            except AttributeError:
+                debug('skipping %r: %r.__spec__ is absent',
+                      parent_name, parent)
+                continue
+
+            if not spec:
+                debug('skipping %r: %r.__spec__=%r',
+                      parent_name, parent, spec)
+                continue
+
+            if spec.name != parent_name:
+                debug('skipping %r: %r.__spec__.name=%r does not match',
+                      parent_name, parent, spec.name)
+                continue
+
+            if not spec.submodule_search_locations:
+                debug('skipping %r: %r.__spec__.submodule_search_locations=%r',
+                      parent_name, parent, spec.submodule_search_locations)
+                continue
+
+            return spec, children
+
+        raise ValueError('%s._find_parent_spec(%r) unexpectedly reached bottom'
+                         % (self.__class__.__name__, fullname))
+
+    def find(self, fullname):
+        # Returns absolute path, ParentImpEnumerationMethod returns relative
+        # >>> spec_pem.find('six_brokenpkg._six')[::2]
+        # ('/Users/alex/src/mitogen/tests/data/importer/six_brokenpkg/_six.py', False)
+
+        if sys.version_info < (3, 4):
+            return None
+
+        fullname = to_text(fullname)
+        spec, children = self._find_parent_spec(fullname)
+        for child_name in children:
+            if spec:
+                name = '%s.%s' % (spec.name, child_name)
+                submodule_search_locations = spec.submodule_search_locations
+            else:
+                name = child_name
+                submodule_search_locations = None
+            spec = importlib.util._find_spec(name, submodule_search_locations)
+            if spec is None:
+                self.log.debug('%r spec unavailable from %s', fullname, spec)
+                return None
+
+            is_package = spec.submodule_search_locations is not None
+            if name != fullname:
+                if not is_package:
+                    self.log.debug('%r appears to be child of non-package %r',
+                                   fullname, spec)
+                    return None
+                continue
+
+            if not spec.has_location:
+                self.log.debug('%r.origin cannot be read as a file', spec)
+                return None
+
+            if os.path.splitext(spec.origin)[1] != '.py':
+                self.log.debug('%r.origin does not contain Python source code',
+                               spec)
+                return None
+
+            # FIXME This should use loader.get_source()
+            with open(spec.origin, 'rb') as f:
+                source = f.read()
+
+            return spec.origin, source, is_package
+
+        raise ValueError('%s.find(%r) unexpectedly reached bottom'
+                         % (self.__class__.__name__, fullname))
+
+
 class ModuleFinder(object):
     """
     Given the name of a loaded module, make a best-effort attempt at finding
@@ -845,7 +949,8 @@ class ModuleFinder(object):
         DefectivePython3xMainMethod(),
         PkgutilMethod(),
         SysModulesMethod(),
-        ParentEnumerationMethod(),
+        ParentSpecEnumerationMethod(),
+        ParentImpEnumerationMethod(),
     ]
 
     def get_module_source(self, fullname):

tests/ansible/integration/connection_delegation/delegate_to_template.yml

@@ -41,7 +41,7 @@
               'keepalive_count': 10,
               'password': null,
               'port': null,
-              'python_path': ["/usr/bin/python"],
+              'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"],
               'remote_name': null,
               'ssh_args': [
                 -o, ControlMaster=auto,
@@ -69,7 +69,7 @@
               'keepalive_count': 10,
               'password': null,
               'port': null,
-              'python_path': ["/usr/bin/python"],
+              'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"],
               'remote_name': null,
               'ssh_args': [
                 -o, ControlMaster=auto,

tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml

@@ -195,13 +195,6 @@
               - distro == 'ubuntu'
               - distro_version is version('16.04', '>=', strict=True)
 
-          - name: mac assertions
-            assert:
-              that:
-              - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python'
-              fail_msg: auto_out={{auto_out}}
-            when: os_family == 'Darwin'
-
           always:
           - meta: clear_facts
       when:

tests/ansible/lib/modules/module_finder_test.py

@@ -0,0 +1,12 @@
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+import ansible.module_utils.external1
+
+from ansible.module_utils.externalpkg.extmod import path as epem_path
+
+def main():
+    pass

tests/ansible/lib/modules/test_echo_module.py

@@ -51,6 +51,7 @@ def main():
                     'as_seen': sys.executable,
                     'resolved': os.path.realpath(sys.executable),
                 },
+                'platform': sys.platform,
             },
         },
     }

tests/ansible/tests/module_finder_test.py

@@ -0,0 +1,80 @@
+import os.path
+import sys
+import textwrap
+import unittest
+
+import ansible_mitogen.module_finder
+
+import testlib
+
+
+class ScanFromListTest(testlib.TestCase):
+    def test_absolute_imports(self):
+        source = textwrap.dedent('''\
+            from __future__ import absolute_import
+            import a; import b.c; from d.e import f; from g import h, i
+        ''')
+        code = compile(source, '<str>', 'exec')
+        self.assertEqual(
+            list(ansible_mitogen.module_finder.scan_fromlist(code)),
+            [(0, '__future__.absolute_import'), (0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')],
+        )
+
+
+class WalkImportsTest(testlib.TestCase):
+    def test_absolute_imports(self):
+        source = textwrap.dedent('''\
+            from __future__ import absolute_import
+            import a; import b; import b.c; from b.d import e, f
+        ''')
+        code = compile(source, '<str>', 'exec')
+
+        self.assertEqual(
+            list(ansible_mitogen.module_finder.walk_imports(code)),
+            ['__future__', '__future__.absolute_import', 'a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f'],
+        )
+        self.assertEqual(
+            list(ansible_mitogen.module_finder.walk_imports(code, prefix='b')),
+            ['b.c', 'b.d', 'b.d.e', 'b.d.f'],
+        )
+
+
+class ScanTest(testlib.TestCase):
+    module_name = 'ansible_module_module_finder_test__this_should_not_matter'
+    module_path = os.path.join(testlib.ANSIBLE_MODULES_DIR, 'module_finder_test.py')
+    search_path = (
+        'does_not_exist/module_utils',
+        testlib.ANSIBLE_MODULE_UTILS_DIR,
+    )
+
+    @staticmethod
+    def relpath(path):
+        return os.path.relpath(path, testlib.ANSIBLE_MODULE_UTILS_DIR)
+
+    @unittest.skipIf(sys.version_info < (3, 4), 'find spec() unavailable')
+    def test_importlib_find_spec(self):
+        scan = ansible_mitogen.module_finder._scan_importlib_find_spec
+        actual = scan(self.module_name, self.module_path, self.search_path)
+        self.assertEqual(
+            [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual],
+            [
+                ('ansible.module_utils.external1', 'external1.py', False),
+                ('ansible.module_utils.external2', 'external2.py', False),
+                ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True),
+                ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False),
+            ],
+        )
+
+    @unittest.skipIf(sys.version_info >= (3, 4), 'find spec() preferred')
+    def test_imp_find_module(self):
+        scan = ansible_mitogen.module_finder._scan_imp_find_module
+        actual = scan(self.module_name, self.module_path, self.search_path)
+        self.assertEqual(
+            [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual],
+            [
+                ('ansible.module_utils.external1', 'external1.py', False),
+                ('ansible.module_utils.external2', 'external2.py', False),
+                ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True),
+                ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False),
+            ],
+        )

tests/importer_test.py

@@ -2,6 +2,7 @@ import sys
 import threading
 import types
 import zlib
+import unittest
 
 import mock
 
@@ -42,6 +43,49 @@ class ImporterMixin(testlib.RouterMixin):
         super(ImporterMixin, self).tearDown()
 
 
+class InvalidNameTest(ImporterMixin, testlib.TestCase):
+    modname = 'trailingdot.'
+    # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+    response = (modname, None, None, None, None)
+
+    @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+    def test_find_spec_invalid(self):
+        self.set_get_module_response(self.response)
+        self.assertEqual(self.importer.find_spec(self.modname, path=None), None)
+
+
+class MissingModuleTest(ImporterMixin, testlib.TestCase):
+    modname = 'missing'
+    # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+    response = (modname, None, None, None, None)
+
+    @unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+')
+    def test_load_module_missing(self):
+        self.set_get_module_response(self.response)
+        self.assertRaises(ImportError, self.importer.load_module, self.modname)
+
+    @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+    def test_find_spec_missing(self):
+        """
+        Importer should optimistically offer itself as a module loader
+        when there are no disqualifying criteria.
+        """
+        import importlib.machinery
+        self.set_get_module_response(self.response)
+        spec = self.importer.find_spec(self.modname, path=None)
+        self.assertIsInstance(spec, importlib.machinery.ModuleSpec)
+        self.assertEqual(spec.name, self.modname)
+        self.assertEqual(spec.loader, self.importer)
+
+    @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+    def test_create_module_missing(self):
+        import importlib.machinery
+        self.set_get_module_response(self.response)
+        spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+        self.assertRaises(ImportError, self.importer.create_module, spec)
+
+
+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+')
 class LoadModuleTest(ImporterMixin, testlib.TestCase):
     data = zlib.compress(b("data = 1\n\n"))
     path = 'fake_module.py'
@@ -50,14 +94,6 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase):
     # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
     response = (modname, None, path, data, [])
 
-    def test_no_such_module(self):
-        self.set_get_module_response(
-            # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
-            (self.modname, None, None, None, None)
-        )
-        self.assertRaises(ImportError,
-            lambda: self.importer.load_module(self.modname))
-
     def test_module_added_to_sys_modules(self):
         self.set_get_module_response(self.response)
         mod = self.importer.load_module(self.modname)
@@ -80,6 +116,26 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase):
         self.assertIsNone(mod.__package__)
 
 
+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+class ModuleSpecTest(ImporterMixin, testlib.TestCase):
+    data = zlib.compress(b("data = 1\n\n"))
+    path = 'fake_module.py'
+    modname = 'fake_module'
+
+    # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+    response = (modname, None, path, data, [])
+
+    def test_module_attributes(self):
+        import importlib.machinery
+        self.set_get_module_response(self.response)
+        spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+        mod = self.importer.create_module(spec)
+        self.assertIsInstance(mod, types.ModuleType)
+        self.assertEqual(mod.__name__, 'fake_module')
+        #self.assertFalse(hasattr(mod, '__file__'))
+
+
+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+')
 class LoadSubmoduleTest(ImporterMixin, testlib.TestCase):
     data = zlib.compress(b("data = 1\n\n"))
     path = 'fake_module.py'
@@ -93,6 +149,25 @@ class LoadSubmoduleTest(ImporterMixin, testlib.TestCase):
         self.assertEqual(mod.__package__, 'mypkg')
 
 
+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+class SubmoduleSpecTest(ImporterMixin, testlib.TestCase):
+    data = zlib.compress(b("data = 1\n\n"))
+    path = 'fake_module.py'
+    modname = 'mypkg.fake_module'
+    # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+    response = (modname, None, path, data, [])
+
+    def test_module_attributes(self):
+        import importlib.machinery
+        self.set_get_module_response(self.response)
+        spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+        mod = self.importer.create_module(spec)
+        self.assertIsInstance(mod, types.ModuleType)
+        self.assertEqual(mod.__name__, 'mypkg.fake_module')
+        #self.assertFalse(hasattr(mod, '__file__'))
+
+
+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+')
 class LoadModulePackageTest(ImporterMixin, testlib.TestCase):
     data = zlib.compress(b("func = lambda: 1\n\n"))
     path = 'fake_pkg/__init__.py'
@@ -140,6 +215,41 @@ class LoadModulePackageTest(ImporterMixin, testlib.TestCase):
         self.assertEqual(mod.func.__module__, self.modname)
 
 
+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+class PackageSpecTest(ImporterMixin, testlib.TestCase):
+    data = zlib.compress(b("func = lambda: 1\n\n"))
+    path = 'fake_pkg/__init__.py'
+    modname = 'fake_pkg'
+    # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
+    response = (modname, [], path, data, [])
+
+    def test_module_attributes(self):
+        import importlib.machinery
+        self.set_get_module_response(self.response)
+        spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+        mod = self.importer.create_module(spec)
+        self.assertIsInstance(mod, types.ModuleType)
+        self.assertEqual(mod.__name__, 'fake_pkg')
+        #self.assertFalse(hasattr(mod, '__file__'))
+
+    def test_get_filename(self):
+        import importlib.machinery
+        self.set_get_module_response(self.response)
+        spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+        _ = self.importer.create_module(spec)
+        filename = self.importer.get_filename(self.modname)
+        self.assertEqual('master:fake_pkg/__init__.py', filename)
+
+    def test_get_source(self):
+        import importlib.machinery
+        self.set_get_module_response(self.response)
+        spec = importlib.machinery.ModuleSpec(self.modname, self.importer)
+        _ = self.importer.create_module(spec)
+        source = self.importer.get_source(self.modname)
+        self.assertEqual(source,
+            mitogen.core.to_text(zlib.decompress(self.data)))
+
+
 class EmailParseAddrSysTest(testlib.RouterMixin, testlib.TestCase):
     def initdir(self, caplog):
         self.caplog = caplog

tests/module_finder_test.py

@@ -140,9 +140,7 @@ class SysModulesMethodTest(testlib.TestCase):
         self.assertIsNone(tup)
 
 
-class GetModuleViaParentEnumerationTest(testlib.TestCase):
-    klass = mitogen.master.ParentEnumerationMethod
-
+class ParentEnumerationMixin(object):
     def call(self, fullname):
         return self.klass().find(fullname)
 
@@ -232,6 +230,16 @@ class GetModuleViaParentEnumerationTest(testlib.TestCase):
         self.assertEqual(is_pkg, False)
 
 
+@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python >= 3.4')
+class ParentImpEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase):
+    klass = mitogen.master.ParentImpEnumerationMethod
+
+
+@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+')
+class ParentSpecEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase):
+    klass = mitogen.master.ParentSpecEnumerationMethod
+
+
 class ResolveRelPathTest(testlib.TestCase):
     klass = mitogen.master.ModuleFinder
 

tests/testlib.py

@@ -50,8 +50,13 @@ except NameError:
 
 
 LOG = logging.getLogger(__name__)
-DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
-MODS_DIR = os.path.join(DATA_DIR, 'importer')
+
+TESTS_DIR =                     os.path.join(os.path.dirname(__file__))
+ANSIBLE_LIB_DIR =               os.path.join(TESTS_DIR, 'ansible', 'lib')
+ANSIBLE_MODULE_UTILS_DIR =      os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils')
+ANSIBLE_MODULES_DIR =           os.path.join(TESTS_DIR, 'ansible', 'lib', 'modules')
+DATA_DIR =                      os.path.join(TESTS_DIR, 'data')
+MODS_DIR =                      os.path.join(TESTS_DIR, 'data', 'importer')
 
 sys.path.append(DATA_DIR)
 sys.path.append(MODS_DIR)

tox.ini

@@ -3,18 +3,34 @@
 #
 #   sudo add-apt-repository ppa:deadsnakes/ppa
 #   sudo apt update
-#   sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python2.7 python3.{6..11} python-is-python3 sshpass tox
+#   sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python2.7 python3.{6..13}{,-venv} python-is-python3 sshpass tox
 
 # Py   A cntrllr  A target   coverage   Django     Jinja2     pip        psutil     pytest     tox        virtualenv
 # ==== ========== ========== ========== ========== ========== ========== ========== ========== ========== ==========
 # 2.4             2.3?       <= 3.7.1   <= 1.3.7              <= 1.1     <= 2.1.3              <= 1.4     <= 1.8
 # 2.5                        <= 3.7.1   <= 1.4.22             <= 1.3.1   <= 2.1.3   <= 2.8.7   <= 1.6.1   <= 1.9.1
-# 2.6  <= 2.6.20  <= 2.13    <= 4.5.4   <= 1.6.11  <= 2.10.3  <= 9.0.3   <= 5.9.0   <= 3.2.5   <= 2.9.1   <= 15.2.0
+# 2.6  <= 2.6.20  <= 2.12    <= 4.5.4   <= 1.6.11  <= 2.10.3  <= 9.0.3   <= 5.9.0   <= 3.2.5   <= 2.9.1   <= 15.2.0
 # 2.7  <= 2.11               <= 5.6     <= 1.11.29 <= 2.11.3  <= 20                 <= 4.6.11  <= 3.28    <= 20.3?
 # 3.5  <= 2.11    <= 2.13    <= 5.6     <= 2.2.28  <= 2.11.3  <= 20      <= 5.9.5   <= 6.1.0   <= 3.28    <= 20.15
 # 3.6  <= 2.11               <= 6.2     <= 3.2.20  <= 3.0.3   <= 21      <= 5.9.5   <= 7.0.1   <= 3.28    <= 20.16
 # 3.7  <= 2.12                          <= 3.2.20
 # 3.8  <= 2.12
+# 3.9  <= 2.15
+# 3.10
+# 3.11
+# 3.12            >= 2.13¹
+#
+# Notes
+# 1. Python 3.12 on a target requires Ansible >= 6 (ansible-core >= 2.13).
+#    Python 3.12 removed support for find_module(), replaced by find_spec().
+#    In Ansible <= 4.x ansible.module_utils.six lacks find_spec().
+#    https://github.com/ansible/ansible/commit/d6e28e68599e703c153914610152cf4492851eb3
+#    In Ansible <= 5.x ansible.utils.collection_loader._AnsibleCollectionFinder
+#    lacks find_spec(). https://github.com/ansible/ansible/pull/76225
+#
+#    Python 3.12 + get_uri requires Ansible >= 8 (ansible-core >= 2.15).
+#    Python 3.12 removed deprecated httplib.HTTPSConnection() arguments.
+#    https://github.com/ansible/ansible/pull/80751
 
 # Ansible            Dependency
 # ================== ======================
@@ -24,6 +40,9 @@
 # ansible == 4.*     ansible-core ~= 2.11.0
 # ansible == 5.*     ansible-core ~= 2.12.0
 # ansible == 6.*     ansible-core ~= 2.13.0
+# ansible == 7.x     ansible-core ~= 2.14.0
+# ansible == 8.x     ansible-core ~= 2.15.0
+# ansible == 9.x     ansible-core ~= 2.16.0
 
 # pip --no-python-version-warning
 # pip --disable-pip-version-check
@@ -51,6 +70,7 @@ basepython =
     py39: python3.9
     py310: python3.10
     py311: python3.11
+    py312: python3.12
 deps =
     -r{toxinidir}/tests/requirements.txt
     mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt
@@ -79,7 +99,6 @@ passenv =
     HOME
 setenv =
     # See also azure-pipelines.yml
-    ANSIBLE_SKIP_TAGS = requires_local_sudo,resource_intensive
     ANSIBLE_STRATEGY = mitogen_linear
     NOCOVERAGE_ERASE = 1
     NOCOVERAGE_REPORT = 1
@@ -111,7 +130,9 @@ setenv =
     distros_ubuntu1804: DISTROS=ubuntu1804
     distros_ubuntu2004: DISTROS=ubuntu2004
     mode_ansible: MODE=ansible
+    mode_ansible: ANSIBLE_SKIP_TAGS=resource_intensive
     mode_debops_common: MODE=debops_common
+    mode_localhost: ANSIBLE_SKIP_TAGS=issue_776,resource_intensive
     mode_mitogen: MODE=mitogen
     strategy_linear: ANSIBLE_STRATEGY=linear
 whitelist_externals =