From 5ad3d14ceb758de7928897a66b235247aee6e1ff Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 17 Mar 2024 14:55:15 +0000 Subject: [PATCH] mitogen: Support PEP 451 ModuleSpec API, required for Python 3.12 importlib.machinery.ModuleSpec and find_spec() were introduced in Python 3.4 under PEP 451. They replace the find_module() API of PEP 302, which was deprecated from Python 3.4. They were removed in Python 3.12 along with the imp module. This change adds support for the PEP 451 APIs. Mitogen should no longer import imp on Python versions that support ModuleSpec. Tests have been added to cover the new APIs. CI jobs have been added to cover Python 3.x on macOS. Refs #1033 Co-authored-by: Witold Baryluk --- .ci/azure-pipelines.yml | 18 +- .ci/localhost_ansible_tests.py | 36 ++++ ansible_mitogen/module_finder.py | 124 +++++++++++++- ansible_mitogen/runner.py | 92 ++++++++++- docs/changelog.rst | 2 + docs/contributors.rst | 1 + mitogen/core.py | 156 ++++++++++++++++-- mitogen/master.py | 123 +++++++++++++- .../delegate_to_template.yml | 4 +- .../ansible_2_8_tests.yml | 7 - .../ansible/lib/modules/module_finder_test.py | 12 ++ tests/ansible/lib/modules/test_echo_module.py | 1 + tests/ansible/tests/module_finder_test.py | 80 +++++++++ tests/importer_test.py | 126 +++++++++++++- tests/module_finder_test.py | 14 +- tests/testlib.py | 9 +- tox.ini | 27 ++- 17 files changed, 766 insertions(+), 66 deletions(-) create mode 100644 tests/ansible/lib/modules/module_finder_test.py create mode 100644 tests/ansible/tests/module_finder_test.py diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index f672240d..0bedaa03 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -25,21 +25,17 @@ jobs: python.version: '3.11' tox.env: py311-mode_mitogen - # TODO: test python3, python3 tests are broken Loc_27_210: tox.env: py27-mode_localhost-ansible2.10 - Loc_27_4: - tox.env: py27-mode_localhost-ansible4 + Loc_311_6: + python.version: '3.11' + tox.env: py311-mode_localhost-ansible6 - # NOTE: this hangs when ran in Ubuntu 18.04 Van_27_210: - tox.env: py27-mode_localhost-ansible2.10 - STRATEGY: linear - ANSIBLE_SKIP_TAGS: resource_intensive - Van_27_4: - tox.env: py27-mode_localhost-ansible4 - STRATEGY: linear - ANSIBLE_SKIP_TAGS: resource_intensive + tox.env: py27-mode_localhost-ansible2.10-strategy_linear + Van_311_6: + python.version: '3.11' + tox.env: py311-mode_localhost-ansible6-strategy_linear - job: Linux pool: diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index 69d67cd1..c50ef220 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -1,6 +1,10 @@ #!/usr/bin/env python # Run tests/ansible/all.yml under Ansible and Ansible-Mitogen +from __future__ import print_function + +import getpass +import io import os import subprocess import sys @@ -53,6 +57,38 @@ with ci_lib.Fold('machine_prep'): os.chdir(IMAGE_PREP_DIR) ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml") + # FIXME Don't hardcode https://github.com/mitogen-hq/mitogen/issues/1022 + # and os.environ['USER'] is not populated on Azure macOS runners. + os.chdir(HOSTS_DIR) + with io.open('default.hosts', 'r+', encoding='utf-8') as f: + user = getpass.getuser() + content = f.read() + content = content.replace("{{ lookup('pipe', 'whoami') }}", user) + f.seek(0) + f.write(content) + f.truncate() + ci_lib.dump_file('default.hosts') + + cmd = ';'.join([ + 'from __future__ import print_function', + 'import os, sys', + 'print(sys.executable, os.path.realpath(sys.executable))', + ]) + for interpreter in ['/usr/bin/python', '/usr/bin/python2', '/usr/bin/python2.7']: + print(interpreter) + try: + subprocess.call([interpreter, '-c', cmd]) + except OSError as exc: + print(exc) + + print(interpreter, 'with PYTHON_LAUNCHED_FROM_WRAPPER=1') + environ = os.environ.copy() + environ['PYTHON_LAUNCHED_FROM_WRAPPER'] = '1' + try: + subprocess.call([interpreter, '-c', cmd], env=environ) + except OSError as exc: + print(exc) + with ci_lib.Fold('ansible'): os.chdir(TESTS_DIR) diff --git a/ansible_mitogen/module_finder.py b/ansible_mitogen/module_finder.py index 19dcef7f..a1870833 100644 --- a/ansible_mitogen/module_finder.py +++ b/ansible_mitogen/module_finder.py @@ -31,12 +31,23 @@ from __future__ import unicode_literals __metaclass__ = type import collections -import imp +import logging import os +import re +import sys + +try: + # Python >= 3.4, PEP 451 ModuleSpec API + import importlib.machinery + import importlib.util +except ImportError: + # Python < 3.4, PEP 302 Import Hooks + import imp import mitogen.master +LOG = logging.getLogger(__name__) PREFIX = 'ansible.module_utils.' @@ -45,8 +56,6 @@ PREFIX = 'ansible.module_utils.' # path Filesystem path of the module. # kind One of the constants in `imp`, as returned in `imp.find_module()` # parent `ansible_mitogen.module_finder.Module` of parent package (if any). -# -# FIXME Python 3.12 removed `imp`, leaving no constants for `Module.kind`. Module = collections.namedtuple('Module', 'name path kind parent') @@ -126,14 +135,121 @@ def find_relative(parent, name, path=()): def scan_fromlist(code): + """Return an iterator of (level, name) for explicit imports in a code + object. + + Not all names identify a module. `from os import name, path` generates + `(0, 'os.name'), (0, 'os.path')`, but `os.name` is usually a string. + + >>> src = 'import a; import b.c; from d.e import f; from g import h, i\\n' + >>> code = compile(src, '', 'exec') + >>> list(scan_fromlist(code)) + [(0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')] + """ for level, modname_s, fromlist in mitogen.master.scan_code_imports(code): for name in fromlist: - yield level, '%s.%s' % (modname_s, name) + yield level, str('%s.%s' % (modname_s, name)) if not fromlist: yield level, modname_s +def walk_imports(code, prefix=None): + """Return an iterator of names for implicit parent imports & explicit + imports in a code object. + + If a prefix is provided, then only children of that prefix are included. + Not all names identify a module. `from os import name, path` generates + `'os', 'os.name', 'os.path'`, but `os.name` is usually a string. + + >>> source = 'import a; import b; import b.c; from b.d import e, f\\n' + >>> code = compile(source, '', 'exec') + >>> list(walk_imports(code)) + ['a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f'] + >>> list(walk_imports(code, prefix='b')) + ['b.c', 'b.d', 'b.d.e', 'b.d.f'] + """ + if prefix is None: + prefix = '' + pattern = re.compile(r'(^|\.)(\w+)') + start = len(prefix) + for _, name, fromlist in mitogen.master.scan_code_imports(code): + if not name.startswith(prefix): + continue + for match in pattern.finditer(name, start): + yield name[:match.end()] + for leaf in fromlist: + yield str('%s.%s' % (name, leaf)) + + def scan(module_name, module_path, search_path): + # type: (str, str, list[str]) -> list[(str, str, bool)] + """Return a list of (name, path, is_package) for ansible.module_utils + imports used by an Ansible module. + """ + log = LOG.getChild('scan') + log.debug('%r, %r, %r', module_name, module_path, search_path) + + if sys.version_info >= (3, 4): + result = _scan_importlib_find_spec( + module_name, module_path, search_path, + ) + log.debug('_scan_importlib_find_spec %r', result) + else: + result = _scan_imp_find_module(module_name, module_path, search_path) + log.debug('_scan_imp_find_module %r', result) + return result + + +def _scan_importlib_find_spec(module_name, module_path, search_path): + # type: (str, str, list[str]) -> list[(str, str, bool)] + module = importlib.machinery.ModuleSpec( + module_name, loader=None, origin=module_path, + ) + prefix = importlib.machinery.ModuleSpec( + PREFIX.rstrip('.'), loader=None, + ) + prefix.submodule_search_locations = search_path + queue = collections.deque([module]) + specs = {prefix.name: prefix} + while queue: + spec = queue.popleft() + if spec.origin is None: + continue + try: + with open(spec.origin, 'rb') as f: + code = compile(f.read(), spec.name, 'exec') + except Exception as exc: + raise ValueError((exc, module, spec, specs)) + + for name in walk_imports(code, prefix.name): + if name in specs: + continue + + parent_name = name.rpartition('.')[0] + parent = specs[parent_name] + if parent is None or not parent.submodule_search_locations: + specs[name] = None + continue + + child = importlib.util._find_spec( + name, parent.submodule_search_locations, + ) + if child is None or child.origin is None: + specs[name] = None + continue + + specs[name] = child + queue.append(child) + + del specs[prefix.name] + return sorted( + (spec.name, spec.origin, spec.submodule_search_locations is not None) + for spec in specs.values() if spec is not None + ) + + +def _scan_imp_find_module(module_name, module_path, search_path): + # type: (str, str, list[str]) -> list[(str, str, bool)] module = Module(module_name, module_path, imp.PY_SOURCE, None) stack = [module] seen = set() diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 6986ddcd..8da1b670 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -40,7 +40,6 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type import atexit -import imp import json import os import re @@ -64,6 +63,14 @@ except ImportError: # Python 2.4 ctypes = None +try: + # Python >= 3.4, PEP 451 ModuleSpec API + import importlib.machinery + import importlib.util +except ImportError: + # Python < 3.4, PEP 302 Import Hooks + import imp + try: # Cannot use cStringIO as it does not support Unicode. from StringIO import StringIO @@ -514,14 +521,74 @@ class ModuleUtilsImporter(object): sys.modules.pop(fullname, None) def find_module(self, fullname, path=None): + """ + Return a loader for the module with fullname, if we will load it. + + Implements importlib.abc.MetaPathFinder.find_module(). + Deprecrated in Python 3.4+, replaced by find_spec(). + Raises ImportWarning in Python 3.10+. Removed in Python 3.12. + """ if fullname in self._by_fullname: return self + def find_spec(self, fullname, path, target=None): + """ + Return a `ModuleSpec` for module with `fullname` if we will load it. + Otherwise return `None`. + + Implements importlib.abc.MetaPathFinder.find_spec(). Python 3.4+. + """ + if fullname.endswith('.'): + return None + + try: + module_path, is_package = self._by_fullname[fullname] + except KeyError: + LOG.debug('Skipping %s: not present', fullname) + return None + + LOG.debug('Handling %s', fullname) + origin = 'master:%s' % (module_path,) + return importlib.machinery.ModuleSpec( + fullname, loader=self, origin=origin, is_package=is_package, + ) + + def create_module(self, spec): + """ + Return a module object for the given ModuleSpec. + + Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4. + Unlike Loader.load_module() this shouldn't populate sys.modules or + set module attributes. Both are done by Python. + """ + module = types.ModuleType(spec.name) + # FIXME create_module() shouldn't initialise module attributes + module.__file__ = spec.origin + return module + + def exec_module(self, module): + """ + Execute the module to initialise it. Don't return anything. + + Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4. + """ + spec = module.__spec__ + path, _ = self._by_fullname[spec.name] + source = ansible_mitogen.target.get_small_file(self._context, path) + code = compile(source, path, 'exec', 0, 1) + exec(code, module.__dict__) + self._loaded.add(spec.name) + def load_module(self, fullname): + """ + Return the loaded module specified by fullname. + + Implements PEP 302 importlib.abc.Loader.load_module(). + Deprecated in Python 3.4+, replaced by create_module() & exec_module(). + """ path, is_pkg = self._by_fullname[fullname] source = ansible_mitogen.target.get_small_file(self._context, path) code = compile(source, path, 'exec', 0, 1) - # FIXME Python 3.12 removed `imp` mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) mod.__file__ = "master:%s" % (path,) mod.__loader__ = self @@ -819,12 +886,17 @@ class NewStyleRunner(ScriptRunner): synchronization mechanism by importing everything the module will need prior to detaching. """ + # I think "custom" means "found in custom module_utils search path", + # e.g. playbook relative dir, ~/.ansible/..., Ansible collection. for fullname, _, _ in self.module_map['custom']: mitogen.core.import_module(fullname) + + # I think "builtin" means "part of ansible/ansible-base/ansible-core", + # as opposed to Python builtin modules such as sys. for fullname in self.module_map['builtin']: try: mitogen.core.import_module(fullname) - except ImportError: + except ImportError as exc: # #590: Ansible 2.8 module_utils.distro is a package that # replaces itself in sys.modules with a non-package during # import. Prior to replacement, it is a real package containing @@ -835,8 +907,18 @@ class NewStyleRunner(ScriptRunner): # loop progresses to the next entry and attempts to preload # 'distro._distro', the import mechanism will fail. So here we # silently ignore any failure for it. - if fullname != 'ansible.module_utils.distro._distro': - raise + if fullname == 'ansible.module_utils.distro._distro': + continue + + # ansible.module_utils.compat.selinux raises ImportError if it + # can't load libselinux.so. The importer would usually catch + # this & skip selinux operations. We don't care about selinux, + # we're using import to get a copy of the module. + if (fullname == 'ansible.module_utils.compat.selinux' + and exc.msg == 'unable to load libselinux.so'): + continue + + raise def _setup_excepthook(self): """ diff --git a/docs/changelog.rst b/docs/changelog.rst index 32829ef6..940e4900 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,6 +23,8 @@ Unreleased * :gh:issue:`987` Support Python 3.11 * :gh:issue:`885` Fix :py:exc:`PermissionError` in :py:mod:`importlib` when becoming an unprivileged user with Python 3.x +* :gh:issue:`1033` Support `PEP 451 , + required by Python 3.12 v0.3.4 (2023-07-02) diff --git a/docs/contributors.rst b/docs/contributors.rst index 584c4cd4..61a9eb1b 100644 --- a/docs/contributors.rst +++ b/docs/contributors.rst @@ -138,4 +138,5 @@ sponsorship and outstanding future-thinking of its early adopters.
  • randy — desperate for automation
  • Michael & Vicky Twomey-Lee
  • Wesley Moore
    • +
  • Witold Baryluk
    • diff --git a/mitogen/core.py b/mitogen/core.py index 6a3f3da7..cd02012f 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -81,13 +81,18 @@ import syslog import threading import time import traceback +import types import warnings import weakref import zlib -# Python >3.7 deprecated the imp module. -warnings.filterwarnings('ignore', message='the imp module is deprecated') -import imp +try: + # Python >= 3.4, PEP 451 ModuleSpec API + import importlib.machinery + import importlib.util +except ImportError: + # Python < 3.4, PEP 302 Import Hooks + import imp # Absolute imports for <2.5. select = __import__('select') @@ -1380,6 +1385,19 @@ class Importer(object): def __repr__(self): return 'Importer' + @staticmethod + def _loader_from_module(module, default=None): + """Return the loader for a module object.""" + try: + return module.__spec__.loader + except AttributeError: + pass + try: + return module.__loader__ + except AttributeError: + pass + return default + def builtin_find_module(self, fullname): # imp.find_module() will always succeed for __main__, because it is a # built-in module. That means it exists on a special linked list deep @@ -1400,7 +1418,6 @@ class Importer(object): # Otherwise use search path of the parent package. # Works for both stdlib modules & third-party modules. # If the search is unsuccessful then raises ImportError. - # FIXME Python 3.12 removed `imp`. fp, pathname, description = imp.find_module(modname, path) if fp: fp.close() @@ -1424,14 +1441,13 @@ class Importer(object): try: #_v and self._log.debug('Python requested %r', fullname) fullname = to_text(fullname) - pkgname, dot, _ = str_rpartition(fullname, '.') + pkgname, _, suffix = str_rpartition(fullname, '.') pkg = sys.modules.get(pkgname) if pkgname and getattr(pkg, '__loader__', None) is not self: self._log.debug('%s is submodule of a locally loaded package', fullname) return None - suffix = fullname[len(pkgname+dot):] if pkgname and suffix not in self._present.get(pkgname, ()): self._log.debug('%s has no submodule %s', pkgname, suffix) return None @@ -1451,6 +1467,66 @@ class Importer(object): finally: del _tls.running + def find_spec(self, fullname, path, target=None): + """ + Return a `ModuleSpec` for module with `fullname` if we will load it. + Otherwise return `None`, allowing other finders to try. + + fullname Fully qualified name of the module (e.g. foo.bar.baz) + path Path entries to search. None for a top-level module. + target Existing module to be reloaded (if any). + + Implements importlib.abc.MetaPathFinder.find_spec() + Python 3.4+. + """ + # Presence of _tls.running indicates we've re-invoked importlib. + # Abort early to prevent infinite recursion. See below. + if hasattr(_tls, 'running'): + return None + + log = self._log.getChild('find_spec') + + if fullname.endswith('.'): + return None + + pkgname, _, modname = fullname.rpartition('.') + if pkgname and modname not in self._present.get(pkgname, ()): + log.debug('Skipping %s. Parent %s has no submodule %s', + fullname, pkgname, modname) + return None + + pkg = sys.modules.get(pkgname) + pkg_loader = self._loader_from_module(pkg) + if pkgname and pkg_loader is not self: + log.debug('Skipping %s. Parent %s was loaded by %r', + fullname, pkgname, pkg_loader) + return None + + # #114: whitelisted prefixes override any system-installed package. + if self.whitelist != ['']: + if any(s and fullname.startswith(s) for s in self.whitelist): + log.debug('Handling %s. It is whitelisted', fullname) + return importlib.machinery.ModuleSpec(fullname, loader=self) + + if fullname == '__main__': + log.debug('Handling %s. A special case', fullname) + return importlib.machinery.ModuleSpec(fullname, loader=self) + + # Re-invoke the import machinery to allow other finders to try. + # Set a guard, so we don't infinitely recurse. See top of this method. + _tls.running = True + try: + spec = importlib.util._find_spec(fullname, path, target) + finally: + del _tls.running + + if spec: + log.debug('Skipping %s. Available as %r', fullname, spec) + return spec + + log.debug('Handling %s. Unavailable locally', fullname) + return importlib.machinery.ModuleSpec(fullname, loader=self) + blacklisted_msg = ( '%r is present in the Mitogen importer blacklist, therefore this ' 'context will not attempt to request it from the master, as the ' @@ -1537,6 +1613,64 @@ class Importer(object): if present: callback() + def create_module(self, spec): + """ + Return a module object for the given ModuleSpec. + + Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4. + Unlike Loader.load_module() this shouldn't populate sys.modules or + set module attributes. Both are done by Python. + """ + self._log.debug('Creating module for %r', spec) + + # FIXME Should this be done in find_spec()? Can it? + self._refuse_imports(spec.name) + + # FIXME "create_module() should properly handle the case where it is + # called more than once for the same spec/module." -- PEP-451 + event = threading.Event() + self._request_module(spec.name, callback=event.set) + event.wait() + + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + _, pkg_present, path, _, _ = self._cache[spec.name] + + if path is None: + raise ImportError(self.absent_msg % (spec.name)) + + spec.origin = self.get_filename(spec.name) + if pkg_present is not None: + # TODO Namespace packages + spec.submodule_search_locations = [] + self._present[spec.name] = pkg_present + + module = types.ModuleType(spec.name) + # FIXME create_module() shouldn't initialise module attributes + module.__file__ = spec.origin + return module + + def exec_module(self, module): + """ + Execute the module to initialise it. Don't return anything. + + Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4. + """ + name = module.__spec__.name + origin = module.__spec__.origin + self._log.debug('Executing %s from %s', name, origin) + source = self.get_source(name) + try: + # Compile the source into a code object. Don't add any __future__ + # flags and don't inherit any from this module. + # FIXME Should probably be exposed as get_code() + code = compile(source, origin, 'exec', flags=0, dont_inherit=True) + except SyntaxError: + # FIXME Why is this LOG, rather than self._log? + LOG.exception('while importing %r', name) + raise + + exec(code, module.__dict__) + def load_module(self, fullname): """ Return the loaded module specified by fullname. @@ -1552,12 +1686,11 @@ class Importer(object): self._request_module(fullname, event.set) event.wait() - ret = self._cache[fullname] - if ret[2] is None: + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + _, pkg_present, path, _, _ = self._cache[fullname] + if path is None: raise ModuleNotFoundError(self.absent_msg % (fullname,)) - pkg_present = ret[1] - # FIXME Python 3.12 removed `imp` mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) mod.__file__ = self.get_filename(fullname) mod.__loader__ = self @@ -3958,8 +4091,7 @@ class ExternalContext(object): def _setup_package(self): global mitogen - # FIXME Python 3.12 removed `imp` - mitogen = imp.new_module('mitogen') + mitogen = types.ModuleType('mitogen') mitogen.__package__ = 'mitogen' mitogen.__path__ = [] mitogen.__loader__ = self.importer diff --git a/mitogen/master.py b/mitogen/master.py index 8cd1d27c..b1e0a1de 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -37,7 +37,6 @@ contexts. import dis import errno -import imp import inspect import itertools import logging @@ -50,6 +49,16 @@ import threading import types import zlib +try: + # Python >= 3.4, PEP 451 ModuleSpec API + import importlib.machinery + import importlib.util + from _imp import is_builtin as _is_builtin +except ImportError: + # Python < 3.4, PEP 302 Import Hooks + import imp + from imp import is_builtin as _is_builtin + try: import sysconfig except ImportError: @@ -122,18 +131,16 @@ def is_stdlib_name(modname): """ Return :data:`True` if `modname` appears to come from the standard library. """ - # `imp.is_builtin()` isn't a documented as part of Python's stdlib API. + # `(_imp|imp).is_builtin()` isn't a documented part of Python's stdlib. # Returns 1 if modname names a module that is "builtin" to the the Python # interpreter (e.g. '_sre'). Otherwise 0 (e.g. 're', 'netifaces'). - # FIXME Python 3.12 removed `imp`, but `_imp.is_builtin()` remains. - # `sys.builtin_module_names` (Python 2.2+) may be an alternative. # # """ # Main is a little special - imp.is_builtin("__main__") will return False, # but BuiltinImporter is still the most appropriate initial setting for # its __loader__ attribute. # """ -- comment in CPython pylifecycle.c:add_main_module() - if imp.is_builtin(modname) != 0: + if _is_builtin(modname) != 0: return True module = sys.modules.get(modname) @@ -464,6 +471,9 @@ class FinderMethod(object): name according to the running Python interpreter. You'd think this was a simple task, right? Naive young fellow, welcome to the real world. """ + def __init__(self): + self.log = LOG.getChild(self.__class__.__name__) + def __repr__(self): return '%s()' % (type(self).__name__,) @@ -645,7 +655,7 @@ class SysModulesMethod(FinderMethod): return path, source, is_pkg -class ParentEnumerationMethod(FinderMethod): +class ParentImpEnumerationMethod(FinderMethod): """ Attempt to fetch source code by examining the module's (hopefully less insane) parent package, and if no insane parents exist, simply use @@ -775,6 +785,9 @@ class ParentEnumerationMethod(FinderMethod): """ See implementation for a description of how this works. """ + if sys.version_info >= (3, 4): + return None + #if fullname not in sys.modules: # Don't attempt this unless a module really exists in sys.modules, # else we could return junk. @@ -792,19 +805,110 @@ class ParentEnumerationMethod(FinderMethod): # Still more components to descent. Result must be a package if fp: fp.close() - # FIXME The imp module was removed in Python 3.12. if kind != imp.PKG_DIRECTORY: LOG.debug('%r: %r appears to be child of non-package %r', self, fullname, path) return None search_path = [path] - # FIXME The imp module was removed in Python 3.12. elif kind == imp.PKG_DIRECTORY: return self._found_package(fullname, path) else: return self._found_module(fullname, path, fp) +class ParentSpecEnumerationMethod(ParentImpEnumerationMethod): + def _find_parent_spec(self, fullname): + #history = [] + debug = self.log.debug + children = [] + for parent_name, child_name in self._iter_parents(fullname): + children.insert(0, child_name) + if not parent_name: + debug('abandoning %r, reached top-level', fullname) + return None, children + + try: + parent = sys.modules[parent_name] + except KeyError: + debug('skipping %r, not in sys.modules', parent_name) + continue + + try: + spec = parent.__spec__ + except AttributeError: + debug('skipping %r: %r.__spec__ is absent', + parent_name, parent) + continue + + if not spec: + debug('skipping %r: %r.__spec__=%r', + parent_name, parent, spec) + continue + + if spec.name != parent_name: + debug('skipping %r: %r.__spec__.name=%r does not match', + parent_name, parent, spec.name) + continue + + if not spec.submodule_search_locations: + debug('skipping %r: %r.__spec__.submodule_search_locations=%r', + parent_name, parent, spec.submodule_search_locations) + continue + + return spec, children + + raise ValueError('%s._find_parent_spec(%r) unexpectedly reached bottom' + % (self.__class__.__name__, fullname)) + + def find(self, fullname): + # Returns absolute path, ParentImpEnumerationMethod returns relative + # >>> spec_pem.find('six_brokenpkg._six')[::2] + # ('/Users/alex/src/mitogen/tests/data/importer/six_brokenpkg/_six.py', False) + + if sys.version_info < (3, 4): + return None + + fullname = to_text(fullname) + spec, children = self._find_parent_spec(fullname) + for child_name in children: + if spec: + name = '%s.%s' % (spec.name, child_name) + submodule_search_locations = spec.submodule_search_locations + else: + name = child_name + submodule_search_locations = None + spec = importlib.util._find_spec(name, submodule_search_locations) + if spec is None: + self.log.debug('%r spec unavailable from %s', fullname, spec) + return None + + is_package = spec.submodule_search_locations is not None + if name != fullname: + if not is_package: + self.log.debug('%r appears to be child of non-package %r', + fullname, spec) + return None + continue + + if not spec.has_location: + self.log.debug('%r.origin cannot be read as a file', spec) + return None + + if os.path.splitext(spec.origin)[1] != '.py': + self.log.debug('%r.origin does not contain Python source code', + spec) + return None + + # FIXME This should use loader.get_source() + with open(spec.origin, 'rb') as f: + source = f.read() + + return spec.origin, source, is_package + + raise ValueError('%s.find(%r) unexpectedly reached bottom' + % (self.__class__.__name__, fullname)) + + class ModuleFinder(object): """ Given the name of a loaded module, make a best-effort attempt at finding @@ -845,7 +949,8 @@ class ModuleFinder(object): DefectivePython3xMainMethod(), PkgutilMethod(), SysModulesMethod(), - ParentEnumerationMethod(), + ParentSpecEnumerationMethod(), + ParentImpEnumerationMethod(), ] def get_module_source(self, fullname): diff --git a/tests/ansible/integration/connection_delegation/delegate_to_template.yml b/tests/ansible/integration/connection_delegation/delegate_to_template.yml index 3776a7db..7d33a161 100644 --- a/tests/ansible/integration/connection_delegation/delegate_to_template.yml +++ b/tests/ansible/integration/connection_delegation/delegate_to_template.yml @@ -41,7 +41,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - 'python_path': ["/usr/bin/python"], + 'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, @@ -69,7 +69,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - 'python_path': ["/usr/bin/python"], + 'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, diff --git a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml index 5b695c42..ce2b3d3a 100644 --- a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml +++ b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml @@ -195,13 +195,6 @@ - distro == 'ubuntu' - distro_version is version('16.04', '>=', strict=True) - - name: mac assertions - assert: - that: - - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python' - fail_msg: auto_out={{auto_out}} - when: os_family == 'Darwin' - always: - meta: clear_facts when: diff --git a/tests/ansible/lib/modules/module_finder_test.py b/tests/ansible/lib/modules/module_finder_test.py new file mode 100644 index 00000000..41cf1c1c --- /dev/null +++ b/tests/ansible/lib/modules/module_finder_test.py @@ -0,0 +1,12 @@ +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +import os +import sys + +import ansible.module_utils.external1 + +from ansible.module_utils.externalpkg.extmod import path as epem_path + +def main(): + pass diff --git a/tests/ansible/lib/modules/test_echo_module.py b/tests/ansible/lib/modules/test_echo_module.py index d44b85ab..9d1d99b1 100644 --- a/tests/ansible/lib/modules/test_echo_module.py +++ b/tests/ansible/lib/modules/test_echo_module.py @@ -51,6 +51,7 @@ def main(): 'as_seen': sys.executable, 'resolved': os.path.realpath(sys.executable), }, + 'platform': sys.platform, }, }, } diff --git a/tests/ansible/tests/module_finder_test.py b/tests/ansible/tests/module_finder_test.py new file mode 100644 index 00000000..79e8fdbd --- /dev/null +++ b/tests/ansible/tests/module_finder_test.py @@ -0,0 +1,80 @@ +import os.path +import sys +import textwrap +import unittest + +import ansible_mitogen.module_finder + +import testlib + + +class ScanFromListTest(testlib.TestCase): + def test_absolute_imports(self): + source = textwrap.dedent('''\ + from __future__ import absolute_import + import a; import b.c; from d.e import f; from g import h, i + ''') + code = compile(source, '', 'exec') + self.assertEqual( + list(ansible_mitogen.module_finder.scan_fromlist(code)), + [(0, '__future__.absolute_import'), (0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')], + ) + + +class WalkImportsTest(testlib.TestCase): + def test_absolute_imports(self): + source = textwrap.dedent('''\ + from __future__ import absolute_import + import a; import b; import b.c; from b.d import e, f + ''') + code = compile(source, '', 'exec') + + self.assertEqual( + list(ansible_mitogen.module_finder.walk_imports(code)), + ['__future__', '__future__.absolute_import', 'a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f'], + ) + self.assertEqual( + list(ansible_mitogen.module_finder.walk_imports(code, prefix='b')), + ['b.c', 'b.d', 'b.d.e', 'b.d.f'], + ) + + +class ScanTest(testlib.TestCase): + module_name = 'ansible_module_module_finder_test__this_should_not_matter' + module_path = os.path.join(testlib.ANSIBLE_MODULES_DIR, 'module_finder_test.py') + search_path = ( + 'does_not_exist/module_utils', + testlib.ANSIBLE_MODULE_UTILS_DIR, + ) + + @staticmethod + def relpath(path): + return os.path.relpath(path, testlib.ANSIBLE_MODULE_UTILS_DIR) + + @unittest.skipIf(sys.version_info < (3, 4), 'find spec() unavailable') + def test_importlib_find_spec(self): + scan = ansible_mitogen.module_finder._scan_importlib_find_spec + actual = scan(self.module_name, self.module_path, self.search_path) + self.assertEqual( + [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual], + [ + ('ansible.module_utils.external1', 'external1.py', False), + ('ansible.module_utils.external2', 'external2.py', False), + ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True), + ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False), + ], + ) + + @unittest.skipIf(sys.version_info >= (3, 4), 'find spec() preferred') + def test_imp_find_module(self): + scan = ansible_mitogen.module_finder._scan_imp_find_module + actual = scan(self.module_name, self.module_path, self.search_path) + self.assertEqual( + [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual], + [ + ('ansible.module_utils.external1', 'external1.py', False), + ('ansible.module_utils.external2', 'external2.py', False), + ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True), + ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False), + ], + ) diff --git a/tests/importer_test.py b/tests/importer_test.py index e48c02a4..e86af8af 100644 --- a/tests/importer_test.py +++ b/tests/importer_test.py @@ -2,6 +2,7 @@ import sys import threading import types import zlib +import unittest import mock @@ -42,6 +43,49 @@ class ImporterMixin(testlib.RouterMixin): super(ImporterMixin, self).tearDown() +class InvalidNameTest(ImporterMixin, testlib.TestCase): + modname = 'trailingdot.' + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, None, None, None, None) + + @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') + def test_find_spec_invalid(self): + self.set_get_module_response(self.response) + self.assertEqual(self.importer.find_spec(self.modname, path=None), None) + + +class MissingModuleTest(ImporterMixin, testlib.TestCase): + modname = 'missing' + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, None, None, None, None) + + @unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') + def test_load_module_missing(self): + self.set_get_module_response(self.response) + self.assertRaises(ImportError, self.importer.load_module, self.modname) + + @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') + def test_find_spec_missing(self): + """ + Importer should optimistically offer itself as a module loader + when there are no disqualifying criteria. + """ + import importlib.machinery + self.set_get_module_response(self.response) + spec = self.importer.find_spec(self.modname, path=None) + self.assertIsInstance(spec, importlib.machinery.ModuleSpec) + self.assertEqual(spec.name, self.modname) + self.assertEqual(spec.loader, self.importer) + + @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') + def test_create_module_missing(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + self.assertRaises(ImportError, self.importer.create_module, spec) + + +@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') class LoadModuleTest(ImporterMixin, testlib.TestCase): data = zlib.compress(b("data = 1\n\n")) path = 'fake_module.py' @@ -50,14 +94,6 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase): # 0:fullname 1:pkg_present 2:path 3:compressed 4:related response = (modname, None, path, data, []) - def test_no_such_module(self): - self.set_get_module_response( - # 0:fullname 1:pkg_present 2:path 3:compressed 4:related - (self.modname, None, None, None, None) - ) - self.assertRaises(ImportError, - lambda: self.importer.load_module(self.modname)) - def test_module_added_to_sys_modules(self): self.set_get_module_response(self.response) mod = self.importer.load_module(self.modname) @@ -80,6 +116,26 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase): self.assertIsNone(mod.__package__) +@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') +class ModuleSpecTest(ImporterMixin, testlib.TestCase): + data = zlib.compress(b("data = 1\n\n")) + path = 'fake_module.py' + modname = 'fake_module' + + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, None, path, data, []) + + def test_module_attributes(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + mod = self.importer.create_module(spec) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'fake_module') + #self.assertFalse(hasattr(mod, '__file__')) + + +@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') class LoadSubmoduleTest(ImporterMixin, testlib.TestCase): data = zlib.compress(b("data = 1\n\n")) path = 'fake_module.py' @@ -93,6 +149,25 @@ class LoadSubmoduleTest(ImporterMixin, testlib.TestCase): self.assertEqual(mod.__package__, 'mypkg') +@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') +class SubmoduleSpecTest(ImporterMixin, testlib.TestCase): + data = zlib.compress(b("data = 1\n\n")) + path = 'fake_module.py' + modname = 'mypkg.fake_module' + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, None, path, data, []) + + def test_module_attributes(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + mod = self.importer.create_module(spec) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'mypkg.fake_module') + #self.assertFalse(hasattr(mod, '__file__')) + + +@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') class LoadModulePackageTest(ImporterMixin, testlib.TestCase): data = zlib.compress(b("func = lambda: 1\n\n")) path = 'fake_pkg/__init__.py' @@ -140,6 +215,41 @@ class LoadModulePackageTest(ImporterMixin, testlib.TestCase): self.assertEqual(mod.func.__module__, self.modname) +@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') +class PackageSpecTest(ImporterMixin, testlib.TestCase): + data = zlib.compress(b("func = lambda: 1\n\n")) + path = 'fake_pkg/__init__.py' + modname = 'fake_pkg' + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, [], path, data, []) + + def test_module_attributes(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + mod = self.importer.create_module(spec) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'fake_pkg') + #self.assertFalse(hasattr(mod, '__file__')) + + def test_get_filename(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + _ = self.importer.create_module(spec) + filename = self.importer.get_filename(self.modname) + self.assertEqual('master:fake_pkg/__init__.py', filename) + + def test_get_source(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + _ = self.importer.create_module(spec) + source = self.importer.get_source(self.modname) + self.assertEqual(source, + mitogen.core.to_text(zlib.decompress(self.data))) + + class EmailParseAddrSysTest(testlib.RouterMixin, testlib.TestCase): def initdir(self, caplog): self.caplog = caplog diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py index 02b8b886..67e937ed 100644 --- a/tests/module_finder_test.py +++ b/tests/module_finder_test.py @@ -140,9 +140,7 @@ class SysModulesMethodTest(testlib.TestCase): self.assertIsNone(tup) -class GetModuleViaParentEnumerationTest(testlib.TestCase): - klass = mitogen.master.ParentEnumerationMethod - +class ParentEnumerationMixin(object): def call(self, fullname): return self.klass().find(fullname) @@ -232,6 +230,16 @@ class GetModuleViaParentEnumerationTest(testlib.TestCase): self.assertEqual(is_pkg, False) +@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python >= 3.4') +class ParentImpEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase): + klass = mitogen.master.ParentImpEnumerationMethod + + +@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') +class ParentSpecEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase): + klass = mitogen.master.ParentSpecEnumerationMethod + + class ResolveRelPathTest(testlib.TestCase): klass = mitogen.master.ModuleFinder diff --git a/tests/testlib.py b/tests/testlib.py index ec0a7443..8c40e7ff 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -50,8 +50,13 @@ except NameError: LOG = logging.getLogger(__name__) -DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') -MODS_DIR = os.path.join(DATA_DIR, 'importer') + +TESTS_DIR = os.path.join(os.path.dirname(__file__)) +ANSIBLE_LIB_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib') +ANSIBLE_MODULE_UTILS_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils') +ANSIBLE_MODULES_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'modules') +DATA_DIR = os.path.join(TESTS_DIR, 'data') +MODS_DIR = os.path.join(TESTS_DIR, 'data', 'importer') sys.path.append(DATA_DIR) sys.path.append(MODS_DIR) diff --git a/tox.ini b/tox.ini index e4c26c39..ae4f35c9 100644 --- a/tox.ini +++ b/tox.ini @@ -3,18 +3,34 @@ # # sudo add-apt-repository ppa:deadsnakes/ppa # sudo apt update -# sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python2.7 python3.{6..11} python-is-python3 sshpass tox +# sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python2.7 python3.{6..13}{,-venv} python-is-python3 sshpass tox # Py A cntrllr A target coverage Django Jinja2 pip psutil pytest tox virtualenv # ==== ========== ========== ========== ========== ========== ========== ========== ========== ========== ========== # 2.4 2.3? <= 3.7.1 <= 1.3.7 <= 1.1 <= 2.1.3 <= 1.4 <= 1.8 # 2.5 <= 3.7.1 <= 1.4.22 <= 1.3.1 <= 2.1.3 <= 2.8.7 <= 1.6.1 <= 1.9.1 -# 2.6 <= 2.6.20 <= 2.13 <= 4.5.4 <= 1.6.11 <= 2.10.3 <= 9.0.3 <= 5.9.0 <= 3.2.5 <= 2.9.1 <= 15.2.0 +# 2.6 <= 2.6.20 <= 2.12 <= 4.5.4 <= 1.6.11 <= 2.10.3 <= 9.0.3 <= 5.9.0 <= 3.2.5 <= 2.9.1 <= 15.2.0 # 2.7 <= 2.11 <= 5.6 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.3? # 3.5 <= 2.11 <= 2.13 <= 5.6 <= 2.2.28 <= 2.11.3 <= 20 <= 5.9.5 <= 6.1.0 <= 3.28 <= 20.15 # 3.6 <= 2.11 <= 6.2 <= 3.2.20 <= 3.0.3 <= 21 <= 5.9.5 <= 7.0.1 <= 3.28 <= 20.16 # 3.7 <= 2.12 <= 3.2.20 # 3.8 <= 2.12 +# 3.9 <= 2.15 +# 3.10 +# 3.11 +# 3.12 >= 2.13ยน +# +# Notes +# 1. Python 3.12 on a target requires Ansible >= 6 (ansible-core >= 2.13). +# Python 3.12 removed support for find_module(), replaced by find_spec(). +# In Ansible <= 4.x ansible.module_utils.six lacks find_spec(). +# https://github.com/ansible/ansible/commit/d6e28e68599e703c153914610152cf4492851eb3 +# In Ansible <= 5.x ansible.utils.collection_loader._AnsibleCollectionFinder +# lacks find_spec(). https://github.com/ansible/ansible/pull/76225 +# +# Python 3.12 + get_uri requires Ansible >= 8 (ansible-core >= 2.15). +# Python 3.12 removed deprecated httplib.HTTPSConnection() arguments. +# https://github.com/ansible/ansible/pull/80751 # Ansible Dependency # ================== ====================== @@ -24,6 +40,9 @@ # ansible == 4.* ansible-core ~= 2.11.0 # ansible == 5.* ansible-core ~= 2.12.0 # ansible == 6.* ansible-core ~= 2.13.0 +# ansible == 7.x ansible-core ~= 2.14.0 +# ansible == 8.x ansible-core ~= 2.15.0 +# ansible == 9.x ansible-core ~= 2.16.0 # pip --no-python-version-warning # pip --disable-pip-version-check @@ -51,6 +70,7 @@ basepython = py39: python3.9 py310: python3.10 py311: python3.11 + py312: python3.12 deps = -r{toxinidir}/tests/requirements.txt mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt @@ -79,7 +99,6 @@ passenv = HOME setenv = # See also azure-pipelines.yml - ANSIBLE_SKIP_TAGS = requires_local_sudo,resource_intensive ANSIBLE_STRATEGY = mitogen_linear NOCOVERAGE_ERASE = 1 NOCOVERAGE_REPORT = 1 @@ -111,7 +130,9 @@ setenv = distros_ubuntu1804: DISTROS=ubuntu1804 distros_ubuntu2004: DISTROS=ubuntu2004 mode_ansible: MODE=ansible + mode_ansible: ANSIBLE_SKIP_TAGS=resource_intensive mode_debops_common: MODE=debops_common + mode_localhost: ANSIBLE_SKIP_TAGS=issue_776,resource_intensive mode_mitogen: MODE=mitogen strategy_linear: ANSIBLE_STRATEGY=linear whitelist_externals =