diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml index ed516d72..919b992b 100644 --- a/.ci/azure-pipelines-steps.yml +++ b/.ci/azure-pipelines-steps.yml @@ -2,6 +2,7 @@ # https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/?view=azure-pipelines&viewFallbackFrom=azure-devops#tool # `{script: ...}` is shorthand for `{task: CmdLine@, inputs: {script: ...}}`. +# The shell is bash. # https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/steps-script?view=azure-pipelines # https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/cmd-line-v2?view=azure-pipelines @@ -14,20 +15,80 @@ steps: condition: ne(variables['python.version'], '') - script: | - type python - python --version - displayName: Show python version + set -o errexit + set -o nounset + set -o pipefail -- script: | sudo apt-get update sudo apt-get install -y python2-dev python3-pip virtualenv displayName: Install build deps condition: and(eq(variables['python.version'], ''), eq(variables['Agent.OS'], 'Linux')) -- script: python -mpip install "tox<4.0" +- script: | + set -o errexit + set -o nounset + set -o pipefail + + # macOS builders lack a realpath command + type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version + type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version + type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version + echo + + if [ -e /usr/bin/python ]; then + echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2 ]; then + echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2.7 ]; then + echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')" + fi + displayName: Show python versions + +- script: | + set -o errexit + set -o nounset + set -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "$(tox.env)"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then + "$PYTHON" -m ensurepip --user --altinstall --no-default-pip + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + elif [[ $PYTHON == "python2.7" ]]; then + curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py" + "$PYTHON" get-pip.py --user --no-python-version-warning + # Avoid Python 2.x pip masking system pip + rm -f ~/.local/bin/{easy_install,pip,wheel} + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + else + "$PYTHON" -m pip install -r "tests/requirements-tox.txt" + fi displayName: Install tooling -- script: python -mtox -e "$(tox.env)" +- script: | + set -o errexit + set -o nounset + set -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "$(tox.env)"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + "$PYTHON" -m tox -e "$(tox.env)" displayName: "Run tests" env: AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index f672240d..3630f83e 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -21,25 +21,21 @@ jobs: matrix: Mito_27: tox.env: py27-mode_mitogen - Mito_311: - python.version: '3.11' - tox.env: py311-mode_mitogen + Mito_312: + python.version: '3.12' + tox.env: py312-mode_mitogen - # TODO: test python3, python3 tests are broken Loc_27_210: tox.env: py27-mode_localhost-ansible2.10 - Loc_27_4: - tox.env: py27-mode_localhost-ansible4 + Loc_312_6: + python.version: '3.12' + tox.env: py312-mode_localhost-ansible6 - # NOTE: this hangs when ran in Ubuntu 18.04 Van_27_210: - tox.env: py27-mode_localhost-ansible2.10 - STRATEGY: linear - ANSIBLE_SKIP_TAGS: resource_intensive - Van_27_4: - tox.env: py27-mode_localhost-ansible4 - STRATEGY: linear - ANSIBLE_SKIP_TAGS: resource_intensive + tox.env: py27-mode_localhost-ansible2.10-strategy_linear + Van_312_6: + python.version: '3.12' + tox.env: py312-mode_localhost-ansible6-strategy_linear - job: Linux pool: @@ -96,33 +92,33 @@ jobs: python.version: '3.6' tox.env: py36-mode_mitogen-distro_ubuntu2004 - Mito_311_centos6: - python.version: '3.11' - tox.env: py311-mode_mitogen-distro_centos6 - Mito_311_centos7: - python.version: '3.11' - tox.env: py311-mode_mitogen-distro_centos7 - Mito_311_centos8: - python.version: '3.11' - tox.env: py311-mode_mitogen-distro_centos8 - Mito_311_debian9: - python.version: '3.11' - tox.env: py311-mode_mitogen-distro_debian9 - Mito_311_debian10: - python.version: '3.11' - tox.env: py311-mode_mitogen-distro_debian10 - Mito_311_debian11: - python.version: '3.11' - tox.env: py311-mode_mitogen-distro_debian11 - Mito_311_ubuntu1604: - python.version: '3.11' - tox.env: py311-mode_mitogen-distro_ubuntu1604 - Mito_311_ubuntu1804: - python.version: '3.11' - tox.env: py311-mode_mitogen-distro_ubuntu1804 - Mito_311_ubuntu2004: - python.version: '3.11' - tox.env: py311-mode_mitogen-distro_ubuntu2004 + Mito_312_centos6: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_centos6 + Mito_312_centos7: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_centos7 + Mito_312_centos8: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_centos8 + Mito_312_debian9: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_debian9 + Mito_312_debian10: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_debian10 + Mito_312_debian11: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_debian11 + Mito_312_ubuntu1604: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_ubuntu1604 + Mito_312_ubuntu1804: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_ubuntu1804 + Mito_312_ubuntu2004: + python.version: '3.12' + tox.env: py312-mode_mitogen-distro_ubuntu2004 Ans_27_210: tox.env: py27-mode_ansible-ansible2.10 @@ -148,6 +144,6 @@ jobs: Ans_311_5: python.version: '3.11' tox.env: py311-mode_ansible-ansible5 - Ans_311_6: - python.version: '3.11' - tox.env: py311-mode_ansible-ansible6 + Ans_312_6: + python.version: '3.12' + tox.env: py312-mode_ansible-ansible6 diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index 69d67cd1..c50ef220 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -1,6 +1,10 @@ #!/usr/bin/env python # Run tests/ansible/all.yml under Ansible and Ansible-Mitogen +from __future__ import print_function + +import getpass +import io import os import subprocess import sys @@ -53,6 +57,38 @@ with ci_lib.Fold('machine_prep'): os.chdir(IMAGE_PREP_DIR) ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml") + # FIXME Don't hardcode https://github.com/mitogen-hq/mitogen/issues/1022 + # and os.environ['USER'] is not populated on Azure macOS runners. + os.chdir(HOSTS_DIR) + with io.open('default.hosts', 'r+', encoding='utf-8') as f: + user = getpass.getuser() + content = f.read() + content = content.replace("{{ lookup('pipe', 'whoami') }}", user) + f.seek(0) + f.write(content) + f.truncate() + ci_lib.dump_file('default.hosts') + + cmd = ';'.join([ + 'from __future__ import print_function', + 'import os, sys', + 'print(sys.executable, os.path.realpath(sys.executable))', + ]) + for interpreter in ['/usr/bin/python', '/usr/bin/python2', '/usr/bin/python2.7']: + print(interpreter) + try: + subprocess.call([interpreter, '-c', cmd]) + except OSError as exc: + print(exc) + + print(interpreter, 'with PYTHON_LAUNCHED_FROM_WRAPPER=1') + environ = os.environ.copy() + environ['PYTHON_LAUNCHED_FROM_WRAPPER'] = '1' + try: + subprocess.call([interpreter, '-c', cmd], env=environ) + except OSError as exc: + print(exc) + with ci_lib.Fold('ansible'): os.chdir(TESTS_DIR) diff --git a/.gitignore b/.gitignore index aa75f691..7297d720 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ venvs/** *.pyc *.pyd *.pyo +*.retry MANIFEST build/ dist/ diff --git a/ansible_mitogen/module_finder.py b/ansible_mitogen/module_finder.py index cec465c1..a1870833 100644 --- a/ansible_mitogen/module_finder.py +++ b/ansible_mitogen/module_finder.py @@ -31,15 +31,31 @@ from __future__ import unicode_literals __metaclass__ = type import collections -import imp +import logging import os +import re +import sys + +try: + # Python >= 3.4, PEP 451 ModuleSpec API + import importlib.machinery + import importlib.util +except ImportError: + # Python < 3.4, PEP 302 Import Hooks + import imp import mitogen.master +LOG = logging.getLogger(__name__) PREFIX = 'ansible.module_utils.' +# Analog of `importlib.machinery.ModuleSpec` or `pkgutil.ModuleInfo`. +# name Unqualified name of the module. +# path Filesystem path of the module. +# kind One of the constants in `imp`, as returned in `imp.find_module()` +# parent `ansible_mitogen.module_finder.Module` of parent package (if any). Module = collections.namedtuple('Module', 'name path kind parent') @@ -119,14 +135,121 @@ def find_relative(parent, name, path=()): def scan_fromlist(code): + """Return an iterator of (level, name) for explicit imports in a code + object. + + Not all names identify a module. `from os import name, path` generates + `(0, 'os.name'), (0, 'os.path')`, but `os.name` is usually a string. + + >>> src = 'import a; import b.c; from d.e import f; from g import h, i\\n' + >>> code = compile(src, '', 'exec') + >>> list(scan_fromlist(code)) + [(0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')] + """ for level, modname_s, fromlist in mitogen.master.scan_code_imports(code): for name in fromlist: - yield level, '%s.%s' % (modname_s, name) + yield level, str('%s.%s' % (modname_s, name)) if not fromlist: yield level, modname_s +def walk_imports(code, prefix=None): + """Return an iterator of names for implicit parent imports & explicit + imports in a code object. + + If a prefix is provided, then only children of that prefix are included. + Not all names identify a module. `from os import name, path` generates + `'os', 'os.name', 'os.path'`, but `os.name` is usually a string. + + >>> source = 'import a; import b; import b.c; from b.d import e, f\\n' + >>> code = compile(source, '', 'exec') + >>> list(walk_imports(code)) + ['a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f'] + >>> list(walk_imports(code, prefix='b')) + ['b.c', 'b.d', 'b.d.e', 'b.d.f'] + """ + if prefix is None: + prefix = '' + pattern = re.compile(r'(^|\.)(\w+)') + start = len(prefix) + for _, name, fromlist in mitogen.master.scan_code_imports(code): + if not name.startswith(prefix): + continue + for match in pattern.finditer(name, start): + yield name[:match.end()] + for leaf in fromlist: + yield str('%s.%s' % (name, leaf)) + + def scan(module_name, module_path, search_path): + # type: (str, str, list[str]) -> list[(str, str, bool)] + """Return a list of (name, path, is_package) for ansible.module_utils + imports used by an Ansible module. + """ + log = LOG.getChild('scan') + log.debug('%r, %r, %r', module_name, module_path, search_path) + + if sys.version_info >= (3, 4): + result = _scan_importlib_find_spec( + module_name, module_path, search_path, + ) + log.debug('_scan_importlib_find_spec %r', result) + else: + result = _scan_imp_find_module(module_name, module_path, search_path) + log.debug('_scan_imp_find_module %r', result) + return result + + +def _scan_importlib_find_spec(module_name, module_path, search_path): + # type: (str, str, list[str]) -> list[(str, str, bool)] + module = importlib.machinery.ModuleSpec( + module_name, loader=None, origin=module_path, + ) + prefix = importlib.machinery.ModuleSpec( + PREFIX.rstrip('.'), loader=None, + ) + prefix.submodule_search_locations = search_path + queue = collections.deque([module]) + specs = {prefix.name: prefix} + while queue: + spec = queue.popleft() + if spec.origin is None: + continue + try: + with open(spec.origin, 'rb') as f: + code = compile(f.read(), spec.name, 'exec') + except Exception as exc: + raise ValueError((exc, module, spec, specs)) + + for name in walk_imports(code, prefix.name): + if name in specs: + continue + + parent_name = name.rpartition('.')[0] + parent = specs[parent_name] + if parent is None or not parent.submodule_search_locations: + specs[name] = None + continue + + child = importlib.util._find_spec( + name, parent.submodule_search_locations, + ) + if child is None or child.origin is None: + specs[name] = None + continue + + specs[name] = child + queue.append(child) + + del specs[prefix.name] + return sorted( + (spec.name, spec.origin, spec.submodule_search_locations is not None) + for spec in specs.values() if spec is not None + ) + + +def _scan_imp_find_module(module_name, module_path, search_path): + # type: (str, str, list[str]) -> list[(str, str, bool)] module = Module(module_name, module_path, imp.PY_SOURCE, None) stack = [module] seen = set() diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index c4cb71ff..8da1b670 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -40,7 +40,6 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type import atexit -import imp import json import os import re @@ -64,6 +63,14 @@ except ImportError: # Python 2.4 ctypes = None +try: + # Python >= 3.4, PEP 451 ModuleSpec API + import importlib.machinery + import importlib.util +except ImportError: + # Python < 3.4, PEP 302 Import Hooks + import imp + try: # Cannot use cStringIO as it does not support Unicode. from StringIO import StringIO @@ -514,10 +521,71 @@ class ModuleUtilsImporter(object): sys.modules.pop(fullname, None) def find_module(self, fullname, path=None): + """ + Return a loader for the module with fullname, if we will load it. + + Implements importlib.abc.MetaPathFinder.find_module(). + Deprecrated in Python 3.4+, replaced by find_spec(). + Raises ImportWarning in Python 3.10+. Removed in Python 3.12. + """ if fullname in self._by_fullname: return self + def find_spec(self, fullname, path, target=None): + """ + Return a `ModuleSpec` for module with `fullname` if we will load it. + Otherwise return `None`. + + Implements importlib.abc.MetaPathFinder.find_spec(). Python 3.4+. + """ + if fullname.endswith('.'): + return None + + try: + module_path, is_package = self._by_fullname[fullname] + except KeyError: + LOG.debug('Skipping %s: not present', fullname) + return None + + LOG.debug('Handling %s', fullname) + origin = 'master:%s' % (module_path,) + return importlib.machinery.ModuleSpec( + fullname, loader=self, origin=origin, is_package=is_package, + ) + + def create_module(self, spec): + """ + Return a module object for the given ModuleSpec. + + Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4. + Unlike Loader.load_module() this shouldn't populate sys.modules or + set module attributes. Both are done by Python. + """ + module = types.ModuleType(spec.name) + # FIXME create_module() shouldn't initialise module attributes + module.__file__ = spec.origin + return module + + def exec_module(self, module): + """ + Execute the module to initialise it. Don't return anything. + + Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4. + """ + spec = module.__spec__ + path, _ = self._by_fullname[spec.name] + source = ansible_mitogen.target.get_small_file(self._context, path) + code = compile(source, path, 'exec', 0, 1) + exec(code, module.__dict__) + self._loaded.add(spec.name) + def load_module(self, fullname): + """ + Return the loaded module specified by fullname. + + Implements PEP 302 importlib.abc.Loader.load_module(). + Deprecated in Python 3.4+, replaced by create_module() & exec_module(). + """ path, is_pkg = self._by_fullname[fullname] source = ansible_mitogen.target.get_small_file(self._context, path) code = compile(source, path, 'exec', 0, 1) @@ -818,12 +886,17 @@ class NewStyleRunner(ScriptRunner): synchronization mechanism by importing everything the module will need prior to detaching. """ + # I think "custom" means "found in custom module_utils search path", + # e.g. playbook relative dir, ~/.ansible/..., Ansible collection. for fullname, _, _ in self.module_map['custom']: mitogen.core.import_module(fullname) + + # I think "builtin" means "part of ansible/ansible-base/ansible-core", + # as opposed to Python builtin modules such as sys. for fullname in self.module_map['builtin']: try: mitogen.core.import_module(fullname) - except ImportError: + except ImportError as exc: # #590: Ansible 2.8 module_utils.distro is a package that # replaces itself in sys.modules with a non-package during # import. Prior to replacement, it is a real package containing @@ -834,8 +907,18 @@ class NewStyleRunner(ScriptRunner): # loop progresses to the next entry and attempts to preload # 'distro._distro', the import mechanism will fail. So here we # silently ignore any failure for it. - if fullname != 'ansible.module_utils.distro._distro': - raise + if fullname == 'ansible.module_utils.distro._distro': + continue + + # ansible.module_utils.compat.selinux raises ImportError if it + # can't load libselinux.so. The importer would usually catch + # this & skip selinux operations. We don't care about selinux, + # we're using import to get a copy of the module. + if (fullname == 'ansible.module_utils.compat.selinux' + and exc.msg == 'unable to load libselinux.so'): + continue + + raise def _setup_excepthook(self): """ diff --git a/ansible_mitogen/utils.py b/ansible_mitogen/utils.py index 25c5a692..a01b261d 100644 --- a/ansible_mitogen/utils.py +++ b/ansible_mitogen/utils.py @@ -1,7 +1,7 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type -import distutils.version +import re import ansible @@ -9,6 +9,21 @@ __all__ = [ 'ansible_version', ] -ansible_version = tuple(distutils.version.LooseVersion(ansible.__version__).version) -del distutils + +def _parse(v_string): + # Adapted from distutils.version.LooseVersion.parse() + component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE) + for component in component_re.split(v_string): + if not component or component == '.': + continue + try: + yield int(component) + except ValueError: + yield component + + +ansible_version = tuple(_parse(ansible.__version__)) + +del _parse +del re del ansible diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index 5679537e..cb83aa71 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -149,7 +149,8 @@ Noteworthy Differences Mitogen 0.3.1+ supports - Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.11 - - Ansible 5 and 6; with Python 3.8-3.11 + - Ansible 5; with Python 3.8-3.11 + - Ansible 6; with Python 3.8-3.12 Verify your installation is running one of these versions by checking ``ansible --version`` output. diff --git a/docs/changelog.rst b/docs/changelog.rst index ef26a047..126095fd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,11 @@ Unreleased ---------- * :gh:issue:`987` Support Python 3.11 +* :gh:issue:`885` Fix :py:exc:`PermissionError` in :py:mod:`importlib` when + becoming an unprivileged user with Python 3.x +* :gh:issue:`1033` Support `PEP 451 , + required by Python 3.12 +* :gh:issue:`1033` Support Python 3.12 v0.3.4 (2023-07-02) diff --git a/docs/conf.py b/docs/conf.py index 8350c79e..3a7fc002 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,9 +1,8 @@ import sys -sys.path.append('..') sys.path.append('.') -import mitogen -VERSION = '.'.join(str(part) for part in mitogen.__version__) + +VERSION = '0.3.4' author = u'Network Genomics' copyright = u'2021, the Mitogen authors' @@ -83,10 +82,16 @@ domainrefs = { }, } +# > ## Official guidance +# > Query PyPI’s JSON API to determine where to download files from. +# > ## Predictable URLs +# > You can use our conveyor service to fetch this file, which exists for +# > cases where using the API is impractical or impossible. +# > -- https://warehouse.pypa.io/api-reference/integration-guide.html#predictable-urls rst_epilog = """ .. |mitogen_version| replace:: %(VERSION)s -.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz `__ +.. |mitogen_url| replace:: `mitogen-%(VERSION)s.tar.gz `__ """ % locals() diff --git a/docs/contributors.rst b/docs/contributors.rst index 584c4cd4..61a9eb1b 100644 --- a/docs/contributors.rst +++ b/docs/contributors.rst @@ -138,4 +138,5 @@ sponsorship and outstanding future-thinking of its early adopters.
  • randy — desperate for automation
  • Michael & Vicky Twomey-Lee
  • Wesley Moore
    • +
  • Witold Baryluk
    • diff --git a/mitogen/core.py b/mitogen/core.py index bee722e6..cd02012f 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -34,6 +34,34 @@ non-essential code in order to reduce its size, since it is also serves as the bootstrap implementation sent to every new slave context. """ +import sys +try: + import _frozen_importlib_external +except ImportError: + pass +else: + class MonkeyPatchedPathFinder(_frozen_importlib_external.PathFinder): + """ + Meta path finder for sys.path and package __path__ attributes. + + Patched for https://github.com/python/cpython/issues/115911. + """ + @classmethod + def _path_importer_cache(cls, path): + if path == '': + try: + path = _frozen_importlib_external._os.getcwd() + except (FileNotFoundError, PermissionError): + return None + return super()._path_importer_cache(path) + + if sys.version_info[:2] <= (3, 12): + for i, mpf in enumerate(sys.meta_path): + if mpf is _frozen_importlib_external.PathFinder: + sys.meta_path[i] = MonkeyPatchedPathFinder + del i, mpf + + import binascii import collections import encodings.latin_1 @@ -49,18 +77,22 @@ import pstats import signal import socket import struct -import sys import syslog import threading import time import traceback +import types import warnings import weakref import zlib -# Python >3.7 deprecated the imp module. -warnings.filterwarnings('ignore', message='the imp module is deprecated') -import imp +try: + # Python >= 3.4, PEP 451 ModuleSpec API + import importlib.machinery + import importlib.util +except ImportError: + # Python < 3.4, PEP 302 Import Hooks + import imp # Absolute imports for <2.5. select = __import__('select') @@ -1353,6 +1385,19 @@ class Importer(object): def __repr__(self): return 'Importer' + @staticmethod + def _loader_from_module(module, default=None): + """Return the loader for a module object.""" + try: + return module.__spec__.loader + except AttributeError: + pass + try: + return module.__loader__ + except AttributeError: + pass + return default + def builtin_find_module(self, fullname): # imp.find_module() will always succeed for __main__, because it is a # built-in module. That means it exists on a special linked list deep @@ -1360,12 +1405,19 @@ class Importer(object): if fullname == '__main__': raise ModuleNotFoundError() + # For a module inside a package (e.g. pkg_a.mod_b) use the search path + # of that package (e.g. ['/usr/lib/python3.11/site-packages/pkg_a']). parent, _, modname = str_rpartition(fullname, '.') if parent: path = sys.modules[parent].__path__ else: path = None + # For a top-level module search builtin modules, frozen modules, + # system specific locations (e.g. Windows registry, site-packages). + # Otherwise use search path of the parent package. + # Works for both stdlib modules & third-party modules. + # If the search is unsuccessful then raises ImportError. fp, pathname, description = imp.find_module(modname, path) if fp: fp.close() @@ -1377,8 +1429,9 @@ class Importer(object): Implements importlib.abc.MetaPathFinder.find_module(). Deprecrated in Python 3.4+, replaced by find_spec(). Raises ImportWarning in Python 3.10+. + Removed in Python 3.12. - fullname A (fully qualified?) module name, e.g. "os.path". + fullname Fully qualified module name, e.g. "os.path". path __path__ of parent packge. None for a top level module. """ if hasattr(_tls, 'running'): @@ -1388,14 +1441,13 @@ class Importer(object): try: #_v and self._log.debug('Python requested %r', fullname) fullname = to_text(fullname) - pkgname, dot, _ = str_rpartition(fullname, '.') + pkgname, _, suffix = str_rpartition(fullname, '.') pkg = sys.modules.get(pkgname) if pkgname and getattr(pkg, '__loader__', None) is not self: self._log.debug('%s is submodule of a locally loaded package', fullname) return None - suffix = fullname[len(pkgname+dot):] if pkgname and suffix not in self._present.get(pkgname, ()): self._log.debug('%s has no submodule %s', pkgname, suffix) return None @@ -1415,6 +1467,66 @@ class Importer(object): finally: del _tls.running + def find_spec(self, fullname, path, target=None): + """ + Return a `ModuleSpec` for module with `fullname` if we will load it. + Otherwise return `None`, allowing other finders to try. + + fullname Fully qualified name of the module (e.g. foo.bar.baz) + path Path entries to search. None for a top-level module. + target Existing module to be reloaded (if any). + + Implements importlib.abc.MetaPathFinder.find_spec() + Python 3.4+. + """ + # Presence of _tls.running indicates we've re-invoked importlib. + # Abort early to prevent infinite recursion. See below. + if hasattr(_tls, 'running'): + return None + + log = self._log.getChild('find_spec') + + if fullname.endswith('.'): + return None + + pkgname, _, modname = fullname.rpartition('.') + if pkgname and modname not in self._present.get(pkgname, ()): + log.debug('Skipping %s. Parent %s has no submodule %s', + fullname, pkgname, modname) + return None + + pkg = sys.modules.get(pkgname) + pkg_loader = self._loader_from_module(pkg) + if pkgname and pkg_loader is not self: + log.debug('Skipping %s. Parent %s was loaded by %r', + fullname, pkgname, pkg_loader) + return None + + # #114: whitelisted prefixes override any system-installed package. + if self.whitelist != ['']: + if any(s and fullname.startswith(s) for s in self.whitelist): + log.debug('Handling %s. It is whitelisted', fullname) + return importlib.machinery.ModuleSpec(fullname, loader=self) + + if fullname == '__main__': + log.debug('Handling %s. A special case', fullname) + return importlib.machinery.ModuleSpec(fullname, loader=self) + + # Re-invoke the import machinery to allow other finders to try. + # Set a guard, so we don't infinitely recurse. See top of this method. + _tls.running = True + try: + spec = importlib.util._find_spec(fullname, path, target) + finally: + del _tls.running + + if spec: + log.debug('Skipping %s. Available as %r', fullname, spec) + return spec + + log.debug('Handling %s. Unavailable locally', fullname) + return importlib.machinery.ModuleSpec(fullname, loader=self) + blacklisted_msg = ( '%r is present in the Mitogen importer blacklist, therefore this ' 'context will not attempt to request it from the master, as the ' @@ -1501,6 +1613,64 @@ class Importer(object): if present: callback() + def create_module(self, spec): + """ + Return a module object for the given ModuleSpec. + + Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4. + Unlike Loader.load_module() this shouldn't populate sys.modules or + set module attributes. Both are done by Python. + """ + self._log.debug('Creating module for %r', spec) + + # FIXME Should this be done in find_spec()? Can it? + self._refuse_imports(spec.name) + + # FIXME "create_module() should properly handle the case where it is + # called more than once for the same spec/module." -- PEP-451 + event = threading.Event() + self._request_module(spec.name, callback=event.set) + event.wait() + + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + _, pkg_present, path, _, _ = self._cache[spec.name] + + if path is None: + raise ImportError(self.absent_msg % (spec.name)) + + spec.origin = self.get_filename(spec.name) + if pkg_present is not None: + # TODO Namespace packages + spec.submodule_search_locations = [] + self._present[spec.name] = pkg_present + + module = types.ModuleType(spec.name) + # FIXME create_module() shouldn't initialise module attributes + module.__file__ = spec.origin + return module + + def exec_module(self, module): + """ + Execute the module to initialise it. Don't return anything. + + Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4. + """ + name = module.__spec__.name + origin = module.__spec__.origin + self._log.debug('Executing %s from %s', name, origin) + source = self.get_source(name) + try: + # Compile the source into a code object. Don't add any __future__ + # flags and don't inherit any from this module. + # FIXME Should probably be exposed as get_code() + code = compile(source, origin, 'exec', flags=0, dont_inherit=True) + except SyntaxError: + # FIXME Why is this LOG, rather than self._log? + LOG.exception('while importing %r', name) + raise + + exec(code, module.__dict__) + def load_module(self, fullname): """ Return the loaded module specified by fullname. @@ -1516,11 +1686,11 @@ class Importer(object): self._request_module(fullname, event.set) event.wait() - ret = self._cache[fullname] - if ret[2] is None: + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + _, pkg_present, path, _, _ = self._cache[fullname] + if path is None: raise ModuleNotFoundError(self.absent_msg % (fullname,)) - pkg_present = ret[1] mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) mod.__file__ = self.get_filename(fullname) mod.__loader__ = self @@ -3921,7 +4091,7 @@ class ExternalContext(object): def _setup_package(self): global mitogen - mitogen = imp.new_module('mitogen') + mitogen = types.ModuleType('mitogen') mitogen.__package__ = 'mitogen' mitogen.__path__ = [] mitogen.__loader__ = self.importer diff --git a/mitogen/master.py b/mitogen/master.py index 4fb535f0..b1e0a1de 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -37,7 +37,6 @@ contexts. import dis import errno -import imp import inspect import itertools import logging @@ -50,6 +49,16 @@ import threading import types import zlib +try: + # Python >= 3.4, PEP 451 ModuleSpec API + import importlib.machinery + import importlib.util + from _imp import is_builtin as _is_builtin +except ImportError: + # Python < 3.4, PEP 302 Import Hooks + import imp + from imp import is_builtin as _is_builtin + try: import sysconfig except ImportError: @@ -122,14 +131,16 @@ def is_stdlib_name(modname): """ Return :data:`True` if `modname` appears to come from the standard library. """ - # `imp.is_builtin()` isn't a documented as part of Python's stdlib API. + # `(_imp|imp).is_builtin()` isn't a documented part of Python's stdlib. + # Returns 1 if modname names a module that is "builtin" to the the Python + # interpreter (e.g. '_sre'). Otherwise 0 (e.g. 're', 'netifaces'). # # """ # Main is a little special - imp.is_builtin("__main__") will return False, # but BuiltinImporter is still the most appropriate initial setting for # its __loader__ attribute. # """ -- comment in CPython pylifecycle.c:add_main_module() - if imp.is_builtin(modname) != 0: + if _is_builtin(modname) != 0: return True module = sys.modules.get(modname) @@ -460,6 +471,9 @@ class FinderMethod(object): name according to the running Python interpreter. You'd think this was a simple task, right? Naive young fellow, welcome to the real world. """ + def __init__(self): + self.log = LOG.getChild(self.__class__.__name__) + def __repr__(self): return '%s()' % (type(self).__name__,) @@ -641,7 +655,7 @@ class SysModulesMethod(FinderMethod): return path, source, is_pkg -class ParentEnumerationMethod(FinderMethod): +class ParentImpEnumerationMethod(FinderMethod): """ Attempt to fetch source code by examining the module's (hopefully less insane) parent package, and if no insane parents exist, simply use @@ -759,6 +773,7 @@ class ParentEnumerationMethod(FinderMethod): def _find_one_component(self, modname, search_path): try: #fp, path, (suffix, _, kind) = imp.find_module(modname, search_path) + # FIXME The imp module was removed in Python 3.12. return imp.find_module(modname, search_path) except ImportError: e = sys.exc_info()[1] @@ -770,6 +785,9 @@ class ParentEnumerationMethod(FinderMethod): """ See implementation for a description of how this works. """ + if sys.version_info >= (3, 4): + return None + #if fullname not in sys.modules: # Don't attempt this unless a module really exists in sys.modules, # else we could return junk. @@ -798,6 +816,99 @@ class ParentEnumerationMethod(FinderMethod): return self._found_module(fullname, path, fp) +class ParentSpecEnumerationMethod(ParentImpEnumerationMethod): + def _find_parent_spec(self, fullname): + #history = [] + debug = self.log.debug + children = [] + for parent_name, child_name in self._iter_parents(fullname): + children.insert(0, child_name) + if not parent_name: + debug('abandoning %r, reached top-level', fullname) + return None, children + + try: + parent = sys.modules[parent_name] + except KeyError: + debug('skipping %r, not in sys.modules', parent_name) + continue + + try: + spec = parent.__spec__ + except AttributeError: + debug('skipping %r: %r.__spec__ is absent', + parent_name, parent) + continue + + if not spec: + debug('skipping %r: %r.__spec__=%r', + parent_name, parent, spec) + continue + + if spec.name != parent_name: + debug('skipping %r: %r.__spec__.name=%r does not match', + parent_name, parent, spec.name) + continue + + if not spec.submodule_search_locations: + debug('skipping %r: %r.__spec__.submodule_search_locations=%r', + parent_name, parent, spec.submodule_search_locations) + continue + + return spec, children + + raise ValueError('%s._find_parent_spec(%r) unexpectedly reached bottom' + % (self.__class__.__name__, fullname)) + + def find(self, fullname): + # Returns absolute path, ParentImpEnumerationMethod returns relative + # >>> spec_pem.find('six_brokenpkg._six')[::2] + # ('/Users/alex/src/mitogen/tests/data/importer/six_brokenpkg/_six.py', False) + + if sys.version_info < (3, 4): + return None + + fullname = to_text(fullname) + spec, children = self._find_parent_spec(fullname) + for child_name in children: + if spec: + name = '%s.%s' % (spec.name, child_name) + submodule_search_locations = spec.submodule_search_locations + else: + name = child_name + submodule_search_locations = None + spec = importlib.util._find_spec(name, submodule_search_locations) + if spec is None: + self.log.debug('%r spec unavailable from %s', fullname, spec) + return None + + is_package = spec.submodule_search_locations is not None + if name != fullname: + if not is_package: + self.log.debug('%r appears to be child of non-package %r', + fullname, spec) + return None + continue + + if not spec.has_location: + self.log.debug('%r.origin cannot be read as a file', spec) + return None + + if os.path.splitext(spec.origin)[1] != '.py': + self.log.debug('%r.origin does not contain Python source code', + spec) + return None + + # FIXME This should use loader.get_source() + with open(spec.origin, 'rb') as f: + source = f.read() + + return spec.origin, source, is_package + + raise ValueError('%s.find(%r) unexpectedly reached bottom' + % (self.__class__.__name__, fullname)) + + class ModuleFinder(object): """ Given the name of a loaded module, make a best-effort attempt at finding @@ -838,7 +949,8 @@ class ModuleFinder(object): DefectivePython3xMainMethod(), PkgutilMethod(), SysModulesMethod(), - ParentEnumerationMethod(), + ParentSpecEnumerationMethod(), + ParentImpEnumerationMethod(), ] def get_module_source(self, fullname): diff --git a/mitogen/parent.py b/mitogen/parent.py index 59ee1685..29bcf66d 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -34,7 +34,7 @@ sent to any child context that is due to become a parent, due to recursive connection. """ -import codecs +import binascii import errno import fcntl import getpass @@ -1405,10 +1405,14 @@ class Connection(object): # file descriptor 0 as 100, creates a pipe, then execs a new interpreter # with a custom argv. # * Optimized for minimum byte count after minification & compression. + # The script preamble_size.py measures this. # * 'CONTEXT_NAME' and 'PREAMBLE_COMPRESSED_LEN' are substituted with # their respective values. # * CONTEXT_NAME must be prefixed with the name of the Python binary in # order to allow virtualenvs to detect their install prefix. + # + # macOS tweaks for Python 2.7 must be kept in sync with the the Ansible + # module test_echo_module, used by the integration tests. # * macOS <= 10.14 (Darwin <= 18) install an unreliable Python version # switcher as /usr/bin/python, which introspects argv0. To workaround # it we redirect attempts to call /usr/bin/python with an explicit @@ -1417,7 +1421,8 @@ class Connection(object): # do something slightly different. The Python executable is patched to # perform an extra execvp(). I don't fully understand the details, but # setting PYTHON_LAUNCHED_FROM_WRAPPER=1 avoids it. - # * macOS 13.x (Darwin 22?) may remove python 2.x entirely. + # * macOS 12.3+ (Darwin 21.4+, Monterey) doesn't ship Python. + # https://developer.apple.com/documentation/macos-release-notes/macos-12_3-release-notes#Python # # Locals: # R: read side of interpreter stdin. @@ -1445,7 +1450,7 @@ class Connection(object): os.environ['ARGV0']=sys.executable os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)') os.write(1,'MITO000\n'.encode()) - C=_(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN),'zip') + C=zlib.decompress(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN)) fp=os.fdopen(W,'wb',0) fp.write(C) fp.close() @@ -1477,16 +1482,16 @@ class Connection(object): source = source.replace('PREAMBLE_COMPRESSED_LEN', str(len(preamble_compressed))) compressed = zlib.compress(source.encode(), 9) - encoded = codecs.encode(compressed, 'base64').replace(b('\n'), b('')) - # We can't use bytes.decode() in 3.x since it was restricted to always - # return unicode, so codecs.decode() is used instead. In 3.x - # codecs.decode() requires a bytes object. Since we must be compatible - # with 2.4 (no bytes literal), an extra .encode() either returns the - # same str (2.x) or an equivalent bytes (3.x). + encoded = binascii.b2a_base64(compressed).replace(b('\n'), b('')) + + # Just enough to decode, decompress, and exec the first stage. + # Priorities: wider compatibility, faster startup, shorter length. + # `import os` here, instead of stage 1, to save a few bytes. + # `sys.path=...` for https://github.com/python/cpython/issues/115911. return self.get_python_argv() + [ '-c', - 'import codecs,os,sys;_=codecs.decode;' - 'exec(_(_("%s".encode(),"base64"),"zip"))' % (encoded.decode(),) + 'import sys;sys.path=[p for p in sys.path if p];import binascii,os,zlib;' + 'exec(zlib.decompress(binascii.a2b_base64("%s")))' % (encoded.decode(),), ] def get_econtext_config(self): diff --git a/setup.py b/setup.py index 4d7fadfc..b17dab9d 100644 --- a/setup.py +++ b/setup.py @@ -78,6 +78,7 @@ setup( 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: System :: Distributed Computing', 'Topic :: System :: Systems Administration', diff --git a/tests/ansible/files/cwd_show b/tests/ansible/files/cwd_show new file mode 100755 index 00000000..42ef3194 --- /dev/null +++ b/tests/ansible/files/cwd_show @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# Show permissions and identities that impact the current working directory. +# On macOS libc cwd() can return EACCES after su or sudo. +# See also +# - https://github.com/ansible/ansible/pull/7078 +# - https://github.com/python/cpython/issues/115911 + +set -o errexit +set -o nounset +set -o pipefail + +whoami +groups +pwd + +d=$(pwd) +while [[ "$d" != "/" && -n "$d" ]]; do + ls -ld "$d" + d=$(dirname "$d") +done +ls -ld / diff --git a/tests/ansible/hosts/become_same_user.hosts b/tests/ansible/hosts/become_same_user.hosts index a18b90d2..ac744ed7 100644 --- a/tests/ansible/hosts/become_same_user.hosts +++ b/tests/ansible/hosts/become_same_user.hosts @@ -1,3 +1,5 @@ +# code: language=ini +# vim: syntax=dosini # become_same_user.yml bsu-joe ansible_user=joe diff --git a/tests/ansible/hosts/connection_delegation.hosts b/tests/ansible/hosts/connection_delegation.hosts index a22bd5df..4ae861b0 100644 --- a/tests/ansible/hosts/connection_delegation.hosts +++ b/tests/ansible/hosts/connection_delegation.hosts @@ -1,3 +1,4 @@ +# code: language=ini # vim: syntax=dosini # Connection delegation scenarios. It's impossible to connect to them, but their would-be diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts index d40c3dd0..1bec0014 100644 --- a/tests/ansible/hosts/default.hosts +++ b/tests/ansible/hosts/default.hosts @@ -1,9 +1,12 @@ +# code: language=ini # vim: syntax=dosini # When running the tests outside CI, make a single 'target' host which is the # local machine. The ansible_user override is necessary since some tests want a # fixed ansible.cfg remote_user setting to test against. -target ansible_host=localhost ansible_user="{{lookup('env', 'USER')}}" +# FIXME Hardcoded by replacement in some CI runs https://github.com/mitogen-hq/mitogen/issues/1022 +# and os.environ['USER'] is not populated on Azure macOS runners. +target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}" [test-targets] target diff --git a/tests/ansible/hosts/k3.hosts b/tests/ansible/hosts/k3.hosts index 34e1ff95..b210164b 100644 --- a/tests/ansible/hosts/k3.hosts +++ b/tests/ansible/hosts/k3.hosts @@ -1,3 +1,4 @@ +# code: language=ini # vim: syntax=dosini # Used for manual testing. diff --git a/tests/ansible/hosts/localhost.hosts b/tests/ansible/hosts/localhost.hosts index 41af412e..e42221e7 100644 --- a/tests/ansible/hosts/localhost.hosts +++ b/tests/ansible/hosts/localhost.hosts @@ -1,3 +1,4 @@ +# code: language=ini # vim: syntax=dosini # issue #511, #536: we must not define an explicit localhost, as some diff --git a/tests/ansible/hosts/transport_config.hosts b/tests/ansible/hosts/transport_config.hosts index 7d7b526a..1c1c2e10 100644 --- a/tests/ansible/hosts/transport_config.hosts +++ b/tests/ansible/hosts/transport_config.hosts @@ -1,3 +1,6 @@ +# code: language=ini +# vim: syntax=dosini + # integration/transport_config # Hosts with twiddled configs that need to be checked somehow. @@ -17,11 +20,12 @@ tc_remote_user tc_transport [transport_config_undiscover:vars] -# If python interpreter path is unset, Ansible tries to connect & discover it. -# That causes approx 10 seconds timeout per task - there's no host to connect to. +# If ansible_*_interpreter isn't set Ansible tries to connect & discover it. +# If that target doesn't exist we must wait $timeout seconds for each attempt. +# Setting a known (invalid) interpreter skips discovery & the many timeouts. # This optimisation should not be relied in any test. # Note: tc-python-path-* are intentionally not included. -ansible_python_interpreter = python3000 # Not expected to exist +ansible_python_interpreter = python3000 [tc_transport] tc-transport-unset diff --git a/tests/ansible/integration/become/su_password.yml b/tests/ansible/integration/become/su_password.yml index bd6a0aee..52d420db 100644 --- a/tests/ansible/integration/become/su_password.yml +++ b/tests/ansible/integration/become/su_password.yml @@ -1,5 +1,4 @@ # Verify passwordful su behaviour - # Ansible can't handle this on OS X. I don't care why. - name: integration/become/su_password.yml hosts: test-targets @@ -44,20 +43,54 @@ fail_msg: out={{out}} when: is_mitogen - - name: Ensure password su succeeds. + - name: Ensure password su with chdir succeeds shell: whoami + args: + chdir: ~mitogen__user1 become: true become_user: mitogen__user1 register: out vars: ansible_become_pass: user1_password - when: is_mitogen + when: + # https://github.com/ansible/ansible/pull/70785 + - ansible_facts.distribution not in ["MacOSX"] + or ansible_version.full is version("2.11", ">=", strict=True) + or is_mitogen - assert: that: - out.stdout == 'mitogen__user1' fail_msg: out={{out}} - when: is_mitogen + when: + # https://github.com/ansible/ansible/pull/70785 + - ansible_facts.distribution not in ["MacOSX"] + or ansible_version.full is version("2.11", ">=", strict=True) + or is_mitogen + + - name: Ensure password su without chdir succeeds + shell: whoami + become: true + become_user: mitogen__user1 + register: out + vars: + ansible_become_pass: user1_password + when: + # https://github.com/ansible/ansible/pull/70785 + - ansible_facts.distribution not in ["MacOSX"] + or ansible_version.full is version("2.11", ">=", strict=True) + or is_mitogen + + - assert: + that: + - out.stdout == 'mitogen__user1' + fail_msg: out={{out}} + when: + # https://github.com/ansible/ansible/pull/70785 + - ansible_facts.distribution not in ["MacOSX"] + or ansible_version.full is version("2.11", ">=", strict=True) + or is_mitogen + tags: - su - su_password diff --git a/tests/ansible/integration/connection_delegation/delegate_to_template.yml b/tests/ansible/integration/connection_delegation/delegate_to_template.yml index 3776a7db..7d33a161 100644 --- a/tests/ansible/integration/connection_delegation/delegate_to_template.yml +++ b/tests/ansible/integration/connection_delegation/delegate_to_template.yml @@ -41,7 +41,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - 'python_path': ["/usr/bin/python"], + 'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, @@ -69,7 +69,7 @@ 'keepalive_count': 10, 'password': null, 'port': null, - 'python_path': ["/usr/bin/python"], + 'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"], 'remote_name': null, 'ssh_args': [ -o, ControlMaster=auto, diff --git a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml index ea828aa8..6ac9bada 100644 --- a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml +++ b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml @@ -37,6 +37,7 @@ vars: ansible_python_interpreter: auto test_echo_module: + facts_copy: "{{ ansible_facts }}" register: echoout # can't test this assertion: @@ -44,11 +45,24 @@ # because Mitogen's ansible_python_interpreter is a connection-layer configurable that # "must be extracted during each task execution to form the complete connection-layer configuration". # Discovery won't be reran though; the ansible_python_interpreter is read from the cache if already discovered - - assert: + - name: assert discovered python matches invoked python + assert: that: - auto_out.ansible_facts.discovered_interpreter_python is defined - - echoout.running_python_interpreter == auto_out.ansible_facts.discovered_interpreter_python - fail_msg: auto_out={{auto_out}} echoout={{echoout}} + - auto_out.ansible_facts.discovered_interpreter_python == echoout.discovered_python.as_seen + - echoout.discovered_python.resolved == echoout.running_python.sys.executable.resolved + fail_msg: + - "auto_out: {{ auto_out }}" + - "echoout: {{ echoout }}" + when: + # On macOS 11 (Darwin 20) CI runners the Python 2.7 binary always + # reports the same path. I can't reach via symlinks. + # >>> sys.executable + # /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python + - is_mitogen + or echoout.running_python.sys.version_info.major != 2 + or not (echoout.running_python.sys.platform == "darwin" + and echoout.running_python.platform.release.major == 20) - name: test that auto_legacy gives a dep warning when /usr/bin/python present but != auto result @@ -128,7 +142,8 @@ - name: ensure modules can't set discovered_interpreter_X or ansible_X_interpreter block: - test_echo_module: - facts: + facts_copy: "{{ ansible_facts }}" + facts_to_override: ansible_discovered_interpreter_bogus: from module discovered_interpreter_bogus: from_module ansible_bogus_interpreter: from_module @@ -189,13 +204,6 @@ - distro == 'ubuntu' - distro_version is version('16.04', '>=', strict=True) - - name: mac assertions - assert: - that: - - auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python' - fail_msg: auto_out={{auto_out}} - when: os_family == 'Darwin' - always: - meta: clear_facts when: diff --git a/tests/ansible/integration/interpreter_discovery/complex_args.yml b/tests/ansible/integration/interpreter_discovery/complex_args.yml index 38d10124..6ffff5f4 100644 --- a/tests/ansible/integration/interpreter_discovery/complex_args.yml +++ b/tests/ansible/integration/interpreter_discovery/complex_args.yml @@ -4,6 +4,10 @@ - name: integration/interpreter_discovery/complex_args.yml hosts: test-targets gather_facts: true + environment: + http_proxy: "{{ lookup('env', 'http_proxy') | default(omit) }}" + https_proxy: "{{ lookup('env', 'https_proxy') | default(omit) }}" + no_proxy: "{{ lookup('env', 'no_proxy') | default(omit) }}" tasks: - name: create temp file to source file: @@ -21,28 +25,24 @@ # special_python: source /tmp/fake && python - name: set python using sourced file set_fact: - special_python: source /tmp/fake || true && python + # Avoid 2.x vs 3.x cross-compatiblity issues (that I can't remember the exact details of). + special_python: "source /tmp/fake || true && python{{ ansible_facts.python.version.major }}" - name: run get_url with specially-sourced python get_url: - url: https://google.com + # Plain http for wider Ansible & Python version compatibility + url: http://httpbin.org/get dest: "/tmp/" mode: 0644 - # this url is the build pic from mitogen's github site; some python versions require ssl stuff installed so will disable need to validate certs - validate_certs: no vars: ansible_python_interpreter: "{{ special_python }}" - environment: - https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}" - no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}" - name: run get_url with specially-sourced python including jinja get_url: - url: https://google.com + # Plain http for wider Ansible & Python version compatibility + url: http://httpbin.org/get dest: "/tmp/" mode: 0644 - # this url is the build pic from mitogen's github site; some python versions require ssl stuff installed so will disable need to validate certs - validate_certs: no vars: ansible_python_interpreter: > {% if "1" == "1" %} @@ -50,8 +50,5 @@ {% else %} python {% endif %} - environment: - https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}" - no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}" tags: - complex_args diff --git a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml index 0c620dac..0d7cf1b6 100644 --- a/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml +++ b/tests/ansible/integration/runner/custom_python_new_style_missing_interpreter.yml @@ -2,6 +2,11 @@ - name: integration/runner/custom_python_new_style_module.yml hosts: test-targets tasks: + # FIXME Without Mitogen Ansible often reads stdin before the module. + # Either don't read directly from stdin, or figure out the cause. + - meta: end_play + when: not is_mitogen + - custom_python_new_style_missing_interpreter: foo: true with_sequence: start=0 end={{end|default(1)}} diff --git a/tests/ansible/integration/runner/custom_python_new_style_module.yml b/tests/ansible/integration/runner/custom_python_new_style_module.yml index e2384f81..8435b158 100644 --- a/tests/ansible/integration/runner/custom_python_new_style_module.yml +++ b/tests/ansible/integration/runner/custom_python_new_style_module.yml @@ -1,7 +1,8 @@ - name: integration/runner/custom_python_new_style_module.yml hosts: test-targets tasks: - # without Mitogen Ansible 2.10 hangs on this play + # FIXME Without Mitogen Ansible often reads stdin before the module. + # Either don't read directly from stdin, or figure out the cause. - meta: end_play when: not is_mitogen diff --git a/tests/ansible/integration/runner/custom_python_prehistoric_module.yml b/tests/ansible/integration/runner/custom_python_prehistoric_module.yml index f2a3eefd..ebe34cc8 100644 --- a/tests/ansible/integration/runner/custom_python_prehistoric_module.yml +++ b/tests/ansible/integration/runner/custom_python_prehistoric_module.yml @@ -1,3 +1,7 @@ +# Test functionality of ansible_mitogen.runner.PREHISTORIC_HACK_RE, which +# removes `reload(sys); sys.setdefaultencoding(...)` from an Ansible module +# as it is sent to a target. There are probably very few modules in the wild +# that still do this, if any - reload() is a Python 2.x builtin function. # issue #555 - name: integration/runner/custom_python_prehistoric_module.yml @@ -5,9 +9,11 @@ tasks: - custom_python_prehistoric_module: register: out + when: is_mitogen - assert: that: out.ok fail_msg: out={{out}} + when: is_mitogen tags: - custom_python_prehistoric_module diff --git a/tests/ansible/lib/modules/custom_python_detect_environment.py b/tests/ansible/lib/modules/custom_python_detect_environment.py index c7a222e7..d2ceaf0a 100644 --- a/tests/ansible/lib/modules/custom_python_detect_environment.py +++ b/tests/ansible/lib/modules/custom_python_detect_environment.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible new-style Python module. I return details about the Python # interpreter I run within. @@ -25,6 +25,11 @@ except NameError: def main(): module = AnsibleModule(argument_spec={}) module.exit_json( + fs={ + '/tmp': { + 'resolved': os.path.realpath('/tmp'), + }, + }, python={ 'version': { 'full': '%i.%i.%i' % sys.version_info[:3], diff --git a/tests/ansible/lib/modules/custom_python_external_module.py b/tests/ansible/lib/modules/custom_python_external_module.py index ae1b78cb..507e53dd 100644 --- a/tests/ansible/lib/modules/custom_python_external_module.py +++ b/tests/ansible/lib/modules/custom_python_external_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I expect the quote from modules2/module_utils/joker.py. from ansible.module_utils.basic import AnsibleModule diff --git a/tests/ansible/lib/modules/custom_python_external_pkg.py b/tests/ansible/lib/modules/custom_python_external_pkg.py index be9acb24..95bd0c7b 100644 --- a/tests/ansible/lib/modules/custom_python_external_pkg.py +++ b/tests/ansible/lib/modules/custom_python_external_pkg.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python from ansible.module_utils.basic import AnsibleModule from ansible.module_utils.externalpkg import extmod diff --git a/tests/ansible/lib/modules/custom_python_json_args_module.py b/tests/ansible/lib/modules/custom_python_json_args_module.py index 61640579..846037ec 100755 --- a/tests/ansible/lib/modules/custom_python_json_args_module.py +++ b/tests/ansible/lib/modules/custom_python_json_args_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible Python JSONARGS module. I should receive an encoding string. json_arguments = """<>""" diff --git a/tests/ansible/lib/modules/custom_python_leaky_class_vars.py b/tests/ansible/lib/modules/custom_python_leaky_class_vars.py index 255e3729..1d342329 100644 --- a/tests/ansible/lib/modules/custom_python_leaky_class_vars.py +++ b/tests/ansible/lib/modules/custom_python_leaky_class_vars.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible new-style Python module. I leak state from each invocation # into a class variable and a global variable. diff --git a/tests/ansible/lib/modules/custom_python_modify_environ.py b/tests/ansible/lib/modules/custom_python_modify_environ.py index 51b74526..9767f855 100644 --- a/tests/ansible/lib/modules/custom_python_modify_environ.py +++ b/tests/ansible/lib/modules/custom_python_modify_environ.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible new-style Python module. I modify the process environment and # don't clean up after myself. diff --git a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py index 2e0ef0da..728685f4 100644 --- a/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py +++ b/tests/ansible/lib/modules/custom_python_new_style_missing_interpreter.py @@ -1,6 +1,20 @@ # I am an Ansible new-style Python module, but I lack an interpreter. +# See also custom_python_new_style_module, we should be updated in tandem. +import io +import json +import select +import signal import sys +import warnings + +# Ansible 2.7 changed how new style modules are invoked. It seems that module +# parameters are *sometimes* read before the module runs. Modules that try +# to read directly from stdin, such as this, are unable to. However it doesn't +# always fail, influences seem to include Ansible & Python version. As noted +# in ansible.module_utils.basic._load_params() we should probably use that. +# I think (medium confidence) I narrowed the inflection (with git bisect) to +# https://github.com/ansible/ansible/commit/52449cc01a71778ef94ea0237eed0284f5d75582 # As of Ansible 2.10, Ansible changed new-style detection: # https://github.com/ansible/ansible/pull/61196/files#diff-5675e463b6ce1fbe274e5e7453f83cd71e61091ea211513c93e7c0b4d527d637L828-R980 # NOTE: this import works for Mitogen, and the import below matches new-style Ansible 2.10 @@ -8,11 +22,46 @@ import sys # from ansible.module_utils. # import ansible.module_utils. +# These timeouts should prevent hard-to-attribute, 2+ hour CI job timeouts. +# Previously this module has waited on stdin forever (timeoutInMinutes=120). +SELECT_TIMEOUT = 5.0 # seconds +SIGNAL_TIMEOUT = 10 # seconds + + +def fail_json(msg, **kwargs): + kwargs.update(failed=True, msg=msg) + print(json.dumps(kwargs, sys.stdout, indent=2, sort_keys=True)) + sys.exit(1) + + +def sigalrm_handler(signum, frame): + fail_json("Still executing after SIGNAL_TIMEOUT=%ds" % (SIGNAL_TIMEOUT,)) + def usage(): sys.stderr.write('Usage: %s \n' % (sys.argv[0],)) sys.exit(1) + +# Wait SIGNAL_TIMEOUT seconds, exit with failure if still running. +signal.signal(signal.SIGALRM, sigalrm_handler) +signal.alarm(SIGNAL_TIMEOUT) + +# Wait SELECT_TIMEOUT seconds, exit with failure if no data appears on stdin. +# TODO Combine select() & read() in a loop, to handle slow trickle of data. +# Consider buffering, line buffering, `f.read()` vs `f.read1()`. +# TODO Document that sys.stdin may be a StringIO under Ansible + Mitogen. +try: + inputs_ready, _, _ = select.select([sys.stdin], [], [], SELECT_TIMEOUT) +except (AttributeError, TypeError, io.UnsupportedOperation) as exc: + # sys.stdin.fileno() doesn't exist or can't return a real file descriptor. + warnings.warn("Could not wait on sys.stdin=%r: %r" % (sys.stdin, exc)) +else: + if not inputs_ready: + fail_json("Gave up waiting on sys.stdin after SELECT_TIMEOUT=%ds" + % (SELECT_TIMEOUT,)) + +# Read all data on stdin. May block forever, if EOF is not reached. input_json = sys.stdin.read() print("{") diff --git a/tests/ansible/lib/modules/custom_python_new_style_module.py b/tests/ansible/lib/modules/custom_python_new_style_module.py index 1e7270cd..c84d241a 100755 --- a/tests/ansible/lib/modules/custom_python_new_style_module.py +++ b/tests/ansible/lib/modules/custom_python_new_style_module.py @@ -1,16 +1,65 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible new-style Python module. I should receive an encoding string. +# See also custom_python_new_style_module, we should be updated in tandem. +import io +import json +import select +import signal import sys +import warnings + +# Ansible 2.7 changed how new style modules are invoked. It seems that module +# parameters are *sometimes* read before the module runs. Modules that try +# to read directly from stdin, such as this, are unable to. However it doesn't +# always fail, influences seem to include Ansible & Python version. As noted +# in ansible.module_utils.basic._load_params() we should probably use that. +# I think (medium confidence) I narrowed the inflection (with git bisect) to +# https://github.com/ansible/ansible/commit/52449cc01a71778ef94ea0237eed0284f5d75582 # This is the magic marker Ansible looks for: # from ansible.module_utils. +# These timeouts should prevent hard-to-attribute, 2+ hour CI job timeouts. +# Previously this module has waited on stdin forever (timeoutInMinutes=120). +SELECT_TIMEOUT = 5.0 # seconds +SIGNAL_TIMEOUT = 10 # seconds + + +def fail_json(msg, **kwargs): + kwargs.update(failed=True, msg=msg) + print(json.dumps(kwargs, sys.stdout, indent=2, sort_keys=True)) + sys.exit(1) + + +def sigalrm_handler(signum, frame): + fail_json("Still executing after SIGNAL_TIMEOUT=%ds" % (SIGNAL_TIMEOUT,)) + def usage(): sys.stderr.write('Usage: %s \n' % (sys.argv[0],)) sys.exit(1) + +# Wait SIGNAL_TIMEOUT seconds, exit with failure if still running. +signal.signal(signal.SIGALRM, sigalrm_handler) +signal.alarm(SIGNAL_TIMEOUT) + +# Wait SELECT_TIMEOUT seconds, exit with failure if no data appears on stdin. +# TODO Combine select() & read() in a loop, to handle slow trickle of data. +# Consider buffering, line buffering, `f.read()` vs `f.read1()`. +# TODO Document that sys.stdin may be a StringIO under Ansible + Mitogen. +try: + inputs_ready, _, _ = select.select([sys.stdin], [], [], SELECT_TIMEOUT) +except (AttributeError, TypeError, io.UnsupportedOperation) as exc: + # sys.stdin.fileno() doesn't exist or can't return a real file descriptor. + warnings.warn("Could not wait on sys.stdin=%r: %r" % (sys.stdin, exc)) +else: + if not inputs_ready: + fail_json("Gave up waiting on sys.stdin after SELECT_TIMEOUT=%ds" + % (SELECT_TIMEOUT,)) + +# Read all data on stdin. May block forever, if EOF is not reached. input_json = sys.stdin.read() print("{") diff --git a/tests/ansible/lib/modules/custom_python_os_getcwd.py b/tests/ansible/lib/modules/custom_python_os_getcwd.py index c5e264ae..d465ac9e 100644 --- a/tests/ansible/lib/modules/custom_python_os_getcwd.py +++ b/tests/ansible/lib/modules/custom_python_os_getcwd.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # #591: call os.getcwd() before AnsibleModule ever gets a chance to fix up the # process environment. diff --git a/tests/ansible/lib/modules/custom_python_prehistoric_module.py b/tests/ansible/lib/modules/custom_python_prehistoric_module.py index 0cf9774d..61397488 100644 --- a/tests/ansible/lib/modules/custom_python_prehistoric_module.py +++ b/tests/ansible/lib/modules/custom_python_prehistoric_module.py @@ -1,4 +1,9 @@ -#!/usr/bin/env python +#!/usr/bin/python + +# Test functionality of ansible_mitogen.runner.PREHISTORIC_HACK_RE, which +# removes `reload(sys); sys.setdefaultencoding(...)` from an Ansible module +# as it is sent to a target. There are probably very few modules in the wild +# that still do this, reload() is a Python 2.x builtin function. # issue #555: I'm a module that cutpastes an old hack. from ansible.module_utils.basic import AnsibleModule diff --git a/tests/ansible/lib/modules/custom_python_run_script.py b/tests/ansible/lib/modules/custom_python_run_script.py index d6a839ae..4a6243d0 100644 --- a/tests/ansible/lib/modules/custom_python_run_script.py +++ b/tests/ansible/lib/modules/custom_python_run_script.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible new-style Python module. I run the script provided in the # parameter. diff --git a/tests/ansible/lib/modules/custom_python_uses_distro.py b/tests/ansible/lib/modules/custom_python_uses_distro.py index 03f3b6aa..6b3a356b 100644 --- a/tests/ansible/lib/modules/custom_python_uses_distro.py +++ b/tests/ansible/lib/modules/custom_python_uses_distro.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # issue #590: I am an Ansible new-style Python module that tries to use # ansible.module_utils.distro. diff --git a/tests/ansible/lib/modules/custom_python_want_json_module.py b/tests/ansible/lib/modules/custom_python_want_json_module.py index f5e33862..23eeeb55 100755 --- a/tests/ansible/lib/modules/custom_python_want_json_module.py +++ b/tests/ansible/lib/modules/custom_python_want_json_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am an Ansible Python WANT_JSON module. I should receive a JSON-encoded file. import json diff --git a/tests/ansible/lib/modules/mitogen_test_gethostbyname.py b/tests/ansible/lib/modules/mitogen_test_gethostbyname.py index 1b80a48b..289e9662 100644 --- a/tests/ansible/lib/modules/mitogen_test_gethostbyname.py +++ b/tests/ansible/lib/modules/mitogen_test_gethostbyname.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # I am a module that indirectly depends on glibc cached /etc/resolv.conf state. diff --git a/tests/ansible/lib/modules/module_finder_test.py b/tests/ansible/lib/modules/module_finder_test.py new file mode 100644 index 00000000..41cf1c1c --- /dev/null +++ b/tests/ansible/lib/modules/module_finder_test.py @@ -0,0 +1,12 @@ +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +import os +import sys + +import ansible.module_utils.external1 + +from ansible.module_utils.externalpkg.extmod import path as epem_path + +def main(): + pass diff --git a/tests/ansible/lib/modules/test_echo_module.py b/tests/ansible/lib/modules/test_echo_module.py index 1f71e879..d6a5fb9e 100644 --- a/tests/ansible/lib/modules/test_echo_module.py +++ b/tests/ansible/lib/modules/test_echo_module.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python # -*- coding: utf-8 -*- # (c) 2012, Michael DeHaan @@ -9,28 +9,61 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type +import os import platform import sys from ansible.module_utils.basic import AnsibleModule def main(): - result = dict(changed=False) - module = AnsibleModule(argument_spec=dict( - facts=dict(type=dict, default={}) + facts_copy=dict(type=dict, default={}), + facts_to_override=dict(type=dict, default={}) )) - result['ansible_facts'] = module.params['facts'] # revert the Mitogen OSX tweak since discover_interpreter() doesn't return this info - if sys.platform == 'darwin' and sys.executable != '/usr/bin/python': - if int(platform.release()[:2]) < 19: - sys.executable = sys.executable[:-3] - else: + # NB This must be synced with mitogen.parent.Connection.get_boot_command() + platform_release_major = int(platform.release().partition('.')[0]) + if sys.modules.get('mitogen') and sys.platform == 'darwin': + if platform_release_major < 19 and sys.executable == '/usr/bin/python2.7': + sys.executable = '/usr/bin/python' + if platform_release_major in (20, 21) and sys.version_info[:2] == (2, 7): # only for tests to check version of running interpreter -- Mac 10.15+ changed python2 # so it looks like it's /usr/bin/python but actually it's /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python sys.executable = "/usr/bin/python" - result['running_python_interpreter'] = sys.executable + + facts_copy = module.params['facts_copy'] + discovered_interpreter_python = facts_copy['discovered_interpreter_python'] + result = { + 'changed': False, + 'ansible_facts': module.params['facts_to_override'], + 'discovered_and_running_samefile': os.path.samefile( + os.path.realpath(discovered_interpreter_python), + os.path.realpath(sys.executable), + ), + 'discovered_python': { + 'as_seen': discovered_interpreter_python, + 'resolved': os.path.realpath(discovered_interpreter_python), + }, + 'running_python': { + 'platform': { + 'release': { + 'major': platform_release_major, + }, + }, + 'sys': { + 'executable': { + 'as_seen': sys.executable, + 'resolved': os.path.realpath(sys.executable), + }, + 'platform': sys.platform, + 'version_info': { + 'major': sys.version_info[0], + 'minor': sys.version_info[1], + }, + }, + }, + } module.exit_json(**result) diff --git a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml index f4c47aba..610eaf33 100644 --- a/tests/ansible/regression/issue_152__virtualenv_python_fails.yml +++ b/tests/ansible/regression/issue_152__virtualenv_python_fails.yml @@ -23,9 +23,16 @@ when: - lout.python.version.full is version('2.7', '>=', strict=True) - - assert: + - name: Check virtualenv was used + # On macOS runners a symlink /tmp -> /private/tmp has been seen + vars: + requested_executable: /tmp/issue_152_virtualenv/bin/python + expected_executables: + - "{{ requested_executable }}" + - "{{ requested_executable.replace('/tmp', out.fs['/tmp'].resolved) }}" + assert: that: - - out.sys_executable == "/tmp/issue_152_virtualenv/bin/python" + - out.sys_executable in expected_executables fail_msg: out={{out}} when: - lout.python.version.full is version('2.7', '>=', strict=True) diff --git a/tests/ansible/setup/all.yml b/tests/ansible/setup/all.yml index 2ca6b97c..8903494d 100644 --- a/tests/ansible/setup/all.yml +++ b/tests/ansible/setup/all.yml @@ -1 +1,2 @@ -- import_playbook: report.yml +- import_playbook: report_controller.yml +- import_playbook: report_targets.yml diff --git a/tests/ansible/setup/report_controller.yml b/tests/ansible/setup/report_controller.yml new file mode 100644 index 00000000..d0d5cc15 --- /dev/null +++ b/tests/ansible/setup/report_controller.yml @@ -0,0 +1,17 @@ +- name: Report controller parameters + hosts: localhost + gather_facts: false + tasks: + - debug: + msg: + - ${ANSIBLE_STRATEGY}: "{{ lookup('env', 'ANSIBLE_STRATEGY') | default('') }}" + - ${USER}: "{{ lookup('env', 'USER') | default('') }}" + - $(groups): "{{ lookup('pipe', 'groups') }}" + - $(pwd): "{{ lookup('pipe', 'pwd') }}" + - $(whoami): "{{ lookup('pipe', 'whoami') }}" + - ansible_run_tags: "{{ ansible_run_tags | default('') }}" + - ansible_playbook_python: "{{ ansible_playbook_python | default('') }}" + - ansible_skip_tags: "{{ ansible_skip_tags | default('') }}" + - ansible_version.full: "{{ ansible_version.full | default('') }}" + - is_mitogen: "{{ is_mitogen | default('') }}" + - playbook_dir: "{{ playbook_dir | default('') }}" diff --git a/tests/ansible/setup/report.yml b/tests/ansible/setup/report_targets.yml similarity index 73% rename from tests/ansible/setup/report.yml rename to tests/ansible/setup/report_targets.yml index 450e4fb0..5aa67124 100644 --- a/tests/ansible/setup/report.yml +++ b/tests/ansible/setup/report_targets.yml @@ -1,4 +1,4 @@ -- name: Report runtime settings +- name: Report target facts hosts: localhost:test-targets gather_facts: true tasks: @@ -13,8 +13,3 @@ - debug: {var: ansible_facts.osversion} - debug: {var: ansible_facts.python} - debug: {var: ansible_facts.system} - - debug: {var: ansible_forks} - - debug: {var: ansible_run_tags} - - debug: {var: ansible_skip_tags} - - debug: {var: ansible_version.full} - - debug: {var: is_mitogen} diff --git a/tests/ansible/tests/module_finder_test.py b/tests/ansible/tests/module_finder_test.py new file mode 100644 index 00000000..79e8fdbd --- /dev/null +++ b/tests/ansible/tests/module_finder_test.py @@ -0,0 +1,80 @@ +import os.path +import sys +import textwrap +import unittest + +import ansible_mitogen.module_finder + +import testlib + + +class ScanFromListTest(testlib.TestCase): + def test_absolute_imports(self): + source = textwrap.dedent('''\ + from __future__ import absolute_import + import a; import b.c; from d.e import f; from g import h, i + ''') + code = compile(source, '', 'exec') + self.assertEqual( + list(ansible_mitogen.module_finder.scan_fromlist(code)), + [(0, '__future__.absolute_import'), (0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')], + ) + + +class WalkImportsTest(testlib.TestCase): + def test_absolute_imports(self): + source = textwrap.dedent('''\ + from __future__ import absolute_import + import a; import b; import b.c; from b.d import e, f + ''') + code = compile(source, '', 'exec') + + self.assertEqual( + list(ansible_mitogen.module_finder.walk_imports(code)), + ['__future__', '__future__.absolute_import', 'a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f'], + ) + self.assertEqual( + list(ansible_mitogen.module_finder.walk_imports(code, prefix='b')), + ['b.c', 'b.d', 'b.d.e', 'b.d.f'], + ) + + +class ScanTest(testlib.TestCase): + module_name = 'ansible_module_module_finder_test__this_should_not_matter' + module_path = os.path.join(testlib.ANSIBLE_MODULES_DIR, 'module_finder_test.py') + search_path = ( + 'does_not_exist/module_utils', + testlib.ANSIBLE_MODULE_UTILS_DIR, + ) + + @staticmethod + def relpath(path): + return os.path.relpath(path, testlib.ANSIBLE_MODULE_UTILS_DIR) + + @unittest.skipIf(sys.version_info < (3, 4), 'find spec() unavailable') + def test_importlib_find_spec(self): + scan = ansible_mitogen.module_finder._scan_importlib_find_spec + actual = scan(self.module_name, self.module_path, self.search_path) + self.assertEqual( + [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual], + [ + ('ansible.module_utils.external1', 'external1.py', False), + ('ansible.module_utils.external2', 'external2.py', False), + ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True), + ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False), + ], + ) + + @unittest.skipIf(sys.version_info >= (3, 4), 'find spec() preferred') + def test_imp_find_module(self): + scan = ansible_mitogen.module_finder._scan_imp_find_module + actual = scan(self.module_name, self.module_path, self.search_path) + self.assertEqual( + [(name, self.relpath(path), is_pkg) for name, path, is_pkg in actual], + [ + ('ansible.module_utils.external1', 'external1.py', False), + ('ansible.module_utils.external2', 'external2.py', False), + ('ansible.module_utils.externalpkg', 'externalpkg/__init__.py', True), + ('ansible.module_utils.externalpkg.extmod', 'externalpkg/extmod.py',False), + ], + ) diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 6224b61a..0b6d5e61 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -4,6 +4,8 @@ # WARNING: this creates non-privilged accounts with pre-set passwords! # +- import_playbook: ../ansible/setup/report_controller.yml + - hosts: all gather_facts: true strategy: mitogen_free @@ -37,12 +39,12 @@ normal_users: "{{ lookup('sequence', 'start=1 end=5 format=user%d', wantlist=True) - }}" + }}" all_users: "{{ special_users + normal_users - }}" + }}" tasks: - name: Disable non-localhost SSH for Mitogen users when: false @@ -100,6 +102,7 @@ with_items: "{{all_users}}" copy: dest: /var/lib/AccountsService/users/mitogen__{{item}} + mode: u=rw,go= content: | [User] SystemAccount=true @@ -108,7 +111,7 @@ when: ansible_system == 'Linux' and out.stat.exists service: name: accounts-daemon - restarted: true + state: restarted - name: Readonly homedir for one account shell: "chown -R root: ~mitogen__readonly_homedir" @@ -117,6 +120,9 @@ copy: dest: ~mitogen__slow_user/.{{item}} src: ../data/docker/mitogen__slow_user.profile + owner: mitogen__slow_user + group: mitogen__group + mode: u=rw,go=r with_items: - bashrc - profile @@ -125,6 +131,9 @@ copy: dest: ~mitogen__permdenied/.{{item}} src: ../data/docker/mitogen__permdenied.profile + owner: mitogen__permdenied + group: mitogen__group + mode: u=rw,go=r with_items: - bashrc - profile @@ -136,20 +145,13 @@ state: directory mode: go= owner: mitogen__has_sudo_pubkey + group: mitogen__group - copy: dest: ~mitogen__has_sudo_pubkey/.ssh/authorized_keys src: ../data/docker/mitogen__has_sudo_pubkey.key.pub mode: go= owner: mitogen__has_sudo_pubkey - - - name: Install slow profile for one account - block: - - copy: - dest: ~mitogen__slow_user/.profile - src: ../data/docker/mitogen__slow_user.profile - - copy: - dest: ~mitogen__slow_user/.bashrc - src: ../data/docker/mitogen__slow_user.profile + group: mitogen__group - name: Require a TTY for two accounts lineinfile: diff --git a/tests/importer_test.py b/tests/importer_test.py index e48c02a4..e86af8af 100644 --- a/tests/importer_test.py +++ b/tests/importer_test.py @@ -2,6 +2,7 @@ import sys import threading import types import zlib +import unittest import mock @@ -42,6 +43,49 @@ class ImporterMixin(testlib.RouterMixin): super(ImporterMixin, self).tearDown() +class InvalidNameTest(ImporterMixin, testlib.TestCase): + modname = 'trailingdot.' + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, None, None, None, None) + + @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') + def test_find_spec_invalid(self): + self.set_get_module_response(self.response) + self.assertEqual(self.importer.find_spec(self.modname, path=None), None) + + +class MissingModuleTest(ImporterMixin, testlib.TestCase): + modname = 'missing' + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, None, None, None, None) + + @unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') + def test_load_module_missing(self): + self.set_get_module_response(self.response) + self.assertRaises(ImportError, self.importer.load_module, self.modname) + + @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') + def test_find_spec_missing(self): + """ + Importer should optimistically offer itself as a module loader + when there are no disqualifying criteria. + """ + import importlib.machinery + self.set_get_module_response(self.response) + spec = self.importer.find_spec(self.modname, path=None) + self.assertIsInstance(spec, importlib.machinery.ModuleSpec) + self.assertEqual(spec.name, self.modname) + self.assertEqual(spec.loader, self.importer) + + @unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') + def test_create_module_missing(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + self.assertRaises(ImportError, self.importer.create_module, spec) + + +@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') class LoadModuleTest(ImporterMixin, testlib.TestCase): data = zlib.compress(b("data = 1\n\n")) path = 'fake_module.py' @@ -50,14 +94,6 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase): # 0:fullname 1:pkg_present 2:path 3:compressed 4:related response = (modname, None, path, data, []) - def test_no_such_module(self): - self.set_get_module_response( - # 0:fullname 1:pkg_present 2:path 3:compressed 4:related - (self.modname, None, None, None, None) - ) - self.assertRaises(ImportError, - lambda: self.importer.load_module(self.modname)) - def test_module_added_to_sys_modules(self): self.set_get_module_response(self.response) mod = self.importer.load_module(self.modname) @@ -80,6 +116,26 @@ class LoadModuleTest(ImporterMixin, testlib.TestCase): self.assertIsNone(mod.__package__) +@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') +class ModuleSpecTest(ImporterMixin, testlib.TestCase): + data = zlib.compress(b("data = 1\n\n")) + path = 'fake_module.py' + modname = 'fake_module' + + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, None, path, data, []) + + def test_module_attributes(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + mod = self.importer.create_module(spec) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'fake_module') + #self.assertFalse(hasattr(mod, '__file__')) + + +@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') class LoadSubmoduleTest(ImporterMixin, testlib.TestCase): data = zlib.compress(b("data = 1\n\n")) path = 'fake_module.py' @@ -93,6 +149,25 @@ class LoadSubmoduleTest(ImporterMixin, testlib.TestCase): self.assertEqual(mod.__package__, 'mypkg') +@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') +class SubmoduleSpecTest(ImporterMixin, testlib.TestCase): + data = zlib.compress(b("data = 1\n\n")) + path = 'fake_module.py' + modname = 'mypkg.fake_module' + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, None, path, data, []) + + def test_module_attributes(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + mod = self.importer.create_module(spec) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'mypkg.fake_module') + #self.assertFalse(hasattr(mod, '__file__')) + + +@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python 3.4+') class LoadModulePackageTest(ImporterMixin, testlib.TestCase): data = zlib.compress(b("func = lambda: 1\n\n")) path = 'fake_pkg/__init__.py' @@ -140,6 +215,41 @@ class LoadModulePackageTest(ImporterMixin, testlib.TestCase): self.assertEqual(mod.func.__module__, self.modname) +@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') +class PackageSpecTest(ImporterMixin, testlib.TestCase): + data = zlib.compress(b("func = lambda: 1\n\n")) + path = 'fake_pkg/__init__.py' + modname = 'fake_pkg' + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + response = (modname, [], path, data, []) + + def test_module_attributes(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + mod = self.importer.create_module(spec) + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, 'fake_pkg') + #self.assertFalse(hasattr(mod, '__file__')) + + def test_get_filename(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + _ = self.importer.create_module(spec) + filename = self.importer.get_filename(self.modname) + self.assertEqual('master:fake_pkg/__init__.py', filename) + + def test_get_source(self): + import importlib.machinery + self.set_get_module_response(self.response) + spec = importlib.machinery.ModuleSpec(self.modname, self.importer) + _ = self.importer.create_module(spec) + source = self.importer.get_source(self.modname) + self.assertEqual(source, + mitogen.core.to_text(zlib.decompress(self.data))) + + class EmailParseAddrSysTest(testlib.RouterMixin, testlib.TestCase): def initdir(self, caplog): self.caplog = caplog diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py index 02b8b886..67e937ed 100644 --- a/tests/module_finder_test.py +++ b/tests/module_finder_test.py @@ -140,9 +140,7 @@ class SysModulesMethodTest(testlib.TestCase): self.assertIsNone(tup) -class GetModuleViaParentEnumerationTest(testlib.TestCase): - klass = mitogen.master.ParentEnumerationMethod - +class ParentEnumerationMixin(object): def call(self, fullname): return self.klass().find(fullname) @@ -232,6 +230,16 @@ class GetModuleViaParentEnumerationTest(testlib.TestCase): self.assertEqual(is_pkg, False) +@unittest.skipIf(sys.version_info >= (3, 4), 'Superceded in Python >= 3.4') +class ParentImpEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase): + klass = mitogen.master.ParentImpEnumerationMethod + + +@unittest.skipIf(sys.version_info < (3, 4), 'Requires ModuleSpec, Python 3.4+') +class ParentSpecEnumerationMethodTest(ParentEnumerationMixin, testlib.TestCase): + klass = mitogen.master.ParentSpecEnumerationMethod + + class ResolveRelPathTest(testlib.TestCase): klass = mitogen.master.ModuleFinder diff --git a/tests/requirements-tox.txt b/tests/requirements-tox.txt new file mode 100644 index 00000000..bc7f7c2a --- /dev/null +++ b/tests/requirements-tox.txt @@ -0,0 +1,4 @@ +tox==3.28; python_version == '2.7' +tox==3.28; python_version == '3.6' +tox==4.8.0; python_version == '3.7' +tox>=4.13.0,~=4.0; python_version >= '3.8' diff --git a/tests/requirements.txt b/tests/requirements.txt index 1e5d2a1d..6d87d177 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,12 +1,24 @@ -cffi==1.15.1 -coverage==5.5; python_version < '3.7' -coverage==6.4.4; python_version >= '3.7' +cffi==1.15.1; python_version < '3.8' +cffi==1.16; python_version >= '3.8' + +coverage==5.5; python_version == '2.7' +coverage==6.2; python_version == '3.6' +coverage==7.2.7; python_version == '3.7' +coverage==7.4.3; python_version >= '3.8' + Django==1.11.29; python_version < '3.0' Django==3.2.20; python_version >= '3.6' -mock==2.0.0 -psutil==5.9.5 -pytest-catchlog==1.2.2 -pytest==3.1.2 + +mock==3.0.5; python_version == '2.7' +mock==5.1.0; python_version >= '3.6' + +psutil==5.9.8 + +pytest==4.6.11; python_version == '2.7' +pytest==7.0.1; python_version == '3.6' +pytest==7.4.4; python_version == '3.7' +pytest==8.0.2; python_version >= '3.8' + subprocess32==3.5.4; python_version < '3.0' timeoutcontext==1.2.0 # Fix InsecurePlatformWarning while creating py26 tox environment @@ -15,4 +27,7 @@ urllib3[secure]==1.23; python_version < '2.7' urllib3[secure]==1.26; python_version > '2.6' and python_version < '2.7.9' # Last idna compatible with Python 2.6 was idna 2.7. idna==2.7; python_version < '2.7' -virtualenv==20.10.0 + +virtualenv==20.15.1; python_version == '2.7' +virtualenv==20.17.1; python_version == '3.6' +virtualenv==20.25.1; python_version >= '3.7' diff --git a/tests/testlib.py b/tests/testlib.py index ec0a7443..8c40e7ff 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -50,8 +50,13 @@ except NameError: LOG = logging.getLogger(__name__) -DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') -MODS_DIR = os.path.join(DATA_DIR, 'importer') + +TESTS_DIR = os.path.join(os.path.dirname(__file__)) +ANSIBLE_LIB_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib') +ANSIBLE_MODULE_UTILS_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils') +ANSIBLE_MODULES_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'modules') +DATA_DIR = os.path.join(TESTS_DIR, 'data') +MODS_DIR = os.path.join(TESTS_DIR, 'data', 'importer') sys.path.append(DATA_DIR) sys.path.append(MODS_DIR) diff --git a/tox.ini b/tox.ini index e4c26c39..4a1772a0 100644 --- a/tox.ini +++ b/tox.ini @@ -3,18 +3,37 @@ # # sudo add-apt-repository ppa:deadsnakes/ppa # sudo apt update -# sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python2.7 python3.{6..11} python-is-python3 sshpass tox +# sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python2.7 python3.{6..13}{,-venv} python-is-python3 sshpass tox # Py A cntrllr A target coverage Django Jinja2 pip psutil pytest tox virtualenv # ==== ========== ========== ========== ========== ========== ========== ========== ========== ========== ========== # 2.4 2.3? <= 3.7.1 <= 1.3.7 <= 1.1 <= 2.1.3 <= 1.4 <= 1.8 # 2.5 <= 3.7.1 <= 1.4.22 <= 1.3.1 <= 2.1.3 <= 2.8.7 <= 1.6.1 <= 1.9.1 -# 2.6 <= 2.6.20 <= 2.13 <= 4.5.4 <= 1.6.11 <= 2.10.3 <= 9.0.3 <= 5.9.0 <= 3.2.5 <= 2.9.1 <= 15.2.0 -# 2.7 <= 2.11 <= 5.6 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.3? -# 3.5 <= 2.11 <= 2.13 <= 5.6 <= 2.2.28 <= 2.11.3 <= 20 <= 5.9.5 <= 6.1.0 <= 3.28 <= 20.15 -# 3.6 <= 2.11 <= 6.2 <= 3.2.20 <= 3.0.3 <= 21 <= 5.9.5 <= 7.0.1 <= 3.28 <= 20.16 -# 3.7 <= 2.12 <= 3.2.20 +# 2.6 <= 2.6.20 <= 2.12 <= 4.5.4 <= 1.6.11 <= 2.10.3 <= 9.0.3 <= 5.9.0 <= 3.2.5 <= 2.9.1 <= 15.2.0 +# 2.7 <= 2.11 <= 5.5 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.15² +# 3.5 <= 2.11 <= 2.13 <= 5.5 <= 2.2.28 <= 2.11.3 <= 20 <= 5.9.5 <= 6.1.0 <= 3.28 <= 20.15² +# 3.6 <= 2.11 <= 6.2 <= 3.2.20 <= 3.0.3 <= 21 <= 7.0.1 <= 3.28 <= 20.17² +# 3.7 <= 2.12 <= 7.2.7 <= 3.2.20 <= 7.4.4 <= 4.8.0 # 3.8 <= 2.12 +# 3.9 <= 2.15 +# 3.10 +# 3.11 +# 3.12 >= 2.13¹ +# +# Notes +# 1. Python 3.12 on a target requires Ansible >= 6 (ansible-core >= 2.13). +# Python 3.12 removed support for find_module(), replaced by find_spec(). +# In Ansible <= 4.x ansible.module_utils.six lacks find_spec(). +# https://github.com/ansible/ansible/commit/d6e28e68599e703c153914610152cf4492851eb3 +# In Ansible <= 5.x ansible.utils.collection_loader._AnsibleCollectionFinder +# lacks find_spec(). https://github.com/ansible/ansible/pull/76225 +# +# Python 3.12 + get_uri requires Ansible >= 8 (ansible-core >= 2.15). +# Python 3.12 removed deprecated httplib.HTTPSConnection() arguments. +# https://github.com/ansible/ansible/pull/80751 +# +# 2. Higher virtualenv versions cannot run under this Python version. They can +# still generate virtual environments for it. # Ansible Dependency # ================== ====================== @@ -24,6 +43,9 @@ # ansible == 4.* ansible-core ~= 2.11.0 # ansible == 5.* ansible-core ~= 2.12.0 # ansible == 6.* ansible-core ~= 2.13.0 +# ansible == 7.x ansible-core ~= 2.14.0 +# ansible == 8.x ansible-core ~= 2.15.0 +# ansible == 9.x ansible-core ~= 2.16.0 # pip --no-python-version-warning # pip --disable-pip-version-check @@ -34,10 +56,11 @@ envlist = init, py{27,36}-mode_ansible-ansible{2.10,3,4}, - py{311}-mode_ansible-ansible{2.10,3,4,5,6}, - py{27,36,311}-mode_mitogen-distro_centos{6,7,8}, - py{27,36,311}-mode_mitogen-distro_debian{9,10,11}, - py{27,36,311}-mode_mitogen-distro_ubuntu{1604,1804,2004}, + py{311}-mode_ansible-ansible{2.10,3,4,5}, + py{312}-mode_ansible-ansible{6}, + py{27,36,312}-mode_mitogen-distro_centos{6,7,8}, + py{27,36,312}-mode_mitogen-distro_debian{9,10,11}, + py{27,36,312}-mode_mitogen-distro_ubuntu{1604,1804,2004}, report, [testenv] @@ -51,14 +74,15 @@ basepython = py39: python3.9 py310: python3.10 py311: python3.11 + py312: python3.12 deps = -r{toxinidir}/tests/requirements.txt mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt ansible2.10: ansible==2.10.7 ansible3: ansible==3.4.0 ansible4: ansible==4.10.0 - ansible5: ansible==5.8.0 - ansible6: ansible==6.0.0 + ansible5: ansible~=5.0 + ansible6: ansible~=6.0 install_command = python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages} commands_pre = @@ -79,7 +103,6 @@ passenv = HOME setenv = # See also azure-pipelines.yml - ANSIBLE_SKIP_TAGS = requires_local_sudo,resource_intensive ANSIBLE_STRATEGY = mitogen_linear NOCOVERAGE_ERASE = 1 NOCOVERAGE_REPORT = 1 @@ -111,10 +134,26 @@ setenv = distros_ubuntu1804: DISTROS=ubuntu1804 distros_ubuntu2004: DISTROS=ubuntu2004 mode_ansible: MODE=ansible + mode_ansible: ANSIBLE_SKIP_TAGS=resource_intensive mode_debops_common: MODE=debops_common + mode_localhost: ANSIBLE_SKIP_TAGS=issue_776,resource_intensive mode_mitogen: MODE=mitogen strategy_linear: ANSIBLE_STRATEGY=linear +allowlist_externals = + # Added: Tox 3.18: Tox 4.0+ + *_install.py + *_tests.py + aws + docker + docker-credential-secretservice + echo + gpg2 + pass whitelist_externals = + # Deprecated: Tox 3.18+; Removed: Tox 4.0 + *_install.py + *_tests.py + aws docker docker-credential-secretservice echo