diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index dc7a02a8..84db7a94 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -264,7 +264,7 @@ def start_containers(containers): "docker rm -f %(name)s || true" % container, "docker run " "--rm " - "--cpuset-cpus 0,1 " + # "--cpuset-cpus 0,1 " "--detach " "--privileged " "--cap-add=SYS_PTRACE " diff --git a/.ci/mitogen_py24_install.py b/.ci/mitogen_py24_install.py index 97370806..868ae4e4 100755 --- a/.ci/mitogen_py24_install.py +++ b/.ci/mitogen_py24_install.py @@ -7,7 +7,7 @@ batches = [ 'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),), ], [ - 'sudo tar -C / -jxvf tests/data/ubuntu-python-2.4.6.tar.bz2', + 'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv', ] ] diff --git a/ansible_mitogen/affinity.py b/ansible_mitogen/affinity.py index 67e16d8a..7f4c8db5 100644 --- a/ansible_mitogen/affinity.py +++ b/ansible_mitogen/affinity.py @@ -265,9 +265,19 @@ class LinuxPolicy(FixedPolicy): mask >>= 64 return mitogen.core.b('').join(chunks) + def _get_thread_ids(self): + try: + ents = os.listdir('/proc/self/task') + except OSError: + LOG.debug('cannot fetch thread IDs for current process') + return [os.getpid()] + + return [int(s) for s in ents if s.isdigit()] + def _set_cpu_mask(self, mask): s = self._mask_to_bytes(mask) - _sched_setaffinity(os.getpid(), len(s), s) + for tid in self._get_thread_ids(): + _sched_setaffinity(tid, len(s), s) if _sched_setaffinity is not None: diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 1686cbde..5e08eb15 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -791,7 +791,9 @@ class Connection(ansible.plugins.connection.ConnectionBase): inventory_name, stack = self._build_stack() worker_model = ansible_mitogen.process.get_worker_model() - self.binding = worker_model.get_binding(inventory_name) + self.binding = worker_model.get_binding( + mitogen.utils.cast(inventory_name) + ) self._connect_stack(stack) def _put_connection(self): diff --git a/ansible_mitogen/loaders.py b/ansible_mitogen/loaders.py index 99294c1f..9ce6b1fa 100644 --- a/ansible_mitogen/loaders.py +++ b/ansible_mitogen/loaders.py @@ -55,3 +55,8 @@ except ImportError: # Ansible <2.4 from ansible.plugins import module_utils_loader from ansible.plugins import shell_loader from ansible.plugins import strategy_loader + + +# These are original, unwrapped implementations +action_loader__get = action_loader.get +connection_loader__get = connection_loader.get diff --git a/ansible_mitogen/logging.py b/ansible_mitogen/logging.py index ce6f1659..00a70184 100644 --- a/ansible_mitogen/logging.py +++ b/ansible_mitogen/logging.py @@ -107,8 +107,9 @@ def setup(): l_mitogen = logging.getLogger('mitogen') l_mitogen_io = logging.getLogger('mitogen.io') l_ansible_mitogen = logging.getLogger('ansible_mitogen') + l_operon = logging.getLogger('operon') - for logger in l_mitogen, l_mitogen_io, l_ansible_mitogen: + for logger in l_mitogen, l_mitogen_io, l_ansible_mitogen, l_operon: logger.handlers = [Handler(display.vvv)] logger.propagate = False diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index eee1ecd7..cfdf8384 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -55,6 +55,11 @@ import ansible_mitogen.planner import ansible_mitogen.target from ansible.module_utils._text import to_text +try: + from ansible.utils.unsafe_proxy import wrap_var +except ImportError: + from ansible.vars.unsafe_proxy import wrap_var + LOG = logging.getLogger(__name__) @@ -306,7 +311,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): except AttributeError: return getattr(self._task, 'async') - def _temp_file_gibberish(self, module_args, wrap_async): + def _set_temp_file_args(self, module_args, wrap_async): # Ansible>2.5 module_utils reuses the action's temporary directory if # one exists. Older versions error if this key is present. if ansible.__version__ > '2.5': @@ -343,7 +348,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): self._update_module_args(module_name, module_args, task_vars) env = {} self._compute_environment_string(env) - self._temp_file_gibberish(module_args, wrap_async) + self._set_temp_file_args(module_args, wrap_async) self._connection._connect() result = ansible_mitogen.planner.invoke( @@ -365,7 +370,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): # on _execute_module(). self._remove_tmp_path(tmp) - return result + return wrap_var(result) def _postprocess_response(self, result): """ diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 96b06995..8febbdb3 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -45,6 +45,7 @@ import random from ansible.executor import module_common import ansible.errors import ansible.module_utils +import ansible.release import mitogen.core import mitogen.select @@ -58,6 +59,8 @@ NO_METHOD_MSG = 'Mitogen: no invocation method found for: ' NO_INTERPRETER_MSG = 'module (%s) is missing interpreter line' NO_MODULE_MSG = 'The module %s was not found in configured module paths.' +_planner_by_path = {} + class Invocation(object): """ @@ -92,7 +95,12 @@ class Invocation(object): self.module_path = None #: Initially ``None``, but set by :func:`invoke`. The raw source or #: binary contents of the module. - self.module_source = None + self._module_source = None + + def get_module_source(self): + if self._module_source is None: + self._module_source = read_file(self.module_path) + return self._module_source def __repr__(self): return 'Invocation(module_name=%s)' % (self.module_name,) @@ -107,7 +115,8 @@ class Planner(object): def __init__(self, invocation): self._inv = invocation - def detect(self): + @classmethod + def detect(cls, path, source): """ Return true if the supplied `invocation` matches the module type implemented by this planner. @@ -171,8 +180,9 @@ class BinaryPlanner(Planner): """ runner_name = 'BinaryRunner' - def detect(self): - return module_common._is_binary(self._inv.module_source) + @classmethod + def detect(cls, path, source): + return module_common._is_binary(source) def get_push_files(self): return [mitogen.core.to_text(self._inv.module_path)] @@ -218,7 +228,7 @@ class ScriptPlanner(BinaryPlanner): def _get_interpreter(self): path, arg = ansible_mitogen.parsing.parse_hashbang( - self._inv.module_source + self._inv.get_module_source() ) if path is None: raise ansible.errors.AnsibleError(NO_INTERPRETER_MSG % ( @@ -247,8 +257,9 @@ class JsonArgsPlanner(ScriptPlanner): """ runner_name = 'JsonArgsRunner' - def detect(self): - return module_common.REPLACER_JSONARGS in self._inv.module_source + @classmethod + def detect(cls, path, source): + return module_common.REPLACER_JSONARGS in source class WantJsonPlanner(ScriptPlanner): @@ -265,8 +276,9 @@ class WantJsonPlanner(ScriptPlanner): """ runner_name = 'WantJsonRunner' - def detect(self): - return b'WANT_JSON' in self._inv.module_source + @classmethod + def detect(cls, path, source): + return b'WANT_JSON' in source class NewStylePlanner(ScriptPlanner): @@ -278,8 +290,9 @@ class NewStylePlanner(ScriptPlanner): runner_name = 'NewStyleRunner' marker = b'from ansible.module_utils.' - def detect(self): - return self.marker in self._inv.module_source + @classmethod + def detect(cls, path, source): + return cls.marker in source def _get_interpreter(self): return None, None @@ -323,7 +336,6 @@ class NewStylePlanner(ScriptPlanner): for path in ansible_mitogen.loaders.module_utils_loader._get_paths( subdirs=False ) - if os.path.isdir(path) ) _module_map = None @@ -347,6 +359,10 @@ class NewStylePlanner(ScriptPlanner): def get_kwargs(self): return super(NewStylePlanner, self).get_kwargs( module_map=self.get_module_map(), + py_module_name=py_modname_from_path( + self._inv.module_name, + self._inv.module_path, + ), ) @@ -376,14 +392,16 @@ class ReplacerPlanner(NewStylePlanner): """ runner_name = 'ReplacerRunner' - def detect(self): - return module_common.REPLACER in self._inv.module_source + @classmethod + def detect(cls, path, source): + return module_common.REPLACER in source class OldStylePlanner(ScriptPlanner): runner_name = 'OldStyleRunner' - def detect(self): + @classmethod + def detect(cls, path, source): # Everything else. return True @@ -398,14 +416,54 @@ _planners = [ ] -def get_module_data(name): - path = ansible_mitogen.loaders.module_loader.find_plugin(name, '') - if path is None: - raise ansible.errors.AnsibleError(NO_MODULE_MSG % (name,)) +try: + _get_ansible_module_fqn = module_common._get_ansible_module_fqn +except AttributeError: + _get_ansible_module_fqn = None + - with open(path, 'rb') as fp: - source = fp.read() - return mitogen.core.to_text(path), source +def py_modname_from_path(name, path): + """ + Fetch the logical name of a new-style module as it might appear in + :data:`sys.modules` of the target's Python interpreter. + + * For Ansible <2.7, this is an unpackaged module named like + "ansible_module_%s". + + * For Ansible <2.9, this is an unpackaged module named like + "ansible.modules.%s" + + * Since Ansible 2.9, modules appearing within a package have the original + package hierarchy approximated on the target, enabling relative imports + to function correctly. For example, "ansible.modules.system.setup". + """ + # 2.9+ + if _get_ansible_module_fqn: + try: + return _get_ansible_module_fqn(path) + except ValueError: + pass + + if ansible.__version__ < '2.7': + return 'ansible_module_' + name + + return 'ansible.modules.' + name + + +def read_file(path): + fd = os.open(path, os.O_RDONLY) + try: + bits = [] + chunk = True + while True: + chunk = os.read(fd, 65536) + if not chunk: + break + bits.append(chunk) + finally: + os.close(fd) + + return mitogen.core.b('').join(bits) def _propagate_deps(invocation, planner, context): @@ -466,14 +524,12 @@ def _invoke_isolated_task(invocation, planner): context.shutdown() -def _get_planner(invocation): +def _get_planner(name, path, source): for klass in _planners: - planner = klass(invocation) - if planner.detect(): - LOG.debug('%r accepted %r (filename %r)', planner, - invocation.module_name, invocation.module_path) - return planner - LOG.debug('%r rejected %r', planner, invocation.module_name) + if klass.detect(path, source): + LOG.debug('%r accepted %r (filename %r)', klass, name, path) + return klass + LOG.debug('%r rejected %r', klass, name) raise ansible.errors.AnsibleError(NO_METHOD_MSG + repr(invocation)) @@ -488,10 +544,24 @@ def invoke(invocation): :raises ansible.errors.AnsibleError: Unrecognized/unsupported module type. """ - (invocation.module_path, - invocation.module_source) = get_module_data(invocation.module_name) - planner = _get_planner(invocation) + path = ansible_mitogen.loaders.module_loader.find_plugin( + invocation.module_name, + '', + ) + if path is None: + raise ansible.errors.AnsibleError(NO_MODULE_MSG % ( + invocation.module_name, + )) + + invocation.module_path = mitogen.core.to_text(path) + if invocation.module_path not in _planner_by_path: + _planner_by_path[invocation.module_path] = _get_planner( + invocation.module_name, + invocation.module_path, + invocation.get_module_source() + ) + planner = _planner_by_path[invocation.module_path](invocation) if invocation.wrap_async: response = _invoke_async_task(invocation, planner) elif planner.should_fork(): diff --git a/ansible_mitogen/plugins/connection/mitogen_kubectl.py b/ansible_mitogen/plugins/connection/mitogen_kubectl.py index 2dab131b..44d3b50a 100644 --- a/ansible_mitogen/plugins/connection/mitogen_kubectl.py +++ b/ansible_mitogen/plugins/connection/mitogen_kubectl.py @@ -31,11 +31,6 @@ from __future__ import absolute_import import os.path import sys -try: - from ansible.plugins.connection import kubectl -except ImportError: - kubectl = None - from ansible.errors import AnsibleConnectionFailure from ansible.module_utils.six import iteritems @@ -47,6 +42,19 @@ except ImportError: del base_dir import ansible_mitogen.connection +import ansible_mitogen.loaders + + +_class = ansible_mitogen.loaders.connection_loader__get( + 'kubectl', + class_only=True, +) + +if _class: + kubectl = sys.modules[_class.__module__] + del _class +else: + kubectl = None class Connection(ansible_mitogen.connection.Connection): diff --git a/ansible_mitogen/plugins/connection/mitogen_ssh.py b/ansible_mitogen/plugins/connection/mitogen_ssh.py index df0e87cb..1c81dae5 100644 --- a/ansible_mitogen/plugins/connection/mitogen_ssh.py +++ b/ansible_mitogen/plugins/connection/mitogen_ssh.py @@ -42,21 +42,23 @@ DOCUMENTATION = """ options: """ -import ansible.plugins.connection.ssh - try: - import ansible_mitogen.connection + import ansible_mitogen except ImportError: base_dir = os.path.dirname(__file__) sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) del base_dir import ansible_mitogen.connection +import ansible_mitogen.loaders class Connection(ansible_mitogen.connection.Connection): transport = 'ssh' - vanilla_class = ansible.plugins.connection.ssh.Connection + vanilla_class = ansible_mitogen.loaders.connection_loader__get( + 'ssh', + class_only=True, + ) @staticmethod def _create_control_path(*args, **kwargs): diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 5cf171b6..06402344 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -803,9 +803,10 @@ class NewStyleRunner(ScriptRunner): #: path => new-style module bytecode. _code_by_path = {} - def __init__(self, module_map, **kwargs): + def __init__(self, module_map, py_module_name, **kwargs): super(NewStyleRunner, self).__init__(**kwargs) self.module_map = module_map + self.py_module_name = py_module_name def _setup_imports(self): """ @@ -942,9 +943,22 @@ class NewStyleRunner(ScriptRunner): self._handle_magic_exception(mod, sys.exc_info()[1]) raise + def _get_module_package(self): + """ + Since Ansible 2.9 __package__ must be set in accordance with an + approximation of the original package hierarchy, so that relative + imports function correctly. + """ + pkg, sep, modname = str_rpartition(self.py_module_name, '.') + if not sep: + return None + if mitogen.core.PY3: + return pkg + return pkg.encode() + def _run(self): mod = types.ModuleType(self.main_module_name) - mod.__package__ = None + mod.__package__ = self._get_module_package() # Some Ansible modules use __file__ to find the Ansiballz temporary # directory. We must provide some temporary path in __file__, but we # don't want to pointlessly write the module to disk when it never diff --git a/ansible_mitogen/strategy.py b/ansible_mitogen/strategy.py index 8f093999..d82e6112 100644 --- a/ansible_mitogen/strategy.py +++ b/ansible_mitogen/strategy.py @@ -27,6 +27,7 @@ # POSSIBILITY OF SUCH DAMAGE. from __future__ import absolute_import +import distutils.version import os import signal import threading @@ -52,8 +53,8 @@ except ImportError: Sentinel = None -ANSIBLE_VERSION_MIN = '2.3' -ANSIBLE_VERSION_MAX = '2.8' +ANSIBLE_VERSION_MIN = (2, 3) +ANSIBLE_VERSION_MAX = (2, 9) NEW_VERSION_MSG = ( "Your Ansible version (%s) is too recent. The most recent version\n" "supported by Mitogen for Ansible is %s.x. Please check the Mitogen\n" @@ -76,13 +77,15 @@ def _assert_supported_release(): an unsupported Ansible release. """ v = ansible.__version__ + if not isinstance(v, tuple): + v = tuple(distutils.version.LooseVersion(v).version) - if v[:len(ANSIBLE_VERSION_MIN)] < ANSIBLE_VERSION_MIN: + if v[:2] < ANSIBLE_VERSION_MIN: raise ansible.errors.AnsibleError( OLD_VERSION_MSG % (v, ANSIBLE_VERSION_MIN) ) - if v[:len(ANSIBLE_VERSION_MAX)] > ANSIBLE_VERSION_MAX: + if v[:2] > ANSIBLE_VERSION_MAX: raise ansible.errors.AnsibleError( NEW_VERSION_MSG % (ansible.__version__, ANSIBLE_VERSION_MAX) ) @@ -132,7 +135,7 @@ def wrap_action_loader__get(name, *args, **kwargs): if ansible.__version__ >= '2.8': get_kwargs['collection_list'] = kwargs.pop('collection_list', None) - klass = action_loader__get(name, **get_kwargs) + klass = ansible_mitogen.loaders.action_loader__get(name, **get_kwargs) if klass: bases = (ansible_mitogen.mixins.ActionModuleMixin, klass) adorned_klass = type(str(name), bases, {}) @@ -141,15 +144,29 @@ def wrap_action_loader__get(name, *args, **kwargs): return adorned_klass(*args, **kwargs) +REDIRECTED_CONNECTION_PLUGINS = ( + 'buildah', + 'docker', + 'kubectl', + 'jail', + 'local', + 'lxc', + 'lxd', + 'machinectl', + 'setns', + 'ssh', +) + + def wrap_connection_loader__get(name, *args, **kwargs): """ While a Mitogen strategy is active, rewrite connection_loader.get() calls for some transports into requests for a compatible Mitogen transport. """ - if name in ('buildah', 'docker', 'kubectl', 'jail', 'local', - 'lxc', 'lxd', 'machinectl', 'setns', 'ssh'): + if name in REDIRECTED_CONNECTION_PLUGINS: name = 'mitogen_' + name - return connection_loader__get(name, *args, **kwargs) + + return ansible_mitogen.loaders.connection_loader__get(name, *args, **kwargs) def wrap_worker__run(self): @@ -199,12 +216,7 @@ class AnsibleWrappers(object): Install our PluginLoader monkey patches and update global variables with references to the real functions. """ - global action_loader__get - action_loader__get = ansible_mitogen.loaders.action_loader.get ansible_mitogen.loaders.action_loader.get = wrap_action_loader__get - - global connection_loader__get - connection_loader__get = ansible_mitogen.loaders.connection_loader.get ansible_mitogen.loaders.connection_loader.get = wrap_connection_loader__get global worker__run @@ -215,8 +227,12 @@ class AnsibleWrappers(object): """ Uninstall the PluginLoader monkey patches. """ - ansible_mitogen.loaders.action_loader.get = action_loader__get - ansible_mitogen.loaders.connection_loader.get = connection_loader__get + ansible_mitogen.loaders.action_loader.get = ( + ansible_mitogen.loaders.action_loader__get + ) + ansible_mitogen.loaders.connection_loader.get = ( + ansible_mitogen.loaders.connection_loader__get + ) ansible.executor.process.worker.WorkerProcess.run = worker__run def install(self): diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index f5fe42b0..97771afa 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -7,6 +7,7 @@ {# Alabaster ships a completely useless custom.css, suppress it. #} {%- block extrahead %} + {% endblock %} diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index 22d7223f..2382bbea 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -85,10 +85,15 @@ Installation Get notified of new releases and important fixes.

- +
+ + + + +