diff --git a/ansible_mitogen/action/mitogen.py b/ansible_mitogen/action/mitogen.py deleted file mode 100644 index fb6ce253..00000000 --- a/ansible_mitogen/action/mitogen.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2017, David Wilson -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its contributors -# may be used to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import json -import os - -import ansible -import ansible.plugins -import ansible.plugins.action.normal -import ansible_mitogen.helpers - - -ANSIBLE_BASEDIR = os.path.dirname(ansible.__file__) - - -class ActionModule(ansible.plugins.action.normal.ActionModule): - def get_py_module_name(self, module_name): - path = ansible.plugins.module_loader.find_plugin(module_name, '') - relpath = os.path.relpath(path, ANSIBLE_BASEDIR) - root, _ = os.path.splitext(relpath) - return 'ansible.' + root.replace('/', '.') - - def _execute_module(self, module_name=None, module_args=None, tmp=None, - task_vars=None, persist_files=False, - delete_remote_tmp=True, wrap_async=False): - - module_name = module_name or self._task.action - module_args = module_args or self._task.args - task_vars = task_vars or {} - - self._update_module_args(module_name, module_args, task_vars) - - ##################################################################### - - py_module_name = self.get_py_module_name(module_name) - js = self._connection.py_call(ansible_mitogen.helpers.run_module, py_module_name, - args=json.loads(json.dumps(module_args))) - - ##################################################################### - - data = self._parse_returned_data({ - 'rc': 0, - 'stdout': js, - 'stdout_lines': [js], - 'stderr': '' - }) - - if wrap_async: - data['changed'] = True - - # pre-split stdout/stderr into lines if needed - if 'stdout' in data and 'stdout_lines' not in data: - # if the value is 'False', a default won't catch it. - txt = data.get('stdout', None) or u'' - data['stdout_lines'] = txt.splitlines() - if 'stderr' in data and 'stderr_lines' not in data: - # if the value is 'False', a default won't catch it. - txt = data.get('stderr', None) or u'' - data['stderr_lines'] = txt.splitlines() - - return data diff --git a/ansible_mitogen/connection/mitogen.py b/ansible_mitogen/connection/mitogen.py index 1be08d9e..20147619 100644 --- a/ansible_mitogen/connection/mitogen.py +++ b/ansible_mitogen/connection/mitogen.py @@ -25,23 +25,6 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -""" -Basic Ansible connection plug-in mostly useful for testing functionality, -due to Ansible's use of the multiprocessing package a lot more work is required -to share the mitogen SSH connection across tasks. - -Enable it by: - - $ cat ansible.cfg - [defaults] - connection_plugins = plugins/connection - - $ mkdir -p plugins/connection - $ cat > plugins/connection/mitogen_conn.py <<-EOF - from mitogen.ansible.connection import Connection - EOF -""" - from __future__ import absolute_import import os @@ -50,6 +33,8 @@ import ansible.plugins.connection import ansible_mitogen.helpers import mitogen.unix +from ansible_mitogen.utils import cast + class Connection(ansible.plugins.connection.ConnectionBase): router = None @@ -69,39 +54,50 @@ class Connection(ansible.plugins.connection.ConnectionBase): path = os.environ['LISTENER_SOCKET_PATH'] self.router, self.parent = mitogen.unix.connect(path) - host = mitogen.service.call(self.parent, 500, { + host = mitogen.service.call(self.parent, 500, cast({ 'method': 'ssh', 'hostname': self._play_context.remote_addr, 'username': self._play_context.remote_user, + 'password': self._play_context.password, 'port': self._play_context.port, + 'python_path': '/usr/bin/python', 'ssh_path': self._play_context.ssh_executable, - }) + })) if not self._play_context.become: self.context = host else: - self.context = mitogen.service.call(self.parent, 500, { + self.context = mitogen.service.call(self.parent, 500, cast({ 'method': 'sudo', 'username': self._play_context.become_user, + 'password': self._play_context.password, + 'python_path': '/usr/bin/python', 'via': host, - }) + 'debug': True, + })) - def py_call(self, func, *args, **kwargs): + def call_async(self, func, *args, **kwargs): self._connect() - return self.context.call(func, *args, **kwargs) + print[func, args, kwargs] + return self.context.call_async(func, *args, **kwargs) + + def call(self, func, *args, **kwargs): + return self.call_async(func, *args, **kwargs).get().unpickle() def exec_command(self, cmd, in_data=None, sudoable=True): super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable) if in_data: raise ansible.errors.AnsibleError("does not support module pipelining") - return self.py_call(ansible_mitogen.helpers.exec_command, cmd, in_data) + return self.py_call(ansible_mitogen.helpers.exec_command, + cast(cmd), cast(in_data)) def fetch_file(self, in_path, out_path): - output = self.py_call(ansible_mitogen.helpers.read_path, in_path) + output = self.py_call(ansible_mitogen.helpers.read_path, + cast(in_path)) ansible_mitogen.helpers.write_path(out_path, output) def put_file(self, in_path, out_path): - self.py_call(ansible_mitogen.helpers.write_path, out_path, + self.py_call(ansible_mitogen.helpers.write_path, cast(out_path), ansible_mitogen.helpers.read_path(in_path)) def close(self): diff --git a/ansible_mitogen/helpers.py b/ansible_mitogen/helpers.py index 518c0780..9190a4f8 100644 --- a/ansible_mitogen/helpers.py +++ b/ansible_mitogen/helpers.py @@ -25,15 +25,6 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -""" -Ansible is so poorly layered that attempting to import anything under -ansible.plugins automatically triggers import of __main__, which causes -remote execution of the ansible command-line tool. :( - -So here we define helpers in some sanely layered package where the entirety of -Ansible won't be imported. -""" - import json import subprocess import time @@ -62,10 +53,9 @@ class ModuleError(Exception): def wtf_exit_json(self, **kwargs): """ - Replace AnsibleModule.exit_json() with something that doesn't try to - suicide the process or JSON-encode the dictionary. Instead, cause Exit to - be raised, with a `dct` attribute containing the successful result - dictionary. + Replace AnsibleModule.exit_json() with something that doesn't try to kill + the process or JSON-encode the result dictionary. Instead, cause Exit to be + raised, with a `dct` attribute containing the successful result dictionary. """ self.add_path_info(kwargs) kwargs.setdefault('changed', False) @@ -95,7 +85,7 @@ def wtf_fail_json(self, **kwargs): def run_module(module, raw_params=None, args=None): """ Set up the process environment in preparation for running an Ansible - module. The monkey-patches the Ansible libraries in various places to + module. This monkey-patches the Ansible libraries in various places to prevent it from trying to kill the process on completion, and to prevent it from reading sys.stdin. """ @@ -112,13 +102,13 @@ def run_module(module, raw_params=None, args=None): try: mod = __import__(module, {}, {}, ['']) - # Ansible modules begin execution on import, because they're crap from - # hell. Thus the above __import__ will cause either Exit or - # ModuleError to be raised. If we reach the line below, the module did - # not execute and must already have been imported for a previous - # invocation, so we need to invoke main explicitly. + # Ansible modules begin execution on import. Thus the above __import__ + # will cause either Exit or ModuleError to be raised. If we reach the + # line below, the module did not execute and must already have been + # imported for a previous invocation, so we need to invoke main + # explicitly. mod.main() - except Exit, e: + except (Exit, ModuleError), e: return json.dumps(e.dct) diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py new file mode 100644 index 00000000..cebda09c --- /dev/null +++ b/ansible_mitogen/mixins.py @@ -0,0 +1,166 @@ +# Copyright 2017, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import pwd +import os +import tempfile + +import ansible +import ansible.plugins.action + +import mitogen.master +import ansible_mitogen.helpers +from ansible.module_utils._text import to_text +from ansible_mitogen.utils import cast +from ansible_mitogen.utils import get_command_module_name + + +class ActionModuleMixin(ansible.plugins.action.ActionBase): + def call(self, func, *args, **kwargs): + return self._connection.call(func, *args, **kwargs) + + COMMAND_RESULT = { + 'rc': 0, + 'stdout': '', + 'stdout_lines': [], + 'stderr': '' + } + + def fake_shell(self, func, stdout=False): + dct = self.COMMAND_RESULT.copy() + try: + rc = func() + if stdout: + dct['stdout'] = repr(rc) + except mitogen.core.CallError: + dct['rc'] = 1 + dct['stderr'] = traceback.format_exc() + + return dct + + def _remote_file_exists(self, path): + # replaces 5 lines. + return self.call(os.path.exists, path) + + def _configure_module(self, module_name, module_args, task_vars=None): + # replaces 58 lines + assert False, "_configure_module() should never be called." + + def _is_pipelining_enabled(self, module_style, wrap_async=False): + # replaces 17 lines + return False + + def _make_tmp_path(self, remote_user=None): + # replaces 58 lines + return self.call(tempfile.mkdtemp, prefix='ansible_mitogen') + + def _remove_tmp_path(self, tmp_path): + # replaces 10 lines + if self._should_remove_tmp_path(tmp_path): + return self.call(shutil.rmtree, tmp_path) + + def _transfer_data(self, remote_path, data): + # replaces 20 lines + assert False, "_transfer_data() should never be called." + + def _fixup_perms2(self, remote_paths, remote_user=None, execute=True): + # replaces 83 lines + assert False, "_fixup_perms2() should never be called." + + def _remote_chmod(self, paths, mode, sudoable=False): + return self.fake_shell(lambda: mitogen.master.Select.all( + self._connection.call_async(os.chmod, path, mode) + for path in paths + )) + + def _remote_chown(self, paths, user, sudoable=False): + ent = self.call(pwd.getpwnam, user) + return self.fake_shell(lambda: mitogen.master.Select.all( + self._connection.call_async(os.chown, path, ent.pw_uid, ent.pw_gid) + for path in paths + )) + + def _remote_expand_user(self, path, sudoable=True): + # replaces 25 lines + if path.startswith('~'): + path = self.call(os.path.expanduser, path) + return path + + def _execute_module(self, module_name=None, module_args=None, tmp=None, + task_vars=None, persist_files=False, + delete_remote_tmp=True, wrap_async=False): + module_name = module_name or self._task.action + module_args = module_args or self._task.args + task_vars = task_vars or {} + + self._update_module_args(module_name, module_args, task_vars) + + # replaces 110 lines + js = self._connection.call( + ansible_mitogen.helpers.run_module, + get_command_module_name(module_name), + args=cast(module_args) + ) + + data = self._parse_returned_data({ + 'rc': 0, + 'stdout': js, + 'stdout_lines': [js], + 'stderr': '' + }) + + if wrap_async: + data['changed'] = True + + # pre-split stdout/stderr into lines if needed + if 'stdout' in data and 'stdout_lines' not in data: + # if the value is 'False', a default won't catch it. + txt = data.get('stdout', None) or u'' + data['stdout_lines'] = txt.splitlines() + if 'stderr' in data and 'stderr_lines' not in data: + # if the value is 'False', a default won't catch it. + txt = data.get('stderr', None) or u'' + data['stderr_lines'] = txt.splitlines() + + return data + + def _low_level_execute_command(self, cmd, sudoable=True, in_data=None, + executable=None, + encoding_errors='surrogate_then_replace'): + # replaces 57 lines + # replaces 126 lines of make_become_cmd() + rc, stdout, stderr = self.call( + ansible_mitogen.helpers.exec_command, + cmd, + in_data, + ) + return { + 'rc': rc, + 'stdout': to_text(stdout, encoding_errors), + 'stdout_lines': '\n'.split(to_text(stdout, encoding_errors)), + 'stderr': stderr, + } diff --git a/ansible_mitogen/strategy/mitogen.py b/ansible_mitogen/strategy/mitogen.py index b239e204..53321251 100644 --- a/ansible_mitogen/strategy/mitogen.py +++ b/ansible_mitogen/strategy/mitogen.py @@ -33,30 +33,57 @@ import mitogen.master import mitogen.service import mitogen.unix import mitogen.utils -import ansible_mitogen.action.mitogen import ansible.errors import ansible.plugins.strategy.linear import ansible.plugins +import ansible_mitogen.mixins + + +def wrap_action_loader__get(name, *args, **kwargs): + """ + Trap calls to the action plug-in loader, supplementing the type of any + ActionModule with Mitogen's ActionModuleMixin before constructing it, + causing the mix-in methods to override any inherited from Ansible's base + class, replacing most shell use with pure Python equivalents. + + This is preferred to static subclassing as it generalizes to third party + action modules existing outside the Ansible tree. + """ + klass = action_loader__get(name, class_only=True) + if klass: + wrapped_name = 'MitogenActionModule_' + name + bases = (ansible_mitogen.mixins.ActionModuleMixin, klass) + adorned_klass = type(name, bases, {}) + return adorned_klass(*args, **kwargs) + +action_loader__get = ansible.plugins.action_loader.get +ansible.plugins.action_loader.get = wrap_action_loader__get class ContextProxyService(mitogen.service.Service): + """ + Implement a service accessible from worker processes connecting back into + the top-level process. The service yields an existing context matching a + connection configuration if it exists, otherwise it constructs a new + conncetion before returning it. + """ well_known_id = 500 max_message_size = 1000 def __init__(self, router): super(ContextProxyService, self).__init__(router) - self._context_by_id = {} + self._context_by_key = {} def validate_args(self, args): return isinstance(args, dict) def dispatch(self, dct, msg): key = repr(sorted(dct.items())) - if key not in self._context_by_id: + if key not in self._context_by_key: method = getattr(self.router, dct.pop('method')) - self._context_by_id[key] = method(**dct) - return self._context_by_id[key] + self._context_by_key[key] = method(**dct) + return self._context_by_key[key] class StrategyModule(ansible.plugins.strategy.linear.StrategyModule): @@ -65,8 +92,10 @@ class StrategyModule(ansible.plugins.strategy.linear.StrategyModule): self.add_connection_plugin_path() def add_connection_plugin_path(self): - """Automatically add the connection plug-in directory to the - ModuleLoader path, reduces end-user configuration.""" + """ + Automatically add the connection plug-in directory to the ModuleLoader + path, slightly reduces end-user configuration. + """ # ansible_mitogen base directory: basedir = os.path.dirname(os.path.dirname(__file__)) conn_dir = os.path.join(basedir, 'connection') @@ -74,11 +103,16 @@ class StrategyModule(ansible.plugins.strategy.linear.StrategyModule): def run(self, iterator, play_context, result=0): self.router = mitogen.master.Router() + self.router.responder.blacklist('OpenSSL') + self.router.responder.blacklist('urllib3') + self.router.responder.blacklist('requests') + self.router.responder.blacklist('systemd') + self.router.responder.blacklist('selinux') self.listener = mitogen.unix.Listener(self.router) os.environ['LISTENER_SOCKET_PATH'] = self.listener.path self.service = ContextProxyService(self.router) - mitogen.utils.log_to_file() + mitogen.utils.log_to_file()#level='DEBUG', io=False) if play_context.connection == 'ssh': play_context.connection = 'mitogen' @@ -88,13 +122,6 @@ class StrategyModule(ansible.plugins.strategy.linear.StrategyModule): th.setDaemon(True) th.start() - real_get = ansible.plugins.action_loader.get - def get(name, *args, **kwargs): - if name == 'normal': - return ansible_mitogen.action.mitogen.ActionModule(*args, **kwargs) - return real_get(name, *args, **kwargs) - ansible.plugins.action_loader.get = get - try: return super(StrategyModule, self).run(iterator, play_context) finally: diff --git a/ansible_mitogen/utils.py b/ansible_mitogen/utils.py new file mode 100644 index 00000000..98193a0e --- /dev/null +++ b/ansible_mitogen/utils.py @@ -0,0 +1,75 @@ +# Copyright 2017, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import os + +import ansible +import ansible.plugins +import mitogen.core + + +def cast(obj): + """ + Ansible loves to decorate built-in types to implement useful functionality + like Vault, however cPickle loves to preserve those decorations during + serialization, resulting in CallError. + + So here we recursively undecorate `obj`, ensuring that any instances of + subclasses of built-in types are downcast to the base type. + """ + if isinstance(obj, dict): + return {cast(k): cast(v) for k, v in obj.iteritems()} + if isinstance(obj, (list, tuple)): + return [cast(v) for v in obj] + if obj is None or isinstance(obj, (int, float)): + return obj + if isinstance(obj, unicode): + return unicode(obj) + if isinstance(obj, str): + return str(obj) + if isinstance(obj, (mitogen.core.Context, + mitogen.core.Dead, + mitogen.core.CallError)): + return obj + raise TypeError("Cannot serialize: %r: %r" % (type(obj), obj)) + + +def get_command_module_name(module_name): + """ + Given the name of an Ansible command module, return its canonical module + path within the ansible. + + :param module_name: + "shell" + :return: + "ansible.modules.commands.shell" + """ + path = ansible.plugins.module_loader.find_plugin(module_name, '') + relpath = os.path.relpath(path, os.path.dirname(ansible.__file__)) + root, _ = os.path.splitext(relpath) + return 'ansible.' + root.replace('/', '.')