ansible: generalized action module wrapping.

ansible_mitogen/action/mitogen.py

@@ -1,85 +0,0 @@
-# Copyright 2017, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice, this
-# list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-import json
-import os
-
-import ansible
-import ansible.plugins
-import ansible.plugins.action.normal
-import ansible_mitogen.helpers
-
-
-ANSIBLE_BASEDIR = os.path.dirname(ansible.__file__)
-
-
-class ActionModule(ansible.plugins.action.normal.ActionModule):
-    def get_py_module_name(self, module_name):
-        path = ansible.plugins.module_loader.find_plugin(module_name, '')
-        relpath = os.path.relpath(path, ANSIBLE_BASEDIR)
-        root, _ = os.path.splitext(relpath)
-        return 'ansible.' + root.replace('/', '.')
-
-    def _execute_module(self, module_name=None, module_args=None, tmp=None,
-                        task_vars=None, persist_files=False,
-                        delete_remote_tmp=True, wrap_async=False):
-
-        module_name = module_name or self._task.action
-        module_args = module_args or self._task.args
-        task_vars = task_vars or {}
-
-        self._update_module_args(module_name, module_args, task_vars)
-
-        #####################################################################
-
-        py_module_name = self.get_py_module_name(module_name)
-        js = self._connection.py_call(ansible_mitogen.helpers.run_module, py_module_name,
-                                      args=json.loads(json.dumps(module_args)))
-
-        #####################################################################
-
-        data = self._parse_returned_data({
-            'rc': 0,
-            'stdout': js,
-            'stdout_lines': [js],
-            'stderr': ''
-        })
-
-        if wrap_async:
-            data['changed'] = True
-
-        # pre-split stdout/stderr into lines if needed
-        if 'stdout' in data and 'stdout_lines' not in data:
-            # if the value is 'False', a default won't catch it.
-            txt = data.get('stdout', None) or u''
-            data['stdout_lines'] = txt.splitlines()
-        if 'stderr' in data and 'stderr_lines' not in data:
-            # if the value is 'False', a default won't catch it.
-            txt = data.get('stderr', None) or u''
-            data['stderr_lines'] = txt.splitlines()
-
-        return data

ansible_mitogen/connection/mitogen.py

@@ -25,23 +25,6 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-"""
-Basic Ansible connection plug-in mostly useful for testing functionality,
-due to Ansible's use of the multiprocessing package a lot more work is required
-to share the mitogen SSH connection across tasks.
-
-Enable it by:
-
-    $ cat ansible.cfg
-    [defaults]
-    connection_plugins = plugins/connection
-
-    $ mkdir -p plugins/connection
-    $ cat > plugins/connection/mitogen_conn.py <<-EOF
-    from mitogen.ansible.connection import Connection
-    EOF
-"""
-
 from __future__ import absolute_import
 import os
 
@@ -50,6 +33,8 @@ import ansible.plugins.connection
 import ansible_mitogen.helpers
 import mitogen.unix
 
+from ansible_mitogen.utils import cast
+
 
 class Connection(ansible.plugins.connection.ConnectionBase):
     router = None
@@ -69,39 +54,50 @@ class Connection(ansible.plugins.connection.ConnectionBase):
         path = os.environ['LISTENER_SOCKET_PATH']
         self.router, self.parent = mitogen.unix.connect(path)
 
-        host = mitogen.service.call(self.parent, 500, {
+        host = mitogen.service.call(self.parent, 500, cast({
             'method': 'ssh',
             'hostname': self._play_context.remote_addr,
             'username': self._play_context.remote_user,
+            'password': self._play_context.password,
             'port': self._play_context.port,
+            'python_path': '/usr/bin/python',
             'ssh_path': self._play_context.ssh_executable,
-        })
+        }))
 
         if not self._play_context.become:
             self.context = host
         else:
-            self.context = mitogen.service.call(self.parent, 500, {
+            self.context = mitogen.service.call(self.parent, 500, cast({
                 'method': 'sudo',
                 'username': self._play_context.become_user,
+                'password': self._play_context.password,
+                'python_path': '/usr/bin/python',
                 'via': host,
-            })
+                'debug': True,
+            }))
 
-    def py_call(self, func, *args, **kwargs):
+    def call_async(self, func, *args, **kwargs):
         self._connect()
-        return self.context.call(func, *args, **kwargs)
+        print[func, args, kwargs]
+        return self.context.call_async(func, *args, **kwargs)
+
+    def call(self, func, *args, **kwargs):
+        return self.call_async(func, *args, **kwargs).get().unpickle()
 
     def exec_command(self, cmd, in_data=None, sudoable=True):
         super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)
         if in_data:
             raise ansible.errors.AnsibleError("does not support module pipelining")
-        return self.py_call(ansible_mitogen.helpers.exec_command, cmd, in_data)
+        return self.py_call(ansible_mitogen.helpers.exec_command,
+                            cast(cmd), cast(in_data))
 
     def fetch_file(self, in_path, out_path):
-        output = self.py_call(ansible_mitogen.helpers.read_path, in_path)
+        output = self.py_call(ansible_mitogen.helpers.read_path,
+                              cast(in_path))
         ansible_mitogen.helpers.write_path(out_path, output)
 
     def put_file(self, in_path, out_path):
-        self.py_call(ansible_mitogen.helpers.write_path, out_path,
+        self.py_call(ansible_mitogen.helpers.write_path, cast(out_path),
                      ansible_mitogen.helpers.read_path(in_path))
 
     def close(self):

ansible_mitogen/helpers.py

@@ -25,15 +25,6 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-"""
-Ansible is so poorly layered that attempting to import anything under
-ansible.plugins automatically triggers import of __main__, which causes
-remote execution of the ansible command-line tool. :(
-
-So here we define helpers in some sanely layered package where the entirety of
-Ansible won't be imported.
-"""
-
 import json
 import subprocess
 import time
@@ -62,10 +53,9 @@ class ModuleError(Exception):
 
 def wtf_exit_json(self, **kwargs):
     """
-    Replace AnsibleModule.exit_json() with something that doesn't try to
+    Replace AnsibleModule.exit_json() with something that doesn't try to kill
-    suicide the process or JSON-encode the dictionary. Instead, cause Exit to
+    the process or JSON-encode the result dictionary. Instead, cause Exit to be
-    be raised, with a `dct` attribute containing the successful result
+    raised, with a `dct` attribute containing the successful result dictionary.
-    dictionary.
     """
     self.add_path_info(kwargs)
     kwargs.setdefault('changed', False)
@@ -95,7 +85,7 @@ def wtf_fail_json(self, **kwargs):
 def run_module(module, raw_params=None, args=None):
     """
     Set up the process environment in preparation for running an Ansible
-    module. The monkey-patches the Ansible libraries in various places to
+    module. This monkey-patches the Ansible libraries in various places to
     prevent it from trying to kill the process on completion, and to prevent it
     from reading sys.stdin.
     """
@@ -112,13 +102,13 @@ def run_module(module, raw_params=None, args=None):
 
     try:
         mod = __import__(module, {}, {}, [''])
-        # Ansible modules begin execution on import, because they're crap from
+        # Ansible modules begin execution on import. Thus the above __import__
-        # hell. Thus the above __import__ will cause either Exit or
+        # will cause either Exit or ModuleError to be raised. If we reach the
-        # ModuleError to be raised. If we reach the line below, the module did
+        # line below, the module did not execute and must already have been
-        # not execute and must already have been imported for a previous
+        # imported for a previous invocation, so we need to invoke main
-        # invocation, so we need to invoke main explicitly.
+        # explicitly.
         mod.main()
-    except Exit, e:
+    except (Exit, ModuleError), e:
         return json.dumps(e.dct)
 
 

ansible_mitogen/mixins.py

@@ -0,0 +1,166 @@
+# Copyright 2017, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import pwd
+import os
+import tempfile
+
+import ansible
+import ansible.plugins.action
+
+import mitogen.master
+import ansible_mitogen.helpers
+from ansible.module_utils._text import to_text
+from ansible_mitogen.utils import cast
+from ansible_mitogen.utils import get_command_module_name
+
+
+class ActionModuleMixin(ansible.plugins.action.ActionBase):
+    def call(self, func, *args, **kwargs):
+        return self._connection.call(func, *args, **kwargs)
+
+    COMMAND_RESULT = {
+        'rc': 0,
+        'stdout': '',
+        'stdout_lines': [],
+        'stderr': ''
+    }
+
+    def fake_shell(self, func, stdout=False):
+        dct = self.COMMAND_RESULT.copy()
+        try:
+            rc = func()
+            if stdout:
+                dct['stdout'] = repr(rc)
+        except mitogen.core.CallError:
+            dct['rc'] = 1
+            dct['stderr'] = traceback.format_exc()
+
+        return dct
+
+    def _remote_file_exists(self, path):
+        # replaces 5 lines.
+        return self.call(os.path.exists, path)
+
+    def _configure_module(self, module_name, module_args, task_vars=None):
+        # replaces 58 lines
+        assert False, "_configure_module() should never be called."
+
+    def _is_pipelining_enabled(self, module_style, wrap_async=False):
+        # replaces 17 lines
+        return False
+
+    def _make_tmp_path(self, remote_user=None):
+        # replaces 58 lines
+        return self.call(tempfile.mkdtemp, prefix='ansible_mitogen')
+
+    def _remove_tmp_path(self, tmp_path):
+        # replaces 10 lines
+        if self._should_remove_tmp_path(tmp_path):
+            return self.call(shutil.rmtree, tmp_path)
+
+    def _transfer_data(self, remote_path, data):
+        # replaces 20 lines
+        assert False, "_transfer_data() should never be called."
+
+    def _fixup_perms2(self, remote_paths, remote_user=None, execute=True):
+        # replaces 83 lines
+        assert False, "_fixup_perms2() should never be called."
+
+    def _remote_chmod(self, paths, mode, sudoable=False):
+        return self.fake_shell(lambda: mitogen.master.Select.all(
+            self._connection.call_async(os.chmod, path, mode)
+            for path in paths
+        ))
+
+    def _remote_chown(self, paths, user, sudoable=False):
+        ent = self.call(pwd.getpwnam, user)
+        return self.fake_shell(lambda: mitogen.master.Select.all(
+            self._connection.call_async(os.chown, path, ent.pw_uid, ent.pw_gid)
+            for path in paths
+        ))
+
+    def _remote_expand_user(self, path, sudoable=True):
+        # replaces 25 lines
+        if path.startswith('~'):
+            path = self.call(os.path.expanduser, path)
+        return path
+
+    def _execute_module(self, module_name=None, module_args=None, tmp=None,
+                        task_vars=None, persist_files=False,
+                        delete_remote_tmp=True, wrap_async=False):
+        module_name = module_name or self._task.action
+        module_args = module_args or self._task.args
+        task_vars = task_vars or {}
+
+        self._update_module_args(module_name, module_args, task_vars)
+
+        # replaces 110 lines
+        js = self._connection.call(
+            ansible_mitogen.helpers.run_module,
+            get_command_module_name(module_name),
+            args=cast(module_args)
+        )
+
+        data = self._parse_returned_data({
+            'rc': 0,
+            'stdout': js,
+            'stdout_lines': [js],
+            'stderr': ''
+        })
+
+        if wrap_async:
+            data['changed'] = True
+
+        # pre-split stdout/stderr into lines if needed
+        if 'stdout' in data and 'stdout_lines' not in data:
+            # if the value is 'False', a default won't catch it.
+            txt = data.get('stdout', None) or u''
+            data['stdout_lines'] = txt.splitlines()
+        if 'stderr' in data and 'stderr_lines' not in data:
+            # if the value is 'False', a default won't catch it.
+            txt = data.get('stderr', None) or u''
+            data['stderr_lines'] = txt.splitlines()
+
+        return data
+
+    def _low_level_execute_command(self, cmd, sudoable=True, in_data=None,
+                                   executable=None,
+                                   encoding_errors='surrogate_then_replace'):
+        # replaces 57 lines
+        # replaces 126 lines of make_become_cmd()
+        rc, stdout, stderr = self.call(
+            ansible_mitogen.helpers.exec_command,
+            cmd,
+            in_data,
+        )
+        return {
+            'rc': rc,
+            'stdout': to_text(stdout, encoding_errors),
+            'stdout_lines': '\n'.split(to_text(stdout, encoding_errors)),
+            'stderr': stderr,
+        }

ansible_mitogen/strategy/mitogen.py

@@ -33,30 +33,57 @@ import mitogen.master
 import mitogen.service
 import mitogen.unix
 import mitogen.utils
-import ansible_mitogen.action.mitogen
 
 import ansible.errors
 import ansible.plugins.strategy.linear
 import ansible.plugins
+import ansible_mitogen.mixins
+
+
+def wrap_action_loader__get(name, *args, **kwargs):
+    """
+    Trap calls to the action plug-in loader, supplementing the type of any
+    ActionModule with Mitogen's ActionModuleMixin before constructing it,
+    causing the mix-in methods to override any inherited from Ansible's base
+    class, replacing most shell use with pure Python equivalents.
+
+    This is preferred to static subclassing as it generalizes to third party
+    action modules existing outside the Ansible tree.
+    """
+    klass = action_loader__get(name, class_only=True)
+    if klass:
+        wrapped_name = 'MitogenActionModule_' + name
+        bases = (ansible_mitogen.mixins.ActionModuleMixin, klass)
+        adorned_klass = type(name, bases, {})
+        return adorned_klass(*args, **kwargs)
+
+action_loader__get = ansible.plugins.action_loader.get
+ansible.plugins.action_loader.get = wrap_action_loader__get
 
 
 class ContextProxyService(mitogen.service.Service):
+    """
+    Implement a service accessible from worker processes connecting back into
+    the top-level process. The service yields an existing context matching a
+    connection configuration if it exists, otherwise it constructs a new
+    conncetion before returning it.
+    """
     well_known_id = 500
     max_message_size = 1000
 
     def __init__(self, router):
         super(ContextProxyService, self).__init__(router)
-        self._context_by_id = {}
+        self._context_by_key = {}
 
     def validate_args(self, args):
         return isinstance(args, dict)
 
     def dispatch(self, dct, msg):
         key = repr(sorted(dct.items()))
-        if key not in self._context_by_id:
+        if key not in self._context_by_key:
             method = getattr(self.router, dct.pop('method'))
-            self._context_by_id[key] = method(**dct)
+            self._context_by_key[key] = method(**dct)
-        return self._context_by_id[key]
+        return self._context_by_key[key]
 
 
 class StrategyModule(ansible.plugins.strategy.linear.StrategyModule):
@@ -65,8 +92,10 @@ class StrategyModule(ansible.plugins.strategy.linear.StrategyModule):
         self.add_connection_plugin_path()
 
     def add_connection_plugin_path(self):
-        """Automatically add the connection plug-in directory to the
+        """
-        ModuleLoader path, reduces end-user configuration."""
+        Automatically add the connection plug-in directory to the ModuleLoader
+        path, slightly reduces end-user configuration.
+        """
         # ansible_mitogen base directory:
         basedir = os.path.dirname(os.path.dirname(__file__))
         conn_dir = os.path.join(basedir, 'connection')
@@ -74,11 +103,16 @@ class StrategyModule(ansible.plugins.strategy.linear.StrategyModule):
 
     def run(self, iterator, play_context, result=0):
         self.router = mitogen.master.Router()
+        self.router.responder.blacklist('OpenSSL')
+        self.router.responder.blacklist('urllib3')
+        self.router.responder.blacklist('requests')
+        self.router.responder.blacklist('systemd')
+        self.router.responder.blacklist('selinux')
         self.listener = mitogen.unix.Listener(self.router)
         os.environ['LISTENER_SOCKET_PATH'] = self.listener.path
 
         self.service = ContextProxyService(self.router)
-        mitogen.utils.log_to_file()
+        mitogen.utils.log_to_file()#level='DEBUG', io=False)
 
         if play_context.connection == 'ssh':
             play_context.connection = 'mitogen'
@@ -88,13 +122,6 @@ class StrategyModule(ansible.plugins.strategy.linear.StrategyModule):
         th.setDaemon(True)
         th.start()
 
-        real_get = ansible.plugins.action_loader.get
-        def get(name, *args, **kwargs):
-            if name == 'normal':
-                return ansible_mitogen.action.mitogen.ActionModule(*args, **kwargs)
-            return real_get(name, *args, **kwargs)
-        ansible.plugins.action_loader.get = get
-
         try:
             return super(StrategyModule, self).run(iterator, play_context)
         finally:

ansible_mitogen/utils.py

@@ -0,0 +1,75 @@
+# Copyright 2017, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import absolute_import
+import os
+
+import ansible
+import ansible.plugins
+import mitogen.core
+
+
+def cast(obj):
+    """
+    Ansible loves to decorate built-in types to implement useful functionality
+    like Vault, however cPickle loves to preserve those decorations during
+    serialization, resulting in CallError.
+
+    So here we recursively undecorate `obj`, ensuring that any instances of
+    subclasses of built-in types are downcast to the base type.
+    """
+    if isinstance(obj, dict):
+        return {cast(k): cast(v) for k, v in obj.iteritems()}
+    if isinstance(obj, (list, tuple)):
+        return [cast(v) for v in obj]
+    if obj is None or isinstance(obj, (int, float)):
+        return obj
+    if isinstance(obj, unicode):
+        return unicode(obj)
+    if isinstance(obj, str):
+        return str(obj)
+    if isinstance(obj, (mitogen.core.Context,
+                        mitogen.core.Dead,
+                        mitogen.core.CallError)):
+        return obj
+    raise TypeError("Cannot serialize: %r: %r" % (type(obj), obj))
+
+
+def get_command_module_name(module_name):
+    """
+    Given the name of an Ansible command module, return its canonical module
+    path within the ansible.
+
+    :param module_name:
+        "shell"
+    :return: 
+        "ansible.modules.commands.shell"
+    """
+    path = ansible.plugins.module_loader.find_plugin(module_name, '')
+    relpath = os.path.relpath(path, os.path.dirname(ansible.__file__))
+    root, _ = os.path.splitext(relpath)
+    return 'ansible.' + root.replace('/', '.')