Merge pull request #233 from dw/dmw

Dmw

ansible_mitogen/connection.py

@@ -117,7 +117,6 @@ def _connect_machinectl(spec):
 
 
 def _connect_setns(spec):
-    print 'ULTRAFLEEN', spec['remote_addr'], spec['remote_user']
     return {
         'method': 'setns',
         'kwargs': {
@@ -146,6 +145,21 @@ def _connect_sudo(spec):
     }
 
 
+def _connect_mitogen_sudo(spec):
+    # sudo as a first-class proxied connection, not a become method.
+    return {
+        'method': 'sudo',
+        'kwargs': {
+            'username': spec['remote_user'],
+            'password': spec['password'],
+            'python_path': spec['python_path'],
+            'sudo_path': spec['become_exe'],
+            'connect_timeout': spec['timeout'],
+            'sudo_args': spec['sudo_args'],
+        }
+    }
+
+
 CONNECTION_METHOD = {
     'docker': _connect_docker,
     'jail': _connect_jail,
@@ -156,6 +170,7 @@ CONNECTION_METHOD = {
     'setns': _connect_setns,
     'ssh': _connect_ssh,
     'sudo': _connect_sudo,
+    'mitogen_sudo': _connect_mitogen_sudo,
 }
 
 
@@ -473,8 +488,8 @@ class Connection(ansible.plugins.connection.ConnectionBase):
         try:
             return self.call_async(func, *args, **kwargs).get().unpickle()
         finally:
-            LOG.debug('Call %s%r took %d ms', func.func_name, args,
-                      1000 * (time.time() - t0))
+            LOG.debug('Call took %d ms: %s%r', 1000 * (time.time() - t0),
+                      func.func_name, args)
 
     def exec_command(self, cmd, in_data='', sudoable=True, mitogen_chdir=None):
         """
@@ -576,27 +591,3 @@ class Connection(ansible.plugins.connection.ConnectionBase):
             in_path=in_path,
             out_path=out_path
         )
-
-
-class SshConnection(Connection):
-    transport = 'ssh'
-
-
-class LocalConnection(Connection):
-    transport = 'local'
-
-
-class DockerConnection(Connection):
-    transport = 'docker'
-
-
-class LxcConnection(Connection):
-    transport = 'lxc'
-
-
-class LxdConnection(Connection):
-    transport = 'lxd'
-
-
-class JailConnection(Connection):
-    transport = 'jail'

ansible_mitogen/plugins/connection/mitogen_docker.py

@@ -29,21 +29,6 @@
 import os.path
 import sys
 
-#
-# This is not the real Connection implementation module, it simply exists as a
-# proxy to the real module, which is loaded using Python's regular import
-# mechanism, to prevent Ansible's PluginLoader from making up a fake name that
-# results in ansible_mitogen plugin modules being loaded twice: once by
-# PluginLoader with a name like "ansible.plugins.connection.mitogen", which is
-# stuffed into sys.modules even though attempting to import it will trigger an
-# ImportError, and once under its canonical name, "ansible_mitogen.connection".
-#
-# Therefore we have a proxy module that imports it under the real name, and
-# sets up the duff PluginLoader-imported module to just contain objects from
-# the real module, so duplicate types don't exist in memory, and things like
-# debuggers and isinstance() work predictably.
-#
-
 try:
     import ansible_mitogen
 except ImportError:
@@ -51,6 +36,8 @@ except ImportError:
     sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
     del base_dir
 
-from ansible_mitogen.connection import DockerConnection as Connection
-del os
-del sys
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'docker'

ansible_mitogen/plugins/connection/mitogen_jail.py

@@ -29,21 +29,6 @@
 import os.path
 import sys
 
-#
-# This is not the real Connection implementation module, it simply exists as a
-# proxy to the real module, which is loaded using Python's regular import
-# mechanism, to prevent Ansible's PluginLoader from making up a fake name that
-# results in ansible_mitogen plugin modules being loaded twice: once by
-# PluginLoader with a name like "ansible.plugins.connection.mitogen", which is
-# stuffed into sys.modules even though attempting to import it will trigger an
-# ImportError, and once under its canonical name, "ansible_mitogen.connection".
-#
-# Therefore we have a proxy module that imports it under the real name, and
-# sets up the duff PluginLoader-imported module to just contain objects from
-# the real module, so duplicate types don't exist in memory, and things like
-# debuggers and isinstance() work predictably.
-#
-
 try:
     import ansible_mitogen
 except ImportError:
@@ -51,6 +36,8 @@ except ImportError:
     sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
     del base_dir
 
-from ansible_mitogen.connection import JailConnection as Connection
-del os
-del sys
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'jail'

ansible_mitogen/plugins/connection/mitogen_local.py

@@ -29,28 +29,15 @@
 import os.path
 import sys
 
-#
-# This is not the real Connection implementation module, it simply exists as a
-# proxy to the real module, which is loaded using Python's regular import
-# mechanism, to prevent Ansible's PluginLoader from making up a fake name that
-# results in ansible_mitogen plugin modules being loaded twice: once by
-# PluginLoader with a name like "ansible.plugins.connection.mitogen", which is
-# stuffed into sys.modules even though attempting to import it will trigger an
-# ImportError, and once under its canonical name, "ansible_mitogen.connection".
-#
-# Therefore we have a proxy module that imports it under the real name, and
-# sets up the duff PluginLoader-imported module to just contain objects from
-# the real module, so duplicate types don't exist in memory, and things like
-# debuggers and isinstance() work predictably.
-#
-
 try:
-    import ansible_mitogen
+    import ansible_mitogen.connection
 except ImportError:
     base_dir = os.path.dirname(__file__)
     sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
     del base_dir
 
-from ansible_mitogen.connection import LocalConnection as Connection
-del os
-del sys
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'local'

ansible_mitogen/plugins/connection/mitogen_lxc.py

@@ -29,21 +29,6 @@
 import os.path
 import sys
 
-#
-# This is not the real Connection implementation module, it simply exists as a
-# proxy to the real module, which is loaded using Python's regular import
-# mechanism, to prevent Ansible's PluginLoader from making up a fake name that
-# results in ansible_mitogen plugin modules being loaded twice: once by
-# PluginLoader with a name like "ansible.plugins.connection.mitogen", which is
-# stuffed into sys.modules even though attempting to import it will trigger an
-# ImportError, and once under its canonical name, "ansible_mitogen.connection".
-#
-# Therefore we have a proxy module that imports it under the real name, and
-# sets up the duff PluginLoader-imported module to just contain objects from
-# the real module, so duplicate types don't exist in memory, and things like
-# debuggers and isinstance() work predictably.
-#
-
 try:
     import ansible_mitogen
 except ImportError:
@@ -51,6 +36,8 @@ except ImportError:
     sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
     del base_dir
 
-from ansible_mitogen.connection import LxcConnection as Connection
-del os
-del sys
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'lxc'

ansible_mitogen/plugins/connection/mitogen_lxd.py

@@ -29,21 +29,6 @@
 import os.path
 import sys
 
-#
-# This is not the real Connection implementation module, it simply exists as a
-# proxy to the real module, which is loaded using Python's regular import
-# mechanism, to prevent Ansible's PluginLoader from making up a fake name that
-# results in ansible_mitogen plugin modules being loaded twice: once by
-# PluginLoader with a name like "ansible.plugins.connection.mitogen", which is
-# stuffed into sys.modules even though attempting to import it will trigger an
-# ImportError, and once under its canonical name, "ansible_mitogen.connection".
-#
-# Therefore we have a proxy module that imports it under the real name, and
-# sets up the duff PluginLoader-imported module to just contain objects from
-# the real module, so duplicate types don't exist in memory, and things like
-# debuggers and isinstance() work predictably.
-#
-
 try:
     import ansible_mitogen
 except ImportError:
@@ -51,6 +36,8 @@ except ImportError:
     sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
     del base_dir
 
-from ansible_mitogen.connection import LxdConnection as Connection
-del os
-del sys
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'lxd'

ansible_mitogen/plugins/connection/mitogen_machinectl.py

@@ -0,0 +1,43 @@
+# Copyright 2017, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import os.path
+import sys
+
+try:
+    import ansible_mitogen.connection
+except ImportError:
+    base_dir = os.path.dirname(__file__)
+    sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
+    del base_dir
+
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'machinectl'

ansible_mitogen/plugins/connection/mitogen_setns.py

@@ -0,0 +1,43 @@
+# Copyright 2017, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import os.path
+import sys
+
+try:
+    import ansible_mitogen.connection
+except ImportError:
+    base_dir = os.path.dirname(__file__)
+    sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
+    del base_dir
+
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'setns'

ansible_mitogen/plugins/connection/mitogen_ssh.py

@@ -29,28 +29,27 @@
 import os.path
 import sys
 
-#
-# This is not the real Connection implementation module, it simply exists as a
-# proxy to the real module, which is loaded using Python's regular import
-# mechanism, to prevent Ansible's PluginLoader from making up a fake name that
-# results in ansible_mitogen plugin modules being loaded twice: once by
-# PluginLoader with a name like "ansible.plugins.connection.mitogen", which is
-# stuffed into sys.modules even though attempting to import it will trigger an
-# ImportError, and once under its canonical name, "ansible_mitogen.connection".
-#
-# Therefore we have a proxy module that imports it under the real name, and
-# sets up the duff PluginLoader-imported module to just contain objects from
-# the real module, so duplicate types don't exist in memory, and things like
-# debuggers and isinstance() work predictably.
-#
+DOCUMENTATION = """
+    author: David Wilson <dw@botanicus.net>
+    connection: mitogen_ssh
+    short_description: Connect over SSH via Mitogen
+    description:
+        - This connects using an OpenSSH client controlled by the Mitogen for
+          Ansible extension. It accepts every option the vanilla ssh plugin
+          accepts.
+    version_added: "2.5"
+    options:
+"""
 
 try:
-    import ansible_mitogen
+    import ansible_mitogen.connection
 except ImportError:
     base_dir = os.path.dirname(__file__)
     sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
     del base_dir
 
-from ansible_mitogen.connection import SshConnection as Connection
-del os
-del sys
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'ssh'

ansible_mitogen/plugins/connection/mitogen_sudo.py

@@ -0,0 +1,43 @@
+# Copyright 2017, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import os.path
+import sys
+
+try:
+    import ansible_mitogen.connection
+except ImportError:
+    base_dir = os.path.dirname(__file__)
+    sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
+    del base_dir
+
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'mitogen_sudo'

docs/ansible.rst

@@ -122,8 +122,9 @@ Noteworthy Differences
   `lxc <https://docs.ansible.com/ansible/2.5/plugins/connection/lxc.html>`_,
   `lxd <https://docs.ansible.com/ansible/2.5/plugins/connection/lxd.html>`_,
   and `ssh <https://docs.ansible.com/ansible/2.5/plugins/connection/ssh.html>`_
-  built-in connection types are supported, along with a Mitogen-specific
-  ``setns`` container type. File bugs to register interest in more.
+  built-in connection types are supported, along with Mitogen-specific
+  :ref:`machinectl <machinectl>`, :ref:`mitogen_sudo <sudo>`, and
+  :ref:`setns <setns>` types. File bugs to register interest in others.
 
 * Local commands execute in a reuseable interpreter created identically to
   interpreters on targets. Presently one interpreter per ``become_user``
@@ -256,10 +257,17 @@ command line, or as host and group variables.
 File Transfer
 ~~~~~~~~~~~~~
 
-Normally a tool like ``scp`` is used to copy a file with the ``copy`` or
-``template`` actions, or when uploading modules with pipelining disabled. With
-Mitogen copies are implemented natively using the same interpreters, connection
-tree, and routed message bus that carries RPCs.
+Normally `sftp <https://linux.die.net/man/1/sftp>`_ or
+`scp <https://linux.die.net/man/1/scp>`_ is used to copy a file by the
+`assemble <http://docs.ansible.com/ansible/latest/modules/assemble_module.html>`_,
+`copy <http://docs.ansible.com/ansible/latest/modules/copy_module.html>`_,
+`patch <http://docs.ansible.com/ansible/latest/modules/patch_module.html>`_,
+`script <http://docs.ansible.com/ansible/latest/modules/script_module.html>`_,
+`template <http://docs.ansible.com/ansible/latest/modules/template_module.html>`_, and
+`unarchive <http://docs.ansible.com/ansible/latest/modules/unarchive_module.html>`_
+actions, or when uploading modules with pipelining disabled. With Mitogen
+copies are implemented natively using the same interpreters, connection tree,
+and routed message bus that carries RPCs.
 
 This permits streaming directly between endpoints regardless of execution
 environment, without necessitating temporary copies in intermediary accounts or
@@ -267,16 +275,39 @@ machines, for example when ``become`` is active, or in the presence of
 connection delegation. It also neatly avoids the problem of securely sharing
 temporary files between accounts and machines.
 
-One roundtrip is required to initiate a transfer. For any tool that operates
-via SSH multiplexing, 5 are required to configure the associated IO channel, in
-addition to the time needed to start the local and remote processes. A complete
-localhost invocation of ``scp`` requires around 15 ms.
-
 As the implementation is self-contained, it is simple to make future
 improvements like prioritizing transfers, supporting resume, or displaying
 progress bars.
 
 
+Safety
+^^^^^^
+
+Incomplete transfers proceed to a hidden file in the destination directory,
+with content and metadata synced using `fsync(2)
+<https://linux.die.net/man/2/fsync>`_ prior to rename over any existing file.
+This ensures the file remains consistent in the event of a crash, or when
+overlapping `ansible-playbook` runs deploy differing file contents.
+
+The `sftp <https://linux.die.net/man/1/sftp>`_ and `scp
+<https://linux.die.net/man/1/sftp>`_ tools may cause undetectable data
+corruption in the form of truncated files, or files containing intermingled
+data segments from overlapping runs. In normal operation both tools
+additionally expose a window where users of the file may observe inconsistent
+contents.
+
+
+Performance
+^^^^^^^^^^^
+
+One roundtrip in each direction is required to initiate a transfer larger than
+32KiB. For smaller transfers content is embedded in the RPC towards the target.
+For any tool that operates via SSH multiplexing, 5 roundtrips are required to
+configure the associated IO channel, in addition to the time needed to start
+the local and remote copy subprocesses. A complete localhost invocation of
+``scp`` with an empty ``.profile`` requires around 15 ms.
+
+
 Interpreter Reuse
 ~~~~~~~~~~~~~~~~~
 
@@ -455,13 +486,15 @@ connection delegation is supported.
 * ``ansible_user``: Name of user within the container to execute as.
 
 
+.. _machinectl:
+
 Machinectl
 ~~~~~~~~~~
 
-Behaves like `machinectl
+Behaves like `machinectl third party plugin
 <https://github.com/BaxterStockman/ansible-connection-machinectl>`_ except
-connection delegation is supported. This is a lightweight wrapper around the
-``setns`` method below.
+connection delegation is supported. This is a light wrapper around the
+:ref:`setns <setns>` method.
 
 * ``ansible_host``: Name of Docker container (default: inventory hostname).
 * ``ansible_user``: Name of user within the container to execute as.
@@ -469,17 +502,44 @@ connection delegation is supported. This is a lightweight wrapper around the
   as ``/bin/machinectl``.
 
 
-Sudo
-~~~~
+FreeBSD Jails
+~~~~~~~~~~~~~
+
+Behaves like `jail
+<https://docs.ansible.com/ansible/2.5/plugins/connection/jail.html>`_ except
+connection delegation is supported.
+
+* ``ansible_host``: Name of jail (default: inventory hostname).
+* ``ansible_user``: Name of user within the jail to execute as.
+
+
+Local
+~~~~~
+
+Behaves like `local
+<https://docs.ansible.com/ansible/2.5/plugins/connection/local.html>`_ except
+connection delegation is supported.
 
 * ``ansible_python_interpreter``
-* ``ansible_sudo_exe``, ``ansible_become_exe``
-* ``ansible_sudo_user``, ``ansible_become_user`` (default: ``root``)
-* ``ansible_sudo_pass``, ``ansible_become_pass`` (default: assume passwordless)
-* ``sudo_flags``, ``become_flags``
-* ansible.cfg: ``timeout``
 
 
+LXC
+~~~
+
+Behaves like `lxc
+<https://docs.ansible.com/ansible/2.5/plugins/connection/lxc.html>`_ and `lxd
+<https://docs.ansible.com/ansible/2.5/plugins/connection/lxd.html>`_ except
+connection delegation is supported, and the ``lxc-attach`` tool is always used
+rather than the LXC Python bindings, as is usual with the ``lxc`` method.
+
+The ``lxc-attach`` command must be available on the host machine.
+
+* ``ansible_python_interpreter``
+* ``ansible_host``: Name of LXC container (default: inventory hostname).
+
+
+.. _setns:
+
 Setns
 ~~~~~
 
@@ -503,6 +563,32 @@ root process.
   as ``/bin/machinectl``.
 
 
+.. _sudo:
+
+Sudo
+~~~~
+
+Sudo can be used as a connection method that supports connection delegation, or
+as a become method.
+
+When used as a become method:
+
+* ``ansible_python_interpreter``
+* ``ansible_sudo_exe``, ``ansible_become_exe``
+* ``ansible_sudo_user``, ``ansible_become_user`` (default: ``root``)
+* ``ansible_sudo_pass``, ``ansible_become_pass`` (default: assume passwordless)
+* ``sudo_flags``, ``become_flags``
+* ansible.cfg: ``timeout``
+
+When used as the ``mitogen_sudo`` connection method:
+
+* The inventory hostname is ignored, and may be any value.
+* ``ansible_user``: username to sudo as.
+* ``ansible_password``: password to sudo as.
+* ``sudo_flags``, ``become_flags``
+* ``ansible_python_interpreter``
+
+
 SSH
 ~~~
 
@@ -520,42 +606,6 @@ connection delegation is supported.
 * ``ssh_args``, ``ssh_common_args``, ``ssh_extra_args``
 
 
-FreeBSD Jails
-~~~~~~~~~~~~~
-
-Behaves like `jail
-<https://docs.ansible.com/ansible/2.5/plugins/connection/jail.html>`_ except
-connection delegation is supported.
-
-* ``ansible_host``: Name of jail (default: inventory hostname).
-* ``ansible_user``: Name of user within the jail to execute as.
-
-
-Local
-~~~~~
-
-Behaves like `local
-<https://docs.ansible.com/ansible/2.5/plugins/connection/local.html>`_ except
-connection delegation is supported.
-
-* ``ansible_python_interpreter``
-
-
-LXC
-~~~
-
-Behaves like `lxc
-<https://docs.ansible.com/ansible/2.5/plugins/connection/lxc.html>`_ and `lxd
-<https://docs.ansible.com/ansible/2.5/plugins/connection/lxd.html>`_ except
-conncetion delegation is supported, and the ``lxc-attach`` tool is always used
-rather than the LXC Python bindings, as is usual with the ``lxc`` method.
-
-The ``lxc-attach`` command must be available on the host machine.
-
-* ``ansible_python_interpreter``
-* ``ansible_host``: Name of LXC container (default: inventory hostname).
-
-
 Debugging
 ---------
 

examples/mitogen-fuse.py

@@ -0,0 +1,216 @@
+#!/usr/bin/env python
+# This implementation could improve a /lot/, but the core library is missing
+# some functionality (#213) to make that easy. Additionally it needs a set of
+# Python bindings for FUSE that stupidly require use of a thread pool.
+
+from __future__ import absolute_import, division
+
+import errno
+import logging
+import threading
+import sys
+import time
+
+import fuse
+import mitogen.master
+import mitogen.utils
+
+import __main__
+import posix
+import os
+
+
+LOG = logging.getLogger(__name__)
+
+
+def errno_wrap(modname, func, *args):
+    try:
+        return getattr(globals()[modname], func)(*args), None
+    except (IOError, OSError):
+        LOG.exception('While running %r(**%r)', func, args)
+        e = sys.exc_info()[1]
+        return None, errno.errorcode[e.args[0]]
+
+
+def errno_call(context, func, *args):
+    result, errname = context.call(
+        errno_wrap,
+        func.__module__,
+        func.__name__,
+        *args
+    )
+    if errname:
+        raise fuse.FuseOSError(getattr(errno, errname))
+    return result
+
+
+def _create(path, mode):
+    fd = os.open(path, os.O_WRONLY)
+    try:
+        os.fchmod(fd, mode)
+    finally:
+        os.close(fd)
+
+
+def _stat(path):
+    st = os.lstat(path)
+    keys = ('st_atime', 'st_gid', 'st_mode', 'st_mtime', 'st_size', 'st_uid')
+    dct = dict((key, getattr(st, key)) for key in keys)
+    dct['has_contents'] = os.path.exists(os.path.join(path, 'Contents'))
+    return dct
+
+
+def _listdir(path):
+    return [
+        (name, _stat(os.path.join(path, name)), 0)
+        for name in os.listdir(path)
+    ]
+
+
+def _read(path, size, offset):
+    fd = os.open(path, os.O_RDONLY)
+    try:
+        os.lseek(fd, offset, os.SEEK_SET)
+        return os.read(fd, size)
+    finally:
+        os.close(fd)
+
+
+def _truncate(path, length):
+    fd = os.open(path, os.O_RDWR)
+    try:
+        os.truncate(fd, length)
+    finally:
+        os.close(fd)
+
+
+def _write(path, data, offset):
+    fd = os.open(path, os.O_RDWR)
+    try:
+        os.lseek(fd, offset, os.SEEK_SET)
+        return os.write(fd, data)
+    finally:
+        os.close(fd)
+
+
+def _evil_name(path):
+    if not (os.path.basename(path).startswith('._') or
+            path.endswith('.DS_Store')):
+        return
+    raise fuse.FuseOSError(errno.ENOENT)
+
+
+def _chroot(path):
+    os.chroot(path)
+
+
+class Operations(fuse.Operations):  # fuse.LoggingMixIn, 
+    def __init__(self, host, path='.'):
+        self.host = host
+        self.root = path
+        self.ready = threading.Event()
+
+    def init(self, path):
+        self.broker = mitogen.master.Broker(install_watcher=False)
+        self.router = mitogen.master.Router(self.broker)
+        self.host = self.router.ssh(hostname=self.host)
+        self._context = self.router.sudo(via=self.host)
+        #self._context.call(_chroot , '/home/dmw')
+        self._stat_cache = {}
+        self.ready.set()
+
+    def destroy(self, path):
+        self.broker.shutdown()
+
+    @property
+    def context(self):
+        self.ready.wait()
+        return self._context
+
+    def chmod(self, path, mode):
+        _evil_name(path)
+        return errno_call(self._context, os.chmod, path, mode)
+
+    def chown(self, path, uid, gid):
+        _evil_name(path)
+        return errno_call(self._context, os.chown, path, uid, gid)
+
+    def create(self, path, mode):
+        _evil_name(path)
+        return errno_call(self._context, _create, path, mode) or 0
+
+    def getattr(self, path, fh=None):
+        _evil_name(path)
+        if path in self._stat_cache:
+            now = time.time()
+            then, st = self._stat_cache[path]
+            if now < (then + 2.0):
+                return st
+        basedir = os.path.dirname(path)
+        if path.endswith('/Contents') and basedir in self._stat_cache:
+            now = time.time()
+            then, st = self._stat_cache[basedir]
+            if now < (then + 2.0) and not st['has_contents']:
+                raise fuse.FuseOSError(errno.ENOENT)
+
+        return errno_call(self._context, _stat, path)
+
+    def mkdir(self, path, mode):
+        _evil_name(path)
+        return errno_call(self._context, os.mkdir, path, mode)
+
+    def read(self, path, size, offset, fh):
+        _evil_name(path)
+        return errno_call(self._context, _read, path, size, offset)
+
+    def readdir(self, path, fh):
+        _evil_name(path)
+        lst = errno_call(self._context, _listdir, path)
+        now = time.time()
+        for name, stat, _ in lst:
+            self._stat_cache[os.path.join(path, name)] = (now, stat)
+        return lst
+
+    def readlink(self, path):
+        _evil_name(path)
+        return errno_call(self._context, os.readlink, path)
+
+    def rename(self, old, new):
+        return errno_call(self._context, os.rename, old, new)
+        # TODO return self.sftp.rename(old, self.root + new)
+
+    def rmdir(self, path):
+        _evil_name(path)
+        return errno_call(self._context, os.rmdir, path)
+
+    def symlink(self, target, source):
+        _evil_name(path)
+        return errno_call(self._context, os.symlink, source, target)
+
+    def truncate(self, path, length, fh=None):
+        _evil_name(path)
+        return errno_call(self._context, _truncate, path, length)
+
+    def unlink(self, path):
+        _evil_name(path)
+        return errno_call(self._context, os.unlink, path)
+
+    def utimens(self, path, times=None):
+        _evil_name(path)
+        return errno_call(self._context, os.utime, path, times)
+
+    def write(self, path, data, offset, fh):
+        _evil_name(path)
+        return errno_call(self._context, _write, path, data, offset)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 3:
+        print('usage: %s <host> <mountpoint>' % sys.argv[0])
+        sys.exit(1)
+
+    ops = Operations(sys.argv[1])
+    mount_point = sys.argv[2]
+
+    mitogen.utils.log_to_file(level='DEBUG')
+    blerp = fuse.FUSE(ops, mount_point, foreground=True)

mitogen/debug.py

@@ -33,17 +33,73 @@ Basic signal handler for dumping thread stacks.
 import difflib
 import logging
 import os
+import gc
 import signal
 import sys
 import threading
 import time
 import traceback
 
+import mitogen.core
+import mitogen.master
+import mitogen.parent
+
 
 LOG = logging.getLogger(__name__)
 _last = None
 
 
+def _hex(n):
+    return '%08x' % n
+
+
+def get_routers():
+    return {
+        _hex(id(router)): router
+        for klass in (
+            mitogen.core.Router,
+            mitogen.parent.Router,
+            mitogen.master.Router,
+        )
+        for router in gc.get_referrers(klass)
+        if isinstance(router, mitogen.core.Router)
+    }
+
+
+def get_router_info():
+    return {
+        'routers': {
+            id_: {
+                'id': id_,
+                'streams': len(set(router._stream_by_id.values())),
+                'contexts': len(set(router._context_by_id.values())),
+                'handles': len(router._handle_map),
+            }
+            for id_, router in get_routers().items()
+        }
+    }
+
+
+def get_router_info(router):
+    pass
+
+
+def get_stream_info(router_id):
+    router = get_routers().get(router_id)
+    return {
+        'streams': dict(
+            (_hex(id(stream)), ({
+                'name': stream.name,
+                'remote_id': stream.remote_id,
+                'sent_module_count': len(getattr(stream, 'sent_modules', [])),
+                'routes': sorted(getattr(stream, 'routes', [])),
+                'type': type(stream).__module__,
+            }))
+            for via_id, stream in router._stream_by_id.items()
+        )
+    }
+
+
 def format_stacks():
     name_by_id = {
         t.ident: t.name
@@ -118,3 +174,42 @@ def dump_to_logger():
     th = threading.Thread(target=_logging_main)
     th.setDaemon(True)
     th.start()
+
+
+class ContextDebugger(object):
+    @classmethod
+    @mitogen.core.takes_econtext
+    def _configure_context(cls, econtext):
+        mitogen.parent.upgrade_router(econtext)
+        econtext.debugger = cls(econtext.router)
+
+    def __init__(self, router):
+        self.router = router
+        self.router.add_handler(
+            func=self._on_debug_msg,
+            handle=mitogen.core.DEBUG,
+            persist=True,
+            policy=mitogen.core.has_parent_authority,
+        )
+        mitogen.core.listen(router, 'register', self._on_stream_register)
+        LOG.debug('Context debugging configured.')
+
+    def _on_stream_register(self, context, stream):
+        LOG.debug('_on_stream_register: sending configure() to %r', stream)
+        context.call_async(ContextDebugger._configure_context)
+
+    def _on_debug_msg(self, msg):
+        if msg != mitogen.core._DEAD:
+            threading.Thread(
+                target=self._handle_debug_msg,
+                name='ContextDebuggerHandler',
+                args=(msg,)
+            ).start()
+
+    def _handle_debug_msg(self, msg):
+        try:
+            method, args, kwargs = msg.unpickle()
+            msg.reply(getattr(cls, method)(*args, **kwargs))
+        except Exception:
+            e = sys.exc_info()[1]
+            msg.reply(mitogen.core.CallError(e))

tests/ansible/regression/issue_140__thread_pileup.yml

@@ -10,20 +10,22 @@
   - name: Create file tree
     connection: local
     shell: >
-      mkdir filetree;
-      for i in `seq 1 1000` ; do echo $i > filetree/$i ; done
+      mkdir /tmp/filetree.in;
+      seq -f /tmp/filetree.in/%g 1 1000 | xargs touch;
     args:
-      creates: filetree
-
+      creates: /tmp/filetree.in
 
   - name: Delete remote file tree
-    shell: rm -rf /tmp/filetree
+    shell: rm -rf /tmp/filetree.out
 
+  - file:
+      state: directory
+      path: /tmp/filetree.out
 
   - name: Trigger nasty process pileup
-    synchronize:
-      src: "{{ item.src }}"
-      dest: "/tmp/filetree"
+    copy:
+      src: "{{item.src}}"
+      dest: "/tmp/filetree.out/{{item.path}}"
     with_filetree:
-      - filetree
+      - /tmp/filetree.in
     when: item.state == 'file'