From b61c291fafa741c02466442c7a8eeb837c46d951 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 29 Apr 2018 01:20:26 +0100 Subject: [PATCH 1/9] examples: import mitogen-fuse.py. --- examples/mitogen-fuse.py | 216 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 examples/mitogen-fuse.py diff --git a/examples/mitogen-fuse.py b/examples/mitogen-fuse.py new file mode 100644 index 00000000..4f6bcf81 --- /dev/null +++ b/examples/mitogen-fuse.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python +# This implementation could improve a /lot/, but the core library is missing +# some functionality (#213) to make that easy. Additionally it needs a set of +# Python bindings for FUSE that stupidly require use of a thread pool. + +from __future__ import absolute_import, division + +import errno +import logging +import threading +import sys +import time + +import fuse +import mitogen.master +import mitogen.utils + +import __main__ +import posix +import os + + +LOG = logging.getLogger(__name__) + + +def errno_wrap(modname, func, *args): + try: + return getattr(globals()[modname], func)(*args), None + except (IOError, OSError): + LOG.exception('While running %r(**%r)', func, args) + e = sys.exc_info()[1] + return None, errno.errorcode[e.args[0]] + + +def errno_call(context, func, *args): + result, errname = context.call( + errno_wrap, + func.__module__, + func.__name__, + *args + ) + if errname: + raise fuse.FuseOSError(getattr(errno, errname)) + return result + + +def _create(path, mode): + fd = os.open(path, os.O_WRONLY) + try: + os.fchmod(fd, mode) + finally: + os.close(fd) + + +def _stat(path): + st = os.lstat(path) + keys = ('st_atime', 'st_gid', 'st_mode', 'st_mtime', 'st_size', 'st_uid') + dct = dict((key, getattr(st, key)) for key in keys) + dct['has_contents'] = os.path.exists(os.path.join(path, 'Contents')) + return dct + + +def _listdir(path): + return [ + (name, _stat(os.path.join(path, name)), 0) + for name in os.listdir(path) + ] + + +def _read(path, size, offset): + fd = os.open(path, os.O_RDONLY) + try: + os.lseek(fd, offset, os.SEEK_SET) + return os.read(fd, size) + finally: + os.close(fd) + + +def _truncate(path, length): + fd = os.open(path, os.O_RDWR) + try: + os.truncate(fd, length) + finally: + os.close(fd) + + +def _write(path, data, offset): + fd = os.open(path, os.O_RDWR) + try: + os.lseek(fd, offset, os.SEEK_SET) + return os.write(fd, data) + finally: + os.close(fd) + + +def _evil_name(path): + if not (os.path.basename(path).startswith('._') or + path.endswith('.DS_Store')): + return + raise fuse.FuseOSError(errno.ENOENT) + + +def _chroot(path): + os.chroot(path) + + +class Operations(fuse.Operations): # fuse.LoggingMixIn, + def __init__(self, host, path='.'): + self.host = host + self.root = path + self.ready = threading.Event() + + def init(self, path): + self.broker = mitogen.master.Broker(install_watcher=False) + self.router = mitogen.master.Router(self.broker) + self.host = self.router.ssh(hostname=self.host) + self._context = self.router.sudo(via=self.host) + #self._context.call(_chroot , '/home/dmw') + self._stat_cache = {} + self.ready.set() + + def destroy(self, path): + self.broker.shutdown() + + @property + def context(self): + self.ready.wait() + return self._context + + def chmod(self, path, mode): + _evil_name(path) + return errno_call(self._context, os.chmod, path, mode) + + def chown(self, path, uid, gid): + _evil_name(path) + return errno_call(self._context, os.chown, path, uid, gid) + + def create(self, path, mode): + _evil_name(path) + return errno_call(self._context, _create, path, mode) or 0 + + def getattr(self, path, fh=None): + _evil_name(path) + if path in self._stat_cache: + now = time.time() + then, st = self._stat_cache[path] + if now < (then + 2.0): + return st + basedir = os.path.dirname(path) + if path.endswith('/Contents') and basedir in self._stat_cache: + now = time.time() + then, st = self._stat_cache[basedir] + if now < (then + 2.0) and not st['has_contents']: + raise fuse.FuseOSError(errno.ENOENT) + + return errno_call(self._context, _stat, path) + + def mkdir(self, path, mode): + _evil_name(path) + return errno_call(self._context, os.mkdir, path, mode) + + def read(self, path, size, offset, fh): + _evil_name(path) + return errno_call(self._context, _read, path, size, offset) + + def readdir(self, path, fh): + _evil_name(path) + lst = errno_call(self._context, _listdir, path) + now = time.time() + for name, stat, _ in lst: + self._stat_cache[os.path.join(path, name)] = (now, stat) + return lst + + def readlink(self, path): + _evil_name(path) + return errno_call(self._context, os.readlink, path) + + def rename(self, old, new): + return errno_call(self._context, os.rename, old, new) + # TODO return self.sftp.rename(old, self.root + new) + + def rmdir(self, path): + _evil_name(path) + return errno_call(self._context, os.rmdir, path) + + def symlink(self, target, source): + _evil_name(path) + return errno_call(self._context, os.symlink, source, target) + + def truncate(self, path, length, fh=None): + _evil_name(path) + return errno_call(self._context, _truncate, path, length) + + def unlink(self, path): + _evil_name(path) + return errno_call(self._context, os.unlink, path) + + def utimens(self, path, times=None): + _evil_name(path) + return errno_call(self._context, os.utime, path, times) + + def write(self, path, data, offset, fh): + _evil_name(path) + return errno_call(self._context, _write, path, data, offset) + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print('usage: %s ' % sys.argv[0]) + sys.exit(1) + + ops = Operations(sys.argv[1]) + mount_point = sys.argv[2] + + mitogen.utils.log_to_file(level='DEBUG') + blerp = fuse.FUSE(ops, mount_point, foreground=True) From 0a9126c510a6cd4d8c6c19d8546668c2aed84dce Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 29 Apr 2018 01:22:01 +0100 Subject: [PATCH 2/9] ansible: connection plugins missing from previous commit. --- .../plugins/connection/mitogen_machinectl.py | 41 +++++++++++++++++++ .../plugins/connection/mitogen_setns.py | 41 +++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 ansible_mitogen/plugins/connection/mitogen_machinectl.py create mode 100644 ansible_mitogen/plugins/connection/mitogen_setns.py diff --git a/ansible_mitogen/plugins/connection/mitogen_machinectl.py b/ansible_mitogen/plugins/connection/mitogen_machinectl.py new file mode 100644 index 00000000..d312c819 --- /dev/null +++ b/ansible_mitogen/plugins/connection/mitogen_machinectl.py @@ -0,0 +1,41 @@ +# Copyright 2017, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +import os.path +import sys + +try: + import ansible_mitogen.connection +except ImportError: + base_dir = os.path.dirname(__file__) + sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) + del base_dir + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'machinectl' diff --git a/ansible_mitogen/plugins/connection/mitogen_setns.py b/ansible_mitogen/plugins/connection/mitogen_setns.py new file mode 100644 index 00000000..7221d6b1 --- /dev/null +++ b/ansible_mitogen/plugins/connection/mitogen_setns.py @@ -0,0 +1,41 @@ +# Copyright 2017, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +import os.path +import sys + +try: + import ansible_mitogen.connection +except ImportError: + base_dir = os.path.dirname(__file__) + sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) + del base_dir + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'setns' From bba2a42e4467309d80b0e558f758b77d74500ca7 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 29 Apr 2018 01:37:53 +0100 Subject: [PATCH 3/9] ansible: add mitogen_sudo method, split out connection subclasses. Slowly moving towards real implementations in those files. --- ansible_mitogen/connection.py | 40 +++--- .../plugins/connection/mitogen_docker.py | 23 +--- .../plugins/connection/mitogen_jail.py | 23 +--- .../plugins/connection/mitogen_local.py | 25 +--- .../plugins/connection/mitogen_lxc.py | 23 +--- .../plugins/connection/mitogen_lxd.py | 23 +--- .../plugins/connection/mitogen_machinectl.py | 2 + .../plugins/connection/mitogen_setns.py | 2 + .../plugins/connection/mitogen_ssh.py | 25 +--- .../plugins/connection/mitogen_sudo.py | 43 +++++++ docs/ansible.rst | 116 ++++++++++-------- 11 files changed, 163 insertions(+), 182 deletions(-) create mode 100644 ansible_mitogen/plugins/connection/mitogen_sudo.py diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index e4d2d786..3350fd3b 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -146,6 +146,21 @@ def _connect_sudo(spec): } +def _connect_mitogen_sudo(spec): + # sudo as a first-class proxied connection, not a become method. + return { + 'method': 'sudo', + 'kwargs': { + 'username': spec['remote_user'], + 'password': spec['password'], + 'python_path': spec['python_path'], + 'sudo_path': spec['become_exe'], + 'connect_timeout': spec['timeout'], + 'sudo_args': spec['sudo_args'], + } + } + + CONNECTION_METHOD = { 'docker': _connect_docker, 'jail': _connect_jail, @@ -156,6 +171,7 @@ CONNECTION_METHOD = { 'setns': _connect_setns, 'ssh': _connect_ssh, 'sudo': _connect_sudo, + 'mitogen_sudo': _connect_mitogen_sudo, } @@ -576,27 +592,3 @@ class Connection(ansible.plugins.connection.ConnectionBase): in_path=in_path, out_path=out_path ) - - -class SshConnection(Connection): - transport = 'ssh' - - -class LocalConnection(Connection): - transport = 'local' - - -class DockerConnection(Connection): - transport = 'docker' - - -class LxcConnection(Connection): - transport = 'lxc' - - -class LxdConnection(Connection): - transport = 'lxd' - - -class JailConnection(Connection): - transport = 'jail' diff --git a/ansible_mitogen/plugins/connection/mitogen_docker.py b/ansible_mitogen/plugins/connection/mitogen_docker.py index 70a90147..a98273e0 100644 --- a/ansible_mitogen/plugins/connection/mitogen_docker.py +++ b/ansible_mitogen/plugins/connection/mitogen_docker.py @@ -29,21 +29,6 @@ import os.path import sys -# -# This is not the real Connection implementation module, it simply exists as a -# proxy to the real module, which is loaded using Python's regular import -# mechanism, to prevent Ansible's PluginLoader from making up a fake name that -# results in ansible_mitogen plugin modules being loaded twice: once by -# PluginLoader with a name like "ansible.plugins.connection.mitogen", which is -# stuffed into sys.modules even though attempting to import it will trigger an -# ImportError, and once under its canonical name, "ansible_mitogen.connection". -# -# Therefore we have a proxy module that imports it under the real name, and -# sets up the duff PluginLoader-imported module to just contain objects from -# the real module, so duplicate types don't exist in memory, and things like -# debuggers and isinstance() work predictably. -# - try: import ansible_mitogen except ImportError: @@ -51,6 +36,8 @@ except ImportError: sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) del base_dir -from ansible_mitogen.connection import DockerConnection as Connection -del os -del sys +import ansible_mitogen.connection + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'docker' diff --git a/ansible_mitogen/plugins/connection/mitogen_jail.py b/ansible_mitogen/plugins/connection/mitogen_jail.py index 24fcb91f..1c57bb38 100644 --- a/ansible_mitogen/plugins/connection/mitogen_jail.py +++ b/ansible_mitogen/plugins/connection/mitogen_jail.py @@ -29,21 +29,6 @@ import os.path import sys -# -# This is not the real Connection implementation module, it simply exists as a -# proxy to the real module, which is loaded using Python's regular import -# mechanism, to prevent Ansible's PluginLoader from making up a fake name that -# results in ansible_mitogen plugin modules being loaded twice: once by -# PluginLoader with a name like "ansible.plugins.connection.mitogen", which is -# stuffed into sys.modules even though attempting to import it will trigger an -# ImportError, and once under its canonical name, "ansible_mitogen.connection". -# -# Therefore we have a proxy module that imports it under the real name, and -# sets up the duff PluginLoader-imported module to just contain objects from -# the real module, so duplicate types don't exist in memory, and things like -# debuggers and isinstance() work predictably. -# - try: import ansible_mitogen except ImportError: @@ -51,6 +36,8 @@ except ImportError: sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) del base_dir -from ansible_mitogen.connection import JailConnection as Connection -del os -del sys +import ansible_mitogen.connection + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'jail' diff --git a/ansible_mitogen/plugins/connection/mitogen_local.py b/ansible_mitogen/plugins/connection/mitogen_local.py index fc1a8565..ebc20788 100644 --- a/ansible_mitogen/plugins/connection/mitogen_local.py +++ b/ansible_mitogen/plugins/connection/mitogen_local.py @@ -29,28 +29,15 @@ import os.path import sys -# -# This is not the real Connection implementation module, it simply exists as a -# proxy to the real module, which is loaded using Python's regular import -# mechanism, to prevent Ansible's PluginLoader from making up a fake name that -# results in ansible_mitogen plugin modules being loaded twice: once by -# PluginLoader with a name like "ansible.plugins.connection.mitogen", which is -# stuffed into sys.modules even though attempting to import it will trigger an -# ImportError, and once under its canonical name, "ansible_mitogen.connection". -# -# Therefore we have a proxy module that imports it under the real name, and -# sets up the duff PluginLoader-imported module to just contain objects from -# the real module, so duplicate types don't exist in memory, and things like -# debuggers and isinstance() work predictably. -# - try: - import ansible_mitogen + import ansible_mitogen.connection except ImportError: base_dir = os.path.dirname(__file__) sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) del base_dir -from ansible_mitogen.connection import LocalConnection as Connection -del os -del sys +import ansible_mitogen.connection + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'local' diff --git a/ansible_mitogen/plugins/connection/mitogen_lxc.py b/ansible_mitogen/plugins/connection/mitogen_lxc.py index 48d4e03d..2195aa3c 100644 --- a/ansible_mitogen/plugins/connection/mitogen_lxc.py +++ b/ansible_mitogen/plugins/connection/mitogen_lxc.py @@ -29,21 +29,6 @@ import os.path import sys -# -# This is not the real Connection implementation module, it simply exists as a -# proxy to the real module, which is loaded using Python's regular import -# mechanism, to prevent Ansible's PluginLoader from making up a fake name that -# results in ansible_mitogen plugin modules being loaded twice: once by -# PluginLoader with a name like "ansible.plugins.connection.mitogen", which is -# stuffed into sys.modules even though attempting to import it will trigger an -# ImportError, and once under its canonical name, "ansible_mitogen.connection". -# -# Therefore we have a proxy module that imports it under the real name, and -# sets up the duff PluginLoader-imported module to just contain objects from -# the real module, so duplicate types don't exist in memory, and things like -# debuggers and isinstance() work predictably. -# - try: import ansible_mitogen except ImportError: @@ -51,6 +36,8 @@ except ImportError: sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) del base_dir -from ansible_mitogen.connection import LxcConnection as Connection -del os -del sys +import ansible_mitogen.connection + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'lxc' diff --git a/ansible_mitogen/plugins/connection/mitogen_lxd.py b/ansible_mitogen/plugins/connection/mitogen_lxd.py index 91fcd0b7..5d1391b9 100644 --- a/ansible_mitogen/plugins/connection/mitogen_lxd.py +++ b/ansible_mitogen/plugins/connection/mitogen_lxd.py @@ -29,21 +29,6 @@ import os.path import sys -# -# This is not the real Connection implementation module, it simply exists as a -# proxy to the real module, which is loaded using Python's regular import -# mechanism, to prevent Ansible's PluginLoader from making up a fake name that -# results in ansible_mitogen plugin modules being loaded twice: once by -# PluginLoader with a name like "ansible.plugins.connection.mitogen", which is -# stuffed into sys.modules even though attempting to import it will trigger an -# ImportError, and once under its canonical name, "ansible_mitogen.connection". -# -# Therefore we have a proxy module that imports it under the real name, and -# sets up the duff PluginLoader-imported module to just contain objects from -# the real module, so duplicate types don't exist in memory, and things like -# debuggers and isinstance() work predictably. -# - try: import ansible_mitogen except ImportError: @@ -51,6 +36,8 @@ except ImportError: sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) del base_dir -from ansible_mitogen.connection import LxdConnection as Connection -del os -del sys +import ansible_mitogen.connection + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'lxd' diff --git a/ansible_mitogen/plugins/connection/mitogen_machinectl.py b/ansible_mitogen/plugins/connection/mitogen_machinectl.py index d312c819..e71496a3 100644 --- a/ansible_mitogen/plugins/connection/mitogen_machinectl.py +++ b/ansible_mitogen/plugins/connection/mitogen_machinectl.py @@ -36,6 +36,8 @@ except ImportError: sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) del base_dir +import ansible_mitogen.connection + class Connection(ansible_mitogen.connection.Connection): transport = 'machinectl' diff --git a/ansible_mitogen/plugins/connection/mitogen_setns.py b/ansible_mitogen/plugins/connection/mitogen_setns.py index 7221d6b1..5f131655 100644 --- a/ansible_mitogen/plugins/connection/mitogen_setns.py +++ b/ansible_mitogen/plugins/connection/mitogen_setns.py @@ -36,6 +36,8 @@ except ImportError: sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) del base_dir +import ansible_mitogen.connection + class Connection(ansible_mitogen.connection.Connection): transport = 'setns' diff --git a/ansible_mitogen/plugins/connection/mitogen_ssh.py b/ansible_mitogen/plugins/connection/mitogen_ssh.py index e0a30672..6a5c10f8 100644 --- a/ansible_mitogen/plugins/connection/mitogen_ssh.py +++ b/ansible_mitogen/plugins/connection/mitogen_ssh.py @@ -29,28 +29,15 @@ import os.path import sys -# -# This is not the real Connection implementation module, it simply exists as a -# proxy to the real module, which is loaded using Python's regular import -# mechanism, to prevent Ansible's PluginLoader from making up a fake name that -# results in ansible_mitogen plugin modules being loaded twice: once by -# PluginLoader with a name like "ansible.plugins.connection.mitogen", which is -# stuffed into sys.modules even though attempting to import it will trigger an -# ImportError, and once under its canonical name, "ansible_mitogen.connection". -# -# Therefore we have a proxy module that imports it under the real name, and -# sets up the duff PluginLoader-imported module to just contain objects from -# the real module, so duplicate types don't exist in memory, and things like -# debuggers and isinstance() work predictably. -# - try: - import ansible_mitogen + import ansible_mitogen.connection except ImportError: base_dir = os.path.dirname(__file__) sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) del base_dir -from ansible_mitogen.connection import SshConnection as Connection -del os -del sys +import ansible_mitogen.connection + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'ssh' diff --git a/ansible_mitogen/plugins/connection/mitogen_sudo.py b/ansible_mitogen/plugins/connection/mitogen_sudo.py new file mode 100644 index 00000000..a6cb8bd2 --- /dev/null +++ b/ansible_mitogen/plugins/connection/mitogen_sudo.py @@ -0,0 +1,43 @@ +# Copyright 2017, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +import os.path +import sys + +try: + import ansible_mitogen.connection +except ImportError: + base_dir = os.path.dirname(__file__) + sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) + del base_dir + +import ansible_mitogen.connection + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'mitogen_sudo' diff --git a/docs/ansible.rst b/docs/ansible.rst index a87382e0..ad739cb5 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -122,8 +122,9 @@ Noteworthy Differences `lxc `_, `lxd `_, and `ssh `_ - built-in connection types are supported, along with a Mitogen-specific - ``setns`` container type. File bugs to register interest in more. + built-in connection types are supported, along with Mitogen-specific + :ref:`machinectl `, :ref:`mitogen_sudo `, and + :ref:`setns ` types. File bugs to register interest in others. * Local commands execute in a reuseable interpreter created identically to interpreters on targets. Presently one interpreter per ``become_user`` @@ -455,13 +456,15 @@ connection delegation is supported. * ``ansible_user``: Name of user within the container to execute as. +.. _machinectl: + Machinectl ~~~~~~~~~~ -Behaves like `machinectl +Behaves like `machinectl third party plugin `_ except -connection delegation is supported. This is a lightweight wrapper around the -``setns`` method below. +connection delegation is supported. This is a light wrapper around the +:ref:`setns ` method. * ``ansible_host``: Name of Docker container (default: inventory hostname). * ``ansible_user``: Name of user within the container to execute as. @@ -469,17 +472,44 @@ connection delegation is supported. This is a lightweight wrapper around the as ``/bin/machinectl``. -Sudo -~~~~ +FreeBSD Jails +~~~~~~~~~~~~~ + +Behaves like `jail +`_ except +connection delegation is supported. + +* ``ansible_host``: Name of jail (default: inventory hostname). +* ``ansible_user``: Name of user within the jail to execute as. + + +Local +~~~~~ + +Behaves like `local +`_ except +connection delegation is supported. * ``ansible_python_interpreter`` -* ``ansible_sudo_exe``, ``ansible_become_exe`` -* ``ansible_sudo_user``, ``ansible_become_user`` (default: ``root``) -* ``ansible_sudo_pass``, ``ansible_become_pass`` (default: assume passwordless) -* ``sudo_flags``, ``become_flags`` -* ansible.cfg: ``timeout`` +LXC +~~~ + +Behaves like `lxc +`_ and `lxd +`_ except +connection delegation is supported, and the ``lxc-attach`` tool is always used +rather than the LXC Python bindings, as is usual with the ``lxc`` method. + +The ``lxc-attach`` command must be available on the host machine. + +* ``ansible_python_interpreter`` +* ``ansible_host``: Name of LXC container (default: inventory hostname). + + +.. _setns: + Setns ~~~~~ @@ -503,6 +533,32 @@ root process. as ``/bin/machinectl``. +.. _sudo: + +Sudo +~~~~ + +Sudo can be used as a connection method that supports connection delegation, or +as a become method. + +When used as a become method: + +* ``ansible_python_interpreter`` +* ``ansible_sudo_exe``, ``ansible_become_exe`` +* ``ansible_sudo_user``, ``ansible_become_user`` (default: ``root``) +* ``ansible_sudo_pass``, ``ansible_become_pass`` (default: assume passwordless) +* ``sudo_flags``, ``become_flags`` +* ansible.cfg: ``timeout`` + +When used as the ``mitogen_sudo`` connection method: + +* The inventory hostname is ignored, and may be any value. +* ``ansible_user``: username to sudo as. +* ``ansible_password``: password to sudo as. +* ``sudo_flags``, ``become_flags`` +* ``ansible_python_interpreter`` + + SSH ~~~ @@ -520,42 +576,6 @@ connection delegation is supported. * ``ssh_args``, ``ssh_common_args``, ``ssh_extra_args`` -FreeBSD Jails -~~~~~~~~~~~~~ - -Behaves like `jail -`_ except -connection delegation is supported. - -* ``ansible_host``: Name of jail (default: inventory hostname). -* ``ansible_user``: Name of user within the jail to execute as. - - -Local -~~~~~ - -Behaves like `local -`_ except -connection delegation is supported. - -* ``ansible_python_interpreter`` - - -LXC -~~~ - -Behaves like `lxc -`_ and `lxd -`_ except -conncetion delegation is supported, and the ``lxc-attach`` tool is always used -rather than the LXC Python bindings, as is usual with the ``lxc`` method. - -The ``lxc-attach`` command must be available on the host machine. - -* ``ansible_python_interpreter`` -* ``ansible_host``: Name of LXC container (default: inventory hostname). - - Debugging --------- From 29f6c46e9770f4d9964053cbf970e886ad762584 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 29 Apr 2018 02:14:29 +0100 Subject: [PATCH 4/9] docs: add file transfer safety section. --- docs/ansible.rst | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/docs/ansible.rst b/docs/ansible.rst index ad739cb5..929835f8 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -268,16 +268,37 @@ machines, for example when ``become`` is active, or in the presence of connection delegation. It also neatly avoids the problem of securely sharing temporary files between accounts and machines. -One roundtrip is required to initiate a transfer. For any tool that operates -via SSH multiplexing, 5 are required to configure the associated IO channel, in -addition to the time needed to start the local and remote processes. A complete -localhost invocation of ``scp`` requires around 15 ms. - As the implementation is self-contained, it is simple to make future improvements like prioritizing transfers, supporting resume, or displaying progress bars. +Safety +^^^^^^ + +Incomplete transfers proceed to a hidden file in the destination directory, +with content and metadata synced using `fsync(2) +`_ prior to being renamed over any existing +file. This ensures the file remains consistent in the event of a crash, or when +overlapping `ansible-playbook` runs deploy differing file contents. + +The ``sftp`` and ``scp`` tools may cause undetectable data corruption in the +form of truncated files, or files containing partial data copies from +overlapping runs of `ansible-playbook`. Both tools additionally expose a window +where users of the file may observe inconsistent contents. + + +Performance +^^^^^^^^^^^ + +One roundtrip in each direction is required to initiate a transfer larger than +32KiB. For smaller transfers content is embedded in the RPC towards the target. +For any tool that operates via SSH multiplexing, 5 roundtrips are required to +configure the associated IO channel, in addition to the time needed to start +the local and remote copy subprocesses. A complete localhost invocation of +``scp`` with an empty ``.profile`` requires around 15 ms. + + Interpreter Reuse ~~~~~~~~~~~~~~~~~ From c85a5b6446bac99b20cbc38c1dc7732697bf22bf Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 29 Apr 2018 02:17:10 +0100 Subject: [PATCH 5/9] ansible: make call timing more readable --- ansible_mitogen/connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 3350fd3b..bfcae929 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -489,8 +489,8 @@ class Connection(ansible.plugins.connection.ConnectionBase): try: return self.call_async(func, *args, **kwargs).get().unpickle() finally: - LOG.debug('Call %s%r took %d ms', func.func_name, args, - 1000 * (time.time() - t0)) + LOG.debug('Call took %d ms: %s%r', 1000 * (time.time() - t0), + func.func_name, args) def exec_command(self, cmd, in_data='', sudoable=True, mitogen_chdir=None): """ From 90f7b4baeadb286a4d9c71336d9be2d2a7f63041 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 29 Apr 2018 02:18:26 +0100 Subject: [PATCH 6/9] ansible: stub plugin documentation. --- ansible_mitogen/plugins/connection/mitogen_ssh.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ansible_mitogen/plugins/connection/mitogen_ssh.py b/ansible_mitogen/plugins/connection/mitogen_ssh.py index 6a5c10f8..c0c577c3 100644 --- a/ansible_mitogen/plugins/connection/mitogen_ssh.py +++ b/ansible_mitogen/plugins/connection/mitogen_ssh.py @@ -29,6 +29,18 @@ import os.path import sys +DOCUMENTATION = """ + author: David Wilson + connection: mitogen_ssh + short_description: Connect over SSH via Mitogen + description: + - This connects using an OpenSSH client controlled by the Mitogen for + Ansible extension. It accepts every option the vanilla ssh plugin + accepts. + version_added: "2.5" + options: +""" + try: import ansible_mitogen.connection except ImportError: From baa4e7552642c2be191fd080f37046eff6288a5b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 29 Apr 2018 02:19:32 +0100 Subject: [PATCH 7/9] tests: tidy up thread_pileup for use as a toy benchmark. --- .../regression/issue_140__thread_pileup.yml | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/ansible/regression/issue_140__thread_pileup.yml b/tests/ansible/regression/issue_140__thread_pileup.yml index 0c86b237..99f31896 100644 --- a/tests/ansible/regression/issue_140__thread_pileup.yml +++ b/tests/ansible/regression/issue_140__thread_pileup.yml @@ -10,20 +10,22 @@ - name: Create file tree connection: local shell: > - mkdir filetree; - for i in `seq 1 1000` ; do echo $i > filetree/$i ; done + mkdir /tmp/filetree.in; + seq -f /tmp/filetree.in/%g 1 1000 | xargs touch; args: - creates: filetree - + creates: /tmp/filetree.in - name: Delete remote file tree - shell: rm -rf /tmp/filetree + shell: rm -rf /tmp/filetree.out + - file: + state: directory + path: /tmp/filetree.out - name: Trigger nasty process pileup - synchronize: - src: "{{ item.src }}" - dest: "/tmp/filetree" + copy: + src: "{{item.src}}" + dest: "/tmp/filetree.out/{{item.path}}" with_filetree: - - filetree + - /tmp/filetree.in when: item.state == 'file' From e93ac2f3a7ae7f403b048a9bf08c0acc9baa64a4 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 29 Apr 2018 02:22:22 +0100 Subject: [PATCH 8/9] debug: implement some basic helpers to debugger. --- mitogen/debug.py | 95 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/mitogen/debug.py b/mitogen/debug.py index e1122192..f2746380 100644 --- a/mitogen/debug.py +++ b/mitogen/debug.py @@ -33,17 +33,73 @@ Basic signal handler for dumping thread stacks. import difflib import logging import os +import gc import signal import sys import threading import time import traceback +import mitogen.core +import mitogen.master +import mitogen.parent + LOG = logging.getLogger(__name__) _last = None +def _hex(n): + return '%08x' % n + + +def get_routers(): + return { + _hex(id(router)): router + for klass in ( + mitogen.core.Router, + mitogen.parent.Router, + mitogen.master.Router, + ) + for router in gc.get_referrers(klass) + if isinstance(router, mitogen.core.Router) + } + + +def get_router_info(): + return { + 'routers': { + id_: { + 'id': id_, + 'streams': len(set(router._stream_by_id.values())), + 'contexts': len(set(router._context_by_id.values())), + 'handles': len(router._handle_map), + } + for id_, router in get_routers().items() + } + } + + +def get_router_info(router): + pass + + +def get_stream_info(router_id): + router = get_routers().get(router_id) + return { + 'streams': dict( + (_hex(id(stream)), ({ + 'name': stream.name, + 'remote_id': stream.remote_id, + 'sent_module_count': len(getattr(stream, 'sent_modules', [])), + 'routes': sorted(getattr(stream, 'routes', [])), + 'type': type(stream).__module__, + })) + for via_id, stream in router._stream_by_id.items() + ) + } + + def format_stacks(): name_by_id = { t.ident: t.name @@ -118,3 +174,42 @@ def dump_to_logger(): th = threading.Thread(target=_logging_main) th.setDaemon(True) th.start() + + +class ContextDebugger(object): + @classmethod + @mitogen.core.takes_econtext + def _configure_context(cls, econtext): + mitogen.parent.upgrade_router(econtext) + econtext.debugger = cls(econtext.router) + + def __init__(self, router): + self.router = router + self.router.add_handler( + func=self._on_debug_msg, + handle=mitogen.core.DEBUG, + persist=True, + policy=mitogen.core.has_parent_authority, + ) + mitogen.core.listen(router, 'register', self._on_stream_register) + LOG.debug('Context debugging configured.') + + def _on_stream_register(self, context, stream): + LOG.debug('_on_stream_register: sending configure() to %r', stream) + context.call_async(ContextDebugger._configure_context) + + def _on_debug_msg(self, msg): + if msg != mitogen.core._DEAD: + threading.Thread( + target=self._handle_debug_msg, + name='ContextDebuggerHandler', + args=(msg,) + ).start() + + def _handle_debug_msg(self, msg): + try: + method, args, kwargs = msg.unpickle() + msg.reply(getattr(cls, method)(*args, **kwargs)) + except Exception: + e = sys.exc_info()[1] + msg.reply(mitogen.core.CallError(e)) From 65e6a44fe7f1a20f721d9b7aba0e1a23cd61017a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 29 Apr 2018 02:31:16 +0100 Subject: [PATCH 9/9] docs: add links. --- ansible_mitogen/connection.py | 1 - docs/ansible.rst | 29 +++++++++++++++++++---------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index bfcae929..192c6dd4 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -117,7 +117,6 @@ def _connect_machinectl(spec): def _connect_setns(spec): - print 'ULTRAFLEEN', spec['remote_addr'], spec['remote_user'] return { 'method': 'setns', 'kwargs': { diff --git a/docs/ansible.rst b/docs/ansible.rst index 929835f8..d86a42cb 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -257,10 +257,17 @@ command line, or as host and group variables. File Transfer ~~~~~~~~~~~~~ -Normally a tool like ``scp`` is used to copy a file with the ``copy`` or -``template`` actions, or when uploading modules with pipelining disabled. With -Mitogen copies are implemented natively using the same interpreters, connection -tree, and routed message bus that carries RPCs. +Normally `sftp `_ or +`scp `_ is used to copy a file by the +`assemble `_, +`copy `_, +`patch `_, +`script `_, +`template `_, and +`unarchive `_ +actions, or when uploading modules with pipelining disabled. With Mitogen +copies are implemented natively using the same interpreters, connection tree, +and routed message bus that carries RPCs. This permits streaming directly between endpoints regardless of execution environment, without necessitating temporary copies in intermediary accounts or @@ -278,14 +285,16 @@ Safety Incomplete transfers proceed to a hidden file in the destination directory, with content and metadata synced using `fsync(2) -`_ prior to being renamed over any existing -file. This ensures the file remains consistent in the event of a crash, or when +`_ prior to rename over any existing file. +This ensures the file remains consistent in the event of a crash, or when overlapping `ansible-playbook` runs deploy differing file contents. -The ``sftp`` and ``scp`` tools may cause undetectable data corruption in the -form of truncated files, or files containing partial data copies from -overlapping runs of `ansible-playbook`. Both tools additionally expose a window -where users of the file may observe inconsistent contents. +The `sftp `_ and `scp +`_ tools may cause undetectable data +corruption in the form of truncated files, or files containing intermingled +data segments from overlapping runs. In normal operation both tools +additionally expose a window where users of the file may observe inconsistent +contents. Performance