Merge pull request #220 from dw/dmw

Docker fixes, logging improvements, LXC support

ansible_mitogen/connection.py

@@ -88,6 +88,17 @@ def _connect_docker(spec):
     }
 
 
+def _connect_lxc(spec):
+    return {
+        'method': 'lxc',
+        'kwargs': {
+            'container': spec['remote_addr'],
+            'python_path': spec['python_path'],
+            'connect_timeout': spec['ansible_ssh_timeout'] or spec['timeout'],
+        }
+    }
+
+
 def _connect_sudo(spec):
     return {
         'method': 'sudo',
@@ -103,10 +114,12 @@ def _connect_sudo(spec):
 
 
 CONNECTION_METHOD = {
-    'sudo': _connect_sudo,
-    'ssh': _connect_ssh,
-    'local': _connect_local,
     'docker': _connect_docker,
+    'local': _connect_local,
+    'lxc': _connect_lxc,
+    'lxd': _connect_lxc,
+    'ssh': _connect_ssh,
+    'sudo': _connect_sudo,
 }
 
 
@@ -494,3 +507,11 @@ class LocalConnection(Connection):
 
 class DockerConnection(Connection):
     transport = 'docker'
+
+
+class LxcConnection(Connection):
+    transport = 'lxc'
+
+
+class LxdConnection(Connection):
+    transport = 'lxd'

ansible_mitogen/logging.py

@@ -46,6 +46,9 @@ class Handler(logging.Handler):
         self.normal_method = normal_method
 
     def emit(self, record):
+        if getattr(record, 'mitogen_name', '') == 'stderr':
+            record.levelno = logging.ERROR
+
         s = '[pid %d] %s' % (os.getpid(), self.format(record))
         if record.levelno >= logging.ERROR:
             self.display.error(s, wrap_text=False)

ansible_mitogen/plugins/connection/mitogen_lxc.py

@@ -0,0 +1,56 @@
+# Copyright 2017, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import os.path
+import sys
+
+#
+# This is not the real Connection implementation module, it simply exists as a
+# proxy to the real module, which is loaded using Python's regular import
+# mechanism, to prevent Ansible's PluginLoader from making up a fake name that
+# results in ansible_mitogen plugin modules being loaded twice: once by
+# PluginLoader with a name like "ansible.plugins.connection.mitogen", which is
+# stuffed into sys.modules even though attempting to import it will trigger an
+# ImportError, and once under its canonical name, "ansible_mitogen.connection".
+#
+# Therefore we have a proxy module that imports it under the real name, and
+# sets up the duff PluginLoader-imported module to just contain objects from
+# the real module, so duplicate types don't exist in memory, and things like
+# debuggers and isinstance() work predictably.
+#
+
+try:
+    import ansible_mitogen
+except ImportError:
+    base_dir = os.path.dirname(__file__)
+    sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
+    del base_dir
+
+from ansible_mitogen.connection import LxcConnection as Connection
+del os
+del sys

ansible_mitogen/plugins/connection/mitogen_lxd.py

@@ -0,0 +1,56 @@
+# Copyright 2017, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import os.path
+import sys
+
+#
+# This is not the real Connection implementation module, it simply exists as a
+# proxy to the real module, which is loaded using Python's regular import
+# mechanism, to prevent Ansible's PluginLoader from making up a fake name that
+# results in ansible_mitogen plugin modules being loaded twice: once by
+# PluginLoader with a name like "ansible.plugins.connection.mitogen", which is
+# stuffed into sys.modules even though attempting to import it will trigger an
+# ImportError, and once under its canonical name, "ansible_mitogen.connection".
+#
+# Therefore we have a proxy module that imports it under the real name, and
+# sets up the duff PluginLoader-imported module to just contain objects from
+# the real module, so duplicate types don't exist in memory, and things like
+# debuggers and isinstance() work predictably.
+#
+
+try:
+    import ansible_mitogen
+except ImportError:
+    base_dir = os.path.dirname(__file__)
+    sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
+    del base_dir
+
+from ansible_mitogen.connection import LxdConnection as Connection
+del os
+del sys

ansible_mitogen/strategy.py

@@ -68,7 +68,7 @@ def wrap_connection_loader__get(name, play_context, new_stdin, **kwargs):
     'mitogen' connection type, passing the original transport name into it as
     an argument, so that it can emulate the original type.
     """
-    if name in ('ssh', 'local', 'docker'):
+    if name in ('ssh', 'local', 'docker', 'lxc', 'lxd'):
         name = 'mitogen_' + name
     return connection_loader__get(name, play_context, new_stdin, **kwargs)
 

docs/ansible.rst

@@ -116,8 +116,8 @@ Noteworthy Differences
 * The ``sudo`` become method is available and ``su`` is planned. File bugs to
   register interest in additional methods.
 
-* The ``ssh``, ``local`` and ``docker`` connection types are available, with
+* The ``docker``, ``local``, ``lxc`` and ``ssh`` connection types are
-  more planned. File bugs to register interest.
+  available, with more planned. File bugs to register interest.
 
 * Local commands execute in a reuseable interpreter created identically to
   interpreters on targets. Presently one interpreter per ``become_user``
@@ -167,7 +167,7 @@ Connection Delegation
 
 Included is a preview of **Connection Delegation**, a Mitogen-specific
 implementation of `stackable connection plug-ins`_. This enables multi-hop
-connections via a bastion, or Docker connections delegated via their host
+connections via a bastion, or Docker/LCX connections delegated via their host
 machine, where reaching the host may itself entail recursive delegation.
 
 .. _Stackable connection plug-ins: https://github.com/ansible/proposals/issues/25
@@ -467,10 +467,24 @@ Sudo
 Docker
 ~~~~~~
 
-Docker support has received relatively little testing, expect increased
+Docker support is fairly new, expect increased surprises for now.
-probability of surprises for the time being.
 
-* ``ansible_host``
+* ``ansible_host``: Name of Docker container.
+* ``ansible_user``: Name of user within the container to execute as.
+
+
+LXC
+~~~
+
+LXC support is fairly new, expect increased surprises for now. Both ``lxc`` and
+``lxd`` connection plug-ins are hijacked, however the resulting implementation
+always uses the ``lxc-attach`` command line tool rather than th LXC Python
+bindings.
+
+Consequently the ``lxc-attach`` command is required to be available on the host
+machine.
+
+* ``ansible_host``: Name of LXC container.
 
 
 Debugging

docs/api.rst

@@ -17,6 +17,7 @@ mitogen Package
 
 .. automodule:: mitogen
 
+.. autodata:: mitogen.__version__
 .. autodata:: mitogen.is_master
 .. autodata:: mitogen.context_id
 .. autodata:: mitogen.parent_id
@@ -711,6 +712,20 @@ Router Class
             Filename or complete path to the Docker binary. ``PATH`` will be
             searched if given as a filename. Defaults to ``docker``.
 
+    .. method:: lxc (container, lxc_attach_path=None, \**kwargs)
+
+        Construct a context on the local machine within an LXC container. The
+        ``lxc-attach`` program must be available.
+
+        Accepts all parameters accepted by :py:meth:`local`, in addition to:
+
+        :param str container:
+            Existing container to connect to. Defaults to ``None``.
+        :param str lxc_attach_path:
+            Filename or complete path to the ``lxc-attach`` binary. ``PATH``
+            will be searched if given as a filename. Defaults to
+            ``lxc-attach``.
+
     .. method:: sudo (username=None, sudo_path=None, password=None, \**kwargs)
 
         Construct a context on the local machine over a ``sudo`` invocation.

docs/conf.py

@@ -1,6 +1,17 @@
+import os
 import sys
+
 sys.path.append('..')
 
+def grep_version():
+    path = os.path.join(os.path.dirname(__file__), '../mitogen/__init__.py')
+    with open(path) as fp:
+        for line in fp:
+            if line.startswith('__version__'):
+                _, _, s = line.partition('=')
+                return '.'.join(map(str, eval(s)))
+
+
 author = u'David Wilson'
 copyright = u'2016, David Wilson'
 exclude_patterns = ['_build']
@@ -20,8 +31,8 @@ language = None
 master_doc = 'toc'
 project = u'Mitogen'
 pygments_style = 'sphinx'
-release = u'master'
+release = grep_version()
 source_suffix = '.rst'
 templates_path = ['_templates']
 todo_include_todos = False
-version = u'master'
+version = grep_version()

docs/getting_started.rst

@@ -125,6 +125,19 @@ Logging Environment Variables
     logs of any IO interaction, which is useful when debugging deadlocks.
 
 
+
+Logging Records
+~~~~~~~~~~~~~~~
+
+Messages received from a child context via :class:`mitogen.master.LogForwarder`
+receive extra attributes:
+
+* `mitogen_context`: :class:`mitogen.master.Context` referring to the message
+  source.
+* `mitogen_name`: original logger name in the source context.
+* `mitogen_msg`: original message in the source context.
+
+
 Creating A Context
 ------------------
 

docs/toc.rst

@@ -13,3 +13,4 @@ Table Of Contents
    examples
    internals
    shame
+   changelog

mitogen/__init__.py

@@ -31,6 +31,11 @@ On the Mitogen master, this is imported from ``mitogen/__init__.py`` as would
 be expected. On the slave, it is built dynamically during startup.
 """
 
+
+#: Library version as a tuple.
+__version__ = (0, 0, 2)
+
+
 #: This is ``False`` in slave contexts. It is used in single-file Python
 #: programs to avoid reexecuting the program's :py:func:`main` function in the
 #: slave. For example:

mitogen/core.py

@@ -485,8 +485,10 @@ class Importer(object):
         self._present = {'mitogen': [
             'compat',
             'debug',
+            'docker',
             'fakessh',
             'fork',
+            'lxc',
             'master',
             'parent',
             'service',
@@ -1585,7 +1587,8 @@ class ExternalContext(object):
         sys.modules['mitogen.core'] = mitogen.core
         del sys.modules['__main__']
 
-    def _setup_globals(self, context_id, parent_ids):
+    def _setup_globals(self, version, context_id, parent_ids):
+        mitogen.__version__ = version
         mitogen.is_master = False
         mitogen.context_id = context_id
         mitogen.parent_ids = parent_ids
@@ -1645,7 +1648,7 @@ class ExternalContext(object):
         self.dispatch_stopped = True
 
     def main(self, parent_ids, context_id, debug, profiling, log_level,
-             max_message_size, in_fd=100, out_fd=1, core_src_fd=101,
+             max_message_size, version, in_fd=100, out_fd=1, core_src_fd=101,
              setup_stdio=True, setup_package=True, importer=None,
              whitelist=(), blacklist=()):
         self._setup_master(max_message_size, profiling, parent_ids[0],
@@ -1656,7 +1659,7 @@ class ExternalContext(object):
                 self._setup_importer(importer, core_src_fd, whitelist, blacklist)
                 if setup_package:
                     self._setup_package()
-                self._setup_globals(context_id, parent_ids)
+                self._setup_globals(version, context_id, parent_ids)
                 if setup_stdio:
                     self._setup_stdio()
 

mitogen/docker.py

@@ -38,9 +38,12 @@ LOG = logging.getLogger(__name__)
 class Stream(mitogen.parent.Stream):
     container = None
     image = None
+    username = None
     docker_path = 'docker'
 
-    def construct(self, container=None, image=None, docker_path=None, **kwargs):
+    def construct(self, container=None, image=None,
+                  docker_path=None, username=None,
+                  **kwargs):
         assert container or image
         super(Stream, self).construct(**kwargs)
         if container:
@@ -49,16 +52,22 @@ class Stream(mitogen.parent.Stream):
             self.image = image
         if docker_path:
             self.docker_path = docker_path
+        if username:
+            self.username = username
 
     def connect(self):
         super(Stream, self).connect()
         self.name = 'docker.' + (self.container or self.image)
 
     def get_boot_command(self):
+        args = ['--interactive']
+        if self.username:
+            args += ['--user=' + self.username]
+
         bits = [self.docker_path]
         if self.container:
-            bits += ['exec', '-i', self.container]
+            bits += ['exec'] + args + [self.container]
         elif self.image:
-            bits += ['run', '-i', '--rm', self.image]
+            bits += ['run'] + args + ['--rm', self.image]
-        bits += super(Stream, self).get_boot_command()
+
-        return bits
+        return bits + super(Stream, self).get_boot_command()

mitogen/fakessh.py

@@ -350,6 +350,7 @@ def run(dest, router, args, deadline=None, econtext=None):
                 'parent_ids': parent_ids,
                 'profiling': getattr(router, 'profiling', False),
                 'setup_stdio': False,
+                'version': mitogen.__version__,
             },))
         finally:
             fp.close()

mitogen/lxc.py

@@ -0,0 +1,68 @@
+# Copyright 2017, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import logging
+
+import mitogen.core
+import mitogen.parent
+
+
+LOG = logging.getLogger(__name__)
+
+
+class Stream(mitogen.parent.Stream):
+    create_child_args = {
+        # If lxc-attach finds any of stdin, stdout, stderr connected to a TTY,
+        # to prevent input injection it creates a proxy pty, forcing all IO to
+        # be buffered in <4KiB chunks. So ensure stderr is also routed to the
+        # socketpair.
+        'merge_stdio': True
+    }
+
+    container = None
+    lxc_attach_path = 'lxc-attach'
+
+    def construct(self, container, lxc_attach_path=None, **kwargs):
+        super(Stream, self).construct(**kwargs)
+        if container:
+            self.container = container
+        if lxc_attach_path:
+            self.lxc_attach_path = lxc_attach_apth
+
+    def connect(self):
+        super(Stream, self).connect()
+        self.name = 'lxc.' + self.container
+
+    def get_boot_command(self):
+        bits = [
+            self.lxc_attach_path,
+            '--clear-env',
+            '--name', self.container,
+            '--',
+        ]
+        return bits + super(Stream, self).get_boot_command()

mitogen/master.py

@@ -309,7 +309,11 @@ class LogForwarder(object):
             self._cache[msg.src_id] = logger = logging.getLogger(name)
 
         name, level_s, s = msg.data.split('\x00', 2)
-        logger.log(int(level_s), '%s: %s', name, s)
+        logger.log(int(level_s), '%s: %s', name, s, extra={
+            'mitogen_message': s,
+            'mitogen_context': self._router.context_by_id(msg.src_id),
+            'mitogen_name': name,
+        })
 
     def __repr__(self):
         return 'LogForwarder(%r)' % (self._router,)

mitogen/parent.py

@@ -243,10 +243,17 @@ def create_socketpair():
     return parentfp, childfp
 
 
-def create_child(*args):
+def create_child(args, merge_stdio=False):
     """
     Create a child process whose stdin/stdout is connected to a socket.
 
+    :param args:
+        Argument vector for execv() call.
+    :param bool merge_stdio:
+        If :data:`True`, arrange for `stderr` to be connected to the `stdout`
+        socketpair, rather than inherited from the parent process. This may be
+        necessary to ensure that not TTY is connected to any stdio handle, for
+        instance when using LXC.
     :returns:
         `(pid, socket_obj, :data:`None`)`
     """
@@ -257,11 +264,17 @@ def create_child(*args):
     # O_NONBLOCK from Python's future stdin fd.
     mitogen.core.set_block(childfp.fileno())
 
+    if merge_stdio:
+        extra = {'stderr': childfp}
+    else:
+        extra = {}
+
     proc = subprocess.Popen(
         args=args,
         stdin=childfp,
         stdout=childfp,
         close_fds=True,
+        **extra
     )
     childfp.close()
     # Decouple the socket from the lifetime of the Python socket object.
@@ -284,7 +297,7 @@ def _acquire_controlling_tty():
         fcntl.ioctl(2, termios.TIOCSCTTY)
 
 
-def tty_create_child(*args):
+def tty_create_child(args):
     """
     Return a file descriptor connected to the master end of a pseudo-terminal,
     whose slave end is connected to stdin/stdout/stderr of a new child process.
@@ -318,7 +331,7 @@ def tty_create_child(*args):
     return proc.pid, master_fd, None
 
 
-def hybrid_tty_create_child(*args):
+def hybrid_tty_create_child(args):
     """
     Like :func:`tty_create_child`, except attach stdin/stdout to a socketpair
     like :func:`create_child`, but leave stderr and the controlling TTY
@@ -685,6 +698,7 @@ class Stream(mitogen.core.Stream):
             'whitelist': self._router.get_module_whitelist(),
             'blacklist': self._router.get_module_blacklist(),
             'max_message_size': self.max_message_size,
+            'version': mitogen.__version__,
         }
 
     def get_preamble(self):
@@ -695,12 +709,13 @@ class Stream(mitogen.core.Stream):
         return zlib.compress(minimize_source(source), 9)
 
     create_child = staticmethod(create_child)
+    create_child_args = {}
     name_prefix = 'local'
 
     def start_child(self):
         args = self.get_boot_command()
         try:
-            return self.create_child(*args)
+            return self.create_child(args, **self.create_child_args)
         except OSError:
             e = sys.exc_info()[1]
             msg = 'Child start failed: %s. Command was: %s' % (e, Argv(args))
@@ -993,6 +1008,9 @@ class Router(mitogen.core.Router):
         self._context_by_id[context.context_id] = context
         return context
 
+    def lxc(self, **kwargs):
+        return self.connect('lxc', **kwargs)
+
     def docker(self, **kwargs):
         return self.connect('docker', **kwargs)
 

setup.py

@@ -28,9 +28,19 @@
 
 from setuptools import find_packages, setup
 
+
+def grep_version():
+    path = os.path.join(os.path.dirname(__file__), 'mitogen/__init__.py')
+    with open(path) as fp:
+        for line in fp:
+            if line.startswith('__version__'):
+                _, _, s = line.partition('=')
+                return '.'.join(map(str, eval(s)))
+
+
 setup(
     name = 'mitogen',
-    version = '0.0.2',
+    version = grep_version(),
     description = 'Library for writing distributed self-replicating programs.',
     author = 'David Wilson',
     license = 'New BSD',

tests/parent_test.py

@@ -117,9 +117,9 @@ class TtyCreateChildTest(unittest2.TestCase):
         # read a password.
         tf = tempfile.NamedTemporaryFile()
         try:
-            pid, fd, _ = self.func(
+            pid, fd, _ = self.func([
                 'bash', '-c', 'exec 2>%s; echo hi > /dev/tty' % (tf.name,)
-            )
+            ])
             deadline = time.time() + 5.0
             for line in mitogen.parent.iter_read([fd], deadline):
                 self.assertEquals('hi\n', line)