Merge pull request #228 from dw/dmw

setns method

ansible_mitogen/services.py

@@ -38,6 +38,7 @@ when a child has completed a job.
 """
 
 from __future__ import absolute_import
+import hashlib
 import logging
 import os
 import os.path
@@ -114,11 +115,19 @@ class ContextService(mitogen.service.Service):
 
     def key_from_kwargs(self, **kwargs):
         """
-        Generate a deduplication key from the request. The default
+        Generate a deduplication key from the request.
-        implementation returns a string based on a stable representation of the
-        input dictionary generated by :py:func:`pprint.pformat`.
         """
-        return pprint.pformat(kwargs)
+        out = []
+        stack = [kwargs]
+        while stack:
+            obj = stack.pop()
+            if isinstance(obj, dict):
+                stack.extend(sorted(obj.iteritems()))
+            elif isinstance(obj, (list, tuple)):
+                stack.extend(obj)
+            else:
+                out.append(str(obj))
+        return ''.join(out)
 
     def _produce_response(self, key, response):
         """

ansible_mitogen/target.py

@@ -43,6 +43,7 @@ import re
 import stat
 import subprocess
 import tempfile
+import time
 import traceback
 import zlib
 
@@ -82,6 +83,7 @@ def _get_file(context, path, out_fp):
         interrupted and the output should be discarded.
     """
     LOG.debug('_get_file(): fetching %r from %r', path, context)
+    t0 = time.time()
     recv = mitogen.core.Receiver(router=context.router)
     size = mitogen.service.call(
         context=context,
@@ -102,8 +104,8 @@ def _get_file(context, path, out_fp):
         LOG.error('get_file(%r): receiver was closed early, controller '
                   'is likely shutting down.', path)
 
-    LOG.debug('target.get_file(): fetched %d bytes of %r from %r',
+    LOG.debug('target.get_file(): fetched %d bytes of %r from %r in %dms',
-              size, path, context)
+              size, path, context, 1000*(time.time() - t0))
     return out_fp.tell() == size
 
 

docs/api.rst

@@ -698,8 +698,8 @@ Router Class
     .. method:: docker (container=None, image=None, docker_path=None, \**kwargs)
 
         Construct a context on the local machine within an existing or
-        temporary new Docker container. One of `container` or `image` must be
+        temporary new Docker container using the ``docker`` program. One of
-        specified.
+        `container` or `image` must be specified.
 
         Accepts all parameters accepted by :py:meth:`local`, in addition to:
 
@@ -717,8 +717,8 @@ Router Class
 
     .. method:: jail (container, jexec_path=None, \**kwargs)
 
-        Construct a context on the local machine within a FreeBSD jail. The
+        Construct a context on the local machine within a FreeBSD jail using
-        ``jexec`` program must be available.
+        the ``jexec`` program.
 
         Accepts all parameters accepted by :py:meth:`local`, in addition to:
 
@@ -733,8 +733,8 @@ Router Class
 
     .. method:: lxc (container, lxc_attach_path=None, \**kwargs)
 
-        Construct a context on the local machine within an LXC container. The
+        Construct a context on the local machine within an LXC container using
-        ``lxc-attach`` program must be available.
+        the ``lxc-attach`` program.
 
         Accepts all parameters accepted by :py:meth:`local`, in addition to:
 
@@ -745,6 +745,34 @@ Router Class
             will be searched if given as a filename. Defaults to
             ``lxc-attach``.
 
+    .. method:: setns (container, kind, docker_path=None, lxc_info_path=None, machinectl_path=None, \**kwargs)
+
+        Construct a context in the style of :meth:`local`, but change the
+        active Linux process namespaces via calls to `setns(1)` before
+        executing Python.
+
+        The namespaces to use, and the active root file system are taken from
+        the root PID of a running Docker, LXC, or systemd-nspawn container.
+
+        A program is required only to find the root PID, after which management
+        of the child Python interpreter is handled directly.
+
+        :param str container:
+            Container to connect to.
+        :param str kind:
+            One of ``docker``, ``lxc`` or ``machinectl``.
+        :param str docker_path:
+            Filename or complete path to the Docker binary. ``PATH`` will be
+            searched if given as a filename. Defaults to ``docker``.
+        :param str lxc_info_path:
+            Filename or complete path to the ``lxc-info`` binary. ``PATH``
+            will be searched if given as a filename. Defaults to
+            ``lxc-info``.
+        :param str machinectl_path:
+            Filename or complete path to the ``machinectl`` binary. ``PATH``
+            will be searched if given as a filename. Defaults to
+            ``machinectl``.
+
     .. method:: sudo (username=None, sudo_path=None, password=None, \**kwargs)
 
         Construct a context on the local machine over a ``sudo`` invocation.

mitogen/core.py

@@ -493,6 +493,7 @@ class Importer(object):
             'master',
             'parent',
             'service',
+            'setns',
             'ssh',
             'sudo',
             'utils',

mitogen/parent.py

@@ -243,7 +243,7 @@ def create_socketpair():
     return parentfp, childfp
 
 
-def create_child(args, merge_stdio=False):
+def create_child(args, merge_stdio=False, preexec_fn=None):
     """
     Create a child process whose stdin/stdout is connected to a socket.
 
@@ -274,6 +274,7 @@ def create_child(args, merge_stdio=False):
         stdin=childfp,
         stdout=childfp,
         close_fds=True,
+        preexec_fn=preexec_fn,
         **extra
     )
     childfp.close()
@@ -1023,6 +1024,9 @@ class Router(mitogen.core.Router):
     def lxc(self, **kwargs):
         return self.connect('lxc', **kwargs)
 
+    def setns(self, **kwargs):
+        return self.connect('setns', **kwargs)
+
     def ssh(self, **kwargs):
         return self.connect('ssh', **kwargs)
 

mitogen/setns.py

@@ -0,0 +1,172 @@
+# Copyright 2017, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import ctypes
+import logging
+import os
+import subprocess
+
+import mitogen.core
+import mitogen.parent
+
+
+LOG = logging.getLogger(__name__)
+LIBC = ctypes.CDLL(None, use_errno=True)
+LIBC__strerror = LIBC.strerror
+LIBC__strerror.restype = ctypes.c_char_p
+
+
+class Error(mitogen.core.StreamError):
+    pass
+
+
+def setns(kind, fd):
+    if LIBC.setns(int(fd), 0) == -1:
+        errno = ctypes.get_errno()
+        msg = 'setns(%s, %s): %s' % (fd, kind, LIBC__strerror(errno))
+        raise OSError(errno, msg)
+
+
+def _run_command(args):
+    proc = subprocess.Popen(
+        args=args,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT
+    )
+
+    output, _ = proc.communicate()
+    if not proc.returncode:
+        return output
+
+    raise Error("%s exitted with status %d: %s",
+                mitogen.parent.Argv(args), proc.returncode, output)
+
+
+def get_docker_pid(path, name):
+    args = [path, 'inspect', '--format={{.State.Pid}}', name]
+    try:
+        return int(_run_command(args))
+    except ValueError:
+        raise Error("could not find PID from docker output.\n%s", output)
+
+
+def get_lxc_pid(path, name):
+    output = _run_command([path, '-n', name])
+    for line in output.splitlines():
+        bits = line.split()
+        if bits and bits[0] == 'PID:':
+            return int(bits[1])
+
+    raise Error("could not find PID from lxc-info output.\n%s", output)
+
+
+def get_machinectl_pid(path, name):
+    output = _run_command([path, 'status', name])
+    for line in output.splitlines():
+        bits = line.split()
+        if bits and bits[0] == 'Leader:':
+            return int(bits[1])
+
+    raise Error("could not find PID from machinectl output.\n%s", output)
+
+
+class Stream(mitogen.parent.Stream):
+    container = None
+    kind = None
+    docker_path = 'docker'
+    lxc_info_path = 'lxc-info'
+    machinectl_path = 'machinectl'
+
+    GET_LEADER_BY_KIND = {
+        'docker': ('docker_path', get_docker_pid),
+        'lxc': ('lxc_info_path', get_lxc_pid),
+        'machinectl': ('machinectl_path', get_machinectl_pid),
+    }
+
+    def construct(self, container, kind, docker_path=None,
+                  lxc_info_path=None, machinectl_path=None, **kwargs):
+        super(Stream, self).construct(**kwargs)
+        if kind not in self.GET_LEADER_BY_KIND:
+            raise Error('unsupported container kind: %r', kind)
+
+        self.container = container
+        self.kind = kind
+        if docker_path:
+            self.docker_path = docker_path
+        if lxc_info_path:
+            self.lxc_info_path = lxc_info_path
+        if machinectl_path:
+            self.machinectl_path = lxc_attach_apth
+
+    # Order matters. https://github.com/karelzak/util-linux/commit/854d0fe/
+    NS_ORDER = ('ipc', 'uts', 'net', 'pid', 'mnt', 'user')
+
+    def preexec_fn(self):
+        nspath = '/proc/%d/ns/' % (self.leader_pid,)
+        selfpath = '/proc/self/ns/'
+        try:
+            ns_fps = [
+                open(nspath + name)
+                for name in self.NS_ORDER
+                if os.path.exists(nspath + name) and (
+                    os.readlink(nspath + name) != os.readlink(selfpath + name)
+                )
+            ]
+        except Exception, e:
+            raise Error(str(e))
+
+        os.chroot('/proc/%s/root' % (self.leader_pid,))
+        os.chdir('/')
+        for fp in ns_fps:
+            setns(fp.name, fp.fileno())
+            fp.close()
+
+    def get_boot_command(self):
+        # With setns(CLONE_NEWPID), new children of the caller receive a new
+        # PID namespace, however the caller's namespace won't change. That
+        # causes subsequent calls to clone() specifying CLONE_THREAD to fail
+        # with EINVAL, as threads in the same process can't have varying PID
+        # namespaces, meaning starting new threads in the exec'd program will
+        # fail. The solution is forking, so inject a /bin/sh call to achieve
+        # this.
+        argv = super(Stream, self).get_boot_command()
+        # bash will exec() if a single command was specified and the shell has
+        # nothing left to do, so "; exit $?" gives bash a reason to live.
+        return ['/bin/sh', '-c', '%s; exit $?' % (mitogen.parent.Argv(argv),)]
+
+    def create_child(self, args):
+        return mitogen.parent.create_child(args, preexec_fn=self.preexec_fn)
+
+    def connect(self):
+        attr, func = self.GET_LEADER_BY_KIND[self.kind]
+        tool_path = getattr(self, attr)
+        self.leader_pid = func(tool_path, self.container)
+        LOG.debug('Leader PID for %s container %r: %d',
+                  self.kind, self.container, self.leader_pid)
+        super(Stream, self).connect()
+        self.name = 'setns.' + self.container