issue #155: introduce mitogen.fork / Router.fork()

docs/api.rst

@@ -304,7 +304,7 @@ Sequence:
 
 
 Message Class
-============
+=============
 
 .. currentmodule:: mitogen.core
 
@@ -513,11 +513,29 @@ Router Class
 
     **Context Factories**
 
+    .. method:: fork (new_stack=False, debug=False, profiling=False)
+
+        Construct a context on the local machine by forking the current
+        process. The associated stream implementation is
+        :py:class:`mitogen.fork.Stream`.
+
+        :param bool new_stack:
+            If :py:data:`True`, arrange for the local thread stack to be
+            discarded, by forking from a new thread. Aside from clean
+            tracebacks, this has the effect of causing objects referenced by
+            the stack to cease existing in the child.
+
+        :param bool debug:
+            Same as the `debug` parameter for :py:meth:`local`.
+
+        :param bool profiling:
+            Same as the `profiling` parameter for :py:meth:`local`.
+
     .. method:: local (remote_name=None, python_path=None, debug=False, connect_timeout=None, profiling=False, via=None)
 
-        Arrange for a context to be constructed on the local machine, as an
-        immediate subprocess of the current process. The associated stream
-        implementation is :py:class:`mitogen.master.Stream`.
+        Construct a context on the local machine as a subprocess of the current
+        process. The associated stream implementation is
+        :py:class:`mitogen.master.Stream`.
 
         :param str remote_name:
             The ``argv[0]`` suffix for the new process. If `remote_name` is
@@ -565,8 +583,9 @@ Router Class
 
     .. method:: docker (container=None, image=None, docker_path=None, \**kwargs)
 
-        Arrange for a context to be constructed in an existing or temporary new
-        Docker container. One of `container` or `image` must be specified.
+        Construct a context on the local machine within an existing or
+        temporary new Docker container. One of `container` or `image` must be
+        specified.
 
         Accepts all parameters accepted by :py:meth:`local`, in addition to:
 
@@ -581,9 +600,9 @@ Router Class
 
     .. method:: sudo (username=None, sudo_path=None, password=None, \**kwargs)
 
-        Arrange for a context to be constructed over a ``sudo`` invocation. The
-        ``sudo`` process is started in a newly allocated pseudo-terminal, and
-        supports typing interactive passwords.
+        Construct a context on the local machine over a ``sudo`` invocation.
+        The ``sudo`` process is started in a newly allocated pseudo-terminal,
+        and supports typing interactive passwords.
 
         Accepts all parameters accepted by :py:meth:`local`, in addition to:
 
@@ -613,9 +632,9 @@ Router Class
 
     .. method:: ssh (hostname, username=None, ssh_path=None, port=None, check_host_keys=True, password=None, identity_file=None, compression=True, \**kwargs)
 
-        Arrange for a context to be constructed over a ``ssh`` invocation. The
-        ``ssh`` process is started in a newly allocated pseudo-terminal, and
-        supports typing interactive passwords.
+        Construct a remote context over a ``ssh`` invocation. The ``ssh``
+        process is started in a newly allocated pseudo-terminal, and supports
+        typing interactive passwords.
 
         Accepts all parameters accepted by :py:meth:`local`, in addition to:
 

mitogen/fork.py

@@ -0,0 +1,98 @@
+# Copyright 2017, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import logging
+import os
+import threading
+
+import mitogen.core
+import mitogen.parent
+
+
+LOG = logging.getLogger('mitogen')
+
+
+def break_logging_locks():
+    """
+    After fork, ensure any logging.Handler locks are recreated, as a variety of
+    threads in the parent may have been using the logging package at the moment
+    of fork.
+
+    It is not possible to solve this problem in general; see
+    https://github.com/dw/mitogen/issues/150 for a full discussion.
+    """
+    logging._lock = threading.RLock()
+    for name in logging.Logger.manager.loggerDict:
+        for handler in logging.getLogger(name).handlers:
+            handler.createLock()
+
+
+class Stream(mitogen.parent.Stream):
+    #: Reference to the importer, if any, recovered from the parent.
+    importer = None
+
+    def construct(self, old_router, debug=False, profiling=False):
+        # fork method only supports a tiny subset of options.
+        super(Stream, self).construct(debug=debug, profiling=profiling)
+
+        responder = getattr(old_router, 'responder', None)
+        if isinstance(responder, mitogen.parent.ModuleForwarder):
+            self.importer = responder.importer
+
+    def create_child(self, *_args):
+        parentfp, childfp = mitogen.parent.create_socketpair()
+        self.pid = os.fork()
+        if self.pid:
+            childfp.close()
+            # Decouple the socket from the lifetime of the Python socket object.
+            fd = os.dup(parentfp.fileno())
+            parentfp.close()
+            return self.pid, fd
+        else:
+            parentfp.close()
+            self._child_main(childfp)
+
+    def _child_main(self, childfp):
+        break_logging_locks()
+        mitogen.core.set_block(childfp.fileno())
+        os.dup2(childfp.fileno(), 1)
+        os.dup2(childfp.fileno(), 100)
+        kwargs = self.get_main_kwargs()
+        kwargs['core_src_fd'] = None
+        kwargs['importer'] = self.importer
+        kwargs['setup_package'] = False
+        mitogen.core.ExternalContext().main(**kwargs)
+        sys.exit(0)
+
+    def connect(self):
+        super(Stream, self).connect()
+        self.name = 'fork.' + str(self.pid)
+
+    def _connect_bootstrap(self):
+        # None required.
+        pass

mitogen/master.py

@@ -708,6 +708,9 @@ class Router(mitogen.parent.Router):
     def local(self, **kwargs):
         return self.connect('local', **kwargs)
 
+    def fork(self, **kwargs):
+        return self.connect('fork', **kwargs)
+
     def sudo(self, **kwargs):
         return self.connect('sudo', **kwargs)
 

mitogen/parent.py

@@ -251,6 +251,10 @@ def _docker_method():
     import mitogen.docker
     return mitogen.docker.Stream
 
+def _fork_method():
+    import mitogen.fork
+    return mitogen.fork.Stream
+
 def _local_method():
     return mitogen.parent.Stream
 
@@ -265,6 +269,7 @@ def _sudo_method():
 
 METHOD_NAMES = {
     'docker': _docker_method,
+    'fork': _fork_method,
     'local': _local_method,
     'ssh': _ssh_method,
     'sudo': _sudo_method,