diff --git a/docs/api.rst b/docs/api.rst index 29f88c32..035954da 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -304,7 +304,7 @@ Sequence: Message Class -============ +============= .. currentmodule:: mitogen.core @@ -513,11 +513,29 @@ Router Class **Context Factories** + .. method:: fork (new_stack=False, debug=False, profiling=False) + + Construct a context on the local machine by forking the current + process. The associated stream implementation is + :py:class:`mitogen.fork.Stream`. + + :param bool new_stack: + If :py:data:`True`, arrange for the local thread stack to be + discarded, by forking from a new thread. Aside from clean + tracebacks, this has the effect of causing objects referenced by + the stack to cease existing in the child. + + :param bool debug: + Same as the `debug` parameter for :py:meth:`local`. + + :param bool profiling: + Same as the `profiling` parameter for :py:meth:`local`. + .. method:: local (remote_name=None, python_path=None, debug=False, connect_timeout=None, profiling=False, via=None) - Arrange for a context to be constructed on the local machine, as an - immediate subprocess of the current process. The associated stream - implementation is :py:class:`mitogen.master.Stream`. + Construct a context on the local machine as a subprocess of the current + process. The associated stream implementation is + :py:class:`mitogen.master.Stream`. :param str remote_name: The ``argv[0]`` suffix for the new process. If `remote_name` is @@ -565,8 +583,9 @@ Router Class .. method:: docker (container=None, image=None, docker_path=None, \**kwargs) - Arrange for a context to be constructed in an existing or temporary new - Docker container. One of `container` or `image` must be specified. + Construct a context on the local machine within an existing or + temporary new Docker container. One of `container` or `image` must be + specified. Accepts all parameters accepted by :py:meth:`local`, in addition to: @@ -581,9 +600,9 @@ Router Class .. method:: sudo (username=None, sudo_path=None, password=None, \**kwargs) - Arrange for a context to be constructed over a ``sudo`` invocation. The - ``sudo`` process is started in a newly allocated pseudo-terminal, and - supports typing interactive passwords. + Construct a context on the local machine over a ``sudo`` invocation. + The ``sudo`` process is started in a newly allocated pseudo-terminal, + and supports typing interactive passwords. Accepts all parameters accepted by :py:meth:`local`, in addition to: @@ -613,9 +632,9 @@ Router Class .. method:: ssh (hostname, username=None, ssh_path=None, port=None, check_host_keys=True, password=None, identity_file=None, compression=True, \**kwargs) - Arrange for a context to be constructed over a ``ssh`` invocation. The - ``ssh`` process is started in a newly allocated pseudo-terminal, and - supports typing interactive passwords. + Construct a remote context over a ``ssh`` invocation. The ``ssh`` + process is started in a newly allocated pseudo-terminal, and supports + typing interactive passwords. Accepts all parameters accepted by :py:meth:`local`, in addition to: diff --git a/mitogen/fork.py b/mitogen/fork.py new file mode 100644 index 00000000..a8827185 --- /dev/null +++ b/mitogen/fork.py @@ -0,0 +1,98 @@ +# Copyright 2017, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +import logging +import os +import threading + +import mitogen.core +import mitogen.parent + + +LOG = logging.getLogger('mitogen') + + +def break_logging_locks(): + """ + After fork, ensure any logging.Handler locks are recreated, as a variety of + threads in the parent may have been using the logging package at the moment + of fork. + + It is not possible to solve this problem in general; see + https://github.com/dw/mitogen/issues/150 for a full discussion. + """ + logging._lock = threading.RLock() + for name in logging.Logger.manager.loggerDict: + for handler in logging.getLogger(name).handlers: + handler.createLock() + + +class Stream(mitogen.parent.Stream): + #: Reference to the importer, if any, recovered from the parent. + importer = None + + def construct(self, old_router, debug=False, profiling=False): + # fork method only supports a tiny subset of options. + super(Stream, self).construct(debug=debug, profiling=profiling) + + responder = getattr(old_router, 'responder', None) + if isinstance(responder, mitogen.parent.ModuleForwarder): + self.importer = responder.importer + + def create_child(self, *_args): + parentfp, childfp = mitogen.parent.create_socketpair() + self.pid = os.fork() + if self.pid: + childfp.close() + # Decouple the socket from the lifetime of the Python socket object. + fd = os.dup(parentfp.fileno()) + parentfp.close() + return self.pid, fd + else: + parentfp.close() + self._child_main(childfp) + + def _child_main(self, childfp): + break_logging_locks() + mitogen.core.set_block(childfp.fileno()) + os.dup2(childfp.fileno(), 1) + os.dup2(childfp.fileno(), 100) + kwargs = self.get_main_kwargs() + kwargs['core_src_fd'] = None + kwargs['importer'] = self.importer + kwargs['setup_package'] = False + mitogen.core.ExternalContext().main(**kwargs) + sys.exit(0) + + def connect(self): + super(Stream, self).connect() + self.name = 'fork.' + str(self.pid) + + def _connect_bootstrap(self): + # None required. + pass diff --git a/mitogen/master.py b/mitogen/master.py index 580ab61f..61418792 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -708,6 +708,9 @@ class Router(mitogen.parent.Router): def local(self, **kwargs): return self.connect('local', **kwargs) + def fork(self, **kwargs): + return self.connect('fork', **kwargs) + def sudo(self, **kwargs): return self.connect('sudo', **kwargs) diff --git a/mitogen/parent.py b/mitogen/parent.py index 8c501468..f81c6e92 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -251,6 +251,10 @@ def _docker_method(): import mitogen.docker return mitogen.docker.Stream +def _fork_method(): + import mitogen.fork + return mitogen.fork.Stream + def _local_method(): return mitogen.parent.Stream @@ -265,6 +269,7 @@ def _sudo_method(): METHOD_NAMES = { 'docker': _docker_method, + 'fork': _fork_method, 'local': _local_method, 'ssh': _ssh_method, 'sudo': _sudo_method,