From 8f5b65f7ecc6a3be28d894b660b612d778cc63e1 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 27 Jan 2019 03:00:46 +0000 Subject: [PATCH] issue #477: introduce subprocess isolation. Since Python 2.4 fork is so defective, we must use subprocesses for mitogen_task_isolation=fork. This has plenty of upside, since the long term goal is to dump forking altogether. This allows a gentle introduction of its replacement. --- ansible_mitogen/connection.py | 11 +++++------ ansible_mitogen/planner.py | 8 ++++---- ansible_mitogen/target.py | 28 ++++++++++++++++++++++------ 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 2f30e5b4..cdb83c61 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -819,21 +819,20 @@ class Connection(ansible.plugins.connection.ConnectionBase): self._connect() if use_login: return self.login_context.default_call_chain - if use_fork: + # See FORK_SUPPORTED comments in target.py. + if use_fork and self.init_child_result['fork_context'] is not None: return self.init_child_result['fork_context'].default_call_chain return self.chain - def create_fork_child(self): + def spawn_isolated_child(self): """ - Fork a new child off the target context. The actual fork occurs from - the 'virginal fork parent', which does not any Ansible modules prior to - fork, to avoid conflicts resulting from custom module_utils paths. + Fork or launch a new child off the target context. :returns: mitogen.core.Context of the new child. """ return self.get_chain(use_fork=True).call( - ansible_mitogen.target.create_fork_child + ansible_mitogen.target.spawn_isolated_child ) def get_extra_args(self): diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 03e7ecdf..80cf5f8b 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -414,7 +414,7 @@ def _propagate_deps(invocation, planner, context): def _invoke_async_task(invocation, planner): job_id = '%016x' % random.randint(0, 2**64) - context = invocation.connection.create_fork_child() + context = invocation.connection.spawn_isolated_child() _propagate_deps(invocation, planner, context) context.call_no_reply( ansible_mitogen.target.run_module_async, @@ -434,8 +434,8 @@ def _invoke_async_task(invocation, planner): } -def _invoke_forked_task(invocation, planner): - context = invocation.connection.create_fork_child() +def _invoke_isolated_task(invocation, planner): + context = invocation.connection.spawn_isolated_child() _propagate_deps(invocation, planner, context) try: return context.call( @@ -475,7 +475,7 @@ def invoke(invocation): if invocation.wrap_async: response = _invoke_async_task(invocation, planner) elif planner.should_fork(): - response = _invoke_forked_task(invocation, planner) + response = _invoke_isolated_task(invocation, planner) else: _propagate_deps(invocation, planner, invocation.connection.context) response = invocation.connection.get_chain().call( diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index 65c5750c..2e91ee22 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -96,6 +96,13 @@ MAKE_TEMP_FAILED_MSG = ( u"Please check '-vvv' output for a log of individual path errors." ) +# Python 2.4/2.5 cannot support fork+threads whatsoever, it doesn't even fix up +# interpreter state. So 2.4/2.5 interpreters start .local() contexts for +# isolation instead. Since we don't have any crazy memory sharing problems to +# avoid, there is no virginal fork parent either. The child is started directly +# from the login/become process. In future this will be default everywhere, +# fork is brainwrong from the stone age. +FORK_SUPPORTED = sys.version_info >= (2, 6) #: Initialized to an econtext.parent.Context pointing at a pristine fork of #: the target Python interpreter before it executes any code or imports. @@ -353,8 +360,9 @@ def init_child(econtext, log_level, candidate_temp_dirs): Dict like:: { - 'fork_context': mitogen.core.Context. - 'home_dir': str. + 'fork_context': mitogen.core.Context or None, + 'good_temp_dir': ... + 'home_dir': str } Where `fork_context` refers to the newly forked 'fork parent' context @@ -368,8 +376,9 @@ def init_child(econtext, log_level, candidate_temp_dirs): logging.getLogger('ansible_mitogen').setLevel(log_level) global _fork_parent - mitogen.parent.upgrade_router(econtext) - _fork_parent = econtext.router.fork() + if FORK_SUPPORTED: + mitogen.parent.upgrade_router(econtext) + _fork_parent = econtext.router.fork() global good_temp_dir good_temp_dir = find_good_temp_dir(candidate_temp_dirs) @@ -382,14 +391,21 @@ def init_child(econtext, log_level, candidate_temp_dirs): @mitogen.core.takes_econtext -def create_fork_child(econtext): +def spawn_isolated_child(econtext): """ For helper functions executed in the fork parent context, arrange for the context's router to be upgraded as necessary and for a new child to be prepared. + + The actual fork occurs from the 'virginal fork parent', which does not have + any Ansible modules loaded prior to fork, to avoid conflicts resulting from + custom module_utils paths. """ mitogen.parent.upgrade_router(econtext) - context = econtext.router.fork() + if FORK_SUPPORTED: + context = econtext.router.fork() + else: + context = econtext.router.local() LOG.debug('create_fork_child() -> %r', context) return context