From 6cb81d3cb73fc7e6ffd4a4ee3d6bd41647c11beb Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 27 Jul 2018 00:03:32 +0000 Subject: [PATCH 001/212] tests: allow passing -vvv to ansible_tests.sh. --- .travis/ansible_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis/ansible_tests.sh b/.travis/ansible_tests.sh index a61ed836..e6441343 100755 --- a/.travis/ansible_tests.sh +++ b/.travis/ansible_tests.sh @@ -60,5 +60,5 @@ echo travis_fold:end:job_setup echo travis_fold:start:ansible /usr/bin/time ./run_ansible_playbook.sh \ all.yml \ - -i "${TMPDIR}/hosts" + -i "${TMPDIR}/hosts" "$@" echo travis_fold:end:ansible From 1008cda93b0375f42c86f07001805c3a25157b22 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 27 Jul 2018 12:03:25 -0700 Subject: [PATCH 002/212] tests: add missing debops installs tep --- tests/ansible/gcloud/controller.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/ansible/gcloud/controller.yml b/tests/ansible/gcloud/controller.yml index 48f233d9..b4ce3fcf 100644 --- a/tests/ansible/gcloud/controller.yml +++ b/tests/ansible/gcloud/controller.yml @@ -56,6 +56,10 @@ editable: true name: ~/ansible + - pip: + virtualenv: ~/venv + name: debops + - lineinfile: line: "source $HOME/venv/bin/activate" path: ~/.profile From 9b2417e62d72fe5cbb44fc2fd9b498674b5eda6d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 27 Jul 2018 12:03:38 -0700 Subject: [PATCH 003/212] docs: add funny testimonial --- docs/ansible.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/ansible.rst b/docs/ansible.rst index 6ab6626a..919c6380 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -118,6 +118,7 @@ Testimonials strategy took Clojars' Ansible runs from **14 minutes to 2 minutes**. I still can't quite believe it." +* "Enabling the mitogen plugin in ansible feels like switching from floppy to SSD" .. _noteworthy_differences: From f8b3441431eae5ada0977cb4135fdeef4a1ba369 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 27 Jul 2018 13:32:55 -0700 Subject: [PATCH 004/212] ansible: work around Ansible PR #41749 --- ansible_mitogen/target.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index e5365dd4..47249f25 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -39,6 +39,7 @@ import functools import grp import json import logging +import new import operator import os import pwd @@ -46,16 +47,25 @@ import re import signal import stat import subprocess +import sys import tempfile import traceback -import ansible.module_utils.json_utils -import ansible_mitogen.runner import mitogen.core import mitogen.fork import mitogen.parent import mitogen.service +# Ansible since PR #41749 inserts "import __main__" into +# ansible.module_utils.basic. Mitogen's importer will refuse such an import, so +# we must setup a fake "__main__" before that module is ever imported. The +# str() is to cast Unicode to bytes on Python 2.6. +if not sys.modules.get(str('__main__')): + sys.modules[str('__main__')] = new.module(str('__main__')) + +import ansible.module_utils.json_utils +import ansible_mitogen.runner + LOG = logging.getLogger(__name__) From 196f76ff223e2acac1bb7b5b19030c2acc87be75 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 27 Jul 2018 19:34:22 -0700 Subject: [PATCH 005/212] Remove staticmethod from docs. Can re-add this later for 3.x, but it's pretty impossible in general for 2.x. Closes #313. --- docs/api.rst | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 6efca6dd..92abe1ec 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -831,8 +831,8 @@ Context Class context's main thread. :param fn: - A free function in module scope, or a classmethod or staticmethod - of a class directly reachable from module scope: + A free function in module scope or a class method of a class + directly reachable from module scope: .. code-block:: python @@ -842,10 +842,6 @@ Context Class """A free function reachable as mymodule.my_func""" class MyClass: - @staticmethod - def my_staticmethod(): - """Reachable as mymodule.MyClass.my_staticmethod""" - @classmethod def my_classmethod(cls): """Reachable as mymodule.MyClass.my_classmethod""" From 3b109201572f4eac5614c87bc99a631e62735cb7 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 28 Jul 2018 12:28:59 -0700 Subject: [PATCH 006/212] docs: delete compared.rst because somehow it's in search results. --- docs/compared.rst | 226 ---------------------------------------------- 1 file changed, 226 deletions(-) delete mode 100644 docs/compared.rst diff --git a/docs/compared.rst b/docs/compared.rst deleted file mode 100644 index b75ae3f2..00000000 --- a/docs/compared.rst +++ /dev/null @@ -1,226 +0,0 @@ - -Mitogen Compared To -------------------- - -This provides a little free-text summary of conceptual differences between -Mitogen and other tools, along with some basic perceptual metrics (project -maturity/age, quality of tests, function matrix) - - -Ansible -####### - -Ansible_ is a complete provisioning system, Mitogen is a small component of such a system. - -You should use Ansible if ... - -You should not use Ansible if ... - - -.. _Ansible: https://docs.ansible.com/ansible/latest/index.html -.. _ansible.src: https://github.com/ansible/ansible/ - -Baker -##### - - Baker_ lets you easily add a command line interface to your Python - functions using a simple decorator, to create scripts with "sub-commands", - similar to Django's ``manage.py``, ``svn``, ``hg``, etc. - -- Unmaintained since 2015 -- No obvious remote execution functionality - -.. _Baker: https://bitbucket.org/mchaput/baker - -Chopsticks -########## - -Chopsticks_ also supports recursion! but the recursively executed instance has no special knowledge of its identity in a tree structure, and little support for functions running in the master to directly invoke functions in a recursive context.. effectively each recursion produces a new master, from which function calls must be made. - -executing functions from __main__ entails picking just that function and deps -out of the main module, not transferring the module intact. that approach works -but it's much messier than just arranging for __main__ to be imported and -executed through the import mechanism. - -supports sudo but no support for require_tty or typing a sudo password. also supports SSH and Docker. - -good set of tests - -real PEP-302 module loader, but doesn't try to cope with master also relying on -a PEP-302 module loader (e.g. py2exe). - -Based on the tox configuration Python 2.7, and 3.3 to 3.6 are supported. - -I/O multiplexer in the master, but not in children. - -As with Execnet it includes its own serialization - pencode_ supports - -- most Python primitive types (``bytes``/``str``/``unicode``, ``list``, ``tuple`` ...) -- identity references -- self referencing (recursive) data srtuctures - -pencode lacks support for arbitrary classes. Byte strings require special -treatment if they contain non-ascii characters. Some primitive types -(e.g. ``complex``) are not handled. This would be straightforwar to address. -Values are length-prefixed with a 32 bit unsigned integer, meaning values -are limited to 4 billion bytes or items in length. - -design is reminiscent of Mitogen in places (Tunnel is practically identical to -Mitogen's Stream), and closer to Execnet elsewhere (lack of uniformity, -tendency to prefer logic expressed in if/else special case soup rather than the -type system, though some of that is due to supporting Python 3, so not judging -too harshly!) - -Chopsticks has its own `Chopsticks vs`_ comparisons. - -You should use Chopsticks if you need Python 3 support. - -.. _Chopsticks: https://chopsticks.readthedocs.io/en/stable/ -.. _Chopsticks.src: https://github.com/lordmauve/chopsticks/ -.. _Chopsticks vs: https://chopsticks.readthedocs.io/en/stable/intro.html#chopsticks-vs -.. _pencode: https://github.com/lordmauve/chopsticks/blob/master/doc/pencode.rst -.. _pencode.src: https://github.com/lordmauve/chopsticks/blob/master/chopsticks/pencode.py - -Disco -##### - - Disco_ is a lightweight, open-source framework for distributed computing - based on the MapReduce paradigm. - -- An Erlang core, with Python bindings -- Wire format is pickle, according to `Execnet vs NLTK for distributed NLTK`_ - -.. _Disco: http://discoproject.org/ -.. _Execnet vs NLTK for distributed NLTK: https://streamhacker.com/2009/12/14/execnet-disco-distributed-nltk/ - -Execnet -####### - -Execnet_ - -- Parent and children may use threads, gevent, or eventlet, Mitogen only supports threads. -- No recursion -- Similar Channel abstraction but better developed.. includes waiting for remote to close its end -- Heavier emphasis on passing chunks of Python source code around, modules are loaded one-at-a-time with no dependency resolution mechanism -- Built-in unidirectional rsync-alike, compared to Mitogen's SSH emulation which allows use of real rsync in any supported mode -- no support for sudo, but supports connecting to vagrant -- works with read-only filesystem -- includes its own serialization_ independent of the standard library - - The obj and all contained objects must be of a builtin python type - (so nested dicts, sets, etc. are all ok but not user-level instances). - -- Known uses include `pytest-xdist`_, and `Distributed NLTK`_ - -You should use Execnet if you value code maturity more than featureset. - -.. _Execnet: https://codespeak.net/execnet/ -.. _serialization: https://codespeak.net/execnet/basics.html#dumps-loads -.. _pytest-xdist: https://pypi.python.org/pypi/pytest-xdist -.. _Distributed NLTK: https://streamhacker.com/2009/12/14/execnet-disco-distributed-nltk/ - -Fabric -###### - -Fabric_ allows execution of shell snippets on remote machines, Python functions run -locally, any remote interaction is fundamentally done via shell, with all the -limitations that entails. prefers to depend on SSH features (e.g. tunnelling) -than reinvent them - -You should use Fabric if you enjoy being woken at 4am to pages about broken -shell snippets. - -.. _fabric: http://www.fabfile.org/ - -Invoke -###### - -Invoke_ - -Python 2.6+, 3.3+ - -Basically a Fabric-alike - -.. _invoke: http://www.pyinvoke.org/ - -Multiprocessing -############### - -multiprocessing_ was added to the stdlib in Python 2.6. - - multiprocessing is a package that supports spawning processes using an - API similar to the threading module. The multiprocessing package offers - both local and remote concurrency - -There is a backport_ for Python 2.4 & 2.5, but it is not pure Python. -pymultiprocessing_ appears to be a pure Python implementation. -An ecosystem_ of packages has built up around multiprocessing. - -The `programming guidelines`_ section notes - -- Arguments to proxies must be picklable. On Windows this also applies to - ``multiprocessing.Process.__init__()`` arguments. -- Callers should beware replacing ``sys.stdin``, because - ``multiprocessing.Process._bootstrap()`` - will close it and open /dev/null instead - -.. _programming guidelines: https://docs.python.org/2/library/multiprocessing.html#programming-guidelines -.. _backport: https://pypi.python.org/pypi/multiprocessing -.. _pymultiprocessing: https://pypi.python.org/pypi/pymultiprocessing -.. _ecosystem: https://pypi.python.org/pypi?%3Aaction=search&term=multiprocessing&submit=search - -Paver -##### - -Paver_ - -More or less another task execution framework / make-alike, doesn't really deal -with remote execution at all. - -.. _Paver: https://github.com/paver/paver/ - -Plumbum -####### - -Plumbum_ - -Shell-only - -Basically syntax sugar for running shell commands. Nicer than raw shell -(depending on your opinions of operating overloading), but it's still shell. - -.. _Plumbum: https://pypi.python.org/pypi/plumbum - -Pyro4 -##### - -Pyro4_ -... - -.. _Pyro4: https://pythonhosted.org/Pyro4/ - -RPyC -#### - -RPyC_ - -- supports transparent object proxies similar to Pyro (with all the pain and suffering hidden network IO entails) -- significantly more 'frameworkey' feel -- runs multiplexer in a thread too? -- bootstrap over SSH only, no recursion and no sudo -- requires a writable filesystem - -.. _RPyC: https://rpyc.readthedocs.io/en/latest/ - -Salt -#### - -Salt_ - -- no crappy deps - -You should use Salt if you enjoy firefighting endless implementation bugs, -otherwise you should prefer Ansible. - -.. _Salt: https://docs.saltstack.com/en/latest/topics/ -.. _Salt.src: https://github.com/saltstack/salt From 6c03b83748e27a027145f925801a49c3747e6739 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 28 Jul 2018 12:35:03 -0700 Subject: [PATCH 007/212] issue #291: don't attempt mitogen import until sys.path modified. Given an extracted download of mitogen-2.2.tar.gz, with strategy_plugins pointing into it, if an old version of the package was pip-installed, then the old pip-installed package would be imported and override whatever came from the tarball. Instead, modify sys.path before attempting any import. This still isn't perfect, but it's better. --- ansible_mitogen/plugins/strategy/mitogen.py | 12 ++++++------ ansible_mitogen/plugins/strategy/mitogen_free.py | 12 ++++++------ ansible_mitogen/plugins/strategy/mitogen_linear.py | 12 ++++++------ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/ansible_mitogen/plugins/strategy/mitogen.py b/ansible_mitogen/plugins/strategy/mitogen.py index 3ef522b4..4f595161 100644 --- a/ansible_mitogen/plugins/strategy/mitogen.py +++ b/ansible_mitogen/plugins/strategy/mitogen.py @@ -44,12 +44,12 @@ import sys # debuggers and isinstance() work predictably. # -try: - import ansible_mitogen -except ImportError: - base_dir = os.path.dirname(__file__) - sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) - del base_dir +BASE_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__), '../../..') +) + +if BASE_DIR not in sys.path: + sys.path.insert(0, BASE_DIR) import ansible_mitogen.strategy import ansible.plugins.strategy.linear diff --git a/ansible_mitogen/plugins/strategy/mitogen_free.py b/ansible_mitogen/plugins/strategy/mitogen_free.py index 34f959ca..8dfaa16e 100644 --- a/ansible_mitogen/plugins/strategy/mitogen_free.py +++ b/ansible_mitogen/plugins/strategy/mitogen_free.py @@ -44,12 +44,12 @@ import sys # debuggers and isinstance() work predictably. # -try: - import ansible_mitogen -except ImportError: - base_dir = os.path.dirname(__file__) - sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) - del base_dir +BASE_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__), '../../..') +) + +if BASE_DIR not in sys.path: + sys.path.insert(0, BASE_DIR) import ansible_mitogen.loaders import ansible_mitogen.strategy diff --git a/ansible_mitogen/plugins/strategy/mitogen_linear.py b/ansible_mitogen/plugins/strategy/mitogen_linear.py index a5ea2a3d..d995b67b 100644 --- a/ansible_mitogen/plugins/strategy/mitogen_linear.py +++ b/ansible_mitogen/plugins/strategy/mitogen_linear.py @@ -44,12 +44,12 @@ import sys # debuggers and isinstance() work predictably. # -try: - import ansible_mitogen -except ImportError: - base_dir = os.path.dirname(__file__) - sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) - del base_dir +BASE_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__), '../../..') +) + +if BASE_DIR not in sys.path: + sys.path.insert(0, BASE_DIR) import ansible_mitogen.loaders import ansible_mitogen.strategy From f4a66194e4387192539ff7228e91b7c764e8ceac Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 28 Jul 2018 12:37:26 -0700 Subject: [PATCH 008/212] ansible: Py3.x fixes for Ansible PR #41749 workaround. --- ansible_mitogen/target.py | 4 ++-- docs/changelog.rst | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index 47249f25..582bf85e 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -39,7 +39,6 @@ import functools import grp import json import logging -import new import operator import os import pwd @@ -50,6 +49,7 @@ import subprocess import sys import tempfile import traceback +import types import mitogen.core import mitogen.fork @@ -61,7 +61,7 @@ import mitogen.service # we must setup a fake "__main__" before that module is ever imported. The # str() is to cast Unicode to bytes on Python 2.6. if not sys.modules.get(str('__main__')): - sys.modules[str('__main__')] = new.module(str('__main__')) + sys.modules[str('__main__')] = types.ModuleType(str('__main__')) import ansible.module_utils.json_utils import ansible_mitogen.runner diff --git a/docs/changelog.rst b/docs/changelog.rst index 2efdb035..6588629b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -24,6 +24,8 @@ Release Notes supported under Ansible 2.6. Contributed by `Dan Quackenbush `_. + * Compatible with development versions of Ansible post https://github.com/ansible/ansible/pull/41749 + v0.2.2 (2018-07-26) ------------------- From 3e0de9790c16297daf33cb0969246a34eeccc040 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 28 Jul 2018 14:09:21 -0700 Subject: [PATCH 009/212] issue #324: fix Python 3 fallout for custom module_utils. Also enable at last one of its tests. --- ansible_mitogen/runner.py | 4 ++-- tests/ansible/ansible.cfg | 2 +- tests/ansible/integration/all.yml | 2 +- tests/ansible/integration/module_utils/all.yml | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index ca3928b3..86f7b329 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -264,9 +264,9 @@ class ModuleUtilsImporter(object): mod.__loader__ = self if is_pkg: mod.__path__ = [] - mod.__package__ = fullname + mod.__package__ = str(fullname) else: - mod.__package__ = fullname.rpartition('.')[0] + mod.__package__ = str(fullname.rpartition('.')[0]) exec(code, mod.__dict__) self._loaded.add(fullname) return mod diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index 7bf849d5..d9224ab7 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -7,7 +7,7 @@ callback_plugins = lib/callback stdout_callback = nice_stdout vars_plugins = lib/vars library = lib/modules -# module_utils = lib/module_utils +module_utils = lib/module_utils retry_files_enabled = False forks = 50 diff --git a/tests/ansible/integration/all.yml b/tests/ansible/integration/all.yml index 264ae716..bf534aed 100644 --- a/tests/ansible/integration/all.yml +++ b/tests/ansible/integration/all.yml @@ -9,7 +9,7 @@ - import_playbook: connection_loader/all.yml - import_playbook: context_service/all.yml - import_playbook: local/all.yml -#- import_playbook: module_utils/all.yml +- import_playbook: module_utils/all.yml - import_playbook: playbook_semantics/all.yml - import_playbook: remote_tmp/all.yml - import_playbook: runner/all.yml diff --git a/tests/ansible/integration/module_utils/all.yml b/tests/ansible/integration/module_utils/all.yml index 920b5d1c..c8b8f2fb 100644 --- a/tests/ansible/integration/module_utils/all.yml +++ b/tests/ansible/integration/module_utils/all.yml @@ -1,6 +1,6 @@ -- import_playbook: from_config_path.yml -- import_playbook: from_config_path_pkg.yml -- import_playbook: adjacent_to_playbook.yml +#- import_playbook: from_config_path.yml +#- import_playbook: from_config_path_pkg.yml +#- import_playbook: adjacent_to_playbook.yml - import_playbook: adjacent_to_role.yml -- import_playbook: overrides_builtin.yml +#- import_playbook: overrides_builtin.yml From af2ded663da2fcbc4ce813f7aafbb64fc7906f9c Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 29 Jul 2018 19:21:42 -0700 Subject: [PATCH 010/212] fork: public on_fork() function. Generally useful, and needed for ongoing Ansible work. --- mitogen/fork.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/mitogen/fork.py b/mitogen/fork.py index 12bb7dfa..cf769788 100644 --- a/mitogen/fork.py +++ b/mitogen/fork.py @@ -75,6 +75,17 @@ def reset_logging_framework(): ] +def on_fork(): + """ + Should be called by any program integrating Mitogen each time the process + is forked, in the context of the new child. + """ + reset_logging_framework() # Must be first! + fixup_prngs() + mitogen.core.Latch._on_fork() + mitogen.core.Side._on_fork() + + def handle_child_crash(): """ Respond to _child_main() crashing by ensuring the relevant exception is @@ -134,10 +145,7 @@ class Stream(mitogen.parent.Stream): handle_child_crash() def _child_main(self, childfp): - reset_logging_framework() # Must be first! - fixup_prngs() - mitogen.core.Latch._on_fork() - mitogen.core.Side._on_fork() + on_fork() if self.on_fork: self.on_fork() mitogen.core.set_block(childfp.fileno()) From a05835c46e0281d5e8fd1a0a307eaab344a47350 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 29 Jul 2018 19:24:01 -0700 Subject: [PATCH 011/212] tests: more stable roundtrip.py. --- tests/bench/roundtrip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bench/roundtrip.py b/tests/bench/roundtrip.py index 40582d46..13b9413d 100644 --- a/tests/bench/roundtrip.py +++ b/tests/bench/roundtrip.py @@ -12,6 +12,6 @@ def do_nothing(): def main(router): f = router.fork() t0 = time.time() - for x in xrange(1000): + for x in xrange(10000): f.call(do_nothing) print '++', int(1e6 * ((time.time() - t0) / (1.0+x))), 'usec' From d62e6e2a7fc4642cb95ddbe81438bc3e37cffd9d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 31 Jul 2018 13:05:07 -0700 Subject: [PATCH 012/212] ansible: serialize calls to ModuleDepService. Concurrent calls to ModuleDepService would cause significant wasted work, as potentially all pool threads run the same uncached module dep scan. Without: 3243581 function calls (3233009 primitive calls) in 4770.672 seconds ncalls tottime percall cumtime percall filename:lineno(function) 2523 0.011 0.000 39.849 0.016 services.py:409(scan) With: 2801561 function calls (2800042 primitive calls) in 5166.843 seconds ncalls tottime percall cumtime percall filename:lineno(function) 2506 0.009 0.000 1.967 0.001 services.py:411(scan) Ignore timing variance due to problems with the test job. --- ansible_mitogen/services.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index a7bb7db1..e95fc226 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -388,6 +388,8 @@ class ModuleDepService(mitogen.service.Service): Scan a new-style module and produce a cached mapping of module_utils names to their resolved filesystem paths. """ + invoker_class = mitogen.service.SerializedInvoker + def __init__(self, *args, **kwargs): super(ModuleDepService, self).__init__(*args, **kwargs) self._cache = {} From 5c573f7fcb7c4f6369872aa907b4a65a5eb446b6 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 31 Jul 2018 23:27:36 -0700 Subject: [PATCH 013/212] ansible: insert short sleep when MITOGEN_PROFILING active. Hacky, but works fine. --- ansible_mitogen/process.py | 10 ++++++++++ mitogen/core.py | 1 + 2 files changed, 11 insertions(+) diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index f19079ee..4724ca93 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -33,6 +33,7 @@ import os import signal import socket import sys +import time import mitogen import mitogen.core @@ -114,6 +115,9 @@ class MuxProcess(object): mitogen.core.set_cloexec(cls.worker_sock.fileno()) mitogen.core.set_cloexec(cls.child_sock.fileno()) + if os.environ.get('MITOGEN_PROFILING', '1'): + mitogen.core.enable_profiling() + cls.original_env = dict(os.environ) cls.child_pid = os.fork() ansible_mitogen.logging.setup() @@ -199,4 +203,10 @@ class MuxProcess(object): ourself. In future this should gracefully join the pool, but TERM is fine for now. """ + if os.environ.get('MITOGEN_PROFILING'): + # TODO: avoid killing pool threads before they have written their + # .pstats. Really shouldn't be using kill() here at all, but hard + # to guarantee services can always be unblocked during shutdown. + time.sleep(1) + os.kill(os.getpid(), signal.SIGTERM) diff --git a/mitogen/core.py b/mitogen/core.py index dd706311..cfcd70d2 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -350,6 +350,7 @@ def enable_profiling(): try: return func(*args) finally: + profiler.dump_stats('/tmp/mitogen.%d.%s.pstat' % (os.getpid(), name)) profiler.create_stats() fp = open('/tmp/mitogen.stats.%d.%s.log' % (os.getpid(), name), 'w') try: From 898c06f1b9f1417b9f7c18465bee78eda7df2ec0 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 5 Aug 2018 10:51:26 +0100 Subject: [PATCH 014/212] docs: host demo on Vimeo. --- docs/ansible.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/ansible.rst b/docs/ansible.rst index 919c6380..5bec89e7 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -91,9 +91,7 @@ concurrent to an equivalent run using the extension. .. raw:: html - + Testimonials From 4077182fb2d05aba4cf20969b7956c031052c1a5 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 5 Aug 2018 00:27:43 +0100 Subject: [PATCH 015/212] ansible: plugins were missing absolute_import. --- ansible_mitogen/plugins/connection/mitogen_doas.py | 1 + ansible_mitogen/plugins/connection/mitogen_docker.py | 1 + ansible_mitogen/plugins/connection/mitogen_jail.py | 1 + ansible_mitogen/plugins/connection/mitogen_local.py | 1 + ansible_mitogen/plugins/connection/mitogen_lxc.py | 1 + ansible_mitogen/plugins/connection/mitogen_lxd.py | 1 + ansible_mitogen/plugins/connection/mitogen_machinectl.py | 1 + ansible_mitogen/plugins/connection/mitogen_setns.py | 1 + ansible_mitogen/plugins/connection/mitogen_ssh.py | 1 + ansible_mitogen/plugins/connection/mitogen_su.py | 1 + ansible_mitogen/plugins/connection/mitogen_sudo.py | 1 + ansible_mitogen/plugins/strategy/mitogen.py | 1 + ansible_mitogen/plugins/strategy/mitogen_free.py | 1 + ansible_mitogen/plugins/strategy/mitogen_linear.py | 1 + 14 files changed, 14 insertions(+) diff --git a/ansible_mitogen/plugins/connection/mitogen_doas.py b/ansible_mitogen/plugins/connection/mitogen_doas.py index 7d60b482..873b0d9d 100644 --- a/ansible_mitogen/plugins/connection/mitogen_doas.py +++ b/ansible_mitogen/plugins/connection/mitogen_doas.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_docker.py b/ansible_mitogen/plugins/connection/mitogen_docker.py index a98273e0..8af42711 100644 --- a/ansible_mitogen/plugins/connection/mitogen_docker.py +++ b/ansible_mitogen/plugins/connection/mitogen_docker.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_jail.py b/ansible_mitogen/plugins/connection/mitogen_jail.py index 1c57bb38..fb7bce54 100644 --- a/ansible_mitogen/plugins/connection/mitogen_jail.py +++ b/ansible_mitogen/plugins/connection/mitogen_jail.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_local.py b/ansible_mitogen/plugins/connection/mitogen_local.py index 35504d4d..fcd9c030 100644 --- a/ansible_mitogen/plugins/connection/mitogen_local.py +++ b/ansible_mitogen/plugins/connection/mitogen_local.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_lxc.py b/ansible_mitogen/plugins/connection/mitogen_lxc.py index 2195aa3c..ce394102 100644 --- a/ansible_mitogen/plugins/connection/mitogen_lxc.py +++ b/ansible_mitogen/plugins/connection/mitogen_lxc.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_lxd.py b/ansible_mitogen/plugins/connection/mitogen_lxd.py index 5d1391b9..77efe6c1 100644 --- a/ansible_mitogen/plugins/connection/mitogen_lxd.py +++ b/ansible_mitogen/plugins/connection/mitogen_lxd.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_machinectl.py b/ansible_mitogen/plugins/connection/mitogen_machinectl.py index e71496a3..9b332a3f 100644 --- a/ansible_mitogen/plugins/connection/mitogen_machinectl.py +++ b/ansible_mitogen/plugins/connection/mitogen_machinectl.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_setns.py b/ansible_mitogen/plugins/connection/mitogen_setns.py index 5f131655..23f62135 100644 --- a/ansible_mitogen/plugins/connection/mitogen_setns.py +++ b/ansible_mitogen/plugins/connection/mitogen_setns.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_ssh.py b/ansible_mitogen/plugins/connection/mitogen_ssh.py index c0c577c3..d2af109c 100644 --- a/ansible_mitogen/plugins/connection/mitogen_ssh.py +++ b/ansible_mitogen/plugins/connection/mitogen_ssh.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_su.py b/ansible_mitogen/plugins/connection/mitogen_su.py index fd09d0f0..104a7190 100644 --- a/ansible_mitogen/plugins/connection/mitogen_su.py +++ b/ansible_mitogen/plugins/connection/mitogen_su.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/connection/mitogen_sudo.py b/ansible_mitogen/plugins/connection/mitogen_sudo.py index a6cb8bd2..367dd61b 100644 --- a/ansible_mitogen/plugins/connection/mitogen_sudo.py +++ b/ansible_mitogen/plugins/connection/mitogen_sudo.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/strategy/mitogen.py b/ansible_mitogen/plugins/strategy/mitogen.py index 4f595161..f8608745 100644 --- a/ansible_mitogen/plugins/strategy/mitogen.py +++ b/ansible_mitogen/plugins/strategy/mitogen.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/strategy/mitogen_free.py b/ansible_mitogen/plugins/strategy/mitogen_free.py index 8dfaa16e..d3b1cdc6 100644 --- a/ansible_mitogen/plugins/strategy/mitogen_free.py +++ b/ansible_mitogen/plugins/strategy/mitogen_free.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys diff --git a/ansible_mitogen/plugins/strategy/mitogen_linear.py b/ansible_mitogen/plugins/strategy/mitogen_linear.py index d995b67b..51b03096 100644 --- a/ansible_mitogen/plugins/strategy/mitogen_linear.py +++ b/ansible_mitogen/plugins/strategy/mitogen_linear.py @@ -26,6 +26,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import import os.path import sys From 81c8156965420449d9978e2b778a5fdd11c97041 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 10 Aug 2018 08:43:13 +0100 Subject: [PATCH 016/212] Support LXD; closes #339. --- docs/ansible.rst | 38 +++++++++++++++++-------- docs/api.rst | 30 +++++++++++++++----- mitogen/core.py | 1 + mitogen/lxd.py | 70 +++++++++++++++++++++++++++++++++++++++++++++++ mitogen/parent.py | 3 ++ mitogen/setns.py | 18 +++++++++++- 6 files changed, 141 insertions(+), 19 deletions(-) create mode 100644 mitogen/lxd.py diff --git a/docs/ansible.rst b/docs/ansible.rst index 5bec89e7..a6130873 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -569,10 +569,10 @@ additional differences exist that may break existing playbooks. LXC ~~~ -Like `lxc `_ -and `lxd `_ -except connection delegation is supported, and ``lxc-attach`` is always used -rather than the LXC Python bindings, as is usual with ``lxc``. +Connect to classic LXC containers, like `lxc +`_ except +connection delegation is supported, and ``lxc-attach`` is always used rather +than the LXC Python bindings, as is usual with ``lxc``. The ``lxc-attach`` command must be available on the host machine. @@ -580,6 +580,20 @@ The ``lxc-attach`` command must be available on the host machine. * ``ansible_host``: Name of LXC container (default: inventory hostname). +.. _method-lxd: + +LXD +~~~ + +Connect to modern LXD containers, like `lxd +`_ except +connection delegation is supported. The ``lxc`` command must be available on +the host machine. + +* ``ansible_python_interpreter`` +* ``ansible_host``: Name of LXC container (default: inventory hostname). + + .. _machinectl: Machinectl @@ -602,21 +616,23 @@ Setns ~~~~~ The ``setns`` method connects to Linux containers via `setns(2) -`_. Unlike :ref:`method-docker` and -:ref:`method-lxc` the namespace transition is handled internally, ensuring -optimal throughput to the child. This is necessary for :ref:`machinectl` where -only PTY channels are supported. +`_. Unlike :ref:`method-docker`, +:ref:`method-lxc`, and :ref:`method-lxd` the namespace transition is handled +internally, ensuring optimal throughput to the child. This is necessary for +:ref:`machinectl` where only PTY channels are supported. A utility program must be installed to discover the PID of the container's root process. -* ``mitogen_kind``: one of ``docker``, ``lxc`` or ``machinectl``. +* ``mitogen_kind``: one of ``docker``, ``lxc``, ``lxd`` or ``machinectl``. * ``ansible_host``: Name of container as it is known to the corresponding tool (default: inventory hostname). * ``ansible_user``: Name of user within the container to execute as. * ``mitogen_docker_path``: path to Docker if not available on the system path. -* ``mitogen_lxc_info_path``: path to ``lxc-info`` command if not available as - ``/usr/bin/lxc-info``. +* ``mitogen_lxc_path``: path to LXD's ``lxc`` command if not available as + ``lxc-info``. +* ``mitogen_lxc_info_path``: path to LXC classic's ``lxc-info`` command if not + available as ``lxc-info``. * ``mitogen_machinectl_path``: path to ``machinectl`` command if not available as ``/bin/machinectl``. diff --git a/docs/api.rst b/docs/api.rst index 92abe1ec..ed9509fc 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -590,8 +590,8 @@ Router Class .. method:: lxc (container, lxc_attach_path=None, \**kwargs) - Construct a context on the local machine within an LXC container using - the ``lxc-attach`` program. + Construct a context on the local machine within an LXC classic + container using the ``lxc-attach`` program. Accepts all parameters accepted by :py:meth:`local`, in addition to: @@ -602,6 +602,19 @@ Router Class will be searched if given as a filename. Defaults to ``lxc-attach``. + .. method:: lxc (container, lxc_attach_path=None, \**kwargs) + + Construct a context on the local machine within a LXD container using + the ``lxc`` program. + + Accepts all parameters accepted by :py:meth:`local`, in addition to: + + :param str container: + Existing container to connect to. Defaults to ``None``. + :param str lxc_path: + Filename or complete path to the ``lxc`` binary. ``PATH`` will be + searched if given as a filename. Defaults to ``lxc``. + .. method:: setns (container, kind, docker_path=None, lxc_info_path=None, machinectl_path=None, \**kwargs) Construct a context in the style of :meth:`local`, but change the @@ -609,7 +622,8 @@ Router Class executing Python. The namespaces to use, and the active root file system are taken from - the root PID of a running Docker, LXC, or systemd-nspawn container. + the root PID of a running Docker, LXC, LXD, or systemd-nspawn + container. A program is required only to find the root PID, after which management of the child Python interpreter is handled directly. @@ -617,14 +631,16 @@ Router Class :param str container: Container to connect to. :param str kind: - One of ``docker``, ``lxc`` or ``machinectl``. + One of ``docker``, ``lxc``, ``lxd`` or ``machinectl``. :param str docker_path: Filename or complete path to the Docker binary. ``PATH`` will be searched if given as a filename. Defaults to ``docker``. + :param str lxc_path: + Filename or complete path to the LXD ``lxc`` binary. ``PATH`` will + be searched if given as a filename. Defaults to ``lxc``. :param str lxc_info_path: - Filename or complete path to the ``lxc-info`` binary. ``PATH`` - will be searched if given as a filename. Defaults to - ``lxc-info``. + Filename or complete path to the LXC ``lxc-info`` binary. ``PATH`` + will be searched if given as a filename. Defaults to ``lxc-info``. :param str machinectl_path: Filename or complete path to the ``machinectl`` binary. ``PATH`` will be searched if given as a filename. Defaults to diff --git a/mitogen/core.py b/mitogen/core.py index cfcd70d2..261f2621 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -614,6 +614,7 @@ class Importer(object): 'fork', 'jail', 'lxc', + 'lxd', 'master', 'minify', 'parent', diff --git a/mitogen/lxd.py b/mitogen/lxd.py new file mode 100644 index 00000000..6e8e8b18 --- /dev/null +++ b/mitogen/lxd.py @@ -0,0 +1,70 @@ +# Copyright 2017, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +import logging + +import mitogen.core +import mitogen.parent + + +LOG = logging.getLogger(__name__) + + +class Stream(mitogen.parent.Stream): + child_is_immediate_subprocess = False + create_child_args = { + # If lxc finds any of stdin, stdout, stderr connected to a TTY, to + # prevent input injection it creates a proxy pty, forcing all IO to be + # buffered in <4KiB chunks. So ensure stderr is also routed to the + # socketpair. + 'merge_stdio': True + } + + container = None + lxc_path = 'lxc' + python_path = 'python' + + def construct(self, container, lxc_path=None, **kwargs): + super(Stream, self).construct(**kwargs) + self.container = container + if lxc_path: + self.lxc_path = lxc_path + + def connect(self): + super(Stream, self).connect() + self.name = u'lxd.' + self.container + + def get_boot_command(self): + bits = [ + self.lxc_path, + 'exec', + '--force-noninteractive', + self.container, + '--', + ] + return bits + super(Stream, self).get_boot_command() diff --git a/mitogen/parent.py b/mitogen/parent.py index 4299d3cd..14e0ef6d 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1288,6 +1288,9 @@ class Router(mitogen.core.Router): def lxc(self, **kwargs): return self.connect(u'lxc', **kwargs) + def lxd(self, **kwargs): + return self.connect(u'lxd', **kwargs) + def setns(self, **kwargs): return self.connect(u'setns', **kwargs) diff --git a/mitogen/setns.py b/mitogen/setns.py index 1779ca77..224550ce 100644 --- a/mitogen/setns.py +++ b/mitogen/setns.py @@ -94,6 +94,16 @@ def get_lxc_pid(path, name): raise Error("could not find PID from lxc-info output.\n%s", output) +def get_lxd_pid(path, name): + output = _run_command([path, 'info', name]) + for line in output.splitlines(): + bits = line.split() + if bits and bits[0] == 'Pid:': + return int(bits[1]) + + raise Error("could not find PID from lxc output.\n%s", output) + + def get_machinectl_pid(path, name): output = _run_command([path, 'status', name]) for line in output.splitlines(): @@ -110,18 +120,22 @@ class Stream(mitogen.parent.Stream): container = None username = None kind = None + python_path = 'python' docker_path = 'docker' + lxc_path = 'lxc' lxc_info_path = 'lxc-info' machinectl_path = 'machinectl' GET_LEADER_BY_KIND = { 'docker': ('docker_path', get_docker_pid), 'lxc': ('lxc_info_path', get_lxc_pid), + 'lxd': ('lxc_path', get_lxd_pid), 'machinectl': ('machinectl_path', get_machinectl_pid), } def construct(self, container, kind, username=None, docker_path=None, - lxc_info_path=None, machinectl_path=None, **kwargs): + lxc_path=None, lxc_info_path=None, machinectl_path=None, + **kwargs): super(Stream, self).construct(**kwargs) if kind not in self.GET_LEADER_BY_KIND: raise Error('unsupported container kind: %r', kind) @@ -132,6 +146,8 @@ class Stream(mitogen.parent.Stream): self.username = username if docker_path: self.docker_path = docker_path + if lxc_path: + self.lxc_path = lxc_path if lxc_info_path: self.lxc_info_path = lxc_info_path if machinectl_path: From 1473f495059e4c6fc0b049267b392cac6bd976c0 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 10 Aug 2018 10:06:57 +0100 Subject: [PATCH 017/212] ansible: emulate /etc/environment reloading behaviour of vanilla. This change is relatively incomplete -- ideally we could snapshot os.environ and /etc/environment at startup and respect key deletions too, but that's a lot more work. Wait for a bug report instead. Closes #338. --- ansible_mitogen/runner.py | 79 +++++++++++++++++++ tests/ansible/integration/runner/all.yml | 3 +- .../integration/runner/etc_environment.yml | 66 ++++++++++++++++ 3 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 tests/ansible/integration/runner/etc_environment.yml diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 86f7b329..1cc02198 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -44,6 +44,7 @@ import imp import json import logging import os +import shlex import sys import tempfile import types @@ -78,6 +79,22 @@ for symbol in 'res_init', '__res_init': except AttributeError: pass +# For tasks running on Linux machines, with vanilla Ansible, edits to +# /etc/environment and ~/.pam_environment are reflected if become:true, due to +# sudo reinvoking pam_env. If multiplexing is disabled, then edits are also +# reflected with become:false. Rather than emulate existing semantics, simply +# always ensure edits are reflects for the next task. +try: + etc_env_st = os.stat('/etc/environment') +except OSError: + etc_env_st = None + +try: + pam_env_st = os.stat(os.path.expanduser('~/.pam_environment')) +except OSError: + pam_env_st = None + + iteritems = getattr(dict, 'iteritems', dict.items) LOG = logging.getLogger(__name__) @@ -104,6 +121,54 @@ def reopen_readonly(fp): os.close(fd) +def parse_env(fp): + """ + Parse /etc/environ using roughly the same syntax as pam_env. + """ + # https://github.com/linux-pam/linux-pam/blob/v1.3.1/modules/pam_env/pam_env.c#L207 + for line in fp: + # ' #export foo=some var ' -> ['#export', 'foo=some var '] + bits = shlex.split(line, comments=True) + if not bits: + continue + + if bits[0] == 'export': + bits.pop(0) + + key, sep, value = (' '.join(bits)).partition('=') + if sep: + os.environ[key] = value + + +def reload_env(old_st, path): + """ + Compare the :func:`os.stat` for the pam_env style environmnt file `path` + with the previous result `old_st`, which may be :data:`None` if the + previous stat attempt failed. Reload its contents if the file has changed + or appeared since last attempt. + + :returns: + New :func:`os.stat` result. The new call to :func:`reload_env` should + pass it as the value of `old_st`. + """ + try: + path = os.path.expanduser(path) + st = os.stat(path) + except OSError: + return None + + if old_st == st: + return old_st + if st is None: + LOG.debug('reload_env(%r): file has disappeared', path) + return st + + LOG.debug('reload_env(%r): file has changed or appeared, reloading', path) + with open(path) as fp: + parse_env(fp) + return st + + class Runner(object): """ Ansible module runner. After instantiation (with kwargs supplied by the @@ -163,8 +228,22 @@ class Runner(object): env = dict(self.extra_env or {}) if self.env: env.update(self.env) + self._setup_environ() self._env = TemporaryEnvironment(env) + def _setup_environ(self): + """ + Ensure /etc/environment and ~/.pam_environment are reloaded if their + content appears to differ since execution of the previous task. This + must happen before TemporaryEnvironment is installed, to ensure changes + persist across tasks. + """ + global etc_env_st + etc_env_st = reload_env(etc_env_st, '/etc/environment') + + global pam_env_st + pam_env_st = reload_env(pam_env_st, '~/.pam_environment') + def revert(self): """ Revert any changes made to the process after running a module. The base diff --git a/tests/ansible/integration/runner/all.yml b/tests/ansible/integration/runner/all.yml index 5242a405..8f4f3426 100644 --- a/tests/ansible/integration/runner/all.yml +++ b/tests/ansible/integration/runner/all.yml @@ -1,7 +1,7 @@ - import_playbook: builtin_command_module.yml +- import_playbook: custom_bash_hashbang_argument.yml - import_playbook: custom_bash_old_style_module.yml - import_playbook: custom_bash_want_json_module.yml -- import_playbook: custom_bash_hashbang_argument.yml - import_playbook: custom_binary_producing_json.yml - import_playbook: custom_binary_producing_junk.yml - import_playbook: custom_binary_single_null.yml @@ -13,4 +13,5 @@ - import_playbook: custom_python_want_json_module.yml - import_playbook: custom_script_interpreter.yml - import_playbook: environment_isolation.yml +- import_playbook: etc_environment.yml - import_playbook: forking_behaviour.yml diff --git a/tests/ansible/integration/runner/etc_environment.yml b/tests/ansible/integration/runner/etc_environment.yml new file mode 100644 index 00000000..4c7f64d8 --- /dev/null +++ b/tests/ansible/integration/runner/etc_environment.yml @@ -0,0 +1,66 @@ +# issue #338: ensure /etc/environment is reloaded if it changes. +# Actually this test uses ~/.pam_environment, which is using the same logic, +# but less likely to brick a development workstation + +- name: integration/runner/etc_environment.yml + hosts: test-targets + any_errors_fatal: true + gather_facts: true + tasks: + - meta: end_play + when: ansible_virtualization_type != "docker" + + + # ~/.pam_environment + + - file: + path: ~/.pam_environment + state: absent + + - shell: echo $MAGIC_NEW_ENV + register: echo + + - assert: + that: echo.stdout == "" + + - copy: + dest: ~/.pam_environment + content: | + MAGIC_NEW_ENV=321 + + - shell: echo $MAGIC_NEW_ENV + register: echo + + - assert: + that: echo.stdout == "321" + + - file: + path: ~/.pam_environment + state: absent + + # /etc/environment + + - file: + path: /etc/environment + state: absent + + - shell: echo $MAGIC_ETC_ENV + register: echo + + - assert: + that: echo.stdout == "" + + - copy: + dest: /etc/environment + content: | + MAGIC_ETC_ENV=555 + + - shell: echo $MAGIC_ENV_ENV + register: echo + + - assert: + that: echo.stdout == "555" + + - file: + path: /etc/environment + state: absent From b964e647d27e4b412bdbd76c310a1d9fb2211dd7 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 10 Aug 2018 14:07:23 +0100 Subject: [PATCH 018/212] tests: split inventory up slightly. This makes it easier to run connection delegation tests against either the local machine or a container. --- .travis/ansible_tests.sh | 6 ++++-- tests/ansible/README.md | 15 +++++++++++++++ tests/ansible/{hosts => common-hosts} | 7 +++---- tests/ansible/hosts/common-hosts | 1 + tests/ansible/hosts/localhost | 2 ++ 5 files changed, 25 insertions(+), 6 deletions(-) rename tests/ansible/{hosts => common-hosts} (81%) create mode 120000 tests/ansible/hosts/common-hosts create mode 100644 tests/ansible/hosts/localhost diff --git a/.travis/ansible_tests.sh b/.travis/ansible_tests.sh index e6441343..bc119149 100755 --- a/.travis/ansible_tests.sh +++ b/.travis/ansible_tests.sh @@ -40,14 +40,16 @@ pip install ansible=="${ANSIBLE_VERSION}" cd ${TRAVIS_BUILD_DIR}/tests/ansible chmod go= ${TRAVIS_BUILD_DIR}/tests/data/docker/mitogen__has_sudo_pubkey.key -echo '[test-targets]' > ${TMPDIR}/hosts +mkdir ${TMPDIR}/hosts +ln -s ${TRAVIS_BUILD_DIR}/tests/ansible/common-hosts ${TMPDIR}/hosts/common-hosts +echo '[test-targets]' > ${TMPDIR}/hosts/target echo \ target \ ansible_host=$DOCKER_HOSTNAME \ ansible_port=2201 \ ansible_user=mitogen__has_sudo_nopw \ ansible_password=has_sudo_nopw_password \ - >> ${TMPDIR}/hosts + >> ${TMPDIR}/hosts/target # Build the binaries. make -C ${TRAVIS_BUILD_DIR}/tests/ansible diff --git a/tests/ansible/README.md b/tests/ansible/README.md index a76c7c1f..8051ef85 100644 --- a/tests/ansible/README.md +++ b/tests/ansible/README.md @@ -25,3 +25,18 @@ environment before the Mitogen connection process forks. ``` ANSIBLE_STRATEGY=mitogen_linear ./run_ansible_playbook.sh all.yml ``` + + +## ``hosts/`` and ``common-hosts`` + +To support running the tests against a dev machine that has the requisite user +accounts, the the default inventory is a directory containing a 'localhost' +file that defines 'localhost' to be named 'target' in Ansible inventory, and a +symlink to 'common-hosts', which defines additional targets that all derive +from 'target'. + +This allows ``ansible_tests.sh`` to reuse the common-hosts definitions while +replacing localhost as the test target by creating a new directory that +similarly symlinks in common-hosts. + +There may be a better solution for this, but it works fine for now. diff --git a/tests/ansible/hosts b/tests/ansible/common-hosts similarity index 81% rename from tests/ansible/hosts rename to tests/ansible/common-hosts index 45bfb9ef..0aead1be 100644 --- a/tests/ansible/hosts +++ b/tests/ansible/common-hosts @@ -1,7 +1,3 @@ - -[test-targets] -localhost - [connection-delegation-test] cd-bastion cd-rack11 mitogen_via=ssh-user@cd-bastion @@ -14,3 +10,6 @@ cdc-bastion mitogen_via=cdc-rack11a-docker cdc-rack11 mitogen_via=ssh-user@cdc-bastion cdc-rack11a mitogen_via=root@cdc-rack11 cdc-rack11a-docker mitogen_via=docker-admin@cdc-rack11a ansible_connection=docker + +[conn-delegation] +cd-user1 ansible_user=mitogen__user1 ansible_connection=mitogen_sudo mitogen_via=target diff --git a/tests/ansible/hosts/common-hosts b/tests/ansible/hosts/common-hosts new file mode 120000 index 00000000..f3cc7f59 --- /dev/null +++ b/tests/ansible/hosts/common-hosts @@ -0,0 +1 @@ +../common-hosts \ No newline at end of file diff --git a/tests/ansible/hosts/localhost b/tests/ansible/hosts/localhost new file mode 100644 index 00000000..6e022a01 --- /dev/null +++ b/tests/ansible/hosts/localhost @@ -0,0 +1,2 @@ +[test-targets] +target ansible_hostname=localhost From 34a9f6711529952ff762ef2582c61fee74821d4a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 10 Aug 2018 14:21:10 +0100 Subject: [PATCH 019/212] issue #339: whoops, actually wire up new connection method. --- ansible_mitogen/connection.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index c45a8aa7..c0a17c17 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -126,6 +126,17 @@ def _connect_lxc(spec): } +def _connect_lxd(spec): + return { + 'method': 'lxd', + 'kwargs': { + 'container': spec['remote_addr'], + 'python_path': spec['python_path'], + 'connect_timeout': spec['ansible_ssh_timeout'] or spec['timeout'], + } + } + + def _connect_machinectl(spec): return _connect_setns(dict(spec, mitogen_kind='machinectl')) @@ -236,7 +247,7 @@ CONNECTION_METHOD = { 'jail': _connect_jail, 'local': _connect_local, 'lxc': _connect_lxc, - 'lxd': _connect_lxc, + 'lxd': _connect_lxd, 'machinectl': _connect_machinectl, 'setns': _connect_setns, 'ssh': _connect_ssh, From 6c4b01642c15978db9544adb701cee688469b7b9 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 10 Aug 2018 15:29:42 +0100 Subject: [PATCH 020/212] ansible: don't crash when adhoc tries to run a missing module. ansible-playbook prints a separate error during parsing stage, adhoc performs no such check. --- ansible_mitogen/planner.py | 4 ++++ tests/ansible/integration/runner/all.yml | 1 + .../ansible/integration/runner/missing_module.yml | 15 +++++++++++++++ 3 files changed, 20 insertions(+) create mode 100644 tests/ansible/integration/runner/missing_module.yml diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index c297ad8f..8ebf4f67 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -55,6 +55,7 @@ import ansible_mitogen.target LOG = logging.getLogger(__name__) NO_METHOD_MSG = 'Mitogen: no invocation method found for: ' NO_INTERPRETER_MSG = 'module (%s) is missing interpreter line' +NO_MODULE_MSG = 'The module %s was not found in configured module paths.' class Invocation(object): @@ -393,6 +394,9 @@ _planners = [ def get_module_data(name): path = ansible_mitogen.loaders.module_loader.find_plugin(name, '') + if path is None: + raise ansible.errors.AnsibleError(NO_MODULE_MSG % (name,)) + with open(path, 'rb') as fp: source = fp.read() return mitogen.core.to_text(path), source diff --git a/tests/ansible/integration/runner/all.yml b/tests/ansible/integration/runner/all.yml index 8f4f3426..ffb263fb 100644 --- a/tests/ansible/integration/runner/all.yml +++ b/tests/ansible/integration/runner/all.yml @@ -15,3 +15,4 @@ - import_playbook: environment_isolation.yml - import_playbook: etc_environment.yml - import_playbook: forking_behaviour.yml +- import_playbook: missing_module.yml diff --git a/tests/ansible/integration/runner/missing_module.yml b/tests/ansible/integration/runner/missing_module.yml new file mode 100644 index 00000000..ac72f69e --- /dev/null +++ b/tests/ansible/integration/runner/missing_module.yml @@ -0,0 +1,15 @@ + +- name: integration/runner/missing_module.yml + hosts: test-targets + connection: local + tasks: + - connection: local + command: ansible -i localhost, localhost -m missing_module + args: + chdir: ../.. + register: out + ignore_errors: true + + - assert: + that: | + 'The module missing_module was not found in configured module paths.' in out.stdout From 1f21a30e7fd2e9ba832294fc4541eb88a9328458 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 18 May 2018 19:00:57 +0100 Subject: [PATCH 021/212] issue #251: ansible: watch for delegate_to during connection delegation. This needs more work -- pretty certain that python_path and suchlike are coming from the wrong place. Possibly we need another config_from_..() specialized for delegate_to. --- ansible_mitogen/connection.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index c0a17c17..12616dbf 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -524,13 +524,23 @@ class Connection(ansible.plugins.connection.ConnectionBase): broker=self.broker, ) - stack, _ = self._stack_from_config( - config_from_play_context( + if hasattr(self._play_context, 'delegate_to'): + target_config = config_from_hostvars( + transport=self._play_context.connection, + inventory_name=self._play_context.delegate_to, + connection=self, + hostvars=self.host_vars[self._play_context.delegate_to], + become_user=(self._play_context.become_user + if self._play_context.become + else None), + ) + else: + target_config = config_from_play_context( transport=self.transport, inventory_name=self.inventory_hostname, connection=self ) - ) + stack, _ = self._stack_from_config(target_config) dct = self.parent.call_service( service_name='ansible_mitogen.services.ContextService', From 2c2fc73b0a523874d5056aef4e05fd386ed35bbb Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 10 Aug 2018 19:29:25 +0100 Subject: [PATCH 022/212] tests: whups, s/ansible_hostname/ansible_host/ --- tests/ansible/hosts/localhost | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ansible/hosts/localhost b/tests/ansible/hosts/localhost index 6e022a01..dc7df668 100644 --- a/tests/ansible/hosts/localhost +++ b/tests/ansible/hosts/localhost @@ -1,2 +1,2 @@ [test-targets] -target ansible_hostname=localhost +target ansible_host=localhost From 6f524d3ff80d366121694b44d3f082cb08a8c01b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 10 Aug 2018 20:18:41 +0100 Subject: [PATCH 023/212] issue #339: minimal tests for lxc/lxd modules. --- mitogen/lxc.py | 2 +- tests/data/fake_lxc.py | 7 +++++++ tests/data/fake_lxc_attach.py | 7 +++++++ tests/lxc_test.py | 29 +++++++++++++++++++++++++++++ tests/lxd_test.py | 26 ++++++++++++++++++++++++++ 5 files changed, 70 insertions(+), 1 deletion(-) create mode 100755 tests/data/fake_lxc.py create mode 100755 tests/data/fake_lxc_attach.py create mode 100644 tests/lxc_test.py create mode 100644 tests/lxd_test.py diff --git a/mitogen/lxc.py b/mitogen/lxc.py index 4d6c21db..71b12221 100644 --- a/mitogen/lxc.py +++ b/mitogen/lxc.py @@ -52,7 +52,7 @@ class Stream(mitogen.parent.Stream): super(Stream, self).construct(**kwargs) self.container = container if lxc_attach_path: - self.lxc_attach_path = lxc_attach_apth + self.lxc_attach_path = lxc_attach_path def connect(self): super(Stream, self).connect() diff --git a/tests/data/fake_lxc.py b/tests/data/fake_lxc.py new file mode 100755 index 00000000..2fedb961 --- /dev/null +++ b/tests/data/fake_lxc.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python + +import sys +import os + +os.environ['ORIGINAL_ARGV'] = repr(sys.argv) +os.execv(sys.executable, sys.argv[sys.argv.index('--') + 1:]) diff --git a/tests/data/fake_lxc_attach.py b/tests/data/fake_lxc_attach.py new file mode 100755 index 00000000..2fedb961 --- /dev/null +++ b/tests/data/fake_lxc_attach.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python + +import sys +import os + +os.environ['ORIGINAL_ARGV'] = repr(sys.argv) +os.execv(sys.executable, sys.argv[sys.argv.index('--') + 1:]) diff --git a/tests/lxc_test.py b/tests/lxc_test.py new file mode 100644 index 00000000..a30cd186 --- /dev/null +++ b/tests/lxc_test.py @@ -0,0 +1,29 @@ +import os + +import mitogen + +import unittest2 + +import testlib + + +def has_subseq(seq, subseq): + return any(seq[x:x+len(subseq)] == subseq for x in range(0, len(seq))) + + +class FakeLxcAttachTest(testlib.RouterMixin, unittest2.TestCase): + def test_okay(self): + lxc_attach_path = testlib.data_path('fake_lxc_attach.py') + context = self.router.lxc( + container='container_name', + lxc_attach_path=lxc_attach_path, + ) + + argv = eval(context.call(os.getenv, 'ORIGINAL_ARGV')) + self.assertEquals(argv[0], lxc_attach_path) + self.assertTrue('--clear-env' in argv) + self.assertTrue(has_subseq(argv, ['--name', 'container_name'])) + + +if __name__ == '__main__': + unittest2.main() diff --git a/tests/lxd_test.py b/tests/lxd_test.py new file mode 100644 index 00000000..c5e4c485 --- /dev/null +++ b/tests/lxd_test.py @@ -0,0 +1,26 @@ +import os + +import mitogen + +import unittest2 + +import testlib + + +class FakeLxcTest(testlib.RouterMixin, unittest2.TestCase): + def test_okay(self): + lxc_path = testlib.data_path('fake_lxc.py') + context = self.router.lxd( + container='container_name', + lxc_path=lxc_path, + ) + + argv = eval(context.call(os.getenv, 'ORIGINAL_ARGV')) + self.assertEquals(argv[0], lxc_path) + self.assertEquals(argv[1], 'exec') + self.assertEquals(argv[2], '--force-noninteractive') + self.assertEquals(argv[3], 'container_name') + + +if __name__ == '__main__': + unittest2.main() From 053c594d65f9651c97f83423a8bf27bf87c9bd7a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 11 Aug 2018 10:11:56 +0100 Subject: [PATCH 024/212] ansible: prevent logs spamming user console on exit. Closes #331. --- ansible_mitogen/process.py | 19 +++++++++++++++++++ docs/changelog.rst | 22 +++++++++++++++++----- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index 4724ca93..2f275dc0 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -27,6 +27,7 @@ # POSSIBILITY OF SUCH DAMAGE. from __future__ import absolute_import +import atexit import errno import logging import os @@ -53,6 +54,23 @@ from mitogen.core import b LOG = logging.getLogger(__name__) +def clean_shutdown(sock): + """ + Shut the write end of `sock`, causing `recv` in the worker process to wake + up with a 0-byte read and initiate mux process exit, then wait for a 0-byte + read from the read end, which will occur after the the child closes the + descriptor on exit. + + This is done using :mod:`atexit` since Ansible lacks any more sensible hook + to run code during exit, and unless some synchronization exists with + MuxProcess, debug logs may appear on the user's terminal *after* the prompt + has been printed. + """ + sock.shutdown(socket.SHUT_WR) + while sock.recv(1): + pass + + class MuxProcess(object): """ Implement a subprocess forked from the Ansible top-level, as a safe place @@ -112,6 +130,7 @@ class MuxProcess(object): cls.unix_listener_path = mitogen.unix.make_socket_path() cls.worker_sock, cls.child_sock = socket.socketpair() + atexit.register(lambda: clean_shutdown(cls.worker_sock)) mitogen.core.set_cloexec(cls.worker_sock.fileno()) mitogen.core.set_cloexec(cls.child_sock.fileno()) diff --git a/docs/changelog.rst b/docs/changelog.rst index fd098e6a..27571ad2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -27,6 +27,18 @@ Release Notes * Compatible with development versions of Ansible post https://github.com/ansible/ansible/pull/41749 +v0.2.3 (2018-08-??) +------------------- + +Mitogen for Ansible +~~~~~~~~~~~~~~~~~~~ + +* `#331 `_: fixed known issue: the + connection multiplexer subprocess always exits before the main Ansible + process exits, ensuring logs generated by it do not overwrite the user's + prompt when ``-vvv`` is enabled. + + v0.2.2 (2018-07-26) ------------------- @@ -204,11 +216,11 @@ Mitogen for Ansible for Message(..., 102, ...), my ID is ...* may be visible. These are due to a minor race while initializing logging and can be ignored. -* When running with ``-vvv``, log messages will be printed to the console - *after* the Ansible run completes, as connection multiplexer shutdown only - begins after Ansible exits. This is due to a lack of suitable shutdown hook - in Ansible, and is fairly harmless, albeit cosmetically annoying. A future - release may include a solution. +.. * When running with ``-vvv``, log messages will be printed to the console + *after* the Ansible run completes, as connection multiplexer shutdown only + begins after Ansible exits. This is due to a lack of suitable shutdown hook + in Ansible, and is fairly harmless, albeit cosmetically annoying. A future + release may include a solution. * Performance does not scale linearly with target count. This requires significant additional work, as major bottlenecks exist in the surrounding From a192935daf09e46ffc1f36b3d447336228145632 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 11 Aug 2018 13:53:07 +0100 Subject: [PATCH 025/212] tests: merge build_docker_images.py with osx_setup.yml Hooray! --- tests/ansible/README.md | 14 +- tests/ansible/osx_setup.yml | 155 ----------------------- tests/build_docker_images.py | 120 ------------------ tests/data/{ => docker}/001-mitogen.sudo | 5 + tests/image_prep/README.md | 25 ++++ tests/image_prep/_container_setup.yml | 116 +++++++++++++++++ tests/image_prep/_user_accounts.yml | 139 ++++++++++++++++++++ tests/image_prep/ansible.cfg | 4 + tests/image_prep/build_docker_images.py | 43 +++++++ tests/image_prep/setup.yml | 13 ++ 10 files changed, 351 insertions(+), 283 deletions(-) delete mode 100644 tests/ansible/osx_setup.yml delete mode 100755 tests/build_docker_images.py rename tests/data/{ => docker}/001-mitogen.sudo (68%) create mode 100644 tests/image_prep/README.md create mode 100644 tests/image_prep/_container_setup.yml create mode 100644 tests/image_prep/_user_accounts.yml create mode 100644 tests/image_prep/ansible.cfg create mode 100755 tests/image_prep/build_docker_images.py create mode 100644 tests/image_prep/setup.yml diff --git a/tests/ansible/README.md b/tests/ansible/README.md index 8051ef85..46320951 100644 --- a/tests/ansible/README.md +++ b/tests/ansible/README.md @@ -1,5 +1,5 @@ -# ``tests/ansible`` Directory +# `tests/ansible` Directory This is an an organically growing collection of integration and regression tests used for development and end-user bug reports. @@ -10,10 +10,10 @@ demonstrator for what does and doesn't work. ## Preparation -For OS X, run the ``osx_setup.yml`` script to create a bunch of users. +See `../image_prep/README.md`. -## ``run_ansible_playbook.sh`` +## `run_ansible_playbook.sh` This is necessary to set some environment variables used by future tests, as there appears to be no better way to inject them into the top-level process @@ -22,12 +22,10 @@ environment before the Mitogen connection process forks. ## Running Everything -``` -ANSIBLE_STRATEGY=mitogen_linear ./run_ansible_playbook.sh all.yml -``` +`ANSIBLE_STRATEGY=mitogen_linear ./run_ansible_playbook.sh all.yml` -## ``hosts/`` and ``common-hosts`` +## `hosts/` and `common-hosts` To support running the tests against a dev machine that has the requisite user accounts, the the default inventory is a directory containing a 'localhost' @@ -35,7 +33,7 @@ file that defines 'localhost' to be named 'target' in Ansible inventory, and a symlink to 'common-hosts', which defines additional targets that all derive from 'target'. -This allows ``ansible_tests.sh`` to reuse the common-hosts definitions while +This allows `ansible_tests.sh` to reuse the common-hosts definitions while replacing localhost as the test target by creating a new directory that similarly symlinks in common-hosts. diff --git a/tests/ansible/osx_setup.yml b/tests/ansible/osx_setup.yml deleted file mode 100644 index 7a6ff23f..00000000 --- a/tests/ansible/osx_setup.yml +++ /dev/null @@ -1,155 +0,0 @@ - -# -# Add users expected by tests to an OS X machine. Assumes passwordless sudo to -# root. -# -# WARNING: this creates non-privilged accounts with pre-set passwords! -# - -- hosts: test-targets - gather_facts: true - become: true - tasks: - - name: Disable non-localhost SSH for Mitogen users - blockinfile: - path: /etc/ssh/sshd_config - block: | - Match User mitogen__* Address !127.0.0.1 - DenyUsers * - - # - # Hashed passwords. - # - - name: Create Mitogen test group - group: - name: "mitogen__group" - - - name: Create Mitogen test users - user: - name: "mitogen__{{item}}" - shell: /bin/bash - groups: mitogen__group - password: "{{ (item + '_password') | password_hash('sha256') }}" - with_items: - - has_sudo - - has_sudo_pubkey - - require_tty - - pw_required - - readonly_homedir - - require_tty_pw_required - - slow_user - when: ansible_system != 'Darwin' - - - name: Create Mitogen test users - user: - name: "mitogen__user{{item}}" - shell: /bin/bash - password: "{{ ('user' + item + '_password') | password_hash('sha256') }}" - with_sequence: start=1 end=21 - when: ansible_system != 'Darwin' - - # - # Plaintext passwords - # - - name: Create Mitogen test users - user: - name: "mitogen__{{item}}" - shell: /bin/bash - groups: mitogen__group - password: "{{item}}_password" - with_items: - - has_sudo - - has_sudo_pubkey - - require_tty - - pw_required - - require_tty_pw_required - - readonly_homedir - - slow_user - when: ansible_system == 'Darwin' - - - name: Create Mitogen test users - user: - name: "mitogen__user{{item}}" - shell: /bin/bash - password: "user{{item}}_password" - with_sequence: start=1 end=21 - when: ansible_system == 'Darwin' - - - name: Hide test users from login window. - shell: > - defaults - write - /Library/Preferences/com.apple.loginwindow - HiddenUsersList - -array-add '{{item}}' - with_items: - - mitogen__require_tty - - mitogen__pw_required - - mitogen__require_tty_pw_required - when: ansible_system == 'Darwin' - - - name: Hide test users from login window. - shell: > - defaults - write - /Library/Preferences/com.apple.loginwindow - HiddenUsersList - -array-add 'mitogen__user{{item}}' - with_sequence: start=1 end=21 - when: ansible_distribution == 'MacOSX' - - - name: Readonly homedir for one account - shell: "chown -R root: ~mitogen__readonly_homedir" - - - name: Slow bash profile for one account - copy: - dest: ~mitogen__slow_user/.{{item}} - src: ../data/docker/mitogen__slow_user.profile - with_items: - - bashrc - - profile - - - name: Install pubkey for one account - file: - path: ~mitogen__has_sudo_pubkey/.ssh - state: directory - mode: go= - owner: mitogen__has_sudo_pubkey - - - name: Install pubkey for one account - copy: - dest: ~mitogen__has_sudo_pubkey/.ssh/authorized_keys - src: ../data/docker/mitogen__has_sudo_pubkey.key.pub - mode: go= - owner: mitogen__has_sudo_pubkey - - - name: Require a TTY for two accounts - lineinfile: - path: /etc/sudoers - line: "{{item}}" - with_items: - - Defaults>mitogen__pw_required targetpw - - Defaults>mitogen__require_tty requiretty - - Defaults>mitogen__require_tty_pw_required requiretty,targetpw - - - name: Require password for two accounts - lineinfile: - path: /etc/sudoers - line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}) ALL" - with_items: - - mitogen__pw_required - - mitogen__require_tty_pw_required - - - name: Allow passwordless for two accounts - lineinfile: - path: /etc/sudoers - line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}) NOPASSWD:ALL" - with_items: - - mitogen__require_tty - - mitogen__readonly_homedir - - - name: Allow passwordless for many accounts - lineinfile: - path: /etc/sudoers - line: "{{lookup('pipe', 'whoami')}} ALL = (mitogen__user{{item}}) NOPASSWD:ALL" - with_sequence: start=1 end=21 diff --git a/tests/build_docker_images.py b/tests/build_docker_images.py deleted file mode 100755 index 7f856b2b..00000000 --- a/tests/build_docker_images.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python - -""" -Build the Docker images used for testing. -""" - -import commands -import os -import shlex -import subprocess -import tempfile - - -DEBIAN_DOCKERFILE = r""" -FROM debian:stretch -RUN apt-get update -RUN \ - apt-get install -y python2.7 openssh-server sudo rsync git strace \ - libjson-perl python-virtualenv && \ - apt-get clean && \ - rm -rf /var/cache/apt -""" - -CENTOS6_DOCKERFILE = r""" -FROM centos:6 -RUN yum clean all && \ - yum -y install -y python2.6 openssh-server sudo rsync git strace sudo \ - perl-JSON python-virtualenv && \ - yum clean all && \ - groupadd sudo && \ - ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key - -""" - -CENTOS7_DOCKERFILE = r""" -FROM centos:7 -RUN yum clean all && \ - yum -y install -y python2.7 openssh-server sudo rsync git strace sudo \ - perl-JSON python-virtualenv && \ - yum clean all && \ - groupadd sudo && \ - ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key - -""" - -DOCKERFILE = r""" -COPY data/001-mitogen.sudo /etc/sudoers.d/001-mitogen -COPY data/docker/ssh_login_banner.txt /etc/ssh/banner.txt -RUN \ - chsh -s /bin/bash && \ - mkdir -p /var/run/sshd && \ - echo i-am-mitogen-test-docker-image > /etc/sentinel && \ - echo "Banner /etc/ssh/banner.txt" >> /etc/ssh/sshd_config && \ - groupadd mitogen__sudo_nopw && \ - useradd -s /bin/bash -m mitogen__has_sudo -G SUDO_GROUP && \ - useradd -s /bin/bash -m mitogen__has_sudo_pubkey -G SUDO_GROUP && \ - useradd -s /bin/bash -m mitogen__has_sudo_nopw -G mitogen__sudo_nopw && \ - useradd -s /bin/bash -m mitogen__webapp && \ - useradd -s /bin/bash -m mitogen__pw_required && \ - useradd -s /bin/bash -m mitogen__require_tty && \ - useradd -s /bin/bash -m mitogen__require_tty_pw_required && \ - useradd -s /bin/bash -m mitogen__readonly_homedir && \ - useradd -s /bin/bash -m mitogen__slow_user && \ - chown -R root: ~mitogen__readonly_homedir && \ - ( for i in `seq 1 21`; do useradd -s /bin/bash -m mitogen__user${i}; done; ) && \ - ( for i in `seq 1 21`; do echo mitogen__user${i}:user${i}_password | chpasswd; done; ) && \ - ( echo 'root:rootpassword' | chpasswd; ) && \ - ( echo 'mitogen__has_sudo:has_sudo_password' | chpasswd; ) && \ - ( echo 'mitogen__has_sudo_pubkey:has_sudo_pubkey_password' | chpasswd; ) && \ - ( echo 'mitogen__has_sudo_nopw:has_sudo_nopw_password' | chpasswd; ) && \ - ( echo 'mitogen__webapp:webapp_password' | chpasswd; ) && \ - ( echo 'mitogen__pw_required:pw_required_password' | chpasswd; ) && \ - ( echo 'mitogen__require_tty:require_tty_password' | chpasswd; ) && \ - ( echo 'mitogen__require_tty_pw_required:require_tty_pw_required_password' | chpasswd; ) && \ - ( echo 'mitogen__readonly_homedir:readonly_homedir_password' | chpasswd; ) && \ - ( echo 'mitogen__slow_user:slow_user_password' | chpasswd; ) && \ - mkdir ~mitogen__has_sudo_pubkey/.ssh && \ - ( echo '#!/bin/bash\nexec strace -ff -o /tmp/pywrap$$.trace python2.7 "$@"' > /usr/local/bin/pywrap; chmod +x /usr/local/bin/pywrap; ) - -COPY data/docker/mitogen__has_sudo_pubkey.key.pub /home/mitogen__has_sudo_pubkey/.ssh/authorized_keys -COPY data/docker/mitogen__slow_user.profile /home/mitogen__slow_user/.profile -COPY data/docker/mitogen__slow_user.profile /home/mitogen__slow_user/.bashrc - -RUN \ - chown -R mitogen__has_sudo_pubkey ~mitogen__has_sudo_pubkey && \ - chmod -R go= ~mitogen__has_sudo_pubkey - -RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config -RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd - -ENV NOTVISIBLE "in users profile" -RUN echo "export VISIBLE=now" >> /etc/profile - -EXPOSE 22 -CMD ["/usr/sbin/sshd", "-D"] -""" - - -def sh(s, *args): - if args: - s %= tuple(map(commands.mkarg, args)) - return shlex.split(s) - - -for (distro, wheel, prefix) in ( - ('debian', 'sudo', DEBIAN_DOCKERFILE), - ('centos6', 'wheel', CENTOS6_DOCKERFILE), - ('centos7', 'wheel', CENTOS7_DOCKERFILE), - ): - mydir = os.path.abspath(os.path.dirname(__file__)) - with tempfile.NamedTemporaryFile(dir=mydir) as dockerfile_fp: - dockerfile_fp.write(prefix) - dockerfile_fp.write(DOCKERFILE.replace('SUDO_GROUP', wheel)) - dockerfile_fp.flush() - - subprocess.check_call(sh('docker build %s -t %s -f %s', - mydir, - 'mitogen/%s-test' % (distro,), - dockerfile_fp.name - )) diff --git a/tests/data/001-mitogen.sudo b/tests/data/docker/001-mitogen.sudo similarity index 68% rename from tests/data/001-mitogen.sudo rename to tests/data/docker/001-mitogen.sudo index 71e20e6a..95b36f3b 100644 --- a/tests/data/001-mitogen.sudo +++ b/tests/data/docker/001-mitogen.sudo @@ -7,3 +7,8 @@ mitogen__has_sudo_nopw ALL = (mitogen__require_tty_pw_required) ALL Defaults>mitogen__pw_required targetpw Defaults>mitogen__require_tty requiretty Defaults>mitogen__require_tty_pw_required requiretty,targetpw + +mitogen__condel1 ALL=(ALL:ALL) NOPASSWD:ALL +mitogen__condel2 ALL=(ALL:ALL) NOPASSWD:ALL +mitogen__condel3 ALL=(ALL:ALL) NOPASSWD:ALL +mitogen__condel4 ALL=(ALL:ALL) NOPASSWD:ALL diff --git a/tests/image_prep/README.md b/tests/image_prep/README.md new file mode 100644 index 00000000..d275672f --- /dev/null +++ b/tests/image_prep/README.md @@ -0,0 +1,25 @@ + +# `image_prep` + +This directory contains Ansible playbooks for building the Docker containers +used for testing, or for setting up an OS X laptop so the tests can (mostly) +run locally. + +The Docker config is more heavily jinxed to trigger adverse conditions in the +code, the OS X config just has the user accounts. + +See ../README.md for a (mostly) description of the accounts created. + + +## Building the containers + +``./build_docker_images.sh`` + + +## Preparing an OS X box + +WARNING: this creates a ton of accounts with preconfigured passwords. It is +generally impossible to restrict remote access to these, so your only option is +to disable remote login and sharing. + +``ansible-playbook -b -c local -i localhost, -l localhost setup.yml`` diff --git a/tests/image_prep/_container_setup.yml b/tests/image_prep/_container_setup.yml new file mode 100644 index 00000000..6e1416df --- /dev/null +++ b/tests/image_prep/_container_setup.yml @@ -0,0 +1,116 @@ + +- hosts: all + strategy: linear + gather_facts: false + tasks: + - raw: > + if ! python -c ''; then + if type -p yum; then + yum -y install python; + else + apt-get -y update && apt-get -y install python; + fi; + fi + +- hosts: all + strategy: mitogen_linear + # Can't gather facts before here. + gather_facts: true + vars: + distro: "{{ansible_distribution}}" + ver: "{{ansible_distribution_major_version}}" + + packages: + common: + - git + - openssh-server + - rsync + - strace + - sudo + Debian: + "9": + - libjson-perl + - python-virtualenv + CentOS: + "6": + - perl-JSON + "7": + - perl-JSON + - python-virtualenv + + tasks: + - when: ansible_virtualization_type != "docker" + meta: end_play + + - apt: + name: "{{packages.common + packages[distro][ver]}}" + state: installed + update_cache: true + when: distro == "Debian" + + - yum: + name: "{{packages.common + packages[distro][ver]}}" + state: installed + update_cache: true + when: distro == "CentOS" + + - command: apt-get clean + when: distro == "Debian" + + - command: yum clean all + when: distro == "CentOS" + + - file: + path: /var/cache/apt + state: absent + when: distro == "Debian" + + - user: + name: root + password: "{{ 'rootpassword' | password_hash('sha256') }}" + shell: /bin/bash + + - file: + path: /var/run/sshd + state: directory + + - command: ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key + args: + creates: /etc/ssh/ssh_host_rsa_key + + - group: + name: "{{sudo_group[distro]}}" + + - copy: + dest: /etc/sentinel + content: | + i-am-mitogen-test-docker-image + + - copy: + dest: /etc/ssh/banner.txt + src: ../data/docker/ssh_login_banner.txt + + - copy: + dest: /etc/sudoers.d/001-mitogen + src: ../data/docker/001-mitogen.sudo + + - lineinfile: + path: /etc/ssh/sshd_config + line: Banner /etc/ssh/banner.txt + + - lineinfile: + path: /etc/ssh/sshd_config + line: PermitRootLogin yes + regexp: '.*PermitRootLogin.*' + + - lineinfile: + path: /etc/pam.d/sshd + regexp: '.*session.*required.*pam_loginuid.so' + line: session optional pam_loginuid.so + + - copy: + mode: 'u+rwx,go=rx' + dest: /usr/local/bin/pywrap + content: | + #!/bin/bash + exec strace -ff -o /tmp/pywrap$$.trace python2.7 "$@"' diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml new file mode 100644 index 00000000..1ed2c61d --- /dev/null +++ b/tests/image_prep/_user_accounts.yml @@ -0,0 +1,139 @@ +# +# Add users expected by tests. Assumes passwordless sudo to root. +# +# WARNING: this creates non-privilged accounts with pre-set passwords! +# + +- hosts: all + gather_facts: true + strategy: mitogen_linear + become: true + vars: + special_users: + - has_sudo + - has_sudo_pubkey + - pw_required + - readonly_homedir + - require_tty + - require_tty_pw_required + - slow_user + - webapp + + groups: + - has_sudo: ['mitogen__group', '{{sudo_group[distro]}}'] + - has_sudo_pubkey: ['mitogen__group', '{{sudo_group[distro]}}'] + - has_sudo_nopw: ['mitogen__group', 'mitogen__sudo_nopw'] + + normal_users: "{{ + lookup('sequence', 'start=1 end=5 format=user%d', wantlist=True) + }}" + + all_users: "{{ + special_users + + normal_users + }}" + tasks: + - name: Disable non-localhost SSH for Mitogen users + blockinfile: + path: /etc/ssh/sshd_config + block: | + Match User mitogen__* Address !127.0.0.1 + DenyUsers * + + - name: Create Mitogen test groups + group: + name: "mitogen__{{item}}" + with_items: + - group + - sudo_nopw + + - name: Create user accounts + block: + - user: + name: "mitogen__{{item}}" + shell: /bin/bash + groups: "{{groups[item]|default(['mitogen__group'])}}" + password: "{{ (item + '_password') | password_hash('sha256') }}" + loop: "{{all_users}}" + when: ansible_system != 'Darwin' + - user: + name: "mitogen__{{item}}" + shell: /bin/bash + groups: "{{groups[item]|default(['mitogen__group'])}}" + password: "{{item}}_password" + loop: "{{all_users}}" + when: ansible_system == 'Darwin' + + - name: Hide users from login window. + loop: "{{all_users}}" + when: ansible_system == 'Darwin' + osx_defaults: + array_add: true + domain: /Library/Preferences/com.apple.loginwindow + type: array + key: HiddenUsersList + value: ['mitogen_{{item}}'] + + - name: Readonly homedir for one account + shell: "chown -R root: ~mitogen__readonly_homedir" + + - name: Slow bash profile for one account + copy: + dest: ~mitogen__slow_user/.{{item}} + src: ../data/docker/mitogen__slow_user.profile + with_items: + - bashrc + - profile + + - name: Install pubkey for mitogen__has_sudo_pubkey + block: + - file: + path: ~mitogen__has_sudo_pubkey/.ssh + state: directory + mode: go= + owner: mitogen__has_sudo_pubkey + - copy: + dest: ~mitogen__has_sudo_pubkey/.ssh/authorized_keys + src: ../data/docker/mitogen__has_sudo_pubkey.key.pub + mode: go= + owner: mitogen__has_sudo_pubkey + + - name: Install slow profile for one account + block: + - copy: + dest: ~mitogen__slow_user/.profile + src: ../data/docker/mitogen__slow_user.profile + - copy: + dest: ~mitogen__slow_user/.bashrc + src: ../data/docker/mitogen__slow_user.profile + + - name: Require a TTY for two accounts + lineinfile: + path: /etc/sudoers + line: "{{item}}" + with_items: + - Defaults>mitogen__pw_required targetpw + - Defaults>mitogen__require_tty requiretty + - Defaults>mitogen__require_tty_pw_required requiretty,targetpw + + - name: Require password for two accounts + lineinfile: + path: /etc/sudoers + line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}) ALL" + with_items: + - mitogen__pw_required + - mitogen__require_tty_pw_required + + - name: Allow passwordless sudo for require_tty/readonly_homedir + lineinfile: + path: /etc/sudoers + line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}) NOPASSWD:ALL" + with_items: + - mitogen__require_tty + - mitogen__readonly_homedir + + - name: Allow passwordless for many accounts + lineinfile: + path: /etc/sudoers + line: "{{lookup('pipe', 'whoami')}} ALL = (mitogen__{{item}}) NOPASSWD:ALL" + loop: "{{normal_users}}" diff --git a/tests/image_prep/ansible.cfg b/tests/image_prep/ansible.cfg new file mode 100644 index 00000000..a3937825 --- /dev/null +++ b/tests/image_prep/ansible.cfg @@ -0,0 +1,4 @@ + +[defaults] +strategy_plugins = ../../ansible_mitogen/plugins/strategy +retry_files_enabled = false diff --git a/tests/image_prep/build_docker_images.py b/tests/image_prep/build_docker_images.py new file mode 100755 index 00000000..8a4161db --- /dev/null +++ b/tests/image_prep/build_docker_images.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python + +""" +Build the Docker images used for testing. +""" + +import commands +import os +import shlex +import subprocess + + +BASEDIR = os.path.dirname(os.path.abspath(__file__)) + + +def sh(s, *args): + if args: + s %= args + return shlex.split(s) + + +for base_image, name in [('debian:stretch', 'debian'), + ('centos:6', 'centos6'), + ('centos:7', 'centos7')]: + args = sh('docker run --rm -it -d %s /bin/bash', base_image) + container_id = subprocess.check_output(args).strip() + try: + subprocess.check_call( + cwd=BASEDIR, + args=sh(''' + ansible-playbook -i %s, -c docker setup.yml -vvv + ''', container_id) + ) + + subprocess.check_call(sh(''' + docker commit + --change 'EXPOSE 22' + --change 'CMD ["/usr/sbin/sshd", "-D"]' + %s + mitogen/%s-test + ''', container_id, name)) + finally: + subprocess.check_call(sh('docker rm -f %s', container_id)) diff --git a/tests/image_prep/setup.yml b/tests/image_prep/setup.yml new file mode 100644 index 00000000..168d583c --- /dev/null +++ b/tests/image_prep/setup.yml @@ -0,0 +1,13 @@ + +- hosts: all + gather_facts: false + tasks: + - set_fact: + # Hacktacular.. but easiest place for it with current structure. + sudo_group: + MacOSX: admin + Debian: wheel + CentOS: sudo + +- import_playbook: _container_setup.yml +- import_playbook: _user_accounts.yml From b521f215fd1e4c2109f16cbde26914c6e5b6875c Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 11 Aug 2018 14:48:48 +0100 Subject: [PATCH 026/212] ansible: handle >2.6 magic exceptions + sys.excepthook damage Closes #332. --- ansible_mitogen/runner.py | 39 +++++++++++++++++-- tests/ansible/regression/all.yml | 1 + ...32_ansiblemoduleerror_first_occurrence.yml | 14 +++++++ 3 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 1cc02198..aaac4882 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -627,6 +627,14 @@ class NewStyleRunner(ScriptRunner): for fullname in self.module_map['builtin']: mitogen.core.import_module(fullname) + def _setup_excepthook(self): + """ + Starting with Ansible 2.6, some modules (file.py) install a + sys.excepthook and never clean it up. So we must preserve the original + excepthook and restore it after the run completes. + """ + self.original_excepthook = sys.excepthook + def setup(self): super(NewStyleRunner, self).setup() @@ -640,12 +648,17 @@ class NewStyleRunner(ScriptRunner): module_utils=self.module_map['custom'], ) self._setup_imports() + self._setup_excepthook() if libc__res_init: libc__res_init() + def _revert_excepthook(self): + sys.excepthook = self.original_excepthook + def revert(self): self._argv.revert() self._stdio.revert() + self._revert_excepthook() super(NewStyleRunner, self).revert() def _get_program_filename(self): @@ -679,6 +692,20 @@ class NewStyleRunner(ScriptRunner): else: main_module_name = b'__main__' + def _handle_magic_exception(self, mod, exc): + """ + Beginning with Ansible >2.6, some modules (file.py) install a + sys.excepthook which is a closure over AnsibleModule, redirecting the + magical exception to AnsibleModule.fail_json(). + + For extra special needs bonus points, the class is not defined in + module_utils, but is defined in the module itself, meaning there is no + type for isinstance() that outlasts the invocation. + """ + klass = getattr(mod, 'AnsibleModuleError', None) + if klass and isinstance(exc, klass): + mod.module.fail_json(**exc.results) + def _run(self): code = self._get_code() @@ -695,10 +722,14 @@ class NewStyleRunner(ScriptRunner): exc = None try: - if mitogen.core.PY3: - exec(code, vars(mod)) - else: - exec('exec code in vars(mod)') + try: + if mitogen.core.PY3: + exec(code, vars(mod)) + else: + exec('exec code in vars(mod)') + except Exception as e: + self._handle_magic_exception(mod, e) + raise except SystemExit as e: exc = e diff --git a/tests/ansible/regression/all.yml b/tests/ansible/regression/all.yml index ecb9638c..46798b3e 100644 --- a/tests/ansible/regression/all.yml +++ b/tests/ansible/regression/all.yml @@ -7,3 +7,4 @@ - import_playbook: issue_152__virtualenv_python_fails.yml - import_playbook: issue_154__module_state_leaks.yml - import_playbook: issue_177__copy_module_failing.yml +- import_playbook: issue_332_ansiblemoduleerror_first_occurrence.yml diff --git a/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml b/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml new file mode 100644 index 00000000..3a64455e --- /dev/null +++ b/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml @@ -0,0 +1,14 @@ +# issue #332: Ansible 2.6 file.py started defining an excepthook and private +# AnsibleModuleError. Ensure file fails correctly. + +- name: regression/issue_332_ansiblemoduleerror_first_occurrence.yml + hosts: all + tasks: + - file: path=/usr/bin/does-not-exist mode='a-s' state=file follow=yes + ignore_errors: true + register: out + + - assert: + that: + - out.state == 'absent' + - out.msg == 'file (/usr/bin/does-not-exist) is absent, cannot continue' From 9365f254d287bfcff383801dd1357d666b36c650 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 11 Aug 2018 14:56:13 +0100 Subject: [PATCH 027/212] Update ChangeLog. --- docs/changelog.rst | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 27571ad2..5bf0aaed 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -33,11 +33,48 @@ v0.2.3 (2018-08-??) Mitogen for Ansible ~~~~~~~~~~~~~~~~~~~ +* `#324 `_: plays with a custom + module_utils would fail due to fallout from the Python 3 port and related + tests being disabled. + * `#331 `_: fixed known issue: the connection multiplexer subprocess always exits before the main Ansible process exits, ensuring logs generated by it do not overwrite the user's prompt when ``-vvv`` is enabled. +* `#332 `_: support a new + :data:`sys.excepthook`-based module exit mechanism added in Ansible 2.6. + +* `#338 `_: compatibility: due to + Ansible's implementation, changes to ``/etc/environment`` made by a task are + reflected in the runtime environment of subsequent tasks, but only if those + tasks set ``become: true``, or if SSH multiplexing is disabled. Changes to + ``/etc/environment`` are now monitored and always reflected. + +* Runs with many targets executed the module dependency scanner redundantly, + due to missing synchronization, creating significant extra work in the + connection multiplexer process. For one real-world playbook, the scanner + runtime was reduced by 95%, which may be apparent + + +Core Library +~~~~~~~~~~~~ + +* `#339 `_: the LXD connection method + was erroneously executing LXC Classic commands. + + +Thanks! +~~~~~~~ + +Mitogen would not be possible without the support of users. A huge thanks for +the bug reports in this release contributed by +`Rick Box `_, +`Alex Russu `_, +`Timo Beckers `_, +`Pateek Jain `_, and +`Pierre-Henry Muller `_. + v0.2.2 (2018-07-26) ------------------- From 49c804937d58ca1b5f12f5a54b108820a1fe9f66 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 11 Aug 2018 17:24:31 +0100 Subject: [PATCH 028/212] tests: import 2 more simple benchmarks. --- tests/ansible/bench/loop-100-items.yml | 10 +++ tests/ansible/bench/loop-100-tasks.yml | 112 +++++++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 tests/ansible/bench/loop-100-items.yml create mode 100644 tests/ansible/bench/loop-100-tasks.yml diff --git a/tests/ansible/bench/loop-100-items.yml b/tests/ansible/bench/loop-100-items.yml new file mode 100644 index 00000000..0feb57c5 --- /dev/null +++ b/tests/ansible/bench/loop-100-items.yml @@ -0,0 +1,10 @@ +# Execute 'hostname' 100 times in a loop. Loops execute within TaskExecutor +# within a single WorkerProcess, each iteration is a fair approximation of the +# non-controller overhead involved in executing a task. +# +# See also: loop-100-tasks.yml +# +- hosts: all + tasks: + - command: hostname + with_sequence: start=1 end=100 diff --git a/tests/ansible/bench/loop-100-tasks.yml b/tests/ansible/bench/loop-100-tasks.yml new file mode 100644 index 00000000..bf6e31b8 --- /dev/null +++ b/tests/ansible/bench/loop-100-tasks.yml @@ -0,0 +1,112 @@ +# Execute 'hostname' 100 times, using 100 individual tasks. Each task causes a +# new WorkerProcess to be forked, along with get_vars() calculation, and in the +# Mitogen extension, reestablishment of the UNIX socket connectionto the +# multiplexer process. +# +# It does not measure at least module dependency scanning (cached after first +# iteration). +# +# See also: loop-100-items.yml +# +- hosts: all + tasks: + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname + - command: hostname From 9e572a793949d788000b40a6a064b4688c142fdc Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 11 Aug 2018 17:39:42 +0100 Subject: [PATCH 029/212] ansible: fix duplicate MuxProcess socket write. The while: loop was necessary due to some cutpaste further on down the file. --- ansible_mitogen/process.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index 2f275dc0..5ce1b8be 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -67,8 +67,7 @@ def clean_shutdown(sock): has been printed. """ sock.shutdown(socket.SHUT_WR) - while sock.recv(1): - pass + sock.recv(1) class MuxProcess(object): @@ -162,7 +161,6 @@ class MuxProcess(object): # Let the parent know our listening socket is ready. mitogen.core.io_op(self.child_sock.send, b('1')) - self.child_sock.send(b('1')) # Block until the socket is closed, which happens on parent exit. mitogen.core.io_op(self.child_sock.recv, 1) From df112be704ec3d982417f83e6d595e9c8d294133 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 11 Aug 2018 17:42:52 +0100 Subject: [PATCH 030/212] tests: teach controller.yml to configure git too --- tests/ansible/gcloud/controller.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/ansible/gcloud/controller.yml b/tests/ansible/gcloud/controller.yml index b4ce3fcf..f7989ddf 100644 --- a/tests/ansible/gcloud/controller.yml +++ b/tests/ansible/gcloud/controller.yml @@ -1,5 +1,9 @@ - hosts: controller + vars: + git_username: '{{ lookup("pipe", "git config --global user.name") }}' + git_email: '{{ lookup("pipe", "git config --global user.email") }}' + tasks: - lineinfile: line: "net.ipv4.ip_forward=1" @@ -32,6 +36,11 @@ - shell: "rsync -a ~/.ssh {{inventory_hostname}}:" connection: local + - shell: | + git config --global user.email "{{git_username}}" + git config --global user.name "{{git_email}}" + name: set_git_config + - git: dest: ~/mitogen repo: https://github.com/dw/mitogen.git From 7d62c79ab7606d99ba1db7e54a67d65aee055ddb Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 11 Aug 2018 17:46:44 +0100 Subject: [PATCH 031/212] docker: redirect stderr to stdout for nicer exceptions. Unclear whether or not this is a hack, or whether it should be the default for more connection methods. When enabled, the exception text thrown when bootstrap fails includes the stderr text, which is apparently always useful. --- mitogen/docker.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mitogen/docker.py b/mitogen/docker.py index 38ee9d4e..36b0635b 100644 --- a/mitogen/docker.py +++ b/mitogen/docker.py @@ -43,6 +43,11 @@ class Stream(mitogen.parent.Stream): username = None docker_path = 'docker' + # TODO: better way of capturing errors such as "No such container." + create_child_args = { + 'merge_stdio': True + } + def construct(self, container=None, image=None, docker_path=None, username=None, **kwargs): From 8e35103185ef3ebce030cc11c2b5b128342a75e4 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 11 Aug 2018 18:31:08 +0100 Subject: [PATCH 032/212] docs: Update Changelog. --- docs/changelog.rst | 46 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5bf0aaed..ac74131d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -33,8 +33,14 @@ v0.2.3 (2018-08-??) Mitogen for Ansible ~~~~~~~~~~~~~~~~~~~ +* `#291 `_: when Mitogen had + previously been installed using ``pip`` or ``setuptools``, the globally + installed version could conflict with a newer version bundled with an + extension that had been installed using the documented steps. Now the bundled + library always overrides over any system-installed copy. + * `#324 `_: plays with a custom - module_utils would fail due to fallout from the Python 3 port and related + ``module_utils`` would fail due to fallout from the Python 3 port and related tests being disabled. * `#331 `_: fixed known issue: the @@ -51,18 +57,36 @@ Mitogen for Ansible tasks set ``become: true``, or if SSH multiplexing is disabled. Changes to ``/etc/environment`` are now monitored and always reflected. -* Runs with many targets executed the module dependency scanner redundantly, - due to missing synchronization, creating significant extra work in the - connection multiplexer process. For one real-world playbook, the scanner - runtime was reduced by 95%, which may be apparent +* Runs with many targets executed the module dependency scanner redundantly + due to missing synchronization, causing significant wasted computation in the + connection multiplexer subprocess. For one real-world playbook the scanner + runtime was reduced by 95%, which may manifest as shorter runs. + +* A missing check caused an exception traceback to appear when using the + ``ansible`` command-line tool with a missing or misspelled module name. + +* Ansible since >2.6 began importing ``__main__`` from + ``ansible.module_utils.basic``, causing an error during execution, due to the + controller being configured to refuse network imports outside the + ``ansible.*`` namespace. Update the target implementation to construct a stub + ``__main__`` module to satisfy the otherwise seemingly vestigial import. Core Library ~~~~~~~~~~~~ +* `#313 `_: + :meth:`mitogen.parent.Context.call` was documented as capable of accepting + static methods. While possible on Python 2.x the result is very ugly, and in + every case it should be trivially possible to replace with a class method. + The API docs were updated to remove mention of static methods. + * `#339 `_: the LXD connection method was erroneously executing LXC Classic commands. +* Add a :func:`mitogen.fork.on_fork` function to allow non-Mitogen managed + process forks to clean up Mitogen resources in the forked chlid. + Thanks! ~~~~~~~ @@ -70,8 +94,10 @@ Thanks! Mitogen would not be possible without the support of users. A huge thanks for the bug reports in this release contributed by `Rick Box `_, +`Dan Quackenbush `_, `Alex Russu `_, `Timo Beckers `_, +`Jesse London `_, `Pateek Jain `_, and `Pierre-Henry Muller `_. @@ -259,6 +285,11 @@ Mitogen for Ansible in Ansible, and is fairly harmless, albeit cosmetically annoying. A future release may include a solution. +.. * Configurations will break that rely on the `hashbang argument splitting + behaviour `_ of the + ``ansible_python_interpreter`` setting, contrary to the Ansible + documentation. This will be addressed in a future 0.2 release. + * Performance does not scale linearly with target count. This requires significant additional work, as major bottlenecks exist in the surrounding Ansible code. Performance-related bug reports for any scenario remain @@ -286,11 +317,6 @@ Mitogen for Ansible actions, such as the ``synchronize`` module. This will be addressed in the 0.3 series. -* Configurations will break that rely on the `hashbang argument splitting - behaviour `_ of the - ``ansible_python_interpreter`` setting, contrary to the Ansible - documentation. This will be addressed in a future 0.2 release. - Core Library ~~~~~~~~~~~~ From e1306bb03d89c21e14dce8e7380b5515ff9ca77a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 11 Aug 2018 19:00:02 +0100 Subject: [PATCH 033/212] tests: build Docker images in parallel --- tests/image_prep/_container_setup.yml | 5 ++- tests/image_prep/_user_accounts.yml | 1 + tests/image_prep/build_docker_images.py | 42 ++++++++++++++++--------- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/tests/image_prep/_container_setup.yml b/tests/image_prep/_container_setup.yml index 6e1416df..eabd97a8 100644 --- a/tests/image_prep/_container_setup.yml +++ b/tests/image_prep/_container_setup.yml @@ -61,8 +61,11 @@ when: distro == "CentOS" - file: - path: /var/cache/apt + path: "{{item}}" state: absent + with_items: + - /var/cache/apt + - /var/lib/apt when: distro == "Debian" - user: diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 1ed2c61d..0167a09f 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -34,6 +34,7 @@ }}" tasks: - name: Disable non-localhost SSH for Mitogen users + when: false blockinfile: path: /etc/ssh/sshd_config block: | diff --git a/tests/image_prep/build_docker_images.py b/tests/image_prep/build_docker_images.py index 8a4161db..44c4e2dd 100755 --- a/tests/image_prep/build_docker_images.py +++ b/tests/image_prep/build_docker_images.py @@ -6,6 +6,7 @@ Build the Docker images used for testing. import commands import os +import tempfile import shlex import subprocess @@ -19,25 +20,36 @@ def sh(s, *args): return shlex.split(s) -for base_image, name in [('debian:stretch', 'debian'), - ('centos:6', 'centos6'), - ('centos:7', 'centos7')]: - args = sh('docker run --rm -it -d %s /bin/bash', base_image) + +label_by_id = {} + +for base_image, label in [('debian:stretch', 'debian'), + ('centos:6', 'centos6'), + ('centos:7', 'centos7')]: + args = sh('docker run --rm -it -d -h mitogen-%s %s /bin/bash', + label, base_image) container_id = subprocess.check_output(args).strip() + label_by_id[container_id] = label + +with tempfile.NamedTemporaryFile() as fp: + fp.write('[all]\n') + for id_, label in label_by_id.items(): + fp.write('%s ansible_host=%s\n' % (label, id_)) + fp.flush() + try: subprocess.check_call( cwd=BASEDIR, - args=sh(''' - ansible-playbook -i %s, -c docker setup.yml -vvv - ''', container_id) + args=sh('ansible-playbook -i %s -c docker setup.yml', fp.name), ) - subprocess.check_call(sh(''' - docker commit - --change 'EXPOSE 22' - --change 'CMD ["/usr/sbin/sshd", "-D"]' - %s - mitogen/%s-test - ''', container_id, name)) + for container_id, label in label_by_id.items(): + subprocess.check_call(sh(''' + docker commit + --change 'EXPOSE 22' + --change 'CMD ["/usr/sbin/sshd", "-D"]' + %s + mitogen/%s-test + ''', container_id, label)) finally: - subprocess.check_call(sh('docker rm -f %s', container_id)) + subprocess.check_call(sh('docker rm -f %s', ' '.join(label_by_id))) From e48e32cd0ca36c134094eed6e6eececec285d2a0 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 11 Aug 2018 20:40:27 +0100 Subject: [PATCH 034/212] tests: image_prep fixes. --- tests/README.md | 2 +- tests/image_prep/_user_accounts.yml | 16 ++++++++++------ tests/image_prep/build_docker_images.py | 8 +++++--- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/tests/README.md b/tests/README.md index f5bbbc41..11a87022 100644 --- a/tests/README.md +++ b/tests/README.md @@ -73,7 +73,7 @@ also by Ansible's `osx_setup.yml`. used to target this account, the parent session requires a TTY and the account password must be entered. -`mitogen__user1` .. `mitogen__user21` +`mitogen__user1` .. `mitogen__user5` These accounts do not have passwords set. They exist to test the Ansible interpreter recycling logic. diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 0167a09f..0c1f6045 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -9,8 +9,12 @@ strategy: mitogen_linear become: true vars: + distro: "{{ansible_distribution}}" + ver: "{{ansible_distribution_major_version}}" + special_users: - has_sudo + - has_sudo_nopw - has_sudo_pubkey - pw_required - readonly_homedir @@ -19,10 +23,10 @@ - slow_user - webapp - groups: - - has_sudo: ['mitogen__group', '{{sudo_group[distro]}}'] - - has_sudo_pubkey: ['mitogen__group', '{{sudo_group[distro]}}'] - - has_sudo_nopw: ['mitogen__group', 'mitogen__sudo_nopw'] + user_groups: + has_sudo: ['mitogen__group', '{{sudo_group[distro]}}'] + has_sudo_pubkey: ['mitogen__group', '{{sudo_group[distro]}}'] + has_sudo_nopw: ['mitogen__group', 'mitogen__sudo_nopw'] normal_users: "{{ lookup('sequence', 'start=1 end=5 format=user%d', wantlist=True) @@ -53,14 +57,14 @@ - user: name: "mitogen__{{item}}" shell: /bin/bash - groups: "{{groups[item]|default(['mitogen__group'])}}" + groups: "{{user_groups[item]|default(['mitogen__group'])}}" password: "{{ (item + '_password') | password_hash('sha256') }}" loop: "{{all_users}}" when: ansible_system != 'Darwin' - user: name: "mitogen__{{item}}" shell: /bin/bash - groups: "{{groups[item]|default(['mitogen__group'])}}" + groups: "{{user_groups[item]|default(['mitogen__group'])}}" password: "{{item}}_password" loop: "{{all_users}}" when: ansible_system == 'Darwin' diff --git a/tests/image_prep/build_docker_images.py b/tests/image_prep/build_docker_images.py index 44c4e2dd..c085d29e 100755 --- a/tests/image_prep/build_docker_images.py +++ b/tests/image_prep/build_docker_images.py @@ -23,9 +23,11 @@ def sh(s, *args): label_by_id = {} -for base_image, label in [('debian:stretch', 'debian'), - ('centos:6', 'centos6'), - ('centos:7', 'centos7')]: +for base_image, label in [ + ('debian:stretch', 'debian'), + ('centos:6', 'centos6'), + ('centos:7', 'centos7') + ]: args = sh('docker run --rm -it -d -h mitogen-%s %s /bin/bash', label, base_image) container_id = subprocess.check_output(args).strip() From d39efd9f54c35c8ab6deb035665c21babd78a58e Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 11 Aug 2018 21:49:08 +0100 Subject: [PATCH 035/212] tests: add new users for conndel tests. --- tests/README.md | 5 ++++- tests/data/docker/001-mitogen.sudo | 5 ----- tests/image_prep/_user_accounts.yml | 8 ++++++++ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/README.md b/tests/README.md index 11a87022..51464989 100644 --- a/tests/README.md +++ b/tests/README.md @@ -77,9 +77,12 @@ also by Ansible's `osx_setup.yml`. These accounts do not have passwords set. They exist to test the Ansible interpreter recycling logic. +`mitogen__sudo1` .. `mitogen__sudo4` + May passwordless sudo to any account. + `mitogen__webapp` A plain old account with no sudo access, used as the target for fakessh - tddests. + tests. # Ansible Integration Test Environment diff --git a/tests/data/docker/001-mitogen.sudo b/tests/data/docker/001-mitogen.sudo index 95b36f3b..71e20e6a 100644 --- a/tests/data/docker/001-mitogen.sudo +++ b/tests/data/docker/001-mitogen.sudo @@ -7,8 +7,3 @@ mitogen__has_sudo_nopw ALL = (mitogen__require_tty_pw_required) ALL Defaults>mitogen__pw_required targetpw Defaults>mitogen__require_tty requiretty Defaults>mitogen__require_tty_pw_required requiretty,targetpw - -mitogen__condel1 ALL=(ALL:ALL) NOPASSWD:ALL -mitogen__condel2 ALL=(ALL:ALL) NOPASSWD:ALL -mitogen__condel3 ALL=(ALL:ALL) NOPASSWD:ALL -mitogen__condel4 ALL=(ALL:ALL) NOPASSWD:ALL diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 0c1f6045..1cb41a86 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -22,11 +22,19 @@ - require_tty_pw_required - slow_user - webapp + - sudo1 + - sudo2 + - sudo3 + - sudo4 user_groups: has_sudo: ['mitogen__group', '{{sudo_group[distro]}}'] has_sudo_pubkey: ['mitogen__group', '{{sudo_group[distro]}}'] has_sudo_nopw: ['mitogen__group', 'mitogen__sudo_nopw'] + sudo1: ['mitogen__group', 'mitogen__sudo_nopw'] + sudo2: ['mitogen__group', '{{sudo_group[distro]}}'] + sudo3: ['mitogen__group', '{{sudo_group[distro]}}'] + sudo4: ['mitogen__group', '{{sudo_group[distro]}}'] normal_users: "{{ lookup('sequence', 'start=1 end=5 format=user%d', wantlist=True) From 370b98f960cb93c8e95952bf474ddf599722cf9b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 12 Aug 2018 10:35:32 +0100 Subject: [PATCH 036/212] ansible: tidy up connection.py. - more docstrings. - _wrap_or_none -> optional_secret() --- ansible_mitogen/connection.py | 100 +++++++++++++++++++++++++--------- 1 file changed, 74 insertions(+), 26 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 12616dbf..0823736f 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -53,7 +53,28 @@ import ansible_mitogen.target LOG = logging.getLogger(__name__) +def optional_secret(value): + """ + Wrap `value` in :class:`mitogen.core.Secret` if it is not :data:`None`, + otherwise return :data:`None`. + """ + if value is not None: + return mitogen.core.Secret(value) + + +def parse_python_path(s): + """ + Given the string set for ansible_python_interpeter, parse it using shell + syntax and return an appropriate argument vector. + """ + if s: + return ansible.utils.shlex.shlex_split(s) + + def _connect_local(spec): + """ + Return ContextService arguments for a local connection. + """ return { 'method': 'local', 'kwargs': { @@ -62,12 +83,10 @@ def _connect_local(spec): } -def wrap_or_none(klass, value): - if value is not None: - return klass(value) - - def _connect_ssh(spec): + """ + Return ContextService arguments for an SSH connection. + """ if C.HOST_KEY_CHECKING: check_host_keys = 'enforce' else: @@ -79,7 +98,7 @@ def _connect_ssh(spec): 'check_host_keys': check_host_keys, 'hostname': spec['remote_addr'], 'username': spec['remote_user'], - 'password': wrap_or_none(mitogen.core.Secret, spec['password']), + 'password': optional_secret(spec['password']), 'port': spec['port'], 'python_path': spec['python_path'], 'identity_file': spec['private_key_file'], @@ -92,6 +111,9 @@ def _connect_ssh(spec): def _connect_docker(spec): + """ + Return ContextService arguments for a Docker connection. + """ return { 'method': 'docker', 'kwargs': { @@ -104,6 +126,9 @@ def _connect_docker(spec): def _connect_jail(spec): + """ + Return ContextService arguments for a FreeBSD jail connection. + """ return { 'method': 'jail', 'kwargs': { @@ -116,6 +141,9 @@ def _connect_jail(spec): def _connect_lxc(spec): + """ + Return ContextService arguments for an LXC Classic container connection. + """ return { 'method': 'lxc', 'kwargs': { @@ -127,6 +155,9 @@ def _connect_lxc(spec): def _connect_lxd(spec): + """ + Return ContextService arguments for an LXD container connection. + """ return { 'method': 'lxd', 'kwargs': { @@ -138,10 +169,16 @@ def _connect_lxd(spec): def _connect_machinectl(spec): + """ + Return ContextService arguments for a machinectl connection. + """ return _connect_setns(dict(spec, mitogen_kind='machinectl')) def _connect_setns(spec): + """ + Return ContextService arguments for a mitogen_setns connection. + """ return { 'method': 'setns', 'kwargs': { @@ -157,12 +194,15 @@ def _connect_setns(spec): def _connect_su(spec): + """ + Return ContextService arguments for su as a become method. + """ return { 'method': 'su', 'enable_lru': True, 'kwargs': { 'username': spec['become_user'], - 'password': wrap_or_none(mitogen.core.Secret, spec['become_pass']), + 'password': optional_secret(spec['become_pass']), 'python_path': spec['python_path'], 'su_path': spec['become_exe'], 'connect_timeout': spec['timeout'], @@ -171,12 +211,15 @@ def _connect_su(spec): def _connect_sudo(spec): + """ + Return ContextService arguments for sudo as a become method. + """ return { 'method': 'sudo', 'enable_lru': True, 'kwargs': { 'username': spec['become_user'], - 'password': wrap_or_none(mitogen.core.Secret, spec['become_pass']), + 'password': optional_secret(spec['become_pass']), 'python_path': spec['python_path'], 'sudo_path': spec['become_exe'], 'connect_timeout': spec['timeout'], @@ -186,12 +229,15 @@ def _connect_sudo(spec): def _connect_doas(spec): + """ + Return ContextService arguments for doas as a become method. + """ return { 'method': 'doas', 'enable_lru': True, 'kwargs': { 'username': spec['become_user'], - 'password': wrap_or_none(mitogen.core.Secret, spec['become_pass']), + 'password': optional_secret(spec['become_pass']), 'python_path': spec['python_path'], 'doas_path': spec['become_exe'], 'connect_timeout': spec['timeout'], @@ -200,12 +246,14 @@ def _connect_doas(spec): def _connect_mitogen_su(spec): - # su as a first-class proxied connection, not a become method. + """ + Return ContextService arguments for su as a first class connection. + """ return { 'method': 'su', 'kwargs': { 'username': spec['remote_user'], - 'password': wrap_or_none(mitogen.core.Secret, spec['password']), + 'password': optional_secret(spec['password']), 'python_path': spec['python_path'], 'su_path': spec['become_exe'], 'connect_timeout': spec['timeout'], @@ -214,12 +262,14 @@ def _connect_mitogen_su(spec): def _connect_mitogen_sudo(spec): - # sudo as a first-class proxied connection, not a become method. + """ + Return ContextService arguments for sudo as a first class connection. + """ return { 'method': 'sudo', 'kwargs': { 'username': spec['remote_user'], - 'password': wrap_or_none(mitogen.core.Secret, spec['password']), + 'password': optional_secret(spec['password']), 'python_path': spec['python_path'], 'sudo_path': spec['become_exe'], 'connect_timeout': spec['timeout'], @@ -229,12 +279,14 @@ def _connect_mitogen_sudo(spec): def _connect_mitogen_doas(spec): - # doas as a first-class proxied connection, not a become method. + """ + Return ContextService arguments for doas as a first class connection. + """ return { 'method': 'doas', 'kwargs': { 'username': spec['remote_user'], - 'password': wrap_or_none(mitogen.core.Secret, spec['password']), + 'password': optional_secret(spec['password']), 'python_path': spec['python_path'], 'doas_path': spec['become_exe'], 'connect_timeout': spec['timeout'], @@ -242,6 +294,9 @@ def _connect_mitogen_doas(spec): } +#: Mapping of connection method names to functions invoked as `func(spec)` +#: generating ContextService keyword arguments matching a connection +#: specification. CONNECTION_METHOD = { 'docker': _connect_docker, 'jail': _connect_jail, @@ -260,17 +315,6 @@ CONNECTION_METHOD = { } -def parse_python_path(s): - """ - Given the string set for ansible_python_interpeter, parse it using shell - syntax and return an appropriate argument vector. - """ - if not s: - return None - - return ansible.utils.shlex.shlex_split(s) - - def config_from_play_context(transport, inventory_name, connection): """ Return a dict representing all important connection configuration, allowing @@ -457,6 +501,10 @@ class Connection(ansible.plugins.connection.ConnectionBase): return self.context is not None def _config_from_via(self, via_spec): + """ + Produce a dict connection specifiction given a string `via_spec`, of + the form `[become_user@]inventory_hostname`. + """ become_user, _, inventory_name = via_spec.rpartition('@') via_vars = self.host_vars[inventory_name] if isinstance(via_vars, jinja2.runtime.Undefined): From a1e653978be38e677db8acf31f439eb8daeee7aa Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 12 Aug 2018 10:37:32 +0100 Subject: [PATCH 037/212] issue #340: connection delegation used wrong variable name. When inventory name did not match remote_addr, it would attempt to SSH to the inventory name. --- ansible_mitogen/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 0823736f..7099c1a2 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -373,7 +373,7 @@ def config_from_hostvars(transport, inventory_name, connection, config = config_from_play_context(transport, inventory_name, connection) hostvars = dict(hostvars) return dict(config, **{ - 'remote_addr': hostvars.get('ansible_hostname', inventory_name), + 'remote_addr': hostvars.get('ansible_host', inventory_name), 'become': bool(become_user), 'become_user': become_user, 'become_pass': None, From 6dcd5f89985ab56691e937dc23faf4634078d559 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 12 Aug 2018 10:39:50 +0100 Subject: [PATCH 038/212] issue #340: split up Connection._connect() The logic was getting too busy. --- ansible_mitogen/connection.py | 45 +++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 7099c1a2..f3a6dac2 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -551,20 +551,11 @@ class Connection(ansible.plugins.connection.ConnectionBase): return stack, seen_names - def _connect(self): + def _connect_broker(self): """ - Establish a connection to the master process's UNIX listener socket, - constructing a mitogen.master.Router to communicate with the master, - and a mitogen.parent.Context to represent it. - - Depending on the original transport we should emulate, trigger one of - the _connect_*() service calls defined above to cause the master - process to establish the real connection on our behalf, or return a - reference to the existing one. + Establish a reference to the Broker, Router and parent context used for + connections. """ - if self.connected: - return - if not self.broker: self.broker = mitogen.master.Broker() self.router, self.parent = mitogen.unix.connect( @@ -572,6 +563,11 @@ class Connection(ansible.plugins.connection.ConnectionBase): broker=self.broker, ) + def _build_stack(self): + """ + Construct a list of dictionaries representing the connection + configuration between the controller and the target. + """ if hasattr(self._play_context, 'delegate_to'): target_config = config_from_hostvars( transport=self._play_context.connection, @@ -589,7 +585,14 @@ class Connection(ansible.plugins.connection.ConnectionBase): connection=self ) stack, _ = self._stack_from_config(target_config) + return stack + def _connect_stack(self, stack): + """ + Pass `stack` to ContextService, requesting a copy of the context object + representing the target. If no connection exists yet, ContextService + will establish it before returning it or throwing an error. + """ dct = self.parent.call_service( service_name='ansible_mitogen.services.ContextService', method_name='get', @@ -610,6 +613,24 @@ class Connection(ansible.plugins.connection.ConnectionBase): self.fork_context = dct['init_child_result']['fork_context'] self.home_dir = dct['init_child_result']['home_dir'] + def _connect(self): + """ + Establish a connection to the master process's UNIX listener socket, + constructing a mitogen.master.Router to communicate with the master, + and a mitogen.parent.Context to represent it. + + Depending on the original transport we should emulate, trigger one of + the _connect_*() service calls defined above to cause the master + process to establish the real connection on our behalf, or return a + reference to the existing one. + """ + if self.connected: + return + + self._connect_broker() + stack = self._build_stack() + self._connect_stack(stack) + def close(self, new_task=False): """ Arrange for the mitogen.master.Router running in the worker to From aed8fb531b394eb627cfe7960de9400f3bb0528d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 12 Aug 2018 11:32:49 +0100 Subject: [PATCH 039/212] tests: unused imports --- tests/ansible/lib/action/mitogen_shutdown_all.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/ansible/lib/action/mitogen_shutdown_all.py b/tests/ansible/lib/action/mitogen_shutdown_all.py index 6ebdbf5c..4909dfe9 100644 --- a/tests/ansible/lib/action/mitogen_shutdown_all.py +++ b/tests/ansible/lib/action/mitogen_shutdown_all.py @@ -3,9 +3,6 @@ Arrange for all ContextService connections to be torn down unconditionally, required for reliable LRU tests. """ -import traceback -import sys - import ansible_mitogen.connection import ansible_mitogen.services import mitogen.service From 916e46621b879b09c389d97bb6d3415ca1edd17f Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 12 Aug 2018 11:53:06 +0100 Subject: [PATCH 040/212] issue #340: add connection delegation tests. --- ansible_mitogen/connection.py | 4 +- tests/ansible/common-hosts | 36 ++ tests/ansible/integration/all.yml | 3 +- tests/ansible/integration/delegation/all.yml | 2 + .../delegation/stack_construction.yml | 311 ++++++++++++++++++ tests/ansible/lib/action/mitogen_get_stack.py | 22 ++ 6 files changed, 376 insertions(+), 2 deletions(-) create mode 100644 tests/ansible/integration/delegation/all.yml create mode 100644 tests/ansible/integration/delegation/stack_construction.yml create mode 100644 tests/ansible/lib/action/mitogen_get_stack.py diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index f3a6dac2..ccfc12b4 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -566,7 +566,9 @@ class Connection(ansible.plugins.connection.ConnectionBase): def _build_stack(self): """ Construct a list of dictionaries representing the connection - configuration between the controller and the target. + configuration between the controller and the target. This is + additionally used by the integration tests "mitogen_get_stack" action + to fetch the would-be connection configuration. """ if hasattr(self._play_context, 'delegate_to'): target_config = config_from_hostvars( diff --git a/tests/ansible/common-hosts b/tests/ansible/common-hosts index 0aead1be..6dafaf47 100644 --- a/tests/ansible/common-hosts +++ b/tests/ansible/common-hosts @@ -1,3 +1,5 @@ +# vim: syntax=dosini + [connection-delegation-test] cd-bastion cd-rack11 mitogen_via=ssh-user@cd-bastion @@ -13,3 +15,37 @@ cdc-rack11a-docker mitogen_via=docker-admin@cdc-rack11a ansible_connection=docke [conn-delegation] cd-user1 ansible_user=mitogen__user1 ansible_connection=mitogen_sudo mitogen_via=target + + +# Connection delegation scenarios. It's impossible to connection to them, but +# you can inspect the would-be config via "mitogen_get_stack" action. +[cd-no-connect] +# Normal inventory host, no aliasing. +cd-normal ansible_connection=mitogen_doas ansible_user=normal-user +# Inventory host that is really a different host. +cd-alias ansible_connection=ssh ansible_user=alias-user ansible_host=alias-host + +# Via one normal host. +cd-normal-normal mitogen_via=cd-normal +# Via one aliased host. +cd-normal-alias mitogen_via=cd-alias + +# newuser@host via host with explicit username. +cd-newuser-normal-normal mitogen_via=cd-normal ansible_user=newuser-normal-normal-user + +# doas:newuser via host. +cd-newuser-doas-normal mitogen_via=cd-normal ansible_connection=mitogen_doas ansible_user=newuser-doas-normal-user + + +# Connection Delegation issue #340 reproduction. +# Path to jails is SSH to H -> mitogen_sudo to root -> jail to J + +[issue340] +# 'target' plays the role of the normal host machine H. +# 'mitogen__sudo1' plays the role of root@H via mitogen_sudo. +# 'mitogen__user1' plays the role of root@J via mitogen__user1. +# 'mitogen__user2' plays the role of E, the delgate_to target for certs. + +i340-root ansible_user=mitogen__sudo1 ansible_connection=mitogen_sudo mitogen_via=target +i340-jail ansible_user=mitogen__user1 ansible_connection=mitogen_sudo mitogen_via=i340-root +i340-certs ansible_user=mitogen__user2 ansible_connection=mitogen_sudo mitogen_via=target diff --git a/tests/ansible/integration/all.yml b/tests/ansible/integration/all.yml index bf534aed..ffea7a46 100644 --- a/tests/ansible/integration/all.yml +++ b/tests/ansible/integration/all.yml @@ -8,6 +8,8 @@ - import_playbook: become/all.yml - import_playbook: connection_loader/all.yml - import_playbook: context_service/all.yml +- import_playbook: delegation/all.yml +- import_playbook: glibc_caches/all.yml - import_playbook: local/all.yml - import_playbook: module_utils/all.yml - import_playbook: playbook_semantics/all.yml @@ -15,4 +17,3 @@ - import_playbook: runner/all.yml - import_playbook: ssh/all.yml - import_playbook: strategy/all.yml -- import_playbook: glibc_caches/all.yml diff --git a/tests/ansible/integration/delegation/all.yml b/tests/ansible/integration/delegation/all.yml new file mode 100644 index 00000000..9646d09c --- /dev/null +++ b/tests/ansible/integration/delegation/all.yml @@ -0,0 +1,2 @@ + +- import_playbook: stack_construction.yml diff --git a/tests/ansible/integration/delegation/stack_construction.yml b/tests/ansible/integration/delegation/stack_construction.yml new file mode 100644 index 00000000..c6851a4c --- /dev/null +++ b/tests/ansible/integration/delegation/stack_construction.yml @@ -0,0 +1,311 @@ +# https://github.com/dw/mitogen/issues/251 + +# ansible_mitogen.connection internally reinterprets Ansible state into a +# 'connection stack' -- this is just a list of dictionaries specifying a +# sequence of proxied Router connection methods and their kwargs used to +# establish the connection. That list is passed to ContextService, which loops +# over the stack specifying via=(None or previous entry) for each connection +# method. + +# mitogen_get_stack is a magic action that returns the stack, so we can test +# all kinds of scenarios without actually needing a real environmnt. + +# Updating this file? Install 'pprintpp' and hack lib/callbacks/nice_stdout.py +# to use it instead of the built-in function, then simply s/'/'/ to get the +# cutpasteable formatted dicts below. WARNING: remove the trailing comma from +# the result list element, it seems to cause assert to silently succeed! + + +- name: integration/delegation/stack_construction.yml + hosts: cd-normal + any_errors_fatal: true + tasks: + - mitogen_get_stack: + register: out + - assert: + that: | + out.result == [ + { + "kwargs": { + "connect_timeout": 10, + "doas_path": None, + "password": None, + "python_path": ["/usr/bin/python"], + "username": "normal-user", + }, + "method": "doas", + } + ] + + +- hosts: cd-normal + tasks: + - mitogen_get_stack: + delegate_to: cd-alias + register: out + - assert: + that: | + out.result == [ + { + 'kwargs': { + 'check_host_keys': 'ignore', + 'connect_timeout': 10, + 'hostname': 'alias-host', + 'identity_file': None, + 'password': None, + 'port': None, + 'python_path': None, + 'ssh_args': [ + '-o', + 'ForwardAgent=yes', + '-o', + 'ControlMaster=auto', + '-o', + 'ControlPersist=60s', + ], + 'ssh_debug_level': None, + 'ssh_path': 'ssh', + 'username': 'alias-user', + }, + 'method': 'ssh', + }, + ] + + +- hosts: cd-alias + tasks: + - mitogen_get_stack: + register: out + - assert: + that: | + out.result == [ + { + 'kwargs': { + 'check_host_keys': 'ignore', + 'connect_timeout': 10, + 'hostname': 'alias-host', + 'identity_file': None, + 'password': None, + 'port': None, + 'python_path': ['/usr/bin/python'], + 'ssh_args': [ + '-o', + 'ForwardAgent=yes', + '-o', + 'ControlMaster=auto', + '-o', + 'ControlPersist=60s', + ], + 'ssh_debug_level': None, + 'ssh_path': 'ssh', + 'username': 'alias-user', + }, + 'method': 'ssh', + }, + ] + + +- hosts: cd-normal-normal + tasks: + - mitogen_get_stack: + register: out + - assert: + that: | + out.result == [ + { + 'kwargs': { + 'connect_timeout': 10, + 'doas_path': None, + 'password': None, + 'python_path': None, + 'username': 'normal-user', + }, + 'method': 'doas', + }, + { + 'kwargs': { + 'check_host_keys': 'ignore', + 'connect_timeout': 10, + 'hostname': 'cd-normal-normal', + 'identity_file': None, + 'password': None, + 'port': None, + 'python_path': ['/usr/bin/python'], + 'ssh_args': [ + '-o', + 'ForwardAgent=yes', + '-o', + 'ControlMaster=auto', + '-o', + 'ControlPersist=60s', + ], + 'ssh_debug_level': None, + 'ssh_path': 'ssh', + 'username': None, + }, + 'method': 'ssh', + }, + ] + + +- hosts: cd-normal-alias + tasks: + - mitogen_get_stack: + register: out + - assert: + that: | + out.result == [ + { + 'kwargs': { + 'check_host_keys': 'ignore', + 'connect_timeout': 10, + 'hostname': 'alias-host', + 'identity_file': None, + 'password': None, + 'port': None, + 'python_path': None, + 'ssh_args': [ + '-o', + 'ForwardAgent=yes', + '-o', + 'ControlMaster=auto', + '-o', + 'ControlPersist=60s', + ], + 'ssh_debug_level': None, + 'ssh_path': 'ssh', + 'username': 'alias-user', + }, + 'method': 'ssh', + }, + { + 'kwargs': { + 'check_host_keys': 'ignore', + 'connect_timeout': 10, + 'hostname': 'cd-normal-alias', + 'identity_file': None, + 'password': None, + 'port': None, + 'python_path': ['/usr/bin/python'], + 'ssh_args': [ + '-o', + 'ForwardAgent=yes', + '-o', + 'ControlMaster=auto', + '-o', + 'ControlPersist=60s', + ], + 'ssh_debug_level': None, + 'ssh_path': 'ssh', + 'username': None, + }, + 'method': 'ssh', + }, + ] + + +- hosts: cd-newuser-normal-normal + tasks: + - mitogen_get_stack: + register: out + - assert: + that: | + out.result == [ + { + 'kwargs': { + 'connect_timeout': 10, + 'doas_path': None, + 'password': None, + 'python_path': None, + 'username': 'normal-user', + }, + 'method': 'doas', + }, + { + 'kwargs': { + 'check_host_keys': 'ignore', + 'connect_timeout': 10, + 'hostname': 'cd-newuser-normal-normal', + 'identity_file': None, + 'password': None, + 'port': None, + 'python_path': ['/usr/bin/python'], + 'ssh_args': [ + '-o', + 'ForwardAgent=yes', + '-o', + 'ControlMaster=auto', + '-o', + 'ControlPersist=60s', + ], + 'ssh_debug_level': None, + 'ssh_path': 'ssh', + 'username': 'newuser-normal-normal-user', + }, + 'method': 'ssh', + }, + ] + + +- hosts: cd-newuser-normal-normal + tasks: + - mitogen_get_stack: + delegate_to: cd-alias + register: out + - assert: + that: | + out.result == [ + { + 'kwargs': { + 'check_host_keys': 'ignore', + 'connect_timeout': 10, + 'hostname': 'alias-host', + 'identity_file': None, + 'password': None, + 'port': None, + 'python_path': None, + 'ssh_args': [ + '-o', + 'ForwardAgent=yes', + '-o', + 'ControlMaster=auto', + '-o', + 'ControlPersist=60s', + ], + 'ssh_debug_level': None, + 'ssh_path': 'ssh', + 'username': 'alias-user', + }, + 'method': 'ssh', + }, + ] + + +- hosts: cd-newuser-doas-normal + tasks: + - mitogen_get_stack: + register: out + - assert: + that: | + out.result == [ + { + 'kwargs': { + 'connect_timeout': 10, + 'doas_path': None, + 'password': None, + 'python_path': None, + 'username': 'normal-user', + }, + 'method': 'doas', + }, + { + 'kwargs': { + 'connect_timeout': 10, + 'doas_path': None, + 'password': None, + 'python_path': ['/usr/bin/python'], + 'username': 'newuser-doas-normal-user', + }, + 'method': 'doas', + }, + ] diff --git a/tests/ansible/lib/action/mitogen_get_stack.py b/tests/ansible/lib/action/mitogen_get_stack.py new file mode 100644 index 00000000..f1b87f35 --- /dev/null +++ b/tests/ansible/lib/action/mitogen_get_stack.py @@ -0,0 +1,22 @@ +""" +Fetch the connection configuration stack that would be used to connect to a +target, without actually connecting to it. +""" + +import ansible_mitogen.connection + +from ansible.plugins.action import ActionBase + + +class ActionModule(ActionBase): + def run(self, tmp=None, task_vars=None): + if not isinstance(self._connection, + ansible_mitogen.connection.Connection): + return { + 'skipped': True, + } + + return { + 'changed': True, + 'result': self._connection._build_stack(), + } From ad365dad56ad07da781165c9222b6da04dfae70d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 12 Aug 2018 12:03:11 +0100 Subject: [PATCH 041/212] issue #340: one more test, update Changelog. --- docs/changelog.rst | 5 ++++ .../delegation/stack_construction.yml | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index ac74131d..9a78790b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -33,6 +33,11 @@ v0.2.3 (2018-08-??) Mitogen for Ansible ~~~~~~~~~~~~~~~~~~~ +* `#251 `_, + `#340 `_: Connection Delegation + could establish connections to the wrong target when ``delegate_to:`` is + present. + * `#291 `_: when Mitogen had previously been installed using ``pip`` or ``setuptools``, the globally installed version could conflict with a newer version bundled with an diff --git a/tests/ansible/integration/delegation/stack_construction.yml b/tests/ansible/integration/delegation/stack_construction.yml index c6851a4c..8ab6e51b 100644 --- a/tests/ansible/integration/delegation/stack_construction.yml +++ b/tests/ansible/integration/delegation/stack_construction.yml @@ -18,6 +18,13 @@ - name: integration/delegation/stack_construction.yml hosts: cd-normal + tasks: + # used later for local_action test. + - local_action: custom_python_detect_environment + register: local_env + + +- hosts: cd-normal any_errors_fatal: true tasks: - mitogen_get_stack: @@ -281,6 +288,24 @@ ] +- hosts: cd-newuser-normal-normal + tasks: + - local_action: mitogen_get_stack + register: out + - assert: + that: | + out.result == [ + { + 'kwargs': { + 'python_path': [ + hostvars['cd-normal'].local_env.sys_executable + ], + }, + 'method': 'local', + }, + ] + + - hosts: cd-newuser-doas-normal tasks: - mitogen_get_stack: From 8eb288856c84a7017e59d1c7fa5d0e25a6bd95b4 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 12 Aug 2018 12:44:57 +0100 Subject: [PATCH 042/212] issue #338: run /etc/environment test with become:true. --- tests/ansible/integration/runner/etc_environment.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/ansible/integration/runner/etc_environment.yml b/tests/ansible/integration/runner/etc_environment.yml index 4c7f64d8..c1195b75 100644 --- a/tests/ansible/integration/runner/etc_environment.yml +++ b/tests/ansible/integration/runner/etc_environment.yml @@ -7,10 +7,6 @@ any_errors_fatal: true gather_facts: true tasks: - - meta: end_play - when: ansible_virtualization_type != "docker" - - # ~/.pam_environment - file: @@ -38,10 +34,14 @@ path: ~/.pam_environment state: absent + # /etc/environment + - meta: end_play + when: ansible_virtualization_type != "docker" - file: path: /etc/environment + become: true state: absent - shell: echo $MAGIC_ETC_ENV @@ -52,6 +52,7 @@ - copy: dest: /etc/environment + become: true content: | MAGIC_ETC_ENV=555 @@ -63,4 +64,5 @@ - file: path: /etc/environment + become: true state: absent From ce058eb8bdde58a1209672e1c6177174d1ecc919 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 12 Aug 2018 13:17:31 +0100 Subject: [PATCH 043/212] Add 'clean' target to makefile. --- tests/ansible/Makefile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/ansible/Makefile b/tests/ansible/Makefile index d9bdc521..00d6a8ab 100644 --- a/tests/ansible/Makefile +++ b/tests/ansible/Makefile @@ -1,10 +1,14 @@ -all: \ - lib/modules/custom_binary_producing_junk \ - lib/modules/custom_binary_producing_json +TARGETS+=lib/modules/custom_binary_producing_junk +TARGETS+=lib/modules/custom_binary_producing_json + +all: clean $(TARGETS) lib/modules/custom_binary_producing_junk: lib/modules.src/custom_binary_producing_junk.c $(CC) -o $@ $< lib/modules/custom_binary_producing_json: lib/modules.src/custom_binary_producing_json.c $(CC) -o $@ $< + +clean: + rm -f $(TARGETS) From da391f054212b22995c091447f80862be9339085 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 12 Aug 2018 13:46:46 +0100 Subject: [PATCH 044/212] tests: fix host limit. --- .../issue_332_ansiblemoduleerror_first_occurrence.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml b/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml index 3a64455e..0162c210 100644 --- a/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml +++ b/tests/ansible/regression/issue_332_ansiblemoduleerror_first_occurrence.yml @@ -2,7 +2,7 @@ # AnsibleModuleError. Ensure file fails correctly. - name: regression/issue_332_ansiblemoduleerror_first_occurrence.yml - hosts: all + hosts: test-targets tasks: - file: path=/usr/bin/does-not-exist mode='a-s' state=file follow=yes ignore_errors: true From 06ae59702c52fc3d9ed554010a260d45733d9012 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 12 Aug 2018 14:01:04 +0100 Subject: [PATCH 045/212] tests: rationalize matrix and rewrite ansible_tests Now all distros run in parallel. --- .travis.yml | 93 +++++----------- .travis/ansible_tests.py | 65 ++++++++++++ .travis/ansible_tests.sh | 66 ------------ .travis/ci_lib.py | 100 ++++++++++++++++++ .../integration/runner/etc_environment.yml | 6 +- tests/image_prep/setup.yml | 4 +- 6 files changed, 195 insertions(+), 139 deletions(-) create mode 100755 .travis/ansible_tests.py delete mode 100755 .travis/ansible_tests.sh create mode 100644 .travis/ci_lib.py diff --git a/.travis.yml b/.travis.yml index 4aed8d0a..1f8c5c6c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,11 +17,21 @@ install: - pip install -r dev_requirements.txt script: -- ${TRAVIS_BUILD_DIR}/.travis/${MODE}_tests.sh +- | + if [ -f "${TRAVIS_BUILD_DIR}/.travis/${MODE}_tests.sh" ]; then + ${TRAVIS_BUILD_DIR}/.travis/${MODE}_tests.sh; + else + ${TRAVIS_BUILD_DIR}/.travis/${MODE}_tests.py; + fi + services: - docker + +# To avoid matrix explosion, just test against oldest->newest and +# newest->oldest in various configuartions. + matrix: include: # Mitogen tests. @@ -34,85 +44,32 @@ matrix: # 2.6 -> 2.7 - python: "2.6" env: MODE=mitogen DISTRO=centos7 - # 2.6 -> 2.6 - - python: "2.6" - env: MODE=mitogen DISTRO=centos6 - # 3.6 -> 2.7 + # 3.6 -> 2.6 - python: "3.6" - env: MODE=mitogen DISTRO=debian + env: MODE=mitogen DISTRO=centos6 # Debops tests. - # 2.4.3.0; 2.7 -> 2.7 - - python: "2.7" - env: MODE=debops_common VER=2.4.3.0 - # 2.5.5; 2.7 -> 2.7 + # 2.4.6.0; 2.7 -> 2.7 - python: "2.7" - env: MODE=debops_common VER=2.6.1 - # 2.5.5; 3.6 -> 2.7 + env: MODE=debops_common VER=2.4.6.0 + # 2.5.7; 3.6 -> 2.7 - python: "3.6" - env: MODE=debops_common VER=2.6.1 + env: MODE=debops_common VER=2.6.2 # ansible_mitogen tests. - # 2.4.3.0; Debian; 2.7 -> 2.7 - - python: "2.7" - env: MODE=ansible VER=2.4.3.0 DISTRO=debian - # 2.5.5; Debian; 2.7 -> 2.7 - - python: "2.7" - env: MODE=ansible VER=2.5.5 DISTRO=debian - # 2.6.0; Debian; 2.7 -> 2.7 - - python: "2.7" - env: MODE=ansible VER=2.6.0 DISTRO=debian - # 2.6.1; Debian; 2.7 -> 2.7 - - python: "2.7" - env: MODE=ansible VER=2.6.1 DISTRO=debian - # Centos 7 Python2 - # Latest + # 2.6 -> {debian, centos6, centos7} - python: "2.6" - env: MODE=ansible VER=2.6.1 DISTRO=centos7 - # Backward Compatiability - - python: "2.7" - env: MODE=ansible VER=2.5.5 DISTRO=centos7 - - python: "2.7" - env: MODE=ansible VER=2.6.0 DISTRO=centos7 - - python: "2.7" - env: MODE=ansible VER=2.6.1 DISTRO=centos7 - - # Centos 7 Python3 - - python: "3.6" - env: MODE=ansible VER=2.5.5 DISTRO=centos7 - - python: "3.6" - env: MODE=ansible VER=2.6.0 DISTRO=centos7 - - python: "3.6" - env: MODE=ansible VER=2.6.1 DISTRO=centos7 - - - # Centos 6 Python2 - # Latest - - python: "2.6" - env: MODE=ansible VER=2.6.1 DISTRO=centos6 - # Backward Compatiability + env: MODE=ansible VER=2.4.6.0 - python: "2.6" - env: MODE=ansible VER=2.5.5 DISTRO=centos6 - - python: "2.6" - env: MODE=ansible VER=2.6.0 DISTRO=centos6 - - python: "2.7" - env: MODE=ansible VER=2.6.1 DISTRO=centos6 + env: MODE=ansible VER=2.6.2 - # Centos 6 Python3 + # 3.6 -> {debian, centos6, centos7} - python: "3.6" - env: MODE=ansible VER=2.5.5 DISTRO=centos6 + env: MODE=ansible VER=2.4.6.0 - python: "3.6" - env: MODE=ansible VER=2.6.0 DISTRO=centos6 - - python: "3.6" - env: MODE=ansible VER=2.6.1 DISTRO=centos6 + env: MODE=ansible VER=2.6.2 - # Sanity check our tests against vanilla Ansible, they should pass. - - python: "2.7" - env: MODE=ansible VER=2.5.5 DISTRO=debian STRATEGY=linear + # Sanity check against vanilla Ansible. One job suffices. - python: "2.7" - env: MODE=ansible VER=2.6.0 DISTRO=debian STRATEGY=linear - - python: "2.7" - env: MODE=ansible VER=2.6.1 DISTRO=debian STRATEGY=linear - - + env: MODE=ansible VER=2.6.2 DISTRO=debian STRATEGY=linear diff --git a/.travis/ansible_tests.py b/.travis/ansible_tests.py new file mode 100755 index 00000000..0c47ab27 --- /dev/null +++ b/.travis/ansible_tests.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# Run tests/ansible/all.yml under Ansible and Ansible-Mitogen + +import os +import sys + +import ci_lib +from ci_lib import run + + +BASE_PORT = 2201 +TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible') +HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts') + + +with ci_lib.Fold('docker_setup'): + for i, distro in enumerate(ci_lib.DISTROS): + try: + run("docker rm -f target-%s", distro) + except: pass + + run(""" + docker run + --rm + --detach + --publish 0.0.0.0:%s:22/tcp + --name=target-%s + mitogen/%s-test + """, BASE_PORT + i, distro, distro,) + + +with ci_lib.Fold('job_setup'): + os.chdir(TESTS_DIR) + os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 7)) + + # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version. + run("pip install -q ansible==%s", ci_lib.ANSIBLE_VERSION) + + run("mkdir %s", HOSTS_DIR) + run("ln -s %s/common-hosts %s", TESTS_DIR, HOSTS_DIR) + + with open(os.path.join(HOSTS_DIR, 'target'), 'w') as fp: + fp.write('[test-targets]\n') + for i, distro in enumerate(ci_lib.DISTROS): + fp.write("target-%s " + "ansible_host=%s " + "ansible_port=%s " + "ansible_user=mitogen__has_sudo_nopw " + "ansible_password=has_sudo_nopw_password" + "\n" % ( + distro, + ci_lib.DOCKER_HOSTNAME, + BASE_PORT + i, + )) + + # Build the binaries. + run("make -C %s", TESTS_DIR) + if not ci_lib.exists_in_path('sshpass'): + run("sudo apt-get update") + run("sudo apt-get install -y sshpass") + + +with ci_lib.Fold('ansible'): + run('/usr/bin/time ./run_ansible_playbook.sh all.yml -i "%s" %s', + HOSTS_DIR, ' '.join(sys.argv[1:])) diff --git a/.travis/ansible_tests.sh b/.travis/ansible_tests.sh deleted file mode 100755 index bc119149..00000000 --- a/.travis/ansible_tests.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash -ex -# Run tests/ansible/all.yml under Ansible and Ansible-Mitogen - -TRAVIS_BUILD_DIR="${TRAVIS_BUILD_DIR:-`pwd`}" -TMPDIR="/tmp/ansible-tests-$$" -ANSIBLE_VERSION="${VER:-2.6.1}" -export ANSIBLE_STRATEGY="${STRATEGY:-mitogen_linear}" -DISTRO="${DISTRO:-debian}" - -export PYTHONPATH="${PYTHONPATH}:${TRAVIS_BUILD_DIR}" - -# SSH passes these through to the container when run interactively, causing -# stdout to get messed up with libc warnings. -unset LANG LC_ALL - -function on_exit() -{ - rm -rf "$TMPDIR" - docker kill target || true -} - -trap on_exit EXIT -mkdir "$TMPDIR" - - -echo travis_fold:start:docker_setup -DOCKER_HOSTNAME="$(python ${TRAVIS_BUILD_DIR}/tests/show_docker_hostname.py)" - -docker run \ - --rm \ - --detach \ - --publish 0.0.0.0:2201:22/tcp \ - --name=target \ - mitogen/${DISTRO}-test -echo travis_fold:end:docker_setup - - -echo travis_fold:start:job_setup -pip install ansible=="${ANSIBLE_VERSION}" -cd ${TRAVIS_BUILD_DIR}/tests/ansible - -chmod go= ${TRAVIS_BUILD_DIR}/tests/data/docker/mitogen__has_sudo_pubkey.key -mkdir ${TMPDIR}/hosts -ln -s ${TRAVIS_BUILD_DIR}/tests/ansible/common-hosts ${TMPDIR}/hosts/common-hosts -echo '[test-targets]' > ${TMPDIR}/hosts/target -echo \ - target \ - ansible_host=$DOCKER_HOSTNAME \ - ansible_port=2201 \ - ansible_user=mitogen__has_sudo_nopw \ - ansible_password=has_sudo_nopw_password \ - >> ${TMPDIR}/hosts/target - -# Build the binaries. -make -C ${TRAVIS_BUILD_DIR}/tests/ansible - -[ ! "$(type -p sshpass)" ] && sudo apt install -y sshpass - -echo travis_fold:end:job_setup - - -echo travis_fold:start:ansible -/usr/bin/time ./run_ansible_playbook.sh \ - all.yml \ - -i "${TMPDIR}/hosts" "$@" -echo travis_fold:end:ansible diff --git a/.travis/ci_lib.py b/.travis/ci_lib.py new file mode 100644 index 00000000..828cae39 --- /dev/null +++ b/.travis/ci_lib.py @@ -0,0 +1,100 @@ + +from __future__ import absolute_import +from __future__ import print_function + +import atexit +import os +import subprocess +import sys +import shlex +import shutil +import tempfile + + +# +# check_output() monkeypatch cutpasted from testlib.py +# + +def subprocess__check_output(*popenargs, **kwargs): + # Missing from 2.6. + process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) + output, _ = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + raise subprocess.CalledProcessError(retcode, cmd) + return output + +if not hasattr(subprocess, 'check_output'): + subprocess.check_output = subprocess__check_output + +# ----------------- + +def _argv(s, *args): + if args: + s %= args + return shlex.split(s) + + +def run(s, *args, **kwargs): + argv = _argv(s, *args) + print('Running: %s' % (argv,)) + return subprocess.check_call(argv, **kwargs) + + +def get_output(s, *args, **kwargs): + argv = _argv(s, *args) + print('Running: %s' % (argv,)) + return subprocess.check_output(argv, **kwargs) + + +def exists_in_path(progname): + return any(os.path.exists(os.path.join(dirname, progname)) + for dirname in os.environ['PATH'].split(os.pathsep)) + + +class TempDir(object): + def __init__(self): + self.path = tempfile.mkdtemp(prefix='mitogen_ci_lib') + atexit.register(self.destroy) + + def destroy(self, rmtree=shutil.rmtree): + rmtree(self.path) + + +class Fold(object): + def __init__(self, name): + self.name = name + + def __enter__(self): + print('travis_fold:start:%s' % (self.name)) + + def __exit__(self, _1, _2, _3): + print('') + print('travis_fold:end:%s' % (self.name)) + + +os.environ.setdefault('ANSIBLE_STRATEGY', + os.environ.get('STRATEGY', 'mitogen_linear')) +ANSIBLE_VERSION = os.environ.get('VER', '2.6.2') +GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +DISTROS = os.environ.get('DISTROS', 'debian centos6 centos7').split() +TMP = TempDir().path + +os.environ['PYTHONDONTWRITEBYTECODE'] = 'x' +os.environ['PYTHONPATH'] = '%s:%s' % ( + os.environ.get('PYTHONPATH', ''), + GIT_ROOT +) + +DOCKER_HOSTNAME = subprocess.check_output([ + sys.executable, + os.path.join(GIT_ROOT, 'tests/show_docker_hostname.py'), +]).decode().strip() + +# SSH passes these through to the container when run interactively, causing +# stdout to get messed up with libc warnings. +os.environ.pop('LANG', None) +os.environ.pop('LC_ALL', None) diff --git a/tests/ansible/integration/runner/etc_environment.yml b/tests/ansible/integration/runner/etc_environment.yml index c1195b75..68ec980a 100644 --- a/tests/ansible/integration/runner/etc_environment.yml +++ b/tests/ansible/integration/runner/etc_environment.yml @@ -41,8 +41,8 @@ - file: path: /etc/environment - become: true state: absent + become: true - shell: echo $MAGIC_ETC_ENV register: echo @@ -52,9 +52,9 @@ - copy: dest: /etc/environment - become: true content: | MAGIC_ETC_ENV=555 + become: true - shell: echo $MAGIC_ENV_ENV register: echo @@ -64,5 +64,5 @@ - file: path: /etc/environment - become: true state: absent + become: true diff --git a/tests/image_prep/setup.yml b/tests/image_prep/setup.yml index 168d583c..7a589239 100644 --- a/tests/image_prep/setup.yml +++ b/tests/image_prep/setup.yml @@ -6,8 +6,8 @@ # Hacktacular.. but easiest place for it with current structure. sudo_group: MacOSX: admin - Debian: wheel - CentOS: sudo + Debian: sudo + CentOS: wheel - import_playbook: _container_setup.yml - import_playbook: _user_accounts.yml From a6995a52883a31ec53c999d5b83d4bf9ffe83834 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 13 Aug 2018 10:36:27 +0100 Subject: [PATCH 046/212] issue #338: refactor env handling into class and fix tests. --- ansible_mitogen/runner.py | 201 +++++++++++------- docs/ansible.rst | 55 +++++ docs/changelog.rst | 9 +- .../integration/runner/etc_environment.yml | 20 +- 4 files changed, 195 insertions(+), 90 deletions(-) diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index aaac4882..26cce71d 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -79,24 +79,112 @@ for symbol in 'res_init', '__res_init': except AttributeError: pass -# For tasks running on Linux machines, with vanilla Ansible, edits to -# /etc/environment and ~/.pam_environment are reflected if become:true, due to -# sudo reinvoking pam_env. If multiplexing is disabled, then edits are also -# reflected with become:false. Rather than emulate existing semantics, simply -# always ensure edits are reflects for the next task. -try: - etc_env_st = os.stat('/etc/environment') -except OSError: - etc_env_st = None +iteritems = getattr(dict, 'iteritems', dict.items) +LOG = logging.getLogger(__name__) -try: - pam_env_st = os.stat(os.path.expanduser('~/.pam_environment')) -except OSError: - pam_env_st = None +class EnvironmentFileWatcher(object): + """ + Usually Ansible edits to /etc/environment and ~/.pam_environment are + reflected in subsequent tasks if become:true or SSH multiplexing is + disabled, due to sudo and/or SSH reinvoking pam_env. Rather than emulate + existing semantics, do our best to ensure edits are always reflected. -iteritems = getattr(dict, 'iteritems', dict.items) -LOG = logging.getLogger(__name__) + This can't perfectly replicate the existing behaviour, but it can safely + update and remove keys that appear to originate in `path`, and that do not + conflict with any existing environment key inherited from elsewhere. + + A more robust future approach may simply be to arrange for the persistent + interpreter to restart when a change is detected. + """ + def __init__(self, path): + self.path = os.path.expanduser(path) + #: Inode data at time of last check. + self._st = self._stat() + #: List of inherited keys appearing to originated from this file. + self._keys = [key for key, value in self._load() + if value == os.environ.get(key)] + LOG.debug('%r installed; existing keys: %r', self, self._keys) + + def __repr__(self): + return 'EnvironmentFileWatcher(%r)' % (self.path,) + + def _stat(self): + try: + return os.stat(self.path) + except OSError: + return None + + def _load(self): + try: + with open(self.path, 'r') as fp: + return list(self._parse(fp)) + except IOError: + return [] + + def _parse(self, fp): + """ + linux-pam-1.3.1/modules/pam_env/pam_env.c#L207 + """ + for line in fp: + # ' #export foo=some var ' -> ['#export', 'foo=some var '] + bits = shlex.split(line, comments=True) + if (not bits) or bits[0].startswith('#'): + continue + + if bits[0] == 'export': + bits.pop(0) + + key, sep, value = (' '.join(bits)).partition('=') + if key and sep: + yield key, value + + def _on_file_changed(self): + LOG.debug('%r: file changed, reloading', self) + for key, value in self._load(): + if key in os.environ: + LOG.debug('%r: existing key %r=%r exists, not setting %r', + self, key, os.environ[key], value) + else: + LOG.debug('%r: setting key %r to %r', self, key, value) + self._keys.append(key) + os.environ[key] = value + + def _remove_existing(self): + """ + When a change is detected, remove keys that existed in the old file. + """ + for key in self._keys: + if key in os.environ: + LOG.debug('%r: removing old key %r', self, key) + del os.environ[key] + self._keys = [] + + def check(self): + """ + Compare the :func:`os.stat` for the pam_env style environmnt file + `path` with the previous result `old_st`, which may be :data:`None` if + the previous stat attempt failed. Reload its contents if the file has + changed or appeared since last attempt. + + :returns: + New :func:`os.stat` result. The new call to :func:`reload_env` should + pass it as the value of `old_st`. + """ + st = self._stat() + if self._st == st: + return + + self._st = st + self._remove_existing() + + if st is None: + LOG.debug('%r: file has disappeared', self) + else: + self._on_file_changed() + +_pam_env_watcher = EnvironmentFileWatcher('~/.pam_environment') +_etc_env_watcher = EnvironmentFileWatcher('/etc/environment') def utf8(s): @@ -121,54 +209,6 @@ def reopen_readonly(fp): os.close(fd) -def parse_env(fp): - """ - Parse /etc/environ using roughly the same syntax as pam_env. - """ - # https://github.com/linux-pam/linux-pam/blob/v1.3.1/modules/pam_env/pam_env.c#L207 - for line in fp: - # ' #export foo=some var ' -> ['#export', 'foo=some var '] - bits = shlex.split(line, comments=True) - if not bits: - continue - - if bits[0] == 'export': - bits.pop(0) - - key, sep, value = (' '.join(bits)).partition('=') - if sep: - os.environ[key] = value - - -def reload_env(old_st, path): - """ - Compare the :func:`os.stat` for the pam_env style environmnt file `path` - with the previous result `old_st`, which may be :data:`None` if the - previous stat attempt failed. Reload its contents if the file has changed - or appeared since last attempt. - - :returns: - New :func:`os.stat` result. The new call to :func:`reload_env` should - pass it as the value of `old_st`. - """ - try: - path = os.path.expanduser(path) - st = os.stat(path) - except OSError: - return None - - if old_st == st: - return old_st - if st is None: - LOG.debug('reload_env(%r): file has disappeared', path) - return st - - LOG.debug('reload_env(%r): file has changed or appeared, reloading', path) - with open(path) as fp: - parse_env(fp) - return st - - class Runner(object): """ Ansible module runner. After instantiation (with kwargs supplied by the @@ -219,30 +259,29 @@ class Runner(object): from the parent, as :meth:`run` may detach prior to beginning execution. The base implementation simply prepares the environment. """ + self._setup_cwd() + self._setup_environ() + + def _setup_cwd(self): + """ + For situations like sudo to a non-privileged account, CWD could be + $HOME of the old account, which could have mode go=, which means it is + impossible to restore the old directory, so don't even try. + """ if self.cwd: - # For situations like sudo to another non-privileged account, the - # CWD could be $HOME of the old account, which could have mode go=, - # which means it is impossible to restore the old directory, so - # don't even bother. os.chdir(self.cwd) - env = dict(self.extra_env or {}) - if self.env: - env.update(self.env) - self._setup_environ() - self._env = TemporaryEnvironment(env) def _setup_environ(self): """ - Ensure /etc/environment and ~/.pam_environment are reloaded if their - content appears to differ since execution of the previous task. This - must happen before TemporaryEnvironment is installed, to ensure changes - persist across tasks. + Apply changes from /etc/environment files before creating a + TemporaryEnvironment to snapshot environment state prior to module run. """ - global etc_env_st - etc_env_st = reload_env(etc_env_st, '/etc/environment') - - global pam_env_st - pam_env_st = reload_env(pam_env_st, '~/.pam_environment') + _pam_env_watcher.check() + _etc_env_watcher.check() + env = dict(self.extra_env or {}) + if self.env: + env.update(self.env) + self._env = TemporaryEnvironment(env) def revert(self): """ diff --git a/docs/ansible.rst b/docs/ansible.rst index a6130873..d66743f4 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -401,6 +401,61 @@ this precisely, to avoid breaking playbooks that expect text to appear in specific variables with a particular linefeed style. +.. _ansible_process_env: + +Process Environment Emulation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Since Ansible discards processes after each module invocation, follow-up tasks +often (but not always) receive a new environment that will usually include +changes made by previous tasks. As such modifications are common, for +compatibility the extension emulates the existing behaviour as closely as +possible. + +Some scenarios exist where emulation is impossible, for example, applying +``nsswitch.conf`` changes when ``nscd`` is not in use. If future scenarios +appear that cannot be solved through emulation, the extension will be updated +to automatically restart affected interpreters instead. + + +DNS Resolution +^^^^^^^^^^^^^^ + +Modifications to ``/etc/resolv.conf`` cause the glibc resolver configuration to +be reloaded via `res_init(3) `_. This +isn't necessary on some Linux distributions carrying glibc patches to +automatically check ``/etc/resolv.conf`` periodically, however it is necessary +on at least Debian and the BSD derivatives. + + +``/etc/environment`` +^^^^^^^^^^^^^^^^^^^^ + +When ``become: true`` is active or SSH multiplexing is disabled, modifications +by previous tasks to ``/etc/environment`` and ``$HOME/.pam_environment`` are +reflected, since the content of those files is reapplied by `PAM +`_ via `pam_env` +on each authentication of ``sudo`` or ``sshd``. + +Both files are monitored for changes, and changes are applied where it appears +safe to do so: + +* New keys are added if they did not otherwise exist in the inherited + environment, or previously had the same value as found in the file before it + changed. + +* Given a key (such as ``http_proxy``) added to the file where no such key + exists in the environment, the key will be added. + +* Given a key (such as ``PATH``) where an existing environment key exists with + a different value, the update or deletion will be ignored, as it is likely + the key was overridden elsewhere after `pam_env` ran, such as by + ``/etc/profile``. + +* Given a key removed from the file that had the same value as the existing + environment key, the key will be removed. + + How Modules Execute ~~~~~~~~~~~~~~~~~~~ diff --git a/docs/changelog.rst b/docs/changelog.rst index 9a78790b..0c1df179 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -56,11 +56,10 @@ Mitogen for Ansible * `#332 `_: support a new :data:`sys.excepthook`-based module exit mechanism added in Ansible 2.6. -* `#338 `_: compatibility: due to - Ansible's implementation, changes to ``/etc/environment`` made by a task are - reflected in the runtime environment of subsequent tasks, but only if those - tasks set ``become: true``, or if SSH multiplexing is disabled. Changes to - ``/etc/environment`` are now monitored and always reflected. +* `#338 `_: compatibility: changes to + ``/etc/environment`` and ``~/.pam_environment`` made by a task are reflected + in the runtime environment of subsequent tasks. See + :ref:`ansible_process_env` for a complete description. * Runs with many targets executed the module dependency scanner redundantly due to missing synchronization, causing significant wasted computation in the diff --git a/tests/ansible/integration/runner/etc_environment.yml b/tests/ansible/integration/runner/etc_environment.yml index 68ec980a..0037698a 100644 --- a/tests/ansible/integration/runner/etc_environment.yml +++ b/tests/ansible/integration/runner/etc_environment.yml @@ -13,7 +13,7 @@ path: ~/.pam_environment state: absent - - shell: echo $MAGIC_NEW_ENV + - shell: echo $MAGIC_PAM_ENV register: echo - assert: @@ -22,9 +22,9 @@ - copy: dest: ~/.pam_environment content: | - MAGIC_NEW_ENV=321 + MAGIC_PAM_ENV=321 - - shell: echo $MAGIC_NEW_ENV + - shell: echo $MAGIC_PAM_ENV register: echo - assert: @@ -34,6 +34,12 @@ path: ~/.pam_environment state: absent + - shell: echo $MAGIC_PAM_ENV + register: echo + + - assert: + that: echo.stdout == "" + # /etc/environment - meta: end_play @@ -56,7 +62,7 @@ MAGIC_ETC_ENV=555 become: true - - shell: echo $MAGIC_ENV_ENV + - shell: echo $MAGIC_ETC_ENV register: echo - assert: @@ -66,3 +72,9 @@ path: /etc/environment state: absent become: true + + - shell: echo $MAGIC_ETC_ENV + register: echo + + - assert: + that: echo.stdout == "" From 2d50270781264fa5ee57080b18619a7d997b1a8e Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 13 Aug 2018 11:29:37 +0100 Subject: [PATCH 047/212] sudo: support '-i' flag. Closes #343. --- mitogen/sudo.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/mitogen/sudo.py b/mitogen/sudo.py index 1ec5c2f6..89291acd 100644 --- a/mitogen/sudo.py +++ b/mitogen/sudo.py @@ -49,7 +49,7 @@ SUDO_OPTIONS = [ #(False, 'str', '--group', '-g') (True, 'bool', '--set-home', '-H'), #(False, 'str', '--host', '-h') - #(False, 'bool', '--login', '-i') + (False, 'bool', '--login', '-i') #(False, 'bool', '--remove-timestamp', '-K') #(False, 'bool', '--reset-timestamp', '-k') #(False, 'bool', '--list', '-l') @@ -116,10 +116,11 @@ class Stream(mitogen.parent.Stream): password = None preserve_env = False set_home = False + login = False def construct(self, username=None, sudo_path=None, password=None, preserve_env=None, set_home=None, sudo_args=None, - **kwargs): + login=None, **kwargs): super(Stream, self).construct(**kwargs) opts = parse_sudo_flags(sudo_args or []) @@ -133,6 +134,8 @@ class Stream(mitogen.parent.Stream): self.preserve_env = preserve_env or opts.preserve_env if (set_home or opts.set_home) is not None: self.set_home = set_home or opts.set_home + if (login or opts.login) is not None: + self.login = True def connect(self): super(Stream, self).connect() @@ -144,13 +147,16 @@ class Stream(mitogen.parent.Stream): def get_boot_command(self): # Note: sudo did not introduce long-format option processing until July - # 2013, so even though we parse long-format options, we always supply - # short-form to the sudo command. + # 2013, so even though we parse long-format options, supply short-form + # to the sudo command. bits = [self.sudo_path, '-u', self.username] if self.preserve_env: bits += ['-E'] if self.set_home: bits += ['-H'] + if self.login: + bits += ['-i'] + bits = bits + super(Stream, self).get_boot_command() LOG.debug('sudo command line: %r', bits) return bits From 30ca569716fd390b8525c37c853de3e609a7fdd1 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 13 Aug 2018 11:33:14 +0100 Subject: [PATCH 048/212] docs: Update Changelog. --- docs/changelog.rst | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0c1df179..f653d366 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -61,6 +61,9 @@ Mitogen for Ansible in the runtime environment of subsequent tasks. See :ref:`ansible_process_env` for a complete description. +* `#343 `_: the sudo ``--login`` + option is supported. + * Runs with many targets executed the module dependency scanner redundantly due to missing synchronization, causing significant wasted computation in the connection multiplexer subprocess. For one real-world playbook the scanner @@ -97,13 +100,14 @@ Thanks! Mitogen would not be possible without the support of users. A huge thanks for the bug reports in this release contributed by -`Rick Box `_, -`Dan Quackenbush `_, `Alex Russu `_, -`Timo Beckers `_, +`Dan Quackenbush `_, `Jesse London `_, -`Pateek Jain `_, and -`Pierre-Henry Muller `_. +`Luca Nunzi `_, +`Pateek Jain `_, +`Pierre-Henry Muller `_, +`Rick Box `_, and +`Timo Beckers `_. v0.2.2 (2018-07-26) From 27c1f3e21e44fd1c5308c64e9d723476db59223d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 13 Aug 2018 12:21:53 +0100 Subject: [PATCH 049/212] sudo: missing comma >:( --- mitogen/sudo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/sudo.py b/mitogen/sudo.py index 89291acd..402d8549 100644 --- a/mitogen/sudo.py +++ b/mitogen/sudo.py @@ -49,7 +49,7 @@ SUDO_OPTIONS = [ #(False, 'str', '--group', '-g') (True, 'bool', '--set-home', '-H'), #(False, 'str', '--host', '-h') - (False, 'bool', '--login', '-i') + (False, 'bool', '--login', '-i'), #(False, 'bool', '--remove-timestamp', '-K') #(False, 'bool', '--reset-timestamp', '-k') #(False, 'bool', '--list', '-l') From 3113bf622830d11739f8a8f40aa6957a4b94ad7b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 13 Aug 2018 13:32:23 +0100 Subject: [PATCH 050/212] tests: fix debops tests (py-apt broken if /var/lbi/apt missing) --- tests/image_prep/_container_setup.yml | 8 +++----- tests/image_prep/_user_accounts.yml | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/image_prep/_container_setup.yml b/tests/image_prep/_container_setup.yml index eabd97a8..db0d3789 100644 --- a/tests/image_prep/_container_setup.yml +++ b/tests/image_prep/_container_setup.yml @@ -13,7 +13,7 @@ fi - hosts: all - strategy: mitogen_linear + strategy: mitogen_free # Can't gather facts before here. gather_facts: true vars: @@ -60,12 +60,10 @@ - command: yum clean all when: distro == "CentOS" - - file: - path: "{{item}}" - state: absent + - shell: rm -rf {{item}}/* with_items: - /var/cache/apt - - /var/lib/apt + - /var/lib/apt/lists when: distro == "Debian" - user: diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 1cb41a86..f9cac85c 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -6,7 +6,7 @@ - hosts: all gather_facts: true - strategy: mitogen_linear + strategy: mitogen_free become: true vars: distro: "{{ansible_distribution}}" From 154dc2e119b8b935afe66b7944d2b051f600bf13 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 13 Aug 2018 13:33:42 +0100 Subject: [PATCH 051/212] tests: fix integration/runner/missing_module.yml on Travis. --- tests/ansible/integration/runner/missing_module.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/ansible/integration/runner/missing_module.yml b/tests/ansible/integration/runner/missing_module.yml index ac72f69e..064a9bf8 100644 --- a/tests/ansible/integration/runner/missing_module.yml +++ b/tests/ansible/integration/runner/missing_module.yml @@ -4,7 +4,11 @@ connection: local tasks: - connection: local - command: ansible -i localhost, localhost -m missing_module + command: | + ansible -vvv + -i "{{inventory_file}}" + test-targets + -m missing_module args: chdir: ../.. register: out From 3d588323ff65045f1ca643f427eec237e9f14cc0 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 17 Aug 2018 11:31:28 +0100 Subject: [PATCH 052/212] issue #340: use expanded delegate_to hostname, not template. PlayContext.delegate_to is the unexpanded template, Ansible doesn't keep a copy of it around anywhere convenient. We either need to re-expand it or take the expanded version that was stored on the Task, which is what is done here. --- ansible_mitogen/connection.py | 55 +++++++++++++------ ansible_mitogen/mixins.py | 1 + tests/ansible/integration/delegation/all.yml | 2 +- .../delegation/delegate_to_template.yml | 41 ++++++++++++++ 4 files changed, 81 insertions(+), 18 deletions(-) create mode 100644 tests/ansible/integration/delegation/delegate_to_template.yml diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index ccfc12b4..d3ae7f04 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -448,7 +448,9 @@ class Connection(ansible.plugins.connection.ConnectionBase): #: Set to 'hostvars' by on_action_run() host_vars = None - #: Set to '_loader.get_basedir()' by on_action_run(). + #: Set to '_loader.get_basedir()' by on_action_run(). Used by mitogen_local + #: to change the working directory to that of the current playbook, + #: matching vanilla Ansible behaviour. loader_basedir = None #: Set after connection to the target context's home directory. @@ -470,11 +472,20 @@ class Connection(ansible.plugins.connection.ConnectionBase): # https://github.com/dw/mitogen/issues/140 self.close() - def on_action_run(self, task_vars, loader_basedir): + def on_action_run(self, task_vars, delegate_to_hostname, loader_basedir): """ Invoked by ActionModuleMixin to indicate a new task is about to start executing. We use the opportunity to grab relevant bits from the task-specific data. + + :param dict task_vars: + Task variable dictionary. + :param str delegate_to_hostname: + :data:`None`, or the template-expanded inventory hostname this task + is being delegated to. A similar variable exists on PlayContext + when ``delegate_to:`` is active, however it is unexpanded. + :param str loader_basedir: + Loader base directory; see :attr:`loader_basedir`. """ self.ansible_ssh_timeout = task_vars.get('ansible_ssh_timeout', C.DEFAULT_TIMEOUT) @@ -488,6 +499,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): self.mitogen_ssh_debug_level = task_vars.get('mitogen_ssh_debug_level') self.inventory_hostname = task_vars['inventory_hostname'] self.host_vars = task_vars['hostvars'] + self.delegate_to_hostname = delegate_to_hostname self.loader_basedir = loader_basedir self.close(new_task=True) @@ -563,6 +575,26 @@ class Connection(ansible.plugins.connection.ConnectionBase): broker=self.broker, ) + def _config_from_direct_connection(self): + """ + """ + return config_from_play_context( + transport=self.transport, + inventory_name=self.inventory_hostname, + connection=self + ) + + def _config_from_delegate_to(self): + return config_from_hostvars( + transport=self._play_context.connection, + inventory_name=self.delegate_to_hostname, + connection=self, + hostvars=self.host_vars[self._play_context.delegate_to], + become_user=(self._play_context.become_user + if self._play_context.become + else None), + ) + def _build_stack(self): """ Construct a list of dictionaries representing the connection @@ -570,22 +602,11 @@ class Connection(ansible.plugins.connection.ConnectionBase): additionally used by the integration tests "mitogen_get_stack" action to fetch the would-be connection configuration. """ - if hasattr(self._play_context, 'delegate_to'): - target_config = config_from_hostvars( - transport=self._play_context.connection, - inventory_name=self._play_context.delegate_to, - connection=self, - hostvars=self.host_vars[self._play_context.delegate_to], - become_user=(self._play_context.become_user - if self._play_context.become - else None), - ) + if self.delegate_to_hostname is not None: + target_config = self._config_from_delegate_to() else: - target_config = config_from_play_context( - transport=self.transport, - inventory_name=self.inventory_hostname, - connection=self - ) + target_config = self._config_from_direct_connection() + stack, _ = self._stack_from_config(target_config) return stack diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 2a9fdac8..f2fa7ec7 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -110,6 +110,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): """ self._connection.on_action_run( task_vars=task_vars, + delegate_to_hostname=self._task.delegate_to, loader_basedir=self._loader.get_basedir(), ) return super(ActionModuleMixin, self).run(tmp, task_vars) diff --git a/tests/ansible/integration/delegation/all.yml b/tests/ansible/integration/delegation/all.yml index 9646d09c..30ea625f 100644 --- a/tests/ansible/integration/delegation/all.yml +++ b/tests/ansible/integration/delegation/all.yml @@ -1,2 +1,2 @@ - +- import_playbook: delegate_to_template.yml - import_playbook: stack_construction.yml diff --git a/tests/ansible/integration/delegation/delegate_to_template.yml b/tests/ansible/integration/delegation/delegate_to_template.yml new file mode 100644 index 00000000..8b1bd9af --- /dev/null +++ b/tests/ansible/integration/delegation/delegate_to_template.yml @@ -0,0 +1,41 @@ +# Ensure templated delegate_to field works. + +- name: integration/delegation/delegate_to_template.yml + vars: + physical_host: "cd-normal-alias" + physical_hosts: ["cd-normal-alias", "cd-normal-normal"] + hosts: test-targets + gather_facts: no + any_errors_fatal: true + tasks: + - mitogen_get_stack: + delegate_to: "{{ physical_host }}" + register: out + + - assert: + that: | + out.result == [ + { + 'kwargs': { + 'check_host_keys': 'ignore', + 'connect_timeout': 10, + 'hostname': 'cd-normal-alias', + 'identity_file': None, + 'password': None, + 'port': None, + 'python_path': None, + 'ssh_args': [ + '-o', + 'ForwardAgent=yes', + '-o', + 'ControlMaster=auto', + '-o', + 'ControlPersist=60s', + ], + 'ssh_debug_level': None, + 'ssh_path': 'ssh', + 'username': None, + }, + 'method': 'ssh', + }, + ] From 06e2e846c50a126d30e8da44f31ceb5008ce5a12 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 17 Aug 2018 11:55:05 +0100 Subject: [PATCH 053/212] parent: don't generate illegal default remote names. getpass.getuser() output may contain slashes, which must be avoided as they break virtualenv when present in argv[0]. Closes #344. --- docs/changelog.rst | 4 ++++ mitogen/parent.py | 16 ++++++++++++++-- tests/parent_test.py | 16 ++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f653d366..9e226f96 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -64,6 +64,9 @@ Mitogen for Ansible * `#343 `_: the sudo ``--login`` option is supported. +* `#344 `_: connections no longer + fail when the parent machine's logged in username contains slashes. + * Runs with many targets executed the module dependency scanner redundantly due to missing synchronization, causing significant wasted computation in the connection multiplexer subprocess. For one real-world playbook the scanner @@ -101,6 +104,7 @@ Thanks! Mitogen would not be possible without the support of users. A huge thanks for the bug reports in this release contributed by `Alex Russu `_, +`atoom `_, `Dan Quackenbush `_, `Jesse London `_, `Luca Nunzi `_, diff --git a/mitogen/parent.py b/mitogen/parent.py index 14e0ef6d..5ff6ced8 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -93,6 +93,19 @@ def get_core_source(): return inspect.getsource(mitogen.core) +def get_default_remote_name(): + """ + Return the default name appearing in argv[0] of remote machines. + """ + s = u'%s@%s:%d' + s %= (getpass.getuser(), socket.gethostname(), os.getpid()) + # In mixed UNIX/Windows environments, the username may contain slashes. + return s.translate({ + ord(u'\\'): ord(u'_'), + ord(u'/'): ord(u'_') + }) + + def is_immediate_child(msg, stream): """ Handler policy that requires messages to arrive only from immediately @@ -765,8 +778,7 @@ class Stream(mitogen.core.Stream): if connect_timeout: self.connect_timeout = connect_timeout if remote_name is None: - remote_name = '%s@%s:%d' - remote_name %= (getpass.getuser(), socket.gethostname(), os.getpid()) + remote_name = get_default_remote_name() if '/' in remote_name or '\\' in remote_name: raise ValueError('remote_name= cannot contain slashes') self.remote_name = remote_name diff --git a/tests/parent_test.py b/tests/parent_test.py index 06eac97e..0b8b5e9a 100644 --- a/tests/parent_test.py +++ b/tests/parent_test.py @@ -5,6 +5,7 @@ import sys import tempfile import time +import mock import unittest2 import testlib @@ -28,6 +29,21 @@ def wait_for_child(pid, timeout=1.0): assert False, "wait_for_child() timed out" +class GetDefaultRemoteNameTest(testlib.TestCase): + func = staticmethod(mitogen.parent.get_default_remote_name) + + @mock.patch('os.getpid') + @mock.patch('getpass.getuser') + @mock.patch('socket.gethostname') + def test_slashes(self, mock_gethostname, mock_getuser, mock_getpid): + # Ensure slashes appearing in the remote name are replaced with + # underscores. + mock_gethostname.return_value = 'box' + mock_getuser.return_value = 'ECORP\\Administrator' + mock_getpid.return_value = 123 + self.assertEquals("ECORP_Administrator@box:123", self.func()) + + class ReapChildTest(testlib.RouterMixin, testlib.TestCase): def test_connect_timeout(self): # Ensure the child process is reaped if the connection times out. From a561fb79e5bdcc60b79fea48e79ec5f437c75dda Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 18 Aug 2018 14:15:37 +0100 Subject: [PATCH 054/212] docs: merge more docs back into mitogen/core.py. --- docs/api.rst | 32 ++++------------- docs/conf.py | 2 +- docs/internals.rst | 47 +++++++++--------------- docs/signals.rst | 12 ++----- mitogen/core.py | 89 ++++++++++++++++++++++++++++++++++++++++++---- 5 files changed, 109 insertions(+), 73 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index ed9509fc..d605e128 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1411,29 +1411,9 @@ Exceptions .. currentmodule:: mitogen.core -.. class:: Error (fmt, \*args) - - Base for all exceptions raised by Mitogen. - -.. class:: CallError (e) - - Raised when :py:meth:`Context.call() ` fails. - A copy of the traceback from the external context is appended to the - exception message. - -.. class:: ChannelError (fmt, \*args) - - Raised when a channel dies or has been closed. - -.. class:: LatchError (fmt, \*args) - - Raised when an attempt is made to use a :py:class:`mitogen.core.Latch` that - has been marked closed. - -.. class:: StreamError (fmt, \*args) - - Raised when a stream cannot be established. - -.. class:: TimeoutError (fmt, \*args) - - Raised when a timeout occurs on a stream. +.. autoclass:: Error +.. autoclass:: CallError +.. autoclass:: ChannelError +.. autoclass:: LatchError +.. autoclass:: StreamError +.. autoclass:: TimeoutError diff --git a/docs/conf.py b/docs/conf.py index 57adf597..abb6e97e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -19,7 +19,7 @@ html_theme_options = { 'head_font_family': "Georgia, serif", } htmlhelp_basename = 'mitogendoc' -intersphinx_mapping = {'python': ('https://docs.python.org/2', None)} +intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} language = None master_doc = 'toc' project = u'Mitogen' diff --git a/docs/internals.rst b/docs/internals.rst index 3d4d4130..7c3809bc 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -8,17 +8,20 @@ Internal API Reference signals -mitogen.core -============ - - Latch Class ------------ +=========== .. currentmodule:: mitogen.core +.. autoclass:: Latch + :members: -.. autoclass:: Latch () +PidfulStreamHandler Class +========================= + +.. currentmodule:: mitogen.core +.. autoclass:: PidfulStreamHandler + :members: Side Class @@ -354,40 +357,22 @@ context. They will eventually be replaced with asynchronous equivalents. Helper Functions ----------------- +================ .. currentmodule:: mitogen.core +.. autofunction:: to_text +.. autofunction:: has_parent_authority +.. autofunction:: set_cloexec +.. autofunction:: set_nonblock +.. autofunction:: set_block +.. autofunction:: io_op -.. function:: io_op (func, \*args) - - Wrap a function that may raise :py:class:`OSError`, trapping common error - codes relating to disconnection events in various subsystems: - - * When performing IO against a TTY, disconnection of the remote end is - signalled by :py:data:`errno.EIO`. - * When performing IO against a socket, disconnection of the remote end is - signalled by :py:data:`errno.ECONNRESET`. - * When performing IO against a pipe, disconnection of the remote end is - signalled by :py:data:`errno.EPIPE`. - - :returns: - Tuple of `(return_value, disconnected)`, where `return_value` is the - return value of `func(\*args)`, and `disconnected` is ``True`` if - disconnection was detected, otherwise ``False``. .. currentmodule:: mitogen.parent .. autofunction:: create_child - - -.. currentmodule:: mitogen.parent - .. autofunction:: tty_create_child - - -.. currentmodule:: mitogen.parent - .. autofunction:: hybrid_tty_create_child diff --git a/docs/signals.rst b/docs/signals.rst index 1c41353a..19533bb1 100644 --- a/docs/signals.rst +++ b/docs/signals.rst @@ -19,16 +19,10 @@ functions registered to receive it will be called back. Functions --------- -.. function:: mitogen.core.listen (obj, name, func) - - Arrange for `func(\*args, \*\*kwargs)` to be invoked when the named signal - is fired by `obj`. - -.. function:: mitogen.core.fire (obj, name, \*args, \*\*kwargs) - - Arrange for `func(\*args, \*\*kwargs)` to be invoked for every function - registered for the named signal on `obj`. +.. currentmodule:: mitogen.core +.. autofunction:: listen +.. autofunction:: fire List diff --git a/mitogen/core.py b/mitogen/core.py index 261f2621..03b892f0 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -131,6 +131,13 @@ else: class Error(Exception): + """Base for all exceptions raised by Mitogen. + + :param str fmt: + Exception text, or format string if `args` is non-empty. + :param tuple args: + Format string arguments. + """ def __init__(self, fmt=None, *args): if args: fmt %= args @@ -140,10 +147,13 @@ class Error(Exception): class LatchError(Error): - pass + """Raised when an attempt is made to use a :py:class:`mitogen.core.Latch` + that has been marked closed.""" class Blob(BytesType): + """A serializable bytes subclass whose content is summarized in repr() + output, making it suitable for logging binary data.""" def __repr__(self): return '[blob: %d bytes]' % len(self) @@ -152,6 +162,8 @@ class Blob(BytesType): class Secret(UnicodeType): + """A serializable unicode subclass whose content is masked in repr() + output, making it suitable for logging passwords.""" def __repr__(self): return '[secret]' @@ -165,6 +177,10 @@ class Secret(UnicodeType): class Kwargs(dict): + """A serializable dict subclass that indicates the contained keys should be + be coerced to Unicode on Python 3 as required. Python 2 produces keyword + argument dicts whose keys are bytestrings, requiring a helper to ensure + compatibility with Python 3.""" if PY3: def __init__(self, dct): for k, v in dct.items(): @@ -181,6 +197,10 @@ class Kwargs(dict): class CallError(Error): + """Serializable :class:`Error` subclass raised when + :py:meth:`Context.call() ` fails. A copy of + the traceback from the external context is appended to the exception + message.""" def __init__(self, fmt=None, *args): if not isinstance(fmt, BaseException): Error.__init__(self, fmt, *args) @@ -207,37 +227,52 @@ def _unpickle_call_error(s): class ChannelError(Error): + """Raised when a channel dies or has been closed.""" remote_msg = 'Channel closed by remote end.' local_msg = 'Channel closed by local end.' class StreamError(Error): - pass + """Raised when a stream cannot be established.""" class TimeoutError(Error): - pass + """Raised when a timeout occurs on a stream.""" def to_text(o): - if isinstance(o, UnicodeType): - return UnicodeType(o) + """Coerce `o` to Unicode by decoding it from UTF-8 if it is an instance of + :class:`bytes`, otherwise pass it to the :class:`str` constructor. The + returned object is always a plain :class:`str`, any subclass is removed.""" if isinstance(o, BytesType): return o.decode('utf-8') return UnicodeType(o) def has_parent_authority(msg, _stream=None): + """Policy function for use with :class:`Receiver` and + :meth:`Router.add_handler` that requires incoming messages to originate + from a parent context, or on a :class:`Stream` whose :attr:`auth_id + ` has been set to that of a parent context or the current + context.""" return (msg.auth_id == mitogen.context_id or msg.auth_id in mitogen.parent_ids) def listen(obj, name, func): + """ + Arrange for `func(*args, **kwargs)` to be invoked when the named signal is + fired by `obj`. + """ signals = vars(obj).setdefault('_signals', {}) signals.setdefault(name, []).append(func) def fire(obj, name, *args, **kwargs): + """ + Arrange for `func(*args, **kwargs)` to be invoked for every function + registered for the named signal on `obj`. + """ signals = vars(obj).get('_signals', {}) return [func(*args, **kwargs) for func in signals.get(name, ())] @@ -253,7 +288,8 @@ def takes_router(func): def is_blacklisted_import(importer, fullname): - """Return ``True`` if `fullname` is part of a blacklisted package, or if + """ + Return :data:`True` if `fullname` is part of a blacklisted package, or if any packages have been whitelisted and `fullname` is not part of one. NB: @@ -266,22 +302,51 @@ def is_blacklisted_import(importer, fullname): def set_cloexec(fd): + """Set the file descriptor `fd` to automatically close on + :func:`os.execve`. This has no effect on file descriptors inherited across + :func:`os.fork`, they must be explicitly closed through some other means, + such as :func:`mitogen.fork.on_fork`.""" flags = fcntl.fcntl(fd, fcntl.F_GETFD) assert fd > 2 fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) def set_nonblock(fd): + """Set the file descriptor `fd` to non-blocking mode. For most underlying + file types, this causes :func:`os.read` or :func:`os.write` to raise + :class:`OSError` with :data:`errno.EAGAIN` rather than block the thread + when the underlying kernel buffer is exhausted.""" flags = fcntl.fcntl(fd, fcntl.F_GETFL) fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) def set_block(fd): + """Inverse of :func:`set_nonblock`, i.e. cause `fd` to block the thread + when the underlying kernel buffer is exhausted.""" flags = fcntl.fcntl(fd, fcntl.F_GETFL) fcntl.fcntl(fd, fcntl.F_SETFL, flags & ~os.O_NONBLOCK) def io_op(func, *args): + """Wrap `func(*args)` that may raise :class:`select.error`, + :class:`IOError`, or :class:`OSError`, trapping UNIX error codes relating + to disconnection and retry events in various subsystems: + + * When a signal is delivered to the process on Python 2, system call retry + is signalled through :data:`errno.EINTR`. The invocation is automatically + restarted. + * When performing IO against a TTY, disconnection of the remote end is + signalled by :data:`errno.EIO`. + * When performing IO against a socket, disconnection of the remote end is + signalled by :data:`errno.ECONNRESET`. + * When performing IO against a pipe, disconnection of the remote end is + signalled by :data:`errno.EPIPE`. + + :returns: + Tuple of `(return_value, disconnected)`, where `return_value` is the + return value of `func(\*args)`, and `disconnected` is :data:`True` if + disconnection was detected, otherwise :data:`False`. + """ while True: try: return func(*args), False @@ -296,7 +361,19 @@ def io_op(func, *args): class PidfulStreamHandler(logging.StreamHandler): + """A :class:`logging.StreamHandler` subclass used when + :meth:`Router.enable_debug() ` has been + called, or the `debug` parameter was specified during context construction. + Verifies the process ID has not changed on each call to :meth:`emit`, + reopening the associated log file when a change is detected. + + This ensures logging to the per-process output files happens correctly even + when uncooperative third party components call :func:`os.fork`. + """ + #: PID that last opened the log file. open_pid = None + + #: Output path template. template = '/tmp/mitogen.%s.%s.log' def _reopen(self): From 8b800e4798c08a3d75a78a818fc8678fd2aeb5c4 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 18 Aug 2018 14:17:41 +0100 Subject: [PATCH 055/212] add --dump to preamble_size.py. --- preamble_size.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/preamble_size.py b/preamble_size.py index 66d5ccf3..bf3b5950 100644 --- a/preamble_size.py +++ b/preamble_size.py @@ -4,6 +4,7 @@ contexts. """ import inspect +import sys import zlib import mitogen.fakessh @@ -24,6 +25,10 @@ print('Preamble size: %s (%.2fKiB)' % ( len(stream.get_preamble()), len(stream.get_preamble()) / 1024.0, )) +if '--dump' in sys.argv: + print(zlib.decompress(stream.get_preamble())) + exit() + print( ' ' From 442d88e3d7d9b1be19e52e2fd92d58f9b30aae11 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 18 Aug 2018 14:33:55 +0100 Subject: [PATCH 056/212] docs: many more fixes/merges. --- docs/api.rst | 59 +++++++++++++------------- docs/howitworks.rst | 15 +++---- docs/internals.rst | 100 +++++++++----------------------------------- mitogen/core.py | 2 +- mitogen/master.py | 17 ++++++-- mitogen/minify.py | 12 ++++-- mitogen/parent.py | 52 ++++++++++++++++++++++- mitogen/service.py | 5 ++- 8 files changed, 136 insertions(+), 126 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index d605e128..7a4c130c 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -238,16 +238,16 @@ Router Class .. method:: add_handler (fn, handle=None, persist=True, respondent=None, policy=None) Invoke `fn(msg)` for each Message sent to `handle` from this context. - Unregister after one invocation if `persist` is ``False``. If `handle` - is ``None``, a new handle is allocated and returned. + Unregister after one invocation if `persist` is :data:`False`. If + `handle` is :data:`None`, a new handle is allocated and returned. :param int handle: - If not ``None``, an explicit handle to register, usually one of the - ``mitogen.core.*`` constants. If unspecified, a new unused handle - will be allocated. + If not :data:`None`, an explicit handle to register, usually one of + the ``mitogen.core.*`` constants. If unspecified, a new unused + handle will be allocated. :param bool persist: - If ``False``, the handler will be unregistered after a single + If :data:`False`, the handler will be unregistered after a single message has been received. :param mitogen.core.Context respondent: @@ -281,7 +281,8 @@ Router Class sender indicating refusal occurred. :return: - `handle`, or if `handle` was ``None``, the newly allocated handle. + `handle`, or if `handle` was :data:`None`, the newly allocated + handle. .. method:: del_handler (handle) @@ -300,10 +301,10 @@ Router Class called from the I/O multiplexer thread. :param mitogen.core.Stream stream: - If not ``None``, a reference to the stream the message arrived on. - Used for performing source route verification, to ensure sensitive - messages such as ``CALL_FUNCTION`` arrive only from trusted - contexts. + If not :data:`None`, a reference to the stream the message arrived + on. Used for performing source route verification, to ensure + sensitive messages such as ``CALL_FUNCTION`` arrive only from + trusted contexts. .. method:: route(msg) @@ -515,8 +516,8 @@ Router Class otherwise. :param mitogen.core.Context via: - If not ``None``, arrange for construction to occur via RPCs made to - the context `via`, and for :py:data:`ADD_ROUTE + If not :data:`None`, arrange for construction to occur via RPCs + made to the context `via`, and for :py:data:`ADD_ROUTE ` messages to be generated as appropriate. .. code-block:: python @@ -567,7 +568,7 @@ Router Class :data:`None`, which Docker interprets as ``root``. :param str image: Image tag to use to construct a temporary container. Defaults to - ``None``. + :data:`None`. :param str docker_path: Filename or complete path to the Docker binary. ``PATH`` will be searched if given as a filename. Defaults to ``docker``. @@ -596,7 +597,7 @@ Router Class Accepts all parameters accepted by :py:meth:`local`, in addition to: :param str container: - Existing container to connect to. Defaults to ``None``. + Existing container to connect to. Defaults to :data:`None`. :param str lxc_attach_path: Filename or complete path to the ``lxc-attach`` binary. ``PATH`` will be searched if given as a filename. Defaults to @@ -610,7 +611,7 @@ Router Class Accepts all parameters accepted by :py:meth:`local`, in addition to: :param str container: - Existing container to connect to. Defaults to ``None``. + Existing container to connect to. Defaults to :data:`None`. :param str lxc_path: Filename or complete path to the ``lxc`` binary. ``PATH`` will be searched if given as a filename. Defaults to ``lxc``. @@ -790,7 +791,7 @@ Context Class handle which is placed in the message's `reply_to`. :param bool persist: - If ``False``, the handler will be unregistered after a single + If :data:`False`, the handler will be unregistered after a single message has been received. :param mitogen.core.Message msg: @@ -809,7 +810,7 @@ Context Class The message. :param float deadline: - If not ``None``, seconds before timing out waiting for a reply. + If not :data:`None`, seconds before timing out waiting for a reply. :raises mitogen.core.TimeoutError: No message was received and `deadline` passed. @@ -931,8 +932,8 @@ Receiver Class Router to register the handler on. :param int handle: - If not ``None``, an explicit handle to register, otherwise an unused - handle is chosen. + If not :data:`None`, an explicit handle to register, otherwise an + unused handle is chosen. :param bool persist: If :data:`True`, do not unregister the receiver's handler after the @@ -940,13 +941,13 @@ Receiver Class :param mitogen.core.Context respondent: Reference to the context this receiver is receiving from. If not - ``None``, arranges for the receiver to receive a dead message if + :data:`None`, arranges for the receiver to receive a dead message if messages can no longer be routed to the context, due to disconnection or exit. .. attribute:: notify = None - If not ``None``, a reference to a function invoked as + If not :data:`None`, a reference to a function invoked as `notify(receiver)` when a new message is delivered to this receiver. Used by :py:class:`mitogen.select.Select` to implement waiting on multiple receivers. @@ -1000,7 +1001,7 @@ Receiver Class Sleep waiting for a message to arrive on this receiver. :param float timeout: - If not ``None``, specifies a timeout in seconds. + If not :data:`None`, specifies a timeout in seconds. :raises mitogen.core.ChannelError: The remote end indicated the channel should be closed, or @@ -1183,10 +1184,10 @@ Select Class message may be posted at any moment between :py:meth:`empty` and :py:meth:`get`. - :py:meth:`empty` may return ``False`` even when :py:meth:`get` would - block if another thread has drained a receiver added to this select. - This can be avoided by only consuming each receiver from a single - thread. + :py:meth:`empty` may return :data:`False` even when :py:meth:`get` + would block if another thread has drained a receiver added to this + select. This can be avoided by only consuming each receiver from a + single thread. .. py:method:: __iter__ (self) @@ -1370,8 +1371,8 @@ A random assortment of utility functions useful on masters and children. variables. See :ref:`logging-env-vars`. :param str path: - If not ``None``, a filesystem path to write logs to. Otherwise, logs - are written to :py:data:`sys.stderr`. + If not :data:`None`, a filesystem path to write logs to. Otherwise, + logs are written to :py:data:`sys.stderr`. :param bool io: If :data:`True`, include extremely verbose IO logs in the output. diff --git a/docs/howitworks.rst b/docs/howitworks.rst index b14ceab7..1d45647f 100644 --- a/docs/howitworks.rst +++ b/docs/howitworks.rst @@ -332,7 +332,7 @@ Masters listen on the following handles: Receives the name of a module to load `fullname`, locates the source code for `fullname`, and routes one or more :py:data:`LOAD_MODULE` messages back towards the sender of the :py:data:`GET_MODULE` request. If lookup fails, - ``None`` is sent instead. + :data:`None` is sent instead. See :ref:`import-preloading` for a deeper discussion of :py:data:`GET_MODULE`/:py:data:`LOAD_MODULE`. @@ -355,12 +355,13 @@ Children listen on the following handles: Receives `(pkg_present, path, compressed, related)` tuples, composed of: - * **pkg_present**: Either ``None`` for a plain ``.py`` module, or a list of - canonical names of submodules existing witin this package. For example, a - :py:data:`LOAD_MODULE` for the :py:mod:`mitogen` package would return a - list like: `["mitogen.core", "mitogen.fakessh", "mitogen.master", ..]`. - This list is used by children to avoid generating useless round-trips due - to Python 2.x's :keyword:`import` statement behavior. + * **pkg_present**: Either :data:`None` for a plain ``.py`` module, or a + list of canonical names of submodules existing witin this package. For + example, a :py:data:`LOAD_MODULE` for the :py:mod:`mitogen` package would + return a list like: `["mitogen.core", "mitogen.fakessh", + "mitogen.master", ..]`. This list is used by children to avoid generating + useless round-trips due to Python 2.x's :keyword:`import` statement + behavior. * **path**: Original filesystem where the module was found on the master. * **compressed**: :py:mod:`zlib`-compressed module source code. * **related**: list of canonical module names on which this module appears diff --git a/docs/internals.rst b/docs/internals.rst index 7c3809bc..7db4262a 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -53,24 +53,24 @@ Side Class .. attribute:: fd - Integer file descriptor to perform IO on, or ``None`` if + Integer file descriptor to perform IO on, or :data:`None` if :py:meth:`close` has been called. .. attribute:: keep_alive - If ``True``, causes presence of this side in :py:class:`Broker`'s + If :data:`True`, causes presence of this side in :py:class:`Broker`'s active reader set to defer shutdown until the side is disconnected. .. method:: fileno - Return :py:attr:`fd` if it is not ``None``, otherwise raise + Return :py:attr:`fd` if it is not :data:`None`, otherwise raise :py:class:`StreamError`. This method is implemented so that :py:class:`Side` can be used directly by :py:func:`select.select`. .. method:: close - Call :py:func:`os.close` on :py:attr:`fd` if it is not ``None``, then - set it to ``None``. + Call :py:func:`os.close` on :py:attr:`fd` if it is not :data:`None`, + then set it to :data:`None`. .. method:: read (n=CHUNK_SIZE) @@ -97,7 +97,8 @@ Side Class in a 0-sized write. :returns: - Number of bytes written, or ``None`` if disconnection was detected. + Number of bytes written, or :data:`None` if disconnection was + detected. Stream Classes @@ -305,55 +306,25 @@ mitogen.master Blocking I/O Functions ----------------------- +====================== These functions exist to support the blocking phase of setting up a new context. They will eventually be replaced with asynchronous equivalents. -.. currentmodule:: mitogen.master - -.. function:: iter_read(fd, deadline=None) - - Return a generator that arranges for up to 4096-byte chunks to be read at a - time from the file descriptor `fd` until the generator is destroyed. - - :param fd: - File descriptor to read from. - - :param deadline: - If not ``None``, an absolute UNIX timestamp after which timeout should - occur. - - :raises mitogen.core.TimeoutError: - Attempt to read beyond deadline. - - :raises mitogen.core.StreamError: - Attempt to read past end of file. - - -.. currentmodule:: mitogen.master - -.. function:: write_all (fd, s, deadline=None) - - Arrange for all of bytestring `s` to be written to the file descriptor - `fd`. - - :param int fd: - File descriptor to write to. - - :param bytes s: - Bytestring to write to file descriptor. +.. currentmodule:: mitogen.parent +.. autofunction:: discard_until +.. autofunction:: iter_read +.. autofunction:: write_all - :param float deadline: - If not ``None``, an absolute UNIX timestamp after which timeout should - occur. - :raises mitogen.core.TimeoutError: - Bytestring could not be written entirely before deadline was exceeded. +Subprocess Creation Functions +============================= - :raises mitogen.core.StreamError: - File descriptor was disconnected before write could complete. +.. currentmodule:: mitogen.parent +.. autofunction:: create_child +.. autofunction:: hybrid_tty_create_child +.. autofunction:: tty_create_child Helper Functions @@ -368,42 +339,11 @@ Helper Functions .. autofunction:: io_op - -.. currentmodule:: mitogen.parent - -.. autofunction:: create_child -.. autofunction:: tty_create_child -.. autofunction:: hybrid_tty_create_child - - .. currentmodule:: mitogen.master - -.. function:: get_child_modules (path) - - Return the suffixes of submodules directly neated beneath of the package - directory at `path`. - - :param str path: - Path to the module's source code on disk, or some PEP-302-recognized - equivalent. Usually this is the module's ``__file__`` attribute, but - is specified explicitly to avoid loading the module. - - :return: - List of submodule name suffixes. - +.. autofunction:: get_child_modules .. currentmodule:: mitogen.minify - -.. autofunction:: minimize_source (source) - - Remove comments and docstrings from Python `source`, preserving line - numbers and syntax of empty blocks. - - :param str source: - The source to minimize. - - :returns str: - The minimized source. +.. autofunction:: minimize_source Signals diff --git a/mitogen/core.py b/mitogen/core.py index 03b892f0..c651b85c 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1013,7 +1013,7 @@ class Stream(BasicStream): :py:class:`BasicStream` subclass implementing mitogen's :ref:`stream protocol `. """ - #: If not ``None``, :py:class:`Router` stamps this into + #: If not :data:`None`, :py:class:`Router` stamps this into #: :py:attr:`Message.auth_id` of every message received on this stream. auth_id = None diff --git a/mitogen/master.py b/mitogen/master.py index d057f7f1..671bee85 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -84,6 +84,17 @@ def _stdlib_paths(): def get_child_modules(path): + """Return the suffixes of submodules directly neated beneath of the package + directory at `path`. + + :param str path: + Path to the module's source code on disk, or some PEP-302-recognized + equivalent. Usually this is the module's ``__file__`` attribute, but + is specified explicitly to avoid loading the module. + + :return: + List of submodule name suffixes. + """ it = pkgutil.iter_modules([os.path.dirname(path)]) return [to_text(name) for _, name, _ in it] @@ -276,7 +287,7 @@ def is_stdlib_path(path): def is_stdlib_name(modname): - """Return ``True`` if `modname` appears to come from the standard + """Return :data:`True` if `modname` appears to come from the standard library.""" if imp.is_builtin(modname) != 0: return True @@ -412,8 +423,8 @@ class ModuleFinder(object): source code. :returns: - Tuple of `(module path, source text, is package?)`, or ``None`` if - the source cannot be found. + Tuple of `(module path, source text, is package?)`, or :data:`None` + if the source cannot be found. """ tup = self._found_cache.get(fullname) if tup: diff --git a/mitogen/minify.py b/mitogen/minify.py index 26ecf62f..a261bf6a 100644 --- a/mitogen/minify.py +++ b/mitogen/minify.py @@ -48,10 +48,16 @@ except ImportError: @lru_cache() def minimize_source(source): - """Remove most comments and docstrings from Python source code. + """Remove comments and docstrings from Python `source`, preserving line + numbers and syntax of empty blocks. + + :param str source: + The source to minimize. + + :returns str: + The minimized source. """ - if not isinstance(source, mitogen.core.UnicodeType): - source = source.decode('utf-8') + source = mitogen.core.to_text(source) tokens = tokenize.generate_tokens(StringIO(source).readline) tokens = strip_comments(tokens) tokens = strip_docstrings(tokens) diff --git a/mitogen/parent.py b/mitogen/parent.py index 5ff6ced8..ef6c69b2 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -297,6 +297,22 @@ def hybrid_tty_create_child(args): def write_all(fd, s, deadline=None): + """Arrange for all of bytestring `s` to be written to the file descriptor + `fd`. + + :param int fd: + File descriptor to write to. + :param bytes s: + Bytestring to write to file descriptor. + :param float deadline: + If not :data:`None`, absolute UNIX timestamp after which timeout should + occur. + + :raises mitogen.core.TimeoutError: + Bytestring could not be written entirely before deadline was exceeded. + :raises mitogen.core.StreamError: + File descriptor was disconnected before write could complete. + """ timeout = None written = 0 poller = PREFERRED_POLLER() @@ -325,6 +341,20 @@ def write_all(fd, s, deadline=None): def iter_read(fds, deadline=None): + """Return a generator that arranges for up to 4096-byte chunks to be read + at a time from the file descriptor `fd` until the generator is destroyed. + + :param int fd: + File descriptor to read from. + :param float deadline: + If not :data:`None`, an absolute UNIX timestamp after which timeout + should occur. + + :raises mitogen.core.TimeoutError: + Attempt to read beyond deadline. + :raises mitogen.core.StreamError: + Attempt to read past end of file. + """ poller = PREFERRED_POLLER() for fd in fds: poller.start_receive(fd) @@ -359,6 +389,24 @@ def iter_read(fds, deadline=None): def discard_until(fd, s, deadline): + """Read chunks from `fd` until one is encountered that ends with `s`. This + is used to skip output produced by ``/etc/profile``, ``/etc/motd`` and + mandatory SSH banners while waiting for :attr:`Stream.EC0_MARKER` to + appear, indicating the first stage is ready to receive the compressed + :mod:`mitogen.core` source. + + :param int fd: + File descriptor to read from. + :param bytes s: + Marker string to discard until encountered. + :param float deadline: + Absolute UNIX timestamp after which timeout should occur. + + :raises mitogen.core.TimeoutError: + Attempt to read beyond deadline. + :raises mitogen.core.StreamError: + Attempt to read past end of file. + """ for buf in iter_read([fd], deadline): if IOLOG.level == logging.DEBUG: for line in buf.splitlines(): @@ -980,7 +1028,9 @@ class Stream(mitogen.core.Stream): self._reap_child() raise - #: For ssh.py, this must be at least max(len('password'), len('debug1:')) + #: Sentinel value emitted by the first stage to indicate it is ready to + #: receive the compressed bootstrap. For :mod:`mitogen.ssh` this must have + #: length of at least `max(len('password'), len('debug1:'))` EC0_MARKER = mitogen.core.b('MITO000\n') EC1_MARKER = mitogen.core.b('MITO001\n') diff --git a/mitogen/service.py b/mitogen/service.py index 923ec04a..f1ccadde 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -372,8 +372,9 @@ class DeduplicatingInvoker(Invoker): class Service(object): - #: Sentinel object to suppress reply generation, since returning ``None`` - #: will trigger a response message containing the pickled ``None``. + #: Sentinel object to suppress reply generation, since returning + #: :data:`None` will trigger a response message containing the pickled + #: :data:`None`. NO_REPLY = object() invoker_class = Invoker From df5342af227ec645ec48a9efe5ab92c5bb8f0b26 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 18 Aug 2018 14:34:29 +0100 Subject: [PATCH 057/212] core: split out _internal_receive() This is needed for libssh2. --- mitogen/core.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index c651b85c..9ea7b695 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1036,6 +1036,16 @@ class Stream(BasicStream): def construct(self): pass + def _internal_receive(self, broker, buf): + if self._input_buf and self._input_buf_len < 128: + self._input_buf[0] += buf + else: + self._input_buf.append(buf) + + self._input_buf_len += len(buf) + while self._receive_one(broker): + pass + def on_receive(self, broker): """Handle the next complete message on the stream. Raise :py:class:`StreamError` on failure.""" @@ -1045,14 +1055,7 @@ class Stream(BasicStream): if not buf: return self.on_disconnect(broker) - if self._input_buf and self._input_buf_len < 128: - self._input_buf[0] += buf - else: - self._input_buf.append(buf) - - self._input_buf_len += len(buf) - while self._receive_one(broker): - pass + self._internal_receive(broker, buf) HEADER_FMT = '>LLLLLL' HEADER_LEN = struct.calcsize(HEADER_FMT) From ec8d759d46c2bb5ba04a1c82544c441bc12fb7c6 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 18 Aug 2018 15:16:12 +0100 Subject: [PATCH 058/212] docs: document one more. --- docs/internals.rst | 3 +++ mitogen/parent.py | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/docs/internals.rst b/docs/internals.rst index 7db4262a..a0ad342d 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -338,6 +338,9 @@ Helper Functions .. autofunction:: set_block .. autofunction:: io_op +.. currentmodule:: mitogen.parent +.. autofunction:: close_nonstandard_fds +.. autofunction:: create_socketpair .. currentmodule:: mitogen.master .. autofunction:: get_child_modules diff --git a/mitogen/parent.py b/mitogen/parent.py index ef6c69b2..44504258 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -157,6 +157,14 @@ def close_nonstandard_fds(): def create_socketpair(): + """ + Create a :func:`socket.socketpair` to use for use as a child process's UNIX + stdio channels. As socket pairs are bidirectional, they are economical on + file descriptor usage as the same descriptor can be used for ``stdin`` and + ``stdout``. As they are sockets their buffers are tunable, allowing large + buffers to be configured in order to improve throughput for file transfers + and reduce :class:`mitogen.core.Broker` IO loop iterations. + """ parentfp, childfp = socket.socketpair() parentfp.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, From 27b64a484be17a58218a831032c7af711a92f0a4 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 18 Aug 2018 16:11:18 +0100 Subject: [PATCH 059/212] docs: document mitogen.core.CHUNK_SIZE. --- docs/api.rst | 8 ++++---- docs/internals.rst | 11 +++++++---- mitogen/core.py | 27 +++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 7a4c130c..9caf3e13 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -87,10 +87,10 @@ Message Class .. class:: Message - Messages are the fundamental unit of communication, comprising the fields - from in the :ref:`stream-protocol` header, an optional reference to the - receiving :class:`mitogen.core.Router` for ingress messages, and helper - methods for deserialization and generating replies. + Messages are the fundamental unit of communication, comprising fields from + the :ref:`stream-protocol` header, an optional reference to the receiving + :class:`mitogen.core.Router` for ingress messages, and helper methods for + deserialization and generating replies. .. attribute:: router diff --git a/docs/internals.rst b/docs/internals.rst index a0ad342d..03f12e1e 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -8,6 +8,13 @@ Internal API Reference signals +Constants +========= + +.. currentmodule:: mitogen.core +.. autodata:: CHUNK_SIZE + + Latch Class =========== @@ -92,10 +99,6 @@ Side Class wrapping the underlying :py:func:`os.write` call with :py:func:`io_op` to trap common disconnection connditions. - :py:meth:`read` always behaves as if it is writing to a regular UNIX - file; socket, pipe, and TTY disconnection errors are masked and result - in a 0-sized write. - :returns: Number of bytes written, or :data:`None` if disconnection was detected. diff --git a/mitogen/core.py b/mitogen/core.py index 9ea7b695..4bb844ab 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -116,7 +116,34 @@ AnyTextType = (BytesType, UnicodeType) if sys.version_info < (2, 5): next = lambda it: it.next() +#: Default size for calls to :meth:`Side.read` or :meth:`Side.write`, and the +#: size of buffers configured by :func:`mitogen.parent.create_socketpair`. This +#: value has many performance implications, 128KiB seems to be a sweet spot. +#: +#: * When set low, large messages cause many :class:`Broker` IO loop +#: iterations, burning CPU and reducing throughput. +#: * When set high, excessive RAM is reserved by the OS for socket buffers (2x +#: per child), and an identically sized temporary userspace buffer is +#: allocated on each read that requires zeroing, and over a particular size +#: may require two system calls to allocate/deallocate. +#: +#: Care must be taken to ensure the underlying kernel object and receiving +#: program support the desired size. For example, +#: +#: * Most UNIXes have TTYs with fixed 2KiB-4KiB buffers, making them unsuitable +#: for efficient IO. +#: * Different UNIXes have varying presets for pipes, which may not be +#: configurable. On recent Linux the default pipe buffer size is 64KiB, but +#: under memory pressure may be as low as 4KiB for unprivileged processes. +#: * When communication is via an intermediary process, its internal buffers +#: effect the speed OS buffers will drain. For example OpenSSH uses 64KiB +#: reads. +#: +#: An ideal :class:`Message` has a size that is a multiple of +#: :data:`CHUNK_SIZE` inclusive of headers, to avoid wasting IO loop iterations +#: writing small trailer chunks. CHUNK_SIZE = 131072 + _tls = threading.local() From 2fcea4b199c7a20d02d3c448555159db03d2e4fa Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 18 Aug 2018 17:21:56 +0100 Subject: [PATCH 060/212] add extra 'pass' statements to work around minify issues. --- mitogen/core.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mitogen/core.py b/mitogen/core.py index 4bb844ab..12983071 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -176,6 +176,7 @@ class Error(Exception): class LatchError(Error): """Raised when an attempt is made to use a :py:class:`mitogen.core.Latch` that has been marked closed.""" + pass class Blob(BytesType): @@ -261,10 +262,12 @@ class ChannelError(Error): class StreamError(Error): """Raised when a stream cannot be established.""" + pass class TimeoutError(Error): """Raised when a timeout occurs on a stream.""" + pass def to_text(o): From a2686b1a2c49ecbe95575b2e73e554a4c8f42b7b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 19 Aug 2018 15:59:35 +0100 Subject: [PATCH 061/212] issue #321: simplify temp directory handling. --- ansible_mitogen/connection.py | 8 ++ ansible_mitogen/mixins.py | 42 ++---- ansible_mitogen/target.py | 44 +++--- docs/ansible.rst | 71 +++++++++- docs/changelog.rst | 7 + tests/ansible/ansible.cfg | 4 - .../integration/action/make_tmp_path.yml | 132 ++++++++++++------ tests/ansible/integration/all.yml | 1 - tests/ansible/integration/remote_tmp/all.yml | 2 - .../remote_tmp/readonly_homedir.yml | 20 --- .../custom_python_detect_environment.py | 1 + 11 files changed, 209 insertions(+), 123 deletions(-) delete mode 100644 tests/ansible/integration/remote_tmp/all.yml delete mode 100644 tests/ansible/integration/remote_tmp/readonly_homedir.yml diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index d3ae7f04..9b6a36a7 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -456,6 +456,9 @@ class Connection(ansible.plugins.connection.ConnectionBase): #: Set after connection to the target context's home directory. home_dir = None + #: Set after connection to the target context's home directory. + _temp_dir = None + def __init__(self, play_context, new_stdin, **kwargs): assert ansible_mitogen.process.MuxProcess.unix_listener_path, ( 'Mitogen connection types may only be instantiated ' @@ -635,6 +638,11 @@ class Connection(ansible.plugins.connection.ConnectionBase): self.fork_context = dct['init_child_result']['fork_context'] self.home_dir = dct['init_child_result']['home_dir'] + self._temp_dir = dct['init_child_result']['temp_dir'] + + def get_temp_dir(self): + self._connect() + return self._temp_dir def _connect(self): """ diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index f2fa7ec7..98cb134e 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -180,48 +180,26 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): """ assert False, "_is_pipelining_enabled() should never be called." - def _get_remote_tmp(self): - """ - Mitogen-only: return the 'remote_tmp' setting. - """ - try: - s = self._connection._shell.get_option('remote_tmp') - except AttributeError: - s = ansible.constants.DEFAULT_REMOTE_TMP # <=2.4.x - - return self._remote_expand_user(s, sudoable=False) - def _make_tmp_path(self, remote_user=None): """ - Replace the base implementation's use of shell to implement mkdtemp() - with an actual call to mkdtemp(). Like vanilla, the directory is always - created in the login account context. + Return the temporary directory created by the persistent interpreter at + startup. """ LOG.debug('_make_tmp_path(remote_user=%r)', remote_user) - # _make_tmp_path() is basically a global stashed away as Shell.tmpdir. - # The copy action plugin violates layering and grabs this attribute - # directly. - self._connection._shell.tmpdir = self._connection.call( - ansible_mitogen.target.make_temp_directory, - base_dir=self._get_remote_tmp(), - use_login_context=True, - ) + self._connection._shell.tmpdir = self._connection.get_temp_dir() LOG.debug('Temporary directory: %r', self._connection._shell.tmpdir) self._cleanup_remote_tmp = True return self._connection._shell.tmpdir def _remove_tmp_path(self, tmp_path): """ - Replace the base implementation's invocation of rm -rf with a call to - shutil.rmtree(). + Stub out the base implementation's invocation of rm -rf, replacing it + with nothing, as the persistent interpreter automatically cleans up + after itself without introducing roundtrips. """ LOG.debug('_remove_tmp_path(%r)', tmp_path) - if tmp_path is None: - tmp_path = self._connection._shell.tmpdir - if self._should_remove_tmp_path(tmp_path): - self.call(shutil.rmtree, tmp_path) - self._connection._shell.tmpdir = None + self._connection._shell.tmpdir = None def _transfer_data(self, remote_path, data): """ @@ -332,7 +310,13 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): env = {} self._compute_environment_string(env) + # Always set _ansible_tmpdir regardless of whether _make_remote_tmp() + # has ever been called. This short-circuits all the .tmpdir logic in + # module_common and ensures no second temporary directory or atexit + # handler is installed. self._connection._connect() + module_args['_ansible_tmpdir'] = self._connection.get_temp_dir() + return ansible_mitogen.planner.invoke( ansible_mitogen.planner.Invocation( action=self, diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index 582bf85e..c5f85c6d 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -69,6 +69,19 @@ import ansible_mitogen.runner LOG = logging.getLogger(__name__) +MAKE_TEMP_FAILED_MSG = ( + "Unable to create a temporary directory for the persistent interpreter.\n" + "This likely means no system-supplied TMP directory can be written to.\n" + "\n" + "The following paths were tried:\n" + " %(namelist)s\n" + "\n" + "The original exception was:\n" + "\n" + "%(exception)s" +) + + #: Set by init_child() to the single temporary directory that will exist for #: the duration of the process. temp_dir = None @@ -204,7 +217,14 @@ def reset_temp_dir(econtext): """ global temp_dir # https://github.com/dw/mitogen/issues/239 - temp_dir = tempfile.mkdtemp(prefix='ansible_mitogen_') + + try: + temp_dir = tempfile.mkdtemp(prefix='ansible_mitogen_') + except IOError: + raise IOError(MAKE_TEMP_FAILED_MSG % { + 'namelist': '\n '.join(tempfile._candidate_tempdir_list()), + 'exception': traceback.format_exc() + }) # This must be reinstalled in forked children too, since the Broker # instance from the parent process does not carry over to the new child. @@ -252,6 +272,7 @@ def init_child(econtext, log_level): return { 'fork_context': _fork_parent, 'home_dir': mitogen.core.to_text(os.path.expanduser('~')), + 'temp_dir': temp_dir, } @@ -416,27 +437,6 @@ def run_module_async(kwargs, job_id, timeout_secs, econtext): arunner.run() -def make_temp_directory(base_dir): - """ - Handle creation of `base_dir` if it is absent, in addition to a unique - temporary directory within `base_dir`. This is the temporary directory that - becomes 'remote_tmp', not the one used by Ansiballz. It always uses the - system temporary directory. - - :returns: - Newly created temporary directory. - """ - # issue #301: remote_tmp may contain $vars. - base_dir = os.path.expandvars(base_dir) - - if not os.path.exists(base_dir): - os.makedirs(base_dir, mode=int('0700', 8)) - return tempfile.mkdtemp( - dir=base_dir, - prefix='ansible-mitogen-tmp-', - ) - - def get_user_shell(): """ For commands executed directly via an SSH command-line, SSH looks up the diff --git a/docs/ansible.rst b/docs/ansible.rst index d66743f4..e0ecce50 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -271,8 +271,8 @@ command line, or as host and group variables. File Transfer ~~~~~~~~~~~~~ -Normally `sftp `_ or -`scp `_ are used to copy files by the +Normally `sftp(1) `_ or +`scp(1) `_ are used to copy files by the `assemble `_, `copy `_, `patch `_, @@ -302,7 +302,7 @@ to rename over any existing file. This ensures the file remains consistent at all times, in the event of a crash, or when overlapping `ansible-playbook` runs deploy differing file contents. -The `sftp `_ and `scp +The `sftp(1) `_ and `scp(1) `_ tools may cause undetected data corruption in the form of truncated files, or files containing intermingled data segments from overlapping runs. As part of normal operation, both tools expose a window @@ -401,6 +401,67 @@ this precisely, to avoid breaking playbooks that expect text to appear in specific variables with a particular linefeed style. +.. _ansible_tempfiles: + +Temporary Files +~~~~~~~~~~~~~~~ + +Ansible creates a variety of temporary files and directories depending on its +operating mode. + +In the best case when pipelining is enabled and no temporary uploads are +required, for each task Ansible will create one directory below a +system-supplied temporary directory returned by :func:`tempfile.mkdtemp`, owned +by the target user account a new-style module intends to execute in. + +In other cases depending on the task type, whether become is active, whether +the target become user is privileged, whether the associated action plugin +needs to upload files, and whether the associated module needs to store files, +Ansible may: + +* Create a directory owned by the SSH user either under ``remote_tmp``, or a + system-default directory, +* Upload action dependencies such as non-new style modules or rendered + templates to that directory via `sftp(1) `_ + or `scp(1) `_. +* Attempt to modify the directory's access control list to grant access to the + target user using `setfacl(1) `_, + requiring that tool to be installed and a supported filesystem to be in use, + or for the ``allow_world_readable_tmpfiles`` setting to be :data:`True`. +* Create a directory owned by the target user either under ``remote_tmp``, or + a system-default directory, if a new-style module needs a temporary directory + and one was not previously created for a supporting file earlier in the + invocation. + +In summary, for each task Ansible may create one or more of: + +* ``~ssh_user//...`` owned by the login user, +* ``$TMPDIR/ansible-tmp-...`` owned by the login user, +* ``$TMPDIR/ansible-tmp-...`` owned by the login user with ACLs permitting + write access by the become user, +* ``~become_user//...`` owned by the become user, +* ``$TMPDIR/ansible__payload_.../`` owned by the become user, +* ``$TMPDIR/ansible-module-tmp-.../`` owned by the become user. + +The extension must create a temporary directory to maintain compatibility with +Ansible, since many modules introspect :data:`sys.argv` in order to find a +directory where they may write temporary files, however for simplicity only +one such directory exists for the lifetime of each interpreter, stored in a +system-supplied temporary directory, and always privately owned by the target +user account. + +The ``remote_tmp`` path is unused, since Ansible does not make exclusive use of +it, existing semantics are untenable, environments exist with read-only home +directories where the default ``remote_tmp`` path (``~/.ansible/tmp``) cannot +be used, and new-style modules always depended on the existence of a +system-supplied directory anyway, so no requirement is introduced by simply +ignoring ``remote_tmp``. + +As the directory is created once at startup, and its content is managed by code +running remotely, no additional network roundtrips are required to create and +destroy it for each task requiring temporary file storage. + + .. _ansible_process_env: Process Environment Emulation @@ -425,7 +486,7 @@ Modifications to ``/etc/resolv.conf`` cause the glibc resolver configuration to be reloaded via `res_init(3) `_. This isn't necessary on some Linux distributions carrying glibc patches to automatically check ``/etc/resolv.conf`` periodically, however it is necessary -on at least Debian and the BSD derivatives. +on at least Debian and BSD derivatives. ``/etc/environment`` @@ -433,7 +494,7 @@ on at least Debian and the BSD derivatives. When ``become: true`` is active or SSH multiplexing is disabled, modifications by previous tasks to ``/etc/environment`` and ``$HOME/.pam_environment`` are -reflected, since the content of those files is reapplied by `PAM +normally reflected, since the content of those files is reapplied by `PAM `_ via `pam_env` on each authentication of ``sudo`` or ``sshd``. diff --git a/docs/changelog.rst b/docs/changelog.rst index 9e226f96..d4ac261c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -44,6 +44,13 @@ Mitogen for Ansible extension that had been installed using the documented steps. Now the bundled library always overrides over any system-installed copy. +* `#321 `_: temporary file handling + has been simplified and additional network roundtrips have been removed, + undoing earlier damage caused by compatibility bug fixes. A single directory + is created once at startup for each persistent interpreter, and the + ``remote_tmp`` setting is always ignored. See :ref:`ansible_tempfiles` for a + complete description. + * `#324 `_: plays with a custom ``module_utils`` would fail due to fallout from the Python 3 port and related tests being disabled. diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index d9224ab7..539964b8 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -17,10 +17,6 @@ timeout = 10 # On Travis, paramiko check fails due to host key checking enabled. host_key_checking = False -# "mitogen-tests" required by integration/runner/remote_tmp.yml -# "$HOME" required by integration/action/make_tmp_path.yml -remote_tmp = $HOME/.ansible/mitogen-tests/ - [ssh_connection] ssh_args = -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s pipelining = True diff --git a/tests/ansible/integration/action/make_tmp_path.yml b/tests/ansible/integration/action/make_tmp_path.yml index 83261208..779280a3 100644 --- a/tests/ansible/integration/action/make_tmp_path.yml +++ b/tests/ansible/integration/action/make_tmp_path.yml @@ -1,63 +1,115 @@ +# +# Ensure _make_tmp_path returns the same result across invocations for a single +# user account, and that the path returned cleans itself up on connection +# termination. +# +# Related bugs prior to the new-style handling: +# https://github.com/dw/mitogen/issues/239 +# https://github.com/dw/mitogen/issues/301 - name: integration/action/make_tmp_path.yml hosts: test-targets any_errors_fatal: true tasks: - - name: "Find out root's homedir." - # Runs first because it blats regular Ansible facts with junk, so - # non-become run fixes that up. - setup: gather_subset=min - become: true - register: root_facts - - - name: "Find regular homedir" - setup: gather_subset=min - register: user_facts - # - # non-become + # non-root # - - action_passthrough: + - name: "Find regular temp path" + action_passthrough: method: _make_tmp_path - register: out + register: tmp_path + + - name: "Write some junk in regular temp path" + shell: hostname > {{tmp_path.result}}/hostname + + - name: "Verify junk did not persist across tasks" + stat: path={{tmp_path.result}}/hostname + register: junk_stat + + - name: "Verify junk did not persist across tasks" + assert: + that: + - not junk_stat.stat.exists + + - name: "Verify temp path hasn't changed since start" + action_passthrough: + method: _make_tmp_path + register: tmp_path2 + + - name: "Verify temp path hasn't changed since start" + assert: + that: + - tmp_path2.result == tmp_path.result + + - name: "Verify temp path changes across connection reset" + mitogen_shutdown_all: + + - name: "Verify temp path changes across connection reset" + action_passthrough: + method: _make_tmp_path + register: tmp_path2 - - assert: - # This string must match ansible.cfg::remote_tmp - that: out.result.startswith("{{user_facts.ansible_facts.ansible_user_dir}}/.ansible/mitogen-tests/") + - name: "Verify temp path changes across connection reset" + assert: + that: + - tmp_path2.result != tmp_path.result - - stat: - path: "{{out.result}}" - register: st + - name: "Verify old path disappears across connection reset" + stat: path={{tmp_path.result}} + register: junk_stat - - assert: - that: st.stat.exists and st.stat.isdir and st.stat.mode == "0700" + - name: "Verify old path disappears across connection reset" + assert: + that: + - not junk_stat.stat.exists - - file: - path: "{{out.result}}" - state: absent + # + # root + # + + - name: "Find root temp path" + become: true + action_passthrough: + method: _make_tmp_path + register: tmp_path_root + + - name: "Verify root temp path differs from regular path" + assert: + that: + - tmp_path2.result != tmp_path_root.result # - # become. make_tmp_path() must evaluate HOME in the context of the SSH - # user, not the become user. + # readonly homedir # - - action_passthrough: + - name: "Try writing to temp directory for the readonly_homedir user" + become: true + become_user: mitogen__readonly_homedir + action_passthrough: method: _make_tmp_path - register: out + register: tmp_path + + - name: "Try writing to temp directory for the readonly_homedir user" become: true + become_user: mitogen__readonly_homedir + shell: hostname > {{tmp_path.result}}/hostname - - assert: - # This string must match ansible.cfg::remote_tmp - that: out.result.startswith("{{user_facts.ansible_facts.ansible_user_dir}}/.ansible/mitogen-tests/") + # + # modules get the same temp dir + # - - stat: - path: "{{out.result}}" - register: st + - name: "Verify modules get the same tmpdir as the action plugin" + action_passthrough: + method: _make_tmp_path + register: tmp_path - - assert: - that: st.stat.exists and st.stat.isdir and st.stat.mode == "0700" + - name: "Verify modules get the same tmpdir as the action plugin" + custom_python_detect_environment: + register: out - - file: - path: "{{out.result}}" - state: absent + # v2.6 related: https://github.com/ansible/ansible/pull/39833 + - name: "Verify modules get the same tmpdir as the action plugin" + assert: + that: + - out.module_tmpdir == tmp_path.result diff --git a/tests/ansible/integration/all.yml b/tests/ansible/integration/all.yml index ffea7a46..4550e203 100644 --- a/tests/ansible/integration/all.yml +++ b/tests/ansible/integration/all.yml @@ -13,7 +13,6 @@ - import_playbook: local/all.yml - import_playbook: module_utils/all.yml - import_playbook: playbook_semantics/all.yml -- import_playbook: remote_tmp/all.yml - import_playbook: runner/all.yml - import_playbook: ssh/all.yml - import_playbook: strategy/all.yml diff --git a/tests/ansible/integration/remote_tmp/all.yml b/tests/ansible/integration/remote_tmp/all.yml deleted file mode 100644 index 5dff88d8..00000000 --- a/tests/ansible/integration/remote_tmp/all.yml +++ /dev/null @@ -1,2 +0,0 @@ - -- import_playbook: readonly_homedir.yml diff --git a/tests/ansible/integration/remote_tmp/readonly_homedir.yml b/tests/ansible/integration/remote_tmp/readonly_homedir.yml deleted file mode 100644 index ffad455a..00000000 --- a/tests/ansible/integration/remote_tmp/readonly_homedir.yml +++ /dev/null @@ -1,20 +0,0 @@ -# https://github.com/dw/mitogen/issues/239 -# While remote_tmp is used in the context of the SSH user by action code -# running on the controller, Ansiballz ignores it and uses the system default -# instead. - -- name: integration/remote_tmp/readonly_homedir.yml - hosts: test-targets - any_errors_fatal: true - tasks: - - custom_python_detect_environment: - become: true - become_user: mitogen__readonly_homedir - register: out - vars: - ansible_become_pass: readonly_homedir_password - - - name: Verify system temp directory was used. - assert: - that: - - out.__file__.startswith("/tmp/ansible_") diff --git a/tests/ansible/lib/modules/custom_python_detect_environment.py b/tests/ansible/lib/modules/custom_python_detect_environment.py index 8fe50bbc..442a8eff 100644 --- a/tests/ansible/lib/modules/custom_python_detect_environment.py +++ b/tests/ansible/lib/modules/custom_python_detect_environment.py @@ -29,6 +29,7 @@ def main(): mitogen_loaded='mitogen.core' in sys.modules, hostname=socket.gethostname(), username=pwd.getpwuid(os.getuid()).pw_name, + module_tmpdir=getattr(module, 'tmpdir', None), ) if __name__ == '__main__': From f24f02ba06e4d1b83b7cf063096f294c59ec157a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 19 Aug 2018 18:50:53 +0100 Subject: [PATCH 062/212] issue #321: take remote_tmp and system_tmpdirs into account. Can't simply ignore these settings as some users may have weird noexec filesystems. --- ansible_mitogen/loaders.py | 2 + ansible_mitogen/mixins.py | 3 +- ansible_mitogen/services.py | 36 +++++++- ansible_mitogen/target.py | 85 +++++++++++++++---- docs/ansible.rst | 35 +++++--- .../custom_python_detect_environment.py | 2 + 6 files changed, 129 insertions(+), 34 deletions(-) diff --git a/ansible_mitogen/loaders.py b/ansible_mitogen/loaders.py index 441e8113..08c59278 100644 --- a/ansible_mitogen/loaders.py +++ b/ansible_mitogen/loaders.py @@ -37,10 +37,12 @@ try: from ansible.plugins.loader import connection_loader from ansible.plugins.loader import module_loader from ansible.plugins.loader import module_utils_loader + from ansible.plugins.loader import shell_loader from ansible.plugins.loader import strategy_loader except ImportError: # Ansible <2.4 from ansible.plugins import action_loader from ansible.plugins import connection_loader from ansible.plugins import module_loader from ansible.plugins import module_utils_loader + from ansible.plugins import shell_loader from ansible.plugins import strategy_loader diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 98cb134e..e0ca63b8 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -315,7 +315,8 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): # module_common and ensures no second temporary directory or atexit # handler is installed. self._connection._connect() - module_args['_ansible_tmpdir'] = self._connection.get_temp_dir() + if not module_args.get('_ansible_tmpdir', object()): + module_args['_ansible_tmpdir'] = self._connection.get_temp_dir() return ansible_mitogen.planner.invoke( ansible_mitogen.planner.Invocation( diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index e95fc226..6c114060 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -46,8 +46,12 @@ import os.path import sys import threading +import ansible.constants + import mitogen import mitogen.service +import mitogen.utils +import ansible_mitogen.loaders import ansible_mitogen.module_finder import ansible_mitogen.target @@ -69,6 +73,19 @@ else: ) +def _get_candidate_temp_dirs(): + # Force load of plugin to ensure ConfigManager has definitions loaded. + ansible_mitogen.loaders.shell_loader.get('sh') + options = ansible.constants.config.get_plugin_options('shell', 'sh') + + # Pre 2.5 this came from ansible.constants. + remote_tmp = (options.get('remote_tmp') or + ansible.constants.DEFAULT_REMOTE_TMP) + dirs = list(options.get('system_tmpdirs', ('/var/tmp', '/tmp'))) + dirs.insert(0, remote_tmp) + return mitogen.utils.cast(dirs) + + class Error(Exception): pass @@ -252,6 +269,18 @@ class ContextService(mitogen.service.Service): for fullname in self.ALWAYS_PRELOAD: self.router.responder.forward_module(context, fullname) + _candidate_temp_dirs = None + + def _get_candidate_temp_dirs(self): + """ + Return a list of locations to try to create the single temporary + directory used by the run. This simply caches the (expensive) plugin + load of :func:`_get_candidate_temp_dirs`. + """ + if self._candidate_temp_dirs is None: + self._candidate_temp_dirs = _get_candidate_temp_dirs() + return self._candidate_temp_dirs + def _connect(self, key, spec, via=None): """ Actual connect implementation. Arranges for the Mitogen connection to @@ -298,8 +327,11 @@ class ContextService(mitogen.service.Service): lambda: self._on_stream_disconnect(stream)) self._send_module_forwards(context) - init_child_result = context.call(ansible_mitogen.target.init_child, - log_level=LOG.getEffectiveLevel()) + init_child_result = context.call( + ansible_mitogen.target.init_child, + log_level=LOG.getEffectiveLevel(), + candidate_temp_dirs=self._get_candidate_temp_dirs(), + ) if os.environ.get('MITOGEN_DUMP_THREAD_STACKS'): from mitogen import debug diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index c5f85c6d..35863cb2 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -70,26 +70,29 @@ import ansible_mitogen.runner LOG = logging.getLogger(__name__) MAKE_TEMP_FAILED_MSG = ( - "Unable to create a temporary directory for the persistent interpreter.\n" - "This likely means no system-supplied TMP directory can be written to.\n" + "Unable to find a useable temporary directory. This likely means no\n" + "system-supplied TMP directory can be written to, or all directories\n" + "were mounted on 'noexec' filesystems.\n" "\n" "The following paths were tried:\n" " %(namelist)s\n" "\n" - "The original exception was:\n" - "\n" - "%(exception)s" + "Please check '-vvv' output for a log of individual path errors." ) -#: Set by init_child() to the single temporary directory that will exist for -#: the duration of the process. -temp_dir = None - #: Initialized to an econtext.parent.Context pointing at a pristine fork of #: the target Python interpreter before it executes any code or imports. _fork_parent = None +#: Set by init_child() to a list of candidate $variable-expanded and +#: tilde-expanded directory paths that may be usable as a temporary directory. +_candidate_temp_dirs = None + +#: Set by reset_temp_dir() to the single temporary directory that will exist +#: for the duration of the process. +temp_dir = None + def get_small_file(context, path): """ @@ -203,6 +206,53 @@ def _on_broker_shutdown(): prune_tree(temp_dir) +def find_good_temp_dir(): + """ + Given a list of candidate temp directories extracted from ``ansible.cfg`` + and stored in _candidate_temp_dirs, combine it with the Python-builtin list + of candidate directories used by :mod:`tempfile`, then iteratively try each + in turn until one is found that is both writeable and executable. + """ + paths = [os.path.expandvars(os.path.expanduser(p)) + for p in _candidate_temp_dirs] + paths.extend(tempfile._candidate_tempdir_list()) + + for path in paths: + try: + tmp = tempfile.NamedTemporaryFile( + prefix='ansible_mitogen_find_good_temp_dir', + dir=path, + ) + except (OSError, IOError) as e: + LOG.debug('temp dir %r unusable: %s', path, e) + continue + + try: + try: + os.chmod(tmp.name, int('0700', 8)) + except OSError as e: + LOG.debug('temp dir %r unusable: %s: chmod failed: %s', + path, e) + continue + + try: + # access(.., X_OK) is sufficient to detect noexec. + if not os.access(tmp.name, os.X_OK): + raise OSError('filesystem appears to be mounted noexec') + except OSError as e: + LOG.debug('temp dir %r unusable: %s: %s', path, e) + continue + + LOG.debug('Selected temp directory: %r (from %r)', path, paths) + return path + finally: + tmp.close() + + raise IOError(MAKE_TEMP_FAILED_MSG % { + 'paths': '\n '.join(paths), + }) + + @mitogen.core.takes_econtext def reset_temp_dir(econtext): """ @@ -218,13 +268,8 @@ def reset_temp_dir(econtext): global temp_dir # https://github.com/dw/mitogen/issues/239 - try: - temp_dir = tempfile.mkdtemp(prefix='ansible_mitogen_') - except IOError: - raise IOError(MAKE_TEMP_FAILED_MSG % { - 'namelist': '\n '.join(tempfile._candidate_tempdir_list()), - 'exception': traceback.format_exc() - }) + basedir = find_good_temp_dir() + temp_dir = tempfile.mkdtemp(prefix='ansible_mitogen_', dir=basedir) # This must be reinstalled in forked children too, since the Broker # instance from the parent process does not carry over to the new child. @@ -232,7 +277,7 @@ def reset_temp_dir(econtext): @mitogen.core.takes_econtext -def init_child(econtext, log_level): +def init_child(econtext, log_level, candidate_temp_dirs): """ Called by ContextService immediately after connection; arranges for the (presently) spotless Python interpreter to be forked, where the newly @@ -245,6 +290,9 @@ def init_child(econtext, log_level): :param int log_level: Logging package level active in the master. + :param list[str] candidate_temp_dirs: + List of $variable-expanded and tilde-expanded directory names to add to + candidate list of temporary directories. :returns: Dict like:: @@ -258,6 +306,9 @@ def init_child(econtext, log_level): the controller will use to start forked jobs, and `home_dir` is the home directory for the active user account. """ + global _candidate_temp_dirs + _candidate_temp_dirs = candidate_temp_dirs + global _fork_parent mitogen.parent.upgrade_router(econtext) _fork_parent = econtext.router.fork() diff --git a/docs/ansible.rst b/docs/ansible.rst index e0ecce50..24b58b1d 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -443,23 +443,30 @@ In summary, for each task Ansible may create one or more of: * ``$TMPDIR/ansible__payload_.../`` owned by the become user, * ``$TMPDIR/ansible-module-tmp-.../`` owned by the become user. -The extension must create a temporary directory to maintain compatibility with -Ansible, since many modules introspect :data:`sys.argv` in order to find a -directory where they may write temporary files, however for simplicity only -one such directory exists for the lifetime of each interpreter, stored in a -system-supplied temporary directory, and always privately owned by the target -user account. - -The ``remote_tmp`` path is unused, since Ansible does not make exclusive use of -it, existing semantics are untenable, environments exist with read-only home -directories where the default ``remote_tmp`` path (``~/.ansible/tmp``) cannot -be used, and new-style modules always depended on the existence of a -system-supplied directory anyway, so no requirement is introduced by simply -ignoring ``remote_tmp``. +A directory must be created in order to maintain compatibility with Ansible, +as many modules introspect :data:`sys.argv` in order to find a directory where +they may write files, however for only one such directory exists for the +lifetime of each interpreter, its location is consistent for each target +account, and it is always privately owned by that account. + +The following candidate paths are tried until one is found that is writeable +and appears live on a filesystem that does not have ``noexec`` enabled: + +1. The ``$variable`` and tilde-expanded ``remote_tmp`` setting from + ``ansible.cfg``. +2. The ``$variable`` and tilde-expanded ``system_tmpdirs`` setting from + ``ansible.cfg``. +3. The ``TMPDIR`` environment variable. +4. The ``TEMP`` environment variable. +5. The ``TMP`` environment variable. +6. ``/tmp`` +7. ``/var/tmp`` +8. ``/usr/tmp`` +9. The current working directory. As the directory is created once at startup, and its content is managed by code running remotely, no additional network roundtrips are required to create and -destroy it for each task requiring temporary file storage. +destroy it for each task requiring temporary storage. .. _ansible_process_env: diff --git a/tests/ansible/lib/modules/custom_python_detect_environment.py b/tests/ansible/lib/modules/custom_python_detect_environment.py index 442a8eff..2da9cddf 100644 --- a/tests/ansible/lib/modules/custom_python_detect_environment.py +++ b/tests/ansible/lib/modules/custom_python_detect_environment.py @@ -3,6 +3,7 @@ # interpreter I run within. from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.basic import get_module_path from ansible.module_utils import six import os @@ -30,6 +31,7 @@ def main(): hostname=socket.gethostname(), username=pwd.getpwuid(os.getuid()).pw_name, module_tmpdir=getattr(module, 'tmpdir', None), + module_path=get_module_path(), ) if __name__ == '__main__': From 4afe2fdc5560c551e4d75dd3b3eb8ebb487c5c77 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 19 Aug 2018 19:03:20 +0100 Subject: [PATCH 063/212] issue #321: fix probable threading issue. --- ansible_mitogen/services.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index 6c114060..952e991a 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -58,6 +58,11 @@ import ansible_mitogen.target LOG = logging.getLogger(__name__) +# Force load of plugin to ensure ConfigManager has definitions loaded. Done +# during module import to ensure a single-threaded environment; PluginLoader +# is not thread-safe. +ansible_mitogen.loaders.shell_loader.get('sh') + if sys.version_info[0] == 3: def reraise(tp, value, tb): @@ -74,8 +79,6 @@ else: def _get_candidate_temp_dirs(): - # Force load of plugin to ensure ConfigManager has definitions loaded. - ansible_mitogen.loaders.shell_loader.get('sh') options = ansible.constants.config.get_plugin_options('shell', 'sh') # Pre 2.5 this came from ansible.constants. From ac9b84d2379a3c54d7bc18b571e6b69b9929506e Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 19 Aug 2018 19:23:40 +0100 Subject: [PATCH 064/212] issue #321: 2.4+ compatibility fixes, disable test on Vanilla. --- ansible_mitogen/mixins.py | 3 +- ansible_mitogen/runner.py | 3 ++ docs/ansible.rst | 38 +++++++++---------- docs/changelog.rst | 5 +-- .../integration/action/make_tmp_path.yml | 14 ++++++- 5 files changed, 39 insertions(+), 24 deletions(-) diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index e0ca63b8..7d8f5518 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -315,7 +315,8 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): # module_common and ensures no second temporary directory or atexit # handler is installed. self._connection._connect() - if not module_args.get('_ansible_tmpdir', object()): + + if ansible.__version__ > '2.5': module_args['_ansible_tmpdir'] = self._connection.get_temp_dir() return ansible_mitogen.planner.invoke( diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 26cce71d..97a3cb36 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -66,6 +66,9 @@ except ImportError: # Prevent accidental import of an Ansible module from hanging on stdin read. import ansible.module_utils.basic ansible.module_utils.basic._ANSIBLE_ARGS = '{}' +ansible.module_utils.basic.get_module_path = lambda: ( + ansible_mitogen.target.temp_dir +) # For tasks that modify /etc/resolv.conf, non-Debian derivative glibcs cache # resolv.conf at startup and never implicitly reload it. Cope with that via an diff --git a/docs/ansible.rst b/docs/ansible.rst index 24b58b1d..556fd686 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -443,30 +443,30 @@ In summary, for each task Ansible may create one or more of: * ``$TMPDIR/ansible__payload_.../`` owned by the become user, * ``$TMPDIR/ansible-module-tmp-.../`` owned by the become user. -A directory must be created in order to maintain compatibility with Ansible, -as many modules introspect :data:`sys.argv` in order to find a directory where -they may write files, however for only one such directory exists for the -lifetime of each interpreter, its location is consistent for each target -account, and it is always privately owned by that account. - -The following candidate paths are tried until one is found that is writeable -and appears live on a filesystem that does not have ``noexec`` enabled: - -1. The ``$variable`` and tilde-expanded ``remote_tmp`` setting from - ``ansible.cfg``. -2. The ``$variable`` and tilde-expanded ``system_tmpdirs`` setting from - ``ansible.cfg``. -3. The ``TMPDIR`` environment variable. -4. The ``TEMP`` environment variable. -5. The ``TMP`` environment variable. +A directory must exist to maintain compatibility with Ansible, as many modules +introspect :data:`sys.argv` to find a directory where they may write files, +however only one directory exists for the lifetime of each interpreter, its +location is consistent for each target account, and it is always privately +owned by that account. + +The paths below are tried until one is found that is writeable and appears live +on a filesystem with ``noexec`` disabled: + +1. ``$variable`` and tilde-expanded ``remote_tmp`` setting from + ``ansible.cfg`` +2. ``$variable`` and tilde-expanded ``system_tmpdirs`` setting from + ``ansible.cfg`` +3. ``TMPDIR`` environment variable +4. ``TEMP`` environment variable +5. ``TMP`` environment variable 6. ``/tmp`` 7. ``/var/tmp`` 8. ``/usr/tmp`` -9. The current working directory. +9. Current working directory As the directory is created once at startup, and its content is managed by code -running remotely, no additional network roundtrips are required to create and -destroy it for each task requiring temporary storage. +running remotely, no additional network roundtrips are required to manage it +for each task requiring temporary storage. .. _ansible_process_env: diff --git a/docs/changelog.rst b/docs/changelog.rst index d4ac261c..d86b3358 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -47,9 +47,8 @@ Mitogen for Ansible * `#321 `_: temporary file handling has been simplified and additional network roundtrips have been removed, undoing earlier damage caused by compatibility bug fixes. A single directory - is created once at startup for each persistent interpreter, and the - ``remote_tmp`` setting is always ignored. See :ref:`ansible_tempfiles` for a - complete description. + is created once at startup for each persistent interpreter. See + :ref:`ansible_tempfiles` for a complete description. * `#324 `_: plays with a custom ``module_utils`` would fail due to fallout from the Python 3 port and related diff --git a/tests/ansible/integration/action/make_tmp_path.yml b/tests/ansible/integration/action/make_tmp_path.yml index 779280a3..eb39068b 100644 --- a/tests/ansible/integration/action/make_tmp_path.yml +++ b/tests/ansible/integration/action/make_tmp_path.yml @@ -11,6 +11,9 @@ hosts: test-targets any_errors_fatal: true tasks: + - meta: end_play + when: not is_mitogen + # # non-root # @@ -109,7 +112,16 @@ register: out # v2.6 related: https://github.com/ansible/ansible/pull/39833 - - name: "Verify modules get the same tmpdir as the action plugin" + - name: "Verify modules get the same tmpdir as the action plugin (<2.5)" + when: ansible_version.full < '2.5' + assert: + that: + - out.module_path == tmp_path.result + - out.module_tmpdir == None + + - name: "Verify modules get the same tmpdir as the action plugin (>2.5)" + when: ansible_version.full > '2.5' assert: that: + - out.module_path == tmp_path.result - out.module_tmpdir == tmp_path.result From 77b68f9b9d9957fcf578c2b8d8db95a578d2c4e2 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 19 Aug 2018 19:32:38 +0100 Subject: [PATCH 065/212] issue #321: docs fixes --- docs/ansible.rst | 6 +++--- docs/changelog.rst | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/ansible.rst b/docs/ansible.rst index 556fd686..bdd05307 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -412,7 +412,7 @@ operating mode. In the best case when pipelining is enabled and no temporary uploads are required, for each task Ansible will create one directory below a system-supplied temporary directory returned by :func:`tempfile.mkdtemp`, owned -by the target user account a new-style module intends to execute in. +by the target account a new-style module will execute in. In other cases depending on the task type, whether become is active, whether the target become user is privileged, whether the associated action plugin @@ -449,8 +449,8 @@ however only one directory exists for the lifetime of each interpreter, its location is consistent for each target account, and it is always privately owned by that account. -The paths below are tried until one is found that is writeable and appears live -on a filesystem with ``noexec`` disabled: +The paths below are tried until one is found that is writeable and lives on a +filesystem with ``noexec`` disabled: 1. ``$variable`` and tilde-expanded ``remote_tmp`` setting from ``ansible.cfg`` diff --git a/docs/changelog.rst b/docs/changelog.rst index d86b3358..f3e0dab5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -46,9 +46,9 @@ Mitogen for Ansible * `#321 `_: temporary file handling has been simplified and additional network roundtrips have been removed, - undoing earlier damage caused by compatibility bug fixes. A single directory - is created once at startup for each persistent interpreter. See - :ref:`ansible_tempfiles` for a complete description. + undoing earlier damage caused by compatibility fixes, and improving 2.6 + compatibility. One directory is created at startup for each persistent + interpreter. See :ref:`ansible_tempfiles` for a complete description. * `#324 `_: plays with a custom ``module_utils`` would fail due to fallout from the Python 3 port and related From 4318c578d3b3eef827c9e338507328378d79ed37 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 19 Aug 2018 19:53:02 +0100 Subject: [PATCH 066/212] tests: add playbook step to ensure key file perms. --- tests/ansible/integration/ssh/variables.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/ansible/integration/ssh/variables.yml b/tests/ansible/integration/ssh/variables.yml index 110d3340..dc4fe434 100644 --- a/tests/ansible/integration/ssh/variables.yml +++ b/tests/ansible/integration/ssh/variables.yml @@ -101,6 +101,11 @@ when: is_mitogen + - name: ansible_ssh_private_key_file + shell: chmod 0600 ../data/docker/mitogen__has_sudo_pubkey.key + args: + chdir: ../.. + - name: ansible_ssh_private_key_file shell: > ANSIBLE_STRATEGY=mitogen_linear From a4ed27fa63be76882dea0e10dbe118a13066c114 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 19 Aug 2018 19:53:02 +0100 Subject: [PATCH 067/212] tests: add playbook step to ensure key file perms. --- tests/ansible/integration/ssh/variables.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/ansible/integration/ssh/variables.yml b/tests/ansible/integration/ssh/variables.yml index 110d3340..dc4fe434 100644 --- a/tests/ansible/integration/ssh/variables.yml +++ b/tests/ansible/integration/ssh/variables.yml @@ -101,6 +101,11 @@ when: is_mitogen + - name: ansible_ssh_private_key_file + shell: chmod 0600 ../data/docker/mitogen__has_sudo_pubkey.key + args: + chdir: ../.. + - name: ansible_ssh_private_key_file shell: > ANSIBLE_STRATEGY=mitogen_linear From bc682ce5a0a63bf6f563bb726d1d7f2fa5f3b82d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 19 Aug 2018 20:20:22 +0100 Subject: [PATCH 068/212] docs: update supported versions. --- docs/ansible.rst | 2 +- docs/changelog.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ansible.rst b/docs/ansible.rst index bdd05307..790d0369 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -124,7 +124,7 @@ Testimonials Noteworthy Differences ---------------------- -* Ansible 2.3-2.5 are supported along with Python 2.6, 2.7 or 3.6. Verify your +* Ansible 2.3-2.6 are supported along with Python 2.6, 2.7 or 3.6. Verify your installation is running one of these versions by checking ``ansible --version`` output. diff --git a/docs/changelog.rst b/docs/changelog.rst index f3e0dab5..d755875a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -261,7 +261,7 @@ within a stable series. Mitogen for Ansible ~~~~~~~~~~~~~~~~~~~ -* Support for Ansible 2.3 - 2.5.x and any mixture of Python 2.6, 2.7 or 3.6 on +* Support for Ansible 2.3 - 2.6.x and any mixture of Python 2.6, 2.7 or 3.6 on controller and target nodes. * Drop-in support for many Ansible connection types. From 2e3d04bbb82321e13df137e3348b887c2606da94 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 19 Aug 2018 21:26:40 +0100 Subject: [PATCH 069/212] issue #342: forward _create_control_path() to SSH plugin. network_cli connection type loads the "ssh" (mitogen_ssh) plugin and expects a private method to exist. --- ansible_mitogen/plugins/connection/mitogen_ssh.py | 9 +++++++++ docs/changelog.rst | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/ansible_mitogen/plugins/connection/mitogen_ssh.py b/ansible_mitogen/plugins/connection/mitogen_ssh.py index d2af109c..dbaba407 100644 --- a/ansible_mitogen/plugins/connection/mitogen_ssh.py +++ b/ansible_mitogen/plugins/connection/mitogen_ssh.py @@ -42,6 +42,8 @@ DOCUMENTATION = """ options: """ +import ansible.plugins.connection.ssh + try: import ansible_mitogen.connection except ImportError: @@ -54,3 +56,10 @@ import ansible_mitogen.connection class Connection(ansible_mitogen.connection.Connection): transport = 'ssh' + vanilla_class = ansible.plugins.connection.ssh.Connection + + @staticmethod + def _create_control_path(*args, **kwargs): + """Forward _create_control_path() to the implementation in ssh.py.""" + # https://github.com/dw/mitogen/issues/342 + return Connection.vanilla_class._create_control_path(*args, **kwargs) diff --git a/docs/changelog.rst b/docs/changelog.rst index d755875a..fea03073 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -185,6 +185,9 @@ Core Library could spuriously wake up due to ignoring an error bit set on events returned by the kernel, manifesting as a failure to read from an unrelated descriptor. +* `#342 `_: The ``network_cli`` + connection type would fail due to a missing internal SSH plugin method. + * Standard IO forwarding accidentally configured the replacement ``stdout`` and ``stderr`` write descriptors as non-blocking, causing subprocesses that generate more output than kernel buffer space existed to throw errors. The @@ -210,6 +213,7 @@ the bug reports and pull requests in this release contributed by `Ayaz Ahmed Khan `_, `Colin McCarthy `_, `Dan Quackenbush `_, +`dsgnr `_, `Duane Zamrok `_, `falbanese `_, `Gonzalo Servat `_, From 06cae11e520d55f77a96bec61126dce2db5a0669 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 19 Aug 2018 21:45:35 +0100 Subject: [PATCH 070/212] Add freze alabaster version to try fix layout issue. --- docs/docs-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/docs-requirements.txt b/docs/docs-requirements.txt index e4ec81c7..f0bddf36 100644 --- a/docs/docs-requirements.txt +++ b/docs/docs-requirements.txt @@ -1,3 +1,4 @@ Sphinx==1.7.1 sphinx-autobuild==0.6.0 # Last version to support Python 2.6 sphinxcontrib-programoutput==0.11 +alabaster==0.7.10 From e84de489eb8fe0008d79322fde64d8c6dae8df48 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 19 Aug 2018 21:55:36 +0100 Subject: [PATCH 071/212] issue #336: update changelog. Closes #336. --- docs/changelog.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index fea03073..8b13f431 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -44,7 +44,8 @@ Mitogen for Ansible extension that had been installed using the documented steps. Now the bundled library always overrides over any system-installed copy. -* `#321 `_: temporary file handling +* `#321 `_, + `#336 `_: temporary file handling has been simplified and additional network roundtrips have been removed, undoing earlier damage caused by compatibility fixes, and improving 2.6 compatibility. One directory is created at startup for each persistent @@ -112,8 +113,10 @@ the bug reports in this release contributed by `Alex Russu `_, `atoom `_, `Dan Quackenbush `_, +`dsgnr `_, `Jesse London `_, `Luca Nunzi `_, +`nikitakazantsev12 `_, `Pateek Jain `_, `Pierre-Henry Muller `_, `Rick Box `_, and @@ -213,7 +216,6 @@ the bug reports and pull requests in this release contributed by `Ayaz Ahmed Khan `_, `Colin McCarthy `_, `Dan Quackenbush `_, -`dsgnr `_, `Duane Zamrok `_, `falbanese `_, `Gonzalo Servat `_, From bce4f59138d7b7eaaad21e71da5eb1aac4c783e7 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 19 Aug 2018 22:11:24 +0100 Subject: [PATCH 072/212] issue #345: disable IdentitiesOnly by default. --- ansible_mitogen/connection.py | 1 + docs/api.rst | 9 ++++++++- docs/changelog.rst | 7 +++++++ mitogen/ssh.py | 5 +++-- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 9b6a36a7..30d23a4f 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -102,6 +102,7 @@ def _connect_ssh(spec): 'port': spec['port'], 'python_path': spec['python_path'], 'identity_file': spec['private_key_file'], + 'identities_only': False, 'ssh_path': spec['ssh_executable'], 'connect_timeout': spec['ansible_ssh_timeout'], 'ssh_args': spec['ssh_args'], diff --git a/docs/api.rst b/docs/api.rst index 9caf3e13..69e3c07b 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -706,7 +706,7 @@ Router Class :py:class:`mitogen.core.StreamError` to be raised, and that attributes of the stream match the actual behaviour of ``sudo``. - .. method:: ssh (hostname, username=None, ssh_path=None, port=None, check_host_keys='enforce', password=None, identity_file=None, compression=True, \**kwargs) + .. method:: ssh (hostname, username=None, ssh_path=None, port=None, check_host_keys='enforce', password=None, identity_file=None, identities_only=True, compression=True, \**kwargs) Construct a remote context over a ``ssh`` invocation. The ``ssh`` process is started in a newly allocated pseudo-terminal, and supports @@ -744,6 +744,13 @@ Router Class the SSH client to perform authenticaion; agent authentication is automatically disabled, as is reading the default private key from ``~/.ssh/id_rsa``, or ``~/.ssh/id_dsa``. + :param bool identities_only: + If :data:`True` and a password or explicit identity file is + specified, instruct the SSH client to disable any authentication + identities inherited from the surrounding environment, such as + those loaded in any running ``ssh-agent``, or default key files + present in ``~/.ssh``. This ensures authentication attempts only + occur using the supplied password or SSH key. :param bool compression: If :py:data:`True`, enable ``ssh`` compression support. Compression has a minimal effect on the size of modules transmitted, as they diff --git a/docs/changelog.rst b/docs/changelog.rst index 8b13f431..d592b08d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -74,6 +74,10 @@ Mitogen for Ansible * `#344 `_: connections no longer fail when the parent machine's logged in username contains slashes. +* `#345 `_: the ``IdentitiesOnly + yes`` option is no longer supplied to OpenSSH by default, more closely + mimicking Ansible's default behaviour. + * Runs with many targets executed the module dependency scanner redundantly due to missing synchronization, causing significant wasted computation in the connection multiplexer subprocess. For one real-world playbook the scanner @@ -101,6 +105,9 @@ Core Library * `#339 `_: the LXD connection method was erroneously executing LXC Classic commands. +* `#345 `_: the SSH connection method + allows optionally disabling ``IdentitiesOnly yes``. + * Add a :func:`mitogen.fork.on_fork` function to allow non-Mitogen managed process forks to clean up Mitogen resources in the forked chlid. diff --git a/mitogen/ssh.py b/mitogen/ssh.py index 25928b45..38e12531 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -142,7 +142,7 @@ class Stream(mitogen.parent.Stream): check_host_keys='enforce', password=None, identity_file=None, compression=True, ssh_args=None, keepalive_enabled=True, keepalive_count=3, keepalive_interval=15, - ssh_debug_level=None, **kwargs): + identities_only=True, ssh_debug_level=None, **kwargs): super(Stream, self).construct(**kwargs) if check_host_keys not in ('accept', 'enforce', 'ignore'): raise ValueError(self.check_host_keys_msg) @@ -153,6 +153,7 @@ class Stream(mitogen.parent.Stream): self.check_host_keys = check_host_keys self.password = password self.identity_file = identity_file + self.identities_only = identities_only self.compression = compression self.keepalive_enabled = keepalive_enabled self.keepalive_count = keepalive_count @@ -181,7 +182,7 @@ class Stream(mitogen.parent.Stream): bits += ['-l', self.username] if self.port is not None: bits += ['-p', str(self.port)] - if self.identity_file or self.password: + if self.identities_only and (self.identity_file or self.password): bits += ['-o', 'IdentitiesOnly yes'] if self.identity_file: bits += ['-i', self.identity_file] From 76c4cf57bd03b745c65f0f8dcdd7b36a6c7858e9 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 19 Aug 2018 22:42:56 +0100 Subject: [PATCH 073/212] docs: update changelog --- docs/changelog.rst | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d592b08d..64dfd471 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -15,18 +15,6 @@ Release Notes -.. comment - - v0.2.3 (2018-07-??) - ------------------- - - * `#315 `_: Mitogen for Ansible is - supported under Ansible 2.6. Contributed by `Dan Quackenbush - `_. - - * Compatible with development versions of Ansible post https://github.com/ansible/ansible/pull/41749 - - v0.2.3 (2018-08-??) ------------------- @@ -44,6 +32,9 @@ Mitogen for Ansible extension that had been installed using the documented steps. Now the bundled library always overrides over any system-installed copy. +* `#315 `_: Mitogen for Ansible is + supported under Ansible 2.6. + * `#321 `_, `#336 `_: temporary file handling has been simplified and additional network roundtrips have been removed, From d580351db41cef775a989a3c6b64e91114a16c33 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 19 Aug 2018 23:04:40 +0100 Subject: [PATCH 074/212] Correct DISTROS variable name for ansible_tests. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1f8c5c6c..5dfdae00 100644 --- a/.travis.yml +++ b/.travis.yml @@ -72,4 +72,4 @@ matrix: # Sanity check against vanilla Ansible. One job suffices. - python: "2.7" - env: MODE=ansible VER=2.6.2 DISTRO=debian STRATEGY=linear + env: MODE=ansible VER=2.6.2 DISTROS=debian STRATEGY=linear From 50e285f7bac96f3d6b0868957b94fa5cf9680fb0 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 19 Aug 2018 23:17:27 +0100 Subject: [PATCH 075/212] tests: update for identities_only change. --- .../integration/delegation/delegate_to_template.yml | 1 + .../ansible/integration/delegation/stack_construction.yml | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/tests/ansible/integration/delegation/delegate_to_template.yml b/tests/ansible/integration/delegation/delegate_to_template.yml index 8b1bd9af..b7dcc246 100644 --- a/tests/ansible/integration/delegation/delegate_to_template.yml +++ b/tests/ansible/integration/delegation/delegate_to_template.yml @@ -20,6 +20,7 @@ 'check_host_keys': 'ignore', 'connect_timeout': 10, 'hostname': 'cd-normal-alias', + 'identities_only': False, 'identity_file': None, 'password': None, 'port': None, diff --git a/tests/ansible/integration/delegation/stack_construction.yml b/tests/ansible/integration/delegation/stack_construction.yml index 8ab6e51b..f62b0e82 100644 --- a/tests/ansible/integration/delegation/stack_construction.yml +++ b/tests/ansible/integration/delegation/stack_construction.yml @@ -58,6 +58,7 @@ 'check_host_keys': 'ignore', 'connect_timeout': 10, 'hostname': 'alias-host', + 'identities_only': False, 'identity_file': None, 'password': None, 'port': None, @@ -91,6 +92,7 @@ 'check_host_keys': 'ignore', 'connect_timeout': 10, 'hostname': 'alias-host', + 'identities_only': False, 'identity_file': None, 'password': None, 'port': None, @@ -134,6 +136,7 @@ 'check_host_keys': 'ignore', 'connect_timeout': 10, 'hostname': 'cd-normal-normal', + 'identities_only': False, 'identity_file': None, 'password': None, 'port': None, @@ -167,6 +170,7 @@ 'check_host_keys': 'ignore', 'connect_timeout': 10, 'hostname': 'alias-host', + 'identities_only': False, 'identity_file': None, 'password': None, 'port': None, @@ -190,6 +194,7 @@ 'check_host_keys': 'ignore', 'connect_timeout': 10, 'hostname': 'cd-normal-alias', + 'identities_only': False, 'identity_file': None, 'password': None, 'port': None, @@ -233,6 +238,7 @@ 'check_host_keys': 'ignore', 'connect_timeout': 10, 'hostname': 'cd-newuser-normal-normal', + 'identities_only': False, 'identity_file': None, 'password': None, 'port': None, @@ -267,6 +273,7 @@ 'check_host_keys': 'ignore', 'connect_timeout': 10, 'hostname': 'alias-host', + 'identities_only': False, 'identity_file': None, 'password': None, 'port': None, From 4098d45dac605f3bdfe7cbf957c8a34f3b54eb53 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 20 Aug 2018 00:21:34 +0100 Subject: [PATCH 076/212] tests: disable delegation tests on vanilla. --- .../delegation/delegate_to_template.yml | 3 ++ .../delegation/stack_construction.yml | 30 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/tests/ansible/integration/delegation/delegate_to_template.yml b/tests/ansible/integration/delegation/delegate_to_template.yml index b7dcc246..8c968640 100644 --- a/tests/ansible/integration/delegation/delegate_to_template.yml +++ b/tests/ansible/integration/delegation/delegate_to_template.yml @@ -8,6 +8,9 @@ gather_facts: no any_errors_fatal: true tasks: + - meta: end_play + when: not is_mitogen + - mitogen_get_stack: delegate_to: "{{ physical_host }}" register: out diff --git a/tests/ansible/integration/delegation/stack_construction.yml b/tests/ansible/integration/delegation/stack_construction.yml index f62b0e82..beb9a9d1 100644 --- a/tests/ansible/integration/delegation/stack_construction.yml +++ b/tests/ansible/integration/delegation/stack_construction.yml @@ -19,6 +19,9 @@ - name: integration/delegation/stack_construction.yml hosts: cd-normal tasks: + - meta: end_play + when: not is_mitogen + # used later for local_action test. - local_action: custom_python_detect_environment register: local_env @@ -27,6 +30,9 @@ - hosts: cd-normal any_errors_fatal: true tasks: + - meta: end_play + when: not is_mitogen + - mitogen_get_stack: register: out - assert: @@ -47,6 +53,9 @@ - hosts: cd-normal tasks: + - meta: end_play + when: not is_mitogen + - mitogen_get_stack: delegate_to: cd-alias register: out @@ -82,6 +91,9 @@ - hosts: cd-alias tasks: + - meta: end_play + when: not is_mitogen + - mitogen_get_stack: register: out - assert: @@ -116,6 +128,9 @@ - hosts: cd-normal-normal tasks: + - meta: end_play + when: not is_mitogen + - mitogen_get_stack: register: out - assert: @@ -160,6 +175,9 @@ - hosts: cd-normal-alias tasks: + - meta: end_play + when: not is_mitogen + - mitogen_get_stack: register: out - assert: @@ -218,6 +236,9 @@ - hosts: cd-newuser-normal-normal tasks: + - meta: end_play + when: not is_mitogen + - mitogen_get_stack: register: out - assert: @@ -262,6 +283,9 @@ - hosts: cd-newuser-normal-normal tasks: + - meta: end_play + when: not is_mitogen + - mitogen_get_stack: delegate_to: cd-alias register: out @@ -297,6 +321,9 @@ - hosts: cd-newuser-normal-normal tasks: + - meta: end_play + when: not is_mitogen + - local_action: mitogen_get_stack register: out - assert: @@ -315,6 +342,9 @@ - hosts: cd-newuser-doas-normal tasks: + - meta: end_play + when: not is_mitogen + - mitogen_get_stack: register: out - assert: From 49f3a61164f99722485a1e2393361e99214fcedc Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 20 Aug 2018 00:33:52 +0100 Subject: [PATCH 077/212] parent: prevent subprocess.Popen.__del__ from calling waitpid(). Closes #253. --- mitogen/parent.py | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index 44504258..ca57efea 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -175,6 +175,30 @@ def create_socketpair(): return parentfp, childfp +def detach_popen(*args, **kwargs): + """ + Use :class:`subprocess.Popen` to construct a child process, then hack the + Popen so that it forgets the child it created, allowing it to survive a + call to Popen.__del__. + + If the child process is not detached, there is a race between it exitting + and __del__ being called. If it exits before __del__ runs, then __del__'s + call to :func:`os.waitpid` will capture the one and only exit event + delivered to this process, causing later 'legitimate' calls to fail with + ECHILD. + + :returns: + Process ID of the new child. + """ + # This allows Popen() to be used for e.g. graceful post-fork error + # handling, without tying the surrounding code into managing a Popen + # object, which isn't possible for at least :mod:`mitogen.fork`. This + # should be replaced by a swappable helper class in a future version. + proc = subprocess.Popen(*args, **kwargs) + proc._child_created = False + return proc.pid + + def create_child(args, merge_stdio=False, preexec_fn=None): """ Create a child process whose stdin/stdout is connected to a socket. @@ -201,7 +225,7 @@ def create_child(args, merge_stdio=False, preexec_fn=None): else: extra = {} - proc = subprocess.Popen( + pid = detach_popen( args=args, stdin=childfp, stdout=childfp, @@ -215,8 +239,8 @@ def create_child(args, merge_stdio=False, preexec_fn=None): parentfp.close() LOG.debug('create_child() child %d fd %d, parent %d, cmd: %s', - proc.pid, fd, os.getpid(), Argv(args)) - return proc.pid, fd, None + pid, fd, os.getpid(), Argv(args)) + return pid, fd, None def _acquire_controlling_tty(): @@ -250,7 +274,7 @@ def tty_create_child(args): disable_echo(master_fd) disable_echo(slave_fd) - proc = subprocess.Popen( + pid = detach_popen( args=args, stdin=slave_fd, stdout=slave_fd, @@ -261,8 +285,8 @@ def tty_create_child(args): os.close(slave_fd) LOG.debug('tty_create_child() child %d fd %d, parent %d, cmd: %s', - proc.pid, master_fd, os.getpid(), Argv(args)) - return proc.pid, master_fd, None + pid, master_fd, os.getpid(), Argv(args)) + return pid, master_fd, None def hybrid_tty_create_child(args): @@ -284,7 +308,7 @@ def hybrid_tty_create_child(args): mitogen.core.set_block(childfp) disable_echo(master_fd) disable_echo(slave_fd) - proc = subprocess.Popen( + pid = detach_popen( args=args, stdin=childfp, stdout=childfp, @@ -300,8 +324,8 @@ def hybrid_tty_create_child(args): parentfp.close() LOG.debug('hybrid_tty_create_child() pid=%d stdio=%d, tty=%d, cmd: %s', - proc.pid, stdio_fd, master_fd, Argv(args)) - return proc.pid, stdio_fd, master_fd + pid, stdio_fd, master_fd, Argv(args)) + return pid, stdio_fd, master_fd def write_all(fd, s, deadline=None): From e18396d54d9e24f9815bbc42d0764683383cddb7 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 20 Aug 2018 12:48:52 +0100 Subject: [PATCH 078/212] ansible: enable profiling by default! Thankfully this never made it into a release --- ansible_mitogen/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index 5ce1b8be..e7ceb2ce 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -133,7 +133,7 @@ class MuxProcess(object): mitogen.core.set_cloexec(cls.worker_sock.fileno()) mitogen.core.set_cloexec(cls.child_sock.fileno()) - if os.environ.get('MITOGEN_PROFILING', '1'): + if os.environ.get('MITOGEN_PROFILING'): mitogen.core.enable_profiling() cls.original_env = dict(os.environ) From 084c0ac06513196d6524add5ce4021b156d06f19 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 20 Aug 2018 13:24:09 +0100 Subject: [PATCH 079/212] ansible: avoid roundtrip in copy action due to fixup_perms2(). On top of existing temporary files work, this reduces the number of roundtrips required for "copy" and "template" actions from 6 to 3. --- ansible_mitogen/mixins.py | 7 +- tests/ansible/integration/action/all.yml | 5 +- .../integration/action/fixup_perms2__copy.yml | 104 ++++++++++++++++++ 3 files changed, 113 insertions(+), 3 deletions(-) create mode 100644 tests/ansible/integration/action/fixup_perms2__copy.yml diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 7d8f5518..7be5444f 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -216,6 +216,11 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): self._connection.put_data(remote_path, data) return remote_path + #: Actions listed here cause :func:`_fixup_perms2` to avoid a needless + #: roundtrip, as they modify file modes separately afterwards. This is due + #: to the method prototype having a default of `execute=True`. + FIXUP_PERMS_RED_HERRING = set(['copy']) + def _fixup_perms2(self, remote_paths, remote_user=None, execute=True): """ Mitogen always executes ActionBase helper methods in the context of the @@ -224,7 +229,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): """ LOG.debug('_fixup_perms2(%r, remote_user=%r, execute=%r)', remote_paths, remote_user, execute) - if execute: + if execute and self._load_name not in self.FIXUP_PERMS_RED_HERRING: return self._remote_chmod(remote_paths, mode='u+x') return self.COMMAND_RESULT.copy() diff --git a/tests/ansible/integration/action/all.yml b/tests/ansible/integration/action/all.yml index ebbff26a..22904c23 100644 --- a/tests/ansible/integration/action/all.yml +++ b/tests/ansible/integration/action/all.yml @@ -1,5 +1,6 @@ -- import_playbook: remote_file_exists.yml -- import_playbook: remote_expand_user.yml +- import_playbook: fixup_perms2__copy.yml - import_playbook: low_level_execute_command.yml - import_playbook: make_tmp_path.yml +- import_playbook: remote_expand_user.yml +- import_playbook: remote_file_exists.yml - import_playbook: transfer_data.yml diff --git a/tests/ansible/integration/action/fixup_perms2__copy.yml b/tests/ansible/integration/action/fixup_perms2__copy.yml new file mode 100644 index 00000000..17bfbed9 --- /dev/null +++ b/tests/ansible/integration/action/fixup_perms2__copy.yml @@ -0,0 +1,104 @@ +# Verify action plugins still set file modes correctly even though +# fixup_perms2() avoids setting execute bit despite being asked to. + +- name: integration/action/fixup_perms2__copy.yml + hosts: test-targets + any_errors_fatal: true + tasks: + - name: Get default remote file mode + shell: python -c 'import os; print("%04o" % (int("0666", 8) & ~os.umask(0)))' + register: py_umask + + - name: Set default file mode + set_fact: + mode: "{{py_umask.stdout}}" + + # + # copy module (no mode). + # + + - name: "Copy files (no mode)" + copy: + content: "" + dest: /tmp/copy-no-mode + + - stat: path=/tmp/copy-no-mode + register: out + - assert: + that: + - out.stat.mode == mode + + # + # copy module (explicit mode). + # + + - name: "Copy files from content: arg" + copy: + content: "" + mode: 0400 + dest: /tmp/copy-with-mode + + - stat: path=/tmp/copy-with-mode + register: out + - assert: + that: + - out.stat.mode == "0400" + + # + # copy module (existing disk files, no mode). + # + + - file: + path: /tmp/weird-mode + state: absent + + - name: Create local test file. + connection: local + copy: + content: "weird mode" + dest: "/tmp/weird-mode" + mode: "1462" + + - copy: + src: "/tmp/weird-mode" + dest: "/tmp/weird-mode" + + - stat: + path: "/tmp/weird-mode" + register: out + - assert: + that: + - out.stat.mode == mode + + # + # copy module (existing disk files, preserve mode). + # + + - copy: + src: "/tmp/weird-mode" + dest: "/tmp/weird-mode" + mode: preserve + + - stat: + path: "/tmp/weird-mode" + register: out + - assert: + that: + - out.stat.mode == "1462" + + # + # copy module (existing disk files, explicit mode). + # + + - copy: + src: "/tmp/weird-mode" + dest: "/tmp/weird-mode" + mode: "1461" + + - stat: + path: "/tmp/weird-mode" + register: out + + - assert: + that: + - out.stat.mode == "1461" From 84521b714fa79404b6d7b3d9e397eb0bd956431d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 20 Aug 2018 13:40:48 +0100 Subject: [PATCH 080/212] docs: update changelog. --- docs/changelog.rst | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 64dfd471..ea44f73c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -37,10 +37,18 @@ Mitogen for Ansible * `#321 `_, `#336 `_: temporary file handling - has been simplified and additional network roundtrips have been removed, - undoing earlier damage caused by compatibility fixes, and improving 2.6 - compatibility. One directory is created at startup for each persistent - interpreter. See :ref:`ansible_tempfiles` for a complete description. + was simplified, undoing earlier damage caused by compatibility fixes, + improving 2.6 compatibility, and avoiding two network roundtrips for every + related action + (`assemble `_, + `aws_s3 `_, + `copy `_, + `patch `_, + `script `_, + `template `_, + `unarchive `_, + `uri `_). See + :ref:`ansible_tempfiles` for a complete description. * `#324 `_: plays with a custom ``module_utils`` would fail due to fallout from the Python 3 port and related @@ -48,8 +56,8 @@ Mitogen for Ansible * `#331 `_: fixed known issue: the connection multiplexer subprocess always exits before the main Ansible - process exits, ensuring logs generated by it do not overwrite the user's - prompt when ``-vvv`` is enabled. + process, ensuring logs generated by it do not overwrite the user's prompt + when ``-vvv`` is enabled. * `#332 `_: support a new :data:`sys.excepthook`-based module exit mechanism added in Ansible 2.6. @@ -69,6 +77,12 @@ Mitogen for Ansible yes`` option is no longer supplied to OpenSSH by default, more closely mimicking Ansible's default behaviour. +* `084c0ac0 `_: avoid a + needless roundtrip for each invocation of the + `copy `_ and + `template `_ + actions, due to an unfortunate default parameter. + * Runs with many targets executed the module dependency scanner redundantly due to missing synchronization, causing significant wasted computation in the connection multiplexer subprocess. For one real-world playbook the scanner @@ -77,7 +91,7 @@ Mitogen for Ansible * A missing check caused an exception traceback to appear when using the ``ansible`` command-line tool with a missing or misspelled module name. -* Ansible since >2.6 began importing ``__main__`` from +* Ansible since >=2.7 began importing ``__main__`` from ``ansible.module_utils.basic``, causing an error during execution, due to the controller being configured to refuse network imports outside the ``ansible.*`` namespace. Update the target implementation to construct a stub From 8e9b5ad5766fec79aa34badab3f95dedd7310163 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 20 Aug 2018 13:44:33 +0100 Subject: [PATCH 081/212] tests: import template benchmark script. --- tests/ansible/bench/loop-20-templates.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 tests/ansible/bench/loop-20-templates.yml diff --git a/tests/ansible/bench/loop-20-templates.yml b/tests/ansible/bench/loop-20-templates.yml new file mode 100644 index 00000000..df994bd8 --- /dev/null +++ b/tests/ansible/bench/loop-20-templates.yml @@ -0,0 +1,14 @@ + +- hosts: all + tasks: + - file: + dest: /tmp/templates + state: "{{item}}" + with_items: ["absent", "directory"] + + - copy: + dest: /tmp/templates/{{item}} + mode: 0755 + content: + Hello from {{item}} + with_sequence: start=1 end=20 From 7458dfae855d86512e85c2a1068ec80eac925ad7 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 20 Aug 2018 14:11:46 +0100 Subject: [PATCH 082/212] ansible: avoid roundtrip for small file transfers. Calls to connect.put_file() where the file is sufficiently small enough to fit in a single RPC proceed without waiting for an RPC response. If the write fails the target context will log an exception, and any subsequent step depending on the written file will fail. I verified every built-in action plugin for file transfer calls, and they all depend on the transferred file in the following step, so this should be safe. Reduces template/copy actions to 2-RTT, loop-20-templates.yml runtime reduced from 30 seconds to 10 seconds over a 250ms link compared to v0.2.2, and from 123 seconds compared to vanilla with pipelining enabled. --- ansible_mitogen/connection.py | 49 ++++++++++++++++++++++++++++------- docs/ansible.rst | 8 +++--- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 30d23a4f..966f86c1 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -692,15 +692,29 @@ class Connection(ansible.plugins.connection.ConnectionBase): :param bool use_login_context: If present and :data:`True`, send the call to the login account context rather than the optional become user context. + + :param bool no_reply: + If present and :data:`True`, send the call with no ``reply_to`` + header, causing the context to execute it entirely asynchronously, + and to log any exception thrown. This allows avoiding a roundtrip + in places where the outcome of a call is highly likely to succeed, + and subsequent actions will fail regardless with a meaningful + exception if the no_reply call failed. + :returns: - mitogen.core.Receiver that receives the function call result. + :class:`mitogen.core.Receiver` that receives the function call result. """ self._connect() + if kwargs.pop('use_login_context', None): call_context = self.login_context else: call_context = self.context - return call_context.call_async(func, *args, **kwargs) + + if kwargs.pop('no_reply', None): + return call_context.call_no_reply(func, *args, **kwargs) + else: + return call_context.call_async(func, *args, **kwargs) def call(self, func, *args, **kwargs): """ @@ -713,7 +727,10 @@ class Connection(ansible.plugins.connection.ConnectionBase): """ t0 = time.time() try: - return self.call_async(func, *args, **kwargs).get().unpickle() + recv = self.call_async(func, *args, **kwargs) + if recv is None: # no_reply=True + return None + return recv.get().unpickle() finally: LOG.debug('Call took %d ms: %r', 1000 * (time.time() - t0), mitogen.parent.CallSpec(func, args, kwargs)) @@ -786,19 +803,33 @@ class Connection(ansible.plugins.connection.ConnectionBase): def put_data(self, out_path, data, mode=None, utimes=None): """ - Implement put_file() by caling the corresponding - ansible_mitogen.target function in the target. + Implement put_file() by caling the corresponding ansible_mitogen.target + function in the target, transferring small files inline. :param str out_path: Remote filesystem path to write. :param byte data: File contents to put. """ + # no_reply=True here avoids a roundrip that 99% of the time will report + # a successful response. If the file transfer fails, the target context + # will dump an exception into the logging framework, which will appear + # on console, and the missing file will cause the subsequent task step + # to fail regardless. This is safe since CALL_FUNCTION is presently + # single-threaded for each target, so subsequent steps cannot execute + # until the transfer RPC has completed. self.call(ansible_mitogen.target.write_path, mitogen.utils.cast(out_path), mitogen.core.Blob(data), mode=mode, - utimes=utimes) + utimes=utimes, + no_reply=True) + + #: Maximum size of a small file before switching to streaming file + #: transfer. This should really be the same as + #: mitogen.services.FileService.IO_SIZE, however the message format has + #: slightly more overhead, so just randomly subtract 4KiB. + SMALL_FILE_LIMIT = mitogen.core.CHUNK_SIZE - 4096 def put_file(self, in_path, out_path): """ @@ -817,14 +848,14 @@ class Connection(ansible.plugins.connection.ConnectionBase): # If the file is sufficiently small, just ship it in the argument list # rather than introducing an extra RTT for the child to request it from # FileService. - if st.st_size <= 32768: + if st.st_size <= self.SMALL_FILE_LIMIT: fp = open(in_path, 'rb') try: - s = fp.read(32769) + s = fp.read(self.SMALL_FILE_LIMIT + 1) finally: fp.close() - # Ensure file was not growing during call. + # Ensure did not grow during read. if len(s) == st.st_size: return self.put_data(out_path, s, mode=st.st_mode, utimes=(st.st_atime, st.st_mtime)) diff --git a/docs/ansible.rst b/docs/ansible.rst index 790d0369..514708bd 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -312,10 +312,10 @@ where readers may observe inconsistent file contents. Performance ^^^^^^^^^^^ -One roundtrip initiates a transfer larger than 32KiB, while smaller transfers -are embedded in the initiating RPC. For tools operating via SSH multiplexing, 4 -roundtrips are required to configure the IO channel, in addition to the time to -start the local and remote processes. +One roundtrip initiates a transfer larger than 124 KiB, while smaller transfers +are embedded in a 0-roundtrip remote call. For tools operating via SSH +multiplexing, 4 roundtrips are required to configure the IO channel, in +addition to the time to start the local and remote processes. An invocation of ``scp`` with an empty ``.profile`` over a 30 ms link takes ~140 ms, wasting 110 ms per invocation, rising to ~2,000 ms over a 400 ms From 52cd7fddc12cdabd0eded3d637a5e4b849a7b975 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 20 Aug 2018 14:23:30 +0100 Subject: [PATCH 083/212] docs: update changelog. --- docs/changelog.rst | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ea44f73c..2a094476 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -32,8 +32,8 @@ Mitogen for Ansible extension that had been installed using the documented steps. Now the bundled library always overrides over any system-installed copy. -* `#315 `_: Mitogen for Ansible is - supported under Ansible 2.6. +* `#315 `_: Ansible 2.6 is now + supported. * `#321 `_, `#336 `_: temporary file handling @@ -50,9 +50,10 @@ Mitogen for Ansible `uri `_). See :ref:`ansible_tempfiles` for a complete description. -* `#324 `_: plays with a custom - ``module_utils`` would fail due to fallout from the Python 3 port and related - tests being disabled. +* `#324 `_: plays with a + `custom module_utils `_ + would fail due to fallout from the Python 3 port and related tests being + disabled. * `#331 `_: fixed known issue: the connection multiplexer subprocess always exits before the main Ansible @@ -60,7 +61,7 @@ Mitogen for Ansible when ``-vvv`` is enabled. * `#332 `_: support a new - :data:`sys.excepthook`-based module exit mechanism added in Ansible 2.6. + :func:`sys.excepthook`-based module exit mechanism added in Ansible 2.6. * `#338 `_: compatibility: changes to ``/etc/environment`` and ``~/.pam_environment`` made by a task are reflected @@ -71,22 +72,28 @@ Mitogen for Ansible option is supported. * `#344 `_: connections no longer - fail when the parent machine's logged in username contains slashes. + fail when the controller's login username contains slashes. * `#345 `_: the ``IdentitiesOnly - yes`` option is no longer supplied to OpenSSH by default, more closely - mimicking Ansible's default behaviour. + yes`` option is no longer supplied to OpenSSH by default, better matching + Ansible's behaviour. * `084c0ac0 `_: avoid a - needless roundtrip for each invocation of the + roundtrip in `copy `_ and `template `_ - actions, due to an unfortunate default parameter. - -* Runs with many targets executed the module dependency scanner redundantly - due to missing synchronization, causing significant wasted computation in the - connection multiplexer subprocess. For one real-world playbook the scanner - runtime was reduced by 95%, which may manifest as shorter runs. + due to an unfortunate default. + +* `7458dfae `_: avoid a + roundtrip when transferring files smaller than 124KiB. Copy and template + actions are now 2-RTT, reducing runtime for a 20-iteration template loop over + a 250 ms link from 30 seconds to 10 seconds compared to v0.2.2, down from 120 + seconds compared to vanilla. + +* `d62e6e2a `_: many-target + runs executed the dependency scanner redundantly due to missing + synchronization, wasting significant runtime in the connection multiplexer. + In one case work was reduced by 95%, which may manifest as faster runs. * A missing check caused an exception traceback to appear when using the ``ansible`` command-line tool with a missing or misspelled module name. From fcc7429111591bdfe93906796557bb44152a567c Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 20 Aug 2018 15:34:43 +0100 Subject: [PATCH 084/212] docs: changelog: split out enhancements --- docs/changelog.rst | 60 +++++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2a094476..7eff2ee9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,16 +21,8 @@ v0.2.3 (2018-08-??) Mitogen for Ansible ~~~~~~~~~~~~~~~~~~~ -* `#251 `_, - `#340 `_: Connection Delegation - could establish connections to the wrong target when ``delegate_to:`` is - present. - -* `#291 `_: when Mitogen had - previously been installed using ``pip`` or ``setuptools``, the globally - installed version could conflict with a newer version bundled with an - extension that had been installed using the documented steps. Now the bundled - library always overrides over any system-installed copy. +Enhancements +^^^^^^^^^^^^ * `#315 `_: Ansible 2.6 is now supported. @@ -50,6 +42,37 @@ Mitogen for Ansible `uri `_). See :ref:`ansible_tempfiles` for a complete description. +* `084c0ac0 `_: avoid a + roundtrip in + `copy `_ and + `template `_ + due to an unfortunate default. + +* `7458dfae `_: avoid a + roundtrip when transferring files smaller than 124KiB. Copy and template + actions are now 2-RTT, reducing runtime for a 20-iteration template loop over + a 250 ms link from 30 seconds to 10 seconds compared to v0.2.2, down from 120 + seconds compared to vanilla. + +* `d62e6e2a `_: many-target + runs executed the dependency scanner redundantly due to missing + synchronization, wasting significant runtime in the connection multiplexer. + In one case work was reduced by 95%, which may manifest as faster runs. + +Fixes +^^^^^ + +* `#251 `_, + `#340 `_: Connection Delegation + could establish connections to the wrong target when ``delegate_to:`` is + present. + +* `#291 `_: when Mitogen had + previously been installed using ``pip`` or ``setuptools``, the globally + installed version could conflict with a newer version bundled with an + extension that had been installed using the documented steps. Now the bundled + library always overrides over any system-installed copy. + * `#324 `_: plays with a `custom module_utils `_ would fail due to fallout from the Python 3 port and related tests being @@ -78,23 +101,6 @@ Mitogen for Ansible yes`` option is no longer supplied to OpenSSH by default, better matching Ansible's behaviour. -* `084c0ac0 `_: avoid a - roundtrip in - `copy `_ and - `template `_ - due to an unfortunate default. - -* `7458dfae `_: avoid a - roundtrip when transferring files smaller than 124KiB. Copy and template - actions are now 2-RTT, reducing runtime for a 20-iteration template loop over - a 250 ms link from 30 seconds to 10 seconds compared to v0.2.2, down from 120 - seconds compared to vanilla. - -* `d62e6e2a `_: many-target - runs executed the dependency scanner redundantly due to missing - synchronization, wasting significant runtime in the connection multiplexer. - In one case work was reduced by 95%, which may manifest as faster runs. - * A missing check caused an exception traceback to appear when using the ``ansible`` command-line tool with a missing or misspelled module name. From 90c2ed03d0a842d04444e81a5c5c058511e5891b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 20 Aug 2018 15:43:56 +0100 Subject: [PATCH 085/212] ansible: fix synchronize module Broken by recent connection delegation fixes. --- ansible_mitogen/connection.py | 9 ++++++ tests/ansible/integration/action/all.yml | 1 + .../integration/action/synchronize.yml | 28 +++++++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 tests/ansible/integration/action/synchronize.yml diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 966f86c1..71eb78e0 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -419,6 +419,12 @@ class Connection(ansible.plugins.connection.ConnectionBase): #: Only sudo, su, and doas are supported for now. become_methods = ['sudo', 'su', 'doas'] + # + # Note: any of the attributes below may be :data:`None` if the connection + # plugin was constructed directly by a non-cooperative action, such as in + # the case of the synchronize module. + # + #: Set to 'ansible_python_interpreter' by on_action_run(). python_path = None @@ -449,6 +455,9 @@ class Connection(ansible.plugins.connection.ConnectionBase): #: Set to 'hostvars' by on_action_run() host_vars = None + #: Set by on_action_run() + delegate_to_hostname = None + #: Set to '_loader.get_basedir()' by on_action_run(). Used by mitogen_local #: to change the working directory to that of the current playbook, #: matching vanilla Ansible behaviour. diff --git a/tests/ansible/integration/action/all.yml b/tests/ansible/integration/action/all.yml index 22904c23..c5cb80d7 100644 --- a/tests/ansible/integration/action/all.yml +++ b/tests/ansible/integration/action/all.yml @@ -3,4 +3,5 @@ - import_playbook: make_tmp_path.yml - import_playbook: remote_expand_user.yml - import_playbook: remote_file_exists.yml +- import_playbook: synchronize.yml - import_playbook: transfer_data.yml diff --git a/tests/ansible/integration/action/synchronize.yml b/tests/ansible/integration/action/synchronize.yml new file mode 100644 index 00000000..f9a8cb2c --- /dev/null +++ b/tests/ansible/integration/action/synchronize.yml @@ -0,0 +1,28 @@ +# Verify basic operation of the synchronize module. + +- name: integration/action/synchronize.yml + hosts: test-targets + any_errors_fatal: true + tasks: + - file: + path: /tmp/sync-test + state: directory + connection: local + + - copy: + dest: /tmp/sync-test/item + content: "item!" + connection: local + + - synchronize: + dest: /tmp/sync-test + src: /tmp/sync-test + + - slurp: + src: /tmp/sync-test/item + register: out + + - set_fact: outout="{{out.content|b64decode}}" + + - assert: + that: outout == "item!" From d36a320e7f2afaff410ef8a6bd947eb1aa3347f8 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 21 Aug 2018 01:39:16 +0100 Subject: [PATCH 086/212] docs: update contributors. --- docs/contributors.rst | 15 ++++++++++++++- docs/images/sponsors/grx.svg | 37 ++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 docs/images/sponsors/grx.svg diff --git a/docs/contributors.rst b/docs/contributors.rst index ee5c3132..dcfb50fa 100644 --- a/docs/contributors.rst +++ b/docs/contributors.rst @@ -10,6 +10,7 @@ sponsorship and outstanding future-thinking of its early adopters. + + + + + + + + + + + + + From 3453d4d7d0b9bf0224c6a2178249a353a68dc962 Mon Sep 17 00:00:00 2001 From: Jesse London Date: Wed, 5 Sep 2018 16:41:05 -0500 Subject: [PATCH 087/212] Python 3 support for classmethod call targets There were two problems with detection and handling of class methods as call targets in Python 3: * Methods no longer define `im_self` -- this is now only `__self__` * The `types` module no longer defines a `ClassType` The universally-compatible (v2.6+) solution was to switch to using the `inspect` module -- whose interface has been stable -- and to checking the method attribute `__self__`. (It doesn't hurt that `inspect` checks are more brief and we now no longer need the `types` module here.) --- mitogen/parent.py | 6 ++---- tests/call_function_test.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index ca57efea..0dfc4f07 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -47,7 +47,6 @@ import termios import textwrap import threading import time -import types import zlib # Absolute imports for <2.5. @@ -490,9 +489,8 @@ def upgrade_router(econtext): def make_call_msg(fn, *args, **kwargs): - if isinstance(fn, types.MethodType) and \ - isinstance(fn.im_self, (type, types.ClassType)): - klass = mitogen.core.to_text(fn.im_self.__name__) + if inspect.ismethod(fn) and inspect.isclass(fn.__self__): + klass = mitogen.core.to_text(fn.__self__.__name__) else: klass = None diff --git a/tests/call_function_test.py b/tests/call_function_test.py index f0074258..eb83dff5 100644 --- a/tests/call_function_test.py +++ b/tests/call_function_test.py @@ -36,7 +36,17 @@ def func_accepts_returns_sender(sender): return sender +class TargetClass: + + offset = 100 + + @classmethod + def add_numbers_with_offset(cls, x, y): + return cls.offset + x + y + + class CallFunctionTest(testlib.RouterMixin, testlib.TestCase): + def setUp(self): super(CallFunctionTest, self).setUp() self.local = self.router.fork() @@ -44,6 +54,12 @@ class CallFunctionTest(testlib.RouterMixin, testlib.TestCase): def test_succeeds(self): self.assertEqual(3, self.local.call(function_that_adds_numbers, 1, 2)) + def test_succeeds_class_method(self): + self.assertEqual( + self.local.call(TargetClass.add_numbers_with_offset, 1, 2), + 103, + ) + def test_crashes(self): exc = self.assertRaises(mitogen.core.CallError, lambda: self.local.call(function_that_fails)) From 4134218ef420c9059e40496658147cade5186009 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 20 Aug 2018 19:32:14 +0100 Subject: [PATCH 088/212] docs: update changelog. --- docs/changelog.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7eff2ee9..e41f416d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -24,8 +24,7 @@ Mitogen for Ansible Enhancements ^^^^^^^^^^^^ -* `#315 `_: Ansible 2.6 is now - supported. +* `#315 `_: Ansible 2.6 is supported. * `#321 `_, `#336 `_: temporary file handling @@ -78,10 +77,10 @@ Fixes would fail due to fallout from the Python 3 port and related tests being disabled. -* `#331 `_: fixed known issue: the - connection multiplexer subprocess always exits before the main Ansible - process, ensuring logs generated by it do not overwrite the user's prompt - when ``-vvv`` is enabled. +* `#331 `_: the connection + multiplexer subprocess always exits before the main Ansible process, ensuring + logs generated by it do not overwrite the user's prompt when ``-vvv`` is + enabled. * `#332 `_: support a new :func:`sys.excepthook`-based module exit mechanism added in Ansible 2.6. @@ -140,9 +139,10 @@ the bug reports in this release contributed by `Dan Quackenbush `_, `dsgnr `_, `Jesse London `_, +`Jonathan Rosser `_, `Luca Nunzi `_, `nikitakazantsev12 `_, -`Pateek Jain `_, +`Prateek Jain `_, `Pierre-Henry Muller `_, `Rick Box `_, and `Timo Beckers `_. From 3b012e5bceeb276f8982340dd8112e834eefed12 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 20 Aug 2018 21:29:57 +0100 Subject: [PATCH 089/212] tests: allow plugging in pprint/pprintpp via env. --- tests/ansible/lib/callback/nice_stdout.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/ansible/lib/callback/nice_stdout.py b/tests/ansible/lib/callback/nice_stdout.py index fa720fd2..1f7d7c89 100644 --- a/tests/ansible/lib/callback/nice_stdout.py +++ b/tests/ansible/lib/callback/nice_stdout.py @@ -1,4 +1,5 @@ from __future__ import unicode_literals +import os import io from ansible.module_utils import six @@ -8,6 +9,11 @@ try: except ImportError: from ansible.plugins.loader import callback_loader +try: + pprint = __import__(os.environ['NICE_STDOUT_PPRINT']) +except KeyError: + pprint = None + def printi(tio, obj, key=None, indent=0): def write(s, *args): @@ -50,7 +56,10 @@ class CallbackModule(DefaultModule): def _dump_results(self, result, *args, **kwargs): try: tio = io.StringIO() - printi(tio, result) + if pprint: + pprint.pprint(result, stream=tio) + else: + printi(tio, result) return tio.getvalue() #.encode('ascii', 'replace') except: import traceback From 9792b8b54f0155c0252f7cdb49804cd91fe0b35a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 20 Aug 2018 22:54:55 +0100 Subject: [PATCH 090/212] ansible: use template-expanded delegate_to hostname in one more location. --- ansible_mitogen/connection.py | 2 +- tests/ansible/common-hosts | 14 --------- .../hosts/group_vars/osa-all-containers.yml | 4 +++ tests/ansible/hosts/issue340 | 12 +++++++ tests/ansible/hosts/osa-containers | 9 ++++++ tests/ansible/integration/delegation/all.yml | 2 ++ .../delegation/delegate_to_template.yml | 26 +++++++++++++++- .../delegation/osa_container_standalone.yml | 29 +++++++++++++++++ .../delegation/osa_delegate_to_self.yml | 31 +++++++++++++++++++ 9 files changed, 113 insertions(+), 16 deletions(-) create mode 100644 tests/ansible/hosts/group_vars/osa-all-containers.yml create mode 100644 tests/ansible/hosts/issue340 create mode 100644 tests/ansible/hosts/osa-containers create mode 100644 tests/ansible/integration/delegation/osa_container_standalone.yml create mode 100644 tests/ansible/integration/delegation/osa_delegate_to_self.yml diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 71eb78e0..bf11bccb 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -602,7 +602,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): transport=self._play_context.connection, inventory_name=self.delegate_to_hostname, connection=self, - hostvars=self.host_vars[self._play_context.delegate_to], + hostvars=self.host_vars[self.delegate_to_hostname], become_user=(self._play_context.become_user if self._play_context.become else None), diff --git a/tests/ansible/common-hosts b/tests/ansible/common-hosts index 6dafaf47..449442f6 100644 --- a/tests/ansible/common-hosts +++ b/tests/ansible/common-hosts @@ -35,17 +35,3 @@ cd-newuser-normal-normal mitogen_via=cd-normal ansible_user=newuser-normal-norma # doas:newuser via host. cd-newuser-doas-normal mitogen_via=cd-normal ansible_connection=mitogen_doas ansible_user=newuser-doas-normal-user - - -# Connection Delegation issue #340 reproduction. -# Path to jails is SSH to H -> mitogen_sudo to root -> jail to J - -[issue340] -# 'target' plays the role of the normal host machine H. -# 'mitogen__sudo1' plays the role of root@H via mitogen_sudo. -# 'mitogen__user1' plays the role of root@J via mitogen__user1. -# 'mitogen__user2' plays the role of E, the delgate_to target for certs. - -i340-root ansible_user=mitogen__sudo1 ansible_connection=mitogen_sudo mitogen_via=target -i340-jail ansible_user=mitogen__user1 ansible_connection=mitogen_sudo mitogen_via=i340-root -i340-certs ansible_user=mitogen__user2 ansible_connection=mitogen_sudo mitogen_via=target diff --git a/tests/ansible/hosts/group_vars/osa-all-containers.yml b/tests/ansible/hosts/group_vars/osa-all-containers.yml new file mode 100644 index 00000000..4f38fcb4 --- /dev/null +++ b/tests/ansible/hosts/group_vars/osa-all-containers.yml @@ -0,0 +1,4 @@ +--- + +ansible_connection: setns +mitogen_kind: lxc diff --git a/tests/ansible/hosts/issue340 b/tests/ansible/hosts/issue340 new file mode 100644 index 00000000..3caa95a9 --- /dev/null +++ b/tests/ansible/hosts/issue340 @@ -0,0 +1,12 @@ +# Connection Delegation issue #340 reproduction. +# Path to jails is SSH to H -> mitogen_sudo to root -> jail to J + +[issue340] +# 'target' plays the role of the normal host machine H. +# 'mitogen__sudo1' plays the role of root@H via mitogen_sudo. +# 'mitogen__user1' plays the role of root@J via mitogen__user1. +# 'mitogen__user2' plays the role of E, the delgate_to target for certs. + +i340-root ansible_user=mitogen__sudo1 ansible_connection=mitogen_sudo mitogen_via=target +i340-jail ansible_user=mitogen__user1 ansible_connection=mitogen_sudo mitogen_via=i340-root +i340-certs ansible_user=mitogen__user2 ansible_connection=mitogen_sudo mitogen_via=target diff --git a/tests/ansible/hosts/osa-containers b/tests/ansible/hosts/osa-containers new file mode 100644 index 00000000..7ff2c2b6 --- /dev/null +++ b/tests/ansible/hosts/osa-containers @@ -0,0 +1,9 @@ +# integration/delegation/delegate_to_container.yml + +# Patterned after openstack-ansible/all_containers.yml +osa-host-machine ansible_host=172.29.236.100 + +[osa-all-containers] +osa-container-1 container_tech=lxc +osa-container-2 container_tech=lxc +osa-container-3 container_tech=lxc diff --git a/tests/ansible/integration/delegation/all.yml b/tests/ansible/integration/delegation/all.yml index 30ea625f..743ce157 100644 --- a/tests/ansible/integration/delegation/all.yml +++ b/tests/ansible/integration/delegation/all.yml @@ -1,2 +1,4 @@ - import_playbook: delegate_to_template.yml +- import_playbook: osa_container_standalone.yml +- import_playbook: osa_delegate_to_self.yml - import_playbook: stack_construction.yml diff --git a/tests/ansible/integration/delegation/delegate_to_template.yml b/tests/ansible/integration/delegation/delegate_to_template.yml index 8c968640..2f0830c4 100644 --- a/tests/ansible/integration/delegation/delegate_to_template.yml +++ b/tests/ansible/integration/delegation/delegate_to_template.yml @@ -18,6 +18,30 @@ - assert: that: | out.result == [ + { + 'kwargs': { + 'check_host_keys': 'ignore', + 'connect_timeout': 10, + 'hostname': 'alias-host', + 'identities_only': False, + 'identity_file': None, + 'password': None, + 'port': None, + 'python_path': None, + 'ssh_args': [ + '-o', + 'ForwardAgent=yes', + '-o', + 'ControlMaster=auto', + '-o', + 'ControlPersist=60s', + ], + 'ssh_debug_level': None, + 'ssh_path': 'ssh', + 'username': 'alias-user', + }, + 'method': 'ssh', + }, { 'kwargs': { 'check_host_keys': 'ignore', @@ -41,5 +65,5 @@ 'username': None, }, 'method': 'ssh', - }, + } ] diff --git a/tests/ansible/integration/delegation/osa_container_standalone.yml b/tests/ansible/integration/delegation/osa_container_standalone.yml new file mode 100644 index 00000000..97830d28 --- /dev/null +++ b/tests/ansible/integration/delegation/osa_container_standalone.yml @@ -0,0 +1,29 @@ +# Verify one OSA-style container has the correct config. + +- name: integration/delegation/container_standalone.yml + hosts: dtc-container-1 + gather_facts: false + tasks: + - meta: end_play + when: not is_mitogen + + - mitogen_get_stack: + register: out + + - debug: msg={{out}} + - assert: + that: | + out.result == [ + { + 'kwargs': { + 'container': 'dtc-container-1', + 'docker_path': None, + 'kind': 'lxc', + 'lxc_info_path': None, + 'machinectl_path': None, + 'python_path': ['/usr/bin/python'], + 'username': None, + }, + 'method': 'setns', + }, + ] diff --git a/tests/ansible/integration/delegation/osa_delegate_to_self.yml b/tests/ansible/integration/delegation/osa_delegate_to_self.yml new file mode 100644 index 00000000..0915bbb8 --- /dev/null +++ b/tests/ansible/integration/delegation/osa_delegate_to_self.yml @@ -0,0 +1,31 @@ +# OSA: Verify delegating the connection back to the container succeeds. + +- name: integration/delegation/osa_delegate_to_self.yml + hosts: osa-container-1 + vars: + target: osa-container-1 + gather_facts: false + tasks: + - meta: end_play + when: not is_mitogen + + - mitogen_get_stack: + delegate_to: "{{target}}" + register: out + + - assert: + that: | + out.result == [ + { + 'kwargs': { + 'container': 'osa-container-1', + 'docker_path': None, + 'kind': 'lxc', + 'lxc_info_path': None, + 'machinectl_path': None, + 'python_path': None, + 'username': None, + }, + 'method': 'setns', + }, + ] From c32b8d972889b7eb15d8497cb83f082f2b46eef0 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 21 Aug 2018 00:56:13 +0100 Subject: [PATCH 091/212] docs: fix up doas documentation. --- docs/api.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 69e3c07b..b58069fc 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -528,11 +528,11 @@ Router Class # Use the SSH connection to create a sudo connection. remote_root = router.sudo(username='root', via=remote_machine) - .. method:: dos (username=None, password=None, su_path=None, password_prompt=None, incorrect_prompts=None, \**kwargs) + .. method:: doas (username=None, password=None, doas_path=None, password_prompt=None, incorrect_prompts=None, \**kwargs) - Construct a context on the local machine over a ``su`` invocation. The - ``su`` process is started in a newly allocated pseudo-terminal, and - supports typing interactive passwords. + Construct a context on the local machine over a ``doas`` invocation. + The ``doas`` process is started in a newly allocated pseudo-terminal, + and supports typing interactive passwords. Accepts all parameters accepted by :py:meth:`local`, in addition to: @@ -540,16 +540,16 @@ Router Class Username to use, defaults to ``root``. :param str password: The account password to use if requested. - :param str su_path: - Filename or complete path to the ``su`` binary. ``PATH`` will be - searched if given as a filename. Defaults to ``su``. + :param str doas_path: + Filename or complete path to the ``doas`` binary. ``PATH`` will be + searched if given as a filename. Defaults to ``doas``. :param bytes password_prompt: A string that indicates ``doas`` is requesting a password. Defaults to ``Password:``. :param list incorrect_prompts: List of bytestrings indicating the password is incorrect. Defaults to `(b"doas: authentication failed")`. - :raises mitogen.su.PasswordError: + :raises mitogen.doas.PasswordError: A password was requested but none was provided, the supplied password was incorrect, or the target account did not exist. From 42f07466d2a48d47e7f07944172aef878716a5cb Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 21 Aug 2018 01:16:01 +0100 Subject: [PATCH 092/212] setns: always assume a user identity, default to root. Without this, an invocation like: sudo ansible-playbook foo.yml Where foo.yml uses setns, could inherit the HOME environment variable from the external non-root user, which broke /usr/bin/mysql_upgrade and plenty more. --- docs/api.rst | 5 ++++- mitogen/setns.py | 43 +++++++++++++++++++++---------------------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index b58069fc..67c61dee 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -616,7 +616,7 @@ Router Class Filename or complete path to the ``lxc`` binary. ``PATH`` will be searched if given as a filename. Defaults to ``lxc``. - .. method:: setns (container, kind, docker_path=None, lxc_info_path=None, machinectl_path=None, \**kwargs) + .. method:: setns (container, kind, username=None, docker_path=None, lxc_info_path=None, machinectl_path=None, \**kwargs) Construct a context in the style of :meth:`local`, but change the active Linux process namespaces via calls to `setns(1)` before @@ -633,6 +633,9 @@ Router Class Container to connect to. :param str kind: One of ``docker``, ``lxc``, ``lxd`` or ``machinectl``. + :param str username: + Username within the container to :func:`setuid` to. Defaults to + ``root``. :param str docker_path: Filename or complete path to the Docker binary. ``PATH`` will be searched if given as a filename. Defaults to ``docker``. diff --git a/mitogen/setns.py b/mitogen/setns.py index 224550ce..be87e063 100644 --- a/mitogen/setns.py +++ b/mitogen/setns.py @@ -118,7 +118,7 @@ class Stream(mitogen.parent.Stream): child_is_immediate_subprocess = False container = None - username = None + username = 'root' kind = None python_path = 'python' docker_path = 'docker' @@ -184,27 +184,26 @@ class Stream(mitogen.parent.Stream): except AttributeError: pass - if self.username: - try: - os.setgroups([grent.gr_gid - for grent in grp.getgrall() - if self.username in grent.gr_mem]) - pwent = pwd.getpwnam(self.username) - os.setreuid(pwent.pw_uid, pwent.pw_uid) - # shadow-4.4/libmisc/setupenv.c. Not done: MAIL, PATH - os.environ.update({ - 'HOME': pwent.pw_dir, - 'SHELL': pwent.pw_shell or '/bin/sh', - 'LOGNAME': self.username, - 'USER': self.username, - }) - if ((os.path.exists(pwent.pw_dir) and - os.access(pwent.pw_dir, os.X_OK))): - os.chdir(pwent.pw_dir) - except Exception: - e = sys.exc_info()[1] - raise Error(self.username_msg, self.username, self.container, - type(e).__name__, e) + try: + os.setgroups([grent.gr_gid + for grent in grp.getgrall() + if self.username in grent.gr_mem]) + pwent = pwd.getpwnam(self.username) + os.setreuid(pwent.pw_uid, pwent.pw_uid) + # shadow-4.4/libmisc/setupenv.c. Not done: MAIL, PATH + os.environ.update({ + 'HOME': pwent.pw_dir, + 'SHELL': pwent.pw_shell or '/bin/sh', + 'LOGNAME': self.username, + 'USER': self.username, + }) + if ((os.path.exists(pwent.pw_dir) and + os.access(pwent.pw_dir, os.X_OK))): + os.chdir(pwent.pw_dir) + except Exception: + e = sys.exc_info()[1] + raise Error(self.username_msg, self.username, self.container, + type(e).__name__, e) username_msg = 'while transitioning to user %r in container %r: %s: %s' From 897bc07ea023244d2f2b557854f58cf81ab29b9d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 21 Aug 2018 11:52:34 +0100 Subject: [PATCH 093/212] docs: update changelog. --- docs/changelog.rst | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e41f416d..482d1460 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -103,21 +103,21 @@ Fixes * A missing check caused an exception traceback to appear when using the ``ansible`` command-line tool with a missing or misspelled module name. -* Ansible since >=2.7 began importing ``__main__`` from - ``ansible.module_utils.basic``, causing an error during execution, due to the - controller being configured to refuse network imports outside the +* Ansible since >=2.7 began importing :mod:`__main__` from + :mod:`ansible.module_utils.basic`, causing an error during execution, due to + the controller being configured to refuse network imports outside the ``ansible.*`` namespace. Update the target implementation to construct a stub - ``__main__`` module to satisfy the otherwise seemingly vestigial import. + :mod:`__main__` module to satisfy the otherwise seemingly vestigial import. Core Library ~~~~~~~~~~~~ * `#313 `_: - :meth:`mitogen.parent.Context.call` was documented as capable of accepting - static methods. While possible on Python 2.x the result is very ugly, and in - every case it should be trivially possible to replace with a class method. - The API docs were updated to remove mention of static methods. + :meth:`mitogen.parent.Context.call` was accidentally documented as capable of + accepting static methods. While possible on Python 2.x the result is ugly, + and in every case it should be trivial to replace with a classmethod. The + documentation was fixed. * `#339 `_: the LXD connection method was erroneously executing LXC Classic commands. @@ -125,8 +125,13 @@ Core Library * `#345 `_: the SSH connection method allows optionally disabling ``IdentitiesOnly yes``. -* Add a :func:`mitogen.fork.on_fork` function to allow non-Mitogen managed - process forks to clean up Mitogen resources in the forked chlid. +* `af2ded66 `_: add + :func:`mitogen.fork.on_fork` to allow non-Mitogen managed process forks to + clean up Mitogen resources in the child. + +* `d6784242 `_: the setns method + always resets ``HOME``, ``SHELL``, ``LOGNAME`` and ``USER`` environment + variables to an account in the target container, defaulting to ``root``. Thanks! From 5bac246676ce9a2aa187112fee7cea3ecaa59a06 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 22 Aug 2018 13:09:30 +0100 Subject: [PATCH 094/212] tests: make nice_stdout print failing task line number --- tests/ansible/lib/callback/nice_stdout.py | 32 +++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/ansible/lib/callback/nice_stdout.py b/tests/ansible/lib/callback/nice_stdout.py index 1f7d7c89..1884ee5d 100644 --- a/tests/ansible/lib/callback/nice_stdout.py +++ b/tests/ansible/lib/callback/nice_stdout.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import os import io +from ansible import constants as C from ansible.module_utils import six try: @@ -65,3 +66,34 @@ class CallbackModule(DefaultModule): import traceback traceback.print_exc() raise + + def v2_runner_on_failed(self, result, ignore_errors=False): + delegated_vars = result._result.get('_ansible_delegated_vars') + self._clean_results(result._result, result._task.action) + + if self._play.strategy == 'free' and self._last_task_banner != result._task._uuid: + self._print_task_banner(result._task) + + self._handle_exception(result._result) + self._handle_warnings(result._result) + + if result._task.loop and 'results' in result._result: + return + + if delegated_vars: + msg = "[%s -> %s]: FAILED! => %s" % ( + result._host.get_name(), + delegated_vars['ansible_host'], + self._dump_results(result._result), + ) + else: + msg = "[%s]: FAILED! => %s" % ( + result._host.get_name(), + self._dump_results(result._result), + ) + + s = "fatal: %s: %s" % ( + result._task.get_path() or '(dynamic task)', + msg, + ) + self._display.display(s, color=C.COLOR_ERROR) From 72fa129f8ad9e6a0d6afd62ffba82d8a9f212418 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 22 Aug 2018 13:17:16 +0100 Subject: [PATCH 095/212] tests: fix clash when localhost is test-target --- tests/ansible/integration/action/fixup_perms2__copy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ansible/integration/action/fixup_perms2__copy.yml b/tests/ansible/integration/action/fixup_perms2__copy.yml index 17bfbed9..7e6ef522 100644 --- a/tests/ansible/integration/action/fixup_perms2__copy.yml +++ b/tests/ansible/integration/action/fixup_perms2__copy.yml @@ -49,7 +49,7 @@ # - file: - path: /tmp/weird-mode + path: /tmp/weird-mode.out state: absent - name: Create local test file. @@ -61,10 +61,10 @@ - copy: src: "/tmp/weird-mode" - dest: "/tmp/weird-mode" + dest: "/tmp/weird-mode.out" - stat: - path: "/tmp/weird-mode" + path: "/tmp/weird-mode.out" register: out - assert: that: From 0a2ae4d5978e7343b1fb87e902209b34a8b9b63c Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 22 Aug 2018 13:22:46 +0100 Subject: [PATCH 096/212] tests: tidy up issue_140.yml --- tests/ansible/regression/issue_140__thread_pileup.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/ansible/regression/issue_140__thread_pileup.yml b/tests/ansible/regression/issue_140__thread_pileup.yml index 99f31896..c0158018 100644 --- a/tests/ansible/regression/issue_140__thread_pileup.yml +++ b/tests/ansible/regression/issue_140__thread_pileup.yml @@ -16,7 +16,7 @@ creates: /tmp/filetree.in - name: Delete remote file tree - shell: rm -rf /tmp/filetree.out + file: path=/tmp/filetree.out state=absent - file: state: directory @@ -26,6 +26,5 @@ copy: src: "{{item.src}}" dest: "/tmp/filetree.out/{{item.path}}" - with_filetree: - - /tmp/filetree.in + with_filetree: /tmp/filetree.in when: item.state == 'file' From e52684c186eaa2334c25099ddf7bc70821a54ef5 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 22 Aug 2018 13:36:33 +0100 Subject: [PATCH 097/212] tests: enable display_args_to_stdout --- tests/ansible/ansible.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index 539964b8..ff7ab25b 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -9,6 +9,7 @@ vars_plugins = lib/vars library = lib/modules module_utils = lib/module_utils retry_files_enabled = False +display_args_to_stdout = True forks = 50 # Required by integration/ssh/timeouts.yml From 8ab11f415f7a2bba4b5b8a1687e76814301bf614 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 24 Aug 2018 16:10:46 +0100 Subject: [PATCH 098/212] ansible: better support for diagnosing hangs * Always enable the faulthandler module in the top-level process if it is available. * Make MITOGEN_DUMP_THREAD_STACKS interval configurable, to better handle larger runs. * Add docs subsection on diagnosing hangs. Conflicts: ansible_mitogen/process.py --- ansible_mitogen/process.py | 36 ++++++++++++++++++++---- docs/ansible.rst | 56 +++++++++++++++++++++++++------------- mitogen/debug.py | 7 +++-- 3 files changed, 72 insertions(+), 27 deletions(-) diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index e7ceb2ce..c5e4c016 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -36,6 +36,11 @@ import socket import sys import time +try: + import faulthandler +except ImportError: + faulthandler = None + import mitogen import mitogen.core import mitogen.debug @@ -70,6 +75,17 @@ def clean_shutdown(sock): sock.recv(1) +def getenv_int(key, default=0): + """ + Get an integer-valued environment variable `key`, if it exists and parses + as an integer, otherwise return `default`. + """ + try: + return int(os.environ.get(key, str(default))) + except ValueError: + return default + + class MuxProcess(object): """ Implement a subprocess forked from the Ansible top-level, as a safe place @@ -127,6 +143,9 @@ class MuxProcess(object): if cls.worker_sock is not None: return + if faulthandler is not None: + faulthandler.enable() + cls.unix_listener_path = mitogen.unix.make_socket_path() cls.worker_sock, cls.child_sock = socket.socketpair() atexit.register(lambda: clean_shutdown(cls.worker_sock)) @@ -164,6 +183,15 @@ class MuxProcess(object): # Block until the socket is closed, which happens on parent exit. mitogen.core.io_op(self.child_sock.recv, 1) + def _enable_router_debug(self): + if 'MITOGEN_ROUTER_DEBUG' in os.environ: + self.router.enable_debug() + + def _enable_stack_dumps(self): + secs = getenv_int('MITOGEN_DUMP_THREAD_STACKS', default=0) + if secs: + mitogen.debug.dump_to_logger(secs=secs) + def _setup_master(self): """ Construct a Router, Broker, and mitogen.unix listener @@ -177,10 +205,8 @@ class MuxProcess(object): router=self.router, path=self.unix_listener_path, ) - if 'MITOGEN_ROUTER_DEBUG' in os.environ: - self.router.enable_debug() - if 'MITOGEN_DUMP_THREAD_STACKS' in os.environ: - mitogen.debug.dump_to_logger() + self._enable_router_debug() + self._enable_stack_dumps() def _setup_services(self): """ @@ -195,7 +221,7 @@ class MuxProcess(object): ansible_mitogen.services.ContextService(self.router), ansible_mitogen.services.ModuleDepService(self.router), ], - size=int(os.environ.get('MITOGEN_POOL_SIZE', '16')), + size=getenv_int('MITOGEN_POOL_SIZE', default=16), ) LOG.debug('Service pool configured: size=%d', self.pool.size) diff --git a/docs/ansible.rst b/docs/ansible.rst index 514708bd..14f6e553 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -832,25 +832,43 @@ except connection delegation is supported. Debugging --------- -Diagnostics and use of the :py:mod:`logging` package output on the target -machine are usually discarded. With Mitogen, all of this is captured and -returned to the controller, where it can be viewed as desired with ``-vvv``. -Basic high level logs are produced with ``-vvv``, with logging of all IO on the -controller with ``-vvvv`` or higher. - -Although use of standard IO and the logging package on the target is forwarded -to the controller, it is not possible to receive IO activity logs, as the -process of receiving those logs would would itself generate IO activity. To -receive a complete trace of every process on every machine, file-based logging -is necessary. File-based logging can be enabled by setting -``MITOGEN_ROUTER_DEBUG=1`` in your environment. - -When file-based logging is enabled, one file per context will be created on the -local machine and every target machine, as ``/tmp/mitogen..log``. - -If you are experiencing a hang, ``MITOGEN_DUMP_THREAD_STACKS=1`` causes every -process on every machine to dump every thread stack into the logging framework -every 5 seconds. +Diagnostics and :py:mod:`logging` package output on targets are usually +discarded. With Mitogen, these are captured and forwarded to the controller +where they can be viewed with ``-vvv``. Basic high level logs are produced with +``-vvv``, with logging of all IO on the controller with ``-vvvv`` or higher. + +While uncaptured standard IO and the logging package on targets is forwarded, +it is not possible to receive IO activity logs, as the forwarding process would +would itself generate additional IO. + +To receive a complete trace of every process on every machine, file-based +logging is necessary. File-based logging can be enabled by setting +``MITOGEN_ROUTER_DEBUG=1`` in your environment. When file-based logging is +enabled, one file per context will be created on the local machine and every +target machine, as ``/tmp/mitogen..log``. + +Diagnosing Hangs +~~~~~~~~~~~~~~~~ + +If you encounter a hang, the ``MITOGEN_DUMP_THREAD_STACKS=`` environment +variable arranges for each process on each machine to dump each thread stack +into the logging framework every `secs` seconds, which is visible when running +with ``-vvv``. + +However, certain controller hangs may render ``MITOGEN_DUMP_THREAD_STACKS`` +ineffective, or occur too infrequently for interactive reproduction. In these +cases `faulthandler `_ may be used: + +1. For Python 2, ``pip install faulthandler``. This is unnecessary on Python 3. +2. Once the hang occurs, observe the process tree using ``pstree`` or ``ps + --forest``. +3. The most likely process to be hung is the connection multiplexer, which can + easily be identified as the parent of all SSH client processes. +4. Send ``kill -SEGV `` to the multiplexer PID, causing it to print all + thread stacks. +5. `File a bug `_ including a copy + of the stacks, along with a description of the last task executing prior to + the hang. Getting Help diff --git a/mitogen/debug.py b/mitogen/debug.py index 64d2292d..19cf1a89 100644 --- a/mitogen/debug.py +++ b/mitogen/debug.py @@ -183,15 +183,16 @@ def install_handler(): signal.signal(signal.SIGUSR2, _handler) -def _logging_main(): +def _logging_main(secs): while True: - time.sleep(5) + time.sleep(secs) LOG.info('PERIODIC THREAD DUMP\n\n%s', get_snapshot()) -def dump_to_logger(): +def dump_to_logger(secs=5): th = threading.Thread( target=_logging_main, + kwargs={'secs': secs}, name='mitogen.debug.dump_to_logger', ) th.setDaemon(True) From 946c4964bb1c4340366d3770e350d338c572bc81 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 24 Aug 2018 16:42:54 +0100 Subject: [PATCH 099/212] issue #349: master: fix broken importer logging format string --- mitogen/master.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/master.py b/mitogen/master.py index 671bee85..d8d7e2e8 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -371,7 +371,7 @@ class ModuleFinder(object): # requests.packages.urllib3.contrib.pyopenssl" e = sys.exc_info()[1] LOG.debug('%r: loading %r using %r failed: %s', - self, fullname, loader) + self, fullname, loader, e) return if path is None or source is None: From b0ffc4e209e5d4fbd697ca39f7b536dbe2ea7993 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 1 Sep 2018 15:04:32 +0100 Subject: [PATCH 100/212] Copy random test setup changes out of linear2 branch. --- tests/ansible/.gitignore | 1 + tests/ansible/ansible.cfg | 2 +- tests/ansible/gcloud/controller.yml | 18 ++++ tests/ansible/gcloud/hosts | 2 +- tests/ansible/gcloud/templates/ansible.cfg.j2 | 19 ++++ tests/ansible/gcloud/templates/ssh_config.j2 | 6 ++ tests/ansible/hosts.docker | 100 ------------------ tests/ansible/hosts/connection-delegation | 12 +++ tests/ansible/hosts/localhost | 12 +++ tests/ansible/lib/inventory/gcloud.py | 2 +- 10 files changed, 71 insertions(+), 103 deletions(-) create mode 100644 tests/ansible/gcloud/templates/ansible.cfg.j2 create mode 100644 tests/ansible/gcloud/templates/ssh_config.j2 delete mode 100644 tests/ansible/hosts.docker create mode 100644 tests/ansible/hosts/connection-delegation diff --git a/tests/ansible/.gitignore b/tests/ansible/.gitignore index 1ea0ada7..8d473777 100644 --- a/tests/ansible/.gitignore +++ b/tests/ansible/.gitignore @@ -1,2 +1,3 @@ lib/modules/custom_binary_producing_junk lib/modules/custom_binary_producing_json +hosts/*.local diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index ff7ab25b..68c3ad19 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -10,7 +10,7 @@ library = lib/modules module_utils = lib/module_utils retry_files_enabled = False display_args_to_stdout = True -forks = 50 +forks = 200 # Required by integration/ssh/timeouts.yml timeout = 10 diff --git a/tests/ansible/gcloud/controller.yml b/tests/ansible/gcloud/controller.yml index f7989ddf..494c2164 100644 --- a/tests/ansible/gcloud/controller.yml +++ b/tests/ansible/gcloud/controller.yml @@ -5,6 +5,24 @@ git_email: '{{ lookup("pipe", "git config --global user.email") }}' tasks: + - lineinfile: + line: "{{item}}" + path: /etc/sysctl.conf + register: sysctl_conf + become: true + with_items: + - "net.ipv4.ip_forward=1" + - "kernel.perf_event_paranoid=-1" + + - copy: + src: ~/.ssh/id_gitlab + dest: ~/.ssh/id_gitlab + mode: 0600 + + - template: + dest: ~/.ssh/config + src: ssh_config.j2 + - lineinfile: line: "net.ipv4.ip_forward=1" path: /etc/sysctl.conf diff --git a/tests/ansible/gcloud/hosts b/tests/ansible/gcloud/hosts index b4562cb5..453320e6 100644 --- a/tests/ansible/gcloud/hosts +++ b/tests/ansible/gcloud/hosts @@ -1,2 +1,2 @@ [controller] -35.206.145.240 +c diff --git a/tests/ansible/gcloud/templates/ansible.cfg.j2 b/tests/ansible/gcloud/templates/ansible.cfg.j2 new file mode 100644 index 00000000..aa31c571 --- /dev/null +++ b/tests/ansible/gcloud/templates/ansible.cfg.j2 @@ -0,0 +1,19 @@ +[defaults] +inventory = hosts,~/mitogen/tests/ansible/lib/inventory +gathering = explicit +strategy_plugins = ~/mitogen/ansible_mitogen/plugins/strategy +action_plugins = ~/mitogen/tests/ansible/lib/action +callback_plugins = ~/mitogen/tests/ansible/lib/callback +stdout_callback = nice_stdout +vars_plugins = ~/mitogen/tests/ansible/lib/vars +library = ~/mitogen/tests/ansible/lib/modules +retry_files_enabled = False +forks = 50 + +strategy = mitogen_linear + +host_key_checking = False + +[ssh_connection] +ssh_args = -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s +pipelining = True diff --git a/tests/ansible/gcloud/templates/ssh_config.j2 b/tests/ansible/gcloud/templates/ssh_config.j2 new file mode 100644 index 00000000..2a65bfe7 --- /dev/null +++ b/tests/ansible/gcloud/templates/ssh_config.j2 @@ -0,0 +1,6 @@ + +Host localhost-* + Hostname localhost + +Host gitlab.com + IdentityFile ~/.ssh/id_gitlab diff --git a/tests/ansible/hosts.docker b/tests/ansible/hosts.docker deleted file mode 100644 index 01a2aff7..00000000 --- a/tests/ansible/hosts.docker +++ /dev/null @@ -1,100 +0,0 @@ -mydeb9-1 ansible_connection=docker -mydeb9-2 ansible_connection=docker -mydeb9-3 ansible_connection=docker -mydeb9-4 ansible_connection=docker -mydeb9-5 ansible_connection=docker -mydeb9-6 ansible_connection=docker -mydeb9-7 ansible_connection=docker -mydeb9-8 ansible_connection=docker -mydeb9-9 ansible_connection=docker -mydeb9-10 ansible_connection=docker -mydeb9-11 ansible_connection=docker -mydeb9-12 ansible_connection=docker -mydeb9-13 ansible_connection=docker -mydeb9-14 ansible_connection=docker -mydeb9-15 ansible_connection=docker -mydeb9-16 ansible_connection=docker -mydeb9-17 ansible_connection=docker -mydeb9-18 ansible_connection=docker -mydeb9-19 ansible_connection=docker -mydeb9-20 ansible_connection=docker -mydeb9-21 ansible_connection=docker -mydeb9-22 ansible_connection=docker -mydeb9-23 ansible_connection=docker -mydeb9-24 ansible_connection=docker -mydeb9-25 ansible_connection=docker -mydeb9-26 ansible_connection=docker -mydeb9-27 ansible_connection=docker -mydeb9-28 ansible_connection=docker -mydeb9-29 ansible_connection=docker -mydeb9-30 ansible_connection=docker -mydeb9-31 ansible_connection=docker -mydeb9-32 ansible_connection=docker -mydeb9-33 ansible_connection=docker -mydeb9-34 ansible_connection=docker -mydeb9-35 ansible_connection=docker -mydeb9-36 ansible_connection=docker -mydeb9-37 ansible_connection=docker -mydeb9-38 ansible_connection=docker -mydeb9-39 ansible_connection=docker -mydeb9-40 ansible_connection=docker -mydeb9-41 ansible_connection=docker -mydeb9-42 ansible_connection=docker -mydeb9-43 ansible_connection=docker -mydeb9-44 ansible_connection=docker -mydeb9-45 ansible_connection=docker -mydeb9-46 ansible_connection=docker -mydeb9-47 ansible_connection=docker -mydeb9-48 ansible_connection=docker -mydeb9-49 ansible_connection=docker -mydeb9-50 ansible_connection=docker -mydeb9-51 ansible_connection=docker -mydeb9-52 ansible_connection=docker -mydeb9-53 ansible_connection=docker -mydeb9-54 ansible_connection=docker -mydeb9-55 ansible_connection=docker -mydeb9-56 ansible_connection=docker -mydeb9-57 ansible_connection=docker -mydeb9-58 ansible_connection=docker -mydeb9-59 ansible_connection=docker -mydeb9-60 ansible_connection=docker -mydeb9-61 ansible_connection=docker -mydeb9-62 ansible_connection=docker -mydeb9-63 ansible_connection=docker -mydeb9-64 ansible_connection=docker -mydeb9-65 ansible_connection=docker -mydeb9-66 ansible_connection=docker -mydeb9-67 ansible_connection=docker -mydeb9-68 ansible_connection=docker -mydeb9-69 ansible_connection=docker -mydeb9-70 ansible_connection=docker -mydeb9-71 ansible_connection=docker -mydeb9-72 ansible_connection=docker -mydeb9-73 ansible_connection=docker -mydeb9-74 ansible_connection=docker -mydeb9-75 ansible_connection=docker -mydeb9-76 ansible_connection=docker -mydeb9-77 ansible_connection=docker -mydeb9-78 ansible_connection=docker -mydeb9-79 ansible_connection=docker -mydeb9-80 ansible_connection=docker -mydeb9-81 ansible_connection=docker -mydeb9-82 ansible_connection=docker -mydeb9-83 ansible_connection=docker -mydeb9-84 ansible_connection=docker -mydeb9-85 ansible_connection=docker -mydeb9-86 ansible_connection=docker -mydeb9-87 ansible_connection=docker -mydeb9-88 ansible_connection=docker -mydeb9-89 ansible_connection=docker -mydeb9-90 ansible_connection=docker -mydeb9-91 ansible_connection=docker -mydeb9-92 ansible_connection=docker -mydeb9-93 ansible_connection=docker -mydeb9-94 ansible_connection=docker -mydeb9-95 ansible_connection=docker -mydeb9-96 ansible_connection=docker -mydeb9-97 ansible_connection=docker -mydeb9-98 ansible_connection=docker -mydeb9-99 ansible_connection=docker -mydeb9-100 ansible_connection=docker diff --git a/tests/ansible/hosts/connection-delegation b/tests/ansible/hosts/connection-delegation new file mode 100644 index 00000000..2fb87455 --- /dev/null +++ b/tests/ansible/hosts/connection-delegation @@ -0,0 +1,12 @@ +[connection-delegation-test] +cd-bastion +cd-rack11 mitogen_via=ssh-user@cd-bastion +cd-rack11a mitogen_via=root@cd-rack11 +cd-rack11a-docker mitogen_via=docker-admin@cd-rack11a ansible_connection=docker + +[connection-delegation-cycle] +# Create cycle with Docker container. +cdc-bastion mitogen_via=cdc-rack11a-docker +cdc-rack11 mitogen_via=ssh-user@cdc-bastion +cdc-rack11a mitogen_via=root@cdc-rack11 +cdc-rack11a-docker mitogen_via=docker-admin@cdc-rack11a ansible_connection=docker diff --git a/tests/ansible/hosts/localhost b/tests/ansible/hosts/localhost index dc7df668..d656b43e 100644 --- a/tests/ansible/hosts/localhost +++ b/tests/ansible/hosts/localhost @@ -1,2 +1,14 @@ [test-targets] target ansible_host=localhost + +[localhost-x10] +localhost-1 +localhost-2 +localhost-3 +localhost-4 +localhost-5 +localhost-6 +localhost-7 +localhost-8 +localhost-9 +localhost-10 diff --git a/tests/ansible/lib/inventory/gcloud.py b/tests/ansible/lib/inventory/gcloud.py index 2135d913..73e083f4 100755 --- a/tests/ansible/lib/inventory/gcloud.py +++ b/tests/ansible/lib/inventory/gcloud.py @@ -14,7 +14,7 @@ import googleapiclient.discovery def main(): project = 'mitogen-load-testing' zone = 'europe-west1-d' - group_name = 'target' + group_name = 'micro-debian9' client = googleapiclient.discovery.build('compute', 'v1') resp = client.instances().list(project=project, zone=zone).execute() From acf7fe56eef276446432edfb0371c33a4f41817c Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 5 Sep 2018 21:54:43 +0100 Subject: [PATCH 101/212] tests: a few more inventory helpers. --- tests/ansible/common-hosts | 37 ------------------------------- tests/ansible/hosts/common-hosts | 38 +++++++++++++++++++++++++++++++- tests/ansible/hosts/k3 | 13 +++++++++++ tests/ansible/hosts/localhost | 16 +++++--------- tests/ansible/hosts/nessy | 10 +++++++++ 5 files changed, 65 insertions(+), 49 deletions(-) delete mode 100644 tests/ansible/common-hosts mode change 120000 => 100644 tests/ansible/hosts/common-hosts create mode 100644 tests/ansible/hosts/k3 create mode 100644 tests/ansible/hosts/nessy diff --git a/tests/ansible/common-hosts b/tests/ansible/common-hosts deleted file mode 100644 index 449442f6..00000000 --- a/tests/ansible/common-hosts +++ /dev/null @@ -1,37 +0,0 @@ -# vim: syntax=dosini - -[connection-delegation-test] -cd-bastion -cd-rack11 mitogen_via=ssh-user@cd-bastion -cd-rack11a mitogen_via=root@cd-rack11 -cd-rack11a-docker mitogen_via=docker-admin@cd-rack11a ansible_connection=docker - -[connection-delegation-cycle] -# Create cycle with Docker container. -cdc-bastion mitogen_via=cdc-rack11a-docker -cdc-rack11 mitogen_via=ssh-user@cdc-bastion -cdc-rack11a mitogen_via=root@cdc-rack11 -cdc-rack11a-docker mitogen_via=docker-admin@cdc-rack11a ansible_connection=docker - -[conn-delegation] -cd-user1 ansible_user=mitogen__user1 ansible_connection=mitogen_sudo mitogen_via=target - - -# Connection delegation scenarios. It's impossible to connection to them, but -# you can inspect the would-be config via "mitogen_get_stack" action. -[cd-no-connect] -# Normal inventory host, no aliasing. -cd-normal ansible_connection=mitogen_doas ansible_user=normal-user -# Inventory host that is really a different host. -cd-alias ansible_connection=ssh ansible_user=alias-user ansible_host=alias-host - -# Via one normal host. -cd-normal-normal mitogen_via=cd-normal -# Via one aliased host. -cd-normal-alias mitogen_via=cd-alias - -# newuser@host via host with explicit username. -cd-newuser-normal-normal mitogen_via=cd-normal ansible_user=newuser-normal-normal-user - -# doas:newuser via host. -cd-newuser-doas-normal mitogen_via=cd-normal ansible_connection=mitogen_doas ansible_user=newuser-doas-normal-user diff --git a/tests/ansible/hosts/common-hosts b/tests/ansible/hosts/common-hosts deleted file mode 120000 index f3cc7f59..00000000 --- a/tests/ansible/hosts/common-hosts +++ /dev/null @@ -1 +0,0 @@ -../common-hosts \ No newline at end of file diff --git a/tests/ansible/hosts/common-hosts b/tests/ansible/hosts/common-hosts new file mode 100644 index 00000000..449442f6 --- /dev/null +++ b/tests/ansible/hosts/common-hosts @@ -0,0 +1,37 @@ +# vim: syntax=dosini + +[connection-delegation-test] +cd-bastion +cd-rack11 mitogen_via=ssh-user@cd-bastion +cd-rack11a mitogen_via=root@cd-rack11 +cd-rack11a-docker mitogen_via=docker-admin@cd-rack11a ansible_connection=docker + +[connection-delegation-cycle] +# Create cycle with Docker container. +cdc-bastion mitogen_via=cdc-rack11a-docker +cdc-rack11 mitogen_via=ssh-user@cdc-bastion +cdc-rack11a mitogen_via=root@cdc-rack11 +cdc-rack11a-docker mitogen_via=docker-admin@cdc-rack11a ansible_connection=docker + +[conn-delegation] +cd-user1 ansible_user=mitogen__user1 ansible_connection=mitogen_sudo mitogen_via=target + + +# Connection delegation scenarios. It's impossible to connection to them, but +# you can inspect the would-be config via "mitogen_get_stack" action. +[cd-no-connect] +# Normal inventory host, no aliasing. +cd-normal ansible_connection=mitogen_doas ansible_user=normal-user +# Inventory host that is really a different host. +cd-alias ansible_connection=ssh ansible_user=alias-user ansible_host=alias-host + +# Via one normal host. +cd-normal-normal mitogen_via=cd-normal +# Via one aliased host. +cd-normal-alias mitogen_via=cd-alias + +# newuser@host via host with explicit username. +cd-newuser-normal-normal mitogen_via=cd-normal ansible_user=newuser-normal-normal-user + +# doas:newuser via host. +cd-newuser-doas-normal mitogen_via=cd-normal ansible_connection=mitogen_doas ansible_user=newuser-doas-normal-user diff --git a/tests/ansible/hosts/k3 b/tests/ansible/hosts/k3 new file mode 100644 index 00000000..d39cf399 --- /dev/null +++ b/tests/ansible/hosts/k3 @@ -0,0 +1,13 @@ +k3 + +[k3-x10] +k3-[01:10] + +[k3-x20] +k3-[01:20] + +[k3-x40] +k3-[01:40] + +[k3-x80] +k3-[01:80] diff --git a/tests/ansible/hosts/localhost b/tests/ansible/hosts/localhost index d656b43e..f4dab2ab 100644 --- a/tests/ansible/hosts/localhost +++ b/tests/ansible/hosts/localhost @@ -1,14 +1,8 @@ -[test-targets] +localhost target ansible_host=localhost +[test-targets] +target + [localhost-x10] -localhost-1 -localhost-2 -localhost-3 -localhost-4 -localhost-5 -localhost-6 -localhost-7 -localhost-8 -localhost-9 -localhost-10 +localhost-[01:10] diff --git a/tests/ansible/hosts/nessy b/tests/ansible/hosts/nessy new file mode 100644 index 00000000..5cdef123 --- /dev/null +++ b/tests/ansible/hosts/nessy @@ -0,0 +1,10 @@ +nessy + +[nessy-x10] +nessy-[00:10] + +[nessy-x20] +nessy-[00:20] + +[nessy-x50] +nessy-[00:50] From 2647f735010c532dae4e6bcce3d7203097e72499 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 7 Sep 2018 13:43:39 +0100 Subject: [PATCH 102/212] ansible: bump UNIX listener default backlog, and set it to match forks. The connection multiplexer can expect to not be scheduled at least until every $forks worker processes has attempted a connection, so the backlog must be able to hold every worker. --- ansible_mitogen/process.py | 2 ++ mitogen/unix.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index c5e4c016..daffe7cf 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -50,6 +50,7 @@ import mitogen.service import mitogen.unix import mitogen.utils +import ansible.constants as C import ansible_mitogen.logging import ansible_mitogen.services @@ -204,6 +205,7 @@ class MuxProcess(object): self.listener = mitogen.unix.Listener( router=self.router, path=self.unix_listener_path, + backlog=C.DEFAULT_FORKS, ) self._enable_router_debug() self._enable_stack_dumps() diff --git a/mitogen/unix.py b/mitogen/unix.py index efcc59cc..1097596b 100644 --- a/mitogen/unix.py +++ b/mitogen/unix.py @@ -63,7 +63,7 @@ def make_socket_path(): class Listener(mitogen.core.BasicStream): keep_alive = True - def __init__(self, router, path=None, backlog=30): + def __init__(self, router, path=None, backlog=100): self._router = router self.path = path or make_socket_path() self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) From c4c6ae88a43b58b3b86fa875b7bd73769d090cee Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 7 Sep 2018 15:37:03 +0100 Subject: [PATCH 103/212] parent: raise a descriptive error when openpty fails. --- mitogen/parent.py | 27 +++++++++++++++++++++++++-- tests/parent_test.py | 19 +++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index 0dfc4f07..67b7bf1d 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -78,6 +78,13 @@ try: except: SC_OPEN_MAX = 1024 +OPENPTY_MSG = ( + "Failed to create a PTY: %s. It is likely the maximum number of PTYs has " + "been reached. Consider increasing the 'kern.tty.ptmx_max' sysctl on OS " + "X, the 'kernel.pty.max' sysctl on Linux, or modifying your configuration " + "to avoid PTY use." +) + def get_log_level(): return (LOG.level or logging.getLogger().level or logging.INFO) @@ -254,6 +261,22 @@ def _acquire_controlling_tty(): fcntl.ioctl(2, termios.TIOCSCTTY) +def openpty(): + """ + Call :func:`os.openpty`, raising a descriptive error if the call fails. + + :raises mitogen.core.StreamError: + Creating a PTY failed. + :returns: + See :func`os.openpty`. + """ + try: + return os.openpty() + except OSError: + e = sys.exc_info()[1] + raise mitogen.core.StreamError(OPENPTY_MSG, e) + + def tty_create_child(args): """ Return a file descriptor connected to the master end of a pseudo-terminal, @@ -268,7 +291,7 @@ def tty_create_child(args): :returns: `(pid, tty_fd, None)` """ - master_fd, slave_fd = os.openpty() + master_fd, slave_fd = openpty() mitogen.core.set_block(slave_fd) disable_echo(master_fd) disable_echo(slave_fd) @@ -300,7 +323,7 @@ def hybrid_tty_create_child(args): :returns: `(pid, socketpair_fd, tty_fd)` """ - master_fd, slave_fd = os.openpty() + master_fd, slave_fd = openpty() parentfp, childfp = create_socketpair() mitogen.core.set_block(slave_fd) diff --git a/tests/parent_test.py b/tests/parent_test.py index 0b8b5e9a..53b66c1d 100644 --- a/tests/parent_test.py +++ b/tests/parent_test.py @@ -121,6 +121,25 @@ class ContextTest(testlib.RouterMixin, unittest2.TestCase): self.assertRaises(OSError, lambda: os.kill(pid, 0)) +class OpenPtyTest(testlib.TestCase): + func = staticmethod(mitogen.parent.openpty) + + def test_pty_returned(self): + master_fd, slave_fd = self.func() + self.assertTrue(isinstance(master_fd, int)) + self.assertTrue(isinstance(slave_fd, int)) + os.close(master_fd) + os.close(slave_fd) + + @mock.patch('os.openpty') + def test_max_reached(self, openpty): + openpty.side_effect = OSError(errno.ENXIO) + e = self.assertRaises(mitogen.core.StreamError, + lambda: self.func()) + msg = mitogen.parent.OPENPTY_MSG % (openpty.side_effect,) + self.assertEquals(e.args[0], msg) + + class TtyCreateChildTest(unittest2.TestCase): func = staticmethod(mitogen.parent.tty_create_child) From 50042077051e70c48ae21af54950ba9378766d2b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 7 Sep 2018 19:09:17 +0100 Subject: [PATCH 104/212] issue #337: ssh: support disabling PTY allocation `.ssh(batch_mode=True)` --- docs/api.rst | 315 ++++++++++++++++++++++-------------------- mitogen/ssh.py | 55 ++++++-- tests/data/fakessh.py | 39 ++++++ tests/ssh_test.py | 87 +++++++++++- 4 files changed, 327 insertions(+), 169 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 67c61dee..008ae247 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -35,9 +35,9 @@ mitogen.core Decorator that marks a function or class method to automatically receive a kwarg named `econtext`, referencing the - :py:class:`mitogen.core.ExternalContext` active in the context in which the + :class:`mitogen.core.ExternalContext` active in the context in which the function is being invoked in. The decorator is only meaningful when the - function is invoked via :py:data:`CALL_FUNCTION + function is invoked via :data:`CALL_FUNCTION `. When the function is invoked directly, `econtext` must still be passed to @@ -47,10 +47,10 @@ mitogen.core .. decorator:: takes_router Decorator that marks a function or class method to automatically receive a - kwarg named `router`, referencing the :py:class:`mitogen.core.Router` + kwarg named `router`, referencing the :class:`mitogen.core.Router` active in the context in which the function is being invoked in. The decorator is only meaningful when the function is invoked via - :py:data:`CALL_FUNCTION `. + :data:`CALL_FUNCTION `. When the function is invoked directly, `router` must still be passed to it explicitly. @@ -94,18 +94,18 @@ Message Class .. attribute:: router - The :py:class:`mitogen.core.Router` responsible for routing the - message. This is :py:data:`None` for locally originated messages. + The :class:`mitogen.core.Router` responsible for routing the + message. This is :data:`None` for locally originated messages. .. attribute:: receiver - The :py:class:`mitogen.core.Receiver` over which the message was last - received. Part of the :py:class:`mitogen.select.Select` interface. - Defaults to :py:data:`None`. + The :class:`mitogen.core.Receiver` over which the message was last + received. Part of the :class:`mitogen.select.Select` interface. + Defaults to :data:`None`. .. attribute:: dst_id - Integer target context ID. :py:class:`mitogen.core.Router` delivers + Integer target context ID. :class:`mitogen.core.Router` delivers messages locally when their :attr:`dst_id` matches :data:`mitogen.context_id`, otherwise they are routed up or downstream. @@ -117,12 +117,12 @@ Message Class .. attribute:: auth_id The context ID under whose authority the message is acting. See - :py:ref:`source-verification`. + :ref:`source-verification`. .. attribute:: handle Integer target handle in the destination context. This is one of the - :py:ref:`standard-handles`, or a dynamically generated handle used to + :ref:`standard-handles`, or a dynamically generated handle used to receive a one-time reply, such as the return value of a function call. .. attribute:: reply_to @@ -143,12 +143,12 @@ Message Class .. py:method:: __init__ (\**kwargs) - Construct a message from from the supplied `kwargs`. :py:attr:`src_id` - and :py:attr:`auth_id` are always set to :py:data:`mitogen.context_id`. + Construct a message from from the supplied `kwargs`. :attr:`src_id` + and :attr:`auth_id` are always set to :data:`mitogen.context_id`. .. py:classmethod:: pickled (obj, \**kwargs) - Construct a pickled message, setting :py:attr:`data` to the + Construct a pickled message, setting :attr:`data` to the serialization of `obj`, and setting remaining fields using `kwargs`. :returns: @@ -156,10 +156,10 @@ Message Class .. method:: unpickle (throw=True) - Unpickle :py:attr:`data`, optionally raising any exceptions present. + Unpickle :attr:`data`, optionally raising any exceptions present. :param bool throw: - If :py:data:`True`, raise exceptions, otherwise it is the caller's + If :data:`True`, raise exceptions, otherwise it is the caller's responsibility. :raises mitogen.core.CallError: @@ -169,8 +169,8 @@ Message Class .. method:: reply (obj, router=None, \**kwargs) - Compose a reply to this message and send it using :py:attr:`router`, or - `router` is :py:attr:`router` is :data:`None`. + Compose a reply to this message and send it using :attr:`router`, or + `router` is :attr:`router` is :data:`None`. :param obj: Either a :class:`Message`, or an object to be serialized in order @@ -190,8 +190,8 @@ Router Class .. class:: Router Route messages between parent and child contexts, and invoke handlers - defined on our parent context. :py:meth:`Router.route() ` straddles - the :py:class:`Broker ` and user threads, it is safe + defined on our parent context. :meth:`Router.route() ` straddles + the :class:`Broker ` and user threads, it is safe to call anywhere. **Note:** This is the somewhat limited core version of the Router class @@ -217,7 +217,7 @@ Router Class .. method:: stream_by_id (dst_id) - Return the :py:class:`mitogen.core.Stream` that should be used to + Return the :class:`mitogen.core.Stream` that should be used to communicate with `dst_id`. If a specific route for `dst_id` is not known, a reference to the parent context's stream is returned. @@ -260,24 +260,24 @@ Router Class :param function policy: Function invoked as `policy(msg, stream)` where `msg` is a - :py:class:`mitogen.core.Message` about to be delivered, and - `stream` is the :py:class:`mitogen.core.Stream` on which it was - received. The function must return :py:data:`True`, otherwise an + :class:`mitogen.core.Message` about to be delivered, and + `stream` is the :class:`mitogen.core.Stream` on which it was + received. The function must return :data:`True`, otherwise an error is logged and delivery is refused. Two built-in policy functions exist: - * :py:func:`mitogen.core.has_parent_authority`: requires the + * :func:`mitogen.core.has_parent_authority`: requires the message arrived from a parent context, or a context acting with a parent context's authority (``auth_id``). - * :py:func:`mitogen.parent.is_immediate_child`: requires the + * :func:`mitogen.parent.is_immediate_child`: requires the message arrived from an immediately connected child, for use in messaging patterns where either something becomes buggy or insecure by permitting indirect upstream communication. In case of refusal, and the message's ``reply_to`` field is - nonzero, a :py:class:`mitogen.core.CallError` is delivered to the + nonzero, a :class:`mitogen.core.CallError` is delivered to the sender indicating refusal occurred. :return: @@ -297,7 +297,7 @@ Router Class destination is the local context, then arrange for it to be dispatched using the local handlers. - This is a lower overhead version of :py:meth:`route` that may only be + This is a lower overhead version of :meth:`route` that may only be called from the I/O multiplexer thread. :param mitogen.core.Stream stream: @@ -308,11 +308,11 @@ Router Class .. method:: route(msg) - Arrange for the :py:class:`Message` `msg` to be delivered to its + Arrange for the :class:`Message` `msg` to be delivered to its destination using any relevant downstream context, or if none is found, by forwarding the message upstream towards the master context. If `msg` is destined for the local context, it is dispatched using the handles - registered with :py:meth:`add_handler`. + registered with :meth:`add_handler`. This may be called from any thread. @@ -321,7 +321,7 @@ Router Class .. class:: Router (broker=None) - Extend :py:class:`mitogen.core.Router` with functionality useful to + Extend :class:`mitogen.core.Router` with functionality useful to masters, and child contexts who later become masters. Currently when this class is required, the target context's router is upgraded at runtime. @@ -334,16 +334,16 @@ Router Class customers or projects. :param mitogen.master.Broker broker: - :py:class:`Broker` instance to use. If not specified, a private - :py:class:`Broker` is created. + :class:`Broker` instance to use. If not specified, a private + :class:`Broker` is created. .. attribute:: profiling When :data:`True`, cause the broker thread and any subsequent broker and main threads existing in any child to write ``/tmp/mitogen.stats...log`` containing a - :py:mod:`cProfile` dump on graceful exit. Must be set prior to - construction of any :py:class:`Broker`, e.g. via: + :mod:`cProfile` dump on graceful exit. Must be set prior to + construction of any :class:`Broker`, e.g. via: .. code:: @@ -378,7 +378,7 @@ Router Class and router, and responds to function calls identically to children created using other methods. - For long-lived processes, :py:meth:`local` is always better as it + For long-lived processes, :meth:`local` is always better as it guarantees a pristine interpreter state that inherited little from the parent. Forking should only be used in performance-sensitive scenarios where short-lived children must be spawned to isolate potentially buggy @@ -420,10 +420,10 @@ Router Class immediate copy-on-write to large portions of the process heap. * Locks held in the parent causing random deadlocks in the child, such - as when another thread emits a log entry via the :py:mod:`logging` - package concurrent to another thread calling :py:meth:`fork`. + as when another thread emits a log entry via the :mod:`logging` + package concurrent to another thread calling :meth:`fork`. - * Objects existing in Thread-Local Storage of every non-:py:meth:`fork` + * Objects existing in Thread-Local Storage of every non-:meth:`fork` thread becoming permanently inaccessible, and never having their object destructors called, including TLS usage by native extension code, triggering many new variants of all the issues above. @@ -434,16 +434,16 @@ Router Class case, children continually reuse the same state due to repeatedly forking from a static parent. - :py:meth:`fork` cleans up Mitogen-internal objects, in addition to - locks held by the :py:mod:`logging` package, reseeds - :py:func:`random.random`, and the OpenSSL PRNG via - :py:func:`ssl.RAND_add`, but only if the :py:mod:`ssl` module is + :meth:`fork` cleans up Mitogen-internal objects, in addition to + locks held by the :mod:`logging` package, reseeds + :func:`random.random`, and the OpenSSL PRNG via + :func:`ssl.RAND_add`, but only if the :mod:`ssl` module is already loaded. You must arrange for your program's state, including any third party packages in use, to be cleaned up by specifying an `on_fork` function. The associated stream implementation is - :py:class:`mitogen.fork.Stream`. + :class:`mitogen.fork.Stream`. :param function on_fork: Function invoked as `on_fork()` from within the child process. This @@ -459,19 +459,19 @@ Router Class serialization. :param Context via: - Same as the `via` parameter for :py:meth:`local`. + Same as the `via` parameter for :meth:`local`. :param bool debug: - Same as the `debug` parameter for :py:meth:`local`. + Same as the `debug` parameter for :meth:`local`. :param bool profiling: - Same as the `profiling` parameter for :py:meth:`local`. + Same as the `profiling` parameter for :meth:`local`. .. method:: local (remote_name=None, python_path=None, debug=False, connect_timeout=None, profiling=False, via=None) Construct a context on the local machine as a subprocess of the current process. The associated stream implementation is - :py:class:`mitogen.master.Stream`. + :class:`mitogen.master.Stream`. :param str remote_name: The ``argv[0]`` suffix for the new process. If `remote_name` is @@ -493,9 +493,9 @@ Router Class another tool, such as ``["/usr/bin/env", "python"]``. :param bool debug: - If :data:`True`, arrange for debug logging (:py:meth:`enable_debug`) to + If :data:`True`, arrange for debug logging (:meth:`enable_debug`) to be enabled in the new context. Automatically :data:`True` when - :py:meth:`enable_debug` has been called, but may be used + :meth:`enable_debug` has been called, but may be used selectively otherwise. :param bool unidirectional: @@ -510,14 +510,14 @@ Router Class healthy. Defaults to 30 seconds. :param bool profiling: - If :data:`True`, arrange for profiling (:py:data:`profiling`) to be + If :data:`True`, arrange for profiling (:data:`profiling`) to be enabled in the new context. Automatically :data:`True` when - :py:data:`profiling` is :data:`True`, but may be used selectively + :data:`profiling` is :data:`True`, but may be used selectively otherwise. :param mitogen.core.Context via: If not :data:`None`, arrange for construction to occur via RPCs - made to the context `via`, and for :py:data:`ADD_ROUTE + made to the context `via`, and for :data:`ADD_ROUTE ` messages to be generated as appropriate. .. code-block:: python @@ -534,7 +534,7 @@ Router Class The ``doas`` process is started in a newly allocated pseudo-terminal, and supports typing interactive passwords. - Accepts all parameters accepted by :py:meth:`local`, in addition to: + Accepts all parameters accepted by :meth:`local`, in addition to: :param str username: Username to use, defaults to ``root``. @@ -559,7 +559,7 @@ Router Class temporary new Docker container using the ``docker`` program. One of `container` or `image` must be specified. - Accepts all parameters accepted by :py:meth:`local`, in addition to: + Accepts all parameters accepted by :meth:`local`, in addition to: :param str container: Existing container to connect to. Defaults to :data:`None`. @@ -578,7 +578,7 @@ Router Class Construct a context on the local machine within a FreeBSD jail using the ``jexec`` program. - Accepts all parameters accepted by :py:meth:`local`, in addition to: + Accepts all parameters accepted by :meth:`local`, in addition to: :param str container: Existing container to connect to. Defaults to :data:`None`. @@ -594,7 +594,7 @@ Router Class Construct a context on the local machine within an LXC classic container using the ``lxc-attach`` program. - Accepts all parameters accepted by :py:meth:`local`, in addition to: + Accepts all parameters accepted by :meth:`local`, in addition to: :param str container: Existing container to connect to. Defaults to :data:`None`. @@ -608,7 +608,7 @@ Router Class Construct a context on the local machine within a LXD container using the ``lxc`` program. - Accepts all parameters accepted by :py:meth:`local`, in addition to: + Accepts all parameters accepted by :meth:`local`, in addition to: :param str container: Existing container to connect to. Defaults to :data:`None`. @@ -656,7 +656,7 @@ Router Class ``su`` process is started in a newly allocated pseudo-terminal, and supports typing interactive passwords. - Accepts all parameters accepted by :py:meth:`local`, in addition to: + Accepts all parameters accepted by :meth:`local`, in addition to: :param str username: Username to pass to ``su``, defaults to ``root``. @@ -683,7 +683,7 @@ Router Class The ``sudo`` process is started in a newly allocated pseudo-terminal, and supports typing interactive passwords. - Accepts all parameters accepted by :py:meth:`local`, in addition to: + Accepts all parameters accepted by :meth:`local`, in addition to: :param str username: Username to pass to sudo as the ``-u`` parameter, defaults to @@ -694,28 +694,32 @@ Router Class :param str password: The password to use if/when sudo requests it. Depending on the sudo configuration, this is either the current account password or the - target account password. :py:class:`mitogen.sudo.PasswordError` + target account password. :class:`mitogen.sudo.PasswordError` will be raised if sudo requests a password but none is provided. :param bool set_home: - If :py:data:`True`, request ``sudo`` set the ``HOME`` environment + If :data:`True`, request ``sudo`` set the ``HOME`` environment variable to match the target UNIX account. :param bool preserve_env: - If :py:data:`True`, request ``sudo`` to preserve the environment of + If :data:`True`, request ``sudo`` to preserve the environment of the parent process. :param list sudo_args: - Arguments in the style of :py:data:`sys.argv` that would normally + Arguments in the style of :data:`sys.argv` that would normally be passed to ``sudo``. The arguments are parsed in-process to set equivalent parameters. Re-parsing ensures unsupported options cause - :py:class:`mitogen.core.StreamError` to be raised, and that + :class:`mitogen.core.StreamError` to be raised, and that attributes of the stream match the actual behaviour of ``sudo``. .. method:: ssh (hostname, username=None, ssh_path=None, port=None, check_host_keys='enforce', password=None, identity_file=None, identities_only=True, compression=True, \**kwargs) - Construct a remote context over a ``ssh`` invocation. The ``ssh`` - process is started in a newly allocated pseudo-terminal, and supports - typing interactive passwords. + Construct a remote context over an OpenSSH ``ssh`` invocation. - Accepts all parameters accepted by :py:meth:`local`, in addition to: + By default, the ``ssh`` process is started in a newly allocated + pseudo-terminal to support typing interactive passwords, however when + making many connections, this may be disabled by specifying + `batch_mode=True`, as most operating systems have a conservative upper + limit on the number of pseudo-terminals that may exist. + + Accepts all parameters accepted by :meth:`local`, in addition to: :param str username: The SSH username; default is unspecified, which causes SSH to pick @@ -737,7 +741,7 @@ Router Class unknown hosts cause a connection failure. :param str password: Password to type if/when ``ssh`` requests it. If not specified and - a password is requested, :py:class:`mitogen.ssh.PasswordError` is + a password is requested, :class:`mitogen.ssh.PasswordError` is raised. :param str identity_file: Path to an SSH private key file to use for authentication. Default @@ -755,12 +759,17 @@ Router Class present in ``~/.ssh``. This ensures authentication attempts only occur using the supplied password or SSH key. :param bool compression: - If :py:data:`True`, enable ``ssh`` compression support. Compression + If :data:`True`, enable ``ssh`` compression support. Compression has a minimal effect on the size of modules transmitted, as they are already compressed, however it has a large effect on every remaining message in the otherwise uncompressed stream protocol, such as function call arguments and return values. - :parama int ssh_debug_level: + :param bool batch_mode: + If :data:`True`, disable pseudo-terminal allocation. When + :data:`True`, the `password=` parameter may not be used, since no + PTY exists to enter the password, and the `check_host_keys=` + parameter may not be set to `accept`. + :param int ssh_debug_level: Optional integer `0..3` indicating the SSH client debug level. :raises mitogen.ssh.PasswordError: A password was requested but none was specified, or the specified @@ -808,12 +817,12 @@ Context Class The message. :returns: - :py:class:`mitogen.core.Receiver` configured to receive any replies + :class:`mitogen.core.Receiver` configured to receive any replies sent to the message's `reply_to` handle. .. method:: send_await (msg, deadline=None) - As with :py:meth:`send_async`, but expect a single reply + As with :meth:`send_async`, but expect a single reply (`persist=False`) delivered within `deadline` seconds. :param mitogen.core.Message msg: @@ -830,7 +839,7 @@ Context Class .. class:: Context - Extend :py:class:`mitogen.core.Router` with functionality useful to + Extend :class:`mitogen.core.Router` with functionality useful to masters, and child contexts who later become parents. Currently when this class is required, the target context's router is upgraded at runtime. @@ -843,7 +852,7 @@ Context Class terminate a hung context using this method. This will be fixed shortly. :param bool wait: - If :py:data:`True`, block the calling thread until the context has + If :data:`True`, block the calling thread until the context has completely terminated. :returns: If `wait` is :data:`False`, returns a :class:`mitogen.core.Latch` @@ -888,7 +897,7 @@ Context Class Function keyword arguments, if any. See :ref:`serialization-rules` for permitted types. :returns: - :py:class:`mitogen.core.Receiver` configured to receive the result + :class:`mitogen.core.Receiver` configured to receive the result of the invocation: .. code-block:: python @@ -903,11 +912,11 @@ Context Class Asynchronous calls may be dispatched in parallel to multiple contexts and consumed as they complete using - :py:class:`mitogen.select.Select`. + :class:`mitogen.select.Select`. .. method:: call (fn, \*args, \*\*kwargs) - Equivalent to :py:meth:`call_async(fn, \*args, \**kwargs).get().unpickle() + Equivalent to :meth:`call_async(fn, \*args, \**kwargs).get().unpickle() `. :returns: @@ -935,7 +944,7 @@ Receiver Class Receivers are used to wait for pickled responses from another context to be sent to a handle registered in this context. A receiver may be single-use - (as in the case of :py:meth:`mitogen.parent.Context.call_async`) or + (as in the case of :meth:`mitogen.parent.Context.call_async`) or multiple use. :param mitogen.core.Router router: @@ -959,12 +968,12 @@ Receiver Class If not :data:`None`, a reference to a function invoked as `notify(receiver)` when a new message is delivered to this receiver. - Used by :py:class:`mitogen.select.Select` to implement waiting on + Used by :class:`mitogen.select.Select` to implement waiting on multiple receivers. .. py:method:: to_sender () - Return a :py:class:`mitogen.core.Sender` configured to deliver messages + Return a :class:`mitogen.core.Sender` configured to deliver messages to this receiver. Since a Sender can be serialized, this makes it convenient to pass `(context_id, handle)` pairs around:: @@ -981,15 +990,15 @@ Receiver Class .. py:method:: empty () - Return :data:`True` if calling :py:meth:`get` would block. + Return :data:`True` if calling :meth:`get` would block. - As with :py:class:`Queue.Queue`, :data:`True` may be returned even - though a subsequent call to :py:meth:`get` will succeed, since a - message may be posted at any moment between :py:meth:`empty` and - :py:meth:`get`. + As with :class:`Queue.Queue`, :data:`True` may be returned even + though a subsequent call to :meth:`get` will succeed, since a + message may be posted at any moment between :meth:`empty` and + :meth:`get`. - :py:meth:`empty` is only useful to avoid a race while installing - :py:attr:`notify`: + :meth:`empty` is only useful to avoid a race while installing + :attr:`notify`: .. code-block:: python @@ -1003,8 +1012,8 @@ Receiver Class .. py:method:: close () - Cause :py:class:`mitogen.core.ChannelError` to be raised in any thread - waiting in :py:meth:`get` on this receiver. + Cause :class:`mitogen.core.ChannelError` to be raised in any thread + waiting in :meth:`get` on this receiver. .. py:method:: get (timeout=None) @@ -1022,17 +1031,17 @@ Receiver Class :returns: `(msg, data)` tuple, where `msg` is the - :py:class:`mitogen.core.Message` that was received, and `data` is + :class:`mitogen.core.Message` that was received, and `data` is its unpickled data part. .. py:method:: get_data (timeout=None) - Like :py:meth:`get`, except only return the data part. + Like :meth:`get`, except only return the data part. .. py:method:: __iter__ () Block and yield `(msg, data)` pairs delivered to this receiver until - :py:class:`mitogen.core.ChannelError` is raised. + :class:`mitogen.core.ChannelError` is raised. Sender Class @@ -1043,10 +1052,10 @@ Sender Class .. class:: Sender (context, dst_handle) Senders are used to send pickled messages to a handle in another context, - it is the inverse of :py:class:`mitogen.core.Sender`. + it is the inverse of :class:`mitogen.core.Sender`. Senders may be serialized, making them convenient to wire up data flows. - See :py:meth:`mitogen.core.Receiver.to_sender` for more information. + See :meth:`mitogen.core.Receiver.to_sender` for more information. :param mitogen.core.Context context: Context to send messages to. @@ -1055,7 +1064,7 @@ Sender Class .. py:method:: close () - Send a dead message to the remote end, causing :py:meth:`ChannelError` + Send a dead message to the remote end, causing :meth:`ChannelError` to be raised in any waiting thread. .. py:method:: send (data) @@ -1074,11 +1083,11 @@ Select Class Support scatter/gather asynchronous calls and waiting on multiple receivers, channels, and sub-Selects. Accepts a sequence of - :py:class:`mitogen.core.Receiver` or :py:class:`mitogen.select.Select` + :class:`mitogen.core.Receiver` or :class:`mitogen.select.Select` instances and returns the first value posted to any receiver or select. If `oneshot` is :data:`True`, then remove each receiver as it yields a - result; since :py:meth:`__iter__` terminates once the final receiver is + result; since :meth:`__iter__` terminates once the final receiver is removed, this makes it convenient to respond to calls made in parallel: .. code-block:: python @@ -1093,7 +1102,7 @@ Select Class # Iteration ends when last Receiver yields a result. print('Received total %s from %s receivers' % (total, len(recvs))) - :py:class:`Select` may drive a long-running scheduler: + :class:`Select` may drive a long-running scheduler: .. code-block:: python @@ -1104,7 +1113,7 @@ Select Class for context, workfunc in get_new_work(): select.add(context.call_async(workfunc)) - :py:class:`Select` may be nested: + :class:`Select` may be nested: .. code-block:: python @@ -1122,11 +1131,11 @@ Select Class .. py:classmethod:: all (it) - Take an iterable of receivers and retrieve a :py:class:`Message` from + Take an iterable of receivers and retrieve a :class:`Message` from each, returning the result of calling `msg.unpickle()` on each in turn. Results are returned in the order they arrived. - This is sugar for handling batch :py:class:`Context.call_async` + This is sugar for handling batch :class:`Context.call_async` invocations: .. code-block:: python @@ -1146,28 +1155,28 @@ Select Class for context in contexts) Result processing happens concurrently to new results arriving, so - :py:meth:`all` should always be faster. + :meth:`all` should always be faster. .. py:method:: get (timeout=None, block=True) Fetch the next available value from any receiver, or raise - :py:class:`mitogen.core.TimeoutError` if no value is available within + :class:`mitogen.core.TimeoutError` if no value is available within `timeout` seconds. - On success, the message's :py:attr:`receiver + On success, the message's :attr:`receiver ` attribute is set to the receiver. :param float timeout: Timeout in seconds. :param bool block: - If :py:data:`False`, immediately raise - :py:class:`mitogen.core.TimeoutError` if the select is empty. + If :data:`False`, immediately raise + :class:`mitogen.core.TimeoutError` if the select is empty. :return: - :py:class:`mitogen.core.Message` + :class:`mitogen.core.Message` :raises mitogen.core.TimeoutError: Timeout was reached. :raises mitogen.core.LatchError: - :py:meth:`close` has been called, and the underlying latch is no + :meth:`close` has been called, and the underlying latch is no longer valid. .. py:method:: __bool__ () @@ -1178,8 +1187,8 @@ Select Class Remove the select's notifier function from each registered receiver, mark the associated latch as closed, and cause any thread currently - sleeping in :py:meth:`get` to be woken with - :py:class:`mitogen.core.LatchError`. + sleeping in :meth:`get` to be woken with + :class:`mitogen.core.LatchError`. This is necessary to prevent memory leaks in long-running receivers. It is called automatically when the Python :keyword:`with` statement is @@ -1187,35 +1196,35 @@ Select Class .. py:method:: empty () - Return :data:`True` if calling :py:meth:`get` would block. + Return :data:`True` if calling :meth:`get` would block. - As with :py:class:`Queue.Queue`, :data:`True` may be returned even - though a subsequent call to :py:meth:`get` will succeed, since a - message may be posted at any moment between :py:meth:`empty` and - :py:meth:`get`. + As with :class:`Queue.Queue`, :data:`True` may be returned even + though a subsequent call to :meth:`get` will succeed, since a + message may be posted at any moment between :meth:`empty` and + :meth:`get`. - :py:meth:`empty` may return :data:`False` even when :py:meth:`get` + :meth:`empty` may return :data:`False` even when :meth:`get` would block if another thread has drained a receiver added to this select. This can be avoided by only consuming each receiver from a single thread. .. py:method:: __iter__ (self) - Yield the result of :py:meth:`get` until no receivers remain in the + Yield the result of :meth:`get` until no receivers remain in the select, either because `oneshot` is :data:`True`, or each receiver was - explicitly removed via :py:meth:`remove`. + explicitly removed via :meth:`remove`. .. py:method:: add (recv) - Add the :py:class:`mitogen.core.Receiver` or - :py:class:`mitogen.core.Channel` `recv` to the select. + Add the :class:`mitogen.core.Receiver` or + :class:`mitogen.core.Channel` `recv` to the select. .. py:method:: remove (recv) - Remove the :py:class:`mitogen.core.Receiver` or - :py:class:`mitogen.core.Channel` `recv` from the select. Note that if - the receiver has notified prior to :py:meth:`remove`, then it will - still be returned by a subsequent :py:meth:`get`. This may change in a + Remove the :class:`mitogen.core.Receiver` or + :class:`mitogen.core.Channel` `recv` from the select. Note that if + the receiver has notified prior to :meth:`remove`, then it will + still be returned by a subsequent :meth:`get`. This may change in a future version. @@ -1226,7 +1235,7 @@ Channel Class .. class:: Channel (router, context, dst_handle, handle=None) - A channel inherits from :py:class:`mitogen.core.Sender` and + A channel inherits from :class:`mitogen.core.Sender` and `mitogen.core.Receiver` to provide bidirectional functionality. Since all handles aren't known until after both ends are constructed, for @@ -1248,8 +1257,8 @@ Broker Class .. attribute:: shutdown_timeout = 3.0 - Seconds grace to allow :py:class:`streams ` to shutdown - gracefully before force-disconnecting them during :py:meth:`shutdown`. + Seconds grace to allow :class:`streams ` to shutdown + gracefully before force-disconnecting them during :meth:`shutdown`. .. method:: defer (func, \*args, \*kwargs) @@ -1259,26 +1268,26 @@ Broker Class .. method:: start_receive (stream) - Mark the :py:attr:`receive_side ` on `stream` as + Mark the :attr:`receive_side ` on `stream` as ready for reading. Safe to call from any thread. When the associated file descriptor becomes ready for reading, - :py:meth:`BasicStream.on_receive` will be called. + :meth:`BasicStream.on_receive` will be called. .. method:: stop_receive (stream) - Mark the :py:attr:`receive_side ` on `stream` as + Mark the :attr:`receive_side ` on `stream` as not ready for reading. Safe to call from any thread. .. method:: _start_transmit (stream) - Mark the :py:attr:`transmit_side ` on `stream` as + Mark the :attr:`transmit_side ` on `stream` as ready for writing. Must only be called from the Broker thread. When the associated file descriptor becomes ready for writing, - :py:meth:`BasicStream.on_transmit` will be called. + :meth:`BasicStream.on_transmit` will be called. .. method:: stop_receive (stream) - Mark the :py:attr:`transmit_side ` on `stream` as + Mark the :attr:`transmit_side ` on `stream` as not ready for writing. Safe to call from any thread. .. method:: shutdown @@ -1288,12 +1297,12 @@ Broker Class .. method:: join Wait for the broker to stop, expected to be called after - :py:meth:`shutdown`. + :meth:`shutdown`. .. method:: keep_alive - Return :data:`True` if any reader's :py:attr:`Side.keep_alive` - attribute is :data:`True`, or any :py:class:`Context` is still + Return :data:`True` if any reader's :attr:`Side.keep_alive` + attribute is :data:`True`, or any :class:`Context` is still registered that is not the master. Used to delay shutdown while some important work is in progress (e.g. log draining). @@ -1301,11 +1310,11 @@ Broker Class .. method:: _broker_main - Handle events until :py:meth:`shutdown`. On shutdown, invoke - :py:meth:`Stream.on_shutdown` for every active stream, then allow up to - :py:attr:`shutdown_timeout` seconds for the streams to unregister + Handle events until :meth:`shutdown`. On shutdown, invoke + :meth:`Stream.on_shutdown` for every active stream, then allow up to + :attr:`shutdown_timeout` seconds for the streams to unregister themselves before forcefully calling - :py:meth:`Stream.on_disconnect`. + :meth:`Stream.on_disconnect`. .. currentmodule:: mitogen.master @@ -1321,7 +1330,7 @@ Broker Class :param bool install_watcher: If :data:`True`, an additional thread is started to monitor the - lifetime of the main thread, triggering :py:meth:`shutdown` + lifetime of the main thread, triggering :meth:`shutdown` automatically in case the user forgets to call it, or their code crashed. @@ -1332,8 +1341,8 @@ Broker Class .. attribute:: shutdown_timeout = 5.0 - Seconds grace to allow :py:class:`streams ` to shutdown - gracefully before force-disconnecting them during :py:meth:`shutdown`. + Seconds grace to allow :class:`streams ` to shutdown + gracefully before force-disconnecting them during :meth:`shutdown`. Utility Functions @@ -1349,7 +1358,7 @@ A random assortment of utility functions useful on masters and children. Many tools love to subclass built-in types in order to implement useful functionality, such as annotating the safety of a Unicode string, or adding additional methods to a dict. However, cPickle loves to preserve those - subtypes during serialization, resulting in CallError during :py:meth:`call + subtypes during serialization, resulting in CallError during :meth:`call ` in the target when it tries to deserialize the data. @@ -1369,12 +1378,12 @@ A random assortment of utility functions useful on masters and children. Remove all entries mentioning ``site-packages`` or ``Extras`` from the system path. Used primarily for testing on OS X within a virtualenv, where - OS X bundles some ancient version of the :py:mod:`six` module. + OS X bundles some ancient version of the :mod:`six` module. .. currentmodule:: mitogen.utils .. function:: log_to_file (path=None, io=False, level='INFO') - Install a new :py:class:`logging.Handler` writing applications logs to the + Install a new :class:`logging.Handler` writing applications logs to the filesystem. Useful when debugging slave IO problems. Parameters to this function may be overridden at runtime using environment @@ -1382,14 +1391,14 @@ A random assortment of utility functions useful on masters and children. :param str path: If not :data:`None`, a filesystem path to write logs to. Otherwise, - logs are written to :py:data:`sys.stderr`. + logs are written to :data:`sys.stderr`. :param bool io: If :data:`True`, include extremely verbose IO logs in the output. Useful for debugging hangs, less useful for debugging application code. :param str level: - Name of the :py:mod:`logging` package constant that is the minimum + Name of the :mod:`logging` package constant that is the minimum level to log at. Useful levels are ``DEBUG``, ``INFO``, ``WARNING``, and ``ERROR``. @@ -1397,7 +1406,7 @@ A random assortment of utility functions useful on masters and children. .. function:: run_with_router(func, \*args, \**kwargs) Arrange for `func(router, \*args, \**kwargs)` to run with a temporary - :py:class:`mitogen.master.Router`, ensuring the Router and Broker are + :class:`mitogen.master.Router`, ensuring the Router and Broker are correctly shut down during normal or exceptional return. :returns: @@ -1406,7 +1415,7 @@ A random assortment of utility functions useful on masters and children. .. currentmodule:: mitogen.utils .. decorator:: with_router - Decorator version of :py:func:`run_with_router`. Example: + Decorator version of :func:`run_with_router`. Example: .. code-block:: python diff --git a/mitogen/ssh.py b/mitogen/ssh.py index 38e12531..2650d32f 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -111,7 +111,6 @@ class HostKeyError(mitogen.core.StreamError): class Stream(mitogen.parent.Stream): - create_child = staticmethod(mitogen.parent.hybrid_tty_create_child) child_is_immediate_subprocess = False #: Default to whatever is available as 'python' on the remote machine, @@ -121,8 +120,8 @@ class Stream(mitogen.parent.Stream): #: Number of -v invocations to pass on command line. ssh_debug_level = 0 - #: Once connected, points to the corresponding TtyLogStream, allowing it to - #: be disconnected at the same time this stream is being torn down. + #: If batch_mode=False, points to the corresponding TtyLogStream, allowing + #: it to be disconnected at the same time this stream is being torn down. tty_stream = None #: The path to the SSH binary. @@ -137,15 +136,27 @@ class Stream(mitogen.parent.Stream): ssh_args = None check_host_keys_msg = 'check_host_keys= must be set to accept, enforce or ignore' + batch_mode_check_host_keys_msg = ( + 'check_host_keys cannot be set to "accept" when batch mode is ' + 'enabled, since batch mode disables PTY allocation.' + ) + batch_mode_password_msg = ( + 'A password cannot be set when batch mode is enabled, ' + 'since batch mode disables PTY allocation.' + ) def construct(self, hostname, username=None, ssh_path=None, port=None, check_host_keys='enforce', password=None, identity_file=None, compression=True, ssh_args=None, keepalive_enabled=True, - keepalive_count=3, keepalive_interval=15, + keepalive_count=3, keepalive_interval=15, batch_mode=False, identities_only=True, ssh_debug_level=None, **kwargs): super(Stream, self).construct(**kwargs) if check_host_keys not in ('accept', 'enforce', 'ignore'): raise ValueError(self.check_host_keys_msg) + if check_host_keys == 'accept' and batch_mode: + raise ValueError(self.batch_mode_check_host_keys_msg) + if password is not None and batch_mode: + raise ValueError(self.batch_mode_password_msg) self.hostname = hostname self.username = username @@ -158,6 +169,14 @@ class Stream(mitogen.parent.Stream): self.keepalive_enabled = keepalive_enabled self.keepalive_count = keepalive_count self.keepalive_interval = keepalive_interval + self.batch_mode = batch_mode + if self.batch_mode: + self.create_child = mitogen.parent.create_child + self.create_child_args = { + 'merge_stdio': True, + } + else: + self.create_child = mitogen.parent.hybrid_tty_create_child if ssh_path: self.ssh_path = ssh_path if ssh_args: @@ -166,7 +185,8 @@ class Stream(mitogen.parent.Stream): self.ssh_debug_level = ssh_debug_level def on_disconnect(self, broker): - self.tty_stream.on_disconnect(broker) + if self.tty_stream is not None: + self.tty_stream.on_disconnect(broker) super(Stream, self).on_disconnect(broker) def get_boot_command(self): @@ -193,10 +213,15 @@ class Stream(mitogen.parent.Stream): '-o', 'ServerAliveInterval %s' % (self.keepalive_interval,), '-o', 'ServerAliveCountMax %s' % (self.keepalive_count,), ] + if self.batch_mode: + bits += ['-o', 'BatchMode yes'] if self.check_host_keys == 'enforce': bits += ['-o', 'StrictHostKeyChecking yes'] if self.check_host_keys == 'accept': - bits += ['-o', 'StrictHostKeyChecking ask'] + if self.batch_mode: + bits += ['-o', 'StrictHostKeyChecking no'] + else: + bits += ['-o', 'StrictHostKeyChecking ask'] elif self.check_host_keys == 'ignore': bits += [ '-o', 'StrictHostKeyChecking no', @@ -240,19 +265,23 @@ class Stream(mitogen.parent.Stream): # with ours. raise HostKeyError(self.hostkey_config_msg) + def _ec0_received(self): + if self.tty_stream is not None: + self._router.broker.start_receive(self.tty_stream) + return super(Stream, self)._ec0_received() + def _connect_bootstrap(self, extra_fd): - self.tty_stream = mitogen.parent.TtyLogStream(extra_fd, self) + fds = [self.receive_side.fd] + if extra_fd is not None: + self.tty_stream = mitogen.parent.TtyLogStream(extra_fd, self) + fds.append(extra_fd) - password_sent = False - it = mitogen.parent.iter_read( - fds=[self.receive_side.fd, extra_fd], - deadline=self.connect_deadline - ) + it = mitogen.parent.iter_read(fds=fds, deadline=self.connect_deadline) + password_sent = False for buf, partial in filter_debug(self, it): LOG.debug('%r: received %r', self, buf) if buf.endswith(self.EC0_MARKER): - self._router.broker.start_receive(self.tty_stream) self._ec0_received() return elif HOSTKEY_REQ_PROMPT in buf.lower(): diff --git a/tests/data/fakessh.py b/tests/data/fakessh.py index 08a5da3e..415425af 100755 --- a/tests/data/fakessh.py +++ b/tests/data/fakessh.py @@ -6,6 +6,45 @@ import shlex import subprocess import sys + +HOST_KEY_ASK_MSG = """ +The authenticity of host '[91.121.165.123]:9122 ([91.121.165.123]:9122)' can't be established. +ECDSA key fingerprint is SHA256:JvfPvazZzQ9/CUdKN7tiYlNZtDRdEgDsYVIzOgPrsR4. +Are you sure you want to continue connecting (yes/no)? +""".strip('\n') + +HOST_KEY_STRICT_MSG = """Host key verification failed.\n""" + + +def tty(msg): + fp = open('/dev/tty', 'w', 0) + fp.write(msg) + fp.close() + + +def stderr(msg): + fp = open('/dev/stderr', 'w', 0) + fp.write(msg) + fp.close() + + +def confirm(msg): + tty(msg) + fp = open('/dev/tty', 'r', 0) + try: + return fp.readline() + finally: + fp.close() + + +if os.getenv('FAKESSH_MODE') == 'ask': + assert 'y\n' == confirm(HOST_KEY_ASK_MSG) + +if os.getenv('FAKESSH_MODE') == 'strict': + stderr(HOST_KEY_STRICT_MSG) + sys.exit(255) + + parser = optparse.OptionParser() parser.add_option('--user', '-l', action='store') parser.add_option('-o', dest='options', action='append') diff --git a/tests/ssh_test.py b/tests/ssh_test.py index a514c8ea..a305ec70 100644 --- a/tests/ssh_test.py +++ b/tests/ssh_test.py @@ -1,3 +1,4 @@ +import os import sys import mitogen @@ -13,9 +14,9 @@ import plain_old_module class FakeSshTest(testlib.RouterMixin, unittest2.TestCase): def test_okay(self): context = self.router.ssh( - hostname='hostname', - username='mitogen__has_sudo', - ssh_path=testlib.data_path('fakessh.py'), + hostname='hostname', + username='mitogen__has_sudo', + ssh_path=testlib.data_path('fakessh.py'), ) #context.call(mitogen.utils.log_to_file, '/tmp/log') #context.call(mitogen.utils.disable_site_packages) @@ -123,6 +124,86 @@ class BannerTest(testlib.DockerMixin, unittest2.TestCase): self.assertEquals(name, context.name) +class BatchModeTest(testlib.DockerMixin, testlib.TestCase): + stream_class = mitogen.ssh.Stream + # + # Test that: + # + # - batch_mode=false, host_key_checking=accept + # - batch_mode=false, host_key_checking=enforce + # - batch_mode=false, host_key_checking=ignore + # + # - batch_mode=true, host_key_checking=accept + # - batch_mode=true, host_key_checking=enforce + # - batch_mode=true, host_key_checking=ignore + # - batch_mode=true, password is not None + # + def fake_ssh(self, FAKESSH_MODE=None, **kwargs): + os.environ['FAKESSH_MODE'] = str(FAKESSH_MODE) + try: + return self.router.ssh( + hostname='hostname', + username='mitogen__has_sudo', + ssh_path=testlib.data_path('fakessh.py'), + **kwargs + ) + finally: + del os.environ['FAKESSH_MODE'] + + def test_false_accept(self): + # Should succeed. + self.fake_ssh(FAKESSH_MODE='ask', check_host_keys='accept') + + def test_false_enforce(self): + # Should succeed. + self.fake_ssh(check_host_keys='enforce') + + def test_false_ignore(self): + # Should succeed. + self.fake_ssh(check_host_keys='ignore') + + def test_false_password(self): + # Should succeed. + self.docker_ssh(username='mitogen__has_sudo_nopw', + password='has_sudo_nopw_password') + + def test_true_accept(self): + e = self.assertRaises(ValueError, + lambda: self.fake_ssh(check_host_keys='accept', batch_mode=True) + ) + self.assertEquals(e.args[0], + self.stream_class.batch_mode_check_host_keys_msg) + + def test_true_enforce(self): + e = self.assertRaises(mitogen.ssh.HostKeyError, + lambda: self.docker_ssh( + batch_mode=True, + check_host_keys='enforce', + ssh_args=['-o', 'UserKnownHostsFile /dev/null'], + ) + ) + self.assertEquals(e.args[0], self.stream_class.hostkey_failed_msg) + + def test_true_ignore(self): + e = self.assertRaises(mitogen.ssh.HostKeyError, + lambda: self.fake_ssh( + FAKESSH_MODE='strict', + batch_mode=True, + check_host_keys='ignore', + ) + ) + self.assertEquals(e.args[0], self.stream_class.hostkey_failed_msg) + + def test_true_password(self): + e = self.assertRaises(ValueError, + lambda: self.fake_ssh( + password='nope', + batch_mode=True, + ) + ) + self.assertEquals(e.args[0], self.stream_class.batch_mode_password_msg) + + if __name__ == '__main__': unittest2.main() From 7d62a53264599e780ba6b075e6fc6a78d430f287 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 7 Sep 2018 20:44:08 +0100 Subject: [PATCH 105/212] issue #337: ssh: disabling PTYs round 2: make it automatic. --- docs/api.rst | 16 +++----- mitogen/core.py | 5 ++- mitogen/doas.py | 6 +-- mitogen/parent.py | 34 ++++++++++++----- mitogen/ssh.py | 60 +++++++++++++++-------------- mitogen/su.py | 2 +- mitogen/sudo.py | 4 +- tests/data/fakessh.py | 7 ++++ tests/ssh_test.py | 87 +++++++++++-------------------------------- 9 files changed, 99 insertions(+), 122 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 008ae247..e823b877 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -713,11 +713,12 @@ Router Class Construct a remote context over an OpenSSH ``ssh`` invocation. - By default, the ``ssh`` process is started in a newly allocated - pseudo-terminal to support typing interactive passwords, however when - making many connections, this may be disabled by specifying - `batch_mode=True`, as most operating systems have a conservative upper - limit on the number of pseudo-terminals that may exist. + The ``ssh`` process is started in a newly allocated pseudo-terminal to + support typing interactive passwords and responding to prompts, if a + password is specified, or `check_host_keys=accept`. In other scenarios, + ``BatchMode`` is enabled and no PTY is allocated. For many-target + configurations, both options should be avoided as most systems have a + conservative limit on the number of pseudo-terminals that may exist. Accepts all parameters accepted by :meth:`local`, in addition to: @@ -764,11 +765,6 @@ Router Class are already compressed, however it has a large effect on every remaining message in the otherwise uncompressed stream protocol, such as function call arguments and return values. - :param bool batch_mode: - If :data:`True`, disable pseudo-terminal allocation. When - :data:`True`, the `password=` parameter may not be used, since no - PTY exists to enter the password, and the `check_host_keys=` - parameter may not be set to `accept`. :param int ssh_debug_level: Optional integer `0..3` indicating the SSH client debug level. :raises mitogen.ssh.PasswordError: diff --git a/mitogen/core.py b/mitogen/core.py index 12983071..9df0fa1d 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1103,8 +1103,9 @@ class Stream(BasicStream): ) if msg_len > self._router.max_message_size: - LOG.error('Maximum message size exceeded (got %d, max %d)', - msg_len, self._router.max_message_size) + LOG.error('Maximum message size exceeded (got %d, max %d) %r', + msg_len, self._router.max_message_size, + self._input_buf[0]) self.on_disconnect(broker) return False diff --git a/mitogen/doas.py b/mitogen/doas.py index 1d9d04eb..cdcee0b0 100644 --- a/mitogen/doas.py +++ b/mitogen/doas.py @@ -45,8 +45,8 @@ class Stream(mitogen.parent.Stream): create_child = staticmethod(mitogen.parent.hybrid_tty_create_child) child_is_immediate_subprocess = False - #: Once connected, points to the corresponding TtyLogStream, allowing it to - #: be disconnected at the same time this stream is being torn down. + #: Once connected, points to the corresponding DiagLogStream, allowing it + #: to be disconnected at the same time this stream is being torn down. tty_stream = None username = 'root' @@ -89,7 +89,7 @@ class Stream(mitogen.parent.Stream): password_required_msg = 'doas password is required' def _connect_bootstrap(self, extra_fd): - self.tty_stream = mitogen.parent.TtyLogStream(extra_fd, self) + self.tty_stream = mitogen.parent.DiagLogStream(extra_fd, self) password_sent = False it = mitogen.parent.iter_read( diff --git a/mitogen/parent.py b/mitogen/parent.py index 67b7bf1d..d7177dde 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -205,7 +205,7 @@ def detach_popen(*args, **kwargs): return proc.pid -def create_child(args, merge_stdio=False, preexec_fn=None): +def create_child(args, merge_stdio=False, stderr_pipe=False, preexec_fn=None): """ Create a child process whose stdin/stdout is connected to a socket. @@ -216,8 +216,13 @@ def create_child(args, merge_stdio=False, preexec_fn=None): socketpair, rather than inherited from the parent process. This may be necessary to ensure that not TTY is connected to any stdio handle, for instance when using LXC. + :param bool stderr_pipe: + If :data:`True` and `merge_stdio` is :data:`False`, arrange for + `stderr` to be connected to a separate pipe, to allow any ongoing debug + logs generated by e.g. SSH to be outpu as the session progresses, + without interfering with `stdout`. :returns: - `(pid, socket_obj, :data:`None`)` + `(pid, socket_obj, :data:`None` or pipe_fd)` """ parentfp, childfp = create_socketpair() # When running under a monkey patches-enabled gevent, the socket module @@ -228,8 +233,12 @@ def create_child(args, merge_stdio=False, preexec_fn=None): if merge_stdio: extra = {'stderr': childfp} - else: - extra = {} + stderr_r = None + elif stderr_pipe: + stderr_r, stderr_w = os.pipe() + mitogen.core.set_cloexec(stderr_r) + mitogen.core.set_cloexec(stderr_w) + extra = {'stderr': stderr_w} pid = detach_popen( args=args, @@ -239,6 +248,8 @@ def create_child(args, merge_stdio=False, preexec_fn=None): preexec_fn=preexec_fn, **extra ) + if stderr_pipe: + os.close(stderr_w) childfp.close() # Decouple the socket from the lifetime of the Python socket object. fd = os.dup(parentfp.fileno()) @@ -246,7 +257,7 @@ def create_child(args, merge_stdio=False, preexec_fn=None): LOG.debug('create_child() child %d fd %d, parent %d, cmd: %s', pid, fd, os.getpid(), Argv(args)) - return pid, fd, None + return pid, fd, stderr_r def _acquire_controlling_tty(): @@ -792,7 +803,7 @@ PREFERRED_POLLER = POLLER_BY_SYSNAME.get( mitogen.core.Latch.poller_class = PREFERRED_POLLER -class TtyLogStream(mitogen.core.BasicStream): +class DiagLogStream(mitogen.core.BasicStream): """ For "hybrid TTY/socketpair" mode, after a connection has been setup, a spare TTY file descriptor will exist that cannot be closed, and to which @@ -802,18 +813,21 @@ class TtyLogStream(mitogen.core.BasicStream): termination signal to any processes whose controlling TTY is the TTY that has been closed. - TtyLogStream takes over this descriptor and creates corresponding log + DiagLogStream takes over this descriptor and creates corresponding log messages for anything written to it. """ - def __init__(self, tty_fd, stream): - self.receive_side = mitogen.core.Side(self, tty_fd) + def __init__(self, fd, stream): + self.receive_side = mitogen.core.Side(self, fd) self.transmit_side = self.receive_side self.stream = stream self.buf = '' def __repr__(self): - return 'mitogen.parent.TtyLogStream(%r)' % (self.stream.name,) + return 'mitogen.parent.DiagLogStream(fd=%r, %r)' % ( + self.receive_side.fd, + self.stream.name, + ) def on_receive(self, broker): """ diff --git a/mitogen/ssh.py b/mitogen/ssh.py index 2650d32f..670294f1 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -120,7 +120,7 @@ class Stream(mitogen.parent.Stream): #: Number of -v invocations to pass on command line. ssh_debug_level = 0 - #: If batch_mode=False, points to the corresponding TtyLogStream, allowing + #: If batch_mode=False, points to the corresponding DiagLogStream, allowing #: it to be disconnected at the same time this stream is being torn down. tty_stream = None @@ -136,27 +136,15 @@ class Stream(mitogen.parent.Stream): ssh_args = None check_host_keys_msg = 'check_host_keys= must be set to accept, enforce or ignore' - batch_mode_check_host_keys_msg = ( - 'check_host_keys cannot be set to "accept" when batch mode is ' - 'enabled, since batch mode disables PTY allocation.' - ) - batch_mode_password_msg = ( - 'A password cannot be set when batch mode is enabled, ' - 'since batch mode disables PTY allocation.' - ) def construct(self, hostname, username=None, ssh_path=None, port=None, check_host_keys='enforce', password=None, identity_file=None, compression=True, ssh_args=None, keepalive_enabled=True, - keepalive_count=3, keepalive_interval=15, batch_mode=False, + keepalive_count=3, keepalive_interval=15, identities_only=True, ssh_debug_level=None, **kwargs): super(Stream, self).construct(**kwargs) if check_host_keys not in ('accept', 'enforce', 'ignore'): raise ValueError(self.check_host_keys_msg) - if check_host_keys == 'accept' and batch_mode: - raise ValueError(self.batch_mode_check_host_keys_msg) - if password is not None and batch_mode: - raise ValueError(self.batch_mode_password_msg) self.hostname = hostname self.username = username @@ -169,14 +157,6 @@ class Stream(mitogen.parent.Stream): self.keepalive_enabled = keepalive_enabled self.keepalive_count = keepalive_count self.keepalive_interval = keepalive_interval - self.batch_mode = batch_mode - if self.batch_mode: - self.create_child = mitogen.parent.create_child - self.create_child_args = { - 'merge_stdio': True, - } - else: - self.create_child = mitogen.parent.hybrid_tty_create_child if ssh_path: self.ssh_path = ssh_path if ssh_args: @@ -184,6 +164,30 @@ class Stream(mitogen.parent.Stream): if ssh_debug_level: self.ssh_debug_level = ssh_debug_level + self._init_create_child() + + def _requires_pty(self): + """ + Return :data:`True` if the configuration requires a PTY to be + allocated. This is only true if we must interactively accept host keys, + or type a password. + """ + return (self.check_host_keys == 'accept' or + self.password is not None) + + def _init_create_child(self): + """ + Initialize the base class :attr:`create_child` and + :attr:`create_child_args` according to whether we need a PTY or not. + """ + if self._requires_pty(): + self.create_child = mitogen.parent.hybrid_tty_create_child + else: + self.create_child = mitogen.parent.create_child + self.create_child_args = { + 'stderr_pipe': True, + } + def on_disconnect(self, broker): if self.tty_stream is not None: self.tty_stream.on_disconnect(broker) @@ -213,15 +217,12 @@ class Stream(mitogen.parent.Stream): '-o', 'ServerAliveInterval %s' % (self.keepalive_interval,), '-o', 'ServerAliveCountMax %s' % (self.keepalive_count,), ] - if self.batch_mode: + if not self._requires_pty(): bits += ['-o', 'BatchMode yes'] if self.check_host_keys == 'enforce': bits += ['-o', 'StrictHostKeyChecking yes'] if self.check_host_keys == 'accept': - if self.batch_mode: - bits += ['-o', 'StrictHostKeyChecking no'] - else: - bits += ['-o', 'StrictHostKeyChecking ask'] + bits += ['-o', 'StrictHostKeyChecking ask'] elif self.check_host_keys == 'ignore': bits += [ '-o', 'StrictHostKeyChecking no', @@ -273,7 +274,7 @@ class Stream(mitogen.parent.Stream): def _connect_bootstrap(self, extra_fd): fds = [self.receive_side.fd] if extra_fd is not None: - self.tty_stream = mitogen.parent.TtyLogStream(extra_fd, self) + self.tty_stream = mitogen.parent.DiagLogStream(extra_fd, self) fds.append(extra_fd) it = mitogen.parent.iter_read(fds=fds, deadline=self.connect_deadline) @@ -294,6 +295,9 @@ class Stream(mitogen.parent.Stream): # it at the start of the line. if self.password is not None and password_sent: raise PasswordError(self.password_incorrect_msg) + elif 'password' in buf and self.password is None: + # Permission denied (password,pubkey) + raise PasswordError(self.password_required_msg) else: raise PasswordError(self.auth_incorrect_msg) elif partial and PASSWORD_PROMPT in buf.lower(): diff --git a/mitogen/su.py b/mitogen/su.py index 45229d6d..7e2e5f08 100644 --- a/mitogen/su.py +++ b/mitogen/su.py @@ -49,7 +49,7 @@ class Stream(mitogen.parent.Stream): create_child = staticmethod(mitogen.parent.tty_create_child) child_is_immediate_subprocess = False - #: Once connected, points to the corresponding TtyLogStream, allowing it to + #: Once connected, points to the corresponding DiagLogStream, allowing it to #: be disconnected at the same time this stream is being torn down. username = 'root' diff --git a/mitogen/sudo.py b/mitogen/sudo.py index 402d8549..c410dac9 100644 --- a/mitogen/sudo.py +++ b/mitogen/sudo.py @@ -107,7 +107,7 @@ class Stream(mitogen.parent.Stream): create_child = staticmethod(mitogen.parent.hybrid_tty_create_child) child_is_immediate_subprocess = False - #: Once connected, points to the corresponding TtyLogStream, allowing it to + #: Once connected, points to the corresponding DiagLogStream, allowing it to #: be disconnected at the same time this stream is being torn down. tty_stream = None @@ -165,7 +165,7 @@ class Stream(mitogen.parent.Stream): password_required_msg = 'sudo password is required' def _connect_bootstrap(self, extra_fd): - self.tty_stream = mitogen.parent.TtyLogStream(extra_fd, self) + self.tty_stream = mitogen.parent.DiagLogStream(extra_fd, self) password_sent = False it = mitogen.parent.iter_read( diff --git a/tests/data/fakessh.py b/tests/data/fakessh.py index 415425af..69d47339 100755 --- a/tests/data/fakessh.py +++ b/tests/data/fakessh.py @@ -45,6 +45,13 @@ if os.getenv('FAKESSH_MODE') == 'strict': sys.exit(255) +# +# Set an env var if stderr was a TTY to make ssh_test tests easier to write. +# +if os.isatty(2): + os.environ['STDERR_WAS_TTY'] = '1' + + parser = optparse.OptionParser() parser.add_option('--user', '-l', action='store') parser.add_option('-o', dest='options', action='append') diff --git a/tests/ssh_test.py b/tests/ssh_test.py index a305ec70..efca057d 100644 --- a/tests/ssh_test.py +++ b/tests/ssh_test.py @@ -124,20 +124,9 @@ class BannerTest(testlib.DockerMixin, unittest2.TestCase): self.assertEquals(name, context.name) -class BatchModeTest(testlib.DockerMixin, testlib.TestCase): +class RequirePtyTest(testlib.DockerMixin, testlib.TestCase): stream_class = mitogen.ssh.Stream - # - # Test that: - # - # - batch_mode=false, host_key_checking=accept - # - batch_mode=false, host_key_checking=enforce - # - batch_mode=false, host_key_checking=ignore - # - # - batch_mode=true, host_key_checking=accept - # - batch_mode=true, host_key_checking=enforce - # - batch_mode=true, host_key_checking=ignore - # - batch_mode=true, password is not None - # + def fake_ssh(self, FAKESSH_MODE=None, **kwargs): os.environ['FAKESSH_MODE'] = str(FAKESSH_MODE) try: @@ -150,59 +139,25 @@ class BatchModeTest(testlib.DockerMixin, testlib.TestCase): finally: del os.environ['FAKESSH_MODE'] - def test_false_accept(self): - # Should succeed. - self.fake_ssh(FAKESSH_MODE='ask', check_host_keys='accept') - - def test_false_enforce(self): - # Should succeed. - self.fake_ssh(check_host_keys='enforce') - - def test_false_ignore(self): - # Should succeed. - self.fake_ssh(check_host_keys='ignore') - - def test_false_password(self): - # Should succeed. - self.docker_ssh(username='mitogen__has_sudo_nopw', - password='has_sudo_nopw_password') - - def test_true_accept(self): - e = self.assertRaises(ValueError, - lambda: self.fake_ssh(check_host_keys='accept', batch_mode=True) - ) - self.assertEquals(e.args[0], - self.stream_class.batch_mode_check_host_keys_msg) - - def test_true_enforce(self): - e = self.assertRaises(mitogen.ssh.HostKeyError, - lambda: self.docker_ssh( - batch_mode=True, - check_host_keys='enforce', - ssh_args=['-o', 'UserKnownHostsFile /dev/null'], - ) - ) - self.assertEquals(e.args[0], self.stream_class.hostkey_failed_msg) - - def test_true_ignore(self): - e = self.assertRaises(mitogen.ssh.HostKeyError, - lambda: self.fake_ssh( - FAKESSH_MODE='strict', - batch_mode=True, - check_host_keys='ignore', - ) - ) - self.assertEquals(e.args[0], self.stream_class.hostkey_failed_msg) - - def test_true_password(self): - e = self.assertRaises(ValueError, - lambda: self.fake_ssh( - password='nope', - batch_mode=True, - ) - ) - self.assertEquals(e.args[0], self.stream_class.batch_mode_password_msg) - + def test_check_host_keys_accept(self): + # required=true, host_key_checking=accept + context = self.fake_ssh(FAKESSH_MODE='ask', check_host_keys='accept') + self.assertEquals('1', context.call(os.getenv, 'STDERR_WAS_TTY')) + + def test_check_host_keys_enforce(self): + # required=false, host_key_checking=enforce + context = self.fake_ssh(check_host_keys='enforce') + self.assertEquals(None, context.call(os.getenv, 'STDERR_WAS_TTY')) + + def test_check_host_keys_ignore(self): + # required=false, host_key_checking=ignore + context = self.fake_ssh(check_host_keys='ignore') + self.assertEquals(None, context.call(os.getenv, 'STDERR_WAS_TTY')) + + def test_password_present(self): + # required=true, password is not None + context = self.fake_ssh(check_host_keys='ignore', password='willick') + self.assertEquals('1', context.call(os.getenv, 'STDERR_WAS_TTY')) if __name__ == '__main__': From 6c8a66769134200c988f79d4c47f260e4182c093 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 7 Sep 2018 21:09:41 +0100 Subject: [PATCH 106/212] docs: update Changelog. --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 482d1460..b79fa2dd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -119,6 +119,13 @@ Core Library and in every case it should be trivial to replace with a classmethod. The documentation was fixed. +* `#337 `_: to avoid a scaling + limitation, SSH no longer allocates a PTY for every OpenSSH client. PTYs are + only allocated if a password is supplied, or when `host_key_checking=accept`. + This is since Linux has a default of 4096 PTYs (``kernel.pty.max``), while OS + X has a default of 127 and an absolute maximum of 999 + (``kern.tty.ptmx_max``). + * `#339 `_: the LXD connection method was erroneously executing LXC Classic commands. From 870bbe0eaef7e5413727662a3d7cede45375f53b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 7 Sep 2018 21:30:32 +0100 Subject: [PATCH 107/212] unix: don't crash listener if remote end disconnects. In some scenarios, Ansible's worker seems to exit early, resulting in EPIPE during .recv() or .send(). Log an error and gracefully disconnect in that case. --- mitogen/unix.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/mitogen/unix.py b/mitogen/unix.py index 1097596b..4a4dfb65 100644 --- a/mitogen/unix.py +++ b/mitogen/unix.py @@ -78,22 +78,39 @@ class Listener(mitogen.core.BasicStream): self.receive_side = mitogen.core.Side(self, self._sock.fileno()) router.broker.start_receive(self) - def on_receive(self, broker): - sock, _ = self._sock.accept() + def _accept_client(self, sock): sock.setblocking(True) - pid, = struct.unpack('>L', sock.recv(4)) + try: + pid, = struct.unpack('>L', sock.recv(4)) + except socket.error: + LOG.error('%r: failed to read remote identity: %s', + self, sys.exc_info()[1]) + return context_id = self._router.id_allocator.allocate() context = mitogen.parent.Context(self._router, context_id) stream = mitogen.core.Stream(self._router, context_id) - stream.accept(sock.fileno(), sock.fileno()) stream.name = u'unix_client.%d' % (pid,) stream.auth_id = mitogen.context_id stream.is_privileged = True + + try: + sock.send(struct.pack('>LLL', context_id, mitogen.context_id, + os.getpid())) + except socket.error: + LOG.error('%r: failed to assign identity to PID %d: %s', + self, pid, sys.exc_info()[1]) + return + + stream.accept(sock.fileno(), sock.fileno()) self._router.register(context, stream) - sock.send(struct.pack('>LLL', context_id, mitogen.context_id, - os.getpid())) - sock.close() + + def on_receive(self, broker): + sock, _ = self._sock.accept() + try: + self._accept_client(sock) + finally: + sock.close() def connect(path, broker=None): From 1b6dea24baff5501eab73f49e2b2a92eeb1d77cb Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 7 Sep 2018 21:32:35 +0100 Subject: [PATCH 108/212] docs: update changelog. --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index b79fa2dd..b410dae9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -140,6 +140,10 @@ Core Library always resets ``HOME``, ``SHELL``, ``LOGNAME`` and ``USER`` environment variables to an account in the target container, defaulting to ``root``. +* `830966bf `_: the UNIX + listener no longer crashes if the peer process disappears in the middle of + connection setup. + Thanks! ~~~~~~~ From c6159c9154b2d7c908382160e475b6fdd4c7187d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 7 Sep 2018 22:21:09 +0100 Subject: [PATCH 109/212] core: fix startup logging race. Closes #305. --- mitogen/core.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index 9df0fa1d..a67dc290 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -950,6 +950,16 @@ class LogHandler(logging.Handler): logging.Handler.__init__(self) self.context = context self.local = threading.local() + self._buffer = [] + + def uncork(self): + self._send = self.context.send + for msg in self._buffer: + self._send(msg) + self._buffer = None + + def _send(self, msg): + self._buffer.append(msg) def emit(self, rec): if rec.name == 'mitogen.io' or \ @@ -963,7 +973,7 @@ class LogHandler(logging.Handler): if isinstance(encoded, UnicodeType): # Logging package emits both :( encoded = encoded.encode('utf-8') - self.context.send(Message(data=encoded, handle=FORWARD_LOG)) + self._send(Message(data=encoded, handle=FORWARD_LOG)) finally: self.local.in_emit = False @@ -2053,9 +2063,10 @@ class ExternalContext(object): pass # No first stage exists (e.g. fakessh) def _setup_logging(self): + self.log_handler = LogHandler(self.master) root = logging.getLogger() root.setLevel(self.config['log_level']) - root.handlers = [LogHandler(self.master)] + root.handlers = [self.log_handler] if self.config['debug']: enable_debug_logging() @@ -2186,6 +2197,7 @@ class ExternalContext(object): self._setup_stdio() self.router.register(self.parent, self.stream) + self.log_handler.uncork() sys.executable = os.environ.pop('ARGV0', sys.executable) _v and LOG.debug('Connected to %s; my ID is %r, PID is %r', From 57fb00cf6b7952d6ae7eaff1f59652c2387aeb25 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 7 Sep 2018 22:23:08 +0100 Subject: [PATCH 110/212] docs: update changelog. --- docs/changelog.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b410dae9..a60f5319 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -113,6 +113,10 @@ Fixes Core Library ~~~~~~~~~~~~ +* `#305 `_: fix a long-standing minor + race relating to the logging framework, where *no route for Message(...)(* + would appear fruequently during startup. + * `#313 `_: :meth:`mitogen.parent.Context.call` was accidentally documented as capable of accepting static methods. While possible on Python 2.x the result is ugly, @@ -340,10 +344,6 @@ Mitogen for Ansible - initech_app - y2k_fix -* When running with ``-vvv``, log messages such as *mitogen: Router(Broker(0x7f5a48921590)): no route - for Message(..., 102, ...), my ID is ...* may be visible. These are due to a - minor race while initializing logging and can be ignored. - .. * When running with ``-vvv``, log messages will be printed to the console *after* the Ansible run completes, as connection multiplexer shutdown only begins after Ansible exits. This is due to a lack of suitable shutdown hook From e647adc62e7219735b89982c3dfa12e4e5641dae Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 7 Sep 2018 22:32:00 +0100 Subject: [PATCH 111/212] ansible: copy GIL change from linear2 branch. Reduces runtime by 25% given 100 25ms SSH targets: ANSIBLE_STRATEGY=mitogen \ MITOGEN_POOL_SIZE=100 \ /usr/bin/time -l ansible k3-x100 -m shell -a hostname Before: 39.56 real 35.29 user 17.24 sys 59600896 maximum resident set size 1784252 page reclaims 9016 messages sent 10382 messages received 18774 voluntary context switches 770070 involuntary context switches After: 29.79 real 22.10 user 11.77 sys 59281408 maximum resident set size 1725268 page reclaims 8582 messages sent 9959 messages received 14582 voluntary context switches 75280 involuntary context switches --- ansible_mitogen/process.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index daffe7cf..6e18a863 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -87,6 +87,27 @@ def getenv_int(key, default=0): return default +def setup_gil(): + """ + Set extremely long GIL release interval to let threads naturally progress + through CPU-heavy sequences without forcing the wake of another thread that + may contend trying to run the same CPU-heavy code. For the new-style work, + this drops runtime ~33% and involuntary context switches by >80%, + essentially making threads cooperatively scheduled. + """ + try: + # Python 2. + sys.setcheckinterval(100000) + except AttributeError: + pass + + try: + # Python 3. + sys.setswitchinterval(10) + except AttributeError: + pass + + class MuxProcess(object): """ Implement a subprocess forked from the Ansible top-level, as a safe place @@ -147,6 +168,7 @@ class MuxProcess(object): if faulthandler is not None: faulthandler.enable() + setup_gil() cls.unix_listener_path = mitogen.unix.make_socket_path() cls.worker_sock, cls.child_sock = socket.socketpair() atexit.register(lambda: clean_shutdown(cls.worker_sock)) From 07845d2f595ca4b4d0527cbe2140b18b959da8d0 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 7 Sep 2018 22:53:26 +0100 Subject: [PATCH 112/212] docs: update changelog. --- docs/changelog.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index a60f5319..326f15d9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -58,6 +58,14 @@ Enhancements synchronization, wasting significant runtime in the connection multiplexer. In one case work was reduced by 95%, which may manifest as faster runs. +* `5189408e `_: threads are + cooperatively scheduled, minimizing `GIL + `_ contention, and + reducing context switching by an order of magnitude. This manifests as an + overall improvement, but is easily noticeable on short many-target + runs, where startup overhead dominates runtime. + + Fixes ^^^^^ From 426cffd9f548590167875031bc45afceb2d88946 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 7 Sep 2018 22:56:25 +0100 Subject: [PATCH 113/212] tests: set no_target_syslog --- tests/ansible/ansible.cfg | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index 68c3ad19..3897519b 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -10,7 +10,9 @@ library = lib/modules module_utils = lib/module_utils retry_files_enabled = False display_args_to_stdout = True -forks = 200 +forks = 100 + +no_target_syslog = True # Required by integration/ssh/timeouts.yml timeout = 10 From e5d421e5f42804e7f4a013702f7a56b86c1ca833 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 7 Sep 2018 22:56:39 +0100 Subject: [PATCH 114/212] Update k3 inventory. --- tests/ansible/hosts/k3 | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/ansible/hosts/k3 b/tests/ansible/hosts/k3 index d39cf399..1a7190d8 100644 --- a/tests/ansible/hosts/k3 +++ b/tests/ansible/hosts/k3 @@ -6,8 +6,20 @@ k3-[01:10] [k3-x20] k3-[01:20] -[k3-x40] -k3-[01:40] +[k3-x50] +k3-[01:50] -[k3-x80] -k3-[01:80] +[k3-x100] +k3-[001:100] + +[k3-x200] +k3-[001:200] + +[k3-x300] +k3-[001:300] + +[k3-x400] +k3-[001:400] + +[k3-x500] +k3-[001:500] From 3c6b72b452050800bef16b24092bdef58e6a3abd Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 7 Sep 2018 23:46:56 +0100 Subject: [PATCH 115/212] ansible: gracefully return (and explain) ChannelError in ContextService. When Ansible abnormally shuts down, the broker begins force-disconnecting every context, including those for which connection is currently in-progress. When that happens, .call(init_child) throws ChannelError, and that needs returned back to the worker, assuming the worker still even exists. This solution is incomplete: with sick nodes, it's also possible the worker died naturally, and so the worker should perhaps respond by retrying the connection. Previously, the unhandled ChannelError would spam the console when e.g. fork() began returning EAGAIN. --- ansible_mitogen/services.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index 952e991a..15c1e8b0 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -380,6 +380,12 @@ class ContextService(mitogen.service.Service): return latch + disconnect_msg = ( + 'Channel was disconnected while connection attempt was in progress; ' + 'this may be caused by an abnormal Ansible exit, or due to an ' + 'unreliable target.' + ) + @mitogen.service.expose(mitogen.service.AllowParents()) @mitogen.service.arg_spec({ 'stack': list @@ -407,6 +413,13 @@ class ContextService(mitogen.service.Service): if isinstance(result, tuple): # exc_info() reraise(*result) via = result['context'] + except mitogen.core.ChannelError: + return { + 'context': None, + 'init_child_result': None, + 'method_name': spec['method'], + 'msg': self.disconnect_msg, + } except mitogen.core.StreamError as e: return { 'context': None, From ba0b3af20550bc893b2234b6a73929deae4459b7 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 8 Sep 2018 00:27:38 +0100 Subject: [PATCH 116/212] core: remove accidentally checked in debug crap (#337) --- mitogen/core.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index a67dc290..3990819f 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1113,9 +1113,8 @@ class Stream(BasicStream): ) if msg_len > self._router.max_message_size: - LOG.error('Maximum message size exceeded (got %d, max %d) %r', - msg_len, self._router.max_message_size, - self._input_buf[0]) + LOG.error('Maximum message size exceeded (got %d, max %d)', + msg_len, self._router.max_message_size) self.on_disconnect(broker) return False From 86942b6bf92832079e247325de5fa82231674d23 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 8 Sep 2018 01:09:55 +0100 Subject: [PATCH 117/212] ansible: add explanatory exception If disconnection occurs during a Connection.call(), return AnsibleConnectionFailure. --- ansible_mitogen/connection.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index bf11bccb..c35717dc 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -725,6 +725,20 @@ class Connection(ansible.plugins.connection.ConnectionBase): else: return call_context.call_async(func, *args, **kwargs) + call_aborted_msg = ( + 'Mitogen was disconnected from the remote environment while a call ' + 'was in-progress. If you feel this is in error, please file a bug. ' + 'Original error was: %s' + ) + + def _call_rethrow(self, recv): + try: + return recv.get().unpickle() + except mitogen.core.ChannelError as e: + raise ansible.errors.AnsibleConnectionFailure( + self.call_aborted_msg % (e,) + ) + def call(self, func, *args, **kwargs): """ Start and wait for completion of a function call in the target. @@ -739,7 +753,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): recv = self.call_async(func, *args, **kwargs) if recv is None: # no_reply=True return None - return recv.get().unpickle() + return self._call_rethrow(recv) finally: LOG.debug('Call took %d ms: %r', 1000 * (time.time() - t0), mitogen.parent.CallSpec(func, args, kwargs)) From 32751cd35606de38b592f7218f1ade0e79ab0fd2 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 8 Sep 2018 16:39:45 +0100 Subject: [PATCH 118/212] master: allow batching context switches for forward_modules() -7 switches per task. --- ansible_mitogen/services.py | 3 +-- mitogen/master.py | 12 ++++++++---- mitogen/service.py | 3 +-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index 15c1e8b0..59c26ba2 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -269,8 +269,7 @@ class ContextService(mitogen.service.Service): ) def _send_module_forwards(self, context): - for fullname in self.ALWAYS_PRELOAD: - self.router.responder.forward_module(context, fullname) + self.router.responder.forward_modules(context, self.ALWAYS_PRELOAD) _candidate_temp_dirs = None diff --git a/mitogen/master.py b/mitogen/master.py index d8d7e2e8..8df4cd5d 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -681,8 +681,7 @@ class ModuleResponder(object): ) ) - def _forward_module(self, context, fullname): - IOLOG.debug('%r._forward_module(%r, %r)', self, context, fullname) + def _forward_one_module(self, context, fullname): path = [] while fullname: path.append(fullname) @@ -693,8 +692,13 @@ class ModuleResponder(object): self._send_module_and_related(stream, fullname) self._send_forward_module(stream, context, fullname) - def forward_module(self, context, fullname): - self._router.broker.defer(self._forward_module, context, fullname) + def _forward_modules(self, context, fullnames): + IOLOG.debug('%r._forward_modules(%r, %r)', self, context, fullnames) + for fullname in fullnames: + self._forward_one_module(context, fullname) + + def forward_modules(self, context, fullnames): + self._router.broker.defer(self._forward_modules, context, fullnames) class Broker(mitogen.core.Broker): diff --git a/mitogen/service.py b/mitogen/service.py index f1ccadde..3713bdea 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -636,8 +636,7 @@ class PushFileService(Service): """ for path in paths: self.propagate_to(context, path) - for fullname in modules: - self.router.responder.forward_module(context, fullname) + self.router.responder.forward_modules(context, modules) @expose(policy=AllowParents()) @arg_spec({ From 92c092d27b9aa460c788f3bb200dd614a169b5da Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 8 Sep 2018 18:27:38 +0100 Subject: [PATCH 119/212] core: split Dispatcher out into own class. --- mitogen/core.py | 92 +++++++++++++++++++++++++++---------------------- 1 file changed, 50 insertions(+), 42 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index 3990819f..1f5ad0a9 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1949,15 +1949,60 @@ class Broker(object): return 'Broker(%#x)' % (id(self),) +class Dispatcher(object): + def __init__(self, econtext): + self.econtext = econtext + self.recv = Receiver(router=econtext.router, + handle=CALL_FUNCTION, + policy=has_parent_authority) + listen(econtext.broker, 'shutdown', self._on_broker_shutdown) + + def _on_broker_shutdown(self): + self.recv.close() + + def _dispatch_one(self, msg): + data = msg.unpickle(throw=False) + _v and LOG.debug('_dispatch_one(%r)', data) + + modname, klass, func, args, kwargs = data + obj = import_module(modname) + if klass: + obj = getattr(obj, klass) + fn = getattr(obj, func) + if getattr(fn, 'mitogen_takes_econtext', None): + kwargs.setdefault('econtext', self.econtext) + if getattr(fn, 'mitogen_takes_router', None): + kwargs.setdefault('router', self.econtext.router) + return fn(*args, **kwargs) + + def _dispatch_calls(self): + for msg in self.recv: + try: + ret = self._dispatch_one(msg) + _v and LOG.debug('_dispatch_calls: %r -> %r', msg, ret) + if msg.reply_to: + msg.reply(ret) + except Exception: + e = sys.exc_info()[1] + if msg.reply_to: + _v and LOG.debug('_dispatch_calls: %s', e) + msg.reply(CallError(e)) + else: + LOG.exception('_dispatch_calls: %r', msg) + + def run(self): + if self.econtext.config.get('on_start'): + self.econtext.config['on_start'](self) + + _profile_hook('main', self._dispatch_calls) + + class ExternalContext(object): detached = False def __init__(self, config): self.config = config - def _on_broker_shutdown(self): - self.recv.close() - def _on_broker_exit(self): if not self.config['profiling']: os.kill(os.getpid(), signal.SIGTERM) @@ -2041,16 +2086,12 @@ class ExternalContext(object): in_fd = self.config.get('in_fd', 100) out_fd = self.config.get('out_fd', 1) - self.recv = Receiver(router=self.router, - handle=CALL_FUNCTION, - policy=has_parent_authority) self.stream = Stream(self.router, parent_id) self.stream.name = 'parent' self.stream.accept(in_fd, out_fd) self.stream.receive_side.keep_alive = False listen(self.stream, 'disconnect', self._on_parent_disconnect) - listen(self.broker, 'shutdown', self._on_broker_shutdown) listen(self.broker, 'exit', self._on_broker_exit) os.close(in_fd) @@ -2148,40 +2189,6 @@ class ExternalContext(object): # Reopen with line buffering. sys.stdout = os.fdopen(1, 'w', 1) - def _dispatch_one(self, msg): - data = msg.unpickle(throw=False) - _v and LOG.debug('_dispatch_calls(%r)', data) - - modname, klass, func, args, kwargs = data - obj = import_module(modname) - if klass: - obj = getattr(obj, klass) - fn = getattr(obj, func) - if getattr(fn, 'mitogen_takes_econtext', None): - kwargs.setdefault('econtext', self) - if getattr(fn, 'mitogen_takes_router', None): - kwargs.setdefault('router', self.router) - return fn(*args, **kwargs) - - def _dispatch_calls(self): - if self.config.get('on_start'): - self.config['on_start'](self) - - for msg in self.recv: - try: - ret = self._dispatch_one(msg) - _v and LOG.debug('_dispatch_calls: %r -> %r', msg, ret) - if msg.reply_to: - msg.reply(ret) - except Exception: - e = sys.exc_info()[1] - if msg.reply_to: - _v and LOG.debug('_dispatch_calls: %s', e) - msg.reply(CallError(e)) - else: - LOG.exception('_dispatch_calls: %r', msg) - self.dispatch_stopped = True - def main(self): self._setup_master() try: @@ -2203,7 +2210,8 @@ class ExternalContext(object): self.parent, mitogen.context_id, os.getpid()) _v and LOG.debug('Recovered sys.executable: %r', sys.executable) - _profile_hook('main', self._dispatch_calls) + self.dispatcher = Dispatcher(self) + self.dispatcher.run() _v and LOG.debug('ExternalContext.main() normal exit') except KeyboardInterrupt: LOG.debug('KeyboardInterrupt received, exiting gracefully.') From 42b1b3d2867a6c6026b59ccc4f99e6248b783248 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 8 Sep 2018 19:19:14 +0100 Subject: [PATCH 120/212] core: support mitogen_chain dispatcher option. --- docs/api.rst | 52 +++++++++++++++++++++++++++++++++++-- mitogen/core.py | 48 +++++++++++++++++++++------------- tests/call_function_test.py | 34 +++++++++++++++++++++--- 3 files changed, 110 insertions(+), 24 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index e823b877..395147b2 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -892,6 +892,52 @@ Context Class :param dict kwargs: Function keyword arguments, if any. See :ref:`serialization-rules` for permitted types. + :param str mitogen_chain: + Optional cancellation key for threading unrelated asynchronous + requests to one context. If any prior call in the chain raised an + exception, subsequent calls with the same key immediately produce + the same exception. + + This permits a sequence of :meth:`no-reply ` or + pipelined asynchronous calls to be made without wasting network + round-trips to discover if prior calls succeeded, while allowing + such chains to overlap concurrently from multiple unrelated source + contexts. The chain is cancelled on first exception, enabling + patterns like:: + + # Must be distinct for each overlapping sequence, and cannot be + # reused. + chain = 'make-dirs-and-do-stuff-%s-%s-%s-%s' % ( + socket.gethostname(), + os.getpid(), + threading.currentThread().id, + time.time(), + ) + context.call_no_reply(os.mkdir, '/tmp/foo', + mitogen_chain=chain) + + # If os.mkdir() fails, this never runs: + context.call_no_reply(os.mkdir, '/tmp/foo/bar', + mitogen_chain=chain) + + # If either os.mkdir() fails, this never runs, and returns the + # exception. + recv = context.call_async(subprocess.check_output, '/tmp/foo', + mitogen_chain=chain) + + # If os.mkdir() or check_call() failed, this never runs, and + # the exception that occurred is raised. + context.call(do_something, mitogen_chain=chain) + + # The receiver also got a copy of the exception, so if this + # code was executed, the exception would also be raised. + if recv.get().unpickle() == 'baz': + pass + + Note that for long-lived programs, there is presently no mechanism + for clearing the chain history on a target. This will be addressed + in future. + :returns: :class:`mitogen.core.Receiver` configured to receive the result of the invocation: @@ -923,8 +969,10 @@ Context Class .. method:: call_no_reply (fn, \*args, \*\*kwargs) - Send a function call, but expect no return value. If the call fails, - the full exception will be logged to the target context's logging framework. + Like :meth:`call_async`, but do not wait for a return value, and inform + the target context no such reply is expected. If the call fails, the + full exception will be logged to the target context's logging + framework, unless the `mitogen_chain` argument was present. :raises mitogen.core.CallError: An exception was raised in the remote context during execution. diff --git a/mitogen/core.py b/mitogen/core.py index 1f5ad0a9..4092db4a 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1952,15 +1952,14 @@ class Broker(object): class Dispatcher(object): def __init__(self, econtext): self.econtext = econtext + #: Chain ID -> CallError if prior call failed. + self._error_by_chain_id = {} self.recv = Receiver(router=econtext.router, handle=CALL_FUNCTION, policy=has_parent_authority) - listen(econtext.broker, 'shutdown', self._on_broker_shutdown) + listen(econtext.broker, 'shutdown', self.recv.close) - def _on_broker_shutdown(self): - self.recv.close() - - def _dispatch_one(self, msg): + def _parse_request(self, msg): data = msg.unpickle(throw=False) _v and LOG.debug('_dispatch_one(%r)', data) @@ -1973,22 +1972,35 @@ class Dispatcher(object): kwargs.setdefault('econtext', self.econtext) if getattr(fn, 'mitogen_takes_router', None): kwargs.setdefault('router', self.econtext.router) - return fn(*args, **kwargs) + + return fn, args, kwargs + + def _dispatch_one(self, msg): + try: + fn, args, kwargs = self._parse_request(msg) + except Exception: + return None, CallError(sys.exc_info()[1]) + + chain_id = kwargs.pop('mitogen_chain', None) + if chain_id in self._error_by_chain_id: + return chain_id, self._error_by_chain_id[chain_id] + + try: + return chain_id, fn(*args, **kwargs) + except Exception: + e = CallError(sys.exc_info()[1]) + if chain_id is not None: + self._error_by_chain_id[chain_id] = e + return chain_id, e def _dispatch_calls(self): for msg in self.recv: - try: - ret = self._dispatch_one(msg) - _v and LOG.debug('_dispatch_calls: %r -> %r', msg, ret) - if msg.reply_to: - msg.reply(ret) - except Exception: - e = sys.exc_info()[1] - if msg.reply_to: - _v and LOG.debug('_dispatch_calls: %s', e) - msg.reply(CallError(e)) - else: - LOG.exception('_dispatch_calls: %r', msg) + chain_id, ret = self._dispatch_one(msg) + _v and LOG.debug('_dispatch_calls: %r -> %r', msg, ret) + if msg.reply_to: + msg.reply(ret) + elif isinstance(ret, CallError) and chain_id is None: + LOG.error('No-reply function call failed: %s', ret) def run(self): if self.econtext.config.get('on_start'): diff --git a/tests/call_function_test.py b/tests/call_function_test.py index eb83dff5..b8b07283 100644 --- a/tests/call_function_test.py +++ b/tests/call_function_test.py @@ -18,15 +18,15 @@ def function_that_adds_numbers(x, y): return x + y -def function_that_fails(): - raise plain_old_module.MyError('exception text') +def function_that_fails(s=''): + raise plain_old_module.MyError('exception text'+s) def func_with_bad_return_value(): return CrazyType() -def func_accepts_returns_context(context): +def func_returns_arg(context): return context @@ -101,7 +101,7 @@ class CallFunctionTest(testlib.RouterMixin, testlib.TestCase): self.assertEquals(exc.args[0], mitogen.core.ChannelError.local_msg) def test_accepts_returns_context(self): - context = self.local.call(func_accepts_returns_context, self.local) + context = self.local.call(func_returns_arg, self.local) self.assertIsNot(context, self.local) self.assertEqual(context.context_id, self.local.context_id) self.assertEqual(context.name, self.local.name) @@ -118,5 +118,31 @@ class CallFunctionTest(testlib.RouterMixin, testlib.TestCase): lambda: recv.get().unpickle()) +class ChainTest(testlib.RouterMixin, testlib.TestCase): + # Verify mitogen_chain functionality. + + def setUp(self): + super(ChainTest, self).setUp() + self.local = self.router.fork() + + def test_subsequent_calls_produce_same_error(self): + self.assertEquals('xx', + self.local.call(func_returns_arg, 'xx', mitogen_chain='c1')) + self.local.call_no_reply(function_that_fails, 'x1', mitogen_chain='c1') + e1 = self.assertRaises(mitogen.core.CallError, + lambda: self.local.call(function_that_fails, 'x2', mitogen_chain='c1')) + e2 = self.assertRaises(mitogen.core.CallError, + lambda: self.local.call(func_returns_arg, 'x3', mitogen_chain='c1')) + self.assertEquals(str(e1), str(e2)) + + def test_unrelated_overlapping_failed_chains(self): + self.local.call_no_reply(function_that_fails, 'c1', mitogen_chain='c1') + self.assertEquals('yes', + self.local.call(func_returns_arg, 'yes', mitogen_chain='c2')) + self.assertRaises(mitogen.core.CallError, + lambda: self.local.call(func_returns_arg, 'yes', mitogen_chain='c1')) + self.local.call_no_reply(function_that_fails, 'c2', mitogen_chain='c2') + + if __name__ == '__main__': unittest2.main() From 1247d1fce69d6026a80802528df91e73076bb467 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 8 Sep 2018 19:20:19 +0100 Subject: [PATCH 121/212] docs: update changelog. --- docs/changelog.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 326f15d9..90d6256a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -156,6 +156,12 @@ Core Library listener no longer crashes if the peer process disappears in the middle of connection setup. +* `5adae88d `_ a new + `mitogen_chain` keyword argument is accepted by + :meth:`mitogen.master.Context.call_async`, allowing overlapping chains of + function calls to be pipelined to a context, while cancelling the chain on + the first exception. + Thanks! ~~~~~~~ From 37223adacd149cd38116ad3fc45a264edc755c06 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 8 Sep 2018 19:22:33 +0100 Subject: [PATCH 122/212] core: fix Dispatcher race introduced in 3a7815e5ca6255272334415916b6289378173859 It must be constructed before are messages pumped. --- mitogen/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/core.py b/mitogen/core.py index 4092db4a..c959ea5d 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -2214,6 +2214,7 @@ class ExternalContext(object): if self.config.get('setup_stdio', True): self._setup_stdio() + self.dispatcher = Dispatcher(self) self.router.register(self.parent, self.stream) self.log_handler.uncork() @@ -2222,7 +2223,6 @@ class ExternalContext(object): self.parent, mitogen.context_id, os.getpid()) _v and LOG.debug('Recovered sys.executable: %r', sys.executable) - self.dispatcher = Dispatcher(self) self.dispatcher.run() _v and LOG.debug('ExternalContext.main() normal exit') except KeyboardInterrupt: From a3957d6aafca2985ad7a15b70a9cc0dc563f5a0f Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 8 Sep 2018 19:32:20 +0100 Subject: [PATCH 123/212] parent: add Context.forget_chain(). --- docs/api.rst | 11 ++++++++--- mitogen/core.py | 5 +++++ mitogen/parent.py | 3 +++ tests/call_function_test.py | 8 ++++++++ 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 395147b2..56bb5c1f 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -934,9 +934,9 @@ Context Class if recv.get().unpickle() == 'baz': pass - Note that for long-lived programs, there is presently no mechanism - for clearing the chain history on a target. This will be addressed - in future. + It is necessary to explicitly clean up the chain history on a + target, otherwise unbounded memory usage is possible. See + :meth:`forget_chain`. :returns: :class:`mitogen.core.Receiver` configured to receive the result @@ -977,6 +977,11 @@ Context Class :raises mitogen.core.CallError: An exception was raised in the remote context during execution. + .. method:: forget_chain (chain_id) + + Instruct the target to forget any exception related to `chain_id`, a + key previously used as the `mitogen_chain` parameter to + :meth:`call_async`. Receiver Class diff --git a/mitogen/core.py b/mitogen/core.py index c959ea5d..0142be7c 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1959,6 +1959,11 @@ class Dispatcher(object): policy=has_parent_authority) listen(econtext.broker, 'shutdown', self.recv.close) + @classmethod + @takes_econtext + def forget_chain(cls, chain_id, econtext): + econtext.dispatcher._error_by_chain_id.pop(chain_id, None) + def _parse_request(self, msg): data = msg.unpickle(throw=False) _v and LOG.debug('_dispatch_one(%r)', data) diff --git a/mitogen/parent.py b/mitogen/parent.py index d7177dde..5c7fb642 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1160,6 +1160,9 @@ class Context(mitogen.core.Context): self, fn, args, kwargs) self.send(make_call_msg(fn, *args, **kwargs)) + def forget_chain(self, chain_id): + self.call_no_reply(mitogen.core.Dispatcher.forget_chain, chain_id) + def shutdown(self, wait=False): LOG.debug('%r.shutdown() sending SHUTDOWN', self) latch = mitogen.core.Latch() diff --git a/tests/call_function_test.py b/tests/call_function_test.py index b8b07283..ca56f07a 100644 --- a/tests/call_function_test.py +++ b/tests/call_function_test.py @@ -143,6 +143,14 @@ class ChainTest(testlib.RouterMixin, testlib.TestCase): lambda: self.local.call(func_returns_arg, 'yes', mitogen_chain='c1')) self.local.call_no_reply(function_that_fails, 'c2', mitogen_chain='c2') + def test_forget(self): + self.local.call_no_reply(function_that_fails, 'x1', mitogen_chain='c1') + e1 = self.assertRaises(mitogen.core.CallError, + lambda: self.local.call(function_that_fails, 'x2', mitogen_chain='c1')) + self.local.forget_chain('c1') + self.assertEquals('x3', + self.local.call(func_returns_arg, 'x3', mitogen_chain='c1')) + if __name__ == '__main__': unittest2.main() From aa9400a3b9e1e28a87d1855d61b2a14fed1d4562 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 8 Sep 2018 20:52:53 +0100 Subject: [PATCH 124/212] docs: fix changelog --- docs/changelog.rst | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 90d6256a..57a7029b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -157,10 +157,10 @@ Core Library connection setup. * `5adae88d `_ a new - `mitogen_chain` keyword argument is accepted by - :meth:`mitogen.master.Context.call_async`, allowing overlapping chains of - function calls to be pipelined to a context, while cancelling the chain on - the first exception. + `mitogen_chain` keyword argument is accepted by + :meth:`mitogen.parent.Context.call_async`, allowing overlapping chains of + function calls to be pipelined to a context, while cancelling the chain on + the first exception. Thanks! @@ -170,14 +170,18 @@ Mitogen would not be possible without the support of users. A huge thanks for the bug reports in this release contributed by `Alex Russu `_, `atoom `_, +`Berend De Schouwer `_, `Dan Quackenbush `_, `dsgnr `_, `Jesse London `_, `Jonathan Rosser `_, +`Josh Smift `_, `Luca Nunzi `_, `nikitakazantsev12 `_, -`Prateek Jain `_, +`Peter V. Saveliev `_, `Pierre-Henry Muller `_, +`Pierre-Louis Bonicoli `_, +`Prateek Jain `_, `Rick Box `_, and `Timo Beckers `_. From da8c6b45b04c33b3b15cc0cfb52c44f0de525881 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 8 Sep 2018 21:59:17 +0100 Subject: [PATCH 125/212] ansible: remove task_vars aliasing from connection.py. Crazy spam creep. --- ansible_mitogen/connection.py | 68 ++++++++++++----------------------- 1 file changed, 22 insertions(+), 46 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index c35717dc..fd81b99c 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -31,7 +31,6 @@ from __future__ import unicode_literals import logging import os -import shlex import stat import time @@ -333,11 +332,15 @@ def config_from_play_context(transport, inventory_name, connection): 'become_pass': connection._play_context.become_pass, 'password': connection._play_context.password, 'port': connection._play_context.port, - 'python_path': parse_python_path(connection.python_path), + 'python_path': parse_python_path( + connection.task_vars.get('ansible_python_interpreter', + '/usr/bin/python') + ), 'private_key_file': connection._play_context.private_key_file, 'ssh_executable': connection._play_context.ssh_executable, 'timeout': connection._play_context.timeout, - 'ansible_ssh_timeout': connection.ansible_ssh_timeout, + 'ansible_ssh_timeout': + connection.task_vars.get('ansible_ssh_timeout', C.DEFAULT_TIMEOUT), 'ssh_args': [ mitogen.core.to_text(term) for s in ( @@ -356,12 +359,18 @@ def config_from_play_context(transport, inventory_name, connection): ) for term in ansible.utils.shlex.shlex_split(s or '') ], - 'mitogen_via': connection.mitogen_via, - 'mitogen_kind': connection.mitogen_kind, - 'mitogen_docker_path': connection.mitogen_docker_path, - 'mitogen_lxc_info_path': connection.mitogen_lxc_info_path, - 'mitogen_machinectl_path': connection.mitogen_machinectl_path, - 'mitogen_ssh_debug_level': connection.mitogen_ssh_debug_level, + 'mitogen_via': + connection.task_vars.get('mitogen_via'), + 'mitogen_kind': + connection.task_vars.get('mitogen_kind'), + 'mitogen_docker_path': + connection.task_vars.get('mitogen_docker_path'), + 'mitogen_lxc_info_path': + connection.task_vars.get('mitogen_lxc_info_path'), + 'mitogen_machinectl_path': + connection.task_vars.get('mitogen_machinectl_path'), + 'mitogen_ssh_debug_level': + connection.task_vars.get('mitogen_ssh_debug_level'), } @@ -425,32 +434,8 @@ class Connection(ansible.plugins.connection.ConnectionBase): # the case of the synchronize module. # - #: Set to 'ansible_python_interpreter' by on_action_run(). - python_path = None - - #: Set to 'ansible_ssh_timeout' by on_action_run(). - ansible_ssh_timeout = None - - #: Set to 'mitogen_via' by on_action_run(). - mitogen_via = None - - #: Set to 'mitogen_kind' by on_action_run(). - mitogen_kind = None - - #: Set to 'mitogen_docker_path' by on_action_run(). - mitogen_docker_path = None - - #: Set to 'mitogen_lxc_info_path' by on_action_run(). - mitogen_lxc_info_path = None - - #: Set to 'mitogen_lxc_info_path' by on_action_run(). - mitogen_machinectl_path = None - - #: Set to 'mitogen_ssh_debug_level' by on_action_run(). - mitogen_ssh_debug_level = None - - #: Set to 'inventory_hostname' by on_action_run(). - inventory_hostname = None + #: Set to task_vars by on_action_run(). + task_vars = None #: Set to 'hostvars' by on_action_run() host_vars = None @@ -500,17 +485,8 @@ class Connection(ansible.plugins.connection.ConnectionBase): :param str loader_basedir: Loader base directory; see :attr:`loader_basedir`. """ - self.ansible_ssh_timeout = task_vars.get('ansible_ssh_timeout', - C.DEFAULT_TIMEOUT) - self.python_path = task_vars.get('ansible_python_interpreter', - '/usr/bin/python') - self.mitogen_via = task_vars.get('mitogen_via') - self.mitogen_kind = task_vars.get('mitogen_kind') - self.mitogen_docker_path = task_vars.get('mitogen_docker_path') - self.mitogen_lxc_info_path = task_vars.get('mitogen_lxc_info_path') - self.mitogen_machinectl_path = task_vars.get('mitogen_machinectl_path') - self.mitogen_ssh_debug_level = task_vars.get('mitogen_ssh_debug_level') self.inventory_hostname = task_vars['inventory_hostname'] + self.task_vars = task_vars self.host_vars = task_vars['hostvars'] self.delegate_to_hostname = delegate_to_hostname self.loader_basedir = loader_basedir @@ -535,7 +511,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): if isinstance(via_vars, jinja2.runtime.Undefined): raise ansible.errors.AnsibleConnectionFailure( self.unknown_via_msg % ( - self.mitogen_via, + via_spec, inventory_name, ) ) From a52f66328b69537a31fd9f8cd3512e131b534477 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 8 Sep 2018 22:12:29 +0100 Subject: [PATCH 126/212] parent: test fixes. --- mitogen/parent.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index 5c7fb642..29decbc1 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -231,9 +231,10 @@ def create_child(args, merge_stdio=False, stderr_pipe=False, preexec_fn=None): # O_NONBLOCK from Python's future stdin fd. mitogen.core.set_block(childfp.fileno()) + stderr_r = None + extra = {} if merge_stdio: extra = {'stderr': childfp} - stderr_r = None elif stderr_pipe: stderr_r, stderr_w = os.pipe() mitogen.core.set_cloexec(stderr_r) From 66142e7d75f76b86d5022c26150e1c2df81db97a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 8 Sep 2018 22:17:39 +0100 Subject: [PATCH 127/212] ansible: fork isolated tasks from correct parent. Closes #355. --- ansible_mitogen/connection.py | 5 +++- tests/ansible/integration/runner/all.yml | 4 ++- ...rking_behaviour.yml => forking_active.yml} | 21 ++------------- .../runner/forking_correct_parent.yml | 26 +++++++++++++++++++ .../integration/runner/forking_inactive.yml | 23 ++++++++++++++++ 5 files changed, 58 insertions(+), 21 deletions(-) rename tests/ansible/integration/runner/{forking_behaviour.yml => forking_active.yml} (59%) create mode 100644 tests/ansible/integration/runner/forking_correct_parent.yml create mode 100644 tests/ansible/integration/runner/forking_inactive.yml diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index fd81b99c..21ecb5f0 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -693,6 +693,8 @@ class Connection(ansible.plugins.connection.ConnectionBase): if kwargs.pop('use_login_context', None): call_context = self.login_context + elif kwargs.pop('use_fork_context', None): + call_context = self.fork_context else: call_context = self.context @@ -743,7 +745,8 @@ class Connection(ansible.plugins.connection.ConnectionBase): :returns: mitogen.core.Context of the new child. """ - return self.call(ansible_mitogen.target.create_fork_child) + return self.call(ansible_mitogen.target.create_fork_child, + use_fork_context=True) def get_default_cwd(self): """ diff --git a/tests/ansible/integration/runner/all.yml b/tests/ansible/integration/runner/all.yml index ffb263fb..69c22edb 100644 --- a/tests/ansible/integration/runner/all.yml +++ b/tests/ansible/integration/runner/all.yml @@ -14,5 +14,7 @@ - import_playbook: custom_script_interpreter.yml - import_playbook: environment_isolation.yml - import_playbook: etc_environment.yml -- import_playbook: forking_behaviour.yml +- import_playbook: forking_active.yml +- import_playbook: forking_inactive.yml +- import_playbook: forking_correct_parent.yml - import_playbook: missing_module.yml diff --git a/tests/ansible/integration/runner/forking_behaviour.yml b/tests/ansible/integration/runner/forking_active.yml similarity index 59% rename from tests/ansible/integration/runner/forking_behaviour.yml rename to tests/ansible/integration/runner/forking_active.yml index 7268fce0..15e6d2bc 100644 --- a/tests/ansible/integration/runner/forking_behaviour.yml +++ b/tests/ansible/integration/runner/forking_active.yml @@ -1,26 +1,8 @@ - -- name: integration/runner/forking_behaviour.yml +- name: integration/runner/forking_active.yml hosts: test-targets any_errors_fatal: true tasks: - # Verify non-async jobs run in-process. - - - name: get process ID. - custom_python_detect_environment: - register: sync_proc1 - when: is_mitogen - - - name: get process ID again. - custom_python_detect_environment: - register: sync_proc2 - when: is_mitogen - - - assert: - that: - - sync_proc1.pid == sync_proc2.pid - when: is_mitogen - # Verify mitogen_task_isolation=fork triggers forking. - name: get force-forked process ID. @@ -42,3 +24,4 @@ - fork_proc1.pid != sync_proc1.pid - fork_proc1.pid != fork_proc2.pid when: is_mitogen + diff --git a/tests/ansible/integration/runner/forking_correct_parent.yml b/tests/ansible/integration/runner/forking_correct_parent.yml new file mode 100644 index 00000000..e8207676 --- /dev/null +++ b/tests/ansible/integration/runner/forking_correct_parent.yml @@ -0,0 +1,26 @@ + +- name: integration/runner/forking_correct_parent.yml + hosts: test-targets + any_errors_fatal: true + tasks: + + # Verify mitogen_task_isolation=fork forks from "virginal fork parent", not + # shared interpreter. + + - name: get regular process ID. + custom_python_detect_environment: + register: regular_proc + when: is_mitogen + + - name: get force-forked process ID again. + custom_python_detect_environment: + register: fork_proc + vars: + mitogen_task_isolation: fork + when: is_mitogen + + - assert: + that: + - fork_proc.pid != regular_proc.pid + - fork_proc.ppid != regular_proc.pid + when: is_mitogen diff --git a/tests/ansible/integration/runner/forking_inactive.yml b/tests/ansible/integration/runner/forking_inactive.yml new file mode 100644 index 00000000..b84cec7e --- /dev/null +++ b/tests/ansible/integration/runner/forking_inactive.yml @@ -0,0 +1,23 @@ +# Verify non-async jobs run in-process. + +- name: integration/runner/forking_inactive.yml + hosts: test-targets + any_errors_fatal: true + tasks: + + - name: get process ID. + custom_python_detect_environment: + register: sync_proc1 + when: is_mitogen + + - name: get process ID again. + custom_python_detect_environment: + register: sync_proc2 + when: is_mitogen + + - assert: + that: + - sync_proc1.pid == sync_proc2.pid + when: is_mitogen + + From 1bf9b2c1a326dd55117c4ce46b6f3f57600a1442 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 8 Sep 2018 22:19:37 +0100 Subject: [PATCH 128/212] docs: update changelog. --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 57a7029b..44480118 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -108,6 +108,11 @@ Fixes yes`` option is no longer supplied to OpenSSH by default, better matching Ansible's behaviour. +* `#355 `_: tasks configured to run + in an isolated forked subprocess were being forked from the wrong parent + context. This meant built-in modules overridden via a custom ``module_utils`` + search path may not have had any effect. + * A missing check caused an exception traceback to appear when using the ``ansible`` command-line tool with a missing or misspelled module name. From 705d77a9bec7196c5bf35351ad8807b05ed41019 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 8 Sep 2018 22:31:19 +0100 Subject: [PATCH 129/212] ansible: remove a bunch more aliasing from connection.py. --- ansible_mitogen/connection.py | 34 ++++++++----------- .../integration/runner/forking_active.yml | 4 +++ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 21ecb5f0..a56850cf 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -417,13 +417,18 @@ class Connection(ansible.plugins.connection.ConnectionBase): #: reached via become. context = None - #: mitogen.parent.Context for the login account on the target. This is - #: always the login account, even when become=True. + #: Context for the login account on the target. This is always the login + #: account, even when become=True. login_context = None - #: mitogen.parent.Context connected to the fork parent process in the - #: target user account. - fork_context = None + #: Dict containing init_child() return vaue as recorded at startup by + #: ContextService. Contains: + #: + #: fork_context: Context connected to the fork parent : process in the + #: target account. + #: home_dir: Target context's home directory. + #: temp_dir: A writeable temporary directory path. + init_child_result = None #: Only sudo, su, and doas are supported for now. become_methods = ['sudo', 'su', 'doas'] @@ -448,12 +453,6 @@ class Connection(ansible.plugins.connection.ConnectionBase): #: matching vanilla Ansible behaviour. loader_basedir = None - #: Set after connection to the target context's home directory. - home_dir = None - - #: Set after connection to the target context's home directory. - _temp_dir = None - def __init__(self, play_context, new_stdin, **kwargs): assert ansible_mitogen.process.MuxProcess.unix_listener_path, ( 'Mitogen connection types may only be instantiated ' @@ -495,7 +494,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): @property def homedir(self): self._connect() - return self.home_dir + return self.init_child_result['home_dir'] @property def connected(self): @@ -622,13 +621,11 @@ class Connection(ansible.plugins.connection.ConnectionBase): else: self.login_context = self.context - self.fork_context = dct['init_child_result']['fork_context'] - self.home_dir = dct['init_child_result']['home_dir'] - self._temp_dir = dct['init_child_result']['temp_dir'] + self.init_child_result = dct['init_child_result'] def get_temp_dir(self): self._connect() - return self._temp_dir + return self.init_child_result['temp_dir'] def _connect(self): """ @@ -662,8 +659,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): ) self.context = None - self.fork_context = None - self.login_context = None + self.init_child_result = None if self.broker and not new_task: self.broker.shutdown() self.broker.join() @@ -694,7 +690,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): if kwargs.pop('use_login_context', None): call_context = self.login_context elif kwargs.pop('use_fork_context', None): - call_context = self.fork_context + call_context = self.init_child_result['fork_context'] else: call_context = self.context diff --git a/tests/ansible/integration/runner/forking_active.yml b/tests/ansible/integration/runner/forking_active.yml index 15e6d2bc..e3e63b71 100644 --- a/tests/ansible/integration/runner/forking_active.yml +++ b/tests/ansible/integration/runner/forking_active.yml @@ -5,6 +5,10 @@ # Verify mitogen_task_isolation=fork triggers forking. + - name: get regular process ID. + custom_python_detect_environment: + register: sync_proc1 + - name: get force-forked process ID. custom_python_detect_environment: register: fork_proc1 From b254eb33990de723484cc374e390b90a883573af Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sat, 8 Sep 2018 22:35:42 +0100 Subject: [PATCH 130/212] ansible: fix non-action connection instantiation. e.g. by synchronize module. --- ansible_mitogen/connection.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index a56850cf..2b4ed61b 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -333,14 +333,15 @@ def config_from_play_context(transport, inventory_name, connection): 'password': connection._play_context.password, 'port': connection._play_context.port, 'python_path': parse_python_path( - connection.task_vars.get('ansible_python_interpreter', - '/usr/bin/python') + connection.get_task_var('ansible_python_interpreter', + default='/usr/bin/python') ), 'private_key_file': connection._play_context.private_key_file, 'ssh_executable': connection._play_context.ssh_executable, 'timeout': connection._play_context.timeout, 'ansible_ssh_timeout': - connection.task_vars.get('ansible_ssh_timeout', C.DEFAULT_TIMEOUT), + connection.get_task_var('ansible_ssh_timeout', + default=C.DEFAULT_TIMEOUT), 'ssh_args': [ mitogen.core.to_text(term) for s in ( @@ -360,17 +361,17 @@ def config_from_play_context(transport, inventory_name, connection): for term in ansible.utils.shlex.shlex_split(s or '') ], 'mitogen_via': - connection.task_vars.get('mitogen_via'), + connection.get_task_var('mitogen_via'), 'mitogen_kind': - connection.task_vars.get('mitogen_kind'), + connection.get_task_var('mitogen_kind'), 'mitogen_docker_path': - connection.task_vars.get('mitogen_docker_path'), + connection.get_task_var('mitogen_docker_path'), 'mitogen_lxc_info_path': - connection.task_vars.get('mitogen_lxc_info_path'), + connection.get_task_var('mitogen_lxc_info_path'), 'mitogen_machinectl_path': - connection.task_vars.get('mitogen_machinectl_path'), + connection.get_task_var('mitogen_machinectl_path'), 'mitogen_ssh_debug_level': - connection.task_vars.get('mitogen_ssh_debug_level'), + connection.get_task_var('mitogen_ssh_debug_level'), } @@ -439,8 +440,11 @@ class Connection(ansible.plugins.connection.ConnectionBase): # the case of the synchronize module. # + #: Set to the host name as it appears in inventory by on_action_run(). + inventory_hostname = None + #: Set to task_vars by on_action_run(). - task_vars = None + _task_vars = None #: Set to 'hostvars' by on_action_run() host_vars = None @@ -485,12 +489,17 @@ class Connection(ansible.plugins.connection.ConnectionBase): Loader base directory; see :attr:`loader_basedir`. """ self.inventory_hostname = task_vars['inventory_hostname'] - self.task_vars = task_vars + self._task_vars = task_vars self.host_vars = task_vars['hostvars'] self.delegate_to_hostname = delegate_to_hostname self.loader_basedir = loader_basedir self.close(new_task=True) + def get_task_var(self, key, default=None): + if self._task_vars and key in self._task_vars: + return self._task_vars[key] + return default + @property def homedir(self): self._connect() From 4d3873c784df2799e75b331ca0110a3f3d9ab15b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 9 Sep 2018 18:51:03 +0100 Subject: [PATCH 131/212] core: call chains v3: abstract it into a new CallChain class. --- docs/api.rst | 130 ++------------------ docs/howitworks.rst | 8 +- mitogen/core.py | 7 +- mitogen/parent.py | 236 ++++++++++++++++++++++++++++++++---- tests/call_function_test.py | 34 +++--- 5 files changed, 246 insertions(+), 169 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 56bb5c1f..4ad1db5e 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -833,12 +833,21 @@ Context Class .. currentmodule:: mitogen.parent +.. autoclass:: CallChain + :members: + .. class:: Context Extend :class:`mitogen.core.Router` with functionality useful to masters, and child contexts who later become parents. Currently when this class is required, the target context's router is upgraded at runtime. + .. attribute:: default_call_chain + + A :class:`CallChain` instance constructed by default, with pipelining + disabled. :meth:`call`, :meth:`call_async` and :meth:`call_no_reply` + use this instance. + .. method:: shutdown (wait=False) Arrange for the context to receive a ``SHUTDOWN`` message, triggering @@ -858,130 +867,15 @@ Context Class .. method:: call_async (fn, \*args, \*\*kwargs) - Arrange for the context's ``CALL_FUNCTION`` handle to receive a - message that causes `fn(\*args, \**kwargs)` to be invoked on the - context's main thread. - - :param fn: - A free function in module scope or a class method of a class - directly reachable from module scope: - - .. code-block:: python - - # mymodule.py - - def my_func(): - """A free function reachable as mymodule.my_func""" - - class MyClass: - @classmethod - def my_classmethod(cls): - """Reachable as mymodule.MyClass.my_classmethod""" - - def my_instancemethod(self): - """Unreachable: requires a class instance!""" - - class MyEmbeddedClass: - @classmethod - def my_classmethod(cls): - """Not directly reachable from module scope!""" - - :param tuple args: - Function arguments, if any. See :ref:`serialization-rules` for - permitted types. - :param dict kwargs: - Function keyword arguments, if any. See :ref:`serialization-rules` - for permitted types. - :param str mitogen_chain: - Optional cancellation key for threading unrelated asynchronous - requests to one context. If any prior call in the chain raised an - exception, subsequent calls with the same key immediately produce - the same exception. - - This permits a sequence of :meth:`no-reply ` or - pipelined asynchronous calls to be made without wasting network - round-trips to discover if prior calls succeeded, while allowing - such chains to overlap concurrently from multiple unrelated source - contexts. The chain is cancelled on first exception, enabling - patterns like:: - - # Must be distinct for each overlapping sequence, and cannot be - # reused. - chain = 'make-dirs-and-do-stuff-%s-%s-%s-%s' % ( - socket.gethostname(), - os.getpid(), - threading.currentThread().id, - time.time(), - ) - context.call_no_reply(os.mkdir, '/tmp/foo', - mitogen_chain=chain) - - # If os.mkdir() fails, this never runs: - context.call_no_reply(os.mkdir, '/tmp/foo/bar', - mitogen_chain=chain) - - # If either os.mkdir() fails, this never runs, and returns the - # exception. - recv = context.call_async(subprocess.check_output, '/tmp/foo', - mitogen_chain=chain) - - # If os.mkdir() or check_call() failed, this never runs, and - # the exception that occurred is raised. - context.call(do_something, mitogen_chain=chain) - - # The receiver also got a copy of the exception, so if this - # code was executed, the exception would also be raised. - if recv.get().unpickle() == 'baz': - pass - - It is necessary to explicitly clean up the chain history on a - target, otherwise unbounded memory usage is possible. See - :meth:`forget_chain`. - - :returns: - :class:`mitogen.core.Receiver` configured to receive the result - of the invocation: - - .. code-block:: python - - recv = context.call_async(os.check_output, 'ls /tmp/') - try: - # Prints output once it is received. - msg = recv.get() - print(msg.unpickle()) - except mitogen.core.CallError, e: - print('Call failed:', str(e)) - - Asynchronous calls may be dispatched in parallel to multiple - contexts and consumed as they complete using - :class:`mitogen.select.Select`. + See :meth:`CallChain.call_async`. .. method:: call (fn, \*args, \*\*kwargs) - Equivalent to :meth:`call_async(fn, \*args, \**kwargs).get().unpickle() - `. - - :returns: - The function's return value. - - :raises mitogen.core.CallError: - An exception was raised in the remote context during execution. + See :meth:`CallChain.call`. .. method:: call_no_reply (fn, \*args, \*\*kwargs) - Like :meth:`call_async`, but do not wait for a return value, and inform - the target context no such reply is expected. If the call fails, the - full exception will be logged to the target context's logging - framework, unless the `mitogen_chain` argument was present. - - :raises mitogen.core.CallError: - An exception was raised in the remote context during execution. - - .. method:: forget_chain (chain_id) - - Instruct the target to forget any exception related to `chain_id`, a - key previously used as the `mitogen_chain` parameter to - :meth:`call_async`. + See :meth:`CallChain.call_no_reply`. Receiver Class diff --git a/docs/howitworks.rst b/docs/howitworks.rst index 1d45647f..1e3d2768 100644 --- a/docs/howitworks.rst +++ b/docs/howitworks.rst @@ -373,11 +373,9 @@ Children listen on the following handles: .. currentmodule:: mitogen.core .. data:: CALL_FUNCTION - Receives `(mod_name, class_name, func_name, args, kwargs)` - 5-tuples from - :py:meth:`call_async() `, - imports ``mod_name``, then attempts to execute - `class_name.func_name(\*args, \**kwargs)`. + Receives `(chain_id, mod_name, class_name, func_name, args, kwargs)` + 6-tuples from :class:`mitogen.parent.CallChain`, imports ``mod_name``, then + attempts to execute `class_name.func_name(\*args, \**kwargs)`. When this channel is closed (by way of receiving a dead message), the child's main thread begins graceful shutdown of its own :py:class:`Broker` diff --git a/mitogen/core.py b/mitogen/core.py index 0142be7c..07cfcf37 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -1968,7 +1968,7 @@ class Dispatcher(object): data = msg.unpickle(throw=False) _v and LOG.debug('_dispatch_one(%r)', data) - modname, klass, func, args, kwargs = data + chain_id, modname, klass, func, args, kwargs = data obj = import_module(modname) if klass: obj = getattr(obj, klass) @@ -1978,15 +1978,14 @@ class Dispatcher(object): if getattr(fn, 'mitogen_takes_router', None): kwargs.setdefault('router', self.econtext.router) - return fn, args, kwargs + return chain_id, fn, args, kwargs def _dispatch_one(self, msg): try: - fn, args, kwargs = self._parse_request(msg) + chain_id, fn, args, kwargs = self._parse_request(msg) except Exception: return None, CallError(sys.exc_info()[1]) - chain_id = kwargs.pop('mitogen_chain', None) if chain_id in self._error_by_chain_id: return chain_id, self._error_by_chain_id[chain_id] diff --git a/mitogen/parent.py b/mitogen/parent.py index 29decbc1..556afb22 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -523,22 +523,6 @@ def upgrade_router(econtext): ) -def make_call_msg(fn, *args, **kwargs): - if inspect.ismethod(fn) and inspect.isclass(fn.__self__): - klass = mitogen.core.to_text(fn.__self__.__name__) - else: - klass = None - - tup = ( - mitogen.core.to_text(fn.__module__), - klass, - mitogen.core.to_text(fn.__name__), - args, - mitogen.core.Kwargs(kwargs) - ) - return mitogen.core.Message.pickled(tup, handle=mitogen.core.CALL_FUNCTION) - - def stream_by_method_name(name): """ Given the name of a Mitogen connection method, import its implementation @@ -1137,9 +1121,216 @@ class ChildIdAllocator(object): return self.allocate() +class CallChain(object): + """ + Construct :data:`mitogen.core.CALL_FUNCTION` messages and deliver them to a + target context, optionally threading related calls such that an exception + in an earlier call cancels subsequent calls. + + :param mitogen.core.Context context: + Target context. + :param bool pipelined: + Enable pipelined mode. + + By default, :meth:`call`, :meth:`call_no_reply` and :meth:`call_async` + issue calls and produce responses, with no memory of prior exceptions. If a + call made with :meth:`call_no_reply` fails, the traceback is logged to the + target context's logging framework. + + **Pipelined Mode** + + When `pipelined=True`, if an exception occurs in a call, + all subsequent calls made by the same :class:`CallChain` instance will fail + with the same exception, including those already in-flight on the network, + and no further calls will execute until :meth:`reset` is invoked. + + No traceback is logged for calls made with :meth:`call_no_reply`, instead + the exception is saved and reported as the result of subsequent + :meth:`call` or :meth:`call_async` calls. + + A sequence of pipelined asynchronous calls can be made without wasting + network round-trips to discover if prior calls succeeded, while allowing + such chains to overlap concurrently at a target context from multiple + unrelated source contexts. This enables many calls to safely progress in + one network round-trip, like:: + + chain = mitogen.parent.CallChain(context, pipelined=True) + chain.call_no_reply(os.mkdir, '/tmp/foo') + + # If previous mkdir() failed, this never runs: + chain.call_no_reply(os.mkdir, '/tmp/foo/bar') + + # If either mkdir() failed, this never runs, and returns the exception. + recv = chain.call_async(subprocess.check_output, '/tmp/foo') + + # If mkdir() or check_call() failed, this never runs, and returns the + # exception. + chain.call(do_something) + + # The receiver also got a copy of the exception, so if this + # code was executed, the exception would also be raised. + if recv.get().unpickle() == 'baz': + pass + + When pipelining is enabled, :meth:`reset` must be called to ensure the last + exception is discarded, otherwise unbounded memory usage is possible in + long-running programs. :class:`CallChain` supports the context manager + protocol to ensure :meth:`reset` is always invoked:: + + with mitogen.parent.CallChain(context, pipelined=True) as chain: + chain.call_no_reply(...) + chain.call_no_reply(...) + chain.call_no_reply(...) + chain.call(...) + + # chain.reset() automatically invoked. + """ + def __init__(self, context, pipelined=False): + self.context = context + if pipelined: + self.chain_id = self.make_chain_id() + else: + self.chain_id = None + + @classmethod + def make_chain_id(cls): + return '%s-%s-%s-%s' % ( + socket.gethostname(), + os.getpid(), + threading.currentThread().ident, + int(1e6 * time.time()), + ) + + def __enter__(self): + return self + + def __exit__(self, _1, _2, _3): + self.reset() + + def reset(self): + """ + Instruct the target to forget any related exception. + """ + if not self.chain_id: + return + + saved, self.chain_id = self.chain_id, None + try: + self.call_no_reply(mitogen.core.Dispatcher.forget_chain, saved) + finally: + self.chain_id = saved + + def make_msg(self, fn, *args, **kwargs): + if inspect.ismethod(fn) and inspect.isclass(fn.__self__): + klass = mitogen.core.to_text(fn.__self__.__name__) + else: + klass = None + + tup = ( + self.chain_id, + mitogen.core.to_text(fn.__module__), + klass, + mitogen.core.to_text(fn.__name__), + args, + mitogen.core.Kwargs(kwargs) + ) + return mitogen.core.Message.pickled(tup, + handle=mitogen.core.CALL_FUNCTION) + + def call_no_reply(self, fn, *args, **kwargs): + """ + Like :meth:`call_async`, but do not wait for a return value, and inform + the target context no such reply is expected. If the call fails and + pipelining is disabled, the full exception will be logged to the target + context's logging framework. + + :raises mitogen.core.CallError: + An exception was raised in the remote context during execution. + """ + LOG.debug('%r.call_no_reply(%r, *%r, **%r)', + self, fn, args, kwargs) + self.context.send(self.make_msg(fn, *args, **kwargs)) + + def call_async(self, fn, *args, **kwargs): + """ + Arrange for the context's ``CALL_FUNCTION`` handle to receive a message + that causes `fn(\*args, \**kwargs)` to be invoked on the context's main + thread. + + :param fn: + A free function in module scope or a class method of a class + directly reachable from module scope: + + .. code-block:: python + + # mymodule.py + + def my_func(): + '''A free function reachable as mymodule.my_func''' + + class MyClass: + @classmethod + def my_classmethod(cls): + '''Reachable as mymodule.MyClass.my_classmethod''' + + def my_instancemethod(self): + '''Unreachable: requires a class instance!''' + + class MyEmbeddedClass: + @classmethod + def my_classmethod(cls): + '''Not directly reachable from module scope!''' + + :param tuple args: + Function arguments, if any. See :ref:`serialization-rules` for + permitted types. + :param dict kwargs: + Function keyword arguments, if any. See :ref:`serialization-rules` + for permitted types. + :returns: + :class:`mitogen.core.Receiver` configured to receive the result of + the invocation: + + .. code-block:: python + + recv = context.call_async(os.check_output, 'ls /tmp/') + try: + # Prints output once it is received. + msg = recv.get() + print(msg.unpickle()) + except mitogen.core.CallError, e: + print('Call failed:', str(e)) + + Asynchronous calls may be dispatched in parallel to multiple + contexts and consumed as they complete using + :class:`mitogen.select.Select`. + """ + LOG.debug('%r.call_async(): %r', self, CallSpec(fn, args, kwargs)) + return self.context.send_async(self.make_msg(fn, *args, **kwargs)) + + def call(self, fn, *args, **kwargs): + """ + Equivalent to :meth:`call_async(fn, \*args, \**kwargs).get().unpickle() + `. + + :returns: + The function's return value. + + :raises mitogen.core.CallError: + An exception was raised in the remote context during execution. + """ + receiver = self.call_async(fn, *args, **kwargs) + return receiver.get().unpickle(throw_dead=False) + + class Context(mitogen.core.Context): + call_chain_class = CallChain via = None + def __init__(self, *args, **kwargs): + super(Context, self).__init__(*args, **kwargs) + self.default_call_chain = self.call_chain_class(self) + def __eq__(self, other): return (isinstance(other, mitogen.core.Context) and (other.context_id == self.context_id) and @@ -1149,20 +1340,13 @@ class Context(mitogen.core.Context): return hash((self.router, self.context_id)) def call_async(self, fn, *args, **kwargs): - LOG.debug('%r.call_async(): %r', self, CallSpec(fn, args, kwargs)) - return self.send_async(make_call_msg(fn, *args, **kwargs)) + return self.default_call_chain.call_async(fn, *args, **kwargs) def call(self, fn, *args, **kwargs): - receiver = self.call_async(fn, *args, **kwargs) - return receiver.get().unpickle(throw_dead=False) + return self.default_call_chain.call(fn, *args, **kwargs) def call_no_reply(self, fn, *args, **kwargs): - LOG.debug('%r.call_no_reply(%r, *%r, **%r)', - self, fn, args, kwargs) - self.send(make_call_msg(fn, *args, **kwargs)) - - def forget_chain(self, chain_id): - self.call_no_reply(mitogen.core.Dispatcher.forget_chain, chain_id) + self.default_call_chain.call_no_reply(fn, *args, **kwargs) def shutdown(self, wait=False): LOG.debug('%r.shutdown() sending SHUTDOWN', self) diff --git a/tests/call_function_test.py b/tests/call_function_test.py index ca56f07a..dc9a2298 100644 --- a/tests/call_function_test.py +++ b/tests/call_function_test.py @@ -4,6 +4,7 @@ import time import unittest2 import mitogen.core +import mitogen.parent import mitogen.master import testlib @@ -120,36 +121,37 @@ class CallFunctionTest(testlib.RouterMixin, testlib.TestCase): class ChainTest(testlib.RouterMixin, testlib.TestCase): # Verify mitogen_chain functionality. + klass = mitogen.parent.CallChain def setUp(self): super(ChainTest, self).setUp() self.local = self.router.fork() def test_subsequent_calls_produce_same_error(self): - self.assertEquals('xx', - self.local.call(func_returns_arg, 'xx', mitogen_chain='c1')) - self.local.call_no_reply(function_that_fails, 'x1', mitogen_chain='c1') + chain = self.klass(self.local, pipelined=True) + self.assertEquals('xx', chain.call(func_returns_arg, 'xx')) + chain.call_no_reply(function_that_fails, 'x1') e1 = self.assertRaises(mitogen.core.CallError, - lambda: self.local.call(function_that_fails, 'x2', mitogen_chain='c1')) + lambda: chain.call(function_that_fails, 'x2')) e2 = self.assertRaises(mitogen.core.CallError, - lambda: self.local.call(func_returns_arg, 'x3', mitogen_chain='c1')) + lambda: chain.call(func_returns_arg, 'x3')) self.assertEquals(str(e1), str(e2)) def test_unrelated_overlapping_failed_chains(self): - self.local.call_no_reply(function_that_fails, 'c1', mitogen_chain='c1') - self.assertEquals('yes', - self.local.call(func_returns_arg, 'yes', mitogen_chain='c2')) + c1 = self.klass(self.local, pipelined=True) + c2 = self.klass(self.local, pipelined=True) + c1.call_no_reply(function_that_fails, 'c1') + self.assertEquals('yes', c2.call(func_returns_arg, 'yes')) self.assertRaises(mitogen.core.CallError, - lambda: self.local.call(func_returns_arg, 'yes', mitogen_chain='c1')) - self.local.call_no_reply(function_that_fails, 'c2', mitogen_chain='c2') + lambda: c1.call(func_returns_arg, 'yes')) - def test_forget(self): - self.local.call_no_reply(function_that_fails, 'x1', mitogen_chain='c1') + def test_reset(self): + c1 = self.klass(self.local, pipelined=True) + c1.call_no_reply(function_that_fails, 'x1') e1 = self.assertRaises(mitogen.core.CallError, - lambda: self.local.call(function_that_fails, 'x2', mitogen_chain='c1')) - self.local.forget_chain('c1') - self.assertEquals('x3', - self.local.call(func_returns_arg, 'x3', mitogen_chain='c1')) + lambda: c1.call(function_that_fails, 'x2')) + c1.reset() + self.assertEquals('x3', c1.call(func_returns_arg, 'x3')) if __name__ == '__main__': From 020482e5540a94b4d1896060e4124a0fec06f9e3 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 9 Sep 2018 18:52:50 +0100 Subject: [PATCH 132/212] dosc: update changelog --- docs/changelog.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 44480118..46d44f62 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -161,11 +161,9 @@ Core Library listener no longer crashes if the peer process disappears in the middle of connection setup. -* `5adae88d `_ a new - `mitogen_chain` keyword argument is accepted by - :meth:`mitogen.parent.Context.call_async`, allowing overlapping chains of - function calls to be pipelined to a context, while cancelling the chain on - the first exception. +* A new :class:`mitogen.parent.CallChain` class abstracts safe pipelining of + overlapping chains of unrelated function calls to a target context, + cancelling the chain if an exception occurs. Thanks! From 43d9815f6d2f9bc1b10fd72b5e00c44045266c92 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 9 Sep 2018 20:29:02 +0100 Subject: [PATCH 133/212] ansible: use CallChain everywhere. This replaces the 'dump to logger' behaviour of pipelined calls from before with a call chain that returns any exception on next synchronized call. --- ansible_mitogen/connection.py | 183 ++++++++++++++++++---------------- ansible_mitogen/mixins.py | 26 ++--- ansible_mitogen/planner.py | 2 +- mitogen/parent.py | 6 +- 4 files changed, 113 insertions(+), 104 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 2b4ed61b..52a61b77 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -403,6 +403,34 @@ def config_from_hostvars(transport, inventory_name, connection, }) +class CallChain(mitogen.parent.CallChain): + call_aborted_msg = ( + 'Mitogen was disconnected from the remote environment while a call ' + 'was in-progress. If you feel this is in error, please file a bug. ' + 'Original error was: %s' + ) + + def _rethrow(self, recv): + try: + return recv.get().unpickle() + except mitogen.core.ChannelError as e: + raise ansible.errors.AnsibleConnectionFailure( + self.call_aborted_msg % (e,) + ) + + def call(self, func, *args, **kwargs): + """ + Like :meth:`mitogen.parent.CallChain.call`, but log timings. + """ + t0 = time.time() + try: + recv = self.call_async(func, *args, **kwargs) + return self._rethrow(recv) + finally: + LOG.debug('Call took %d ms: %r', 1000 * (time.time() - t0), + mitogen.parent.CallSpec(func, args, kwargs)) + + class Connection(ansible.plugins.connection.ConnectionBase): #: mitogen.master.Broker for this worker. broker = None @@ -422,17 +450,30 @@ class Connection(ansible.plugins.connection.ConnectionBase): #: account, even when become=True. login_context = None + #: Only sudo, su, and doas are supported for now. + become_methods = ['sudo', 'su', 'doas'] + #: Dict containing init_child() return vaue as recorded at startup by #: ContextService. Contains: #: #: fork_context: Context connected to the fork parent : process in the #: target account. #: home_dir: Target context's home directory. - #: temp_dir: A writeable temporary directory path. + #: temp_dir: A writeable temporary directory managed by the + #: target, automatically destroyed at shutdown. init_child_result = None - #: Only sudo, su, and doas are supported for now. - become_methods = ['sudo', 'su', 'doas'] + #: After :meth:`get_temp_dir` is called, a private temporary directory, + #: destroyed during :meth:`close`, or automatically during shutdown if + #: :meth:`close` failed or was never called. + _temp_dir = None + + #: A :class:`mitogen.parent.CallChain` to use for calls made to the target + #: account, to ensure subsequent calls fail if pipelined directory creation + #: or file transfer fails. This eliminates roundtrips when a call is likely + #: to succeed, and ensures subsequent actions will fail with the original + #: exception if the pipelined call failed. + chain = None # # Note: any of the attributes below may be :data:`None` if the connection @@ -625,6 +666,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): raise ansible.errors.AnsibleConnectionFailure(dct['msg']) self.context = dct['context'] + self.chain = CallChain(self.context, pipelined=True) if self._play_context.become: self.login_context = dct['via'] else: @@ -633,8 +675,16 @@ class Connection(ansible.plugins.connection.ConnectionBase): self.init_child_result = dct['init_child_result'] def get_temp_dir(self): - self._connect() - return self.init_child_result['temp_dir'] + """ + """ + if self._temp_dir is None: + self._temp_dir = os.path.join( + self.init_child_result['temp_dir'], + 'worker-%d-%x' % (os.getpid(), id(self)) + ) + self.get_chain().call_no_reply(os.mkdir, self._temp_dir) + + return self._temp_dir def _connect(self): """ @@ -661,6 +711,13 @@ class Connection(ansible.plugins.connection.ConnectionBase): multiple times. """ if self.context: + self.chain.reset() + if self._temp_dir: + # Don't pipeline here to ensure exception is dumped into + # logging framework on failure. + self.context.call_no_reply(ansible_mitogen.target.prune_tree, + self._temp_dir) + self._temp_dir = None self.parent.call_service( service_name='ansible_mitogen.services.ContextService', method_name='put', @@ -668,78 +725,33 @@ class Connection(ansible.plugins.connection.ConnectionBase): ) self.context = None + self.login_context = None self.init_child_result = None + self.chain = None if self.broker and not new_task: self.broker.shutdown() self.broker.join() self.broker = None self.router = None - def call_async(self, func, *args, **kwargs): + def get_chain(self, use_login=False, use_fork=False): """ - Start a function call to the target. - - :param bool use_login_context: - If present and :data:`True`, send the call to the login account - context rather than the optional become user context. + Return the :class:`mitogen.parent.CallChain` to use for executing + function calls. - :param bool no_reply: - If present and :data:`True`, send the call with no ``reply_to`` - header, causing the context to execute it entirely asynchronously, - and to log any exception thrown. This allows avoiding a roundtrip - in places where the outcome of a call is highly likely to succeed, - and subsequent actions will fail regardless with a meaningful - exception if the no_reply call failed. - - :returns: - :class:`mitogen.core.Receiver` that receives the function call result. + :param bool use_login: + If :data:`True`, always return the chain for the login account + rather than any active become user. + :param bool use_fork: + If :data:`True`, return the chain for the fork parent. + :returns mitogen.parent.CallChain: """ self._connect() - - if kwargs.pop('use_login_context', None): - call_context = self.login_context - elif kwargs.pop('use_fork_context', None): - call_context = self.init_child_result['fork_context'] - else: - call_context = self.context - - if kwargs.pop('no_reply', None): - return call_context.call_no_reply(func, *args, **kwargs) - else: - return call_context.call_async(func, *args, **kwargs) - - call_aborted_msg = ( - 'Mitogen was disconnected from the remote environment while a call ' - 'was in-progress. If you feel this is in error, please file a bug. ' - 'Original error was: %s' - ) - - def _call_rethrow(self, recv): - try: - return recv.get().unpickle() - except mitogen.core.ChannelError as e: - raise ansible.errors.AnsibleConnectionFailure( - self.call_aborted_msg % (e,) - ) - - def call(self, func, *args, **kwargs): - """ - Start and wait for completion of a function call in the target. - - :raises mitogen.core.CallError: - The function call failed. - :returns: - Function return value. - """ - t0 = time.time() - try: - recv = self.call_async(func, *args, **kwargs) - if recv is None: # no_reply=True - return None - return self._call_rethrow(recv) - finally: - LOG.debug('Call took %d ms: %r', 1000 * (time.time() - t0), - mitogen.parent.CallSpec(func, args, kwargs)) + if use_login: + return self.login_context.default_call_chain + if use_fork: + return self.init_child_result['fork_context'].default_call_chain + return self.chain def create_fork_child(self): """ @@ -750,8 +762,9 @@ class Connection(ansible.plugins.connection.ConnectionBase): :returns: mitogen.core.Context of the new child. """ - return self.call(ansible_mitogen.target.create_fork_child, - use_fork_context=True) + return self.get_chain(use_fork=True).call( + ansible_mitogen.target.create_fork_child + ) def get_default_cwd(self): """ @@ -804,35 +817,33 @@ class Connection(ansible.plugins.connection.ConnectionBase): :param str out_path: Local filesystem path to write. """ - output = self.call(ansible_mitogen.target.read_path, - mitogen.utils.cast(in_path)) + output = self.get_chain().call( + ansible_mitogen.target.read_path, + mitogen.utils.cast(in_path), + ) ansible_mitogen.target.write_path(out_path, output) def put_data(self, out_path, data, mode=None, utimes=None): """ Implement put_file() by caling the corresponding ansible_mitogen.target - function in the target, transferring small files inline. + function in the target, transferring small files inline. This is + pipelined and will return immediately; failed transfers are reported as + exceptions in subsequent functon calls. :param str out_path: Remote filesystem path to write. :param byte data: File contents to put. """ - # no_reply=True here avoids a roundrip that 99% of the time will report - # a successful response. If the file transfer fails, the target context - # will dump an exception into the logging framework, which will appear - # on console, and the missing file will cause the subsequent task step - # to fail regardless. This is safe since CALL_FUNCTION is presently - # single-threaded for each target, so subsequent steps cannot execute - # until the transfer RPC has completed. - self.call(ansible_mitogen.target.write_path, - mitogen.utils.cast(out_path), - mitogen.core.Blob(data), - mode=mode, - utimes=utimes, - no_reply=True) - - #: Maximum size of a small file before switching to streaming file + self.get_chain().call_no_reply( + ansible_mitogen.target.write_path, + mitogen.utils.cast(out_path), + mitogen.core.Blob(data), + mode=mode, + utimes=utimes, + ) + + #: Maximum size of a small file before switching to streaming #: transfer. This should really be the same as #: mitogen.services.FileService.IO_SIZE, however the message format has #: slightly more overhead, so just randomly subtract 4KiB. diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 7be5444f..6ab8fec7 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -115,15 +115,6 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): ) return super(ActionModuleMixin, self).run(tmp, task_vars) - def call(self, func, *args, **kwargs): - """ - Arrange for a Python function to be called in the target context, which - should be some function from the standard library or - ansible_mitogen.target module. This junction point exists mainly as a - nice place to insert print statements during debugging. - """ - return self._connection.call(func, *args, **kwargs) - COMMAND_RESULT = { 'rc': 0, 'stdout': '', @@ -164,7 +155,10 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): target user account. """ LOG.debug('_remote_file_exists(%r)', path) - return self.call(os.path.exists, mitogen.utils.cast(path)) + return self._connection.get_chain().call( + os.path.exists, + mitogen.utils.cast(path) + ) def _configure_module(self, module_name, module_args, task_vars=None): """ @@ -241,7 +235,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): LOG.debug('_remote_chmod(%r, mode=%r, sudoable=%r)', paths, mode, sudoable) return self.fake_shell(lambda: mitogen.select.Select.all( - self._connection.call_async( + self._connection.get_chain().call_async( ansible_mitogen.target.set_file_mode, path, mode ) for path in paths @@ -254,9 +248,9 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): """ LOG.debug('_remote_chown(%r, user=%r, sudoable=%r)', paths, user, sudoable) - ent = self.call(pwd.getpwnam, user) + ent = self._connection.get_chain().call(pwd.getpwnam, user) return self.fake_shell(lambda: mitogen.select.Select.all( - self._connection.call_async( + self._connection.get_chain().call_async( os.chown, path, ent.pw_uid, ent.pw_gid ) for path in paths @@ -284,8 +278,10 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): # ~/.ansible -> /home/dmw/.ansible return os.path.join(self._connection.homedir, path[2:]) # ~root/.ansible -> /root/.ansible - return self.call(os.path.expanduser, mitogen.utils.cast(path), - use_login_context=not sudoable) + return self._connection.get_chain(login=(not sudoable)).call( + os.path.expanduser, + mitogen.utils.cast(path), + ) def get_task_timeout_secs(self): """ diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 8ebf4f67..c709bf7d 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -478,7 +478,7 @@ def invoke(invocation): response = _invoke_forked_task(invocation, planner) else: _propagate_deps(invocation, planner, invocation.connection.context) - response = invocation.connection.call( + response = invocation.connection.get_chain().call( ansible_mitogen.target.run_module, kwargs=planner.get_kwargs(), ) diff --git a/mitogen/parent.py b/mitogen/parent.py index 556afb22..bb0eb0fb 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1201,6 +1201,9 @@ class CallChain(object): int(1e6 * time.time()), ) + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, self.context) + def __enter__(self): return self @@ -1247,8 +1250,7 @@ class CallChain(object): :raises mitogen.core.CallError: An exception was raised in the remote context during execution. """ - LOG.debug('%r.call_no_reply(%r, *%r, **%r)', - self, fn, args, kwargs) + LOG.debug('%r.call_no_reply(): %r', self, CallSpec(fn, args, kwargs)) self.context.send(self.make_msg(fn, *args, **kwargs)) def call_async(self, fn, *args, **kwargs): From 42d3f96d146f068a4d90a30159f223857606963e Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 9 Sep 2018 21:43:46 +0100 Subject: [PATCH 134/212] parent: do updates --- mitogen/parent.py | 74 ++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 39 deletions(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index bb0eb0fb..b291447c 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1123,36 +1123,36 @@ class ChildIdAllocator(object): class CallChain(object): """ - Construct :data:`mitogen.core.CALL_FUNCTION` messages and deliver them to a - target context, optionally threading related calls such that an exception - in an earlier call cancels subsequent calls. + Deliver :data:`mitogen.core.CALL_FUNCTION` messages to a target context, + optionally threading related calls so an exception in an earlier call + cancels subsequent calls. :param mitogen.core.Context context: Target context. :param bool pipelined: - Enable pipelined mode. + Enable pipelining. - By default, :meth:`call`, :meth:`call_no_reply` and :meth:`call_async` - issue calls and produce responses, with no memory of prior exceptions. If a - call made with :meth:`call_no_reply` fails, the traceback is logged to the - target context's logging framework. + :meth:`call`, :meth:`call_no_reply` and :meth:`call_async` + normally issue calls and produce responses with no memory of prior + exceptions. If a call made with :meth:`call_no_reply` fails, the exception + is logged to the target context's logging framework. - **Pipelined Mode** + **Pipelining** - When `pipelined=True`, if an exception occurs in a call, - all subsequent calls made by the same :class:`CallChain` instance will fail - with the same exception, including those already in-flight on the network, - and no further calls will execute until :meth:`reset` is invoked. + When pipelining is enabled, if an exception occurs during a call, + subsequent calls made by the same :class:`CallChain` fail with the same + exception, including those already in-flight on the network, and no further + calls execute until :meth:`reset` is invoked. - No traceback is logged for calls made with :meth:`call_no_reply`, instead - the exception is saved and reported as the result of subsequent - :meth:`call` or :meth:`call_async` calls. + No exception is logged for calls made with :meth:`call_no_reply`, instead + it is saved and reported as the result of subsequent :meth:`call` or + :meth:`call_async` calls. - A sequence of pipelined asynchronous calls can be made without wasting - network round-trips to discover if prior calls succeeded, while allowing - such chains to overlap concurrently at a target context from multiple - unrelated source contexts. This enables many calls to safely progress in - one network round-trip, like:: + Sequences of asynchronous calls can be made without wasting network + round-trips to discover if prior calls succeed, and chains originating from + multiple unrelated source contexts may overlap concurrently at a target + context without interference. In this example, 4 calls complete in one + round-trip:: chain = mitogen.parent.CallChain(context, pipelined=True) chain.call_no_reply(os.mkdir, '/tmp/foo') @@ -1160,22 +1160,21 @@ class CallChain(object): # If previous mkdir() failed, this never runs: chain.call_no_reply(os.mkdir, '/tmp/foo/bar') - # If either mkdir() failed, this never runs, and returns the exception. + # If either mkdir() failed, this never runs, and the exception is + # asynchronously delivered to the receiver. recv = chain.call_async(subprocess.check_output, '/tmp/foo') - # If mkdir() or check_call() failed, this never runs, and returns the - # exception. + # If anything so far failed, this never runs, and raises the exception. chain.call(do_something) - # The receiver also got a copy of the exception, so if this - # code was executed, the exception would also be raised. + # If this code was executed, the exception would also be raised. if recv.get().unpickle() == 'baz': pass - When pipelining is enabled, :meth:`reset` must be called to ensure the last + When pipelining is enabled, :meth:`reset` must be invoked to ensure any exception is discarded, otherwise unbounded memory usage is possible in - long-running programs. :class:`CallChain` supports the context manager - protocol to ensure :meth:`reset` is always invoked:: + long-running programs. The context manager protocol is supported to ensure + :meth:`reset` is always invoked:: with mitogen.parent.CallChain(context, pipelined=True) as chain: chain.call_no_reply(...) @@ -1243,20 +1242,16 @@ class CallChain(object): def call_no_reply(self, fn, *args, **kwargs): """ Like :meth:`call_async`, but do not wait for a return value, and inform - the target context no such reply is expected. If the call fails and - pipelining is disabled, the full exception will be logged to the target + the target context no reply is expected. If the call fails and + pipelining is disabled, the exception will be logged to the target context's logging framework. - - :raises mitogen.core.CallError: - An exception was raised in the remote context during execution. """ LOG.debug('%r.call_no_reply(): %r', self, CallSpec(fn, args, kwargs)) self.context.send(self.make_msg(fn, *args, **kwargs)) def call_async(self, fn, *args, **kwargs): """ - Arrange for the context's ``CALL_FUNCTION`` handle to receive a message - that causes `fn(\*args, \**kwargs)` to be invoked on the context's main + Arrange for `fn(\*args, \**kwargs)` to be invoked on the context's main thread. :param fn: @@ -1312,12 +1307,13 @@ class CallChain(object): def call(self, fn, *args, **kwargs): """ - Equivalent to :meth:`call_async(fn, \*args, \**kwargs).get().unpickle() - `. + Like :meth:`call_async`, but block until the return value is available. + Equivalent to:: + + call_async(fn, *args, **kwargs).get().unpickle() :returns: The function's return value. - :raises mitogen.core.CallError: An exception was raised in the remote context during execution. """ From e241081cae51f7abcfeb5de547bf5fb16ea19cc6 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 9 Sep 2018 23:39:08 +0100 Subject: [PATCH 135/212] ansible: stop sharing target temp_dir in runner. This cannot work with delegate_to, since delegate_to permits multiple concurrent tasks to be executing on the same target. --- ansible_mitogen/connection.py | 34 ++++++++++------------ ansible_mitogen/mixins.py | 9 +++--- ansible_mitogen/planner.py | 1 + ansible_mitogen/runner.py | 55 ++++++++++++----------------------- docs/ansible.rst | 7 +++-- mitogen/parent.py | 2 +- 6 files changed, 44 insertions(+), 64 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 52a61b77..6c2ef4b0 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -463,10 +463,10 @@ class Connection(ansible.plugins.connection.ConnectionBase): #: target, automatically destroyed at shutdown. init_child_result = None - #: After :meth:`get_temp_dir` is called, a private temporary directory, - #: destroyed during :meth:`close`, or automatically during shutdown if - #: :meth:`close` failed or was never called. - _temp_dir = None + #: A private temporary directory destroyed during :meth:`close`, or + #: automatically during shutdown if :meth:`close` failed or was never + #: called. + temp_dir = None #: A :class:`mitogen.parent.CallChain` to use for calls made to the target #: account, to ensure subsequent calls fail if pipelined directory creation @@ -674,17 +674,14 @@ class Connection(ansible.plugins.connection.ConnectionBase): self.init_child_result = dct['init_child_result'] - def get_temp_dir(self): + def _init_temp_dir(self): """ """ - if self._temp_dir is None: - self._temp_dir = os.path.join( - self.init_child_result['temp_dir'], - 'worker-%d-%x' % (os.getpid(), id(self)) - ) - self.get_chain().call_no_reply(os.mkdir, self._temp_dir) - - return self._temp_dir + self.temp_dir = os.path.join( + self.init_child_result['temp_dir'], + 'worker-%d-%x' % (os.getpid(), id(self)) + ) + self.get_chain().call_no_reply(os.mkdir, self.temp_dir) def _connect(self): """ @@ -703,6 +700,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): self._connect_broker() stack = self._build_stack() self._connect_stack(stack) + self._init_temp_dir() def close(self, new_task=False): """ @@ -712,18 +710,16 @@ class Connection(ansible.plugins.connection.ConnectionBase): """ if self.context: self.chain.reset() - if self._temp_dir: - # Don't pipeline here to ensure exception is dumped into - # logging framework on failure. - self.context.call_no_reply(ansible_mitogen.target.prune_tree, - self._temp_dir) - self._temp_dir = None + # No pipelining to ensure exception is logged on failure. + self.context.call_no_reply(ansible_mitogen.target.prune_tree, + self.temp_dir) self.parent.call_service( service_name='ansible_mitogen.services.ContextService', method_name='put', context=self.context ) + self.temp_dir = None self.context = None self.login_context = None self.init_child_result = None diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 6ab8fec7..2ebffb84 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -176,12 +176,13 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): def _make_tmp_path(self, remote_user=None): """ - Return the temporary directory created by the persistent interpreter at - startup. + Return the directory created by the Connection instance during + connection. """ LOG.debug('_make_tmp_path(remote_user=%r)', remote_user) + self._connection._connect() # _make_tmp_path() is basically a global stashed away as Shell.tmpdir. - self._connection._shell.tmpdir = self._connection.get_temp_dir() + self._connection._shell.tmpdir = self._connection.temp_dir LOG.debug('Temporary directory: %r', self._connection._shell.tmpdir) self._cleanup_remote_tmp = True return self._connection._shell.tmpdir @@ -318,7 +319,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): self._connection._connect() if ansible.__version__ > '2.5': - module_args['_ansible_tmpdir'] = self._connection.get_temp_dir() + module_args['_ansible_tmpdir'] = self._connection.temp_dir return ansible_mitogen.planner.invoke( ansible_mitogen.planner.Invocation( diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index c709bf7d..5b3e5547 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -149,6 +149,7 @@ class Planner(object): """ new = dict((mitogen.core.UnicodeType(k), kwargs[k]) for k in kwargs) + new.setdefault('temp_dir', self._inv.connection.temp_dir) new.setdefault('cwd', self._inv.connection.get_default_cwd()) new.setdefault('extra_env', self._inv.connection.get_default_env()) new.setdefault('emulate_tty', True) diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 97a3cb36..fe5f4c46 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -66,9 +66,6 @@ except ImportError: # Prevent accidental import of an Ansible module from hanging on stdin read. import ansible.module_utils.basic ansible.module_utils.basic._ANSIBLE_ARGS = '{}' -ansible.module_utils.basic.get_module_path = lambda: ( - ansible_mitogen.target.temp_dir -) # For tasks that modify /etc/resolv.conf, non-Debian derivative glibcs cache # resolv.conf at startup and never implicitly reload it. Cope with that via an @@ -245,13 +242,15 @@ class Runner(object): When :data:`True`, indicate the runner should detach the context from its parent after setup has completed successfully. """ - def __init__(self, module, service_context, json_args, extra_env=None, - cwd=None, env=None, econtext=None, detach=False): + def __init__(self, module, service_context, json_args, temp_dir, + extra_env=None, cwd=None, env=None, econtext=None, + detach=False): self.module = module self.service_context = service_context self.econtext = econtext self.detach = detach self.args = json.loads(json_args) + self.temp_dir = temp_dir self.extra_env = extra_env self.env = env self.cwd = cwd @@ -292,33 +291,6 @@ class Runner(object): implementation simply restores the original environment. """ self._env.revert() - self._try_cleanup_temp() - - def _cleanup_temp(self): - """ - Empty temp_dir in time for the next module invocation. - """ - for name in os.listdir(ansible_mitogen.target.temp_dir): - if name in ('.', '..'): - continue - - path = os.path.join(ansible_mitogen.target.temp_dir, name) - LOG.debug('Deleting %r', path) - ansible_mitogen.target.prune_tree(path) - - def _try_cleanup_temp(self): - """ - During broker shutdown triggered by async task timeout or loss of - connection to the parent, it is possible for prune_tree() in - target.py::_on_broker_shutdown() to run before _cleanup_temp(), so skip - cleanup if the directory or a file disappears from beneath us. - """ - try: - self._cleanup_temp() - except (IOError, OSError) as e: - if e.args[0] == errno.ENOENT: - return - raise def _run(self): """ @@ -431,7 +403,8 @@ class NewStyleStdio(object): """ Patch ansible.module_utils.basic argument globals. """ - def __init__(self, args): + def __init__(self, args, temp_dir): + self.temp_dir = temp_dir self.original_stdout = sys.stdout self.original_stderr = sys.stderr self.original_stdin = sys.stdin @@ -441,7 +414,15 @@ class NewStyleStdio(object): ansible.module_utils.basic._ANSIBLE_ARGS = utf8(encoded) sys.stdin = StringIO(mitogen.core.to_text(encoded)) + self.original_get_path = getattr(ansible.module_utils.basic, + 'get_module_path', None) + ansible.module_utils.basic.get_module_path = self._get_path + + def _get_path(self): + return self.temp_dir + def revert(self): + ansible.module_utils.basic.get_module_path = self.original_get_path sys.stdout = self.original_stdout sys.stderr = self.original_stderr sys.stdin = self.original_stdin @@ -485,7 +466,7 @@ class ProgramRunner(Runner): fetched via :meth:`_get_program`. """ filename = self._get_program_filename() - path = os.path.join(ansible_mitogen.target.temp_dir, filename) + path = os.path.join(self.temp_dir, filename) self.program_fp = open(path, 'wb') self.program_fp.write(self._get_program()) self.program_fp.flush() @@ -565,7 +546,7 @@ class ArgsFileRunner(Runner): self.args_fp = tempfile.NamedTemporaryFile( prefix='ansible_mitogen', suffix='-args', - dir=ansible_mitogen.target.temp_dir, + dir=self.temp_dir, ) self.args_fp.write(utf8(self._get_args_contents())) self.args_fp.flush() @@ -680,7 +661,7 @@ class NewStyleRunner(ScriptRunner): def setup(self): super(NewStyleRunner, self).setup() - self._stdio = NewStyleStdio(self.args) + self._stdio = NewStyleStdio(self.args, self.temp_dir) # It is possible that not supplying the script filename will break some # module, but this has never been a bug report. Instead act like an # interpreter that had its script piped on stdin. @@ -758,7 +739,7 @@ class NewStyleRunner(ScriptRunner): # don't want to pointlessly write the module to disk when it never # actually needs to exist. So just pass the filename as it would exist. mod.__file__ = os.path.join( - ansible_mitogen.target.temp_dir, + self.temp_dir, 'ansible_module_' + os.path.basename(self.path), ) diff --git a/docs/ansible.rst b/docs/ansible.rst index 14f6e553..7a06c120 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -464,9 +464,10 @@ filesystem with ``noexec`` disabled: 8. ``/usr/tmp`` 9. Current working directory -As the directory is created once at startup, and its content is managed by code -running remotely, no additional network roundtrips are required to manage it -for each task requiring temporary storage. +The directory is created once at startup, and subdirectories are automatically +created and destroyed for every new task. Management of subdirectories happens +on the controller, but management of the parent directory happens entirely on +the target. .. _ansible_process_env: diff --git a/mitogen/parent.py b/mitogen/parent.py index b291447c..1bda68a2 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1193,7 +1193,7 @@ class CallChain(object): @classmethod def make_chain_id(cls): - return '%s-%s-%s-%s' % ( + return '%s-%s-%x-%x' % ( socket.gethostname(), os.getpid(), threading.currentThread().ident, From c8081e7ca144ab65638552c8a1accf80f7550a34 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 9 Sep 2018 23:42:27 +0100 Subject: [PATCH 136/212] docs: typo --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 46d44f62..818242ca 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -162,7 +162,7 @@ Core Library connection setup. * A new :class:`mitogen.parent.CallChain` class abstracts safe pipelining of - overlapping chains of unrelated function calls to a target context, + overlapping chains of related function calls to a target context, cancelling the chain if an exception occurs. From 2c0244eea71132b81c39d8a14c088a30d549fb5a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Sun, 9 Sep 2018 23:51:48 +0100 Subject: [PATCH 137/212] docs: more tweaks --- docs/api.rst | 8 ++++---- docs/changelog.rst | 32 ++++++++++++++++---------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 4ad1db5e..cb980b55 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1094,11 +1094,11 @@ Select Class .. code-block:: python - sum(context.call_async(get_disk_usage).get().unpickle() - for context in contexts) + recvs = [c.call_async(get_disk_usage) for c in contexts] + sum(recv.get().unpickle() for recv in recvs) - Result processing happens concurrently to new results arriving, so - :meth:`all` should always be faster. + Result processing happens in the order results arrive, rather than the + order requests were issued, so :meth:`all` should always be faster. .. py:method:: get (timeout=None, block=True) diff --git a/docs/changelog.rst b/docs/changelog.rst index 818242ca..20ed9a5d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -61,9 +61,9 @@ Enhancements * `5189408e `_: threads are cooperatively scheduled, minimizing `GIL `_ contention, and - reducing context switching by an order of magnitude. This manifests as an - overall improvement, but is easily noticeable on short many-target - runs, where startup overhead dominates runtime. + reducing context switching by around 90%. This manifests as an overall + improvement, but is easily noticeable on short many-target runs, where + startup overhead dominates runtime. Fixes @@ -109,9 +109,9 @@ Fixes Ansible's behaviour. * `#355 `_: tasks configured to run - in an isolated forked subprocess were being forked from the wrong parent - context. This meant built-in modules overridden via a custom ``module_utils`` - search path may not have had any effect. + in an isolated forked subprocess were forked from the wrong parent context. + This meant built-in modules overridden via a custom ``module_utils`` search + path may not have had any effect. * A missing check caused an exception traceback to appear when using the ``ansible`` command-line tool with a missing or misspelled module name. @@ -126,15 +126,19 @@ Fixes Core Library ~~~~~~~~~~~~ +* A new :class:`mitogen.parent.CallChain` class abstracts safe pipelining of + related function calls to a target context, cancelling the chain if an + exception occurs. + * `#305 `_: fix a long-standing minor - race relating to the logging framework, where *no route for Message(...)(* - would appear fruequently during startup. + race relating to the logging framework, where *no route for Message..* + would frequently appear during startup. * `#313 `_: - :meth:`mitogen.parent.Context.call` was accidentally documented as capable of - accepting static methods. While possible on Python 2.x the result is ugly, - and in every case it should be trivial to replace with a classmethod. The - documentation was fixed. + :meth:`mitogen.parent.Context.call` was documented as capable of accepting + static methods. While possible on Python 2.x the result is ugly, and in every + case it should be trivial to replace with a classmethod. The documentation + was fixed. * `#337 `_: to avoid a scaling limitation, SSH no longer allocates a PTY for every OpenSSH client. PTYs are @@ -161,10 +165,6 @@ Core Library listener no longer crashes if the peer process disappears in the middle of connection setup. -* A new :class:`mitogen.parent.CallChain` class abstracts safe pipelining of - overlapping chains of related function calls to a target context, - cancelling the chain if an exception occurs. - Thanks! ~~~~~~~ From 863c1b75975e9329d0e5456ad8c6532ca40c19f4 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 00:23:32 +0100 Subject: [PATCH 138/212] parent: correct CallSpec name formatting for class methods. --- mitogen/parent.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index 1bda68a2..8e9f53a5 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -596,8 +596,12 @@ class CallSpec(object): self.kwargs = kwargs def _get_name(self): - return u'%s.%s' % (self.func.__module__, - self.func.__name__) + bits = [self.func.__module__] + if inspect.ismethod(self.func): + bits.append(getattr(self.func.__self__, '__name__', None) or + getattr(type(self.func.__self__), '__name__', None)) + bits.append(self.func.__name__) + return u'.'.join(bits) def _get_args(self): return u', '.join(repr(a) for a in self.args) From 24a44499cac0f06683f85ba00b9e85993e38f325 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 00:26:14 +0100 Subject: [PATCH 139/212] tests: verify Connection.put_file() for small/large files. --- tests/ansible/integration/all.yml | 1 + .../integration/connection/_put_file.yml | 23 +++++++++++++++++++ tests/ansible/integration/connection/all.yml | 4 ++++ .../integration/connection/put_large_file.yml | 12 ++++++++++ .../integration/connection/put_small_file.yml | 12 ++++++++++ 5 files changed, 52 insertions(+) create mode 100644 tests/ansible/integration/connection/_put_file.yml create mode 100644 tests/ansible/integration/connection/all.yml create mode 100644 tests/ansible/integration/connection/put_large_file.yml create mode 100644 tests/ansible/integration/connection/put_small_file.yml diff --git a/tests/ansible/integration/all.yml b/tests/ansible/integration/all.yml index 4550e203..e9a12ec8 100644 --- a/tests/ansible/integration/all.yml +++ b/tests/ansible/integration/all.yml @@ -6,6 +6,7 @@ - import_playbook: action/all.yml - import_playbook: async/all.yml - import_playbook: become/all.yml +- import_playbook: connection/all.yml - import_playbook: connection_loader/all.yml - import_playbook: context_service/all.yml - import_playbook: delegation/all.yml diff --git a/tests/ansible/integration/connection/_put_file.yml b/tests/ansible/integration/connection/_put_file.yml new file mode 100644 index 00000000..e93a1e68 --- /dev/null +++ b/tests/ansible/integration/connection/_put_file.yml @@ -0,0 +1,23 @@ +--- + +- shell: dd if=/dev/urandom of=/tmp/{{file_name}} bs=1024 count={{file_size}} + args: + creates: /tmp/{{file_name}} + connection: local + +- copy: + dest: /tmp/{{file_name}}.out + src: /tmp/{{file_name}} + +- stat: path=/tmp/{{file_name}} + register: original + connection: local + +- stat: path=/tmp/{{file_name}} + register: copied + +- assert: + that: + - original.stat.checksum == copied.stat.checksum + #- original.stat.atime == copied.stat.atime + - original.stat.mtime == copied.stat.mtime diff --git a/tests/ansible/integration/connection/all.yml b/tests/ansible/integration/connection/all.yml new file mode 100644 index 00000000..96018d1a --- /dev/null +++ b/tests/ansible/integration/connection/all.yml @@ -0,0 +1,4 @@ +--- + +- import_playbook: put_small_file.yml +- import_playbook: put_large_file.yml diff --git a/tests/ansible/integration/connection/put_large_file.yml b/tests/ansible/integration/connection/put_large_file.yml new file mode 100644 index 00000000..210c5d6a --- /dev/null +++ b/tests/ansible/integration/connection/put_large_file.yml @@ -0,0 +1,12 @@ +# Test transfers made via FileService. +--- + +- name: integration/connection/put_large_file.yml + hosts: test-targets + gather_facts: no + any_errors_fatal: true + vars: + file_name: large-file + file_size: 512 + tasks: + - include_tasks: _put_file.yml diff --git a/tests/ansible/integration/connection/put_small_file.yml b/tests/ansible/integration/connection/put_small_file.yml new file mode 100644 index 00000000..aa6cc0d7 --- /dev/null +++ b/tests/ansible/integration/connection/put_small_file.yml @@ -0,0 +1,12 @@ +# Test small transfers made via RPC. +--- + +- name: integration/connection/put_small_file.yml + hosts: test-targets + gather_facts: no + any_errors_fatal: true + vars: + file_name: small-file + file_size: 123 + tasks: + - include_tasks: _put_file.yml From 49736b3ae835c3bf6d44f005d29447b714da2dca Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 00:31:16 +0100 Subject: [PATCH 140/212] ansible: fix FileService call, and remove another roundtrip. --- ansible_mitogen/connection.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 6c2ef4b0..4ec00308 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -879,7 +879,14 @@ class Connection(ansible.plugins.connection.ConnectionBase): method_name='register', path=mitogen.utils.cast(in_path) ) - self.call( + + # A roundtrip is always necessary for the target to request the file + # from FileService, however, by pipelining the transfer function, the + # subsequent step (probably a module invocation) can get its + # dependencies and function call in-flight before the transfer is + # complete. This saves at least 1 RTT between the transfer completing + # and the start of the follow-up task. + self.get_chain().call_no_reply( ansible_mitogen.target.transfer_file, context=self.parent, in_path=in_path, From c9596568bee8af8ef414ac3cd7448f3a03256c18 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 00:33:54 +0100 Subject: [PATCH 141/212] docs: update changelog. --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 20ed9a5d..827ac609 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -53,6 +53,10 @@ Enhancements a 250 ms link from 30 seconds to 10 seconds compared to v0.2.2, down from 120 seconds compared to vanilla. +* `49736b3a `_: avoid a + roundtrip when transferring files larger than 124KiB, in between waiting for + the transfer to complete and start of the follow-up action.. + * `d62e6e2a `_: many-target runs executed the dependency scanner redundantly due to missing synchronization, wasting significant runtime in the connection multiplexer. From ae446ad7c8d36b08d3a435204f92cbe3076cacf6 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 00:37:24 +0100 Subject: [PATCH 142/212] docs: fix changelog --- docs/changelog.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 827ac609..81027572 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -54,8 +54,9 @@ Enhancements seconds compared to vanilla. * `49736b3a `_: avoid a - roundtrip when transferring files larger than 124KiB, in between waiting for - the transfer to complete and start of the follow-up action.. + roundtrip when transferring files larger than 124KiB, avoiding a delay + between waiting for the transfer to complete and start of the follow-up + action. * `d62e6e2a `_: many-target runs executed the dependency scanner redundantly due to missing From f0f828033fc374195ef380fb9d6b9814ae9e3f92 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 00:41:52 +0100 Subject: [PATCH 143/212] docs: update changelog. --- docs/changelog.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 81027572..14403877 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -146,11 +146,11 @@ Core Library was fixed. * `#337 `_: to avoid a scaling - limitation, SSH no longer allocates a PTY for every OpenSSH client. PTYs are - only allocated if a password is supplied, or when `host_key_checking=accept`. - This is since Linux has a default of 4096 PTYs (``kernel.pty.max``), while OS - X has a default of 127 and an absolute maximum of 999 - (``kern.tty.ptmx_max``). + limitation, SSH now longer allocates a PTY for every OpenSSH client if it can + be avoided. PTYs are only allocated if a password is supplied, or when + `host_key_checking=accept`. This is since Linux has a default of 4096 PTYs + (``kernel.pty.max``), while OS X has a default of 127 and an absolute maximum + of 999 (``kern.tty.ptmx_max``). * `#339 `_: the LXD connection method was erroneously executing LXC Classic commands. From cc4835ce9960748c0892a237c1dd82d1b7ce79c0 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 00:44:12 +0100 Subject: [PATCH 144/212] docs: update changelog. --- docs/changelog.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 14403877..c88d2436 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -54,10 +54,14 @@ Enhancements seconds compared to vanilla. * `49736b3a `_: avoid a - roundtrip when transferring files larger than 124KiB, avoiding a delay + roundtrip when transferring files larger than 124KiB, removing a delay between waiting for the transfer to complete and start of the follow-up action. +* `#337 `_: To avoid a scaling + limitation, Mitogen no longer requires a PTY for every SSH client unless an + SSH password has been specified. + * `d62e6e2a `_: many-target runs executed the dependency scanner redundantly due to missing synchronization, wasting significant runtime in the connection multiplexer. From 294f17e491986a2f6298ab8a78dd778abab24ea5 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 01:07:58 +0100 Subject: [PATCH 145/212] core: fix econtext on_start parameter, used by fork_test. --- mitogen/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/core.py b/mitogen/core.py index 07cfcf37..a6ee1896 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -2008,7 +2008,7 @@ class Dispatcher(object): def run(self): if self.econtext.config.get('on_start'): - self.econtext.config['on_start'](self) + self.econtext.config['on_start'](self.econtext) _profile_hook('main', self._dispatch_calls) From b3be182795f32da644354ad7d2818647e740fe00 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 01:13:27 +0100 Subject: [PATCH 146/212] ssh: fix 2/3 regression. --- mitogen/ssh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/ssh.py b/mitogen/ssh.py index 670294f1..50990a43 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -295,7 +295,7 @@ class Stream(mitogen.parent.Stream): # it at the start of the line. if self.password is not None and password_sent: raise PasswordError(self.password_incorrect_msg) - elif 'password' in buf and self.password is None: + elif PASSWORD_PROMPT in buf and self.password is None: # Permission denied (password,pubkey) raise PasswordError(self.password_required_msg) else: From 7cd4d0828db788dbb141765edeab362fb3d88f03 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 01:23:29 +0100 Subject: [PATCH 147/212] tests: data/fakessh.py 3.x fixes. --- tests/data/fakessh.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/data/fakessh.py b/tests/data/fakessh.py index 69d47339..8df5aa39 100755 --- a/tests/data/fakessh.py +++ b/tests/data/fakessh.py @@ -17,22 +17,22 @@ HOST_KEY_STRICT_MSG = """Host key verification failed.\n""" def tty(msg): - fp = open('/dev/tty', 'w', 0) - fp.write(msg) + fp = open('/dev/tty', 'wb', 0) + fp.write(msg.encode()) fp.close() def stderr(msg): - fp = open('/dev/stderr', 'w', 0) - fp.write(msg) + fp = open('/dev/stderr', 'wb', 0) + fp.write(msg.encode()) fp.close() def confirm(msg): tty(msg) - fp = open('/dev/tty', 'r', 0) + fp = open('/dev/tty', 'rb', 0) try: - return fp.readline() + return fp.readline().decode() finally: fp.close() From 90f89f95fbcda4059b1306d4f980b87759c38ca8 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 01:31:15 +0100 Subject: [PATCH 148/212] ansible: fix exec_command() regression. --- ansible_mitogen/connection.py | 2 +- tests/ansible/integration/connection/all.yml | 1 + .../integration/connection/exec_command.yml | 19 +++++++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 tests/ansible/integration/connection/exec_command.yml diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 4ec00308..c5c96f24 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -789,7 +789,7 @@ class Connection(ansible.plugins.connection.ConnectionBase): (return code, stdout bytes, stderr bytes) """ emulate_tty = (not in_data and sudoable) - rc, stdout, stderr = self.call( + rc, stdout, stderr = self.get_chain().call( ansible_mitogen.target.exec_command, cmd=mitogen.utils.cast(cmd), in_data=mitogen.utils.cast(in_data), diff --git a/tests/ansible/integration/connection/all.yml b/tests/ansible/integration/connection/all.yml index 96018d1a..123e11c4 100644 --- a/tests/ansible/integration/connection/all.yml +++ b/tests/ansible/integration/connection/all.yml @@ -1,4 +1,5 @@ --- +- import_playbook: exec_command.yml - import_playbook: put_small_file.yml - import_playbook: put_large_file.yml diff --git a/tests/ansible/integration/connection/exec_command.yml b/tests/ansible/integration/connection/exec_command.yml new file mode 100644 index 00000000..6a632961 --- /dev/null +++ b/tests/ansible/integration/connection/exec_command.yml @@ -0,0 +1,19 @@ +# Test basic functinality of exec_command. +--- + +- name: integration/connection/exec_command.yml + hosts: test-targets + gather_facts: no + any_errors_fatal: true + tasks: + - connection_passthrough: + method: exec_command + kwargs: + cmd: echo "hello, world" + register: out + + - assert: + that: + - out.result[0] == 0 + - out.result[1] == "hello, world\r\n" + - out.result[2].startswith("Shared connection to ") From 8e9605db0227a705332b86cb1e3ae29980e51d71 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 01:34:04 +0100 Subject: [PATCH 149/212] ssh: fix another 3.x regression. --- mitogen/ssh.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mitogen/ssh.py b/mitogen/ssh.py index 50990a43..2e723d67 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -258,7 +258,7 @@ class Stream(mitogen.parent.Stream): def _host_key_prompt(self): if self.check_host_keys == 'accept': LOG.debug('%r: accepting host key', self) - self.tty_stream.transmit_side.write('y\n') + self.tty_stream.transmit_side.write(b('y\n')) return # _host_key_prompt() should never be reached with ignore or enforce @@ -304,7 +304,9 @@ class Stream(mitogen.parent.Stream): if self.password is None: raise PasswordError(self.password_required_msg) LOG.debug('%r: sending password', self) - self.tty_stream.transmit_side.write((self.password + '\n').encode()) + self.tty_stream.transmit_side.write( + (self.password + '\n').encode() + ) password_sent = True raise mitogen.core.StreamError('bootstrap failed') From 5eb41751f5a9ca11421c8d1454ff014d39649da9 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 01:40:49 +0100 Subject: [PATCH 150/212] tests: import missing connection_passthrough --- .../lib/action/connection_passthrough.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 tests/ansible/lib/action/connection_passthrough.py diff --git a/tests/ansible/lib/action/connection_passthrough.py b/tests/ansible/lib/action/connection_passthrough.py new file mode 100644 index 00000000..1e9211e4 --- /dev/null +++ b/tests/ansible/lib/action/connection_passthrough.py @@ -0,0 +1,28 @@ + +import traceback +import sys + +from ansible.plugins.strategy import StrategyBase +from ansible.plugins.action import ActionBase + + +class ActionModule(ActionBase): + def run(self, tmp=None, task_vars=None): + try: + method = getattr(self._connection, self._task.args['method']) + args = tuple(self._task.args.get('args', ())) + kwargs = self._task.args.get('kwargs', {}) + + return { + 'changed': False, + 'failed': False, + 'result': method(*args, **kwargs) + } + except Exception as e: + traceback.print_exc() + return { + 'changed': False, + 'failed': True, + 'msg': str(e), + 'result': e, + } From 65f03e03f501339ebf45136c7b0cd91d27c8d2e5 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 02:17:01 +0100 Subject: [PATCH 151/212] tests: remote_tmp test fixes. --- docs/changelog.rst | 3 +- .../integration/action/make_tmp_path.yml | 121 +++++++++++++----- .../lib/modules/custom_python_run_script.py | 39 ++++++ 3 files changed, 131 insertions(+), 32 deletions(-) create mode 100644 tests/ansible/lib/modules/custom_python_run_script.py diff --git a/docs/changelog.rst b/docs/changelog.rst index c88d2436..9d99d414 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -194,7 +194,8 @@ the bug reports in this release contributed by `Pierre-Henry Muller `_, `Pierre-Louis Bonicoli `_, `Prateek Jain `_, -`Rick Box `_, and +`Rick Box `_, +`Tawana Musewe `_, and `Timo Beckers `_. diff --git a/tests/ansible/integration/action/make_tmp_path.yml b/tests/ansible/integration/action/make_tmp_path.yml index eb39068b..668bd83f 100644 --- a/tests/ansible/integration/action/make_tmp_path.yml +++ b/tests/ansible/integration/action/make_tmp_path.yml @@ -23,27 +23,88 @@ method: _make_tmp_path register: tmp_path - - name: "Write some junk in regular temp path" - shell: hostname > {{tmp_path.result}}/hostname + - name: "Find regular temp path (new task)" + action_passthrough: + method: _make_tmp_path + register: tmp_path2 - - name: "Verify junk did not persist across tasks" - stat: path={{tmp_path.result}}/hostname - register: junk_stat + - name: "Find parent temp path" + set_fact: + parent_temp_path: "{{tmp_path.result|dirname}}" - - name: "Verify junk did not persist across tasks" + - name: "Find parent temp path (new task)" + set_fact: + parent_temp_path2: "{{tmp_path2.result|dirname}}" + + - name: "Verify common base path for both tasks" assert: that: - - not junk_stat.stat.exists + - parent_temp_path == parent_temp_path2 - - name: "Verify temp path hasn't changed since start" - action_passthrough: - method: _make_tmp_path - register: tmp_path2 + - name: "Verify different subdir for both tasks" + assert: + that: + - tmp_path.result != tmp_path2.result - - name: "Verify temp path hasn't changed since start" + # + # Verify subdirectory removal. + # + + - name: Stat temp path + stat: + path: "{{tmp_path.result}}" + register: stat1 + + - name: Stat temp path (new task) + stat: + path: "{{tmp_path2.result}}" + register: stat2 + + - name: "Verify neither subdir exists any more" + assert: + that: + - not stat1.stat.exists + - not stat2.stat.exists + + # + # Verify parent directory persistence. + # + + - name: Stat parent temp path (new task) + stat: + path: "{{parent_temp_path}}" + register: stat + + - name: "Verify parent temp path is persistent" assert: that: - - tmp_path2.result == tmp_path.result + - stat.stat.exists + + # + # Write some junk into the temp path. + # + + - name: "Write junk to temp path and verify it disappears" + custom_python_run_script: + script: | + from ansible.module_utils.basic import get_module_path + path = get_module_path() + '/foo.txt' + result['path'] = path + open(path, 'w').write("bar") + register: out + + - name: "Verify junk disappeared." + stat: + path: "{{out.path}}" + register: out + + - assert: + that: + - not out.stat.exists + + # + # + # - name: "Verify temp path changes across connection reset" mitogen_shutdown_all: @@ -53,13 +114,17 @@ method: _make_tmp_path register: tmp_path2 + - name: "Verify temp path changes across connection reset" + set_fact: + parent_temp_path2: "{{tmp_path2.result|dirname}}" + - name: "Verify temp path changes across connection reset" assert: that: - - tmp_path2.result != tmp_path.result + - parent_temp_path != parent_temp_path2 - name: "Verify old path disappears across connection reset" - stat: path={{tmp_path.result}} + stat: path={{parent_temp_path}} register: junk_stat - name: "Verify old path disappears across connection reset" @@ -89,24 +154,18 @@ - name: "Try writing to temp directory for the readonly_homedir user" become: true become_user: mitogen__readonly_homedir - action_passthrough: - method: _make_tmp_path + custom_python_run_script: + script: | + from ansible.module_utils.basic import get_module_path + path = get_module_path() + '/foo.txt' + result['path'] = path + open(path, 'w').write("bar") register: tmp_path - - name: "Try writing to temp directory for the readonly_homedir user" - become: true - become_user: mitogen__readonly_homedir - shell: hostname > {{tmp_path.result}}/hostname - # - # modules get the same temp dir + # modules get the same base dir # - - name: "Verify modules get the same tmpdir as the action plugin" - action_passthrough: - method: _make_tmp_path - register: tmp_path - - name: "Verify modules get the same tmpdir as the action plugin" custom_python_detect_environment: register: out @@ -116,12 +175,12 @@ when: ansible_version.full < '2.5' assert: that: - - out.module_path == tmp_path.result + - out.module_path.startswith( == tmp_path.result - out.module_tmpdir == None - name: "Verify modules get the same tmpdir as the action plugin (>2.5)" when: ansible_version.full > '2.5' assert: that: - - out.module_path == tmp_path.result - - out.module_tmpdir == tmp_path.result + - out.module_path.startswith(parent_temp_path2) + - out.module_tmpdir.startswith(parent_temp_path2) diff --git a/tests/ansible/lib/modules/custom_python_run_script.py b/tests/ansible/lib/modules/custom_python_run_script.py new file mode 100644 index 00000000..2313291b --- /dev/null +++ b/tests/ansible/lib/modules/custom_python_run_script.py @@ -0,0 +1,39 @@ +#!/usr/bin/python +# I am an Ansible new-style Python module. I run the script provided in the +# parameter. + +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.basic import get_module_path +from ansible.module_utils import six + +import os +import pwd +import socket +import sys + + +def execute(s, gbls, lcls): + if sys.version_info > (3,): + exec(s, gbls, lcls) + else: + exec('exec s in gbls, lcls') + + +def main(): + module = AnsibleModule(argument_spec={ + 'script': { + 'type': str + } + }) + + lcls = { + 'module': module, + 'result': {} + } + execute(module.params['script'], globals(), lcls) + del lcls['module'] + module.exit_json(**lcls['result']) + + +if __name__ == '__main__': + main() From d5524178bf365813ee3c979e6e3fc416be5f7f43 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 02:27:19 +0100 Subject: [PATCH 152/212] tests: fix bonehead syntax error. --- tests/ansible/integration/action/make_tmp_path.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ansible/integration/action/make_tmp_path.yml b/tests/ansible/integration/action/make_tmp_path.yml index 668bd83f..95f790ab 100644 --- a/tests/ansible/integration/action/make_tmp_path.yml +++ b/tests/ansible/integration/action/make_tmp_path.yml @@ -175,7 +175,7 @@ when: ansible_version.full < '2.5' assert: that: - - out.module_path.startswith( == tmp_path.result + - out.module_path.startswith(tmp_path.result) - out.module_tmpdir == None - name: "Verify modules get the same tmpdir as the action plugin (>2.5)" From 9ff34afafe63c59edcf1a6f991b4682e0d164c5a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 02:28:29 +0100 Subject: [PATCH 153/212] ansible: fix regression. --- ansible_mitogen/mixins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 2ebffb84..faadbc15 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -279,7 +279,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): # ~/.ansible -> /home/dmw/.ansible return os.path.join(self._connection.homedir, path[2:]) # ~root/.ansible -> /root/.ansible - return self._connection.get_chain(login=(not sudoable)).call( + return self._connection.get_chain(use_login=(not sudoable)).call( os.path.expanduser, mitogen.utils.cast(path), ) From 6cb0e422e9e508b93321b8a524e355397912280e Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 02:30:25 +0100 Subject: [PATCH 154/212] docs: changelog typo. --- docs/changelog.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9d99d414..7beb043c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -59,8 +59,8 @@ Enhancements action. * `#337 `_: To avoid a scaling - limitation, Mitogen no longer requires a PTY for every SSH client unless an - SSH password has been specified. + limitation, a PTY is no longer allocated for an SSH connection unless the + configuration specifies a password. * `d62e6e2a `_: many-target runs executed the dependency scanner redundantly due to missing @@ -150,8 +150,8 @@ Core Library was fixed. * `#337 `_: to avoid a scaling - limitation, SSH now longer allocates a PTY for every OpenSSH client if it can - be avoided. PTYs are only allocated if a password is supplied, or when + limitation, a PTY is no longer allocated for each OpenSSH client if it can be + avoided. PTYs are only allocated if a password is supplied, or when `host_key_checking=accept`. This is since Linux has a default of 4096 PTYs (``kernel.pty.max``), while OS X has a default of 127 and an absolute maximum of 999 (``kern.tty.ptmx_max``). From 001b63074c7ccb3c5402030af181f3035b4c7ba7 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 03:18:57 +0100 Subject: [PATCH 155/212] tests: fix another typo. --- tests/ansible/integration/action/make_tmp_path.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ansible/integration/action/make_tmp_path.yml b/tests/ansible/integration/action/make_tmp_path.yml index 95f790ab..97da070d 100644 --- a/tests/ansible/integration/action/make_tmp_path.yml +++ b/tests/ansible/integration/action/make_tmp_path.yml @@ -175,7 +175,7 @@ when: ansible_version.full < '2.5' assert: that: - - out.module_path.startswith(tmp_path.result) + - out.module_path.startswith(parent_temp_path2) - out.module_tmpdir == None - name: "Verify modules get the same tmpdir as the action plugin (>2.5)" From dfc67b89fdb055010c8a05987eb5d2be142e1775 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 18:32:23 +0100 Subject: [PATCH 156/212] docs: some more cleanups - add faulthandler/thread stacks to changelog. - various api.rst cleanups. - docs: explain chain_id in howitworks. --- docs/ansible.rst | 9 ++++++--- docs/api.rst | 26 +++++++++++++------------- docs/changelog.rst | 8 ++++++++ docs/howitworks.rst | 7 +++++++ mitogen/parent.py | 9 +++++---- 5 files changed, 39 insertions(+), 20 deletions(-) diff --git a/docs/ansible.rst b/docs/ansible.rst index 7a06c120..485c24dc 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -313,9 +313,10 @@ Performance ^^^^^^^^^^^ One roundtrip initiates a transfer larger than 124 KiB, while smaller transfers -are embedded in a 0-roundtrip remote call. For tools operating via SSH -multiplexing, 4 roundtrips are required to configure the IO channel, in -addition to the time to start the local and remote processes. +are embedded in a 0-roundtrip pipelined call. For tools operating via SSH +multiplexing, 4 roundtrips are required to configure the IO channel, followed +by 6 roundtrips to transfer the file in the case of ``sftp``, in addition to +the time to start the local and remote processes. An invocation of ``scp`` with an empty ``.profile`` over a 30 ms link takes ~140 ms, wasting 110 ms per invocation, rising to ~2,000 ms over a 400 ms @@ -848,6 +849,8 @@ logging is necessary. File-based logging can be enabled by setting enabled, one file per context will be created on the local machine and every target machine, as ``/tmp/mitogen..log``. +.. _diagnosing-hangs: + Diagnosing Hangs ~~~~~~~~~~~~~~~~ diff --git a/docs/api.rst b/docs/api.rst index cb980b55..c74193e3 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -792,8 +792,8 @@ Context Class .. method:: send (msg) - Arrange for `msg` to be delivered to this context. Updates the - message's `dst_id` prior to routing it via the associated router. + Arrange for `msg` to be delivered to this context. + :attr:`dst_id ` is set to the target context ID. :param mitogen.core.Message msg: The message. @@ -801,9 +801,9 @@ Context Class .. method:: send_async (msg, persist=False) Arrange for `msg` to be delivered to this context, with replies - delivered to a newly constructed Receiver. Updates the message's - `dst_id` prior to routing it via the associated router and registers a - handle which is placed in the message's `reply_to`. + directed to a newly constructed receiver. :attr:`dst_id + ` is set to the target context ID, and :attr:`reply_to + ` is set to the newly constructed receiver's handle. :param bool persist: If :data:`False`, the handler will be unregistered after a single @@ -818,15 +818,15 @@ Context Class .. method:: send_await (msg, deadline=None) - As with :meth:`send_async`, but expect a single reply - (`persist=False`) delivered within `deadline` seconds. + Like :meth:`send_async`, but expect a single reply (`persist=False`) + delivered within `deadline` seconds. :param mitogen.core.Message msg: The message. - :param float deadline: If not :data:`None`, seconds before timing out waiting for a reply. - + :returns: + The deserialized reply. :raises mitogen.core.TimeoutError: No message was received and `deadline` passed. @@ -838,9 +838,9 @@ Context Class .. class:: Context - Extend :class:`mitogen.core.Router` with functionality useful to - masters, and child contexts who later become parents. Currently when this - class is required, the target context's router is upgraded at runtime. + Extend :class:`mitogen.core.Context` with functionality useful to masters, + and child contexts who later become parents. Currently when this class is + required, the target context's router is upgraded at runtime. .. attribute:: default_call_chain @@ -1196,7 +1196,7 @@ Broker Class Responsible for handling I/O multiplexing in a private thread. **Note:** This is the somewhat limited core version of the Broker class - used by child contexts. The master subclass is documented below this one. + used by child contexts. The master subclass is documented below. .. attribute:: shutdown_timeout = 3.0 diff --git a/docs/changelog.rst b/docs/changelog.rst index 7beb043c..f4c46976 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -74,6 +74,14 @@ Enhancements improvement, but is easily noticeable on short many-target runs, where startup overhead dominates runtime. +* The `faulthandler `_ module is + automatically activated if it is installed, simplifying debugging of hangs. + See :ref:`diagnosing-hangs` for more information. + +* The ``MITOGEN_DUMP_THREAD_STACKS`` environment variable's value now indicates + the number of seconds between stack dumps. See :ref:`diagnosing-hangs` for + more information. + Fixes ^^^^^ diff --git a/docs/howitworks.rst b/docs/howitworks.rst index 1e3d2768..a3b08eac 100644 --- a/docs/howitworks.rst +++ b/docs/howitworks.rst @@ -377,6 +377,13 @@ Children listen on the following handles: 6-tuples from :class:`mitogen.parent.CallChain`, imports ``mod_name``, then attempts to execute `class_name.func_name(\*args, \**kwargs)`. + * `chain_id`: if not :data:`None`, an identifier unique to the originating + :class:`mitogen.parent.CallChain`. When set, if an exception occurs + during a call, future calls with the same ID automatically fail with the + same exception without ever executing, and failed calls with no + `reply_to` set are not dumped to the logging framework as they otherwise + would. This is used to implement pipelining. + When this channel is closed (by way of receiving a dead message), the child's main thread begins graceful shutdown of its own :py:class:`Broker` and :py:class:`Router`. diff --git a/mitogen/parent.py b/mitogen/parent.py index 8e9f53a5..bb2b5d1e 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1149,14 +1149,15 @@ class CallChain(object): calls execute until :meth:`reset` is invoked. No exception is logged for calls made with :meth:`call_no_reply`, instead - it is saved and reported as the result of subsequent :meth:`call` or - :meth:`call_async` calls. + the exception is saved and reported as the result of subsequent + :meth:`call` or :meth:`call_async` calls. Sequences of asynchronous calls can be made without wasting network round-trips to discover if prior calls succeed, and chains originating from multiple unrelated source contexts may overlap concurrently at a target - context without interference. In this example, 4 calls complete in one - round-trip:: + context without interference. + + In this example, 4 calls complete in one round-trip:: chain = mitogen.parent.CallChain(context, pipelined=True) chain.call_no_reply(os.mkdir, '/tmp/foo') From 530fd18e4c9aa2f39f6e9ddf013f423ca13fa5c9 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 18:59:29 +0100 Subject: [PATCH 157/212] service: wake FileService client on file open failure. Previously client would hang and excption woud be dumped to logger. --- mitogen/service.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mitogen/service.py b/mitogen/service.py index 3713bdea..ffb7308e 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -873,7 +873,14 @@ class FileService(Service): raise Error(self.context_mismatch_msg) LOG.debug('Serving %r', path) - fp = open(path, 'rb', self.IO_SIZE) + try: + fp = open(path, 'rb', self.IO_SIZE) + except IOError: + msg.reply(mitogen.core.CallError( + sys.exc_info()[1] + )) + return + # Response must arrive first so requestee can begin receive loop, # otherwise first ack won't arrive until all pending chunks were # delivered. In that case max BDP would always be 128KiB, aka. max From 638b196a4538926c125b9258db68c537a3098ee3 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 19:05:27 +0100 Subject: [PATCH 158/212] ansible: fix put_file() for large temporary files. Reverts 49736b3a, large file copies can't avoid the RTT. The parent stack must be blocked while FileService progresses, as unlike the small file path, it does not make a snapshot of the (possibly temporary) file passed by the action plug-in. So we need to keep that file alive while the service runs. Add a new integration test and a new soak test to cover both. --- ansible_mitogen/connection.py | 12 ++--- docs/changelog.rst | 5 -- tests/ansible/integration/action/all.yml | 1 + tests/ansible/integration/action/copy.yml | 66 +++++++++++++++++++++++ tests/ansible/soak/_file_service_loop.yml | 6 +++ tests/ansible/soak/file_service.yml | 6 +++ 6 files changed, 84 insertions(+), 12 deletions(-) create mode 100644 tests/ansible/integration/action/copy.yml create mode 100644 tests/ansible/soak/_file_service_loop.yml create mode 100644 tests/ansible/soak/file_service.yml diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index c5c96f24..12626485 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -880,13 +880,11 @@ class Connection(ansible.plugins.connection.ConnectionBase): path=mitogen.utils.cast(in_path) ) - # A roundtrip is always necessary for the target to request the file - # from FileService, however, by pipelining the transfer function, the - # subsequent step (probably a module invocation) can get its - # dependencies and function call in-flight before the transfer is - # complete. This saves at least 1 RTT between the transfer completing - # and the start of the follow-up task. - self.get_chain().call_no_reply( + # For now this must remain synchronous, as the action plug-in may have + # passed us a temporary file to transfer. A future FileService could + # maintain an LRU list of open file descriptors to keep the temporary + # file alive, but that requires more work. + self.get_chain().call( ansible_mitogen.target.transfer_file, context=self.parent, in_path=in_path, diff --git a/docs/changelog.rst b/docs/changelog.rst index f4c46976..162a7c48 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -53,11 +53,6 @@ Enhancements a 250 ms link from 30 seconds to 10 seconds compared to v0.2.2, down from 120 seconds compared to vanilla. -* `49736b3a `_: avoid a - roundtrip when transferring files larger than 124KiB, removing a delay - between waiting for the transfer to complete and start of the follow-up - action. - * `#337 `_: To avoid a scaling limitation, a PTY is no longer allocated for an SSH connection unless the configuration specifies a password. diff --git a/tests/ansible/integration/action/all.yml b/tests/ansible/integration/action/all.yml index c5cb80d7..75f77243 100644 --- a/tests/ansible/integration/action/all.yml +++ b/tests/ansible/integration/action/all.yml @@ -1,3 +1,4 @@ +- import_playbook: copy.yml - import_playbook: fixup_perms2__copy.yml - import_playbook: low_level_execute_command.yml - import_playbook: make_tmp_path.yml diff --git a/tests/ansible/integration/action/copy.yml b/tests/ansible/integration/action/copy.yml new file mode 100644 index 00000000..e3fca87f --- /dev/null +++ b/tests/ansible/integration/action/copy.yml @@ -0,0 +1,66 @@ +# Verify copy module for small and large files, and inline content. + +- name: integration/action/synchronize.yml + hosts: test-targets + any_errors_fatal: true + tasks: + - copy: + dest: /tmp/copy-tiny-file + content: + this is a tiny file. + connection: local + + - copy: + dest: /tmp/copy-large-file + # Must be larger than Connection.SMALL_SIZE_LIMIT. + content: "{% for x in range(200000) %}x{% endfor %}" + connection: local + + # end of making files + + - file: + state: absent + path: "{{item}}" + with_items: + - /tmp/copy-tiny-file.out + - /tmp/copy-large-file.out + - /tmp/copy-tiny-inline-file.out + - /tmp/copy-large-inline-file.out + + # end of cleaning out files + + - copy: + dest: /tmp/copy-large-file.out + src: /tmp/copy-large-file + + - copy: + dest: /tmp/copy-tiny-file.out + src: /tmp/copy-tiny-file + + - copy: + dest: /tmp/copy-tiny-inline-file.out + content: "tiny inline content" + + - copy: + dest: /tmp/copy-large-inline-file.out + content: | + {% for x in range(200000) %}y{% endfor %} + + # stat results + + - stat: + path: "{{item}}" + with_items: + - /tmp/copy-tiny-file.out + - /tmp/copy-large-file.out + - /tmp/copy-tiny-inline-file.out + - /tmp/copy-large-inline-file.out + register: stat + + - assert: + that: + - stat.results[0].stat.checksum == "f29faa9a6f19a700a941bf2aa5b281643c4ec8a0" + - stat.results[1].stat.checksum == "62951f943c41cdd326e5ce2b53a779e7916a820d" + - stat.results[2].stat.checksum == "b26dd6444595e2bdb342aa0a91721b57478b5029" + - stat.results[3].stat.checksum == "d675f47e467eae19e49032a2cc39118e12a6ee72" + diff --git a/tests/ansible/soak/_file_service_loop.yml b/tests/ansible/soak/_file_service_loop.yml new file mode 100644 index 00000000..96111b3c --- /dev/null +++ b/tests/ansible/soak/_file_service_loop.yml @@ -0,0 +1,6 @@ + - file: + path: /tmp/foo-{{inventory_hostname}} + state: absent + - copy: + dest: /tmp/foo-{{inventory_hostname}} + content: "{{content}}" diff --git a/tests/ansible/soak/file_service.yml b/tests/ansible/soak/file_service.yml new file mode 100644 index 00000000..3b338b3c --- /dev/null +++ b/tests/ansible/soak/file_service.yml @@ -0,0 +1,6 @@ +- hosts: all + tasks: + - set_fact: + content: "{% for x in range(126977) %}x{% endfor %}" + - include_tasks: _file_service_loop.yml + with_sequence: start=1 end=100 From d1c84552ecf306cc6901f967104cbcd22985a107 Mon Sep 17 00:00:00 2001 From: Brian Candler Date: Mon, 17 Sep 2018 21:45:40 +0100 Subject: [PATCH 159/212] Use `lxc exec --mode=noninteractive` which is more widely compatible Closes #371 --- mitogen/lxd.py | 2 +- tests/lxd_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mitogen/lxd.py b/mitogen/lxd.py index 6e8e8b18..9e6702f4 100644 --- a/mitogen/lxd.py +++ b/mitogen/lxd.py @@ -63,7 +63,7 @@ class Stream(mitogen.parent.Stream): bits = [ self.lxc_path, 'exec', - '--force-noninteractive', + '--mode=noninteractive', self.container, '--', ] diff --git a/tests/lxd_test.py b/tests/lxd_test.py index c5e4c485..9c2397a2 100644 --- a/tests/lxd_test.py +++ b/tests/lxd_test.py @@ -18,7 +18,7 @@ class FakeLxcTest(testlib.RouterMixin, unittest2.TestCase): argv = eval(context.call(os.getenv, 'ORIGINAL_ARGV')) self.assertEquals(argv[0], lxc_path) self.assertEquals(argv[1], 'exec') - self.assertEquals(argv[2], '--force-noninteractive') + self.assertEquals(argv[2], '--mode=noninteractive') self.assertEquals(argv[3], 'container_name') From 6828926a36985bef17c7788a3a4df0b242449437 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannig=20Perr=C3=A9?= Date: Wed, 19 Sep 2018 16:52:20 +0200 Subject: [PATCH 160/212] Kubernetes connection support for mitogen. --- ansible_mitogen/connection.py | 21 +++++ .../plugins/connection/mitogen_kubectl.py | 45 +++++++++ ansible_mitogen/strategy.py | 2 +- mitogen/core.py | 1 + mitogen/kubectl.py | 78 ++++++++++++++++ mitogen/parent.py | 3 + tests/ansible/test-kubectl.yml | 91 +++++++++++++++++++ 7 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 ansible_mitogen/plugins/connection/mitogen_kubectl.py create mode 100644 mitogen/kubectl.py create mode 100644 tests/ansible/test-kubectl.yml diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 12626485..60724fa3 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -125,6 +125,22 @@ def _connect_docker(spec): } +def _connect_kubectl(spec): + """ + Return ContextService arguments for a Kubernetes connection. + """ + return { + 'method': 'kubectl', + 'kwargs': { + 'username': spec['remote_user'], + 'pod': spec['remote_addr'], + #'container': spec['container'], + 'python_path': spec['python_path'], + 'connect_timeout': spec['ansible_ssh_timeout'] or spec['timeout'], + } + } + + def _connect_jail(spec): """ Return ContextService arguments for a FreeBSD jail connection. @@ -187,6 +203,7 @@ def _connect_setns(spec): 'python_path': spec['python_path'], 'kind': spec['mitogen_kind'], 'docker_path': spec['mitogen_docker_path'], + 'kubectl_path': spec['mitogen_kubectl_path'], 'lxc_info_path': spec['mitogen_lxc_info_path'], 'machinectl_path': spec['mitogen_machinectl_path'], } @@ -299,6 +316,7 @@ def _connect_mitogen_doas(spec): #: specification. CONNECTION_METHOD = { 'docker': _connect_docker, + 'kubectl': _connect_kubectl, 'jail': _connect_jail, 'local': _connect_local, 'lxc': _connect_lxc, @@ -366,6 +384,8 @@ def config_from_play_context(transport, inventory_name, connection): connection.get_task_var('mitogen_kind'), 'mitogen_docker_path': connection.get_task_var('mitogen_docker_path'), + 'mitogen_kubectl_path': + connection.get_task_var('mitogen_kubectl_path'), 'mitogen_lxc_info_path': connection.get_task_var('mitogen_lxc_info_path'), 'mitogen_machinectl_path': @@ -398,6 +418,7 @@ def config_from_hostvars(transport, inventory_name, connection, 'mitogen_via': hostvars.get('mitogen_via'), 'mitogen_kind': hostvars.get('mitogen_kind'), 'mitogen_docker_path': hostvars.get('mitogen_docker_path'), + 'mitogen_kubectl_path': hostvars.get('mitogen_kubectl_path'), 'mitogen_lxc_info_path': hostvars.get('mitogen_lxc_info_path'), 'mitogen_machinectl_path': hostvars.get('mitogen_machinctl_path'), }) diff --git a/ansible_mitogen/plugins/connection/mitogen_kubectl.py b/ansible_mitogen/plugins/connection/mitogen_kubectl.py new file mode 100644 index 00000000..43d162f8 --- /dev/null +++ b/ansible_mitogen/plugins/connection/mitogen_kubectl.py @@ -0,0 +1,45 @@ +# coding: utf-8 +# Copyright 2018, Yannig Perré +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import os.path +import sys + +try: + import ansible_mitogen +except ImportError: + base_dir = os.path.dirname(__file__) + sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) + del base_dir + +import ansible_mitogen.connection + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'kubectl' diff --git a/ansible_mitogen/strategy.py b/ansible_mitogen/strategy.py index fbe23ef7..e105984c 100644 --- a/ansible_mitogen/strategy.py +++ b/ansible_mitogen/strategy.py @@ -59,7 +59,7 @@ def wrap_connection_loader__get(name, *args, **kwargs): While the strategy is active, rewrite connection_loader.get() calls for some transports into requests for a compatible Mitogen transport. """ - if name in ('docker', 'jail', 'local', 'lxc', + if name in ('docker', 'kubectl', 'jail', 'local', 'lxc', 'lxd', 'machinectl', 'setns', 'ssh'): name = 'mitogen_' + name return connection_loader__get(name, *args, **kwargs) diff --git a/mitogen/core.py b/mitogen/core.py index a6ee1896..d829d624 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -717,6 +717,7 @@ class Importer(object): 'debug', 'doas', 'docker', + 'kubectl', 'fakessh', 'fork', 'jail', diff --git a/mitogen/kubectl.py b/mitogen/kubectl.py new file mode 100644 index 00000000..2dfaa232 --- /dev/null +++ b/mitogen/kubectl.py @@ -0,0 +1,78 @@ +# coding: utf-8 +# Copyright 2018, Yannig Perré +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +import logging + +import mitogen.core +import mitogen.parent + + +LOG = logging.getLogger(__name__) + +class Stream(mitogen.parent.Stream): + child_is_immediate_subprocess = True + + pod = None + container = None + username = None + kubectl_path = 'kubectl' + + # TODO: better way of capturing errors such as "No such container." + create_child_args = { + 'merge_stdio': True + } + + def construct(self, pod = None, container=None, + kubectl_path=None, username=None, + **kwargs): + assert pod + super(Stream, self).construct(**kwargs) + if pod: + self.pod = pod + if container: + self.container = container + if kubectl_path: + self.kubectl_path = kubectl_path + if username: + self.username = username + + def connect(self): + super(Stream, self).connect() + self.name = u'kubectl.' + (self.pod) + str(self.container) + + def get_boot_command(self): + args = ['exec', '-it', self.pod] + if self.username: + args += ['--username=' + self.username] + + if self.container: + args += ['--container=' + self.container] + bits = [self.kubectl_path] + + return bits + args + [ "--" ] + super(Stream, self).get_boot_command() diff --git a/mitogen/parent.py b/mitogen/parent.py index bb2b5d1e..fe5e6889 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1596,6 +1596,9 @@ class Router(mitogen.core.Router): def docker(self, **kwargs): return self.connect(u'docker', **kwargs) + def kubectl(self, **kwargs): + return self.connect(u'kubectl', **kwargs) + def fork(self, **kwargs): return self.connect(u'fork', **kwargs) diff --git a/tests/ansible/test-kubectl.yml b/tests/ansible/test-kubectl.yml new file mode 100644 index 00000000..05fc6517 --- /dev/null +++ b/tests/ansible/test-kubectl.yml @@ -0,0 +1,91 @@ +--- + +- name: "Create pod" + tags: always + hosts: localhost + gather_facts: no + tasks: + - name: Create a test pod + k8s: + state: present + definition: + apiVersion: v1 + kind: Pod + metadata: + name: test-pod-{{item}} + namespace: default + spec: + containers: + - name: python2 + image: python:2 + args: [ "sleep", "100000" ] + loop: "{{ range(10)|list }}" + + - name: "Wait pod to be running" + debug: { msg: "pod is running" } + # status and availableReplicas might not be there. Using default value (d(default_value)) + until: "pod_def.status.containerStatuses[0].ready" + # Waiting 100 s + retries: 50 + delay: 2 + vars: + pod_def: "{{lookup('k8s', kind='Pod', namespace='default', resource_name='test-pod-' ~ item)}}" + loop: "{{ range(10)|list }}" + + - name: "Add pod to pods group" + add_host: + name: "test-pod-{{item}}" + groups: [ "pods" ] + ansible_connection: "kubectl" + changed_when: no + tags: "always" + loop: "{{ range(10)|list }}" + +- name: "Test kubectl connection (default strategy)" + tags: default + hosts: pods + strategy: "linear" + gather_facts: no + tasks: + - name: "Simple shell with linear" + shell: ls /tmp + loop: [ 1, 2, 3, 4, 5 ] + + - name: "Simple file with linear" + file: + path: "/etc" + state: directory + loop: [ 1, 2, 3, 4, 5 ] + +- name: "Test kubectl connection (mitogen strategy)" + tags: mitogen + hosts: pods + strategy: "mitogen_linear" + gather_facts: no + tasks: + - name: "Simple shell with mitogen" + shell: ls /tmp + loop: [ 1, 2, 3, 4, 5 ] + + - name: "Simple file with mitogen" + file: + path: "/etc" + state: directory + loop: [ 1, 2, 3, 4, 5 ] + register: _ + +- name: "Destroy pod" + tags: cleanup + hosts: localhost + gather_facts: no + tasks: + - name: Destroy pod + k8s: + state: absent + definition: + apiVersion: v1 + kind: Pod + metadata: + name: test-pod-{{item}} + namespace: default + loop: "{{ range(10)|list }}" From 2c2878012d4aad55e2d7fdc0c5aadb61657d9e5e Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 30 Sep 2018 21:23:10 +0100 Subject: [PATCH 161/212] Match "user@host: Permission denied ..." messages OpenSSH 7.5 changed the text of the permission denied message. As a result ssh_test.SshTest.test_password_required and test_pubkey_required were failing on an Ubuntu 18.04 client, which ships OpenSSH 7.6. Refs - https://bugzilla.mindrot.org/show_bug.cgi?id=2720 --- mitogen/ssh.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mitogen/ssh.py b/mitogen/ssh.py index 2e723d67..f8255865 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -289,7 +289,11 @@ class Stream(mitogen.parent.Stream): self._host_key_prompt() elif HOSTKEY_FAIL in buf.lower(): raise HostKeyError(self.hostkey_failed_msg) - elif buf.lower().startswith(PERMDENIED_PROMPT): + elif buf.lower().startswith(( + PERMDENIED_PROMPT, + b("%s@%s: %s" % (self.username, self.hostname, + PERMDENIED_PROMPT)), + )): # issue #271: work around conflict with user shell reporting # 'permission denied' e.g. during chdir($HOME) by only matching # it at the start of the line. From b3d8c947c7798d29e0c03c8ef99002d57d0580bc Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 30 Sep 2018 21:26:34 +0100 Subject: [PATCH 162/212] Ignore built doc artifacts --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 458cf82c..6092d04e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ MANIFEST build/ dist/ +docs/_build/ htmlcov/ *.egg-info __pycache__/ From 9fbcb67665231f8e7d2cd375f4bec70ab124b4b4 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 30 Sep 2018 21:32:45 +0100 Subject: [PATCH 163/212] Update pytz to 2018.05 (needed by babel 2.6.0) On Ubuntu 18.04 (others not tested) installing the dev requirements fails with the following error babel 2.6.0 has requirement pytz>=0a, but you'll have pytz 2012d which is incompatible. Despite the comment in dev_requirements.txt pytz-2012d is not the most recent version to support Python 2.6. In fact the latest release of pytz supports Python 2.6. --- dev_requirements.txt | 2 +- tests/module_finder_test.py | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index f093721b..68f0422a 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -3,7 +3,7 @@ ansible==2.6.1 coverage==4.5.1 Django==1.6.11 # Last version supporting 2.6. mock==2.0.0 -pytz==2012d # Last 2.6-compat version. +pytz==2018.5 paramiko==2.3.2 # Last 2.6-compat version. pytest-catchlog==1.2.2 pytest==3.1.2 diff --git a/tests/module_finder_test.py b/tests/module_finder_test.py index 9c85e26c..1d5a0796 100644 --- a/tests/module_finder_test.py +++ b/tests/module_finder_test.py @@ -353,13 +353,9 @@ class DjangoFindRelatedTest(DjangoMixin, testlib.TestCase): 'django.utils.translation', 'django.utils.tree', 'django.utils.tzinfo', - 'pkg_resources', - 'pkg_resources.extern', - 'pkg_resources.extern.appdirs', - 'pkg_resources.extern.packaging', - 'pkg_resources.extern.six', 'pytz', 'pytz.exceptions', + 'pytz.lazy', 'pytz.tzfile', 'pytz.tzinfo', ]) From 03be0afeebbc163cd915f36e70e9ea970fe5f8e4 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Sun, 30 Sep 2018 21:37:15 +0100 Subject: [PATCH 164/212] tests: Add tests of mitogen.utils.cast() --- tests/utils_test.py | 60 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/utils_test.py b/tests/utils_test.py index 4c2e2e0f..b2e0aa9e 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -2,6 +2,7 @@ import unittest2 +import mitogen.core import mitogen.master import mitogen.utils @@ -32,5 +33,64 @@ class WithRouterTest(unittest2.TestCase): self.assertFalse(router.broker._thread.isAlive()) +class Dict(dict): pass +class List(list): pass +class Tuple(tuple): pass +class Unicode(mitogen.core.UnicodeType): pass +class Bytes(mitogen.core.BytesType): pass + + +class CastTest(unittest2.TestCase): + def test_dict(self): + self.assertEqual(type(mitogen.utils.cast({})), dict) + self.assertEqual(type(mitogen.utils.cast(Dict())), dict) + + def test_nested_dict(self): + specimen = mitogen.utils.cast(Dict({'k': Dict({'k2': 'v2'})})) + self.assertEqual(type(specimen), dict) + self.assertEqual(type(specimen['k']), dict) + + def test_list(self): + self.assertEqual(type(mitogen.utils.cast([])), list) + self.assertEqual(type(mitogen.utils.cast(List())), list) + + def test_nested_list(self): + specimen = mitogen.utils.cast(List((0, 1, List((None,))))) + self.assertEqual(type(specimen), list) + self.assertEqual(type(specimen[2]), list) + + def test_tuple(self): + self.assertEqual(type(mitogen.utils.cast(())), list) + self.assertEqual(type(mitogen.utils.cast(Tuple())), list) + + def test_nested_tuple(self): + specimen = mitogen.utils.cast(Tuple((0, 1, Tuple((None,))))) + self.assertEqual(type(specimen), list) + self.assertEqual(type(specimen[2]), list) + + def assertUnchanged(self, v): + self.assertIs(mitogen.utils.cast(v), v) + + def test_passthrough(self): + self.assertUnchanged(0) + self.assertUnchanged(0.0) + self.assertUnchanged(float('inf')) + self.assertUnchanged(True) + self.assertUnchanged(False) + self.assertUnchanged(None) + + def test_unicode(self): + self.assertEqual(type(mitogen.utils.cast(u'')), mitogen.core.UnicodeType) + self.assertEqual(type(mitogen.utils.cast(Unicode())), mitogen.core.UnicodeType) + + def test_bytes(self): + self.assertEqual(type(mitogen.utils.cast(b'')), mitogen.core.BytesType) + self.assertEqual(type(mitogen.utils.cast(Bytes())), mitogen.core.BytesType) + + def test_unknown(self): + self.assertRaises(TypeError, mitogen.utils.cast, set()) + self.assertRaises(TypeError, mitogen.utils.cast, 4j) + + if __name__ == '__main__': unittest2.main() From 17548d1e49ab548379d4579c8db6ddec823aba33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannig=20Perr=C3=A9?= Date: Thu, 20 Sep 2018 21:20:39 +0200 Subject: [PATCH 165/212] [Enhancement] handle kubectl vars from Ansible connector. This change allows the kubectl connector to support the same options as Ansible's original connector. The playbook sample comes with an example of a pod containing two containers and checking that moving from one container to another, the version of Python changes as expected. --- ansible_mitogen/connection.py | 8 +- .../plugins/connection/mitogen_kubectl.py | 11 +++ mitogen/kubectl.py | 34 +++----- tests/ansible/test-kubectl.yml | 79 ++++++++++++++++--- 4 files changed, 97 insertions(+), 35 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 60724fa3..ce36bc53 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -132,11 +132,10 @@ def _connect_kubectl(spec): return { 'method': 'kubectl', 'kwargs': { - 'username': spec['remote_user'], 'pod': spec['remote_addr'], - #'container': spec['container'], 'python_path': spec['python_path'], 'connect_timeout': spec['ansible_ssh_timeout'] or spec['timeout'], + 'additional_parameters': spec['additional_parameters'], } } @@ -392,6 +391,8 @@ def config_from_play_context(transport, inventory_name, connection): connection.get_task_var('mitogen_machinectl_path'), 'mitogen_ssh_debug_level': connection.get_task_var('mitogen_ssh_debug_level'), + 'additional_parameters': + connection.get_additional_parameters(), } @@ -571,6 +572,9 @@ class Connection(ansible.plugins.connection.ConnectionBase): def connected(self): return self.context is not None + def get_additional_parameters(self): + return [] + def _config_from_via(self, via_spec): """ Produce a dict connection specifiction given a string `via_spec`, of diff --git a/ansible_mitogen/plugins/connection/mitogen_kubectl.py b/ansible_mitogen/plugins/connection/mitogen_kubectl.py index 43d162f8..acbe9a39 100644 --- a/ansible_mitogen/plugins/connection/mitogen_kubectl.py +++ b/ansible_mitogen/plugins/connection/mitogen_kubectl.py @@ -31,6 +31,8 @@ from __future__ import absolute_import import os.path import sys +iteritems = getattr(dict, 'iteritems', dict.items) + try: import ansible_mitogen except ImportError: @@ -40,6 +42,15 @@ except ImportError: import ansible_mitogen.connection +import ansible.plugins.connection.kubectl class Connection(ansible_mitogen.connection.Connection): transport = 'kubectl' + + def get_additional_parameters(self): + parameters = [] + for key, option in iteritems(ansible.plugins.connection.kubectl.CONNECTION_OPTIONS): + if self.get_task_var('ansible_' + key) is not None: + parameters += [ option, self.get_task_var('ansible_' + key) ] + + return parameters diff --git a/mitogen/kubectl.py b/mitogen/kubectl.py index 2dfaa232..d9a70927 100644 --- a/mitogen/kubectl.py +++ b/mitogen/kubectl.py @@ -32,6 +32,9 @@ import logging import mitogen.core import mitogen.parent +import ansible.plugins.connection.kubectl + +from ansible.module_utils.six import iteritems LOG = logging.getLogger(__name__) @@ -42,37 +45,26 @@ class Stream(mitogen.parent.Stream): container = None username = None kubectl_path = 'kubectl' + kubectl_options = [] # TODO: better way of capturing errors such as "No such container." create_child_args = { 'merge_stdio': True } - def construct(self, pod = None, container=None, - kubectl_path=None, username=None, - **kwargs): - assert pod + def construct(self, pod = None, kubectl_path=None, + additional_parameters=None, **kwargs): + assert(pod) super(Stream, self).construct(**kwargs) - if pod: - self.pod = pod - if container: - self.container = container - if kubectl_path: - self.kubectl_path = kubectl_path - if username: - self.username = username + + self.kubectl_options = additional_parameters + self.pod = pod def connect(self): super(Stream, self).connect() - self.name = u'kubectl.' + (self.pod) + str(self.container) + self.name = u'kubectl.' + (self.pod) + str(self.container) + str(self.kubectl_options) def get_boot_command(self): - args = ['exec', '-it', self.pod] - if self.username: - args += ['--username=' + self.username] - - if self.container: - args += ['--container=' + self.container] - bits = [self.kubectl_path] + bits = [self.kubectl_path] + self.kubectl_options + ['exec', '-it', self.pod] - return bits + args + [ "--" ] + super(Stream, self).get_boot_command() + return bits + [ "--" ] + super(Stream, self).get_boot_command() diff --git a/tests/ansible/test-kubectl.yml b/tests/ansible/test-kubectl.yml index 05fc6517..d2be9ba5 100644 --- a/tests/ansible/test-kubectl.yml +++ b/tests/ansible/test-kubectl.yml @@ -1,8 +1,11 @@ --- - name: "Create pod" - tags: always + tags: create hosts: localhost + vars: + pod_count: 10 + loop_count: 5 gather_facts: no tasks: - name: Create a test pod @@ -19,7 +22,10 @@ - name: python2 image: python:2 args: [ "sleep", "100000" ] - loop: "{{ range(10)|list }}" + - name: python3 + image: python:3 + args: [ "sleep", "100000" ] + loop: "{{ range(pod_count|int)|list }}" - name: "Wait pod to be running" debug: { msg: "pod is running" } @@ -30,7 +36,7 @@ delay: 2 vars: pod_def: "{{lookup('k8s', kind='Pod', namespace='default', resource_name='test-pod-' ~ item)}}" - loop: "{{ range(10)|list }}" + loop: "{{ range(pod_count|int)|list }}" - name: "Add pod to pods group" add_host: @@ -39,45 +45,95 @@ ansible_connection: "kubectl" changed_when: no tags: "always" - loop: "{{ range(10)|list }}" + loop: "{{ range(pod_count|int)|list }}" - name: "Test kubectl connection (default strategy)" tags: default hosts: pods strategy: "linear" + vars: + pod_count: 10 + loop_count: 5 gather_facts: no tasks: - name: "Simple shell with linear" shell: ls /tmp - loop: [ 1, 2, 3, 4, 5 ] + loop: "{{ range(loop_count|int)|list }}" - name: "Simple file with linear" file: path: "/etc" state: directory - loop: [ 1, 2, 3, 4, 5 ] + loop: "{{ range(loop_count|int)|list }}" + + - block: + - name: "Check python version on python3 container" + command: python --version + vars: + ansible_kubectl_container: python3 + register: _ + + - assert: { that: "'Python 3' in _.stdout" } + + - debug: var=_.stdout,_.stderr + run_once: yes + + - name: "Check python version on default container" + command: python --version + register: _ + + - assert: { that: "'Python 2' in _.stderr" } + + - debug: var=_.stdout,_.stderr + run_once: yes - name: "Test kubectl connection (mitogen strategy)" tags: mitogen hosts: pods strategy: "mitogen_linear" + vars: + pod_count: 10 + loop_count: 5 gather_facts: no tasks: - name: "Simple shell with mitogen" shell: ls /tmp - loop: [ 1, 2, 3, 4, 5 ] + loop: "{{ range(loop_count|int)|list }}" - name: "Simple file with mitogen" file: path: "/etc" state: directory - loop: [ 1, 2, 3, 4, 5 ] - register: _ + loop: "{{ range(loop_count|int)|list }}" + + - block: + - name: "Check python version on python3 container" + command: python --version + vars: + ansible_kubectl_container: python3 + register: _ + + - assert: { that: "'Python 3' in _.stdout" } + + - debug: var=_.stdout,_.stderr + run_once: yes + + - name: "Check python version on default container" + command: python --version + register: _ + + - assert: { that: "'Python 2' in _.stderr" } + + - debug: var=_.stdout,_.stderr + run_once: yes + tags: check - name: "Destroy pod" tags: cleanup - hosts: localhost + hosts: pods gather_facts: no + vars: + ansible_connection: "local" tasks: - name: Destroy pod k8s: @@ -86,6 +142,5 @@ apiVersion: v1 kind: Pod metadata: - name: test-pod-{{item}} + name: "{{inventory_hostname}}" namespace: default - loop: "{{ range(10)|list }}" From 3660febeb239dedbc09ffc234484fe5f9f10ff0d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 19:23:37 +0100 Subject: [PATCH 166/212] docs: add inline subscribe form to installation instructions --- docs/ansible.rst | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/docs/ansible.rst b/docs/ansible.rst index 485c24dc..e4619ef9 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -78,9 +78,27 @@ Installation deploy = (ALL) NOPASSWD:/usr/bin/python -c* -5. Subscribe to the `mitogen-announce mailing list - `_ to stay updated with new - releases and important bug fixes. +5. + + .. raw:: html + +
+ Releases occur frequently and often include important fixes. Subscribe + to the mitogen-announce + mailing list be notified of new releases. + +

+ + + + + +

+
+ Demo From 43ad23946e45ca6cfd482254386a82713647f93e Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 19:25:31 +0100 Subject: [PATCH 167/212] docs: tidy up wording. --- docs/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 162a7c48..2c219c4d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -71,11 +71,11 @@ Enhancements * The `faulthandler `_ module is automatically activated if it is installed, simplifying debugging of hangs. - See :ref:`diagnosing-hangs` for more information. + See :ref:`diagnosing-hangs` for details. * The ``MITOGEN_DUMP_THREAD_STACKS`` environment variable's value now indicates the number of seconds between stack dumps. See :ref:`diagnosing-hangs` for - more information. + details. Fixes From 638e473ff1233a198a2bf0bb4e9e961fb74c1ec6 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 10 Sep 2018 19:43:32 +0100 Subject: [PATCH 168/212] tests: hacksmash synchronize test to work Avoid password typing idiocy. --- .../integration/action/synchronize.yml | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/ansible/integration/action/synchronize.yml b/tests/ansible/integration/action/synchronize.yml index f9a8cb2c..2672b08e 100644 --- a/tests/ansible/integration/action/synchronize.yml +++ b/tests/ansible/integration/action/synchronize.yml @@ -3,7 +3,22 @@ - name: integration/action/synchronize.yml hosts: test-targets any_errors_fatal: true + vars: + ansible_user: mitogen__has_sudo_pubkey + ansible_ssh_private_key_file: /tmp/synchronize-action-key tasks: + # must copy git file to set proper file mode. + - copy: + dest: /tmp/synchronize-action-key + src: ../../../data/docker/mitogen__has_sudo_pubkey.key + mode: u=rw,go= + connection: local + + - file: + path: /tmp/sync-test + state: absent + connection: local + - file: path: /tmp/sync-test state: directory @@ -14,12 +29,17 @@ content: "item!" connection: local + - file: + path: /tmp/sync-test.out + state: absent + - synchronize: - dest: /tmp/sync-test - src: /tmp/sync-test + private_key: /tmp/synchronize-action-key + dest: /tmp/sync-test.out + src: /tmp/sync-test/ - slurp: - src: /tmp/sync-test/item + src: /tmp/sync-test.out/item register: out - set_fact: outout="{{out.content|b64decode}}" From 0cf908661ea2f53f26aa940a08d5d91ea88cfa3b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 11 Sep 2018 03:36:32 +0000 Subject: [PATCH 169/212] tests: set Docker hostname for more readable exceptions --- .travis/ansible_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis/ansible_tests.py b/.travis/ansible_tests.py index 0c47ab27..c7ddb989 100755 --- a/.travis/ansible_tests.py +++ b/.travis/ansible_tests.py @@ -24,9 +24,10 @@ with ci_lib.Fold('docker_setup'): --rm --detach --publish 0.0.0.0:%s:22/tcp + --hostname=target-%s --name=target-%s mitogen/%s-test - """, BASE_PORT + i, distro, distro,) + """, BASE_PORT + i, distro, distro, distro) with ci_lib.Fold('job_setup'): From e85760477bbdf91e1f554c2afc167fa6ca43f318 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 11 Sep 2018 03:39:39 +0000 Subject: [PATCH 170/212] tests: fix connection/_put_file.yml Was statting wrong destination path, and comparing floats that don't roundtrip serialization reliably. --- tests/ansible/integration/connection/_put_file.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/ansible/integration/connection/_put_file.yml b/tests/ansible/integration/connection/_put_file.yml index e93a1e68..a0fea4ed 100644 --- a/tests/ansible/integration/connection/_put_file.yml +++ b/tests/ansible/integration/connection/_put_file.yml @@ -13,11 +13,10 @@ register: original connection: local -- stat: path=/tmp/{{file_name}} +- stat: path=/tmp/{{file_name}}.out register: copied - assert: that: - original.stat.checksum == copied.stat.checksum - #- original.stat.atime == copied.stat.atime - - original.stat.mtime == copied.stat.mtime + - original.stat.mtime|int == copied.stat.mtime|int From f6b74992e1861caf738c228ff36fa9a4cff44be9 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 11 Sep 2018 05:22:49 +0100 Subject: [PATCH 171/212] tests: fix apparently erroneous localhost delegation. The stack delegates to localhost, which has ansible_python_interpreter set. --- tests/ansible/integration/delegation/stack_construction.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/ansible/integration/delegation/stack_construction.yml b/tests/ansible/integration/delegation/stack_construction.yml index beb9a9d1..4d9c75f4 100644 --- a/tests/ansible/integration/delegation/stack_construction.yml +++ b/tests/ansible/integration/delegation/stack_construction.yml @@ -331,9 +331,7 @@ out.result == [ { 'kwargs': { - 'python_path': [ - hostvars['cd-normal'].local_env.sys_executable - ], + 'python_path': None }, 'method': 'local', }, From 5521945bd2e9a601fa834dad8075824af4e74ed9 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 11 Sep 2018 03:44:17 +0100 Subject: [PATCH 172/212] ansible: temporary files take 5. --- ansible_mitogen/connection.py | 57 +++++++++------- ansible_mitogen/mixins.py | 36 ++++++---- ansible_mitogen/planner.py | 3 +- ansible_mitogen/runner.py | 40 +++++++++-- ansible_mitogen/target.py | 67 ++++++------------- docs/ansible.rst | 22 ++++-- .../integration/action/make_tmp_path.yml | 56 ++++------------ 7 files changed, 142 insertions(+), 139 deletions(-) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index 60724fa3..ae6268de 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -31,6 +31,7 @@ from __future__ import unicode_literals import logging import os +import random import stat import time @@ -474,26 +475,19 @@ class Connection(ansible.plugins.connection.ConnectionBase): #: Only sudo, su, and doas are supported for now. become_methods = ['sudo', 'su', 'doas'] - #: Dict containing init_child() return vaue as recorded at startup by + #: Dict containing init_child() return value as recorded at startup by #: ContextService. Contains: #: #: fork_context: Context connected to the fork parent : process in the #: target account. #: home_dir: Target context's home directory. - #: temp_dir: A writeable temporary directory managed by the - #: target, automatically destroyed at shutdown. + #: good_temp_dir: A writeable directory where new temporary directories + #: can be created. init_child_result = None - #: A private temporary directory destroyed during :meth:`close`, or - #: automatically during shutdown if :meth:`close` failed or was never - #: called. - temp_dir = None - - #: A :class:`mitogen.parent.CallChain` to use for calls made to the target - #: account, to ensure subsequent calls fail if pipelined directory creation - #: or file transfer fails. This eliminates roundtrips when a call is likely - #: to succeed, and ensures subsequent actions will fail with the original - #: exception if the pipelined call failed. + #: A :class:`mitogen.parent.CallChain` for calls made to the target + #: account, to ensure subsequent calls fail with the original exception if + #: pipelined directory creation or file transfer fails. chain = None # @@ -695,14 +689,24 @@ class Connection(ansible.plugins.connection.ConnectionBase): self.init_child_result = dct['init_child_result'] - def _init_temp_dir(self): - """ - """ - self.temp_dir = os.path.join( - self.init_child_result['temp_dir'], - 'worker-%d-%x' % (os.getpid(), id(self)) + def get_good_temp_dir(self): + self._connect() + return self.init_child_result['good_temp_dir'] + + def _generate_tmp_path(self): + return os.path.join( + self.get_good_temp_dir(), + 'ansible_mitogen_action_%016x' % ( + random.getrandbits(8*8), + ) ) - self.get_chain().call_no_reply(os.mkdir, self.temp_dir) + + def _make_tmp_path(self): + assert getattr(self._shell, 'tmpdir', None) is None + self._shell.tmpdir = self._generate_tmp_path() + LOG.debug('Temporary directory: %r', self._shell.tmpdir) + self.get_chain().call_no_reply(os.mkdir, self._shell.tmpdir) + return self._shell.tmpdir def _connect(self): """ @@ -721,7 +725,6 @@ class Connection(ansible.plugins.connection.ConnectionBase): self._connect_broker() stack = self._build_stack() self._connect_stack(stack) - self._init_temp_dir() def close(self, new_task=False): """ @@ -729,18 +732,22 @@ class Connection(ansible.plugins.connection.ConnectionBase): gracefully shut down, and wait for shutdown to complete. Safe to call multiple times. """ + if getattr(self._shell, 'tmpdir', None) is not None: + # Avoid CallChain to ensure exception is logged on failure. + self.context.call_no_reply( + ansible_mitogen.target.prune_tree, + self._shell.tmpdir, + ) + self._shell.tmpdir = None + if self.context: self.chain.reset() - # No pipelining to ensure exception is logged on failure. - self.context.call_no_reply(ansible_mitogen.target.prune_tree, - self.temp_dir) self.parent.call_service( service_name='ansible_mitogen.services.ContextService', method_name='put', context=self.context ) - self.temp_dir = None self.context = None self.login_context = None self.init_child_result = None diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index faadbc15..d4fcbd0d 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -180,12 +180,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): connection. """ LOG.debug('_make_tmp_path(remote_user=%r)', remote_user) - self._connection._connect() - # _make_tmp_path() is basically a global stashed away as Shell.tmpdir. - self._connection._shell.tmpdir = self._connection.temp_dir - LOG.debug('Temporary directory: %r', self._connection._shell.tmpdir) - self._cleanup_remote_tmp = True - return self._connection._shell.tmpdir + return self._connection._make_tmp_path() def _remove_tmp_path(self, tmp_path): """ @@ -193,6 +188,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): with nothing, as the persistent interpreter automatically cleans up after itself without introducing roundtrips. """ + # The actual removal is pipelined by Connection.close(). LOG.debug('_remove_tmp_path(%r)', tmp_path) self._connection._shell.tmpdir = None @@ -293,6 +289,25 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): except AttributeError: return getattr(self._task, 'async') + def _temp_file_gibberish(self, module_args, wrap_async): + # Ansible>2.5 module_utils reuses the action's temporary directory if + # one exists. Older versions error if this key is present. + if ansible.__version__ > '2.5': + if wrap_async: + # Sharing is not possible with async tasks, as in that case, + # the directory must outlive the action plug-in. + module_args['_ansible_tmpdir'] = None + else: + module_args['_ansible_tmpdir'] = self._connection._shell.tmpdir + + # If _ansible_tmpdir is unset, Ansible>2.6 module_utils will use + # _ansible_remote_tmp as the location to create the module's temporary + # directory. Older versions error if this key is present. + if ansible.__version__ > '2.6': + module_args['_ansible_remote_tmp'] = ( + self._connection.get_good_temp_dir() + ) + def _execute_module(self, module_name=None, module_args=None, tmp=None, task_vars=None, persist_files=False, delete_remote_tmp=True, wrap_async=False): @@ -311,16 +326,9 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): self._update_module_args(module_name, module_args, task_vars) env = {} self._compute_environment_string(env) + self._temp_file_gibberish(module_args, wrap_async) - # Always set _ansible_tmpdir regardless of whether _make_remote_tmp() - # has ever been called. This short-circuits all the .tmpdir logic in - # module_common and ensures no second temporary directory or atexit - # handler is installed. self._connection._connect() - - if ansible.__version__ > '2.5': - module_args['_ansible_tmpdir'] = self._connection.temp_dir - return ansible_mitogen.planner.invoke( ansible_mitogen.planner.Invocation( action=self, diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 5b3e5547..caf40af3 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -149,7 +149,8 @@ class Planner(object): """ new = dict((mitogen.core.UnicodeType(k), kwargs[k]) for k in kwargs) - new.setdefault('temp_dir', self._inv.connection.temp_dir) + new.setdefault('good_temp_dir', + self._inv.connection.get_good_temp_dir()) new.setdefault('cwd', self._inv.connection.get_default_cwd()) new.setdefault('extra_env', self._inv.connection.get_default_env()) new.setdefault('emulate_tty', True) diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index fe5f4c46..44780aa2 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -230,6 +230,11 @@ class Runner(object): This is passed as a string rather than a dict in order to mimic the implicit bytes/str conversion behaviour of a 2.x controller running against a 3.x target. + :param str good_temp_dir: + The writeable temporary directory for this user account reported by + :func:`ansible_mitogen.target.init_child` passed via the controller. + This is specified explicitly to remain compatible with Ansible<2.5, and + for forked tasks where init_child never runs. :param dict env: Additional environment variables to set during the run. Keys with :data:`None` are unset if present. @@ -242,7 +247,7 @@ class Runner(object): When :data:`True`, indicate the runner should detach the context from its parent after setup has completed successfully. """ - def __init__(self, module, service_context, json_args, temp_dir, + def __init__(self, module, service_context, json_args, good_temp_dir, extra_env=None, cwd=None, env=None, econtext=None, detach=False): self.module = module @@ -250,10 +255,32 @@ class Runner(object): self.econtext = econtext self.detach = detach self.args = json.loads(json_args) - self.temp_dir = temp_dir + self.good_temp_dir = good_temp_dir self.extra_env = extra_env self.env = env self.cwd = cwd + #: If not :data:`None`, :meth:`get_temp_dir` had to create a temporary + #: directory for this run, because we're in an asynchronous task, or + #: because the originating action did not create a directory. + self._temp_dir = None + + def get_temp_dir(self): + path = self.args.get('_ansible_tmpdir') + if path is not None: + return path + + if self._temp_dir is None: + self._temp_dir = tempfile.mkdtemp( + prefix='ansible_mitogen_runner_', + dir=self.good_temp_dir, + ) + + return self._temp_dir + + def revert_temp_dir(self): + if self._temp_dir is not None: + ansible_mitogen.target.prune_tree(self._temp_dir) + self._temp_dir = None def setup(self): """ @@ -291,6 +318,7 @@ class Runner(object): implementation simply restores the original environment. """ self._env.revert() + self.revert_temp_dir() def _run(self): """ @@ -466,7 +494,7 @@ class ProgramRunner(Runner): fetched via :meth:`_get_program`. """ filename = self._get_program_filename() - path = os.path.join(self.temp_dir, filename) + path = os.path.join(self.get_temp_dir(), filename) self.program_fp = open(path, 'wb') self.program_fp.write(self._get_program()) self.program_fp.flush() @@ -546,7 +574,7 @@ class ArgsFileRunner(Runner): self.args_fp = tempfile.NamedTemporaryFile( prefix='ansible_mitogen', suffix='-args', - dir=self.temp_dir, + dir=self.get_temp_dir(), ) self.args_fp.write(utf8(self._get_args_contents())) self.args_fp.flush() @@ -661,7 +689,7 @@ class NewStyleRunner(ScriptRunner): def setup(self): super(NewStyleRunner, self).setup() - self._stdio = NewStyleStdio(self.args, self.temp_dir) + self._stdio = NewStyleStdio(self.args, self.get_temp_dir()) # It is possible that not supplying the script filename will break some # module, but this has never been a bug report. Instead act like an # interpreter that had its script piped on stdin. @@ -739,7 +767,7 @@ class NewStyleRunner(ScriptRunner): # don't want to pointlessly write the module to disk when it never # actually needs to exist. So just pass the filename as it would exist. mod.__file__ = os.path.join( - self.temp_dir, + self.get_temp_dir(), 'ansible_module_' + os.path.basename(self.path), ) diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index 35863cb2..ae7990a9 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -85,13 +85,9 @@ MAKE_TEMP_FAILED_MSG = ( #: the target Python interpreter before it executes any code or imports. _fork_parent = None -#: Set by init_child() to a list of candidate $variable-expanded and -#: tilde-expanded directory paths that may be usable as a temporary directory. -_candidate_temp_dirs = None - -#: Set by reset_temp_dir() to the single temporary directory that will exist -#: for the duration of the process. -temp_dir = None +#: Set by :func:`init_child` to the name of a writeable and executable +#: temporary directory accessible by the active user account. +good_temp_dir = None def get_small_file(context, path): @@ -206,15 +202,19 @@ def _on_broker_shutdown(): prune_tree(temp_dir) -def find_good_temp_dir(): +def find_good_temp_dir(candidate_temp_dirs): """ - Given a list of candidate temp directories extracted from ``ansible.cfg`` - and stored in _candidate_temp_dirs, combine it with the Python-builtin list - of candidate directories used by :mod:`tempfile`, then iteratively try each - in turn until one is found that is both writeable and executable. + Given a list of candidate temp directories extracted from ``ansible.cfg``, + combine it with the Python-builtin list of candidate directories used by + :mod:`tempfile`, then iteratively try each until one is found that is both + writeable and executable. + + :param list candidate_temp_dirs: + List of candidate $variable-expanded and tilde-expanded directory paths + that may be usable as a temporary directory. """ paths = [os.path.expandvars(os.path.expanduser(p)) - for p in _candidate_temp_dirs] + for p in candidate_temp_dirs] paths.extend(tempfile._candidate_tempdir_list()) for path in paths: @@ -253,29 +253,6 @@ def find_good_temp_dir(): }) -@mitogen.core.takes_econtext -def reset_temp_dir(econtext): - """ - Create one temporary directory to be reused by all runner.py invocations - for the lifetime of the process. The temporary directory is changed for - each forked job, and emptied as necessary by runner.py::_cleanup_temp() - after each module invocation. - - The result is that a context need only create and delete one directory - during startup and shutdown, and no further filesystem writes need occur - assuming no modules execute that create temporary files. - """ - global temp_dir - # https://github.com/dw/mitogen/issues/239 - - basedir = find_good_temp_dir() - temp_dir = tempfile.mkdtemp(prefix='ansible_mitogen_', dir=basedir) - - # This must be reinstalled in forked children too, since the Broker - # instance from the parent process does not carry over to the new child. - mitogen.core.listen(econtext.broker, 'shutdown', _on_broker_shutdown) - - @mitogen.core.takes_econtext def init_child(econtext, log_level, candidate_temp_dirs): """ @@ -306,24 +283,23 @@ def init_child(econtext, log_level, candidate_temp_dirs): the controller will use to start forked jobs, and `home_dir` is the home directory for the active user account. """ - global _candidate_temp_dirs - _candidate_temp_dirs = candidate_temp_dirs - - global _fork_parent - mitogen.parent.upgrade_router(econtext) - _fork_parent = econtext.router.fork() - reset_temp_dir(econtext) - # Copying the master's log level causes log messages to be filtered before # they reach LogForwarder, thus reducing an influx of tiny messges waking # the connection multiplexer process in the master. LOG.setLevel(log_level) logging.getLogger('ansible_mitogen').setLevel(log_level) + global _fork_parent + mitogen.parent.upgrade_router(econtext) + _fork_parent = econtext.router.fork() + + global good_temp_dir + good_temp_dir = find_good_temp_dir(candidate_temp_dirs) + return { 'fork_context': _fork_parent, 'home_dir': mitogen.core.to_text(os.path.expanduser('~')), - 'temp_dir': temp_dir, + 'good_temp_dir': good_temp_dir, } @@ -336,7 +312,6 @@ def create_fork_child(econtext): """ mitogen.parent.upgrade_router(econtext) context = econtext.router.fork() - context.call(reset_temp_dir) LOG.debug('create_fork_child() -> %r', context) return context diff --git a/docs/ansible.rst b/docs/ansible.rst index e4619ef9..d2138f21 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -425,6 +425,9 @@ specific variables with a particular linefeed style. Temporary Files ~~~~~~~~~~~~~~~ +Temporary file handling in Ansible is incredibly tricky business, and the exact +behaviour varies across major releases. + Ansible creates a variety of temporary files and directories depending on its operating mode. @@ -462,11 +465,20 @@ In summary, for each task Ansible may create one or more of: * ``$TMPDIR/ansible__payload_.../`` owned by the become user, * ``$TMPDIR/ansible-module-tmp-.../`` owned by the become user. -A directory must exist to maintain compatibility with Ansible, as many modules -introspect :data:`sys.argv` to find a directory where they may write files, -however only one directory exists for the lifetime of each interpreter, its -location is consistent for each target account, and it is always privately -owned by that account. + +Mitogen for Ansible +^^^^^^^^^^^^^^^^^^^ + +Temporary h +Temporary directory handling is fiddly and varies across major Ansible +releases. + + +Temporary directories must exist to maintain compatibility with Ansible, as +many modules introspect :data:`sys.argv` to find a directory where they may +write files, however only one directory exists for the lifetime of each +interpreter, its location is consistent for each target account, and it is +always privately owned by that account. The paths below are tried until one is found that is writeable and lives on a filesystem with ``noexec`` disabled: diff --git a/tests/ansible/integration/action/make_tmp_path.yml b/tests/ansible/integration/action/make_tmp_path.yml index 97da070d..dc713c31 100644 --- a/tests/ansible/integration/action/make_tmp_path.yml +++ b/tests/ansible/integration/action/make_tmp_path.yml @@ -28,18 +28,18 @@ method: _make_tmp_path register: tmp_path2 - - name: "Find parent temp path" + - name: "Find good temp path" set_fact: - parent_temp_path: "{{tmp_path.result|dirname}}" + good_temp_path: "{{tmp_path.result|dirname}}" - - name: "Find parent temp path (new task)" + - name: "Find good temp path (new task)" set_fact: - parent_temp_path2: "{{tmp_path2.result|dirname}}" + good_temp_path2: "{{tmp_path2.result|dirname}}" - name: "Verify common base path for both tasks" assert: that: - - parent_temp_path == parent_temp_path2 + - good_temp_path == good_temp_path2 - name: "Verify different subdir for both tasks" assert: @@ -60,6 +60,8 @@ path: "{{tmp_path2.result}}" register: stat2 + - debug: msg={{stat1}} + - name: "Verify neither subdir exists any more" assert: that: @@ -67,15 +69,15 @@ - not stat2.stat.exists # - # Verify parent directory persistence. + # Verify good directory persistence. # - - name: Stat parent temp path (new task) + - name: Stat good temp path (new task) stat: - path: "{{parent_temp_path}}" + path: "{{good_temp_path}}" register: stat - - name: "Verify parent temp path is persistent" + - name: "Verify good temp path is persistent" assert: that: - stat.stat.exists @@ -102,36 +104,6 @@ that: - not out.stat.exists - # - # - # - - - name: "Verify temp path changes across connection reset" - mitogen_shutdown_all: - - - name: "Verify temp path changes across connection reset" - action_passthrough: - method: _make_tmp_path - register: tmp_path2 - - - name: "Verify temp path changes across connection reset" - set_fact: - parent_temp_path2: "{{tmp_path2.result|dirname}}" - - - name: "Verify temp path changes across connection reset" - assert: - that: - - parent_temp_path != parent_temp_path2 - - - name: "Verify old path disappears across connection reset" - stat: path={{parent_temp_path}} - register: junk_stat - - - name: "Verify old path disappears across connection reset" - assert: - that: - - not junk_stat.stat.exists - # # root # @@ -175,12 +147,12 @@ when: ansible_version.full < '2.5' assert: that: - - out.module_path.startswith(parent_temp_path2) + - out.module_path.startswith(good_temp_path2) - out.module_tmpdir == None - name: "Verify modules get the same tmpdir as the action plugin (>2.5)" when: ansible_version.full > '2.5' assert: that: - - out.module_path.startswith(parent_temp_path2) - - out.module_tmpdir.startswith(parent_temp_path2) + - out.module_path.startswith(good_temp_path2) + - out.module_tmpdir.startswith(good_temp_path2) From e58b6a8f056eb8f202fb9d543ba823dabf99384d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 11 Sep 2018 05:14:30 +0000 Subject: [PATCH 173/212] tests: correct path for common-hosts --- .travis/ansible_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis/ansible_tests.py b/.travis/ansible_tests.py index c7ddb989..723f9c1a 100755 --- a/.travis/ansible_tests.py +++ b/.travis/ansible_tests.py @@ -38,7 +38,7 @@ with ci_lib.Fold('job_setup'): run("pip install -q ansible==%s", ci_lib.ANSIBLE_VERSION) run("mkdir %s", HOSTS_DIR) - run("ln -s %s/common-hosts %s", TESTS_DIR, HOSTS_DIR) + run("ln -s %s/hosts/common-hosts %s", TESTS_DIR, HOSTS_DIR) with open(os.path.join(HOSTS_DIR, 'target'), 'w') as fp: fp.write('[test-targets]\n') From 564113874be90ead2394efbd0a5a1a3d4a4aa008 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 11 Sep 2018 06:21:16 +0100 Subject: [PATCH 174/212] tests: explicitly define localhost in common-hosts --- tests/ansible/hosts/common-hosts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/ansible/hosts/common-hosts b/tests/ansible/hosts/common-hosts index 449442f6..cf84d2d1 100644 --- a/tests/ansible/hosts/common-hosts +++ b/tests/ansible/hosts/common-hosts @@ -1,5 +1,11 @@ # vim: syntax=dosini + +# This must be defined explicitly, otherwise _create_implicit_localhost() +# generates its own copy, which includes an ansible_python_interpreter that +# varies according to host machine. +localhost + [connection-delegation-test] cd-bastion cd-rack11 mitogen_via=ssh-user@cd-bastion From dfb4930fce82cc0f59e3907c995ffbae3d8a116a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 11 Sep 2018 06:40:56 +0100 Subject: [PATCH 175/212] tests: import custom binaries for tests It means Linux<->OS X runs work fine without manual hackery. --- .travis/ansible_tests.py | 2 +- tests/ansible/Makefile | 10 +++++--- .../runner/custom_binary_producing_json.yml | 21 +++++++++++++--- .../runner/custom_binary_producing_junk.yml | 23 ++++++++++++++---- .../custom_binary_producing_json_Darwin | Bin 0 -> 8540 bytes .../custom_binary_producing_json_Linux | Bin 0 -> 8400 bytes .../custom_binary_producing_junk_Darwin | Bin 0 -> 8540 bytes .../custom_binary_producing_junk_Linux | Bin 0 -> 8400 bytes 8 files changed, 42 insertions(+), 14 deletions(-) create mode 100755 tests/ansible/lib/modules/custom_binary_producing_json_Darwin create mode 100755 tests/ansible/lib/modules/custom_binary_producing_json_Linux create mode 100755 tests/ansible/lib/modules/custom_binary_producing_junk_Darwin create mode 100755 tests/ansible/lib/modules/custom_binary_producing_junk_Linux diff --git a/.travis/ansible_tests.py b/.travis/ansible_tests.py index 723f9c1a..3b5e40db 100755 --- a/.travis/ansible_tests.py +++ b/.travis/ansible_tests.py @@ -55,7 +55,7 @@ with ci_lib.Fold('job_setup'): )) # Build the binaries. - run("make -C %s", TESTS_DIR) + # run("make -C %s", TESTS_DIR) if not ci_lib.exists_in_path('sshpass'): run("sudo apt-get update") run("sudo apt-get install -y sshpass") diff --git a/tests/ansible/Makefile b/tests/ansible/Makefile index 00d6a8ab..1d4ab1dd 100644 --- a/tests/ansible/Makefile +++ b/tests/ansible/Makefile @@ -1,13 +1,15 @@ -TARGETS+=lib/modules/custom_binary_producing_junk -TARGETS+=lib/modules/custom_binary_producing_json +SYSTEM=$(shell uname -s) + +TARGETS+=lib/modules/custom_binary_producing_junk_$(SYSTEM) +TARGETS+=lib/modules/custom_binary_producing_json_$(SYSTEM) all: clean $(TARGETS) -lib/modules/custom_binary_producing_junk: lib/modules.src/custom_binary_producing_junk.c +lib/modules/custom_binary_producing_junk_$(SYSTEM): lib/modules.src/custom_binary_producing_junk.c $(CC) -o $@ $< -lib/modules/custom_binary_producing_json: lib/modules.src/custom_binary_producing_json.c +lib/modules/custom_binary_producing_json_$(SYSTEM): lib/modules.src/custom_binary_producing_json.c $(CC) -o $@ $< clean: diff --git a/tests/ansible/integration/runner/custom_binary_producing_json.yml b/tests/ansible/integration/runner/custom_binary_producing_json.yml index 00f03f07..4fe09f0d 100644 --- a/tests/ansible/integration/runner/custom_binary_producing_json.yml +++ b/tests/ansible/integration/runner/custom_binary_producing_json.yml @@ -1,12 +1,25 @@ - name: integration/runner/custom_binary_producing_json.yml hosts: test-targets any_errors_fatal: true + gather_facts: true tasks: - - custom_binary_producing_json: - foo: true - with_sequence: start=1 end={{end|default(1)}} - register: out + - block: + - custom_binary_producing_json_Darwin: + foo: true + with_sequence: start=1 end={{end|default(1)}} + register: out_darwin + - set_fact: out={{out_darwin}} + when: ansible_system == "Darwin" + - block: + - custom_binary_producing_json_Linux: + foo: true + with_sequence: start=1 end={{end|default(1)}} + register: out_linux + - set_fact: out={{out_linux}} + when: ansible_system == "Linux" + + - debug: msg={{out}} - assert: that: | out.changed and diff --git a/tests/ansible/integration/runner/custom_binary_producing_junk.yml b/tests/ansible/integration/runner/custom_binary_producing_junk.yml index 93d98065..b1672ad9 100644 --- a/tests/ansible/integration/runner/custom_binary_producing_junk.yml +++ b/tests/ansible/integration/runner/custom_binary_producing_junk.yml @@ -1,11 +1,24 @@ - name: integration/runner/custom_binary_producing_junk.yml hosts: test-targets + gather_facts: true tasks: - - custom_binary_producing_junk: - foo: true - with_sequence: start=1 end={{end|default(1)}} - ignore_errors: true - register: out + - block: + - custom_binary_producing_junk_Darwin: + foo: true + with_sequence: start=1 end={{end|default(1)}} + ignore_errors: true + register: out_darwin + - set_fact: out={{out_darwin}} + when: ansible_system == "Darwin" + + - block: + - custom_binary_producing_junk_Linux: + foo: true + with_sequence: start=1 end={{end|default(1)}} + ignore_errors: true + register: out_linux + - set_fact: out={{out_linux}} + when: ansible_system == "Linux" - hosts: test-targets diff --git a/tests/ansible/lib/modules/custom_binary_producing_json_Darwin b/tests/ansible/lib/modules/custom_binary_producing_json_Darwin new file mode 100755 index 0000000000000000000000000000000000000000..69de2fea2ef06ce88f94230d7f0eb94e2da4b653 GIT binary patch literal 8540 zcmeHMQEL=e6uxU?8&c6ILQ`6C+%}?;W>u0l1cgL1bVac-MhIfL>}F@PE0dXJW)=;l z5D~#3f&2ttee=P8pkn*fzO?w@gAY{<#J=c*6zTeXckXOvC&Bl?J?zF%Nx6b#ecEtH)&2bw}0ADuWkk0q5jwdGXZP`%YY#0S~Z%zkf^Udgd zS9CvdS~_y0L&~Oit?pH1-K+Wvch1KHosV-)Axi1yiSg<2R1fBlye0FP zwVYQxb4nQ|<(!|IR^ZL^bu*ZP{2ZSM`A^p=rRU0UVZQ9UvJnLv@^$O_^kXK7v-#3k zO1Jc}e0`aG#Od+LbDY$>zOK%lIe*rkIFqaiE9UKrA`j8Plms(R(nWtB>-H!cmJsBo z8}<-vB1f$_#Uw1_$Wb5kHZVEsIv~Uh?B~!)D24eGWqil_VL(b=SwgU;tYF!(B(-h| z%`hmq^>U%I;8t|tSFo((?A_C&$N&7ny*IGuyYibG|8QM^It=VldL+U0L2z%29LIH? z7Ou|`DEnCRwiAD&?3`#{8`rQ7uhczZYMhpXn0mFb{ zz%XDKFbo(5wjKkQic2qwH`jWL%foe4U%a!u2d}nI?DxEt;+>KE;?;h(mQP-VS+d{3 zu@|jlzc1NumwqeWeAN5%!puLvOz5xUN57UGoFCGMN9hN4=YW&1*u^mhZiMFCK!W@>`X{Horx7U}UcvP49X` zq0HZe{ti5^ZKm5m!+>GHFkl!k3>XFs1BL;^fMLKeU>GnA7zVZw1H1EmQ!ID)nB>uQ z#R-CjxDbeyF&@^b!*Pkd4EV(g>lbhYZPEe`@qlLoo_s52YhmJF)_y9Q#_#^-T literal 0 HcmV?d00001 diff --git a/tests/ansible/lib/modules/custom_binary_producing_json_Linux b/tests/ansible/lib/modules/custom_binary_producing_json_Linux new file mode 100755 index 0000000000000000000000000000000000000000..16e6d046516121ce619146739eb42eacdab12d0c GIT binary patch literal 8400 zcmeHMYit}>6~60@6X)Ue#!Xu10U1mpk~W?=FULt6;$1&7rp|-ZE>zT+tasP;mVH>e zvyB}eNe5I8s#8u23m5rO~_P*a?$6{e3Ufa2wxJNL6^ zX4e!X#IM}d%(>^g=bU@)bI!f{TzgkXh0iCD{Nkg6x$#OHX;B4dSEvk;7SSvg()$Bq zwU|%*Jc-%mEjFQ2JKa^LL8}z*C(3%w6*l0dtCbumJY=d|Qxa~{ywK)Bh0r9xdhF}@ zYRMASWCrwz@?%s{=rPNB<4SK_>48ou$3W3OZW}+{%AS|DQ5h#tmgRKtmKEdty9*>+ zLLakX0Sdj3lOFr|pQfM^Jf!NIt6vT(|3OuA#inGUuW8$+WUL{XNau$ehMSujnzre= zjJ`#-o9(mg-u*qYCLV8eE&IHT9>%!*EB7wE-tzKaPF+b~uRU_oS3g|&g;!aY`x4_O zGs*YfWa-@$N`pk~IfXnY7Y0ZPBuV>|BK$K^_@9 z$Cyw4gur~tuT0PitPp#+kP_Ax+BL2w=|#1UfR8G?%F_?vZ`5eg9*gmuK)UI`c|IXc zI&iEJE}L@T&h>KFfvcq_O<`Pk&w;O8@N*^4vY6M5Uv=QleaaYnv&uMGnO~#{C)74ug+5I}n6qnywsEK!L*0gcS?iFO9$2eWNl&G=uu2HyU ztsvK~kPN$4-CMlbuh*Hn4mPjCMvksf2iH0c|cru{}?%D6PS$MC&k zT${G)$lw>1!6j^PE_%(A1m|C*{6)%$-ig_! zNqeDBgr4jy{3g^BqP|>rX1?sI2BuGh3imn-ce*y*l4E)4o`3QqZ1TigmbUqKXkS<1 zA6Z7%t z9WjTpnOHuWNDr9LjzNtlvgY70FexoJy?wG(C&R&mD|~0>S4iTC3-0$suA{ z=d<)QQhA)H2nT{MRD=U{XZ-B}?ex5GVEw5|BhdK#d?V02wqReNB^ziC1sX$v_2Ga< zawH!PRLOnqGO2%(>^Rpso4lXy_rU!gxZeZ+&mKVhgm{U+aGXGw*sK^-!G}7wh~p}# z!7&bT*(xQEy>+GH5ueFDnlO~VH(kuI$T_tcBE~tVf-l5W4WRzY!#a?}=)T;5eL9vvLIZk$E-_zi-$qom=gM`Z+G=1i60}?eT28oV~ZTb(gljr!Swj z^4bo4tKQhKxhXH%ts89MeJt_0&Np+ecMAI{w!t~rjr)a#fp!%5=_HAH=77n5rNI1h zcxJu1@g-uk%y{@kwK!iE56s-Vxb4); ztY0@C6lXEv9R+?mQR4o<0aL%#ibhOyM}c2FFmq0Gz$Cv+U>|Vf%jrHub3!uj7Y_=o zTW6u4SE)MJx$)U^uN%iMHH-3knWE zrJd68`lFJ^cR;fAdRt~n=T9X$>MJw8s|c^9{vTEI5X<-x!ZjFm(9cV}bUy5rb{v8# z`xN0pnm<9${8=aEOV`DbIqW>A?0DzlDZ;hVYQ*l-?46TorOYVbI_2obDV8w@l9|3p z(u`S|Y|f12heb4#8cN2kcue2Xw6m$a5Z`wt%t$sH88PE&D?1|kvyoKXjOA0Q5mIq* zCP`W@(P%zrWm0C@V>P*InjMEi``XR+{cR>)gSqu%!fbnLe`sH4tGh(rw-H2lm}a}7 zEEsKvgt@otKseN89_Z+JqP^Sf4u!kg*{Zyyi{|pOCI9wbj_>?hT3oky@mR!)z{u=X zpGV4MDf>Q;WZjp0W-OO6>DC~|7k`}xNGg^{oB3Rv8aV44cOCJ4xtua1FBI*IM{i|x zmxwOoOUAM{hwi&dw*~uF(~Su|Hp<5+P*Jab8{u~- zhDQB}|3IOS&%HOC*-!d>#(@5~%7Ee=z}qPI6ZCN&c#<%V-Jy?o5)|hc5|N2*IMZQb z_>2I3#GRn{?*aZp4)oKc&u0$sh)+Q^Rhid+1&d>p;XZ}_5!Jz zSApUj#`+J^J29-GVXW~s&0qTJP=z*W2 z3}dKYQ*mQ(4t))ZioQpb>u>bn&|mApS(cNNV(!@`_PbftU#B{bzcU=O!hh_aJip<) gt!cjx@@7FP>@kjd?NW{@%j*}v>U|sRf2`WX( zOS!x9uKm5Klk^f+xQf5tAC`~1Erp(*cqIk_-etufS;dntTrJ49!ytYt0 z;$VOOJ@(UI*jGb?Cywg_gZ`e-T;9$*2Ianq6$@6(%p_x} zl9>XgY=kn8*}2(ePrn^`a`)o82XAig{KS0$>d>*p(Y-#V4T8Cq*^m1=C>*B;$~NZQ zaOf@4jx3Cg@kdGp+nDM++KD_eNjO1vK{;MobNSojVe+FyF`yVw3@8Q^1BwB~fMVeP zXW&eH`a}HsmuP&ZJA>KAZ#BI~@k0N_smE{i+!ym(S(`aLiDFv+0mE9<_P?6ezfZr6 zUwhz{{q2i#x!$v=jam_0OO{%IGPz)#r6a$I@#eiZ!F`yVw3@8Q^1BwB~z(2-7 z>-M(OOu4mB^6ENeSXN$~vP7kimvzqNxWw56?4m+Hs|}tLdE7|b6R=nsU=7hBBIQ(C z8dr_+qHW+FJi)H(c`UJ{RJOQg-q#<39@P`P?sx7S*F>C~K<6BS_j)*w;E{wQJMawR zju7T~N3fk|AH2t8BF+Fp^!YZJa6h~_>i{FQDf^RM#Ez9nS-r7X_rd*L-CYM5Qv3l+ CsOnPy literal 0 HcmV?d00001 diff --git a/tests/ansible/lib/modules/custom_binary_producing_junk_Linux b/tests/ansible/lib/modules/custom_binary_producing_junk_Linux new file mode 100755 index 0000000000000000000000000000000000000000..4aadc9c1d36a55b07f12a215347b8067891dba00 GIT binary patch literal 8400 zcmeHMU2I%O6`u9RiIdptO`5d%DdbWT;UswD{5ejWQ1ALD*VIW&VwYCbxmkPH_O|;I z?cPo72&qO%DpvU+AXEh*9uN|ON{GNqBObuPrG!2uARq`30TJX>Etr-@0TeIi%srod z_wJg4!~+i*Ywwx!%{gbzoS8fK%=pEQ?#?QoPoenL#}soD)eh1s4Hs8Sg-EMvQH$yM z0rjX_Nc;kgIqj_up;o8d6VsqIg8PZGUQ3k&crIH>gy?}zi({Y|ACHZHy<*Qx+o_EUD9du`xoyX|er=Iv z%jg~lR-n-P1nIG#|7i(|;34U6zPR*=|DZBlwI!JtXx_0U8EZ@?()rQG(U#`M<{hD2 zCbUhDo9(mgz5{)_C!TMFmVI79KgPKIl|7NQo4$2?a`V*To@>*wFP-~yLkG)pEHQ7o zA|@E${mxk|4dTknQv@01uv$vk@(Ot4Joqr-Mg{pt=8-=>kNgJ-f2@KXj?*%=N*O^z zw_^P$itiwYvsoR>+c}l9WASWO4V}m)?6|V5!O^HSlt@RDiIdE-w3V}?S=&lQ6KS=t zyDQRWZ3}G+?NnA*?|zH4y9*gmsK)L0@c|D;_ zxp2HA+&1mP-S5ji7cQ@!HidcNJqNyW!!?m-S*&Zuue)&fK4qSItHwNAonK-o^UQ=@ zRlIJVdaY(s6?bkUsJLMVJ?q!CQjPf$Hg)G#k;;Zm%=3`%T-SVr_@Rp|z-8;(U4FqvW{R+vN=j*m>viDCUQ(Sp>v7!D}zpkBI_O2!aedhVWtqkOq*Ayo-}>0 zn>S|cRb=q{f_pv+U z?AW`LWRk|i;{A7qNP=tB>r*6V7OsA89SMD(!$ux|o5<96XNpCtPjdZbs$Zds>Ytn& znzk4IWccZ>!Y{*pVT$F33k!9q8krsr7w&fz?sjjyt>^OUegD))*yNeFZDZ@NF~07? zKe`Kd+Y5gQ7gzkoJT>VvcmFp3CeQV!`oqtK`@=`V*5t*xPTwVev^~8Vcx^kYX@K7C z?3i^dn~CKI6X{{=`F#3?$BoR0kbA9L}YJfwW1LEqZiMKu?Fi-L^vMtg~D zY-Rx|mF1OMPwgz4}+8w?WpY=~GQ4SHp5rSob;l zZKKfcVk{g8zEss#w=hc%5z9KCrr$Hv9w(|If#Bz=B7s#G{2c+~{DMfJ;as&DXnJX( z8E84RXn&wJ8)yj!n!=0vsP^oS%Ma%@qjC8@zN4td#Hk;fjqMtJ0BydQHx`+GCR4D**{(?pJQNs=$* zR1?y^goUXfVOi?1l<_)crBD#$aF->I!rrXEB_xFJjtoa}9?qpj907iGfrG>En+{9o zR;QqU&da$#?w>_xd=ys=(dR*rw^i)XCsXr?`?T;4(*AehF%HN2|L%!%r`O`kGXAe) zLc9X+!z0h$*Vgv9(a<-LPuqE8S7>{vsc~y_UbEXbI>7r_;&YvE_FV54_ET#G!BsIU%8@kI*rQ;OFp%xfvWSYiE?;!9@NSt-6$;e9K` zYt?v#`S7bcb-5xQn7wx?wNpR){+8lFbrB2R)!?TSCGHPgFvV@Ts=-2cHTcy-v*$Dy zO!6xf_JLA-CEbS@Zb;Yt>S2ZV*4^mmw^Yt`rTE;rw-m=NHHY&1wm|*0IEOej-tVeZ zKuvq%84$b)8dS;L zt9nFXUxdgv7$=YL-pRJFUL~C45R^E?Md1d~Ag!w=kN;;x9`~#`Tm6`DmiLbLXCnWG ztY==YTyARIsk@(V5zhX5?eIOU;TDwoa+Ap&dv`Fg$C8! zos(s!tUfT%G%d`FcR*z4t91P?&!68!;$U|wyJOG26K5m zl7D+I$9I0MttGd3@mSQ3!pPiJpGV4KDd#?qWJ@pitXM8%jYQKizWD1pNK&yx+REqR zG{8CEICR7h5GMP1cPWQP|H%J9p^wkKH=Q{^`h3QK{)ALOaSq^Zl;Z?_ zoClsJjB|JBBcBAtIfg`ZVH@spgcv>}Kp%N0DE{{V{~-tZ0_pRa13dCmP(wQN`Y*6L zNfnMM^p8q_K{0;x?;ZbX(T|8hoU1@_4rBd?>F3qIKp4kFK2x)KG4f;j@|P5GZx$<1 z&8?6885I3PAt)8lFM9NmXM@6D=z|CSibo&cXV4#dCLI03-q$?(ICq13(>d?>zbX2d zKjayp&FDxd{096Tk3P-;pnD)F)Z70LJo-2<%u>(zz5V}$^f~v&`4{;m{=evz2M_vl z(ye#thJbQSvpc3&IN54)h=aiN8zaSmVKv1Yx|6e)`PrLvC literal 0 HcmV?d00001 From 21a7aac220ef5b054ea127cd11136e5c8b5211dd Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 11 Sep 2018 06:40:56 +0100 Subject: [PATCH 176/212] tests: import custom binaries for tests Same for async tests. --- .../async/result_binary_producing_json.yml | 19 +++++++++++++++---- .../async/result_binary_producing_junk.yml | 19 +++++++++++++++---- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/tests/ansible/integration/async/result_binary_producing_json.yml b/tests/ansible/integration/async/result_binary_producing_json.yml index 61d63a08..a53923d6 100644 --- a/tests/ansible/integration/async/result_binary_producing_json.yml +++ b/tests/ansible/integration/async/result_binary_producing_json.yml @@ -5,10 +5,21 @@ any_errors_fatal: true tasks: - - custom_binary_producing_json: - async: 100 - poll: 0 - register: job + - block: + - custom_binary_producing_json_Darwin: + async: 100 + poll: 0 + register: job_darwin + - set_fact: job={{job_darwin}} + when: ansible_system == "Darwin" + + - block: + - custom_binary_producing_json_Linux: + async: 100 + poll: 0 + register: job_linux + - set_fact: job={{job_linux}} + when: ansible_system == "Linux" - assert: that: | diff --git a/tests/ansible/integration/async/result_binary_producing_junk.yml b/tests/ansible/integration/async/result_binary_producing_junk.yml index 37f31704..e1628501 100644 --- a/tests/ansible/integration/async/result_binary_producing_junk.yml +++ b/tests/ansible/integration/async/result_binary_producing_junk.yml @@ -5,10 +5,21 @@ any_errors_fatal: true tasks: - - custom_binary_producing_junk: - async: 100 - poll: 0 - register: job + - block: + - custom_binary_producing_junk_Darwin: + async: 100 + poll: 0 + register: job_darwin + - set_fact: job={{job_darwin}} + when: ansible_system == "Darwin" + + - block: + - custom_binary_producing_junk_Linux: + async: 100 + poll: 0 + register: job_linux + - set_fact: job={{job_linux}} + when: ansible_system == "Linux" - shell: sleep 1 From 2eb3ea78d642f0c165e876fdd4a532283b35ff07 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 11 Sep 2018 06:59:12 +0100 Subject: [PATCH 177/212] tests: remove a bunch of stray debug --- .../integration/action/low_level_execute_command.yml | 1 - tests/ansible/integration/action/make_tmp_path.yml | 2 -- tests/ansible/integration/action/transfer_data.yml | 2 -- .../integration/async/result_binary_producing_json.yml | 6 +++--- .../integration/async/result_binary_producing_junk.yml | 6 +++--- tests/ansible/integration/async/result_shell_echo_hi.yml | 6 +++--- tests/ansible/integration/become/sudo_flags_failure.yml | 3 +-- .../integration/delegation/osa_container_standalone.yml | 1 - tests/ansible/integration/glibc_caches/resolv_conf.yml | 1 - .../integration/module_utils/adjacent_to_playbook.yml | 1 - .../integration/module_utils/roles/modrole/tasks/main.yml | 1 - .../module_utils/roles/overrides_modrole/tasks/main.yml | 1 - .../ansible/integration/playbook_semantics/environment.yml | 2 -- .../integration/runner/custom_binary_producing_json.yml | 1 - .../integration/runner/custom_binary_producing_junk.yml | 1 - 15 files changed, 10 insertions(+), 25 deletions(-) diff --git a/tests/ansible/integration/action/low_level_execute_command.yml b/tests/ansible/integration/action/low_level_execute_command.yml index 842d99d2..a42fa877 100644 --- a/tests/ansible/integration/action/low_level_execute_command.yml +++ b/tests/ansible/integration/action/low_level_execute_command.yml @@ -23,7 +23,6 @@ register: raw # Can't test stdout because TTY inserts \r in Ansible version. - - debug: msg={{raw}} - name: Verify raw module output. assert: that: diff --git a/tests/ansible/integration/action/make_tmp_path.yml b/tests/ansible/integration/action/make_tmp_path.yml index dc713c31..0631727d 100644 --- a/tests/ansible/integration/action/make_tmp_path.yml +++ b/tests/ansible/integration/action/make_tmp_path.yml @@ -60,8 +60,6 @@ path: "{{tmp_path2.result}}" register: stat2 - - debug: msg={{stat1}} - - name: "Verify neither subdir exists any more" assert: that: diff --git a/tests/ansible/integration/action/transfer_data.yml b/tests/ansible/integration/action/transfer_data.yml index c6845cff..bbd39309 100644 --- a/tests/ansible/integration/action/transfer_data.yml +++ b/tests/ansible/integration/action/transfer_data.yml @@ -37,8 +37,6 @@ src: /tmp/transfer-data register: out - - debug: msg={{out}} - - assert: that: out.content|b64decode == 'I am text.' diff --git a/tests/ansible/integration/async/result_binary_producing_json.yml b/tests/ansible/integration/async/result_binary_producing_json.yml index a53923d6..f81d0bb2 100644 --- a/tests/ansible/integration/async/result_binary_producing_json.yml +++ b/tests/ansible/integration/async/result_binary_producing_json.yml @@ -41,9 +41,9 @@ src: "{{ansible_user_dir}}/.ansible_async/{{job.ansible_job_id}}" register: result - - debug: msg={{async_out}} - vars: - async_out: "{{result.content|b64decode|from_json}}" + #- debug: msg={{async_out}} + #vars: + #async_out: "{{result.content|b64decode|from_json}}" - assert: that: diff --git a/tests/ansible/integration/async/result_binary_producing_junk.yml b/tests/ansible/integration/async/result_binary_producing_junk.yml index e1628501..87877db7 100644 --- a/tests/ansible/integration/async/result_binary_producing_junk.yml +++ b/tests/ansible/integration/async/result_binary_producing_junk.yml @@ -27,9 +27,9 @@ src: "{{ansible_user_dir}}/.ansible_async/{{job.ansible_job_id}}" register: result - - debug: msg={{async_out}} - vars: - async_out: "{{result.content|b64decode|from_json}}" + #- debug: msg={{async_out}} + #vars: + #async_out: "{{result.content|b64decode|from_json}}" - assert: that: diff --git a/tests/ansible/integration/async/result_shell_echo_hi.yml b/tests/ansible/integration/async/result_shell_echo_hi.yml index 77678318..8858037a 100644 --- a/tests/ansible/integration/async/result_shell_echo_hi.yml +++ b/tests/ansible/integration/async/result_shell_echo_hi.yml @@ -16,9 +16,9 @@ src: "{{ansible_user_dir}}/.ansible_async/{{job.ansible_job_id}}" register: result - - debug: msg={{async_out}} - vars: - async_out: "{{result.content|b64decode|from_json}}" + #- debug: msg={{async_out}} + #vars: + #async_out: "{{result.content|b64decode|from_json}}" - assert: that: diff --git a/tests/ansible/integration/become/sudo_flags_failure.yml b/tests/ansible/integration/become/sudo_flags_failure.yml index 484134c5..52404019 100644 --- a/tests/ansible/integration/become/sudo_flags_failure.yml +++ b/tests/ansible/integration/become/sudo_flags_failure.yml @@ -11,12 +11,11 @@ vars: ansible_become_flags: --derps - - debug: msg={{out}} - name: Verify raw module output. assert: that: - out.failed - | ('sudo: no such option: --derps' in out.msg) or - ("sudo: unrecognized option `--derps'" in out.module_stderr) or + ("sudo: unrecognized option `--derps'" in out.module_stderr) or ("sudo: unrecognized option '--derps'" in out.module_stderr) diff --git a/tests/ansible/integration/delegation/osa_container_standalone.yml b/tests/ansible/integration/delegation/osa_container_standalone.yml index 97830d28..b942ef63 100644 --- a/tests/ansible/integration/delegation/osa_container_standalone.yml +++ b/tests/ansible/integration/delegation/osa_container_standalone.yml @@ -10,7 +10,6 @@ - mitogen_get_stack: register: out - - debug: msg={{out}} - assert: that: | out.result == [ diff --git a/tests/ansible/integration/glibc_caches/resolv_conf.yml b/tests/ansible/integration/glibc_caches/resolv_conf.yml index d1a466e9..643b83ec 100644 --- a/tests/ansible/integration/glibc_caches/resolv_conf.yml +++ b/tests/ansible/integration/glibc_caches/resolv_conf.yml @@ -9,7 +9,6 @@ ansible_become_pass: has_sudo_pubkey_password tasks: - - debug: msg={{hostvars}} - mitogen_test_gethostbyname: name: www.google.com register: out diff --git a/tests/ansible/integration/module_utils/adjacent_to_playbook.yml b/tests/ansible/integration/module_utils/adjacent_to_playbook.yml index 34cf1c5d..63bd90b2 100644 --- a/tests/ansible/integration/module_utils/adjacent_to_playbook.yml +++ b/tests/ansible/integration/module_utils/adjacent_to_playbook.yml @@ -9,7 +9,6 @@ - custom_python_external_module: register: out - - debug: msg={{out}} - assert: that: - out.external1_path == "ansible/integration/module_utils/module_utils/external1.py" diff --git a/tests/ansible/integration/module_utils/roles/modrole/tasks/main.yml b/tests/ansible/integration/module_utils/roles/modrole/tasks/main.yml index 857abae5..2c7c3372 100644 --- a/tests/ansible/integration/module_utils/roles/modrole/tasks/main.yml +++ b/tests/ansible/integration/module_utils/roles/modrole/tasks/main.yml @@ -3,7 +3,6 @@ - uses_external3: register: out -- debug: msg={{out}} - assert: that: - out.external3_path == "integration/module_utils/roles/modrole/module_utils/external3.py" diff --git a/tests/ansible/integration/module_utils/roles/overrides_modrole/tasks/main.yml b/tests/ansible/integration/module_utils/roles/overrides_modrole/tasks/main.yml index 24717693..6ef4703a 100644 --- a/tests/ansible/integration/module_utils/roles/overrides_modrole/tasks/main.yml +++ b/tests/ansible/integration/module_utils/roles/overrides_modrole/tasks/main.yml @@ -3,7 +3,6 @@ - uses_custom_known_hosts: register: out -- debug: msg={{out}} - assert: that: - out.path == "ansible/integration/module_utils/roles/override_modrole/module_utils/known_hosts.py" diff --git a/tests/ansible/integration/playbook_semantics/environment.yml b/tests/ansible/integration/playbook_semantics/environment.yml index 1c183a5a..1ac7f71d 100644 --- a/tests/ansible/integration/playbook_semantics/environment.yml +++ b/tests/ansible/integration/playbook_semantics/environment.yml @@ -9,7 +9,5 @@ SOME_ENV: 123 register: result - - debug: msg={{result}} - - assert: that: "result.stdout == '123'" diff --git a/tests/ansible/integration/runner/custom_binary_producing_json.yml b/tests/ansible/integration/runner/custom_binary_producing_json.yml index 4fe09f0d..a3b8a224 100644 --- a/tests/ansible/integration/runner/custom_binary_producing_json.yml +++ b/tests/ansible/integration/runner/custom_binary_producing_json.yml @@ -19,7 +19,6 @@ - set_fact: out={{out_linux}} when: ansible_system == "Linux" - - debug: msg={{out}} - assert: that: | out.changed and diff --git a/tests/ansible/integration/runner/custom_binary_producing_junk.yml b/tests/ansible/integration/runner/custom_binary_producing_junk.yml index b1672ad9..41572aad 100644 --- a/tests/ansible/integration/runner/custom_binary_producing_junk.yml +++ b/tests/ansible/integration/runner/custom_binary_producing_junk.yml @@ -24,7 +24,6 @@ - hosts: test-targets any_errors_fatal: true tasks: - - debug: msg={{out}} - assert: that: | out.failed and From 86ff2cc7685ab59906cc0b26347ff3dc98ba4519 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 11 Sep 2018 07:44:43 +0100 Subject: [PATCH 178/212] bodge. --- .travis/ci_lib.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis/ci_lib.py b/.travis/ci_lib.py index 828cae39..eb130a14 100644 --- a/.travis/ci_lib.py +++ b/.travis/ci_lib.py @@ -10,6 +10,8 @@ import shlex import shutil import tempfile +import os +os.system('curl -H Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/machine-type') # # check_output() monkeypatch cutpasted from testlib.py From f8b6c774ddd81e8f3cb1fd18b1a7211aaaf0f031 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 11 Sep 2018 18:22:58 +0100 Subject: [PATCH 179/212] issue #362: cap max open files in children. --- ansible_mitogen/target.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index ae7990a9..347dc3c2 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -43,6 +43,7 @@ import operator import os import pwd import re +import resource import signal import stat import subprocess @@ -90,6 +91,15 @@ _fork_parent = None good_temp_dir = None +# subprocess.Popen(close_fds=True) aka. AnsibleModule.run_command() loops the +# entire SC_OPEN_MAX space. CentOS>5 ships with 1,048,576 FDs by default, +# resulting in huge (>500ms) runtime waste running many commands. Therefore if +# we are a child context, cap the insane FD count to something reasonable. +if subprocess.MAXFD > 512 and not mitogen.is_master: + subprocess.MAXFD = 512 + resource.setrlimit(resource.RLIMIT_NOFILE, (512, 512)) + + def get_small_file(context, path): """ Basic in-memory caching module fetcher. This generates an one roundtrip for From 9fadd22396b03935c5e7eb8ce529d1a1245f9e41 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 11 Sep 2018 18:37:34 +0100 Subject: [PATCH 180/212] docs: update Changelog; closes #362. --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2c219c4d..0027ee7d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -125,6 +125,11 @@ Fixes This meant built-in modules overridden via a custom ``module_utils`` search path may not have had any effect. +* `#362 `_: to work around a slow + algorithm in the :mod:`subprocess` module, the maximum number of open files + in processes running on the target is capped to 512, reducing the work + required to start a subprocess by >2000x in default CentOS configurations. + * A missing check caused an exception traceback to appear when using the ``ansible`` command-line tool with a missing or misspelled module name. From f8bf780e212ecaf95222b8cae335945ac23e6ba1 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 11 Sep 2018 21:01:51 +0100 Subject: [PATCH 181/212] issue #362: Py3.x fixes. --- ansible_mitogen/target.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index 347dc3c2..026ddda7 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -91,13 +91,15 @@ _fork_parent = None good_temp_dir = None -# subprocess.Popen(close_fds=True) aka. AnsibleModule.run_command() loops the -# entire SC_OPEN_MAX space. CentOS>5 ships with 1,048,576 FDs by default, -# resulting in huge (>500ms) runtime waste running many commands. Therefore if -# we are a child context, cap the insane FD count to something reasonable. -if subprocess.MAXFD > 512 and not mitogen.is_master: - subprocess.MAXFD = 512 +# issue #362: subprocess.Popen(close_fds=True) aka. AnsibleModule.run_command() +# loops the entire SC_OPEN_MAX space. CentOS>5 ships with 1,048,576 FDs by +# default, resulting in huge (>500ms) runtime waste running many commands. +# Therefore if we are a child, cap the range to something reasonable. +rlimit = resource.getrlimit(resource.RLIMIT_NOFILE) +if (rlimit[0] > 512 or rlimit[1] > 512) and not mitogen.is_master: resource.setrlimit(resource.RLIMIT_NOFILE, (512, 512)) + subprocess.MAXFD = 512 # Python <3.x +del rlimit def get_small_file(context, path): From 498db57ec84083d92a9f512fa3ac1ed90625eb69 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 12 Sep 2018 00:34:53 +0100 Subject: [PATCH 182/212] issue #360: ansible: missing lock around ContextService.put(). --- ansible_mitogen/services.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index 59c26ba2..fdc7e2a7 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -139,11 +139,15 @@ class ContextService(mitogen.service.Service): count reaches zero. """ LOG.debug('%r.put(%r)', self, context) - if self._refs_by_context.get(context, 0) == 0: - LOG.warning('%r.put(%r): refcount was 0. shutdown_all called?', - self, context) - return - self._refs_by_context[context] -= 1 + self._lock.acquire() + try: + if self._refs_by_context.get(context, 0) == 0: + LOG.warning('%r.put(%r): refcount was 0. shutdown_all called?', + self, context) + return + self._refs_by_context[context] -= 1 + finally: + self._lock.release() def key_from_kwargs(self, **kwargs): """ From 7a00e1cc87f510e41578bc1323e2e68fb8749c6b Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 12 Sep 2018 18:56:06 +0100 Subject: [PATCH 183/212] issue #360: missing locks around shutdown and LRU management. --- ansible_mitogen/services.py | 44 +++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index fdc7e2a7..199f2116 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -187,29 +187,24 @@ class ContextService(mitogen.service.Service): self._lock.release() return count - def _shutdown(self, context, lru=None, new_context=None): + def _shutdown_unlocked(self, context, lru=None, new_context=None): """ Arrange for `context` to be shut down, and optionally add `new_context` to the LRU list while holding the lock. """ - LOG.info('%r._shutdown(): shutting down %r', self, context) + LOG.info('%r._shutdown_unlocked(): shutting down %r', self, context) context.shutdown() key = self._key_by_context[context] + del self._response_by_key[key] + del self._refs_by_context[context] + del self._key_by_context[context] + if lru and context in lru: + lru.remove(context) + if new_context: + lru.append(new_context) - self._lock.acquire() - try: - del self._response_by_key[key] - del self._refs_by_context[context] - del self._key_by_context[context] - if lru and context in lru: - lru.remove(context) - if new_context: - lru.append(new_context) - finally: - self._lock.release() - - def _update_lru(self, new_context, spec, via): + def _update_lru_unlocked(self, new_context, spec, via): """ Update the LRU ("MRU"?) list associated with the connection described by `kwargs`, destroying the most recently created context if the list @@ -228,16 +223,27 @@ class ContextService(mitogen.service.Service): 'but they are all marked as in-use.', via) return - self._shutdown(context, lru=lru, new_context=new_context) + self._shutdown_unlocked(context, lru=lru, new_context=new_context) + + def _update_lru(self, new_context, spec, via): + self._lock.acquire() + try: + self._update_lru_unlocked(new_context, spec, via) + finally: + self._lock.release() @mitogen.service.expose(mitogen.service.AllowParents()) def shutdown_all(self): """ For testing use, arrange for all connections to be shut down. """ - for context in list(self._key_by_context): - self._shutdown(context) - self._lru_by_via = {} + self._lock.acquire() + try: + for context in list(self._key_by_context): + self._shutdown_unlocked(context) + self._lru_by_via = {} + finally: + self._lock.release() def _on_stream_disconnect(self, stream): """ From 6dddef0c453eaeed8611183ca1a3f408a434cc5d Mon Sep 17 00:00:00 2001 From: David Wilson Date: Fri, 14 Sep 2018 18:10:16 +0000 Subject: [PATCH 184/212] Make image_prep work on Ubuntu. --- tests/image_prep/setup.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/image_prep/setup.yml b/tests/image_prep/setup.yml index 7a589239..77a80e3b 100644 --- a/tests/image_prep/setup.yml +++ b/tests/image_prep/setup.yml @@ -7,6 +7,7 @@ sudo_group: MacOSX: admin Debian: sudo + Ubuntu: sudo CentOS: wheel - import_playbook: _container_setup.yml From f6b201bdfc69c72b72e082476ce9d551f7e05617 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 19 Sep 2018 19:52:14 +0100 Subject: [PATCH 185/212] docs: updates for #376 and #371 --- docs/ansible.rst | 16 ++++++++++++++++ docs/api.rst | 20 ++++++++++++++++++++ docs/changelog.rst | 13 +++++++++++-- mitogen/kubectl.py | 7 ++----- 4 files changed, 49 insertions(+), 7 deletions(-) diff --git a/docs/ansible.rst b/docs/ansible.rst index d2138f21..529dd4a6 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -155,6 +155,7 @@ Noteworthy Differences * The `docker `_, `jail `_, + `kubectl `_, `local `_, `lxc `_, `lxd `_, @@ -681,6 +682,8 @@ connection delegation is supported. * ``ansible_user``: Name of user within the container to execute as. +.. _method-jail: + FreeBSD Jail ~~~~~~~~~~~~ @@ -692,6 +695,19 @@ connection delegation is supported. * ``ansible_user``: Name of user within the jail to execute as. +.. _method-kubectl: + +Kubernetes Pod +~~~~~~~~~~~~~~ + +Like `kubectl +`_ except +connection delegation is supported. + +* ``ansible_host``: Name of pod (default: inventory hostname). +* ``ansible_user``: Name of user to authenticate to API as. + + Local ~~~~~ diff --git a/docs/api.rst b/docs/api.rst index c74193e3..f365372d 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -589,6 +589,26 @@ Router Class Filename or complete path to the ``jexec`` binary. ``PATH`` will be searched if given as a filename. Defaults to ``/usr/sbin/jexec``. + .. method:: kubectl (pid=None, container=None, kubectl_path=None, username=None, \**kwargs) + + Construct a context in a container via the Kubernetes ``kubectl`` + program. + + Accepts all parameters accepted by :meth:`local`, in addition to: + + :param str pod: + Kubernetes pod to connect to. + :param str container: + Optional container within pod to connect to. If the pod has only + one container, this parameter is not required. Defaults to + :data:`None`. + :param str kubectl_path: + Filename or complete path to the ``kubectl`` binary. ``PATH`` will + be searched if given as a filename. Defaults to ``kubectl``. + :param str username: + Optional username to authenticate to the Kubernetes API server + with. within the container to :func:`setuid` to. + .. method:: lxc (container, lxc_attach_path=None, \**kwargs) Construct a context on the local machine within an LXC classic diff --git a/docs/changelog.rst b/docs/changelog.rst index 0027ee7d..4ba20f8b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -41,6 +41,9 @@ Enhancements `uri `_). See :ref:`ansible_tempfiles` for a complete description. +* `#376 `_: the ``kubectl`` connection + type is now supported. Contributed by Yannig Perré. + * `084c0ac0 `_: avoid a roundtrip in `copy `_ and @@ -170,6 +173,10 @@ Core Library * `#345 `_: the SSH connection method allows optionally disabling ``IdentitiesOnly yes``. +* `#371 `_: the LXC connection method + uses a more compatible method to establish an non-interactive session. + Contributed by Brian Candler. + * `af2ded66 `_: add :func:`mitogen.fork.on_fork` to allow non-Mitogen managed process forks to clean up Mitogen resources in the child. @@ -191,6 +198,7 @@ the bug reports in this release contributed by `Alex Russu `_, `atoom `_, `Berend De Schouwer `_, +`Brian Candler `_, `Dan Quackenbush `_, `dsgnr `_, `Jesse London `_, @@ -203,8 +211,9 @@ the bug reports in this release contributed by `Pierre-Louis Bonicoli `_, `Prateek Jain `_, `Rick Box `_, -`Tawana Musewe `_, and -`Timo Beckers `_. +`Tawana Musewe `_, +`Timo Beckers `_, and +`Yannig Perré `_. v0.2.2 (2018-07-26) diff --git a/mitogen/kubectl.py b/mitogen/kubectl.py index 2dfaa232..26480f7d 100644 --- a/mitogen/kubectl.py +++ b/mitogen/kubectl.py @@ -48,13 +48,10 @@ class Stream(mitogen.parent.Stream): 'merge_stdio': True } - def construct(self, pod = None, container=None, - kubectl_path=None, username=None, + def construct(self, pod, container=None, kubectl_path=None, username=None, **kwargs): - assert pod super(Stream, self).construct(**kwargs) - if pod: - self.pod = pod + self.pod = pod if container: self.container = container if kubectl_path: From 41466487597f4807e46aabf84f91e3f239757422 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 1 Oct 2018 20:16:19 +0100 Subject: [PATCH 186/212] master: log error an refuse __main__ import if no guard detected. Closes #366. --- mitogen/master.py | 19 +++++++++++--- tests/responder_test.py | 55 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/mitogen/master.py b/mitogen/master.py index 8df4cd5d..0a434a94 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -553,6 +553,14 @@ class ModuleResponder(object): return 'ModuleResponder(%r)' % (self._router,) MAIN_RE = re.compile(b(r'^if\s+__name__\s*==\s*.__main__.\s*:'), re.M) + main_guard_msg = ( + "A child context attempted to import __main__, however the main " + "module present in the master process lacks an execution guard. " + "Update %r to prevent unintended execution, using a guard like:\n" + "\n" + " if __name__ == '__main__':\n" + " # your code here.\n" + ) def whitelist_prefix(self, fullname): if self.whitelist == ['']: @@ -562,14 +570,19 @@ class ModuleResponder(object): def blacklist_prefix(self, fullname): self.blacklist.append(fullname) - def neutralize_main(self, src): + def neutralize_main(self, path, src): """Given the source for the __main__ module, try to find where it begins conditional execution based on a "if __name__ == '__main__'" guard, and remove any code after that point.""" match = self.MAIN_RE.search(src) if match: return src[:match.start()] - return src + + if b('mitogen.main(') in src: + return src + + LOG.error(self.main_guard_msg, path) + raise ImportError('refused') def _make_negative_response(self, fullname): return (fullname, None, None, None, ()) @@ -596,7 +609,7 @@ class ModuleResponder(object): pkg_present = None if fullname == '__main__': - source = self.neutralize_main(source) + source = self.neutralize_main(path, source) compressed = mitogen.core.Blob(zlib.compress(source, 9)) related = [ to_text(name) diff --git a/tests/responder_test.py b/tests/responder_test.py index dfdd67fa..46400fce 100644 --- a/tests/responder_test.py +++ b/tests/responder_test.py @@ -1,5 +1,6 @@ import mock +import textwrap import subprocess import sys @@ -12,6 +13,60 @@ import plain_old_module import simple_pkg.a +class NeutralizeMainTest(testlib.RouterMixin, unittest2.TestCase): + klass = mitogen.master.ModuleResponder + + def call(self, *args, **kwargs): + return self.klass(self.router).neutralize_main(*args, **kwargs) + + def test_missing_exec_guard(self): + path = testlib.data_path('main_with_no_exec_guard.py') + args = [sys.executable, path] + proc = subprocess.Popen(args, stderr=subprocess.PIPE) + _, stderr = proc.communicate() + self.assertEquals(1, proc.returncode) + expect = self.klass.main_guard_msg % (path,) + self.assertTrue(expect in stderr.decode()) + + HAS_MITOGEN_MAIN = mitogen.core.b( + textwrap.dedent(""" + herp derp + + def myprog(): + pass + + @mitogen.main(maybe_some_option=True) + def main(router): + pass + """) + ) + + def test_mitogen_main(self): + untouched = self.call("derp.py", self.HAS_MITOGEN_MAIN) + self.assertEquals(untouched, self.HAS_MITOGEN_MAIN) + + HAS_EXEC_GUARD = mitogen.core.b( + textwrap.dedent(""" + herp derp + + def myprog(): + pass + + def main(): + pass + + if __name__ == '__main__': + main() + """) + ) + + def test_exec_guard(self): + touched = self.call("derp.py", self.HAS_EXEC_GUARD) + bits = touched.decode().split() + self.assertEquals(bits[-3:], ['def', 'main():', 'pass']) + + + class GoodModulesTest(testlib.RouterMixin, unittest2.TestCase): def test_plain_old_module(self): # The simplest case: a top-level module with no interesting imports or From 0abb6b0880e8a4db72d334a3a8297b36c546fca0 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Mon, 1 Oct 2018 20:21:29 +0100 Subject: [PATCH 187/212] issue 366: update changelog. --- docs/changelog.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4ba20f8b..c64e999b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -173,6 +173,12 @@ Core Library * `#345 `_: the SSH connection method allows optionally disabling ``IdentitiesOnly yes``. +* `#366 `_, + `#380 `_: attempts by children to + import :mod:`__main__` where the main program module lacks an execution guard + are refused, and an error is logged. This prevents a common and highly + confusing error when prototyping new scripts. + * `#371 `_: the LXC connection method uses a more compatible method to establish an non-interactive session. Contributed by Brian Candler. @@ -202,6 +208,7 @@ the bug reports in this release contributed by `Dan Quackenbush `_, `dsgnr `_, `Jesse London `_, +`John McGrath `_, `Jonathan Rosser `_, `Josh Smift `_, `Luca Nunzi `_, @@ -210,6 +217,7 @@ the bug reports in this release contributed by `Pierre-Henry Muller `_, `Pierre-Louis Bonicoli `_, `Prateek Jain `_, +`RedheatWei `_, `Rick Box `_, `Tawana Musewe `_, `Timo Beckers `_, and From 130e42a932c794059bb11299bdffec4b1a661b88 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 2 Oct 2018 20:30:18 +0100 Subject: [PATCH 188/212] tests: prevent compare_output_test running on import. --- tests/ansible/compare_output_test.py | 38 ++++++++++++++-------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/ansible/compare_output_test.py b/tests/ansible/compare_output_test.py index 5a091f15..e4a3565f 100755 --- a/tests/ansible/compare_output_test.py +++ b/tests/ansible/compare_output_test.py @@ -6,7 +6,6 @@ import re import subprocess import tempfile - LOG = logging.getLogger(__name__) suffixes = [ @@ -42,21 +41,22 @@ def run(s): return fp.read() -logging.basicConfig(level=logging.DEBUG) - -for suffix in suffixes: - ansible = run('ansible localhost %s' % (suffix,)) - mitogen = run('ANSIBLE_STRATEGY=mitogen ansible localhost %s' % (suffix,)) - - diff = list(difflib.unified_diff( - a=fixup(ansible).splitlines(), - b=fixup(mitogen).splitlines(), - fromfile='ansible-output.txt', - tofile='mitogen-output.txt', - )) - if diff: - print('++ differ! suffix: %r' % (suffix,)) - for line in diff: - print(line) - print - print +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) + + for suffix in suffixes: + ansible = run('ansible localhost %s' % (suffix,)) + mitogen = run('ANSIBLE_STRATEGY=mitogen ansible localhost %s' % (suffix,)) + + diff = list(difflib.unified_diff( + a=fixup(ansible).splitlines(), + b=fixup(mitogen).splitlines(), + fromfile='ansible-output.txt', + tofile='mitogen-output.txt', + )) + if diff: + print('++ differ! suffix: %r' % (suffix,)) + for line in diff: + print(line) + print + print From 6dd1001d7af05054e950f8af9730d0a158b50f82 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 2 Oct 2018 20:31:17 +0100 Subject: [PATCH 189/212] tests: move kubectl into new subdir Fixes tab completion with tests/ dir :) CC @yannig --- tests/ansible/integration/transport/README.md | 2 ++ tests/ansible/integration/transport/all.yml | 2 ++ .../{test-kubectl.yml => integration/transport/kubectl.yml} | 0 3 files changed, 4 insertions(+) create mode 100644 tests/ansible/integration/transport/README.md create mode 100644 tests/ansible/integration/transport/all.yml rename tests/ansible/{test-kubectl.yml => integration/transport/kubectl.yml} (100%) diff --git a/tests/ansible/integration/transport/README.md b/tests/ansible/integration/transport/README.md new file mode 100644 index 00000000..9a31a530 --- /dev/null +++ b/tests/ansible/integration/transport/README.md @@ -0,0 +1,2 @@ + +# Integration tests that require a real target available. diff --git a/tests/ansible/integration/transport/all.yml b/tests/ansible/integration/transport/all.yml new file mode 100644 index 00000000..89949b58 --- /dev/null +++ b/tests/ansible/integration/transport/all.yml @@ -0,0 +1,2 @@ + +- import_playbook: kubectl.yml diff --git a/tests/ansible/test-kubectl.yml b/tests/ansible/integration/transport/kubectl.yml similarity index 100% rename from tests/ansible/test-kubectl.yml rename to tests/ansible/integration/transport/kubectl.yml From 62f7963da9618c8036d79ac0e3df1b60aef01168 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 2 Oct 2018 20:42:24 +0100 Subject: [PATCH 190/212] tests: make ansible/tests/ run in run_tests. --- run_tests | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/run_tests b/run_tests index 122cd79e..65bf1fef 100755 --- a/run_tests +++ b/run_tests @@ -6,15 +6,32 @@ echo '-------------------' echo set -o errexit -set -o nounset set -o pipefail UNIT2="$(which unit2)" coverage erase -coverage run "${UNIT2}" discover \ - --start-directory "tests" \ - --pattern '*_test.py' \ - "$@" + +# First run overwites coverage output. +[ "$SKIP_MITOGEN" ] || { + coverage run "${UNIT2}" discover \ + --start-directory "tests" \ + --pattern '*_test.py' \ + "$@" +} + +# Second run appends. This is since 'discover' treats subdirs as packages and +# the 'ansible' subdir shadows the real Ansible package when it contains +# __init__.py, so hack around it by just running again with 'ansible' as the +# start directory. Alternative seems to be renaming tests/ansible/ and making a +# mess of Git history. +[ "$SKIP_ANSIBLE" ] || { + export PYTHONPATH=`pwd`/tests:$PYTHONPATH + coverage run -a "${UNIT2}" discover \ + --start-directory "tests/ansible" \ + --pattern '*_test.py' \ + "$@" +} + coverage html echo coverage report is at "file://$(pwd)/htmlcov/index.html" From 9d070541d92234a398169c6652abff3d80359fc4 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 2 Oct 2018 21:06:00 +0100 Subject: [PATCH 191/212] ansible: try to create tempdir if missing. Closes #358. --- ansible_mitogen/target.py | 72 +++++++++++++++++---------- tests/ansible/tests/helpers_test.py | 20 -------- tests/ansible/tests/target_test.py | 77 +++++++++++++++++++++++++++++ tests/testlib.py | 30 ++++++++++- 4 files changed, 150 insertions(+), 49 deletions(-) delete mode 100644 tests/ansible/tests/helpers_test.py create mode 100644 tests/ansible/tests/target_test.py diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index 026ddda7..ff6ed083 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -214,6 +214,50 @@ def _on_broker_shutdown(): prune_tree(temp_dir) +def is_good_temp_dir(path): + """ + Return :data:`True` if `path` can be used as a temporary directory, logging + any failures that may cause it to be unsuitable. If the directory doesn't + exist, we attempt to create it using :func:`os.makedirs`. + """ + if not os.path.exists(path): + try: + os.makedirs(path, mode=int('0700', 8)) + except OSError as e: + LOG.debug('temp dir %r unusable: did not exist and attempting ' + 'to create it failed: %s', path, e) + return False + + try: + tmp = tempfile.NamedTemporaryFile( + prefix='ansible_mitogen_is_good_temp_dir', + dir=path, + ) + except (OSError, IOError) as e: + LOG.debug('temp dir %r unusable: %s', path, e) + return False + + try: + try: + os.chmod(tmp.name, int('0700', 8)) + except OSError as e: + LOG.debug('temp dir %r unusable: %s: chmod failed: %s', + path, e) + return False + + try: + # access(.., X_OK) is sufficient to detect noexec. + if not os.access(tmp.name, os.X_OK): + raise OSError('filesystem appears to be mounted noexec') + except OSError as e: + LOG.debug('temp dir %r unusable: %s: %s', path, e) + return False + finally: + tmp.close() + + return True + + def find_good_temp_dir(candidate_temp_dirs): """ Given a list of candidate temp directories extracted from ``ansible.cfg``, @@ -230,35 +274,9 @@ def find_good_temp_dir(candidate_temp_dirs): paths.extend(tempfile._candidate_tempdir_list()) for path in paths: - try: - tmp = tempfile.NamedTemporaryFile( - prefix='ansible_mitogen_find_good_temp_dir', - dir=path, - ) - except (OSError, IOError) as e: - LOG.debug('temp dir %r unusable: %s', path, e) - continue - - try: - try: - os.chmod(tmp.name, int('0700', 8)) - except OSError as e: - LOG.debug('temp dir %r unusable: %s: chmod failed: %s', - path, e) - continue - - try: - # access(.., X_OK) is sufficient to detect noexec. - if not os.access(tmp.name, os.X_OK): - raise OSError('filesystem appears to be mounted noexec') - except OSError as e: - LOG.debug('temp dir %r unusable: %s: %s', path, e) - continue - + if is_good_temp_dir(path): LOG.debug('Selected temp directory: %r (from %r)', path, paths) return path - finally: - tmp.close() raise IOError(MAKE_TEMP_FAILED_MSG % { 'paths': '\n '.join(paths), diff --git a/tests/ansible/tests/helpers_test.py b/tests/ansible/tests/helpers_test.py deleted file mode 100644 index 95973b1f..00000000 --- a/tests/ansible/tests/helpers_test.py +++ /dev/null @@ -1,20 +0,0 @@ - -import unittest2 - -import ansible_mitogen.helpers -import testlib - - -class ApplyModeSpecTest(unittest2.TestCase): - func = staticmethod(ansible_mitogen.helpers.apply_mode_spec) - - def test_simple(self): - spec = 'u+rwx,go=x' - self.assertEquals(0711, self.func(spec, 0)) - - spec = 'g-rw' - self.assertEquals(0717, self.func(spec, 0777)) - - -if __name__ == '__main__': - unittest2.main() diff --git a/tests/ansible/tests/target_test.py b/tests/ansible/tests/target_test.py new file mode 100644 index 00000000..e3d59433 --- /dev/null +++ b/tests/ansible/tests/target_test.py @@ -0,0 +1,77 @@ + +from __future__ import absolute_import +import os.path +import subprocess +import tempfile +import unittest2 + +import mock + +import ansible_mitogen.target +import testlib + + +LOGGER_NAME = ansible_mitogen.target.LOG.name + + +class NamedTemporaryDirectory(object): + def __enter__(self): + self.path = tempfile.mkdtemp() + return self.path + + def __exit__(self, _1, _2, _3): + subprocess.check_call(['rm', '-rf', self.path]) + + +class ApplyModeSpecTest(unittest2.TestCase): + func = staticmethod(ansible_mitogen.target.apply_mode_spec) + + def test_simple(self): + spec = 'u+rwx,go=x' + self.assertEquals(0711, self.func(spec, 0)) + + spec = 'g-rw' + self.assertEquals(0717, self.func(spec, 0777)) + + +class IsGoodTempDirTest(unittest2.TestCase): + func = staticmethod(ansible_mitogen.target.is_good_temp_dir) + + def test_creates(self): + with NamedTemporaryDirectory() as temp_path: + bleh = os.path.join(temp_path, 'bleh') + self.assertFalse(os.path.exists(bleh)) + self.assertTrue(self.func(bleh)) + self.assertTrue(os.path.exists(bleh)) + + def test_file_exists(self): + with NamedTemporaryDirectory() as temp_path: + bleh = os.path.join(temp_path, 'bleh') + with open(bleh, 'w') as fp: + fp.write('derp') + self.assertTrue(os.path.isfile(bleh)) + self.assertFalse(self.func(bleh)) + self.assertEquals(open(bleh).read(), 'derp') + + def test_unwriteable(self): + with NamedTemporaryDirectory() as temp_path: + os.chmod(temp_path, 0) + self.assertFalse(self.func(temp_path)) + os.chmod(temp_path, int('0700', 8)) + + @mock.patch('os.chmod') + def test_weird_filesystem(self, os_chmod): + os_chmod.side_effect = OSError('nope') + with NamedTemporaryDirectory() as temp_path: + self.assertFalse(self.func(temp_path)) + + @mock.patch('os.access') + def test_noexec(self, os_access): + os_access.return_value = False + with NamedTemporaryDirectory() as temp_path: + self.assertFalse(self.func(temp_path)) + + + +if __name__ == '__main__': + unittest2.main() diff --git a/tests/testlib.py b/tests/testlib.py index d812609c..63d96233 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -158,22 +158,48 @@ def sync_with_broker(broker, timeout=10.0): sem.get(timeout=10.0) +class CaptureStreamHandler(logging.StreamHandler): + def __init__(self, *args, **kwargs): + super(CaptureStreamHandler, self).__init__(*args, **kwargs) + self.msgs = [] + + def emit(self, msg): + self.msgs.append(msg) + return super(CaptureStreamHandler, self).emit(msg) + + class LogCapturer(object): def __init__(self, name=None): self.sio = StringIO() self.logger = logging.getLogger(name) - self.handler = logging.StreamHandler(self.sio) + self.handler = CaptureStreamHandler(self.sio) self.old_propagate = self.logger.propagate self.old_handlers = self.logger.handlers + self.old_level = self.logger.level def start(self): self.logger.handlers = [self.handler] self.logger.propagate = False + self.logger.level = logging.DEBUG + + def raw(self): + return self.sio.getvalue() + + def msgs(self): + return self.handler.msgs + + def __enter__(self): + self.start() + return self + + def __exit__(self, _1, _2, _3): + self.stop() def stop(self): + self.logger.level = self.old_level self.logger.handlers = self.old_handlers self.logger.propagate = self.old_propagate - return self.sio.getvalue() + return self.raw() class TestCase(unittest2.TestCase): From 0fa5fe5559341bf73414246cda1bd9ec32334b64 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 2 Oct 2018 21:13:05 +0100 Subject: [PATCH 192/212] parent: handle masters with blank sys.executable; closes #356. --- docs/changelog.rst | 5 +++++ mitogen/fakessh.py | 4 ++-- mitogen/parent.py | 26 +++++++++++++++++++++++++- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6f5ac94e..aeaa36f5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -174,6 +174,11 @@ Core Library * `#345 `_: the SSH connection method allows optionally disabling ``IdentitiesOnly yes``. +* `#356 `_: if the master Python + process does not have :data:`sys.executable` set, the default Python + interpreter used for new children on the local machine defaults to + ``"/usr/bin/python"``. + * `#366 `_, `#380 `_: attempts by children to import :mod:`__main__` where the main program module lacks an execution guard diff --git a/mitogen/fakessh.py b/mitogen/fakessh.py index 5667bcad..582017bc 100644 --- a/mitogen/fakessh.py +++ b/mitogen/fakessh.py @@ -436,7 +436,7 @@ def run(dest, router, args, deadline=None, econtext=None): ssh_path = os.path.join(tmp_path, 'ssh') fp = open(ssh_path, 'w') try: - fp.write('#!%s\n' % (sys.executable,)) + fp.write('#!%s\n' % (mitogen.parent.get_sys_executable(),)) fp.write(inspect.getsource(mitogen.core)) fp.write('\n') fp.write('ExternalContext(%r).main()\n' % ( @@ -449,7 +449,7 @@ def run(dest, router, args, deadline=None, econtext=None): env = os.environ.copy() env.update({ 'PATH': '%s:%s' % (tmp_path, env.get('PATH', '')), - 'ARGV0': sys.executable, + 'ARGV0': mitogen.parent.get_sys_executable(), 'SSH_PATH': ssh_path, }) diff --git a/mitogen/parent.py b/mitogen/parent.py index fe5e6889..a4e17b93 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -85,11 +85,35 @@ OPENPTY_MSG = ( "to avoid PTY use." ) +SYS_EXECUTABLE_MSG = ( + "The Python sys.executable variable is unset, indicating Python was " + "unable to determine its original program name. Unless explicitly " + "configured otherwise, child contexts will be started using " + "'/usr/bin/python'" +) +_sys_executable_warning_logged = False + def get_log_level(): return (LOG.level or logging.getLogger().level or logging.INFO) +def get_sys_executable(): + """ + Return :data:`sys.executable` if it is set, otherwise return + ``"/usr/bin/python"`` and log a warning. + """ + if sys.executable: + return sys.executable + + global _sys_executable_warning_logged + if not _sys_executable_warning_logged: + LOG.warn(SYS_EXECUTABLE_MSG) + _sys_executable_warning_logged = True + + return '/usr/bin/python' + + def get_core_source(): """ In non-masters, simply fetch the cached mitogen.core source code via the @@ -841,7 +865,7 @@ class Stream(mitogen.core.Stream): Base for streams capable of starting new slaves. """ #: The path to the remote Python interpreter. - python_path = sys.executable + python_path = get_sys_executable() #: Maximum time to wait for a connection attempt. connect_timeout = 30.0 From 67f26434cbe347c85ad76210b1cafe263fb4432c Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 3 Oct 2018 00:02:42 +0100 Subject: [PATCH 193/212] tests: Run tests on CPython 3.5 and 3.6 with Tox Python 3.0 to 3.4 are excluded because no version of Ansible supports them. Due to their setup.py declarations pip refuses to install Ansible on these versions of Python. --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index ae761121..6bf8bb53 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,8 @@ envlist = py26, py27, + py35, + py36, [testenv] deps = From b9112a9cbb10da3200d07dcc5acc16b2a01b4af9 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 3 Oct 2018 00:06:53 +0100 Subject: [PATCH 194/212] ssh: Fix password authentication with Python 3.x & OpenSSH 7.5+ Since PERMDENIED_PROMPT is a byte string the interpolation was resulting in: b"user@host: b'permission denied'". Needless to say this didn't match. --- mitogen/ssh.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mitogen/ssh.py b/mitogen/ssh.py index f8255865..ee97425b 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -291,8 +291,8 @@ class Stream(mitogen.parent.Stream): raise HostKeyError(self.hostkey_failed_msg) elif buf.lower().startswith(( PERMDENIED_PROMPT, - b("%s@%s: %s" % (self.username, self.hostname, - PERMDENIED_PROMPT)), + b("%s@%s: " % (self.username, self.hostname)) + + PERMDENIED_PROMPT, )): # issue #271: work around conflict with user shell reporting # 'permission denied' e.g. during chdir($HOME) by only matching From 6da31c9deef1dbb001ebabc0c13377b0f44e7e7c Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 3 Oct 2018 00:10:41 +0100 Subject: [PATCH 195/212] docs: Remove unneeded backslash escapes Python 3.x was emitting a DeprecationWarning. AFAICT there has been no impact on the HTML rendering. --- mitogen/core.py | 2 +- mitogen/parent.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mitogen/core.py b/mitogen/core.py index d829d624..9aa95973 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -374,7 +374,7 @@ def io_op(func, *args): :returns: Tuple of `(return_value, disconnected)`, where `return_value` is the - return value of `func(\*args)`, and `disconnected` is :data:`True` if + return value of `func(*args)`, and `disconnected` is :data:`True` if disconnection was detected, otherwise :data:`False`. """ while True: diff --git a/mitogen/parent.py b/mitogen/parent.py index fe5e6889..c0d2d294 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -1256,7 +1256,7 @@ class CallChain(object): def call_async(self, fn, *args, **kwargs): """ - Arrange for `fn(\*args, \**kwargs)` to be invoked on the context's main + Arrange for `fn(*args, **kwargs)` to be invoked on the context's main thread. :param fn: From 191a327d9df10b0eca7af5b8a4751e4e78ce69df Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 3 Oct 2018 00:21:18 +0100 Subject: [PATCH 196/212] docs: Don't redefine links to scp and sftp This addresses an error found while running tox -edocs Warning, treated as error: ./docs/ansible.rst:6:Duplicate explicit target name: "scp(1)". --- docs/ansible.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/ansible.rst b/docs/ansible.rst index 485c24dc..06d4b270 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -271,8 +271,7 @@ command line, or as host and group variables. File Transfer ~~~~~~~~~~~~~ -Normally `sftp(1) `_ or -`scp(1) `_ are used to copy files by the +Normally `sftp(1)`_ or `scp(1)`_ are used to copy files by the `assemble `_, `copy `_, `patch `_, @@ -283,6 +282,9 @@ actions, or when uploading modules with pipelining disabled. With Mitogen copies are implemented natively using the same interpreters, connection tree, and routed message bus that carries RPCs. +.. _scp(1): https://linux.die.net/man/1/scp +.. _sftp(1): https://linux.die.net/man/1/sftp + This permits direct streaming between endpoints regardless of execution environment, without necessitating temporary copies in intermediary accounts or machines, for example when ``become`` is active, or in the presence of @@ -302,8 +304,7 @@ to rename over any existing file. This ensures the file remains consistent at all times, in the event of a crash, or when overlapping `ansible-playbook` runs deploy differing file contents. -The `sftp(1) `_ and `scp(1) -`_ tools may cause undetected data corruption +The `sftp(1)`_ and `scp(1)`_ tools may cause undetected data corruption in the form of truncated files, or files containing intermingled data segments from overlapping runs. As part of normal operation, both tools expose a window where readers may observe inconsistent file contents. @@ -423,8 +424,7 @@ Ansible may: * Create a directory owned by the SSH user either under ``remote_tmp``, or a system-default directory, * Upload action dependencies such as non-new style modules or rendered - templates to that directory via `sftp(1) `_ - or `scp(1) `_. + templates to that directory via `sftp(1)`_ or `scp(1)`_. * Attempt to modify the directory's access control list to grant access to the target user using `setfacl(1) `_, requiring that tool to be installed and a supported filesystem to be in use, From bf34b383eb4169c192725fce744fc300502eb6b3 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 3 Oct 2018 00:22:18 +0100 Subject: [PATCH 197/212] docs: Disambiguate references to Context This addresses an error found while running tox -edocs Warning, treated as error: mitogen/docs/api.rst:469:more than one target found for cross-reference u'Context': mitogen.core.Context, mitogen.parent.Context --- docs/api.rst | 8 +++++--- docs/getting_started.rst | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index c74193e3..6f114a0a 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -458,7 +458,7 @@ Router Class rich data structures that cannot normally be passed via a serialization. - :param Context via: + :param mitogen.core.Context via: Same as the `via` parameter for :meth:`local`. :param bool debug: @@ -1078,7 +1078,8 @@ Select Class each, returning the result of calling `msg.unpickle()` on each in turn. Results are returned in the order they arrived. - This is sugar for handling batch :class:`Context.call_async` + This is sugar for handling batch + :meth:`Context.call_async ` invocations: .. code-block:: python @@ -1245,7 +1246,8 @@ Broker Class .. method:: keep_alive Return :data:`True` if any reader's :attr:`Side.keep_alive` - attribute is :data:`True`, or any :class:`Context` is still + attribute is :data:`True`, or any + :class:`Context ` is still registered that is not the master. Used to delay shutdown while some important work is in progress (e.g. log draining). diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 436b34a1..1c92a559 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -208,7 +208,7 @@ started the context, however as shown, this can be overridden. Calling A Function ------------------ -.. currentmodule:: mitogen.master +.. currentmodule:: mitogen.parent Now that some contexts exist, it is time to execute code in them. Any regular function, static method, or class method reachable directly from module scope From 90823231f9b356254098a4810d7e93bec3c1c7fd Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 3 Oct 2018 00:24:11 +0100 Subject: [PATCH 198/212] docs: Add services page to a hidden table of contents This fixes an error reported by tox -edocs Warning, treated as error: /home/alex/src/mitogen/docs/services.rst:document isn't included in any toctree without promoting the page to a top level TOC entry, since the page appears to be work in progress. --- docs/toc.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/toc.rst b/docs/toc.rst index 357fea3f..7b3274a9 100644 --- a/docs/toc.rst +++ b/docs/toc.rst @@ -15,3 +15,8 @@ Table Of Contents examples internals shame + +.. toctree:: + :hidden: + + services From f5e933e8a2a6b9c41ef5620871a6fd58c99993c0 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 3 Oct 2018 00:47:13 +0100 Subject: [PATCH 199/212] docs: add Alex's fix to changelog --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index aeaa36f5..fa9351f3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -134,6 +134,10 @@ Fixes in processes running on the target is capped to 512, reducing the work required to start a subprocess by >2000x in default CentOS configurations. +* `b9112a9c `_, + `2c287801 `_: OpenSSH 7.5 + permission denied prompts are now recognized. Contributed by Alex Willmer. + * A missing check caused an exception traceback to appear when using the ``ansible`` command-line tool with a missing or misspelled module name. @@ -208,6 +212,7 @@ Thanks! Mitogen would not be possible without the support of users. A huge thanks for the bug reports in this release contributed by `Alex Russu `_, +`Alex Willmer `_, `atoom `_, `Berend De Schouwer `_, `Brian Candler `_, From 48f9fc89308b72df438d415ce4f3dc805169dab5 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 3 Oct 2018 00:50:44 +0100 Subject: [PATCH 200/212] docs: tweak thanks text --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index fa9351f3..1e7fb913 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -210,7 +210,7 @@ Thanks! ~~~~~~~ Mitogen would not be possible without the support of users. A huge thanks for -the bug reports in this release contributed by +bug reports, features and fixes in this release contributed by `Alex Russu `_, `Alex Willmer `_, `atoom `_, From 3aa5c4c53d51ca9103237ea9eae36d5c95478462 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 3 Oct 2018 13:51:11 +0100 Subject: [PATCH 201/212] issue #373: parse the child process wait status Don't log the raw waitpid() result, convert it to a useful string first. --- mitogen/parent.py | 23 ++++++++++++++++++++++- tests/parent_test.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/mitogen/parent.py b/mitogen/parent.py index 45c8e2f2..a57ca20b 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -93,6 +93,12 @@ SYS_EXECUTABLE_MSG = ( ) _sys_executable_warning_logged = False +SIGNAL_BY_NUM = dict( + (getattr(signal, name), name) + for name in sorted(vars(signal), reverse=True) + if name.startswith('SIG') and not name.startswith('SIG_') +) + def get_log_level(): return (LOG.level or logging.getLogger().level or logging.INFO) @@ -584,6 +590,21 @@ def _proxy_connect(name, method_name, kwargs, econtext): } +def wstatus_to_str(status): + """ + Parse and format a :func:`os.waitpid` exit status. + """ + if os.WIFEXITED(status): + return 'exited with return code %d' % (os.WEXITSTATUS(status),) + if os.WIFSIGNALED(status): + n = os.WTERMSIG(status) + return 'exited due to signal %d (%s)' % (n, SIGNAL_BY_NUM.get(n)) + if os.WIFSTOPPED(status): + n = os.WSTOPSIG(status) + return 'stopped due to signal %d (%s)' % (n, SIGNAL_BY_NUM.get(n)) + return 'unknown wait status (%d)' % (status,) + + class Argv(object): """ Wrapper to defer argv formatting when debug logging is disabled. @@ -961,7 +982,7 @@ class Stream(mitogen.core.Stream): self._reaped = True if pid: - LOG.debug('%r: child process exit status was %d', self, status) + LOG.debug('%r: PID %d %s', self, pid, wstatus_to_str(status)) return # For processes like sudo we cannot actually send sudo a signal, diff --git a/tests/parent_test.py b/tests/parent_test.py index 53b66c1d..c9ccaf3f 100644 --- a/tests/parent_test.py +++ b/tests/parent_test.py @@ -1,5 +1,6 @@ import errno import os +import signal import subprocess import sys import tempfile @@ -44,6 +45,41 @@ class GetDefaultRemoteNameTest(testlib.TestCase): self.assertEquals("ECORP_Administrator@box:123", self.func()) +class WstatusToStrTest(testlib.TestCase): + func = staticmethod(mitogen.parent.wstatus_to_str) + + def test_return_zero(self): + pid = os.fork() + if not pid: + os._exit(0) + (pid, status), _ = mitogen.core.io_op(os.waitpid, pid, 0) + self.assertEquals(self.func(status), + 'exited with return code 0') + + def test_return_one(self): + pid = os.fork() + if not pid: + os._exit(1) + (pid, status), _ = mitogen.core.io_op(os.waitpid, pid, 0) + self.assertEquals( + self.func(status), + 'exited with return code 1' + ) + + def test_sigkill(self): + pid = os.fork() + if not pid: + time.sleep(600) + os.kill(pid, signal.SIGKILL) + (pid, status), _ = mitogen.core.io_op(os.waitpid, pid, 0) + self.assertEquals( + self.func(status), + 'exited due to signal %s (SIGKILL)' % (signal.SIGKILL,) + ) + + # can't test SIGSTOP without POSIX sessions rabbithole + + class ReapChildTest(testlib.RouterMixin, testlib.TestCase): def test_connect_timeout(self): # Ensure the child process is reaped if the connection times out. From a7b1831ddf2623c89be3c21972301ddf0c8a465e Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 3 Oct 2018 14:45:23 +0100 Subject: [PATCH 202/212] core: move IS_DEAD doc into core.py. --- docs/howitworks.rst | 20 ++++++-------------- mitogen/core.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/docs/howitworks.rst b/docs/howitworks.rst index a3b08eac..2a4623eb 100644 --- a/docs/howitworks.rst +++ b/docs/howitworks.rst @@ -298,8 +298,9 @@ parent and child. Integers use big endian in their encoded form. * - `reply_to` - 4 - Integer target handle to direct any reply to this message. Used to - receive a one-time reply, such as the return value of a function call. - :data:`IS_DEAD` has a special meaning when it appears in this field. + receive a one-time reply, such as the return value of a function call, + or to signal a special condition for the message. :ref:`See below + ` for special values for this field. * - `length` - 4 @@ -472,23 +473,14 @@ Non-master parents also listen on the following handles: ensuring they are cached and deduplicated at each hop in the chain leading to the target context. +.. _reply_to_values: + Special values for the `reply_to` field: .. _IS_DEAD: .. currentmodule:: mitogen.core -.. data:: IS_DEAD - - Special value used to signal disconnection or the inability to route a - message, when it appears in the `reply_to` field. Usually causes - :class:`mitogen.core.ChannelError` to be raised when it is received. - - It indicates the sender did not know how to process the message, or wishes - no further messages to be delivered to it. It is used when: +.. autodata:: IS_DEAD - * a remote receiver is disconnected or explicitly closed. - * a related message could not be delivered due to no route existing for it. - * a router is being torn down, as a sentinel value to notify - :py:meth:`mitogen.core.Router.add_handler` callbacks to clean up. Additional handles are created to receive the result of every function call diff --git a/mitogen/core.py b/mitogen/core.py index 9aa95973..dadf0924 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -89,6 +89,18 @@ LOAD_MODULE = 107 FORWARD_MODULE = 108 DETACHING = 109 CALL_SERVICE = 110 + +#: Special value used to signal disconnection or the inability to route a +#: message, when it appears in the `reply_to` field. Usually causes +#: :class:`mitogen.core.ChannelError` to be raised when it is received. +#: +#: It indicates the sender did not know how to process the message, or wishes +#: no further messages to be delivered to it. It is used when: +#: +#: * a remote receiver is disconnected or explicitly closed. +#: * a related message could not be delivered due to no route existing for it. +#: * a router is being torn down, as a sentinel value to notify +#: :py:meth:`mitogen.core.Router.add_handler` callbacks to clean up. IS_DEAD = 999 try: From 74cf9c3c962961fab97c9ef5a7e21ad1f245ad98 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 4 Oct 2018 19:15:59 +0000 Subject: [PATCH 203/212] master: document ThreadWatcher --- mitogen/master.py | 56 ++++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/mitogen/master.py b/mitogen/master.py index 0a434a94..22ea0fc1 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -179,24 +179,35 @@ def scan_code_imports(co): class ThreadWatcher(object): """ - Manage threads that waits for nother threads to shutdown, before invoking - `on_join()`. In CPython it seems possible to use this method to ensure a - non-main thread is signalled when the main thread has exitted, using yet - another thread as a proxy. + Manage threads that wait for another thread to shut down, before invoking + `on_join()` for each associated ThreadWatcher. + + In CPython it seems possible to use this method to ensure a non-main thread + is signalled when the main thread has exited, using a third thread as a + proxy. """ - _lock = threading.Lock() - _pid = None - _instances_by_target = {} - _thread_by_target = {} + #: Protects remaining _cls_* members. + _cls_lock = threading.Lock() + + #: PID of the process that last modified the class data. If the PID + #: changes, it means the thread watch dict refers to threads that no longer + #: exist in the current process (since it forked), and so must be reset. + _cls_pid = None + + #: Map watched Thread -> list of ThreadWatcher instances. + _cls_instances_by_target = {} + + #: Map watched Thread -> watcher Thread for each watched thread. + _cls_thread_by_target = {} @classmethod def _reset(cls): """If we have forked since the watch dictionaries were initialized, all that has is garbage, so clear it.""" - if os.getpid() != cls._pid: - cls._pid = os.getpid() - cls._instances_by_target.clear() - cls._thread_by_target.clear() + if os.getpid() != cls._cls_pid: + cls._cls_pid = os.getpid() + cls._cls_instances_by_target.clear() + cls._cls_thread_by_target.clear() def __init__(self, target, on_join): self.target = target @@ -205,33 +216,34 @@ class ThreadWatcher(object): @classmethod def _watch(cls, target): target.join() - for watcher in cls._instances_by_target[target]: + for watcher in cls._cls_instances_by_target[target]: watcher.on_join() def install(self): - self._lock.acquire() + self._cls_lock.acquire() try: self._reset() - self._instances_by_target.setdefault(self.target, []).append(self) - if self.target not in self._thread_by_target: - self._thread_by_target[self.target] = threading.Thread( + lst = self._cls_instances_by_target.setdefault(self.target, []) + lst.append(self) + if self.target not in self._cls_thread_by_target: + self._cls_thread_by_target[self.target] = threading.Thread( name='mitogen.master.join_thread_async', target=self._watch, args=(self.target,) ) - self._thread_by_target[self.target].start() + self._cls_thread_by_target[self.target].start() finally: - self._lock.release() + self._cls_lock.release() def remove(self): - self._lock.acquire() + self._cls_lock.acquire() try: self._reset() - lst = self._instances_by_target.get(self.target, []) + lst = self._cls_instances_by_target.get(self.target, []) if self in lst: lst.remove(self) finally: - self._lock.release() + self._cls_lock.release() @classmethod def watch(cls, target, on_join): From bf597d257f6661bb8307419a7ba8fd0c9295ff87 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 4 Oct 2018 19:21:23 +0000 Subject: [PATCH 204/212] master: document LogForwarder. --- mitogen/master.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/mitogen/master.py b/mitogen/master.py index 22ea0fc1..4e0745c8 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -253,6 +253,25 @@ class ThreadWatcher(object): class LogForwarder(object): + """ + Install a :data:`mitogen.core.FORWARD_LOG` handler that delivers forwarded + log events into the local logging framework. This is used by the master's + :class:`Router`. + + The forwarded :class:`logging.LogRecord` objects are delivered to loggers + under ``mitogen.ctx.*`` corresponding to their + :attr:`mitogen.core.Context.name`, with the message prefixed with the + logger name used in the child. The records include some extra attributes: + + * ``mitogen_message``: Unicode original message without the logger name + prepended. + * ``mitogen_context``: :class:`mitogen.parent.Context` reference to the + source context. + * ``mitogen_name``: Original logger name. + + :param mitogen.master.Router router: + Router to install the handler on. + """ def __init__(self, router): self._router = router self._cache = {} @@ -269,7 +288,8 @@ class LogForwarder(object): if logger is None: context = self._router.context_by_id(msg.src_id) if context is None: - LOG.error('FORWARD_LOG received from src_id %d', msg.src_id) + LOG.error('%s: dropping log from unknown context ID %d', + self, msg.src_id) return name = '%s.%s' % (RLOG.name, context.name) From 9828588e9795640c4df1e626d3a7bf2edbc2cb74 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Thu, 4 Oct 2018 19:22:19 +0000 Subject: [PATCH 205/212] master: group is_stdlib_name() with other module functions. --- mitogen/master.py | 55 ++++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/mitogen/master.py b/mitogen/master.py index 4e0745c8..d4ee607a 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -83,6 +83,34 @@ def _stdlib_paths(): for p in prefixes) +def is_stdlib_name(modname): + """Return :data:`True` if `modname` appears to come from the standard + library. + """ + if imp.is_builtin(modname) != 0: + return True + + module = sys.modules.get(modname) + if module is None: + return False + + # six installs crap with no __file__ + modpath = os.path.abspath(getattr(module, '__file__', '')) + return is_stdlib_path(modpath) + + +_STDLIB_PATHS = _stdlib_paths() + + +def is_stdlib_path(path): + return any( + os.path.commonprefix((libpath, path)) == libpath + and 'site-packages' not in path + and 'dist-packages' not in path + for libpath in _STDLIB_PATHS + ) + + def get_child_modules(path): """Return the suffixes of submodules directly neated beneath of the package directory at `path`. @@ -306,33 +334,6 @@ class LogForwarder(object): return 'LogForwarder(%r)' % (self._router,) -_STDLIB_PATHS = _stdlib_paths() - - -def is_stdlib_path(path): - return any( - os.path.commonprefix((libpath, path)) == libpath - and 'site-packages' not in path - and 'dist-packages' not in path - for libpath in _STDLIB_PATHS - ) - - -def is_stdlib_name(modname): - """Return :data:`True` if `modname` appears to come from the standard - library.""" - if imp.is_builtin(modname) != 0: - return True - - module = sys.modules.get(modname) - if module is None: - return False - - # six installs crap with no __file__ - modpath = os.path.abspath(getattr(module, '__file__', '')) - return is_stdlib_path(modpath) - - class ModuleFinder(object): def __init__(self): #: Import machinery is expensive, keep :py:meth`:get_module_source` From e45e5d3e064c00b38c6fe1e9714b759eb6c16543 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Wed, 3 Oct 2018 18:08:51 +0100 Subject: [PATCH 206/212] tests: Document Python versions in build_docker_images.py --- tests/image_prep/build_docker_images.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/image_prep/build_docker_images.py b/tests/image_prep/build_docker_images.py index c085d29e..94a17104 100755 --- a/tests/image_prep/build_docker_images.py +++ b/tests/image_prep/build_docker_images.py @@ -24,9 +24,9 @@ def sh(s, *args): label_by_id = {} for base_image, label in [ - ('debian:stretch', 'debian'), - ('centos:6', 'centos6'), - ('centos:7', 'centos7') + ('debian:stretch', 'debian'), # Python 2.7.13, 3.5.3 + ('centos:6', 'centos6'), # Python 2.6.6 + ('centos:7', 'centos7') # Python 2.7.5 ]: args = sh('docker run --rm -it -d -h mitogen-%s %s /bin/bash', label, base_image) From 1b17aa1d1a02346b35cdd69acc7d41e6306ebdbb Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 23 Oct 2018 14:42:44 +0100 Subject: [PATCH 207/212] ansible: fix temp cleanup regression and add test; closes #397. --- ansible_mitogen/mixins.py | 5 ++- tests/ansible/integration/action/all.yml | 1 + .../integration/action/remove_tmp_path.yml | 40 +++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 tests/ansible/integration/action/remove_tmp_path.yml diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index d4fcbd0d..4c06063b 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -190,7 +190,10 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): """ # The actual removal is pipelined by Connection.close(). LOG.debug('_remove_tmp_path(%r)', tmp_path) - self._connection._shell.tmpdir = None + # Upstream _remove_tmp_path resets shell.tmpdir here, however + # connection.py uses that as the sole location of the temporary + # directory, if one exists. + # self._connection._shell.tmpdir = None def _transfer_data(self, remote_path, data): """ diff --git a/tests/ansible/integration/action/all.yml b/tests/ansible/integration/action/all.yml index 75f77243..018973a9 100644 --- a/tests/ansible/integration/action/all.yml +++ b/tests/ansible/integration/action/all.yml @@ -4,5 +4,6 @@ - import_playbook: make_tmp_path.yml - import_playbook: remote_expand_user.yml - import_playbook: remote_file_exists.yml +- import_playbook: remove_tmp_path.yml - import_playbook: synchronize.yml - import_playbook: transfer_data.yml diff --git a/tests/ansible/integration/action/remove_tmp_path.yml b/tests/ansible/integration/action/remove_tmp_path.yml new file mode 100644 index 00000000..566e4f3f --- /dev/null +++ b/tests/ansible/integration/action/remove_tmp_path.yml @@ -0,0 +1,40 @@ +# +# Ensure _remove_tmp_path cleans up the temporary path. +# +# +- name: integration/action/remove_tmp_path.yml + hosts: test-targets + any_errors_fatal: true + tasks: + - meta: end_play + when: not is_mitogen + + # + # Use the copy module to cause a temporary directory to be created, and + # return a result with a 'src' attribute pointing into that directory. + # + + - copy: + dest: /tmp/remove_tmp_path_test + content: "{{ 123123 | random }}" + register: out + + - stat: + path: "{{out.src}}" + register: out2 + + - assert: + that: + - not out2.stat.exists + + - stat: + path: "{{out.src|dirname}}" + register: out2 + + - assert: + that: + - not out2.stat.exists + + - file: + path: /tmp/remove_tmp_path_test + state: absent From 7fd9fb001486b9695f715a1cb080cbc5f982c87f Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 23 Oct 2018 15:29:03 +0100 Subject: [PATCH 208/212] issue #397: fix another case where stray tmpdirs can be left behind. Newer Ansibles use atexit.register() to invoke cleanup, so we need to run those registrations after each run. --- ansible_mitogen/runner.py | 32 +++++++++++++++------ tests/ansible/integration/runner/all.yml | 3 +- tests/ansible/integration/runner/atexit.yml | 31 ++++++++++++++++++++ 3 files changed, 56 insertions(+), 10 deletions(-) create mode 100644 tests/ansible/integration/runner/atexit.yml diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 44780aa2..45bb5f0b 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -38,6 +38,7 @@ how to build arguments for it, preseed related data, etc. from __future__ import absolute_import from __future__ import unicode_literals +import atexit import ctypes import errno import imp @@ -757,9 +758,25 @@ class NewStyleRunner(ScriptRunner): if klass and isinstance(exc, klass): mod.module.fail_json(**exc.results) - def _run(self): - code = self._get_code() + def _run_code(self, code, mod): + try: + if mitogen.core.PY3: + exec(code, vars(mod)) + else: + exec('exec code in vars(mod)') + except Exception as e: + self._handle_magic_exception(mod, e) + raise + def _run_atexit_funcs(self): + """ + Newer Ansibles use atexit.register() to trigger tmpdir cleanup, when + AnsibleModule.tmpdir is responsible for creating its own temporary + directory. + """ + atexit._run_exitfuncs() + + def _run(self): mod = types.ModuleType(self.main_module_name) mod.__package__ = None # Some Ansible modules use __file__ to find the Ansiballz temporary @@ -771,16 +788,13 @@ class NewStyleRunner(ScriptRunner): 'ansible_module_' + os.path.basename(self.path), ) + code = self._get_code() exc = None try: try: - if mitogen.core.PY3: - exec(code, vars(mod)) - else: - exec('exec code in vars(mod)') - except Exception as e: - self._handle_magic_exception(mod, e) - raise + self._run_code(code, mod) + finally: + self._run_atexit_funcs() except SystemExit as e: exc = e diff --git a/tests/ansible/integration/runner/all.yml b/tests/ansible/integration/runner/all.yml index 69c22edb..9dd209d7 100644 --- a/tests/ansible/integration/runner/all.yml +++ b/tests/ansible/integration/runner/all.yml @@ -1,3 +1,4 @@ +- import_playbook: atexit.yml - import_playbook: builtin_command_module.yml - import_playbook: custom_bash_hashbang_argument.yml - import_playbook: custom_bash_old_style_module.yml @@ -15,6 +16,6 @@ - import_playbook: environment_isolation.yml - import_playbook: etc_environment.yml - import_playbook: forking_active.yml -- import_playbook: forking_inactive.yml - import_playbook: forking_correct_parent.yml +- import_playbook: forking_inactive.yml - import_playbook: missing_module.yml diff --git a/tests/ansible/integration/runner/atexit.yml b/tests/ansible/integration/runner/atexit.yml new file mode 100644 index 00000000..872cdd57 --- /dev/null +++ b/tests/ansible/integration/runner/atexit.yml @@ -0,0 +1,31 @@ +# issue #397: newer Ansibles rely on atexit to cleanup their temporary +# directories. Ensure atexit handlers run during runner completion. + +- name: integration/runner/atexit.yml + hosts: test-targets + gather_facts: false + any_errors_fatal: false + tasks: + + # + # Verify a run with a healthy atexit handler. Broken handlers cause an + # exception to be raised. + # + + - custom_python_run_script: + script: | + import atexit + atexit.register(lambda: + open('/tmp/atexit-was-triggered', 'w').write('yep')) + + - slurp: + path: /tmp/atexit-was-triggered + register: out + + - assert: + that: + - out.content|b64decode == "yep" + + - file: + path: /tmp/atexit-was-triggered + state: absent From 40d2cf7e25a2efb6245bc5d3aa2ba9fcf507773a Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 23 Oct 2018 15:34:06 +0100 Subject: [PATCH 209/212] docs: update changelog. --- docs/changelog.rst | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1e7fb913..c58950a0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -134,6 +134,11 @@ Fixes in processes running on the target is capped to 512, reducing the work required to start a subprocess by >2000x in default CentOS configurations. +* `#397 `_: recent Mitogen master + versions could fail to clean up temporary directories in a number of + circumstances, and newer Ansibles moved to using :mod:`atexit` to effect + temporary directory cleanup in some circumstances. + * `b9112a9c `_, `2c287801 `_: OpenSSH 7.5 permission denied prompts are now recognized. Contributed by Alex Willmer. @@ -223,13 +228,14 @@ bug reports, features and fixes in this release contributed by `Jonathan Rosser `_, `Josh Smift `_, `Luca Nunzi `_, -`nikitakazantsev12 `_, +`Orion Poplawski `_, `Peter V. Saveliev `_, `Pierre-Henry Muller `_, `Pierre-Louis Bonicoli `_, `Prateek Jain `_, `RedheatWei `_, `Rick Box `_, +`nikitakazantsev12 `_, `Tawana Musewe `_, `Timo Beckers `_, and `Yannig Perré `_. @@ -329,12 +335,13 @@ the bug reports and pull requests in this release contributed by `Colin McCarthy `_, `Dan Quackenbush `_, `Duane Zamrok `_, -`falbanese `_, `Gonzalo Servat `_, `Guy Knights `_, `Josh Smift `_, `Mark Janssen `_, `Mike Walker `_, +`Orion Poplawski `_, +`falbanese `_, `Tawana Musewe `_, and `Zach Swanson `_. From fd5066d6711fb86152bb5798c976b7e40a5ea140 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 23 Oct 2018 15:34:18 +0100 Subject: [PATCH 210/212] tests: teach various tests to cleanup /tmp when they're done. --- tests/ansible/integration/action/copy.yml | 17 +++++++++++++++++ .../integration/action/fixup_perms2__copy.yml | 13 +++++++++++++ .../ansible/integration/action/synchronize.yml | 10 ++++++++++ 3 files changed, 40 insertions(+) diff --git a/tests/ansible/integration/action/copy.yml b/tests/ansible/integration/action/copy.yml index e3fca87f..d799be90 100644 --- a/tests/ansible/integration/action/copy.yml +++ b/tests/ansible/integration/action/copy.yml @@ -64,3 +64,20 @@ - stat.results[2].stat.checksum == "b26dd6444595e2bdb342aa0a91721b57478b5029" - stat.results[3].stat.checksum == "d675f47e467eae19e49032a2cc39118e12a6ee72" + - file: + state: absent + path: "{{item}}" + with_items: + - /tmp/copy-tiny-file + - /tmp/copy-tiny-file.out + - /tmp/copy-no-mode + - /tmp/copy-no-mode.out + - /tmp/copy-with-mode + - /tmp/copy-with-mode.out + - /tmp/copy-large-file + - /tmp/copy-large-file.out + - /tmp/copy-tiny-inline-file.out + - /tmp/copy-large-inline-file + - /tmp/copy-large-inline-file.out + + # end of cleaning out files (again) diff --git a/tests/ansible/integration/action/fixup_perms2__copy.yml b/tests/ansible/integration/action/fixup_perms2__copy.yml index 7e6ef522..c92b158e 100644 --- a/tests/ansible/integration/action/fixup_perms2__copy.yml +++ b/tests/ansible/integration/action/fixup_perms2__copy.yml @@ -102,3 +102,16 @@ - assert: that: - out.stat.mode == "1461" + + - file: + state: absent + path: "{{item}}" + with_items: + - /tmp/weird-mode + - /tmp/weird-mode.out + - /tmp/copy-no-mode + - /tmp/copy-no-mode.out + - /tmp/copy-with-mode + - /tmp/copy-with-mode.out + + # end of cleaning out files diff --git a/tests/ansible/integration/action/synchronize.yml b/tests/ansible/integration/action/synchronize.yml index 2672b08e..25649fbf 100644 --- a/tests/ansible/integration/action/synchronize.yml +++ b/tests/ansible/integration/action/synchronize.yml @@ -32,6 +32,7 @@ - file: path: /tmp/sync-test.out state: absent + become: true - synchronize: private_key: /tmp/synchronize-action-key @@ -46,3 +47,12 @@ - assert: that: outout == "item!" + + - file: + path: "{{item}}" + state: absent + become: true + with_items: + - /tmp/synchronize-action-key + - /tmp/sync-test + - /tmp/sync-test.out From 48bc91550b65a6f4f0958de79646f76e254e5630 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 23 Oct 2018 16:14:07 +0100 Subject: [PATCH 211/212] docs: update changelog. --- docs/ansible.rst | 2 +- docs/changelog.rst | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/ansible.rst b/docs/ansible.rst index 517446f8..263d2b10 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -142,7 +142,7 @@ Testimonials Noteworthy Differences ---------------------- -* Ansible 2.3-2.6 are supported along with Python 2.6, 2.7 or 3.6. Verify your +* Ansible 2.3-2.7 are supported along with Python 2.6, 2.7 or 3.6. Verify your installation is running one of these versions by checking ``ansible --version`` output. diff --git a/docs/changelog.rst b/docs/changelog.rst index c58950a0..099b253b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -15,7 +15,7 @@ Release Notes -v0.2.3 (2018-08-??) +v0.2.3 (2018-10-23) ------------------- Mitogen for Ansible @@ -24,7 +24,9 @@ Mitogen for Ansible Enhancements ^^^^^^^^^^^^ -* `#315 `_: Ansible 2.6 is supported. +* `#315 `_, + `#392 `_: Ansible 2.6 and 2.7 are + supported. * `#321 `_, `#336 `_: temporary file handling @@ -386,7 +388,7 @@ within a stable series. Mitogen for Ansible ~~~~~~~~~~~~~~~~~~~ -* Support for Ansible 2.3 - 2.6.x and any mixture of Python 2.6, 2.7 or 3.6 on +* Support for Ansible 2.3 - 2.7.x and any mixture of Python 2.6, 2.7 or 3.6 on controller and target nodes. * Drop-in support for many Ansible connection types. @@ -429,6 +431,8 @@ Mitogen for Ansible ``ansible_python_interpreter`` setting, contrary to the Ansible documentation. This will be addressed in a future 0.2 release. +* The Ansible 2.7 ``reboot`` module is not yet supported. + * Performance does not scale linearly with target count. This requires significant additional work, as major bottlenecks exist in the surrounding Ansible code. Performance-related bug reports for any scenario remain From cfcc7c0273801e07f09339ee8d50c725d7dd5d56 Mon Sep 17 00:00:00 2001 From: David Wilson Date: Tue, 23 Oct 2018 16:14:40 +0100 Subject: [PATCH 212/212] Bump version for release. --- mitogen/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 3fc02433..58ef2030 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -33,7 +33,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 2, 2) +__version__ = (0, 2, 3) #: This is :data:`False` in slave contexts. Previously it was used to prevent