diff --git a/.travis/ansible_tests.sh b/.travis/ansible_tests.sh index a61ed836..e6441343 100755 --- a/.travis/ansible_tests.sh +++ b/.travis/ansible_tests.sh @@ -60,5 +60,5 @@ echo travis_fold:end:job_setup echo travis_fold:start:ansible /usr/bin/time ./run_ansible_playbook.sh \ all.yml \ - -i "${TMPDIR}/hosts" + -i "${TMPDIR}/hosts" "$@" echo travis_fold:end:ansible diff --git a/ansible_mitogen/plugins/strategy/mitogen.py b/ansible_mitogen/plugins/strategy/mitogen.py index 3ef522b4..4f595161 100644 --- a/ansible_mitogen/plugins/strategy/mitogen.py +++ b/ansible_mitogen/plugins/strategy/mitogen.py @@ -44,12 +44,12 @@ import sys # debuggers and isinstance() work predictably. # -try: - import ansible_mitogen -except ImportError: - base_dir = os.path.dirname(__file__) - sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) - del base_dir +BASE_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__), '../../..') +) + +if BASE_DIR not in sys.path: + sys.path.insert(0, BASE_DIR) import ansible_mitogen.strategy import ansible.plugins.strategy.linear diff --git a/ansible_mitogen/plugins/strategy/mitogen_free.py b/ansible_mitogen/plugins/strategy/mitogen_free.py index 34f959ca..8dfaa16e 100644 --- a/ansible_mitogen/plugins/strategy/mitogen_free.py +++ b/ansible_mitogen/plugins/strategy/mitogen_free.py @@ -44,12 +44,12 @@ import sys # debuggers and isinstance() work predictably. # -try: - import ansible_mitogen -except ImportError: - base_dir = os.path.dirname(__file__) - sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) - del base_dir +BASE_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__), '../../..') +) + +if BASE_DIR not in sys.path: + sys.path.insert(0, BASE_DIR) import ansible_mitogen.loaders import ansible_mitogen.strategy diff --git a/ansible_mitogen/plugins/strategy/mitogen_linear.py b/ansible_mitogen/plugins/strategy/mitogen_linear.py index a5ea2a3d..d995b67b 100644 --- a/ansible_mitogen/plugins/strategy/mitogen_linear.py +++ b/ansible_mitogen/plugins/strategy/mitogen_linear.py @@ -44,12 +44,12 @@ import sys # debuggers and isinstance() work predictably. # -try: - import ansible_mitogen -except ImportError: - base_dir = os.path.dirname(__file__) - sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) - del base_dir +BASE_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__), '../../..') +) + +if BASE_DIR not in sys.path: + sys.path.insert(0, BASE_DIR) import ansible_mitogen.loaders import ansible_mitogen.strategy diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index f19079ee..4724ca93 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -33,6 +33,7 @@ import os import signal import socket import sys +import time import mitogen import mitogen.core @@ -114,6 +115,9 @@ class MuxProcess(object): mitogen.core.set_cloexec(cls.worker_sock.fileno()) mitogen.core.set_cloexec(cls.child_sock.fileno()) + if os.environ.get('MITOGEN_PROFILING', '1'): + mitogen.core.enable_profiling() + cls.original_env = dict(os.environ) cls.child_pid = os.fork() ansible_mitogen.logging.setup() @@ -199,4 +203,10 @@ class MuxProcess(object): ourself. In future this should gracefully join the pool, but TERM is fine for now. """ + if os.environ.get('MITOGEN_PROFILING'): + # TODO: avoid killing pool threads before they have written their + # .pstats. Really shouldn't be using kill() here at all, but hard + # to guarantee services can always be unblocked during shutdown. + time.sleep(1) + os.kill(os.getpid(), signal.SIGTERM) diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index ca3928b3..86f7b329 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -264,9 +264,9 @@ class ModuleUtilsImporter(object): mod.__loader__ = self if is_pkg: mod.__path__ = [] - mod.__package__ = fullname + mod.__package__ = str(fullname) else: - mod.__package__ = fullname.rpartition('.')[0] + mod.__package__ = str(fullname.rpartition('.')[0]) exec(code, mod.__dict__) self._loaded.add(fullname) return mod diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index a7bb7db1..e95fc226 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -388,6 +388,8 @@ class ModuleDepService(mitogen.service.Service): Scan a new-style module and produce a cached mapping of module_utils names to their resolved filesystem paths. """ + invoker_class = mitogen.service.SerializedInvoker + def __init__(self, *args, **kwargs): super(ModuleDepService, self).__init__(*args, **kwargs) self._cache = {} diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index e5365dd4..582bf85e 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -46,16 +46,26 @@ import re import signal import stat import subprocess +import sys import tempfile import traceback +import types -import ansible.module_utils.json_utils -import ansible_mitogen.runner import mitogen.core import mitogen.fork import mitogen.parent import mitogen.service +# Ansible since PR #41749 inserts "import __main__" into +# ansible.module_utils.basic. Mitogen's importer will refuse such an import, so +# we must setup a fake "__main__" before that module is ever imported. The +# str() is to cast Unicode to bytes on Python 2.6. +if not sys.modules.get(str('__main__')): + sys.modules[str('__main__')] = types.ModuleType(str('__main__')) + +import ansible.module_utils.json_utils +import ansible_mitogen.runner + LOG = logging.getLogger(__name__) diff --git a/docs/ansible.rst b/docs/ansible.rst index 6ab6626a..919c6380 100644 --- a/docs/ansible.rst +++ b/docs/ansible.rst @@ -118,6 +118,7 @@ Testimonials strategy took Clojars' Ansible runs from **14 minutes to 2 minutes**. I still can't quite believe it." +* "Enabling the mitogen plugin in ansible feels like switching from floppy to SSD" .. _noteworthy_differences: diff --git a/docs/api.rst b/docs/api.rst index 6efca6dd..92abe1ec 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -831,8 +831,8 @@ Context Class context's main thread. :param fn: - A free function in module scope, or a classmethod or staticmethod - of a class directly reachable from module scope: + A free function in module scope or a class method of a class + directly reachable from module scope: .. code-block:: python @@ -842,10 +842,6 @@ Context Class """A free function reachable as mymodule.my_func""" class MyClass: - @staticmethod - def my_staticmethod(): - """Reachable as mymodule.MyClass.my_staticmethod""" - @classmethod def my_classmethod(cls): """Reachable as mymodule.MyClass.my_classmethod""" diff --git a/docs/changelog.rst b/docs/changelog.rst index 14eeaae6..fd098e6a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -24,6 +24,8 @@ Release Notes supported under Ansible 2.6. Contributed by `Dan Quackenbush `_. + * Compatible with development versions of Ansible post https://github.com/ansible/ansible/pull/41749 + v0.2.2 (2018-07-26) ------------------- diff --git a/docs/compared.rst b/docs/compared.rst deleted file mode 100644 index b75ae3f2..00000000 --- a/docs/compared.rst +++ /dev/null @@ -1,226 +0,0 @@ - -Mitogen Compared To -------------------- - -This provides a little free-text summary of conceptual differences between -Mitogen and other tools, along with some basic perceptual metrics (project -maturity/age, quality of tests, function matrix) - - -Ansible -####### - -Ansible_ is a complete provisioning system, Mitogen is a small component of such a system. - -You should use Ansible if ... - -You should not use Ansible if ... - - -.. _Ansible: https://docs.ansible.com/ansible/latest/index.html -.. _ansible.src: https://github.com/ansible/ansible/ - -Baker -##### - - Baker_ lets you easily add a command line interface to your Python - functions using a simple decorator, to create scripts with "sub-commands", - similar to Django's ``manage.py``, ``svn``, ``hg``, etc. - -- Unmaintained since 2015 -- No obvious remote execution functionality - -.. _Baker: https://bitbucket.org/mchaput/baker - -Chopsticks -########## - -Chopsticks_ also supports recursion! but the recursively executed instance has no special knowledge of its identity in a tree structure, and little support for functions running in the master to directly invoke functions in a recursive context.. effectively each recursion produces a new master, from which function calls must be made. - -executing functions from __main__ entails picking just that function and deps -out of the main module, not transferring the module intact. that approach works -but it's much messier than just arranging for __main__ to be imported and -executed through the import mechanism. - -supports sudo but no support for require_tty or typing a sudo password. also supports SSH and Docker. - -good set of tests - -real PEP-302 module loader, but doesn't try to cope with master also relying on -a PEP-302 module loader (e.g. py2exe). - -Based on the tox configuration Python 2.7, and 3.3 to 3.6 are supported. - -I/O multiplexer in the master, but not in children. - -As with Execnet it includes its own serialization - pencode_ supports - -- most Python primitive types (``bytes``/``str``/``unicode``, ``list``, ``tuple`` ...) -- identity references -- self referencing (recursive) data srtuctures - -pencode lacks support for arbitrary classes. Byte strings require special -treatment if they contain non-ascii characters. Some primitive types -(e.g. ``complex``) are not handled. This would be straightforwar to address. -Values are length-prefixed with a 32 bit unsigned integer, meaning values -are limited to 4 billion bytes or items in length. - -design is reminiscent of Mitogen in places (Tunnel is practically identical to -Mitogen's Stream), and closer to Execnet elsewhere (lack of uniformity, -tendency to prefer logic expressed in if/else special case soup rather than the -type system, though some of that is due to supporting Python 3, so not judging -too harshly!) - -Chopsticks has its own `Chopsticks vs`_ comparisons. - -You should use Chopsticks if you need Python 3 support. - -.. _Chopsticks: https://chopsticks.readthedocs.io/en/stable/ -.. _Chopsticks.src: https://github.com/lordmauve/chopsticks/ -.. _Chopsticks vs: https://chopsticks.readthedocs.io/en/stable/intro.html#chopsticks-vs -.. _pencode: https://github.com/lordmauve/chopsticks/blob/master/doc/pencode.rst -.. _pencode.src: https://github.com/lordmauve/chopsticks/blob/master/chopsticks/pencode.py - -Disco -##### - - Disco_ is a lightweight, open-source framework for distributed computing - based on the MapReduce paradigm. - -- An Erlang core, with Python bindings -- Wire format is pickle, according to `Execnet vs NLTK for distributed NLTK`_ - -.. _Disco: http://discoproject.org/ -.. _Execnet vs NLTK for distributed NLTK: https://streamhacker.com/2009/12/14/execnet-disco-distributed-nltk/ - -Execnet -####### - -Execnet_ - -- Parent and children may use threads, gevent, or eventlet, Mitogen only supports threads. -- No recursion -- Similar Channel abstraction but better developed.. includes waiting for remote to close its end -- Heavier emphasis on passing chunks of Python source code around, modules are loaded one-at-a-time with no dependency resolution mechanism -- Built-in unidirectional rsync-alike, compared to Mitogen's SSH emulation which allows use of real rsync in any supported mode -- no support for sudo, but supports connecting to vagrant -- works with read-only filesystem -- includes its own serialization_ independent of the standard library - - The obj and all contained objects must be of a builtin python type - (so nested dicts, sets, etc. are all ok but not user-level instances). - -- Known uses include `pytest-xdist`_, and `Distributed NLTK`_ - -You should use Execnet if you value code maturity more than featureset. - -.. _Execnet: https://codespeak.net/execnet/ -.. _serialization: https://codespeak.net/execnet/basics.html#dumps-loads -.. _pytest-xdist: https://pypi.python.org/pypi/pytest-xdist -.. _Distributed NLTK: https://streamhacker.com/2009/12/14/execnet-disco-distributed-nltk/ - -Fabric -###### - -Fabric_ allows execution of shell snippets on remote machines, Python functions run -locally, any remote interaction is fundamentally done via shell, with all the -limitations that entails. prefers to depend on SSH features (e.g. tunnelling) -than reinvent them - -You should use Fabric if you enjoy being woken at 4am to pages about broken -shell snippets. - -.. _fabric: http://www.fabfile.org/ - -Invoke -###### - -Invoke_ - -Python 2.6+, 3.3+ - -Basically a Fabric-alike - -.. _invoke: http://www.pyinvoke.org/ - -Multiprocessing -############### - -multiprocessing_ was added to the stdlib in Python 2.6. - - multiprocessing is a package that supports spawning processes using an - API similar to the threading module. The multiprocessing package offers - both local and remote concurrency - -There is a backport_ for Python 2.4 & 2.5, but it is not pure Python. -pymultiprocessing_ appears to be a pure Python implementation. -An ecosystem_ of packages has built up around multiprocessing. - -The `programming guidelines`_ section notes - -- Arguments to proxies must be picklable. On Windows this also applies to - ``multiprocessing.Process.__init__()`` arguments. -- Callers should beware replacing ``sys.stdin``, because - ``multiprocessing.Process._bootstrap()`` - will close it and open /dev/null instead - -.. _programming guidelines: https://docs.python.org/2/library/multiprocessing.html#programming-guidelines -.. _backport: https://pypi.python.org/pypi/multiprocessing -.. _pymultiprocessing: https://pypi.python.org/pypi/pymultiprocessing -.. _ecosystem: https://pypi.python.org/pypi?%3Aaction=search&term=multiprocessing&submit=search - -Paver -##### - -Paver_ - -More or less another task execution framework / make-alike, doesn't really deal -with remote execution at all. - -.. _Paver: https://github.com/paver/paver/ - -Plumbum -####### - -Plumbum_ - -Shell-only - -Basically syntax sugar for running shell commands. Nicer than raw shell -(depending on your opinions of operating overloading), but it's still shell. - -.. _Plumbum: https://pypi.python.org/pypi/plumbum - -Pyro4 -##### - -Pyro4_ -... - -.. _Pyro4: https://pythonhosted.org/Pyro4/ - -RPyC -#### - -RPyC_ - -- supports transparent object proxies similar to Pyro (with all the pain and suffering hidden network IO entails) -- significantly more 'frameworkey' feel -- runs multiplexer in a thread too? -- bootstrap over SSH only, no recursion and no sudo -- requires a writable filesystem - -.. _RPyC: https://rpyc.readthedocs.io/en/latest/ - -Salt -#### - -Salt_ - -- no crappy deps - -You should use Salt if you enjoy firefighting endless implementation bugs, -otherwise you should prefer Ansible. - -.. _Salt: https://docs.saltstack.com/en/latest/topics/ -.. _Salt.src: https://github.com/saltstack/salt diff --git a/mitogen/core.py b/mitogen/core.py index dd706311..cfcd70d2 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -350,6 +350,7 @@ def enable_profiling(): try: return func(*args) finally: + profiler.dump_stats('/tmp/mitogen.%d.%s.pstat' % (os.getpid(), name)) profiler.create_stats() fp = open('/tmp/mitogen.stats.%d.%s.log' % (os.getpid(), name), 'w') try: diff --git a/mitogen/fork.py b/mitogen/fork.py index 12bb7dfa..cf769788 100644 --- a/mitogen/fork.py +++ b/mitogen/fork.py @@ -75,6 +75,17 @@ def reset_logging_framework(): ] +def on_fork(): + """ + Should be called by any program integrating Mitogen each time the process + is forked, in the context of the new child. + """ + reset_logging_framework() # Must be first! + fixup_prngs() + mitogen.core.Latch._on_fork() + mitogen.core.Side._on_fork() + + def handle_child_crash(): """ Respond to _child_main() crashing by ensuring the relevant exception is @@ -134,10 +145,7 @@ class Stream(mitogen.parent.Stream): handle_child_crash() def _child_main(self, childfp): - reset_logging_framework() # Must be first! - fixup_prngs() - mitogen.core.Latch._on_fork() - mitogen.core.Side._on_fork() + on_fork() if self.on_fork: self.on_fork() mitogen.core.set_block(childfp.fileno()) diff --git a/tests/ansible/ansible.cfg b/tests/ansible/ansible.cfg index 7bf849d5..d9224ab7 100644 --- a/tests/ansible/ansible.cfg +++ b/tests/ansible/ansible.cfg @@ -7,7 +7,7 @@ callback_plugins = lib/callback stdout_callback = nice_stdout vars_plugins = lib/vars library = lib/modules -# module_utils = lib/module_utils +module_utils = lib/module_utils retry_files_enabled = False forks = 50 diff --git a/tests/ansible/gcloud/controller.yml b/tests/ansible/gcloud/controller.yml index 48f233d9..b4ce3fcf 100644 --- a/tests/ansible/gcloud/controller.yml +++ b/tests/ansible/gcloud/controller.yml @@ -56,6 +56,10 @@ editable: true name: ~/ansible + - pip: + virtualenv: ~/venv + name: debops + - lineinfile: line: "source $HOME/venv/bin/activate" path: ~/.profile diff --git a/tests/ansible/integration/all.yml b/tests/ansible/integration/all.yml index 264ae716..bf534aed 100644 --- a/tests/ansible/integration/all.yml +++ b/tests/ansible/integration/all.yml @@ -9,7 +9,7 @@ - import_playbook: connection_loader/all.yml - import_playbook: context_service/all.yml - import_playbook: local/all.yml -#- import_playbook: module_utils/all.yml +- import_playbook: module_utils/all.yml - import_playbook: playbook_semantics/all.yml - import_playbook: remote_tmp/all.yml - import_playbook: runner/all.yml diff --git a/tests/ansible/integration/module_utils/all.yml b/tests/ansible/integration/module_utils/all.yml index 920b5d1c..c8b8f2fb 100644 --- a/tests/ansible/integration/module_utils/all.yml +++ b/tests/ansible/integration/module_utils/all.yml @@ -1,6 +1,6 @@ -- import_playbook: from_config_path.yml -- import_playbook: from_config_path_pkg.yml -- import_playbook: adjacent_to_playbook.yml +#- import_playbook: from_config_path.yml +#- import_playbook: from_config_path_pkg.yml +#- import_playbook: adjacent_to_playbook.yml - import_playbook: adjacent_to_role.yml -- import_playbook: overrides_builtin.yml +#- import_playbook: overrides_builtin.yml diff --git a/tests/bench/roundtrip.py b/tests/bench/roundtrip.py index 40582d46..13b9413d 100644 --- a/tests/bench/roundtrip.py +++ b/tests/bench/roundtrip.py @@ -12,6 +12,6 @@ def do_nothing(): def main(router): f = router.fork() t0 = time.time() - for x in xrange(1000): + for x in xrange(10000): f.call(do_nothing) print '++', int(1e6 * ((time.time() - t0) / (1.0+x))), 'usec'