diff --git a/ansible_mitogen/loaders.py b/ansible_mitogen/loaders.py new file mode 100644 index 00000000..441e8113 --- /dev/null +++ b/ansible_mitogen/loaders.py @@ -0,0 +1,46 @@ +# Copyright 2017, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +""" +Stable names for PluginLoader instances across Ansible versions. +""" + +from __future__ import absolute_import + +try: + from ansible.plugins.loader import action_loader + from ansible.plugins.loader import connection_loader + from ansible.plugins.loader import module_loader + from ansible.plugins.loader import module_utils_loader + from ansible.plugins.loader import strategy_loader +except ImportError: # Ansible <2.4 + from ansible.plugins import action_loader + from ansible.plugins import connection_loader + from ansible.plugins import module_loader + from ansible.plugins import module_utils_loader + from ansible.plugins import strategy_loader diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 007974ad..676ca933 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -47,13 +47,7 @@ import ansible.errors import ansible.module_utils import mitogen.core -try: - from ansible.plugins.loader import module_loader - from ansible.plugins.loader import module_utils_loader -except ImportError: # Ansible <2.4 - from ansible.plugins import module_loader - from ansible.plugins import module_utils_loader - +import ansible_mitogen.loaders import ansible_mitogen.target @@ -322,7 +316,9 @@ class NewStylePlanner(ScriptPlanner): def get_search_path(self): return tuple( path - for path in module_utils_loader._get_paths(subdirs=False) + for path in ansible_mitogen.loaders.module_utils_loader._get_paths( + subdirs=False + ) if os.path.isdir(path) ) @@ -397,7 +393,7 @@ _planners = [ def get_module_data(name): - path = module_loader.find_plugin(name, '') + path = ansible_mitogen.loaders.module_loader.find_plugin(name, '') with open(path, 'rb') as fp: source = fp.read() return mitogen.core.to_text(path), source diff --git a/ansible_mitogen/plugins/strategy/mitogen_free.py b/ansible_mitogen/plugins/strategy/mitogen_free.py index b7ee03bc..34f959ca 100644 --- a/ansible_mitogen/plugins/strategy/mitogen_free.py +++ b/ansible_mitogen/plugins/strategy/mitogen_free.py @@ -51,10 +51,11 @@ except ImportError: sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) del base_dir +import ansible_mitogen.loaders import ansible_mitogen.strategy -import ansible.plugins.strategy.free -class StrategyModule(ansible_mitogen.strategy.StrategyMixin, - ansible.plugins.strategy.free.StrategyModule): +Base = ansible_mitogen.loaders.strategy_loader.get('free', class_only=True) + +class StrategyModule(ansible_mitogen.strategy.StrategyMixin, Base): pass diff --git a/ansible_mitogen/plugins/strategy/mitogen_linear.py b/ansible_mitogen/plugins/strategy/mitogen_linear.py index 3ef522b4..a5ea2a3d 100644 --- a/ansible_mitogen/plugins/strategy/mitogen_linear.py +++ b/ansible_mitogen/plugins/strategy/mitogen_linear.py @@ -51,10 +51,11 @@ except ImportError: sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) del base_dir +import ansible_mitogen.loaders import ansible_mitogen.strategy -import ansible.plugins.strategy.linear -class StrategyModule(ansible_mitogen.strategy.StrategyMixin, - ansible.plugins.strategy.linear.StrategyModule): +Base = ansible_mitogen.loaders.strategy_loader.get('linear', class_only=True) + +class StrategyModule(ansible_mitogen.strategy.StrategyMixin, Base): pass diff --git a/ansible_mitogen/strategy.py b/ansible_mitogen/strategy.py index 2fd4cb91..6dd8ec04 100644 --- a/ansible_mitogen/strategy.py +++ b/ansible_mitogen/strategy.py @@ -29,16 +29,10 @@ from __future__ import absolute_import import os +import ansible_mitogen.loaders import ansible_mitogen.mixins import ansible_mitogen.process -try: - from ansible.plugins.loader import action_loader - from ansible.plugins.loader import connection_loader -except ImportError: # Ansible <2.4 - from ansible.plugins import action_loader - from ansible.plugins import connection_loader - def wrap_action_loader__get(name, *args, **kwargs): """ @@ -138,19 +132,19 @@ class StrategyMixin(object): with references to the real functions. """ global action_loader__get - action_loader__get = action_loader.get - action_loader.get = wrap_action_loader__get + action_loader__get = ansible_mitogen.loaders.action_loader.get + ansible_mitogen.loaders.action_loader.get = wrap_action_loader__get global connection_loader__get - connection_loader__get = connection_loader.get - connection_loader.get = wrap_connection_loader__get + connection_loader__get = ansible_mitogen.loaders.connection_loader.get + ansible_mitogen.loaders.connection_loader.get = wrap_connection_loader__get def _remove_wrappers(self): """ Uninstall the PluginLoader monkey patches. """ - action_loader.get = action_loader__get - connection_loader.get = connection_loader__get + ansible_mitogen.loaders.action_loader.get = action_loader__get + ansible_mitogen.loaders.connection_loader.get = connection_loader__get def _add_connection_plugin_path(self): """ @@ -158,7 +152,9 @@ class StrategyMixin(object): avoiding the need for manual configuration. """ base_dir = os.path.join(os.path.dirname(__file__), 'plugins') - connection_loader.add_directory(os.path.join(base_dir, 'connection')) + ansible_mitogen.loaders.connection_loader.add_directory( + os.path.join(base_dir, 'connection') + ) def run(self, iterator, play_context, result=0): """ diff --git a/docs/changelog.rst b/docs/changelog.rst index e06c1cfc..658ab66c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -55,9 +55,25 @@ Mitogen for Ansible **Known Issues** -* The Ansible ``raw`` action executes as a regular Mitogen connection, - precluding its use for installing Python on a target. This will be addressed - in a future 0.2 release. +* The ``raw`` action executes as a regular Mitogen connection, which requires + Python on the target, precluding its use for installing Python. This will be + addressed in a future 0.2 release. For now, simply mix Mitogen and vanilla + Ansible strategies in your playbook: + + .. code-block:: yaml + + - hosts: web-servers + strategy: linear + tasks: + - name: Install Python if necessary. + raw: test -e /usr/bin/python || apt install -y python-minimal + + - hosts: web-servers + strategy: mitogen_linear + roles: + - nginx + - initech_app + - y2k_fix * Performance does not scale linearly with target count. This requires significant additional work, as major bottlenecks exist in the surrounding @@ -72,10 +88,10 @@ Mitogen for Ansible * *Module Replacer* style Ansible modules are not supported. * Actions are single-threaded for each `(host, user account)` combination, - including actions that execute on the local machine. Certain styles of - playbook may experience slowdown compared to vanilla Ansible if they employ - long-running ``local_action`` or ``delegate_to`` tasks delegating many target - hosts to a single machine and user account. + including actions that execute on the local machine. Playbooks may experience + slowdown compared to vanilla Ansible if they employ long-running + ``local_action`` or ``delegate_to`` tasks delegating many target hosts to a + single machine and user account. * Connection Delegation remains in preview and has bugs around how it infers connections. Connection establishment will remain single-threaded for the 0.2 @@ -83,8 +99,13 @@ Mitogen for Ansible release. * Connection Delegation does not support automatic tunnelling of SSH-dependent - actions, such as the ``synchronize`` module. This will be added in the 0.3 - series. + actions, such as the ``synchronize`` module. This will be addressed in the + 0.3 series. + +* Configurations will break that rely on the `hashbang argument splitting + behaviour `_ of the + ``ansible_python_interpreter`` setting, contrary to the Ansible + documentation. This will be addressed in a future 0.2 release. Core Library diff --git a/mitogen/debug.py b/mitogen/debug.py index a9a06763..64d2292d 100644 --- a/mitogen/debug.py +++ b/mitogen/debug.py @@ -190,7 +190,10 @@ def _logging_main(): def dump_to_logger(): - th = threading.Thread(target=_logging_main) + th = threading.Thread( + target=_logging_main, + name='mitogen.debug.dump_to_logger', + ) th.setDaemon(True) th.start() diff --git a/tests/ansible/debug_run_ansible_playbook.sh b/tests/ansible/debug_run_ansible_playbook.sh new file mode 100755 index 00000000..ab2c9385 --- /dev/null +++ b/tests/ansible/debug_run_ansible_playbook.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Wrap ansible-playbook, setting up some test of the test environment. + +# Used by delegate_to.yml to ensure "sudo -E" preserves environment. +export I_WAS_PRESERVED=1 +export MITOGEN_MAX_INTERPRETERS=3 + +if [ "${ANSIBLE_STRATEGY:0:7}" = "mitogen" ] +then + EXTRA='{"is_mitogen": true}' +else + EXTRA='{"is_mitogen": false}' +fi + +exec ~/src/cpython/venv/bin/ansible-playbook -e "$EXTRA" -e ansible_python_interpreter=/Users/dmw/src/cpython/venv/bin/python2.7 "$@" diff --git a/tests/ansible/gcloud/ansible.cfg b/tests/ansible/gcloud/ansible.cfg index 75be745c..b7d93d3d 100644 --- a/tests/ansible/gcloud/ansible.cfg +++ b/tests/ansible/gcloud/ansible.cfg @@ -4,3 +4,5 @@ strategy = mitogen inventory = hosts retry_files_enabled = False host_key_checking = False +callback_plugins = ../lib/callback +stdout_callback = nice_stdout diff --git a/tests/ansible/gcloud/gce.yml b/tests/ansible/gcloud/gce.yml new file mode 100644 index 00000000..e3f64c23 --- /dev/null +++ b/tests/ansible/gcloud/gce.yml @@ -0,0 +1,11 @@ + +- hosts: localhost + tasks: + - command: date +%Y%m%d-%H%M%S + register: out + + - set_fact: + instance_name: "controller-{{out.stdout}}" + + - command: > + gcloud compute instances create {{instance_name}} --can-ip-forward --machine-type=n1-standard-8 --preemptible --scopes=compute-ro --image-project=debian-cloud --image-family=debian-9 diff --git a/tests/ansible/integration/all.yml b/tests/ansible/integration/all.yml index 3451f464..b7b7cc96 100644 --- a/tests/ansible/integration/all.yml +++ b/tests/ansible/integration/all.yml @@ -13,4 +13,5 @@ - import_playbook: remote_tmp/all.yml - import_playbook: runner/all.yml - import_playbook: ssh/all.yml +- import_playbook: strategy/all.yml - import_playbook: glibc_caches/all.yml diff --git a/tests/ansible/integration/strategy/all.yml b/tests/ansible/integration/strategy/all.yml new file mode 100644 index 00000000..3e1b1360 --- /dev/null +++ b/tests/ansible/integration/strategy/all.yml @@ -0,0 +1 @@ +- import_playbook: mixed_vanilla_ansible.yml diff --git a/tests/ansible/integration/strategy/mixed_vanilla_ansible.yml b/tests/ansible/integration/strategy/mixed_vanilla_ansible.yml new file mode 100644 index 00000000..f9cda746 --- /dev/null +++ b/tests/ansible/integration/strategy/mixed_vanilla_ansible.yml @@ -0,0 +1,45 @@ + +# issue #294: ensure running mixed vanilla/Mitogen succeeds. + +- name: integration/strategy/mixed_vanilla_ansible.yml (linear) + hosts: test-targets + any_errors_fatal: true + strategy: linear + tasks: + - custom_python_detect_environment: + register: out + - assert: + that: not out.mitogen_loaded + + - determine_strategy: + - assert: + that: strategy == 'ansible.plugins.strategy.linear.StrategyModule' + +- name: integration/strategy/mixed_vanilla_ansible.yml (mitogen_linear) + hosts: test-targets + any_errors_fatal: true + strategy: mitogen_linear + tasks: + - custom_python_detect_environment: + register: out + - assert: + that: out.mitogen_loaded + + - determine_strategy: + - assert: + that: strategy == 'ansible.plugins.strategy.mitogen_linear.StrategyModule' + + +- name: integration/strategy/mixed_vanilla_ansible.yml (linear) + hosts: test-targets + any_errors_fatal: true + strategy: linear + tasks: + - custom_python_detect_environment: + register: out + - assert: + that: not out.mitogen_loaded + + - determine_strategy: + - assert: + that: strategy == 'ansible.plugins.strategy.linear.StrategyModule' diff --git a/tests/bench/fork.py b/tests/bench/fork.py new file mode 100644 index 00000000..b2f2382c --- /dev/null +++ b/tests/bench/fork.py @@ -0,0 +1,15 @@ +""" +Measure latency of .fork() setup/teardown. +""" + +import mitogen +import time + +@mitogen.main() +def main(router): + t0 = time.time() + for x in xrange(200): + t = time.time() + ctx = router.fork() + ctx.shutdown(wait=True) + print '++', 1000 * ((time.time() - t0) / (1.0+x)) diff --git a/tests/bench/local.py b/tests/bench/local.py index 31f3c952..a4ec2428 100644 --- a/tests/bench/local.py +++ b/tests/bench/local.py @@ -2,13 +2,11 @@ Measure latency of .local() setup. """ -import os -import socket import mitogen import time -@mitogen.main() #(log_level='DEBUG') +@mitogen.main() def main(router): for x in range(1000): t = time.time() diff --git a/tests/bench/roundtrip.py b/tests/bench/roundtrip.py new file mode 100644 index 00000000..40582d46 --- /dev/null +++ b/tests/bench/roundtrip.py @@ -0,0 +1,17 @@ +""" +Measure latency of local RPC. +""" + +import mitogen +import time + +def do_nothing(): + pass + +@mitogen.main() +def main(router): + f = router.fork() + t0 = time.time() + for x in xrange(1000): + f.call(do_nothing) + print '++', int(1e6 * ((time.time() - t0) / (1.0+x))), 'usec'