Merge pull request #296 from dw/dmw

Dmw
pull/303/head
dw 6 years ago committed by GitHub
commit 239ef65652
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -0,0 +1,46 @@
# Copyright 2017, David Wilson
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
"""
Stable names for PluginLoader instances across Ansible versions.
"""
from __future__ import absolute_import
try:
from ansible.plugins.loader import action_loader
from ansible.plugins.loader import connection_loader
from ansible.plugins.loader import module_loader
from ansible.plugins.loader import module_utils_loader
from ansible.plugins.loader import strategy_loader
except ImportError: # Ansible <2.4
from ansible.plugins import action_loader
from ansible.plugins import connection_loader
from ansible.plugins import module_loader
from ansible.plugins import module_utils_loader
from ansible.plugins import strategy_loader

@ -47,13 +47,7 @@ import ansible.errors
import ansible.module_utils
import mitogen.core
try:
from ansible.plugins.loader import module_loader
from ansible.plugins.loader import module_utils_loader
except ImportError: # Ansible <2.4
from ansible.plugins import module_loader
from ansible.plugins import module_utils_loader
import ansible_mitogen.loaders
import ansible_mitogen.target
@ -322,7 +316,9 @@ class NewStylePlanner(ScriptPlanner):
def get_search_path(self):
return tuple(
path
for path in module_utils_loader._get_paths(subdirs=False)
for path in ansible_mitogen.loaders.module_utils_loader._get_paths(
subdirs=False
)
if os.path.isdir(path)
)
@ -397,7 +393,7 @@ _planners = [
def get_module_data(name):
path = module_loader.find_plugin(name, '')
path = ansible_mitogen.loaders.module_loader.find_plugin(name, '')
with open(path, 'rb') as fp:
source = fp.read()
return mitogen.core.to_text(path), source

@ -51,10 +51,11 @@ except ImportError:
sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
del base_dir
import ansible_mitogen.loaders
import ansible_mitogen.strategy
import ansible.plugins.strategy.free
class StrategyModule(ansible_mitogen.strategy.StrategyMixin,
ansible.plugins.strategy.free.StrategyModule):
Base = ansible_mitogen.loaders.strategy_loader.get('free', class_only=True)
class StrategyModule(ansible_mitogen.strategy.StrategyMixin, Base):
pass

@ -51,10 +51,11 @@ except ImportError:
sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
del base_dir
import ansible_mitogen.loaders
import ansible_mitogen.strategy
import ansible.plugins.strategy.linear
class StrategyModule(ansible_mitogen.strategy.StrategyMixin,
ansible.plugins.strategy.linear.StrategyModule):
Base = ansible_mitogen.loaders.strategy_loader.get('linear', class_only=True)
class StrategyModule(ansible_mitogen.strategy.StrategyMixin, Base):
pass

@ -29,16 +29,10 @@
from __future__ import absolute_import
import os
import ansible_mitogen.loaders
import ansible_mitogen.mixins
import ansible_mitogen.process
try:
from ansible.plugins.loader import action_loader
from ansible.plugins.loader import connection_loader
except ImportError: # Ansible <2.4
from ansible.plugins import action_loader
from ansible.plugins import connection_loader
def wrap_action_loader__get(name, *args, **kwargs):
"""
@ -138,19 +132,19 @@ class StrategyMixin(object):
with references to the real functions.
"""
global action_loader__get
action_loader__get = action_loader.get
action_loader.get = wrap_action_loader__get
action_loader__get = ansible_mitogen.loaders.action_loader.get
ansible_mitogen.loaders.action_loader.get = wrap_action_loader__get
global connection_loader__get
connection_loader__get = connection_loader.get
connection_loader.get = wrap_connection_loader__get
connection_loader__get = ansible_mitogen.loaders.connection_loader.get
ansible_mitogen.loaders.connection_loader.get = wrap_connection_loader__get
def _remove_wrappers(self):
"""
Uninstall the PluginLoader monkey patches.
"""
action_loader.get = action_loader__get
connection_loader.get = connection_loader__get
ansible_mitogen.loaders.action_loader.get = action_loader__get
ansible_mitogen.loaders.connection_loader.get = connection_loader__get
def _add_connection_plugin_path(self):
"""
@ -158,7 +152,9 @@ class StrategyMixin(object):
avoiding the need for manual configuration.
"""
base_dir = os.path.join(os.path.dirname(__file__), 'plugins')
connection_loader.add_directory(os.path.join(base_dir, 'connection'))
ansible_mitogen.loaders.connection_loader.add_directory(
os.path.join(base_dir, 'connection')
)
def run(self, iterator, play_context, result=0):
"""

@ -55,9 +55,25 @@ Mitogen for Ansible
**Known Issues**
* The Ansible ``raw`` action executes as a regular Mitogen connection,
precluding its use for installing Python on a target. This will be addressed
in a future 0.2 release.
* The ``raw`` action executes as a regular Mitogen connection, which requires
Python on the target, precluding its use for installing Python. This will be
addressed in a future 0.2 release. For now, simply mix Mitogen and vanilla
Ansible strategies in your playbook:
.. code-block:: yaml
- hosts: web-servers
strategy: linear
tasks:
- name: Install Python if necessary.
raw: test -e /usr/bin/python || apt install -y python-minimal
- hosts: web-servers
strategy: mitogen_linear
roles:
- nginx
- initech_app
- y2k_fix
* Performance does not scale linearly with target count. This requires
significant additional work, as major bottlenecks exist in the surrounding
@ -72,10 +88,10 @@ Mitogen for Ansible
* *Module Replacer* style Ansible modules are not supported.
* Actions are single-threaded for each `(host, user account)` combination,
including actions that execute on the local machine. Certain styles of
playbook may experience slowdown compared to vanilla Ansible if they employ
long-running ``local_action`` or ``delegate_to`` tasks delegating many target
hosts to a single machine and user account.
including actions that execute on the local machine. Playbooks may experience
slowdown compared to vanilla Ansible if they employ long-running
``local_action`` or ``delegate_to`` tasks delegating many target hosts to a
single machine and user account.
* Connection Delegation remains in preview and has bugs around how it infers
connections. Connection establishment will remain single-threaded for the 0.2
@ -83,8 +99,13 @@ Mitogen for Ansible
release.
* Connection Delegation does not support automatic tunnelling of SSH-dependent
actions, such as the ``synchronize`` module. This will be added in the 0.3
series.
actions, such as the ``synchronize`` module. This will be addressed in the
0.3 series.
* Configurations will break that rely on the `hashbang argument splitting
behaviour <https://github.com/ansible/ansible/issues/15635>`_ of the
``ansible_python_interpreter`` setting, contrary to the Ansible
documentation. This will be addressed in a future 0.2 release.
Core Library

@ -190,7 +190,10 @@ def _logging_main():
def dump_to_logger():
th = threading.Thread(target=_logging_main)
th = threading.Thread(
target=_logging_main,
name='mitogen.debug.dump_to_logger',
)
th.setDaemon(True)
th.start()

@ -0,0 +1,15 @@
#!/bin/bash
# Wrap ansible-playbook, setting up some test of the test environment.
# Used by delegate_to.yml to ensure "sudo -E" preserves environment.
export I_WAS_PRESERVED=1
export MITOGEN_MAX_INTERPRETERS=3
if [ "${ANSIBLE_STRATEGY:0:7}" = "mitogen" ]
then
EXTRA='{"is_mitogen": true}'
else
EXTRA='{"is_mitogen": false}'
fi
exec ~/src/cpython/venv/bin/ansible-playbook -e "$EXTRA" -e ansible_python_interpreter=/Users/dmw/src/cpython/venv/bin/python2.7 "$@"

@ -4,3 +4,5 @@ strategy = mitogen
inventory = hosts
retry_files_enabled = False
host_key_checking = False
callback_plugins = ../lib/callback
stdout_callback = nice_stdout

@ -0,0 +1,11 @@
- hosts: localhost
tasks:
- command: date +%Y%m%d-%H%M%S
register: out
- set_fact:
instance_name: "controller-{{out.stdout}}"
- command: >
gcloud compute instances create {{instance_name}} --can-ip-forward --machine-type=n1-standard-8 --preemptible --scopes=compute-ro --image-project=debian-cloud --image-family=debian-9

@ -13,4 +13,5 @@
- import_playbook: remote_tmp/all.yml
- import_playbook: runner/all.yml
- import_playbook: ssh/all.yml
- import_playbook: strategy/all.yml
- import_playbook: glibc_caches/all.yml

@ -0,0 +1 @@
- import_playbook: mixed_vanilla_ansible.yml

@ -0,0 +1,45 @@
# issue #294: ensure running mixed vanilla/Mitogen succeeds.
- name: integration/strategy/mixed_vanilla_ansible.yml (linear)
hosts: test-targets
any_errors_fatal: true
strategy: linear
tasks:
- custom_python_detect_environment:
register: out
- assert:
that: not out.mitogen_loaded
- determine_strategy:
- assert:
that: strategy == 'ansible.plugins.strategy.linear.StrategyModule'
- name: integration/strategy/mixed_vanilla_ansible.yml (mitogen_linear)
hosts: test-targets
any_errors_fatal: true
strategy: mitogen_linear
tasks:
- custom_python_detect_environment:
register: out
- assert:
that: out.mitogen_loaded
- determine_strategy:
- assert:
that: strategy == 'ansible.plugins.strategy.mitogen_linear.StrategyModule'
- name: integration/strategy/mixed_vanilla_ansible.yml (linear)
hosts: test-targets
any_errors_fatal: true
strategy: linear
tasks:
- custom_python_detect_environment:
register: out
- assert:
that: not out.mitogen_loaded
- determine_strategy:
- assert:
that: strategy == 'ansible.plugins.strategy.linear.StrategyModule'

@ -0,0 +1,15 @@
"""
Measure latency of .fork() setup/teardown.
"""
import mitogen
import time
@mitogen.main()
def main(router):
t0 = time.time()
for x in xrange(200):
t = time.time()
ctx = router.fork()
ctx.shutdown(wait=True)
print '++', 1000 * ((time.time() - t0) / (1.0+x))

@ -2,13 +2,11 @@
Measure latency of .local() setup.
"""
import os
import socket
import mitogen
import time
@mitogen.main() #(log_level='DEBUG')
@mitogen.main()
def main(router):
for x in range(1000):
t = time.time()

@ -0,0 +1,17 @@
"""
Measure latency of local RPC.
"""
import mitogen
import time
def do_nothing():
pass
@mitogen.main()
def main(router):
f = router.fork()
t0 = time.time()
for x in xrange(1000):
f.call(do_nothing)
print '++', int(1e6 * ((time.time() - t0) / (1.0+x))), 'usec'
Loading…
Cancel
Save