Merge pull request #296 from dw/dmw

Dmw

ansible_mitogen/loaders.py

@@ -0,0 +1,46 @@
+# Copyright 2017, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+"""
+Stable names for PluginLoader instances across Ansible versions.
+"""
+
+from __future__ import absolute_import
+
+try:
+    from ansible.plugins.loader import action_loader
+    from ansible.plugins.loader import connection_loader
+    from ansible.plugins.loader import module_loader
+    from ansible.plugins.loader import module_utils_loader
+    from ansible.plugins.loader import strategy_loader
+except ImportError:  # Ansible <2.4
+    from ansible.plugins import action_loader
+    from ansible.plugins import connection_loader
+    from ansible.plugins import module_loader
+    from ansible.plugins import module_utils_loader
+    from ansible.plugins import strategy_loader

ansible_mitogen/planner.py

@@ -47,13 +47,7 @@ import ansible.errors
 import ansible.module_utils
 import mitogen.core
 
-try:
+import ansible_mitogen.loaders
-    from ansible.plugins.loader import module_loader
-    from ansible.plugins.loader import module_utils_loader
-except ImportError:  # Ansible <2.4
-    from ansible.plugins import module_loader
-    from ansible.plugins import module_utils_loader
-
 import ansible_mitogen.target
 
 
@@ -322,7 +316,9 @@ class NewStylePlanner(ScriptPlanner):
     def get_search_path(self):
         return tuple(
             path
-            for path in module_utils_loader._get_paths(subdirs=False)
+            for path in ansible_mitogen.loaders.module_utils_loader._get_paths(
+                subdirs=False
+            )
             if os.path.isdir(path)
         )
 
@@ -397,7 +393,7 @@ _planners = [
 
 
 def get_module_data(name):
-    path = module_loader.find_plugin(name, '')
+    path = ansible_mitogen.loaders.module_loader.find_plugin(name, '')
     with open(path, 'rb') as fp:
         source = fp.read()
     return mitogen.core.to_text(path), source

ansible_mitogen/plugins/strategy/mitogen_free.py

@@ -51,10 +51,11 @@ except ImportError:
     sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
     del base_dir
 
+import ansible_mitogen.loaders
 import ansible_mitogen.strategy
-import ansible.plugins.strategy.free
 
 
-class StrategyModule(ansible_mitogen.strategy.StrategyMixin,
+Base = ansible_mitogen.loaders.strategy_loader.get('free', class_only=True)
-                     ansible.plugins.strategy.free.StrategyModule):
+
+class StrategyModule(ansible_mitogen.strategy.StrategyMixin, Base):
     pass

ansible_mitogen/plugins/strategy/mitogen_linear.py

@@ -51,10 +51,11 @@ except ImportError:
     sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
     del base_dir
 
+import ansible_mitogen.loaders
 import ansible_mitogen.strategy
-import ansible.plugins.strategy.linear
 
 
-class StrategyModule(ansible_mitogen.strategy.StrategyMixin,
+Base = ansible_mitogen.loaders.strategy_loader.get('linear', class_only=True)
-                     ansible.plugins.strategy.linear.StrategyModule):
+
+class StrategyModule(ansible_mitogen.strategy.StrategyMixin, Base):
     pass

ansible_mitogen/strategy.py

@@ -29,16 +29,10 @@
 from __future__ import absolute_import
 import os
 
+import ansible_mitogen.loaders
 import ansible_mitogen.mixins
 import ansible_mitogen.process
 
-try:
-    from ansible.plugins.loader import action_loader
-    from ansible.plugins.loader import connection_loader
-except ImportError:  # Ansible <2.4
-    from ansible.plugins import action_loader
-    from ansible.plugins import connection_loader
-
 
 def wrap_action_loader__get(name, *args, **kwargs):
     """
@@ -138,19 +132,19 @@ class StrategyMixin(object):
         with references to the real functions.
         """
         global action_loader__get
-        action_loader__get = action_loader.get
+        action_loader__get = ansible_mitogen.loaders.action_loader.get
-        action_loader.get = wrap_action_loader__get
+        ansible_mitogen.loaders.action_loader.get = wrap_action_loader__get
 
         global connection_loader__get
-        connection_loader__get = connection_loader.get
+        connection_loader__get = ansible_mitogen.loaders.connection_loader.get
-        connection_loader.get = wrap_connection_loader__get
+        ansible_mitogen.loaders.connection_loader.get = wrap_connection_loader__get
 
     def _remove_wrappers(self):
         """
         Uninstall the PluginLoader monkey patches.
         """
-        action_loader.get = action_loader__get
+        ansible_mitogen.loaders.action_loader.get = action_loader__get
-        connection_loader.get = connection_loader__get
+        ansible_mitogen.loaders.connection_loader.get = connection_loader__get
 
     def _add_connection_plugin_path(self):
         """
@@ -158,7 +152,9 @@ class StrategyMixin(object):
         avoiding the need for manual configuration.
         """
         base_dir = os.path.join(os.path.dirname(__file__), 'plugins')
-        connection_loader.add_directory(os.path.join(base_dir, 'connection'))
+        ansible_mitogen.loaders.connection_loader.add_directory(
+            os.path.join(base_dir, 'connection')
+        )
 
     def run(self, iterator, play_context, result=0):
         """

docs/changelog.rst

@@ -55,9 +55,25 @@ Mitogen for Ansible
 
 **Known Issues**
 
-* The Ansible ``raw`` action executes as a regular Mitogen connection,
+* The ``raw`` action executes as a regular Mitogen connection, which requires
-  precluding its use for installing Python on a target. This will be addressed
+  Python on the target, precluding its use for installing Python. This will be
-  in a future 0.2 release.
+  addressed in a future 0.2 release. For now, simply mix Mitogen and vanilla
+  Ansible strategies in your playbook:
+
+  .. code-block:: yaml
+
+    - hosts: web-servers
+      strategy: linear
+      tasks:
+      - name: Install Python if necessary.
+        raw: test -e /usr/bin/python || apt install -y python-minimal
+
+    - hosts: web-servers
+      strategy: mitogen_linear
+      roles:
+      - nginx
+      - initech_app
+      - y2k_fix
 
 * Performance does not scale linearly with target count. This requires
   significant additional work, as major bottlenecks exist in the surrounding
@@ -72,10 +88,10 @@ Mitogen for Ansible
 * *Module Replacer* style Ansible modules are not supported.
 
 * Actions are single-threaded for each `(host, user account)` combination,
-  including actions that execute on the local machine. Certain styles of
+  including actions that execute on the local machine. Playbooks may experience
-  playbook may experience slowdown compared to vanilla Ansible if they employ
+  slowdown compared to vanilla Ansible if they employ long-running
-  long-running ``local_action`` or ``delegate_to`` tasks delegating many target
+  ``local_action`` or ``delegate_to`` tasks delegating many target hosts to a
-  hosts to a single machine and user account.
+  single machine and user account.
 
 * Connection Delegation remains in preview and has bugs around how it infers
   connections. Connection establishment will remain single-threaded for the 0.2
@@ -83,8 +99,13 @@ Mitogen for Ansible
   release.
 
 * Connection Delegation does not support automatic tunnelling of SSH-dependent
-  actions, such as the ``synchronize`` module. This will be added in the 0.3
+  actions, such as the ``synchronize`` module. This will be addressed in the
-  series.
+  0.3 series.
+
+* Configurations will break that rely on the `hashbang argument splitting
+  behaviour <https://github.com/ansible/ansible/issues/15635>`_ of the
+  ``ansible_python_interpreter`` setting, contrary to the Ansible
+  documentation. This will be addressed in a future 0.2 release.
 
 
 Core Library

mitogen/debug.py

@@ -190,7 +190,10 @@ def _logging_main():
 
 
 def dump_to_logger():
-    th = threading.Thread(target=_logging_main)
+    th = threading.Thread(
+        target=_logging_main,
+        name='mitogen.debug.dump_to_logger',
+    )
     th.setDaemon(True)
     th.start()
 

tests/ansible/debug_run_ansible_playbook.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Wrap ansible-playbook, setting up some test of the test environment.
+
+# Used by delegate_to.yml to ensure "sudo -E" preserves environment.
+export I_WAS_PRESERVED=1
+export MITOGEN_MAX_INTERPRETERS=3
+
+if [ "${ANSIBLE_STRATEGY:0:7}" = "mitogen" ]
+then
+    EXTRA='{"is_mitogen": true}'
+else
+    EXTRA='{"is_mitogen": false}'
+fi
+
+exec ~/src/cpython/venv/bin/ansible-playbook -e "$EXTRA" -e ansible_python_interpreter=/Users/dmw/src/cpython/venv/bin/python2.7 "$@"

tests/ansible/gcloud/ansible.cfg

@@ -4,3 +4,5 @@ strategy = mitogen
 inventory = hosts
 retry_files_enabled = False
 host_key_checking = False
+callback_plugins = ../lib/callback
+stdout_callback = nice_stdout

tests/ansible/gcloud/gce.yml

@@ -0,0 +1,11 @@
+
+- hosts: localhost
+  tasks:
+    - command: date +%Y%m%d-%H%M%S
+      register: out
+
+    - set_fact:
+        instance_name: "controller-{{out.stdout}}"
+
+    - command: >
+        gcloud compute instances create {{instance_name}} --can-ip-forward --machine-type=n1-standard-8 --preemptible --scopes=compute-ro --image-project=debian-cloud --image-family=debian-9

tests/ansible/integration/all.yml

@@ -13,4 +13,5 @@
 - import_playbook: remote_tmp/all.yml
 - import_playbook: runner/all.yml
 - import_playbook: ssh/all.yml
+- import_playbook: strategy/all.yml
 - import_playbook: glibc_caches/all.yml

tests/ansible/integration/strategy/all.yml

@@ -0,0 +1 @@
+- import_playbook: mixed_vanilla_ansible.yml

tests/ansible/integration/strategy/mixed_vanilla_ansible.yml

@@ -0,0 +1,45 @@
+
+# issue #294: ensure running mixed vanilla/Mitogen succeeds.
+
+- name: integration/strategy/mixed_vanilla_ansible.yml (linear)
+  hosts: test-targets
+  any_errors_fatal: true
+  strategy: linear
+  tasks:
+    - custom_python_detect_environment:
+      register: out
+    - assert:
+        that: not out.mitogen_loaded
+
+    - determine_strategy:
+    - assert:
+        that: strategy == 'ansible.plugins.strategy.linear.StrategyModule'
+
+- name: integration/strategy/mixed_vanilla_ansible.yml (mitogen_linear)
+  hosts: test-targets
+  any_errors_fatal: true
+  strategy: mitogen_linear
+  tasks:
+    - custom_python_detect_environment:
+      register: out
+    - assert:
+        that: out.mitogen_loaded
+
+    - determine_strategy:
+    - assert:
+        that: strategy == 'ansible.plugins.strategy.mitogen_linear.StrategyModule'
+
+
+- name: integration/strategy/mixed_vanilla_ansible.yml (linear)
+  hosts: test-targets
+  any_errors_fatal: true
+  strategy: linear
+  tasks:
+    - custom_python_detect_environment:
+      register: out
+    - assert:
+        that: not out.mitogen_loaded
+
+    - determine_strategy:
+    - assert:
+        that: strategy == 'ansible.plugins.strategy.linear.StrategyModule'

tests/bench/fork.py

@@ -0,0 +1,15 @@
+"""
+Measure latency of .fork() setup/teardown.
+"""
+
+import mitogen
+import time
+
+@mitogen.main()
+def main(router):
+    t0 = time.time()
+    for x in xrange(200):
+        t = time.time()
+        ctx = router.fork()
+        ctx.shutdown(wait=True)
+    print '++', 1000 * ((time.time() - t0) / (1.0+x))

tests/bench/local.py

@@ -2,13 +2,11 @@
 Measure latency of .local() setup.
 """
 
-import os
-import socket
 import mitogen
 import time
 
 
-@mitogen.main() #(log_level='DEBUG')
+@mitogen.main()
 def main(router):
     for x in range(1000):
         t = time.time()

tests/bench/roundtrip.py

@@ -0,0 +1,17 @@
+"""
+Measure latency of local RPC.
+"""
+
+import mitogen
+import time
+
+def do_nothing():
+    pass
+
+@mitogen.main()
+def main(router):
+    f = router.fork()
+    t0 = time.time()
+    for x in xrange(1000):
+        f.call(do_nothing)
+    print '++', int(1e6 * ((time.time() - t0) / (1.0+x))), 'usec'