Merge branch 'master' into collectionsSupport

.ci/azure-pipelines.yml

@@ -13,10 +13,10 @@ jobs:
   strategy:
     matrix:
       Mito27_27:
-        python.version: '2.7'
+        python.version: '2.7.18'
         MODE: mitogen
       Ans288_27:
-        python.version: '2.7'
+        python.version: '2.7.18'
         MODE: localhost_ansible
         VER: 2.8.8
 

.ci/localhost_ansible_tests.py

@@ -1,9 +1,7 @@
 #!/usr/bin/env python
 # Run tests/ansible/all.yml under Ansible and Ansible-Mitogen
 
-import glob
 import os
-import shutil
 import sys
 
 import ci_lib
@@ -31,16 +29,17 @@ with ci_lib.Fold('job_setup'):
 
 
 with ci_lib.Fold('machine_prep'):
-    ssh_dir = os.path.expanduser('~/.ssh')
-    if not os.path.exists(ssh_dir):
-        os.makedirs(ssh_dir, int('0700', 8))
-
-    key_path = os.path.expanduser('~/.ssh/id_rsa')
-    shutil.copy(KEY_PATH, key_path)
-
-    auth_path = os.path.expanduser('~/.ssh/authorized_keys')
-    os.system('ssh-keygen -y -f %s >> %s' % (key_path, auth_path))
-    os.chmod(auth_path, int('0600', 8))
+    # generate a new ssh key for localhost ssh
+    os.system("ssh-keygen -P '' -m pem -f ~/.ssh/id_rsa")
+    os.system("cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys")
+    # also generate it for the sudo user
+    os.system("sudo ssh-keygen -P '' -m pem -f /var/root/.ssh/id_rsa")
+    os.system("sudo cat /var/root/.ssh/id_rsa.pub | sudo tee -a /var/root/.ssh/authorized_keys")
+    os.chmod(os.path.expanduser('~/.ssh'), int('0700', 8))
+    os.chmod(os.path.expanduser('~/.ssh/authorized_keys'), int('0600', 8))
+    # run chmod through sudo since it's owned by root
+    os.system('sudo chmod 600 /var/root/.ssh')
+    os.system('sudo chmod 600 /var/root/.ssh/authorized_keys')
 
     if os.path.expanduser('~mitogen__user1') == '~mitogen__user1':
         os.chdir(IMAGE_PREP_DIR)

README.md

@@ -1,4 +1,3 @@
-
 # Mitogen
 
 <!-- [![Build Status](https://travis-ci.org/dw/mitogen.png?branch=master)](https://travis-ci.org/dw/mitogen}) -->

ansible_mitogen/planner.py

@@ -96,6 +96,9 @@ class Invocation(object):
         #: Initially ``None``, but set by :func:`invoke`. The raw source or
         #: binary contents of the module.
         self._module_source = None
+        #: Initially ``{}``, but set by :func:`invoke`. Optional source to send
+        #: to :func:`propagate_paths_and_modules` to fix Python3.5 relative import errors
+        self._overridden_sources = {}
 
     def get_module_source(self):
         if self._module_source is None:
@@ -476,6 +479,7 @@ def _propagate_deps(invocation, planner, context):
         context=context,
         paths=planner.get_push_files(),
         modules=planner.get_module_deps(),
+        overridden_sources=invocation._overridden_sources
     )
 
 
@@ -533,6 +537,26 @@ def _get_planner(name, path, source):
     raise ansible.errors.AnsibleError(NO_METHOD_MSG + repr(invocation))
 
 
+def _fix_py35(invocation, module_source):
+    """
+    super edge case with a relative import error in Python 3.5.1-3.5.3
+    in Ansible's setup module when using Mitogen
+    https://github.com/dw/mitogen/issues/672#issuecomment-636408833
+    We replace a relative import in the setup module with the actual full file path
+    This works in vanilla Ansible but not in Mitogen otherwise
+    """
+    if invocation.module_name == 'setup' and \
+            invocation.module_path not in invocation._overridden_sources:
+        # in-memory replacement of setup module's relative import
+        # would check for just python3.5 and run this then but we don't know the
+        # target python at this time yet
+        module_source = module_source.replace(
+            b"from ...module_utils.basic import AnsibleModule",
+            b"from ansible.module_utils.basic import AnsibleModule"
+        )
+        invocation._overridden_sources[invocation.module_path] = module_source
+
+
 def invoke(invocation):
     """
     Find a Planner subclass corresponding to `invocation` and use it to invoke
@@ -558,10 +582,12 @@ def invoke(invocation):
     # if 'ansible_collections' in invocation.module_path:
     #     import epdb; epdb.set_trace()
     if invocation.module_path not in _planner_by_path:
+        module_source = invocation.get_module_source()
+        _fix_py35(invocation, module_source)
         _planner_by_path[invocation.module_path] = _get_planner(
             invocation.module_name,
             invocation.module_path,
-            invocation.get_module_source()
+            module_source
         )
 
     planner = _planner_by_path[invocation.module_path](invocation)

docs/ansible_detailed.rst

@@ -9,7 +9,7 @@ Mitogen for Ansible
 
 **Mitogen for Ansible** is a completely redesigned UNIX connection layer and
 module runtime for `Ansible`_. Requiring minimal configuration changes, it
-updates Ansible's slow and wasteful shell-centic implementation with
+updates Ansible's slow and wasteful shell-centric implementation with
 pure-Python equivalents, invoked via highly efficient remote procedure calls to
 persistent interpreters tunnelled over SSH. No changes are required to target
 hosts.
@@ -1009,7 +1009,7 @@ Like the :ans:conn:`ssh` except connection delegation is supported.
 * ``mitogen_ssh_keepalive_count``: integer count of server keepalive messages to
   which no reply is received before considering the SSH server dead. Defaults
   to 10.
-* ``mitogen_ssh_keepalive_count``: integer seconds delay between keepalive
+* ``mitogen_ssh_keepalive_interval``: integer seconds delay between keepalive
   messages. Defaults to 30.
 
 

mitogen/service.py

@@ -746,13 +746,18 @@ class PushFileService(Service):
         'paths': list,
         'modules': list,
     })
-    def propagate_paths_and_modules(self, context, paths, modules):
+    def propagate_paths_and_modules(self, context, paths, modules, overridden_sources=None):
         """
         One size fits all method to ensure a target context has been preloaded
         with a set of small files and Python modules.
+
+        overridden_sources: optional dict containing source code to override path's source code
         """
         for path in paths:
-            self.propagate_to(context, mitogen.core.to_text(path))
+            overridden_source = None
+            if overridden_sources is not None and path in overridden_sources:
+                overridden_source = overridden_sources[path]
+            self.propagate_to(context, mitogen.core.to_text(path), overridden_source)
         #self.router.responder.forward_modules(context, modules) TODO
 
     @expose(policy=AllowParents())
@@ -760,14 +765,22 @@ class PushFileService(Service):
         'context': mitogen.core.Context,
         'path': mitogen.core.FsPathTypes,
     })
-    def propagate_to(self, context, path):
+    def propagate_to(self, context, path, overridden_source=None):
+        """
+        If the optional parameter 'overridden_source' is passed, use
+        that instead of the path's code as source code. This works around some bugs
+        of source modules such as relative imports on unsupported Python versions
+        """
         if path not in self._cache:
             LOG.debug('caching small file %s', path)
-            fp = open(path, 'rb')
-            try:
-                self._cache[path] = mitogen.core.Blob(fp.read())
-            finally:
-                fp.close()
+            if overridden_source is None:
+                fp = open(path, 'rb')
+                try:
+                    self._cache[path] = mitogen.core.Blob(fp.read())
+                finally:
+                    fp.close()
+            else:
+                self._cache[path] = mitogen.core.Blob(overridden_source)
         self._forward(context, path)
 
     @expose(policy=AllowParents())