Merge branch 'master' into collectionsSupport

pull/715/head
Steven Robertson 4 years ago committed by GitHub
commit e34cf8667f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -13,10 +13,10 @@ jobs:
strategy:
matrix:
Mito27_27:
python.version: '2.7'
python.version: '2.7.18'
MODE: mitogen
Ans288_27:
python.version: '2.7'
python.version: '2.7.18'
MODE: localhost_ansible
VER: 2.8.8

@ -1,9 +1,7 @@
#!/usr/bin/env python
# Run tests/ansible/all.yml under Ansible and Ansible-Mitogen
import glob
import os
import shutil
import sys
import ci_lib
@ -31,16 +29,17 @@ with ci_lib.Fold('job_setup'):
with ci_lib.Fold('machine_prep'):
ssh_dir = os.path.expanduser('~/.ssh')
if not os.path.exists(ssh_dir):
os.makedirs(ssh_dir, int('0700', 8))
key_path = os.path.expanduser('~/.ssh/id_rsa')
shutil.copy(KEY_PATH, key_path)
auth_path = os.path.expanduser('~/.ssh/authorized_keys')
os.system('ssh-keygen -y -f %s >> %s' % (key_path, auth_path))
os.chmod(auth_path, int('0600', 8))
# generate a new ssh key for localhost ssh
os.system("ssh-keygen -P '' -m pem -f ~/.ssh/id_rsa")
os.system("cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys")
# also generate it for the sudo user
os.system("sudo ssh-keygen -P '' -m pem -f /var/root/.ssh/id_rsa")
os.system("sudo cat /var/root/.ssh/id_rsa.pub | sudo tee -a /var/root/.ssh/authorized_keys")
os.chmod(os.path.expanduser('~/.ssh'), int('0700', 8))
os.chmod(os.path.expanduser('~/.ssh/authorized_keys'), int('0600', 8))
# run chmod through sudo since it's owned by root
os.system('sudo chmod 600 /var/root/.ssh')
os.system('sudo chmod 600 /var/root/.ssh/authorized_keys')
if os.path.expanduser('~mitogen__user1') == '~mitogen__user1':
os.chdir(IMAGE_PREP_DIR)

@ -1,4 +1,3 @@
# Mitogen
<!-- [![Build Status](https://travis-ci.org/dw/mitogen.png?branch=master)](https://travis-ci.org/dw/mitogen}) -->

@ -96,6 +96,9 @@ class Invocation(object):
#: Initially ``None``, but set by :func:`invoke`. The raw source or
#: binary contents of the module.
self._module_source = None
#: Initially ``{}``, but set by :func:`invoke`. Optional source to send
#: to :func:`propagate_paths_and_modules` to fix Python3.5 relative import errors
self._overridden_sources = {}
def get_module_source(self):
if self._module_source is None:
@ -476,6 +479,7 @@ def _propagate_deps(invocation, planner, context):
context=context,
paths=planner.get_push_files(),
modules=planner.get_module_deps(),
overridden_sources=invocation._overridden_sources
)
@ -533,6 +537,26 @@ def _get_planner(name, path, source):
raise ansible.errors.AnsibleError(NO_METHOD_MSG + repr(invocation))
def _fix_py35(invocation, module_source):
"""
super edge case with a relative import error in Python 3.5.1-3.5.3
in Ansible's setup module when using Mitogen
https://github.com/dw/mitogen/issues/672#issuecomment-636408833
We replace a relative import in the setup module with the actual full file path
This works in vanilla Ansible but not in Mitogen otherwise
"""
if invocation.module_name == 'setup' and \
invocation.module_path not in invocation._overridden_sources:
# in-memory replacement of setup module's relative import
# would check for just python3.5 and run this then but we don't know the
# target python at this time yet
module_source = module_source.replace(
b"from ...module_utils.basic import AnsibleModule",
b"from ansible.module_utils.basic import AnsibleModule"
)
invocation._overridden_sources[invocation.module_path] = module_source
def invoke(invocation):
"""
Find a Planner subclass corresponding to `invocation` and use it to invoke
@ -558,10 +582,12 @@ def invoke(invocation):
# if 'ansible_collections' in invocation.module_path:
# import epdb; epdb.set_trace()
if invocation.module_path not in _planner_by_path:
module_source = invocation.get_module_source()
_fix_py35(invocation, module_source)
_planner_by_path[invocation.module_path] = _get_planner(
invocation.module_name,
invocation.module_path,
invocation.get_module_source()
module_source
)
planner = _planner_by_path[invocation.module_path](invocation)

@ -9,7 +9,7 @@ Mitogen for Ansible
**Mitogen for Ansible** is a completely redesigned UNIX connection layer and
module runtime for `Ansible`_. Requiring minimal configuration changes, it
updates Ansible's slow and wasteful shell-centic implementation with
updates Ansible's slow and wasteful shell-centric implementation with
pure-Python equivalents, invoked via highly efficient remote procedure calls to
persistent interpreters tunnelled over SSH. No changes are required to target
hosts.
@ -1009,7 +1009,7 @@ Like the :ans:conn:`ssh` except connection delegation is supported.
* ``mitogen_ssh_keepalive_count``: integer count of server keepalive messages to
which no reply is received before considering the SSH server dead. Defaults
to 10.
* ``mitogen_ssh_keepalive_count``: integer seconds delay between keepalive
* ``mitogen_ssh_keepalive_interval``: integer seconds delay between keepalive
messages. Defaults to 30.

@ -746,13 +746,18 @@ class PushFileService(Service):
'paths': list,
'modules': list,
})
def propagate_paths_and_modules(self, context, paths, modules):
def propagate_paths_and_modules(self, context, paths, modules, overridden_sources=None):
"""
One size fits all method to ensure a target context has been preloaded
with a set of small files and Python modules.
overridden_sources: optional dict containing source code to override path's source code
"""
for path in paths:
self.propagate_to(context, mitogen.core.to_text(path))
overridden_source = None
if overridden_sources is not None and path in overridden_sources:
overridden_source = overridden_sources[path]
self.propagate_to(context, mitogen.core.to_text(path), overridden_source)
#self.router.responder.forward_modules(context, modules) TODO
@expose(policy=AllowParents())
@ -760,14 +765,22 @@ class PushFileService(Service):
'context': mitogen.core.Context,
'path': mitogen.core.FsPathTypes,
})
def propagate_to(self, context, path):
def propagate_to(self, context, path, overridden_source=None):
"""
If the optional parameter 'overridden_source' is passed, use
that instead of the path's code as source code. This works around some bugs
of source modules such as relative imports on unsupported Python versions
"""
if path not in self._cache:
LOG.debug('caching small file %s', path)
if overridden_source is None:
fp = open(path, 'rb')
try:
self._cache[path] = mitogen.core.Blob(fp.read())
finally:
fp.close()
else:
self._cache[path] = mitogen.core.Blob(overridden_source)
self._forward(context, path)
@expose(policy=AllowParents())

Loading…
Cancel
Save