From 0e47280e43216b328ed55ffa88464cd7104de9ef Mon Sep 17 00:00:00 2001 From: Luiz Ribeiro Date: Sun, 7 Jun 2020 11:00:32 -0400 Subject: [PATCH 1/4] Fix mitogen_ssh_keepalive_interval documentation --- docs/ansible_detailed.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index 27798faa..28197ab9 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -1009,7 +1009,7 @@ Like the :ans:conn:`ssh` except connection delegation is supported. * ``mitogen_ssh_keepalive_count``: integer count of server keepalive messages to which no reply is received before considering the SSH server dead. Defaults to 10. -* ``mitogen_ssh_keepalive_count``: integer seconds delay between keepalive +* ``mitogen_ssh_keepalive_interval``: integer seconds delay between keepalive messages. Defaults to 30. From 81076c9da81c214e4af12112c1a3fe5010e35a4a Mon Sep 17 00:00:00 2001 From: Steven Robertson Date: Sat, 30 May 2020 18:25:46 -0700 Subject: [PATCH 2/4] fixes setup module relative import fail on some pythons --- ansible_mitogen/planner.py | 28 +++++++++++++++++++++++++++- mitogen/service.py | 29 +++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 460e5be4..d070daeb 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -96,6 +96,9 @@ class Invocation(object): #: Initially ``None``, but set by :func:`invoke`. The raw source or #: binary contents of the module. self._module_source = None + #: Initially ``{}``, but set by :func:`invoke`. Optional source to send + #: to :func:`propagate_paths_and_modules` to fix Python3.5 relative import errors + self._overridden_sources = {} def get_module_source(self): if self._module_source is None: @@ -476,6 +479,7 @@ def _propagate_deps(invocation, planner, context): context=context, paths=planner.get_push_files(), modules=planner.get_module_deps(), + overridden_sources=invocation._overridden_sources ) @@ -533,6 +537,26 @@ def _get_planner(name, path, source): raise ansible.errors.AnsibleError(NO_METHOD_MSG + repr(invocation)) +def _fix_py35(invocation, module_source): + """ + super edge case with a relative import error in Python 3.5.1-3.5.3 + in Ansible's setup module when using Mitogen + https://github.com/dw/mitogen/issues/672#issuecomment-636408833 + We replace a relative import in the setup module with the actual full file path + This works in vanilla Ansible but not in Mitogen otherwise + """ + if invocation.module_name == 'setup' and \ + invocation.module_path not in invocation._overridden_sources: + # in-memory replacement of setup module's relative import + # would check for just python3.5 and run this then but we don't know the + # target python at this time yet + module_source = module_source.replace( + b"from ...module_utils.basic import AnsibleModule", + b"from ansible.module_utils.basic import AnsibleModule" + ) + invocation._overridden_sources[invocation.module_path] = module_source + + def invoke(invocation): """ Find a Planner subclass corresponding to `invocation` and use it to invoke @@ -555,10 +579,12 @@ def invoke(invocation): invocation.module_path = mitogen.core.to_text(path) if invocation.module_path not in _planner_by_path: + module_source = invocation.get_module_source() + _fix_py35(invocation, module_source) _planner_by_path[invocation.module_path] = _get_planner( invocation.module_name, invocation.module_path, - invocation.get_module_source() + module_source ) planner = _planner_by_path[invocation.module_path](invocation) diff --git a/mitogen/service.py b/mitogen/service.py index 3b244414..69376a80 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -746,13 +746,18 @@ class PushFileService(Service): 'paths': list, 'modules': list, }) - def propagate_paths_and_modules(self, context, paths, modules): + def propagate_paths_and_modules(self, context, paths, modules, overridden_sources=None): """ One size fits all method to ensure a target context has been preloaded with a set of small files and Python modules. + + overridden_sources: optional dict containing source code to override path's source code """ for path in paths: - self.propagate_to(context, mitogen.core.to_text(path)) + overridden_source = None + if overridden_sources is not None and path in overridden_sources: + overridden_source = overridden_sources[path] + self.propagate_to(context, mitogen.core.to_text(path), overridden_source) #self.router.responder.forward_modules(context, modules) TODO @expose(policy=AllowParents()) @@ -760,14 +765,22 @@ class PushFileService(Service): 'context': mitogen.core.Context, 'path': mitogen.core.FsPathTypes, }) - def propagate_to(self, context, path): + def propagate_to(self, context, path, overridden_source=None): + """ + If the optional parameter 'overridden_source' is passed, use + that instead of the path's code as source code. This works around some bugs + of source modules such as relative imports on unsupported Python versions + """ if path not in self._cache: LOG.debug('caching small file %s', path) - fp = open(path, 'rb') - try: - self._cache[path] = mitogen.core.Blob(fp.read()) - finally: - fp.close() + if overridden_source is None: + fp = open(path, 'rb') + try: + self._cache[path] = mitogen.core.Blob(fp.read()) + finally: + fp.close() + else: + self._cache[path] = mitogen.core.Blob(overridden_source) self._forward(context, path) @expose(policy=AllowParents()) From 4d48f140a8e8e2dad878f37b4c2b565d437eb67d Mon Sep 17 00:00:00 2001 From: Nicolas Mattia Date: Fri, 26 Jun 2020 12:21:44 +0200 Subject: [PATCH 3/4] Fix typo in Ansible documentation --- docs/ansible_detailed.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index 28197ab9..12d89c9c 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -9,7 +9,7 @@ Mitogen for Ansible **Mitogen for Ansible** is a completely redesigned UNIX connection layer and module runtime for `Ansible`_. Requiring minimal configuration changes, it -updates Ansible's slow and wasteful shell-centic implementation with +updates Ansible's slow and wasteful shell-centric implementation with pure-Python equivalents, invoked via highly efficient remote procedure calls to persistent interpreters tunnelled over SSH. No changes are required to target hosts. From bacc752ff261d927a2a5d4cdd668a5e984f0924c Mon Sep 17 00:00:00 2001 From: Steven Robertson Date: Thu, 11 Jun 2020 21:17:31 -0700 Subject: [PATCH 4/4] Pegs python 2 version, removes unused imports, fixes sudden ssh dir with bad perms, generates key for sudo user as well in tests --- .ci/azure-pipelines.yml | 4 ++-- .ci/localhost_ansible_tests.py | 23 +++++++++++------------ README.md | 1 - 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index a537b0f5..c23974df 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -13,10 +13,10 @@ jobs: strategy: matrix: Mito27_27: - python.version: '2.7' + python.version: '2.7.18' MODE: mitogen Ans288_27: - python.version: '2.7' + python.version: '2.7.18' MODE: localhost_ansible VER: 2.8.8 diff --git a/.ci/localhost_ansible_tests.py b/.ci/localhost_ansible_tests.py index 888d2e44..b4d6a542 100755 --- a/.ci/localhost_ansible_tests.py +++ b/.ci/localhost_ansible_tests.py @@ -1,9 +1,7 @@ #!/usr/bin/env python # Run tests/ansible/all.yml under Ansible and Ansible-Mitogen -import glob import os -import shutil import sys import ci_lib @@ -31,16 +29,17 @@ with ci_lib.Fold('job_setup'): with ci_lib.Fold('machine_prep'): - ssh_dir = os.path.expanduser('~/.ssh') - if not os.path.exists(ssh_dir): - os.makedirs(ssh_dir, int('0700', 8)) - - key_path = os.path.expanduser('~/.ssh/id_rsa') - shutil.copy(KEY_PATH, key_path) - - auth_path = os.path.expanduser('~/.ssh/authorized_keys') - os.system('ssh-keygen -y -f %s >> %s' % (key_path, auth_path)) - os.chmod(auth_path, int('0600', 8)) + # generate a new ssh key for localhost ssh + os.system("ssh-keygen -P '' -m pem -f ~/.ssh/id_rsa") + os.system("cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys") + # also generate it for the sudo user + os.system("sudo ssh-keygen -P '' -m pem -f /var/root/.ssh/id_rsa") + os.system("sudo cat /var/root/.ssh/id_rsa.pub | sudo tee -a /var/root/.ssh/authorized_keys") + os.chmod(os.path.expanduser('~/.ssh'), int('0700', 8)) + os.chmod(os.path.expanduser('~/.ssh/authorized_keys'), int('0600', 8)) + # run chmod through sudo since it's owned by root + os.system('sudo chmod 600 /var/root/.ssh') + os.system('sudo chmod 600 /var/root/.ssh/authorized_keys') if os.path.expanduser('~mitogen__user1') == '~mitogen__user1': os.chdir(IMAGE_PREP_DIR) diff --git a/README.md b/README.md index da93a80b..c7d8b03f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ - # Mitogen