Merge branch 'master' into patch-1

pull/714/head
Steven Robertson 4 years ago committed by GitHub
commit 8d3da2dbd2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -11,7 +11,8 @@ batches = [
'pip install '
'-r tests/requirements.txt '
'-r tests/ansible/requirements.txt',
'pip install -q ansible=={0}'.format(ci_lib.ANSIBLE_VERSION)
# encoding is required for installing ansible 2.10 with pip2, otherwise we get a UnicodeDecode error
'LC_CTYPE=en_US.UTF-8 LANG=en_US.UTF-8 pip install -q ansible=={0}'.format(ci_lib.ANSIBLE_VERSION)
]
]

@ -37,9 +37,6 @@ with ci_lib.Fold('docker_setup'):
with ci_lib.Fold('job_setup'):
# Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version.
run("pip install -q ansible==%s", ci_lib.ANSIBLE_VERSION)
os.chdir(TESTS_DIR)
os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 7))
@ -75,7 +72,7 @@ with ci_lib.Fold('job_setup'):
with ci_lib.Fold('ansible'):
playbook = os.environ.get('PLAYBOOK', 'all.yml')
try:
run('./run_ansible_playbook.py %s -i "%s" %s',
run('./run_ansible_playbook.py %s -i "%s" -vvv %s',
playbook, HOSTS_DIR, ' '.join(sys.argv[1:]))
except:
pause_if_interactive()

@ -8,23 +8,7 @@ steps:
- script: "PYTHONVERSION=$(python.version) .ci/prep_azure.py"
displayName: "Run prep_azure.py"
# The VSTS-shipped Pythons available via UsePythonVErsion are pure garbage,
# broken symlinks, incorrect permissions and missing codecs. So we use the
# deadsnakes PPA to get sane Pythons, and setup a virtualenv to install our
# stuff into. The virtualenv can probably be removed again, but this was a
# hard-fought battle and for now I am tired of this crap.
- script: |
# need wheel before building virtualenv because of bdist_wheel and setuptools deps
# Mac's System Integrity Protection prevents symlinking /usr/bin
# and Azure isn't allowing disabling it apparently: https://developercommunityapi.westus.cloudapp.azure.com/idea/558702/allow-disabling-sip-on-microsoft-hosted-macos-agen.html
# the || will activate when running python3 tests
# TODO: get python3 tests passing
(sudo ln -fs /usr/bin/python$(python.version) /usr/bin/python &&
/usr/bin/python -m pip install -U pip wheel setuptools &&
/usr/bin/python -m pip install -U virtualenv &&
/usr/bin/python -m virtualenv /tmp/venv -p /usr/bin/python$(python.version)) ||
(sudo /usr/bin/python$(python.version) -m pip install -U pip wheel setuptools &&
/usr/bin/python$(python.version) -m venv /tmp/venv)
echo "##vso[task.prependpath]/tmp/venv/bin"
displayName: activate venv

@ -6,19 +6,30 @@
jobs:
- job: Mac
# vanilla Ansible is really slow
timeoutInMinutes: 120
steps:
- template: azure-pipelines-steps.yml
pool:
vmImage: macOS-10.14
vmImage: macOS-10.15
strategy:
matrix:
Mito27_27:
python.version: '2.7.18'
python.version: '2.7'
MODE: mitogen
Ans288_27:
python.version: '2.7.18'
VER: 2.10.0
# TODO: test python3, python3 tests are broken
Ans210_27:
python.version: '2.7'
MODE: localhost_ansible
VER: 2.10.0
# NOTE: this hangs when ran in Ubuntu 18.04
Vanilla_210_27:
python.version: '2.7'
MODE: localhost_ansible
VER: 2.8.8
VER: 2.10.0
STRATEGY: linear
- job: Linux
@ -35,6 +46,7 @@ jobs:
python.version: '2.7'
MODE: mitogen
DISTRO: debian
VER: 2.10.0
#MitoPy27CentOS6_26:
#python.version: '2.7'
@ -45,12 +57,13 @@ jobs:
python.version: '3.6'
MODE: mitogen
DISTRO: centos6
VER: 2.10.0
Mito37Debian_27:
python.version: '3.7'
MODE: mitogen
DISTRO: debian
VER: 2.9.6
VER: 2.10.0
#Py26CentOS7:
#python.version: '2.7'
@ -94,17 +107,12 @@ jobs:
#DISTROS: debian
#STRATEGY: linear
Ansible_280_27:
Ansible_210_27:
python.version: '2.7'
MODE: ansible
VER: 2.8.0
VER: 2.10.0
Ansible_280_35:
Ansible_210_35:
python.version: '3.5'
MODE: ansible
VER: 2.8.0
Ansible_296_37:
python.version: '3.7'
MODE: ansible
VER: 2.9.6
VER: 2.10.0

@ -49,6 +49,10 @@ def have_apt():
proc = subprocess.Popen('apt --help >/dev/null 2>/dev/null', shell=True)
return proc.wait() == 0
def have_brew():
proc = subprocess.Popen('brew help >/dev/null 2>/dev/null', shell=True)
return proc.wait() == 0
def have_docker():
proc = subprocess.Popen('docker info >/dev/null 2>/dev/null', shell=True)

@ -10,9 +10,11 @@ ci_lib.run_batches([
# Must be installed separately, as PyNACL indirect requirement causes
# newer version to be installed if done in a single pip run.
'pip install "pycparser<2.19"',
'pip install -qqqU debops==0.7.2 ansible==%s' % ci_lib.ANSIBLE_VERSION,
'pip install -qqq debops[ansible]==2.1.2 ansible==%s' % ci_lib.ANSIBLE_VERSION,
],
[
'docker pull %s' % (ci_lib.image_for_distro('debian'),),
],
])
ci_lib.run('ansible-galaxy collection install debops.debops:==2.1.2')

@ -26,12 +26,14 @@ with ci_lib.Fold('job_setup'):
ci_lib.run('debops-init %s', project_dir)
os.chdir(project_dir)
ansible_strategy_plugin = "{}/ansible_mitogen/plugins/strategy".format(ci_lib.GIT_ROOT)
with open('.debops.cfg', 'w') as fp:
fp.write(
"[ansible defaults]\n"
"strategy_plugins = %s/ansible_mitogen/plugins/strategy\n"
"strategy_plugins = {}\n"
"strategy = mitogen_linear\n"
% (ci_lib.GIT_ROOT,)
.format(ansible_strategy_plugin)
)
with open(vars_path, 'w') as fp:

@ -7,7 +7,8 @@ batches = [
# Must be installed separately, as PyNACL indirect requirement causes
# newer version to be installed if done in a single pip run.
# Separately install ansible based on version passed in from azure-pipelines.yml or .travis.yml
'pip install "pycparser<2.19" "idna<2.7"',
# Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version.
'pip install "pycparser<2.19" "idna<2.7" virtualenv',
'pip install '
'-r tests/requirements.txt '
'-r tests/ansible/requirements.txt',

@ -20,12 +20,15 @@ with ci_lib.Fold('unit_tests'):
with ci_lib.Fold('job_setup'):
# Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version.
run("pip install -q virtualenv ansible==%s", ci_lib.ANSIBLE_VERSION)
os.chmod(KEY_PATH, int('0600', 8))
# NOTE: sshpass v1.06 causes errors so pegging to 1.05 -> "msg": "Error when changing password","out": "passwd: DS error: eDSAuthFailed\n",
# there's a checksum error with "brew install http://git.io/sshpass.rb" though, so installing manually
if not ci_lib.exists_in_path('sshpass'):
run("brew install http://git.io/sshpass.rb")
os.system("curl -O -L https://sourceforge.net/projects/sshpass/files/sshpass/1.05/sshpass-1.05.tar.gz && \
tar xvf sshpass-1.05.tar.gz && \
cd sshpass-1.05 && \
./configure && \
sudo make install")
with ci_lib.Fold('machine_prep'):

@ -30,8 +30,20 @@ if 0 and os.uname()[0] == 'Linux':
]
]
# setup venv, need all python commands in 1 list to be subprocessed at the same time
venv_steps = []
need_to_fix_psycopg2 = False
is_python3 = os.environ['PYTHONVERSION'].startswith('3')
# @dw: The VSTS-shipped Pythons available via UsePythonVErsion are pure garbage,
# broken symlinks, incorrect permissions and missing codecs. So we use the
# deadsnakes PPA to get sane Pythons, and setup a virtualenv to install our
# stuff into. The virtualenv can probably be removed again, but this was a
# hard-fought battle and for now I am tired of this crap.
if ci_lib.have_apt():
batches.append([
venv_steps.extend([
'echo force-unsafe-io | sudo tee /etc/dpkg/dpkg.cfg.d/nosync',
'sudo add-apt-repository ppa:deadsnakes/ppa',
'sudo apt-get update',
@ -40,8 +52,39 @@ if ci_lib.have_apt():
'python{pv}-dev '
'libsasl2-dev '
'libldap2-dev '
.format(pv=os.environ['PYTHONVERSION'])
.format(pv=os.environ['PYTHONVERSION']),
'sudo ln -fs /usr/bin/python{pv} /usr/local/bin/python{pv}'
.format(pv=os.environ['PYTHONVERSION'])
])
if is_python3:
venv_steps.append('sudo apt-get -y install python{pv}-venv'.format(pv=os.environ['PYTHONVERSION']))
# TODO: somehow `Mito36CentOS6_26` has both brew and apt installed https://dev.azure.com/dw-mitogen/Mitogen/_build/results?buildId=1031&view=logs&j=7bdbcdc6-3d3e-568d-ccf8-9ddca1a9623a&t=73d379b6-4eea-540f-c97e-046a2f620483
elif is_python3 and ci_lib.have_brew():
# Mac's System Integrity Protection prevents symlinking /usr/bin
# and Azure isn't allowing disabling it apparently: https://developercommunityapi.westus.cloudapp.azure.com/idea/558702/allow-disabling-sip-on-microsoft-hosted-macos-agen.html
# so we'll use /usr/local/bin/python for everything
# /usr/local/bin/python2.7 already exists!
need_to_fix_psycopg2 = True
venv_steps.append(
'brew install python@{pv} postgresql'
.format(pv=os.environ['PYTHONVERSION'])
)
# need wheel before building virtualenv because of bdist_wheel and setuptools deps
venv_steps.append('/usr/local/bin/python{pv} -m pip install -U pip wheel setuptools'.format(pv=os.environ['PYTHONVERSION']))
if os.environ['PYTHONVERSION'].startswith('2'):
venv_steps.extend([
'/usr/local/bin/python{pv} -m pip install -U virtualenv'.format(pv=os.environ['PYTHONVERSION']),
'/usr/local/bin/python{pv} -m virtualenv /tmp/venv -p /usr/local/bin/python{pv}'.format(pv=os.environ['PYTHONVERSION'])
])
else:
venv_steps.append('/usr/local/bin/python{pv} -m venv /tmp/venv'.format(pv=os.environ['PYTHONVERSION']))
# fixes https://stackoverflow.com/questions/59595649/can-not-install-psycopg2-on-macos-catalina https://github.com/Azure/azure-cli/issues/12854#issuecomment-619213863
if need_to_fix_psycopg2:
venv_steps.append('/tmp/venv/bin/pip3 install psycopg2==2.8.5 psycopg2-binary')
batches.append(venv_steps)
if ci_lib.have_docker():

@ -0,0 +1,35 @@
#!/bin/bash
# workaround from https://stackoverflow.com/a/26082445 to handle Travis 4MB log limit
set -e
export PING_SLEEP=30s
export WORKDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
export BUILD_OUTPUT=$WORKDIR/build.out
touch $BUILD_OUTPUT
dump_output() {
echo Tailing the last 1000 lines of output:
tail -1000 $BUILD_OUTPUT
}
error_handler() {
echo ERROR: An error was encountered with the build.
dump_output
kill $PING_LOOP_PID
exit 1
}
# If an error occurs, run our error handler to output a tail of the build
trap 'error_handler' ERR
# Set up a repeating loop to send some output to Travis.
bash -c "while true; do echo \$(date) - building ...; sleep $PING_SLEEP; done" &
PING_LOOP_PID=$!
.ci/${MODE}_tests.py >> $BUILD_OUTPUT 2>&1
# The build finished without returning an error so dump a tail of the output
dump_output
# nicely terminate the ping output loop
kill $PING_LOOP_PID

@ -18,75 +18,65 @@ cache:
install:
- grep -Erl git-lfs\|couchdb /etc/apt | sudo xargs rm -v
- pip install -U pip==20.2.1
- .ci/${MODE}_install.py
# Travis has a 4MB log limit (https://github.com/travis-ci/travis-ci/issues/1382), but verbose Mitogen logs run larger than that
# in order to keep verbosity to debug a build failure, will run with this workaround: https://stackoverflow.com/a/26082445
script:
- .ci/spawn_reverse_shell.py
- .ci/${MODE}_tests.py
- MODE=${MODE} .ci/travis.sh
# To avoid matrix explosion, just test against oldest->newest and
# newest->oldest in various configuartions.
matrix:
allow_failures:
# Python 2.4 tests are still unreliable
- language: c
env: MODE=mitogen_py24 DISTRO=centos5
include:
# Debops tests.
# 2.9.6; 3.6 -> 2.7
- python: "3.6"
env: MODE=debops_common VER=2.9.6
# 2.8.3; 3.6 -> 2.7
- python: "3.6"
env: MODE=debops_common VER=2.8.3
# 2.4.6.0; 2.7 -> 2.7
- python: "2.7"
env: MODE=debops_common VER=2.4.6.0
# NOTE: debops tests turned off for Ansible 2.10: https://github.com/debops/debops/issues/1521
# 2.10; 3.6 -> 2.7
# - python: "3.6"
# env: MODE=debops_common VER=2.10.0
# 2.10; 2.7 -> 2.7
# - python: "2.7"
# env: MODE=debops_common VER=2.10.0
# Sanity check against vanilla Ansible. One job suffices.
- python: "2.7"
env: MODE=ansible VER=2.8.3 DISTROS=debian STRATEGY=linear
# https://github.com/dw/mitogen/pull/715#issuecomment-719266420 migrating to Azure for now due to Travis 50 min time limit cap
# azure lets us adjust the cap, and the current STRATEGY=linear tests take up to 1.5 hours to finish
# - python: "2.7"
# env: MODE=ansible VER=2.10.0 DISTROS=debian STRATEGY=linear
# ansible_mitogen tests.
# 2.9.6 -> {debian, centos6, centos7}
- python: "3.6"
env: MODE=ansible VER=2.9.6
# 2.8.3 -> {debian, centos6, centos7}
# 2.10 -> {debian, centos6, centos7}
- python: "3.6"
env: MODE=ansible VER=2.8.3
# 2.8.3 -> {debian, centos6, centos7}
env: MODE=ansible VER=2.10.0
# 2.10 -> {debian, centos6, centos7}
- python: "2.7"
env: MODE=ansible VER=2.8.3
# 2.4.6.0 -> {debian, centos6, centos7}
- python: "3.6"
env: MODE=ansible VER=2.4.6.0
# 2.4.6.0 -> {debian, centos6, centos7}
- python: "2.6"
env: MODE=ansible VER=2.4.6.0
env: MODE=ansible VER=2.10.0
# 2.10 -> {debian, centos6, centos7}
# - python: "2.6"
# env: MODE=ansible VER=2.10.0
# 2.3 -> {centos5}
- python: "2.6"
env: MODE=ansible VER=2.3.3.0 DISTROS=centos5
# 2.10 -> {centos5}
# - python: "2.6"
# env: MODE=ansible DISTROS=centos5 VER=2.10.0
# Mitogen tests.
# 2.4 -> 2.4
- language: c
env: MODE=mitogen_py24 DISTRO=centos5
# - language: c
# env: MODE=mitogen_py24 DISTROS=centos5 VER=2.10.0
# 2.7 -> 2.7 -- moved to Azure
# 2.7 -> 2.6
#- python: "2.7"
#env: MODE=mitogen DISTRO=centos6
- python: "3.6"
env: MODE=mitogen DISTRO=centos7
env: MODE=mitogen DISTROS=centos7 VER=2.10.0
# 2.6 -> 2.7
- python: "2.6"
env: MODE=mitogen DISTRO=centos7
# - python: "2.6"
# env: MODE=mitogen DISTROS=centos7 VER=2.10.0
# 2.6 -> 3.5
- python: "2.6"
env: MODE=mitogen DISTRO=debian-py3
# - python: "2.6"
# env: MODE=mitogen DISTROS=debian-py3 VER=2.10.0
# 3.6 -> 2.6 -- moved to Azure

@ -59,4 +59,6 @@ except ImportError: # Ansible <2.4
# These are original, unwrapped implementations
action_loader__get = action_loader.get
connection_loader__get = connection_loader.get
# NOTE: this used to be `connection_loader.get`; breaking change unless we do a hack based on
# ansible version again
connection_loader__get = connection_loader.get_with_context

@ -375,7 +375,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
# wait_for_connection, the `ping` test from Ansible won't pass because we lost connection
# clearing out context forces a reconnect
# see https://github.com/dw/mitogen/issues/655 and Ansible's `wait_for_connection` module for more info
if module_name == 'ping' and type(self).__name__ == 'wait_for_connection':
if module_name == 'ansible.legacy.ping' and type(self).__name__ == 'wait_for_connection':
self._connection.context = None
self._connection._connect()

@ -43,6 +43,7 @@ import os
import random
from ansible.executor import module_common
from ansible.collections.list import list_collection_dirs
import ansible.errors
import ansible.module_utils
import ansible.release
@ -57,7 +58,8 @@ import ansible_mitogen.target
LOG = logging.getLogger(__name__)
NO_METHOD_MSG = 'Mitogen: no invocation method found for: '
NO_INTERPRETER_MSG = 'module (%s) is missing interpreter line'
NO_MODULE_MSG = 'The module %s was not found in configured module paths.'
# NOTE: Ansible 2.10 no longer has a `.` at the end of NO_MODULE_MSG error
NO_MODULE_MSG = 'The module %s was not found in configured module paths'
_planner_by_path = {}
@ -99,6 +101,10 @@ class Invocation(object):
#: Initially ``{}``, but set by :func:`invoke`. Optional source to send
#: to :func:`propagate_paths_and_modules` to fix Python3.5 relative import errors
self._overridden_sources = {}
#: Initially ``set()``, but set by :func:`invoke`. Optional source paths to send
#: to :func:`propagate_paths_and_modules` to handle loading source dependencies from
#: places outside of the main source path, such as collections
self._extra_sys_paths = set()
def get_module_source(self):
if self._module_source is None:
@ -478,8 +484,10 @@ def _propagate_deps(invocation, planner, context):
context=context,
paths=planner.get_push_files(),
modules=planner.get_module_deps(),
overridden_sources=invocation._overridden_sources
# modules=planner.get_module_deps(), TODO
overridden_sources=invocation._overridden_sources,
# needs to be a list because can't unpickle() a set()
extra_sys_paths=list(invocation._extra_sys_paths),
)
@ -545,18 +553,29 @@ def _fix_py35(invocation, module_source):
We replace a relative import in the setup module with the actual full file path
This works in vanilla Ansible but not in Mitogen otherwise
"""
if invocation.module_name == 'setup' and \
if invocation.module_name in {'ansible.builtin.setup', 'setup'} and \
invocation.module_path not in invocation._overridden_sources:
# in-memory replacement of setup module's relative import
# would check for just python3.5 and run this then but we don't know the
# target python at this time yet
# NOTE: another ansible 2.10-specific fix: `from ..module_utils` used to be `from ...module_utils`
module_source = module_source.replace(
b"from ...module_utils.basic import AnsibleModule",
b"from ..module_utils.basic import AnsibleModule",
b"from ansible.module_utils.basic import AnsibleModule"
)
invocation._overridden_sources[invocation.module_path] = module_source
def _load_collections(invocation):
"""
Special loader that ensures that `ansible_collections` exist as a module path for import
Goes through all collection path possibilities and stores paths to installed collections
Stores them on the current invocation to later be passed to the master service
"""
for collection_path in list_collection_dirs():
invocation._extra_sys_paths.add(collection_path.decode('utf-8'))
def invoke(invocation):
"""
Find a Planner subclass corresponding to `invocation` and use it to invoke
@ -579,6 +598,9 @@ def invoke(invocation):
invocation.module_path = mitogen.core.to_text(path)
if invocation.module_path not in _planner_by_path:
if 'ansible_collections' in invocation.module_path:
_load_collections(invocation)
module_source = invocation.get_module_source()
_fix_py35(invocation, module_source)
_planner_by_path[invocation.module_path] = _get_planner(

@ -53,8 +53,10 @@ except ImportError:
Sentinel = None
ANSIBLE_VERSION_MIN = (2, 3)
ANSIBLE_VERSION_MAX = (2, 9)
# TODO: might be possible to lower this back to 2.3 if collection support works without hacks
ANSIBLE_VERSION_MIN = (2, 10)
ANSIBLE_VERSION_MAX = (2, 10)
NEW_VERSION_MSG = (
"Your Ansible version (%s) is too recent. The most recent version\n"
"supported by Mitogen for Ansible is %s.x. Please check the Mitogen\n"
@ -132,8 +134,7 @@ def wrap_action_loader__get(name, *args, **kwargs):
get_kwargs = {'class_only': True}
if name in ('fetch',):
name = 'mitogen_' + name
if ansible.__version__ >= '2.8':
get_kwargs['collection_list'] = kwargs.pop('collection_list', None)
get_kwargs['collection_list'] = kwargs.pop('collection_list', None)
klass = ansible_mitogen.loaders.action_loader__get(name, **get_kwargs)
if klass:
@ -217,7 +218,9 @@ class AnsibleWrappers(object):
with references to the real functions.
"""
ansible_mitogen.loaders.action_loader.get = wrap_action_loader__get
ansible_mitogen.loaders.connection_loader.get = wrap_connection_loader__get
# NOTE: this used to be `connection_loader.get`; breaking change unless we do a hack based on
# ansible version again
ansible_mitogen.loaders.connection_loader.get_with_context = wrap_connection_loader__get
global worker__run
worker__run = ansible.executor.process.worker.WorkerProcess.run
@ -230,7 +233,7 @@ class AnsibleWrappers(object):
ansible_mitogen.loaders.action_loader.get = (
ansible_mitogen.loaders.action_loader__get
)
ansible_mitogen.loaders.connection_loader.get = (
ansible_mitogen.loaders.connection_loader.get_with_context = (
ansible_mitogen.loaders.connection_loader__get
)
ansible.executor.process.worker.WorkerProcess.run = worker__run

@ -21,7 +21,11 @@ v0.2.10 (unreleased)
To avail of fixes in an unreleased version, please download a ZIP file
`directly from GitHub <https://github.com/dw/mitogen/>`_.
*(no changes)*
* :gh:issue:`756` ssh connections with `check_host_keys='accept'` would
timeout, when using recent OpenSSH client versions.
* :gh:issue:`758` fix initilialisation of callback plugins in test suite, to
to address a `KeyError` in
:method:`ansible.plugins.callback.CallbackBase.v2_runner_on_start`
v0.2.9 (2019-11-02)

@ -89,6 +89,14 @@ except NameError:
RLOG = logging.getLogger('mitogen.ctx')
# there are some cases where modules are loaded in memory only, such as
# ansible collections, and the module "filename" doesn't actually exist
SPECIAL_FILE_PATHS = {
"__synthetic__",
"<ansible_synthetic_collection_package>"
}
def _stdlib_paths():
"""
Return a set of paths from which Python imports the standard library.
@ -138,7 +146,7 @@ def is_stdlib_path(path):
)
def get_child_modules(path):
def get_child_modules(path, fullname):
"""
Return the suffixes of submodules directly neated beneath of the package
directory at `path`.
@ -147,12 +155,19 @@ def get_child_modules(path):
Path to the module's source code on disk, or some PEP-302-recognized
equivalent. Usually this is the module's ``__file__`` attribute, but
is specified explicitly to avoid loading the module.
:param str fullname:
Name of the package we're trying to get child modules for
:return:
List of submodule name suffixes.
"""
it = pkgutil.iter_modules([os.path.dirname(path)])
return [to_text(name) for _, name, _ in it]
mod_path = os.path.dirname(path)
if mod_path != '':
return [to_text(name) for _, name, _ in pkgutil.iter_modules([mod_path])]
else:
# we loaded some weird package in memory, so we'll see if it has a custom loader we can use
loader = pkgutil.find_loader(fullname)
return [to_text(name) for name, _ in loader.iter_modules(None)] if loader else []
def _looks_like_script(path):
@ -177,17 +192,31 @@ def _looks_like_script(path):
def _py_filename(path):
"""
Returns a tuple of a Python path (if the file looks Pythonic) and whether or not
the Python path is special. Special file paths/modules might only exist in memory
"""
if not path:
return None
return None, False
if path[-4:] in ('.pyc', '.pyo'):
path = path.rstrip('co')
if path.endswith('.py'):
return path
return path, False
if os.path.exists(path) and _looks_like_script(path):
return path
return path, False
basepath = os.path.basename(path)
if basepath in SPECIAL_FILE_PATHS:
return path, True
# return None, False means that the filename passed to _py_filename does not appear
# to be python, and code later will handle when this function returns None
# see https://github.com/dw/mitogen/pull/715#discussion_r532380528 for how this
# decision was made to handle non-python files in this manner
return None, False
def _get_core_source():
@ -498,9 +527,13 @@ class PkgutilMethod(FinderMethod):
return
try:
path = _py_filename(loader.get_filename(fullname))
path, is_special = _py_filename(loader.get_filename(fullname))
source = loader.get_source(fullname)
is_pkg = loader.is_package(fullname)
# workaround for special python modules that might only exist in memory
if is_special and is_pkg and not source:
source = '\n'
except (AttributeError, ImportError):
# - Per PEP-302, get_source() and is_package() are optional,
# calling them may throw AttributeError.
@ -549,7 +582,7 @@ class SysModulesMethod(FinderMethod):
fullname, alleged_name, module)
return
path = _py_filename(getattr(module, '__file__', ''))
path, _ = _py_filename(getattr(module, '__file__', ''))
if not path:
return
@ -639,7 +672,7 @@ class ParentEnumerationMethod(FinderMethod):
def _found_module(self, fullname, path, fp, is_pkg=False):
try:
path = _py_filename(path)
path, _ = _py_filename(path)
if not path:
return
@ -971,7 +1004,7 @@ class ModuleResponder(object):
self.minify_secs += mitogen.core.now() - t0
if is_pkg:
pkg_present = get_child_modules(path)
pkg_present = get_child_modules(path, fullname)
self._log.debug('%s is a package at %s with submodules %r',
fullname, path, pkg_present)
else:

@ -42,6 +42,7 @@ import heapq
import inspect
import logging
import os
import platform
import re
import signal
import socket
@ -1434,7 +1435,10 @@ class Connection(object):
os.close(r)
os.close(W)
os.close(w)
if sys.platform == 'darwin' and sys.executable == '/usr/bin/python':
# this doesn't apply anymore to Mac OSX 10.15+ (Darwin 19+), new interpreter looks like this:
# /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python
if sys.platform == 'darwin' and sys.executable == '/usr/bin/python' and \
int(platform.release()[:2]) < 19:
sys.executable += sys.version[:3]
os.environ['ARGV0']=sys.executable
os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)')

@ -691,6 +691,7 @@ class PushFileService(Service):
super(PushFileService, self).__init__(**kwargs)
self._lock = threading.Lock()
self._cache = {}
self._extra_sys_paths = set()
self._waiters = {}
self._sent_by_stream = {}
@ -744,21 +745,35 @@ class PushFileService(Service):
@arg_spec({
'context': mitogen.core.Context,
'paths': list,
'modules': list,
# 'modules': list, TODO, modules was passed into this func but it's not used yet
})
def propagate_paths_and_modules(self, context, paths, modules, overridden_sources=None):
def propagate_paths_and_modules(self, context, paths, overridden_sources=None, extra_sys_paths=None):
"""
One size fits all method to ensure a target context has been preloaded
with a set of small files and Python modules.
overridden_sources: optional dict containing source code to override path's source code
extra_sys_paths: loads additional sys paths for use in finding modules; beneficial
in situations like loading Ansible Collections because source code
dependencies come from different file paths than where the source lives
"""
for path in paths:
overridden_source = None
if overridden_sources is not None and path in overridden_sources:
overridden_source = overridden_sources[path]
self.propagate_to(context, mitogen.core.to_text(path), overridden_source)
#self.router.responder.forward_modules(context, modules) TODO
# self.router.responder.forward_modules(context, modules) TODO
# NOTE: could possibly be handled by the above TODO, but not sure how forward_modules works enough
# to know for sure, so for now going to pass the sys paths themselves and have `propagate_to`
# load them up in sys.path for later import
# ensure we don't add to sys.path the same path we've already seen
for extra_path in extra_sys_paths:
# store extra paths in cached set for O(1) lookup
if extra_path not in self._extra_sys_paths:
# not sure if it matters but we could prepend to sys.path instead if we need to
sys.path.append(extra_path)
self._extra_sys_paths.add(extra_path)
@expose(policy=AllowParents())
@arg_spec({

@ -72,7 +72,10 @@ PASSWORD_PROMPT_PATTERN = re.compile(
)
HOSTKEY_REQ_PATTERN = re.compile(
b(r'are you sure you want to continue connecting \(yes/no\)\?'),
b(
r'are you sure you want to continue connecting '
r'\(yes/no(?:/\[fingerprint\])?\)\?'
),
re.I
)

@ -21,5 +21,6 @@
copy:
src: "{{item.src}}"
dest: "/tmp/filetree.out/{{item.path}}"
mode: 0644
with_filetree: /tmp/filetree.in
when: item.state == 'file'

@ -1,18 +1,12 @@
# Verify action plugins still set file modes correctly even though
# fixup_perms2() avoids setting execute bit despite being asked to.
# As of Ansible 2.10.0, default perms vary based on OS. On debian systems it's 0644 and on centos it's 0664 based on test output
# regardless, we're testing that no execute bit is set here so either check is ok
- name: integration/action/fixup_perms2__copy.yml
hosts: test-targets
any_errors_fatal: true
tasks:
- name: Get default remote file mode
shell: python -c 'import os; print("%04o" % (int("0666", 8) & ~os.umask(0)))'
register: py_umask
- name: Set default file mode
set_fact:
mode: "{{py_umask.stdout}}"
#
# copy module (no mode).
#
@ -26,7 +20,7 @@
register: out
- assert:
that:
- out.stat.mode == mode
- out.stat.mode in ("0644", "0664")
#
# copy module (explicit mode).
@ -68,7 +62,7 @@
register: out
- assert:
that:
- out.stat.mode == mode
- out.stat.mode in ("0644", "0664")
#
# copy module (existing disk files, preserve mode).

@ -148,16 +148,7 @@
custom_python_detect_environment:
register: out
# v2.6 related: https://github.com/ansible/ansible/pull/39833
- name: "Verify modules get the same tmpdir as the action plugin (<2.5)"
when: ansible_version.full < '2.5'
assert:
that:
- out.module_path.startswith(good_temp_path2)
- out.module_tmpdir == None
- name: "Verify modules get the same tmpdir as the action plugin (>2.5)"
when: ansible_version.full > '2.5'
- name: "Verify modules get the same tmpdir as the action plugin"
assert:
that:
- out.module_path.startswith(good_temp_path2)

@ -40,19 +40,27 @@
# state: absent
# become: true
- synchronize:
private_key: /tmp/synchronize-action-key
dest: /tmp/sync-test.out
src: /tmp/sync-test/
# exception: File "/tmp/venv/lib/python2.7/site-packages/ansible/plugins/action/__init__.py", line 129, in cleanup
# exception: self._remove_tmp_path(self._connection._shell.tmpdir)
# exception: AttributeError: 'get_with_context_result' object has no attribute '_shell'
# TODO: looks like a bug on Ansible's end with 2.10? Maybe 2.10.1 will fix it
# https://github.com/dw/mitogen/issues/746
- name: do synchronize test
block:
- synchronize:
private_key: /tmp/synchronize-action-key
dest: /tmp/sync-test.out
src: /tmp/sync-test/
- slurp:
src: /tmp/sync-test.out/item
register: out
- slurp:
src: /tmp/sync-test.out/item
register: out
- set_fact: outout="{{out.content|b64decode}}"
- set_fact: outout="{{out.content|b64decode}}"
- assert:
that: outout == "item!"
- assert:
that: outout == "item!"
when: False
# TODO: https://github.com/dw/mitogen/issues/692
# - file:

@ -40,15 +40,14 @@
- result1.changed == True
# ansible/b72e989e1837ccad8dcdc926c43ccbc4d8cdfe44
- |
(ansible_version.full >= '2.8' and
(ansible_version.full is version('2.8', ">=") and
result1.cmd == "echo alldone;\nsleep 1;\n") or
(ansible_version.full < '2.8' and
(ansible_version.full is version('2.8', '<') and
result1.cmd == "echo alldone;\n sleep 1;")
- result1.delta|length == 14
- result1.start|length == 26
- result1.finished == 1
- result1.rc == 0
- result1.start|length == 26
- assert:
that:
@ -56,10 +55,9 @@
- result1.stderr_lines == []
- result1.stdout == "alldone"
- result1.stdout_lines == ["alldone"]
when: ansible_version.full > '2.8' # ansible#51393
when: ansible_version.full is version('2.8', '>') # ansible#51393
- assert:
that:
- result1.failed == False
when: ansible_version.full > '2.4'
when: ansible_version.full is version('2.4', '>')

@ -1,12 +1,18 @@
# Ensure paramiko connections aren't grabbed.
---
- name: integration/connection_loader/paramiko_unblemished.yml
hosts: test-targets
any_errors_fatal: true
tasks:
- custom_python_detect_environment:
connection: paramiko
register: out
- debug:
msg: "skipped for now"
- name: this is flaky -> https://github.com/dw/mitogen/issues/747
block:
- custom_python_detect_environment:
connection: paramiko
register: out
- assert:
that: not out.mitogen_loaded
- assert:
that: not out.mitogen_loaded
when: False

@ -14,8 +14,8 @@
- out.rc == 1
# ansible/62d8c8fde6a76d9c567ded381e9b34dad69afcd6
- |
(ansible_version.full < '2.7' and out.msg == "MODULE FAILURE") or
(ansible_version.full >= '2.7' and
(ansible_version.full is version('2.7', '<') and out.msg == "MODULE FAILURE") or
(ansible_version.full is version('2.7', '>=') and
out.msg == (
"MODULE FAILURE\n" +
"See stdout/stderr for the exact error"

@ -2,6 +2,10 @@
hosts: test-targets
any_errors_fatal: true
tasks:
# without Mitogen Ansible 2.10 hangs on this play
- meta: end_play
when: not is_mitogen
- custom_python_new_style_module:
foo: true
with_sequence: start=0 end={{end|default(1)}}

@ -16,4 +16,4 @@
- assert:
that: |
'The module missing_module was not found in configured module paths.' in out.stdout
'The module missing_module was not found in configured module paths' in out.stdout

@ -113,7 +113,8 @@
# ansible_become_pass & ansible_become_password set, password takes precedence
# ansible_become_pass & ansible_become_password set, password used to take precedence
# but it's possible since https://github.com/ansible/ansible/pull/69629/files#r428376864, now it doesn't
- hosts: tc-become-pass-both
become: true
tasks:
@ -124,7 +125,7 @@
- out.result|length == 2
- out.result[0].method == "ssh"
- out.result[1].method == "sudo"
- out.result[1].kwargs.password == "a.b.c"
- out.result[1].kwargs.password == "c.b.a"
# both, mitogen_via

@ -20,6 +20,8 @@ DefaultModule = callback_loader.get('default', class_only=True)
DOCUMENTATION = '''
callback: nice_stdout
type: stdout
extends_documentation_fragment:
- default_callback
options:
check_mode_markers:
name: Show markers when running in check mode
@ -74,6 +76,10 @@ def printi(tio, obj, key=None, indent=0):
class CallbackModule(DefaultModule):
CALLBACK_VERSION = 2.0
CALLBACK_TYPE = 'stdout'
CALLBACK_NAME = 'nice_stdout'
def _dump_results(self, result, *args, **kwargs):
try:
tio = io.StringIO()

@ -37,6 +37,7 @@ class CallbackModule(CallbackBase):
A plugin for timing tasks
"""
def __init__(self):
super(CallbackModule, self).__init__()
self.stats = {}
self.current = None

@ -2,8 +2,11 @@
import sys
# This is the magic marker Ansible looks for:
# As of Ansible 2.10, Ansible changed new-style detection: # https://github.com/ansible/ansible/pull/61196/files#diff-5675e463b6ce1fbe274e5e7453f83cd71e61091ea211513c93e7c0b4d527d637L828-R980
# NOTE: this import works for Mitogen, and the import below matches new-style Ansible 2.10
# TODO: find out why 1 import won't work for both Mitogen and Ansible
# from ansible.module_utils.
# import ansible.module_utils.
def usage():

@ -9,6 +9,7 @@
from __future__ import absolute_import, division, print_function
__metaclass__ = type
import platform
import sys
from ansible.module_utils.basic import AnsibleModule
@ -23,7 +24,12 @@ def main():
result['ansible_facts'] = module.params['facts']
# revert the Mitogen OSX tweak since discover_interpreter() doesn't return this info
if sys.platform == 'darwin' and sys.executable != '/usr/bin/python':
sys.executable = sys.executable[:-3]
if int(platform.release()[:2]) < 19:
sys.executable = sys.executable[:-3]
else:
# only for tests to check version of running interpreter -- Mac 10.15+ changed python2
# so it looks like it's /usr/bin/python but actually it's /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python
sys.executable = "/usr/bin/python"
result['running_python_interpreter'] = sys.executable
module.exit_json(**result)

@ -26,5 +26,6 @@
copy:
src: "{{item.src}}"
dest: "/tmp/filetree.out/{{item.path}}"
mode: 0644
with_filetree: /tmp/filetree.in
when: item.state == 'file'

@ -1,11 +1,9 @@
#!/usr/bin/env python
# Wrap ansible-playbook, setting up some test of the test environment.
import json
import os
import sys
GIT_BASEDIR = os.path.dirname(
os.path.abspath(
os.path.join(__file__, '..', '..')

@ -3,7 +3,7 @@ coverage==4.5.1
Django==1.6.11 # Last version supporting 2.6.
mock==2.0.0
pytz==2018.5
cffi==1.11.2 # Random pin to try and fix pyparser==2.18 not having effect
cffi==1.14.3 # Random pin to try and fix pyparser==2.18 not having effect
pycparser==2.18 # Last version supporting 2.6.
faulthandler==3.1; python_version < '3.3' # used by testlib
pytest-catchlog==1.2.2

Loading…
Cancel
Save