Merge branch 'master' into become_chdir

pull/1080/head
Alex Willmer 2 months ago committed by GitHub
commit 9fffb6c2f2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -28,7 +28,6 @@ for doing `setup.py install` while pulling a Docker container, for example.
### Environment Variables
* `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2.
* `DISTRO`: the `mitogen_` tests need a target Docker container distro. This
name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test`
* `DISTROS`: the `ansible_` tests can run against multiple targets

@ -1,11 +0,0 @@
#!/usr/bin/env python
import ci_lib
batches = [
[
'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
]
]
ci_lib.run_batches(batches)

@ -6,11 +6,13 @@ import glob
import os
import signal
import sys
import textwrap
import jinja2
import ci_lib
TEMPLATES_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible/templates')
TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible')
HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts')
@ -33,7 +35,7 @@ ci_lib.check_stray_processes(interesting)
with ci_lib.Fold('docker_setup'):
containers = ci_lib.make_containers()
containers = ci_lib.container_specs(ci_lib.DISTROS)
ci_lib.start_containers(containers)
@ -52,37 +54,19 @@ with ci_lib.Fold('job_setup'):
distros[container['distro']].append(container['name'])
families[container['family']].append(container['name'])
jinja_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(searchpath=TEMPLATES_DIR),
lstrip_blocks=True, # Remove spaces and tabs from before a block
trim_blocks=True, # Remove first newline after a block
)
inventory_template = jinja_env.get_template('test-targets.j2')
inventory_path = os.path.join(HOSTS_DIR, 'target')
with open(inventory_path, 'w') as fp:
fp.write('[test-targets]\n')
fp.writelines(
"%(name)s "
"ansible_host=%(hostname)s "
"ansible_port=%(port)s "
"ansible_python_interpreter=%(python_path)s "
"ansible_user=mitogen__has_sudo_nopw "
"ansible_password=has_sudo_nopw_password"
"\n"
% container
for container in containers
)
for distro, hostnames in sorted(distros.items(), key=lambda t: t[0]):
fp.write('\n[%s]\n' % distro)
fp.writelines('%s\n' % name for name in hostnames)
for family, hostnames in sorted(families.items(), key=lambda t: t[0]):
fp.write('\n[%s]\n' % family)
fp.writelines('%s\n' % name for name in hostnames)
fp.write(textwrap.dedent(
'''
[linux:children]
test-targets
[linux_containers:children]
test-targets
'''
fp.write(inventory_template.render(
containers=containers,
distros=distros,
families=families,
))
ci_lib.dump_file(inventory_path)

@ -14,6 +14,19 @@ steps:
versionSpec: '$(python.version)'
condition: ne(variables['python.version'], '')
- script: |
set -o errexit
set -o nounset
set -o pipefail
aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws
displayName: Authenticate to container registry
condition: eq(variables['Agent.OS'], 'Linux')
env:
AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)
- script: |
set -o errexit
set -o nounset
@ -90,7 +103,3 @@ steps:
"$PYTHON" -m tox -e "$(tox.env)"
displayName: "Run tests"
env:
AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)

@ -16,37 +16,26 @@ trigger:
- docs-master
jobs:
- job: mac11
- job: mac12
# vanilla Ansible is really slow
timeoutInMinutes: 120
steps:
- template: azure-pipelines-steps.yml
pool:
# https://github.com/actions/runner-images/blob/main/images/macos/macos-11-Readme.md
vmImage: macOS-11
# https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md
vmImage: macOS-12
strategy:
matrix:
Mito_27:
tox.env: py27-mode_mitogen
Mito_312:
python.version: '3.12'
tox.env: py312-mode_mitogen
Loc_27_210:
tox.env: py27-mode_localhost-ansible2.10
Loc_312_9:
python.version: '3.12'
tox.env: py312-mode_localhost-ansible9
Van_27_210:
tox.env: py27-mode_localhost-ansible2.10-strategy_linear
Van_312_9:
python.version: '3.12'
tox.env: py312-mode_localhost-ansible9-strategy_linear
Loc_312_10:
tox.env: py312-mode_localhost-ansible10
Van_312_10:
tox.env: py312-mode_localhost-ansible10-strategy_linear
- job: Linux
pool:
# https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2004-Readme.md
# https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md
vmImage: ubuntu-20.04
steps:
- template: azure-pipelines-steps.yml
@ -163,3 +152,6 @@ jobs:
Ans_312_9:
python.version: '3.12'
tox.env: py312-mode_ansible-ansible9
Ans_312_10:
python.version: '3.12'
tox.env: py312-mode_ansible-ansible10

@ -27,6 +27,13 @@ os.chdir(
)
)
IMAGE_TEMPLATE = os.environ.get(
'MITOGEN_TEST_IMAGE_TEMPLATE',
'public.ecr.aws/n5z0e8q9/%(distro)s-test',
)
_print = print
def print(*args, **kwargs):
file = kwargs.get('file', sys.stdout)
@ -193,8 +200,6 @@ GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
DISTRO = os.environ.get('DISTRO', 'debian9')
# Used only when MODE=ansible
DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split()
TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2'))
BASE_PORT = 2200
TMP = TempDir().path
@ -217,6 +222,7 @@ os.environ['PYTHONPATH'] = '%s:%s' % (
def get_docker_hostname():
"""Return the hostname where the docker daemon is running.
"""
# Duplicated in testlib
url = os.environ.get('DOCKER_HOST')
if url in (None, 'http+docker://localunixsocket'):
return 'localhost'
@ -225,27 +231,34 @@ def get_docker_hostname():
return parsed.netloc.partition(':')[0]
def make_containers(name_prefix='', port_offset=0):
def container_specs(
distros,
base_port=2200,
image_template=IMAGE_TEMPLATE,
name_template='target-%(distro)s-%(index)d',
):
"""
>>> import pprint
>>> BASE_PORT=2200; DISTROS=['debian11', 'centos6']
>>> pprint.pprint(make_containers())
>>> pprint.pprint(container_specs(['debian11-py3', 'centos6']))
[{'distro': 'debian11',
'family': 'debian',
'hostname': 'localhost',
'image': 'public.ecr.aws/n5z0e8q9/debian11-test',
'index': 1,
'name': 'target-debian11-1',
'port': 2201,
'python_path': '/usr/bin/python'},
'python_path': '/usr/bin/python3'},
{'distro': 'centos6',
'family': 'centos',
'hostname': 'localhost',
'image': 'public.ecr.aws/n5z0e8q9/centos6-test',
'index': 2,
'name': 'target-centos6-2',
'port': 2202,
'python_path': '/usr/bin/python'}]
"""
docker_hostname = get_docker_hostname()
# Code duplicated in testlib.py, both should be updated together
distro_pattern = re.compile(r'''
(?P<distro>(?P<family>[a-z]+)[0-9]+)
(?:-(?P<py>py3))?
@ -256,30 +269,27 @@ def make_containers(name_prefix='', port_offset=0):
i = 1
lst = []
for distro in DISTROS:
for distro in distros:
# Code duplicated in testlib.py, both should be updated together
d = distro_pattern.match(distro).groupdict(default=None)
distro = d['distro']
family = d['family']
image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,)
if d['py'] == 'py3':
if d.pop('py') == 'py3':
python_path = '/usr/bin/python3'
else:
python_path = '/usr/bin/python'
if d['count']:
count = int(count)
else:
count = 1
count = int(d.pop('count') or '1', 10)
for x in range(count):
lst.append({
"distro": distro, "family": family, "image": image,
"name": name_prefix + ("target-%s-%s" % (distro, i)),
d['index'] = i
d.update({
'image': image_template % d,
'name': name_template % d,
"hostname": docker_hostname,
"port": BASE_PORT + i + port_offset,
'port': base_port + i,
"python_path": python_path,
})
lst.append(d)
i += 1
return lst

@ -2,16 +2,10 @@
import ci_lib
# Naturally DebOps only supports Debian.
ci_lib.DISTROS = ['debian']
ci_lib.run_batches([
[
'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"',
],
[
'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
],
])
ci_lib.run('ansible-galaxy collection install debops.debops:==2.1.2')

@ -6,9 +6,6 @@ import sys
import ci_lib
# DebOps only supports Debian.
ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT
project_dir = os.path.join(ci_lib.TMP, 'project')
vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml'
inventory_path = 'ansible/inventory/hosts'
@ -16,7 +13,11 @@ docker_hostname = ci_lib.get_docker_hostname()
with ci_lib.Fold('docker_setup'):
containers = ci_lib.make_containers(port_offset=500, name_prefix='debops-')
containers = ci_lib.container_specs(
['debian*2'],
base_port=2700,
name_template='debops-target-%(distro)s-%(index)d',
)
ci_lib.start_containers(containers)

@ -1,8 +0,0 @@
#!/usr/bin/env python
import ci_lib
batches = [
]
ci_lib.run_batches(batches)

@ -1,14 +0,0 @@
#!/usr/bin/env python
import ci_lib
batches = [
]
if ci_lib.have_docker():
batches.append([
'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
])
ci_lib.run_batches(batches)

@ -3,9 +3,6 @@
import ci_lib
batches = [
[
'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
],
[
'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv',
]

@ -0,0 +1,326 @@
# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
name: Tests
on:
pull_request:
push:
branches-ignore:
- docs-master
env:
#ANSIBLE_VERBOSITY: 3
#MITOGEN_LOG_LEVEL: DEBUG
MITOGEN_TEST_IMAGE_TEMPLATE: "ghcr.io/mitogen-hq/%(distro)s-test"
# https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners
# https://github.com/actions/runner-images/blob/main/README.md#software-and-image-support
jobs:
linux:
# https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
include:
- name: Ans_27_210
tox_env: py27-mode_ansible-ansible2.10
- name: Ans_27_4
tox_env: py27-mode_ansible-ansible4
- name: Ans_36_210
python_version: '3.6'
tox_env: py36-mode_ansible-ansible2.10
- name: Ans_36_4
python_version: '3.6'
tox_env: py36-mode_ansible-ansible4
- name: Ans_311_210
python_version: '3.11'
tox_env: py311-mode_ansible-ansible2.10
- name: Ans_311_3
python_version: '3.11'
tox_env: py311-mode_ansible-ansible3
- name: Ans_311_4
python_version: '3.11'
tox_env: py311-mode_ansible-ansible4
- name: Ans_311_5
python_version: '3.11'
tox_env: py311-mode_ansible-ansible5
- name: Ans_312_6
python_version: '3.12'
tox_env: py312-mode_ansible-ansible6
- name: Ans_312_7
python_version: '3.12'
tox_env: py312-mode_ansible-ansible7
- name: Ans_312_8
python_version: '3.12'
tox_env: py312-mode_ansible-ansible8
- name: Ans_312_9
python_version: '3.12'
tox_env: py312-mode_ansible-ansible9
- name: Ans_312_10
python_version: '3.12'
tox_env: py312-mode_ansible-ansible10
- name: Van_312_10
python_version: '3.12'
tox_env: py312-mode_ansible-ansible10-strategy_linear
- name: Mito_27_centos6
tox_env: py27-mode_mitogen-distro_centos6
- name: Mito_27_centos7
tox_env: py27-mode_mitogen-distro_centos7
- name: Mito_27_centos8
tox_env: py27-mode_mitogen-distro_centos8
- name: Mito_27_debian9
tox_env: py27-mode_mitogen-distro_debian9
- name: Mito_27_debian10
tox_env: py27-mode_mitogen-distro_debian10
- name: Mito_27_debian11
tox_env: py27-mode_mitogen-distro_debian11
- name: Mito_27_ubuntu1604
tox_env: py27-mode_mitogen-distro_ubuntu1604
- name: Mito_27_ubuntu1804
tox_env: py27-mode_mitogen-distro_ubuntu1804
- name: Mito_27_ubuntu2004
tox_env: py27-mode_mitogen-distro_ubuntu2004
- name: Mito_36_centos6
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_centos6
- name: Mito_36_centos7
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_centos7
- name: Mito_36_centos8
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_centos8
- name: Mito_36_debian9
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_debian9
- name: Mito_36_debian10
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_debian10
- name: Mito_36_debian11
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_debian11
- name: Mito_36_ubuntu1604
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_ubuntu1604
- name: Mito_36_ubuntu1804
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_ubuntu1804
- name: Mito_36_ubuntu2004
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_ubuntu2004
- name: Mito_312_centos6
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_centos6
- name: Mito_312_centos7
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_centos7
- name: Mito_312_centos8
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_centos8
- name: Mito_312_debian9
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_debian9
- name: Mito_312_debian10
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_debian10
- name: Mito_312_debian11
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_debian11
- name: Mito_312_ubuntu1604
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_ubuntu1604
- name: Mito_312_ubuntu1804
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_ubuntu1804
- name: Mito_312_ubuntu2004
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_ubuntu2004
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python_version }}
if: ${{ matrix.python_version }}
- uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Install build deps
run: |
set -o errexit -o nounset -o pipefail
sudo apt-get update
sudo apt-get install -y python2-dev python3-pip virtualenv
- name: Show Python versions
run: |
set -o errexit -o nounset -o pipefail
# macOS builders lack a realpath command
type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version
type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version
type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version
echo
if [ -e /usr/bin/python ]; then
echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')"
fi
if [ -e /usr/bin/python2 ]; then
echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')"
fi
if [ -e /usr/bin/python2.7 ]; then
echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')"
fi
- name: Install tooling
run: |
set -o errexit -o nounset -o pipefail
# Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))')
if [[ -z $PYTHON ]]; then
echo 1>&2 "Python interpreter could not be determined"
exit 1
fi
if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then
"$PYTHON" -m ensurepip --user --altinstall --no-default-pip
"$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
elif [[ $PYTHON == "python2.7" ]]; then
curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py"
"$PYTHON" get-pip.py --user --no-python-version-warning
# Avoid Python 2.x pip masking system pip
rm -f ~/.local/bin/{easy_install,pip,wheel}
"$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
else
"$PYTHON" -m pip install -r "tests/requirements-tox.txt"
fi
- name: Run tests
env:
GITHUB_ACTOR: ${{ github.actor }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -o errexit -o nounset -o pipefail
# Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))')
if [[ -z $PYTHON ]]; then
echo 1>&2 "Python interpreter could not be determined"
exit 1
fi
"$PYTHON" -m tox -e "${{ matrix.tox_env }}"
macos:
# https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md
runs-on: macos-12
timeout-minutes: 120
strategy:
fail-fast: false
matrix:
include:
- name: Mito_27
tox_env: py27-mode_mitogen
- name: Mito_312
tox_env: py312-mode_mitogen
- name: Loc_27_210
tox_env: py27-mode_localhost-ansible2.10
- name: Loc_312_10
tox_env: py312-mode_localhost-ansible10
- name: Van_27_210
tox_env: py27-mode_localhost-ansible2.10-strategy_linear
- name: Van_312_10
tox_env: py312-mode_localhost-ansible10-strategy_linear
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python_version }}
if: ${{ matrix.python_version }}
- name: Show Python versions
run: |
set -o errexit -o nounset -o pipefail
# macOS builders lack a realpath command
type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version
type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version
type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version
echo
if [ -e /usr/bin/python ]; then
echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')"
fi
if [ -e /usr/bin/python2 ]; then
echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')"
fi
if [ -e /usr/bin/python2.7 ]; then
echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')"
fi
if [ -e /Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7 ]; then
# GitHub macOS 12 images: python2.7 is installed, but not on $PATH
echo "/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7: sys.executable: $(/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7 -c 'import sys; print(sys.executable)')"
fi
- name: Install tooling
run: |
set -o errexit -o nounset -o pipefail
# Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))')
if [[ -z $PYTHON ]]; then
echo 1>&2 "Python interpreter could not be determined"
exit 1
fi
if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then
# GitHub macOS 12 images: python2.7 is installed, but not on $PATH
PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7"
"$PYTHON" -m ensurepip --user --altinstall --no-default-pip
"$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
elif [[ $PYTHON == "python2.7" ]]; then
curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py"
"$PYTHON" get-pip.py --user --no-python-version-warning
# Avoid Python 2.x pip masking system pip
rm -f ~/.local/bin/{easy_install,pip,wheel}
"$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
else
"$PYTHON" -m pip install -r "tests/requirements-tox.txt"
fi
- name: Run tests
env:
GITHUB_ACTOR: ${{ github.actor }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -o errexit -o nounset -o pipefail
# Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))')
if [[ -z $PYTHON ]]; then
echo 1>&2 "Python interpreter could not be determined"
exit 1
fi
if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then
# GitHub macOS 12 images: python2.7 is installed, but not on $PATH
PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7"
fi
"$PYTHON" -m tox -e "${{ matrix.tox_env }}"

@ -83,7 +83,6 @@ import multiprocessing
import os
import struct
import mitogen.core
import mitogen.parent
@ -265,7 +264,7 @@ class LinuxPolicy(FixedPolicy):
for x in range(16):
chunks.append(struct.pack('<Q', mask & shiftmask))
mask >>= 64
return mitogen.core.b('').join(chunks)
return b''.join(chunks)
def _get_thread_ids(self):
try:

@ -1129,6 +1129,6 @@ class Connection(ansible.plugins.connection.ConnectionBase):
self.get_chain().call(
ansible_mitogen.target.transfer_file,
context=self.binding.get_child_service_context(),
in_path=in_path,
out_path=out_path
in_path=ansible_mitogen.utils.unsafe.cast(in_path),
out_path=ansible_mitogen.utils.unsafe.cast(out_path)
)

@ -49,7 +49,7 @@ __all__ = [
ANSIBLE_VERSION_MIN = (2, 10)
ANSIBLE_VERSION_MAX = (2, 16)
ANSIBLE_VERSION_MAX = (2, 17)
NEW_VERSION_MSG = (
"Your Ansible version (%s) is too recent. The most recent version\n"

@ -32,15 +32,13 @@ __metaclass__ = type
import logging
import os
import ansible.utils.display
import mitogen.core
import mitogen.utils
try:
from __main__ import display
except ImportError:
import ansible.utils.display
display = ansible.utils.display.Display()
display = ansible.utils.display.Display()
#: The process name set via :func:`set_process_name`.
_process_name = None

@ -35,18 +35,16 @@ import pwd
import random
import traceback
try:
from shlex import quote as shlex_quote
except ImportError:
from pipes import quote as shlex_quote
from ansible.module_utils._text import to_bytes
from ansible.parsing.utils.jsonify import jsonify
import ansible
import ansible.constants
import ansible.plugins
import ansible.plugins.action
import ansible.utils.unsafe_proxy
import ansible.vars.clean
from ansible.module_utils.common.text.converters import to_bytes, to_text
from ansible.module_utils.six.moves import shlex_quote
from ansible.parsing.utils.jsonify import jsonify
import mitogen.core
import mitogen.select
@ -57,24 +55,6 @@ import ansible_mitogen.target
import ansible_mitogen.utils
import ansible_mitogen.utils.unsafe
from ansible.module_utils._text import to_text
try:
from ansible.utils.unsafe_proxy import wrap_var
except ImportError:
from ansible.vars.unsafe_proxy import wrap_var
try:
# ansible 2.8 moved remove_internal_keys to the clean module
from ansible.vars.clean import remove_internal_keys
except ImportError:
try:
from ansible.vars.manager import remove_internal_keys
except ImportError:
# ansible 2.3.3 has remove_internal_keys as a protected func on the action class
# we'll fallback to calling self._remove_internal_keys in this case
remove_internal_keys = lambda a: "Not found"
LOG = logging.getLogger(__name__)
@ -280,7 +260,9 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
paths, mode, sudoable)
return self.fake_shell(lambda: mitogen.select.Select.all(
self._connection.get_chain().call_async(
ansible_mitogen.target.set_file_mode, path, mode
ansible_mitogen.target.set_file_mode,
ansible_mitogen.utils.unsafe.cast(path),
mode,
)
for path in paths
))
@ -357,7 +339,9 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
def _execute_module(self, module_name=None, module_args=None, tmp=None,
task_vars=None, persist_files=False,
delete_remote_tmp=True, wrap_async=False):
delete_remote_tmp=True, wrap_async=False,
ignore_unknown_opts=False,
):
"""
Collect up a module's execution environment then use it to invoke
target.run_module() or helpers.run_module_async() in the target
@ -370,7 +354,13 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
if task_vars is None:
task_vars = {}
self._update_module_args(module_name, module_args, task_vars)
if ansible_mitogen.utils.ansible_version[:2] >= (2, 17):
self._update_module_args(
module_name, module_args, task_vars,
ignore_unknown_opts=ignore_unknown_opts,
)
else:
self._update_module_args(module_name, module_args, task_vars)
env = {}
self._compute_environment_string(env)
self._set_temp_file_args(module_args, wrap_async)
@ -403,10 +393,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
self._remove_tmp_path(tmp)
# prevents things like discovered_interpreter_* or ansible_discovered_interpreter_* from being set
# handle ansible 2.3.3 that has remove_internal_keys in a different place
check = remove_internal_keys(result)
if check == 'Not found':
self._remove_internal_keys(result)
ansible.vars.clean.remove_internal_keys(result)
# taken from _execute_module of ansible 2.8.6
# propagate interpreter discovery results back to the controller
@ -430,7 +417,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
result['deprecations'] = []
result['deprecations'].extend(self._discovery_deprecation_warnings)
return wrap_var(result)
return ansible.utils.unsafe_proxy.wrap_var(result)
def _postprocess_response(self, result):
"""

@ -54,6 +54,7 @@ import mitogen.select
import ansible_mitogen.loaders
import ansible_mitogen.parsing
import ansible_mitogen.target
import ansible_mitogen.utils.unsafe
LOG = logging.getLogger(__name__)
@ -215,12 +216,15 @@ class ScriptPlanner(BinaryPlanner):
"""
def _rewrite_interpreter(self, path):
"""
Given the original interpreter binary extracted from the script's
interpreter line, look up the associated `ansible_*_interpreter`
variable, render it and return it.
Given the interpreter path (from the script's hashbang line), return
the desired interpreter path. This tries, in order
1. Look up & render the `ansible_*_interpreter` variable, if set
2. Look up the `discovered_interpreter_*` fact, if present
3. The unmodified path from the hashbang line.
:param str path:
Absolute UNIX path to original interpreter.
Absolute path to original interpreter (e.g. '/usr/bin/python').
:returns:
Shell fragment prefix used to execute the script via "/bin/sh -c".
@ -228,13 +232,25 @@ class ScriptPlanner(BinaryPlanner):
involved here, the vanilla implementation uses it and that use is
exploited in common playbooks.
"""
key = u'ansible_%s_interpreter' % os.path.basename(path).strip()
interpreter_name = os.path.basename(path).strip()
key = u'ansible_%s_interpreter' % interpreter_name
try:
template = self._inv.task_vars[key]
except KeyError:
return path
pass
else:
configured_interpreter = self._inv.templar.template(template)
return ansible_mitogen.utils.unsafe.cast(configured_interpreter)
key = u'discovered_interpreter_%s' % interpreter_name
try:
discovered_interpreter = self._inv.task_vars['ansible_facts'][key]
except KeyError:
pass
else:
return ansible_mitogen.utils.unsafe.cast(discovered_interpreter)
return mitogen.utils.cast(self._inv.templar.template(template))
return path
def _get_interpreter(self):
path, arg = ansible_mitogen.parsing.parse_hashbang(
@ -249,7 +265,8 @@ class ScriptPlanner(BinaryPlanner):
if arg:
fragment += ' ' + arg
return fragment, path.startswith('python')
is_python = path.startswith('python')
return fragment, is_python
def get_kwargs(self, **kwargs):
interpreter_fragment, is_python = self._get_interpreter()
@ -460,7 +477,7 @@ def read_file(path):
finally:
os.close(fd)
return mitogen.core.b('').join(bits)
return b''.join(bits)
def _propagate_deps(invocation, planner, context):

@ -42,13 +42,7 @@ except ImportError:
import ansible_mitogen.connection
import ansible_mitogen.process
if sys.version_info > (3,):
viewkeys = dict.keys
elif sys.version_info > (2, 7):
viewkeys = dict.viewkeys
else:
viewkeys = lambda dct: set(dct)
viewkeys = getattr(dict, 'viewkeys', dict.keys)
def dict_diff(old, new):

@ -61,10 +61,9 @@ import mitogen.utils
import ansible
import ansible.constants as C
import ansible.errors
import ansible_mitogen.logging
import ansible_mitogen.services
from mitogen.core import b
import ansible_mitogen.affinity
@ -639,7 +638,7 @@ class MuxProcess(object):
try:
# Let the parent know our listening socket is ready.
mitogen.core.io_op(self.model.child_sock.send, b('1'))
mitogen.core.io_op(self.model.child_sock.send, b'1')
# Block until the socket is closed, which happens on parent exit.
mitogen.core.io_op(self.model.child_sock.recv, 1)
finally:

@ -40,7 +40,9 @@ from __future__ import absolute_import, division, print_function
__metaclass__ = type
import atexit
import ctypes
import json
import logging
import os
import re
import shlex
@ -50,19 +52,12 @@ import tempfile
import traceback
import types
from ansible.module_utils.six.moves import shlex_quote
import mitogen.core
import ansible_mitogen.target # TODO: circular import
from mitogen.core import b
from mitogen.core import bytes_partition
from mitogen.core import str_rpartition
from mitogen.core import to_text
try:
import ctypes
except ImportError:
# Python 2.4
ctypes = None
try:
# Python >= 3.4, PEP 451 ModuleSpec API
import importlib.machinery
@ -77,15 +72,6 @@ try:
except ImportError:
from io import StringIO
try:
from shlex import quote as shlex_quote
except ImportError:
from pipes import quote as shlex_quote
# Absolute imports for <2.5.
logging = __import__('logging')
# Prevent accidental import of an Ansible module from hanging on stdin read.
import ansible.module_utils.basic
ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
@ -95,15 +81,13 @@ ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
# explicit call to res_init() on each task invocation. BSD-alikes export it
# directly, Linux #defines it as "__res_init".
libc__res_init = None
if ctypes:
libc = ctypes.CDLL(None)
for symbol in 'res_init', '__res_init':
try:
libc__res_init = getattr(libc, symbol)
except AttributeError:
pass
libc = ctypes.CDLL(None)
for symbol in 'res_init', '__res_init':
try:
libc__res_init = getattr(libc, symbol)
except AttributeError:
pass
iteritems = getattr(dict, 'iteritems', dict.items)
LOG = logging.getLogger(__name__)
@ -217,13 +201,13 @@ class EnvironmentFileWatcher(object):
for line in fp:
# ' #export foo=some var ' -> ['#export', 'foo=some var ']
bits = shlex_split_b(line)
if (not bits) or bits[0].startswith(b('#')):
if (not bits) or bits[0].startswith(b'#'):
continue
if bits[0] == b('export'):
if bits[0] == b'export':
bits.pop(0)
key, sep, value = bytes_partition(b(' ').join(bits), b('='))
key, sep, value = b' '.join(bits).partition(b'=')
if key and sep:
yield key, value
@ -601,7 +585,7 @@ class ModuleUtilsImporter(object):
mod.__path__ = []
mod.__package__ = str(fullname)
else:
mod.__package__ = str(str_rpartition(to_text(fullname), '.')[0])
mod.__package__ = str(to_text(fullname).rpartition('.')[0])
exec(code, mod.__dict__)
self._loaded.add(fullname)
return mod
@ -616,7 +600,7 @@ class TemporaryEnvironment(object):
def __init__(self, env=None):
self.original = dict(os.environ)
self.env = env or {}
for key, value in iteritems(self.env):
for key, value in mitogen.core.iteritems(self.env):
key = mitogen.core.to_text(key)
value = mitogen.core.to_text(value)
if value is None:
@ -824,7 +808,7 @@ class ScriptRunner(ProgramRunner):
self.interpreter_fragment = interpreter_fragment
self.is_python = is_python
b_ENCODING_STRING = b('# -*- coding: utf-8 -*-')
b_ENCODING_STRING = b'# -*- coding: utf-8 -*-'
def _get_program(self):
return self._rewrite_source(
@ -857,13 +841,13 @@ class ScriptRunner(ProgramRunner):
# While Ansible rewrites the #! using ansible_*_interpreter, it is
# never actually used to execute the script, instead it is a shell
# fragment consumed by shell/__init__.py::build_module_command().
new = [b('#!') + utf8(self.interpreter_fragment)]
new = [b'#!' + utf8(self.interpreter_fragment)]
if self.is_python:
new.append(self.b_ENCODING_STRING)
_, _, rest = bytes_partition(s, b('\n'))
_, _, rest = s.partition(b'\n')
new.append(rest)
return b('\n').join(new)
return b'\n'.join(new)
class NewStyleRunner(ScriptRunner):
@ -976,8 +960,7 @@ class NewStyleRunner(ScriptRunner):
# change the default encoding. This hack was removed from Ansible long ago,
# but not before permeating into many third party modules.
PREHISTORIC_HACK_RE = re.compile(
b(r'reload\s*\(\s*sys\s*\)\s*'
r'sys\s*\.\s*setdefaultencoding\([^)]+\)')
br'reload\s*\(\s*sys\s*\)\s*sys\s*\.\s*setdefaultencoding\([^)]+\)',
)
def _setup_program(self):
@ -985,7 +968,7 @@ class NewStyleRunner(ScriptRunner):
context=self.service_context,
path=self.path,
)
self.source = self.PREHISTORIC_HACK_RE.sub(b(''), source)
self.source = self.PREHISTORIC_HACK_RE.sub(b'', source)
def _get_code(self):
try:
@ -1003,7 +986,7 @@ class NewStyleRunner(ScriptRunner):
if mitogen.core.PY3:
main_module_name = '__main__'
else:
main_module_name = b('__main__')
main_module_name = b'__main__'
def _handle_magic_exception(self, mod, exc):
"""
@ -1035,7 +1018,7 @@ class NewStyleRunner(ScriptRunner):
approximation of the original package hierarchy, so that relative
imports function correctly.
"""
pkg, sep, modname = str_rpartition(self.py_module_name, '.')
pkg, sep, _ = self.py_module_name.rpartition('.')
if not sep:
return None
if mitogen.core.PY3:
@ -1078,7 +1061,7 @@ class NewStyleRunner(ScriptRunner):
class JsonArgsRunner(ScriptRunner):
JSON_ARGS = b('<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>')
JSON_ARGS = b'<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>'
def _get_args_contents(self):
return json.dumps(self.args).encode()

@ -50,6 +50,8 @@ import threading
import ansible.constants
from ansible.module_utils.six import reraise
import mitogen.core
import mitogen.service
import ansible_mitogen.loaders
@ -66,20 +68,6 @@ LOG = logging.getLogger(__name__)
ansible_mitogen.loaders.shell_loader.get('sh')
if sys.version_info[0] == 3:
def reraise(tp, value, tb):
if value is None:
value = tp()
if value.__traceback__ is not tb:
raise value.with_traceback(tb)
raise value
else:
exec(
"def reraise(tp, value, tb=None):\n"
" raise tp, value, tb\n"
)
def _get_candidate_temp_dirs():
try:
# >=2.5

@ -39,6 +39,7 @@ __metaclass__ = type
import errno
import grp
import json
import logging
import operator
import os
import pwd
@ -51,26 +52,9 @@ import tempfile
import traceback
import types
# Absolute imports for <2.5.
logging = __import__('logging')
import mitogen.core
import mitogen.parent
import mitogen.service
from mitogen.core import b
try:
reduce
except NameError:
# Python 3.x.
from functools import reduce
try:
BaseException
except NameError:
# Python 2.4
BaseException = Exception
# Ansible since PR #41749 inserts "import __main__" into
# ansible.module_utils.basic. Mitogen's importer will refuse such an import, so
@ -80,6 +64,9 @@ if not sys.modules.get(str('__main__')):
sys.modules[str('__main__')] = types.ModuleType(str('__main__'))
import ansible.module_utils.json_utils
from ansible.module_utils.six.moves import reduce
import ansible_mitogen.runner
@ -615,8 +602,8 @@ def exec_args(args, in_data='', chdir=None, shell=None, emulate_tty=False):
stdout, stderr = proc.communicate(in_data)
if emulate_tty:
stdout = stdout.replace(b('\n'), b('\r\n'))
return proc.returncode, stdout, stderr or b('')
stdout = stdout.replace(b'\n', b'\r\n')
return proc.returncode, stdout, stderr or b''
def exec_command(cmd, in_data='', chdir=None, shell=None, emulate_tty=False):
@ -746,9 +733,7 @@ def set_file_mode(path, spec, fd=None):
"""
Update the permissions of a file using the same syntax as chmod(1).
"""
if isinstance(spec, int):
new_mode = spec
elif not mitogen.core.PY3 and isinstance(spec, long):
if isinstance(spec, mitogen.core.integer_types):
new_mode = spec
elif spec.isdigit():
new_mode = int(spec, 8)

@ -65,21 +65,12 @@ import abc
import os
import ansible.utils.shlex
import ansible.constants as C
import ansible.executor.interpreter_discovery
import ansible.utils.unsafe_proxy
from ansible.module_utils.six import with_metaclass
from ansible.module_utils.parsing.convert_bool import boolean
# this was added in Ansible >= 2.8.0; fallback to the default interpreter if necessary
try:
from ansible.executor.interpreter_discovery import discover_interpreter
except ImportError:
discover_interpreter = lambda action,interpreter_name,discovery_mode,task_vars: '/usr/bin/python'
try:
from ansible.utils.unsafe_proxy import AnsibleUnsafeText
except ImportError:
from ansible.vars.unsafe_proxy import AnsibleUnsafeText
import mitogen.core
@ -115,12 +106,13 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth
action._finding_python_interpreter = True
# fake pipelining so discover_interpreter can be happy
action._connection.has_pipelining = True
s = AnsibleUnsafeText(discover_interpreter(
s = ansible.executor.interpreter_discovery.discover_interpreter(
action=action,
interpreter_name=interpreter_name,
discovery_mode=s,
task_vars=task_vars))
task_vars=task_vars,
)
s = ansible.utils.unsafe_proxy.AnsibleUnsafeText(s)
# cache discovered interpreter
task_vars['ansible_facts'][discovered_interpreter_config] = s
action._connection.has_pipelining = False
@ -498,12 +490,13 @@ class PlayContextSpec(Spec):
)
def ssh_args(self):
local_vars = self._task_vars.get("hostvars", {}).get(self._inventory_name, {})
return [
mitogen.core.to_text(term)
for s in (
C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=local_vars),
C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=local_vars),
C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=local_vars)
)
for term in ansible.utils.shlex.shlex_split(s or '')
]
@ -738,12 +731,13 @@ class MitogenViaSpec(Spec):
)
def ssh_args(self):
local_vars = self._task_vars.get("hostvars", {}).get(self._inventory_name, {})
return [
mitogen.core.to_text(term)
for s in (
C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=local_vars),
C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=local_vars),
C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=local_vars)
)
for term in ansible.utils.shlex.shlex_split(s)
if s

@ -1,14 +0,0 @@
<!doctype html>
<title>Mitogen for Ansible (Redirect)</title>
<script>
{% include "piwik-config.js" %}
var u="https://networkgenomics.com/p/tr/";
_paq.push(['setTrackerUrl', u+'ep']);
</script>
<script src="https://networkgenomics.com/p/tr/js"></script>
<script>
setTimeout(function() {
window.location = 'https://networkgenomics.com/ansible/';
}, 0);
</script>
<meta http-equiv="Refresh" content="0; url=https://networkgenomics.com/ansible/">

@ -14,23 +14,5 @@
{% block footer %}
{{ super() }}
<script>
(function() {
{% include "piwik-config.js" %}
var u="https://networkgenomics.com/p/tr/";
_paq.push(['setTrackerUrl', u+'ep']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; g.type='text/javascript';
g.defer=true; g.async=true; g.src=u+'js'; s.parentNode.insertBefore(g,s);
})();
</script>
<noscript>
<p>
{% set fulltitle = (title|striptags|e) + titlesuffix -%}
<img src="https://networkgenomics.com/p/tr/ep?idsite=6&action_name={{fulltitle}}" style="border:0" alt="">
</p>
</noscript>
<script async defer src="https://buttons.github.io/buttons.js"></script>
{% endblock %}

@ -1,5 +0,0 @@
window._paq = [];
window._paq.push(['trackPageView']);
window._paq.push(['enableLinkTracking']);
window._paq.push(['enableHeartBeatTimer', 30]);
window._paq.push(['setSiteId', 6]);

@ -75,34 +75,6 @@ Installation
``mitogen_host_pinned`` strategies exists to mimic the ``free`` and
``host_pinned`` strategies.
4.
.. raw:: html
<form action="https://networkgenomics.com/save-email/" method="post" id="emailform">
<input type=hidden name="list_name" value="mitogen-announce">
Get notified of new releases and important fixes.
<p>
<input type="email" placeholder="E-mail Address" name="email" style="font-size: 105%;"><br>
<input name="captcha_1" placeholder="Captcha" style="width: 10ex;">
<img class="captcha-image">
<a class="captcha-refresh" href="#">&#x21bb</a>
<button type="submit" style="font-size: 105%;">
Subscribe
</button>
</p>
<div id="emailthanks" style="display:none">
Thanks!
</div>
<p>
</form>
Demo
~~~~
@ -165,7 +137,9 @@ Noteworthy Differences
+-----------------+-----------------+
| 8 | 3.9 - 3.12 |
+-----------------+-----------------+
| 9 | 3.10 - 3.12 |
| 9 | |
+-----------------+ 3.10 - 3.12 |
| 10 | |
+-----------------+-----------------+
Verify your installation is running one of these versions by checking
@ -271,15 +245,14 @@ Noteworthy Differences
* "Module Replacer" style modules are not supported. These rarely appear in
practice, and light web searches failed to reveal many examples of them.
..
* The ``ansible_python_interpreter`` variable is parsed using a restrictive
:mod:`shell-like <shlex>` syntax, permitting values such as ``/usr/bin/env
FOO=bar python`` or ``source /opt/rh/rh-python36/enable && python``, which
occur in practice. Jinja2 templating is also supported for complex task-level
interpreter settings. Ansible `documents this
<https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html#ansible-python-interpreter>`_
as an absolute path, however the implementation passes it unquoted through
the shell, permitting arbitrary code to be injected.
* The ``ansible_python_interpreter`` variable is parsed using a restrictive
:mod:`shell-like <shlex>` syntax, permitting values such as ``/usr/bin/env
FOO=bar python`` or ``source /opt/rh/rh-python36/enable && python``.
Jinja2 templating is also supported for complex task-level
interpreter settings. Ansible documents `ansible_python_interpreter
<https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html#ansible-python-interpreter>`_
as an absolute path and releases since June 2024 (e.g. Ansible 10.1)
reflect this. Older Ansible releases passed it to the shell unquoted.
..
* Configurations will break that rely on the `hashbang argument splitting
@ -1418,20 +1391,3 @@ Despite the small margin for optimization, Mitogen still manages **6.2x less
bandwidth and 1.8x less time**.
.. image:: images/ansible/pcaps/costapp-uk-india.svg
.. raw:: html
<script src="https://networkgenomics.com/static/js/public_all.js?92d49a3a"></script>
<script>
NetGen = {
public: {
page_id: "operon",
urls: {
save_email: "https://networkgenomics.com/save-email/",
save_email_captcha: "https://networkgenomics.com/save-email/captcha/",
}
}
};
setupEmailForm();
</script>

@ -22,10 +22,49 @@ Unreleased
----------
* :gh:issue:`636` os.chdir fails if the sudo/become user lacks adequate permissions to chdir prior to task
* :gh:issue:`1127` :mod:`mitogen`: Consolidate mitogen backward compatibility
fallbacks and polyfills into :mod:`mitogen.core`
* :gh:issue:`1127` :mod:`ansible_mitogen`: Remove backward compatibility
fallbacks for Python 2.4 & 2.5.
* :gh:issue:`1127` :mod:`ansible_mitogen`: Remove fallback imports for Ansible
releases before 2.10
* :gh:issue:`1127` :mod:`ansible_mitogen`: Consolidate Python 2 & 3
compatibility
* :gh:issue:`1128` CI: Start migration from Azure DevOps to GitHub Actions
v0.3.10 (2024-09-20)
--------------------
* :gh:issue:`950` Fix Solaris/Illumos/SmartOS compatibility with become
* :gh:issue:`1087` Fix :exc:`mitogen.core.StreamError` when Ansible template
module is called with a ``dest:`` filename that has an extension
* :gh:issue:`1110` Fix :exc:`mitogen.core.StreamError` when Ansible copy
module is called with a file larger than 124 kibibytes
(:data:`ansible_mitogen.connection.Connection.SMALL_FILE_LIMIT`)
* :gh:issue:`905` Initial support for templated ``ansible_ssh_args``,
``ansible_ssh_common_args``, and ``ansible_ssh_extra_args`` variables.
NB: play or task scoped variables will probably still fail.
* :gh:issue:`694` CI: Fixed a race condition and some resource leaks causing
some of intermittent failures when running the test suite.
v0.3.9 (2024-08-13)
-------------------
* :gh:issue:`1097` Respect `ansible_facts.discovered_interpreter_python` when
executing non new-style modules (e.g. JSONARGS style, WANT_JSON style).
* :gh:issue:`1074` Support Ansible 10 (ansible-core 2.17)
v0.3.8 (2024-07-30)
-------------------
* :gh:issue:`952` Fix Ansible `--ask-become-pass`, add test coverage
* :gh:issue:`957` Fix Ansible exception when executing against 10s of hosts
"ValueError: filedescriptor out of range in select()"
* :gh:issue:`1066` Support Ansible `ansible_host_key_checking` & `ansible_ssh_host_key_checking`
* :gh:issue:`1090` CI: Migrate macOS integration tests to macOS 12, drop Python 2.7 jobs
v0.3.7 (2024-04-08)

@ -2,7 +2,7 @@ import sys
sys.path.append('.')
VERSION = '0.3.7'
VERSION = '0.3.9'
author = u'Network Genomics'
copyright = u'2021, the Mitogen authors'
@ -16,7 +16,6 @@ html_show_copyright = False
html_show_sourcelink = False
html_show_sphinx = False
html_sidebars = {'**': ['globaltoc.html', 'github.html']}
html_additional_pages = {'ansible': 'ansible.html'}
html_static_path = ['_static']
html_theme = 'alabaster'
html_theme_options = {

@ -116,6 +116,7 @@ sponsorship and outstanding future-thinking of its early adopters.
<ul>
<li>Alex Willmer</li>
<li><a href="https://github.com/momiji">Christian Bourgeois </a></li>
<li><a href="https://underwhelm.net/">Dan Dorman</a> &mdash; - <em>When I truly understand my enemy … then in that very moment I also love him.</em></li>
<li>Daniel Foerster</li>
<li><a href="https://www.deps.co/">Deps</a> &mdash; <em>Private Maven Repository Hosting for Java, Scala, Groovy, Clojure</em></li>
@ -125,6 +126,7 @@ sponsorship and outstanding future-thinking of its early adopters.
<li><a href="https://www.channable.com">rkrzr</a></li>
<li>jgadling</li>
<li>John F Wall &mdash; <em>Making Ansible Great with Massive Parallelism</em></li>
<li><a href="https://github.com/jrosser">Jonathan Rosser</a></li>
<li>KennethC</li>
<li><a href="https://github.com/lberruti">Luca Berruti</li>
<li>Lewis Bellwood &mdash; <em>Happy to be apart of a great project.</em></li>

@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup.
#: Library version as a tuple.
__version__ = (0, 3, 8, 'dev')
__version__ = (0, 3, 11, 'dev')
#: This is :data:`False` in slave contexts. Previously it was used to prevent

@ -102,21 +102,6 @@ try:
except ImportError:
cProfile = None
try:
import thread
except ImportError:
import threading as thread
try:
import cPickle as pickle
except ImportError:
import pickle
try:
from cStringIO import StringIO as BytesIO
except ImportError:
from io import BytesIO
try:
BaseException
except NameError:
@ -169,31 +154,35 @@ STUB_CALL_SERVICE = 111
#: :meth:`mitogen.core.Router.add_handler` callbacks to clean up.
IS_DEAD = 999
try:
BaseException
except NameError:
BaseException = Exception
PY24 = sys.version_info < (2, 5)
PY3 = sys.version_info > (3,)
if PY3:
import pickle
import _thread as thread
from io import BytesIO
b = str.encode
BytesType = bytes
UnicodeType = str
FsPathTypes = (str,)
BufferType = lambda buf, start: memoryview(buf)[start:]
long = int
integer_types = (int,)
iteritems, iterkeys, itervalues = dict.items, dict.keys, dict.values
else:
import cPickle as pickle
import thread
from cStringIO import StringIO as BytesIO
b = str
BytesType = str
FsPathTypes = (str, unicode)
BufferType = buffer
UnicodeType = unicode
integer_types = (int, long)
iteritems, iterkeys, itervalues = dict.iteritems, dict.iterkeys, dict.itervalues
AnyTextType = (BytesType, UnicodeType)
try:
next
next = next
except NameError:
next = lambda it: it.next()
@ -400,12 +389,19 @@ now = getattr(time, 'monotonic', time.time)
# Python 2.4
try:
any
all, any = all, any
except NameError:
def all(it):
for elem in it:
if not elem:
return False
return True
def any(it):
for elem in it:
if elem:
return True
return False
def _partition(s, sep, find):
@ -1065,8 +1061,8 @@ class Sender(object):
def _unpickle_sender(router, context_id, dst_handle):
if not (isinstance(router, Router) and
isinstance(context_id, (int, long)) and context_id >= 0 and
isinstance(dst_handle, (int, long)) and dst_handle > 0):
isinstance(context_id, integer_types) and context_id >= 0 and
isinstance(dst_handle, integer_types) and dst_handle > 0):
raise TypeError('cannot unpickle Sender: bad input or missing router')
return Sender(Context(router, context_id), dst_handle)
@ -2508,7 +2504,7 @@ class Context(object):
def _unpickle_context(context_id, name, router=None):
if not (isinstance(context_id, (int, long)) and context_id >= 0 and (
if not (isinstance(context_id, integer_types) and context_id >= 0 and (
(name is None) or
(isinstance(name, UnicodeType) and len(name) < 100))
):

@ -74,9 +74,11 @@ import mitogen.core
import mitogen.minify
import mitogen.parent
from mitogen.core import any
from mitogen.core import b
from mitogen.core import IOLOG
from mitogen.core import LOG
from mitogen.core import next
from mitogen.core import str_partition
from mitogen.core import str_rpartition
from mitogen.core import to_text
@ -84,17 +86,6 @@ from mitogen.core import to_text
imap = getattr(itertools, 'imap', map)
izip = getattr(itertools, 'izip', zip)
try:
any
except NameError:
from mitogen.core import any
try:
next
except NameError:
from mitogen.core import next
RLOG = logging.getLogger('mitogen.ctx')

@ -56,15 +56,13 @@ import zlib
# Absolute imports for <2.5.
select = __import__('select')
try:
import thread
except ImportError:
import threading as thread
import mitogen.core
from mitogen.core import b
from mitogen.core import bytes_partition
from mitogen.core import IOLOG
from mitogen.core import itervalues
from mitogen.core import next
from mitogen.core import thread
LOG = logging.getLogger(__name__)
@ -80,15 +78,6 @@ except IOError:
SELINUX_ENABLED = False
try:
next
except NameError:
# Python 2.4/2.5
from mitogen.core import next
itervalues = getattr(dict, 'itervalues', dict.values)
if mitogen.core.PY3:
xrange = range
closure_attr = '__closure__'
@ -147,6 +136,8 @@ LINUX_TIOCGPTN = _ioctl_cast(2147767344)
LINUX_TIOCSPTLCK = _ioctl_cast(1074025521)
IS_LINUX = os.uname()[0] == 'Linux'
IS_SOLARIS = os.uname()[0] == 'SunOS'
SIGNAL_BY_NUM = dict(
(getattr(signal, name), name)
@ -411,7 +402,7 @@ def _acquire_controlling_tty():
# On Linux, the controlling tty becomes the first tty opened by a
# process lacking any prior tty.
os.close(os.open(os.ttyname(2), os.O_RDWR))
if hasattr(termios, 'TIOCSCTTY') and not mitogen.core.IS_WSL:
if hasattr(termios, 'TIOCSCTTY') and not mitogen.core.IS_WSL and not IS_SOLARIS:
# #550: prehistoric WSL does not like TIOCSCTTY.
# On BSD an explicit ioctl is required. For some inexplicable reason,
# Python 2.6 on Travis also requires it.
@ -479,7 +470,8 @@ def openpty():
master_fp = os.fdopen(master_fd, 'r+b', 0)
slave_fp = os.fdopen(slave_fd, 'r+b', 0)
disable_echo(master_fd)
if not IS_SOLARIS:
disable_echo(master_fd)
disable_echo(slave_fd)
mitogen.core.set_block(slave_fd)
return master_fp, slave_fp
@ -2542,7 +2534,7 @@ class Reaper(object):
# because it is setuid, so this is best-effort only.
LOG.debug('%r: sending %s', self.proc, SIGNAL_BY_NUM[signum])
try:
os.kill(self.proc.pid, signum)
self.proc.send_signal(signum)
except OSError:
e = sys.exc_info()[1]
if e.args[0] != errno.EPERM:
@ -2662,6 +2654,17 @@ class Process(object):
"""
raise NotImplementedError()
def send_signal(self, sig):
os.kill(self.pid, sig)
def terminate(self):
"Ask the process to gracefully shutdown."
self.send_signal(signal.SIGTERM)
def kill(self):
"Ask the operating system to forcefully destroy the process."
self.send_signal(signal.SIGKILL)
class PopenProcess(Process):
"""
@ -2678,6 +2681,9 @@ class PopenProcess(Process):
def poll(self):
return self.proc.poll()
def send_signal(self, sig):
self.proc.send_signal(sig)
class ModuleForwarder(object):
"""

@ -39,18 +39,10 @@ import threading
import mitogen.core
import mitogen.select
from mitogen.core import all
from mitogen.core import b
from mitogen.core import str_rpartition
try:
all
except NameError:
def all(it):
for elem in it:
if not elem:
return False
return True
LOG = logging.getLogger(__name__)

@ -43,11 +43,6 @@ except ImportError:
import mitogen.parent
from mitogen.core import b
try:
any
except NameError:
from mitogen.core import any
LOG = logging.getLogger(__name__)

@ -34,11 +34,6 @@ import re
import mitogen.core
import mitogen.parent
try:
any
except NameError:
from mitogen.core import any
LOG = logging.getLogger(__name__)

@ -143,19 +143,23 @@ class Listener(mitogen.core.Protocol):
def on_accept_client(self, sock):
sock.setblocking(True)
try:
pid, = struct.unpack('>L', sock.recv(4))
data = sock.recv(4)
pid, = struct.unpack('>L', data)
except (struct.error, socket.error):
LOG.error('listener: failed to read remote identity: %s',
sys.exc_info()[1])
LOG.error('listener: failed to read remote identity, got %d bytes: %s',
len(data), sys.exc_info()[1])
sock.close()
return
context_id = self._router.id_allocator.allocate()
try:
# FIXME #1109 send() returns number of bytes sent, check it
sock.send(struct.pack('>LLL', context_id, mitogen.context_id,
os.getpid()))
except socket.error:
LOG.error('listener: failed to assign identity to PID %d: %s',
pid, sys.exc_info()[1])
sock.close()
return
context = mitogen.parent.Context(self._router, context_id)

@ -37,13 +37,7 @@ import sys
import mitogen.core
import mitogen.master
iteritems = getattr(dict, 'iteritems', dict.items)
if mitogen.core.PY3:
iteritems = dict.items
else:
iteritems = dict.iteritems
from mitogen.core import iteritems
def setup_gil():

@ -1,6 +1,9 @@
# code: language=ini
# vim: syntax=dosini
[become_same_user]
# become_same_user.yml
bsu-joe ansible_user=joe
[become_same_user:vars]
ansible_python_interpreter=python3000

@ -4,7 +4,7 @@
# Connection delegation scenarios. It's impossible to connect to them, but their would-be
# config can be inspected using "mitogen_get_stack" action.
[cd]
# Normal inventory host, no aliasing.
cd-normal ansible_connection=mitogen_doas ansible_user=normal-user
@ -23,6 +23,8 @@ cd-newuser-normal-normal mitogen_via=cd-normal ansible_user=newuser-normal-norma
# doas:newuser via host.
cd-newuser-doas-normal mitogen_via=cd-normal ansible_connection=mitogen_doas ansible_user=newuser-doas-normal-user
[cd:vars]
ansible_python_interpreter = python3000
[connection-delegation-test]
cd-bastion

@ -12,3 +12,10 @@ target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}"
target
[linux_containers]
[issue905]
ssh-common-args ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}"
[issue905:vars]
ansible_ssh_common_args=-o PermitLocalCommand=yes -o LocalCommand="touch {{ ssh_args_canary_file }}"
ssh_args_canary_file=/tmp/ssh_args_{{ inventory_hostname }}

@ -1,2 +1,3 @@
---
pkg_mgr_python_interpreter: python
pkg_repos_overrides: []

@ -1,2 +1,28 @@
---
pkg_mgr_python_interpreter: /usr/libexec/platform-python
pkg_repos_overrides:
- dest: /etc/yum.repos.d/CentOS-Linux-AppStream.repo
content: |
[appstream]
name=CentOS Linux $releasever - AppStream
baseurl=http://vault.centos.org/$contentdir/$releasever/AppStream/$basearch/os/
enabled=1
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial
- dest: /etc/yum.repos.d/CentOS-Linux-BaseOS.repo
content: |
[baseos]
name=CentOS Linux $releasever - BaseOS
baseurl=http://vault.centos.org/$contentdir/$releasever/BaseOS/$basearch/os/
enabled=1
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial
- dest: /etc/yum.repos.d/CentOS-Linux-Extras.repo
content: |
[extras]
name=CentOS Linux $releasever - Extras
baseurl=http://vault.centos.org/$contentdir/$releasever/extras/$basearch/os/
enabled=1
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial

@ -0,0 +1,4 @@
pkg_repos_overrides:
- dest: /etc/apt/sources.list
content: |
deb http://archive.debian.org/debian stretch main contrib non-free

@ -1,92 +1,94 @@
# Verify copy module for small and large files, and inline content.
# To exercise https://github.com/mitogen-hq/mitogen/pull/1110 destination
# files must have extensions and loops must use `with_items:`.
- name: integration/action/copy.yml
hosts: test-targets
tasks:
- name: Create tiny file
copy:
dest: /tmp/copy-tiny-file
content:
this is a tiny file.
delegate_to: localhost
run_once: true
vars:
sourced_files:
- src: /tmp/copy-tiny-file
dest: /tmp/copy-tiny-file.out
content: this is a tiny file.
expected_checksum: f29faa9a6f19a700a941bf2aa5b281643c4ec8a0
- src: /tmp/copy-large-file
dest: /tmp/copy-large-file.out
content: "{{ 'x' * 200000 }}"
expected_checksum: 62951f943c41cdd326e5ce2b53a779e7916a820d
inline_files:
- dest: /tmp/copy-tiny-inline-file.out
content: tiny inline content
expected_checksum: b26dd6444595e2bdb342aa0a91721b57478b5029
- dest: /tmp/copy-large-inline-file.out
content: |
{{ 'y' * 200000 }}
expected_checksum: d675f47e467eae19e49032a2cc39118e12a6ee72
- name: Create large file
files: "{{ sourced_files + inline_files }}"
tasks:
- name: Create sourced files
copy:
dest: /tmp/copy-large-file
# Must be larger than Connection.SMALL_SIZE_LIMIT.
content: "{% for x in range(200000) %}x{% endfor %}"
dest: "{{ item.src }}"
content: "{{ item.content }}"
mode: u=rw,go=r
with_items: "{{ sourced_files }}"
loop_control:
label: "{{ item.src }}"
delegate_to: localhost
run_once: true
- name: Cleanup copied files
- name: Cleanup lingering destination files
file:
path: "{{ item.dest }}"
state: absent
path: "{{item}}"
with_items:
- /tmp/copy-tiny-file.out
- /tmp/copy-large-file.out
- /tmp/copy-tiny-inline-file.out
- /tmp/copy-large-inline-file.out
with_items: "{{ files }}"
loop_control:
label: "{{ item.dest }}"
- name: Copy large file
- name: Copy sourced files
copy:
dest: /tmp/copy-large-file.out
src: /tmp/copy-large-file
- name: Copy tiny file
copy:
dest: /tmp/copy-tiny-file.out
src: /tmp/copy-tiny-file
src: "{{ item.src }}"
dest: "{{ item.dest }}"
mode: u=rw,go=r
with_items: "{{ sourced_files }}"
loop_control:
label: "{{ item.dest }}"
- name: Copy tiny inline file
- name: Copy inline files
copy:
dest: /tmp/copy-tiny-inline-file.out
content: "tiny inline content"
- name: Copy large inline file
copy:
dest: /tmp/copy-large-inline-file.out
content: |
{% for x in range(200000) %}y{% endfor %}
dest: "{{ item.dest }}"
content: "{{ item.content }}"
mode: u=rw,go=r
with_items: "{{ inline_files }}"
loop_control:
label: "{{ item.dest }}"
# stat results
- name: Stat copied files
stat:
path: "{{item}}"
with_items:
- /tmp/copy-tiny-file.out
- /tmp/copy-large-file.out
- /tmp/copy-tiny-inline-file.out
- /tmp/copy-large-inline-file.out
path: "{{ item.dest }}"
with_items: "{{ files }}"
loop_control:
label: "{{ item.dest }}"
register: stat
- assert:
that:
- stat.results[0].stat.checksum == "f29faa9a6f19a700a941bf2aa5b281643c4ec8a0"
- stat.results[1].stat.checksum == "62951f943c41cdd326e5ce2b53a779e7916a820d"
- stat.results[2].stat.checksum == "b26dd6444595e2bdb342aa0a91721b57478b5029"
- stat.results[3].stat.checksum == "d675f47e467eae19e49032a2cc39118e12a6ee72"
fail_msg: stat={{stat}}
- item.stat.checksum == item.item.expected_checksum
quiet: true # Avoid spamming stdout with 400 kB of item.item.content
fail_msg: item={{ item }}
with_items: "{{ stat.results }}"
loop_control:
label: "{{ item.stat.path }}"
- name: Cleanup files
- name: Cleanup destination files
file:
path: "{{ item.dest }}"
state: absent
path: "{{item}}"
with_items:
- /tmp/copy-tiny-file
- /tmp/copy-tiny-file.out
- /tmp/copy-no-mode
- /tmp/copy-no-mode.out
- /tmp/copy-with-mode
- /tmp/copy-with-mode.out
- /tmp/copy-large-file
- /tmp/copy-large-file.out
- /tmp/copy-tiny-inline-file.out
- /tmp/copy-large-inline-file
- /tmp/copy-large-inline-file.out
# end of cleaning out files (again)
with_items: "{{ files }}"
loop_control:
label: "{{ item.dest }}"
tags:
- copy
- issue_1110

@ -53,20 +53,22 @@
vars:
ansible_become_pass: user1_password
when:
# https://github.com/ansible/ansible/pull/70785
- ansible_facts.distribution not in ["MacOSX"]
or ansible_version.full is version("2.11", ">=", strict=True)
or is_mitogen
# CI containers lack `setfacl` for unpriv -> unpriv
# https://github.com/mitogen-hq/mitogen/issues/1118
- is_mitogen
or (ansible_facts.distribution in ["MacOSX"]
and ansible_version.full is version("2.11", ">=", strict=True))
- assert:
that:
- out.stdout == 'mitogen__user1'
fail_msg: out={{out}}
when:
# https://github.com/ansible/ansible/pull/70785
- ansible_facts.distribution not in ["MacOSX"]
or ansible_version.full is version("2.11", ">=", strict=True)
or is_mitogen
# CI containers lack `setfacl` for unpriv -> unpriv
# https://github.com/mitogen-hq/mitogen/issues/1118
- is_mitogen
or (ansible_facts.distribution in ["MacOSX"]
and ansible_version.full is version("2.11", ">=", strict=True))
- name: Ensure password su without chdir succeeds
shell: whoami
@ -76,20 +78,22 @@
vars:
ansible_become_pass: user1_password
when:
# https://github.com/ansible/ansible/pull/70785
- ansible_facts.distribution not in ["MacOSX"]
or ansible_version.full is version("2.11", ">=", strict=True)
or is_mitogen
# CI containers lack `setfacl` for unpriv -> unpriv
# https://github.com/mitogen-hq/mitogen/issues/1118
- is_mitogen
or (ansible_facts.distribution in ["MacOSX"]
and ansible_version.full is version("2.11", ">=", strict=True))
- assert:
that:
- out.stdout == 'mitogen__user1'
fail_msg: out={{out}}
when:
# https://github.com/ansible/ansible/pull/70785
- ansible_facts.distribution not in ["MacOSX"]
or ansible_version.full is version("2.11", ">=", strict=True)
or is_mitogen
# CI containers lack `setfacl` for unpriv -> unpriv
# https://github.com/mitogen-hq/mitogen/issues/1118
- is_mitogen
or (ansible_facts.distribution in ["MacOSX"]
and ansible_version.full is version("2.11", ">=", strict=True))
tags:
- su

@ -21,8 +21,7 @@
# sudo-1.8.6p3-29.el6_10.3 on RHEL & CentOS 6.10 (final release)
# removed user/group error messages, as defence against CVE-2019-14287.
- >-
'sudo: unknown user: slartibartfast' in out.module_stdout | default(out.msg)
or 'sudo: unknown user: slartibartfast' in out.module_stderr | default(out.msg)
(out.module_stderr | default(out.module_stdout, true) | default(out.msg, true)) is search('sudo: unknown user:? slartibartfast')
or (ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_version == '6.10')
fail_msg: out={{out}}
when:

@ -40,7 +40,7 @@
'keepalive_count': 10,
'password': null,
'port': null,
'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"],
'python_path': ['python3000'],
'remote_name': null,
'ssh_args': [
-o, ControlMaster=auto,
@ -68,7 +68,7 @@
'keepalive_count': 10,
'password': null,
'port': null,
'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"],
'python_path': ['python3000'],
'remote_name': null,
'ssh_args': [
-o, ControlMaster=auto,

@ -19,7 +19,7 @@
'kind': 'lxc',
'lxc_info_path': null,
'machinectl_path': null,
'python_path': ['/usr/bin/python'],
'python_path': ['python3000'],
'remote_name': null,
'username': null,
},

@ -23,7 +23,7 @@
'lxc_info_path': null,
'lxc_path': null,
'machinectl_path': null,
'python_path': ["/usr/bin/python"],
'python_path': ["python3000"],
'username': 'ansible-cfg-remote-user',
},
'method': 'setns',

@ -40,7 +40,7 @@
"connect_timeout": 30,
"doas_path": null,
"password": null,
"python_path": ["/usr/bin/python"],
"python_path": ["python3000"],
'remote_name': null,
"username": "normal-user",
},
@ -73,7 +73,7 @@
'keepalive_count': 10,
'password': null,
'port': null,
"python_path": ["/usr/bin/python"],
"python_path": ["python3000"],
'remote_name': null,
'ssh_args': [
-o, ControlMaster=auto,
@ -115,7 +115,7 @@
'keepalive_count': 10,
'password': null,
'port': null,
"python_path": ["/usr/bin/python"],
"python_path": ["python3000"],
'remote_name': null,
'ssh_args': [
-o, ControlMaster=auto,
@ -150,7 +150,7 @@
'connect_timeout': 30,
'doas_path': null,
'password': null,
"python_path": ["/usr/bin/python"],
"python_path": ["python3000"],
'remote_name': null,
'username': 'normal-user',
},
@ -168,7 +168,7 @@
'keepalive_count': 10,
'password': null,
'port': null,
"python_path": ["/usr/bin/python"],
"python_path": ["python3000"],
'remote_name': null,
'ssh_args': [
-o, ControlMaster=auto,
@ -210,7 +210,7 @@
'keepalive_count': 10,
'password': null,
'port': null,
"python_path": ["/usr/bin/python"],
"python_path": ["python3000"],
'remote_name': null,
'ssh_args': [
-o, ControlMaster=auto,
@ -238,7 +238,7 @@
'keepalive_count': 10,
'password': null,
'port': null,
"python_path": ["/usr/bin/python"],
"python_path": ["python3000"],
'remote_name': null,
'ssh_args': [
-o, ControlMaster=auto,
@ -272,7 +272,7 @@
'connect_timeout': 30,
'doas_path': null,
'password': null,
"python_path": ["/usr/bin/python"],
"python_path": ["python3000"],
'remote_name': null,
'username': 'normal-user',
},
@ -290,7 +290,7 @@
'keepalive_count': 10,
'password': null,
'port': null,
"python_path": ["/usr/bin/python"],
"python_path": ["python3000"],
'remote_name': null,
'ssh_args': [
-o, ControlMaster=auto,
@ -333,7 +333,7 @@
'keepalive_count': 10,
'password': null,
'port': null,
"python_path": ["/usr/bin/python"],
"python_path": ["python3000"],
'remote_name': null,
'ssh_args': [
-o, ControlMaster=auto,
@ -388,7 +388,7 @@
'connect_timeout': 30,
'doas_path': null,
'password': null,
'python_path': ["/usr/bin/python"],
'python_path': ["python3000"],
'remote_name': null,
'username': 'normal-user',
},
@ -399,7 +399,7 @@
'connect_timeout': 30,
'doas_path': null,
'password': null,
'python_path': ["/usr/bin/python"],
'python_path': ["python3000"],
'remote_name': null,
'username': 'newuser-doas-normal-user',
},

@ -4,6 +4,55 @@
- name: integration/interpreter_discovery/ansible_2_8_tests.yml
hosts: test-targets
gather_facts: true
vars:
DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_lt_2_12:
centos:
'6': /usr/bin/python
'7': /usr/bin/python
'8': /usr/libexec/platform-python
debian:
'9': /usr/bin/python
'10': /usr/bin/python3
'11': /usr/bin/python
'NA': /usr/bin/python # Debian 11, Ansible <= 7 (ansible-core <= 2.14)
'bullseye/sid': /usr/bin/python # Debian 11, Ansible 8 - 9 (ansible-core 2.15 - 2.16)
ubuntu:
'16': /usr/bin/python3
'18': /usr/bin/python3
'20': /usr/bin/python3
DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_2_12_to_2_16:
centos:
'6': /usr/bin/python
'7': /usr/bin/python
'8': /usr/libexec/platform-python
debian:
'9': /usr/bin/python
'10': /usr/bin/python3
'11': /usr/bin/python3.9
'NA': /usr/bin/python3.9 # Debian 11, Ansible <= 7 (ansible-core <= 2.14)
'bullseye/sid': /usr/bin/python3.9 # Debian 11, Ansible 8 - 9 (ansible-core 2.15 - 2.16)
ubuntu:
'16': /usr/bin/python3
'18': /usr/bin/python3
'20': /usr/bin/python3
DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_ge_2_17:
debian:
'10': /usr/bin/python3.7
'11': /usr/bin/python3.9
'bullseye/sid': /usr/bin/python3.9
ubuntu:
'20': /usr/bin/python3.8
discovered_interpreter_expected: >-
{%- if ansible_version.full is version('2.12', '<', strict=True) -%}
{{ DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_lt_2_12[distro][distro_major] }}
{%- elif ansible_version.full is version('2.17', '<', strict=True) -%}
{{ DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_2_12_to_2_16[distro][distro_major] }}
{%- else -%}
{{ DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_ge_2_17[distro][distro_major] }}
{%- endif -%}
tasks:
- name: can only run these tests on ansible >= 2.8.0
block:
@ -18,9 +67,9 @@
- name: snag some facts to validate for later
set_fact:
distro: '{{ ansible_distribution | default("unknown") | lower }}'
distro_version: '{{ ansible_distribution_version | default("unknown") }}'
os_family: '{{ ansible_os_family | default("unknown") }}'
distro: '{{ ansible_facts.distribution | lower }}'
distro_major: '{{ ansible_facts.distribution_major_version }}'
system: '{{ ansible_facts.system }}'
- name: test that python discovery is working and that fact persistence makes it only run once
block:
@ -50,7 +99,7 @@
that:
- auto_out.ansible_facts.discovered_interpreter_python is defined
- auto_out.ansible_facts.discovered_interpreter_python == echoout.discovered_python.as_seen
- echoout.discovered_python.resolved == echoout.running_python.sys.executable.resolved
- echoout.discovered_python.sys.executable.as_seen == echoout.running_python.sys.executable.as_seen
fail_msg:
- "auto_out: {{ auto_out }}"
- "echoout: {{ echoout }}"
@ -159,50 +208,19 @@
- ansible_facts['ansible_bogus_interpreter'] | default('nope') == 'nope'
- ansible_facts['discovered_interpreter_bogus'] | default('nope') == 'nope'
- name: fedora assertions
- name: Check discovered interpreter matches expected
assert:
that:
- auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python3'
fail_msg: auto_out={{auto_out}}
- auto_out.ansible_facts.discovered_interpreter_python == discovered_interpreter_expected
fail_msg: >-
distro={{ distro }}
distro_major= {{ distro_major }}
system={{ system }}
auto_out={{ auto_out }}
discovered_interpreter_expected={{ discovered_interpreter_expected }}
ansible_version.full={{ ansible_version.full }}
when:
- distro == 'fedora'
- distro_version is version('23.0', '>=', strict=True)
- name: rhel < 8 assertions
assert:
that:
- auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python'
fail_msg: auto_out={{auto_out}}
when:
- distro in ('redhat', 'centos')
- distro_version is version('8.0', '<', strict=true)
- name: rhel 8+ assertions
assert:
that:
- auto_out.ansible_facts.discovered_interpreter_python == '/usr/libexec/platform-python'
fail_msg: auto_out={{auto_out}}
when:
- distro in ('redhat', 'centos')
- distro_version is version('8.0', '>=', strict=true)
- name: ubuntu < 16.04 assertions
assert:
that:
- auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python'
fail_msg: auto_out={{auto_out}}
when:
- distro == 'ubuntu'
- distro_version is version('16.04', '<', strict=true)
- name: ubuntu 16.04+ assertions
assert:
that:
- auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python3'
fail_msg: auto_out={{auto_out}}
when:
- distro == 'ubuntu'
- distro_version is version('16.04', '>=', strict=True)
- system in ['Linux']
always:
- meta: clear_facts

@ -9,6 +9,13 @@
https_proxy: "{{ lookup('env', 'https_proxy') | default(omit) }}"
no_proxy: "{{ lookup('env', 'no_proxy') | default(omit) }}"
tasks:
# Ansible releases after June 2024 quote ansible_python_interpreter
# https://github.com/ansible/ansible/pull/83365
- meta: end_play
when:
- not is_mitogen
- ansible_version.full is version('2.17.1', '>=', strict=True)
- name: create temp file to source
file:
path: /tmp/fake
@ -25,8 +32,7 @@
# special_python: source /tmp/fake && python
- name: set python using sourced file
set_fact:
# Avoid 2.x vs 3.x cross-compatiblity issues (that I can't remember the exact details of).
special_python: "source /tmp/fake || true && python{{ ansible_facts.python.version.major }}"
special_python: "source /tmp/fake || true && {{ ansible_facts.python.executable }}"
- name: run get_url with specially-sourced python
uri:

@ -1,7 +1,7 @@
# external2 is loaded from config path.
# external1 is loaded from integration/module_utils/roles/modrole/module_utils/..
- name: integration/module_utils/adjacent_to_playbook.yml
- name: integration/module_utils/adjacent_to_role.yml
hosts: test-targets
roles:
- modrole

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/python
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils import external3

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/python
import json
import ansible.module_utils.basic

@ -2,6 +2,12 @@
- name: integration/runner/custom_bash_hashbang_argument.yml
hosts: test-targets
tasks:
# Ansible releases after June 2024 quote ansible_python_interpreter
# https://github.com/ansible/ansible/pull/83365
- meta: end_play
when:
- not is_mitogen
- ansible_version.full is version('2.17.1', '>=', strict=True)
- custom_bash_old_style_module:
foo: true

@ -1,3 +1,5 @@
- import_playbook: args.yml
- import_playbook: config.yml
- import_playbook: password.yml
- import_playbook: timeouts.yml
- import_playbook: variables.yml

@ -0,0 +1,48 @@
- name: integration/ssh/args.yml
hosts: issue905
gather_facts: false
tasks:
# Test that ansible_ssh_common_args are templated; ansible_ssh_args &
# ansible_ssh_extra_args aren't directly tested, we assume they're similar.
# FIXME This test currently relies on variables set in the host group.
# Ideally they'd be set here, and the host group eliminated, but
# Mitogen currently fails to template when defined in the play.
# TODO Replace LocalCommand canary with SetEnv canary, to simplify test.
# Requires modification of sshd_config files to add AcceptEnv ...
- name: Test templating of ansible_ssh_common_args et al
block:
- name: Ensure no lingering canary files
file:
path: "{{ ssh_args_canary_file }}"
state: absent
delegate_to: localhost
- name: Reset connections to force new ssh execution
meta: reset_connection
- name: Perform SSH connection, to trigger side effect
ping:
# LocalCommand="touch {{ ssh_args_canary_file }}" in ssh_*_args
- name: Stat for canary file created by side effect
stat:
path: "{{ ssh_args_canary_file }}"
delegate_to: localhost
register: ssh_args_canary_stat
- assert:
that:
- ssh_args_canary_stat.stat.exists == true
quiet: true
success_msg: "Canary found: {{ ssh_args_canary_file }}"
fail_msg: |
ssh_args_canary_file={{ ssh_args_canary_file }}
ssh_args_canary_stat={{ ssh_args_canary_stat }}
always:
- name: Cleanup canary files
file:
path: "{{ ssh_args_canary_file }}"
state: absent
delegate_to: localhost
tags:
- issue_905

@ -0,0 +1,51 @@
- name: integration/ssh/password.yml
hosts: test-targets[0]
gather_facts: false
vars:
ansible_user: mitogen__user1
tasks:
- meta: reset_connection
- name: ansible_password
vars:
ansible_password: user1_password
ping:
- meta: reset_connection
- name: ansible_ssh_pass
vars:
ansible_ssh_pass: user1_password
ping:
- meta: reset_connection
- name: absent password should fail
ping:
ignore_errors: true
ignore_unreachable: true
register: ssh_no_password_result
- assert:
that:
- ssh_no_password_result.unreachable == True
fail_msg: ssh_no_password_result={{ ssh_no_password_result }}
- meta: reset_connection
- name: ansible_ssh_pass should override ansible_password
ping:
vars:
ansible_password: wrong
ansible_ssh_pass: user1_password
# Tests that ansible_ssh_pass has priority over ansible_password
# and that a wrong password causes a target to be marked unreachable.
- meta: reset_connection
- name: ansible_password should not override
vars:
ansible_password: user1_password
ansible_ssh_pass: wrong
ping:
ignore_errors: true
ignore_unreachable: true
register: ssh_wrong_password_result
- assert:
that:
- ssh_wrong_password_result.unreachable == True
fail_msg: ssh_wrong_password_result={{ ssh_wrong_password_result }}

@ -13,134 +13,6 @@
-o "ControlPath /tmp/mitogen-ansible-test-{{18446744073709551615|random}}"
tasks:
- include_tasks: ../_mitogen_only.yml
- name: ansible_ssh_user, ansible_ssh_pass
shell: >
ANSIBLE_ANY_ERRORS_FATAL=false
ANSIBLE_STRATEGY=mitogen_linear
ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
ansible -m shell -a whoami
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
{% endfor %}
test-targets
-e ansible_ssh_user=mitogen__has_sudo
-e ansible_ssh_pass=has_sudo_password
args:
chdir: ../..
register: out
- name: ansible_ssh_user, wrong ansible_ssh_pass
shell: >
ANSIBLE_ANY_ERRORS_FATAL=false
ANSIBLE_STRATEGY=mitogen_linear
ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
ansible -m shell -a whoami
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
{% endfor %}
test-targets
-e ansible_ssh_user=mitogen__has_sudo
-e ansible_ssh_pass=wrong_password
-e ansible_python_interpreter=python3000
args:
chdir: ../..
register: out
ignore_errors: true
- assert:
that:
- out.rc == 4 # ansible.executor.task_queue_manager.TaskQueueManager.RUN_UNREACHABLE_HOSTS
fail_msg: out={{out}}
- name: ansible_user, ansible_ssh_pass
shell: >
ANSIBLE_ANY_ERRORS_FATAL=false
ANSIBLE_STRATEGY=mitogen_linear
ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
ansible -m shell -a whoami
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
{% endfor %}
test-targets
-e ansible_user=mitogen__has_sudo
-e ansible_ssh_pass=has_sudo_password
args:
chdir: ../..
register: out
- name: ansible_user, wrong ansible_ssh_pass
shell: >
ANSIBLE_ANY_ERRORS_FATAL=false
ANSIBLE_STRATEGY=mitogen_linear
ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
ansible -m shell -a whoami
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
{% endfor %}
test-targets
-e ansible_user=mitogen__has_sudo
-e ansible_ssh_pass=wrong_password
-e ansible_python_interpreter=python3000
args:
chdir: ../..
register: out
ignore_errors: true
- assert:
that:
- out.rc == 4 # ansible.executor.task_queue_manager.TaskQueueManager.RUN_UNREACHABLE_HOSTS
fail_msg: out={{out}}
- name: ansible_user, ansible_password
shell: >
ANSIBLE_ANY_ERRORS_FATAL=false
ANSIBLE_STRATEGY=mitogen_linear
ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
ansible -m shell -a whoami
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
{% endfor %}
test-targets
-e ansible_user=mitogen__has_sudo
-e ansible_password=has_sudo_password
args:
chdir: ../..
register: out
- name: ansible_user, wrong ansible_password
shell: >
ANSIBLE_ANY_ERRORS_FATAL=false
ANSIBLE_STRATEGY=mitogen_linear
ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
ansible -m shell -a whoami
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
{% endfor %}
test-targets
-e ansible_user=mitogen__has_sudo
-e ansible_password=wrong_password
-e ansible_python_interpreter=python3000
args:
chdir: ../..
register: out
ignore_errors: true
- assert:
that:
- out.rc == 4 # ansible.executor.task_queue_manager.TaskQueueManager.RUN_UNREACHABLE_HOSTS
fail_msg: out={{out}}
- name: setup ansible_ssh_private_key_file
shell: chmod 0600 ../data/docker/mitogen__has_sudo_pubkey.key
args:

@ -12,7 +12,6 @@
- custom_python_detect_environment:
vars:
ansible_connection: kubectl
ansible_python_interpreter: python # avoid Travis virtualenv breakage
mitogen_kubectl_path: stub-kubectl.py
register: out

@ -8,7 +8,6 @@
- custom_python_detect_environment:
vars:
ansible_connection: lxc
ansible_python_interpreter: python # avoid Travis virtualenv breakage
mitogen_lxc_attach_path: stub-lxc-attach.py
register: out

@ -8,7 +8,6 @@
- custom_python_detect_environment:
vars:
ansible_connection: lxd
ansible_python_interpreter: python # avoid Travis virtualenv breakage
mitogen_lxc_path: stub-lxc.py
register: out

@ -8,7 +8,6 @@
- custom_python_detect_environment:
vars:
ansible_connection: mitogen_doas
ansible_python_interpreter: python # avoid Travis virtualenv breakage
ansible_doas_exe: stub-doas.py
ansible_user: someuser
register: out

@ -8,7 +8,6 @@
- custom_python_detect_environment:
vars:
ansible_connection: mitogen_sudo
ansible_python_interpreter: python # avoid Travis virtualenv breakage
ansible_user: root
ansible_become_exe: stub-sudo.py
ansible_become_flags: -H --type=sometype --role=somerole

@ -2,12 +2,17 @@
# Each case is followed by mitogen_via= case to test hostvars method.
# When no ansible_python_interpreter is set, ansible 2.8+ automatically
# tries to detect the desired interpreter, falling back to "/usr/bin/python" if necessary
# If ansible_python_interpreter isn't set, Ansible 2.8+ tries to connect and
# detect the interpreter. If that fails (e.g. connection timeout)
# - Ansible 2.8 - 9 (ansible-core 2.8 - 2.16) assumes "/usr/bin/python"
# - Ansible 10+ (ansible-core 2.17+) marks the target unreachable
- name: integration/transport_config/python_path.yml
hosts: tc-python-path-unset
tasks:
- include_tasks: ../_mitogen_only.yml
- meta: end_play
when:
- ansible_version.full is version('2.17', '>=', strict=True)
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.python_path
@ -19,6 +24,9 @@
vars: {mitogen_via: tc-python-path-unset}
tasks:
- include_tasks: ../_mitogen_only.yml
- meta: end_play
when:
- ansible_version.full is version('2.17', '>=', strict=True)
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.python_path
@ -45,6 +53,9 @@
vars: {mitogen_via: tc-python-path-hostvar}
tasks:
- include_tasks: ../_mitogen_only.yml
- meta: end_play
when:
- ansible_version.full is version('2.17', '>=', strict=True)
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.python_path
@ -71,6 +82,9 @@
vars: {mitogen_via: localhost}
tasks:
- include_tasks: ../_mitogen_only.yml
- meta: end_play
when:
- ansible_version.full is version('2.17', '>=', strict=True)
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.python_path

@ -11,17 +11,6 @@ import socket
import sys
try:
all
except NameError:
# Python 2.4
def all(it):
for elem in it:
if not elem:
return False
return True
def main():
module = AnsibleModule(argument_spec={})
module.exit_json(

@ -10,11 +10,97 @@ from __future__ import absolute_import, division, print_function
__metaclass__ = type
import os
import stat
import platform
import subprocess
import sys
from ansible.module_utils.basic import AnsibleModule
# trace_realpath() and _join_tracepath() adapated from stdlib posixpath.py
# https://github.com/python/cpython/blob/v3.12.6/Lib/posixpath.py#L423-L492
# Copyright (c) 2001 - 2023 Python Software Foundation
# Copyright (c) 2024 Alex Willmer <alex@moreati.org.uk>
# License: Python Software Foundation License Version 2
def trace_realpath(filename, strict=False):
"""
Return the canonical path of the specified filename, and a trace of
the route taken, eliminating any symbolic links encountered in the path.
"""
path, trace, ok = _join_tracepath(filename[:0], filename, strict, seen={}, trace=[])
return os.path.abspath(path), trace
def _join_tracepath(path, rest, strict, seen, trace):
"""
Join two paths, normalizing and eliminating any symbolic links encountered
in the second path.
"""
trace.append(rest)
if isinstance(path, bytes):
sep = b'/'
curdir = b'.'
pardir = b'..'
else:
sep = '/'
curdir = '.'
pardir = '..'
if os.path.isabs(rest):
rest = rest[1:]
path = sep
while rest:
name, _, rest = rest.partition(sep)
if not name or name == curdir:
# current dir
continue
if name == pardir:
# parent dir
if path:
path, name = os.path.split(path)
if name == pardir:
path = os.path.join(path, pardir, pardir)
else:
path = pardir
continue
newpath = os.path.join(path, name)
try:
st = os.lstat(newpath)
except OSError:
if strict:
raise
is_link = False
else:
is_link = stat.S_ISLNK(st.st_mode)
if not is_link:
path = newpath
continue
# Resolve the symbolic link
if newpath in seen:
# Already seen this path
path = seen[newpath]
if path is not None:
# use cached value
continue
# The symlink is not resolved, so we must have a symlink loop.
if strict:
# Raise OSError(errno.ELOOP)
os.stat(newpath)
else:
# Return already resolved part + rest of the path unchanged.
return os.path.join(newpath, rest), trace, False
seen[newpath] = None # not resolved symlink
path, trace, ok = _join_tracepath(path, os.readlink(newpath), strict, seen, trace)
if not ok:
return os.path.join(path, rest), False
seen[newpath] = path # resolved symlink
return path, trace, True
def main():
module = AnsibleModule(argument_spec=dict(
facts_copy=dict(type=dict, default={}),
@ -33,7 +119,18 @@ def main():
sys.executable = "/usr/bin/python"
facts_copy = module.params['facts_copy']
discovered_interpreter_python = facts_copy['discovered_interpreter_python']
d_i_p_realpath, d_i_p_trace = trace_realpath(discovered_interpreter_python)
d_i_p_proc = subprocess.Popen(
[discovered_interpreter_python, '-c', 'import sys; print(sys.executable)'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
)
d_i_p_stdout, d_i_p_stderr = d_i_p_proc.communicate()
sys_exec_realpath, sys_exec_trace = trace_realpath(sys.executable)
result = {
'changed': False,
'ansible_facts': module.params['facts_to_override'],
@ -43,7 +140,17 @@ def main():
),
'discovered_python': {
'as_seen': discovered_interpreter_python,
'resolved': os.path.realpath(discovered_interpreter_python),
'resolved': d_i_p_realpath,
'trace': [os.path.abspath(p) for p in d_i_p_trace],
'sys': {
'executable': {
'as_seen': d_i_p_stdout.decode('ascii').rstrip('\n'),
'proc': {
'stderr': d_i_p_stderr.decode('ascii'),
'returncode': d_i_p_proc.returncode,
},
},
},
},
'running_python': {
'platform': {
@ -54,7 +161,8 @@ def main():
'sys': {
'executable': {
'as_seen': sys.executable,
'resolved': os.path.realpath(sys.executable),
'resolved': sys_exec_realpath,
'trace': [os.path.abspath(p) for p in sys_exec_trace],
},
'platform': sys.platform,
'version_info': {

@ -16,3 +16,4 @@
- import_playbook: issue_776__load_plugins_called_twice.yml
- import_playbook: issue_952__ask_become_pass.yml
- import_playbook: issue_1066__add_host__host_key_checking.yml
- import_playbook: issue_1087__template_streamerror.yml

@ -28,6 +28,10 @@
become: false
serial: 1
tasks:
# FIXME https://github.com/mitogen-hq/mitogen/issues/1096
- meta: end_play
when:
- ansible_version.full is version('2.17', '>=', strict=True)
- meta: reset_connection
# The host key might be in ~/.ssh/known_hosts. If it's removed then no

@ -0,0 +1,43 @@
- name: regression/issue_1087__template_streamerror.yml
# Ansible's template module has been seen to raise mitogen.core.StreamError
# iif there is a with_items loop and the destination path has an extension.
# This printed an error message and left file permissions incorrect,
# but did not cause the task/playbook to fail.
hosts: test-targets
gather_facts: false
become: false
vars:
foos:
- dest: /tmp/foo
- dest: /tmp/foo.txt
foo: Foo
bar: Bar
tasks:
- block:
- name: Test template does not cause StreamError
delegate_to: localhost
run_once: true
environment:
ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
command:
cmd: >
ansible-playbook
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
{% endfor %}
regression/template_test.yml
chdir: ../
register: issue_1087_cmd
failed_when:
- issue_1087_cmd is failed
or issue_1087_cmd.stdout is search('ERROR|mitogen\.core\.CallError')
or issue_1087_cmd.stderr is search('ERROR|mitogen\.core\.CallError')
always:
- name: Cleanup
file:
path: "{{ item.dest }}"
state: absent
with_items: "{{ foos }}"
tags:
- issue_1087

@ -12,7 +12,9 @@
https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}"
no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}"
PATH: "{{ lookup('env', 'PATH') }}"
shell: virtualenv /tmp/issue_152_virtualenv
command:
cmd: virtualenv -p "{{ ansible_facts.python.executable }}" /tmp/issue_152_virtualenv
creates: /tmp/issue_152_virtualenv
when:
- lout.python.version.full is version('2.7', '>=', strict=True)

@ -19,9 +19,6 @@
# Will crash if process has a nonexistent CWD.
- custom_python_os_getcwd:
script: |
import os
self._connection.get_chain().call(os.getcwd)
tags:
- issue_591
- mitogen_only

@ -11,11 +11,10 @@
tasks:
- meta: end_play
when:
# TODO CI currently runs on macOS 11 images in Azure DevOps. MacOS 11
# is no longer supported by homebrew, so the following install
# task fails.
# TODO CI currently runs on macOS 12 & which isn't supported by Podman
# version available in Homebrew.
- ansible_facts.system == 'Darwin'
- ansible_facts.distribution_major_version == '11'
- ansible_facts.distribution_version is version('13.0', '<', strict=True)
- name: set up test container and run tests inside it
block:

@ -3,28 +3,33 @@
- name: regression/issue_776__load_plugins_called_twice.yml
hosts: test-targets
become: "{{ groups.linux is defined and inventory_hostname in groups.linux }}"
gather_facts: yes
# Delayed until after the end_play, due to Python version requirements
gather_facts: false
tags:
- issue_776
vars:
ansible_python_interpreter: "{{ pkg_mgr_python_interpreter }}"
package: rsync # Chosen to exist in all tested distros/package managers
tasks:
- name: Switch to centos-stream
command: dnf --assumeyes --disablerepo="*" --enablerepo=extras swap centos-linux-repos centos-stream-repos
# The package management modules require using the same Python version
# as the target's package manager libraries. This is sometimes in conflict
# with Ansible requirements, e.g. Ansible 10 (ansible-core 2.17) does not
# support Python 2.x on targets.
- meta: end_play
when:
- ansible_facts.pkg_mgr in ["dnf"]
- ansible_version.full is version('2.17', '>=', strict=True)
- name: Switch to archive.debian.org
# Debian 9 has been archived https://lists.debian.org/debian-devel-announce/2023/03/msg00006.html
- name: Gather facts manually
setup:
- name: Switch to archived package repositories
copy:
content: |
deb http://archive.debian.org/debian stretch main contrib non-free
dest: /etc/apt/sources.list
dest: "{{ item.dest }}"
content: "{{ item.content }}"
mode: u=rw,go=r
when:
- ansible_facts.distribution == "Debian"
- ansible_facts.distribution_major_version == "9"
loop: "{{ pkg_repos_overrides }}"
loop_control:
label: "{{ item.dest }}"
- name: Add signing keys
copy:

@ -0,0 +1,28 @@
- name: regression/template_test.yml
# Ansible's template module has been seen to raise mitogen.core.StreamError
# iif there is a with_items loop and the destination path has an extension
hosts: test-targets
gather_facts: false
become: false
vars:
foos:
- dest: /tmp/foo
- dest: /tmp/foo.txt
foo: Foo
bar: Bar
tasks:
- block:
- name: Template files
template:
src: foo.bar.j2
dest: "{{ item.dest }}"
mode: u=rw,go=r
# This has to be with_items, loop: doesn't trigger the bug
with_items: "{{ foos }}"
always:
- name: Cleanup
file:
path: "{{ item.dest }}"
state: absent
with_items: "{{ foos }}"

@ -0,0 +1 @@
A {{ foo }} walks into a {{ bar }}. Ow!

@ -1,4 +1,7 @@
paramiko==2.3.2 # Last 2.6-compat version.
# Incompatible with pip >= 72, due to removal of `setup.py test`:
# ModuleNotFoundError: No module named 'setuptools.command.test'
# https://github.com/pypa/setuptools/issues/4519
hdrhistogram==0.6.1
PyYAML==3.11; python_version < '2.7'
PyYAML==5.3.1; python_version >= '2.7' # Latest release (Jan 2021)

@ -0,0 +1,39 @@
[test-targets]
{% for c in containers %}
{{ c.name }} ansible_host={{ c.hostname }} ansible_port={{ c.port }} ansible_python_interpreter={{ c.python_path }}
{% endfor %}
[test-targets:vars]
ansible_user=mitogen__has_sudo_nopw
ansible_password=has_sudo_nopw_password
{% for distro, hostnames in distros | dictsort %}
[{{ distro }}]
{% for hostname in hostnames %}
{{ hostname }}
{% endfor %}
{% endfor %}
{% for family, hostnames in families | dictsort %}
[{{ family }}]
{% for hostname in hostnames %}
{{ hostname }}
{% endfor %}
{% endfor %}
[linux:children]
test-targets
[linux_containers:children]
test-targets
[issue905]
{% for c in containers[:1] %}
ssh-common-args ansible_host={{ c.hostname }} ansible_port={{ c.port }} ansible_python_interpreter={{ c.python_path }}
{% endfor %}
[issue905:vars]
ansible_user=mitogen__has_sudo_nopw
ansible_password=has_sudo_nopw_password
ansible_ssh_common_args=-o PermitLocalCommand=yes -o LocalCommand="touch {{ '{{' }} ssh_args_canary_file {{ '}}' }}"
ssh_args_canary_file=/tmp/ssh_args_{{ '{{' }} inventory_hostname {{ '}}' }}

@ -1,5 +1,5 @@
import logging
import os
import signal
import sys
import tempfile
import threading
@ -54,7 +54,9 @@ def do_detach(econtext):
class DetachReapTest(testlib.RouterMixin, testlib.TestCase):
def test_subprocess_preserved_on_shutdown(self):
c1 = self.router.local()
c1_stream = self.router.stream_by_id(c1.context_id)
pid = c1.call(os.getpid)
self.assertEqual(pid, c1_stream.conn.proc.pid)
l = mitogen.core.Latch()
mitogen.core.listen(c1, 'disconnect', l.put)
@ -64,8 +66,8 @@ class DetachReapTest(testlib.RouterMixin, testlib.TestCase):
self.broker.shutdown()
self.broker.join()
os.kill(pid, 0) # succeeds if process still alive
self.assertIsNone(os.kill(pid, 0)) # succeeds if process still alive
# now clean up
os.kill(pid, signal.SIGTERM)
os.waitpid(pid, 0)
c1_stream.conn.proc.terminate()
c1_stream.conn.proc.proc.wait()

@ -0,0 +1,3 @@
# Setuptools 72 removed `setup.py test`. hdrhistogram 0.6.1 still depends on it.
# TODO Bump dependencies and unconstrain Pip.
setuptools<72

@ -76,6 +76,7 @@ def close_proc(proc):
proc.stdout.close()
if proc.stderr:
proc.stderr.close()
proc.proc.wait()
def wait_read(fp, n):

@ -53,4 +53,4 @@ if _system_six:
else:
from . import _six as six
six_py_file = '{0}.py'.format(os.path.splitext(six.__file__)[0])
exec(open(six_py_file, 'rb').read())
with open(six_py_file, 'rb') as f: exec(f.read())

@ -27,3 +27,6 @@ class SlaveTest(testlib.RouterMixin, testlib.TestCase):
# Subsequent master allocation does not collide
c2 = self.router.local()
self.assertEqual(1002, c2.context_id)
context.shutdown()
c2.shutdown()

@ -9,7 +9,7 @@
--change 'EXPOSE 22'
--change 'CMD ["/usr/sbin/sshd", "-D"]'
{{ inventory_hostname }}
public.ecr.aws/n5z0e8q9/{{ inventory_hostname }}-test
{{ container_image_name }}
delegate_to: localhost
- name: Stop containers

@ -4,6 +4,9 @@ common_packages:
- strace
- sudo
container_image_name: "{{ container_registry }}/{{ inventory_hostname }}-test"
container_registry: public.ecr.aws/n5z0e8q9
sudo_group:
MacOSX: admin
Debian: sudo

@ -2,11 +2,7 @@ import unittest
import mitogen.core
try:
next
except NameError:
def next(it):
return it.next()
from mitogen.core import next
class IterSplitTest(unittest.TestCase):

@ -3,10 +3,7 @@ import os
import mitogen.lxc
import mitogen.parent
try:
any
except NameError:
from mitogen.core import any
from mitogen.core import any
import testlib

@ -8,13 +8,9 @@ import unittest
import mitogen.core
import mitogen.parent
import testlib
from mitogen.core import next
try:
next
except NameError:
# Python 2.4
from mitogen.core import next
import testlib
class SockMixin(object):

@ -10,8 +10,7 @@ import mitogen.parent
class ReaperTest(testlib.TestCase):
@mock.patch('os.kill')
def test_calc_delay(self, kill):
def test_calc_delay(self):
broker = mock.Mock()
proc = mock.Mock()
proc.poll.return_value = None
@ -24,8 +23,7 @@ class ReaperTest(testlib.TestCase):
self.assertEqual(752, int(1000 * reaper._calc_delay(5)))
self.assertEqual(1294, int(1000 * reaper._calc_delay(6)))
@mock.patch('os.kill')
def test_reap_calls(self, kill):
def test_reap_calls(self):
broker = mock.Mock()
proc = mock.Mock()
proc.poll.return_value = None
@ -33,20 +31,20 @@ class ReaperTest(testlib.TestCase):
reaper = mitogen.parent.Reaper(broker, proc, True, True)
reaper.reap()
self.assertEqual(0, kill.call_count)
self.assertEqual(0, proc.send_signal.call_count)
reaper.reap()
self.assertEqual(1, kill.call_count)
self.assertEqual(1, proc.send_signal.call_count)
reaper.reap()
reaper.reap()
reaper.reap()
self.assertEqual(1, kill.call_count)
self.assertEqual(1, proc.send_signal.call_count)
reaper.reap()
self.assertEqual(2, kill.call_count)
self.assertEqual(2, proc.send_signal.call_count)
self.assertEqual(kill.mock_calls, [
mock.call(proc.pid, signal.SIGTERM),
mock.call(proc.pid, signal.SIGKILL),
self.assertEqual(proc.send_signal.mock_calls, [
mock.call(signal.SIGTERM),
mock.call(signal.SIGKILL),
])

@ -190,6 +190,7 @@ class BannerTest(testlib.DockerMixin, testlib.TestCase):
self.dockerized_ssh.port,
)
self.assertEqual(name, context.name)
context.shutdown(wait=True)
class StubPermissionDeniedTest(StubSshMixin, testlib.TestCase):

@ -51,6 +51,12 @@ except NameError:
LOG = logging.getLogger(__name__)
DISTRO = os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')
IMAGE_TEMPLATE = os.environ.get(
'MITOGEN_TEST_IMAGE_TEMPLATE',
'public.ecr.aws/n5z0e8q9/%(distro)s-test',
)
TESTS_DIR = os.path.join(os.path.dirname(__file__))
ANSIBLE_LIB_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib')
ANSIBLE_MODULE_UTILS_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils')
@ -146,6 +152,17 @@ def data_path(suffix):
return path
def retry(fn, on, max_attempts, delay):
for i in range(max_attempts):
try:
return fn()
except on:
if i >= max_attempts - 1:
raise
else:
time.sleep(delay)
def threading__thread_is_alive(thread):
"""Return whether the thread is alive (Python version compatibility shim).
@ -498,6 +515,7 @@ class TestCase(unittest.TestCase):
def get_docker_host():
# Duplicated in ci_lib
url = os.environ.get('DOCKER_HOST')
if url in (None, 'http+docker://localunixsocket'):
return 'localhost'
@ -538,19 +556,23 @@ class DockerizedSshDaemon(object):
]
subprocess.check_output(args)
def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')):
if '-' in mitogen_test_distro:
distro, _py3 = mitogen_test_distro.split('-')
else:
distro = mitogen_test_distro
_py3 = None
def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE):
# Code duplicated in ci_lib.py, both should be updated together
distro_pattern = re.compile(r'''
(?P<distro>(?P<family>[a-z]+)[0-9]+)
(?:-(?P<py>py3))?
(?:\*(?P<count>[0-9]+))?
''',
re.VERBOSE,
)
d = distro_pattern.match(distro).groupdict(default=None)
if _py3 == 'py3':
if d.pop('py') == 'py3':
self.python_path = '/usr/bin/python3'
else:
self.python_path = '/usr/bin/python'
self.image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,)
self.image = image_template % d
self.start_container()
self.host = self.get_host()
self.port = self.get_port(self.container_name)
@ -562,18 +584,24 @@ class DockerizedSshDaemon(object):
wait_for_port(self.get_host(), self.port, pattern='OpenSSH')
def check_processes(self):
args = ['docker', 'exec', self.container_name, 'ps', '-o', 'comm=']
# Get Accounting name (ucomm) & command line (args) of each process
# in the container. No truncation (-ww). No column headers (foo=).
ps_output = subprocess.check_output([
'docker', 'exec', self.container_name,
'ps', '-w', '-w', '-o', 'ucomm=', '-o', 'args=',
])
ps_lines = ps_output.decode().splitlines()
processes = [tuple(line.split(None, 1)) for line in ps_lines]
counts = {}
for comm in subprocess.check_output(args).decode().splitlines():
comm = comm.strip()
counts[comm] = counts.get(comm, 0) + 1
for ucomm, _ in processes:
counts[ucomm] = counts.get(ucomm, 0) + 1
if counts != {'ps': 1, 'sshd': 1}:
assert 0, (
'Docker container %r contained extra running processes '
'after test completed: %r' % (
self.container_name,
counts
processes,
)
)
@ -584,6 +612,9 @@ class DockerizedSshDaemon(object):
class BrokerMixin(object):
broker_class = mitogen.master.Broker
# Flag for tests that shutdown the broker themself
# e.g. unix_test.ListenerTest
broker_shutdown = False
def setUp(self):
@ -630,7 +661,12 @@ class DockerMixin(RouterMixin):
@classmethod
def tearDownClass(cls):
cls.dockerized_ssh.check_processes()
retry(
cls.dockerized_ssh.check_processes,
on=AssertionError,
max_attempts=5,
delay=0.1,
)
cls.dockerized_ssh.close()
super(DockerMixin, cls).tearDownClass()

@ -1,3 +1,6 @@
import os
import unittest
import mitogen.core
import testlib
@ -7,6 +10,10 @@ import simple_pkg.ping
# TODO: this is a joke. 2/3 interop is one of the hardest bits to get right.
# There should be 100 tests in this file.
@unittest.skipIf(
os.uname()[0] == 'Darwin' and int(os.uname()[2].partition('.')[0]) >= 21,
"Python 2.x not shipped on macOS 12.3+ (Darwin 21.4+, Monterey)",
)
class TwoThreeCompatTest(testlib.RouterMixin, testlib.TestCase):
if mitogen.core.PY3:
python_path = 'python2'

@ -65,17 +65,13 @@ class ListenerTest(testlib.RouterMixin, testlib.TestCase):
def test_constructor_basic(self):
listener = self.klass.build_stream(router=self.router)
capture = testlib.LogCapturer()
capture.start()
try:
self.assertFalse(mitogen.unix.is_path_dead(listener.protocol.path))
os.unlink(listener.protocol.path)
# ensure we catch 0 byte read error log message
self.broker.shutdown()
self.broker.join()
self.broker_shutdown = True
finally:
capture.stop()
self.assertFalse(mitogen.unix.is_path_dead(listener.protocol.path))
os.unlink(listener.protocol.path)
# ensure we catch 0 byte read error log message
self.broker.shutdown()
self.broker.join()
self.broker_shutdown = True
class ClientTest(testlib.TestCase):

@ -10,9 +10,9 @@
# 2.4 2.3? <= 3.7.1 <= 1.3.7 <= 1.1 <= 2.1.3 <= 1.4 <= 1.8
# 2.5 <= 3.7.1 <= 1.4.22 <= 1.3.1 <= 2.1.3 <= 2.8.7 <= 1.6.1 <= 1.9.1
# 2.6 <= 2.6.20 <= 2.12 <= 4.5.4 <= 1.6.11 <= 2.10.3 <= 9.0.3 <= 5.9.0 <= 3.2.5 <= 2.9.1 <= 15.2.0
# 2.7 <= 2.11 <= 5.5 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.15²
# 2.7 <= 2.11 <= 2.16 <= 5.5 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.15²
# 3.5 <= 2.11 <= 2.15 <= 5.5 <= 2.2.28 <= 2.11.3 <= 20 <= 5.9.5 <= 6.1.0 <= 3.28 <= 20.15²
# 3.6 <= 2.11 <= 6.2 <= 3.2.20 <= 3.0.3 <= 21 <= 7.0.1 <= 3.28 <= 20.17²
# 3.6 <= 2.11 <= 2.16 <= 6.2 <= 3.2.20 <= 3.0.3 <= 21 <= 7.0.1 <= 3.28 <= 20.17²
# 3.7 <= 2.12 <= 7.2.7 <= 3.2.20 <= 7.4.4 <= 4.8.0
# 3.8 <= 2.12
# 3.9 <= 2.15
@ -46,18 +46,17 @@
# ansible == 7.x ansible-core ~= 2.14.0
# ansible == 8.x ansible-core ~= 2.15.0
# ansible == 9.x ansible-core ~= 2.16.0
# ansible == 10.x ansible-core ~= 2.17.0
# pip --no-python-version-warning
# pip --disable-pip-version-check
# TODO distros=-py3
# See also
# - https://docs.ansible.com/ansible/latest/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix
[tox]
envlist =
init,
py{27,36}-mode_ansible-ansible{2.10,3,4},
py{311}-mode_ansible-ansible{2.10,3,4,5},
py{312}-mode_ansible-ansible{6,7,8,9},
py{312}-mode_ansible-ansible{6,7,8,9,10},
py{27,36,312}-mode_mitogen-distro_centos{6,7,8},
py{27,36,312}-mode_mitogen-distro_debian{9,10,11},
py{27,36,312}-mode_mitogen-distro_ubuntu{1604,1804,2004},
@ -78,21 +77,19 @@ basepython =
deps =
-r{toxinidir}/tests/requirements.txt
mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt
ansible2.10: ansible==2.10.7
ansible3: ansible==3.4.0
ansible4: ansible==4.10.0
ansible2.10: ansible~=2.10.0
ansible3: ansible~=3.0
ansible4: ansible~=4.0
ansible5: ansible~=5.0
ansible6: ansible~=6.0
ansible7: ansible~=7.0
ansible8: ansible~=8.0
ansible9: ansible~=9.0
ansible10: ansible~=10.0
install_command =
python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages}
commands_pre =
mode_ansible: {toxinidir}/.ci/ansible_install.py
mode_debops_common: {toxinidir}/.ci/debops_common_install.py
mode_localhost: {toxinidir}/.ci/localhost_ansible_install.py
mode_mitogen: {toxinidir}/.ci/mitogen_install.py
commands =
mode_ansible: {toxinidir}/.ci/ansible_tests.py
mode_debops_common: {toxinidir}/.ci/debops_common_tests.py
@ -100,15 +97,17 @@ commands =
mode_mitogen: {toxinidir}/.ci/mitogen_tests.py
passenv =
ANSIBLE_*
AWS_ACCESS_KEY_ID
AWS_DEFAULT_REGION
AWS_SECRET_ACCESS_KEY
HOME
MITOGEN_*
# Azure DevOps, TF_BUILD is set to 'True' when running in a build task
# https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables
TF_BUILD
setenv =
# See also azure-pipelines.yml
ANSIBLE_STRATEGY = mitogen_linear
NOCOVERAGE_ERASE = 1
NOCOVERAGE_REPORT = 1
PIP_CONSTRAINT={toxinidir}/tests/constraints.txt
# Only applicable to MODE=mitogen
distro_centos5: DISTRO=centos5
distro_centos6: DISTRO=centos6
@ -125,8 +124,10 @@ setenv =
ansible6: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004
ansible7: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004
ansible8: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004
# Ansible >= 9 (ansible-core >= 2.16) require Python 2.7 or >= 3.6 on targets
# Ansible 9 (ansible-core 2.16) requires Python 2.7 or >= 3.6 on targets
ansible9: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1804 ubuntu2004
# Ansible 10 (ansible-core 2.17) requires Python >= 3.7 on targets
ansible10: DISTROS=debian10-py3 debian11-py3 ubuntu2004-py3
distros_centos: DISTROS=centos6 centos7 centos8
distros_centos5: DISTROS=centos5
distros_centos6: DISTROS=centos6

Loading…
Cancel
Save