Merge branch 'master' into become_chdir

pull/1080/head
Alex Willmer 2 months ago committed by GitHub
commit 9fffb6c2f2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -28,7 +28,6 @@ for doing `setup.py install` while pulling a Docker container, for example.
### Environment Variables
* `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2.
* `DISTRO`: the `mitogen_` tests need a target Docker container distro. This
name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test`
* `DISTROS`: the `ansible_` tests can run against multiple targets

@ -1,11 +0,0 @@
#!/usr/bin/env python
import ci_lib
batches = [
[
'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
]
]
ci_lib.run_batches(batches)

@ -6,11 +6,13 @@ import glob
import os
import signal
import sys
import textwrap
import jinja2
import ci_lib
TEMPLATES_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible/templates')
TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible')
HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts')
@ -33,7 +35,7 @@ ci_lib.check_stray_processes(interesting)
with ci_lib.Fold('docker_setup'):
containers = ci_lib.make_containers()
containers = ci_lib.container_specs(ci_lib.DISTROS)
ci_lib.start_containers(containers)
@ -52,37 +54,19 @@ with ci_lib.Fold('job_setup'):
distros[container['distro']].append(container['name'])
families[container['family']].append(container['name'])
inventory_path = os.path.join(HOSTS_DIR, 'target')
with open(inventory_path, 'w') as fp:
fp.write('[test-targets]\n')
fp.writelines(
"%(name)s "
"ansible_host=%(hostname)s "
"ansible_port=%(port)s "
"ansible_python_interpreter=%(python_path)s "
"ansible_user=mitogen__has_sudo_nopw "
"ansible_password=has_sudo_nopw_password"
"\n"
% container
for container in containers
jinja_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(searchpath=TEMPLATES_DIR),
lstrip_blocks=True, # Remove spaces and tabs from before a block
trim_blocks=True, # Remove first newline after a block
)
inventory_template = jinja_env.get_template('test-targets.j2')
inventory_path = os.path.join(HOSTS_DIR, 'target')
for distro, hostnames in sorted(distros.items(), key=lambda t: t[0]):
fp.write('\n[%s]\n' % distro)
fp.writelines('%s\n' % name for name in hostnames)
for family, hostnames in sorted(families.items(), key=lambda t: t[0]):
fp.write('\n[%s]\n' % family)
fp.writelines('%s\n' % name for name in hostnames)
fp.write(textwrap.dedent(
'''
[linux:children]
test-targets
[linux_containers:children]
test-targets
'''
with open(inventory_path, 'w') as fp:
fp.write(inventory_template.render(
containers=containers,
distros=distros,
families=families,
))
ci_lib.dump_file(inventory_path)

@ -14,6 +14,19 @@ steps:
versionSpec: '$(python.version)'
condition: ne(variables['python.version'], '')
- script: |
set -o errexit
set -o nounset
set -o pipefail
aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws
displayName: Authenticate to container registry
condition: eq(variables['Agent.OS'], 'Linux')
env:
AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)
- script: |
set -o errexit
set -o nounset
@ -90,7 +103,3 @@ steps:
"$PYTHON" -m tox -e "$(tox.env)"
displayName: "Run tests"
env:
AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)

@ -16,37 +16,26 @@ trigger:
- docs-master
jobs:
- job: mac11
- job: mac12
# vanilla Ansible is really slow
timeoutInMinutes: 120
steps:
- template: azure-pipelines-steps.yml
pool:
# https://github.com/actions/runner-images/blob/main/images/macos/macos-11-Readme.md
vmImage: macOS-11
# https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md
vmImage: macOS-12
strategy:
matrix:
Mito_27:
tox.env: py27-mode_mitogen
Mito_312:
python.version: '3.12'
tox.env: py312-mode_mitogen
Loc_27_210:
tox.env: py27-mode_localhost-ansible2.10
Loc_312_9:
python.version: '3.12'
tox.env: py312-mode_localhost-ansible9
Van_27_210:
tox.env: py27-mode_localhost-ansible2.10-strategy_linear
Van_312_9:
python.version: '3.12'
tox.env: py312-mode_localhost-ansible9-strategy_linear
Loc_312_10:
tox.env: py312-mode_localhost-ansible10
Van_312_10:
tox.env: py312-mode_localhost-ansible10-strategy_linear
- job: Linux
pool:
# https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2004-Readme.md
# https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md
vmImage: ubuntu-20.04
steps:
- template: azure-pipelines-steps.yml
@ -163,3 +152,6 @@ jobs:
Ans_312_9:
python.version: '3.12'
tox.env: py312-mode_ansible-ansible9
Ans_312_10:
python.version: '3.12'
tox.env: py312-mode_ansible-ansible10

@ -27,6 +27,13 @@ os.chdir(
)
)
IMAGE_TEMPLATE = os.environ.get(
'MITOGEN_TEST_IMAGE_TEMPLATE',
'public.ecr.aws/n5z0e8q9/%(distro)s-test',
)
_print = print
def print(*args, **kwargs):
file = kwargs.get('file', sys.stdout)
@ -193,8 +200,6 @@ GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
DISTRO = os.environ.get('DISTRO', 'debian9')
# Used only when MODE=ansible
DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split()
TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2'))
BASE_PORT = 2200
TMP = TempDir().path
@ -217,6 +222,7 @@ os.environ['PYTHONPATH'] = '%s:%s' % (
def get_docker_hostname():
"""Return the hostname where the docker daemon is running.
"""
# Duplicated in testlib
url = os.environ.get('DOCKER_HOST')
if url in (None, 'http+docker://localunixsocket'):
return 'localhost'
@ -225,27 +231,34 @@ def get_docker_hostname():
return parsed.netloc.partition(':')[0]
def make_containers(name_prefix='', port_offset=0):
def container_specs(
distros,
base_port=2200,
image_template=IMAGE_TEMPLATE,
name_template='target-%(distro)s-%(index)d',
):
"""
>>> import pprint
>>> BASE_PORT=2200; DISTROS=['debian11', 'centos6']
>>> pprint.pprint(make_containers())
>>> pprint.pprint(container_specs(['debian11-py3', 'centos6']))
[{'distro': 'debian11',
'family': 'debian',
'hostname': 'localhost',
'image': 'public.ecr.aws/n5z0e8q9/debian11-test',
'index': 1,
'name': 'target-debian11-1',
'port': 2201,
'python_path': '/usr/bin/python'},
'python_path': '/usr/bin/python3'},
{'distro': 'centos6',
'family': 'centos',
'hostname': 'localhost',
'image': 'public.ecr.aws/n5z0e8q9/centos6-test',
'index': 2,
'name': 'target-centos6-2',
'port': 2202,
'python_path': '/usr/bin/python'}]
"""
docker_hostname = get_docker_hostname()
# Code duplicated in testlib.py, both should be updated together
distro_pattern = re.compile(r'''
(?P<distro>(?P<family>[a-z]+)[0-9]+)
(?:-(?P<py>py3))?
@ -256,30 +269,27 @@ def make_containers(name_prefix='', port_offset=0):
i = 1
lst = []
for distro in DISTROS:
for distro in distros:
# Code duplicated in testlib.py, both should be updated together
d = distro_pattern.match(distro).groupdict(default=None)
distro = d['distro']
family = d['family']
image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,)
if d['py'] == 'py3':
if d.pop('py') == 'py3':
python_path = '/usr/bin/python3'
else:
python_path = '/usr/bin/python'
if d['count']:
count = int(count)
else:
count = 1
count = int(d.pop('count') or '1', 10)
for x in range(count):
lst.append({
"distro": distro, "family": family, "image": image,
"name": name_prefix + ("target-%s-%s" % (distro, i)),
d['index'] = i
d.update({
'image': image_template % d,
'name': name_template % d,
"hostname": docker_hostname,
"port": BASE_PORT + i + port_offset,
'port': base_port + i,
"python_path": python_path,
})
lst.append(d)
i += 1
return lst

@ -2,16 +2,10 @@
import ci_lib
# Naturally DebOps only supports Debian.
ci_lib.DISTROS = ['debian']
ci_lib.run_batches([
[
'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"',
],
[
'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
],
])
ci_lib.run('ansible-galaxy collection install debops.debops:==2.1.2')

@ -6,9 +6,6 @@ import sys
import ci_lib
# DebOps only supports Debian.
ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT
project_dir = os.path.join(ci_lib.TMP, 'project')
vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml'
inventory_path = 'ansible/inventory/hosts'
@ -16,7 +13,11 @@ docker_hostname = ci_lib.get_docker_hostname()
with ci_lib.Fold('docker_setup'):
containers = ci_lib.make_containers(port_offset=500, name_prefix='debops-')
containers = ci_lib.container_specs(
['debian*2'],
base_port=2700,
name_template='debops-target-%(distro)s-%(index)d',
)
ci_lib.start_containers(containers)

@ -1,8 +0,0 @@
#!/usr/bin/env python
import ci_lib
batches = [
]
ci_lib.run_batches(batches)

@ -1,14 +0,0 @@
#!/usr/bin/env python
import ci_lib
batches = [
]
if ci_lib.have_docker():
batches.append([
'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
])
ci_lib.run_batches(batches)

@ -3,9 +3,6 @@
import ci_lib
batches = [
[
'aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws',
],
[
'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv',
]

@ -0,0 +1,326 @@
# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
name: Tests
on:
pull_request:
push:
branches-ignore:
- docs-master
env:
#ANSIBLE_VERBOSITY: 3
#MITOGEN_LOG_LEVEL: DEBUG
MITOGEN_TEST_IMAGE_TEMPLATE: "ghcr.io/mitogen-hq/%(distro)s-test"
# https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners
# https://github.com/actions/runner-images/blob/main/README.md#software-and-image-support
jobs:
linux:
# https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
include:
- name: Ans_27_210
tox_env: py27-mode_ansible-ansible2.10
- name: Ans_27_4
tox_env: py27-mode_ansible-ansible4
- name: Ans_36_210
python_version: '3.6'
tox_env: py36-mode_ansible-ansible2.10
- name: Ans_36_4
python_version: '3.6'
tox_env: py36-mode_ansible-ansible4
- name: Ans_311_210
python_version: '3.11'
tox_env: py311-mode_ansible-ansible2.10
- name: Ans_311_3
python_version: '3.11'
tox_env: py311-mode_ansible-ansible3
- name: Ans_311_4
python_version: '3.11'
tox_env: py311-mode_ansible-ansible4
- name: Ans_311_5
python_version: '3.11'
tox_env: py311-mode_ansible-ansible5
- name: Ans_312_6
python_version: '3.12'
tox_env: py312-mode_ansible-ansible6
- name: Ans_312_7
python_version: '3.12'
tox_env: py312-mode_ansible-ansible7
- name: Ans_312_8
python_version: '3.12'
tox_env: py312-mode_ansible-ansible8
- name: Ans_312_9
python_version: '3.12'
tox_env: py312-mode_ansible-ansible9
- name: Ans_312_10
python_version: '3.12'
tox_env: py312-mode_ansible-ansible10
- name: Van_312_10
python_version: '3.12'
tox_env: py312-mode_ansible-ansible10-strategy_linear
- name: Mito_27_centos6
tox_env: py27-mode_mitogen-distro_centos6
- name: Mito_27_centos7
tox_env: py27-mode_mitogen-distro_centos7
- name: Mito_27_centos8
tox_env: py27-mode_mitogen-distro_centos8
- name: Mito_27_debian9
tox_env: py27-mode_mitogen-distro_debian9
- name: Mito_27_debian10
tox_env: py27-mode_mitogen-distro_debian10
- name: Mito_27_debian11
tox_env: py27-mode_mitogen-distro_debian11
- name: Mito_27_ubuntu1604
tox_env: py27-mode_mitogen-distro_ubuntu1604
- name: Mito_27_ubuntu1804
tox_env: py27-mode_mitogen-distro_ubuntu1804
- name: Mito_27_ubuntu2004
tox_env: py27-mode_mitogen-distro_ubuntu2004
- name: Mito_36_centos6
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_centos6
- name: Mito_36_centos7
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_centos7
- name: Mito_36_centos8
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_centos8
- name: Mito_36_debian9
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_debian9
- name: Mito_36_debian10
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_debian10
- name: Mito_36_debian11
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_debian11
- name: Mito_36_ubuntu1604
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_ubuntu1604
- name: Mito_36_ubuntu1804
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_ubuntu1804
- name: Mito_36_ubuntu2004
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_ubuntu2004
- name: Mito_312_centos6
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_centos6
- name: Mito_312_centos7
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_centos7
- name: Mito_312_centos8
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_centos8
- name: Mito_312_debian9
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_debian9
- name: Mito_312_debian10
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_debian10
- name: Mito_312_debian11
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_debian11
- name: Mito_312_ubuntu1604
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_ubuntu1604
- name: Mito_312_ubuntu1804
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_ubuntu1804
- name: Mito_312_ubuntu2004
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_ubuntu2004
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python_version }}
if: ${{ matrix.python_version }}
- uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Install build deps
run: |
set -o errexit -o nounset -o pipefail
sudo apt-get update
sudo apt-get install -y python2-dev python3-pip virtualenv
- name: Show Python versions
run: |
set -o errexit -o nounset -o pipefail
# macOS builders lack a realpath command
type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version
type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version
type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version
echo
if [ -e /usr/bin/python ]; then
echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')"
fi
if [ -e /usr/bin/python2 ]; then
echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')"
fi
if [ -e /usr/bin/python2.7 ]; then
echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')"
fi
- name: Install tooling
run: |
set -o errexit -o nounset -o pipefail
# Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))')
if [[ -z $PYTHON ]]; then
echo 1>&2 "Python interpreter could not be determined"
exit 1
fi
if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then
"$PYTHON" -m ensurepip --user --altinstall --no-default-pip
"$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
elif [[ $PYTHON == "python2.7" ]]; then
curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py"
"$PYTHON" get-pip.py --user --no-python-version-warning
# Avoid Python 2.x pip masking system pip
rm -f ~/.local/bin/{easy_install,pip,wheel}
"$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
else
"$PYTHON" -m pip install -r "tests/requirements-tox.txt"
fi
- name: Run tests
env:
GITHUB_ACTOR: ${{ github.actor }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -o errexit -o nounset -o pipefail
# Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))')
if [[ -z $PYTHON ]]; then
echo 1>&2 "Python interpreter could not be determined"
exit 1
fi
"$PYTHON" -m tox -e "${{ matrix.tox_env }}"
macos:
# https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md
runs-on: macos-12
timeout-minutes: 120
strategy:
fail-fast: false
matrix:
include:
- name: Mito_27
tox_env: py27-mode_mitogen
- name: Mito_312
tox_env: py312-mode_mitogen
- name: Loc_27_210
tox_env: py27-mode_localhost-ansible2.10
- name: Loc_312_10
tox_env: py312-mode_localhost-ansible10
- name: Van_27_210
tox_env: py27-mode_localhost-ansible2.10-strategy_linear
- name: Van_312_10
tox_env: py312-mode_localhost-ansible10-strategy_linear
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python_version }}
if: ${{ matrix.python_version }}
- name: Show Python versions
run: |
set -o errexit -o nounset -o pipefail
# macOS builders lack a realpath command
type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version
type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version
type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version
echo
if [ -e /usr/bin/python ]; then
echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')"
fi
if [ -e /usr/bin/python2 ]; then
echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')"
fi
if [ -e /usr/bin/python2.7 ]; then
echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')"
fi
if [ -e /Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7 ]; then
# GitHub macOS 12 images: python2.7 is installed, but not on $PATH
echo "/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7: sys.executable: $(/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7 -c 'import sys; print(sys.executable)')"
fi
- name: Install tooling
run: |
set -o errexit -o nounset -o pipefail
# Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))')
if [[ -z $PYTHON ]]; then
echo 1>&2 "Python interpreter could not be determined"
exit 1
fi
if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then
# GitHub macOS 12 images: python2.7 is installed, but not on $PATH
PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7"
"$PYTHON" -m ensurepip --user --altinstall --no-default-pip
"$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
elif [[ $PYTHON == "python2.7" ]]; then
curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py"
"$PYTHON" get-pip.py --user --no-python-version-warning
# Avoid Python 2.x pip masking system pip
rm -f ~/.local/bin/{easy_install,pip,wheel}
"$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
else
"$PYTHON" -m pip install -r "tests/requirements-tox.txt"
fi
- name: Run tests
env:
GITHUB_ACTOR: ${{ github.actor }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -o errexit -o nounset -o pipefail
# Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))')
if [[ -z $PYTHON ]]; then
echo 1>&2 "Python interpreter could not be determined"
exit 1
fi
if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then
# GitHub macOS 12 images: python2.7 is installed, but not on $PATH
PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7"
fi
"$PYTHON" -m tox -e "${{ matrix.tox_env }}"

@ -83,7 +83,6 @@ import multiprocessing
import os
import struct
import mitogen.core
import mitogen.parent
@ -265,7 +264,7 @@ class LinuxPolicy(FixedPolicy):
for x in range(16):
chunks.append(struct.pack('<Q', mask & shiftmask))
mask >>= 64
return mitogen.core.b('').join(chunks)
return b''.join(chunks)
def _get_thread_ids(self):
try:

@ -1129,6 +1129,6 @@ class Connection(ansible.plugins.connection.ConnectionBase):
self.get_chain().call(
ansible_mitogen.target.transfer_file,
context=self.binding.get_child_service_context(),
in_path=in_path,
out_path=out_path
in_path=ansible_mitogen.utils.unsafe.cast(in_path),
out_path=ansible_mitogen.utils.unsafe.cast(out_path)
)

@ -49,7 +49,7 @@ __all__ = [
ANSIBLE_VERSION_MIN = (2, 10)
ANSIBLE_VERSION_MAX = (2, 16)
ANSIBLE_VERSION_MAX = (2, 17)
NEW_VERSION_MSG = (
"Your Ansible version (%s) is too recent. The most recent version\n"

@ -32,15 +32,13 @@ __metaclass__ = type
import logging
import os
import ansible.utils.display
import mitogen.core
import mitogen.utils
try:
from __main__ import display
except ImportError:
import ansible.utils.display
display = ansible.utils.display.Display()
display = ansible.utils.display.Display()
#: The process name set via :func:`set_process_name`.
_process_name = None

@ -35,18 +35,16 @@ import pwd
import random
import traceback
try:
from shlex import quote as shlex_quote
except ImportError:
from pipes import quote as shlex_quote
from ansible.module_utils._text import to_bytes
from ansible.parsing.utils.jsonify import jsonify
import ansible
import ansible.constants
import ansible.plugins
import ansible.plugins.action
import ansible.utils.unsafe_proxy
import ansible.vars.clean
from ansible.module_utils.common.text.converters import to_bytes, to_text
from ansible.module_utils.six.moves import shlex_quote
from ansible.parsing.utils.jsonify import jsonify
import mitogen.core
import mitogen.select
@ -57,24 +55,6 @@ import ansible_mitogen.target
import ansible_mitogen.utils
import ansible_mitogen.utils.unsafe
from ansible.module_utils._text import to_text
try:
from ansible.utils.unsafe_proxy import wrap_var
except ImportError:
from ansible.vars.unsafe_proxy import wrap_var
try:
# ansible 2.8 moved remove_internal_keys to the clean module
from ansible.vars.clean import remove_internal_keys
except ImportError:
try:
from ansible.vars.manager import remove_internal_keys
except ImportError:
# ansible 2.3.3 has remove_internal_keys as a protected func on the action class
# we'll fallback to calling self._remove_internal_keys in this case
remove_internal_keys = lambda a: "Not found"
LOG = logging.getLogger(__name__)
@ -280,7 +260,9 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
paths, mode, sudoable)
return self.fake_shell(lambda: mitogen.select.Select.all(
self._connection.get_chain().call_async(
ansible_mitogen.target.set_file_mode, path, mode
ansible_mitogen.target.set_file_mode,
ansible_mitogen.utils.unsafe.cast(path),
mode,
)
for path in paths
))
@ -357,7 +339,9 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
def _execute_module(self, module_name=None, module_args=None, tmp=None,
task_vars=None, persist_files=False,
delete_remote_tmp=True, wrap_async=False):
delete_remote_tmp=True, wrap_async=False,
ignore_unknown_opts=False,
):
"""
Collect up a module's execution environment then use it to invoke
target.run_module() or helpers.run_module_async() in the target
@ -370,6 +354,12 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
if task_vars is None:
task_vars = {}
if ansible_mitogen.utils.ansible_version[:2] >= (2, 17):
self._update_module_args(
module_name, module_args, task_vars,
ignore_unknown_opts=ignore_unknown_opts,
)
else:
self._update_module_args(module_name, module_args, task_vars)
env = {}
self._compute_environment_string(env)
@ -403,10 +393,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
self._remove_tmp_path(tmp)
# prevents things like discovered_interpreter_* or ansible_discovered_interpreter_* from being set
# handle ansible 2.3.3 that has remove_internal_keys in a different place
check = remove_internal_keys(result)
if check == 'Not found':
self._remove_internal_keys(result)
ansible.vars.clean.remove_internal_keys(result)
# taken from _execute_module of ansible 2.8.6
# propagate interpreter discovery results back to the controller
@ -430,7 +417,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase):
result['deprecations'] = []
result['deprecations'].extend(self._discovery_deprecation_warnings)
return wrap_var(result)
return ansible.utils.unsafe_proxy.wrap_var(result)
def _postprocess_response(self, result):
"""

@ -54,6 +54,7 @@ import mitogen.select
import ansible_mitogen.loaders
import ansible_mitogen.parsing
import ansible_mitogen.target
import ansible_mitogen.utils.unsafe
LOG = logging.getLogger(__name__)
@ -215,12 +216,15 @@ class ScriptPlanner(BinaryPlanner):
"""
def _rewrite_interpreter(self, path):
"""
Given the original interpreter binary extracted from the script's
interpreter line, look up the associated `ansible_*_interpreter`
variable, render it and return it.
Given the interpreter path (from the script's hashbang line), return
the desired interpreter path. This tries, in order
1. Look up & render the `ansible_*_interpreter` variable, if set
2. Look up the `discovered_interpreter_*` fact, if present
3. The unmodified path from the hashbang line.
:param str path:
Absolute UNIX path to original interpreter.
Absolute path to original interpreter (e.g. '/usr/bin/python').
:returns:
Shell fragment prefix used to execute the script via "/bin/sh -c".
@ -228,13 +232,25 @@ class ScriptPlanner(BinaryPlanner):
involved here, the vanilla implementation uses it and that use is
exploited in common playbooks.
"""
key = u'ansible_%s_interpreter' % os.path.basename(path).strip()
interpreter_name = os.path.basename(path).strip()
key = u'ansible_%s_interpreter' % interpreter_name
try:
template = self._inv.task_vars[key]
except KeyError:
return path
pass
else:
configured_interpreter = self._inv.templar.template(template)
return ansible_mitogen.utils.unsafe.cast(configured_interpreter)
key = u'discovered_interpreter_%s' % interpreter_name
try:
discovered_interpreter = self._inv.task_vars['ansible_facts'][key]
except KeyError:
pass
else:
return ansible_mitogen.utils.unsafe.cast(discovered_interpreter)
return mitogen.utils.cast(self._inv.templar.template(template))
return path
def _get_interpreter(self):
path, arg = ansible_mitogen.parsing.parse_hashbang(
@ -249,7 +265,8 @@ class ScriptPlanner(BinaryPlanner):
if arg:
fragment += ' ' + arg
return fragment, path.startswith('python')
is_python = path.startswith('python')
return fragment, is_python
def get_kwargs(self, **kwargs):
interpreter_fragment, is_python = self._get_interpreter()
@ -460,7 +477,7 @@ def read_file(path):
finally:
os.close(fd)
return mitogen.core.b('').join(bits)
return b''.join(bits)
def _propagate_deps(invocation, planner, context):

@ -42,13 +42,7 @@ except ImportError:
import ansible_mitogen.connection
import ansible_mitogen.process
if sys.version_info > (3,):
viewkeys = dict.keys
elif sys.version_info > (2, 7):
viewkeys = dict.viewkeys
else:
viewkeys = lambda dct: set(dct)
viewkeys = getattr(dict, 'viewkeys', dict.keys)
def dict_diff(old, new):

@ -61,10 +61,9 @@ import mitogen.utils
import ansible
import ansible.constants as C
import ansible.errors
import ansible_mitogen.logging
import ansible_mitogen.services
from mitogen.core import b
import ansible_mitogen.affinity
@ -639,7 +638,7 @@ class MuxProcess(object):
try:
# Let the parent know our listening socket is ready.
mitogen.core.io_op(self.model.child_sock.send, b('1'))
mitogen.core.io_op(self.model.child_sock.send, b'1')
# Block until the socket is closed, which happens on parent exit.
mitogen.core.io_op(self.model.child_sock.recv, 1)
finally:

@ -40,7 +40,9 @@ from __future__ import absolute_import, division, print_function
__metaclass__ = type
import atexit
import ctypes
import json
import logging
import os
import re
import shlex
@ -50,19 +52,12 @@ import tempfile
import traceback
import types
from ansible.module_utils.six.moves import shlex_quote
import mitogen.core
import ansible_mitogen.target # TODO: circular import
from mitogen.core import b
from mitogen.core import bytes_partition
from mitogen.core import str_rpartition
from mitogen.core import to_text
try:
import ctypes
except ImportError:
# Python 2.4
ctypes = None
try:
# Python >= 3.4, PEP 451 ModuleSpec API
import importlib.machinery
@ -77,15 +72,6 @@ try:
except ImportError:
from io import StringIO
try:
from shlex import quote as shlex_quote
except ImportError:
from pipes import quote as shlex_quote
# Absolute imports for <2.5.
logging = __import__('logging')
# Prevent accidental import of an Ansible module from hanging on stdin read.
import ansible.module_utils.basic
ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
@ -95,7 +81,6 @@ ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
# explicit call to res_init() on each task invocation. BSD-alikes export it
# directly, Linux #defines it as "__res_init".
libc__res_init = None
if ctypes:
libc = ctypes.CDLL(None)
for symbol in 'res_init', '__res_init':
try:
@ -103,7 +88,6 @@ if ctypes:
except AttributeError:
pass
iteritems = getattr(dict, 'iteritems', dict.items)
LOG = logging.getLogger(__name__)
@ -217,13 +201,13 @@ class EnvironmentFileWatcher(object):
for line in fp:
# ' #export foo=some var ' -> ['#export', 'foo=some var ']
bits = shlex_split_b(line)
if (not bits) or bits[0].startswith(b('#')):
if (not bits) or bits[0].startswith(b'#'):
continue
if bits[0] == b('export'):
if bits[0] == b'export':
bits.pop(0)
key, sep, value = bytes_partition(b(' ').join(bits), b('='))
key, sep, value = b' '.join(bits).partition(b'=')
if key and sep:
yield key, value
@ -601,7 +585,7 @@ class ModuleUtilsImporter(object):
mod.__path__ = []
mod.__package__ = str(fullname)
else:
mod.__package__ = str(str_rpartition(to_text(fullname), '.')[0])
mod.__package__ = str(to_text(fullname).rpartition('.')[0])
exec(code, mod.__dict__)
self._loaded.add(fullname)
return mod
@ -616,7 +600,7 @@ class TemporaryEnvironment(object):
def __init__(self, env=None):
self.original = dict(os.environ)
self.env = env or {}
for key, value in iteritems(self.env):
for key, value in mitogen.core.iteritems(self.env):
key = mitogen.core.to_text(key)
value = mitogen.core.to_text(value)
if value is None:
@ -824,7 +808,7 @@ class ScriptRunner(ProgramRunner):
self.interpreter_fragment = interpreter_fragment
self.is_python = is_python
b_ENCODING_STRING = b('# -*- coding: utf-8 -*-')
b_ENCODING_STRING = b'# -*- coding: utf-8 -*-'
def _get_program(self):
return self._rewrite_source(
@ -857,13 +841,13 @@ class ScriptRunner(ProgramRunner):
# While Ansible rewrites the #! using ansible_*_interpreter, it is
# never actually used to execute the script, instead it is a shell
# fragment consumed by shell/__init__.py::build_module_command().
new = [b('#!') + utf8(self.interpreter_fragment)]
new = [b'#!' + utf8(self.interpreter_fragment)]
if self.is_python:
new.append(self.b_ENCODING_STRING)
_, _, rest = bytes_partition(s, b('\n'))
_, _, rest = s.partition(b'\n')
new.append(rest)
return b('\n').join(new)
return b'\n'.join(new)
class NewStyleRunner(ScriptRunner):
@ -976,8 +960,7 @@ class NewStyleRunner(ScriptRunner):
# change the default encoding. This hack was removed from Ansible long ago,
# but not before permeating into many third party modules.
PREHISTORIC_HACK_RE = re.compile(
b(r'reload\s*\(\s*sys\s*\)\s*'
r'sys\s*\.\s*setdefaultencoding\([^)]+\)')
br'reload\s*\(\s*sys\s*\)\s*sys\s*\.\s*setdefaultencoding\([^)]+\)',
)
def _setup_program(self):
@ -985,7 +968,7 @@ class NewStyleRunner(ScriptRunner):
context=self.service_context,
path=self.path,
)
self.source = self.PREHISTORIC_HACK_RE.sub(b(''), source)
self.source = self.PREHISTORIC_HACK_RE.sub(b'', source)
def _get_code(self):
try:
@ -1003,7 +986,7 @@ class NewStyleRunner(ScriptRunner):
if mitogen.core.PY3:
main_module_name = '__main__'
else:
main_module_name = b('__main__')
main_module_name = b'__main__'
def _handle_magic_exception(self, mod, exc):
"""
@ -1035,7 +1018,7 @@ class NewStyleRunner(ScriptRunner):
approximation of the original package hierarchy, so that relative
imports function correctly.
"""
pkg, sep, modname = str_rpartition(self.py_module_name, '.')
pkg, sep, _ = self.py_module_name.rpartition('.')
if not sep:
return None
if mitogen.core.PY3:
@ -1078,7 +1061,7 @@ class NewStyleRunner(ScriptRunner):
class JsonArgsRunner(ScriptRunner):
JSON_ARGS = b('<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>')
JSON_ARGS = b'<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>'
def _get_args_contents(self):
return json.dumps(self.args).encode()

@ -50,6 +50,8 @@ import threading
import ansible.constants
from ansible.module_utils.six import reraise
import mitogen.core
import mitogen.service
import ansible_mitogen.loaders
@ -66,20 +68,6 @@ LOG = logging.getLogger(__name__)
ansible_mitogen.loaders.shell_loader.get('sh')
if sys.version_info[0] == 3:
def reraise(tp, value, tb):
if value is None:
value = tp()
if value.__traceback__ is not tb:
raise value.with_traceback(tb)
raise value
else:
exec(
"def reraise(tp, value, tb=None):\n"
" raise tp, value, tb\n"
)
def _get_candidate_temp_dirs():
try:
# >=2.5

@ -39,6 +39,7 @@ __metaclass__ = type
import errno
import grp
import json
import logging
import operator
import os
import pwd
@ -51,26 +52,9 @@ import tempfile
import traceback
import types
# Absolute imports for <2.5.
logging = __import__('logging')
import mitogen.core
import mitogen.parent
import mitogen.service
from mitogen.core import b
try:
reduce
except NameError:
# Python 3.x.
from functools import reduce
try:
BaseException
except NameError:
# Python 2.4
BaseException = Exception
# Ansible since PR #41749 inserts "import __main__" into
# ansible.module_utils.basic. Mitogen's importer will refuse such an import, so
@ -80,6 +64,9 @@ if not sys.modules.get(str('__main__')):
sys.modules[str('__main__')] = types.ModuleType(str('__main__'))
import ansible.module_utils.json_utils
from ansible.module_utils.six.moves import reduce
import ansible_mitogen.runner
@ -615,8 +602,8 @@ def exec_args(args, in_data='', chdir=None, shell=None, emulate_tty=False):
stdout, stderr = proc.communicate(in_data)
if emulate_tty:
stdout = stdout.replace(b('\n'), b('\r\n'))
return proc.returncode, stdout, stderr or b('')
stdout = stdout.replace(b'\n', b'\r\n')
return proc.returncode, stdout, stderr or b''
def exec_command(cmd, in_data='', chdir=None, shell=None, emulate_tty=False):
@ -746,9 +733,7 @@ def set_file_mode(path, spec, fd=None):
"""
Update the permissions of a file using the same syntax as chmod(1).
"""
if isinstance(spec, int):
new_mode = spec
elif not mitogen.core.PY3 and isinstance(spec, long):
if isinstance(spec, mitogen.core.integer_types):
new_mode = spec
elif spec.isdigit():
new_mode = int(spec, 8)

@ -65,21 +65,12 @@ import abc
import os
import ansible.utils.shlex
import ansible.constants as C
import ansible.executor.interpreter_discovery
import ansible.utils.unsafe_proxy
from ansible.module_utils.six import with_metaclass
from ansible.module_utils.parsing.convert_bool import boolean
# this was added in Ansible >= 2.8.0; fallback to the default interpreter if necessary
try:
from ansible.executor.interpreter_discovery import discover_interpreter
except ImportError:
discover_interpreter = lambda action,interpreter_name,discovery_mode,task_vars: '/usr/bin/python'
try:
from ansible.utils.unsafe_proxy import AnsibleUnsafeText
except ImportError:
from ansible.vars.unsafe_proxy import AnsibleUnsafeText
import mitogen.core
@ -115,12 +106,13 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth
action._finding_python_interpreter = True
# fake pipelining so discover_interpreter can be happy
action._connection.has_pipelining = True
s = AnsibleUnsafeText(discover_interpreter(
s = ansible.executor.interpreter_discovery.discover_interpreter(
action=action,
interpreter_name=interpreter_name,
discovery_mode=s,
task_vars=task_vars))
task_vars=task_vars,
)
s = ansible.utils.unsafe_proxy.AnsibleUnsafeText(s)
# cache discovered interpreter
task_vars['ansible_facts'][discovered_interpreter_config] = s
action._connection.has_pipelining = False
@ -498,12 +490,13 @@ class PlayContextSpec(Spec):
)
def ssh_args(self):
local_vars = self._task_vars.get("hostvars", {}).get(self._inventory_name, {})
return [
mitogen.core.to_text(term)
for s in (
C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=local_vars),
C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=local_vars),
C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=local_vars)
)
for term in ansible.utils.shlex.shlex_split(s or '')
]
@ -738,12 +731,13 @@ class MitogenViaSpec(Spec):
)
def ssh_args(self):
local_vars = self._task_vars.get("hostvars", {}).get(self._inventory_name, {})
return [
mitogen.core.to_text(term)
for s in (
C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=local_vars),
C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=local_vars),
C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=local_vars)
)
for term in ansible.utils.shlex.shlex_split(s)
if s

@ -1,14 +0,0 @@
<!doctype html>
<title>Mitogen for Ansible (Redirect)</title>
<script>
{% include "piwik-config.js" %}
var u="https://networkgenomics.com/p/tr/";
_paq.push(['setTrackerUrl', u+'ep']);
</script>
<script src="https://networkgenomics.com/p/tr/js"></script>
<script>
setTimeout(function() {
window.location = 'https://networkgenomics.com/ansible/';
}, 0);
</script>
<meta http-equiv="Refresh" content="0; url=https://networkgenomics.com/ansible/">

@ -14,23 +14,5 @@
{% block footer %}
{{ super() }}
<script>
(function() {
{% include "piwik-config.js" %}
var u="https://networkgenomics.com/p/tr/";
_paq.push(['setTrackerUrl', u+'ep']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; g.type='text/javascript';
g.defer=true; g.async=true; g.src=u+'js'; s.parentNode.insertBefore(g,s);
})();
</script>
<noscript>
<p>
{% set fulltitle = (title|striptags|e) + titlesuffix -%}
<img src="https://networkgenomics.com/p/tr/ep?idsite=6&action_name={{fulltitle}}" style="border:0" alt="">
</p>
</noscript>
<script async defer src="https://buttons.github.io/buttons.js"></script>
{% endblock %}

@ -1,5 +0,0 @@
window._paq = [];
window._paq.push(['trackPageView']);
window._paq.push(['enableLinkTracking']);
window._paq.push(['enableHeartBeatTimer', 30]);
window._paq.push(['setSiteId', 6]);

@ -75,34 +75,6 @@ Installation
``mitogen_host_pinned`` strategies exists to mimic the ``free`` and
``host_pinned`` strategies.
4.
.. raw:: html
<form action="https://networkgenomics.com/save-email/" method="post" id="emailform">
<input type=hidden name="list_name" value="mitogen-announce">
Get notified of new releases and important fixes.
<p>
<input type="email" placeholder="E-mail Address" name="email" style="font-size: 105%;"><br>
<input name="captcha_1" placeholder="Captcha" style="width: 10ex;">
<img class="captcha-image">
<a class="captcha-refresh" href="#">&#x21bb</a>
<button type="submit" style="font-size: 105%;">
Subscribe
</button>
</p>
<div id="emailthanks" style="display:none">
Thanks!
</div>
<p>
</form>
Demo
~~~~
@ -165,7 +137,9 @@ Noteworthy Differences
+-----------------+-----------------+
| 8 | 3.9 - 3.12 |
+-----------------+-----------------+
| 9 | 3.10 - 3.12 |
| 9 | |
+-----------------+ 3.10 - 3.12 |
| 10 | |
+-----------------+-----------------+
Verify your installation is running one of these versions by checking
@ -271,15 +245,14 @@ Noteworthy Differences
* "Module Replacer" style modules are not supported. These rarely appear in
practice, and light web searches failed to reveal many examples of them.
..
* The ``ansible_python_interpreter`` variable is parsed using a restrictive
:mod:`shell-like <shlex>` syntax, permitting values such as ``/usr/bin/env
FOO=bar python`` or ``source /opt/rh/rh-python36/enable && python``, which
occur in practice. Jinja2 templating is also supported for complex task-level
interpreter settings. Ansible `documents this
FOO=bar python`` or ``source /opt/rh/rh-python36/enable && python``.
Jinja2 templating is also supported for complex task-level
interpreter settings. Ansible documents `ansible_python_interpreter
<https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html#ansible-python-interpreter>`_
as an absolute path, however the implementation passes it unquoted through
the shell, permitting arbitrary code to be injected.
as an absolute path and releases since June 2024 (e.g. Ansible 10.1)
reflect this. Older Ansible releases passed it to the shell unquoted.
..
* Configurations will break that rely on the `hashbang argument splitting
@ -1418,20 +1391,3 @@ Despite the small margin for optimization, Mitogen still manages **6.2x less
bandwidth and 1.8x less time**.
.. image:: images/ansible/pcaps/costapp-uk-india.svg
.. raw:: html
<script src="https://networkgenomics.com/static/js/public_all.js?92d49a3a"></script>
<script>
NetGen = {
public: {
page_id: "operon",
urls: {
save_email: "https://networkgenomics.com/save-email/",
save_email_captcha: "https://networkgenomics.com/save-email/captcha/",
}
}
};
setupEmailForm();
</script>

@ -22,10 +22,49 @@ Unreleased
----------
* :gh:issue:`636` os.chdir fails if the sudo/become user lacks adequate permissions to chdir prior to task
* :gh:issue:`1127` :mod:`mitogen`: Consolidate mitogen backward compatibility
fallbacks and polyfills into :mod:`mitogen.core`
* :gh:issue:`1127` :mod:`ansible_mitogen`: Remove backward compatibility
fallbacks for Python 2.4 & 2.5.
* :gh:issue:`1127` :mod:`ansible_mitogen`: Remove fallback imports for Ansible
releases before 2.10
* :gh:issue:`1127` :mod:`ansible_mitogen`: Consolidate Python 2 & 3
compatibility
* :gh:issue:`1128` CI: Start migration from Azure DevOps to GitHub Actions
v0.3.10 (2024-09-20)
--------------------
* :gh:issue:`950` Fix Solaris/Illumos/SmartOS compatibility with become
* :gh:issue:`1087` Fix :exc:`mitogen.core.StreamError` when Ansible template
module is called with a ``dest:`` filename that has an extension
* :gh:issue:`1110` Fix :exc:`mitogen.core.StreamError` when Ansible copy
module is called with a file larger than 124 kibibytes
(:data:`ansible_mitogen.connection.Connection.SMALL_FILE_LIMIT`)
* :gh:issue:`905` Initial support for templated ``ansible_ssh_args``,
``ansible_ssh_common_args``, and ``ansible_ssh_extra_args`` variables.
NB: play or task scoped variables will probably still fail.
* :gh:issue:`694` CI: Fixed a race condition and some resource leaks causing
some of intermittent failures when running the test suite.
v0.3.9 (2024-08-13)
-------------------
* :gh:issue:`1097` Respect `ansible_facts.discovered_interpreter_python` when
executing non new-style modules (e.g. JSONARGS style, WANT_JSON style).
* :gh:issue:`1074` Support Ansible 10 (ansible-core 2.17)
v0.3.8 (2024-07-30)
-------------------
* :gh:issue:`952` Fix Ansible `--ask-become-pass`, add test coverage
* :gh:issue:`957` Fix Ansible exception when executing against 10s of hosts
"ValueError: filedescriptor out of range in select()"
* :gh:issue:`1066` Support Ansible `ansible_host_key_checking` & `ansible_ssh_host_key_checking`
* :gh:issue:`1090` CI: Migrate macOS integration tests to macOS 12, drop Python 2.7 jobs
v0.3.7 (2024-04-08)

@ -2,7 +2,7 @@ import sys
sys.path.append('.')
VERSION = '0.3.7'
VERSION = '0.3.9'
author = u'Network Genomics'
copyright = u'2021, the Mitogen authors'
@ -16,7 +16,6 @@ html_show_copyright = False
html_show_sourcelink = False
html_show_sphinx = False
html_sidebars = {'**': ['globaltoc.html', 'github.html']}
html_additional_pages = {'ansible': 'ansible.html'}
html_static_path = ['_static']
html_theme = 'alabaster'
html_theme_options = {

@ -116,6 +116,7 @@ sponsorship and outstanding future-thinking of its early adopters.
<ul>
<li>Alex Willmer</li>
<li><a href="https://github.com/momiji">Christian Bourgeois </a></li>
<li><a href="https://underwhelm.net/">Dan Dorman</a> &mdash; - <em>When I truly understand my enemy … then in that very moment I also love him.</em></li>
<li>Daniel Foerster</li>
<li><a href="https://www.deps.co/">Deps</a> &mdash; <em>Private Maven Repository Hosting for Java, Scala, Groovy, Clojure</em></li>
@ -125,6 +126,7 @@ sponsorship and outstanding future-thinking of its early adopters.
<li><a href="https://www.channable.com">rkrzr</a></li>
<li>jgadling</li>
<li>John F Wall &mdash; <em>Making Ansible Great with Massive Parallelism</em></li>
<li><a href="https://github.com/jrosser">Jonathan Rosser</a></li>
<li>KennethC</li>
<li><a href="https://github.com/lberruti">Luca Berruti</li>
<li>Lewis Bellwood &mdash; <em>Happy to be apart of a great project.</em></li>

@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup.
#: Library version as a tuple.
__version__ = (0, 3, 8, 'dev')
__version__ = (0, 3, 11, 'dev')
#: This is :data:`False` in slave contexts. Previously it was used to prevent

@ -102,21 +102,6 @@ try:
except ImportError:
cProfile = None
try:
import thread
except ImportError:
import threading as thread
try:
import cPickle as pickle
except ImportError:
import pickle
try:
from cStringIO import StringIO as BytesIO
except ImportError:
from io import BytesIO
try:
BaseException
except NameError:
@ -169,31 +154,35 @@ STUB_CALL_SERVICE = 111
#: :meth:`mitogen.core.Router.add_handler` callbacks to clean up.
IS_DEAD = 999
try:
BaseException
except NameError:
BaseException = Exception
PY24 = sys.version_info < (2, 5)
PY3 = sys.version_info > (3,)
if PY3:
import pickle
import _thread as thread
from io import BytesIO
b = str.encode
BytesType = bytes
UnicodeType = str
FsPathTypes = (str,)
BufferType = lambda buf, start: memoryview(buf)[start:]
long = int
integer_types = (int,)
iteritems, iterkeys, itervalues = dict.items, dict.keys, dict.values
else:
import cPickle as pickle
import thread
from cStringIO import StringIO as BytesIO
b = str
BytesType = str
FsPathTypes = (str, unicode)
BufferType = buffer
UnicodeType = unicode
integer_types = (int, long)
iteritems, iterkeys, itervalues = dict.iteritems, dict.iterkeys, dict.itervalues
AnyTextType = (BytesType, UnicodeType)
try:
next
next = next
except NameError:
next = lambda it: it.next()
@ -400,12 +389,19 @@ now = getattr(time, 'monotonic', time.time)
# Python 2.4
try:
any
all, any = all, any
except NameError:
def all(it):
for elem in it:
if not elem:
return False
return True
def any(it):
for elem in it:
if elem:
return True
return False
def _partition(s, sep, find):
@ -1065,8 +1061,8 @@ class Sender(object):
def _unpickle_sender(router, context_id, dst_handle):
if not (isinstance(router, Router) and
isinstance(context_id, (int, long)) and context_id >= 0 and
isinstance(dst_handle, (int, long)) and dst_handle > 0):
isinstance(context_id, integer_types) and context_id >= 0 and
isinstance(dst_handle, integer_types) and dst_handle > 0):
raise TypeError('cannot unpickle Sender: bad input or missing router')
return Sender(Context(router, context_id), dst_handle)
@ -2508,7 +2504,7 @@ class Context(object):
def _unpickle_context(context_id, name, router=None):
if not (isinstance(context_id, (int, long)) and context_id >= 0 and (
if not (isinstance(context_id, integer_types) and context_id >= 0 and (
(name is None) or
(isinstance(name, UnicodeType) and len(name) < 100))
):

@ -74,9 +74,11 @@ import mitogen.core
import mitogen.minify
import mitogen.parent
from mitogen.core import any
from mitogen.core import b
from mitogen.core import IOLOG
from mitogen.core import LOG
from mitogen.core import next
from mitogen.core import str_partition
from mitogen.core import str_rpartition
from mitogen.core import to_text
@ -84,17 +86,6 @@ from mitogen.core import to_text
imap = getattr(itertools, 'imap', map)
izip = getattr(itertools, 'izip', zip)
try:
any
except NameError:
from mitogen.core import any
try:
next
except NameError:
from mitogen.core import next
RLOG = logging.getLogger('mitogen.ctx')

@ -56,15 +56,13 @@ import zlib
# Absolute imports for <2.5.
select = __import__('select')
try:
import thread
except ImportError:
import threading as thread
import mitogen.core
from mitogen.core import b
from mitogen.core import bytes_partition
from mitogen.core import IOLOG
from mitogen.core import itervalues
from mitogen.core import next
from mitogen.core import thread
LOG = logging.getLogger(__name__)
@ -80,15 +78,6 @@ except IOError:
SELINUX_ENABLED = False
try:
next
except NameError:
# Python 2.4/2.5
from mitogen.core import next
itervalues = getattr(dict, 'itervalues', dict.values)
if mitogen.core.PY3:
xrange = range
closure_attr = '__closure__'
@ -147,6 +136,8 @@ LINUX_TIOCGPTN = _ioctl_cast(2147767344)
LINUX_TIOCSPTLCK = _ioctl_cast(1074025521)
IS_LINUX = os.uname()[0] == 'Linux'
IS_SOLARIS = os.uname()[0] == 'SunOS'
SIGNAL_BY_NUM = dict(
(getattr(signal, name), name)
@ -411,7 +402,7 @@ def _acquire_controlling_tty():
# On Linux, the controlling tty becomes the first tty opened by a
# process lacking any prior tty.
os.close(os.open(os.ttyname(2), os.O_RDWR))
if hasattr(termios, 'TIOCSCTTY') and not mitogen.core.IS_WSL:
if hasattr(termios, 'TIOCSCTTY') and not mitogen.core.IS_WSL and not IS_SOLARIS:
# #550: prehistoric WSL does not like TIOCSCTTY.
# On BSD an explicit ioctl is required. For some inexplicable reason,
# Python 2.6 on Travis also requires it.
@ -479,6 +470,7 @@ def openpty():
master_fp = os.fdopen(master_fd, 'r+b', 0)
slave_fp = os.fdopen(slave_fd, 'r+b', 0)
if not IS_SOLARIS:
disable_echo(master_fd)
disable_echo(slave_fd)
mitogen.core.set_block(slave_fd)
@ -2542,7 +2534,7 @@ class Reaper(object):
# because it is setuid, so this is best-effort only.
LOG.debug('%r: sending %s', self.proc, SIGNAL_BY_NUM[signum])
try:
os.kill(self.proc.pid, signum)
self.proc.send_signal(signum)
except OSError:
e = sys.exc_info()[1]
if e.args[0] != errno.EPERM:
@ -2662,6 +2654,17 @@ class Process(object):
"""
raise NotImplementedError()
def send_signal(self, sig):
os.kill(self.pid, sig)
def terminate(self):
"Ask the process to gracefully shutdown."
self.send_signal(signal.SIGTERM)
def kill(self):
"Ask the operating system to forcefully destroy the process."
self.send_signal(signal.SIGKILL)
class PopenProcess(Process):
"""
@ -2678,6 +2681,9 @@ class PopenProcess(Process):
def poll(self):
return self.proc.poll()
def send_signal(self, sig):
self.proc.send_signal(sig)
class ModuleForwarder(object):
"""

@ -39,18 +39,10 @@ import threading
import mitogen.core
import mitogen.select
from mitogen.core import all
from mitogen.core import b
from mitogen.core import str_rpartition
try:
all
except NameError:
def all(it):
for elem in it:
if not elem:
return False
return True
LOG = logging.getLogger(__name__)

@ -43,11 +43,6 @@ except ImportError:
import mitogen.parent
from mitogen.core import b
try:
any
except NameError:
from mitogen.core import any
LOG = logging.getLogger(__name__)

@ -34,11 +34,6 @@ import re
import mitogen.core
import mitogen.parent
try:
any
except NameError:
from mitogen.core import any
LOG = logging.getLogger(__name__)

@ -143,19 +143,23 @@ class Listener(mitogen.core.Protocol):
def on_accept_client(self, sock):
sock.setblocking(True)
try:
pid, = struct.unpack('>L', sock.recv(4))
data = sock.recv(4)
pid, = struct.unpack('>L', data)
except (struct.error, socket.error):
LOG.error('listener: failed to read remote identity: %s',
sys.exc_info()[1])
LOG.error('listener: failed to read remote identity, got %d bytes: %s',
len(data), sys.exc_info()[1])
sock.close()
return
context_id = self._router.id_allocator.allocate()
try:
# FIXME #1109 send() returns number of bytes sent, check it
sock.send(struct.pack('>LLL', context_id, mitogen.context_id,
os.getpid()))
except socket.error:
LOG.error('listener: failed to assign identity to PID %d: %s',
pid, sys.exc_info()[1])
sock.close()
return
context = mitogen.parent.Context(self._router, context_id)

@ -37,13 +37,7 @@ import sys
import mitogen.core
import mitogen.master
iteritems = getattr(dict, 'iteritems', dict.items)
if mitogen.core.PY3:
iteritems = dict.items
else:
iteritems = dict.iteritems
from mitogen.core import iteritems
def setup_gil():

@ -1,6 +1,9 @@
# code: language=ini
# vim: syntax=dosini
[become_same_user]
# become_same_user.yml
bsu-joe ansible_user=joe
[become_same_user:vars]
ansible_python_interpreter=python3000

@ -4,7 +4,7 @@
# Connection delegation scenarios. It's impossible to connect to them, but their would-be
# config can be inspected using "mitogen_get_stack" action.
[cd]
# Normal inventory host, no aliasing.
cd-normal ansible_connection=mitogen_doas ansible_user=normal-user
@ -23,6 +23,8 @@ cd-newuser-normal-normal mitogen_via=cd-normal ansible_user=newuser-normal-norma
# doas:newuser via host.
cd-newuser-doas-normal mitogen_via=cd-normal ansible_connection=mitogen_doas ansible_user=newuser-doas-normal-user
[cd:vars]
ansible_python_interpreter = python3000
[connection-delegation-test]
cd-bastion

@ -12,3 +12,10 @@ target ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}"
target
[linux_containers]
[issue905]
ssh-common-args ansible_host=localhost ansible_user="{{ lookup('pipe', 'whoami') }}"
[issue905:vars]
ansible_ssh_common_args=-o PermitLocalCommand=yes -o LocalCommand="touch {{ ssh_args_canary_file }}"
ssh_args_canary_file=/tmp/ssh_args_{{ inventory_hostname }}

@ -1,2 +1,3 @@
---
pkg_mgr_python_interpreter: python
pkg_repos_overrides: []

@ -1,2 +1,28 @@
---
pkg_mgr_python_interpreter: /usr/libexec/platform-python
pkg_repos_overrides:
- dest: /etc/yum.repos.d/CentOS-Linux-AppStream.repo
content: |
[appstream]
name=CentOS Linux $releasever - AppStream
baseurl=http://vault.centos.org/$contentdir/$releasever/AppStream/$basearch/os/
enabled=1
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial
- dest: /etc/yum.repos.d/CentOS-Linux-BaseOS.repo
content: |
[baseos]
name=CentOS Linux $releasever - BaseOS
baseurl=http://vault.centos.org/$contentdir/$releasever/BaseOS/$basearch/os/
enabled=1
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial
- dest: /etc/yum.repos.d/CentOS-Linux-Extras.repo
content: |
[extras]
name=CentOS Linux $releasever - Extras
baseurl=http://vault.centos.org/$contentdir/$releasever/extras/$basearch/os/
enabled=1
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial

@ -0,0 +1,4 @@
pkg_repos_overrides:
- dest: /etc/apt/sources.list
content: |
deb http://archive.debian.org/debian stretch main contrib non-free

@ -1,92 +1,94 @@
# Verify copy module for small and large files, and inline content.
# To exercise https://github.com/mitogen-hq/mitogen/pull/1110 destination
# files must have extensions and loops must use `with_items:`.
- name: integration/action/copy.yml
hosts: test-targets
tasks:
- name: Create tiny file
copy:
dest: /tmp/copy-tiny-file
content:
this is a tiny file.
delegate_to: localhost
run_once: true
vars:
sourced_files:
- src: /tmp/copy-tiny-file
dest: /tmp/copy-tiny-file.out
content: this is a tiny file.
expected_checksum: f29faa9a6f19a700a941bf2aa5b281643c4ec8a0
- src: /tmp/copy-large-file
dest: /tmp/copy-large-file.out
content: "{{ 'x' * 200000 }}"
expected_checksum: 62951f943c41cdd326e5ce2b53a779e7916a820d
inline_files:
- dest: /tmp/copy-tiny-inline-file.out
content: tiny inline content
expected_checksum: b26dd6444595e2bdb342aa0a91721b57478b5029
- dest: /tmp/copy-large-inline-file.out
content: |
{{ 'y' * 200000 }}
expected_checksum: d675f47e467eae19e49032a2cc39118e12a6ee72
- name: Create large file
files: "{{ sourced_files + inline_files }}"
tasks:
- name: Create sourced files
copy:
dest: /tmp/copy-large-file
# Must be larger than Connection.SMALL_SIZE_LIMIT.
content: "{% for x in range(200000) %}x{% endfor %}"
dest: "{{ item.src }}"
content: "{{ item.content }}"
mode: u=rw,go=r
with_items: "{{ sourced_files }}"
loop_control:
label: "{{ item.src }}"
delegate_to: localhost
run_once: true
- name: Cleanup copied files
- name: Cleanup lingering destination files
file:
path: "{{ item.dest }}"
state: absent
path: "{{item}}"
with_items:
- /tmp/copy-tiny-file.out
- /tmp/copy-large-file.out
- /tmp/copy-tiny-inline-file.out
- /tmp/copy-large-inline-file.out
with_items: "{{ files }}"
loop_control:
label: "{{ item.dest }}"
- name: Copy large file
- name: Copy sourced files
copy:
dest: /tmp/copy-large-file.out
src: /tmp/copy-large-file
- name: Copy tiny file
copy:
dest: /tmp/copy-tiny-file.out
src: /tmp/copy-tiny-file
src: "{{ item.src }}"
dest: "{{ item.dest }}"
mode: u=rw,go=r
with_items: "{{ sourced_files }}"
loop_control:
label: "{{ item.dest }}"
- name: Copy tiny inline file
- name: Copy inline files
copy:
dest: /tmp/copy-tiny-inline-file.out
content: "tiny inline content"
- name: Copy large inline file
copy:
dest: /tmp/copy-large-inline-file.out
content: |
{% for x in range(200000) %}y{% endfor %}
dest: "{{ item.dest }}"
content: "{{ item.content }}"
mode: u=rw,go=r
with_items: "{{ inline_files }}"
loop_control:
label: "{{ item.dest }}"
# stat results
- name: Stat copied files
stat:
path: "{{item}}"
with_items:
- /tmp/copy-tiny-file.out
- /tmp/copy-large-file.out
- /tmp/copy-tiny-inline-file.out
- /tmp/copy-large-inline-file.out
path: "{{ item.dest }}"
with_items: "{{ files }}"
loop_control:
label: "{{ item.dest }}"
register: stat
- assert:
that:
- stat.results[0].stat.checksum == "f29faa9a6f19a700a941bf2aa5b281643c4ec8a0"
- stat.results[1].stat.checksum == "62951f943c41cdd326e5ce2b53a779e7916a820d"
- stat.results[2].stat.checksum == "b26dd6444595e2bdb342aa0a91721b57478b5029"
- stat.results[3].stat.checksum == "d675f47e467eae19e49032a2cc39118e12a6ee72"
fail_msg: stat={{stat}}
- item.stat.checksum == item.item.expected_checksum
quiet: true # Avoid spamming stdout with 400 kB of item.item.content
fail_msg: item={{ item }}
with_items: "{{ stat.results }}"
loop_control:
label: "{{ item.stat.path }}"
- name: Cleanup files
- name: Cleanup destination files
file:
path: "{{ item.dest }}"
state: absent
path: "{{item}}"
with_items:
- /tmp/copy-tiny-file
- /tmp/copy-tiny-file.out
- /tmp/copy-no-mode
- /tmp/copy-no-mode.out
- /tmp/copy-with-mode
- /tmp/copy-with-mode.out
- /tmp/copy-large-file
- /tmp/copy-large-file.out
- /tmp/copy-tiny-inline-file.out
- /tmp/copy-large-inline-file
- /tmp/copy-large-inline-file.out
# end of cleaning out files (again)
with_items: "{{ files }}"
loop_control:
label: "{{ item.dest }}"
tags:
- copy
- issue_1110

@ -53,20 +53,22 @@
vars:
ansible_become_pass: user1_password
when:
# https://github.com/ansible/ansible/pull/70785
- ansible_facts.distribution not in ["MacOSX"]
or ansible_version.full is version("2.11", ">=", strict=True)
or is_mitogen
# CI containers lack `setfacl` for unpriv -> unpriv
# https://github.com/mitogen-hq/mitogen/issues/1118
- is_mitogen
or (ansible_facts.distribution in ["MacOSX"]
and ansible_version.full is version("2.11", ">=", strict=True))
- assert:
that:
- out.stdout == 'mitogen__user1'
fail_msg: out={{out}}
when:
# https://github.com/ansible/ansible/pull/70785
- ansible_facts.distribution not in ["MacOSX"]
or ansible_version.full is version("2.11", ">=", strict=True)
or is_mitogen
# CI containers lack `setfacl` for unpriv -> unpriv
# https://github.com/mitogen-hq/mitogen/issues/1118
- is_mitogen
or (ansible_facts.distribution in ["MacOSX"]
and ansible_version.full is version("2.11", ">=", strict=True))
- name: Ensure password su without chdir succeeds
shell: whoami
@ -76,20 +78,22 @@
vars:
ansible_become_pass: user1_password
when:
# https://github.com/ansible/ansible/pull/70785
- ansible_facts.distribution not in ["MacOSX"]
or ansible_version.full is version("2.11", ">=", strict=True)
or is_mitogen
# CI containers lack `setfacl` for unpriv -> unpriv
# https://github.com/mitogen-hq/mitogen/issues/1118
- is_mitogen
or (ansible_facts.distribution in ["MacOSX"]
and ansible_version.full is version("2.11", ">=", strict=True))
- assert:
that:
- out.stdout == 'mitogen__user1'
fail_msg: out={{out}}
when:
# https://github.com/ansible/ansible/pull/70785
- ansible_facts.distribution not in ["MacOSX"]
or ansible_version.full is version("2.11", ">=", strict=True)
or is_mitogen
# CI containers lack `setfacl` for unpriv -> unpriv
# https://github.com/mitogen-hq/mitogen/issues/1118
- is_mitogen
or (ansible_facts.distribution in ["MacOSX"]
and ansible_version.full is version("2.11", ">=", strict=True))
tags:
- su

@ -21,8 +21,7 @@
# sudo-1.8.6p3-29.el6_10.3 on RHEL & CentOS 6.10 (final release)
# removed user/group error messages, as defence against CVE-2019-14287.
- >-
'sudo: unknown user: slartibartfast' in out.module_stdout | default(out.msg)
or 'sudo: unknown user: slartibartfast' in out.module_stderr | default(out.msg)
(out.module_stderr | default(out.module_stdout, true) | default(out.msg, true)) is search('sudo: unknown user:? slartibartfast')
or (ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_version == '6.10')
fail_msg: out={{out}}
when:

@ -40,7 +40,7 @@
'keepalive_count': 10,
'password': null,
'port': null,
'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"],
'python_path': ['python3000'],
'remote_name': null,
'ssh_args': [
-o, ControlMaster=auto,
@ -68,7 +68,7 @@
'keepalive_count': 10,
'password': null,
'port': null,
'python_path': ["{{ ansible_facts.discovered_interpreter_python | default('/usr/bin/python') }}"],
'python_path': ['python3000'],
'remote_name': null,
'ssh_args': [
-o, ControlMaster=auto,

@ -19,7 +19,7 @@
'kind': 'lxc',
'lxc_info_path': null,
'machinectl_path': null,
'python_path': ['/usr/bin/python'],
'python_path': ['python3000'],
'remote_name': null,
'username': null,
},

@ -23,7 +23,7 @@
'lxc_info_path': null,
'lxc_path': null,
'machinectl_path': null,
'python_path': ["/usr/bin/python"],
'python_path': ["python3000"],
'username': 'ansible-cfg-remote-user',
},
'method': 'setns',

@ -40,7 +40,7 @@
"connect_timeout": 30,
"doas_path": null,
"password": null,
"python_path": ["/usr/bin/python"],
"python_path": ["python3000"],
'remote_name': null,
"username": "normal-user",
},
@ -73,7 +73,7 @@
'keepalive_count': 10,
'password': null,
'port': null,
"python_path": ["/usr/bin/python"],
"python_path": ["python3000"],
'remote_name': null,
'ssh_args': [
-o, ControlMaster=auto,
@ -115,7 +115,7 @@
'keepalive_count': 10,
'password': null,
'port': null,
"python_path": ["/usr/bin/python"],
"python_path": ["python3000"],
'remote_name': null,
'ssh_args': [
-o, ControlMaster=auto,
@ -150,7 +150,7 @@
'connect_timeout': 30,
'doas_path': null,
'password': null,
"python_path": ["/usr/bin/python"],
"python_path": ["python3000"],
'remote_name': null,
'username': 'normal-user',
},
@ -168,7 +168,7 @@
'keepalive_count': 10,
'password': null,
'port': null,
"python_path": ["/usr/bin/python"],
"python_path": ["python3000"],
'remote_name': null,
'ssh_args': [
-o, ControlMaster=auto,
@ -210,7 +210,7 @@
'keepalive_count': 10,
'password': null,
'port': null,
"python_path": ["/usr/bin/python"],
"python_path": ["python3000"],
'remote_name': null,
'ssh_args': [
-o, ControlMaster=auto,
@ -238,7 +238,7 @@
'keepalive_count': 10,
'password': null,
'port': null,
"python_path": ["/usr/bin/python"],
"python_path": ["python3000"],
'remote_name': null,
'ssh_args': [
-o, ControlMaster=auto,
@ -272,7 +272,7 @@
'connect_timeout': 30,
'doas_path': null,
'password': null,
"python_path": ["/usr/bin/python"],
"python_path": ["python3000"],
'remote_name': null,
'username': 'normal-user',
},
@ -290,7 +290,7 @@
'keepalive_count': 10,
'password': null,
'port': null,
"python_path": ["/usr/bin/python"],
"python_path": ["python3000"],
'remote_name': null,
'ssh_args': [
-o, ControlMaster=auto,
@ -333,7 +333,7 @@
'keepalive_count': 10,
'password': null,
'port': null,
"python_path": ["/usr/bin/python"],
"python_path": ["python3000"],
'remote_name': null,
'ssh_args': [
-o, ControlMaster=auto,
@ -388,7 +388,7 @@
'connect_timeout': 30,
'doas_path': null,
'password': null,
'python_path': ["/usr/bin/python"],
'python_path': ["python3000"],
'remote_name': null,
'username': 'normal-user',
},
@ -399,7 +399,7 @@
'connect_timeout': 30,
'doas_path': null,
'password': null,
'python_path': ["/usr/bin/python"],
'python_path': ["python3000"],
'remote_name': null,
'username': 'newuser-doas-normal-user',
},

@ -4,6 +4,55 @@
- name: integration/interpreter_discovery/ansible_2_8_tests.yml
hosts: test-targets
gather_facts: true
vars:
DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_lt_2_12:
centos:
'6': /usr/bin/python
'7': /usr/bin/python
'8': /usr/libexec/platform-python
debian:
'9': /usr/bin/python
'10': /usr/bin/python3
'11': /usr/bin/python
'NA': /usr/bin/python # Debian 11, Ansible <= 7 (ansible-core <= 2.14)
'bullseye/sid': /usr/bin/python # Debian 11, Ansible 8 - 9 (ansible-core 2.15 - 2.16)
ubuntu:
'16': /usr/bin/python3
'18': /usr/bin/python3
'20': /usr/bin/python3
DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_2_12_to_2_16:
centos:
'6': /usr/bin/python
'7': /usr/bin/python
'8': /usr/libexec/platform-python
debian:
'9': /usr/bin/python
'10': /usr/bin/python3
'11': /usr/bin/python3.9
'NA': /usr/bin/python3.9 # Debian 11, Ansible <= 7 (ansible-core <= 2.14)
'bullseye/sid': /usr/bin/python3.9 # Debian 11, Ansible 8 - 9 (ansible-core 2.15 - 2.16)
ubuntu:
'16': /usr/bin/python3
'18': /usr/bin/python3
'20': /usr/bin/python3
DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_ge_2_17:
debian:
'10': /usr/bin/python3.7
'11': /usr/bin/python3.9
'bullseye/sid': /usr/bin/python3.9
ubuntu:
'20': /usr/bin/python3.8
discovered_interpreter_expected: >-
{%- if ansible_version.full is version('2.12', '<', strict=True) -%}
{{ DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_lt_2_12[distro][distro_major] }}
{%- elif ansible_version.full is version('2.17', '<', strict=True) -%}
{{ DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_2_12_to_2_16[distro][distro_major] }}
{%- else -%}
{{ DISCOVERED_INTERPRETER_EXPECTED_MAP__ANSIBLE_ge_2_17[distro][distro_major] }}
{%- endif -%}
tasks:
- name: can only run these tests on ansible >= 2.8.0
block:
@ -18,9 +67,9 @@
- name: snag some facts to validate for later
set_fact:
distro: '{{ ansible_distribution | default("unknown") | lower }}'
distro_version: '{{ ansible_distribution_version | default("unknown") }}'
os_family: '{{ ansible_os_family | default("unknown") }}'
distro: '{{ ansible_facts.distribution | lower }}'
distro_major: '{{ ansible_facts.distribution_major_version }}'
system: '{{ ansible_facts.system }}'
- name: test that python discovery is working and that fact persistence makes it only run once
block:
@ -50,7 +99,7 @@
that:
- auto_out.ansible_facts.discovered_interpreter_python is defined
- auto_out.ansible_facts.discovered_interpreter_python == echoout.discovered_python.as_seen
- echoout.discovered_python.resolved == echoout.running_python.sys.executable.resolved
- echoout.discovered_python.sys.executable.as_seen == echoout.running_python.sys.executable.as_seen
fail_msg:
- "auto_out: {{ auto_out }}"
- "echoout: {{ echoout }}"
@ -159,50 +208,19 @@
- ansible_facts['ansible_bogus_interpreter'] | default('nope') == 'nope'
- ansible_facts['discovered_interpreter_bogus'] | default('nope') == 'nope'
- name: fedora assertions
assert:
that:
- auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python3'
fail_msg: auto_out={{auto_out}}
when:
- distro == 'fedora'
- distro_version is version('23.0', '>=', strict=True)
- name: rhel < 8 assertions
assert:
that:
- auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python'
fail_msg: auto_out={{auto_out}}
when:
- distro in ('redhat', 'centos')
- distro_version is version('8.0', '<', strict=true)
- name: rhel 8+ assertions
assert:
that:
- auto_out.ansible_facts.discovered_interpreter_python == '/usr/libexec/platform-python'
fail_msg: auto_out={{auto_out}}
when:
- distro in ('redhat', 'centos')
- distro_version is version('8.0', '>=', strict=true)
- name: ubuntu < 16.04 assertions
assert:
that:
- auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python'
fail_msg: auto_out={{auto_out}}
when:
- distro == 'ubuntu'
- distro_version is version('16.04', '<', strict=true)
- name: ubuntu 16.04+ assertions
- name: Check discovered interpreter matches expected
assert:
that:
- auto_out.ansible_facts.discovered_interpreter_python == '/usr/bin/python3'
fail_msg: auto_out={{auto_out}}
- auto_out.ansible_facts.discovered_interpreter_python == discovered_interpreter_expected
fail_msg: >-
distro={{ distro }}
distro_major= {{ distro_major }}
system={{ system }}
auto_out={{ auto_out }}
discovered_interpreter_expected={{ discovered_interpreter_expected }}
ansible_version.full={{ ansible_version.full }}
when:
- distro == 'ubuntu'
- distro_version is version('16.04', '>=', strict=True)
- system in ['Linux']
always:
- meta: clear_facts

@ -9,6 +9,13 @@
https_proxy: "{{ lookup('env', 'https_proxy') | default(omit) }}"
no_proxy: "{{ lookup('env', 'no_proxy') | default(omit) }}"
tasks:
# Ansible releases after June 2024 quote ansible_python_interpreter
# https://github.com/ansible/ansible/pull/83365
- meta: end_play
when:
- not is_mitogen
- ansible_version.full is version('2.17.1', '>=', strict=True)
- name: create temp file to source
file:
path: /tmp/fake
@ -25,8 +32,7 @@
# special_python: source /tmp/fake && python
- name: set python using sourced file
set_fact:
# Avoid 2.x vs 3.x cross-compatiblity issues (that I can't remember the exact details of).
special_python: "source /tmp/fake || true && python{{ ansible_facts.python.version.major }}"
special_python: "source /tmp/fake || true && {{ ansible_facts.python.executable }}"
- name: run get_url with specially-sourced python
uri:

@ -1,7 +1,7 @@
# external2 is loaded from config path.
# external1 is loaded from integration/module_utils/roles/modrole/module_utils/..
- name: integration/module_utils/adjacent_to_playbook.yml
- name: integration/module_utils/adjacent_to_role.yml
hosts: test-targets
roles:
- modrole

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/python
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils import external3

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/python
import json
import ansible.module_utils.basic

@ -2,6 +2,12 @@
- name: integration/runner/custom_bash_hashbang_argument.yml
hosts: test-targets
tasks:
# Ansible releases after June 2024 quote ansible_python_interpreter
# https://github.com/ansible/ansible/pull/83365
- meta: end_play
when:
- not is_mitogen
- ansible_version.full is version('2.17.1', '>=', strict=True)
- custom_bash_old_style_module:
foo: true

@ -1,3 +1,5 @@
- import_playbook: args.yml
- import_playbook: config.yml
- import_playbook: password.yml
- import_playbook: timeouts.yml
- import_playbook: variables.yml

@ -0,0 +1,48 @@
- name: integration/ssh/args.yml
hosts: issue905
gather_facts: false
tasks:
# Test that ansible_ssh_common_args are templated; ansible_ssh_args &
# ansible_ssh_extra_args aren't directly tested, we assume they're similar.
# FIXME This test currently relies on variables set in the host group.
# Ideally they'd be set here, and the host group eliminated, but
# Mitogen currently fails to template when defined in the play.
# TODO Replace LocalCommand canary with SetEnv canary, to simplify test.
# Requires modification of sshd_config files to add AcceptEnv ...
- name: Test templating of ansible_ssh_common_args et al
block:
- name: Ensure no lingering canary files
file:
path: "{{ ssh_args_canary_file }}"
state: absent
delegate_to: localhost
- name: Reset connections to force new ssh execution
meta: reset_connection
- name: Perform SSH connection, to trigger side effect
ping:
# LocalCommand="touch {{ ssh_args_canary_file }}" in ssh_*_args
- name: Stat for canary file created by side effect
stat:
path: "{{ ssh_args_canary_file }}"
delegate_to: localhost
register: ssh_args_canary_stat
- assert:
that:
- ssh_args_canary_stat.stat.exists == true
quiet: true
success_msg: "Canary found: {{ ssh_args_canary_file }}"
fail_msg: |
ssh_args_canary_file={{ ssh_args_canary_file }}
ssh_args_canary_stat={{ ssh_args_canary_stat }}
always:
- name: Cleanup canary files
file:
path: "{{ ssh_args_canary_file }}"
state: absent
delegate_to: localhost
tags:
- issue_905

@ -0,0 +1,51 @@
- name: integration/ssh/password.yml
hosts: test-targets[0]
gather_facts: false
vars:
ansible_user: mitogen__user1
tasks:
- meta: reset_connection
- name: ansible_password
vars:
ansible_password: user1_password
ping:
- meta: reset_connection
- name: ansible_ssh_pass
vars:
ansible_ssh_pass: user1_password
ping:
- meta: reset_connection
- name: absent password should fail
ping:
ignore_errors: true
ignore_unreachable: true
register: ssh_no_password_result
- assert:
that:
- ssh_no_password_result.unreachable == True
fail_msg: ssh_no_password_result={{ ssh_no_password_result }}
- meta: reset_connection
- name: ansible_ssh_pass should override ansible_password
ping:
vars:
ansible_password: wrong
ansible_ssh_pass: user1_password
# Tests that ansible_ssh_pass has priority over ansible_password
# and that a wrong password causes a target to be marked unreachable.
- meta: reset_connection
- name: ansible_password should not override
vars:
ansible_password: user1_password
ansible_ssh_pass: wrong
ping:
ignore_errors: true
ignore_unreachable: true
register: ssh_wrong_password_result
- assert:
that:
- ssh_wrong_password_result.unreachable == True
fail_msg: ssh_wrong_password_result={{ ssh_wrong_password_result }}

@ -13,134 +13,6 @@
-o "ControlPath /tmp/mitogen-ansible-test-{{18446744073709551615|random}}"
tasks:
- include_tasks: ../_mitogen_only.yml
- name: ansible_ssh_user, ansible_ssh_pass
shell: >
ANSIBLE_ANY_ERRORS_FATAL=false
ANSIBLE_STRATEGY=mitogen_linear
ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
ansible -m shell -a whoami
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
{% endfor %}
test-targets
-e ansible_ssh_user=mitogen__has_sudo
-e ansible_ssh_pass=has_sudo_password
args:
chdir: ../..
register: out
- name: ansible_ssh_user, wrong ansible_ssh_pass
shell: >
ANSIBLE_ANY_ERRORS_FATAL=false
ANSIBLE_STRATEGY=mitogen_linear
ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
ansible -m shell -a whoami
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
{% endfor %}
test-targets
-e ansible_ssh_user=mitogen__has_sudo
-e ansible_ssh_pass=wrong_password
-e ansible_python_interpreter=python3000
args:
chdir: ../..
register: out
ignore_errors: true
- assert:
that:
- out.rc == 4 # ansible.executor.task_queue_manager.TaskQueueManager.RUN_UNREACHABLE_HOSTS
fail_msg: out={{out}}
- name: ansible_user, ansible_ssh_pass
shell: >
ANSIBLE_ANY_ERRORS_FATAL=false
ANSIBLE_STRATEGY=mitogen_linear
ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
ansible -m shell -a whoami
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
{% endfor %}
test-targets
-e ansible_user=mitogen__has_sudo
-e ansible_ssh_pass=has_sudo_password
args:
chdir: ../..
register: out
- name: ansible_user, wrong ansible_ssh_pass
shell: >
ANSIBLE_ANY_ERRORS_FATAL=false
ANSIBLE_STRATEGY=mitogen_linear
ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
ansible -m shell -a whoami
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
{% endfor %}
test-targets
-e ansible_user=mitogen__has_sudo
-e ansible_ssh_pass=wrong_password
-e ansible_python_interpreter=python3000
args:
chdir: ../..
register: out
ignore_errors: true
- assert:
that:
- out.rc == 4 # ansible.executor.task_queue_manager.TaskQueueManager.RUN_UNREACHABLE_HOSTS
fail_msg: out={{out}}
- name: ansible_user, ansible_password
shell: >
ANSIBLE_ANY_ERRORS_FATAL=false
ANSIBLE_STRATEGY=mitogen_linear
ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
ansible -m shell -a whoami
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
{% endfor %}
test-targets
-e ansible_user=mitogen__has_sudo
-e ansible_password=has_sudo_password
args:
chdir: ../..
register: out
- name: ansible_user, wrong ansible_password
shell: >
ANSIBLE_ANY_ERRORS_FATAL=false
ANSIBLE_STRATEGY=mitogen_linear
ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
ansible -m shell -a whoami
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
{% endfor %}
test-targets
-e ansible_user=mitogen__has_sudo
-e ansible_password=wrong_password
-e ansible_python_interpreter=python3000
args:
chdir: ../..
register: out
ignore_errors: true
- assert:
that:
- out.rc == 4 # ansible.executor.task_queue_manager.TaskQueueManager.RUN_UNREACHABLE_HOSTS
fail_msg: out={{out}}
- name: setup ansible_ssh_private_key_file
shell: chmod 0600 ../data/docker/mitogen__has_sudo_pubkey.key
args:

@ -12,7 +12,6 @@
- custom_python_detect_environment:
vars:
ansible_connection: kubectl
ansible_python_interpreter: python # avoid Travis virtualenv breakage
mitogen_kubectl_path: stub-kubectl.py
register: out

@ -8,7 +8,6 @@
- custom_python_detect_environment:
vars:
ansible_connection: lxc
ansible_python_interpreter: python # avoid Travis virtualenv breakage
mitogen_lxc_attach_path: stub-lxc-attach.py
register: out

@ -8,7 +8,6 @@
- custom_python_detect_environment:
vars:
ansible_connection: lxd
ansible_python_interpreter: python # avoid Travis virtualenv breakage
mitogen_lxc_path: stub-lxc.py
register: out

@ -8,7 +8,6 @@
- custom_python_detect_environment:
vars:
ansible_connection: mitogen_doas
ansible_python_interpreter: python # avoid Travis virtualenv breakage
ansible_doas_exe: stub-doas.py
ansible_user: someuser
register: out

@ -8,7 +8,6 @@
- custom_python_detect_environment:
vars:
ansible_connection: mitogen_sudo
ansible_python_interpreter: python # avoid Travis virtualenv breakage
ansible_user: root
ansible_become_exe: stub-sudo.py
ansible_become_flags: -H --type=sometype --role=somerole

@ -2,12 +2,17 @@
# Each case is followed by mitogen_via= case to test hostvars method.
# When no ansible_python_interpreter is set, ansible 2.8+ automatically
# tries to detect the desired interpreter, falling back to "/usr/bin/python" if necessary
# If ansible_python_interpreter isn't set, Ansible 2.8+ tries to connect and
# detect the interpreter. If that fails (e.g. connection timeout)
# - Ansible 2.8 - 9 (ansible-core 2.8 - 2.16) assumes "/usr/bin/python"
# - Ansible 10+ (ansible-core 2.17+) marks the target unreachable
- name: integration/transport_config/python_path.yml
hosts: tc-python-path-unset
tasks:
- include_tasks: ../_mitogen_only.yml
- meta: end_play
when:
- ansible_version.full is version('2.17', '>=', strict=True)
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.python_path
@ -19,6 +24,9 @@
vars: {mitogen_via: tc-python-path-unset}
tasks:
- include_tasks: ../_mitogen_only.yml
- meta: end_play
when:
- ansible_version.full is version('2.17', '>=', strict=True)
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.python_path
@ -45,6 +53,9 @@
vars: {mitogen_via: tc-python-path-hostvar}
tasks:
- include_tasks: ../_mitogen_only.yml
- meta: end_play
when:
- ansible_version.full is version('2.17', '>=', strict=True)
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.python_path
@ -71,6 +82,9 @@
vars: {mitogen_via: localhost}
tasks:
- include_tasks: ../_mitogen_only.yml
- meta: end_play
when:
- ansible_version.full is version('2.17', '>=', strict=True)
- {mitogen_get_stack: {}, register: out}
- assert_equal:
left: out.result[0].kwargs.python_path

@ -11,17 +11,6 @@ import socket
import sys
try:
all
except NameError:
# Python 2.4
def all(it):
for elem in it:
if not elem:
return False
return True
def main():
module = AnsibleModule(argument_spec={})
module.exit_json(

@ -10,11 +10,97 @@ from __future__ import absolute_import, division, print_function
__metaclass__ = type
import os
import stat
import platform
import subprocess
import sys
from ansible.module_utils.basic import AnsibleModule
# trace_realpath() and _join_tracepath() adapated from stdlib posixpath.py
# https://github.com/python/cpython/blob/v3.12.6/Lib/posixpath.py#L423-L492
# Copyright (c) 2001 - 2023 Python Software Foundation
# Copyright (c) 2024 Alex Willmer <alex@moreati.org.uk>
# License: Python Software Foundation License Version 2
def trace_realpath(filename, strict=False):
"""
Return the canonical path of the specified filename, and a trace of
the route taken, eliminating any symbolic links encountered in the path.
"""
path, trace, ok = _join_tracepath(filename[:0], filename, strict, seen={}, trace=[])
return os.path.abspath(path), trace
def _join_tracepath(path, rest, strict, seen, trace):
"""
Join two paths, normalizing and eliminating any symbolic links encountered
in the second path.
"""
trace.append(rest)
if isinstance(path, bytes):
sep = b'/'
curdir = b'.'
pardir = b'..'
else:
sep = '/'
curdir = '.'
pardir = '..'
if os.path.isabs(rest):
rest = rest[1:]
path = sep
while rest:
name, _, rest = rest.partition(sep)
if not name or name == curdir:
# current dir
continue
if name == pardir:
# parent dir
if path:
path, name = os.path.split(path)
if name == pardir:
path = os.path.join(path, pardir, pardir)
else:
path = pardir
continue
newpath = os.path.join(path, name)
try:
st = os.lstat(newpath)
except OSError:
if strict:
raise
is_link = False
else:
is_link = stat.S_ISLNK(st.st_mode)
if not is_link:
path = newpath
continue
# Resolve the symbolic link
if newpath in seen:
# Already seen this path
path = seen[newpath]
if path is not None:
# use cached value
continue
# The symlink is not resolved, so we must have a symlink loop.
if strict:
# Raise OSError(errno.ELOOP)
os.stat(newpath)
else:
# Return already resolved part + rest of the path unchanged.
return os.path.join(newpath, rest), trace, False
seen[newpath] = None # not resolved symlink
path, trace, ok = _join_tracepath(path, os.readlink(newpath), strict, seen, trace)
if not ok:
return os.path.join(path, rest), False
seen[newpath] = path # resolved symlink
return path, trace, True
def main():
module = AnsibleModule(argument_spec=dict(
facts_copy=dict(type=dict, default={}),
@ -33,7 +119,18 @@ def main():
sys.executable = "/usr/bin/python"
facts_copy = module.params['facts_copy']
discovered_interpreter_python = facts_copy['discovered_interpreter_python']
d_i_p_realpath, d_i_p_trace = trace_realpath(discovered_interpreter_python)
d_i_p_proc = subprocess.Popen(
[discovered_interpreter_python, '-c', 'import sys; print(sys.executable)'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
)
d_i_p_stdout, d_i_p_stderr = d_i_p_proc.communicate()
sys_exec_realpath, sys_exec_trace = trace_realpath(sys.executable)
result = {
'changed': False,
'ansible_facts': module.params['facts_to_override'],
@ -43,7 +140,17 @@ def main():
),
'discovered_python': {
'as_seen': discovered_interpreter_python,
'resolved': os.path.realpath(discovered_interpreter_python),
'resolved': d_i_p_realpath,
'trace': [os.path.abspath(p) for p in d_i_p_trace],
'sys': {
'executable': {
'as_seen': d_i_p_stdout.decode('ascii').rstrip('\n'),
'proc': {
'stderr': d_i_p_stderr.decode('ascii'),
'returncode': d_i_p_proc.returncode,
},
},
},
},
'running_python': {
'platform': {
@ -54,7 +161,8 @@ def main():
'sys': {
'executable': {
'as_seen': sys.executable,
'resolved': os.path.realpath(sys.executable),
'resolved': sys_exec_realpath,
'trace': [os.path.abspath(p) for p in sys_exec_trace],
},
'platform': sys.platform,
'version_info': {

@ -16,3 +16,4 @@
- import_playbook: issue_776__load_plugins_called_twice.yml
- import_playbook: issue_952__ask_become_pass.yml
- import_playbook: issue_1066__add_host__host_key_checking.yml
- import_playbook: issue_1087__template_streamerror.yml

@ -28,6 +28,10 @@
become: false
serial: 1
tasks:
# FIXME https://github.com/mitogen-hq/mitogen/issues/1096
- meta: end_play
when:
- ansible_version.full is version('2.17', '>=', strict=True)
- meta: reset_connection
# The host key might be in ~/.ssh/known_hosts. If it's removed then no

@ -0,0 +1,43 @@
- name: regression/issue_1087__template_streamerror.yml
# Ansible's template module has been seen to raise mitogen.core.StreamError
# iif there is a with_items loop and the destination path has an extension.
# This printed an error message and left file permissions incorrect,
# but did not cause the task/playbook to fail.
hosts: test-targets
gather_facts: false
become: false
vars:
foos:
- dest: /tmp/foo
- dest: /tmp/foo.txt
foo: Foo
bar: Bar
tasks:
- block:
- name: Test template does not cause StreamError
delegate_to: localhost
run_once: true
environment:
ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
command:
cmd: >
ansible-playbook
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
{% endfor %}
regression/template_test.yml
chdir: ../
register: issue_1087_cmd
failed_when:
- issue_1087_cmd is failed
or issue_1087_cmd.stdout is search('ERROR|mitogen\.core\.CallError')
or issue_1087_cmd.stderr is search('ERROR|mitogen\.core\.CallError')
always:
- name: Cleanup
file:
path: "{{ item.dest }}"
state: absent
with_items: "{{ foos }}"
tags:
- issue_1087

@ -12,7 +12,9 @@
https_proxy: "{{ lookup('env', 'https_proxy')|default('') }}"
no_proxy: "{{ lookup('env', 'no_proxy')|default('') }}"
PATH: "{{ lookup('env', 'PATH') }}"
shell: virtualenv /tmp/issue_152_virtualenv
command:
cmd: virtualenv -p "{{ ansible_facts.python.executable }}" /tmp/issue_152_virtualenv
creates: /tmp/issue_152_virtualenv
when:
- lout.python.version.full is version('2.7', '>=', strict=True)

@ -19,9 +19,6 @@
# Will crash if process has a nonexistent CWD.
- custom_python_os_getcwd:
script: |
import os
self._connection.get_chain().call(os.getcwd)
tags:
- issue_591
- mitogen_only

@ -11,11 +11,10 @@
tasks:
- meta: end_play
when:
# TODO CI currently runs on macOS 11 images in Azure DevOps. MacOS 11
# is no longer supported by homebrew, so the following install
# task fails.
# TODO CI currently runs on macOS 12 & which isn't supported by Podman
# version available in Homebrew.
- ansible_facts.system == 'Darwin'
- ansible_facts.distribution_major_version == '11'
- ansible_facts.distribution_version is version('13.0', '<', strict=True)
- name: set up test container and run tests inside it
block:

@ -3,28 +3,33 @@
- name: regression/issue_776__load_plugins_called_twice.yml
hosts: test-targets
become: "{{ groups.linux is defined and inventory_hostname in groups.linux }}"
gather_facts: yes
# Delayed until after the end_play, due to Python version requirements
gather_facts: false
tags:
- issue_776
vars:
ansible_python_interpreter: "{{ pkg_mgr_python_interpreter }}"
package: rsync # Chosen to exist in all tested distros/package managers
tasks:
- name: Switch to centos-stream
command: dnf --assumeyes --disablerepo="*" --enablerepo=extras swap centos-linux-repos centos-stream-repos
# The package management modules require using the same Python version
# as the target's package manager libraries. This is sometimes in conflict
# with Ansible requirements, e.g. Ansible 10 (ansible-core 2.17) does not
# support Python 2.x on targets.
- meta: end_play
when:
- ansible_facts.pkg_mgr in ["dnf"]
- ansible_version.full is version('2.17', '>=', strict=True)
- name: Switch to archive.debian.org
# Debian 9 has been archived https://lists.debian.org/debian-devel-announce/2023/03/msg00006.html
- name: Gather facts manually
setup:
- name: Switch to archived package repositories
copy:
content: |
deb http://archive.debian.org/debian stretch main contrib non-free
dest: /etc/apt/sources.list
dest: "{{ item.dest }}"
content: "{{ item.content }}"
mode: u=rw,go=r
when:
- ansible_facts.distribution == "Debian"
- ansible_facts.distribution_major_version == "9"
loop: "{{ pkg_repos_overrides }}"
loop_control:
label: "{{ item.dest }}"
- name: Add signing keys
copy:

@ -0,0 +1,28 @@
- name: regression/template_test.yml
# Ansible's template module has been seen to raise mitogen.core.StreamError
# iif there is a with_items loop and the destination path has an extension
hosts: test-targets
gather_facts: false
become: false
vars:
foos:
- dest: /tmp/foo
- dest: /tmp/foo.txt
foo: Foo
bar: Bar
tasks:
- block:
- name: Template files
template:
src: foo.bar.j2
dest: "{{ item.dest }}"
mode: u=rw,go=r
# This has to be with_items, loop: doesn't trigger the bug
with_items: "{{ foos }}"
always:
- name: Cleanup
file:
path: "{{ item.dest }}"
state: absent
with_items: "{{ foos }}"

@ -0,0 +1 @@
A {{ foo }} walks into a {{ bar }}. Ow!

@ -1,4 +1,7 @@
paramiko==2.3.2 # Last 2.6-compat version.
# Incompatible with pip >= 72, due to removal of `setup.py test`:
# ModuleNotFoundError: No module named 'setuptools.command.test'
# https://github.com/pypa/setuptools/issues/4519
hdrhistogram==0.6.1
PyYAML==3.11; python_version < '2.7'
PyYAML==5.3.1; python_version >= '2.7' # Latest release (Jan 2021)

@ -0,0 +1,39 @@
[test-targets]
{% for c in containers %}
{{ c.name }} ansible_host={{ c.hostname }} ansible_port={{ c.port }} ansible_python_interpreter={{ c.python_path }}
{% endfor %}
[test-targets:vars]
ansible_user=mitogen__has_sudo_nopw
ansible_password=has_sudo_nopw_password
{% for distro, hostnames in distros | dictsort %}
[{{ distro }}]
{% for hostname in hostnames %}
{{ hostname }}
{% endfor %}
{% endfor %}
{% for family, hostnames in families | dictsort %}
[{{ family }}]
{% for hostname in hostnames %}
{{ hostname }}
{% endfor %}
{% endfor %}
[linux:children]
test-targets
[linux_containers:children]
test-targets
[issue905]
{% for c in containers[:1] %}
ssh-common-args ansible_host={{ c.hostname }} ansible_port={{ c.port }} ansible_python_interpreter={{ c.python_path }}
{% endfor %}
[issue905:vars]
ansible_user=mitogen__has_sudo_nopw
ansible_password=has_sudo_nopw_password
ansible_ssh_common_args=-o PermitLocalCommand=yes -o LocalCommand="touch {{ '{{' }} ssh_args_canary_file {{ '}}' }}"
ssh_args_canary_file=/tmp/ssh_args_{{ '{{' }} inventory_hostname {{ '}}' }}

@ -1,5 +1,5 @@
import logging
import os
import signal
import sys
import tempfile
import threading
@ -54,7 +54,9 @@ def do_detach(econtext):
class DetachReapTest(testlib.RouterMixin, testlib.TestCase):
def test_subprocess_preserved_on_shutdown(self):
c1 = self.router.local()
c1_stream = self.router.stream_by_id(c1.context_id)
pid = c1.call(os.getpid)
self.assertEqual(pid, c1_stream.conn.proc.pid)
l = mitogen.core.Latch()
mitogen.core.listen(c1, 'disconnect', l.put)
@ -64,8 +66,8 @@ class DetachReapTest(testlib.RouterMixin, testlib.TestCase):
self.broker.shutdown()
self.broker.join()
os.kill(pid, 0) # succeeds if process still alive
self.assertIsNone(os.kill(pid, 0)) # succeeds if process still alive
# now clean up
os.kill(pid, signal.SIGTERM)
os.waitpid(pid, 0)
c1_stream.conn.proc.terminate()
c1_stream.conn.proc.proc.wait()

@ -0,0 +1,3 @@
# Setuptools 72 removed `setup.py test`. hdrhistogram 0.6.1 still depends on it.
# TODO Bump dependencies and unconstrain Pip.
setuptools<72

@ -76,6 +76,7 @@ def close_proc(proc):
proc.stdout.close()
if proc.stderr:
proc.stderr.close()
proc.proc.wait()
def wait_read(fp, n):

@ -53,4 +53,4 @@ if _system_six:
else:
from . import _six as six
six_py_file = '{0}.py'.format(os.path.splitext(six.__file__)[0])
exec(open(six_py_file, 'rb').read())
with open(six_py_file, 'rb') as f: exec(f.read())

@ -27,3 +27,6 @@ class SlaveTest(testlib.RouterMixin, testlib.TestCase):
# Subsequent master allocation does not collide
c2 = self.router.local()
self.assertEqual(1002, c2.context_id)
context.shutdown()
c2.shutdown()

@ -9,7 +9,7 @@
--change 'EXPOSE 22'
--change 'CMD ["/usr/sbin/sshd", "-D"]'
{{ inventory_hostname }}
public.ecr.aws/n5z0e8q9/{{ inventory_hostname }}-test
{{ container_image_name }}
delegate_to: localhost
- name: Stop containers

@ -4,6 +4,9 @@ common_packages:
- strace
- sudo
container_image_name: "{{ container_registry }}/{{ inventory_hostname }}-test"
container_registry: public.ecr.aws/n5z0e8q9
sudo_group:
MacOSX: admin
Debian: sudo

@ -2,11 +2,7 @@ import unittest
import mitogen.core
try:
next
except NameError:
def next(it):
return it.next()
from mitogen.core import next
class IterSplitTest(unittest.TestCase):

@ -3,9 +3,6 @@ import os
import mitogen.lxc
import mitogen.parent
try:
any
except NameError:
from mitogen.core import any
import testlib

@ -8,14 +8,10 @@ import unittest
import mitogen.core
import mitogen.parent
import testlib
try:
next
except NameError:
# Python 2.4
from mitogen.core import next
import testlib
class SockMixin(object):
def tearDown(self):

@ -10,8 +10,7 @@ import mitogen.parent
class ReaperTest(testlib.TestCase):
@mock.patch('os.kill')
def test_calc_delay(self, kill):
def test_calc_delay(self):
broker = mock.Mock()
proc = mock.Mock()
proc.poll.return_value = None
@ -24,8 +23,7 @@ class ReaperTest(testlib.TestCase):
self.assertEqual(752, int(1000 * reaper._calc_delay(5)))
self.assertEqual(1294, int(1000 * reaper._calc_delay(6)))
@mock.patch('os.kill')
def test_reap_calls(self, kill):
def test_reap_calls(self):
broker = mock.Mock()
proc = mock.Mock()
proc.poll.return_value = None
@ -33,20 +31,20 @@ class ReaperTest(testlib.TestCase):
reaper = mitogen.parent.Reaper(broker, proc, True, True)
reaper.reap()
self.assertEqual(0, kill.call_count)
self.assertEqual(0, proc.send_signal.call_count)
reaper.reap()
self.assertEqual(1, kill.call_count)
self.assertEqual(1, proc.send_signal.call_count)
reaper.reap()
reaper.reap()
reaper.reap()
self.assertEqual(1, kill.call_count)
self.assertEqual(1, proc.send_signal.call_count)
reaper.reap()
self.assertEqual(2, kill.call_count)
self.assertEqual(2, proc.send_signal.call_count)
self.assertEqual(kill.mock_calls, [
mock.call(proc.pid, signal.SIGTERM),
mock.call(proc.pid, signal.SIGKILL),
self.assertEqual(proc.send_signal.mock_calls, [
mock.call(signal.SIGTERM),
mock.call(signal.SIGKILL),
])

@ -190,6 +190,7 @@ class BannerTest(testlib.DockerMixin, testlib.TestCase):
self.dockerized_ssh.port,
)
self.assertEqual(name, context.name)
context.shutdown(wait=True)
class StubPermissionDeniedTest(StubSshMixin, testlib.TestCase):

@ -51,6 +51,12 @@ except NameError:
LOG = logging.getLogger(__name__)
DISTRO = os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')
IMAGE_TEMPLATE = os.environ.get(
'MITOGEN_TEST_IMAGE_TEMPLATE',
'public.ecr.aws/n5z0e8q9/%(distro)s-test',
)
TESTS_DIR = os.path.join(os.path.dirname(__file__))
ANSIBLE_LIB_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib')
ANSIBLE_MODULE_UTILS_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils')
@ -146,6 +152,17 @@ def data_path(suffix):
return path
def retry(fn, on, max_attempts, delay):
for i in range(max_attempts):
try:
return fn()
except on:
if i >= max_attempts - 1:
raise
else:
time.sleep(delay)
def threading__thread_is_alive(thread):
"""Return whether the thread is alive (Python version compatibility shim).
@ -498,6 +515,7 @@ class TestCase(unittest.TestCase):
def get_docker_host():
# Duplicated in ci_lib
url = os.environ.get('DOCKER_HOST')
if url in (None, 'http+docker://localunixsocket'):
return 'localhost'
@ -538,19 +556,23 @@ class DockerizedSshDaemon(object):
]
subprocess.check_output(args)
def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')):
if '-' in mitogen_test_distro:
distro, _py3 = mitogen_test_distro.split('-')
else:
distro = mitogen_test_distro
_py3 = None
def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE):
# Code duplicated in ci_lib.py, both should be updated together
distro_pattern = re.compile(r'''
(?P<distro>(?P<family>[a-z]+)[0-9]+)
(?:-(?P<py>py3))?
(?:\*(?P<count>[0-9]+))?
''',
re.VERBOSE,
)
d = distro_pattern.match(distro).groupdict(default=None)
if _py3 == 'py3':
if d.pop('py') == 'py3':
self.python_path = '/usr/bin/python3'
else:
self.python_path = '/usr/bin/python'
self.image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,)
self.image = image_template % d
self.start_container()
self.host = self.get_host()
self.port = self.get_port(self.container_name)
@ -562,18 +584,24 @@ class DockerizedSshDaemon(object):
wait_for_port(self.get_host(), self.port, pattern='OpenSSH')
def check_processes(self):
args = ['docker', 'exec', self.container_name, 'ps', '-o', 'comm=']
# Get Accounting name (ucomm) & command line (args) of each process
# in the container. No truncation (-ww). No column headers (foo=).
ps_output = subprocess.check_output([
'docker', 'exec', self.container_name,
'ps', '-w', '-w', '-o', 'ucomm=', '-o', 'args=',
])
ps_lines = ps_output.decode().splitlines()
processes = [tuple(line.split(None, 1)) for line in ps_lines]
counts = {}
for comm in subprocess.check_output(args).decode().splitlines():
comm = comm.strip()
counts[comm] = counts.get(comm, 0) + 1
for ucomm, _ in processes:
counts[ucomm] = counts.get(ucomm, 0) + 1
if counts != {'ps': 1, 'sshd': 1}:
assert 0, (
'Docker container %r contained extra running processes '
'after test completed: %r' % (
self.container_name,
counts
processes,
)
)
@ -584,6 +612,9 @@ class DockerizedSshDaemon(object):
class BrokerMixin(object):
broker_class = mitogen.master.Broker
# Flag for tests that shutdown the broker themself
# e.g. unix_test.ListenerTest
broker_shutdown = False
def setUp(self):
@ -630,7 +661,12 @@ class DockerMixin(RouterMixin):
@classmethod
def tearDownClass(cls):
cls.dockerized_ssh.check_processes()
retry(
cls.dockerized_ssh.check_processes,
on=AssertionError,
max_attempts=5,
delay=0.1,
)
cls.dockerized_ssh.close()
super(DockerMixin, cls).tearDownClass()

@ -1,3 +1,6 @@
import os
import unittest
import mitogen.core
import testlib
@ -7,6 +10,10 @@ import simple_pkg.ping
# TODO: this is a joke. 2/3 interop is one of the hardest bits to get right.
# There should be 100 tests in this file.
@unittest.skipIf(
os.uname()[0] == 'Darwin' and int(os.uname()[2].partition('.')[0]) >= 21,
"Python 2.x not shipped on macOS 12.3+ (Darwin 21.4+, Monterey)",
)
class TwoThreeCompatTest(testlib.RouterMixin, testlib.TestCase):
if mitogen.core.PY3:
python_path = 'python2'

@ -65,17 +65,13 @@ class ListenerTest(testlib.RouterMixin, testlib.TestCase):
def test_constructor_basic(self):
listener = self.klass.build_stream(router=self.router)
capture = testlib.LogCapturer()
capture.start()
try:
self.assertFalse(mitogen.unix.is_path_dead(listener.protocol.path))
os.unlink(listener.protocol.path)
# ensure we catch 0 byte read error log message
self.broker.shutdown()
self.broker.join()
self.broker_shutdown = True
finally:
capture.stop()
class ClientTest(testlib.TestCase):

@ -10,9 +10,9 @@
# 2.4 2.3? <= 3.7.1 <= 1.3.7 <= 1.1 <= 2.1.3 <= 1.4 <= 1.8
# 2.5 <= 3.7.1 <= 1.4.22 <= 1.3.1 <= 2.1.3 <= 2.8.7 <= 1.6.1 <= 1.9.1
# 2.6 <= 2.6.20 <= 2.12 <= 4.5.4 <= 1.6.11 <= 2.10.3 <= 9.0.3 <= 5.9.0 <= 3.2.5 <= 2.9.1 <= 15.2.0
# 2.7 <= 2.11 <= 5.5 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.15²
# 2.7 <= 2.11 <= 2.16 <= 5.5 <= 1.11.29 <= 2.11.3 <= 20 <= 4.6.11 <= 3.28 <= 20.15²
# 3.5 <= 2.11 <= 2.15 <= 5.5 <= 2.2.28 <= 2.11.3 <= 20 <= 5.9.5 <= 6.1.0 <= 3.28 <= 20.15²
# 3.6 <= 2.11 <= 6.2 <= 3.2.20 <= 3.0.3 <= 21 <= 7.0.1 <= 3.28 <= 20.17²
# 3.6 <= 2.11 <= 2.16 <= 6.2 <= 3.2.20 <= 3.0.3 <= 21 <= 7.0.1 <= 3.28 <= 20.17²
# 3.7 <= 2.12 <= 7.2.7 <= 3.2.20 <= 7.4.4 <= 4.8.0
# 3.8 <= 2.12
# 3.9 <= 2.15
@ -46,18 +46,17 @@
# ansible == 7.x ansible-core ~= 2.14.0
# ansible == 8.x ansible-core ~= 2.15.0
# ansible == 9.x ansible-core ~= 2.16.0
# ansible == 10.x ansible-core ~= 2.17.0
# pip --no-python-version-warning
# pip --disable-pip-version-check
# TODO distros=-py3
# See also
# - https://docs.ansible.com/ansible/latest/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix
[tox]
envlist =
init,
py{27,36}-mode_ansible-ansible{2.10,3,4},
py{311}-mode_ansible-ansible{2.10,3,4,5},
py{312}-mode_ansible-ansible{6,7,8,9},
py{312}-mode_ansible-ansible{6,7,8,9,10},
py{27,36,312}-mode_mitogen-distro_centos{6,7,8},
py{27,36,312}-mode_mitogen-distro_debian{9,10,11},
py{27,36,312}-mode_mitogen-distro_ubuntu{1604,1804,2004},
@ -78,21 +77,19 @@ basepython =
deps =
-r{toxinidir}/tests/requirements.txt
mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt
ansible2.10: ansible==2.10.7
ansible3: ansible==3.4.0
ansible4: ansible==4.10.0
ansible2.10: ansible~=2.10.0
ansible3: ansible~=3.0
ansible4: ansible~=4.0
ansible5: ansible~=5.0
ansible6: ansible~=6.0
ansible7: ansible~=7.0
ansible8: ansible~=8.0
ansible9: ansible~=9.0
ansible10: ansible~=10.0
install_command =
python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages}
commands_pre =
mode_ansible: {toxinidir}/.ci/ansible_install.py
mode_debops_common: {toxinidir}/.ci/debops_common_install.py
mode_localhost: {toxinidir}/.ci/localhost_ansible_install.py
mode_mitogen: {toxinidir}/.ci/mitogen_install.py
commands =
mode_ansible: {toxinidir}/.ci/ansible_tests.py
mode_debops_common: {toxinidir}/.ci/debops_common_tests.py
@ -100,15 +97,17 @@ commands =
mode_mitogen: {toxinidir}/.ci/mitogen_tests.py
passenv =
ANSIBLE_*
AWS_ACCESS_KEY_ID
AWS_DEFAULT_REGION
AWS_SECRET_ACCESS_KEY
HOME
MITOGEN_*
# Azure DevOps, TF_BUILD is set to 'True' when running in a build task
# https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables
TF_BUILD
setenv =
# See also azure-pipelines.yml
ANSIBLE_STRATEGY = mitogen_linear
NOCOVERAGE_ERASE = 1
NOCOVERAGE_REPORT = 1
PIP_CONSTRAINT={toxinidir}/tests/constraints.txt
# Only applicable to MODE=mitogen
distro_centos5: DISTRO=centos5
distro_centos6: DISTRO=centos6
@ -125,8 +124,10 @@ setenv =
ansible6: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004
ansible7: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004
ansible8: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004
# Ansible >= 9 (ansible-core >= 2.16) require Python 2.7 or >= 3.6 on targets
# Ansible 9 (ansible-core 2.16) requires Python 2.7 or >= 3.6 on targets
ansible9: DISTROS=centos7 centos8 debian9 debian10 debian11 ubuntu1804 ubuntu2004
# Ansible 10 (ansible-core 2.17) requires Python >= 3.7 on targets
ansible10: DISTROS=debian10-py3 debian11-py3 ubuntu2004-py3
distros_centos: DISTROS=centos6 centos7 centos8
distros_centos5: DISTROS=centos5
distros_centos6: DISTROS=centos6

Loading…
Cancel
Save