diff --git a/.ci/README.md b/.ci/README.md index 67a3805b..9248ac58 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -28,7 +28,6 @@ for doing `setup.py install` while pulling a Docker container, for example. ### Environment Variables -* `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2. * `DISTRO`: the `mitogen_` tests need a target Docker container distro. This name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test` * `DISTROS`: the `ansible_` tests can run against multiple targets diff --git a/.ci/ansible_install.py b/.ci/ansible_install.py deleted file mode 100755 index 3b217ff2..00000000 --- a/.ci/ansible_install.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python - -import ci_lib - -batches = [ - [ - 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', - ] -] - -ci_lib.run_batches(batches) diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 102eda9c..3ec48dfd 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -35,7 +35,7 @@ ci_lib.check_stray_processes(interesting) with ci_lib.Fold('docker_setup'): - containers = ci_lib.make_containers() + containers = ci_lib.container_specs(ci_lib.DISTROS) ci_lib.start_containers(containers) diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml index 919b992b..791372af 100644 --- a/.ci/azure-pipelines-steps.yml +++ b/.ci/azure-pipelines-steps.yml @@ -14,6 +14,19 @@ steps: versionSpec: '$(python.version)' condition: ne(variables['python.version'], '') +- script: | + set -o errexit + set -o nounset + set -o pipefail + + aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws + displayName: Authenticate to container registry + condition: eq(variables['Agent.OS'], 'Linux') + env: + AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) + AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) + AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION) + - script: | set -o errexit set -o nounset @@ -90,7 +103,3 @@ steps: "$PYTHON" -m tox -e "$(tox.env)" displayName: "Run tests" - env: - AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) - AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) - AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 3e716385..dfe49b97 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -27,6 +27,13 @@ os.chdir( ) ) + +IMAGE_TEMPLATE = os.environ.get( + 'MITOGEN_TEST_IMAGE_TEMPLATE', + 'public.ecr.aws/n5z0e8q9/%(distro)s-test', +) + + _print = print def print(*args, **kwargs): file = kwargs.get('file', sys.stdout) @@ -193,8 +200,6 @@ GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) DISTRO = os.environ.get('DISTRO', 'debian9') # Used only when MODE=ansible DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split() -TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2')) -BASE_PORT = 2200 TMP = TempDir().path @@ -217,6 +222,7 @@ os.environ['PYTHONPATH'] = '%s:%s' % ( def get_docker_hostname(): """Return the hostname where the docker daemon is running. """ + # Duplicated in testlib url = os.environ.get('DOCKER_HOST') if url in (None, 'http+docker://localunixsocket'): return 'localhost' @@ -225,27 +231,34 @@ def get_docker_hostname(): return parsed.netloc.partition(':')[0] -def make_containers(name_prefix='', port_offset=0): +def container_specs( + distros, + base_port=2200, + image_template=IMAGE_TEMPLATE, + name_template='target-%(distro)s-%(index)d', +): """ >>> import pprint - >>> BASE_PORT=2200; DISTROS=['debian11', 'centos6'] - >>> pprint.pprint(make_containers()) + >>> pprint.pprint(container_specs(['debian11-py3', 'centos6'])) [{'distro': 'debian11', 'family': 'debian', 'hostname': 'localhost', 'image': 'public.ecr.aws/n5z0e8q9/debian11-test', + 'index': 1, 'name': 'target-debian11-1', 'port': 2201, - 'python_path': '/usr/bin/python'}, + 'python_path': '/usr/bin/python3'}, {'distro': 'centos6', 'family': 'centos', 'hostname': 'localhost', 'image': 'public.ecr.aws/n5z0e8q9/centos6-test', + 'index': 2, 'name': 'target-centos6-2', 'port': 2202, 'python_path': '/usr/bin/python'}] """ docker_hostname = get_docker_hostname() + # Code duplicated in testlib.py, both should be updated together distro_pattern = re.compile(r''' (?P(?P[a-z]+)[0-9]+) (?:-(?Ppy3))? @@ -256,30 +269,27 @@ def make_containers(name_prefix='', port_offset=0): i = 1 lst = [] - for distro in DISTROS: + for distro in distros: + # Code duplicated in testlib.py, both should be updated together d = distro_pattern.match(distro).groupdict(default=None) - distro = d['distro'] - family = d['family'] - image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,) - if d['py'] == 'py3': + if d.pop('py') == 'py3': python_path = '/usr/bin/python3' else: python_path = '/usr/bin/python' - if d['count']: - count = int(count) - else: - count = 1 + count = int(d.pop('count') or '1', 10) for x in range(count): - lst.append({ - "distro": distro, "family": family, "image": image, - "name": name_prefix + ("target-%s-%s" % (distro, i)), + d['index'] = i + d.update({ + 'image': image_template % d, + 'name': name_template % d, "hostname": docker_hostname, - "port": BASE_PORT + i + port_offset, + 'port': base_port + i, "python_path": python_path, }) + lst.append(d) i += 1 return lst diff --git a/.ci/debops_common_install.py b/.ci/debops_common_install.py index 565f9488..825126c7 100755 --- a/.ci/debops_common_install.py +++ b/.ci/debops_common_install.py @@ -2,16 +2,10 @@ import ci_lib -# Naturally DebOps only supports Debian. -ci_lib.DISTROS = ['debian'] - ci_lib.run_batches([ [ 'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"', ], - [ - 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', - ], ]) ci_lib.run('ansible-galaxy collection install debops.debops:==2.1.2') diff --git a/.ci/debops_common_tests.py b/.ci/debops_common_tests.py index 7db8a797..b065486f 100755 --- a/.ci/debops_common_tests.py +++ b/.ci/debops_common_tests.py @@ -6,9 +6,6 @@ import sys import ci_lib -# DebOps only supports Debian. -ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT - project_dir = os.path.join(ci_lib.TMP, 'project') vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml' inventory_path = 'ansible/inventory/hosts' @@ -16,7 +13,11 @@ docker_hostname = ci_lib.get_docker_hostname() with ci_lib.Fold('docker_setup'): - containers = ci_lib.make_containers(port_offset=500, name_prefix='debops-') + containers = ci_lib.container_specs( + ['debian*2'], + base_port=2700, + name_template='debops-target-%(distro)s-%(index)d', + ) ci_lib.start_containers(containers) diff --git a/.ci/localhost_ansible_install.py b/.ci/localhost_ansible_install.py deleted file mode 100755 index d08ddafc..00000000 --- a/.ci/localhost_ansible_install.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python - -import ci_lib - -batches = [ -] - -ci_lib.run_batches(batches) diff --git a/.ci/mitogen_install.py b/.ci/mitogen_install.py deleted file mode 100755 index 23ff384b..00000000 --- a/.ci/mitogen_install.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python - -import ci_lib - -batches = [ -] - -if ci_lib.have_docker(): - batches.append([ - 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', - ]) - - -ci_lib.run_batches(batches) diff --git a/.ci/mitogen_py24_install.py b/.ci/mitogen_py24_install.py index 8af90405..85ea013c 100755 --- a/.ci/mitogen_py24_install.py +++ b/.ci/mitogen_py24_install.py @@ -3,9 +3,6 @@ import ci_lib batches = [ - [ - 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', - ], [ 'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv', ] diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..4520c3cf --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,326 @@ +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions + +name: Tests + +on: + pull_request: + push: + branches-ignore: + - docs-master + +env: + #ANSIBLE_VERBOSITY: 3 + #MITOGEN_LOG_LEVEL: DEBUG + MITOGEN_TEST_IMAGE_TEMPLATE: "ghcr.io/mitogen-hq/%(distro)s-test" + +# https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners +# https://github.com/actions/runner-images/blob/main/README.md#software-and-image-support +jobs: + linux: + # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md + runs-on: ubuntu-20.04 + + strategy: + fail-fast: false + matrix: + include: + - name: Ans_27_210 + tox_env: py27-mode_ansible-ansible2.10 + - name: Ans_27_4 + tox_env: py27-mode_ansible-ansible4 + + - name: Ans_36_210 + python_version: '3.6' + tox_env: py36-mode_ansible-ansible2.10 + - name: Ans_36_4 + python_version: '3.6' + tox_env: py36-mode_ansible-ansible4 + + - name: Ans_311_210 + python_version: '3.11' + tox_env: py311-mode_ansible-ansible2.10 + - name: Ans_311_3 + python_version: '3.11' + tox_env: py311-mode_ansible-ansible3 + - name: Ans_311_4 + python_version: '3.11' + tox_env: py311-mode_ansible-ansible4 + - name: Ans_311_5 + python_version: '3.11' + tox_env: py311-mode_ansible-ansible5 + - name: Ans_312_6 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible6 + - name: Ans_312_7 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible7 + - name: Ans_312_8 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible8 + - name: Ans_312_9 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible9 + - name: Ans_312_10 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible10 + - name: Van_312_10 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible10-strategy_linear + + - name: Mito_27_centos6 + tox_env: py27-mode_mitogen-distro_centos6 + - name: Mito_27_centos7 + tox_env: py27-mode_mitogen-distro_centos7 + - name: Mito_27_centos8 + tox_env: py27-mode_mitogen-distro_centos8 + - name: Mito_27_debian9 + tox_env: py27-mode_mitogen-distro_debian9 + - name: Mito_27_debian10 + tox_env: py27-mode_mitogen-distro_debian10 + - name: Mito_27_debian11 + tox_env: py27-mode_mitogen-distro_debian11 + - name: Mito_27_ubuntu1604 + tox_env: py27-mode_mitogen-distro_ubuntu1604 + - name: Mito_27_ubuntu1804 + tox_env: py27-mode_mitogen-distro_ubuntu1804 + - name: Mito_27_ubuntu2004 + tox_env: py27-mode_mitogen-distro_ubuntu2004 + + - name: Mito_36_centos6 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_centos6 + - name: Mito_36_centos7 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_centos7 + - name: Mito_36_centos8 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_centos8 + - name: Mito_36_debian9 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_debian9 + - name: Mito_36_debian10 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_debian10 + - name: Mito_36_debian11 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_debian11 + - name: Mito_36_ubuntu1604 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_ubuntu1604 + - name: Mito_36_ubuntu1804 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_ubuntu1804 + - name: Mito_36_ubuntu2004 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_ubuntu2004 + + - name: Mito_312_centos6 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_centos6 + - name: Mito_312_centos7 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_centos7 + - name: Mito_312_centos8 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_centos8 + - name: Mito_312_debian9 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_debian9 + - name: Mito_312_debian10 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_debian10 + - name: Mito_312_debian11 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_debian11 + - name: Mito_312_ubuntu1604 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_ubuntu1604 + - name: Mito_312_ubuntu1804 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_ubuntu1804 + - name: Mito_312_ubuntu2004 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_ubuntu2004 + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python_version }} + if: ${{ matrix.python_version }} + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Install build deps + run: | + set -o errexit -o nounset -o pipefail + + sudo apt-get update + sudo apt-get install -y python2-dev python3-pip virtualenv + - name: Show Python versions + run: | + set -o errexit -o nounset -o pipefail + + # macOS builders lack a realpath command + type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version + type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version + type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version + echo + + if [ -e /usr/bin/python ]; then + echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2 ]; then + echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2.7 ]; then + echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')" + fi + - name: Install tooling + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then + "$PYTHON" -m ensurepip --user --altinstall --no-default-pip + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + elif [[ $PYTHON == "python2.7" ]]; then + curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py" + "$PYTHON" get-pip.py --user --no-python-version-warning + # Avoid Python 2.x pip masking system pip + rm -f ~/.local/bin/{easy_install,pip,wheel} + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + else + "$PYTHON" -m pip install -r "tests/requirements-tox.txt" + fi + - name: Run tests + env: + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + "$PYTHON" -m tox -e "${{ matrix.tox_env }}" + + macos: + # https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md + runs-on: macos-12 + timeout-minutes: 120 + + strategy: + fail-fast: false + matrix: + include: + - name: Mito_27 + tox_env: py27-mode_mitogen + - name: Mito_312 + tox_env: py312-mode_mitogen + + - name: Loc_27_210 + tox_env: py27-mode_localhost-ansible2.10 + - name: Loc_312_10 + tox_env: py312-mode_localhost-ansible10 + + - name: Van_27_210 + tox_env: py27-mode_localhost-ansible2.10-strategy_linear + - name: Van_312_10 + tox_env: py312-mode_localhost-ansible10-strategy_linear + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python_version }} + if: ${{ matrix.python_version }} + - name: Show Python versions + run: | + set -o errexit -o nounset -o pipefail + + # macOS builders lack a realpath command + type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version + type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version + type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version + echo + + if [ -e /usr/bin/python ]; then + echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2 ]; then + echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2.7 ]; then + echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')" + fi + + if [ -e /Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7 ]; then + # GitHub macOS 12 images: python2.7 is installed, but not on $PATH + echo "/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7: sys.executable: $(/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7 -c 'import sys; print(sys.executable)')" + fi + - name: Install tooling + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then + # GitHub macOS 12 images: python2.7 is installed, but not on $PATH + PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7" + "$PYTHON" -m ensurepip --user --altinstall --no-default-pip + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + elif [[ $PYTHON == "python2.7" ]]; then + curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py" + "$PYTHON" get-pip.py --user --no-python-version-warning + # Avoid Python 2.x pip masking system pip + rm -f ~/.local/bin/{easy_install,pip,wheel} + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + else + "$PYTHON" -m pip install -r "tests/requirements-tox.txt" + fi + - name: Run tests + env: + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then + # GitHub macOS 12 images: python2.7 is installed, but not on $PATH + PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7" + fi + + "$PYTHON" -m tox -e "${{ matrix.tox_env }}" diff --git a/ansible_mitogen/affinity.py b/ansible_mitogen/affinity.py index 635ee7b9..223794ab 100644 --- a/ansible_mitogen/affinity.py +++ b/ansible_mitogen/affinity.py @@ -83,7 +83,6 @@ import multiprocessing import os import struct -import mitogen.core import mitogen.parent @@ -265,7 +264,7 @@ class LinuxPolicy(FixedPolicy): for x in range(16): chunks.append(struct.pack('>= 64 - return mitogen.core.b('').join(chunks) + return b''.join(chunks) def _get_thread_ids(self): try: diff --git a/ansible_mitogen/logging.py b/ansible_mitogen/logging.py index 40b2b339..4d5647a4 100644 --- a/ansible_mitogen/logging.py +++ b/ansible_mitogen/logging.py @@ -32,15 +32,13 @@ __metaclass__ = type import logging import os +import ansible.utils.display + import mitogen.core import mitogen.utils -try: - from __main__ import display -except ImportError: - import ansible.utils.display - display = ansible.utils.display.Display() +display = ansible.utils.display.Display() #: The process name set via :func:`set_process_name`. _process_name = None diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 2cd97a3e..38f351ed 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -35,18 +35,16 @@ import pwd import random import traceback -try: - from shlex import quote as shlex_quote -except ImportError: - from pipes import quote as shlex_quote - -from ansible.module_utils._text import to_bytes -from ansible.parsing.utils.jsonify import jsonify - import ansible import ansible.constants import ansible.plugins import ansible.plugins.action +import ansible.utils.unsafe_proxy +import ansible.vars.clean + +from ansible.module_utils.common.text.converters import to_bytes, to_text +from ansible.module_utils.six.moves import shlex_quote +from ansible.parsing.utils.jsonify import jsonify import mitogen.core import mitogen.select @@ -57,24 +55,6 @@ import ansible_mitogen.target import ansible_mitogen.utils import ansible_mitogen.utils.unsafe -from ansible.module_utils._text import to_text - -try: - from ansible.utils.unsafe_proxy import wrap_var -except ImportError: - from ansible.vars.unsafe_proxy import wrap_var - -try: - # ansible 2.8 moved remove_internal_keys to the clean module - from ansible.vars.clean import remove_internal_keys -except ImportError: - try: - from ansible.vars.manager import remove_internal_keys - except ImportError: - # ansible 2.3.3 has remove_internal_keys as a protected func on the action class - # we'll fallback to calling self._remove_internal_keys in this case - remove_internal_keys = lambda a: "Not found" - LOG = logging.getLogger(__name__) @@ -413,10 +393,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): self._remove_tmp_path(tmp) # prevents things like discovered_interpreter_* or ansible_discovered_interpreter_* from being set - # handle ansible 2.3.3 that has remove_internal_keys in a different place - check = remove_internal_keys(result) - if check == 'Not found': - self._remove_internal_keys(result) + ansible.vars.clean.remove_internal_keys(result) # taken from _execute_module of ansible 2.8.6 # propagate interpreter discovery results back to the controller @@ -440,7 +417,7 @@ class ActionModuleMixin(ansible.plugins.action.ActionBase): result['deprecations'] = [] result['deprecations'].extend(self._discovery_deprecation_warnings) - return wrap_var(result) + return ansible.utils.unsafe_proxy.wrap_var(result) def _postprocess_response(self, result): """ diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py index 0a91039a..4cdc0f20 100644 --- a/ansible_mitogen/planner.py +++ b/ansible_mitogen/planner.py @@ -477,7 +477,7 @@ def read_file(path): finally: os.close(fd) - return mitogen.core.b('').join(bits) + return b''.join(bits) def _propagate_deps(invocation, planner, context): diff --git a/ansible_mitogen/plugins/connection/mitogen_local.py b/ansible_mitogen/plugins/connection/mitogen_local.py index 6ff86733..2d1e7052 100644 --- a/ansible_mitogen/plugins/connection/mitogen_local.py +++ b/ansible_mitogen/plugins/connection/mitogen_local.py @@ -42,13 +42,7 @@ except ImportError: import ansible_mitogen.connection import ansible_mitogen.process - -if sys.version_info > (3,): - viewkeys = dict.keys -elif sys.version_info > (2, 7): - viewkeys = dict.viewkeys -else: - viewkeys = lambda dct: set(dct) +viewkeys = getattr(dict, 'viewkeys', dict.keys) def dict_diff(old, new): diff --git a/ansible_mitogen/process.py b/ansible_mitogen/process.py index 3a41a43d..7ec70f2a 100644 --- a/ansible_mitogen/process.py +++ b/ansible_mitogen/process.py @@ -61,10 +61,9 @@ import mitogen.utils import ansible import ansible.constants as C import ansible.errors + import ansible_mitogen.logging import ansible_mitogen.services - -from mitogen.core import b import ansible_mitogen.affinity @@ -639,7 +638,7 @@ class MuxProcess(object): try: # Let the parent know our listening socket is ready. - mitogen.core.io_op(self.model.child_sock.send, b('1')) + mitogen.core.io_op(self.model.child_sock.send, b'1') # Block until the socket is closed, which happens on parent exit. mitogen.core.io_op(self.model.child_sock.recv, 1) finally: diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py index 8da1b670..b60e537c 100644 --- a/ansible_mitogen/runner.py +++ b/ansible_mitogen/runner.py @@ -40,7 +40,9 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type import atexit +import ctypes import json +import logging import os import re import shlex @@ -50,19 +52,12 @@ import tempfile import traceback import types +from ansible.module_utils.six.moves import shlex_quote + import mitogen.core import ansible_mitogen.target # TODO: circular import -from mitogen.core import b -from mitogen.core import bytes_partition -from mitogen.core import str_rpartition from mitogen.core import to_text -try: - import ctypes -except ImportError: - # Python 2.4 - ctypes = None - try: # Python >= 3.4, PEP 451 ModuleSpec API import importlib.machinery @@ -77,15 +72,6 @@ try: except ImportError: from io import StringIO -try: - from shlex import quote as shlex_quote -except ImportError: - from pipes import quote as shlex_quote - -# Absolute imports for <2.5. -logging = __import__('logging') - - # Prevent accidental import of an Ansible module from hanging on stdin read. import ansible.module_utils.basic ansible.module_utils.basic._ANSIBLE_ARGS = '{}' @@ -95,15 +81,13 @@ ansible.module_utils.basic._ANSIBLE_ARGS = '{}' # explicit call to res_init() on each task invocation. BSD-alikes export it # directly, Linux #defines it as "__res_init". libc__res_init = None -if ctypes: - libc = ctypes.CDLL(None) - for symbol in 'res_init', '__res_init': - try: - libc__res_init = getattr(libc, symbol) - except AttributeError: - pass +libc = ctypes.CDLL(None) +for symbol in 'res_init', '__res_init': + try: + libc__res_init = getattr(libc, symbol) + except AttributeError: + pass -iteritems = getattr(dict, 'iteritems', dict.items) LOG = logging.getLogger(__name__) @@ -217,13 +201,13 @@ class EnvironmentFileWatcher(object): for line in fp: # ' #export foo=some var ' -> ['#export', 'foo=some var '] bits = shlex_split_b(line) - if (not bits) or bits[0].startswith(b('#')): + if (not bits) or bits[0].startswith(b'#'): continue - if bits[0] == b('export'): + if bits[0] == b'export': bits.pop(0) - key, sep, value = bytes_partition(b(' ').join(bits), b('=')) + key, sep, value = b' '.join(bits).partition(b'=') if key and sep: yield key, value @@ -596,7 +580,7 @@ class ModuleUtilsImporter(object): mod.__path__ = [] mod.__package__ = str(fullname) else: - mod.__package__ = str(str_rpartition(to_text(fullname), '.')[0]) + mod.__package__ = str(to_text(fullname).rpartition('.')[0]) exec(code, mod.__dict__) self._loaded.add(fullname) return mod @@ -611,7 +595,7 @@ class TemporaryEnvironment(object): def __init__(self, env=None): self.original = dict(os.environ) self.env = env or {} - for key, value in iteritems(self.env): + for key, value in mitogen.core.iteritems(self.env): key = mitogen.core.to_text(key) value = mitogen.core.to_text(value) if value is None: @@ -819,7 +803,7 @@ class ScriptRunner(ProgramRunner): self.interpreter_fragment = interpreter_fragment self.is_python = is_python - b_ENCODING_STRING = b('# -*- coding: utf-8 -*-') + b_ENCODING_STRING = b'# -*- coding: utf-8 -*-' def _get_program(self): return self._rewrite_source( @@ -852,13 +836,13 @@ class ScriptRunner(ProgramRunner): # While Ansible rewrites the #! using ansible_*_interpreter, it is # never actually used to execute the script, instead it is a shell # fragment consumed by shell/__init__.py::build_module_command(). - new = [b('#!') + utf8(self.interpreter_fragment)] + new = [b'#!' + utf8(self.interpreter_fragment)] if self.is_python: new.append(self.b_ENCODING_STRING) - _, _, rest = bytes_partition(s, b('\n')) + _, _, rest = s.partition(b'\n') new.append(rest) - return b('\n').join(new) + return b'\n'.join(new) class NewStyleRunner(ScriptRunner): @@ -971,8 +955,7 @@ class NewStyleRunner(ScriptRunner): # change the default encoding. This hack was removed from Ansible long ago, # but not before permeating into many third party modules. PREHISTORIC_HACK_RE = re.compile( - b(r'reload\s*\(\s*sys\s*\)\s*' - r'sys\s*\.\s*setdefaultencoding\([^)]+\)') + br'reload\s*\(\s*sys\s*\)\s*sys\s*\.\s*setdefaultencoding\([^)]+\)', ) def _setup_program(self): @@ -980,7 +963,7 @@ class NewStyleRunner(ScriptRunner): context=self.service_context, path=self.path, ) - self.source = self.PREHISTORIC_HACK_RE.sub(b(''), source) + self.source = self.PREHISTORIC_HACK_RE.sub(b'', source) def _get_code(self): try: @@ -998,7 +981,7 @@ class NewStyleRunner(ScriptRunner): if mitogen.core.PY3: main_module_name = '__main__' else: - main_module_name = b('__main__') + main_module_name = b'__main__' def _handle_magic_exception(self, mod, exc): """ @@ -1030,7 +1013,7 @@ class NewStyleRunner(ScriptRunner): approximation of the original package hierarchy, so that relative imports function correctly. """ - pkg, sep, modname = str_rpartition(self.py_module_name, '.') + pkg, sep, _ = self.py_module_name.rpartition('.') if not sep: return None if mitogen.core.PY3: @@ -1073,7 +1056,7 @@ class NewStyleRunner(ScriptRunner): class JsonArgsRunner(ScriptRunner): - JSON_ARGS = b('<>') + JSON_ARGS = b'<>' def _get_args_contents(self): return json.dumps(self.args).encode() diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py index 3e9de652..abc0e379 100644 --- a/ansible_mitogen/services.py +++ b/ansible_mitogen/services.py @@ -50,6 +50,8 @@ import threading import ansible.constants +from ansible.module_utils.six import reraise + import mitogen.core import mitogen.service import ansible_mitogen.loaders @@ -66,20 +68,6 @@ LOG = logging.getLogger(__name__) ansible_mitogen.loaders.shell_loader.get('sh') -if sys.version_info[0] == 3: - def reraise(tp, value, tb): - if value is None: - value = tp() - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value -else: - exec( - "def reraise(tp, value, tb=None):\n" - " raise tp, value, tb\n" - ) - - def _get_candidate_temp_dirs(): try: # >=2.5 diff --git a/ansible_mitogen/target.py b/ansible_mitogen/target.py index 7d907d62..ee4cb398 100644 --- a/ansible_mitogen/target.py +++ b/ansible_mitogen/target.py @@ -39,6 +39,7 @@ __metaclass__ = type import errno import grp import json +import logging import operator import os import pwd @@ -51,26 +52,9 @@ import tempfile import traceback import types -# Absolute imports for <2.5. -logging = __import__('logging') - import mitogen.core import mitogen.parent import mitogen.service -from mitogen.core import b - -try: - reduce -except NameError: - # Python 3.x. - from functools import reduce - -try: - BaseException -except NameError: - # Python 2.4 - BaseException = Exception - # Ansible since PR #41749 inserts "import __main__" into # ansible.module_utils.basic. Mitogen's importer will refuse such an import, so @@ -80,6 +64,9 @@ if not sys.modules.get(str('__main__')): sys.modules[str('__main__')] = types.ModuleType(str('__main__')) import ansible.module_utils.json_utils + +from ansible.module_utils.six.moves import reduce + import ansible_mitogen.runner @@ -615,8 +602,8 @@ def exec_args(args, in_data='', chdir=None, shell=None, emulate_tty=False): stdout, stderr = proc.communicate(in_data) if emulate_tty: - stdout = stdout.replace(b('\n'), b('\r\n')) - return proc.returncode, stdout, stderr or b('') + stdout = stdout.replace(b'\n', b'\r\n') + return proc.returncode, stdout, stderr or b'' def exec_command(cmd, in_data='', chdir=None, shell=None, emulate_tty=False): @@ -746,9 +733,7 @@ def set_file_mode(path, spec, fd=None): """ Update the permissions of a file using the same syntax as chmod(1). """ - if isinstance(spec, int): - new_mode = spec - elif not mitogen.core.PY3 and isinstance(spec, long): + if isinstance(spec, mitogen.core.integer_types): new_mode = spec elif spec.isdigit(): new_mode = int(spec, 8) diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index 144de563..39a4b604 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -65,21 +65,12 @@ import abc import os import ansible.utils.shlex import ansible.constants as C +import ansible.executor.interpreter_discovery +import ansible.utils.unsafe_proxy from ansible.module_utils.six import with_metaclass from ansible.module_utils.parsing.convert_bool import boolean -# this was added in Ansible >= 2.8.0; fallback to the default interpreter if necessary -try: - from ansible.executor.interpreter_discovery import discover_interpreter -except ImportError: - discover_interpreter = lambda action,interpreter_name,discovery_mode,task_vars: '/usr/bin/python' - -try: - from ansible.utils.unsafe_proxy import AnsibleUnsafeText -except ImportError: - from ansible.vars.unsafe_proxy import AnsibleUnsafeText - import mitogen.core @@ -115,12 +106,13 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth action._finding_python_interpreter = True # fake pipelining so discover_interpreter can be happy action._connection.has_pipelining = True - s = AnsibleUnsafeText(discover_interpreter( + s = ansible.executor.interpreter_discovery.discover_interpreter( action=action, interpreter_name=interpreter_name, discovery_mode=s, - task_vars=task_vars)) - + task_vars=task_vars, + ) + s = ansible.utils.unsafe_proxy.AnsibleUnsafeText(s) # cache discovered interpreter task_vars['ansible_facts'][discovered_interpreter_config] = s action._connection.has_pipelining = False diff --git a/docs/changelog.rst b/docs/changelog.rst index 00db0bf6..6f61614a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,20 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. +v0.3.11 (2024-10-30) +-------------------- + +* :gh:issue:`1127` :mod:`mitogen`: Consolidate mitogen backward compatibility + fallbacks and polyfills into :mod:`mitogen.core` +* :gh:issue:`1127` :mod:`ansible_mitogen`: Remove backward compatibility + fallbacks for Python 2.4 & 2.5. +* :gh:issue:`1127` :mod:`ansible_mitogen`: Remove fallback imports for Ansible + releases before 2.10 +* :gh:issue:`1127` :mod:`ansible_mitogen`: Consolidate Python 2 & 3 + compatibility +* :gh:issue:`1128` CI: Start migration from Azure DevOps to GitHub Actions + + v0.3.10 (2024-09-20) -------------------- diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 9c1ddc6c..0cacddfe 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 10) +__version__ = (0, 3, 11) #: This is :data:`False` in slave contexts. Previously it was used to prevent diff --git a/mitogen/core.py b/mitogen/core.py index cdfbbcde..9b225ed7 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -102,21 +102,6 @@ try: except ImportError: cProfile = None -try: - import thread -except ImportError: - import threading as thread - -try: - import cPickle as pickle -except ImportError: - import pickle - -try: - from cStringIO import StringIO as BytesIO -except ImportError: - from io import BytesIO - try: BaseException except NameError: @@ -169,31 +154,35 @@ STUB_CALL_SERVICE = 111 #: :meth:`mitogen.core.Router.add_handler` callbacks to clean up. IS_DEAD = 999 -try: - BaseException -except NameError: - BaseException = Exception - PY24 = sys.version_info < (2, 5) PY3 = sys.version_info > (3,) if PY3: + import pickle + import _thread as thread + from io import BytesIO b = str.encode BytesType = bytes UnicodeType = str FsPathTypes = (str,) BufferType = lambda buf, start: memoryview(buf)[start:] - long = int + integer_types = (int,) + iteritems, iterkeys, itervalues = dict.items, dict.keys, dict.values else: + import cPickle as pickle + import thread + from cStringIO import StringIO as BytesIO b = str BytesType = str FsPathTypes = (str, unicode) BufferType = buffer UnicodeType = unicode + integer_types = (int, long) + iteritems, iterkeys, itervalues = dict.iteritems, dict.iterkeys, dict.itervalues AnyTextType = (BytesType, UnicodeType) try: - next + next = next except NameError: next = lambda it: it.next() @@ -400,12 +389,19 @@ now = getattr(time, 'monotonic', time.time) # Python 2.4 try: - any + all, any = all, any except NameError: + def all(it): + for elem in it: + if not elem: + return False + return True + def any(it): for elem in it: if elem: return True + return False def _partition(s, sep, find): @@ -1065,8 +1061,8 @@ class Sender(object): def _unpickle_sender(router, context_id, dst_handle): if not (isinstance(router, Router) and - isinstance(context_id, (int, long)) and context_id >= 0 and - isinstance(dst_handle, (int, long)) and dst_handle > 0): + isinstance(context_id, integer_types) and context_id >= 0 and + isinstance(dst_handle, integer_types) and dst_handle > 0): raise TypeError('cannot unpickle Sender: bad input or missing router') return Sender(Context(router, context_id), dst_handle) @@ -2508,7 +2504,7 @@ class Context(object): def _unpickle_context(context_id, name, router=None): - if not (isinstance(context_id, (int, long)) and context_id >= 0 and ( + if not (isinstance(context_id, integer_types) and context_id >= 0 and ( (name is None) or (isinstance(name, UnicodeType) and len(name) < 100)) ): diff --git a/mitogen/master.py b/mitogen/master.py index b1e0a1de..51b29b82 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -74,9 +74,11 @@ import mitogen.core import mitogen.minify import mitogen.parent +from mitogen.core import any from mitogen.core import b from mitogen.core import IOLOG from mitogen.core import LOG +from mitogen.core import next from mitogen.core import str_partition from mitogen.core import str_rpartition from mitogen.core import to_text @@ -84,17 +86,6 @@ from mitogen.core import to_text imap = getattr(itertools, 'imap', map) izip = getattr(itertools, 'izip', zip) -try: - any -except NameError: - from mitogen.core import any - -try: - next -except NameError: - from mitogen.core import next - - RLOG = logging.getLogger('mitogen.ctx') diff --git a/mitogen/parent.py b/mitogen/parent.py index 2ed7e8ba..dd51b697 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -56,15 +56,13 @@ import zlib # Absolute imports for <2.5. select = __import__('select') -try: - import thread -except ImportError: - import threading as thread - import mitogen.core from mitogen.core import b from mitogen.core import bytes_partition from mitogen.core import IOLOG +from mitogen.core import itervalues +from mitogen.core import next +from mitogen.core import thread LOG = logging.getLogger(__name__) @@ -80,15 +78,6 @@ except IOError: SELINUX_ENABLED = False -try: - next -except NameError: - # Python 2.4/2.5 - from mitogen.core import next - - -itervalues = getattr(dict, 'itervalues', dict.values) - if mitogen.core.PY3: xrange = range closure_attr = '__closure__' diff --git a/mitogen/service.py b/mitogen/service.py index 7fde9013..ffc1085b 100644 --- a/mitogen/service.py +++ b/mitogen/service.py @@ -39,18 +39,10 @@ import threading import mitogen.core import mitogen.select +from mitogen.core import all from mitogen.core import b from mitogen.core import str_rpartition -try: - all -except NameError: - def all(it): - for elem in it: - if not elem: - return False - return True - LOG = logging.getLogger(__name__) diff --git a/mitogen/ssh.py b/mitogen/ssh.py index 656dc72c..f32d2cab 100644 --- a/mitogen/ssh.py +++ b/mitogen/ssh.py @@ -43,11 +43,6 @@ except ImportError: import mitogen.parent from mitogen.core import b -try: - any -except NameError: - from mitogen.core import any - LOG = logging.getLogger(__name__) diff --git a/mitogen/su.py b/mitogen/su.py index 080c9782..9b908460 100644 --- a/mitogen/su.py +++ b/mitogen/su.py @@ -34,11 +34,6 @@ import re import mitogen.core import mitogen.parent -try: - any -except NameError: - from mitogen.core import any - LOG = logging.getLogger(__name__) diff --git a/mitogen/utils.py b/mitogen/utils.py index 1fbf71fe..9d1c1bc9 100644 --- a/mitogen/utils.py +++ b/mitogen/utils.py @@ -37,13 +37,7 @@ import sys import mitogen.core import mitogen.master - -iteritems = getattr(dict, 'iteritems', dict.items) - -if mitogen.core.PY3: - iteritems = dict.items -else: - iteritems = dict.iteritems +from mitogen.core import iteritems def setup_gil(): diff --git a/tests/ansible/integration/become/su_password.yml b/tests/ansible/integration/become/su_password.yml index 52d420db..207980c4 100644 --- a/tests/ansible/integration/become/su_password.yml +++ b/tests/ansible/integration/become/su_password.yml @@ -53,20 +53,22 @@ vars: ansible_become_pass: user1_password when: - # https://github.com/ansible/ansible/pull/70785 - - ansible_facts.distribution not in ["MacOSX"] - or ansible_version.full is version("2.11", ">=", strict=True) - or is_mitogen + # CI containers lack `setfacl` for unpriv -> unpriv + # https://github.com/mitogen-hq/mitogen/issues/1118 + - is_mitogen + or (ansible_facts.distribution in ["MacOSX"] + and ansible_version.full is version("2.11", ">=", strict=True)) - assert: that: - out.stdout == 'mitogen__user1' fail_msg: out={{out}} when: - # https://github.com/ansible/ansible/pull/70785 - - ansible_facts.distribution not in ["MacOSX"] - or ansible_version.full is version("2.11", ">=", strict=True) - or is_mitogen + # CI containers lack `setfacl` for unpriv -> unpriv + # https://github.com/mitogen-hq/mitogen/issues/1118 + - is_mitogen + or (ansible_facts.distribution in ["MacOSX"] + and ansible_version.full is version("2.11", ">=", strict=True)) - name: Ensure password su without chdir succeeds shell: whoami @@ -76,20 +78,22 @@ vars: ansible_become_pass: user1_password when: - # https://github.com/ansible/ansible/pull/70785 - - ansible_facts.distribution not in ["MacOSX"] - or ansible_version.full is version("2.11", ">=", strict=True) - or is_mitogen + # CI containers lack `setfacl` for unpriv -> unpriv + # https://github.com/mitogen-hq/mitogen/issues/1118 + - is_mitogen + or (ansible_facts.distribution in ["MacOSX"] + and ansible_version.full is version("2.11", ">=", strict=True)) - assert: that: - out.stdout == 'mitogen__user1' fail_msg: out={{out}} when: - # https://github.com/ansible/ansible/pull/70785 - - ansible_facts.distribution not in ["MacOSX"] - or ansible_version.full is version("2.11", ">=", strict=True) - or is_mitogen + # CI containers lack `setfacl` for unpriv -> unpriv + # https://github.com/mitogen-hq/mitogen/issues/1118 + - is_mitogen + or (ansible_facts.distribution in ["MacOSX"] + and ansible_version.full is version("2.11", ">=", strict=True)) tags: - su diff --git a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml index 0c7d30c9..fc0d3cf7 100644 --- a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml +++ b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml @@ -99,7 +99,7 @@ that: - auto_out.ansible_facts.discovered_interpreter_python is defined - auto_out.ansible_facts.discovered_interpreter_python == echoout.discovered_python.as_seen - - echoout.discovered_python.resolved == echoout.running_python.sys.executable.resolved + - echoout.discovered_python.sys.executable.as_seen == echoout.running_python.sys.executable.as_seen fail_msg: - "auto_out: {{ auto_out }}" - "echoout: {{ echoout }}" diff --git a/tests/ansible/lib/modules/custom_python_detect_environment.py b/tests/ansible/lib/modules/custom_python_detect_environment.py index d2ceaf0a..4879ac33 100644 --- a/tests/ansible/lib/modules/custom_python_detect_environment.py +++ b/tests/ansible/lib/modules/custom_python_detect_environment.py @@ -11,17 +11,6 @@ import socket import sys -try: - all -except NameError: - # Python 2.4 - def all(it): - for elem in it: - if not elem: - return False - return True - - def main(): module = AnsibleModule(argument_spec={}) module.exit_json( diff --git a/tests/ansible/lib/modules/test_echo_module.py b/tests/ansible/lib/modules/test_echo_module.py index d6a5fb9e..fe8ed69a 100644 --- a/tests/ansible/lib/modules/test_echo_module.py +++ b/tests/ansible/lib/modules/test_echo_module.py @@ -10,11 +10,97 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type import os +import stat import platform +import subprocess import sys + from ansible.module_utils.basic import AnsibleModule +# trace_realpath() and _join_tracepath() adapated from stdlib posixpath.py +# https://github.com/python/cpython/blob/v3.12.6/Lib/posixpath.py#L423-L492 +# Copyright (c) 2001 - 2023 Python Software Foundation +# Copyright (c) 2024 Alex Willmer +# License: Python Software Foundation License Version 2 + +def trace_realpath(filename, strict=False): + """ + Return the canonical path of the specified filename, and a trace of + the route taken, eliminating any symbolic links encountered in the path. + """ + path, trace, ok = _join_tracepath(filename[:0], filename, strict, seen={}, trace=[]) + return os.path.abspath(path), trace + + +def _join_tracepath(path, rest, strict, seen, trace): + """ + Join two paths, normalizing and eliminating any symbolic links encountered + in the second path. + """ + trace.append(rest) + if isinstance(path, bytes): + sep = b'/' + curdir = b'.' + pardir = b'..' + else: + sep = '/' + curdir = '.' + pardir = '..' + + if os.path.isabs(rest): + rest = rest[1:] + path = sep + + while rest: + name, _, rest = rest.partition(sep) + if not name or name == curdir: + # current dir + continue + if name == pardir: + # parent dir + if path: + path, name = os.path.split(path) + if name == pardir: + path = os.path.join(path, pardir, pardir) + else: + path = pardir + continue + newpath = os.path.join(path, name) + try: + st = os.lstat(newpath) + except OSError: + if strict: + raise + is_link = False + else: + is_link = stat.S_ISLNK(st.st_mode) + if not is_link: + path = newpath + continue + # Resolve the symbolic link + if newpath in seen: + # Already seen this path + path = seen[newpath] + if path is not None: + # use cached value + continue + # The symlink is not resolved, so we must have a symlink loop. + if strict: + # Raise OSError(errno.ELOOP) + os.stat(newpath) + else: + # Return already resolved part + rest of the path unchanged. + return os.path.join(newpath, rest), trace, False + seen[newpath] = None # not resolved symlink + path, trace, ok = _join_tracepath(path, os.readlink(newpath), strict, seen, trace) + if not ok: + return os.path.join(path, rest), False + seen[newpath] = path # resolved symlink + + return path, trace, True + + def main(): module = AnsibleModule(argument_spec=dict( facts_copy=dict(type=dict, default={}), @@ -33,7 +119,18 @@ def main(): sys.executable = "/usr/bin/python" facts_copy = module.params['facts_copy'] + discovered_interpreter_python = facts_copy['discovered_interpreter_python'] + d_i_p_realpath, d_i_p_trace = trace_realpath(discovered_interpreter_python) + d_i_p_proc = subprocess.Popen( + [discovered_interpreter_python, '-c', 'import sys; print(sys.executable)'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + + ) + d_i_p_stdout, d_i_p_stderr = d_i_p_proc.communicate() + + sys_exec_realpath, sys_exec_trace = trace_realpath(sys.executable) + result = { 'changed': False, 'ansible_facts': module.params['facts_to_override'], @@ -43,7 +140,17 @@ def main(): ), 'discovered_python': { 'as_seen': discovered_interpreter_python, - 'resolved': os.path.realpath(discovered_interpreter_python), + 'resolved': d_i_p_realpath, + 'trace': [os.path.abspath(p) for p in d_i_p_trace], + 'sys': { + 'executable': { + 'as_seen': d_i_p_stdout.decode('ascii').rstrip('\n'), + 'proc': { + 'stderr': d_i_p_stderr.decode('ascii'), + 'returncode': d_i_p_proc.returncode, + }, + }, + }, }, 'running_python': { 'platform': { @@ -54,7 +161,8 @@ def main(): 'sys': { 'executable': { 'as_seen': sys.executable, - 'resolved': os.path.realpath(sys.executable), + 'resolved': sys_exec_realpath, + 'trace': [os.path.abspath(p) for p in sys_exec_trace], }, 'platform': sys.platform, 'version_info': { diff --git a/tests/connection_test.py b/tests/connection_test.py index c3146954..79e27aee 100644 --- a/tests/connection_test.py +++ b/tests/connection_test.py @@ -1,6 +1,5 @@ import logging import os -import signal import sys import tempfile import threading diff --git a/tests/image_prep/_container_finalize.yml b/tests/image_prep/_container_finalize.yml index d61d9b3b..5329fefa 100644 --- a/tests/image_prep/_container_finalize.yml +++ b/tests/image_prep/_container_finalize.yml @@ -9,7 +9,7 @@ --change 'EXPOSE 22' --change 'CMD ["/usr/sbin/sshd", "-D"]' {{ inventory_hostname }} - public.ecr.aws/n5z0e8q9/{{ inventory_hostname }}-test + {{ container_image_name }} delegate_to: localhost - name: Stop containers diff --git a/tests/image_prep/group_vars/all.yml b/tests/image_prep/group_vars/all.yml index 5f182f86..91ff934d 100644 --- a/tests/image_prep/group_vars/all.yml +++ b/tests/image_prep/group_vars/all.yml @@ -4,6 +4,9 @@ common_packages: - strace - sudo +container_image_name: "{{ container_registry }}/{{ inventory_hostname }}-test" +container_registry: public.ecr.aws/n5z0e8q9 + sudo_group: MacOSX: admin Debian: sudo diff --git a/tests/iter_split_test.py b/tests/iter_split_test.py index 74c46c0a..39c11a47 100644 --- a/tests/iter_split_test.py +++ b/tests/iter_split_test.py @@ -2,11 +2,7 @@ import unittest import mitogen.core -try: - next -except NameError: - def next(it): - return it.next() +from mitogen.core import next class IterSplitTest(unittest.TestCase): diff --git a/tests/lxc_test.py b/tests/lxc_test.py index b9ebfa53..a613cefa 100644 --- a/tests/lxc_test.py +++ b/tests/lxc_test.py @@ -3,10 +3,7 @@ import os import mitogen.lxc import mitogen.parent -try: - any -except NameError: - from mitogen.core import any +from mitogen.core import any import testlib diff --git a/tests/poller_test.py b/tests/poller_test.py index f915df0a..0abc836d 100644 --- a/tests/poller_test.py +++ b/tests/poller_test.py @@ -8,13 +8,9 @@ import unittest import mitogen.core import mitogen.parent -import testlib +from mitogen.core import next -try: - next -except NameError: - # Python 2.4 - from mitogen.core import next +import testlib class SockMixin(object): diff --git a/tests/testlib.py b/tests/testlib.py index a52292ce..76743e82 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -51,6 +51,12 @@ except NameError: LOG = logging.getLogger(__name__) +DISTRO = os.environ.get('MITOGEN_TEST_DISTRO', 'debian9') +IMAGE_TEMPLATE = os.environ.get( + 'MITOGEN_TEST_IMAGE_TEMPLATE', + 'public.ecr.aws/n5z0e8q9/%(distro)s-test', +) + TESTS_DIR = os.path.join(os.path.dirname(__file__)) ANSIBLE_LIB_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib') ANSIBLE_MODULE_UTILS_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils') @@ -509,6 +515,7 @@ class TestCase(unittest.TestCase): def get_docker_host(): + # Duplicated in ci_lib url = os.environ.get('DOCKER_HOST') if url in (None, 'http+docker://localunixsocket'): return 'localhost' @@ -549,19 +556,23 @@ class DockerizedSshDaemon(object): ] subprocess.check_output(args) - def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')): - if '-' in mitogen_test_distro: - distro, _py3 = mitogen_test_distro.split('-') - else: - distro = mitogen_test_distro - _py3 = None + def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE): + # Code duplicated in ci_lib.py, both should be updated together + distro_pattern = re.compile(r''' + (?P(?P[a-z]+)[0-9]+) + (?:-(?Ppy3))? + (?:\*(?P[0-9]+))? + ''', + re.VERBOSE, + ) + d = distro_pattern.match(distro).groupdict(default=None) - if _py3 == 'py3': + if d.pop('py') == 'py3': self.python_path = '/usr/bin/python3' else: self.python_path = '/usr/bin/python' - self.image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,) + self.image = image_template % d self.start_container() self.host = self.get_host() self.port = self.get_port(self.container_name) @@ -601,6 +612,9 @@ class DockerizedSshDaemon(object): class BrokerMixin(object): broker_class = mitogen.master.Broker + + # Flag for tests that shutdown the broker themself + # e.g. unix_test.ListenerTest broker_shutdown = False def setUp(self): diff --git a/tox.ini b/tox.ini index 870a6345..9fb31bdc 100644 --- a/tox.ini +++ b/tox.ini @@ -48,6 +48,9 @@ # ansible == 9.x ansible-core ~= 2.16.0 # ansible == 10.x ansible-core ~= 2.17.0 +# See also +# - https://docs.ansible.com/ansible/latest/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix + [tox] envlist = init, @@ -86,10 +89,7 @@ deps = install_command = python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages} commands_pre = - mode_ansible: {toxinidir}/.ci/ansible_install.py mode_debops_common: {toxinidir}/.ci/debops_common_install.py - mode_localhost: {toxinidir}/.ci/localhost_ansible_install.py - mode_mitogen: {toxinidir}/.ci/mitogen_install.py commands = mode_ansible: {toxinidir}/.ci/ansible_tests.py mode_debops_common: {toxinidir}/.ci/debops_common_tests.py @@ -97,10 +97,8 @@ commands = mode_mitogen: {toxinidir}/.ci/mitogen_tests.py passenv = ANSIBLE_* - AWS_ACCESS_KEY_ID - AWS_DEFAULT_REGION - AWS_SECRET_ACCESS_KEY HOME + MITOGEN_* # Azure DevOps, TF_BUILD is set to 'True' when running in a build task # https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables TF_BUILD