diff --git a/.ci/README.md b/.ci/README.md index 9248ac58..9a5e8898 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -2,7 +2,7 @@ # `.ci` This directory contains scripts for Continuous Integration platforms. Currently -Azure Pipelines, but they will also happily run on any Debian-like machine. +GitHub Actions, but ideally they will also run on any Debian-like machine. The scripts are usually split into `_install` and `_test` steps. The `_install` step will damage your machine, the `_test` step will just run the tests the way diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml deleted file mode 100644 index 791372af..00000000 --- a/.ci/azure-pipelines-steps.yml +++ /dev/null @@ -1,105 +0,0 @@ -# Each step entry runs a task (Azure Pipelines analog of an Ansible module). -# https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/?view=azure-pipelines&viewFallbackFrom=azure-devops#tool - -# `{script: ...}` is shorthand for `{task: CmdLine@, inputs: {script: ...}}`. -# The shell is bash. -# https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/steps-script?view=azure-pipelines -# https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/cmd-line-v2?view=azure-pipelines - -steps: -- task: UsePythonVersion@0 - displayName: Install python - inputs: - githubToken: '$(GITHUB_PYVER_TOKEN)' - versionSpec: '$(python.version)' - condition: ne(variables['python.version'], '') - -- script: | - set -o errexit - set -o nounset - set -o pipefail - - aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws - displayName: Authenticate to container registry - condition: eq(variables['Agent.OS'], 'Linux') - env: - AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) - AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) - AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION) - -- script: | - set -o errexit - set -o nounset - set -o pipefail - - sudo apt-get update - sudo apt-get install -y python2-dev python3-pip virtualenv - displayName: Install build deps - condition: and(eq(variables['python.version'], ''), eq(variables['Agent.OS'], 'Linux')) - -- script: | - set -o errexit - set -o nounset - set -o pipefail - - # macOS builders lack a realpath command - type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version - type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version - type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version - echo - - if [ -e /usr/bin/python ]; then - echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')" - fi - - if [ -e /usr/bin/python2 ]; then - echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')" - fi - - if [ -e /usr/bin/python2.7 ]; then - echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')" - fi - displayName: Show python versions - -- script: | - set -o errexit - set -o nounset - set -o pipefail - - # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) - PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "$(tox.env)"))') - - if [[ -z $PYTHON ]]; then - echo 1>&2 "Python interpreter could not be determined" - exit 1 - fi - - if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then - "$PYTHON" -m ensurepip --user --altinstall --no-default-pip - "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" - elif [[ $PYTHON == "python2.7" ]]; then - curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py" - "$PYTHON" get-pip.py --user --no-python-version-warning - # Avoid Python 2.x pip masking system pip - rm -f ~/.local/bin/{easy_install,pip,wheel} - "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" - else - "$PYTHON" -m pip install -r "tests/requirements-tox.txt" - fi - displayName: Install tooling - -- script: | - set -o errexit - set -o nounset - set -o pipefail - - # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) - PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "$(tox.env)"))') - - if [[ -z $PYTHON ]]; then - echo 1>&2 "Python interpreter could not be determined" - exit 1 - fi - - "$PYTHON" -m tox -e "$(tox.env)" - displayName: "Run tests" diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml deleted file mode 100644 index 0bf4556b..00000000 --- a/.ci/azure-pipelines.yml +++ /dev/null @@ -1,157 +0,0 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - -# User defined variables are also injected as environment variables -# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables#environment-variables -#variables: - #ANSIBLE_VERBOSITY: 3 - -trigger: - branches: - include: - - "*" - exclude: - - docs-master - -jobs: -- job: mac12 - # vanilla Ansible is really slow - timeoutInMinutes: 120 - steps: - - template: azure-pipelines-steps.yml - pool: - # https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md - vmImage: macOS-12 - strategy: - matrix: - Mito_312: - tox.env: py312-mode_mitogen - Loc_312_10: - tox.env: py312-mode_localhost-ansible10 - Van_312_10: - tox.env: py312-mode_localhost-ansible10-strategy_linear - -- job: Linux - pool: - # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md - vmImage: ubuntu-20.04 - steps: - - template: azure-pipelines-steps.yml - strategy: - matrix: - Mito_27_centos6: - tox.env: py27-mode_mitogen-distro_centos6 - Mito_27_centos7: - tox.env: py27-mode_mitogen-distro_centos7 - Mito_27_centos8: - tox.env: py27-mode_mitogen-distro_centos8 - Mito_27_debian9: - tox.env: py27-mode_mitogen-distro_debian9 - Mito_27_debian10: - tox.env: py27-mode_mitogen-distro_debian10 - Mito_27_debian11: - tox.env: py27-mode_mitogen-distro_debian11 - Mito_27_ubuntu1604: - tox.env: py27-mode_mitogen-distro_ubuntu1604 - Mito_27_ubuntu1804: - tox.env: py27-mode_mitogen-distro_ubuntu1804 - Mito_27_ubuntu2004: - tox.env: py27-mode_mitogen-distro_ubuntu2004 - - Mito_36_centos6: - python.version: '3.6' - tox.env: py36-mode_mitogen-distro_centos6 - Mito_36_centos7: - python.version: '3.6' - tox.env: py36-mode_mitogen-distro_centos7 - Mito_36_centos8: - python.version: '3.6' - tox.env: py36-mode_mitogen-distro_centos8 - Mito_36_debian9: - python.version: '3.6' - tox.env: py36-mode_mitogen-distro_debian9 - Mito_36_debian10: - python.version: '3.6' - tox.env: py36-mode_mitogen-distro_debian10 - Mito_36_debian11: - python.version: '3.6' - tox.env: py36-mode_mitogen-distro_debian11 - Mito_36_ubuntu1604: - python.version: '3.6' - tox.env: py36-mode_mitogen-distro_ubuntu1604 - Mito_36_ubuntu1804: - python.version: '3.6' - tox.env: py36-mode_mitogen-distro_ubuntu1804 - Mito_36_ubuntu2004: - python.version: '3.6' - tox.env: py36-mode_mitogen-distro_ubuntu2004 - - Mito_312_centos6: - python.version: '3.12' - tox.env: py312-mode_mitogen-distro_centos6 - Mito_312_centos7: - python.version: '3.12' - tox.env: py312-mode_mitogen-distro_centos7 - Mito_312_centos8: - python.version: '3.12' - tox.env: py312-mode_mitogen-distro_centos8 - Mito_312_debian9: - python.version: '3.12' - tox.env: py312-mode_mitogen-distro_debian9 - Mito_312_debian10: - python.version: '3.12' - tox.env: py312-mode_mitogen-distro_debian10 - Mito_312_debian11: - python.version: '3.12' - tox.env: py312-mode_mitogen-distro_debian11 - Mito_312_ubuntu1604: - python.version: '3.12' - tox.env: py312-mode_mitogen-distro_ubuntu1604 - Mito_312_ubuntu1804: - python.version: '3.12' - tox.env: py312-mode_mitogen-distro_ubuntu1804 - Mito_312_ubuntu2004: - python.version: '3.12' - tox.env: py312-mode_mitogen-distro_ubuntu2004 - - Ans_27_210: - tox.env: py27-mode_ansible-ansible2.10 - Ans_27_4: - tox.env: py27-mode_ansible-ansible4 - - Ans_36_210: - python.version: '3.6' - tox.env: py36-mode_ansible-ansible2.10 - Ans_36_4: - python.version: '3.6' - tox.env: py36-mode_ansible-ansible4 - - Ans_311_210: - python.version: '3.11' - tox.env: py311-mode_ansible-ansible2.10 - Ans_311_3: - python.version: '3.11' - tox.env: py311-mode_ansible-ansible3 - Ans_311_4: - python.version: '3.11' - tox.env: py311-mode_ansible-ansible4 - Ans_311_5: - python.version: '3.11' - tox.env: py311-mode_ansible-ansible5 - Ans_312_6: - python.version: '3.12' - tox.env: py312-mode_ansible-ansible6 - Ans_312_7: - python.version: '3.12' - tox.env: py312-mode_ansible-ansible7 - Ans_312_8: - python.version: '3.12' - tox.env: py312-mode_ansible-ansible8 - Ans_312_9: - python.version: '3.12' - tox.env: py312-mode_ansible-ansible9 - Ans_312_10: - python.version: '3.12' - tox.env: py312-mode_ansible-ansible10 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2c819e7d..f3f31d82 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,24 +48,24 @@ jobs: - name: Ans_311_5 python_version: '3.11' tox_env: py311-mode_ansible-ansible5 - - name: Ans_312_6 - python_version: '3.12' - tox_env: py312-mode_ansible-ansible6 - - name: Ans_312_7 - python_version: '3.12' - tox_env: py312-mode_ansible-ansible7 - - name: Ans_312_8 - python_version: '3.12' - tox_env: py312-mode_ansible-ansible8 - - name: Ans_312_9 - python_version: '3.12' - tox_env: py312-mode_ansible-ansible9 - - name: Ans_312_10 - python_version: '3.12' - tox_env: py312-mode_ansible-ansible10 - - name: Van_312_10 - python_version: '3.12' - tox_env: py312-mode_ansible-ansible10-strategy_linear + - name: Ans_313_6 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible6 + - name: Ans_313_7 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible7 + - name: Ans_313_8 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible8 + - name: Ans_313_9 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible9 + - name: Ans_313_10 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible10 + - name: Van_313_10 + python_version: '3.13' + tox_env: py313-mode_ansible-ansible10-strategy_linear - name: Mito_27_centos6 tox_env: py27-mode_mitogen-distro_centos6 @@ -114,33 +114,33 @@ jobs: python_version: '3.6' tox_env: py36-mode_mitogen-distro_ubuntu2004 - - name: Mito_312_centos6 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_centos6 - - name: Mito_312_centos7 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_centos7 - - name: Mito_312_centos8 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_centos8 - - name: Mito_312_debian9 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_debian9 - - name: Mito_312_debian10 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_debian10 - - name: Mito_312_debian11 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_debian11 - - name: Mito_312_ubuntu1604 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_ubuntu1604 - - name: Mito_312_ubuntu1804 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_ubuntu1804 - - name: Mito_312_ubuntu2004 - python_version: '3.12' - tox_env: py312-mode_mitogen-distro_ubuntu2004 + - name: Mito_313_centos6 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_centos6 + - name: Mito_313_centos7 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_centos7 + - name: Mito_313_centos8 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_centos8 + - name: Mito_313_debian9 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_debian9 + - name: Mito_313_debian10 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_debian10 + - name: Mito_313_debian11 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_debian11 + - name: Mito_313_ubuntu1604 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_ubuntu1604 + - name: Mito_313_ubuntu1804 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_ubuntu1804 + - name: Mito_313_ubuntu2004 + python_version: '3.13' + tox_env: py313-mode_mitogen-distro_ubuntu2004 steps: - uses: actions/checkout@v4 @@ -232,18 +232,21 @@ jobs: include: - name: Mito_27 tox_env: py27-mode_mitogen - - name: Mito_312 - tox_env: py312-mode_mitogen + - name: Mito_313 + python_version: '3.13' + tox_env: py313-mode_mitogen - name: Loc_27_210 tox_env: py27-mode_localhost-ansible2.10 - - name: Loc_312_10 - tox_env: py312-mode_localhost-ansible10 + - name: Loc_313_10 + python_version: '3.13' + tox_env: py313-mode_localhost-ansible10 - name: Van_27_210 tox_env: py27-mode_localhost-ansible2.10-strategy_linear - - name: Van_312_10 - tox_env: py312-mode_localhost-ansible10-strategy_linear + - name: Van_313_10 + python_version: '3.13' + tox_env: py313-mode_localhost-ansible10-strategy_linear steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index 0d4d1b30..c3cd9a87 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,11 @@ # Mitogen +[![PyPI - Version](https://img.shields.io/pypi/v/mitogen)](https://pypi.org/project/mitogen/) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/mitogen)](https://pypi.org/project/mitogen/) +[![Build Status](https://img.shields.io/github/actions/workflow/status/mitogen-hq/mitogen/tests.yml?branch=master)](https://github.com/mitogen-hq/mitogen/actions?query=branch%3Amaster) + Please see the documentation. ![](https://i.imgur.com/eBM6LhJ.gif) [![Total alerts](https://img.shields.io/lgtm/alerts/g/mitogen-hq/mitogen.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/mitogen-hq/mitogen/alerts/) - -[![Build Status](https://dev.azure.com/mitogen-hq/mitogen/_apis/build/status/mitogen-hq.mitogen?branchName=master)](https://dev.azure.com/mitogen-hq/mitogen/_build/latest?definitionId=1&branchName=master) diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py index a3f66eac..a715b2b0 100644 --- a/ansible_mitogen/connection.py +++ b/ansible_mitogen/connection.py @@ -158,6 +158,7 @@ def _connect_ssh(spec): } } + def _connect_buildah(spec): """ Return ContextService arguments for a Buildah connection. @@ -173,6 +174,7 @@ def _connect_buildah(spec): } } + def _connect_docker(spec): """ Return ContextService arguments for a Docker connection. @@ -276,6 +278,7 @@ def _connect_podman(spec): } } + def _connect_setns(spec, kind=None): """ Return ContextService arguments for a mitogen_setns connection. diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py index 38f351ed..1b6512e8 100644 --- a/ansible_mitogen/mixins.py +++ b/ansible_mitogen/mixins.py @@ -36,8 +36,6 @@ import random import traceback import ansible -import ansible.constants -import ansible.plugins import ansible.plugins.action import ansible.utils.unsafe_proxy import ansible.vars.clean diff --git a/ansible_mitogen/transport_config.py b/ansible_mitogen/transport_config.py index a3336365..a9f67209 100644 --- a/ansible_mitogen/transport_config.py +++ b/ansible_mitogen/transport_config.py @@ -64,6 +64,7 @@ __metaclass__ = type import abc import logging import os + import ansible.utils.shlex import ansible.constants as C import ansible.executor.interpreter_discovery @@ -88,12 +89,12 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth # keep trying different interpreters until we don't error if action._finding_python_interpreter: return action._possible_python_interpreter - + if s in ['auto', 'auto_legacy', 'auto_silent', 'auto_legacy_silent']: # python is the only supported interpreter_name as of Ansible 2.8.8 interpreter_name = 'python' discovered_interpreter_config = u'discovered_interpreter_%s' % interpreter_name - + if task_vars.get('ansible_facts') is None: task_vars['ansible_facts'] = {} @@ -134,7 +135,7 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth def parse_python_path(s, task_vars, action, rediscover_python): """ Given the string set for ansible_python_interpeter, parse it using shell - syntax and return an appropriate argument vector. If the value detected is + syntax and return an appropriate argument vector. If the value detected is one of interpreter discovery then run that first. Caches python interpreter discovery value in `facts_from_task_vars` like how Ansible handles this. """ @@ -433,7 +434,7 @@ class PlayContextSpec(Spec): return self._play_context.remote_addr def remote_user(self): - return self._play_context.remote_user + return self._connection_option('remote_user') def become(self): return self._play_context.become @@ -463,7 +464,7 @@ class PlayContextSpec(Spec): return optional_secret(self._connection_option('password')) def port(self): - return self._play_context.port + return self._connection_option('port') def python_path(self, rediscover_python=False): s = self._connection.get_task_var('ansible_python_interpreter') diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst index d818edd7..73ebba4d 100644 --- a/docs/ansible_detailed.rst +++ b/docs/ansible_detailed.rst @@ -132,13 +132,13 @@ Noteworthy Differences | 5 | 3.8 - 3.11 | +-----------------+-----------------+ | 6 | | - +-----------------+ 3.8 - 3.12 | + +-----------------+ 3.8 - 3.13 | | 7 | | +-----------------+-----------------+ - | 8 | 3.9 - 3.12 | + | 8 | 3.9 - 3.13 | +-----------------+-----------------+ | 9 | | - +-----------------+ 3.10 - 3.12 | + +-----------------+ 3.10 - 3.13 | | 10 | | +-----------------+-----------------+ @@ -1275,7 +1275,7 @@ on each process whose name begins with ``mitogen:``:: [pid 29858] futex(0x55ea9be52f60, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, NULL, 0xffffffff ^C - $ + $ This shows one thread waiting on IO (``poll``) and two more waiting on the same lock. It is taken from a real example of a deadlock due to a forking bug. diff --git a/docs/changelog.rst b/docs/changelog.rst index dabf5f7d..ea45a735 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,7 +18,18 @@ To avail of fixes in an unreleased version, please download a ZIP file `directly from GitHub `_. -v0.3.11 (2024-10-07) +v0.3.13 (2024-10-09) +-------------------- + +* :gh:issue:`1138` CI: Complete migration from Azure DevOps Pipelines to + GitHub Actions +* :gh:issue:`1116` :mod:`ansible_mitogen`: Support for templated variable + `ansible_ssh_user`. +* :gh:issue:`978` :mod:`ansible_mitogen`: Support templated Ansible SSH port. +* :gh:issue:`1073` Python 3.13 support + + +v0.3.12 (2024-10-07) -------------------- * :gh:issue:`1106` :mod:`ansible_mitogen`: Support for `ansible_ssh_password` @@ -112,7 +123,7 @@ v0.3.4 (2023-07-02) * :gh:issue:`929` Support Ansible 6 and ansible-core 2.13 * :gh:issue:`832` Fix runtime error when using the ansible.builtin.dnf module multiple times -* :gh:issue:`925` :class:`ansible_mitogen.connection.Connection` no longer tries to close the +* :gh:issue:`925` :class:`ansible_mitogen.connection.Connection` no longer tries to close the connection on destruction. This is expected to reduce cases of `mitogen.core.Error: An attempt was made to enqueue a message with a Broker that has already exitted`. However it may result in resource leaks. diff --git a/docs/contributors.rst b/docs/contributors.rst index 4e9e58bd..69dc1e76 100644 --- a/docs/contributors.rst +++ b/docs/contributors.rst @@ -127,6 +127,7 @@ sponsorship and outstanding future-thinking of its early adopters.
  • jgadling
  • John F Wall — Making Ansible Great with Massive Parallelism
  • Jonathan Rosser
    • +
  • Joshua M. Keyes
  • KennethC
  • Luca Berruti
  • Lewis Bellwood — Happy to be apart of a great project.
    • diff --git a/docs/howitworks.rst b/docs/howitworks.rst index 27b109fe..d7606b11 100644 --- a/docs/howitworks.rst +++ b/docs/howitworks.rst @@ -1038,7 +1038,7 @@ receive items in the order they are requested, as they become available. Mitogen enables SSH compression by default, there are circumstances where disabling SSH compression is desirable, and many scenarios for future connection methods where transport-layer compression is not supported at - all. + all. .. [#f2] Compression may seem redundant, however it is basically free and reducing IO is always a good idea. The 33% / 200 byte saving may mean the presence or diff --git a/examples/mitogen-fuse.py b/examples/mitogen-fuse.py index 55b272d9..73101fb8 100644 --- a/examples/mitogen-fuse.py +++ b/examples/mitogen-fuse.py @@ -119,7 +119,7 @@ def _chroot(path): os.chroot(path) -class Operations(fuse.Operations): # fuse.LoggingMixIn, +class Operations(fuse.Operations): # fuse.LoggingMixIn, def __init__(self, host, path='.'): self.host = host self.root = path diff --git a/examples/mitop.py b/examples/mitop.py index 8749e12a..72a60bf3 100644 --- a/examples/mitop.py +++ b/examples/mitop.py @@ -61,7 +61,7 @@ def child_main(sender, delay): Executed on the main thread of the Python interpreter running on each target machine, Context.call() from the master. It simply sends the output of the UNIX 'ps' command at regular intervals toward a Receiver on master. - + :param mitogen.core.Sender sender: The Sender to use for delivering our result. This could target anywhere, but the sender supplied by the master simply causes results diff --git a/examples/the_basics.py b/examples/the_basics.py index 0dcd4049..af9903b0 100644 --- a/examples/the_basics.py +++ b/examples/the_basics.py @@ -10,7 +10,6 @@ from __future__ import print_function import hashlib import io import os -import spwd import mitogen.core import mitogen.master @@ -57,21 +56,6 @@ def streamy_download_file(context, path): } -def get_password_hash(username): - """ - Fetch a user's password hash. - """ - try: - h = spwd.getspnam(username) - except KeyError: - return None - - # mitogen.core.Secret() is a Unicode subclass with a repr() that hides the - # secret data. This keeps secret stuff out of logs. Like blobs, secrets can - # also be serialized. - return mitogen.core.Secret(h) - - def md5sum(path): """ Return the MD5 checksum for a file. diff --git a/mitogen/__init__.py b/mitogen/__init__.py index 003ea701..48baa7c8 100644 --- a/mitogen/__init__.py +++ b/mitogen/__init__.py @@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup. #: Library version as a tuple. -__version__ = (0, 3, 12) +__version__ = (0, 3, 13) #: This is :data:`False` in slave contexts. Previously it was used to prevent diff --git a/mitogen/core.py b/mitogen/core.py index 9b225ed7..49f92cae 100644 --- a/mitogen/core.py +++ b/mitogen/core.py @@ -3290,7 +3290,7 @@ class Router(object): This can be used from any thread, but its output is only meaningful from the context of the :class:`Broker` thread, as disconnection or replacement could happen in parallel on the broker thread at any - moment. + moment. """ return ( self._stream_by_id.get(dst_id) or diff --git a/mitogen/master.py b/mitogen/master.py index 51b29b82..865c9dc1 100644 --- a/mitogen/master.py +++ b/mitogen/master.py @@ -652,7 +652,7 @@ class ParentImpEnumerationMethod(FinderMethod): insane) parent package, and if no insane parents exist, simply use :mod:`sys.path` to search for it from scratch on the filesystem using the normal Python lookup mechanism. - + This is required for older versions of :mod:`ansible.compat.six`, :mod:`plumbum.colors`, Ansible 2.8 :mod:`ansible.module_utils.distro` and its submodule :mod:`ansible.module_utils.distro._distro`. diff --git a/mitogen/parent.py b/mitogen/parent.py index dd51b697..fa3092c1 100644 --- a/mitogen/parent.py +++ b/mitogen/parent.py @@ -631,7 +631,7 @@ class TimerList(object): def get_timeout(self): """ Return the floating point seconds until the next event is due. - + :returns: Floating point delay, or 0.0, or :data:`None` if no events are scheduled. diff --git a/setup.py b/setup.py index b17dab9d..9cb30427 100644 --- a/setup.py +++ b/setup.py @@ -79,6 +79,7 @@ setup( 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: System :: Distributed Computing', 'Topic :: System :: Systems Administration', diff --git a/tests/README.md b/tests/README.md index 65226e87..35a8775c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -7,7 +7,7 @@ started in September 2017. Pull requests in this area are very welcome! ## Running The Tests -[![Build Status](https://dev.azure.com/mitogen-hq/mitogen/_apis/build/status/mitogen-hq.mitogen?branchName=master)](https://dev.azure.com/mitogen-hq/mitogen/_build/latest?definitionId=1&branchName=master) +[![Build Status](https://img.shields.io/github/actions/workflow/status/mitogen-hq/mitogen/tests.yml?branch=master)](https://github.com/mitogen-hq/mitogen/actions?query=branch%3Amaster) Your computer should have an Internet connection, and the ``docker`` command line tool should be able to connect to a working Docker daemon (localhost or diff --git a/tests/ansible/hosts/default.hosts b/tests/ansible/hosts/default.hosts index 8ed80787..609cd9f8 100644 --- a/tests/ansible/hosts/default.hosts +++ b/tests/ansible/hosts/default.hosts @@ -25,11 +25,11 @@ tt-bare [tt_targets_bare:vars] ansible_host=localhost -ansible_user=mitogen__has_sudo_nopw [tt_targets_inventory] -tt-password ansible_password="{{ 'has_sudo_nopw_password' | trim }}" +tt-password ansible_password="{{ 'has_sudo_nopw_password' | trim }}" ansible_user=mitogen__has_sudo_nopw +tt-port ansible_password=has_sudo_nopw_password ansible_port="{{ 22 | int }}" ansible_user=mitogen__has_sudo_nopw +tt-remote-user ansible_password=has_sudo_nopw_password ansible_user="{{ 'mitogen__has_sudo_nopw' | trim }}" [tt_targets_inventory:vars] ansible_host=localhost -ansible_user=mitogen__has_sudo_nopw diff --git a/tests/ansible/integration/_expected_ssh_port.yml b/tests/ansible/integration/_expected_ssh_port.yml new file mode 100644 index 00000000..442659a5 --- /dev/null +++ b/tests/ansible/integration/_expected_ssh_port.yml @@ -0,0 +1,18 @@ +# Ansible removed its default SSH port in May 2021, defering to the SSH +# implementation. +# https://github.com/ansible/ansible/commit/45618a6f3856f7332df8afe4adc40d85649a70da + +# Careful templating is needed to preseve the type(s) of expected_ssh_port, +# particularly in combination with the assert_equal action plugin. +# Do: {{ expected_ssh_port }} +# Don't: {{ expected_ssh_port | any_filter }} +# Don't: {% if ...%}{{ expected_ssh_port }}{% else %}...{% endif %} +# https://stackoverflow.com/questions/66102524/ansible-set-fact-type-cast/66104814#66104814 + +- set_fact: + expected_ssh_port: null + when: ansible_version.full is version('2.11.1', '>=', strict=True) + +- set_fact: + expected_ssh_port: 22 + when: ansible_version.full is version('2.11.1', '<', strict=True) diff --git a/tests/ansible/integration/connection_delegation/delegate_to_template.yml b/tests/ansible/integration/connection_delegation/delegate_to_template.yml index f9ad9e0b..60a67b82 100644 --- a/tests/ansible/integration/connection_delegation/delegate_to_template.yml +++ b/tests/ansible/integration/connection_delegation/delegate_to_template.yml @@ -11,11 +11,11 @@ - name: integration/connection_delegation/delegate_to_template.yml vars: physical_host: "cd-normal-alias" - physical_hosts: ["cd-normal-alias", "cd-normal-normal"] hosts: test-targets gather_facts: no tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - meta: end_play when: @@ -67,7 +67,7 @@ 'keepalive_interval': 30, 'keepalive_count': 10, 'password': null, - 'port': null, + 'port': '{{ expected_ssh_port }}', 'python_path': ['python3000'], 'remote_name': null, 'ssh_args': [ diff --git a/tests/ansible/integration/connection_delegation/stack_construction.yml b/tests/ansible/integration/connection_delegation/stack_construction.yml index 8cf064bb..b0475275 100644 --- a/tests/ansible/integration/connection_delegation/stack_construction.yml +++ b/tests/ansible/integration/connection_delegation/stack_construction.yml @@ -55,6 +55,7 @@ - hosts: cd-normal tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - mitogen_get_stack: delegate_to: cd-alias register: out @@ -72,7 +73,7 @@ 'keepalive_interval': 30, 'keepalive_count': 10, 'password': null, - 'port': null, + 'port': '{{ expected_ssh_port }}', "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ @@ -98,6 +99,7 @@ - hosts: cd-alias tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - mitogen_get_stack: register: out - assert_equal: @@ -114,7 +116,7 @@ 'keepalive_interval': 30, 'keepalive_count': 10, 'password': null, - 'port': null, + 'port': '{{ expected_ssh_port }}', "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ @@ -140,6 +142,7 @@ - hosts: cd-normal-normal tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - mitogen_get_stack: register: out - assert_equal: @@ -167,7 +170,7 @@ 'keepalive_interval': 30, 'keepalive_count': 10, 'password': null, - 'port': null, + 'port': '{{ expected_ssh_port }}', "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ @@ -193,6 +196,7 @@ - hosts: cd-normal-alias tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - mitogen_get_stack: register: out - assert_equal: @@ -237,7 +241,7 @@ 'keepalive_interval': 30, 'keepalive_count': 10, 'password': null, - 'port': null, + 'port': '{{ expected_ssh_port }}', "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ @@ -262,6 +266,7 @@ - hosts: cd-newuser-normal-normal tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - mitogen_get_stack: register: out - assert_equal: @@ -289,7 +294,7 @@ 'keepalive_interval': 30, 'keepalive_count': 10, 'password': null, - 'port': null, + 'port': '{{ expected_ssh_port }}', "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ @@ -315,6 +320,7 @@ - hosts: cd-newuser-normal-normal tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - mitogen_get_stack: delegate_to: cd-alias register: out @@ -332,7 +338,7 @@ 'keepalive_interval': 30, 'keepalive_count': 10, 'password': null, - 'port': null, + 'port': '{{ expected_ssh_port }}', "python_path": ["python3000"], 'remote_name': null, 'ssh_args': [ diff --git a/tests/ansible/integration/interpreter_discovery/complex_args.yml b/tests/ansible/integration/interpreter_discovery/complex_args.yml index af2b5e46..f9770876 100644 --- a/tests/ansible/integration/interpreter_discovery/complex_args.yml +++ b/tests/ansible/integration/interpreter_discovery/complex_args.yml @@ -52,7 +52,7 @@ {% if "1" == "1" %} {{ special_python }} {% else %} - python + python {% endif %} tags: - complex_args diff --git a/tests/ansible/integration/ssh/templated_by_play_taskvar.yml b/tests/ansible/integration/ssh/templated_by_play_taskvar.yml index bc4ef1d8..fd4bc848 100644 --- a/tests/ansible/integration/ssh/templated_by_play_taskvar.yml +++ b/tests/ansible/integration/ssh/templated_by_play_taskvar.yml @@ -3,6 +3,8 @@ gather_facts: false vars: ansible_password: "{{ 'has_sudo_nopw_password' | trim }}" + ansible_port: "{{ hostvars[groups['test-targets'][0]].ansible_port | default(22) }}" + ansible_user: "{{ 'mitogen__has_sudo_nopw' | trim }}" tasks: - meta: reset_connection diff --git a/tests/ansible/integration/transport_config/port.yml b/tests/ansible/integration/transport_config/port.yml index 1b8e04a0..abc68058 100644 --- a/tests/ansible/integration/transport_config/port.yml +++ b/tests/ansible/integration/transport_config/port.yml @@ -6,13 +6,15 @@ hosts: tc-port-unset tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - {mitogen_get_stack: {}, register: out} - assert: that: - out.result|length == 1 - out.result[0].method == "ssh" - - out.result[0].kwargs.port == None + - out.result[0].kwargs.port == expected_ssh_port fail_msg: | + expected_ssh_port={{ expected_ssh_port }} out={{ out }} tags: - mitogen_only @@ -54,6 +56,7 @@ vars: {mitogen_via: tc-port-explicit-ssh} tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -61,8 +64,9 @@ - out.result[0].method == "ssh" - out.result[0].kwargs.port == 4321 - out.result[1].method == "ssh" - - out.result[1].kwargs.port == None + - out.result[1].kwargs.port == expected_ssh_port fail_msg: | + expected_ssh_port={{ expected_ssh_port }} out={{ out }} tags: - mitogen_only @@ -86,6 +90,7 @@ vars: {mitogen_via: tc-port-explicit-port} tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -93,8 +98,9 @@ - out.result[0].method == "ssh" - out.result[0].kwargs.port == 1234 - out.result[1].method == "ssh" - - out.result[1].kwargs.port == None + - out.result[1].kwargs.port == expected_ssh_port fail_msg: | + expected_ssh_port={{ expected_ssh_port }} out={{ out }} tags: - mitogen_only @@ -119,6 +125,7 @@ vars: {mitogen_via: tc-port-both} tasks: - include_tasks: ../_mitogen_only.yml + - include_tasks: ../_expected_ssh_port.yml - {mitogen_get_stack: {}, register: out} - assert: that: @@ -126,8 +133,9 @@ - out.result[0].method == "ssh" - out.result[0].kwargs.port == 1532 - out.result[1].method == "ssh" - - out.result[1].kwargs.port == None + - out.result[1].kwargs.port == expected_ssh_port fail_msg: | + expected_ssh_port={{ expected_ssh_port }} out={{ out }} tags: - mitogen_only diff --git a/tests/ansible/templates/test-targets.j2 b/tests/ansible/templates/test-targets.j2 index 37a0725a..0fdef20b 100644 --- a/tests/ansible/templates/test-targets.j2 +++ b/tests/ansible/templates/test-targets.j2 @@ -47,12 +47,12 @@ tt-bare ansible_host={{ tt.hostname }} ansible_port={{ tt.port }} ansible_python_interpreter={{ tt.python_path }} -ansible_user=mitogen__has_sudo_nopw [tt_targets_inventory] -tt-password ansible_password="{{ '{{' }} 'has_sudo_nopw_password' | trim {{ '}}' }}" ansible_port={{ tt.port }} +tt-password ansible_password="{{ '{{' }} 'has_sudo_nopw_password' | trim {{ '}}' }}" ansible_port={{ tt.port }} ansible_user=mitogen__has_sudo_nopw +tt-port ansible_password=has_sudo_nopw_password ansible_port="{{ '{{' }} {{ tt.port }} | int {{ '}}' }}" ansible_user=mitogen__has_sudo_nopw +tt-remote-user ansible_password=has_sudo_nopw_password ansible_port={{ tt.port }} ansible_user="{{ '{{' }} 'mitogen__has_sudo_nopw' | trim {{ '}}' }}" [tt_targets_inventory:vars] ansible_host={{ tt.hostname }} ansible_python_interpreter={{ tt.python_path }} -ansible_user=mitogen__has_sudo_nopw diff --git a/tests/ansible/tests/utils_unsafe_test.py b/tests/ansible/tests/utils_unsafe_test.py index a020f55b..9aa461c5 100644 --- a/tests/ansible/tests/utils_unsafe_test.py +++ b/tests/ansible/tests/utils_unsafe_test.py @@ -48,7 +48,7 @@ class CastTest(unittest.TestCase): self.assertCasts(wrap_var([]), []) self.assertCasts(wrap_var(u''), u'') self.assertCasts(wrap_var(()), []) - + def test_subtypes_roundtrip(self): self.assertCasts(wrap_var(Bytes()), b'') self.assertCasts(wrap_var(Dict()), {}) diff --git a/tests/data/importer/webproject/modules_expected_py3x-new.json b/tests/data/importer/webproject/modules_expected_py3x-new.json index dcbcc785..614a7f9c 100644 --- a/tests/data/importer/webproject/modules_expected_py3x-new.json +++ b/tests/data/importer/webproject/modules_expected_py3x-new.json @@ -205,4 +205,3 @@ ] } } - \ No newline at end of file diff --git a/tests/image_prep/py24-build.sh b/tests/image_prep/py24-build.sh index b30cc24b..b99e36a0 100755 --- a/tests/image_prep/py24-build.sh +++ b/tests/image_prep/py24-build.sh @@ -15,7 +15,7 @@ tar xzvf cpython-2.4.6.tar.gz ( cd cpython-2.4.6 - ./configure --prefix=/usr/local/python2.4.6 --with-pydebug --enable-debug CFLAGS="-g -O0" # --enable-debug + ./configure --prefix=/usr/local/python2.4.6 --with-pydebug --enable-debug CFLAGS="-g -O0" # --enable-debug echo 'zlib zlibmodule.c -I$(prefix)/include -L$(exec_prefix)/lib -lz' >> Modules/Setup.config make -j 8 sudo make install diff --git a/tests/requirements.txt b/tests/requirements.txt index d64d8b87..c5671b37 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,5 +1,5 @@ cffi==1.15.1; python_version < '3.8' -cffi==1.16; python_version >= '3.8' +cffi==1.17.1; python_version >= '3.8' coverage==5.5; python_version == '2.7' coverage==6.2; python_version == '3.6' diff --git a/tox.ini b/tox.ini index 9fb31bdc..6a0eb180 100644 --- a/tox.ini +++ b/tox.ini @@ -2,8 +2,7 @@ # I use this locally on Ubuntu 22.04, with the following additions # # sudo add-apt-repository ppa:deadsnakes/ppa -# sudo apt update -# sudo apt install awscli lib{ldap2,sasl2,ssl}-dev python{2,2.7,3} python3.{6..13}{,-venv} python-is-python3 sshpass tox +# sudo apt install lib{ldap2,sasl2,ssl}-dev python{2,2.7,3}{,-dev} python3.{7..13}{,-dev,-venv} python-is-python3 sshpass tox # Py A cntrllr A target coverage Django Jinja2 pip psutil pytest tox virtualenv # ==== ========== ========== ========== ========== ========== ========== ========== ========== ========== ========== @@ -56,10 +55,10 @@ envlist = init, py{27,36}-mode_ansible-ansible{2.10,3,4}, py{311}-mode_ansible-ansible{2.10,3,4,5}, - py{312}-mode_ansible-ansible{6,7,8,9,10}, - py{27,36,312}-mode_mitogen-distro_centos{6,7,8}, - py{27,36,312}-mode_mitogen-distro_debian{9,10,11}, - py{27,36,312}-mode_mitogen-distro_ubuntu{1604,1804,2004}, + py{313}-mode_ansible-ansible{6,7,8,9,10}, + py{27,36,313}-mode_mitogen-distro_centos{6,7,8}, + py{27,36,313}-mode_mitogen-distro_debian{9,10,11}, + py{27,36,313}-mode_mitogen-distro_ubuntu{1604,1804,2004}, report, [testenv] @@ -74,6 +73,7 @@ basepython = py310: python3.10 py311: python3.11 py312: python3.12 + py313: python3.13 deps = -r{toxinidir}/tests/requirements.txt mode_ansible: -r{toxinidir}/tests/ansible/requirements.txt @@ -99,9 +99,6 @@ passenv = ANSIBLE_* HOME MITOGEN_* - # Azure DevOps, TF_BUILD is set to 'True' when running in a build task - # https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables - TF_BUILD setenv = # See also azure-pipelines.yml ANSIBLE_STRATEGY = mitogen_linear