Merge commit '8e25944' into release-v0.3.24

pull/1262/head
Alex Willmer 7 months ago
commit 9e02175134

@ -12,9 +12,8 @@ import jinja2
import ci_lib
TEMPLATES_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible/templates')
TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible')
HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts')
TMP = ci_lib.TempDir(prefix='mitogen_ci_ansible')
TMP_HOSTS_DIR = os.path.join(TMP.path, 'hosts')
def pause_if_interactive():
@ -40,13 +39,13 @@ with ci_lib.Fold('docker_setup'):
with ci_lib.Fold('job_setup'):
os.chdir(TESTS_DIR)
os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 8))
os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8))
os.chdir(ci_lib.ANSIBLE_TESTS_DIR)
ci_lib.run("mkdir %s", HOSTS_DIR)
for path in glob.glob(TESTS_DIR + '/hosts/*'):
os.mkdir(TMP_HOSTS_DIR)
for path in glob.glob(os.path.join(ci_lib.ANSIBLE_TESTS_HOSTS_DIR, '*')):
if not path.endswith('default.hosts'):
ci_lib.run("ln -s %s %s", path, HOSTS_DIR)
os.symlink(path, os.path.join(TMP_HOSTS_DIR, os.path.basename(path)))
distros = collections.defaultdict(list)
families = collections.defaultdict(list)
@ -55,12 +54,14 @@ with ci_lib.Fold('job_setup'):
families[container['family']].append(container['name'])
jinja_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(searchpath=TEMPLATES_DIR),
loader=jinja2.FileSystemLoader(
searchpath=ci_lib.ANSIBLE_TESTS_TEMPLATES_DIR,
),
lstrip_blocks=True, # Remove spaces and tabs from before a block
trim_blocks=True, # Remove first newline after a block
)
inventory_template = jinja_env.get_template('test-targets.j2')
inventory_path = os.path.join(HOSTS_DIR, 'target')
inventory_path = os.path.join(TMP_HOSTS_DIR, 'test-targets.ini')
with open(inventory_path, 'w') as fp:
fp.write(inventory_template.render(
@ -71,16 +72,12 @@ with ci_lib.Fold('job_setup'):
ci_lib.dump_file(inventory_path)
if not ci_lib.exists_in_path('sshpass'):
ci_lib.run("sudo apt-get update")
ci_lib.run("sudo apt-get install -y sshpass")
with ci_lib.Fold('ansible'):
playbook = os.environ.get('PLAYBOOK', 'all.yml')
try:
ci_lib.run('./run_ansible_playbook.py %s -i "%s" %s',
playbook, HOSTS_DIR, ' '.join(sys.argv[1:]))
playbook, TMP_HOSTS_DIR, ' '.join(sys.argv[1:]),
)
except:
pause_if_interactive()
raise

@ -28,14 +28,20 @@ os.chdir(
)
GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
ANSIBLE_TESTS_DIR = os.path.join(GIT_ROOT, 'tests/ansible')
ANSIBLE_TESTS_HOSTS_DIR = os.path.join(GIT_ROOT, 'tests/ansible/hosts')
ANSIBLE_TESTS_TEMPLATES_DIR = os.path.join(GIT_ROOT, 'tests/ansible/templates')
DISTRO_SPECS = os.environ.get(
'MITOGEN_TEST_DISTRO_SPECS',
'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004',
)
IMAGE_PREP_DIR = os.path.join(GIT_ROOT, 'tests/image_prep')
IMAGE_TEMPLATE = os.environ.get(
'MITOGEN_TEST_IMAGE_TEMPLATE',
'public.ecr.aws/n5z0e8q9/%(distro)s-test',
'ghcr.io/mitogen-hq/%(distro)s-test:2021',
)
TESTS_SSH_PRIVATE_KEY_FILE = os.path.join(GIT_ROOT, 'tests/data/docker/mitogen__has_sudo_pubkey.key')
_print = print
@ -48,6 +54,7 @@ def print(*args, **kwargs):
def _have_cmd(args):
# Code duplicated in testlib.py
try:
subprocess.run(
args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
@ -56,19 +63,11 @@ def _have_cmd(args):
if exc.errno == errno.ENOENT:
return False
raise
except subprocess.CallProcessError:
except subprocess.CalledProcessError:
return False
return True
def have_apt():
return _have_cmd(['apt', '--help'])
def have_brew():
return _have_cmd(['brew', 'help'])
def have_docker():
return _have_cmd(['docker', 'info'])
@ -185,8 +184,8 @@ def exists_in_path(progname):
class TempDir(object):
def __init__(self):
self.path = tempfile.mkdtemp(prefix='mitogen_ci_lib')
def __init__(self, prefix='mitogen_ci_lib'):
self.path = tempfile.mkdtemp(prefix=prefix)
atexit.register(self.destroy)
def destroy(self, rmtree=shutil.rmtree):
@ -199,20 +198,6 @@ class Fold(object):
def __exit__(self, _1, _2, _3): pass
GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
TMP = TempDir().path
# We copy this out of the way to avoid random stuff modifying perms in the Git
# tree (like git pull).
src_key_file = os.path.join(GIT_ROOT,
'tests/data/docker/mitogen__has_sudo_pubkey.key')
key_file = os.path.join(TMP,
'mitogen__has_sudo_pubkey.key')
shutil.copyfile(src_key_file, key_file)
os.chmod(key_file, int('0600', 8))
os.environ['PYTHONDONTWRITEBYTECODE'] = 'x'
os.environ['PYTHONPATH'] = '%s:%s' % (
os.environ.get('PYTHONPATH', ''),
@ -243,7 +228,7 @@ def container_specs(
[{'distro': 'debian11',
'family': 'debian',
'hostname': 'localhost',
'image': 'public.ecr.aws/n5z0e8q9/debian11-test',
'image': 'ghcr.io/mitogen-hq/debian11-test:2021',
'index': 1,
'name': 'target-debian11-1',
'port': 2201,
@ -251,7 +236,7 @@ def container_specs(
{'distro': 'centos6',
'family': 'centos',
'hostname': 'localhost',
'image': 'public.ecr.aws/n5z0e8q9/centos6-test',
'image': 'ghcr.io/mitogen-hq/centos6-test:2021',
'index': 2,
'name': 'target-centos6-2',
'port': 2202,

@ -6,7 +6,8 @@ import sys
import ci_lib
project_dir = os.path.join(ci_lib.TMP, 'project')
TMP = ci_lib.TempDir(prefix='mitogen_ci_debops')
project_dir = os.path.join(TMP.path, 'project')
vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml'
inventory_path = 'ansible/inventory/hosts'
docker_hostname = ci_lib.get_docker_hostname()
@ -22,6 +23,7 @@ with ci_lib.Fold('docker_setup'):
with ci_lib.Fold('job_setup'):
os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8))
ci_lib.run('debops-init %s', project_dir)
os.chdir(project_dir)
@ -45,7 +47,7 @@ with ci_lib.Fold('job_setup'):
"\n"
# Speed up slow DH generation.
"dhparam__bits: ['128', '64']\n"
% (ci_lib.key_file,)
% (ci_lib.TESTS_SSH_PRIVATE_KEY_FILE,)
)
with open(inventory_path, 'a') as fp:

@ -10,19 +10,13 @@ import sys
import ci_lib
TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible')
IMAGE_PREP_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/image_prep')
HOSTS_DIR = os.path.join(TESTS_DIR, 'hosts')
KEY_PATH = os.path.join(TESTS_DIR, '../data/docker/mitogen__has_sudo_pubkey.key')
with ci_lib.Fold('unit_tests'):
os.environ['SKIP_MITOGEN'] = '1'
ci_lib.run('./run_tests -v')
with ci_lib.Fold('job_setup'):
os.chmod(KEY_PATH, int('0600', 8))
os.chmod(ci_lib.TESTS_SSH_PRIVATE_KEY_FILE, int('0600', 8))
# NOTE: sshpass v1.06 causes errors so pegging to 1.05 -> "msg": "Error when changing password","out": "passwd: DS error: eDSAuthFailed\n",
# there's a checksum error with "brew install http://git.io/sshpass.rb" though, so installing manually
if not ci_lib.exists_in_path('sshpass'):
@ -51,11 +45,11 @@ with ci_lib.Fold('machine_prep'):
subprocess.check_call('sudo chmod 700 ~root/.ssh', shell=True)
subprocess.check_call('sudo chmod 600 ~root/.ssh/authorized_keys', shell=True)
os.chdir(IMAGE_PREP_DIR)
os.chdir(ci_lib.IMAGE_PREP_DIR)
ci_lib.run("ansible-playbook -c local -i localhost, macos_localhost.yml")
if os.path.expanduser('~mitogen__user1') == '~mitogen__user1':
os.chdir(IMAGE_PREP_DIR)
os.chdir(ci_lib.IMAGE_PREP_DIR)
ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml")
cmd = ';'.join([
@ -80,7 +74,7 @@ with ci_lib.Fold('machine_prep'):
with ci_lib.Fold('ansible'):
os.chdir(TESTS_DIR)
os.chdir(ci_lib.ANSIBLE_TESTS_DIR)
playbook = os.environ.get('PLAYBOOK', 'all.yml')
ci_lib.run('./run_ansible_playbook.py %s %s',
playbook, ' '.join(sys.argv[1:]))

@ -2,23 +2,23 @@
name: Tests
# env:
# ANSIBLE_VERBOSITY: 3
# MITOGEN_LOG_LEVEL: DEBUG
on:
pull_request:
push:
branches-ignore:
- docs-master
env:
#ANSIBLE_VERBOSITY: 3
#MITOGEN_LOG_LEVEL: DEBUG
MITOGEN_TEST_IMAGE_TEMPLATE: "ghcr.io/mitogen-hq/%(distro)s-test"
# https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners
# https://github.com/actions/runner-images/blob/main/README.md#software-and-image-support
jobs:
linux:
# https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md
runs-on: ubuntu-20.04
u2204:
name: u2204 ${{ matrix.tox_env }}
# https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md
runs-on: ubuntu-22.04
strategy:
fail-fast: false
@ -36,6 +36,115 @@ jobs:
python_version: '3.6'
tox_env: py36-mode_ansible-ansible4
- name: Mito_27
tox_env: py27-mode_mitogen
- name: Mito_36
python_version: '3.6'
tox_env: py36-mode_mitogen
steps:
- uses: actions/checkout@v4
- uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Install build deps
run: |
set -o errexit -o nounset -o pipefail
PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))')
if [[ -z $PYTHON ]]; then
echo 1>&2 "Python interpreter could not be determined"
exit 1
fi
sudo apt-get update
if [[ $PYTHON == "python2.7" ]]; then
sudo apt install -y python2-dev sshpass virtualenv
elif [[ $PYTHON == "python3.6" ]]; then
sudo apt install -y gcc-10 make libbz2-dev liblzma-dev libreadline-dev libsqlite3-dev libssl-dev sshpass virtualenv zlib1g-dev
curl --fail --silent --show-error --location https://pyenv.run | bash
CC=gcc-10 ~/.pyenv/bin/pyenv install --force 3.6
else
echo 1>&2 "Python interpreter $PYTHON not available"
exit 1
fi
- name: Show Python versions
run: |
set -o errexit -o nounset -o pipefail
# macOS builders lack a realpath command
type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version
type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version
type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version
echo
if [ -e /usr/bin/python ]; then
echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')"
fi
if [ -e /usr/bin/python2 ]; then
echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')"
fi
if [ -e /usr/bin/python2.7 ]; then
echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')"
fi
- name: Install tooling
run: |
set -o errexit -o nounset -o pipefail
# Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))')
if [[ -z $PYTHON ]]; then
echo 1>&2 "Python interpreter could not be determined"
exit 1
fi
if [[ $PYTHON == "python2.7" ]]; then
curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py"
"$PYTHON" get-pip.py --user --no-python-version-warning
# Avoid Python 2.x pip masking system pip
rm -f ~/.local/bin/{easy_install,pip,wheel}
elif [[ $PYTHON == "python3.6" ]]; then
PYTHON="$HOME/.pyenv/versions/3.6.15/bin/python3.6"
fi
"$PYTHON" -m pip install -r "tests/requirements-tox.txt"
- name: Run tests
env:
GITHUB_ACTOR: ${{ github.actor }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -o errexit -o nounset -o pipefail
# Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))')
if [[ -z $PYTHON ]]; then
echo 1>&2 "Python interpreter could not be determined"
exit 1
fi
if [[ $PYTHON == "python3.6" ]]; then
PYTHON="$HOME/.pyenv/versions/3.6.15/bin/python3.6"
fi
"$PYTHON" -m tox -e "${{ matrix.tox_env }}"
u2404:
name: u2404 ${{ matrix.tox_env }}
# https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md
runs-on: ubuntu-24.04
strategy:
fail-fast: false
matrix:
include:
- name: Ans_311_210
python_version: '3.11'
tox_env: py311-mode_ansible-ansible2.10
@ -71,11 +180,6 @@ jobs:
python_version: '3.13'
tox_env: py313-mode_ansible-ansible11-strategy_linear
- name: Mito_27
tox_env: py27-mode_mitogen
- name: Mito_36
python_version: '3.6'
tox_env: py36-mode_mitogen
- name: Mito_313
python_version: '3.13'
tox_env: py313-mode_mitogen
@ -96,7 +200,7 @@ jobs:
set -o errexit -o nounset -o pipefail
sudo apt-get update
sudo apt-get install -y python2-dev python3-pip virtualenv
sudo apt-get install -y sshpass virtualenv
- name: Show Python versions
run: |
set -o errexit -o nounset -o pipefail
@ -130,18 +234,7 @@ jobs:
exit 1
fi
if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then
"$PYTHON" -m ensurepip --user --altinstall --no-default-pip
"$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
elif [[ $PYTHON == "python2.7" ]]; then
curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py"
"$PYTHON" get-pip.py --user --no-python-version-warning
# Avoid Python 2.x pip masking system pip
rm -f ~/.local/bin/{easy_install,pip,wheel}
"$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
else
"$PYTHON" -m pip install -r "tests/requirements-tox.txt"
fi
"$PYTHON" -m pip install -r "tests/requirements-tox.txt"
- name: Run tests
env:
GITHUB_ACTOR: ${{ github.actor }}
@ -160,6 +253,7 @@ jobs:
"$PYTHON" -m tox -e "${{ matrix.tox_env }}"
macos:
name: macos ${{ matrix.tox_env }}
# https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md
runs-on: macos-13
timeout-minutes: 15
@ -221,20 +315,7 @@ jobs:
exit 1
fi
if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then
# GitHub macOS 12 images: python2.7 is installed, but not on $PATH
PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7"
"$PYTHON" -m ensurepip --user --altinstall --no-default-pip
"$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
elif [[ $PYTHON == "python2.7" ]]; then
curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py"
"$PYTHON" get-pip.py --user --no-python-version-warning
# Avoid Python 2.x pip masking system pip
rm -f ~/.local/bin/{easy_install,pip,wheel}
"$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
else
"$PYTHON" -m pip install -r "tests/requirements-tox.txt"
fi
"$PYTHON" -m pip install -r "tests/requirements-tox.txt"
- name: Run tests
env:
GITHUB_ACTOR: ${{ github.actor }}
@ -250,18 +331,14 @@ jobs:
exit 1
fi
if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then
# GitHub macOS 12 images: python2.7 is installed, but not on $PATH
PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7"
fi
"$PYTHON" -m tox -e "${{ matrix.tox_env }}"
# https://github.com/marketplace/actions/alls-green
check:
if: always()
needs:
- linux
- u2204
- u2404
- macos
runs-on: ubuntu-latest
steps:

@ -18,6 +18,18 @@ To avail of fixes in an unreleased version, please download a ZIP file
`directly from GitHub <https://github.com/mitogen-hq/mitogen/>`_.
v0.3.23 (2025-04-28)
--------------------
* :gh:issue:`1121` :mod:`mitogen`: Log skipped :py:mod:`termios` attributes
* :gh:issue:`1238` packaging: Avoid :py:mod:`ast`, requires Python = 2.6
* :gh:issue:`1118` CI: Statically specify test usernames and group names
* :gh:issue:`1118` CI: Don't copy SSH private key to temporary dir
* :gh:issue:`1118` CI: Don't share temporary directory between test groupings
* :gh:issue:`1256` CI: Upgrade Github jobs from Ubuntu 20.04 to 22.04 & 24.04
* :gh:issue:`1263` packaging: Fix InvalidVersion in release versions
v0.3.22 (2025-02-04)
--------------------

@ -35,7 +35,7 @@ be expected. On the slave, it is built dynamically during startup.
#: Library version as a tuple.
__version__ = (0, 3, 22)
__version__ = (0, 3, 23)
#: This is :data:`False` in slave contexts. Previously it was used to prevent

@ -224,8 +224,16 @@ def flags(names):
Return the result of ORing a set of (space separated) :py:mod:`termios`
module constants together.
"""
return sum(getattr(termios, name, 0)
for name in names.split())
i = 0
skipped = []
for name in names.split():
try:
i |= getattr(termios, name)
except AttributeError:
skipped.append(name)
if skipped:
LOG.debug('Skipped termios attributes: %s', ', '.join(skipped))
return i
def cfmakeraw(tflags):
@ -234,12 +242,9 @@ def cfmakeraw(tflags):
modified in a manner similar to the `cfmakeraw()` C library function, but
additionally disabling local echo.
"""
# BSD: github.com/freebsd/freebsd/blob/master/lib/libc/gen/termios.c#L162
# Linux: github.com/lattera/glibc/blob/master/termios/cfmakeraw.c#L20
iflag, oflag, cflag, lflag, ispeed, ospeed, cc = tflags
iflag &= ~flags('IMAXBEL IXOFF INPCK BRKINT PARMRK '
'ISTRIP INLCR ICRNL IXON IGNPAR')
iflag &= ~flags('IGNBRK BRKINT PARMRK')
'ISTRIP INLCR ICRNL IXON IGNPAR IGNBRK')
oflag &= ~flags('OPOST')
lflag &= ~flags('ECHO ECHOE ECHOK ECHONL ICANON ISIG '
'IEXTEN NOFLSH TOSTOP PENDIN')

@ -26,20 +26,23 @@
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import ast
import os
import re
from setuptools import find_packages, setup
def grep_version():
path = os.path.join(os.path.dirname(__file__), 'mitogen/__init__.py')
version_pattern = re.compile(
r"__version__ = \((\d+), (\d+), (\d+)(?:, '(dev)')?\)",
)
with open(path) as fp:
for line in fp:
if line.startswith('__version__'):
_, _, s = line.partition('=')
parts = ast.literal_eval(s.strip())
return '.'.join(str(part) for part in parts)
match = version_pattern.search(fp.read())
if match is None:
raise ValueError('Could not find __version__ string in %s', path)
# E.g. '0.1.2', '0.1.3dev'
return '.'.join(str(part) for part in match.groups() if part)
def long_description():

@ -4,6 +4,8 @@
- name: Create file tree
connection: local
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
shell: >
mkdir -p /tmp/filetree.in;
for i in `seq -f /tmp/filetree.in/%g 1 100`; do echo $RANDOM > $i; done;

@ -27,4 +27,3 @@ become_unpriv_available: >-
-}}
pkg_mgr_python_interpreter: python
pkg_repos_overrides: []

@ -1,7 +1,7 @@
---
pkg_mgr_python_interpreter: /usr/libexec/platform-python
pkg_repos_overrides:
package_manager_repos:
- dest: /etc/yum.repos.d/CentOS-Linux-AppStream.repo
content: |
[appstream]

@ -0,0 +1,5 @@
package_manager_keys:
- src: debian-archive-bullseye-automatic.gpg # Debian 11
dest: /etc/apt/trusted.gpg.d/debian-archive-bullseye-automatic.gpg
- src: debian-archive-bookworm-automatic.gpg # Debian 12
dest: /etc/apt/trusted.gpg.d/debian-archive-bookworm-automatic.gpg

@ -1,4 +1,4 @@
pkg_repos_overrides:
package_manager_repos:
- dest: /etc/apt/sources.list
content: |
deb http://archive.debian.org/debian stretch main contrib non-free

@ -10,6 +10,8 @@
- name: Run _disconnect_during_module.yml
delegate_to: localhost
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
environment:
ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
command: |

@ -19,6 +19,8 @@
bash -c "( sleep 3; kill -9 {{ssh_account_env.pid}}; ) & disown"
- connection: local
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
shell: sleep 3
- wait_for_connection:

@ -8,11 +8,15 @@
hosts: test-targets
tasks:
- name: Get local cwd
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
connection: local
command: pwd
register: pwd
- connection: local
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
stat:
path: "{{pwd.stdout}}/cwd_preserved.yml"
register: stat

@ -30,6 +30,8 @@
# connection:local, no sudo
#
- name: "connection:local, no sudo"
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
copy:
dest: "{{ local_path }}"
content: "Hello, world."
@ -43,6 +45,8 @@
fail_msg: "{{ lookup('file', local_path) }}"
- name: "connection:local, no sudo"
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
file:
path: "{{ local_path }}"
state: absent
@ -84,6 +88,8 @@
# connection:local, sudo
#
- name: "connection:local, sudo"
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
shell: |
whoami > "{{ local_path }}"
args:
@ -102,6 +108,8 @@
- requires_local_sudo
- name: "connection:local, sudo"
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
file:
path: "{{ local_path }}"
state: absent

@ -7,7 +7,10 @@
result['sockets'] = glob.glob('/tmp/mitogen_unix*.sock')
register: socks
- shell: >
- name: Run whoami locally in an ansible subprocess
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
shell: >-
ANSIBLE_STRATEGY=mitogen_linear
ANSIBLE_SSH_ARGS="-o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa"
ANSIBLE_VERBOSITY="{{ ansible_verbosity }}"
@ -15,6 +18,7 @@
{% for inv in ansible_inventory_sources %}
-i "{{ inv }}"
{% endfor %}
-e ansible_python_interpreter="{{ ansible_playbook_python }}"
test-targets
args:
chdir: ../..

@ -6,6 +6,8 @@
connection: local
environment:
ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
command: |
ansible
{% for inv in ansible_inventory_sources %}

@ -15,6 +15,8 @@
environment:
ANSIBLE_SSH_TIMEOUT: 10
ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
command: |
ansible
{% for inv in ansible_inventory_sources %}

@ -6,6 +6,7 @@
hosts: test-targets[0]
connection: local
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
# ControlMaster has the effect of caching the previous auth to the same
# account, so disable it. Can't disable with ControlMaster no since that
# already appears on command line, so override ControlPath with junk.

@ -4,7 +4,10 @@
tasks:
- connection: local
environment:
ANSIBLE_PYTHON_INTERPRETER: "{{ ansible_playbook_python }}"
ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
command: |
ansible-playbook
{% for inv in ansible_inventory_sources %}
@ -18,6 +21,8 @@
- connection: local
environment:
ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
command: |
ansible-playbook
{% for inv in ansible_inventory_sources %}

@ -12,6 +12,7 @@
- custom_python_detect_environment:
vars:
ansible_connection: kubectl
ansible_python_interpreter: "{{ ansible_playbook_python }}"
mitogen_kubectl_path: stub-kubectl.py
register: out

@ -8,6 +8,7 @@
- custom_python_detect_environment:
vars:
ansible_connection: lxc
ansible_python_interpreter: "{{ ansible_playbook_python }}"
mitogen_lxc_attach_path: stub-lxc-attach.py
register: out

@ -8,6 +8,7 @@
- custom_python_detect_environment:
vars:
ansible_connection: lxd
ansible_python_interpreter: "{{ ansible_playbook_python }}"
mitogen_lxc_path: stub-lxc.py
register: out

@ -9,6 +9,7 @@
vars:
ansible_connection: mitogen_doas
ansible_doas_exe: stub-doas.py
ansible_python_interpreter: "{{ ansible_playbook_python }}"
ansible_user: someuser
register: out

@ -8,6 +8,7 @@
- custom_python_detect_environment:
vars:
ansible_connection: mitogen_sudo
ansible_python_interpreter: "{{ ansible_playbook_python }}"
ansible_user: root
ansible_become_exe: stub-sudo.py
ansible_become_flags: -H --type=sometype --role=somerole

@ -7,6 +7,8 @@
gather_facts: false
any_errors_fatal: false
connection: local
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
tasks:
- include_tasks: ../_mitogen_only.yml
- include_tasks: _end_play_if_not_sudo_linux.yml
@ -19,7 +21,7 @@
-i localhost,
-c setns
-e mitogen_kind=lxc
-e ansible_python_interpreter=python
-e ansible_python_interpreter="{{ ansible_playbook_python }}"
-e mitogen_lxc_info_path={{git_basedir}}/tests/data/stubs/stub-lxc-info.py
-m shell
-a "echo hi"

@ -7,6 +7,8 @@
gather_facts: false
any_errors_fatal: false
connection: local
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
tasks:
- include_tasks: ../_mitogen_only.yml
- include_tasks: _end_play_if_not_sudo_linux.yml
@ -19,7 +21,7 @@
-i localhost,
-c setns
-e mitogen_kind=lxd
-e ansible_python_interpreter=python
-e ansible_python_interpreter="{{ ansible_playbook_python }}"
-e mitogen_lxc_path={{git_basedir}}/tests/data/stubs/stub-lxc.py
-m shell
-a "echo hi"

@ -17,6 +17,8 @@
- name: Test template does not cause StreamError
delegate_to: localhost
run_once: true
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
environment:
ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
command:

@ -9,6 +9,8 @@
- name: Create file tree
connection: local
run_once: true
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
shell: >
mkdir /tmp/filetree.in;
seq -f /tmp/filetree.in/%g 1 1000 | xargs touch;
@ -37,6 +39,8 @@
- name: Cleanup local file tree
connection: local
run_once: true
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
file:
path: /tmp/filetree.in
state: absent

@ -8,6 +8,8 @@
connection: local
tasks:
- name: Create /tmp/issue_152_interpreter.sh
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
copy:
dest: /tmp/issue_152_interpreter.sh
mode: u+x
@ -28,6 +30,8 @@
out={{ out }}
- name: Cleanup /tmp/issue_152_interpreter.sh
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
file:
path: /tmp/issue_152_interpreter.sh
state: absent

@ -10,7 +10,7 @@
vars:
ansible_python_interpreter: "{{ pkg_mgr_python_interpreter }}"
package: rsync # Chosen to exist in all tested distros/package managers
tasks:
pre_tasks:
# The package management modules require using the same Python version
# as the target's package manager libraries. This is sometimes in conflict
# with Ansible requirements, e.g. Ansible 10 (ansible-core 2.17) does not
@ -19,31 +19,13 @@
when:
- ansible_version.full is version('2.17', '>=', strict=True)
roles:
- role: package_manager
tasks:
- name: Gather facts manually
setup:
- name: Switch to archived package repositories
copy:
dest: "{{ item.dest }}"
content: "{{ item.content }}"
mode: u=rw,go=r
loop: "{{ pkg_repos_overrides }}"
loop_control:
label: "{{ item.dest }}"
- name: Add signing keys
copy:
src: "{{ item.src }}"
dest: "/etc/apt/trusted.gpg.d/{{ item.src | basename }}"
mode: u=rw,go=r
loop:
- src: debian-archive-bullseye-automatic.gpg # Debian 11
- src: debian-archive-bookworm-automatic.gpg # Debian 12
when:
# Ideally this would check for Debian 11, but distribution_major_version
# is unpopulated sometimes.
- ansible_facts.distribution == "Debian"
- name: Update package index
apt:
update_cache: true

@ -6,6 +6,8 @@
tasks:
- name: Test --ask-become-pass
delegate_to: localhost
vars:
ansible_python_interpreter: "{{ ansible_playbook_python }}"
environment:
ANSIBLE_VERBOSITY: "{{ ansible_verbosity }}"
expect:

@ -1,5 +1,5 @@
- hosts: all
- name: Bootstrap containers
hosts: all
strategy: linear
gather_facts: false
tasks:
@ -15,7 +15,8 @@
fi
when: bootstrap_packages | length
- hosts: all
- name: Setup containers
hosts: all
strategy: mitogen_free
# Resource limitation, my laptop freezes doing every container concurrently
serial: 4
@ -30,7 +31,9 @@
- ansible_facts.virtualization_type != "docker"
roles:
- role: package_manager
- role: sshd
- role: sshd_container
tasks:
- name: Ensure requisite apt packages are installed

@ -3,48 +3,53 @@
#
# WARNING: this creates non-privilged accounts with pre-set passwords!
#
- import_playbook: ../ansible/setup/report_controller.yml
- hosts: all
- name: Mitogen test users and groups
hosts: all
gather_facts: true
strategy: mitogen_free
become: true
vars:
distro: "{{ansible_distribution}}"
special_users:
- has_sudo
- has_sudo_nopw
- has_sudo_pubkey
- pw_required
- readonly_homedir
- require_tty
- require_tty_pw_required
- permdenied
- slow_user
- webapp
- sudo1
- sudo2
- sudo3
- sudo4
- name: mitogen__has_sudo
- name: mitogen__has_sudo_nopw
- name: mitogen__has_sudo_pubkey
- name: mitogen__pw_required
- name: mitogen__readonly_homedir
- name: mitogen__require_tty
- name: mitogen__require_tty_pw_required
- name: mitogen__permdenied
- name: mitogen__slow_user
- name: mitogen__webapp
- name: mitogen__sudo1
- name: mitogen__sudo2
- name: mitogen__sudo3
- name: mitogen__sudo4
user_groups:
has_sudo: ['mitogen__group', '{{sudo_group[distro]}}']
has_sudo_pubkey: ['mitogen__group', '{{sudo_group[distro]}}']
has_sudo_nopw: ['mitogen__group', 'mitogen__sudo_nopw']
sudo1: ['mitogen__group', 'mitogen__sudo_nopw']
sudo2: ['mitogen__group', '{{sudo_group[distro]}}']
sudo3: ['mitogen__group', '{{sudo_group[distro]}}']
sudo4: ['mitogen__group', '{{sudo_group[distro]}}']
normal_users: "{{
lookup('sequence', 'start=1 end=5 format=user%d', wantlist=True)
}}"
mitogen__has_sudo: ['mitogen__group', '{{ sudo_group[distro] }}']
mitogen__has_sudo_pubkey: ['mitogen__group', '{{ sudo_group[distro] }}']
mitogen__has_sudo_nopw: ['mitogen__group', 'mitogen__sudo_nopw']
mitogen__sudo1: ['mitogen__group', 'mitogen__sudo_nopw']
mitogen__sudo2: ['mitogen__group', '{{ sudo_group[distro] }}']
mitogen__sudo3: ['mitogen__group', '{{ sudo_group[distro] }}']
mitogen__sudo4: ['mitogen__group', '{{ sudo_group[distro] }}']
normal_users:
- name: mitogen__user1
- name: mitogen__user2
- name: mitogen__user3
- name: mitogen__user4
- name: mitogen__user5
all_users: "{{
special_users +
normal_users
}}"
mitogen_test_groups:
- name: mitogen__group
- name: mitogen__sudo_nopw
tasks:
- name: Disable non-localhost SSH for Mitogen users
when: false
@ -56,43 +61,34 @@
- name: Create Mitogen test groups
group:
name: "mitogen__{{item}}"
with_items:
- group
- sudo_nopw
name: "{{ item.name }}"
with_items: "{{ mitogen_test_groups }}"
- name: Create user accounts
vars:
password: "{{ item.name | replace('mitogen__', '') }}_password"
block:
- user:
name: "mitogen__{{item}}"
name: "{{ item.name }}"
shell: /bin/bash
groups: "{{user_groups[item]|default(['mitogen__group'])}}"
password: "{{ (item + '_password') | password_hash('sha256') }}"
groups: "{{ user_groups[item.name] | default(['mitogen__group']) }}"
password: "{{ password | password_hash('sha256') }}"
with_items: "{{all_users}}"
when: ansible_system != 'Darwin'
- user:
name: "mitogen__{{item}}"
name: "{{ item.name }}"
shell: /bin/bash
group: staff
groups: |
{{
['com.apple.access_ssh'] +
(user_groups[item] | default(['mitogen__group']))
(user_groups[item.name] | default(['mitogen__group']))
}}
password: "{{item}}_password"
hidden: true
password: "{{ password }}"
with_items: "{{all_users}}"
when: ansible_system == 'Darwin'
- name: Hide users from login window (Darwin).
when: ansible_system == 'Darwin'
with_items: "{{all_users}}"
osx_defaults:
array_add: true
domain: /Library/Preferences/com.apple.loginwindow
type: array
key: HiddenUsersList
value: ['mitogen_{{item}}']
- name: Check if AccountsService is used
stat:
path: /var/lib/AccountsService/users
@ -102,7 +98,7 @@
when: ansible_system == 'Linux' and out.stat.exists
with_items: "{{all_users}}"
copy:
dest: /var/lib/AccountsService/users/mitogen__{{item}}
dest: /var/lib/AccountsService/users/{{ item.name }}
mode: u=rw,go=
content: |
[User]
@ -115,7 +111,11 @@
state: restarted
- name: Readonly homedir for one account
shell: "chown -R root: ~mitogen__readonly_homedir"
file:
path: ~mitogen__readonly_homedir
owner: root
recurse: true
state: directory
- name: Slow bash profile for one account
copy:
@ -154,42 +154,29 @@
owner: mitogen__has_sudo_pubkey
group: mitogen__group
- name: Require a TTY for two accounts
lineinfile:
path: /etc/sudoers
line: "{{item}}"
with_items:
- Defaults>mitogen__pw_required targetpw
- Defaults>mitogen__require_tty requiretty
- Defaults>mitogen__require_tty_pw_required requiretty,targetpw
- name: Require password for two accounts
lineinfile:
path: /etc/sudoers
line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}:ALL) ALL"
validate: '/usr/sbin/visudo -cf %s'
with_items:
- mitogen__pw_required
- mitogen__require_tty_pw_required
when:
- ansible_virtualization_type != "docker"
- name: Allow passwordless sudo for require_tty/readonly_homedir
lineinfile:
- name: Configure sudoers defaults
blockinfile:
path: /etc/sudoers
line: "{{lookup('pipe', 'whoami')}} ALL = ({{item}}:ALL) NOPASSWD:ALL"
marker: "# {mark} Mitogen test defaults"
block: |
Defaults>mitogen__pw_required targetpw
Defaults>mitogen__require_tty requiretty
Defaults>mitogen__require_tty_pw_required requiretty,targetpw
validate: '/usr/sbin/visudo -cf %s'
with_items:
- mitogen__require_tty
- mitogen__readonly_homedir
when:
- ansible_virtualization_type != "docker"
- name: Allow passwordless for many accounts
lineinfile:
- name: Configure sudoers users
blockinfile:
path: /etc/sudoers
line: "{{lookup('pipe', 'whoami')}} ALL = (mitogen__{{item}}:ALL) NOPASSWD:ALL"
marker: "# {mark} Mitogen test users"
block: |
# User Host(s) = (runas user:runas group) Command(s)
{{ lookup('pipe', 'whoami') }} ALL = (mitogen__pw_required:ALL) ALL
{{ lookup('pipe', 'whoami') }} ALL = (mitogen__require_tty_pw_required:ALL) ALL
{{ lookup('pipe', 'whoami') }} ALL = (mitogen__require_tty:ALL) NOPASSWD:ALL
{{ lookup('pipe', 'whoami') }} ALL = (mitogen__readonly_homedir:ALL) NOPASSWD:ALL
{% for runas_user in normal_users %}
{{ lookup('pipe', 'whoami') }} ALL = ({{ runas_user.name }}:ALL) NOPASSWD:ALL
{% endfor %}
validate: '/usr/sbin/visudo -cf %s'
with_items: "{{normal_users}}"
when:
- ansible_virtualization_type != "docker"

@ -5,3 +5,4 @@
become: true
roles:
- role: sshd
- role: sshd_macos

@ -0,0 +1,2 @@
package_manager_keys: []
package_manager_repos: []

@ -0,0 +1,13 @@
- name: Add signing keys
copy:
src: "{{ item.src }}"
dest: "{{ item.dest }}"
mode: u=rw,go=r
with_items: "{{ package_manager_keys }}"
- name: Configure package repositories
copy:
dest: "{{ item.dest }}"
content: "{{ item.content }}"
mode: u=rw,go=r
with_items: "{{ package_manager_repos }}"

@ -9,23 +9,12 @@
path: "{{ sshd_config_file }}"
line: "{{ item.line }}"
regexp: "{{ item.regexp }}"
loop:
with_items:
- line: Banner /etc/ssh/banner.txt
regexp: '^#? *Banner.*'
- line: MaxAuthTries {{ sshd_config__max_auth_tries }}
regexp: '^#? *MaxAuthTries.*'
- line: PermitRootLogin yes
regexp: '.*PermitRootLogin.*'
loop_control:
label: "{{ item.line }}"
register: configure_sshd_result
- name: Restart sshd
shell: |
launchctl unload /System/Library/LaunchDaemons/ssh.plist
wait 5
launchctl load -w /System/Library/LaunchDaemons/ssh.plist
changed_when: true
when:
- ansible_facts.distribution == "MacOSX"
- configure_sshd_result is changed
notify:
- Restart sshd # Handler in platform specific role

@ -0,0 +1,2 @@
- name: Restart sshd
meta: noop

@ -0,0 +1,6 @@
- name: Restart sshd
shell: |
launchctl unload /System/Library/LaunchDaemons/ssh.plist
wait 5
launchctl load -w /System/Library/LaunchDaemons/ssh.plist
changed_when: true

@ -57,7 +57,7 @@ DISTRO_SPECS = os.environ.get(
)
IMAGE_TEMPLATE = os.environ.get(
'MITOGEN_TEST_IMAGE_TEMPLATE',
'public.ecr.aws/n5z0e8q9/%(distro)s-test',
'ghcr.io/mitogen-hq/%(distro)s-test:2021',
)
TESTS_DIR = os.path.join(os.path.dirname(__file__))
@ -155,6 +155,29 @@ def data_path(suffix):
return path
def _have_cmd(args):
# Code duplicated in ci_lib.py
try:
subprocess.run(
args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
)
except OSError as exc:
if exc.errno == errno.ENOENT:
return False
raise
except subprocess.CalledProcessError:
return False
return True
def have_python2():
return _have_cmd(['python2'])
def have_python3():
return _have_cmd(['python3'])
def retry(fn, on, max_attempts, delay):
for i in range(max_attempts):
try:

@ -1,4 +1,3 @@
import os
import unittest
import mitogen.core
@ -11,8 +10,8 @@ import simple_pkg.ping
# There should be 100 tests in this file.
@unittest.skipIf(
os.uname()[0] == 'Darwin' and int(os.uname()[2].partition('.')[0]) >= 21,
"Python 2.x not shipped on macOS 12.3+ (Darwin 21.4+, Monterey)",
not testlib.have_python2() or not testlib.have_python3(),
"Python 2/3 compatibility tests require both versions on the controller",
)
class TwoThreeCompatTest(testlib.RouterMixin, testlib.TestCase):
if mitogen.core.PY3:

@ -100,6 +100,7 @@ passenv =
HOME
MITOGEN_*
setenv =
ANSIBLE_ROLES_PATH = {toxinidir}/tests/image_prep/roles
# See also azure-pipelines.yml
ANSIBLE_STRATEGY = mitogen_linear
NOCOVERAGE_ERASE = 1

Loading…
Cancel
Save