Merge pull request #1128 from moreati/github-actions

CI: Begin migration from Azure DevOps to GitHub actions
pull/1080/head^2
Alex Willmer 2 months ago committed by GitHub
commit a51909ea79
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -28,7 +28,6 @@ for doing `setup.py install` while pulling a Docker container, for example.
### Environment Variables ### Environment Variables
* `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2.
* `DISTRO`: the `mitogen_` tests need a target Docker container distro. This * `DISTRO`: the `mitogen_` tests need a target Docker container distro. This
name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test` name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test`
* `DISTROS`: the `ansible_` tests can run against multiple targets * `DISTROS`: the `ansible_` tests can run against multiple targets

@ -1,11 +0,0 @@
#!/usr/bin/env python
import ci_lib
batches = [
[
'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi',
]
]
ci_lib.run_batches(batches)

@ -35,7 +35,7 @@ ci_lib.check_stray_processes(interesting)
with ci_lib.Fold('docker_setup'): with ci_lib.Fold('docker_setup'):
containers = ci_lib.make_containers() containers = ci_lib.container_specs(ci_lib.DISTROS)
ci_lib.start_containers(containers) ci_lib.start_containers(containers)

@ -14,6 +14,19 @@ steps:
versionSpec: '$(python.version)' versionSpec: '$(python.version)'
condition: ne(variables['python.version'], '') condition: ne(variables['python.version'], '')
- script: |
set -o errexit
set -o nounset
set -o pipefail
aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws
displayName: Authenticate to container registry
condition: eq(variables['Agent.OS'], 'Linux')
env:
AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)
- script: | - script: |
set -o errexit set -o errexit
set -o nounset set -o nounset
@ -90,7 +103,3 @@ steps:
"$PYTHON" -m tox -e "$(tox.env)" "$PYTHON" -m tox -e "$(tox.env)"
displayName: "Run tests" displayName: "Run tests"
env:
AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)

@ -27,6 +27,13 @@ os.chdir(
) )
) )
IMAGE_TEMPLATE = os.environ.get(
'MITOGEN_TEST_IMAGE_TEMPLATE',
'public.ecr.aws/n5z0e8q9/%(distro)s-test',
)
_print = print _print = print
def print(*args, **kwargs): def print(*args, **kwargs):
file = kwargs.get('file', sys.stdout) file = kwargs.get('file', sys.stdout)
@ -193,8 +200,6 @@ GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
DISTRO = os.environ.get('DISTRO', 'debian9') DISTRO = os.environ.get('DISTRO', 'debian9')
# Used only when MODE=ansible # Used only when MODE=ansible
DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split() DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split()
TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2'))
BASE_PORT = 2200
TMP = TempDir().path TMP = TempDir().path
@ -217,6 +222,7 @@ os.environ['PYTHONPATH'] = '%s:%s' % (
def get_docker_hostname(): def get_docker_hostname():
"""Return the hostname where the docker daemon is running. """Return the hostname where the docker daemon is running.
""" """
# Duplicated in testlib
url = os.environ.get('DOCKER_HOST') url = os.environ.get('DOCKER_HOST')
if url in (None, 'http+docker://localunixsocket'): if url in (None, 'http+docker://localunixsocket'):
return 'localhost' return 'localhost'
@ -225,27 +231,34 @@ def get_docker_hostname():
return parsed.netloc.partition(':')[0] return parsed.netloc.partition(':')[0]
def make_containers(name_prefix='', port_offset=0): def container_specs(
distros,
base_port=2200,
image_template=IMAGE_TEMPLATE,
name_template='target-%(distro)s-%(index)d',
):
""" """
>>> import pprint >>> import pprint
>>> BASE_PORT=2200; DISTROS=['debian11', 'centos6'] >>> pprint.pprint(container_specs(['debian11-py3', 'centos6']))
>>> pprint.pprint(make_containers())
[{'distro': 'debian11', [{'distro': 'debian11',
'family': 'debian', 'family': 'debian',
'hostname': 'localhost', 'hostname': 'localhost',
'image': 'public.ecr.aws/n5z0e8q9/debian11-test', 'image': 'public.ecr.aws/n5z0e8q9/debian11-test',
'index': 1,
'name': 'target-debian11-1', 'name': 'target-debian11-1',
'port': 2201, 'port': 2201,
'python_path': '/usr/bin/python'}, 'python_path': '/usr/bin/python3'},
{'distro': 'centos6', {'distro': 'centos6',
'family': 'centos', 'family': 'centos',
'hostname': 'localhost', 'hostname': 'localhost',
'image': 'public.ecr.aws/n5z0e8q9/centos6-test', 'image': 'public.ecr.aws/n5z0e8q9/centos6-test',
'index': 2,
'name': 'target-centos6-2', 'name': 'target-centos6-2',
'port': 2202, 'port': 2202,
'python_path': '/usr/bin/python'}] 'python_path': '/usr/bin/python'}]
""" """
docker_hostname = get_docker_hostname() docker_hostname = get_docker_hostname()
# Code duplicated in testlib.py, both should be updated together
distro_pattern = re.compile(r''' distro_pattern = re.compile(r'''
(?P<distro>(?P<family>[a-z]+)[0-9]+) (?P<distro>(?P<family>[a-z]+)[0-9]+)
(?:-(?P<py>py3))? (?:-(?P<py>py3))?
@ -256,30 +269,27 @@ def make_containers(name_prefix='', port_offset=0):
i = 1 i = 1
lst = [] lst = []
for distro in DISTROS: for distro in distros:
# Code duplicated in testlib.py, both should be updated together
d = distro_pattern.match(distro).groupdict(default=None) d = distro_pattern.match(distro).groupdict(default=None)
distro = d['distro']
family = d['family']
image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,)
if d['py'] == 'py3': if d.pop('py') == 'py3':
python_path = '/usr/bin/python3' python_path = '/usr/bin/python3'
else: else:
python_path = '/usr/bin/python' python_path = '/usr/bin/python'
if d['count']: count = int(d.pop('count') or '1', 10)
count = int(count)
else:
count = 1
for x in range(count): for x in range(count):
lst.append({ d['index'] = i
"distro": distro, "family": family, "image": image, d.update({
"name": name_prefix + ("target-%s-%s" % (distro, i)), 'image': image_template % d,
'name': name_template % d,
"hostname": docker_hostname, "hostname": docker_hostname,
"port": BASE_PORT + i + port_offset, 'port': base_port + i,
"python_path": python_path, "python_path": python_path,
}) })
lst.append(d)
i += 1 i += 1
return lst return lst

@ -2,16 +2,10 @@
import ci_lib import ci_lib
# Naturally DebOps only supports Debian.
ci_lib.DISTROS = ['debian']
ci_lib.run_batches([ ci_lib.run_batches([
[ [
'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"', 'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"',
], ],
[
'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi',
],
]) ])
ci_lib.run('ansible-galaxy collection install debops.debops:==2.1.2') ci_lib.run('ansible-galaxy collection install debops.debops:==2.1.2')

@ -6,9 +6,6 @@ import sys
import ci_lib import ci_lib
# DebOps only supports Debian.
ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT
project_dir = os.path.join(ci_lib.TMP, 'project') project_dir = os.path.join(ci_lib.TMP, 'project')
vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml' vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml'
inventory_path = 'ansible/inventory/hosts' inventory_path = 'ansible/inventory/hosts'
@ -16,7 +13,11 @@ docker_hostname = ci_lib.get_docker_hostname()
with ci_lib.Fold('docker_setup'): with ci_lib.Fold('docker_setup'):
containers = ci_lib.make_containers(port_offset=500, name_prefix='debops-') containers = ci_lib.container_specs(
['debian*2'],
base_port=2700,
name_template='debops-target-%(distro)s-%(index)d',
)
ci_lib.start_containers(containers) ci_lib.start_containers(containers)

@ -1,8 +0,0 @@
#!/usr/bin/env python
import ci_lib
batches = [
]
ci_lib.run_batches(batches)

@ -1,14 +0,0 @@
#!/usr/bin/env python
import ci_lib
batches = [
]
if ci_lib.have_docker():
batches.append([
'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi',
])
ci_lib.run_batches(batches)

@ -3,9 +3,6 @@
import ci_lib import ci_lib
batches = [ batches = [
[
'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi',
],
[ [
'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv', 'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv',
] ]

@ -0,0 +1,326 @@
# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
name: Tests
on:
pull_request:
push:
branches-ignore:
- docs-master
env:
#ANSIBLE_VERBOSITY: 3
#MITOGEN_LOG_LEVEL: DEBUG
MITOGEN_TEST_IMAGE_TEMPLATE: "ghcr.io/mitogen-hq/%(distro)s-test"
# https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners
# https://github.com/actions/runner-images/blob/main/README.md#software-and-image-support
jobs:
linux:
# https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
include:
- name: Ans_27_210
tox_env: py27-mode_ansible-ansible2.10
- name: Ans_27_4
tox_env: py27-mode_ansible-ansible4
- name: Ans_36_210
python_version: '3.6'
tox_env: py36-mode_ansible-ansible2.10
- name: Ans_36_4
python_version: '3.6'
tox_env: py36-mode_ansible-ansible4
- name: Ans_311_210
python_version: '3.11'
tox_env: py311-mode_ansible-ansible2.10
- name: Ans_311_3
python_version: '3.11'
tox_env: py311-mode_ansible-ansible3
- name: Ans_311_4
python_version: '3.11'
tox_env: py311-mode_ansible-ansible4
- name: Ans_311_5
python_version: '3.11'
tox_env: py311-mode_ansible-ansible5
- name: Ans_312_6
python_version: '3.12'
tox_env: py312-mode_ansible-ansible6
- name: Ans_312_7
python_version: '3.12'
tox_env: py312-mode_ansible-ansible7
- name: Ans_312_8
python_version: '3.12'
tox_env: py312-mode_ansible-ansible8
- name: Ans_312_9
python_version: '3.12'
tox_env: py312-mode_ansible-ansible9
- name: Ans_312_10
python_version: '3.12'
tox_env: py312-mode_ansible-ansible10
- name: Van_312_10
python_version: '3.12'
tox_env: py312-mode_ansible-ansible10-strategy_linear
- name: Mito_27_centos6
tox_env: py27-mode_mitogen-distro_centos6
- name: Mito_27_centos7
tox_env: py27-mode_mitogen-distro_centos7
- name: Mito_27_centos8
tox_env: py27-mode_mitogen-distro_centos8
- name: Mito_27_debian9
tox_env: py27-mode_mitogen-distro_debian9
- name: Mito_27_debian10
tox_env: py27-mode_mitogen-distro_debian10
- name: Mito_27_debian11
tox_env: py27-mode_mitogen-distro_debian11
- name: Mito_27_ubuntu1604
tox_env: py27-mode_mitogen-distro_ubuntu1604
- name: Mito_27_ubuntu1804
tox_env: py27-mode_mitogen-distro_ubuntu1804
- name: Mito_27_ubuntu2004
tox_env: py27-mode_mitogen-distro_ubuntu2004
- name: Mito_36_centos6
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_centos6
- name: Mito_36_centos7
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_centos7
- name: Mito_36_centos8
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_centos8
- name: Mito_36_debian9
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_debian9
- name: Mito_36_debian10
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_debian10
- name: Mito_36_debian11
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_debian11
- name: Mito_36_ubuntu1604
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_ubuntu1604
- name: Mito_36_ubuntu1804
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_ubuntu1804
- name: Mito_36_ubuntu2004
python_version: '3.6'
tox_env: py36-mode_mitogen-distro_ubuntu2004
- name: Mito_312_centos6
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_centos6
- name: Mito_312_centos7
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_centos7
- name: Mito_312_centos8
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_centos8
- name: Mito_312_debian9
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_debian9
- name: Mito_312_debian10
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_debian10
- name: Mito_312_debian11
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_debian11
- name: Mito_312_ubuntu1604
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_ubuntu1604
- name: Mito_312_ubuntu1804
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_ubuntu1804
- name: Mito_312_ubuntu2004
python_version: '3.12'
tox_env: py312-mode_mitogen-distro_ubuntu2004
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python_version }}
if: ${{ matrix.python_version }}
- uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Install build deps
run: |
set -o errexit -o nounset -o pipefail
sudo apt-get update
sudo apt-get install -y python2-dev python3-pip virtualenv
- name: Show Python versions
run: |
set -o errexit -o nounset -o pipefail
# macOS builders lack a realpath command
type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version
type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version
type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version
echo
if [ -e /usr/bin/python ]; then
echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')"
fi
if [ -e /usr/bin/python2 ]; then
echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')"
fi
if [ -e /usr/bin/python2.7 ]; then
echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')"
fi
- name: Install tooling
run: |
set -o errexit -o nounset -o pipefail
# Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))')
if [[ -z $PYTHON ]]; then
echo 1>&2 "Python interpreter could not be determined"
exit 1
fi
if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then
"$PYTHON" -m ensurepip --user --altinstall --no-default-pip
"$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
elif [[ $PYTHON == "python2.7" ]]; then
curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py"
"$PYTHON" get-pip.py --user --no-python-version-warning
# Avoid Python 2.x pip masking system pip
rm -f ~/.local/bin/{easy_install,pip,wheel}
"$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
else
"$PYTHON" -m pip install -r "tests/requirements-tox.txt"
fi
- name: Run tests
env:
GITHUB_ACTOR: ${{ github.actor }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -o errexit -o nounset -o pipefail
# Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))')
if [[ -z $PYTHON ]]; then
echo 1>&2 "Python interpreter could not be determined"
exit 1
fi
"$PYTHON" -m tox -e "${{ matrix.tox_env }}"
macos:
# https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md
runs-on: macos-12
timeout-minutes: 120
strategy:
fail-fast: false
matrix:
include:
- name: Mito_27
tox_env: py27-mode_mitogen
- name: Mito_312
tox_env: py312-mode_mitogen
- name: Loc_27_210
tox_env: py27-mode_localhost-ansible2.10
- name: Loc_312_10
tox_env: py312-mode_localhost-ansible10
- name: Van_27_210
tox_env: py27-mode_localhost-ansible2.10-strategy_linear
- name: Van_312_10
tox_env: py312-mode_localhost-ansible10-strategy_linear
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python_version }}
if: ${{ matrix.python_version }}
- name: Show Python versions
run: |
set -o errexit -o nounset -o pipefail
# macOS builders lack a realpath command
type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version
type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version
type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version
echo
if [ -e /usr/bin/python ]; then
echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')"
fi
if [ -e /usr/bin/python2 ]; then
echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')"
fi
if [ -e /usr/bin/python2.7 ]; then
echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')"
fi
if [ -e /Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7 ]; then
# GitHub macOS 12 images: python2.7 is installed, but not on $PATH
echo "/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7: sys.executable: $(/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7 -c 'import sys; print(sys.executable)')"
fi
- name: Install tooling
run: |
set -o errexit -o nounset -o pipefail
# Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))')
if [[ -z $PYTHON ]]; then
echo 1>&2 "Python interpreter could not be determined"
exit 1
fi
if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then
# GitHub macOS 12 images: python2.7 is installed, but not on $PATH
PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7"
"$PYTHON" -m ensurepip --user --altinstall --no-default-pip
"$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
elif [[ $PYTHON == "python2.7" ]]; then
curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py"
"$PYTHON" get-pip.py --user --no-python-version-warning
# Avoid Python 2.x pip masking system pip
rm -f ~/.local/bin/{easy_install,pip,wheel}
"$PYTHON" -m pip install --user -r "tests/requirements-tox.txt"
else
"$PYTHON" -m pip install -r "tests/requirements-tox.txt"
fi
- name: Run tests
env:
GITHUB_ACTOR: ${{ github.actor }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -o errexit -o nounset -o pipefail
# Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12)
PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))')
if [[ -z $PYTHON ]]; then
echo 1>&2 "Python interpreter could not be determined"
exit 1
fi
if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then
# GitHub macOS 12 images: python2.7 is installed, but not on $PATH
PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7"
fi
"$PYTHON" -m tox -e "${{ matrix.tox_env }}"

@ -29,6 +29,7 @@ Unreleased
releases before 2.10 releases before 2.10
* :gh:issue:`1127` :mod:`ansible_mitogen`: Consolidate Python 2 & 3 * :gh:issue:`1127` :mod:`ansible_mitogen`: Consolidate Python 2 & 3
compatibility compatibility
* :gh:issue:`1128` CI: Start migration from Azure DevOps to GitHub Actions
v0.3.10 (2024-09-20) v0.3.10 (2024-09-20)

@ -53,20 +53,22 @@
vars: vars:
ansible_become_pass: user1_password ansible_become_pass: user1_password
when: when:
# https://github.com/ansible/ansible/pull/70785 # CI containers lack `setfacl` for unpriv -> unpriv
- ansible_facts.distribution not in ["MacOSX"] # https://github.com/mitogen-hq/mitogen/issues/1118
or ansible_version.full is version("2.11", ">=", strict=True) - is_mitogen
or is_mitogen or (ansible_facts.distribution in ["MacOSX"]
and ansible_version.full is version("2.11", ">=", strict=True))
- assert: - assert:
that: that:
- out.stdout == 'mitogen__user1' - out.stdout == 'mitogen__user1'
fail_msg: out={{out}} fail_msg: out={{out}}
when: when:
# https://github.com/ansible/ansible/pull/70785 # CI containers lack `setfacl` for unpriv -> unpriv
- ansible_facts.distribution not in ["MacOSX"] # https://github.com/mitogen-hq/mitogen/issues/1118
or ansible_version.full is version("2.11", ">=", strict=True) - is_mitogen
or is_mitogen or (ansible_facts.distribution in ["MacOSX"]
and ansible_version.full is version("2.11", ">=", strict=True))
- name: Ensure password su without chdir succeeds - name: Ensure password su without chdir succeeds
shell: whoami shell: whoami
@ -76,20 +78,22 @@
vars: vars:
ansible_become_pass: user1_password ansible_become_pass: user1_password
when: when:
# https://github.com/ansible/ansible/pull/70785 # CI containers lack `setfacl` for unpriv -> unpriv
- ansible_facts.distribution not in ["MacOSX"] # https://github.com/mitogen-hq/mitogen/issues/1118
or ansible_version.full is version("2.11", ">=", strict=True) - is_mitogen
or is_mitogen or (ansible_facts.distribution in ["MacOSX"]
and ansible_version.full is version("2.11", ">=", strict=True))
- assert: - assert:
that: that:
- out.stdout == 'mitogen__user1' - out.stdout == 'mitogen__user1'
fail_msg: out={{out}} fail_msg: out={{out}}
when: when:
# https://github.com/ansible/ansible/pull/70785 # CI containers lack `setfacl` for unpriv -> unpriv
- ansible_facts.distribution not in ["MacOSX"] # https://github.com/mitogen-hq/mitogen/issues/1118
or ansible_version.full is version("2.11", ">=", strict=True) - is_mitogen
or is_mitogen or (ansible_facts.distribution in ["MacOSX"]
and ansible_version.full is version("2.11", ">=", strict=True))
tags: tags:
- su - su

@ -99,7 +99,7 @@
that: that:
- auto_out.ansible_facts.discovered_interpreter_python is defined - auto_out.ansible_facts.discovered_interpreter_python is defined
- auto_out.ansible_facts.discovered_interpreter_python == echoout.discovered_python.as_seen - auto_out.ansible_facts.discovered_interpreter_python == echoout.discovered_python.as_seen
- echoout.discovered_python.resolved == echoout.running_python.sys.executable.resolved - echoout.discovered_python.sys.executable.as_seen == echoout.running_python.sys.executable.as_seen
fail_msg: fail_msg:
- "auto_out: {{ auto_out }}" - "auto_out: {{ auto_out }}"
- "echoout: {{ echoout }}" - "echoout: {{ echoout }}"

@ -10,11 +10,97 @@ from __future__ import absolute_import, division, print_function
__metaclass__ = type __metaclass__ = type
import os import os
import stat
import platform import platform
import subprocess
import sys import sys
from ansible.module_utils.basic import AnsibleModule from ansible.module_utils.basic import AnsibleModule
# trace_realpath() and _join_tracepath() adapated from stdlib posixpath.py
# https://github.com/python/cpython/blob/v3.12.6/Lib/posixpath.py#L423-L492
# Copyright (c) 2001 - 2023 Python Software Foundation
# Copyright (c) 2024 Alex Willmer <alex@moreati.org.uk>
# License: Python Software Foundation License Version 2
def trace_realpath(filename, strict=False):
"""
Return the canonical path of the specified filename, and a trace of
the route taken, eliminating any symbolic links encountered in the path.
"""
path, trace, ok = _join_tracepath(filename[:0], filename, strict, seen={}, trace=[])
return os.path.abspath(path), trace
def _join_tracepath(path, rest, strict, seen, trace):
"""
Join two paths, normalizing and eliminating any symbolic links encountered
in the second path.
"""
trace.append(rest)
if isinstance(path, bytes):
sep = b'/'
curdir = b'.'
pardir = b'..'
else:
sep = '/'
curdir = '.'
pardir = '..'
if os.path.isabs(rest):
rest = rest[1:]
path = sep
while rest:
name, _, rest = rest.partition(sep)
if not name or name == curdir:
# current dir
continue
if name == pardir:
# parent dir
if path:
path, name = os.path.split(path)
if name == pardir:
path = os.path.join(path, pardir, pardir)
else:
path = pardir
continue
newpath = os.path.join(path, name)
try:
st = os.lstat(newpath)
except OSError:
if strict:
raise
is_link = False
else:
is_link = stat.S_ISLNK(st.st_mode)
if not is_link:
path = newpath
continue
# Resolve the symbolic link
if newpath in seen:
# Already seen this path
path = seen[newpath]
if path is not None:
# use cached value
continue
# The symlink is not resolved, so we must have a symlink loop.
if strict:
# Raise OSError(errno.ELOOP)
os.stat(newpath)
else:
# Return already resolved part + rest of the path unchanged.
return os.path.join(newpath, rest), trace, False
seen[newpath] = None # not resolved symlink
path, trace, ok = _join_tracepath(path, os.readlink(newpath), strict, seen, trace)
if not ok:
return os.path.join(path, rest), False
seen[newpath] = path # resolved symlink
return path, trace, True
def main(): def main():
module = AnsibleModule(argument_spec=dict( module = AnsibleModule(argument_spec=dict(
facts_copy=dict(type=dict, default={}), facts_copy=dict(type=dict, default={}),
@ -33,7 +119,18 @@ def main():
sys.executable = "/usr/bin/python" sys.executable = "/usr/bin/python"
facts_copy = module.params['facts_copy'] facts_copy = module.params['facts_copy']
discovered_interpreter_python = facts_copy['discovered_interpreter_python'] discovered_interpreter_python = facts_copy['discovered_interpreter_python']
d_i_p_realpath, d_i_p_trace = trace_realpath(discovered_interpreter_python)
d_i_p_proc = subprocess.Popen(
[discovered_interpreter_python, '-c', 'import sys; print(sys.executable)'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
)
d_i_p_stdout, d_i_p_stderr = d_i_p_proc.communicate()
sys_exec_realpath, sys_exec_trace = trace_realpath(sys.executable)
result = { result = {
'changed': False, 'changed': False,
'ansible_facts': module.params['facts_to_override'], 'ansible_facts': module.params['facts_to_override'],
@ -43,7 +140,17 @@ def main():
), ),
'discovered_python': { 'discovered_python': {
'as_seen': discovered_interpreter_python, 'as_seen': discovered_interpreter_python,
'resolved': os.path.realpath(discovered_interpreter_python), 'resolved': d_i_p_realpath,
'trace': [os.path.abspath(p) for p in d_i_p_trace],
'sys': {
'executable': {
'as_seen': d_i_p_stdout.decode('ascii').rstrip('\n'),
'proc': {
'stderr': d_i_p_stderr.decode('ascii'),
'returncode': d_i_p_proc.returncode,
},
},
},
}, },
'running_python': { 'running_python': {
'platform': { 'platform': {
@ -54,7 +161,8 @@ def main():
'sys': { 'sys': {
'executable': { 'executable': {
'as_seen': sys.executable, 'as_seen': sys.executable,
'resolved': os.path.realpath(sys.executable), 'resolved': sys_exec_realpath,
'trace': [os.path.abspath(p) for p in sys_exec_trace],
}, },
'platform': sys.platform, 'platform': sys.platform,
'version_info': { 'version_info': {

@ -9,7 +9,7 @@
--change 'EXPOSE 22' --change 'EXPOSE 22'
--change 'CMD ["/usr/sbin/sshd", "-D"]' --change 'CMD ["/usr/sbin/sshd", "-D"]'
{{ inventory_hostname }} {{ inventory_hostname }}
public.ecr.aws/n5z0e8q9/{{ inventory_hostname }}-test {{ container_image_name }}
delegate_to: localhost delegate_to: localhost
- name: Stop containers - name: Stop containers

@ -4,6 +4,9 @@ common_packages:
- strace - strace
- sudo - sudo
container_image_name: "{{ container_registry }}/{{ inventory_hostname }}-test"
container_registry: public.ecr.aws/n5z0e8q9
sudo_group: sudo_group:
MacOSX: admin MacOSX: admin
Debian: sudo Debian: sudo

@ -51,6 +51,12 @@ except NameError:
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
DISTRO = os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')
IMAGE_TEMPLATE = os.environ.get(
'MITOGEN_TEST_IMAGE_TEMPLATE',
'public.ecr.aws/n5z0e8q9/%(distro)s-test',
)
TESTS_DIR = os.path.join(os.path.dirname(__file__)) TESTS_DIR = os.path.join(os.path.dirname(__file__))
ANSIBLE_LIB_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib') ANSIBLE_LIB_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib')
ANSIBLE_MODULE_UTILS_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils') ANSIBLE_MODULE_UTILS_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils')
@ -509,6 +515,7 @@ class TestCase(unittest.TestCase):
def get_docker_host(): def get_docker_host():
# Duplicated in ci_lib
url = os.environ.get('DOCKER_HOST') url = os.environ.get('DOCKER_HOST')
if url in (None, 'http+docker://localunixsocket'): if url in (None, 'http+docker://localunixsocket'):
return 'localhost' return 'localhost'
@ -549,19 +556,23 @@ class DockerizedSshDaemon(object):
] ]
subprocess.check_output(args) subprocess.check_output(args)
def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')): def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE):
if '-' in mitogen_test_distro: # Code duplicated in ci_lib.py, both should be updated together
distro, _py3 = mitogen_test_distro.split('-') distro_pattern = re.compile(r'''
else: (?P<distro>(?P<family>[a-z]+)[0-9]+)
distro = mitogen_test_distro (?:-(?P<py>py3))?
_py3 = None (?:\*(?P<count>[0-9]+))?
''',
re.VERBOSE,
)
d = distro_pattern.match(distro).groupdict(default=None)
if _py3 == 'py3': if d.pop('py') == 'py3':
self.python_path = '/usr/bin/python3' self.python_path = '/usr/bin/python3'
else: else:
self.python_path = '/usr/bin/python' self.python_path = '/usr/bin/python'
self.image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,) self.image = image_template % d
self.start_container() self.start_container()
self.host = self.get_host() self.host = self.get_host()
self.port = self.get_port(self.container_name) self.port = self.get_port(self.container_name)
@ -601,6 +612,9 @@ class DockerizedSshDaemon(object):
class BrokerMixin(object): class BrokerMixin(object):
broker_class = mitogen.master.Broker broker_class = mitogen.master.Broker
# Flag for tests that shutdown the broker themself
# e.g. unix_test.ListenerTest
broker_shutdown = False broker_shutdown = False
def setUp(self): def setUp(self):

@ -89,10 +89,7 @@ deps =
install_command = install_command =
python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages} python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages}
commands_pre = commands_pre =
mode_ansible: {toxinidir}/.ci/ansible_install.py
mode_debops_common: {toxinidir}/.ci/debops_common_install.py mode_debops_common: {toxinidir}/.ci/debops_common_install.py
mode_localhost: {toxinidir}/.ci/localhost_ansible_install.py
mode_mitogen: {toxinidir}/.ci/mitogen_install.py
commands = commands =
mode_ansible: {toxinidir}/.ci/ansible_tests.py mode_ansible: {toxinidir}/.ci/ansible_tests.py
mode_debops_common: {toxinidir}/.ci/debops_common_tests.py mode_debops_common: {toxinidir}/.ci/debops_common_tests.py
@ -100,10 +97,8 @@ commands =
mode_mitogen: {toxinidir}/.ci/mitogen_tests.py mode_mitogen: {toxinidir}/.ci/mitogen_tests.py
passenv = passenv =
ANSIBLE_* ANSIBLE_*
AWS_ACCESS_KEY_ID
AWS_DEFAULT_REGION
AWS_SECRET_ACCESS_KEY
HOME HOME
MITOGEN_*
# Azure DevOps, TF_BUILD is set to 'True' when running in a build task # Azure DevOps, TF_BUILD is set to 'True' when running in a build task
# https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables # https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables
TF_BUILD TF_BUILD

Loading…
Cancel
Save