From b926795973906d77918843bf5f750f5c92315204 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Tue, 24 Sep 2024 18:04:18 +0100 Subject: [PATCH 1/5] ci: Move container registry authentication to an Azure Devops step This aims to - Reduce duplication - Seperate CI specific setup from test setup - Prepare for migration from Azure DevOps to GitHub Actions --- .ci/ansible_install.py | 11 ----------- .ci/azure-pipelines-steps.yml | 17 +++++++++++++---- .ci/debops_common_install.py | 3 --- .ci/localhost_ansible_install.py | 8 -------- .ci/mitogen_install.py | 14 -------------- .ci/mitogen_py24_install.py | 3 --- tox.ini | 6 ------ 7 files changed, 13 insertions(+), 49 deletions(-) delete mode 100755 .ci/ansible_install.py delete mode 100755 .ci/localhost_ansible_install.py delete mode 100755 .ci/mitogen_install.py diff --git a/.ci/ansible_install.py b/.ci/ansible_install.py deleted file mode 100755 index 3b217ff2..00000000 --- a/.ci/ansible_install.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python - -import ci_lib - -batches = [ - [ - 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', - ] -] - -ci_lib.run_batches(batches) diff --git a/.ci/azure-pipelines-steps.yml b/.ci/azure-pipelines-steps.yml index 919b992b..791372af 100644 --- a/.ci/azure-pipelines-steps.yml +++ b/.ci/azure-pipelines-steps.yml @@ -14,6 +14,19 @@ steps: versionSpec: '$(python.version)' condition: ne(variables['python.version'], '') +- script: | + set -o errexit + set -o nounset + set -o pipefail + + aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws + displayName: Authenticate to container registry + condition: eq(variables['Agent.OS'], 'Linux') + env: + AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) + AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) + AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION) + - script: | set -o errexit set -o nounset @@ -90,7 +103,3 @@ steps: "$PYTHON" -m tox -e "$(tox.env)" displayName: "Run tests" - env: - AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID) - AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) - AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION) diff --git a/.ci/debops_common_install.py b/.ci/debops_common_install.py index 565f9488..13217133 100755 --- a/.ci/debops_common_install.py +++ b/.ci/debops_common_install.py @@ -9,9 +9,6 @@ ci_lib.run_batches([ [ 'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"', ], - [ - 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', - ], ]) ci_lib.run('ansible-galaxy collection install debops.debops:==2.1.2') diff --git a/.ci/localhost_ansible_install.py b/.ci/localhost_ansible_install.py deleted file mode 100755 index d08ddafc..00000000 --- a/.ci/localhost_ansible_install.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python - -import ci_lib - -batches = [ -] - -ci_lib.run_batches(batches) diff --git a/.ci/mitogen_install.py b/.ci/mitogen_install.py deleted file mode 100755 index 23ff384b..00000000 --- a/.ci/mitogen_install.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python - -import ci_lib - -batches = [ -] - -if ci_lib.have_docker(): - batches.append([ - 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', - ]) - - -ci_lib.run_batches(batches) diff --git a/.ci/mitogen_py24_install.py b/.ci/mitogen_py24_install.py index 8af90405..85ea013c 100755 --- a/.ci/mitogen_py24_install.py +++ b/.ci/mitogen_py24_install.py @@ -3,9 +3,6 @@ import ci_lib batches = [ - [ - 'if [ "${TF_BUILD:-false}" = "True" ]; then aws ecr-public get-login-password | docker login --username AWS --password-stdin public.ecr.aws; fi', - ], [ 'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv', ] diff --git a/tox.ini b/tox.ini index 9bb82def..9820a79f 100644 --- a/tox.ini +++ b/tox.ini @@ -89,10 +89,7 @@ deps = install_command = python -m pip --no-python-version-warning --disable-pip-version-check install {opts} {packages} commands_pre = - mode_ansible: {toxinidir}/.ci/ansible_install.py mode_debops_common: {toxinidir}/.ci/debops_common_install.py - mode_localhost: {toxinidir}/.ci/localhost_ansible_install.py - mode_mitogen: {toxinidir}/.ci/mitogen_install.py commands = mode_ansible: {toxinidir}/.ci/ansible_tests.py mode_debops_common: {toxinidir}/.ci/debops_common_tests.py @@ -100,9 +97,6 @@ commands = mode_mitogen: {toxinidir}/.ci/mitogen_tests.py passenv = ANSIBLE_* - AWS_ACCESS_KEY_ID - AWS_DEFAULT_REGION - AWS_SECRET_ACCESS_KEY HOME # Azure DevOps, TF_BUILD is set to 'True' when running in a build task # https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables From 8b92e09655e31f0404b302e001c1f4f024616c57 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 25 Sep 2024 12:19:39 +0100 Subject: [PATCH 2/5] ci: Extract container registry location into variables Preperation for migrating from Azure DevOps with Amazon Elastic Container Registry (AWS ECR), to GitHub Actions with GitHub Container Registry (GHCR). DebOps tests are not currently being run, the updates to .ci/debops*.py are best effort only. --- .ci/README.md | 1 - .ci/ansible_tests.py | 2 +- .ci/ci_lib.py | 48 ++++++++++++++---------- .ci/debops_common_install.py | 3 -- .ci/debops_common_tests.py | 9 +++-- tests/image_prep/_container_finalize.yml | 2 +- tests/image_prep/group_vars/all.yml | 3 ++ tests/testlib.py | 30 +++++++++++---- tox.ini | 1 + 9 files changed, 62 insertions(+), 37 deletions(-) diff --git a/.ci/README.md b/.ci/README.md index 67a3805b..9248ac58 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -28,7 +28,6 @@ for doing `setup.py install` while pulling a Docker container, for example. ### Environment Variables -* `TARGET_COUNT`: number of targets for `debops_` run. Defaults to 2. * `DISTRO`: the `mitogen_` tests need a target Docker container distro. This name comes from the Docker Hub `mitogen` user, i.e. `mitogen/$DISTRO-test` * `DISTROS`: the `ansible_` tests can run against multiple targets diff --git a/.ci/ansible_tests.py b/.ci/ansible_tests.py index 102eda9c..3ec48dfd 100755 --- a/.ci/ansible_tests.py +++ b/.ci/ansible_tests.py @@ -35,7 +35,7 @@ ci_lib.check_stray_processes(interesting) with ci_lib.Fold('docker_setup'): - containers = ci_lib.make_containers() + containers = ci_lib.container_specs(ci_lib.DISTROS) ci_lib.start_containers(containers) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 3e716385..dfe49b97 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -27,6 +27,13 @@ os.chdir( ) ) + +IMAGE_TEMPLATE = os.environ.get( + 'MITOGEN_TEST_IMAGE_TEMPLATE', + 'public.ecr.aws/n5z0e8q9/%(distro)s-test', +) + + _print = print def print(*args, **kwargs): file = kwargs.get('file', sys.stdout) @@ -193,8 +200,6 @@ GIT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) DISTRO = os.environ.get('DISTRO', 'debian9') # Used only when MODE=ansible DISTROS = os.environ.get('DISTROS', 'centos6 centos8 debian9 debian11 ubuntu1604 ubuntu2004').split() -TARGET_COUNT = int(os.environ.get('TARGET_COUNT', '2')) -BASE_PORT = 2200 TMP = TempDir().path @@ -217,6 +222,7 @@ os.environ['PYTHONPATH'] = '%s:%s' % ( def get_docker_hostname(): """Return the hostname where the docker daemon is running. """ + # Duplicated in testlib url = os.environ.get('DOCKER_HOST') if url in (None, 'http+docker://localunixsocket'): return 'localhost' @@ -225,27 +231,34 @@ def get_docker_hostname(): return parsed.netloc.partition(':')[0] -def make_containers(name_prefix='', port_offset=0): +def container_specs( + distros, + base_port=2200, + image_template=IMAGE_TEMPLATE, + name_template='target-%(distro)s-%(index)d', +): """ >>> import pprint - >>> BASE_PORT=2200; DISTROS=['debian11', 'centos6'] - >>> pprint.pprint(make_containers()) + >>> pprint.pprint(container_specs(['debian11-py3', 'centos6'])) [{'distro': 'debian11', 'family': 'debian', 'hostname': 'localhost', 'image': 'public.ecr.aws/n5z0e8q9/debian11-test', + 'index': 1, 'name': 'target-debian11-1', 'port': 2201, - 'python_path': '/usr/bin/python'}, + 'python_path': '/usr/bin/python3'}, {'distro': 'centos6', 'family': 'centos', 'hostname': 'localhost', 'image': 'public.ecr.aws/n5z0e8q9/centos6-test', + 'index': 2, 'name': 'target-centos6-2', 'port': 2202, 'python_path': '/usr/bin/python'}] """ docker_hostname = get_docker_hostname() + # Code duplicated in testlib.py, both should be updated together distro_pattern = re.compile(r''' (?P(?P[a-z]+)[0-9]+) (?:-(?Ppy3))? @@ -256,30 +269,27 @@ def make_containers(name_prefix='', port_offset=0): i = 1 lst = [] - for distro in DISTROS: + for distro in distros: + # Code duplicated in testlib.py, both should be updated together d = distro_pattern.match(distro).groupdict(default=None) - distro = d['distro'] - family = d['family'] - image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,) - if d['py'] == 'py3': + if d.pop('py') == 'py3': python_path = '/usr/bin/python3' else: python_path = '/usr/bin/python' - if d['count']: - count = int(count) - else: - count = 1 + count = int(d.pop('count') or '1', 10) for x in range(count): - lst.append({ - "distro": distro, "family": family, "image": image, - "name": name_prefix + ("target-%s-%s" % (distro, i)), + d['index'] = i + d.update({ + 'image': image_template % d, + 'name': name_template % d, "hostname": docker_hostname, - "port": BASE_PORT + i + port_offset, + 'port': base_port + i, "python_path": python_path, }) + lst.append(d) i += 1 return lst diff --git a/.ci/debops_common_install.py b/.ci/debops_common_install.py index 13217133..825126c7 100755 --- a/.ci/debops_common_install.py +++ b/.ci/debops_common_install.py @@ -2,9 +2,6 @@ import ci_lib -# Naturally DebOps only supports Debian. -ci_lib.DISTROS = ['debian'] - ci_lib.run_batches([ [ 'python -m pip --no-python-version-warning --disable-pip-version-check "debops[ansible]==2.1.2"', diff --git a/.ci/debops_common_tests.py b/.ci/debops_common_tests.py index 7db8a797..b065486f 100755 --- a/.ci/debops_common_tests.py +++ b/.ci/debops_common_tests.py @@ -6,9 +6,6 @@ import sys import ci_lib -# DebOps only supports Debian. -ci_lib.DISTROS = ['debian'] * ci_lib.TARGET_COUNT - project_dir = os.path.join(ci_lib.TMP, 'project') vars_path = 'ansible/inventory/group_vars/debops_all_hosts.yml' inventory_path = 'ansible/inventory/hosts' @@ -16,7 +13,11 @@ docker_hostname = ci_lib.get_docker_hostname() with ci_lib.Fold('docker_setup'): - containers = ci_lib.make_containers(port_offset=500, name_prefix='debops-') + containers = ci_lib.container_specs( + ['debian*2'], + base_port=2700, + name_template='debops-target-%(distro)s-%(index)d', + ) ci_lib.start_containers(containers) diff --git a/tests/image_prep/_container_finalize.yml b/tests/image_prep/_container_finalize.yml index d61d9b3b..5329fefa 100644 --- a/tests/image_prep/_container_finalize.yml +++ b/tests/image_prep/_container_finalize.yml @@ -9,7 +9,7 @@ --change 'EXPOSE 22' --change 'CMD ["/usr/sbin/sshd", "-D"]' {{ inventory_hostname }} - public.ecr.aws/n5z0e8q9/{{ inventory_hostname }}-test + {{ container_image_name }} delegate_to: localhost - name: Stop containers diff --git a/tests/image_prep/group_vars/all.yml b/tests/image_prep/group_vars/all.yml index 5f182f86..91ff934d 100644 --- a/tests/image_prep/group_vars/all.yml +++ b/tests/image_prep/group_vars/all.yml @@ -4,6 +4,9 @@ common_packages: - strace - sudo +container_image_name: "{{ container_registry }}/{{ inventory_hostname }}-test" +container_registry: public.ecr.aws/n5z0e8q9 + sudo_group: MacOSX: admin Debian: sudo diff --git a/tests/testlib.py b/tests/testlib.py index a52292ce..76743e82 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -51,6 +51,12 @@ except NameError: LOG = logging.getLogger(__name__) +DISTRO = os.environ.get('MITOGEN_TEST_DISTRO', 'debian9') +IMAGE_TEMPLATE = os.environ.get( + 'MITOGEN_TEST_IMAGE_TEMPLATE', + 'public.ecr.aws/n5z0e8q9/%(distro)s-test', +) + TESTS_DIR = os.path.join(os.path.dirname(__file__)) ANSIBLE_LIB_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib') ANSIBLE_MODULE_UTILS_DIR = os.path.join(TESTS_DIR, 'ansible', 'lib', 'module_utils') @@ -509,6 +515,7 @@ class TestCase(unittest.TestCase): def get_docker_host(): + # Duplicated in ci_lib url = os.environ.get('DOCKER_HOST') if url in (None, 'http+docker://localunixsocket'): return 'localhost' @@ -549,19 +556,23 @@ class DockerizedSshDaemon(object): ] subprocess.check_output(args) - def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')): - if '-' in mitogen_test_distro: - distro, _py3 = mitogen_test_distro.split('-') - else: - distro = mitogen_test_distro - _py3 = None + def __init__(self, distro=DISTRO, image_template=IMAGE_TEMPLATE): + # Code duplicated in ci_lib.py, both should be updated together + distro_pattern = re.compile(r''' + (?P(?P[a-z]+)[0-9]+) + (?:-(?Ppy3))? + (?:\*(?P[0-9]+))? + ''', + re.VERBOSE, + ) + d = distro_pattern.match(distro).groupdict(default=None) - if _py3 == 'py3': + if d.pop('py') == 'py3': self.python_path = '/usr/bin/python3' else: self.python_path = '/usr/bin/python' - self.image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,) + self.image = image_template % d self.start_container() self.host = self.get_host() self.port = self.get_port(self.container_name) @@ -601,6 +612,9 @@ class DockerizedSshDaemon(object): class BrokerMixin(object): broker_class = mitogen.master.Broker + + # Flag for tests that shutdown the broker themself + # e.g. unix_test.ListenerTest broker_shutdown = False def setUp(self): diff --git a/tox.ini b/tox.ini index 9820a79f..9fb31bdc 100644 --- a/tox.ini +++ b/tox.ini @@ -98,6 +98,7 @@ commands = passenv = ANSIBLE_* HOME + MITOGEN_* # Azure DevOps, TF_BUILD is set to 'True' when running in a build task # https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables TF_BUILD From c6c8bfb690b8423b140917c9f5e9dcca72f17d30 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 25 Sep 2024 13:30:22 +0100 Subject: [PATCH 3/5] tests: Skip vanilla Ansible on Linux unpriviliged -> unprivileged become CI containers lack the necessary `setfacl` command. This has not previously been noticed because no vanilla Ansible jobs were being run on Linux, only on macOS. refs #1118 --- .../integration/become/su_password.yml | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/tests/ansible/integration/become/su_password.yml b/tests/ansible/integration/become/su_password.yml index 52d420db..207980c4 100644 --- a/tests/ansible/integration/become/su_password.yml +++ b/tests/ansible/integration/become/su_password.yml @@ -53,20 +53,22 @@ vars: ansible_become_pass: user1_password when: - # https://github.com/ansible/ansible/pull/70785 - - ansible_facts.distribution not in ["MacOSX"] - or ansible_version.full is version("2.11", ">=", strict=True) - or is_mitogen + # CI containers lack `setfacl` for unpriv -> unpriv + # https://github.com/mitogen-hq/mitogen/issues/1118 + - is_mitogen + or (ansible_facts.distribution in ["MacOSX"] + and ansible_version.full is version("2.11", ">=", strict=True)) - assert: that: - out.stdout == 'mitogen__user1' fail_msg: out={{out}} when: - # https://github.com/ansible/ansible/pull/70785 - - ansible_facts.distribution not in ["MacOSX"] - or ansible_version.full is version("2.11", ">=", strict=True) - or is_mitogen + # CI containers lack `setfacl` for unpriv -> unpriv + # https://github.com/mitogen-hq/mitogen/issues/1118 + - is_mitogen + or (ansible_facts.distribution in ["MacOSX"] + and ansible_version.full is version("2.11", ">=", strict=True)) - name: Ensure password su without chdir succeeds shell: whoami @@ -76,20 +78,22 @@ vars: ansible_become_pass: user1_password when: - # https://github.com/ansible/ansible/pull/70785 - - ansible_facts.distribution not in ["MacOSX"] - or ansible_version.full is version("2.11", ">=", strict=True) - or is_mitogen + # CI containers lack `setfacl` for unpriv -> unpriv + # https://github.com/mitogen-hq/mitogen/issues/1118 + - is_mitogen + or (ansible_facts.distribution in ["MacOSX"] + and ansible_version.full is version("2.11", ">=", strict=True)) - assert: that: - out.stdout == 'mitogen__user1' fail_msg: out={{out}} when: - # https://github.com/ansible/ansible/pull/70785 - - ansible_facts.distribution not in ["MacOSX"] - or ansible_version.full is version("2.11", ">=", strict=True) - or is_mitogen + # CI containers lack `setfacl` for unpriv -> unpriv + # https://github.com/mitogen-hq/mitogen/issues/1118 + - is_mitogen + or (ansible_facts.distribution in ["MacOSX"] + and ansible_version.full is version("2.11", ">=", strict=True)) tags: - su From 27214517a724431fb5dc73255b608bc4fe078479 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 26 Sep 2024 02:08:17 +0100 Subject: [PATCH 4/5] tests: Use a subprocess to check discovered python == running This replaces the use of `os.path.realpath()` which gave incorrect results on macOS - depending on the exact Python build, Python version, macOS version, installation method, and phase of the moon. realpath information kept around to aid debugging. --- .../ansible_2_8_tests.yml | 2 +- tests/ansible/lib/modules/test_echo_module.py | 112 +++++++++++++++++- 2 files changed, 111 insertions(+), 3 deletions(-) diff --git a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml index 0c7d30c9..fc0d3cf7 100644 --- a/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml +++ b/tests/ansible/integration/interpreter_discovery/ansible_2_8_tests.yml @@ -99,7 +99,7 @@ that: - auto_out.ansible_facts.discovered_interpreter_python is defined - auto_out.ansible_facts.discovered_interpreter_python == echoout.discovered_python.as_seen - - echoout.discovered_python.resolved == echoout.running_python.sys.executable.resolved + - echoout.discovered_python.sys.executable.as_seen == echoout.running_python.sys.executable.as_seen fail_msg: - "auto_out: {{ auto_out }}" - "echoout: {{ echoout }}" diff --git a/tests/ansible/lib/modules/test_echo_module.py b/tests/ansible/lib/modules/test_echo_module.py index d6a5fb9e..fe8ed69a 100644 --- a/tests/ansible/lib/modules/test_echo_module.py +++ b/tests/ansible/lib/modules/test_echo_module.py @@ -10,11 +10,97 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type import os +import stat import platform +import subprocess import sys + from ansible.module_utils.basic import AnsibleModule +# trace_realpath() and _join_tracepath() adapated from stdlib posixpath.py +# https://github.com/python/cpython/blob/v3.12.6/Lib/posixpath.py#L423-L492 +# Copyright (c) 2001 - 2023 Python Software Foundation +# Copyright (c) 2024 Alex Willmer +# License: Python Software Foundation License Version 2 + +def trace_realpath(filename, strict=False): + """ + Return the canonical path of the specified filename, and a trace of + the route taken, eliminating any symbolic links encountered in the path. + """ + path, trace, ok = _join_tracepath(filename[:0], filename, strict, seen={}, trace=[]) + return os.path.abspath(path), trace + + +def _join_tracepath(path, rest, strict, seen, trace): + """ + Join two paths, normalizing and eliminating any symbolic links encountered + in the second path. + """ + trace.append(rest) + if isinstance(path, bytes): + sep = b'/' + curdir = b'.' + pardir = b'..' + else: + sep = '/' + curdir = '.' + pardir = '..' + + if os.path.isabs(rest): + rest = rest[1:] + path = sep + + while rest: + name, _, rest = rest.partition(sep) + if not name or name == curdir: + # current dir + continue + if name == pardir: + # parent dir + if path: + path, name = os.path.split(path) + if name == pardir: + path = os.path.join(path, pardir, pardir) + else: + path = pardir + continue + newpath = os.path.join(path, name) + try: + st = os.lstat(newpath) + except OSError: + if strict: + raise + is_link = False + else: + is_link = stat.S_ISLNK(st.st_mode) + if not is_link: + path = newpath + continue + # Resolve the symbolic link + if newpath in seen: + # Already seen this path + path = seen[newpath] + if path is not None: + # use cached value + continue + # The symlink is not resolved, so we must have a symlink loop. + if strict: + # Raise OSError(errno.ELOOP) + os.stat(newpath) + else: + # Return already resolved part + rest of the path unchanged. + return os.path.join(newpath, rest), trace, False + seen[newpath] = None # not resolved symlink + path, trace, ok = _join_tracepath(path, os.readlink(newpath), strict, seen, trace) + if not ok: + return os.path.join(path, rest), False + seen[newpath] = path # resolved symlink + + return path, trace, True + + def main(): module = AnsibleModule(argument_spec=dict( facts_copy=dict(type=dict, default={}), @@ -33,7 +119,18 @@ def main(): sys.executable = "/usr/bin/python" facts_copy = module.params['facts_copy'] + discovered_interpreter_python = facts_copy['discovered_interpreter_python'] + d_i_p_realpath, d_i_p_trace = trace_realpath(discovered_interpreter_python) + d_i_p_proc = subprocess.Popen( + [discovered_interpreter_python, '-c', 'import sys; print(sys.executable)'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + + ) + d_i_p_stdout, d_i_p_stderr = d_i_p_proc.communicate() + + sys_exec_realpath, sys_exec_trace = trace_realpath(sys.executable) + result = { 'changed': False, 'ansible_facts': module.params['facts_to_override'], @@ -43,7 +140,17 @@ def main(): ), 'discovered_python': { 'as_seen': discovered_interpreter_python, - 'resolved': os.path.realpath(discovered_interpreter_python), + 'resolved': d_i_p_realpath, + 'trace': [os.path.abspath(p) for p in d_i_p_trace], + 'sys': { + 'executable': { + 'as_seen': d_i_p_stdout.decode('ascii').rstrip('\n'), + 'proc': { + 'stderr': d_i_p_stderr.decode('ascii'), + 'returncode': d_i_p_proc.returncode, + }, + }, + }, }, 'running_python': { 'platform': { @@ -54,7 +161,8 @@ def main(): 'sys': { 'executable': { 'as_seen': sys.executable, - 'resolved': os.path.realpath(sys.executable), + 'resolved': sys_exec_realpath, + 'trace': [os.path.abspath(p) for p in sys_exec_trace], }, 'platform': sys.platform, 'version_info': { From 4f60d01f09f2ae31c8f58cddafb17766e1054fa4 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 26 Sep 2024 02:15:39 +0100 Subject: [PATCH 5/5] CI: Enable GitHub Actions testing workflow This replicate the existing Azure DevOps workflow, and adds a couple of new jobs (Python 2.7 on macOS, Python + vanilla Ansible on Linux). The GitHub Actions use container images hosted on GitHub Container Registry (GHCR, ghcr.io/mitogen-hq). These images have been copied straight from the existing Amazon Elastic Cloud Registry (AWS ECR, public.ecr.aws/n5z0e8q9). A short period of parallel running is planned. Then a second PR will remove the Azure DevOps workflow. --- .github/workflows/tests.yml | 326 ++++++++++++++++++++++++++++++++++++ docs/changelog.rst | 1 + 2 files changed, 327 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..4520c3cf --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,326 @@ +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions + +name: Tests + +on: + pull_request: + push: + branches-ignore: + - docs-master + +env: + #ANSIBLE_VERBOSITY: 3 + #MITOGEN_LOG_LEVEL: DEBUG + MITOGEN_TEST_IMAGE_TEMPLATE: "ghcr.io/mitogen-hq/%(distro)s-test" + +# https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners +# https://github.com/actions/runner-images/blob/main/README.md#software-and-image-support +jobs: + linux: + # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md + runs-on: ubuntu-20.04 + + strategy: + fail-fast: false + matrix: + include: + - name: Ans_27_210 + tox_env: py27-mode_ansible-ansible2.10 + - name: Ans_27_4 + tox_env: py27-mode_ansible-ansible4 + + - name: Ans_36_210 + python_version: '3.6' + tox_env: py36-mode_ansible-ansible2.10 + - name: Ans_36_4 + python_version: '3.6' + tox_env: py36-mode_ansible-ansible4 + + - name: Ans_311_210 + python_version: '3.11' + tox_env: py311-mode_ansible-ansible2.10 + - name: Ans_311_3 + python_version: '3.11' + tox_env: py311-mode_ansible-ansible3 + - name: Ans_311_4 + python_version: '3.11' + tox_env: py311-mode_ansible-ansible4 + - name: Ans_311_5 + python_version: '3.11' + tox_env: py311-mode_ansible-ansible5 + - name: Ans_312_6 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible6 + - name: Ans_312_7 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible7 + - name: Ans_312_8 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible8 + - name: Ans_312_9 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible9 + - name: Ans_312_10 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible10 + - name: Van_312_10 + python_version: '3.12' + tox_env: py312-mode_ansible-ansible10-strategy_linear + + - name: Mito_27_centos6 + tox_env: py27-mode_mitogen-distro_centos6 + - name: Mito_27_centos7 + tox_env: py27-mode_mitogen-distro_centos7 + - name: Mito_27_centos8 + tox_env: py27-mode_mitogen-distro_centos8 + - name: Mito_27_debian9 + tox_env: py27-mode_mitogen-distro_debian9 + - name: Mito_27_debian10 + tox_env: py27-mode_mitogen-distro_debian10 + - name: Mito_27_debian11 + tox_env: py27-mode_mitogen-distro_debian11 + - name: Mito_27_ubuntu1604 + tox_env: py27-mode_mitogen-distro_ubuntu1604 + - name: Mito_27_ubuntu1804 + tox_env: py27-mode_mitogen-distro_ubuntu1804 + - name: Mito_27_ubuntu2004 + tox_env: py27-mode_mitogen-distro_ubuntu2004 + + - name: Mito_36_centos6 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_centos6 + - name: Mito_36_centos7 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_centos7 + - name: Mito_36_centos8 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_centos8 + - name: Mito_36_debian9 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_debian9 + - name: Mito_36_debian10 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_debian10 + - name: Mito_36_debian11 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_debian11 + - name: Mito_36_ubuntu1604 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_ubuntu1604 + - name: Mito_36_ubuntu1804 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_ubuntu1804 + - name: Mito_36_ubuntu2004 + python_version: '3.6' + tox_env: py36-mode_mitogen-distro_ubuntu2004 + + - name: Mito_312_centos6 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_centos6 + - name: Mito_312_centos7 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_centos7 + - name: Mito_312_centos8 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_centos8 + - name: Mito_312_debian9 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_debian9 + - name: Mito_312_debian10 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_debian10 + - name: Mito_312_debian11 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_debian11 + - name: Mito_312_ubuntu1604 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_ubuntu1604 + - name: Mito_312_ubuntu1804 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_ubuntu1804 + - name: Mito_312_ubuntu2004 + python_version: '3.12' + tox_env: py312-mode_mitogen-distro_ubuntu2004 + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python_version }} + if: ${{ matrix.python_version }} + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Install build deps + run: | + set -o errexit -o nounset -o pipefail + + sudo apt-get update + sudo apt-get install -y python2-dev python3-pip virtualenv + - name: Show Python versions + run: | + set -o errexit -o nounset -o pipefail + + # macOS builders lack a realpath command + type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version + type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version + type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version + echo + + if [ -e /usr/bin/python ]; then + echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2 ]; then + echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2.7 ]; then + echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')" + fi + - name: Install tooling + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then + "$PYTHON" -m ensurepip --user --altinstall --no-default-pip + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + elif [[ $PYTHON == "python2.7" ]]; then + curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py" + "$PYTHON" get-pip.py --user --no-python-version-warning + # Avoid Python 2.x pip masking system pip + rm -f ~/.local/bin/{easy_install,pip,wheel} + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + else + "$PYTHON" -m pip install -r "tests/requirements-tox.txt" + fi + - name: Run tests + env: + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + "$PYTHON" -m tox -e "${{ matrix.tox_env }}" + + macos: + # https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md + runs-on: macos-12 + timeout-minutes: 120 + + strategy: + fail-fast: false + matrix: + include: + - name: Mito_27 + tox_env: py27-mode_mitogen + - name: Mito_312 + tox_env: py312-mode_mitogen + + - name: Loc_27_210 + tox_env: py27-mode_localhost-ansible2.10 + - name: Loc_312_10 + tox_env: py312-mode_localhost-ansible10 + + - name: Van_27_210 + tox_env: py27-mode_localhost-ansible2.10-strategy_linear + - name: Van_312_10 + tox_env: py312-mode_localhost-ansible10-strategy_linear + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python_version }} + if: ${{ matrix.python_version }} + - name: Show Python versions + run: | + set -o errexit -o nounset -o pipefail + + # macOS builders lack a realpath command + type python && python -c"import os.path;print(os.path.realpath('$(type -p python)'))" && python --version + type python2 && python2 -c"import os.path;print(os.path.realpath('$(type -p python2)'))" && python2 --version + type python3 && python3 -c"import os.path;print(os.path.realpath('$(type -p python3)'))" && python3 --version + echo + + if [ -e /usr/bin/python ]; then + echo "/usr/bin/python: sys.executable: $(/usr/bin/python -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2 ]; then + echo "/usr/bin/python2: sys.executable: $(/usr/bin/python2 -c 'import sys; print(sys.executable)')" + fi + + if [ -e /usr/bin/python2.7 ]; then + echo "/usr/bin/python2.7: sys.executable: $(/usr/bin/python2.7 -c 'import sys; print(sys.executable)')" + fi + + if [ -e /Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7 ]; then + # GitHub macOS 12 images: python2.7 is installed, but not on $PATH + echo "/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7: sys.executable: $(/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7 -c 'import sys; print(sys.executable)')" + fi + - name: Install tooling + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then + # GitHub macOS 12 images: python2.7 is installed, but not on $PATH + PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7" + "$PYTHON" -m ensurepip --user --altinstall --no-default-pip + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + elif [[ $PYTHON == "python2.7" ]]; then + curl "https://bootstrap.pypa.io/pip/2.7/get-pip.py" --output "get-pip.py" + "$PYTHON" get-pip.py --user --no-python-version-warning + # Avoid Python 2.x pip masking system pip + rm -f ~/.local/bin/{easy_install,pip,wheel} + "$PYTHON" -m pip install --user -r "tests/requirements-tox.txt" + else + "$PYTHON" -m pip install -r "tests/requirements-tox.txt" + fi + - name: Run tests + env: + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -o errexit -o nounset -o pipefail + + # Tox environment name (e.g. py312-mode_mitogen) -> Python executable name (e.g. python3.12) + PYTHON=$(python -c 'import re; print(re.sub(r"^py([23])([0-9]{1,2}).*", r"python\1.\2", "${{ matrix.tox_env }}"))') + + if [[ -z $PYTHON ]]; then + echo 1>&2 "Python interpreter could not be determined" + exit 1 + fi + + if [[ $PYTHON == "python2.7" && $(uname) == "Darwin" ]]; then + # GitHub macOS 12 images: python2.7 is installed, but not on $PATH + PYTHON="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7" + fi + + "$PYTHON" -m tox -e "${{ matrix.tox_env }}" diff --git a/docs/changelog.rst b/docs/changelog.rst index 66a8077e..3adaec58 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -29,6 +29,7 @@ Unreleased releases before 2.10 * :gh:issue:`1127` :mod:`ansible_mitogen`: Consolidate Python 2 & 3 compatibility +* :gh:issue:`1128` CI: Start migration from Azure DevOps to GitHub Actions v0.3.10 (2024-09-20)