mirror of https://github.com/ansible/ansible.git
You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1118 lines
40 KiB
Python
1118 lines
40 KiB
Python
#!/usr/bin/env python
|
|
"""Test suite used to verify ansible-test is able to run its containers on various container hosts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import abc
|
|
import dataclasses
|
|
import datetime
|
|
import errno
|
|
import functools
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import pwd
|
|
import re
|
|
import secrets
|
|
import shlex
|
|
import shutil
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import typing as t
|
|
|
|
UNPRIVILEGED_USER_NAME = 'ansible-test'
|
|
CGROUP_SYSTEMD = pathlib.Path('/sys/fs/cgroup/systemd')
|
|
LOG_PATH = pathlib.Path('/tmp/results')
|
|
|
|
# The value of /proc/*/loginuid when it is not set.
|
|
# It is a reserved UID, which is the maximum 32-bit unsigned integer value.
|
|
# See: https://access.redhat.com/solutions/25404
|
|
LOGINUID_NOT_SET = 4294967295
|
|
|
|
UID = os.getuid()
|
|
|
|
try:
|
|
LOGINUID = int(pathlib.Path('/proc/self/loginuid').read_text())
|
|
LOGINUID_MISMATCH = LOGINUID != LOGINUID_NOT_SET and LOGINUID != UID
|
|
except FileNotFoundError:
|
|
LOGINUID = None
|
|
LOGINUID_MISMATCH = False
|
|
|
|
|
|
def main() -> None:
|
|
"""Main program entry point."""
|
|
display.section('Startup check')
|
|
|
|
try:
|
|
bootstrap_type = pathlib.Path('/etc/ansible-test.bootstrap').read_text().strip()
|
|
except FileNotFoundError:
|
|
bootstrap_type = 'undefined'
|
|
|
|
display.info(f'Bootstrap type: {bootstrap_type}')
|
|
|
|
if bootstrap_type != 'remote':
|
|
display.warning('Skipping destructive test on system which is not an ansible-test remote provisioned instance.')
|
|
return
|
|
|
|
display.info(f'UID: {UID} / {LOGINUID}')
|
|
|
|
if UID != 0:
|
|
raise Exception('This test must be run as root.')
|
|
|
|
if not LOGINUID_MISMATCH:
|
|
if LOGINUID is None:
|
|
display.warning('Tests involving loginuid mismatch will be skipped on this host since it does not have audit support.')
|
|
elif LOGINUID == LOGINUID_NOT_SET:
|
|
display.warning('Tests involving loginuid mismatch will be skipped on this host since it is not set.')
|
|
elif LOGINUID == 0:
|
|
raise Exception('Use sudo, su, etc. as a non-root user to become root before running this test.')
|
|
else:
|
|
raise Exception()
|
|
|
|
display.section(f'Bootstrapping {os_release}')
|
|
|
|
bootstrapper = Bootstrapper.init()
|
|
bootstrapper.run()
|
|
|
|
result_dir = LOG_PATH
|
|
|
|
if result_dir.exists():
|
|
shutil.rmtree(result_dir)
|
|
|
|
result_dir.mkdir()
|
|
result_dir.chmod(0o777)
|
|
|
|
scenarios = get_test_scenarios()
|
|
results = [run_test(scenario) for scenario in scenarios]
|
|
error_total = 0
|
|
|
|
for name in sorted(result_dir.glob('*.log')):
|
|
lines = name.read_text().strip().splitlines()
|
|
error_count = len([line for line in lines if line.startswith('FAIL: ')])
|
|
error_total += error_count
|
|
|
|
display.section(f'Log ({error_count=}/{len(lines)}): {name.name}')
|
|
|
|
for line in lines:
|
|
if line.startswith('FAIL: '):
|
|
display.show(line, display.RED)
|
|
else:
|
|
display.show(line)
|
|
|
|
error_count = len([result for result in results if result.message])
|
|
error_total += error_count
|
|
|
|
duration = datetime.timedelta(seconds=int(sum(result.duration.total_seconds() for result in results)))
|
|
|
|
display.section(f'Test Results ({error_count=}/{len(results)}) [{duration}]')
|
|
|
|
for result in results:
|
|
notes = f' <cleanup: {", ".join(result.cleanup)}>' if result.cleanup else ''
|
|
|
|
if result.cgroup_dirs:
|
|
notes += f' <cgroup_dirs: {len(result.cgroup_dirs)}>'
|
|
|
|
notes += f' [{result.duration}]'
|
|
|
|
if result.message:
|
|
display.show(f'FAIL: {result.scenario} {result.message}{notes}', display.RED)
|
|
elif result.duration.total_seconds() >= 90:
|
|
display.show(f'SLOW: {result.scenario}{notes}', display.YELLOW)
|
|
else:
|
|
display.show(f'PASS: {result.scenario}{notes}')
|
|
|
|
if error_total:
|
|
sys.exit(1)
|
|
|
|
|
|
def get_container_completion_entries() -> dict[str, dict[str, str]]:
|
|
"""Parse and return the ansible-test container completion entries."""
|
|
completion_lines = pathlib.Path(os.environ['PYTHONPATH'], '../test/lib/ansible_test/_data/completion/docker.txt').read_text().splitlines()
|
|
|
|
# TODO: consider including testing for the collection default image
|
|
entries = {name: value for name, value in (parse_completion_entry(line) for line in completion_lines) if name != 'default'}
|
|
|
|
return entries
|
|
|
|
|
|
def get_test_scenarios() -> list[TestScenario]:
|
|
"""Generate and return a list of test scenarios."""
|
|
|
|
supported_engines = ('docker', 'podman')
|
|
available_engines = [engine for engine in supported_engines if shutil.which(engine)]
|
|
|
|
if not available_engines:
|
|
raise ApplicationError(f'No supported container engines found: {", ".join(supported_engines)}')
|
|
|
|
entries = get_container_completion_entries()
|
|
|
|
unprivileged_user = User.get(UNPRIVILEGED_USER_NAME)
|
|
|
|
scenarios: list[TestScenario] = []
|
|
|
|
for container_name, settings in entries.items():
|
|
image = settings['image']
|
|
cgroup = settings.get('cgroup', 'v1-v2')
|
|
|
|
if container_name == 'centos6' and os_release.id == 'alpine':
|
|
# Alpine kernels do not emulate vsyscall by default, which causes the centos6 container to fail during init.
|
|
# See: https://unix.stackexchange.com/questions/478387/running-a-centos-docker-image-on-arch-linux-exits-with-code-139
|
|
# Other distributions enable settings which trap vsyscall by default.
|
|
# See: https://www.kernelconfig.io/config_legacy_vsyscall_xonly
|
|
# See: https://www.kernelconfig.io/config_legacy_vsyscall_emulate
|
|
continue
|
|
|
|
for engine in available_engines:
|
|
# TODO: figure out how to get tests passing using docker without disabling selinux
|
|
disable_selinux = os_release.id == 'fedora' and engine == 'docker' and cgroup != 'none'
|
|
expose_cgroup_v1 = cgroup == 'v1-only' and get_docker_info(engine).cgroup_version != 1
|
|
debug_systemd = cgroup != 'none'
|
|
|
|
# The sleep+pkill used to support the cgroup probe causes problems with the centos6 container.
|
|
# It results in sshd connections being refused or reset for many, but not all, container instances.
|
|
# The underlying cause of this issue is unknown.
|
|
probe_cgroups = container_name != 'centos6'
|
|
|
|
# The default RHEL 9 crypto policy prevents use of SHA-1.
|
|
# This results in SSH errors with centos6 containers: ssh_dispatch_run_fatal: Connection to 1.2.3.4 port 22: error in libcrypto
|
|
# See: https://access.redhat.com/solutions/6816771
|
|
enable_sha1 = os_release.id == 'rhel' and os_release.version_id.startswith('9.') and container_name == 'centos6'
|
|
|
|
if cgroup != 'none' and get_docker_info(engine).cgroup_version == 1 and not have_cgroup_systemd():
|
|
expose_cgroup_v1 = True # the host uses cgroup v1 but there is no systemd cgroup and the container requires cgroup support
|
|
|
|
user_scenarios = [
|
|
# TODO: test rootless docker
|
|
UserScenario(ssh=unprivileged_user),
|
|
]
|
|
|
|
if engine == 'podman':
|
|
user_scenarios.append(UserScenario(ssh=ROOT_USER))
|
|
|
|
# TODO: test podman remote on Alpine and Ubuntu hosts
|
|
# TODO: combine remote with ssh using different unprivileged users
|
|
if os_release.id not in ('alpine', 'ubuntu'):
|
|
user_scenarios.append(UserScenario(remote=unprivileged_user))
|
|
|
|
if LOGINUID_MISMATCH:
|
|
user_scenarios.append(UserScenario())
|
|
|
|
for user_scenario in user_scenarios:
|
|
scenarios.append(
|
|
TestScenario(
|
|
user_scenario=user_scenario,
|
|
engine=engine,
|
|
container_name=container_name,
|
|
image=image,
|
|
disable_selinux=disable_selinux,
|
|
expose_cgroup_v1=expose_cgroup_v1,
|
|
enable_sha1=enable_sha1,
|
|
debug_systemd=debug_systemd,
|
|
probe_cgroups=probe_cgroups,
|
|
)
|
|
)
|
|
|
|
return scenarios
|
|
|
|
|
|
def run_test(scenario: TestScenario) -> TestResult:
|
|
"""Run a test scenario and return the test results."""
|
|
display.section(f'Testing {scenario} Started')
|
|
|
|
start = time.monotonic()
|
|
|
|
integration = ['ansible-test', 'integration', 'split']
|
|
integration_options = ['--target', f'docker:{scenario.container_name}', '--color', '--truncate', '0', '-v']
|
|
target_only_options = []
|
|
|
|
if scenario.debug_systemd:
|
|
integration_options.append('--dev-systemd-debug')
|
|
|
|
if scenario.probe_cgroups:
|
|
target_only_options = ['--dev-probe-cgroups', str(LOG_PATH)]
|
|
|
|
entries = get_container_completion_entries()
|
|
alpine_container = [name for name in entries if name.startswith('alpine')][0]
|
|
|
|
commands = [
|
|
# The cgroup probe is only performed for the first test of the target.
|
|
# There's no need to repeat the probe again for the same target.
|
|
# The controller will be tested separately as a target.
|
|
# This ensures that both the probe and no-probe code paths are functional.
|
|
[*integration, *integration_options, *target_only_options],
|
|
# For the split test we'll use Alpine Linux as the controller. There are two reasons for this:
|
|
# 1) It doesn't require the cgroup v1 hack, so we can test a target that doesn't need that.
|
|
# 2) It doesn't require disabling selinux, so we can test a target that doesn't need that.
|
|
[*integration, '--controller', f'docker:{alpine_container}', *integration_options],
|
|
]
|
|
|
|
common_env: dict[str, str] = {}
|
|
test_env: dict[str, str] = {}
|
|
|
|
if scenario.engine == 'podman':
|
|
if scenario.user_scenario.remote:
|
|
common_env.update(
|
|
# Podman 4.3.0 has a regression which requires a port for remote connections to work.
|
|
# See: https://github.com/containers/podman/issues/16509
|
|
CONTAINER_HOST=f'ssh://{scenario.user_scenario.remote.name}@localhost:22'
|
|
f'/run/user/{scenario.user_scenario.remote.pwnam.pw_uid}/podman/podman.sock',
|
|
CONTAINER_SSHKEY=str(pathlib.Path('~/.ssh/id_rsa').expanduser()), # TODO: add support for ssh + remote when the ssh user is not root
|
|
)
|
|
|
|
test_env.update(ANSIBLE_TEST_PREFER_PODMAN='1')
|
|
|
|
test_env.update(common_env)
|
|
|
|
if scenario.user_scenario.ssh:
|
|
client_become_cmd = ['ssh', f'{scenario.user_scenario.ssh.name}@localhost']
|
|
test_commands = [client_become_cmd + [f'cd ~/ansible; {format_env(test_env)}{sys.executable} bin/{shlex.join(command)}'] for command in commands]
|
|
else:
|
|
client_become_cmd = ['sh', '-c']
|
|
test_commands = [client_become_cmd + [f'{format_env(test_env)}{shlex.join(command)}'] for command in commands]
|
|
|
|
prime_storage_command = []
|
|
|
|
if scenario.engine == 'podman' and scenario.user_scenario.actual.name == UNPRIVILEGED_USER_NAME:
|
|
# When testing podman we need to make sure that the overlay filesystem is used instead of vfs.
|
|
# Using the vfs filesystem will result in running out of disk space during the tests.
|
|
# To change the filesystem used, the existing storage directory must be removed before "priming" the storage database.
|
|
#
|
|
# Without this change the following message may be displayed:
|
|
#
|
|
# User-selected graph driver "overlay" overwritten by graph driver "vfs" from database - delete libpod local files to resolve
|
|
#
|
|
# However, with this change it may be replaced with the following message:
|
|
#
|
|
# User-selected graph driver "vfs" overwritten by graph driver "overlay" from database - delete libpod local files to resolve
|
|
|
|
actual_become_cmd = ['ssh', f'{scenario.user_scenario.actual.name}@localhost']
|
|
prime_storage_command = actual_become_cmd + prepare_prime_podman_storage()
|
|
|
|
message = ''
|
|
|
|
if scenario.expose_cgroup_v1:
|
|
prepare_cgroup_systemd(scenario.user_scenario.actual.name, scenario.engine)
|
|
|
|
try:
|
|
if prime_storage_command:
|
|
retry_command(lambda: run_command(*prime_storage_command), retry_any_error=True)
|
|
|
|
if scenario.disable_selinux:
|
|
run_command('setenforce', 'permissive')
|
|
|
|
if scenario.enable_sha1:
|
|
run_command('update-crypto-policies', '--set', 'DEFAULT:SHA1')
|
|
|
|
for test_command in test_commands:
|
|
retry_command(lambda: run_command(*test_command))
|
|
except SubprocessError as ex:
|
|
message = str(ex)
|
|
display.error(f'{scenario} {message}')
|
|
finally:
|
|
if scenario.enable_sha1:
|
|
run_command('update-crypto-policies', '--set', 'DEFAULT')
|
|
|
|
if scenario.disable_selinux:
|
|
run_command('setenforce', 'enforcing')
|
|
|
|
if scenario.expose_cgroup_v1:
|
|
dirs = remove_cgroup_systemd()
|
|
else:
|
|
dirs = list_group_systemd()
|
|
|
|
cleanup_command = [scenario.engine, 'rmi', '-f', scenario.image]
|
|
|
|
try:
|
|
retry_command(lambda: run_command(*client_become_cmd + [f'{format_env(common_env)}{shlex.join(cleanup_command)}']), retry_any_error=True)
|
|
except SubprocessError as ex:
|
|
display.error(str(ex))
|
|
|
|
cleanup = cleanup_podman() if scenario.engine == 'podman' else tuple()
|
|
|
|
finish = time.monotonic()
|
|
duration = datetime.timedelta(seconds=int(finish - start))
|
|
|
|
display.section(f'Testing {scenario} Completed in {duration}')
|
|
|
|
return TestResult(
|
|
scenario=scenario,
|
|
message=message,
|
|
cleanup=cleanup,
|
|
duration=duration,
|
|
cgroup_dirs=tuple(str(path) for path in dirs),
|
|
)
|
|
|
|
|
|
def prepare_prime_podman_storage() -> list[str]:
|
|
"""Partially prime podman storage and return a command to complete the remainder."""
|
|
prime_storage_command = ['rm -rf ~/.local/share/containers; STORAGE_DRIVER=overlay podman pull quay.io/bedrock/alpine:3.16.2']
|
|
|
|
test_containers = pathlib.Path(f'~{UNPRIVILEGED_USER_NAME}/.local/share/containers').expanduser()
|
|
|
|
if test_containers.is_dir():
|
|
# First remove the directory as root, since the user may not have permissions on all the files.
|
|
# The directory will be removed again after login, before initializing the database.
|
|
rmtree(test_containers)
|
|
|
|
return prime_storage_command
|
|
|
|
|
|
def cleanup_podman() -> tuple[str, ...]:
|
|
"""Cleanup podman processes and files on disk."""
|
|
cleanup = []
|
|
|
|
for remaining in range(3, -1, -1):
|
|
processes = [(int(item[0]), item[1]) for item in
|
|
[item.split(maxsplit=1) for item in run_command('ps', '-A', '-o', 'pid,comm', capture=True).stdout.splitlines()]
|
|
if pathlib.Path(item[1].split()[0]).name in ('catatonit', 'podman', 'conmon')]
|
|
|
|
if not processes:
|
|
break
|
|
|
|
for pid, name in processes:
|
|
display.info(f'Killing "{name}" ({pid}) ...')
|
|
|
|
try:
|
|
os.kill(pid, signal.SIGTERM if remaining > 1 else signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
|
|
cleanup.append(name)
|
|
|
|
time.sleep(1)
|
|
else:
|
|
raise Exception('failed to kill all matching processes')
|
|
|
|
uid = pwd.getpwnam(UNPRIVILEGED_USER_NAME).pw_uid
|
|
|
|
container_tmp = pathlib.Path(f'/tmp/containers-user-{uid}')
|
|
podman_tmp = pathlib.Path(f'/tmp/podman-run-{uid}')
|
|
|
|
user_config = pathlib.Path(f'~{UNPRIVILEGED_USER_NAME}/.config').expanduser()
|
|
user_local = pathlib.Path(f'~{UNPRIVILEGED_USER_NAME}/.local').expanduser()
|
|
|
|
if container_tmp.is_dir():
|
|
rmtree(container_tmp)
|
|
|
|
if podman_tmp.is_dir():
|
|
rmtree(podman_tmp)
|
|
|
|
if user_config.is_dir():
|
|
rmtree(user_config)
|
|
|
|
if user_local.is_dir():
|
|
rmtree(user_local)
|
|
|
|
return tuple(sorted(set(cleanup)))
|
|
|
|
|
|
def have_cgroup_systemd() -> bool:
|
|
"""Return True if the container host has a systemd cgroup."""
|
|
return pathlib.Path(CGROUP_SYSTEMD).is_dir()
|
|
|
|
|
|
def prepare_cgroup_systemd(username: str, engine: str) -> None:
|
|
"""Prepare the systemd cgroup."""
|
|
CGROUP_SYSTEMD.mkdir()
|
|
|
|
run_command('mount', 'cgroup', '-t', 'cgroup', str(CGROUP_SYSTEMD), '-o', 'none,name=systemd,xattr', capture=True)
|
|
|
|
if engine == 'podman':
|
|
run_command('chown', '-R', f'{username}:{username}', str(CGROUP_SYSTEMD))
|
|
|
|
run_command('find', str(CGROUP_SYSTEMD), '-type', 'd', '-exec', 'ls', '-l', '{}', ';')
|
|
|
|
|
|
def list_group_systemd() -> list[pathlib.Path]:
|
|
"""List the systemd cgroup."""
|
|
dirs = set()
|
|
|
|
for dirpath, dirnames, filenames in os.walk(CGROUP_SYSTEMD, topdown=False):
|
|
for dirname in dirnames:
|
|
target_path = pathlib.Path(dirpath, dirname)
|
|
display.info(f'dir: {target_path}')
|
|
dirs.add(target_path)
|
|
|
|
return sorted(dirs)
|
|
|
|
|
|
def remove_cgroup_systemd() -> list[pathlib.Path]:
|
|
"""Remove the systemd cgroup."""
|
|
dirs = set()
|
|
|
|
for sleep_seconds in range(1, 10):
|
|
try:
|
|
for dirpath, dirnames, filenames in os.walk(CGROUP_SYSTEMD, topdown=False):
|
|
for dirname in dirnames:
|
|
target_path = pathlib.Path(dirpath, dirname)
|
|
display.info(f'rmdir: {target_path}')
|
|
dirs.add(target_path)
|
|
target_path.rmdir()
|
|
except OSError as ex:
|
|
if ex.errno != errno.EBUSY:
|
|
raise
|
|
|
|
error = str(ex)
|
|
else:
|
|
break
|
|
|
|
display.warning(f'{error} -- sleeping for {sleep_seconds} second(s) before trying again ...') # pylint: disable=used-before-assignment
|
|
|
|
time.sleep(sleep_seconds)
|
|
|
|
time.sleep(1) # allow time for cgroups to be fully removed before unmounting
|
|
|
|
run_command('umount', str(CGROUP_SYSTEMD))
|
|
|
|
CGROUP_SYSTEMD.rmdir()
|
|
|
|
time.sleep(1) # allow time for cgroup hierarchy to be removed after unmounting
|
|
|
|
cgroup = pathlib.Path('/proc/self/cgroup').read_text()
|
|
|
|
if 'systemd' in cgroup:
|
|
raise Exception('systemd hierarchy detected')
|
|
|
|
return sorted(dirs)
|
|
|
|
|
|
def rmtree(path: pathlib.Path) -> None:
|
|
"""Wrapper around shutil.rmtree with additional error handling."""
|
|
for retries in range(10, -1, -1):
|
|
try:
|
|
display.info(f'rmtree: {path} ({retries} attempts remaining) ... ')
|
|
shutil.rmtree(path)
|
|
except Exception:
|
|
if not path.exists():
|
|
display.info(f'rmtree: {path} (not found)')
|
|
return
|
|
|
|
if not path.is_dir():
|
|
display.info(f'rmtree: {path} (not a directory)')
|
|
return
|
|
|
|
if retries:
|
|
continue
|
|
|
|
raise
|
|
else:
|
|
display.info(f'rmtree: {path} (done)')
|
|
return
|
|
|
|
|
|
def format_env(env: dict[str, str]) -> str:
|
|
"""Format an env dict for injection into a shell command and return the resulting string."""
|
|
if env:
|
|
return ' '.join(f'{shlex.quote(key)}={shlex.quote(value)}' for key, value in env.items()) + ' '
|
|
|
|
return ''
|
|
|
|
|
|
class DockerInfo:
|
|
"""The results of `docker info` for the container runtime."""
|
|
|
|
def __init__(self, data: dict[str, t.Any]) -> None:
|
|
self.data = data
|
|
|
|
@property
|
|
def cgroup_version(self) -> int:
|
|
"""The cgroup version of the container host."""
|
|
data = self.data
|
|
host = data.get('host')
|
|
|
|
if host:
|
|
version = int(host['cgroupVersion'].lstrip('v')) # podman
|
|
else:
|
|
version = int(data['CgroupVersion']) # docker
|
|
|
|
return version
|
|
|
|
|
|
@functools.lru_cache
|
|
def get_docker_info(engine: str) -> DockerInfo:
|
|
"""Return info for the current container runtime. The results are cached."""
|
|
return DockerInfo(json.loads(run_command(engine, 'info', '--format', '{{ json . }}', capture=True).stdout))
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class User:
|
|
name: str
|
|
pwnam: pwd.struct_passwd
|
|
|
|
@classmethod
|
|
def get(cls, name: str) -> User:
|
|
return User(
|
|
name=name,
|
|
pwnam=pwd.getpwnam(name),
|
|
)
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class UserScenario:
|
|
ssh: User = None
|
|
remote: User = None
|
|
|
|
@property
|
|
def actual(self) -> User:
|
|
return self.remote or self.ssh or ROOT_USER
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class TestScenario:
|
|
user_scenario: UserScenario
|
|
engine: str
|
|
container_name: str
|
|
image: str
|
|
disable_selinux: bool
|
|
expose_cgroup_v1: bool
|
|
enable_sha1: bool
|
|
debug_systemd: bool
|
|
probe_cgroups: bool
|
|
|
|
@property
|
|
def tags(self) -> tuple[str, ...]:
|
|
tags = []
|
|
|
|
if self.user_scenario.ssh:
|
|
tags.append(f'ssh: {self.user_scenario.ssh.name}')
|
|
|
|
if self.user_scenario.remote:
|
|
tags.append(f'remote: {self.user_scenario.remote.name}')
|
|
|
|
if self.disable_selinux:
|
|
tags.append('selinux: permissive')
|
|
|
|
if self.expose_cgroup_v1:
|
|
tags.append('cgroup: v1')
|
|
|
|
if self.enable_sha1:
|
|
tags.append('sha1: enabled')
|
|
|
|
return tuple(tags)
|
|
|
|
@property
|
|
def tag_label(self) -> str:
|
|
return ' '.join(f'[{tag}]' for tag in self.tags)
|
|
|
|
def __str__(self):
|
|
return f'[{self.container_name}] ({self.engine}) {self.tag_label}'.strip()
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class TestResult:
|
|
scenario: TestScenario
|
|
message: str
|
|
cleanup: tuple[str, ...]
|
|
duration: datetime.timedelta
|
|
cgroup_dirs: tuple[str, ...]
|
|
|
|
|
|
def parse_completion_entry(value: str) -> tuple[str, dict[str, str]]:
|
|
"""Parse the given completion entry, returning the entry name and a dictionary of key/value settings."""
|
|
values = value.split()
|
|
|
|
name = values[0]
|
|
data = {kvp[0]: kvp[1] if len(kvp) > 1 else '' for kvp in [item.split('=', 1) for item in values[1:]]}
|
|
|
|
return name, data
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class SubprocessResult:
|
|
"""Result from execution of a subprocess."""
|
|
|
|
command: list[str]
|
|
stdout: str
|
|
stderr: str
|
|
status: int
|
|
|
|
|
|
class ApplicationError(Exception):
|
|
"""An application error."""
|
|
|
|
def __init__(self, message: str) -> None:
|
|
self.message = message
|
|
|
|
super().__init__(message)
|
|
|
|
|
|
class SubprocessError(ApplicationError):
|
|
"""An error from executing a subprocess."""
|
|
|
|
def __init__(self, result: SubprocessResult) -> None:
|
|
self.result = result
|
|
|
|
message = f'Command `{shlex.join(result.command)}` exited with status: {result.status}'
|
|
|
|
stdout = (result.stdout or '').strip()
|
|
stderr = (result.stderr or '').strip()
|
|
|
|
if stdout:
|
|
message += f'\n>>> Standard Output\n{stdout}'
|
|
|
|
if stderr:
|
|
message += f'\n>>> Standard Error\n{stderr}'
|
|
|
|
super().__init__(message)
|
|
|
|
|
|
class ProgramNotFoundError(ApplicationError):
|
|
"""A required program was not found."""
|
|
|
|
def __init__(self, name: str) -> None:
|
|
self.name = name
|
|
|
|
super().__init__(f'Missing program: {name}')
|
|
|
|
|
|
class Display:
|
|
"""Display interface for sending output to the console."""
|
|
|
|
CLEAR = '\033[0m'
|
|
RED = '\033[31m'
|
|
GREEN = '\033[32m'
|
|
YELLOW = '\033[33m'
|
|
BLUE = '\033[34m'
|
|
PURPLE = '\033[35m'
|
|
CYAN = '\033[36m'
|
|
|
|
def __init__(self) -> None:
|
|
self.sensitive: set[str] = set()
|
|
|
|
def section(self, message: str) -> None:
|
|
"""Print a section message to the console."""
|
|
self.show(f'==> {message}', color=self.BLUE)
|
|
|
|
def subsection(self, message: str) -> None:
|
|
"""Print a subsection message to the console."""
|
|
self.show(f'--> {message}', color=self.CYAN)
|
|
|
|
def fatal(self, message: str) -> None:
|
|
"""Print a fatal message to the console."""
|
|
self.show(f'FATAL: {message}', color=self.RED)
|
|
|
|
def error(self, message: str) -> None:
|
|
"""Print an error message to the console."""
|
|
self.show(f'ERROR: {message}', color=self.RED)
|
|
|
|
def warning(self, message: str) -> None:
|
|
"""Print a warning message to the console."""
|
|
self.show(f'WARNING: {message}', color=self.PURPLE)
|
|
|
|
def info(self, message: str) -> None:
|
|
"""Print an info message to the console."""
|
|
self.show(f'INFO: {message}', color=self.YELLOW)
|
|
|
|
def show(self, message: str, color: str | None = None) -> None:
|
|
"""Print a message to the console."""
|
|
for item in self.sensitive:
|
|
message = message.replace(item, '*' * len(item))
|
|
|
|
print(f'{color or self.CLEAR}{message}{self.CLEAR}', flush=True)
|
|
|
|
|
|
def run_module(
|
|
module: str,
|
|
args: dict[str, t.Any],
|
|
) -> SubprocessResult:
|
|
"""Run the specified Ansible module and return the result."""
|
|
return run_command('ansible', '-m', module, '-v', '-a', json.dumps(args), 'localhost')
|
|
|
|
|
|
def retry_command(func: t.Callable[[], SubprocessResult], attempts: int = 3, retry_any_error: bool = False) -> SubprocessResult:
|
|
"""Run the given command function up to the specified number of attempts when the failure is due to an SSH error."""
|
|
for attempts_remaining in range(attempts - 1, -1, -1):
|
|
try:
|
|
return func()
|
|
except SubprocessError as ex:
|
|
if ex.result.command[0] == 'ssh' and ex.result.status == 255 and attempts_remaining:
|
|
# SSH connections on our Ubuntu 22.04 host sometimes fail for unknown reasons.
|
|
# This retry should allow the test suite to continue, maintaining CI stability.
|
|
# TODO: Figure out why local SSH connections sometimes fail during the test run.
|
|
display.warning('Command failed due to an SSH error. Waiting a few seconds before retrying.')
|
|
time.sleep(3)
|
|
continue
|
|
|
|
if retry_any_error:
|
|
display.warning('Command failed. Waiting a few seconds before retrying.')
|
|
time.sleep(3)
|
|
continue
|
|
|
|
raise
|
|
|
|
|
|
def run_command(
|
|
*command: str,
|
|
data: str | None = None,
|
|
stdin: int | t.IO[bytes] | None = None,
|
|
env: dict[str, str] | None = None,
|
|
capture: bool = False,
|
|
) -> SubprocessResult:
|
|
"""Run the specified command and return the result."""
|
|
stdin = subprocess.PIPE if data else stdin or subprocess.DEVNULL
|
|
stdout = subprocess.PIPE if capture else None
|
|
stderr = subprocess.PIPE if capture else None
|
|
|
|
display.subsection(f'Run command: {shlex.join(command)}')
|
|
|
|
try:
|
|
with subprocess.Popen(args=command, stdin=stdin, stdout=stdout, stderr=stderr, env=env, text=True) as process:
|
|
process_stdout, process_stderr = process.communicate(data)
|
|
process_status = process.returncode
|
|
except FileNotFoundError:
|
|
raise ProgramNotFoundError(command[0]) from None
|
|
|
|
result = SubprocessResult(
|
|
command=list(command),
|
|
stdout=process_stdout,
|
|
stderr=process_stderr,
|
|
status=process_status,
|
|
)
|
|
|
|
if process.returncode != 0:
|
|
raise SubprocessError(result)
|
|
|
|
return result
|
|
|
|
|
|
class Bootstrapper(metaclass=abc.ABCMeta):
|
|
"""Bootstrapper for remote instances."""
|
|
|
|
@classmethod
|
|
def install_podman(cls) -> bool:
|
|
"""Return True if podman will be installed."""
|
|
return False
|
|
|
|
@classmethod
|
|
def install_docker(cls) -> bool:
|
|
"""Return True if docker will be installed."""
|
|
return False
|
|
|
|
@classmethod
|
|
def usable(cls) -> bool:
|
|
"""Return True if the bootstrapper can be used, otherwise False."""
|
|
return False
|
|
|
|
@classmethod
|
|
def init(cls) -> t.Type[Bootstrapper]:
|
|
"""Return a bootstrapper type appropriate for the current system."""
|
|
for bootstrapper in cls.__subclasses__():
|
|
if bootstrapper.usable():
|
|
return bootstrapper
|
|
|
|
display.warning('No supported bootstrapper found.')
|
|
return Bootstrapper
|
|
|
|
@classmethod
|
|
def run(cls) -> None:
|
|
"""Run the bootstrapper."""
|
|
cls.configure_root_user()
|
|
cls.configure_unprivileged_user()
|
|
cls.configure_source_trees()
|
|
cls.configure_ssh_keys()
|
|
cls.configure_podman_remote()
|
|
|
|
@classmethod
|
|
def configure_root_user(cls) -> None:
|
|
"""Configure the root user to run tests."""
|
|
root_password_status = run_command('passwd', '--status', 'root', capture=True)
|
|
root_password_set = root_password_status.stdout.split()[1]
|
|
|
|
if root_password_set not in ('P', 'PS'):
|
|
root_password = run_command('openssl', 'passwd', '-5', '-stdin', data=secrets.token_hex(8), capture=True).stdout.strip()
|
|
|
|
run_module(
|
|
'user',
|
|
dict(
|
|
user='root',
|
|
password=root_password,
|
|
),
|
|
)
|
|
|
|
@classmethod
|
|
def configure_unprivileged_user(cls) -> None:
|
|
"""Configure the unprivileged user to run tests."""
|
|
unprivileged_password = run_command('openssl', 'passwd', '-5', '-stdin', data=secrets.token_hex(8), capture=True).stdout.strip()
|
|
|
|
run_module(
|
|
'user',
|
|
dict(
|
|
user=UNPRIVILEGED_USER_NAME,
|
|
password=unprivileged_password,
|
|
groups=['docker'] if cls.install_docker() else [],
|
|
append=True,
|
|
),
|
|
)
|
|
|
|
if os_release.id == 'alpine':
|
|
# Most distros handle this automatically, but not Alpine.
|
|
# See: https://www.redhat.com/sysadmin/rootless-podman
|
|
start = 165535
|
|
end = start + 65535
|
|
id_range = f'{start}-{end}'
|
|
|
|
run_command(
|
|
'usermod',
|
|
'--add-subuids',
|
|
id_range,
|
|
'--add-subgids',
|
|
id_range,
|
|
UNPRIVILEGED_USER_NAME,
|
|
)
|
|
|
|
@classmethod
|
|
def configure_source_trees(cls):
|
|
"""Configure the source trees needed to run tests for both root and the unprivileged user."""
|
|
current_ansible = pathlib.Path(os.environ['PYTHONPATH']).parent
|
|
|
|
root_ansible = pathlib.Path('~').expanduser() / 'ansible'
|
|
test_ansible = pathlib.Path(f'~{UNPRIVILEGED_USER_NAME}').expanduser() / 'ansible'
|
|
|
|
if current_ansible != root_ansible:
|
|
display.info(f'copying {current_ansible} -> {root_ansible} ...')
|
|
rmtree(root_ansible)
|
|
shutil.copytree(current_ansible, root_ansible)
|
|
run_command('chown', '-R', 'root:root', str(root_ansible))
|
|
|
|
display.info(f'copying {current_ansible} -> {test_ansible} ...')
|
|
rmtree(test_ansible)
|
|
shutil.copytree(current_ansible, test_ansible)
|
|
run_command('chown', '-R', f'{UNPRIVILEGED_USER_NAME}:{UNPRIVILEGED_USER_NAME}', str(test_ansible))
|
|
|
|
paths = [pathlib.Path(test_ansible)]
|
|
|
|
for root, dir_names, file_names in os.walk(test_ansible):
|
|
paths.extend(pathlib.Path(root, dir_name) for dir_name in dir_names)
|
|
paths.extend(pathlib.Path(root, file_name) for file_name in file_names)
|
|
|
|
user = pwd.getpwnam(UNPRIVILEGED_USER_NAME)
|
|
uid = user.pw_uid
|
|
gid = user.pw_gid
|
|
|
|
for path in paths:
|
|
os.chown(path, uid, gid)
|
|
|
|
@classmethod
|
|
def configure_ssh_keys(cls) -> None:
|
|
"""Configure SSH keys needed to run tests."""
|
|
user = pwd.getpwnam(UNPRIVILEGED_USER_NAME)
|
|
uid = user.pw_uid
|
|
gid = user.pw_gid
|
|
|
|
current_rsa_pub = pathlib.Path('~/.ssh/id_rsa.pub').expanduser()
|
|
|
|
test_authorized_keys = pathlib.Path(f'~{UNPRIVILEGED_USER_NAME}/.ssh/authorized_keys').expanduser()
|
|
|
|
test_authorized_keys.parent.mkdir(mode=0o755, parents=True, exist_ok=True)
|
|
os.chown(test_authorized_keys.parent, uid, gid)
|
|
|
|
shutil.copyfile(current_rsa_pub, test_authorized_keys)
|
|
os.chown(test_authorized_keys, uid, gid)
|
|
test_authorized_keys.chmod(mode=0o644)
|
|
|
|
@classmethod
|
|
def configure_podman_remote(cls) -> None:
|
|
"""Configure podman remote support."""
|
|
# TODO: figure out how to support remote podman without systemd (Alpine)
|
|
# TODO: figure out how to support remote podman on Ubuntu
|
|
if os_release.id in ('alpine', 'ubuntu'):
|
|
return
|
|
|
|
# Support podman remote on any host with systemd available.
|
|
retry_command(lambda: run_command('ssh', f'{UNPRIVILEGED_USER_NAME}@localhost', 'systemctl', '--user', 'enable', '--now', 'podman.socket'))
|
|
run_command('loginctl', 'enable-linger', UNPRIVILEGED_USER_NAME)
|
|
|
|
|
|
class DnfBootstrapper(Bootstrapper):
|
|
"""Bootstrapper for dnf based systems."""
|
|
|
|
@classmethod
|
|
def install_podman(cls) -> bool:
|
|
"""Return True if podman will be installed."""
|
|
return True
|
|
|
|
@classmethod
|
|
def install_docker(cls) -> bool:
|
|
"""Return True if docker will be installed."""
|
|
return os_release.id != 'rhel'
|
|
|
|
@classmethod
|
|
def usable(cls) -> bool:
|
|
"""Return True if the bootstrapper can be used, otherwise False."""
|
|
return bool(shutil.which('dnf'))
|
|
|
|
@classmethod
|
|
def run(cls) -> None:
|
|
"""Run the bootstrapper."""
|
|
# NOTE: Install crun to make it available to podman, otherwise installing moby-engine can cause podman to use runc instead.
|
|
packages = ['podman', 'crun']
|
|
|
|
if cls.install_docker():
|
|
packages.append('moby-engine')
|
|
|
|
if os_release.id == 'fedora' and os_release.version_id == '36':
|
|
# In Fedora 36 the current version of netavark, 1.2.0, causes TCP connect to hang between rootfull containers.
|
|
# The previously tested version, 1.1.0, did not have this issue.
|
|
# Unfortunately, with the release of 1.2.0 the 1.1.0 package was removed from the repositories.
|
|
# Thankfully the 1.0.2 version is available and also works, so we'll use that here until a fixed version is available.
|
|
# See: https://github.com/containers/netavark/issues/491
|
|
packages.append('netavark-1.0.2')
|
|
|
|
if os_release.id == 'fedora' and os_release.version_id == '39':
|
|
# In Fedora 39, the current version of containerd, 1.6.23, prevents Docker from working.
|
|
# The previously tested version, 1.6.19, did not have this issue.
|
|
# See: https://bugzilla.redhat.com/show_bug.cgi?id=2237396
|
|
run_command(
|
|
'dnf', 'install', '-y',
|
|
'https://kojipkgs.fedoraproject.org/packages/containerd/1.6.19/2.fc39/x86_64/containerd-1.6.19-2.fc39.x86_64.rpm'
|
|
)
|
|
|
|
if os_release.id == 'rhel':
|
|
# As of the release of RHEL 9.1, installing podman on RHEL 9.0 results in a non-fatal error at install time:
|
|
#
|
|
# libsemanage.semanage_pipe_data: Child process /usr/libexec/selinux/hll/pp failed with code: 255. (No such file or directory).
|
|
# container: libsepol.policydb_read: policydb module version 21 does not match my version range 4-20
|
|
# container: libsepol.sepol_module_package_read: invalid module in module package (at section 0)
|
|
# container: Failed to read policy package
|
|
# libsemanage.semanage_direct_commit: Failed to compile hll files into cil files.
|
|
# (No such file or directory).
|
|
# /usr/sbin/semodule: Failed!
|
|
#
|
|
# Unfortunately this is then fatal when running podman, resulting in no error message and a 127 return code.
|
|
# The solution is to update the policycoreutils package *before* installing podman.
|
|
#
|
|
# NOTE: This work-around can probably be removed once we're testing on RHEL 9.1, as the updated packages should already be installed.
|
|
# Unfortunately at this time there is no RHEL 9.1 AMI available (other than the Beta release).
|
|
|
|
run_command('dnf', 'update', '-y', 'policycoreutils')
|
|
|
|
run_command('dnf', 'install', '-y', *packages)
|
|
|
|
if cls.install_docker():
|
|
run_command('systemctl', 'start', 'docker')
|
|
|
|
if os_release.id == 'rhel' and os_release.version_id.startswith('8.'):
|
|
# RHEL 8 defaults to using runc instead of crun.
|
|
# Unfortunately runc seems to have issues with podman remote.
|
|
# Specifically, it tends to cause conmon to burn CPU until it reaches the specified exit delay.
|
|
# So we'll just change the system default to crun instead.
|
|
# Unfortunately we can't do this with the `--runtime` option since that doesn't work with podman remote.
|
|
|
|
conf = pathlib.Path('/usr/share/containers/containers.conf').read_text()
|
|
|
|
conf = re.sub('^runtime .*', 'runtime = "crun"', conf, flags=re.MULTILINE)
|
|
|
|
pathlib.Path('/etc/containers/containers.conf').write_text(conf)
|
|
|
|
super().run()
|
|
|
|
|
|
class AptBootstrapper(Bootstrapper):
|
|
"""Bootstrapper for apt based systems."""
|
|
|
|
@classmethod
|
|
def install_podman(cls) -> bool:
|
|
"""Return True if podman will be installed."""
|
|
return not (os_release.id == 'ubuntu' and os_release.version_id in {'20.04', '22.04'})
|
|
|
|
@classmethod
|
|
def install_docker(cls) -> bool:
|
|
"""Return True if docker will be installed."""
|
|
return True
|
|
|
|
@classmethod
|
|
def usable(cls) -> bool:
|
|
"""Return True if the bootstrapper can be used, otherwise False."""
|
|
return bool(shutil.which('apt-get'))
|
|
|
|
@classmethod
|
|
def run(cls) -> None:
|
|
"""Run the bootstrapper."""
|
|
apt_env = os.environ.copy()
|
|
apt_env.update(
|
|
DEBIAN_FRONTEND='noninteractive',
|
|
)
|
|
|
|
packages = ['docker.io']
|
|
|
|
if cls.install_podman():
|
|
# NOTE: Install crun to make it available to podman, otherwise installing docker.io can cause podman to use runc instead.
|
|
# Using podman rootless requires the `newuidmap` and `slirp4netns` commands.
|
|
packages.extend(('podman', 'crun', 'uidmap', 'slirp4netns'))
|
|
|
|
run_command('apt-get', 'install', *packages, '-y', '--no-install-recommends', env=apt_env)
|
|
|
|
super().run()
|
|
|
|
|
|
class ApkBootstrapper(Bootstrapper):
|
|
"""Bootstrapper for apk based systems."""
|
|
|
|
@classmethod
|
|
def install_podman(cls) -> bool:
|
|
"""Return True if podman will be installed."""
|
|
return True
|
|
|
|
@classmethod
|
|
def install_docker(cls) -> bool:
|
|
"""Return True if docker will be installed."""
|
|
return True
|
|
|
|
@classmethod
|
|
def usable(cls) -> bool:
|
|
"""Return True if the bootstrapper can be used, otherwise False."""
|
|
return bool(shutil.which('apk'))
|
|
|
|
@classmethod
|
|
def run(cls) -> None:
|
|
"""Run the bootstrapper."""
|
|
# The `openssl` package is used to generate hashed passwords.
|
|
# crun added as podman won't install it as dep if runc is present
|
|
# but we don't want runc as it fails
|
|
# The edge `crun` package installed below requires ip6tables, and in
|
|
# edge, the `iptables` package includes `ip6tables`, but in 3.18 they
|
|
# are separate. Remove `ip6tables` once we update to 3.19.
|
|
packages = ['docker', 'podman', 'openssl', 'crun', 'ip6tables']
|
|
|
|
run_command('apk', 'add', *packages)
|
|
# 3.18 only contains crun 1.8.4, to get 1.9.2 to resolve the run/shm issue, install crun from edge
|
|
# Remove once we update to 3.19
|
|
run_command('apk', 'upgrade', '-U', '--repository=http://dl-cdn.alpinelinux.org/alpine/edge/community', 'crun')
|
|
run_command('service', 'docker', 'start')
|
|
run_command('modprobe', 'tun')
|
|
|
|
super().run()
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class OsRelease:
|
|
"""Operating system identification."""
|
|
|
|
id: str
|
|
version_id: str
|
|
|
|
@staticmethod
|
|
def init() -> OsRelease:
|
|
"""Detect the current OS release and return the result."""
|
|
lines = run_command('sh', '-c', '. /etc/os-release && echo $ID && echo $VERSION_ID', capture=True).stdout.splitlines()
|
|
|
|
result = OsRelease(
|
|
id=lines[0],
|
|
version_id=lines[1],
|
|
)
|
|
|
|
display.show(f'Detected OS "{result.id}" version "{result.version_id}".')
|
|
|
|
return result
|
|
|
|
|
|
display = Display()
|
|
os_release = OsRelease.init()
|
|
|
|
ROOT_USER = User.get('root')
|
|
|
|
if __name__ == '__main__':
|
|
main()
|