Merge pull request #816 from moreati/issue815

Expand coverage of mitogen/<distro>-test docker images
pull/707/head^2
Steven Robertson 4 years ago committed by GitHub
commit 36f3e3b28c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -45,7 +45,7 @@ jobs:
Mito27Debian_27:
python.version: '2.7'
MODE: mitogen
DISTRO: debian
DISTRO: debian9
#MitoPy27CentOS6_26:
#python.version: '2.7'
@ -60,12 +60,12 @@ jobs:
Mito37Debian_27:
python.version: '3.7'
MODE: mitogen
DISTRO: debian
DISTRO: debian9
Mito39Debian_27:
python.version: '3.9'
MODE: mitogen
DISTRO: debian
DISTRO: debian9
VER: 2.10.0
#Py26CentOS7:

@ -12,8 +12,10 @@
- name: Verify raw module output.
assert:
that: |
out.failed and (
('sudo: unknown user: slartibartfast' in out.msg) or
('sudo: unknown user: slartibartfast' in out.module_stderr)
)
that:
- out.failed
# sudo-1.8.6p3-29.el6_10.3 on RHEL & CentOS 6.10 (final release)
# removed user/group error messages, as defence against CVE-2019-14287.
- >-
('sudo: unknown user: slartibartfast' in out.module_stderr | default(out.msg))
or (ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_version == '6.10')

@ -15,10 +15,18 @@
- "out.failed"
- "out.results[0].failed"
- "out.results[0].msg.startswith('MODULE FAILURE')"
- "out.results[0].module_stdout.startswith('/bin/sh: ')"
# On Ubuntu 16.04 /bin/sh is dash 0.5.8. It treats custom_binary_single_null
# as a valid executable. There's no error message, and rc == 0.
- |
out.results[0].module_stdout.endswith('custom_binary_single_null: cannot execute binary file\r\n') or
out.results[0].module_stdout.endswith('custom_binary_single_null: Exec format error\r\n')
out.results[0].module_stdout.startswith('/bin/sh: ')
or (ansible_facts.distribution == 'Ubuntu' and ansible_facts.distribution_version == '16.04')
- |
out.results[0].module_stdout.endswith((
'custom_binary_single_null: cannot execute binary file\r\n',
'custom_binary_single_null: Exec format error\r\n',
'custom_binary_single_null: cannot execute binary file: Exec format error\r\n',
))
or (ansible_facts.distribution == 'Ubuntu' and ansible_facts.distribution_version == '16.04')
# Can't test this: Mitogen returns 126, 2.5.x returns 126, 2.4.x discarded the

@ -14,10 +14,11 @@ See ../README.md for a (mostly) description of the accounts created.
## Building the containers
``./build_docker_images.sh``
Requires Ansible 2.3.x.x in order to target CentOS 5
No single version of Ansible supports every Linux distribution that we target.
To workaround this [Tox](https://tox.readthedocs.io) is used, to install and
run multiple versions of Ansible, in Python virtualenvs.
``tox``
## Preparing an OS X box

@ -0,0 +1,20 @@
- name: Start containers
hosts: all
strategy: mitogen_free
gather_facts: false
tasks:
- name: Fetch container images
docker_image:
name: "{{ docker_base }}"
delegate_to: localhost
- name: Start containers
docker_container:
name: "{{ inventory_hostname }}"
image: "{{ docker_base }}"
command: /bin/bash
hostname: "mitogen-{{ inventory_hostname }}"
detach: true
interactive: true
tty: true
delegate_to: localhost

@ -0,0 +1,18 @@
- name: Prepare images
hosts: all
strategy: mitogen_free
gather_facts: true
tasks:
- name: Commit containers
command: >
docker commit
--change 'EXPOSE 22'
--change 'CMD ["/usr/sbin/sshd", "-D"]'
{{ inventory_hostname }}
public.ecr.aws/n5z0e8q9/{{ inventory_hostname }}-test
delegate_to: localhost
- name: Stop containers
command: >
docker rm -f {{ inventory_hostname }}
delegate_to: localhost

@ -1,83 +1,76 @@
- hosts: all
vars_files:
- shared_vars.yml
strategy: linear
gather_facts: false
tasks:
- raw: >
if ! python -c ''; then
if type -p yum; then
yum -y install python;
else
apt-get -y update && apt-get -y install python;
fi;
- name: Install bootstrap packages
raw: |
set -o errexit
set -o nounset
if type -p yum; then
yum -y install {{ bootstrap_packages | join(' ') }}
else
apt-get -y update
apt-get -y --no-install-recommends install {{ bootstrap_packages | join(' ') }}
fi
when: bootstrap_packages | length
- hosts: all
vars_files:
- shared_vars.yml
strategy: mitogen_free
# Resource limitation, my laptop freezes doing every container concurrently
serial: 4
# Can't gather facts before here.
gather_facts: true
vars:
distro: "{{ansible_distribution}}"
ver: "{{ansible_distribution_major_version}}"
packages:
common:
- openssh-server
- rsync
- strace
- sudo
Debian:
"9":
- libjson-perl
- python-virtualenv
- locales
CentOS:
"5":
- perl
- sudo
#- perl-JSON -- skipped on CentOS 5, packages are a pain.
"6":
- perl-JSON
"7":
- perl-JSON
- python-virtualenv
tasks:
- when: ansible_virtualization_type != "docker"
meta: end_play
- name: Ensure requisite Debian packages are installed
- name: Ensure requisite apt packages are installed
apt:
name: "{{packages.common + packages[distro][ver]}}"
state: installed
name: "{{ common_packages + packages }}"
state: present
install_recommends: false
update_cache: true
when: distro == "Debian"
when: ansible_pkg_mgr == 'apt'
- name: Ensure requisite Red Hat packaed are installed
- name: Ensure requisite yum packages are installed
yum:
name: "{{packages.common + packages[distro][ver]}}"
state: installed
name: "{{ common_packages + packages }}"
state: present
update_cache: true
when: distro == "CentOS"
when: ansible_pkg_mgr == 'yum'
- name: Clean up apt cache
command: apt-get clean
when: distro == "Debian"
- name: Ensure requisite dnf packages are installed
dnf:
name: "{{ common_packages + packages }}"
state: present
update_cache: true
when: ansible_pkg_mgr == 'dnf'
- name: Clean up package cache
vars:
clean_command:
apt: apt-get clean
yum: yum clean all
dnf: dnf clean all
command: "{{ clean_command[ansible_pkg_mgr] }}"
args:
warn: false
- name: Clean up apt package lists
shell: rm -rf {{item}}/*
with_items:
- /var/cache/apt
- /var/lib/apt/lists
when: distro == "Debian"
when: ansible_pkg_mgr == 'apt'
- name: Clean up yum cache
command: yum clean all
when: distro == "CentOS"
- name: Configure /usr/bin/python
command: alternatives --set python /usr/bin/python3.8
args:
creates: /usr/bin/python
when: inventory_hostname in ["centos8"]
- name: Enable UTF-8 locale on Debian
copy:
@ -85,11 +78,11 @@
content: |
en_US.UTF-8 UTF-8
fr_FR.UTF-8 UTF-8
when: distro == "Debian"
when: ansible_pkg_mgr == 'apt'
- name: Generate UTF-8 locale on Debian
shell: locale-gen
when: distro == "Debian"
when: ansible_pkg_mgr == 'apt'
- name: Write Unicode into /etc/environment
copy:
@ -115,16 +108,6 @@
permit :mitogen__group
permit :root
- name: Vanilla Ansible needs simplejson on CentOS 5.
shell: mkdir -p /usr/lib/python2.4/site-packages/simplejson/
when: distro == "CentOS" and ver == "5"
- name: Vanilla Ansible needs simplejson on CentOS 5.
synchronize:
dest: /usr/lib/python2.4/site-packages/simplejson/
src: ../../ansible_mitogen/compat/simplejson/
when: distro == "CentOS" and ver == "5"
- name: Set root user password and shell
user:
name: root
@ -182,8 +165,9 @@
- name: Install CentOS wheel sudo rule
lineinfile:
path: /etc/sudoers
line: "%wheel ALL=(ALL) ALL"
when: distro == "CentOS"
regexp: '#* *%wheel +ALL=(ALL) +ALL'
line: "%wheel ALL=(ALL) ALL"
when: ansible_os_family == 'RedHat'
- name: Enable SSH banner
lineinfile:
@ -202,6 +186,15 @@
regexp: '.*session.*required.*pam_loginuid.so'
line: session optional pam_loginuid.so
# Normally this would be removed by systemd-networkd-wait-online. If
# present ssh works only for root. The message displayed is
# > System is booting up. Unprivileged users are not permitted to log in
# > yet. Please come back later. For technical details, see pam_nologin(8).
- name: Remove login lockout
file:
path: /run/nologin
state: absent
- name: Install convenience script for running an straced Python
copy:
mode: 'u+rwx,go=rx'

@ -5,15 +5,11 @@
#
- hosts: all
vars_files:
- shared_vars.yml
gather_facts: true
strategy: mitogen_free
become: true
vars:
distro: "{{ansible_distribution}}"
ver: "{{ansible_distribution_major_version}}"
special_users:
- has_sudo
- has_sudo_nopw
@ -172,6 +168,8 @@
with_items:
- mitogen__pw_required
- mitogen__require_tty_pw_required
when:
- ansible_virtualization_type != "docker"
- name: Allow passwordless sudo for require_tty/readonly_homedir
lineinfile:
@ -181,6 +179,8 @@
with_items:
- mitogen__require_tty
- mitogen__readonly_homedir
when:
- ansible_virtualization_type != "docker"
- name: Allow passwordless for many accounts
lineinfile:
@ -188,3 +188,5 @@
line: "{{lookup('pipe', 'whoami')}} ALL = (mitogen__{{item}}:ALL) NOPASSWD:ALL"
validate: '/usr/sbin/visudo -cf %s'
with_items: "{{normal_users}}"
when:
- ansible_virtualization_type != "docker"

@ -1,7 +1,11 @@
[defaults]
deprecation_warnings = false
strategy_plugins = ../../ansible_mitogen/plugins/strategy
retry_files_enabled = false
display_args_to_stdout = True
no_target_syslog = True
host_key_checking = False
[inventory]
unparsed_is_fatal = true

@ -1,65 +0,0 @@
#!/usr/bin/env python
"""
Build the Docker images used for testing.
"""
import commands
import os
import shlex
import subprocess
import sys
import tempfile
BASEDIR = os.path.dirname(os.path.abspath(__file__))
def sh(s, *args):
if args:
s %= args
return shlex.split(s)
label_by_id = {}
for base_image, label in [
('astj/centos5-vault', 'centos5'), # Python 2.4.3
# Debian containers later than debuerreotype/debuerreotype#48 no longer
# ship a stub 'initctl', causing (apparently) the Ansible service
# module run at the end of DebOps to trigger a full stop/start of SSHd.
# When SSHd is killed, Docker responds by destroying the container.
# Proper solution is to include a full /bin/init; Docker --init doesn't
# help. In the meantime, just use a fixed older version.
('debian:stretch-20181112', 'debian'), # Python 2.7.13, 3.5.3
('centos:6', 'centos6'), # Python 2.6.6
('centos:7', 'centos7') # Python 2.7.5
]:
args = sh('docker run --rm -it -d -h mitogen-%s %s /bin/bash',
label, base_image)
container_id = subprocess.check_output(args).strip()
label_by_id[container_id] = label
with tempfile.NamedTemporaryFile() as fp:
fp.write('[all]\n')
for id_, label in label_by_id.items():
fp.write('%s ansible_host=%s\n' % (label, id_))
fp.flush()
try:
subprocess.check_call(
cwd=BASEDIR,
args=sh('ansible-playbook -i %s -c docker setup.yml', fp.name) + sys.argv[1:],
)
for container_id, label in label_by_id.items():
subprocess.check_call(sh('''
docker commit
--change 'EXPOSE 22'
--change 'CMD ["/usr/sbin/sshd", "-D"]'
%s
mitogen/%s-test
''', container_id, label))
finally:
subprocess.check_call(sh('docker rm -f %s', ' '.join(label_by_id)))

@ -1,3 +1,9 @@
common_packages:
- openssh-server
- rsync
- strace
- sudo
sudo_group:
MacOSX: admin
Debian: sudo

@ -0,0 +1,6 @@
bootstrap_packages: [python-simplejson]
docker_base: astj/centos5-vault
packages:
- perl

@ -0,0 +1,6 @@
bootstrap_packages: [python]
docker_base: moreati/centos6-vault
packages:
- perl-JSON

@ -0,0 +1,8 @@
bootstrap_packages: [python]
docker_base: centos:7
packages:
- perl-JSON
- python-virtualenv
- python3

@ -0,0 +1,10 @@
bootstrap_packages: [python3]
docker_base: centos:8
packages:
- perl-JSON
- python2-virtualenv
- python3-virtualenv
- python36
- python38

@ -0,0 +1,11 @@
bootstrap_packages: [python]
docker_base: debian:10
packages:
- libjson-perl
- locales
- python-virtualenv
- python3
- python3-virtualenv
- virtualenv

@ -0,0 +1,11 @@
bootstrap_packages: [python3, python3-apt]
docker_base: debian:bullseye
packages:
- libjson-perl
- locales
- python-is-python3
- python2
- python3-virtualenv
- virtualenv

@ -0,0 +1,11 @@
bootstrap_packages: [python]
docker_base: debian:9
packages:
- libjson-perl
- locales
- python-virtualenv
- python3
- python3-virtualenv
- virtualenv

@ -0,0 +1,11 @@
bootstrap_packages: [python]
docker_base: ubuntu:16.04
packages:
- libjson-perl
- locales
- python-virtualenv
- python3
- python3-virtualenv
- virtualenv

@ -0,0 +1,11 @@
bootstrap_packages: [python]
docker_base: ubuntu:18.04
packages:
- libjson-perl
- locales
- python-virtualenv
- python3
- python3-virtualenv
- virtualenv

@ -0,0 +1,11 @@
bootstrap_packages: [python3]
docker_base: ubuntu:20.04
packages:
- libjson-perl
- locales
- python-is-python3
- python2
- python3-virtualenv
- virtualenv

@ -0,0 +1,23 @@
[all:children]
centos
debian
ubuntu
[all:vars]
ansible_connection = docker
[centos]
centos5
centos6
centos7
centos8
[debian]
debian9
debian10
debian11
[ubuntu]
ubuntu1604
ubuntu1804
ubuntu2004

@ -1,3 +1,6 @@
#!/usr/bin/env ansible-playbook
- include: _container_create.yml
- include: _container_setup.yml
- include: _user_accounts.yml
- include: _container_finalize.yml

@ -0,0 +1,29 @@
[tox]
envlist =
ansible2.3,
ansible2.10,
skipsdist = true
[testenv]
setenv =
ANSIBLE_STRATEGY_PLUGINS={envsitepackagesdir}/ansible_mitogen/plugins/strategy
[testenv:ansible2.3]
basepython = python2
deps =
ansible>=2.3,<2.4
docker-py>=1.7.0
mitogen>=0.2.10rc1,<0.3
install_command =
python -m pip --no-python-version-warning install {opts} {packages}
commands =
./setup.yml -i hosts.ini -l 'localhost,centos5' {posargs}
[testenv:ansible2.10]
basepython = python3
deps =
ansible>=2.10,<2.11
docker>=1.8.0
mitogen>=0.3.0rc1,<0.4
commands =
./setup.yml -i hosts.ini -l '!centos5' {posargs}

@ -450,7 +450,7 @@ class DockerizedSshDaemon(object):
subprocess__check_output(args)
self._get_container_port()
def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian')):
def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')):
if '-' in mitogen_test_distro:
distro, _py3 = mitogen_test_distro.split('-')
else:
@ -462,7 +462,7 @@ class DockerizedSshDaemon(object):
else:
self.python_path = '/usr/bin/python'
self.image = 'mitogen/%s-test' % (distro,)
self.image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,)
# 22/tcp -> 0.0.0.0:32771
self.PORT_RE = re.compile(r'([^/]+)/([^ ]+) -> ([^:]+):(.*)')

@ -9,18 +9,29 @@
# Last version to support each python version
#
# tox vir'env pip ansible coverage
# ========== ======== ======== ======== ======== ========
# python2.4 1.4 1.8 1.1 ???
# tox vir'env pip ansible ansible coverage
# control target
# ========== ======== ======== ======== ======== ======== ========
# python2.4 1.4 1.8 1.1 2.3?
# python2.5 1.6.1 1.9.1 1.3.1 ???
# python2.6 2.9.1 15.2.0 9.0.3 2.6.20 4.5.4
# python2.6 2.9.1 15.2.0 9.0.3 2.6.20 4.5.4
# python2.7 20.3 2.10
# python3.5 2.10
# python3.6 2.10
# python3.7 2.10
# pip --no-python-version-warning
# pip --disable-pip-version-check
[tox]
envlist =
init,
py{27,36,39}-mode_ansible-ansible2.10,
py{27,36,39}-mode_mitogen,
py{27,36,39}-mode_mitogen-distro_centos7,
py{27,35,39}-mode_ansible-distros_centos,
py{27,35,39}-mode_ansible-distros_debian,
py{27,35,39}-mode_ansible-distros_ubuntu,
py{27,35,39}-mode_mitogen-distro_centos{6,7,8},
py{27,35,39}-mode_mitogen-distro_debian{9,10,11},
py{27,35,39}-mode_mitogen-distro_ubuntu{1604,1804,2004},
report,
requires =
tox-factor
@ -29,24 +40,29 @@ requires =
basepython =
py26: python2.6
py27: python2.7
py35: python3.5
py36: python3.6
py37: python3.7
py38: python3.8
py39: python3.9
install_command =
python -m pip --no-python-version-warning install {opts} {packages}
commands_pre =
mode_ansible: {toxinidir}/.ci/ansible_install.py
mode_debops_common: {toxinidir}/.ci/debops_common_install.py
mode_mitogen: {toxinidir}/.ci/mitogen_install.py
commands =
mode_ansible: {toxinidir}/.ci/ansible_tests.py \
--skip-tags requires_local_sudo
mode_ansible: {toxinidir}/.ci/ansible_tests.py
mode_debops_common: {toxinidir}/.ci/debops_common_tests.py
mode_mitogen: {toxinidir}/.ci/mitogen_tests.py
passenv =
ANSIBLE_*
HOME
setenv =
ANSIBLE_SKIP_TAGS = requires_local_sudo
NOCOVERAGE_ERASE = 1
NOCOVERAGE_REPORT = 1
VER=2.10.5
ansible2.3: VER=2.3.3.0
ansible2.4: VER=2.4.6.0
ansible2.8: VER=2.8.3
@ -55,10 +71,26 @@ setenv =
distro_centos5: DISTRO=centos5
distro_centos6: DISTRO=centos6
distro_centos7: DISTRO=centos7
distro_debian: DISTRO=debian
distro_debianpy3: DISTRO=debian-py3
distro_centos8: DISTRO=centos8
distro_debian9: DISTRO=debian9
distro_debian10: DISTRO=debian10
distro_debian11: DISTRO=debian11
distro_ubuntu1604: DISTRO=ubuntu1604
distro_ubuntu1804: DISTRO=ubuntu1804
distro_ubuntu2004: DISTRO=ubuntu2004
distros_centos: DISTROS=centos6 centos7 centos8
distros_centos5: DISTROS=centos5
distros_debian: DISTROS=debian
distros_centos6: DISTROS=centos6
distros_centos7: DISTROS=centos7
distros_centos8: DISTROS=centos8
distros_debian: DISTROS=debian9 debian10 debian11
distros_debian9: DISTROS=debian9
distros_debian10: DISTROS=debian10
distros_debian11: DISTROS=debian11
distros_ubuntu: DISTROS=ubuntu1604 ubuntu1804 ubuntu2004
distros_ubuntu1604: DISTROS=ubuntu1604
distros_ubuntu1804: DISTROS=ubuntu1804
distros_ubuntu2004: DISTROS=ubuntu2004
mode_ansible: MODE=ansible
mode_debops_common: MODE=debops_common
mode_mitogen: MODE=mitogen

Loading…
Cancel
Save