diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 3dc2af41..c22dcf6c 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -45,7 +45,7 @@ jobs: Mito27Debian_27: python.version: '2.7' MODE: mitogen - DISTRO: debian + DISTRO: debian9 #MitoPy27CentOS6_26: #python.version: '2.7' @@ -60,12 +60,12 @@ jobs: Mito37Debian_27: python.version: '3.7' MODE: mitogen - DISTRO: debian + DISTRO: debian9 Mito39Debian_27: python.version: '3.9' MODE: mitogen - DISTRO: debian + DISTRO: debian9 VER: 2.10.0 #Py26CentOS7: diff --git a/tests/ansible/integration/become/sudo_nonexistent.yml b/tests/ansible/integration/become/sudo_nonexistent.yml index ccb94e34..09ab91bb 100644 --- a/tests/ansible/integration/become/sudo_nonexistent.yml +++ b/tests/ansible/integration/become/sudo_nonexistent.yml @@ -12,8 +12,10 @@ - name: Verify raw module output. assert: - that: | - out.failed and ( - ('sudo: unknown user: slartibartfast' in out.msg) or - ('sudo: unknown user: slartibartfast' in out.module_stderr) - ) + that: + - out.failed + # sudo-1.8.6p3-29.el6_10.3 on RHEL & CentOS 6.10 (final release) + # removed user/group error messages, as defence against CVE-2019-14287. + - >- + ('sudo: unknown user: slartibartfast' in out.module_stderr | default(out.msg)) + or (ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_version == '6.10') diff --git a/tests/image_prep/README.md b/tests/image_prep/README.md index a970b319..9e0e630a 100644 --- a/tests/image_prep/README.md +++ b/tests/image_prep/README.md @@ -14,10 +14,11 @@ See ../README.md for a (mostly) description of the accounts created. ## Building the containers -``./build_docker_images.sh`` - -Requires Ansible 2.3.x.x in order to target CentOS 5 +No single version of Ansible supports every Linux distribution that we target. +To workaround this [Tox](https://tox.readthedocs.io) is used, to install and +run multiple versions of Ansible, in Python virtualenvs. +``tox`` ## Preparing an OS X box diff --git a/tests/image_prep/_container_create.yml b/tests/image_prep/_container_create.yml new file mode 100644 index 00000000..b07c46eb --- /dev/null +++ b/tests/image_prep/_container_create.yml @@ -0,0 +1,20 @@ +- name: Start containers + hosts: all + strategy: mitogen_free + gather_facts: false + tasks: + - name: Fetch container images + docker_image: + name: "{{ docker_base }}" + delegate_to: localhost + + - name: Start containers + docker_container: + name: "{{ inventory_hostname }}" + image: "{{ docker_base }}" + command: /bin/bash + hostname: "mitogen-{{ inventory_hostname }}" + detach: true + interactive: true + tty: true + delegate_to: localhost diff --git a/tests/image_prep/_container_finalize.yml b/tests/image_prep/_container_finalize.yml new file mode 100644 index 00000000..d61d9b3b --- /dev/null +++ b/tests/image_prep/_container_finalize.yml @@ -0,0 +1,18 @@ +- name: Prepare images + hosts: all + strategy: mitogen_free + gather_facts: true + tasks: + - name: Commit containers + command: > + docker commit + --change 'EXPOSE 22' + --change 'CMD ["/usr/sbin/sshd", "-D"]' + {{ inventory_hostname }} + public.ecr.aws/n5z0e8q9/{{ inventory_hostname }}-test + delegate_to: localhost + + - name: Stop containers + command: > + docker rm -f {{ inventory_hostname }} + delegate_to: localhost diff --git a/tests/image_prep/_container_setup.yml b/tests/image_prep/_container_setup.yml index 65e898a1..c564f1a1 100644 --- a/tests/image_prep/_container_setup.yml +++ b/tests/image_prep/_container_setup.yml @@ -1,83 +1,63 @@ - hosts: all - vars_files: - - shared_vars.yml strategy: linear gather_facts: false tasks: - - raw: > - if ! python -c ''; then - if type -p yum; then - yum -y install python; - else - apt-get -y update && apt-get -y install python; - fi; + - name: Install bootstrap packages + raw: | + set -o errexit + set -o nounset + if type -p yum; then + yum -y install {{ bootstrap_packages | join(' ') }} + else + apt-get -y update + apt-get -y --no-install-recommends install {{ bootstrap_packages | join(' ') }} fi + when: bootstrap_packages | length - hosts: all - vars_files: - - shared_vars.yml strategy: mitogen_free + # Resource limitation, my laptop freezes doing every container concurrently + serial: 4 # Can't gather facts before here. gather_facts: true vars: distro: "{{ansible_distribution}}" - ver: "{{ansible_distribution_major_version}}" - - packages: - common: - - openssh-server - - rsync - - strace - - sudo - Debian: - "9": - - libjson-perl - - python-virtualenv - - locales - CentOS: - "5": - - perl - - sudo - #- perl-JSON -- skipped on CentOS 5, packages are a pain. - "6": - - perl-JSON - "7": - - perl-JSON - - python-virtualenv - tasks: - when: ansible_virtualization_type != "docker" meta: end_play - - name: Ensure requisite Debian packages are installed + - name: Ensure requisite apt packages are installed apt: - name: "{{packages.common + packages[distro][ver]}}" - state: installed + name: "{{ common_packages + packages }}" + state: present + install_recommends: false update_cache: true - when: distro == "Debian" + when: ansible_pkg_mgr == 'apt' - - name: Ensure requisite Red Hat packaed are installed + - name: Ensure requisite yum packages are installed yum: - name: "{{packages.common + packages[distro][ver]}}" - state: installed + name: "{{ common_packages + packages }}" + state: present update_cache: true - when: distro == "CentOS" - - - name: Clean up apt cache - command: apt-get clean - when: distro == "Debian" + when: ansible_pkg_mgr == 'yum' + + - name: Clean up package cache + vars: + clean_command: + apt: apt-get clean + yum: yum clean all + command: "{{ clean_command[ansible_pkg_mgr] }}" + args: + warn: false - name: Clean up apt package lists shell: rm -rf {{item}}/* with_items: - /var/cache/apt - /var/lib/apt/lists - when: distro == "Debian" + when: ansible_pkg_mgr == 'apt' - - name: Clean up yum cache - command: yum clean all - when: distro == "CentOS" - name: Enable UTF-8 locale on Debian copy: @@ -85,11 +65,11 @@ content: | en_US.UTF-8 UTF-8 fr_FR.UTF-8 UTF-8 - when: distro == "Debian" + when: ansible_pkg_mgr == 'apt' - name: Generate UTF-8 locale on Debian shell: locale-gen - when: distro == "Debian" + when: ansible_pkg_mgr == 'apt' - name: Write Unicode into /etc/environment copy: @@ -115,16 +95,6 @@ permit :mitogen__group permit :root - - name: Vanilla Ansible needs simplejson on CentOS 5. - shell: mkdir -p /usr/lib/python2.4/site-packages/simplejson/ - when: distro == "CentOS" and ver == "5" - - - name: Vanilla Ansible needs simplejson on CentOS 5. - synchronize: - dest: /usr/lib/python2.4/site-packages/simplejson/ - src: ../../ansible_mitogen/compat/simplejson/ - when: distro == "CentOS" and ver == "5" - - name: Set root user password and shell user: name: root @@ -182,8 +152,9 @@ - name: Install CentOS wheel sudo rule lineinfile: path: /etc/sudoers - line: "%wheel ALL=(ALL) ALL" - when: distro == "CentOS" + regexp: '#* *%wheel +ALL=(ALL) +ALL' + line: "%wheel ALL=(ALL) ALL" + when: ansible_os_family == 'RedHat' - name: Enable SSH banner lineinfile: diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 150e54b4..5fc97714 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -5,15 +5,11 @@ # - hosts: all - vars_files: - - shared_vars.yml gather_facts: true strategy: mitogen_free become: true vars: distro: "{{ansible_distribution}}" - ver: "{{ansible_distribution_major_version}}" - special_users: - has_sudo - has_sudo_nopw diff --git a/tests/image_prep/ansible.cfg b/tests/image_prep/ansible.cfg index 60f2975e..0745aed1 100644 --- a/tests/image_prep/ansible.cfg +++ b/tests/image_prep/ansible.cfg @@ -1,7 +1,11 @@ [defaults] +deprecation_warnings = false strategy_plugins = ../../ansible_mitogen/plugins/strategy retry_files_enabled = false display_args_to_stdout = True no_target_syslog = True host_key_checking = False + +[inventory] +unparsed_is_fatal = true diff --git a/tests/image_prep/build_docker_images.py b/tests/image_prep/build_docker_images.py deleted file mode 100755 index 76564297..00000000 --- a/tests/image_prep/build_docker_images.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python - -""" -Build the Docker images used for testing. -""" - -import commands -import os -import shlex -import subprocess -import sys -import tempfile - - -BASEDIR = os.path.dirname(os.path.abspath(__file__)) - - -def sh(s, *args): - if args: - s %= args - return shlex.split(s) - - - -label_by_id = {} - -for base_image, label in [ - ('astj/centos5-vault', 'centos5'), # Python 2.4.3 - # Debian containers later than debuerreotype/debuerreotype#48 no longer - # ship a stub 'initctl', causing (apparently) the Ansible service - # module run at the end of DebOps to trigger a full stop/start of SSHd. - # When SSHd is killed, Docker responds by destroying the container. - # Proper solution is to include a full /bin/init; Docker --init doesn't - # help. In the meantime, just use a fixed older version. - ('debian:stretch-20181112', 'debian'), # Python 2.7.13, 3.5.3 - ('centos:6', 'centos6'), # Python 2.6.6 - ('centos:7', 'centos7') # Python 2.7.5 - ]: - args = sh('docker run --rm -it -d -h mitogen-%s %s /bin/bash', - label, base_image) - container_id = subprocess.check_output(args).strip() - label_by_id[container_id] = label - -with tempfile.NamedTemporaryFile() as fp: - fp.write('[all]\n') - for id_, label in label_by_id.items(): - fp.write('%s ansible_host=%s\n' % (label, id_)) - fp.flush() - - try: - subprocess.check_call( - cwd=BASEDIR, - args=sh('ansible-playbook -i %s -c docker setup.yml', fp.name) + sys.argv[1:], - ) - - for container_id, label in label_by_id.items(): - subprocess.check_call(sh(''' - docker commit - --change 'EXPOSE 22' - --change 'CMD ["/usr/sbin/sshd", "-D"]' - %s - mitogen/%s-test - ''', container_id, label)) - finally: - subprocess.check_call(sh('docker rm -f %s', ' '.join(label_by_id))) diff --git a/tests/image_prep/shared_vars.yml b/tests/image_prep/group_vars/all.yml similarity index 52% rename from tests/image_prep/shared_vars.yml rename to tests/image_prep/group_vars/all.yml index 4be7babe..5f182f86 100644 --- a/tests/image_prep/shared_vars.yml +++ b/tests/image_prep/group_vars/all.yml @@ -1,3 +1,9 @@ +common_packages: + - openssh-server + - rsync + - strace + - sudo + sudo_group: MacOSX: admin Debian: sudo diff --git a/tests/image_prep/host_vars/centos5.yml b/tests/image_prep/host_vars/centos5.yml new file mode 100644 index 00000000..1828c29e --- /dev/null +++ b/tests/image_prep/host_vars/centos5.yml @@ -0,0 +1,6 @@ +bootstrap_packages: [python-simplejson] + +docker_base: astj/centos5-vault + +packages: + - perl diff --git a/tests/image_prep/host_vars/centos6.yml b/tests/image_prep/host_vars/centos6.yml new file mode 100644 index 00000000..aae7965f --- /dev/null +++ b/tests/image_prep/host_vars/centos6.yml @@ -0,0 +1,6 @@ +bootstrap_packages: [python] + +docker_base: moreati/centos6-vault + +packages: + - perl-JSON diff --git a/tests/image_prep/host_vars/centos7.yml b/tests/image_prep/host_vars/centos7.yml new file mode 100644 index 00000000..fec83471 --- /dev/null +++ b/tests/image_prep/host_vars/centos7.yml @@ -0,0 +1,8 @@ +bootstrap_packages: [python] + +docker_base: centos:7 + +packages: + - perl-JSON + - python-virtualenv + - python3 diff --git a/tests/image_prep/host_vars/debian9.yml b/tests/image_prep/host_vars/debian9.yml new file mode 100644 index 00000000..cbd22e0f --- /dev/null +++ b/tests/image_prep/host_vars/debian9.yml @@ -0,0 +1,11 @@ +bootstrap_packages: [python] + +docker_base: debian:9 + +packages: + - libjson-perl + - locales + - python-virtualenv + - python3 + - python3-virtualenv + - virtualenv diff --git a/tests/image_prep/hosts.ini b/tests/image_prep/hosts.ini new file mode 100644 index 00000000..9c756f51 --- /dev/null +++ b/tests/image_prep/hosts.ini @@ -0,0 +1,14 @@ +[all:children] +centos +debian + +[all:vars] +ansible_connection = docker + +[centos] +centos5 +centos6 +centos7 + +[debian] +debian9 diff --git a/tests/image_prep/setup.yml b/tests/image_prep/setup.yml old mode 100644 new mode 100755 index 2c37c6bb..9aa3285c --- a/tests/image_prep/setup.yml +++ b/tests/image_prep/setup.yml @@ -1,3 +1,6 @@ +#!/usr/bin/env ansible-playbook +- include: _container_create.yml - include: _container_setup.yml - include: _user_accounts.yml +- include: _container_finalize.yml diff --git a/tests/image_prep/tox.ini b/tests/image_prep/tox.ini new file mode 100644 index 00000000..d28d1512 --- /dev/null +++ b/tests/image_prep/tox.ini @@ -0,0 +1,29 @@ +[tox] +envlist = + ansible2.3, + ansible2.10, +skipsdist = true + +[testenv] +setenv = + ANSIBLE_STRATEGY_PLUGINS={envsitepackagesdir}/ansible_mitogen/plugins/strategy + +[testenv:ansible2.3] +basepython = python2 +deps = + ansible>=2.3,<2.4 + docker-py>=1.7.0 + mitogen>=0.2.10rc1,<0.3 +install_command = + python -m pip --no-python-version-warning install {opts} {packages} +commands = + ./setup.yml -i hosts.ini -l 'localhost,centos5' {posargs} + +[testenv:ansible2.10] +basepython = python3 +deps = + ansible>=2.10,<2.11 + docker>=1.8.0 + mitogen>=0.3.0rc1,<0.4 +commands = + ./setup.yml -i hosts.ini -l '!centos5' {posargs} diff --git a/tests/testlib.py b/tests/testlib.py index ee76a26d..e16309ee 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -450,7 +450,7 @@ class DockerizedSshDaemon(object): subprocess__check_output(args) self._get_container_port() - def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian')): + def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')): if '-' in mitogen_test_distro: distro, _py3 = mitogen_test_distro.split('-') else: @@ -462,7 +462,7 @@ class DockerizedSshDaemon(object): else: self.python_path = '/usr/bin/python' - self.image = 'mitogen/%s-test' % (distro,) + self.image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,) # 22/tcp -> 0.0.0.0:32771 self.PORT_RE = re.compile(r'([^/]+)/([^ ]+) -> ([^:]+):(.*)')