Merge pull request #816 from moreati/issue815

Expand coverage of mitogen/<distro>-test docker images

.ci/azure-pipelines.yml

@@ -45,7 +45,7 @@ jobs:
       Mito27Debian_27:
         python.version: '2.7'
         MODE: mitogen
-        DISTRO: debian
+        DISTRO: debian9
 
       #MitoPy27CentOS6_26:
         #python.version: '2.7'
@@ -60,12 +60,12 @@ jobs:
       Mito37Debian_27:
         python.version: '3.7'
         MODE: mitogen
-        DISTRO: debian
+        DISTRO: debian9
 
       Mito39Debian_27:
         python.version: '3.9'
         MODE: mitogen
-        DISTRO: debian
+        DISTRO: debian9
         VER: 2.10.0
 
       #Py26CentOS7:

tests/ansible/integration/become/sudo_nonexistent.yml

@@ -12,8 +12,10 @@
 
     - name: Verify raw module output.
       assert:
-        that: |
+        that:
-          out.failed and (
+          - out.failed
-            ('sudo: unknown user: slartibartfast' in out.msg) or
+          # sudo-1.8.6p3-29.el6_10.3 on RHEL & CentOS 6.10 (final release)
-            ('sudo: unknown user: slartibartfast' in out.module_stderr)
+          # removed user/group error messages, as defence against CVE-2019-14287.
-          )
+          - >-
+            ('sudo: unknown user: slartibartfast' in out.module_stderr | default(out.msg))
+            or (ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_version == '6.10')

tests/ansible/integration/runner/custom_binary_single_null.yml

@@ -15,10 +15,18 @@
       - "out.failed"
       - "out.results[0].failed"
       - "out.results[0].msg.startswith('MODULE FAILURE')"
-      - "out.results[0].module_stdout.startswith('/bin/sh: ')"
+      # On Ubuntu 16.04 /bin/sh is dash 0.5.8. It treats custom_binary_single_null
+      # as a valid executable. There's no error message, and rc == 0.
       - |
-        out.results[0].module_stdout.endswith('custom_binary_single_null: cannot execute binary file\r\n') or
+        out.results[0].module_stdout.startswith('/bin/sh: ')
-        out.results[0].module_stdout.endswith('custom_binary_single_null: Exec format error\r\n')
+        or (ansible_facts.distribution == 'Ubuntu' and ansible_facts.distribution_version == '16.04')
+      - |
+        out.results[0].module_stdout.endswith((
+          'custom_binary_single_null: cannot execute binary file\r\n',
+          'custom_binary_single_null: Exec format error\r\n',
+          'custom_binary_single_null: cannot execute binary file: Exec format error\r\n',
+        ))
+        or (ansible_facts.distribution == 'Ubuntu' and ansible_facts.distribution_version == '16.04')
 
 
 # Can't test this: Mitogen returns 126, 2.5.x returns 126, 2.4.x discarded the

tests/image_prep/README.md

@@ -14,10 +14,11 @@ See ../README.md for a (mostly) description of the accounts created.
 
 ## Building the containers
 
-``./build_docker_images.sh``
+No single version of Ansible supports every Linux distribution that we target.
-
+To workaround this [Tox](https://tox.readthedocs.io) is used, to install and
-Requires Ansible 2.3.x.x in order to target CentOS 5
+run multiple versions of Ansible, in Python virtualenvs.
 
+``tox``
 
 ## Preparing an OS X box
 

tests/image_prep/_container_create.yml

@@ -0,0 +1,20 @@
+- name: Start containers
+  hosts: all
+  strategy: mitogen_free
+  gather_facts: false
+  tasks:
+    - name: Fetch container images
+      docker_image:
+        name: "{{ docker_base }}"
+      delegate_to: localhost
+
+    - name: Start containers
+      docker_container:
+        name: "{{ inventory_hostname }}"
+        image: "{{ docker_base }}"
+        command: /bin/bash
+        hostname: "mitogen-{{ inventory_hostname }}"
+        detach: true
+        interactive: true
+        tty: true
+      delegate_to: localhost

tests/image_prep/_container_finalize.yml

@@ -0,0 +1,18 @@
+- name: Prepare images
+  hosts: all
+  strategy: mitogen_free
+  gather_facts: true
+  tasks:
+    - name: Commit containers
+      command: >
+        docker commit
+        --change 'EXPOSE 22'
+        --change 'CMD ["/usr/sbin/sshd", "-D"]'
+        {{ inventory_hostname }}
+        public.ecr.aws/n5z0e8q9/{{ inventory_hostname }}-test
+      delegate_to: localhost
+
+    - name: Stop containers
+      command: >
+        docker rm -f {{ inventory_hostname }}
+      delegate_to: localhost

tests/image_prep/_container_setup.yml

@@ -1,83 +1,76 @@
 
 - hosts: all
-  vars_files:
-  - shared_vars.yml
   strategy: linear
   gather_facts: false
   tasks:
-    - raw: >
+    - name: Install bootstrap packages
-        if ! python -c ''; then
+      raw: |
+        set -o errexit
+        set -o nounset
         if type -p yum; then
-            yum -y install python;
+          yum -y install {{ bootstrap_packages | join(' ') }}
         else
-            apt-get -y update && apt-get -y install python;
+          apt-get -y update
-          fi;
+          apt-get -y --no-install-recommends install {{ bootstrap_packages | join(' ') }}
         fi
+      when: bootstrap_packages | length
 
 - hosts: all
-  vars_files:
-  - shared_vars.yml
   strategy: mitogen_free
+  # Resource limitation, my laptop freezes doing every container concurrently
+  serial: 4
   # Can't gather facts before here.
   gather_facts: true
   vars:
     distro: "{{ansible_distribution}}"
-    ver: "{{ansible_distribution_major_version}}"
-
-    packages:
-      common:
-        - openssh-server
-        - rsync
-        - strace
-        - sudo
-      Debian:
-        "9":
-          - libjson-perl
-          - python-virtualenv
-          - locales
-      CentOS:
-        "5":
-        - perl
-        - sudo
-          #- perl-JSON  -- skipped on CentOS 5, packages are a pain.
-        "6":
-          - perl-JSON
-        "7":
-          - perl-JSON
-          - python-virtualenv
-
   tasks:
     - when: ansible_virtualization_type != "docker"
       meta: end_play
 
-    - name: Ensure requisite Debian packages are installed
+    - name: Ensure requisite apt packages are installed
       apt:
-        name: "{{packages.common + packages[distro][ver]}}"
+        name: "{{ common_packages + packages }}"
-        state: installed
+        state: present
+        install_recommends: false
         update_cache: true
-      when: distro == "Debian"
+      when: ansible_pkg_mgr == 'apt'
 
-    - name: Ensure requisite Red Hat packaed are installed
+    - name: Ensure requisite yum packages are installed
       yum:
-        name: "{{packages.common + packages[distro][ver]}}"
+        name: "{{ common_packages + packages }}"
-        state: installed
+        state: present
+        update_cache: true
+      when: ansible_pkg_mgr == 'yum'
+
+    - name: Ensure requisite dnf packages are installed
+      dnf:
+        name: "{{ common_packages + packages }}"
+        state: present
         update_cache: true
-      when: distro == "CentOS"
+      when: ansible_pkg_mgr == 'dnf'
 
-    - name: Clean up apt cache
+    - name: Clean up package cache
-      command: apt-get clean
+      vars:
-      when: distro == "Debian"
+        clean_command:
+          apt: apt-get clean
+          yum: yum clean all
+          dnf: dnf clean all
+      command: "{{ clean_command[ansible_pkg_mgr] }}"
+      args:
+        warn: false
 
     - name: Clean up apt package lists
       shell: rm -rf {{item}}/*
       with_items:
       - /var/cache/apt
       - /var/lib/apt/lists
-      when: distro == "Debian"
+      when: ansible_pkg_mgr == 'apt'
 
-    - name: Clean up yum cache
+    - name: Configure /usr/bin/python
-      command: yum clean all
+      command: alternatives --set python /usr/bin/python3.8
-      when: distro == "CentOS"
+      args:
+        creates: /usr/bin/python
+      when: inventory_hostname in ["centos8"]
 
     - name: Enable UTF-8 locale on Debian
       copy:
@@ -85,11 +78,11 @@
         content: |
           en_US.UTF-8 UTF-8
           fr_FR.UTF-8 UTF-8
-      when: distro == "Debian"
+      when: ansible_pkg_mgr == 'apt'
 
     - name: Generate UTF-8 locale on Debian
       shell: locale-gen
-      when: distro == "Debian"
+      when: ansible_pkg_mgr == 'apt'
 
     - name: Write Unicode into /etc/environment
       copy:
@@ -115,16 +108,6 @@
           permit :mitogen__group
           permit :root
 
-    - name: Vanilla Ansible needs simplejson on CentOS 5.
-      shell: mkdir -p /usr/lib/python2.4/site-packages/simplejson/
-      when: distro == "CentOS" and ver == "5"
-
-    - name: Vanilla Ansible needs simplejson on CentOS 5.
-      synchronize:
-        dest: /usr/lib/python2.4/site-packages/simplejson/
-        src: ../../ansible_mitogen/compat/simplejson/
-      when: distro == "CentOS" and ver == "5"
-
     - name: Set root user password and shell
       user:
         name: root
@@ -182,8 +165,9 @@
     - name: Install CentOS wheel sudo rule
       lineinfile:
         path: /etc/sudoers
+        regexp: '#* *%wheel +ALL=(ALL) +ALL'
         line: "%wheel  ALL=(ALL)       ALL"
-      when: distro == "CentOS"
+      when: ansible_os_family == 'RedHat'
 
     - name: Enable SSH banner
       lineinfile:
@@ -202,6 +186,15 @@
         regexp: '.*session.*required.*pam_loginuid.so'
         line: session optional pam_loginuid.so
 
+    # Normally this would be removed by systemd-networkd-wait-online. If
+    # present ssh works only for root. The message displayed is
+    # > System is booting up. Unprivileged users are not permitted to log in
+    # > yet. Please come back later. For technical details, see pam_nologin(8).
+    - name: Remove login lockout
+      file:
+        path: /run/nologin
+        state: absent
+
     - name: Install convenience script for running an straced Python
       copy:
         mode: 'u+rwx,go=rx'

tests/image_prep/_user_accounts.yml

@@ -5,15 +5,11 @@
 #
 
 - hosts: all
-  vars_files:
-  - shared_vars.yml
   gather_facts: true
   strategy: mitogen_free
   become: true
   vars:
     distro: "{{ansible_distribution}}"
-    ver: "{{ansible_distribution_major_version}}"
-
     special_users:
       - has_sudo
       - has_sudo_nopw
@@ -172,6 +168,8 @@
       with_items:
         - mitogen__pw_required
         - mitogen__require_tty_pw_required
+      when:
+        - ansible_virtualization_type != "docker"
 
     - name: Allow passwordless sudo for require_tty/readonly_homedir
       lineinfile:
@@ -181,6 +179,8 @@
       with_items:
         - mitogen__require_tty
         - mitogen__readonly_homedir
+      when:
+        - ansible_virtualization_type != "docker"
 
     - name: Allow passwordless for many accounts
       lineinfile:
@@ -188,3 +188,5 @@
         line: "{{lookup('pipe', 'whoami')}} ALL = (mitogen__{{item}}:ALL) NOPASSWD:ALL"
         validate: '/usr/sbin/visudo -cf %s'
       with_items: "{{normal_users}}"
+      when:
+        - ansible_virtualization_type != "docker"

tests/image_prep/ansible.cfg

@@ -1,7 +1,11 @@
 
 [defaults]
+deprecation_warnings = false
 strategy_plugins = ../../ansible_mitogen/plugins/strategy
 retry_files_enabled = false
 display_args_to_stdout = True
 no_target_syslog = True
 host_key_checking = False
+
+[inventory]
+unparsed_is_fatal = true

tests/image_prep/build_docker_images.py

@@ -1,65 +0,0 @@
-#!/usr/bin/env python
-
-"""
-Build the Docker images used for testing.
-"""
-
-import commands
-import os
-import shlex
-import subprocess
-import sys
-import tempfile
-
-
-BASEDIR = os.path.dirname(os.path.abspath(__file__))
-
-
-def sh(s, *args):
-    if args:
-        s %= args
-    return shlex.split(s)
-
-
-
-label_by_id = {}
-
-for base_image, label in [
-        ('astj/centos5-vault', 'centos5'),  # Python 2.4.3
-        # Debian containers later than debuerreotype/debuerreotype#48 no longer
-        # ship a stub 'initctl', causing (apparently) the Ansible service
-        # module run at the end of DebOps to trigger a full stop/start of SSHd.
-        # When SSHd is killed, Docker responds by destroying the container.
-        # Proper solution is to include a full /bin/init; Docker --init doesn't
-        # help. In the meantime, just use a fixed older version.
-        ('debian:stretch-20181112', 'debian'),       # Python 2.7.13, 3.5.3
-        ('centos:6', 'centos6'),            # Python 2.6.6
-        ('centos:7', 'centos7')             # Python 2.7.5
-    ]:
-    args = sh('docker run --rm -it -d -h mitogen-%s %s /bin/bash',
-              label, base_image)
-    container_id = subprocess.check_output(args).strip()
-    label_by_id[container_id] = label
-
-with tempfile.NamedTemporaryFile() as fp:
-    fp.write('[all]\n')
-    for id_, label in label_by_id.items():
-        fp.write('%s ansible_host=%s\n' % (label, id_))
-    fp.flush()
-
-    try:
-        subprocess.check_call(
-            cwd=BASEDIR,
-            args=sh('ansible-playbook -i %s -c docker setup.yml', fp.name) + sys.argv[1:],
-        )
-
-        for container_id, label in label_by_id.items():
-            subprocess.check_call(sh('''
-                docker commit
-                --change 'EXPOSE 22'
-                --change 'CMD ["/usr/sbin/sshd", "-D"]'
-                %s
-                mitogen/%s-test
-            ''', container_id, label))
-    finally:
-        subprocess.check_call(sh('docker rm -f %s', ' '.join(label_by_id)))

tests/image_prep/group_vars/all.yml

@@ -1,3 +1,9 @@
+common_packages:
+  - openssh-server
+  - rsync
+  - strace
+  - sudo
+
 sudo_group:
   MacOSX: admin
   Debian: sudo

tests/image_prep/host_vars/centos5.yml

@@ -0,0 +1,6 @@
+bootstrap_packages: [python-simplejson]
+
+docker_base: astj/centos5-vault
+
+packages:
+  - perl

tests/image_prep/host_vars/centos6.yml

@@ -0,0 +1,6 @@
+bootstrap_packages: [python]
+
+docker_base: moreati/centos6-vault
+
+packages:
+  - perl-JSON

tests/image_prep/host_vars/centos7.yml

@@ -0,0 +1,8 @@
+bootstrap_packages: [python]
+
+docker_base: centos:7
+
+packages:
+  - perl-JSON
+  - python-virtualenv
+  - python3

tests/image_prep/host_vars/centos8.yml

@@ -0,0 +1,10 @@
+bootstrap_packages: [python3]
+
+docker_base: centos:8
+
+packages:
+  - perl-JSON
+  - python2-virtualenv
+  - python3-virtualenv
+  - python36
+  - python38

tests/image_prep/host_vars/debian10.yml

@@ -0,0 +1,11 @@
+bootstrap_packages: [python]
+
+docker_base: debian:10
+
+packages:
+  - libjson-perl
+  - locales
+  - python-virtualenv
+  - python3
+  - python3-virtualenv
+  - virtualenv

tests/image_prep/host_vars/debian11.yml

@@ -0,0 +1,11 @@
+bootstrap_packages: [python3, python3-apt]
+
+docker_base: debian:bullseye
+
+packages:
+  - libjson-perl
+  - locales
+  - python-is-python3
+  - python2
+  - python3-virtualenv
+  - virtualenv

tests/image_prep/host_vars/debian9.yml

@@ -0,0 +1,11 @@
+bootstrap_packages: [python]
+
+docker_base: debian:9
+
+packages:
+  - libjson-perl
+  - locales
+  - python-virtualenv
+  - python3
+  - python3-virtualenv
+  - virtualenv

tests/image_prep/host_vars/ubuntu1604.yml

@@ -0,0 +1,11 @@
+bootstrap_packages: [python]
+
+docker_base: ubuntu:16.04
+
+packages:
+  - libjson-perl
+  - locales
+  - python-virtualenv
+  - python3
+  - python3-virtualenv
+  - virtualenv

tests/image_prep/host_vars/ubuntu1804.yml

@@ -0,0 +1,11 @@
+bootstrap_packages: [python]
+
+docker_base: ubuntu:18.04
+
+packages:
+  - libjson-perl
+  - locales
+  - python-virtualenv
+  - python3
+  - python3-virtualenv
+  - virtualenv

tests/image_prep/host_vars/ubuntu2004.yml

@@ -0,0 +1,11 @@
+bootstrap_packages: [python3]
+
+docker_base: ubuntu:20.04
+
+packages:
+  - libjson-perl
+  - locales
+  - python-is-python3
+  - python2
+  - python3-virtualenv
+  - virtualenv

tests/image_prep/hosts.ini

@@ -0,0 +1,23 @@
+[all:children]
+centos
+debian
+ubuntu
+
+[all:vars]
+ansible_connection = docker
+
+[centos]
+centos5
+centos6
+centos7
+centos8
+
+[debian]
+debian9
+debian10
+debian11
+
+[ubuntu]
+ubuntu1604
+ubuntu1804
+ubuntu2004

tests/image_prep/setup.yml

@@ -1,3 +1,6 @@
+#!/usr/bin/env ansible-playbook
 
+- include: _container_create.yml
 - include: _container_setup.yml
 - include: _user_accounts.yml
+- include: _container_finalize.yml

tests/image_prep/tox.ini

@@ -0,0 +1,29 @@
+[tox]
+envlist =
+    ansible2.3,
+    ansible2.10,
+skipsdist = true
+
+[testenv]
+setenv =
+    ANSIBLE_STRATEGY_PLUGINS={envsitepackagesdir}/ansible_mitogen/plugins/strategy
+
+[testenv:ansible2.3]
+basepython = python2
+deps =
+    ansible>=2.3,<2.4
+    docker-py>=1.7.0
+    mitogen>=0.2.10rc1,<0.3
+install_command =
+    python -m pip --no-python-version-warning install {opts} {packages}
+commands =
+    ./setup.yml -i hosts.ini -l 'localhost,centos5' {posargs}
+
+[testenv:ansible2.10]
+basepython = python3
+deps =
+    ansible>=2.10,<2.11
+    docker>=1.8.0
+    mitogen>=0.3.0rc1,<0.4
+commands =
+    ./setup.yml -i hosts.ini  -l '!centos5' {posargs}

tests/testlib.py

@@ -450,7 +450,7 @@ class DockerizedSshDaemon(object):
         subprocess__check_output(args)
         self._get_container_port()
 
-    def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian')):
+    def __init__(self, mitogen_test_distro=os.environ.get('MITOGEN_TEST_DISTRO', 'debian9')):
         if '-'  in mitogen_test_distro:
             distro, _py3 = mitogen_test_distro.split('-')
         else:
@@ -462,7 +462,7 @@ class DockerizedSshDaemon(object):
         else:
             self.python_path = '/usr/bin/python'
 
-        self.image = 'mitogen/%s-test' % (distro,)
+        self.image = 'public.ecr.aws/n5z0e8q9/%s-test' % (distro,)
 
         # 22/tcp -> 0.0.0.0:32771
         self.PORT_RE = re.compile(r'([^/]+)/([^ ]+) -> ([^:]+):(.*)')

tox.ini

@@ -9,18 +9,29 @@
 
 # Last version to support each python version
 #
-#            tox      vir'env  pip      ansible  coverage
+#            tox      vir'env  pip      ansible  ansible  coverage
-# ========== ======== ======== ======== ======== ========
+#                                       control  target
-# python2.4  1.4      1.8      1.1      ???
+# ========== ======== ======== ======== ======== ======== ========
+# python2.4  1.4      1.8      1.1               2.3?
 # python2.5  1.6.1    1.9.1    1.3.1    ???
 # python2.6  2.9.1    15.2.0   9.0.3    2.6.20            4.5.4
+# python2.7                    20.3     2.10
+# python3.5                             2.10
+# python3.6                             2.10
+# python3.7                             2.10
+
+# pip --no-python-version-warning
+# pip --disable-pip-version-check
 
 [tox]
 envlist =
     init,
-    py{27,36,39}-mode_ansible-ansible2.10,
+    py{27,35,39}-mode_ansible-distros_centos,
-    py{27,36,39}-mode_mitogen,
+    py{27,35,39}-mode_ansible-distros_debian,
-    py{27,36,39}-mode_mitogen-distro_centos7,
+    py{27,35,39}-mode_ansible-distros_ubuntu,
+    py{27,35,39}-mode_mitogen-distro_centos{6,7,8},
+    py{27,35,39}-mode_mitogen-distro_debian{9,10,11},
+    py{27,35,39}-mode_mitogen-distro_ubuntu{1604,1804,2004},
     report,
 requires =
     tox-factor
@@ -29,24 +40,29 @@ requires =
 basepython =
     py26: python2.6
     py27: python2.7
+    py35: python3.5
     py36: python3.6
     py37: python3.7
     py38: python3.8
     py39: python3.9
+install_command =
+    python -m pip --no-python-version-warning install {opts} {packages}
 commands_pre =
     mode_ansible: {toxinidir}/.ci/ansible_install.py
     mode_debops_common: {toxinidir}/.ci/debops_common_install.py
     mode_mitogen: {toxinidir}/.ci/mitogen_install.py
 commands =
-    mode_ansible: {toxinidir}/.ci/ansible_tests.py \
+    mode_ansible: {toxinidir}/.ci/ansible_tests.py
-                        --skip-tags requires_local_sudo
     mode_debops_common: {toxinidir}/.ci/debops_common_tests.py
     mode_mitogen: {toxinidir}/.ci/mitogen_tests.py
 passenv =
+    ANSIBLE_*
     HOME
 setenv =
+    ANSIBLE_SKIP_TAGS = requires_local_sudo
     NOCOVERAGE_ERASE = 1
     NOCOVERAGE_REPORT = 1
+    VER=2.10.5
     ansible2.3: VER=2.3.3.0
     ansible2.4: VER=2.4.6.0
     ansible2.8: VER=2.8.3
@@ -55,10 +71,26 @@ setenv =
     distro_centos5: DISTRO=centos5
     distro_centos6: DISTRO=centos6
     distro_centos7: DISTRO=centos7
-    distro_debian: DISTRO=debian
+    distro_centos8: DISTRO=centos8
-    distro_debianpy3: DISTRO=debian-py3
+    distro_debian9: DISTRO=debian9
+    distro_debian10: DISTRO=debian10
+    distro_debian11: DISTRO=debian11
+    distro_ubuntu1604: DISTRO=ubuntu1604
+    distro_ubuntu1804: DISTRO=ubuntu1804
+    distro_ubuntu2004: DISTRO=ubuntu2004
+    distros_centos: DISTROS=centos6 centos7 centos8
     distros_centos5: DISTROS=centos5
-    distros_debian: DISTROS=debian
+    distros_centos6: DISTROS=centos6
+    distros_centos7: DISTROS=centos7
+    distros_centos8: DISTROS=centos8
+    distros_debian: DISTROS=debian9 debian10 debian11
+    distros_debian9: DISTROS=debian9
+    distros_debian10: DISTROS=debian10
+    distros_debian11: DISTROS=debian11
+    distros_ubuntu: DISTROS=ubuntu1604 ubuntu1804 ubuntu2004
+    distros_ubuntu1604: DISTROS=ubuntu1604
+    distros_ubuntu1804: DISTROS=ubuntu1804
+    distros_ubuntu2004: DISTROS=ubuntu2004
     mode_ansible: MODE=ansible
     mode_debops_common: MODE=debops_common
     mode_mitogen: MODE=mitogen