Merge pull request #1252 from moreati/issue1118-update-containers

CI: Mark and use 2021 test containers

.ci/ci_lib.py

@@ -39,7 +39,7 @@ DISTRO_SPECS = os.environ.get(
 IMAGE_PREP_DIR = os.path.join(GIT_ROOT, 'tests/image_prep')
 IMAGE_TEMPLATE = os.environ.get(
     'MITOGEN_TEST_IMAGE_TEMPLATE',
-    'public.ecr.aws/n5z0e8q9/%(distro)s-test',
+    'ghcr.io/mitogen-hq/%(distro)s-test:2021',
 )
 TESTS_SSH_PRIVATE_KEY_FILE = os.path.join(GIT_ROOT, 'tests/data/docker/mitogen__has_sudo_pubkey.key')
 
@@ -227,7 +227,7 @@ def container_specs(
     [{'distro': 'debian11',
       'family': 'debian',
       'hostname': 'localhost',
-      'image': 'public.ecr.aws/n5z0e8q9/debian11-test',
+      'image': 'ghcr.io/mitogen-hq/debian11-test:2021',
       'index': 1,
       'name': 'target-debian11-1',
       'port': 2201,
@@ -235,7 +235,7 @@ def container_specs(
      {'distro': 'centos6',
       'family': 'centos',
       'hostname': 'localhost',
-      'image': 'public.ecr.aws/n5z0e8q9/centos6-test',
+      'image': 'ghcr.io/mitogen-hq/centos6-test:2021',
       'index': 2,
       'name': 'target-centos6-2',
       'port': 2202,

.github/workflows/tests.yml

@@ -8,11 +8,6 @@ on:
     branches-ignore:
       - docs-master
 
-env:
-  #ANSIBLE_VERBOSITY: 3
-  #MITOGEN_LOG_LEVEL: DEBUG
-  MITOGEN_TEST_IMAGE_TEMPLATE: "ghcr.io/mitogen-hq/%(distro)s-test"
-
 # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners
 # https://github.com/actions/runner-images/blob/main/README.md#software-and-image-support
 jobs:

tests/ansible/hosts/group_vars/all.yml

@@ -27,4 +27,3 @@ become_unpriv_available: >-
   -}}
 
 pkg_mgr_python_interpreter: python
-pkg_repos_overrides: []

tests/ansible/hosts/group_vars/centos8.yml

@@ -1,7 +1,7 @@
 ---
 pkg_mgr_python_interpreter: /usr/libexec/platform-python
 
-pkg_repos_overrides:
+package_manager_repos:
   - dest: /etc/yum.repos.d/CentOS-Linux-AppStream.repo
     content: |
       [appstream]

tests/ansible/hosts/group_vars/debian11.yml

@@ -0,0 +1,5 @@
+package_manager_keys:
+  - src: debian-archive-bullseye-automatic.gpg  # Debian 11
+    dest: /etc/apt/trusted.gpg.d/debian-archive-bullseye-automatic.gpg
+  - src: debian-archive-bookworm-automatic.gpg  # Debian 12
+    dest: /etc/apt/trusted.gpg.d/debian-archive-bookworm-automatic.gpg

tests/ansible/hosts/group_vars/debian9.yml

@@ -1,4 +1,4 @@
-pkg_repos_overrides:
+package_manager_repos:
   - dest: /etc/apt/sources.list
     content: |
       deb http://archive.debian.org/debian stretch main contrib non-free

tests/ansible/regression/issue_776__load_plugins_called_twice.yml

@@ -10,7 +10,7 @@
   vars:
     ansible_python_interpreter: "{{ pkg_mgr_python_interpreter }}"
     package: rsync  # Chosen to exist in all tested distros/package managers
-  tasks:
+  pre_tasks:
     # The package management modules require using the same Python version
     # as the target's package manager libraries. This is sometimes in conflict
     # with Ansible requirements, e.g. Ansible 10 (ansible-core 2.17) does not
@@ -19,31 +19,13 @@
       when:
         - ansible_version.full is version('2.17', '>=', strict=True)
 
+  roles:
+    - role: package_manager
+
+  tasks:
     - name: Gather facts manually
       setup:
 
-    - name: Switch to archived package repositories
-      copy:
-        dest: "{{ item.dest }}"
-        content: "{{ item.content }}"
-        mode: u=rw,go=r
-      loop: "{{ pkg_repos_overrides }}"
-      loop_control:
-        label: "{{ item.dest }}"
-
-    - name: Add signing keys
-      copy:
-        src: "{{ item.src }}"
-        dest: "/etc/apt/trusted.gpg.d/{{ item.src | basename }}"
-        mode: u=rw,go=r
-      loop:
-        - src: debian-archive-bullseye-automatic.gpg  # Debian 11
-        - src: debian-archive-bookworm-automatic.gpg  # Debian 12
-      when:
-        # Ideally this would check for Debian 11, but distribution_major_version
-        # is unpopulated sometimes.
-        - ansible_facts.distribution == "Debian"
-
     - name: Update package index
       apt:
         update_cache: true

tests/image_prep/_container_setup.yml

@@ -1,5 +1,5 @@
-
+- name: Bootstrap containers
-- hosts: all
+  hosts: all
   strategy: linear
   gather_facts: false
   tasks:
@@ -15,7 +15,8 @@
         fi
       when: bootstrap_packages | length
 
-- hosts: all
+- name: Setup containers
+  hosts: all
   strategy: mitogen_free
   # Resource limitation, my laptop freezes doing every container concurrently
   serial: 4
@@ -30,7 +31,9 @@
         - ansible_facts.virtualization_type != "docker"
 
   roles:
+    - role: package_manager
     - role: sshd
+    - role: sshd_container
 
   tasks:
     - name: Ensure requisite apt packages are installed

tests/image_prep/_user_accounts.yml

@@ -3,10 +3,8 @@
 #
 # WARNING: this creates non-privilged accounts with pre-set passwords!
 #
-
+- name: Mitogen test users and groups
-- import_playbook: ../ansible/setup/report_controller.yml
+  hosts: all
-
-- hosts: all
   gather_facts: true
   strategy: mitogen_free
   become: true
@@ -64,7 +62,7 @@
     - name: Create Mitogen test groups
       group:
         name: "{{ item.name }}"
-      loop: "{{ mitogen_test_groups }}"
+      with_items: "{{ mitogen_test_groups }}"
 
     - name: Create user accounts
       vars:
@@ -113,7 +111,11 @@
         state: restarted
 
     - name: Readonly homedir for one account
-      shell: "chown -R root: ~mitogen__readonly_homedir"
+      file:
+        path: ~mitogen__readonly_homedir
+        owner: root
+        recurse: true
+        state: directory
 
     - name: Slow bash profile for one account
       copy:
@@ -160,7 +162,6 @@
           Defaults>mitogen__pw_required targetpw
           Defaults>mitogen__require_tty requiretty
           Defaults>mitogen__require_tty_pw_required requiretty,targetpw
-        prepend_newline: true
         validate: '/usr/sbin/visudo -cf %s'
 
     - name: Configure sudoers users
@@ -176,7 +177,6 @@
           {% for runas_user in normal_users %}
           {{ lookup('pipe', 'whoami') }} ALL = ({{ runas_user.name }}:ALL) NOPASSWD:ALL
           {% endfor %}
-        prepend_newline: true
         validate: '/usr/sbin/visudo -cf %s'
       when:
         - ansible_virtualization_type != "docker"

tests/image_prep/macos_localhost.yml

@@ -5,3 +5,4 @@
   become: true
   roles:
     - role: sshd
+    - role: sshd_macos

tests/image_prep/roles/package_manager/defaults/main.yml

@@ -0,0 +1,2 @@
+package_manager_keys: []
+package_manager_repos: []

tests/image_prep/roles/package_manager/tasks/main.yml

@@ -0,0 +1,13 @@
+- name: Add signing keys
+  copy:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    mode: u=rw,go=r
+  with_items: "{{ package_manager_keys }}"
+
+- name: Configure package repositories
+  copy:
+    dest: "{{ item.dest }}"
+    content: "{{ item.content }}"
+    mode: u=rw,go=r
+  with_items: "{{ package_manager_repos }}"

tests/image_prep/roles/sshd/tasks/main.yml

@@ -9,23 +9,12 @@
     path: "{{ sshd_config_file }}"
     line: "{{ item.line }}"
     regexp: "{{ item.regexp }}"
-  loop:
+  with_items:
     - line:     Banner /etc/ssh/banner.txt
       regexp:   '^#? *Banner.*'
     - line:     MaxAuthTries {{ sshd_config__max_auth_tries }}
       regexp:   '^#? *MaxAuthTries.*'
     - line:     PermitRootLogin yes
       regexp:   '.*PermitRootLogin.*'
-  loop_control:
+  notify:
-    label: "{{ item.line }}"
+    - Restart sshd  # Handler in platform specific role
-  register: configure_sshd_result
-
-- name: Restart sshd
-  shell: |
-    launchctl unload /System/Library/LaunchDaemons/ssh.plist
-    wait 5
-    launchctl load -w /System/Library/LaunchDaemons/ssh.plist
-  changed_when: true
-  when:
-    - ansible_facts.distribution == "MacOSX"
-    - configure_sshd_result is changed

tests/image_prep/roles/sshd_container/handlers/main.yml

@@ -0,0 +1,2 @@
+- name: Restart sshd
+  meta: noop

tests/image_prep/roles/sshd_macos/handlers/main.yml

@@ -0,0 +1,6 @@
+- name: Restart sshd
+  shell: |
+    launchctl unload /System/Library/LaunchDaemons/ssh.plist
+    wait 5
+    launchctl load -w /System/Library/LaunchDaemons/ssh.plist
+  changed_when: true

tests/testlib.py

@@ -57,7 +57,7 @@ DISTRO_SPECS = os.environ.get(
 )
 IMAGE_TEMPLATE = os.environ.get(
     'MITOGEN_TEST_IMAGE_TEMPLATE',
-    'public.ecr.aws/n5z0e8q9/%(distro)s-test',
+    'ghcr.io/mitogen-hq/%(distro)s-test:2021',
 )
 
 TESTS_DIR =                     os.path.join(os.path.dirname(__file__))

tox.ini

@@ -100,6 +100,7 @@ passenv =
     HOME
     MITOGEN_*
 setenv =
+    ANSIBLE_ROLES_PATH = {toxinidir}/tests/image_prep/roles
     # See also azure-pipelines.yml
     ANSIBLE_STRATEGY = mitogen_linear
     NOCOVERAGE_ERASE = 1