From 67ececc804bcd207d9429fe9cc0e32ba74bf6ecc Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Fri, 7 Feb 2025 15:42:11 +0000 Subject: [PATCH 1/6] ci: Use GitHub Container Registery images tagged 2021 Previously (and implicitly) used "latest". The tag 2021 is new today, the image contents have not changed since they were generated in 2021. They have moved container registry twice since 2021 - #791 Docker -> Amazon Elastic Container Registry (public.ecr.aws/n5z0e8q) - #1128 Amazon ECR -> GitHub Container Registry (ghcr.io/mitogen-hq) This commit also removes the last references to ECR. --- .ci/ci_lib.py | 6 +++--- .github/workflows/tests.yml | 5 ----- tests/testlib.py | 2 +- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.ci/ci_lib.py b/.ci/ci_lib.py index 9f333dde..96c85811 100644 --- a/.ci/ci_lib.py +++ b/.ci/ci_lib.py @@ -39,7 +39,7 @@ DISTRO_SPECS = os.environ.get( IMAGE_PREP_DIR = os.path.join(GIT_ROOT, 'tests/image_prep') IMAGE_TEMPLATE = os.environ.get( 'MITOGEN_TEST_IMAGE_TEMPLATE', - 'public.ecr.aws/n5z0e8q9/%(distro)s-test', + 'ghcr.io/mitogen-hq/%(distro)s-test:2021', ) TESTS_SSH_PRIVATE_KEY_FILE = os.path.join(GIT_ROOT, 'tests/data/docker/mitogen__has_sudo_pubkey.key') @@ -227,7 +227,7 @@ def container_specs( [{'distro': 'debian11', 'family': 'debian', 'hostname': 'localhost', - 'image': 'public.ecr.aws/n5z0e8q9/debian11-test', + 'image': 'ghcr.io/mitogen-hq/debian11-test:2021', 'index': 1, 'name': 'target-debian11-1', 'port': 2201, @@ -235,7 +235,7 @@ def container_specs( {'distro': 'centos6', 'family': 'centos', 'hostname': 'localhost', - 'image': 'public.ecr.aws/n5z0e8q9/centos6-test', + 'image': 'ghcr.io/mitogen-hq/centos6-test:2021', 'index': 2, 'name': 'target-centos6-2', 'port': 2202, diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bbbc33e8..393a52c7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,11 +8,6 @@ on: branches-ignore: - docs-master -env: - #ANSIBLE_VERBOSITY: 3 - #MITOGEN_LOG_LEVEL: DEBUG - MITOGEN_TEST_IMAGE_TEMPLATE: "ghcr.io/mitogen-hq/%(distro)s-test" - # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners # https://github.com/actions/runner-images/blob/main/README.md#software-and-image-support jobs: diff --git a/tests/testlib.py b/tests/testlib.py index 05779dc0..e4fa25b8 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -57,7 +57,7 @@ DISTRO_SPECS = os.environ.get( ) IMAGE_TEMPLATE = os.environ.get( 'MITOGEN_TEST_IMAGE_TEMPLATE', - 'public.ecr.aws/n5z0e8q9/%(distro)s-test', + 'ghcr.io/mitogen-hq/%(distro)s-test:2021', ) TESTS_DIR = os.path.join(os.path.dirname(__file__)) From 8e58c4a7592c640fb5e5e80c5878c1a79eb89f55 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 26 Feb 2025 11:05:54 +0000 Subject: [PATCH 2/6] ci: Decouple image prep from Ansible controller reporting This will allow image preparation using Ansible versions that predate import_playbook. --- tests/image_prep/_user_accounts.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 10834efc..01747876 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -3,9 +3,6 @@ # # WARNING: this creates non-privilged accounts with pre-set passwords! # - -- import_playbook: ../ansible/setup/report_controller.yml - - hosts: all gather_facts: true strategy: mitogen_free From 20e23b5bd9621f7c5d45e60c38c91dff2bd54150 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 26 Feb 2025 11:11:08 +0000 Subject: [PATCH 3/6] ci: Name all image prep plays --- tests/image_prep/_container_setup.yml | 7 ++++--- tests/image_prep/_user_accounts.yml | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/image_prep/_container_setup.yml b/tests/image_prep/_container_setup.yml index 2972adda..7b466fc0 100644 --- a/tests/image_prep/_container_setup.yml +++ b/tests/image_prep/_container_setup.yml @@ -1,5 +1,5 @@ - -- hosts: all +- name: Bootstrap containers + hosts: all strategy: linear gather_facts: false tasks: @@ -15,7 +15,8 @@ fi when: bootstrap_packages | length -- hosts: all +- name: Setup containers + hosts: all strategy: mitogen_free # Resource limitation, my laptop freezes doing every container concurrently serial: 4 diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 01747876..14ac87f0 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -3,7 +3,8 @@ # # WARNING: this creates non-privilged accounts with pre-set passwords! # -- hosts: all +- name: Mitogen test users and groups + hosts: all gather_facts: true strategy: mitogen_free become: true From f2e0e552accafd924f71c95005f666e37a018386 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 26 Feb 2025 11:20:35 +0000 Subject: [PATCH 4/6] ci: Fix sshd configuration during image prep This will allow image preparation using Ansible versions that - predate ansible_facts.* - predate loop keyword - predate collections --- tests/image_prep/_container_setup.yml | 1 + tests/image_prep/_user_accounts.yml | 4 +--- tests/image_prep/macos_localhost.yml | 1 + tests/image_prep/roles/sshd/tasks/main.yml | 17 +++-------------- .../roles/sshd_container/handlers/main.yml | 2 ++ .../roles/sshd_macos/handlers/main.yml | 6 ++++++ 6 files changed, 14 insertions(+), 17 deletions(-) create mode 100644 tests/image_prep/roles/sshd_container/handlers/main.yml create mode 100644 tests/image_prep/roles/sshd_macos/handlers/main.yml diff --git a/tests/image_prep/_container_setup.yml b/tests/image_prep/_container_setup.yml index 7b466fc0..06cfc967 100644 --- a/tests/image_prep/_container_setup.yml +++ b/tests/image_prep/_container_setup.yml @@ -32,6 +32,7 @@ roles: - role: sshd + - role: sshd_container tasks: - name: Ensure requisite apt packages are installed diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 14ac87f0..3af21db7 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -62,7 +62,7 @@ - name: Create Mitogen test groups group: name: "{{ item.name }}" - loop: "{{ mitogen_test_groups }}" + with_items: "{{ mitogen_test_groups }}" - name: Create user accounts vars: @@ -158,7 +158,6 @@ Defaults>mitogen__pw_required targetpw Defaults>mitogen__require_tty requiretty Defaults>mitogen__require_tty_pw_required requiretty,targetpw - prepend_newline: true validate: '/usr/sbin/visudo -cf %s' - name: Configure sudoers users @@ -174,7 +173,6 @@ {% for runas_user in normal_users %} {{ lookup('pipe', 'whoami') }} ALL = ({{ runas_user.name }}:ALL) NOPASSWD:ALL {% endfor %} - prepend_newline: true validate: '/usr/sbin/visudo -cf %s' when: - ansible_virtualization_type != "docker" diff --git a/tests/image_prep/macos_localhost.yml b/tests/image_prep/macos_localhost.yml index c046a2bc..b93bc780 100644 --- a/tests/image_prep/macos_localhost.yml +++ b/tests/image_prep/macos_localhost.yml @@ -5,3 +5,4 @@ become: true roles: - role: sshd + - role: sshd_macos diff --git a/tests/image_prep/roles/sshd/tasks/main.yml b/tests/image_prep/roles/sshd/tasks/main.yml index 837c7d15..8f3e7e03 100644 --- a/tests/image_prep/roles/sshd/tasks/main.yml +++ b/tests/image_prep/roles/sshd/tasks/main.yml @@ -9,23 +9,12 @@ path: "{{ sshd_config_file }}" line: "{{ item.line }}" regexp: "{{ item.regexp }}" - loop: + with_items: - line: Banner /etc/ssh/banner.txt regexp: '^#? *Banner.*' - line: MaxAuthTries {{ sshd_config__max_auth_tries }} regexp: '^#? *MaxAuthTries.*' - line: PermitRootLogin yes regexp: '.*PermitRootLogin.*' - loop_control: - label: "{{ item.line }}" - register: configure_sshd_result - -- name: Restart sshd - shell: | - launchctl unload /System/Library/LaunchDaemons/ssh.plist - wait 5 - launchctl load -w /System/Library/LaunchDaemons/ssh.plist - changed_when: true - when: - - ansible_facts.distribution == "MacOSX" - - configure_sshd_result is changed + notify: + - Restart sshd # Handler in platform specific role diff --git a/tests/image_prep/roles/sshd_container/handlers/main.yml b/tests/image_prep/roles/sshd_container/handlers/main.yml new file mode 100644 index 00000000..cc7b9166 --- /dev/null +++ b/tests/image_prep/roles/sshd_container/handlers/main.yml @@ -0,0 +1,2 @@ +- name: Restart sshd + meta: noop diff --git a/tests/image_prep/roles/sshd_macos/handlers/main.yml b/tests/image_prep/roles/sshd_macos/handlers/main.yml new file mode 100644 index 00000000..6d4a7e21 --- /dev/null +++ b/tests/image_prep/roles/sshd_macos/handlers/main.yml @@ -0,0 +1,6 @@ +- name: Restart sshd + shell: | + launchctl unload /System/Library/LaunchDaemons/ssh.plist + wait 5 + launchctl load -w /System/Library/LaunchDaemons/ssh.plist + changed_when: true From 1e12edbf9517402fef40c315a4d641b1603828a8 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 26 Feb 2025 11:24:40 +0000 Subject: [PATCH 5/6] ci: Use file module to set mitogen__readonly_homedir permissions --- tests/image_prep/_user_accounts.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/image_prep/_user_accounts.yml b/tests/image_prep/_user_accounts.yml index 3af21db7..bee89084 100644 --- a/tests/image_prep/_user_accounts.yml +++ b/tests/image_prep/_user_accounts.yml @@ -111,7 +111,11 @@ state: restarted - name: Readonly homedir for one account - shell: "chown -R root: ~mitogen__readonly_homedir" + file: + path: ~mitogen__readonly_homedir + owner: root + recurse: true + state: directory - name: Slow bash profile for one account copy: From fdbd1a8c9b73ada07f90cbb4d0637a35cf7134c2 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Wed, 26 Feb 2025 12:13:11 +0000 Subject: [PATCH 6/6] ci: Configure package managers using a role This allows code sharing between integration tests and test image prep. --- tests/ansible/hosts/group_vars/all.yml | 1 - tests/ansible/hosts/group_vars/centos8.yml | 2 +- tests/ansible/hosts/group_vars/debian11.yml | 5 ++++ tests/ansible/hosts/group_vars/debian9.yml | 2 +- .../issue_776__load_plugins_called_twice.yml | 28 ++++-------------- tests/image_prep/_container_setup.yml | 1 + .../roles/package_manager/defaults/main.yml | 2 ++ .../debian-archive-bookworm-automatic.gpg | Bin .../debian-archive-bullseye-automatic.gpg | Bin .../roles/package_manager/tasks/main.yml | 13 ++++++++ tox.ini | 1 + 11 files changed, 29 insertions(+), 26 deletions(-) create mode 100644 tests/ansible/hosts/group_vars/debian11.yml create mode 100644 tests/image_prep/roles/package_manager/defaults/main.yml rename tests/{ansible/regression => image_prep/roles/package_manager}/files/debian-archive-bookworm-automatic.gpg (100%) rename tests/{ansible/regression => image_prep/roles/package_manager}/files/debian-archive-bullseye-automatic.gpg (100%) create mode 100644 tests/image_prep/roles/package_manager/tasks/main.yml diff --git a/tests/ansible/hosts/group_vars/all.yml b/tests/ansible/hosts/group_vars/all.yml index 44e660fc..fa70b89d 100644 --- a/tests/ansible/hosts/group_vars/all.yml +++ b/tests/ansible/hosts/group_vars/all.yml @@ -27,4 +27,3 @@ become_unpriv_available: >- -}} pkg_mgr_python_interpreter: python -pkg_repos_overrides: [] diff --git a/tests/ansible/hosts/group_vars/centos8.yml b/tests/ansible/hosts/group_vars/centos8.yml index c90dd5f4..5ab4577d 100644 --- a/tests/ansible/hosts/group_vars/centos8.yml +++ b/tests/ansible/hosts/group_vars/centos8.yml @@ -1,7 +1,7 @@ --- pkg_mgr_python_interpreter: /usr/libexec/platform-python -pkg_repos_overrides: +package_manager_repos: - dest: /etc/yum.repos.d/CentOS-Linux-AppStream.repo content: | [appstream] diff --git a/tests/ansible/hosts/group_vars/debian11.yml b/tests/ansible/hosts/group_vars/debian11.yml new file mode 100644 index 00000000..9f62f43c --- /dev/null +++ b/tests/ansible/hosts/group_vars/debian11.yml @@ -0,0 +1,5 @@ +package_manager_keys: + - src: debian-archive-bullseye-automatic.gpg # Debian 11 + dest: /etc/apt/trusted.gpg.d/debian-archive-bullseye-automatic.gpg + - src: debian-archive-bookworm-automatic.gpg # Debian 12 + dest: /etc/apt/trusted.gpg.d/debian-archive-bookworm-automatic.gpg diff --git a/tests/ansible/hosts/group_vars/debian9.yml b/tests/ansible/hosts/group_vars/debian9.yml index 4b180b13..e08b1ed2 100644 --- a/tests/ansible/hosts/group_vars/debian9.yml +++ b/tests/ansible/hosts/group_vars/debian9.yml @@ -1,4 +1,4 @@ -pkg_repos_overrides: +package_manager_repos: - dest: /etc/apt/sources.list content: | deb http://archive.debian.org/debian stretch main contrib non-free diff --git a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml index ef573276..d482c41c 100755 --- a/tests/ansible/regression/issue_776__load_plugins_called_twice.yml +++ b/tests/ansible/regression/issue_776__load_plugins_called_twice.yml @@ -10,7 +10,7 @@ vars: ansible_python_interpreter: "{{ pkg_mgr_python_interpreter }}" package: rsync # Chosen to exist in all tested distros/package managers - tasks: + pre_tasks: # The package management modules require using the same Python version # as the target's package manager libraries. This is sometimes in conflict # with Ansible requirements, e.g. Ansible 10 (ansible-core 2.17) does not @@ -19,31 +19,13 @@ when: - ansible_version.full is version('2.17', '>=', strict=True) + roles: + - role: package_manager + + tasks: - name: Gather facts manually setup: - - name: Switch to archived package repositories - copy: - dest: "{{ item.dest }}" - content: "{{ item.content }}" - mode: u=rw,go=r - loop: "{{ pkg_repos_overrides }}" - loop_control: - label: "{{ item.dest }}" - - - name: Add signing keys - copy: - src: "{{ item.src }}" - dest: "/etc/apt/trusted.gpg.d/{{ item.src | basename }}" - mode: u=rw,go=r - loop: - - src: debian-archive-bullseye-automatic.gpg # Debian 11 - - src: debian-archive-bookworm-automatic.gpg # Debian 12 - when: - # Ideally this would check for Debian 11, but distribution_major_version - # is unpopulated sometimes. - - ansible_facts.distribution == "Debian" - - name: Update package index apt: update_cache: true diff --git a/tests/image_prep/_container_setup.yml b/tests/image_prep/_container_setup.yml index 06cfc967..4aa3b46d 100644 --- a/tests/image_prep/_container_setup.yml +++ b/tests/image_prep/_container_setup.yml @@ -31,6 +31,7 @@ - ansible_facts.virtualization_type != "docker" roles: + - role: package_manager - role: sshd - role: sshd_container diff --git a/tests/image_prep/roles/package_manager/defaults/main.yml b/tests/image_prep/roles/package_manager/defaults/main.yml new file mode 100644 index 00000000..c384b19a --- /dev/null +++ b/tests/image_prep/roles/package_manager/defaults/main.yml @@ -0,0 +1,2 @@ +package_manager_keys: [] +package_manager_repos: [] diff --git a/tests/ansible/regression/files/debian-archive-bookworm-automatic.gpg b/tests/image_prep/roles/package_manager/files/debian-archive-bookworm-automatic.gpg similarity index 100% rename from tests/ansible/regression/files/debian-archive-bookworm-automatic.gpg rename to tests/image_prep/roles/package_manager/files/debian-archive-bookworm-automatic.gpg diff --git a/tests/ansible/regression/files/debian-archive-bullseye-automatic.gpg b/tests/image_prep/roles/package_manager/files/debian-archive-bullseye-automatic.gpg similarity index 100% rename from tests/ansible/regression/files/debian-archive-bullseye-automatic.gpg rename to tests/image_prep/roles/package_manager/files/debian-archive-bullseye-automatic.gpg diff --git a/tests/image_prep/roles/package_manager/tasks/main.yml b/tests/image_prep/roles/package_manager/tasks/main.yml new file mode 100644 index 00000000..f0bc8ad8 --- /dev/null +++ b/tests/image_prep/roles/package_manager/tasks/main.yml @@ -0,0 +1,13 @@ +- name: Add signing keys + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: u=rw,go=r + with_items: "{{ package_manager_keys }}" + +- name: Configure package repositories + copy: + dest: "{{ item.dest }}" + content: "{{ item.content }}" + mode: u=rw,go=r + with_items: "{{ package_manager_repos }}" diff --git a/tox.ini b/tox.ini index bd2d65e9..ddb8c88d 100644 --- a/tox.ini +++ b/tox.ini @@ -100,6 +100,7 @@ passenv = HOME MITOGEN_* setenv = + ANSIBLE_ROLES_PATH = {toxinidir}/tests/image_prep/roles # See also azure-pipelines.yml ANSIBLE_STRATEGY = mitogen_linear NOCOVERAGE_ERASE = 1