From d2f6ea4179066e396433370d8c565ba0632e9199 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Tue, 28 Mar 2023 14:34:15 -0500 Subject: [PATCH] New deb822_repository module (#80018) Fixes #77073 --- changelogs/fragments/deb822-repository.yml | 2 + lib/ansible/modules/deb822_repository.py | 554 ++++++++++++++++++ .../targets/deb822_repository/aliases | 6 + .../targets/deb822_repository/meta/main.yml | 4 + .../deb822_repository/tasks/install.yml | 40 ++ .../targets/deb822_repository/tasks/main.yml | 19 + .../targets/deb822_repository/tasks/test.yml | 213 +++++++ .../targets/setup_deb_repo/tasks/main.yml | 1 + 8 files changed, 839 insertions(+) create mode 100644 changelogs/fragments/deb822-repository.yml create mode 100644 lib/ansible/modules/deb822_repository.py create mode 100644 test/integration/targets/deb822_repository/aliases create mode 100644 test/integration/targets/deb822_repository/meta/main.yml create mode 100644 test/integration/targets/deb822_repository/tasks/install.yml create mode 100644 test/integration/targets/deb822_repository/tasks/main.yml create mode 100644 test/integration/targets/deb822_repository/tasks/test.yml diff --git a/changelogs/fragments/deb822-repository.yml b/changelogs/fragments/deb822-repository.yml new file mode 100644 index 00000000000..cffb4ba5c51 --- /dev/null +++ b/changelogs/fragments/deb822-repository.yml @@ -0,0 +1,2 @@ +minor_changes: +- deb822_repository - Add new module for managing DEB822 formatted apt repositories diff --git a/lib/ansible/modules/deb822_repository.py b/lib/ansible/modules/deb822_repository.py new file mode 100644 index 00000000000..0c706ce06ec --- /dev/null +++ b/lib/ansible/modules/deb822_repository.py @@ -0,0 +1,554 @@ +# -*- coding: utf-8 -*- +# Copyright: Contributors to the Ansible project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + +DOCUMENTATION = ''' +author: 'Ansible Core Team (@ansible)' +short_description: 'Add and remove deb822 formatted repositories' +description: +- 'Add and remove deb822 formatted repositories in Debian based distributions' +module: deb822_repository +notes: +- This module will not automatically update caches, call the apt module based + on the changed state. +options: + allow_downgrade_to_insecure: + description: + - Allow downgrading a package that was previously authenticated but + is no longer authenticated + type: bool + allow_insecure: + description: + - Allow insecure repositories + type: bool + allow_weak: + description: + - Allow repositories signed with a key using a weak digest algorithm + type: bool + architectures: + description: + - 'Architectures to search within repository' + type: list + elements: str + by_hash: + description: + - Controls if APT should try to acquire indexes via a URI constructed + from a hashsum of the expected file instead of using the well-known + stable filename of the index. + type: bool + check_date: + description: + - Controls if APT should consider the machine's time correct and hence + perform time related checks, such as verifying that a Release file + is not from the future. + type: bool + check_valid_until: + description: + - Controls if APT should try to detect replay attacks. + type: bool + components: + description: + - Components specify different sections of one distribution version + present in a Suite. + type: list + elements: str + date_max_future: + description: + - Controls how far from the future a repository may be. + type: int + enabled: + description: + - Tells APT whether the source is enabled or not. + type: bool + inrelease_path: + description: + - Determines the path to the InRelease file, relative to the normal + position of an InRelease file. + type: str + languages: + description: + - Defines which languages information such as translated + package descriptions should be downloaded. + type: list + elements: str + name: + description: + - Name of the repo. Specifically used for C(X-Repolib-Name) and in + naming the repository and signing key files. + required: true + type: str + pdiffs: + description: + - Controls if APT should try to use PDiffs to update old indexes + instead of downloading the new indexes entirely + type: bool + signed_by: + description: + - Either a URL to a GPG key, absolute path to a keyring file, one or + more fingerprints of keys either in the C(trusted.gpg) keyring or in + the keyrings in the C(trusted.gpg.d/) directory, or an ASCII armored + GPG public key block. + type: str + suites: + description: + - >- + Suite can specify an exact path in relation to the URI(s) provided, + in which case the Components: must be omitted and suite must end + with a slash C( / ). Alternatively, it may take the form of a + distribution version (e.g. a version codename like disco or artful). + If the suite does not specify a path, at least one component must + be present. + type: list + elements: str + targets: + description: + - Defines which download targets apt will try to acquire from this + source. + type: list + elements: str + trusted: + description: + - Decides if a source is considered trusted or if warnings should be + raised before e.g. packages are installed from this source. + type: bool + types: + choices: + - deb + - deb-src + default: + - deb + type: list + elements: str + description: + - Which types of packages to look for from a given source; either + binary C(deb) or source code C(deb-src) + uris: + description: + - The URIs must specify the base of the Debian distribution archive, + from which APT finds the information it needs. + type: list + elements: str + mode: + description: + - The octal mode for newly created files in sources.list.d. + type: raw + default: '0644' + state: + description: + - A source string state. + type: str + choices: + - absent + - present + default: present +requirements: + - python3-debian / python-debian +version_added: '2.15' +''' + +EXAMPLES = ''' +- name: Add debian repo + deb822_repository: + name: debian + types: deb + uris: http://deb.debian.org/debian + suites: stretch + components: + - main + - contrib + - non-free + +- name: Add debian repo with key + deb822_repository: + name: debian + types: deb + uris: https://deb.debian.org + suites: stable + components: + - main + - contrib + - non-free + signed_by: |- + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mDMEYCQjIxYJKwYBBAHaRw8BAQdAD/P5Nvvnvk66SxBBHDbhRml9ORg1WV5CvzKY + CuMfoIS0BmFiY2RlZoiQBBMWCgA4FiEErCIG1VhKWMWo2yfAREZd5NfO31cFAmAk + IyMCGyMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AACgkQREZd5NfO31fbOwD6ArzS + dM0Dkd5h2Ujy1b6KcAaVW9FOa5UNfJ9FFBtjLQEBAJ7UyWD3dZzhvlaAwunsk7DG + 3bHcln8DMpIJVXht78sL + =IE0r + -----END PGP PUBLIC KEY BLOCK----- + +- name: Add repo using key from URL + deb822_repository: + name: example + types: deb + uris: https://download.example.com/linux/ubuntu + suites: '{{ ansible_distribution_release }}' + components: stable + architectures: amd64 + signed_by: https://download.example.com/linux/ubuntu/gpg +''' + +RETURN = ''' +repo: + description: A source string for the repository + returned: always + type: str + sample: | + X-Repolib-Name: debian + Types: deb + URIs: https://deb.debian.org + Suites: stable + Components: main contrib non-free + Signed-By: + -----BEGIN PGP PUBLIC KEY BLOCK----- + . + mDMEYCQjIxYJKwYBBAHaRw8BAQdAD/P5Nvvnvk66SxBBHDbhRml9ORg1WV5CvzKY + CuMfoIS0BmFiY2RlZoiQBBMWCgA4FiEErCIG1VhKWMWo2yfAREZd5NfO31cFAmAk + IyMCGyMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AACgkQREZd5NfO31fbOwD6ArzS + dM0Dkd5h2Ujy1b6KcAaVW9FOa5UNfJ9FFBtjLQEBAJ7UyWD3dZzhvlaAwunsk7DG + 3bHcln8DMpIJVXht78sL + =IE0r + -----END PGP PUBLIC KEY BLOCK----- + +dest: + description: Path to the repository file + returned: always + type: str + sample: /etc/apt/sources.list.d/focal-archive.sources + +key_filename: + description: Path to the signed_by key file + returned: always + type: str + sample: /etc/apt/keyrings/debian.gpg +''' + +import os +import re +import tempfile +import textwrap +import traceback + +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.basic import missing_required_lib +from ansible.module_utils.common.collections import is_sequence +from ansible.module_utils.common.text.converters import to_bytes +from ansible.module_utils.common.text.converters import to_native +from ansible.module_utils.six import raise_from # type: ignore[attr-defined] +from ansible.module_utils.urls import generic_urlparse +from ansible.module_utils.urls import open_url +from ansible.module_utils.urls import urlparse + +HAS_DEBIAN = True +DEBIAN_IMP_ERR = None +try: + from debian.deb822 import Deb822 # type: ignore[import] +except ImportError: + HAS_DEBIAN = False + DEBIAN_IMP_ERR = traceback.format_exc() + +KEYRINGS_DIR = '/etc/apt/keyrings' + + +def ensure_keyrings_dir(module): + changed = False + if not os.path.isdir(KEYRINGS_DIR): + if not module.check_mode: + os.mkdir(KEYRINGS_DIR, 0o755) + changed |= True + + changed |= module.set_fs_attributes_if_different( + { + 'path': KEYRINGS_DIR, + 'secontext': [None, None, None], + 'owner': 'root', + 'group': 'root', + 'mode': '0755', + 'attributes': None, + }, + changed, + ) + + return changed + + +def make_signed_by_filename(slug, ext): + return os.path.join(KEYRINGS_DIR, '%s.%s' % (slug, ext)) + + +def make_sources_filename(slug): + return os.path.join( + '/etc/apt/sources.list.d', + '%s.sources' % slug + ) + + +def format_bool(v): + return 'yes' if v else 'no' + + +def format_list(v): + return ' '.join(v) + + +def format_multiline(v): + return '\n' + textwrap.indent( + '\n'.join(line.strip() or '.' for line in v.strip().splitlines()), + ' ' + ) + + +def format_field_name(v): + if v == 'name': + return 'X-Repolib-Name' + elif v == 'uris': + return 'URIs' + return v.replace('_', '-').title() + + +def is_armored(b_data): + return b'-----BEGIN PGP PUBLIC KEY BLOCK-----' in b_data + + +def write_signed_by_key(module, v, slug): + changed = False + if os.path.isfile(v): + return changed, v, None + + b_data = None + + parts = generic_urlparse(urlparse(v)) + if parts.scheme: + try: + r = open_url(v) + except Exception as exc: + raise_from(RuntimeError(to_native(exc)), exc) + else: + b_data = r.read() + else: + # Not a file, nor a URL, just pass it through + return changed, None, v + + if not b_data: + return changed, v, None + + tmpfd, tmpfile = tempfile.mkstemp(dir=module.tmpdir) + with os.fdopen(tmpfd, 'wb') as f: + f.write(b_data) + + ext = 'asc' if is_armored(b_data) else 'gpg' + filename = make_signed_by_filename(slug, ext) + + src_chksum = module.sha256(tmpfile) + dest_chksum = module.sha256(filename) + + if src_chksum != dest_chksum: + changed |= ensure_keyrings_dir(module) + if not module.check_mode: + module.atomic_move(tmpfile, filename) + changed |= True + + changed |= module.set_mode_if_different(filename, 0o0644, False) + + return changed, filename, None + + +def main(): + module = AnsibleModule( + argument_spec={ + 'allow_downgrade_to_insecure': { + 'type': 'bool', + }, + 'allow_insecure': { + 'type': 'bool', + }, + 'allow_weak': { + 'type': 'bool', + }, + 'architectures': { + 'elements': 'str', + 'type': 'list', + }, + 'by_hash': { + 'type': 'bool', + }, + 'check_date': { + 'type': 'bool', + }, + 'check_valid_until': { + 'type': 'bool', + }, + 'components': { + 'elements': 'str', + 'type': 'list', + }, + 'date_max_future': { + 'type': 'int', + }, + 'enabled': { + 'type': 'bool', + }, + 'inrelease_path': { + 'type': 'str', + }, + 'languages': { + 'elements': 'str', + 'type': 'list', + }, + 'name': { + 'type': 'str', + 'required': True, + }, + 'pdiffs': { + 'type': 'bool', + }, + 'signed_by': { + 'type': 'str', + }, + 'suites': { + 'elements': 'str', + 'type': 'list', + }, + 'targets': { + 'elements': 'str', + 'type': 'list', + }, + 'trusted': { + 'type': 'bool', + }, + 'types': { + 'choices': [ + 'deb', + 'deb-src', + ], + 'elements': 'str', + 'type': 'list', + 'default': [ + 'deb', + ] + }, + 'uris': { + 'elements': 'str', + 'type': 'list', + }, + # non-deb822 args + 'mode': { + 'type': 'raw', + 'default': '0644', + }, + 'state': { + 'type': 'str', + 'choices': [ + 'present', + 'absent', + ], + 'default': 'present', + }, + }, + supports_check_mode=True, + ) + + if not HAS_DEBIAN: + module.fail_json(msg=missing_required_lib("python3-debian"), + exception=DEBIAN_IMP_ERR) + + check_mode = module.check_mode + + changed = False + + # Make a copy, so we don't mutate module.params to avoid future issues + params = module.params.copy() + + # popped non-deb822 args + mode = params.pop('mode') + state = params.pop('state') + + name = params['name'] + slug = re.sub( + r'[^a-z0-9-]+', + '', + re.sub( + r'[_\s]+', + '-', + name.lower(), + ), + ) + sources_filename = make_sources_filename(slug) + + if state == 'absent': + if os.path.exists(sources_filename): + if not check_mode: + os.unlink(sources_filename) + changed |= True + for ext in ('asc', 'gpg'): + signed_by_filename = make_signed_by_filename(slug, ext) + if os.path.exists(signed_by_filename): + if not check_mode: + os.unlink(signed_by_filename) + changed = True + module.exit_json( + repo=None, + changed=changed, + dest=sources_filename, + key_filename=signed_by_filename, + ) + + deb822 = Deb822() + signed_by_filename = None + for key, value in params.items(): + if value is None: + continue + + if isinstance(value, bool): + value = format_bool(value) + elif isinstance(value, int): + value = to_native(value) + elif is_sequence(value): + value = format_list(value) + elif key == 'signed_by': + try: + key_changed, signed_by_filename, signed_by_data = write_signed_by_key(module, value, slug) + value = signed_by_filename or signed_by_data + changed |= key_changed + except RuntimeError as exc: + module.fail_json( + msg='Could not fetch signed_by key: %s' % to_native(exc) + ) + + if value.count('\n') > 0: + value = format_multiline(value) + + deb822[format_field_name(key)] = value + + repo = deb822.dump() + tmpfd, tmpfile = tempfile.mkstemp(dir=module.tmpdir) + with os.fdopen(tmpfd, 'wb') as f: + f.write(to_bytes(repo)) + + sources_filename = make_sources_filename(slug) + + src_chksum = module.sha256(tmpfile) + dest_chksum = module.sha256(sources_filename) + + if src_chksum != dest_chksum: + if not check_mode: + module.atomic_move(tmpfile, sources_filename) + changed |= True + + changed |= module.set_mode_if_different(sources_filename, mode, False) + + module.exit_json( + repo=repo, + changed=changed, + dest=sources_filename, + key_filename=signed_by_filename, + ) + + +if __name__ == '__main__': + main() diff --git a/test/integration/targets/deb822_repository/aliases b/test/integration/targets/deb822_repository/aliases new file mode 100644 index 00000000000..34e2b54058f --- /dev/null +++ b/test/integration/targets/deb822_repository/aliases @@ -0,0 +1,6 @@ +destructive +shippable/posix/group1 +skip/freebsd +skip/osx +skip/macos +skip/rhel diff --git a/test/integration/targets/deb822_repository/meta/main.yml b/test/integration/targets/deb822_repository/meta/main.yml new file mode 100644 index 00000000000..83e789ee787 --- /dev/null +++ b/test/integration/targets/deb822_repository/meta/main.yml @@ -0,0 +1,4 @@ +dependencies: + - prepare_tests + - role: setup_deb_repo + install_repo: false diff --git a/test/integration/targets/deb822_repository/tasks/install.yml b/test/integration/targets/deb822_repository/tasks/install.yml new file mode 100644 index 00000000000..a5dce4377e1 --- /dev/null +++ b/test/integration/targets/deb822_repository/tasks/install.yml @@ -0,0 +1,40 @@ +- name: Create repo to install from + deb822_repository: + name: ansible-test local + uris: file:{{ repodir }} + suites: + - stable + - testing + components: + - main + architectures: + - all + trusted: yes + register: deb822_install_repo + +- name: Update apt cache + apt: + update_cache: yes + when: deb822_install_repo is changed + +- block: + - name: Install package from local repo + apt: + name: foo=1.0.0 + register: deb822_install_pkg + always: + - name: Uninstall foo + apt: + name: foo + state: absent + when: deb822_install_pkg is changed + + - name: remove repo + deb822_repository: + name: ansible-test local + state: absent + +- assert: + that: + - deb822_install_repo is changed + - deb822_install_pkg is changed diff --git a/test/integration/targets/deb822_repository/tasks/main.yml b/test/integration/targets/deb822_repository/tasks/main.yml new file mode 100644 index 00000000000..561ef2a6f50 --- /dev/null +++ b/test/integration/targets/deb822_repository/tasks/main.yml @@ -0,0 +1,19 @@ +- meta: end_play + when: ansible_os_family != 'Debian' + +- block: + - name: install python3-debian + apt: + name: python3-debian + state: present + register: py3_deb_install + + - import_tasks: test.yml + + - import_tasks: install.yml + always: + - name: uninstall python3-debian + apt: + name: python3-debian + state: absent + when: py3_deb_install is changed diff --git a/test/integration/targets/deb822_repository/tasks/test.yml b/test/integration/targets/deb822_repository/tasks/test.yml new file mode 100644 index 00000000000..3231cbeb95a --- /dev/null +++ b/test/integration/targets/deb822_repository/tasks/test.yml @@ -0,0 +1,213 @@ +- name: Create deb822 repo - check_mode + deb822_repository: + name: ansible-test focal archive + uris: http://us.archive.ubuntu.com/ubuntu + suites: + - focal + - focal-updates + components: + - main + - restricted + register: deb822_check_mode_1 + check_mode: yes + +- name: Create deb822 repo + deb822_repository: + name: ansible-test focal archive + uris: http://us.archive.ubuntu.com/ubuntu + suites: + - focal + - focal-updates + components: + - main + - restricted + date_max_future: 10 + register: deb822_create_1 + +- name: Check file mode + stat: + path: /etc/apt/sources.list.d/ansible-test-focal-archive.sources + register: deb822_create_1_stat_1 + +- name: Create another deb822 repo + deb822_repository: + name: ansible-test focal security + uris: http://security.ubuntu.com/ubuntu + suites: + - focal-security + components: + - main + - restricted + register: deb822_create_2 + +- name: Create deb822 repo idempotency + deb822_repository: + name: ansible-test focal archive + uris: http://us.archive.ubuntu.com/ubuntu + suites: + - focal + - focal-updates + components: + - main + - restricted + date_max_future: 10 + register: deb822_create_1_idem + +- name: Create deb822 repo - check_mode + deb822_repository: + name: ansible-test focal archive + uris: http://us.archive.ubuntu.com/ubuntu + suites: + - focal + - focal-updates + components: + - main + - restricted + date_max_future: 10 + register: deb822_check_mode_2 + check_mode: yes + +- name: Change deb822 repo mode + deb822_repository: + name: ansible-test focal archive + uris: http://us.archive.ubuntu.com/ubuntu + suites: + - focal + - focal-updates + components: + - main + - restricted + date_max_future: 10 + mode: '0600' + register: deb822_create_1_mode + +- name: Check file mode + stat: + path: /etc/apt/sources.list.d/ansible-test-focal-archive.sources + register: deb822_create_1_stat_2 + +- assert: + that: + - deb822_check_mode_1 is changed + + - deb822_check_mode_2 is not changed + + - deb822_create_1 is changed + - deb822_create_1.dest == '/etc/apt/sources.list.d/ansible-test-focal-archive.sources' + - deb822_create_1.repo|trim == focal_archive_expected + + - deb822_create_1_idem is not changed + + - deb822_create_1_mode is changed + - deb822_create_1_stat_1.stat.mode == '0644' + - deb822_create_1_stat_2.stat.mode == '0600' + vars: + focal_archive_expected: |- + X-Repolib-Name: ansible-test focal archive + URIs: http://us.archive.ubuntu.com/ubuntu + Suites: focal focal-updates + Components: main restricted + Date-Max-Future: 10 + Types: deb + +- name: Remove repos + deb822_repository: + name: '{{ item }}' + state: absent + register: remove_repos_1 + loop: + - ansible-test focal archive + - ansible-test focal security + +- name: Check for repo files + stat: + path: /etc/apt/sources.list.d/ansible-test-{{ item }}.sources + register: remove_stats + loop: + - focal-archive + - focal-security + +- assert: + that: + - remove_repos_1 is changed + - remove_stats.results|map(attribute='stat')|selectattr('exists') == [] + +- name: Add repo with signed_by + deb822_repository: + name: ansible-test + types: deb + uris: https://deb.debian.org + suites: stable + components: + - main + - contrib + - non-free + signed_by: |- + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mDMEYCQjIxYJKwYBBAHaRw8BAQdAD/P5Nvvnvk66SxBBHDbhRml9ORg1WV5CvzKY + CuMfoIS0BmFiY2RlZoiQBBMWCgA4FiEErCIG1VhKWMWo2yfAREZd5NfO31cFAmAk + IyMCGyMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AACgkQREZd5NfO31fbOwD6ArzS + dM0Dkd5h2Ujy1b6KcAaVW9FOa5UNfJ9FFBtjLQEBAJ7UyWD3dZzhvlaAwunsk7DG + 3bHcln8DMpIJVXht78sL + =IE0r + -----END PGP PUBLIC KEY BLOCK----- + register: signed_by_inline + +- name: Change signed_by to URL + deb822_repository: + name: ansible-test + types: deb + uris: https://deb.debian.org + suites: stable + components: + - main + - contrib + - non-free + signed_by: https://ci-files.testing.ansible.com/test/integration/targets/apt_key/apt-key-example-binary.gpg + register: signed_by_url + +- assert: + that: + - signed_by_inline.key_filename is none + - signed_by_inline.repo|trim == signed_by_inline_expected + - signed_by_url is changed + - signed_by_url.key_filename == '/etc/apt/keyrings/ansible-test.gpg' + - > + 'BEGIN' not in signed_by_url.repo + vars: + signed_by_inline_expected: |- + X-Repolib-Name: ansible-test + Types: deb + URIs: https://deb.debian.org + Suites: stable + Components: main contrib non-free + Signed-By: + -----BEGIN PGP PUBLIC KEY BLOCK----- + . + mDMEYCQjIxYJKwYBBAHaRw8BAQdAD/P5Nvvnvk66SxBBHDbhRml9ORg1WV5CvzKY + CuMfoIS0BmFiY2RlZoiQBBMWCgA4FiEErCIG1VhKWMWo2yfAREZd5NfO31cFAmAk + IyMCGyMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AACgkQREZd5NfO31fbOwD6ArzS + dM0Dkd5h2Ujy1b6KcAaVW9FOa5UNfJ9FFBtjLQEBAJ7UyWD3dZzhvlaAwunsk7DG + 3bHcln8DMpIJVXht78sL + =IE0r + -----END PGP PUBLIC KEY BLOCK----- + +- name: remove ansible-test repo + deb822_repository: + name: ansible-test + state: absent + register: ansible_test_repo_remove + +- name: check for ansible-test repo and key + stat: + path: '{{ item }}' + register: ansible_test_repo_stats + loop: + - /etc/apt/sources.list.d/ansible-test.sources + - /etc/apt/keyrings/ansible-test.gpg + +- assert: + that: + - ansible_test_repo_remove is changed + - ansible_test_repo_stats.results|map(attribute='stat')|selectattr('exists') == [] diff --git a/test/integration/targets/setup_deb_repo/tasks/main.yml b/test/integration/targets/setup_deb_repo/tasks/main.yml index 471fb2a2c1a..3e640f69e86 100644 --- a/test/integration/targets/setup_deb_repo/tasks/main.yml +++ b/test/integration/targets/setup_deb_repo/tasks/main.yml @@ -59,6 +59,7 @@ loop: - stable - testing + when: install_repo|default(True)|bool is true # Need to uncomment the deb-src for the universe component for build-dep state - name: Ensure deb-src for the universe component