From 73369f53af2fadad75632b3ce62bbd5f02f8ca27 Mon Sep 17 00:00:00 2001 From: Matt Davis <6775756+nitzmahone@users.noreply.github.com> Date: Thu, 26 Jun 2025 11:34:29 -0700 Subject: [PATCH 01/68] Remove redundant vars FA on PlaybookInclude (#85395) * The redundant FA declaration was not static, which broke a number of automatic validation behaviors. * Added tests to assert deferred validation and lack of templating on `import_playbook.vars`. Co-authored-by: Matt Clay --- lib/ansible/playbook/playbook_include.py | 1 - .../include_import/playbook/playbook_using_a_var.yml | 6 ++++++ .../include_import/playbook/test_import_playbook.yml | 12 ++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 test/integration/targets/include_import/playbook/playbook_using_a_var.yml diff --git a/lib/ansible/playbook/playbook_include.py b/lib/ansible/playbook/playbook_include.py index dfa0971263b..4ec9090de65 100644 --- a/lib/ansible/playbook/playbook_include.py +++ b/lib/ansible/playbook/playbook_include.py @@ -34,7 +34,6 @@ from ansible import constants as C class PlaybookInclude(Base, Conditional, Taggable): import_playbook = NonInheritableFieldAttribute(isa='string', required=True) - vars_val = NonInheritableFieldAttribute(isa='dict', default=dict, alias='vars') _post_validate_object = True # manually post_validate to get free arg validation/coercion diff --git a/test/integration/targets/include_import/playbook/playbook_using_a_var.yml b/test/integration/targets/include_import/playbook/playbook_using_a_var.yml new file mode 100644 index 00000000000..01745aeb2a8 --- /dev/null +++ b/test/integration/targets/include_import/playbook/playbook_using_a_var.yml @@ -0,0 +1,6 @@ +- hosts: localhost + gather_facts: no + tasks: + - name: Verify an imported playbook can see a var it was given + assert: + that: pb_var == 'hello' diff --git a/test/integration/targets/include_import/playbook/test_import_playbook.yml b/test/integration/targets/include_import/playbook/test_import_playbook.yml index e5910c4f1a7..9e266b340bb 100644 --- a/test/integration/targets/include_import/playbook/test_import_playbook.yml +++ b/test/integration/targets/include_import/playbook/test_import_playbook.yml @@ -20,3 +20,15 @@ # https://github.com/ansible/ansible/issues/59548 - import_playbook: sub_playbook/sub_playbook.yml + +- name: Use set_fact to declare a variable + hosts: localhost + gather_facts: no + tasks: + - set_fact: + a_var_from_set_fact: hello + +- name: Verify vars for import_playbook are not templated too early + import_playbook: playbook_using_a_var.yml + vars: + pb_var: "{{ a_var_from_set_fact }}" From fcffd707c6f8d959d7dc7c6e7a91fa2f59fd0308 Mon Sep 17 00:00:00 2001 From: Patrick Kingston <66141901+pkingstonxyz@users.noreply.github.com> Date: Thu, 26 Jun 2025 20:14:20 -0400 Subject: [PATCH 02/68] Test multiple inheritance in _utils get_all_subclasses (#85396) Should bring code coverage in _utils.py to 100%. Ensures get_all_subclasses won't enter infinite recursion. --- test/units/module_utils/common/test_utils.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/units/module_utils/common/test_utils.py b/test/units/module_utils/common/test_utils.py index 9fe56c924b4..9657f5d6fdb 100644 --- a/test/units/module_utils/common/test_utils.py +++ b/test/units/module_utils/common/test_utils.py @@ -33,6 +33,18 @@ class TestGetAllSubclasses: class BranchIIB(BranchII): pass + class MultipleInheritanceBase: + pass + + class MultipleInheritanceBranchI(MultipleInheritanceBase): + pass + + class MultipleInheritanceBranchII(MultipleInheritanceBase): + pass + + class MultipleInheritanceChild(MultipleInheritanceBranchI, MultipleInheritanceBranchII): + pass + def test_bottom_level(self): assert get_all_subclasses(self.BranchIIB) == set() @@ -43,3 +55,8 @@ class TestGetAllSubclasses: assert set(get_all_subclasses(self.Base)) == set([self.BranchI, self.BranchII, self.BranchIA, self.BranchIB, self.BranchIIA, self.BranchIIB]) + + def test_multiple_inheritance(self) -> None: + assert get_all_subclasses(self.MultipleInheritanceBase) == {self.MultipleInheritanceBranchI, + self.MultipleInheritanceBranchII, + self.MultipleInheritanceChild} From 54ccad9e460cb6bfa0ff585a46289239966b56cc Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Sat, 28 Jun 2025 04:01:33 +0200 Subject: [PATCH 03/68] Limit askpass prompts to single attempt (#85364) * Limit askpass prompts to single attempt OpenSSH client option NumberOfPasswordPrompts defaults to 3 so in case an incorrect connection password is provided it is excessively tried 3 times. Not only that but running the `_ssh_askpass` entry point multiple times (via ssh) results in `json.decoder.JSONDecodeError` as after the first run the shared memory is zero'd and the subsequent runs end up calling `json.loads` on empty data. `json.decoder.JSONDecodeError` does not happen prior to Python 3.13 as the share memory is unlinked automatically on `.close()` and the `_ssh_askpass` entry point exits with return code 1 before attempting to load zero'd memory. Fixes #85359 * changelog and tests * Update changelogs/fragments/85359-askpass-incorrect-password-retries.yml Co-authored-by: Matt Davis <6775756+nitzmahone@users.noreply.github.com> * Update lib/ansible/cli/_ssh_askpass.py Co-authored-by: Martin Krizek * Avoid race condition in second unlink --------- Co-authored-by: Matt Davis <6775756+nitzmahone@users.noreply.github.com> Co-authored-by: Matt Clay --- ...359-askpass-incorrect-password-retries.yml | 2 + lib/ansible/cli/_ssh_askpass.py | 67 ++++++++++--------- lib/ansible/plugins/connection/ssh.py | 17 +++-- .../connection_ssh/test_ssh_askpass.yml | 37 +++++++++- 4 files changed, 87 insertions(+), 36 deletions(-) create mode 100644 changelogs/fragments/85359-askpass-incorrect-password-retries.yml diff --git a/changelogs/fragments/85359-askpass-incorrect-password-retries.yml b/changelogs/fragments/85359-askpass-incorrect-password-retries.yml new file mode 100644 index 00000000000..9890fa161f2 --- /dev/null +++ b/changelogs/fragments/85359-askpass-incorrect-password-retries.yml @@ -0,0 +1,2 @@ +bugfixes: + - ssh connection plugin - Allow only one password prompt attempt when utilizing ``SSH_ASKPASS`` (https://github.com/ansible/ansible/issues/85359) diff --git a/lib/ansible/cli/_ssh_askpass.py b/lib/ansible/cli/_ssh_askpass.py index c5d414cdbd6..47cb1299780 100644 --- a/lib/ansible/cli/_ssh_askpass.py +++ b/lib/ansible/cli/_ssh_askpass.py @@ -3,45 +3,52 @@ from __future__ import annotations import json +import multiprocessing.resource_tracker import os import re import sys import typing as t -from multiprocessing.shared_memory import SharedMemory -HOST_KEY_RE = re.compile( - r'(The authenticity of host |differs from the key for the IP address)', -) +from multiprocessing.shared_memory import SharedMemory def main() -> t.Never: - try: - if HOST_KEY_RE.search(sys.argv[1]): - sys.stdout.buffer.write(b'no') - sys.stdout.flush() - sys.exit(0) - except IndexError: - pass - - kwargs: dict[str, bool] = {} - if sys.version_info[:2] >= (3, 13): - # deprecated: description='unneeded due to track argument for SharedMemory' python_version='3.12' - kwargs['track'] = False - try: - shm = SharedMemory(name=os.environ['_ANSIBLE_SSH_ASKPASS_SHM'], **kwargs) - except FileNotFoundError: - # We must be running after the ansible fork is shutting down - sys.exit(1) + if len(sys.argv) > 1: + exit_code = 0 if handle_prompt(sys.argv[1]) else 1 + else: + exit_code = 1 + + sys.exit(exit_code) + + +def handle_prompt(prompt: str) -> bool: + if re.search(r'(The authenticity of host |differs from the key for the IP address)', prompt): + sys.stdout.write('no') + sys.stdout.flush() + return True + + # deprecated: description='Python 3.13 and later support track' python_version='3.12' + can_track = sys.version_info[:2] >= (3, 13) + kwargs = dict(track=False) if can_track else {} + + # This SharedMemory instance is intentionally not closed or unlinked. + # Closing will occur naturally in the SharedMemory finalizer. + # Unlinking is the responsibility of the process which created it. + shm = SharedMemory(name=os.environ['_ANSIBLE_SSH_ASKPASS_SHM'], **kwargs) + + if not can_track: + # When track=False is not available, we must unregister explicitly, since it otherwise only occurs during unlink. + # This avoids resource tracker noise on stderr during process exit. + multiprocessing.resource_tracker.unregister(shm._name, 'shared_memory') + cfg = json.loads(shm.buf.tobytes().rstrip(b'\x00')) - try: - if cfg['prompt'] not in sys.argv[1]: - sys.exit(1) - except IndexError: - sys.exit(1) + if cfg['prompt'] not in prompt: + return False - sys.stdout.buffer.write(cfg['password'].encode('utf-8')) + # Report the password provided by the SharedMemory instance. + # The contents are left untouched after consumption to allow subsequent attempts to succeed. + # This can occur when multiple password prompting methods are enabled, such as password and keyboard-interactive, which is the default on macOS. + sys.stdout.write(cfg['password']) sys.stdout.flush() - shm.buf[:] = b'\x00' * shm.size - shm.close() - sys.exit(0) + return True diff --git a/lib/ansible/plugins/connection/ssh.py b/lib/ansible/plugins/connection/ssh.py index 4ef400e8186..08ff188cf6c 100644 --- a/lib/ansible/plugins/connection/ssh.py +++ b/lib/ansible/plugins/connection/ssh.py @@ -640,11 +640,11 @@ def _clean_shm(func): self.shm.close() with contextlib.suppress(FileNotFoundError): self.shm.unlink() - if not _HAS_RESOURCE_TRACK: - # deprecated: description='unneeded due to track argument for SharedMemory' python_version='3.12' - # There is a resource tracking issue where the resource is deleted, but tracking still has a record - # This will effectively overwrite the record and remove it - SharedMemory(name=self.shm.name, create=True, size=1).unlink() + if not _HAS_RESOURCE_TRACK: + # deprecated: description='unneeded due to track argument for SharedMemory' python_version='3.12' + # There is a resource tracking issue where the resource is deleted, but tracking still has a record + # This will effectively overwrite the record and remove it + SharedMemory(name=self.shm.name, create=True, size=1).unlink() return ret return inner @@ -961,6 +961,13 @@ class Connection(ConnectionBase): b_args = (b"-o", b'ControlPath="%s"' % to_bytes(self.control_path % dict(directory=cpdir), errors='surrogate_or_strict')) self._add_args(b_command, b_args, u"found only ControlPersist; added ControlPath") + if password_mechanism == "ssh_askpass": + self._add_args( + b_command, + (b"-o", b"NumberOfPasswordPrompts=1"), + "Restrict number of password prompts in case incorrect password is provided.", + ) + # Finally, we add any caller-supplied extras. if other_args: b_command += [to_bytes(a) for a in other_args] diff --git a/test/integration/targets/connection_ssh/test_ssh_askpass.yml b/test/integration/targets/connection_ssh/test_ssh_askpass.yml index 506a200813c..e89438aaf41 100644 --- a/test/integration/targets/connection_ssh/test_ssh_askpass.yml +++ b/test/integration/targets/connection_ssh/test_ssh_askpass.yml @@ -23,7 +23,42 @@ state: restarted when: ansible_facts.system != 'Darwin' - - command: + - name: Test incorrect password + command: + argv: + - ansible + - localhost + - -m + - command + - -a + - id + - -vvv + - -e + - ansible_pipelining=yes + - -e + - ansible_connection=ssh + - -e + - ansible_ssh_password_mechanism=ssh_askpass + - -e + - ansible_user={{ test_user_name }} + - -e + - ansible_password=INCORRECT_PASSWORD + environment: + ANSIBLE_NOCOLOR: "1" + ANSIBLE_FORCE_COLOR: "0" + register: askpass_out + ignore_errors: true + + - assert: + that: + - askpass_out is failed + - askpass_out.stdout is contains('UNREACHABLE') + - askpass_out.stdout is contains('Permission denied') + - askpass_out.stdout is not contains('Permission denied, please try again.') # password tried only once + - askpass_out.stdout is not contains('Traceback (most recent call last)') + + - name: Test correct password + command: argv: - ansible - localhost From 7216e04139caa70deb0ec638a416dafadb30df63 Mon Sep 17 00:00:00 2001 From: Sloane Hertel <19572925+s-hertel@users.noreply.github.com> Date: Mon, 30 Jun 2025 15:19:57 -0400 Subject: [PATCH 04/68] Remove deprecation warning caused by automatic role argument spec validation (#85401) --- .../fragments/fix-auto-role-spec-validation-deprecation.yml | 2 ++ lib/ansible/playbook/role/__init__.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 changelogs/fragments/fix-auto-role-spec-validation-deprecation.yml diff --git a/changelogs/fragments/fix-auto-role-spec-validation-deprecation.yml b/changelogs/fragments/fix-auto-role-spec-validation-deprecation.yml new file mode 100644 index 00000000000..61575b5065e --- /dev/null +++ b/changelogs/fragments/fix-auto-role-spec-validation-deprecation.yml @@ -0,0 +1,2 @@ +bugfixes: + - Update automatic role argument spec validation to not use deprecated syntax (https://github.com/ansible/ansible/issues/85399). diff --git a/lib/ansible/playbook/role/__init__.py b/lib/ansible/playbook/role/__init__.py index 7a55b52563a..a86bcd9234a 100644 --- a/lib/ansible/playbook/role/__init__.py +++ b/lib/ansible/playbook/role/__init__.py @@ -374,8 +374,8 @@ class Role(Base, Conditional, Taggable, CollectionSearch, Delegatable): task_name = task_name + ' - ' + argument_spec['short_description'] return { - 'action': { - 'module': 'ansible.builtin.validate_argument_spec', + 'action': 'ansible.builtin.validate_argument_spec', + 'args': { # Pass only the 'options' portion of the arg spec to the module. 'argument_spec': argument_spec.get('options', {}), 'provided_arguments': self._role_params, From 3511299e1ea95ff19a6adbda915f159d3da341a9 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Mon, 30 Jun 2025 14:18:49 -0700 Subject: [PATCH 05/68] Fix deprecated pylint plugin version parsing (#85402) --- .../_util/controller/sanity/pylint/plugins/deprecated_calls.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated_calls.py b/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated_calls.py index ae0f5476cfb..000213c4a76 100644 --- a/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated_calls.py +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated_calls.py @@ -9,6 +9,7 @@ import dataclasses import datetime import functools import pathlib +import re import astroid import astroid.context @@ -156,7 +157,7 @@ class AnsibleDeprecatedChecker(pylint.checkers.BaseChecker): ), ) - ANSIBLE_VERSION = StrictVersion('.'.join(ansible.release.__version__.split('.')[:3])) + ANSIBLE_VERSION = StrictVersion(re.match('[0-9.]*[0-9]', ansible.release.__version__)[0]) """The current ansible-core X.Y.Z version.""" DEPRECATION_MODULE_FUNCTIONS: dict[tuple[str, str], tuple[str, ...]] = { From ac8c66d4310eb75ebe8eeb683c989e34c53b899a Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Mon, 30 Jun 2025 16:31:20 -0700 Subject: [PATCH 06/68] Bump devel version to 2.20.0.dev0 (#85409) --- .github/RELEASE_NAMES.txt | 1 + ...9-ansible-galaxy-slow-resolution-hints.yml | 10 -- ...ansible-galaxy-negative-spec-is-pinned.yml | 8 - .../81874-deprecate-datetime-compat.yml | 11 -- .../83642-fix-sanity-ignore-for-uri.yml | 2 - .../83643-fix-sanity-ignore-for-copy.yml | 3 - ...0-get_url-content-disposition-filename.yml | 2 - .../83700-enable-file-disable-diff.yml | 2 - .../fragments/83757-deprecate-paramiko.yml | 2 - changelogs/fragments/83936-ssh-askpass.yml | 3 - .../fragments/83965-action-groups-schema.yml | 2 - .../fragments/84008-additional-logging.yml | 3 - .../84019-ignore_unreachable-loop.yml | 2 - ...149-add-flush-cache-for-adhoc-commands.yml | 3 - ...4206-dnf5-apt-auto-install-module-deps.yml | 2 - .../84213-ansible-galaxy-url-building.yml | 2 - .../fragments/84229-windows-server-2025.yml | 4 - ...et_connection-ssh_executable-templated.yml | 2 - .../fragments/84259-dnf5-latest-fix.yml | 2 - ...84321-added-ansible_uptime_seconds_aix.yml | 2 - .../84325-validate-modules-seealso-fqcn.yml | 2 - .../84334-dnf5-consolidate-settings.yml | 2 - ...4384-fix-undefined-key-host-group-vars.yml | 4 - .../84419-fix-wait_for_connection-warning.yml | 2 - .../84468-timeout_become_unreachable.yml | 2 - .../84473-dict-lookup-type-error-message.yml | 2 - ...llow-iptables-chain-creation-with-wait.yml | 2 - .../84496-CallbackBase-get_options.yml | 4 - .../fragments/84540-uri-relative-redirect.yml | 3 - .../fragments/84547-acme-test-container.yml | 4 - .../84578-dnf5-is_installed-provides.yml | 2 - .../fragments/84634-dnf5-all-exceptions.yml | 2 - ...4660-fix-meta-end_role-linear-strategy.yml | 2 - .../fragments/84685-add-opensuse-microos.yml | 3 - .../84690-ansible-doc-indent-wrapping.yml | 3 - ...message-malformed-plugin-documentation.yml | 2 - .../84725-deprecate-strategy-plugins.yml | 2 - .../84825-get-url-incomplete-read.yml | 2 - ...5-fix-handler-include_tasks-templating.yml | 2 - .../fragments/85046-dnf5-history-entries.yml | 2 - ...117-add-cloudstack-kvm-for-linux-facts.yml | 2 - .../85184-add-linode-for-linux-facts.yml | 2 - ...359-askpass-incorrect-password-retries.yml | 2 - .../Ansible.Basic-required_if-null.yml | 3 - .../add_type_checking_to_role_init.yml | 4 - changelogs/fragments/adoc_noext_fix.yml | 2 - changelogs/fragments/ansiballz.yml | 5 - .../fragments/ansible-doc-jinja-builtins.yml | 2 - .../fragments/ansible-doc-protomatter.yml | 2 - ...sible-galaxy-keycloak-service-accounts.yml | 2 - .../ansible-test-added-macos-15.3.yml | 6 - .../ansible-test-change-detection-fix.yml | 2 - .../fragments/ansible-test-container-stop.yml | 3 - .../fragments/ansible-test-containers.yml | 7 - .../ansible-test-coverage-test-files.yml | 4 - changelogs/fragments/ansible-test-curl.yml | 3 - .../fragments/ansible-test-debugging.yml | 4 - .../ansible-test-delegation-options.yml | 5 - changelogs/fragments/ansible-test-env-set.yml | 3 - .../ansible-test-fix-command-traceback.yml | 2 - .../fragments/ansible-test-freebsd-nss.yml | 2 - .../ansible-test-network-detection.yml | 3 - .../fragments/ansible-test-nios-container.yml | 2 - .../fragments/ansible-test-no-exec-script.yml | 2 - .../ansible-test-probe-error-handling.yml | 3 - .../ansible-test-pylint-deprecated-fix.yml | 2 - .../ansible-test-pylint-fix-inference.yml | 3 - .../fragments/ansible-test-pylint-fix.yml | 4 - changelogs/fragments/ansible-test-remotes.yml | 6 - changelogs/fragments/ansible-test-rhel-10.yml | 2 - changelogs/fragments/ansible-test-update.yml | 7 - changelogs/fragments/apt_key_bye.yml | 5 - .../apt_repository-remove-py2-compat.yml | 2 - changelogs/fragments/apt_timeout.yml | 3 - changelogs/fragments/assemble_check_mode.yml | 3 - changelogs/fragments/async_really_true.yml | 2 - .../fragments/become-runas-system-deux.yml | 3 - changelogs/fragments/buildroot.yml | 3 - changelogs/fragments/callback_base.yml | 8 - changelogs/fragments/comment_fail.yml | 3 - changelogs/fragments/compat_removal.yml | 3 - changelogs/fragments/config.yml | 3 - changelogs/fragments/config_dump.yml | 3 - changelogs/fragments/config_priv.yml | 2 - .../fragments/constructed-default-value.yml | 4 - changelogs/fragments/copy_validate_input.yml | 2 - changelogs/fragments/cron_err.yml | 3 - changelogs/fragments/csvfile-col.yml | 3 - changelogs/fragments/cve-2024-8775.yml | 5 - changelogs/fragments/darwin_pagesize.yml | 3 - .../fragments/debconf_empty_password.yml | 3 - .../deprecate-compat-importlib-resources.yml | 2 - .../fragments/deprecate-shell-checksum.yml | 2 - changelogs/fragments/deprecated.yml | 3 - changelogs/fragments/deprecator.yml | 17 -- .../fragments/detect-variantid-suse.yaml | 3 - .../fragments/display-windows-newline.yml | 4 - changelogs/fragments/display_args.yml | 5 - changelogs/fragments/distro_LMDE_6.yml | 4 - changelogs/fragments/dnf5-advisory-type.yml | 2 - .../fragments/dnf5-exception-forwarding.yml | 2 - changelogs/fragments/dnf5-plugins-compat.yml | 2 - .../dnf5-remove-usage-deprecated-option.yml | 2 - changelogs/fragments/ensure_remote_perms.yml | 2 - changelogs/fragments/ensure_type.yml | 15 -- ...ture-uri-add-option-multipart-encoding.yml | 2 - changelogs/fragments/file_simplify.yml | 3 - changelogs/fragments/find-checksum.yml | 2 - changelogs/fragments/find_enoent.yml | 5 - .../fix-ansible-galaxy-ignore-certs.yml | 2 - ...-auto-role-spec-validation-deprecation.yml | 2 - .../fragments/fix-cli-doc-path_undefined.yaml | 2 - .../fix-display-bug-in-action-plugin.yml | 2 - .../fragments/fix-include_vars-merge-hash.yml | 2 - changelogs/fragments/fix-ipv6-pattern.yml | 2 - .../fragments/fix-is-filter-is-test.yml | 3 - .../fix-lookup-password-lock-acquisition.yml | 2 - .../fix-lookup-sequence-keyword-args-only.yml | 2 - .../fix-module-utils-facts-timeout.yml | 2 - changelogs/fragments/fix_errors.yml | 2 - changelogs/fragments/follow_redirects_url.yml | 3 - changelogs/fragments/from_yaml_all.yml | 2 - .../fragments/gather_facts_netos_fixes.yml | 3 - .../fragments/gather_facts_smart_fix.yml | 2 - ...in_path-remove-use-of-deprecated-param.yml | 2 - .../fragments/get_url_bsd_style_digest.yml | 3 - .../fragments/hide-loop-vars-debug-vars.yml | 3 - .../implicit_flush_handlers_parents.yml | 2 - changelogs/fragments/include_delegate_to.yml | 3 - .../interpreter-discovery-auto-legacy.yml | 3 - changelogs/fragments/jinja-version.yml | 2 - .../jinja2-__version__-deprecated.yml | 2 - changelogs/fragments/libvirt_lxc.yml | 3 - changelogs/fragments/local-become-fixes.yml | 22 --- changelogs/fragments/local_popen_text.yml | 2 - .../lookup-csvfile-remove-py2-compat.yml | 2 - changelogs/fragments/lookup_config.yml | 3 - changelogs/fragments/macos-correct-lock.yml | 2 - changelogs/fragments/macro_support.yml | 3 - ...-common-collections-counter-deprecated.yml | 2 - .../fragments/module_utils_warnings.yml | 5 - changelogs/fragments/no-inherit-stdio.yml | 6 - changelogs/fragments/no-return.yml | 2 - changelogs/fragments/openrc-status.yml | 3 - changelogs/fragments/os_family.yml | 3 - ...kage-dnf-action-plugins-facts-fail-msg.yml | 2 - changelogs/fragments/package_facts_fix.yml | 2 - .../paramiko-global-config-removal.yml | 2 + changelogs/fragments/passlib.yml | 3 - changelogs/fragments/pin-wheel.yml | 2 - changelogs/fragments/pipelining_refactor.yml | 5 - .../playiterator-add_tasks-optimize.yml | 2 - .../fragments/plugin-loader-trust-docs.yml | 2 - .../fragments/post_fork_stdio_deadlock.yml | 2 - .../fragments/preserve_config_origin.yml | 2 - changelogs/fragments/ps-import-sanity.yml | 3 - changelogs/fragments/pull_changed_fix.yml | 2 - .../fragments/remove-warnings-retval.yml | 2 - .../fragments/remove_ini_ignored_dir.yml | 2 - .../fragments/reserved_module_chekc.yml | 2 - changelogs/fragments/respawn-min-python.yml | 2 - changelogs/fragments/respawn_os_env.yml | 3 - changelogs/fragments/sandbox_config.yml | 2 - changelogs/fragments/selector_removal.yml | 3 - changelogs/fragments/service_facts_fbsd.yml | 2 - .../set_ipv4_and_ipv6_simultaneously.yml | 2 - changelogs/fragments/simplify-copy-module.yml | 2 - .../fragments/skip-handlers-tagged-play.yml | 2 - ...skip-implicit-flush_handlers-no-notify.yml | 2 - .../fragments/skip-role-task-iterator.yml | 2 - changelogs/fragments/ssh-agent.yml | 6 - changelogs/fragments/ssh-clixml.yml | 4 - changelogs/fragments/ssh_agent_misc.yml | 4 - changelogs/fragments/ssh_raise_exception.yml | 3 - changelogs/fragments/ssh_verbosity.yml | 4 - changelogs/fragments/string_conversion.yml | 3 - changelogs/fragments/sunos_virtinfo.yml | 3 - .../fragments/task-error-and-timeout.yml | 7 - .../fragments/task_esoterica_deprecation.yml | 4 - changelogs/fragments/template-none.yml | 2 - changelogs/fragments/template-sandbox.yml | 5 - .../fragments/template-tags-on-play-roles.yml | 2 - .../fragments/templates_types_datatagging.yml | 161 ------------------ .../toml-library-support-dropped.yml | 4 - changelogs/fragments/trim_blocks.yml | 3 - changelogs/fragments/truthy_tests.yml | 3 - .../fragments/unarchive_timestamp_t32.yaml | 3 - .../fragments/unmask_ansible_managed.yml | 3 - changelogs/fragments/unnecessary-shebang.yml | 2 - .../fragments/update-resolvelib-lt-2_0_0.yml | 2 - changelogs/fragments/uri_httpexception.yml | 3 - .../fragments/url_safe_b64_encode_decode.yml | 3 - changelogs/fragments/user_action_fix.yml | 2 - changelogs/fragments/user_module.yml | 3 - changelogs/fragments/user_passphrase.yml | 4 - changelogs/fragments/user_ssh_fix.yml | 4 - .../fragments/v2.19.0-initial-commit.yaml | 1 - changelogs/fragments/variable_names.yml | 7 - changelogs/fragments/vault_cli_fix.yml | 2 - changelogs/fragments/vault_docs_fix.yaml | 3 - changelogs/fragments/warn-on-reserved.yml | 2 - changelogs/fragments/win-async-refactor.yml | 3 - changelogs/fragments/win-wdac-audit.yml | 4 - changelogs/fragments/windows-app-control.yml | 9 - changelogs/fragments/windows-exec.yml | 11 -- changelogs/fragments/winrm-kinit-pexpect.yml | 5 - lib/ansible/config/base.yml | 23 --- lib/ansible/release.py | 4 +- test/sanity/code-smell/deprecated-config.py | 2 +- test/sanity/code-smell/update-bundled.py | 1 + test/sanity/ignore.txt | 10 ++ 211 files changed, 17 insertions(+), 846 deletions(-) delete mode 100644 changelogs/fragments/81709-ansible-galaxy-slow-resolution-hints.yml delete mode 100644 changelogs/fragments/81812-ansible-galaxy-negative-spec-is-pinned.yml delete mode 100644 changelogs/fragments/81874-deprecate-datetime-compat.yml delete mode 100644 changelogs/fragments/83642-fix-sanity-ignore-for-uri.yml delete mode 100644 changelogs/fragments/83643-fix-sanity-ignore-for-copy.yml delete mode 100644 changelogs/fragments/83690-get_url-content-disposition-filename.yml delete mode 100644 changelogs/fragments/83700-enable-file-disable-diff.yml delete mode 100644 changelogs/fragments/83757-deprecate-paramiko.yml delete mode 100644 changelogs/fragments/83936-ssh-askpass.yml delete mode 100644 changelogs/fragments/83965-action-groups-schema.yml delete mode 100644 changelogs/fragments/84008-additional-logging.yml delete mode 100644 changelogs/fragments/84019-ignore_unreachable-loop.yml delete mode 100644 changelogs/fragments/84149-add-flush-cache-for-adhoc-commands.yml delete mode 100644 changelogs/fragments/84206-dnf5-apt-auto-install-module-deps.yml delete mode 100644 changelogs/fragments/84213-ansible-galaxy-url-building.yml delete mode 100644 changelogs/fragments/84229-windows-server-2025.yml delete mode 100644 changelogs/fragments/84238-fix-reset_connection-ssh_executable-templated.yml delete mode 100644 changelogs/fragments/84259-dnf5-latest-fix.yml delete mode 100644 changelogs/fragments/84321-added-ansible_uptime_seconds_aix.yml delete mode 100644 changelogs/fragments/84325-validate-modules-seealso-fqcn.yml delete mode 100644 changelogs/fragments/84334-dnf5-consolidate-settings.yml delete mode 100644 changelogs/fragments/84384-fix-undefined-key-host-group-vars.yml delete mode 100644 changelogs/fragments/84419-fix-wait_for_connection-warning.yml delete mode 100644 changelogs/fragments/84468-timeout_become_unreachable.yml delete mode 100644 changelogs/fragments/84473-dict-lookup-type-error-message.yml delete mode 100644 changelogs/fragments/84490-allow-iptables-chain-creation-with-wait.yml delete mode 100644 changelogs/fragments/84496-CallbackBase-get_options.yml delete mode 100644 changelogs/fragments/84540-uri-relative-redirect.yml delete mode 100644 changelogs/fragments/84547-acme-test-container.yml delete mode 100644 changelogs/fragments/84578-dnf5-is_installed-provides.yml delete mode 100644 changelogs/fragments/84634-dnf5-all-exceptions.yml delete mode 100644 changelogs/fragments/84660-fix-meta-end_role-linear-strategy.yml delete mode 100644 changelogs/fragments/84685-add-opensuse-microos.yml delete mode 100644 changelogs/fragments/84690-ansible-doc-indent-wrapping.yml delete mode 100644 changelogs/fragments/84705-error-message-malformed-plugin-documentation.yml delete mode 100644 changelogs/fragments/84725-deprecate-strategy-plugins.yml delete mode 100644 changelogs/fragments/84825-get-url-incomplete-read.yml delete mode 100644 changelogs/fragments/85015-fix-handler-include_tasks-templating.yml delete mode 100644 changelogs/fragments/85046-dnf5-history-entries.yml delete mode 100644 changelogs/fragments/85117-add-cloudstack-kvm-for-linux-facts.yml delete mode 100644 changelogs/fragments/85184-add-linode-for-linux-facts.yml delete mode 100644 changelogs/fragments/85359-askpass-incorrect-password-retries.yml delete mode 100644 changelogs/fragments/Ansible.Basic-required_if-null.yml delete mode 100644 changelogs/fragments/add_type_checking_to_role_init.yml delete mode 100644 changelogs/fragments/adoc_noext_fix.yml delete mode 100644 changelogs/fragments/ansiballz.yml delete mode 100644 changelogs/fragments/ansible-doc-jinja-builtins.yml delete mode 100644 changelogs/fragments/ansible-doc-protomatter.yml delete mode 100644 changelogs/fragments/ansible-galaxy-keycloak-service-accounts.yml delete mode 100644 changelogs/fragments/ansible-test-added-macos-15.3.yml delete mode 100644 changelogs/fragments/ansible-test-change-detection-fix.yml delete mode 100644 changelogs/fragments/ansible-test-container-stop.yml delete mode 100644 changelogs/fragments/ansible-test-containers.yml delete mode 100644 changelogs/fragments/ansible-test-coverage-test-files.yml delete mode 100644 changelogs/fragments/ansible-test-curl.yml delete mode 100644 changelogs/fragments/ansible-test-debugging.yml delete mode 100644 changelogs/fragments/ansible-test-delegation-options.yml delete mode 100644 changelogs/fragments/ansible-test-env-set.yml delete mode 100644 changelogs/fragments/ansible-test-fix-command-traceback.yml delete mode 100644 changelogs/fragments/ansible-test-freebsd-nss.yml delete mode 100644 changelogs/fragments/ansible-test-network-detection.yml delete mode 100644 changelogs/fragments/ansible-test-nios-container.yml delete mode 100644 changelogs/fragments/ansible-test-no-exec-script.yml delete mode 100644 changelogs/fragments/ansible-test-probe-error-handling.yml delete mode 100644 changelogs/fragments/ansible-test-pylint-deprecated-fix.yml delete mode 100644 changelogs/fragments/ansible-test-pylint-fix-inference.yml delete mode 100644 changelogs/fragments/ansible-test-pylint-fix.yml delete mode 100644 changelogs/fragments/ansible-test-remotes.yml delete mode 100644 changelogs/fragments/ansible-test-rhel-10.yml delete mode 100644 changelogs/fragments/ansible-test-update.yml delete mode 100644 changelogs/fragments/apt_key_bye.yml delete mode 100644 changelogs/fragments/apt_repository-remove-py2-compat.yml delete mode 100644 changelogs/fragments/apt_timeout.yml delete mode 100644 changelogs/fragments/assemble_check_mode.yml delete mode 100644 changelogs/fragments/async_really_true.yml delete mode 100644 changelogs/fragments/become-runas-system-deux.yml delete mode 100644 changelogs/fragments/buildroot.yml delete mode 100644 changelogs/fragments/callback_base.yml delete mode 100644 changelogs/fragments/comment_fail.yml delete mode 100644 changelogs/fragments/compat_removal.yml delete mode 100644 changelogs/fragments/config.yml delete mode 100644 changelogs/fragments/config_dump.yml delete mode 100644 changelogs/fragments/config_priv.yml delete mode 100644 changelogs/fragments/constructed-default-value.yml delete mode 100644 changelogs/fragments/copy_validate_input.yml delete mode 100644 changelogs/fragments/cron_err.yml delete mode 100644 changelogs/fragments/csvfile-col.yml delete mode 100644 changelogs/fragments/cve-2024-8775.yml delete mode 100644 changelogs/fragments/darwin_pagesize.yml delete mode 100644 changelogs/fragments/debconf_empty_password.yml delete mode 100644 changelogs/fragments/deprecate-compat-importlib-resources.yml delete mode 100644 changelogs/fragments/deprecate-shell-checksum.yml delete mode 100644 changelogs/fragments/deprecated.yml delete mode 100644 changelogs/fragments/deprecator.yml delete mode 100644 changelogs/fragments/detect-variantid-suse.yaml delete mode 100644 changelogs/fragments/display-windows-newline.yml delete mode 100644 changelogs/fragments/display_args.yml delete mode 100644 changelogs/fragments/distro_LMDE_6.yml delete mode 100644 changelogs/fragments/dnf5-advisory-type.yml delete mode 100644 changelogs/fragments/dnf5-exception-forwarding.yml delete mode 100644 changelogs/fragments/dnf5-plugins-compat.yml delete mode 100644 changelogs/fragments/dnf5-remove-usage-deprecated-option.yml delete mode 100644 changelogs/fragments/ensure_remote_perms.yml delete mode 100644 changelogs/fragments/ensure_type.yml delete mode 100644 changelogs/fragments/feature-uri-add-option-multipart-encoding.yml delete mode 100644 changelogs/fragments/file_simplify.yml delete mode 100644 changelogs/fragments/find-checksum.yml delete mode 100644 changelogs/fragments/find_enoent.yml delete mode 100644 changelogs/fragments/fix-ansible-galaxy-ignore-certs.yml delete mode 100644 changelogs/fragments/fix-auto-role-spec-validation-deprecation.yml delete mode 100644 changelogs/fragments/fix-cli-doc-path_undefined.yaml delete mode 100644 changelogs/fragments/fix-display-bug-in-action-plugin.yml delete mode 100644 changelogs/fragments/fix-include_vars-merge-hash.yml delete mode 100644 changelogs/fragments/fix-ipv6-pattern.yml delete mode 100644 changelogs/fragments/fix-is-filter-is-test.yml delete mode 100644 changelogs/fragments/fix-lookup-password-lock-acquisition.yml delete mode 100644 changelogs/fragments/fix-lookup-sequence-keyword-args-only.yml delete mode 100644 changelogs/fragments/fix-module-utils-facts-timeout.yml delete mode 100644 changelogs/fragments/fix_errors.yml delete mode 100644 changelogs/fragments/follow_redirects_url.yml delete mode 100644 changelogs/fragments/from_yaml_all.yml delete mode 100644 changelogs/fragments/gather_facts_netos_fixes.yml delete mode 100644 changelogs/fragments/gather_facts_smart_fix.yml delete mode 100644 changelogs/fragments/get_bin_path-remove-use-of-deprecated-param.yml delete mode 100644 changelogs/fragments/get_url_bsd_style_digest.yml delete mode 100644 changelogs/fragments/hide-loop-vars-debug-vars.yml delete mode 100644 changelogs/fragments/implicit_flush_handlers_parents.yml delete mode 100644 changelogs/fragments/include_delegate_to.yml delete mode 100644 changelogs/fragments/interpreter-discovery-auto-legacy.yml delete mode 100644 changelogs/fragments/jinja-version.yml delete mode 100644 changelogs/fragments/jinja2-__version__-deprecated.yml delete mode 100644 changelogs/fragments/libvirt_lxc.yml delete mode 100644 changelogs/fragments/local-become-fixes.yml delete mode 100644 changelogs/fragments/local_popen_text.yml delete mode 100644 changelogs/fragments/lookup-csvfile-remove-py2-compat.yml delete mode 100644 changelogs/fragments/lookup_config.yml delete mode 100644 changelogs/fragments/macos-correct-lock.yml delete mode 100644 changelogs/fragments/macro_support.yml delete mode 100644 changelogs/fragments/module_utils-common-collections-counter-deprecated.yml delete mode 100644 changelogs/fragments/module_utils_warnings.yml delete mode 100644 changelogs/fragments/no-inherit-stdio.yml delete mode 100644 changelogs/fragments/no-return.yml delete mode 100644 changelogs/fragments/openrc-status.yml delete mode 100644 changelogs/fragments/os_family.yml delete mode 100644 changelogs/fragments/package-dnf-action-plugins-facts-fail-msg.yml delete mode 100644 changelogs/fragments/package_facts_fix.yml create mode 100644 changelogs/fragments/paramiko-global-config-removal.yml delete mode 100644 changelogs/fragments/passlib.yml delete mode 100644 changelogs/fragments/pin-wheel.yml delete mode 100644 changelogs/fragments/pipelining_refactor.yml delete mode 100644 changelogs/fragments/playiterator-add_tasks-optimize.yml delete mode 100644 changelogs/fragments/plugin-loader-trust-docs.yml delete mode 100644 changelogs/fragments/post_fork_stdio_deadlock.yml delete mode 100644 changelogs/fragments/preserve_config_origin.yml delete mode 100644 changelogs/fragments/ps-import-sanity.yml delete mode 100644 changelogs/fragments/pull_changed_fix.yml delete mode 100644 changelogs/fragments/remove-warnings-retval.yml delete mode 100644 changelogs/fragments/remove_ini_ignored_dir.yml delete mode 100644 changelogs/fragments/reserved_module_chekc.yml delete mode 100644 changelogs/fragments/respawn-min-python.yml delete mode 100644 changelogs/fragments/respawn_os_env.yml delete mode 100644 changelogs/fragments/sandbox_config.yml delete mode 100644 changelogs/fragments/selector_removal.yml delete mode 100644 changelogs/fragments/service_facts_fbsd.yml delete mode 100644 changelogs/fragments/set_ipv4_and_ipv6_simultaneously.yml delete mode 100644 changelogs/fragments/simplify-copy-module.yml delete mode 100644 changelogs/fragments/skip-handlers-tagged-play.yml delete mode 100644 changelogs/fragments/skip-implicit-flush_handlers-no-notify.yml delete mode 100644 changelogs/fragments/skip-role-task-iterator.yml delete mode 100644 changelogs/fragments/ssh-agent.yml delete mode 100644 changelogs/fragments/ssh-clixml.yml delete mode 100644 changelogs/fragments/ssh_agent_misc.yml delete mode 100644 changelogs/fragments/ssh_raise_exception.yml delete mode 100644 changelogs/fragments/ssh_verbosity.yml delete mode 100644 changelogs/fragments/string_conversion.yml delete mode 100644 changelogs/fragments/sunos_virtinfo.yml delete mode 100644 changelogs/fragments/task-error-and-timeout.yml delete mode 100644 changelogs/fragments/task_esoterica_deprecation.yml delete mode 100644 changelogs/fragments/template-none.yml delete mode 100644 changelogs/fragments/template-sandbox.yml delete mode 100644 changelogs/fragments/template-tags-on-play-roles.yml delete mode 100644 changelogs/fragments/templates_types_datatagging.yml delete mode 100644 changelogs/fragments/toml-library-support-dropped.yml delete mode 100644 changelogs/fragments/trim_blocks.yml delete mode 100644 changelogs/fragments/truthy_tests.yml delete mode 100644 changelogs/fragments/unarchive_timestamp_t32.yaml delete mode 100644 changelogs/fragments/unmask_ansible_managed.yml delete mode 100644 changelogs/fragments/unnecessary-shebang.yml delete mode 100644 changelogs/fragments/update-resolvelib-lt-2_0_0.yml delete mode 100644 changelogs/fragments/uri_httpexception.yml delete mode 100644 changelogs/fragments/url_safe_b64_encode_decode.yml delete mode 100644 changelogs/fragments/user_action_fix.yml delete mode 100644 changelogs/fragments/user_module.yml delete mode 100644 changelogs/fragments/user_passphrase.yml delete mode 100644 changelogs/fragments/user_ssh_fix.yml delete mode 100644 changelogs/fragments/v2.19.0-initial-commit.yaml delete mode 100644 changelogs/fragments/variable_names.yml delete mode 100644 changelogs/fragments/vault_cli_fix.yml delete mode 100644 changelogs/fragments/vault_docs_fix.yaml delete mode 100644 changelogs/fragments/warn-on-reserved.yml delete mode 100644 changelogs/fragments/win-async-refactor.yml delete mode 100644 changelogs/fragments/win-wdac-audit.yml delete mode 100644 changelogs/fragments/windows-app-control.yml delete mode 100644 changelogs/fragments/windows-exec.yml delete mode 100644 changelogs/fragments/winrm-kinit-pexpect.yml diff --git a/.github/RELEASE_NAMES.txt b/.github/RELEASE_NAMES.txt index 17d96a6897e..588013b5e9d 100644 --- a/.github/RELEASE_NAMES.txt +++ b/.github/RELEASE_NAMES.txt @@ -1,3 +1,4 @@ +2.20.0 TBD 2.19.0 What Is and What Should Never Be 2.18.0 Fool in the Rain 2.17.0 Gallows Pole diff --git a/changelogs/fragments/81709-ansible-galaxy-slow-resolution-hints.yml b/changelogs/fragments/81709-ansible-galaxy-slow-resolution-hints.yml deleted file mode 100644 index a3823d9e5e0..00000000000 --- a/changelogs/fragments/81709-ansible-galaxy-slow-resolution-hints.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- - -minor_changes: -- >- - ``ansible-galaxy collection install`` — the collection dependency resolver - now prints out conflicts it hits during dependency resolution when it's - taking too long and it ends up backtracking a lot. It also displays - suggestions on how to help it compute the result more quickly. - -... diff --git a/changelogs/fragments/81812-ansible-galaxy-negative-spec-is-pinned.yml b/changelogs/fragments/81812-ansible-galaxy-negative-spec-is-pinned.yml deleted file mode 100644 index a4997347b11..00000000000 --- a/changelogs/fragments/81812-ansible-galaxy-negative-spec-is-pinned.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- - -bugfixes: -- >- - ``ansible-galaxy`` — the collection dependency resolver now treats - version specifiers starting with ``!=`` as unpinned. - -... diff --git a/changelogs/fragments/81874-deprecate-datetime-compat.yml b/changelogs/fragments/81874-deprecate-datetime-compat.yml deleted file mode 100644 index 63f1b259632..00000000000 --- a/changelogs/fragments/81874-deprecate-datetime-compat.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- - -deprecated_features: -- >- - ``ansible.module_utils.compat.datetime`` - The datetime compatibility - shims are now deprecated. They are scheduled to be removed in - ``ansible-core`` v2.21. This includes ``UTC``, ``utcfromtimestamp()`` - and ``utcnow`` importable from said module - (https://github.com/ansible/ansible/pull/81874). - -... diff --git a/changelogs/fragments/83642-fix-sanity-ignore-for-uri.yml b/changelogs/fragments/83642-fix-sanity-ignore-for-uri.yml deleted file mode 100644 index 14ff7a0723e..00000000000 --- a/changelogs/fragments/83642-fix-sanity-ignore-for-uri.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - uri - mark ``url`` as required (https://github.com/ansible/ansible/pull/83642). diff --git a/changelogs/fragments/83643-fix-sanity-ignore-for-copy.yml b/changelogs/fragments/83643-fix-sanity-ignore-for-copy.yml deleted file mode 100644 index 07d6312cb4d..00000000000 --- a/changelogs/fragments/83643-fix-sanity-ignore-for-copy.yml +++ /dev/null @@ -1,3 +0,0 @@ -minor_changes: - - copy - parameter ``local_follow`` was incorrectly documented as having default value ``True`` (https://github.com/ansible/ansible/pull/83643). - - copy - fix sanity test failures (https://github.com/ansible/ansible/pull/83643). diff --git a/changelogs/fragments/83690-get_url-content-disposition-filename.yml b/changelogs/fragments/83690-get_url-content-disposition-filename.yml deleted file mode 100644 index 47f9734c35e..00000000000 --- a/changelogs/fragments/83690-get_url-content-disposition-filename.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - get_url - fix honoring ``filename`` from the ``content-disposition`` header even when the type is ``inline`` (https://github.com/ansible/ansible/issues/83690) diff --git a/changelogs/fragments/83700-enable-file-disable-diff.yml b/changelogs/fragments/83700-enable-file-disable-diff.yml deleted file mode 100644 index 4fdc9feb4c7..00000000000 --- a/changelogs/fragments/83700-enable-file-disable-diff.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - file - enable file module to disable diff_mode (https://github.com/ansible/ansible/issues/80817). diff --git a/changelogs/fragments/83757-deprecate-paramiko.yml b/changelogs/fragments/83757-deprecate-paramiko.yml deleted file mode 100644 index 982fd9d8b5e..00000000000 --- a/changelogs/fragments/83757-deprecate-paramiko.yml +++ /dev/null @@ -1,2 +0,0 @@ -deprecated_features: -- paramiko - The paramiko connection plugin has been deprecated with planned removal in 2.21. diff --git a/changelogs/fragments/83936-ssh-askpass.yml b/changelogs/fragments/83936-ssh-askpass.yml deleted file mode 100644 index 5118d11f1e3..00000000000 --- a/changelogs/fragments/83936-ssh-askpass.yml +++ /dev/null @@ -1,3 +0,0 @@ -minor_changes: -- ssh connection plugin - Support ``SSH_ASKPASS`` mechanism to provide passwords, making it the default, but still offering an explicit choice to use ``sshpass`` - (https://github.com/ansible/ansible/pull/83936) diff --git a/changelogs/fragments/83965-action-groups-schema.yml b/changelogs/fragments/83965-action-groups-schema.yml deleted file mode 100644 index cd4a439044d..00000000000 --- a/changelogs/fragments/83965-action-groups-schema.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - "runtime-metadata sanity test - improve validation of ``action_groups`` (https://github.com/ansible/ansible/pull/83965)." diff --git a/changelogs/fragments/84008-additional-logging.yml b/changelogs/fragments/84008-additional-logging.yml deleted file mode 100644 index 80bd3a7ddd9..00000000000 --- a/changelogs/fragments/84008-additional-logging.yml +++ /dev/null @@ -1,3 +0,0 @@ -minor_changes: - - Added a -vvvvv log message indicating when a host fails to produce output within the timeout period. - - SSH Escalation-related -vvv log messages now include the associated host information. diff --git a/changelogs/fragments/84019-ignore_unreachable-loop.yml b/changelogs/fragments/84019-ignore_unreachable-loop.yml deleted file mode 100644 index da85af7e4b5..00000000000 --- a/changelogs/fragments/84019-ignore_unreachable-loop.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - Fix returning 'unreachable' for the overall task result. This prevents false positives when a looped task has unignored unreachable items (https://github.com/ansible/ansible/issues/84019). diff --git a/changelogs/fragments/84149-add-flush-cache-for-adhoc-commands.yml b/changelogs/fragments/84149-add-flush-cache-for-adhoc-commands.yml deleted file mode 100644 index 854d2628b64..00000000000 --- a/changelogs/fragments/84149-add-flush-cache-for-adhoc-commands.yml +++ /dev/null @@ -1,3 +0,0 @@ -minor_changes: -- > - ansible, ansible-console, ansible-pull - add --flush-cache option (https://github.com/ansible/ansible/issues/83749). diff --git a/changelogs/fragments/84206-dnf5-apt-auto-install-module-deps.yml b/changelogs/fragments/84206-dnf5-apt-auto-install-module-deps.yml deleted file mode 100644 index 14d595449c3..00000000000 --- a/changelogs/fragments/84206-dnf5-apt-auto-install-module-deps.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - dnf5, apt - add ``auto_install_module_deps`` option (https://github.com/ansible/ansible/issues/84206) diff --git a/changelogs/fragments/84213-ansible-galaxy-url-building.yml b/changelogs/fragments/84213-ansible-galaxy-url-building.yml deleted file mode 100644 index 55b0cf80425..00000000000 --- a/changelogs/fragments/84213-ansible-galaxy-url-building.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: -- ansible-galaxy - Small adjustments to URL building for ``download_url`` and relative redirects. diff --git a/changelogs/fragments/84229-windows-server-2025.yml b/changelogs/fragments/84229-windows-server-2025.yml deleted file mode 100644 index 82c16371a34..00000000000 --- a/changelogs/fragments/84229-windows-server-2025.yml +++ /dev/null @@ -1,4 +0,0 @@ -minor_changes: - - >- - Windows - Add support for Windows Server 2025 to Ansible and as an ``ansible-test`` - remote target - https://github.com/ansible/ansible/issues/84229 diff --git a/changelogs/fragments/84238-fix-reset_connection-ssh_executable-templated.yml b/changelogs/fragments/84238-fix-reset_connection-ssh_executable-templated.yml deleted file mode 100644 index ea77b48ddef..00000000000 --- a/changelogs/fragments/84238-fix-reset_connection-ssh_executable-templated.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - ssh - connection options were incorrectly templated during ``reset_connection`` tasks (https://github.com/ansible/ansible/pull/84238). diff --git a/changelogs/fragments/84259-dnf5-latest-fix.yml b/changelogs/fragments/84259-dnf5-latest-fix.yml deleted file mode 100644 index 40f6ddb7408..00000000000 --- a/changelogs/fragments/84259-dnf5-latest-fix.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - "dnf5 - fix installing a package using ``state=latest`` when a binary of the same name as the package is already installed (https://github.com/ansible/ansible/issues/84259)" diff --git a/changelogs/fragments/84321-added-ansible_uptime_seconds_aix.yml b/changelogs/fragments/84321-added-ansible_uptime_seconds_aix.yml deleted file mode 100644 index 2314753c2b9..00000000000 --- a/changelogs/fragments/84321-added-ansible_uptime_seconds_aix.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - ansible_uptime_second - added ansible_uptime_seconds fact support for AIX (https://github.com/ansible/ansible/pull/84321). diff --git a/changelogs/fragments/84325-validate-modules-seealso-fqcn.yml b/changelogs/fragments/84325-validate-modules-seealso-fqcn.yml deleted file mode 100644 index 98bb102fb9e..00000000000 --- a/changelogs/fragments/84325-validate-modules-seealso-fqcn.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - "validate-modules sanity test - make sure that ``module`` and ``plugin`` ``seealso`` entries use FQCNs (https://github.com/ansible/ansible/pull/84325)." diff --git a/changelogs/fragments/84334-dnf5-consolidate-settings.yml b/changelogs/fragments/84334-dnf5-consolidate-settings.yml deleted file mode 100644 index 7873d3ed432..00000000000 --- a/changelogs/fragments/84334-dnf5-consolidate-settings.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - dnf5 - matching on a binary can be achieved only by specifying a full path (https://github.com/ansible/ansible/issues/84334) diff --git a/changelogs/fragments/84384-fix-undefined-key-host-group-vars.yml b/changelogs/fragments/84384-fix-undefined-key-host-group-vars.yml deleted file mode 100644 index 00f7117d802..00000000000 --- a/changelogs/fragments/84384-fix-undefined-key-host-group-vars.yml +++ /dev/null @@ -1,4 +0,0 @@ -# Filename: changelogs/fragments/84384-fix-undefined-key-host-group-vars.yml - -bugfixes: - - host_group_vars - fixed defining the 'key' variable if the get_vars method is called with cache=False (https://github.com/ansible/ansible/issues/84384) diff --git a/changelogs/fragments/84419-fix-wait_for_connection-warning.yml b/changelogs/fragments/84419-fix-wait_for_connection-warning.yml deleted file mode 100644 index 3b34fefc459..00000000000 --- a/changelogs/fragments/84419-fix-wait_for_connection-warning.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - wait_for_connection - a warning was displayed if any hosts used a local connection (https://github.com/ansible/ansible/issues/84419) diff --git a/changelogs/fragments/84468-timeout_become_unreachable.yml b/changelogs/fragments/84468-timeout_become_unreachable.yml deleted file mode 100644 index d1b3d64bf37..00000000000 --- a/changelogs/fragments/84468-timeout_become_unreachable.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - Time out waiting on become is an unreachable error (https://github.com/ansible/ansible/issues/84468) diff --git a/changelogs/fragments/84473-dict-lookup-type-error-message.yml b/changelogs/fragments/84473-dict-lookup-type-error-message.yml deleted file mode 100644 index cefc5b96788..00000000000 --- a/changelogs/fragments/84473-dict-lookup-type-error-message.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - when the ``dict`` lookup is given a non-dict argument, show the value of the argument and its type in the error message. diff --git a/changelogs/fragments/84490-allow-iptables-chain-creation-with-wait.yml b/changelogs/fragments/84490-allow-iptables-chain-creation-with-wait.yml deleted file mode 100644 index 8d8d3fcff2f..00000000000 --- a/changelogs/fragments/84490-allow-iptables-chain-creation-with-wait.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - iptables - Allows the wait parameter to be used with iptables chain creation (https://github.com/ansible/ansible/issues/84490) diff --git a/changelogs/fragments/84496-CallbackBase-get_options.yml b/changelogs/fragments/84496-CallbackBase-get_options.yml deleted file mode 100644 index 09bcd4bb6d3..00000000000 --- a/changelogs/fragments/84496-CallbackBase-get_options.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -minor_changes: - - callback plugins - add has_option() to CallbackBase to match other functions overloaded from AnsiblePlugin - - callback plugins - fix get_options() for CallbackBase diff --git a/changelogs/fragments/84540-uri-relative-redirect.yml b/changelogs/fragments/84540-uri-relative-redirect.yml deleted file mode 100644 index 837530ee329..00000000000 --- a/changelogs/fragments/84540-uri-relative-redirect.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -bugfixes: - - uri - Form location correctly when the server returns a relative redirect (https://github.com/ansible/ansible/issues/84540) diff --git a/changelogs/fragments/84547-acme-test-container.yml b/changelogs/fragments/84547-acme-test-container.yml deleted file mode 100644 index 26335d51fbc..00000000000 --- a/changelogs/fragments/84547-acme-test-container.yml +++ /dev/null @@ -1,4 +0,0 @@ -minor_changes: - - "ansible-test acme test container - bump `version to 2.3.0 `__ - to include newer versions of Pebble, dependencies, and runtimes. This adds support for ACME profiles, ``dns-account-01`` support, - and some smaller improvements (https://github.com/ansible/ansible/pull/84547)." diff --git a/changelogs/fragments/84578-dnf5-is_installed-provides.yml b/changelogs/fragments/84578-dnf5-is_installed-provides.yml deleted file mode 100644 index f2760356099..00000000000 --- a/changelogs/fragments/84578-dnf5-is_installed-provides.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - "dnf5 - fix ``is_installed`` check for packages that are not installed but listed as provided by an installed package (https://github.com/ansible/ansible/issues/84578)" diff --git a/changelogs/fragments/84634-dnf5-all-exceptions.yml b/changelogs/fragments/84634-dnf5-all-exceptions.yml deleted file mode 100644 index f84a93c2ed0..00000000000 --- a/changelogs/fragments/84634-dnf5-all-exceptions.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - "dnf5 - handle all libdnf5 specific exceptions (https://github.com/ansible/ansible/issues/84634)" diff --git a/changelogs/fragments/84660-fix-meta-end_role-linear-strategy.yml b/changelogs/fragments/84660-fix-meta-end_role-linear-strategy.yml deleted file mode 100644 index c9dff03a433..00000000000 --- a/changelogs/fragments/84660-fix-meta-end_role-linear-strategy.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - linear strategy - fix executing ``end_role`` meta tasks for each host, instead of handling these as implicit run_once tasks (https://github.com/ansible/ansible/issues/84660). diff --git a/changelogs/fragments/84685-add-opensuse-microos.yml b/changelogs/fragments/84685-add-opensuse-microos.yml deleted file mode 100644 index 2ea9d096b94..00000000000 --- a/changelogs/fragments/84685-add-opensuse-microos.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -minor_changes: - - distribution - Added openSUSE MicroOS to Suse OS family (#84685). diff --git a/changelogs/fragments/84690-ansible-doc-indent-wrapping.yml b/changelogs/fragments/84690-ansible-doc-indent-wrapping.yml deleted file mode 100644 index e9ffbca5310..00000000000 --- a/changelogs/fragments/84690-ansible-doc-indent-wrapping.yml +++ /dev/null @@ -1,3 +0,0 @@ -bugfixes: - - "ansible-doc - fix indentation for first line of descriptions of suboptions and sub-return values (https://github.com/ansible/ansible/pull/84690)." - - "ansible-doc - fix line wrapping for first line of description of options and return values (https://github.com/ansible/ansible/pull/84690)." diff --git a/changelogs/fragments/84705-error-message-malformed-plugin-documentation.yml b/changelogs/fragments/84705-error-message-malformed-plugin-documentation.yml deleted file mode 100644 index 488cf8bcccf..00000000000 --- a/changelogs/fragments/84705-error-message-malformed-plugin-documentation.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - improved error message for yaml parsing errors in plugin documentation diff --git a/changelogs/fragments/84725-deprecate-strategy-plugins.yml b/changelogs/fragments/84725-deprecate-strategy-plugins.yml deleted file mode 100644 index 972fc61d052..00000000000 --- a/changelogs/fragments/84725-deprecate-strategy-plugins.yml +++ /dev/null @@ -1,2 +0,0 @@ -deprecated_features: -- Strategy Plugins - Use of strategy plugins not provided in ``ansible.builtin`` are deprecated and do not carry any backwards compatibility guarantees going forward. A future release will remove the ability to use external strategy plugins. No alternative for third party strategy plugins is currently planned. diff --git a/changelogs/fragments/84825-get-url-incomplete-read.yml b/changelogs/fragments/84825-get-url-incomplete-read.yml deleted file mode 100644 index 6879bcfd8c1..00000000000 --- a/changelogs/fragments/84825-get-url-incomplete-read.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - get_url - add a check to recognize incomplete data transfers. diff --git a/changelogs/fragments/85015-fix-handler-include_tasks-templating.yml b/changelogs/fragments/85015-fix-handler-include_tasks-templating.yml deleted file mode 100644 index 17e4d6155b5..00000000000 --- a/changelogs/fragments/85015-fix-handler-include_tasks-templating.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - include_tasks - fix templating options when used as a handler (https://github.com/ansible/ansible/pull/85015). diff --git a/changelogs/fragments/85046-dnf5-history-entries.yml b/changelogs/fragments/85046-dnf5-history-entries.yml deleted file mode 100644 index 917b48d83d3..00000000000 --- a/changelogs/fragments/85046-dnf5-history-entries.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - dnf5 - avoid generating excessive transaction entries in the dnf5 history (https://github.com/ansible/ansible/issues/85046) diff --git a/changelogs/fragments/85117-add-cloudstack-kvm-for-linux-facts.yml b/changelogs/fragments/85117-add-cloudstack-kvm-for-linux-facts.yml deleted file mode 100644 index c3513ba65b8..00000000000 --- a/changelogs/fragments/85117-add-cloudstack-kvm-for-linux-facts.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - facts - add "CloudStack KVM Hypervisor" for Linux VM in virtual facts (https://github.com/ansible/ansible/issues/85089). diff --git a/changelogs/fragments/85184-add-linode-for-linux-facts.yml b/changelogs/fragments/85184-add-linode-for-linux-facts.yml deleted file mode 100644 index edd2f6e6b56..00000000000 --- a/changelogs/fragments/85184-add-linode-for-linux-facts.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - facts - add "Linode" for Linux VM in virtual facts diff --git a/changelogs/fragments/85359-askpass-incorrect-password-retries.yml b/changelogs/fragments/85359-askpass-incorrect-password-retries.yml deleted file mode 100644 index 9890fa161f2..00000000000 --- a/changelogs/fragments/85359-askpass-incorrect-password-retries.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - ssh connection plugin - Allow only one password prompt attempt when utilizing ``SSH_ASKPASS`` (https://github.com/ansible/ansible/issues/85359) diff --git a/changelogs/fragments/Ansible.Basic-required_if-null.yml b/changelogs/fragments/Ansible.Basic-required_if-null.yml deleted file mode 100644 index 8cffba09405..00000000000 --- a/changelogs/fragments/Ansible.Basic-required_if-null.yml +++ /dev/null @@ -1,3 +0,0 @@ -bugfixes: - - >- - Ansible.Basic - Fix ``required_if`` check when the option value to check is unset or set to null. diff --git a/changelogs/fragments/add_type_checking_to_role_init.yml b/changelogs/fragments/add_type_checking_to_role_init.yml deleted file mode 100644 index 53ee68d42bf..00000000000 --- a/changelogs/fragments/add_type_checking_to_role_init.yml +++ /dev/null @@ -1,4 +0,0 @@ -minor_changes: - - >- - Added type annotations to the ``Role.__init__()`` method to enable type checking. - (https://github.com/ansible/ansible/pull/85346) diff --git a/changelogs/fragments/adoc_noext_fix.yml b/changelogs/fragments/adoc_noext_fix.yml deleted file mode 100644 index 8dbf06a77d7..00000000000 --- a/changelogs/fragments/adoc_noext_fix.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: -- ansible-doc will no longer ignore docs for modules without an extension (https://github.com/ansible/ansible/issues/85279). diff --git a/changelogs/fragments/ansiballz.yml b/changelogs/fragments/ansiballz.yml deleted file mode 100644 index dc1cf94f0ce..00000000000 --- a/changelogs/fragments/ansiballz.yml +++ /dev/null @@ -1,5 +0,0 @@ -minor_changes: - - ansiballz - Refactored AnsiballZ and module respawn. - - ansiballz - Added support for AnsiballZ extensions. - - ansiballz - Moved AnsiballZ code coverage support into an extension. - - ansiballz - Added an experimental AnsiballZ extension for remote debugging. diff --git a/changelogs/fragments/ansible-doc-jinja-builtins.yml b/changelogs/fragments/ansible-doc-jinja-builtins.yml deleted file mode 100644 index 49df43b56cb..00000000000 --- a/changelogs/fragments/ansible-doc-jinja-builtins.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - ansible-doc - Return dynamic stub when reporting on Jinja filters and tests not explicitly documented in Ansible diff --git a/changelogs/fragments/ansible-doc-protomatter.yml b/changelogs/fragments/ansible-doc-protomatter.yml deleted file mode 100644 index 47e8afdac4f..00000000000 --- a/changelogs/fragments/ansible-doc-protomatter.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - ansible-doc - Skip listing the internal ``ansible._protomatter`` plugins unless explicitly requested diff --git a/changelogs/fragments/ansible-galaxy-keycloak-service-accounts.yml b/changelogs/fragments/ansible-galaxy-keycloak-service-accounts.yml deleted file mode 100644 index 2b9a2fb96ee..00000000000 --- a/changelogs/fragments/ansible-galaxy-keycloak-service-accounts.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: -- ansible-galaxy - Add support for Keycloak service accounts diff --git a/changelogs/fragments/ansible-test-added-macos-15.3.yml b/changelogs/fragments/ansible-test-added-macos-15.3.yml deleted file mode 100644 index 455f06746f0..00000000000 --- a/changelogs/fragments/ansible-test-added-macos-15.3.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -minor_changes: -- ansible-test - Added a macOS 15.3 remote VM, replacing 14.3. - -... diff --git a/changelogs/fragments/ansible-test-change-detection-fix.yml b/changelogs/fragments/ansible-test-change-detection-fix.yml deleted file mode 100644 index 90effab76f6..00000000000 --- a/changelogs/fragments/ansible-test-change-detection-fix.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - ansible-test - Fix Python relative import resolution from ``__init__.py`` files when using change detection. diff --git a/changelogs/fragments/ansible-test-container-stop.yml b/changelogs/fragments/ansible-test-container-stop.yml deleted file mode 100644 index 344ea69c269..00000000000 --- a/changelogs/fragments/ansible-test-container-stop.yml +++ /dev/null @@ -1,3 +0,0 @@ -minor_changes: - - ansible-test - Use the ``-t`` option to set the stop timeout when stopping a container. - This avoids use of the ``--time`` option which was deprecated in Docker v28.0. diff --git a/changelogs/fragments/ansible-test-containers.yml b/changelogs/fragments/ansible-test-containers.yml deleted file mode 100644 index ed1430e5222..00000000000 --- a/changelogs/fragments/ansible-test-containers.yml +++ /dev/null @@ -1,7 +0,0 @@ -minor_changes: - - ansible-test - Replace container Fedora 40 with 41. - - ansible-test - Replace container Alpine 3.20 with 3.21. - - ansible-test - Update distro containers to remove unnecessary packages (apache2, subversion, ruby). - - ansible-test - Update the HTTP test container. - - ansible-test - Update the PyPI test container. - - ansible-test - Update the utility container. diff --git a/changelogs/fragments/ansible-test-coverage-test-files.yml b/changelogs/fragments/ansible-test-coverage-test-files.yml deleted file mode 100644 index 28b35e6cc38..00000000000 --- a/changelogs/fragments/ansible-test-coverage-test-files.yml +++ /dev/null @@ -1,4 +0,0 @@ -bugfixes: - - >- - ansible-test - Fix up coverage reporting to properly translate the temporary path of integration test modules to - the expected static test module path. diff --git a/changelogs/fragments/ansible-test-curl.yml b/changelogs/fragments/ansible-test-curl.yml deleted file mode 100644 index 0e97d874253..00000000000 --- a/changelogs/fragments/ansible-test-curl.yml +++ /dev/null @@ -1,3 +0,0 @@ -minor_changes: - - ansible-test - Use Python's ``urllib`` instead of ``curl`` for HTTP requests. - - ansible-test - Automatically retry HTTP GET/PUT/DELETE requests on exceptions. diff --git a/changelogs/fragments/ansible-test-debugging.yml b/changelogs/fragments/ansible-test-debugging.yml deleted file mode 100644 index 18c65c9b156..00000000000 --- a/changelogs/fragments/ansible-test-debugging.yml +++ /dev/null @@ -1,4 +0,0 @@ -minor_changes: - - ansible-test - Added experimental support for remote debugging. - - ansible-test - The ``shell`` command has been augmented to propagate remote debug configurations and other test-related settings when running on the controller. - Use the ``--raw`` argument to bypass the additional environment configuration. diff --git a/changelogs/fragments/ansible-test-delegation-options.yml b/changelogs/fragments/ansible-test-delegation-options.yml deleted file mode 100644 index 78de3d0f4f6..00000000000 --- a/changelogs/fragments/ansible-test-delegation-options.yml +++ /dev/null @@ -1,5 +0,0 @@ -bugfixes: - - ansible-test - Fix incorrect handling of options with optional args (e.g. ``--color``), - when followed by other options which are omitted during arg filtering (e.g. ``--docker``). - Previously it was possible for non-option arguments to be incorrectly omitted in these cases. - (https://github.com/ansible/ansible/issues/85173) diff --git a/changelogs/fragments/ansible-test-env-set.yml b/changelogs/fragments/ansible-test-env-set.yml deleted file mode 100644 index f39809293bc..00000000000 --- a/changelogs/fragments/ansible-test-env-set.yml +++ /dev/null @@ -1,3 +0,0 @@ -minor_changes: - - ansible-test - Added support for setting static environment variables in integration tests using ``env/set/`` - entries in the ``aliases`` file. For example, ``env/set/MY_KEY/MY_VALUE`` or ``env/set/MY_PATH//an/abs/path``. diff --git a/changelogs/fragments/ansible-test-fix-command-traceback.yml b/changelogs/fragments/ansible-test-fix-command-traceback.yml deleted file mode 100644 index d43294006f9..00000000000 --- a/changelogs/fragments/ansible-test-fix-command-traceback.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - ansible-test - Fix traceback that occurs after an interactive command fails. diff --git a/changelogs/fragments/ansible-test-freebsd-nss.yml b/changelogs/fragments/ansible-test-freebsd-nss.yml deleted file mode 100644 index d879bb0c2b7..00000000000 --- a/changelogs/fragments/ansible-test-freebsd-nss.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - ansible-test - Ensure CA certificates are installed on managed FreeBSD instances. diff --git a/changelogs/fragments/ansible-test-network-detection.yml b/changelogs/fragments/ansible-test-network-detection.yml deleted file mode 100644 index a7277e16a35..00000000000 --- a/changelogs/fragments/ansible-test-network-detection.yml +++ /dev/null @@ -1,3 +0,0 @@ -minor_changes: - - ansible-test - When detection of the current container network fails, a warning is now issued and execution continues. - This simplifies usage in cases where the current container cannot be inspected, such as when running in GitHub Codespaces. diff --git a/changelogs/fragments/ansible-test-nios-container.yml b/changelogs/fragments/ansible-test-nios-container.yml deleted file mode 100644 index ddf11bbc95f..00000000000 --- a/changelogs/fragments/ansible-test-nios-container.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - ansible-test - Update ``nios-test-container`` to version 7.0.0. diff --git a/changelogs/fragments/ansible-test-no-exec-script.yml b/changelogs/fragments/ansible-test-no-exec-script.yml deleted file mode 100644 index 2748a0fb2f4..00000000000 --- a/changelogs/fragments/ansible-test-no-exec-script.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - ansible-test - Show a more user-friendly error message when a ``runme.sh`` script is not executable. diff --git a/changelogs/fragments/ansible-test-probe-error-handling.yml b/changelogs/fragments/ansible-test-probe-error-handling.yml deleted file mode 100644 index bf4301cc48b..00000000000 --- a/changelogs/fragments/ansible-test-probe-error-handling.yml +++ /dev/null @@ -1,3 +0,0 @@ -minor_changes: - - ansible-test - Improve container runtime probe error handling. - When unexpected probe output is encountered, an error with more useful debugging information is provided. diff --git a/changelogs/fragments/ansible-test-pylint-deprecated-fix.yml b/changelogs/fragments/ansible-test-pylint-deprecated-fix.yml deleted file mode 100644 index 9dabcd0c3a3..00000000000 --- a/changelogs/fragments/ansible-test-pylint-deprecated-fix.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - ansible-test - Updated the ``pylint`` sanity test to skip some deprecation validation checks when all arguments are dynamic. diff --git a/changelogs/fragments/ansible-test-pylint-fix-inference.yml b/changelogs/fragments/ansible-test-pylint-fix-inference.yml deleted file mode 100644 index a2870b8f8f9..00000000000 --- a/changelogs/fragments/ansible-test-pylint-fix-inference.yml +++ /dev/null @@ -1,3 +0,0 @@ -bugfixes: - - ansible-test - Improve type inference for pylint deprecated checks to accommodate some type annotations. - - ansible-test - Disabled the ``bad-super-call`` pylint rule due to false positives. diff --git a/changelogs/fragments/ansible-test-pylint-fix.yml b/changelogs/fragments/ansible-test-pylint-fix.yml deleted file mode 100644 index 877a5944967..00000000000 --- a/changelogs/fragments/ansible-test-pylint-fix.yml +++ /dev/null @@ -1,4 +0,0 @@ -bugfixes: - - ansible-test - Enable the ``sys.unraisablehook`` work-around for the ``pylint`` sanity test on Python 3.11. - Previously the work-around was only enabled for Python 3.12 and later. - However, the same issue has been discovered on Python 3.11. diff --git a/changelogs/fragments/ansible-test-remotes.yml b/changelogs/fragments/ansible-test-remotes.yml deleted file mode 100644 index f9991492941..00000000000 --- a/changelogs/fragments/ansible-test-remotes.yml +++ /dev/null @@ -1,6 +0,0 @@ -minor_changes: - - ansible-test - Replace remote FreeBSD 13.3 with 13.5. - - ansible-test - Replace remote FreeBSD 14.1 with 14.2. - - ansible-test - Replace remote Fedora 40 with 41. - - ansible-test - Replace remote Alpine 3.20 with 3.21. - - ansible-test - Replace remote RHEL 9.4 with 9.5. diff --git a/changelogs/fragments/ansible-test-rhel-10.yml b/changelogs/fragments/ansible-test-rhel-10.yml deleted file mode 100644 index 956c929a159..00000000000 --- a/changelogs/fragments/ansible-test-rhel-10.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - ansible-test - Add RHEL 10.0 as a remote platform for testing. diff --git a/changelogs/fragments/ansible-test-update.yml b/changelogs/fragments/ansible-test-update.yml deleted file mode 100644 index ed58e33562a..00000000000 --- a/changelogs/fragments/ansible-test-update.yml +++ /dev/null @@ -1,7 +0,0 @@ -minor_changes: - - ansible-test - Update ``pylint`` sanity test to use version 3.3.1. - - ansible-test - Default to Python 3.13 in the ``base`` and ``default`` containers. - - ansible-test - Disable the ``deprecated-`` prefixed ``pylint`` rules as their results vary by Python version. - - ansible-test - Update the ``base`` and ``default`` containers. - - ansible-test - Update sanity test requirements to latest available versions. - - ansible-test - Disable the ``pep8`` sanity test rules ``E701`` and ``E704`` to improve compatibility with ``black``. diff --git a/changelogs/fragments/apt_key_bye.yml b/changelogs/fragments/apt_key_bye.yml deleted file mode 100644 index a1792fd9c7c..00000000000 --- a/changelogs/fragments/apt_key_bye.yml +++ /dev/null @@ -1,5 +0,0 @@ -minor_changes: - - apt_key module - add notes to docs and errors to point at the CLI tool deprecation by Debian and alternatives - - apt_repository module - add notes to errors to point at the CLI tool deprecation by Debian and alternatives -bugfixes: - - apt_key module - prevent tests from running when apt-key was removed diff --git a/changelogs/fragments/apt_repository-remove-py2-compat.yml b/changelogs/fragments/apt_repository-remove-py2-compat.yml deleted file mode 100644 index 5e3f1e61673..00000000000 --- a/changelogs/fragments/apt_repository-remove-py2-compat.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - "apt_repository - remove Python 2 support" diff --git a/changelogs/fragments/apt_timeout.yml b/changelogs/fragments/apt_timeout.yml deleted file mode 100644 index b7db9d1eb82..00000000000 --- a/changelogs/fragments/apt_timeout.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -minor_changes: - - apt - consider lock timeout while invoking apt-get command (https://github.com/ansible/ansible/issues/78658). diff --git a/changelogs/fragments/assemble_check_mode.yml b/changelogs/fragments/assemble_check_mode.yml deleted file mode 100644 index b90dd7ecb8f..00000000000 --- a/changelogs/fragments/assemble_check_mode.yml +++ /dev/null @@ -1,3 +0,0 @@ -minor_changes: - - assemble action added check_mode support - - module_utils.basic.backup_local enforces check_mode now diff --git a/changelogs/fragments/async_really_true.yml b/changelogs/fragments/async_really_true.yml deleted file mode 100644 index 5bb092ef94e..00000000000 --- a/changelogs/fragments/async_really_true.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: -- async_status module - The ``started`` and ``finished`` return values are now ``True`` or ``False`` instead of ``1`` or ``0``. diff --git a/changelogs/fragments/become-runas-system-deux.yml b/changelogs/fragments/become-runas-system-deux.yml deleted file mode 100644 index e8b17f92a4c..00000000000 --- a/changelogs/fragments/become-runas-system-deux.yml +++ /dev/null @@ -1,3 +0,0 @@ -bugfixes: - - >- - runas become - Fix up become logic to still get the SYSTEM token with the most privileges when running as SYSTEM. diff --git a/changelogs/fragments/buildroot.yml b/changelogs/fragments/buildroot.yml deleted file mode 100644 index 18acd5438e0..00000000000 --- a/changelogs/fragments/buildroot.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -bugfixes: - - user - Create Buildroot subclass as alias to Busybox (https://github.com/ansible/ansible/issues/83665). diff --git a/changelogs/fragments/callback_base.yml b/changelogs/fragments/callback_base.yml deleted file mode 100644 index 7e18c183d6b..00000000000 --- a/changelogs/fragments/callback_base.yml +++ /dev/null @@ -1,8 +0,0 @@ -bugfixes: - - callback plugins - Callback plugins that do not extend ``ansible.plugins.callback.CallbackBase`` will fail to load with a warning. - If the plugin is used as the stdout callback plugin, this will also be a fatal error. - - callback plugins - Removed unused methods - runner_on_no_hosts, playbook_on_setup, playbook_on_import_for_host, playbook_on_not_import_for_host, - v2_playbook_on_cleanup_task_start, v2_playbook_on_import_for_host, v2_playbook_on_not_import_for_host. - - callback plugins - The stdout callback plugin is no longer called twice if it is also in the list of additional callback plugins. - - callback plugins - A more descriptive error is now raised if the stdout callback plugin cannot be loaded. - - plugin loader - A warning is now emitted for any plugin which fails to load due to a missing base class. diff --git a/changelogs/fragments/comment_fail.yml b/changelogs/fragments/comment_fail.yml deleted file mode 100644 index 5b122d4c9b9..00000000000 --- a/changelogs/fragments/comment_fail.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -minor_changes: - - comment filter - Improve the error message shown when an invalid ``style`` argument is provided. diff --git a/changelogs/fragments/compat_removal.yml b/changelogs/fragments/compat_removal.yml deleted file mode 100644 index 86da5d9933a..00000000000 --- a/changelogs/fragments/compat_removal.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -removed_features: - - removed deprecated pycompat24 and compat.importlib. diff --git a/changelogs/fragments/config.yml b/changelogs/fragments/config.yml deleted file mode 100644 index e7b7d6f808a..00000000000 --- a/changelogs/fragments/config.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -removed_features: - - Remove deprecated plural form of collection path (https://github.com/ansible/ansible/pull/84156). diff --git a/changelogs/fragments/config_dump.yml b/changelogs/fragments/config_dump.yml deleted file mode 100644 index bcd5e7e7ad6..00000000000 --- a/changelogs/fragments/config_dump.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -bugfixes: - - ansible-config - format galaxy server configs while dumping in JSON format (https://github.com/ansible/ansible/issues/84840). diff --git a/changelogs/fragments/config_priv.yml b/changelogs/fragments/config_priv.yml deleted file mode 100644 index 367feaed0aa..00000000000 --- a/changelogs/fragments/config_priv.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - ansible-config will now show internal, but not test configuration entries. This allows for debugging but still denoting the configurations as internal use only (_ prefix). diff --git a/changelogs/fragments/constructed-default-value.yml b/changelogs/fragments/constructed-default-value.yml deleted file mode 100644 index 7f030649c55..00000000000 --- a/changelogs/fragments/constructed-default-value.yml +++ /dev/null @@ -1,4 +0,0 @@ -bugfixes: - - >- - constructed inventory - Use the ``default_value`` or ``trailing_separator`` in a ``keyed_groups`` entry if the expression result of ``key`` - is ``None`` and not just an empty string. diff --git a/changelogs/fragments/copy_validate_input.yml b/changelogs/fragments/copy_validate_input.yml deleted file mode 100644 index 6673def54ec..00000000000 --- a/changelogs/fragments/copy_validate_input.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - copy action now prevents user from setting internal options. diff --git a/changelogs/fragments/cron_err.yml b/changelogs/fragments/cron_err.yml deleted file mode 100644 index 5e65a7b68ec..00000000000 --- a/changelogs/fragments/cron_err.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -minor_changes: - - cron - Provide additional error information while writing cron file (https://github.com/ansible/ansible/issues/83223). diff --git a/changelogs/fragments/csvfile-col.yml b/changelogs/fragments/csvfile-col.yml deleted file mode 100644 index 9e371010c04..00000000000 --- a/changelogs/fragments/csvfile-col.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -minor_changes: - - csvfile - let the config system do the typecasting (https://github.com/ansible/ansible/pull/82263). diff --git a/changelogs/fragments/cve-2024-8775.yml b/changelogs/fragments/cve-2024-8775.yml deleted file mode 100644 index a292c997044..00000000000 --- a/changelogs/fragments/cve-2024-8775.yml +++ /dev/null @@ -1,5 +0,0 @@ -security_fixes: - - task result processing - Ensure that action-sourced result masking (``_ansible_no_log=True``) - is preserved. (CVE-2024-8775) - - include_vars action - Ensure that result masking is correctly requested when vault-encrypted - files are read. (CVE-2024-8775) diff --git a/changelogs/fragments/darwin_pagesize.yml b/changelogs/fragments/darwin_pagesize.yml deleted file mode 100644 index fa1b9703449..00000000000 --- a/changelogs/fragments/darwin_pagesize.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -bugfixes: - - facts - gather pagesize and calculate respective values depending upon architecture (https://github.com/ansible/ansible/issues/84773). diff --git a/changelogs/fragments/debconf_empty_password.yml b/changelogs/fragments/debconf_empty_password.yml deleted file mode 100644 index 473dc53e0d5..00000000000 --- a/changelogs/fragments/debconf_empty_password.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -bugfixes: - - debconf - set empty password values (https://github.com/ansible/ansible/issues/83214). diff --git a/changelogs/fragments/deprecate-compat-importlib-resources.yml b/changelogs/fragments/deprecate-compat-importlib-resources.yml deleted file mode 100644 index 3dc0483a075..00000000000 --- a/changelogs/fragments/deprecate-compat-importlib-resources.yml +++ /dev/null @@ -1,2 +0,0 @@ -deprecated_features: - - "``ansible.compat.importlib_resources`` is deprecated and will be removed in ansible-core 2.23. Use ``importlib.resources`` from the Python standard library instead." diff --git a/changelogs/fragments/deprecate-shell-checksum.yml b/changelogs/fragments/deprecate-shell-checksum.yml deleted file mode 100644 index 040f23d3bab..00000000000 --- a/changelogs/fragments/deprecate-shell-checksum.yml +++ /dev/null @@ -1,2 +0,0 @@ -deprecated_features: - - "The ``ShellModule.checksum`` method is now deprecated and will be removed in ansible-core 2.23. Use ``ActionBase._execute_remote_stat()`` instead." diff --git a/changelogs/fragments/deprecated.yml b/changelogs/fragments/deprecated.yml deleted file mode 100644 index aa632c0487d..00000000000 --- a/changelogs/fragments/deprecated.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -minor_changes: - - docs - add collection name in message from which the module is being deprecated (https://github.com/ansible/ansible/issues/84116). diff --git a/changelogs/fragments/deprecator.yml b/changelogs/fragments/deprecator.yml deleted file mode 100644 index 2a5ed82c2d4..00000000000 --- a/changelogs/fragments/deprecator.yml +++ /dev/null @@ -1,17 +0,0 @@ -minor_changes: - - modules - The ``AnsibleModule.deprecate`` function no longer sends deprecation messages to the target host's logging system. - - ansible-test - Improved ``pylint`` checks for Ansible-specific deprecation functions. - - deprecations - Removed support for specifying deprecation dates as a ``datetime.date``, which was included in an earlier 2.19 pre-release. - - deprecations - Some argument names to ``deprecate_value`` for consistency with existing APIs. - An earlier 2.19 pre-release included a ``removal_`` prefix on the ``date`` and ``version`` arguments. - - deprecations - Collection name strings not of the form ``ns.coll`` passed to deprecation API functions will result in an error. - - collection metadata - The collection loader now parses scalar values from ``meta/runtime.yml`` as strings. - This avoids issues caused by unquoted values such as versions or dates being parsed as types other than strings. - - deprecation warnings - Deprecation warning APIs automatically capture the identity of the deprecating plugin. - The ``collection_name`` argument is only required to correctly attribute deprecations that occur in module_utils or other non-plugin code. - - deprecation warnings - Improved deprecation messages to more clearly indicate the affected content, including plugin name when available. - -deprecated_features: - - plugins - Accessing plugins with ``_``-prefixed filenames without the ``_`` prefix is deprecated. - - Passing a ``warnings` or ``deprecations`` key to ``exit_json`` or ``fail_json`` is deprecated. - Use ``AnsibleModule.warn`` or ``AnsibleModule.deprecate`` instead. diff --git a/changelogs/fragments/detect-variantid-suse.yaml b/changelogs/fragments/detect-variantid-suse.yaml deleted file mode 100644 index 940a9154383..00000000000 --- a/changelogs/fragments/detect-variantid-suse.yaml +++ /dev/null @@ -1,3 +0,0 @@ -minor_changes: - - Improved SUSE distribution detection in distribution.py by parsing VARIANT_ID from /etc/os-release - for identifying SLES_SAP and SL-Micro. Falls back to /etc/products.d/baseproduct symlink for older systems. diff --git a/changelogs/fragments/display-windows-newline.yml b/changelogs/fragments/display-windows-newline.yml deleted file mode 100644 index c4d4887d4c5..00000000000 --- a/changelogs/fragments/display-windows-newline.yml +++ /dev/null @@ -1,4 +0,0 @@ -minor_changes: - - display - Replace Windows newlines (``\r\n``) in display output with Unix newlines (``\n``). - This ensures proper display of strings sourced from Windows hosts in environments which treat ``\r`` as ``\n``, - such as Azure Pipelines. diff --git a/changelogs/fragments/display_args.yml b/changelogs/fragments/display_args.yml deleted file mode 100644 index 12ccd8c80f7..00000000000 --- a/changelogs/fragments/display_args.yml +++ /dev/null @@ -1,5 +0,0 @@ -minor_changes: - - display - The ``formatted`` arg to ``warning`` has no effect. - Warning wrapping is left to the consumer (e.g. terminal, browser). - - display - The ``wrap_text`` and ``stderr`` arguments to ``error`` have no effect. - Errors are always sent to stderr and wrapping is left to the consumer (e.g. terminal, browser). diff --git a/changelogs/fragments/distro_LMDE_6.yml b/changelogs/fragments/distro_LMDE_6.yml deleted file mode 100644 index 212f09de837..00000000000 --- a/changelogs/fragments/distro_LMDE_6.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -bugfixes: - - distro - add support for Linux Mint Debian Edition (LMDE) (https://github.com/ansible/ansible/issues/84934). - - distro - detect Debian as os_family for LMDE 6 (https://github.com/ansible/ansible/issues/84934). diff --git a/changelogs/fragments/dnf5-advisory-type.yml b/changelogs/fragments/dnf5-advisory-type.yml deleted file mode 100644 index 625dbd6f8f2..00000000000 --- a/changelogs/fragments/dnf5-advisory-type.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - "dnf5 - when ``bugfix`` and/or ``security`` is specified, skip packages that do not have any such updates, even for new versions of libdnf5 where this functionality changed and it is considered failure" diff --git a/changelogs/fragments/dnf5-exception-forwarding.yml b/changelogs/fragments/dnf5-exception-forwarding.yml deleted file mode 100644 index 023785ae1fb..00000000000 --- a/changelogs/fragments/dnf5-exception-forwarding.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: -- dnf5 - Handle forwarded exceptions from dnf5-5.2.13 where a generic ``RuntimeError`` was previously raised diff --git a/changelogs/fragments/dnf5-plugins-compat.yml b/changelogs/fragments/dnf5-plugins-compat.yml deleted file mode 100644 index 5d42b0f99f1..00000000000 --- a/changelogs/fragments/dnf5-plugins-compat.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - "dnf5 - fix traceback when ``enable_plugins``/``disable_plugins`` is used on ``python3-libdnf5`` versions that do not support this functionality" diff --git a/changelogs/fragments/dnf5-remove-usage-deprecated-option.yml b/changelogs/fragments/dnf5-remove-usage-deprecated-option.yml deleted file mode 100644 index 3c11a9184ee..00000000000 --- a/changelogs/fragments/dnf5-remove-usage-deprecated-option.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - dnf5 - libdnf5 - use ``conf.pkg_gpgcheck`` instead of deprecated ``conf.gpgcheck`` which is used only as a fallback diff --git a/changelogs/fragments/ensure_remote_perms.yml b/changelogs/fragments/ensure_remote_perms.yml deleted file mode 100644 index fe6a30588f5..00000000000 --- a/changelogs/fragments/ensure_remote_perms.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - Ansible will now ensure predictable permissions on remote artifacts, until now it only ensured executable and relied on system masks for the rest. diff --git a/changelogs/fragments/ensure_type.yml b/changelogs/fragments/ensure_type.yml deleted file mode 100644 index 472aeda7068..00000000000 --- a/changelogs/fragments/ensure_type.yml +++ /dev/null @@ -1,15 +0,0 @@ -bugfixes: - - config - ``ensure_type`` correctly propagates trust and other tags on returned values. - - config - Prevented fatal errors when ``MODULE_IGNORE_EXTS`` configuration was set. - - config - ``ensure_type`` with expected type ``int`` now properly converts ``True`` and ``False`` values to ``int``. - Previously, these values were silently returned unmodified. - - config - ``ensure_type`` now reports an error when ``bytes`` are provided for any known ``value_type``. - Previously, the behavior was undefined, but often resulted in an unhandled exception or incorrect return type. - - config - ``ensure_type`` now converts sequences to ``list`` when requested, instead of returning the sequence. - - config - ``ensure_type`` now converts mappings to ``dict`` when requested, instead of returning the mapping. - - config - ``ensure_type`` now correctly errors when ``pathlist`` or ``pathspec`` types encounter non-string list items. - - config - Templating failures on config defaults now issue a warning. - Previously, failures silently returned an unrendered and untrusted template to the caller. - - convert_bool.boolean API conversion function - Unhashable values passed to ``boolean`` behave like other non-boolean convertible values, - returning False or raising ``TypeError`` depending on the value of ``strict``. - Previously, unhashable values always raised ``ValueError`` due to an invalid set membership check. diff --git a/changelogs/fragments/feature-uri-add-option-multipart-encoding.yml b/changelogs/fragments/feature-uri-add-option-multipart-encoding.yml deleted file mode 100644 index be53360b950..00000000000 --- a/changelogs/fragments/feature-uri-add-option-multipart-encoding.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - AnsibleModule.uri - Add option ``multipart_encoding`` for ``form-multipart`` files in body to change default base64 encoding for files diff --git a/changelogs/fragments/file_simplify.yml b/changelogs/fragments/file_simplify.yml deleted file mode 100644 index 63e48fbdb9a..00000000000 --- a/changelogs/fragments/file_simplify.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -minor_changes: - - file - make code more readable and simple. diff --git a/changelogs/fragments/find-checksum.yml b/changelogs/fragments/find-checksum.yml deleted file mode 100644 index c713beabd68..00000000000 --- a/changelogs/fragments/find-checksum.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - find - add a checksum_algorithm parameter to specify which type of checksum the module will return diff --git a/changelogs/fragments/find_enoent.yml b/changelogs/fragments/find_enoent.yml deleted file mode 100644 index 7fa780cd35b..00000000000 --- a/changelogs/fragments/find_enoent.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -bugfixes: - - find - skip ENOENT error code while recursively enumerating files. - find module will now be tolerant to race conditions that remove files or directories - from the target it is currently inspecting. (https://github.com/ansible/ansible/issues/84873). diff --git a/changelogs/fragments/fix-ansible-galaxy-ignore-certs.yml b/changelogs/fragments/fix-ansible-galaxy-ignore-certs.yml deleted file mode 100644 index aba789bdadd..00000000000 --- a/changelogs/fragments/fix-ansible-galaxy-ignore-certs.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - Fix disabling SSL verification when installing collections and roles from git repositories. If ``--ignore-certs`` isn't provided, the value for the ``GALAXY_IGNORE_CERTS`` configuration option will be used (https://github.com/ansible/ansible/issues/83326). diff --git a/changelogs/fragments/fix-auto-role-spec-validation-deprecation.yml b/changelogs/fragments/fix-auto-role-spec-validation-deprecation.yml deleted file mode 100644 index 61575b5065e..00000000000 --- a/changelogs/fragments/fix-auto-role-spec-validation-deprecation.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - Update automatic role argument spec validation to not use deprecated syntax (https://github.com/ansible/ansible/issues/85399). diff --git a/changelogs/fragments/fix-cli-doc-path_undefined.yaml b/changelogs/fragments/fix-cli-doc-path_undefined.yaml deleted file mode 100644 index 9a62bf77383..00000000000 --- a/changelogs/fragments/fix-cli-doc-path_undefined.yaml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: -- ansible-doc - If none of the files in files exists, path will be undefined and a direct reference will throw an UnboundLocalError (https://github.com/ansible/ansible/pull/84464). diff --git a/changelogs/fragments/fix-display-bug-in-action-plugin.yml b/changelogs/fragments/fix-display-bug-in-action-plugin.yml deleted file mode 100644 index 067a7bb8ce6..00000000000 --- a/changelogs/fragments/fix-display-bug-in-action-plugin.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - Fix a display.debug statement with the wrong param in _get_diff_data() method diff --git a/changelogs/fragments/fix-include_vars-merge-hash.yml b/changelogs/fragments/fix-include_vars-merge-hash.yml deleted file mode 100644 index 48f9bea0005..00000000000 --- a/changelogs/fragments/fix-include_vars-merge-hash.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - include_vars - fix including previously undefined hash variables with hash_behaviour merge (https://github.com/ansible/ansible/issues/84295). diff --git a/changelogs/fragments/fix-ipv6-pattern.yml b/changelogs/fragments/fix-ipv6-pattern.yml deleted file mode 100644 index 48b18150527..00000000000 --- a/changelogs/fragments/fix-ipv6-pattern.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - Fix ipv6 pattern bug in lib/ansible/parsing/utils/addresses.py (https://github.com/ansible/ansible/issues/84237) \ No newline at end of file diff --git a/changelogs/fragments/fix-is-filter-is-test.yml b/changelogs/fragments/fix-is-filter-is-test.yml deleted file mode 100644 index e6563846537..00000000000 --- a/changelogs/fragments/fix-is-filter-is-test.yml +++ /dev/null @@ -1,3 +0,0 @@ -bugfixes: - - Correctly return ``False`` when using the ``filter`` and ``test`` Jinja tests on plugin names which are not filters or tests, respectively. - (resolves issue https://github.com/ansible/ansible/issues/82084) diff --git a/changelogs/fragments/fix-lookup-password-lock-acquisition.yml b/changelogs/fragments/fix-lookup-password-lock-acquisition.yml deleted file mode 100644 index 9152917a53b..00000000000 --- a/changelogs/fragments/fix-lookup-password-lock-acquisition.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - password lookup - fix acquiring the lock when human-readable FileExistsError error message is not English. diff --git a/changelogs/fragments/fix-lookup-sequence-keyword-args-only.yml b/changelogs/fragments/fix-lookup-sequence-keyword-args-only.yml deleted file mode 100644 index ae9f8716b94..00000000000 --- a/changelogs/fragments/fix-lookup-sequence-keyword-args-only.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - sequence lookup - sequence query/lookups without positional arguments now return a valid list if their kwargs comprise a valid sequence expression (https://github.com/ansible/ansible/issues/82921). \ No newline at end of file diff --git a/changelogs/fragments/fix-module-utils-facts-timeout.yml b/changelogs/fragments/fix-module-utils-facts-timeout.yml deleted file mode 100644 index 3ecc95dfab3..00000000000 --- a/changelogs/fragments/fix-module-utils-facts-timeout.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - Use the requested error message in the ansible.module_utils.facts.timeout timeout function instead of hardcoding one. diff --git a/changelogs/fragments/fix_errors.yml b/changelogs/fragments/fix_errors.yml deleted file mode 100644 index 995cc28ffda..00000000000 --- a/changelogs/fragments/fix_errors.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - Errors now preserve stacked error messages even when YAML is involved. diff --git a/changelogs/fragments/follow_redirects_url.yml b/changelogs/fragments/follow_redirects_url.yml deleted file mode 100644 index 53e9bfd97a5..00000000000 --- a/changelogs/fragments/follow_redirects_url.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -minor_changes: - - Move ``follow_redirects`` parameter to module_utils so external modules can reuse it. diff --git a/changelogs/fragments/from_yaml_all.yml b/changelogs/fragments/from_yaml_all.yml deleted file mode 100644 index 2e65c15961b..00000000000 --- a/changelogs/fragments/from_yaml_all.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - from_yaml_all filter - `None` and empty string inputs now always return an empty list. Previously, `None` was returned in Jinja native mode and empty list in classic mode. diff --git a/changelogs/fragments/gather_facts_netos_fixes.yml b/changelogs/fragments/gather_facts_netos_fixes.yml deleted file mode 100644 index 77d021e0cfe..00000000000 --- a/changelogs/fragments/gather_facts_netos_fixes.yml +++ /dev/null @@ -1,3 +0,0 @@ -bugfixes: - - gather_facts action will now issues errors and warnings as appropriate if a network OS is detected but no facts modules are defined for it. - - gather_facts action now defaults to `ansible.legacy.setup` if `smart` was set, no network OS was found and no other alias for `setup` was present. diff --git a/changelogs/fragments/gather_facts_smart_fix.yml b/changelogs/fragments/gather_facts_smart_fix.yml deleted file mode 100644 index c6100d0844c..00000000000 --- a/changelogs/fragments/gather_facts_smart_fix.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - gather_facts action, will now add setup when 'smart' appears with other modules in the FACTS_MODULES setting (#84750). diff --git a/changelogs/fragments/get_bin_path-remove-use-of-deprecated-param.yml b/changelogs/fragments/get_bin_path-remove-use-of-deprecated-param.yml deleted file mode 100644 index 5220fb025be..00000000000 --- a/changelogs/fragments/get_bin_path-remove-use-of-deprecated-param.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - "Remove use of `required` parameter in `get_bin_path` which has been deprecated." diff --git a/changelogs/fragments/get_url_bsd_style_digest.yml b/changelogs/fragments/get_url_bsd_style_digest.yml deleted file mode 100644 index fe4a6f288c3..00000000000 --- a/changelogs/fragments/get_url_bsd_style_digest.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -bugfixes: - - get_url - add support for BSD-style checksum digest file (https://github.com/ansible/ansible/issues/84476). diff --git a/changelogs/fragments/hide-loop-vars-debug-vars.yml b/changelogs/fragments/hide-loop-vars-debug-vars.yml deleted file mode 100644 index 975ab2f75ab..00000000000 --- a/changelogs/fragments/hide-loop-vars-debug-vars.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -bugfixes: - - debug - hide loop vars in debug var display (https://github.com/ansible/ansible/issues/65856). diff --git a/changelogs/fragments/implicit_flush_handlers_parents.yml b/changelogs/fragments/implicit_flush_handlers_parents.yml deleted file mode 100644 index ffff595b3c9..00000000000 --- a/changelogs/fragments/implicit_flush_handlers_parents.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - "Implicit ``meta: flush_handlers`` tasks now have a parent block to prevent potential tracebacks when calling methods like ``get_play()`` on them internally." diff --git a/changelogs/fragments/include_delegate_to.yml b/changelogs/fragments/include_delegate_to.yml deleted file mode 100644 index 4887d0f751f..00000000000 --- a/changelogs/fragments/include_delegate_to.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -removed_features: - - manager - remove deprecated include_delegate_to parameter from get_vars API. diff --git a/changelogs/fragments/interpreter-discovery-auto-legacy.yml b/changelogs/fragments/interpreter-discovery-auto-legacy.yml deleted file mode 100644 index abe6b6ccad8..00000000000 --- a/changelogs/fragments/interpreter-discovery-auto-legacy.yml +++ /dev/null @@ -1,3 +0,0 @@ -deprecated_features: - - interpreter discovery - The ``auto_legacy`` and ``auto_legacy_silent`` options for ``INTERPRETER_PYTHON`` are deprecated. - Use ``auto`` or ``auto_silent`` options instead, as they have the same effect. diff --git a/changelogs/fragments/jinja-version.yml b/changelogs/fragments/jinja-version.yml deleted file mode 100644 index d7ec4c1e821..00000000000 --- a/changelogs/fragments/jinja-version.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - Jinja2 version 3.1.0 or later is now required on the controller. diff --git a/changelogs/fragments/jinja2-__version__-deprecated.yml b/changelogs/fragments/jinja2-__version__-deprecated.yml deleted file mode 100644 index 3c84d7baee6..00000000000 --- a/changelogs/fragments/jinja2-__version__-deprecated.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - "Use ``importlib.metadata.version()`` to detect Jinja version as jinja2.__version__ is deprecated and will be removed in Jinja 3.3." diff --git a/changelogs/fragments/libvirt_lxc.yml b/changelogs/fragments/libvirt_lxc.yml deleted file mode 100644 index 7d575756983..00000000000 --- a/changelogs/fragments/libvirt_lxc.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -bugfixes: - - base.yml - deprecated libvirt_lxc_noseclabel config. diff --git a/changelogs/fragments/local-become-fixes.yml b/changelogs/fragments/local-become-fixes.yml deleted file mode 100644 index a4fd90d5062..00000000000 --- a/changelogs/fragments/local-become-fixes.yml +++ /dev/null @@ -1,22 +0,0 @@ -minor_changes: - - local connection plugin - A new ``become_success_timeout`` operation-wide timeout config (default 10s) was added for ``become``. - - local connection plugin - A new ``become_strip_preamble`` config option (default True) was added; disable to preserve diagnostic ``become`` output in task results. - - local connection plugin - When a ``become`` plugin's ``prompt`` value is a non-string after the ``check_password_prompt`` callback has completed, no prompt stripping will occur on stderr. - -bugfixes: - - local connection plugin - Fixed silent ignore of ``become`` failures and loss of task output when data arrived concurrently on stdout and stderr during ``become`` operation validation. - - local connection plugin - Fixed hang or spurious failure when data arrived concurrently on stdout and stderr during a successful ``become`` operation validation. - - local connection plugin - Fixed task output header truncation when post-become data arrived before ``become`` operation validation had completed. - - local connection plugin - Ensure ``become`` success validation always occurs, even when an active plugin does not set ``prompt``. - - local connection plugin - Fixed cases where the internal ``BECOME-SUCCESS`` message appeared in task output. - - local connection plugin - Fixed long timeout/hang for ``become`` plugins that repeat their prompt on failure (e.g., ``sudo``, some ``su`` implementations). - - local connection plugin - Fixed hang when an active become plugin incorrectly signals lack of prompt. - - local connection plugin - Fixed hang when a become plugin expects a prompt but a password was not provided. - - local connection plugin - Fixed hang when an internal become read timeout expired before the password prompt was written. - - local connection plugin - Fixed hang when only one of stdout or stderr was closed by the ``become_exe`` subprocess. - - local connection plugin - Become timeout errors now include all received data. Previously, the most recently-received data was discarded. - - sudo become plugin - The `sudo_chdir` config option allows the current directory to be set to the specified value before executing sudo to avoid permission errors when dropping privileges. - - su become plugin - Ensure generated regex from ``prompt_l10n`` config values is properly escaped. - - su become plugin - Ensure that password prompts are correctly detected in the presence of leading output. Previously, this case resulted in a timeout or hang. - - su become plugin - Ensure that trailing colon is expected on all ``prompt_l10n`` config values. - - ansible-test - Managed macOS instances now use the ``sudo_chdir`` option for the ``sudo`` become plugin to avoid permission errors when dropping privileges. diff --git a/changelogs/fragments/local_popen_text.yml b/changelogs/fragments/local_popen_text.yml deleted file mode 100644 index 76053d2cdf0..00000000000 --- a/changelogs/fragments/local_popen_text.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - local connection plugin - The command-line used to create subprocesses is now always ``str`` to avoid issues with debuggers and profilers. diff --git a/changelogs/fragments/lookup-csvfile-remove-py2-compat.yml b/changelogs/fragments/lookup-csvfile-remove-py2-compat.yml deleted file mode 100644 index 1f74b25f7ee..00000000000 --- a/changelogs/fragments/lookup-csvfile-remove-py2-compat.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - "csvfile lookup - remove Python 2 compat" diff --git a/changelogs/fragments/lookup_config.yml b/changelogs/fragments/lookup_config.yml deleted file mode 100644 index a1315997cd8..00000000000 --- a/changelogs/fragments/lookup_config.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -bugfixes: - - config - various fixes to config lookup plugin (https://github.com/ansible/ansible/pull/84398). diff --git a/changelogs/fragments/macos-correct-lock.yml b/changelogs/fragments/macos-correct-lock.yml deleted file mode 100644 index d764a8eb0bb..00000000000 --- a/changelogs/fragments/macos-correct-lock.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: -- Use consistent multiprocessing context for action write locks diff --git a/changelogs/fragments/macro_support.yml b/changelogs/fragments/macro_support.yml deleted file mode 100644 index 814be35218d..00000000000 --- a/changelogs/fragments/macro_support.yml +++ /dev/null @@ -1,3 +0,0 @@ -bugfixes: - - templating - Jinja macros returned from a template expression can now be called from another template expression. - - templating - Fixed cases where template expression blocks halted prematurely when a Jinja macro invocation returned an undefined value. diff --git a/changelogs/fragments/module_utils-common-collections-counter-deprecated.yml b/changelogs/fragments/module_utils-common-collections-counter-deprecated.yml deleted file mode 100644 index 89b0b8f3e26..00000000000 --- a/changelogs/fragments/module_utils-common-collections-counter-deprecated.yml +++ /dev/null @@ -1,2 +0,0 @@ -deprecated_features: - - "The ``ansible.module_utils.common.collections.count()`` function is deprecated and will be removed in ansible-core 2.23. Use ``collections.Counter()`` from the Python standard library instead." diff --git a/changelogs/fragments/module_utils_warnings.yml b/changelogs/fragments/module_utils_warnings.yml deleted file mode 100644 index 0679481448b..00000000000 --- a/changelogs/fragments/module_utils_warnings.yml +++ /dev/null @@ -1,5 +0,0 @@ -minor_changes: - - module_utils - Add optional ``help_text`` argument to ``AnsibleModule.warn``. - - module_utils - Add ``AnsibleModule.error_as_warning``. - - module_utils - Add ``ansible.module_utils.common.warnings.error_as_warning``. - - display - Add ``help_text`` and ``obj`` to ``Display.error_as_warning``. diff --git a/changelogs/fragments/no-inherit-stdio.yml b/changelogs/fragments/no-inherit-stdio.yml deleted file mode 100644 index 761abe6ea0c..00000000000 --- a/changelogs/fragments/no-inherit-stdio.yml +++ /dev/null @@ -1,6 +0,0 @@ -major_changes: -- Task Execution / Forks - Forks no longer inherit stdio from the parent - ``ansible-playbook`` process. ``stdout``, ``stderr``, and ``stdin`` - within a worker are detached from the terminal, and non-functional. All - needs to access stdio from a fork for controller side plugins requires - use of ``Display``. diff --git a/changelogs/fragments/no-return.yml b/changelogs/fragments/no-return.yml deleted file mode 100644 index b55db43eb2f..00000000000 --- a/changelogs/fragments/no-return.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - module_utils - Add ``NoReturn`` type annotations to functions which never return. diff --git a/changelogs/fragments/openrc-status.yml b/changelogs/fragments/openrc-status.yml deleted file mode 100644 index 48f667817ac..00000000000 --- a/changelogs/fragments/openrc-status.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -bugfixes: - - service_facts - skip lines which does not contain service names in openrc output (https://github.com/ansible/ansible/issues/84512). diff --git a/changelogs/fragments/os_family.yml b/changelogs/fragments/os_family.yml deleted file mode 100644 index 7126a00c27b..00000000000 --- a/changelogs/fragments/os_family.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -bugfixes: - - facts - skip if distribution file path is directory, instead of raising error (https://github.com/ansible/ansible/issues/84006). diff --git a/changelogs/fragments/package-dnf-action-plugins-facts-fail-msg.yml b/changelogs/fragments/package-dnf-action-plugins-facts-fail-msg.yml deleted file mode 100644 index 8dd037a4e02..00000000000 --- a/changelogs/fragments/package-dnf-action-plugins-facts-fail-msg.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - "``package``/``dnf`` action plugins - provide the reason behind the failure to gather the ``ansible_pkg_mgr`` fact to identify the package backend" diff --git a/changelogs/fragments/package_facts_fix.yml b/changelogs/fragments/package_facts_fix.yml deleted file mode 100644 index f1ffbf4d641..00000000000 --- a/changelogs/fragments/package_facts_fix.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - package_facts module when using 'auto' will return the first package manager found that provides an output, instead of just the first one, as this can be foreign and not have any packages. diff --git a/changelogs/fragments/paramiko-global-config-removal.yml b/changelogs/fragments/paramiko-global-config-removal.yml new file mode 100644 index 00000000000..599e4c26805 --- /dev/null +++ b/changelogs/fragments/paramiko-global-config-removal.yml @@ -0,0 +1,2 @@ +removed_features: + - paramiko - Removed the ``PARAMIKO_HOST_KEY_AUTO_ADD`` and ``PARAMIKO_LOOK_FOR_KEYS`` configuration keys, which were previously deprecated. diff --git a/changelogs/fragments/passlib.yml b/changelogs/fragments/passlib.yml deleted file mode 100644 index b6bf883ae6f..00000000000 --- a/changelogs/fragments/passlib.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -removed_features: - - encrypt - passing unsupported passlib hashtype now raises AnsibleFilterError. diff --git a/changelogs/fragments/pin-wheel.yml b/changelogs/fragments/pin-wheel.yml deleted file mode 100644 index c3c7706e3ba..00000000000 --- a/changelogs/fragments/pin-wheel.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - build - Pin ``wheel`` in ``pyproject.toml`` to ensure compatibility with supported ``setuptools`` versions. diff --git a/changelogs/fragments/pipelining_refactor.yml b/changelogs/fragments/pipelining_refactor.yml deleted file mode 100644 index f4d551375f2..00000000000 --- a/changelogs/fragments/pipelining_refactor.yml +++ /dev/null @@ -1,5 +0,0 @@ -minor_changes: - - pipelining logic has mostly moved to connection plugins so they can decide/override settings. - - ssh connection plugin now overrides pipelining when a tty is requested. - - become plugins get new property 'pipelining' to show support or lack there of for the feature. - - removed hardcoding of su plugin as it now works with pipelining. diff --git a/changelogs/fragments/playiterator-add_tasks-optimize.yml b/changelogs/fragments/playiterator-add_tasks-optimize.yml deleted file mode 100644 index a0b69242fe0..00000000000 --- a/changelogs/fragments/playiterator-add_tasks-optimize.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - Optimize the way tasks from within ``include_tasks``/``include_role`` are inserted into the play. diff --git a/changelogs/fragments/plugin-loader-trust-docs.yml b/changelogs/fragments/plugin-loader-trust-docs.yml deleted file mode 100644 index c2bd09498ed..00000000000 --- a/changelogs/fragments/plugin-loader-trust-docs.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - plugin loader - Apply template trust to strings loaded from plugin configuration definitions and doc fragments. diff --git a/changelogs/fragments/post_fork_stdio_deadlock.yml b/changelogs/fragments/post_fork_stdio_deadlock.yml deleted file mode 100644 index fee63456240..00000000000 --- a/changelogs/fragments/post_fork_stdio_deadlock.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - display - Fix hang caused by early post-fork writers to stdout/stderr (e.g., pydevd) encountering an unreleased fork lock. diff --git a/changelogs/fragments/preserve_config_origin.yml b/changelogs/fragments/preserve_config_origin.yml deleted file mode 100644 index 14d87e1c4ae..00000000000 --- a/changelogs/fragments/preserve_config_origin.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - config - Preserve or apply Origin tag to values returned by config. diff --git a/changelogs/fragments/ps-import-sanity.yml b/changelogs/fragments/ps-import-sanity.yml deleted file mode 100644 index 6d77dcbe876..00000000000 --- a/changelogs/fragments/ps-import-sanity.yml +++ /dev/null @@ -1,3 +0,0 @@ -bugfixes: - - ansible-test - Fix support for detecting PowerShell modules importing module utils with the newer ``#AnsibleRequires`` format. - - ansible-test - Fix support for PowerShell module_util imports with the ``-Optional`` flag. diff --git a/changelogs/fragments/pull_changed_fix.yml b/changelogs/fragments/pull_changed_fix.yml deleted file mode 100644 index 17312b07769..00000000000 --- a/changelogs/fragments/pull_changed_fix.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - ansible-pull change detection will now work independently of callback or result format settings. diff --git a/changelogs/fragments/remove-warnings-retval.yml b/changelogs/fragments/remove-warnings-retval.yml deleted file mode 100644 index aa887c021a7..00000000000 --- a/changelogs/fragments/remove-warnings-retval.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - "modules - use ``AnsibleModule.warn`` instead of passing ``warnings`` to ``exit_json`` or ``fail_json`` which is deprecated." diff --git a/changelogs/fragments/remove_ini_ignored_dir.yml b/changelogs/fragments/remove_ini_ignored_dir.yml deleted file mode 100644 index 10a5a8e61ce..00000000000 --- a/changelogs/fragments/remove_ini_ignored_dir.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - INVENTORY_IGNORE_EXTS config, removed ``ini`` from the default list, inventory scripts using a corresponding .ini configuration are rare now and inventory.ini files are more common. Those that need to ignore the ini files for inventory scripts can still add it to configuration. diff --git a/changelogs/fragments/reserved_module_chekc.yml b/changelogs/fragments/reserved_module_chekc.yml deleted file mode 100644 index 81dc79f6a94..00000000000 --- a/changelogs/fragments/reserved_module_chekc.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - Ansible will now also warn when reserved keywords are set via a module (set_fact, include_vars, etc). diff --git a/changelogs/fragments/respawn-min-python.yml b/changelogs/fragments/respawn-min-python.yml deleted file mode 100644 index 400f9587f97..00000000000 --- a/changelogs/fragments/respawn-min-python.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: -- module respawn - limit to supported Python versions diff --git a/changelogs/fragments/respawn_os_env.yml b/changelogs/fragments/respawn_os_env.yml deleted file mode 100644 index fb54fad8dbe..00000000000 --- a/changelogs/fragments/respawn_os_env.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -bugfixes: - - respawn - use copy of env variables to update existing PYTHONPATH value (https://github.com/ansible/ansible/issues/84954). diff --git a/changelogs/fragments/sandbox_config.yml b/changelogs/fragments/sandbox_config.yml deleted file mode 100644 index 0f17016dc97..00000000000 --- a/changelogs/fragments/sandbox_config.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - templating - Added ``_ANSIBLE_TEMPLAR_SANDBOX_MODE=allow_unsafe_attributes`` environment variable to disable Jinja template attribute sandbox. (https://github.com/ansible/ansible/issues/85202) diff --git a/changelogs/fragments/selector_removal.yml b/changelogs/fragments/selector_removal.yml deleted file mode 100644 index 681686f72e4..00000000000 --- a/changelogs/fragments/selector_removal.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -removed_features: - - selector - remove deprecated compat.selector related files (https://github.com/ansible/ansible/pull/84155). diff --git a/changelogs/fragments/service_facts_fbsd.yml b/changelogs/fragments/service_facts_fbsd.yml deleted file mode 100644 index 6f06ab79f23..00000000000 --- a/changelogs/fragments/service_facts_fbsd.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - service_facts module got freebsd support added. diff --git a/changelogs/fragments/set_ipv4_and_ipv6_simultaneously.yml b/changelogs/fragments/set_ipv4_and_ipv6_simultaneously.yml deleted file mode 100644 index a15e7025232..00000000000 --- a/changelogs/fragments/set_ipv4_and_ipv6_simultaneously.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - module - set ipv4 and ipv6 rules simultaneously in iptables module (https://github.com/ansible/ansible/issues/84404). diff --git a/changelogs/fragments/simplify-copy-module.yml b/changelogs/fragments/simplify-copy-module.yml deleted file mode 100644 index 02f33da8d0e..00000000000 --- a/changelogs/fragments/simplify-copy-module.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - copy - refactor copy module for simplicity. diff --git a/changelogs/fragments/skip-handlers-tagged-play.yml b/changelogs/fragments/skip-handlers-tagged-play.yml deleted file mode 100644 index 755308eafbe..00000000000 --- a/changelogs/fragments/skip-handlers-tagged-play.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - "Do not run implicit ``flush_handlers`` meta tasks when the whole play is excluded from the run due to tags specified." diff --git a/changelogs/fragments/skip-implicit-flush_handlers-no-notify.yml b/changelogs/fragments/skip-implicit-flush_handlers-no-notify.yml deleted file mode 100644 index a4c913791d2..00000000000 --- a/changelogs/fragments/skip-implicit-flush_handlers-no-notify.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - "Improve performance on large inventories by reducing the number of implicit meta tasks." diff --git a/changelogs/fragments/skip-role-task-iterator.yml b/changelogs/fragments/skip-role-task-iterator.yml deleted file mode 100644 index 1cf6b4cbb84..00000000000 --- a/changelogs/fragments/skip-role-task-iterator.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - PlayIterator - do not return tasks from already executed roles so specific strategy plugins do not have to do the filtering of such tasks themselves diff --git a/changelogs/fragments/ssh-agent.yml b/changelogs/fragments/ssh-agent.yml deleted file mode 100644 index b849c72fd6f..00000000000 --- a/changelogs/fragments/ssh-agent.yml +++ /dev/null @@ -1,6 +0,0 @@ -minor_changes: -- ssh-agent - ``ansible``, ``ansible-playbook`` and ``ansible-console`` are capable of spawning or reusing an ssh-agent, - allowing plugins to interact with the ssh-agent. - Additionally a pure python ssh-agent client has been added, enabling easy interaction with the agent. The ssh connection plugin contains - new functionality via ``ansible_ssh_private_key`` and ``ansible_ssh_private_key_passphrase``, for loading an SSH private key into - the agent from a variable. diff --git a/changelogs/fragments/ssh-clixml.yml b/changelogs/fragments/ssh-clixml.yml deleted file mode 100644 index 05c7af4f802..00000000000 --- a/changelogs/fragments/ssh-clixml.yml +++ /dev/null @@ -1,4 +0,0 @@ -bugfixes: - - >- - ssh - Improve the logic for parsing CLIXML data in stderr when working with Windows host. This fixes issues when - the raw stderr contains invalid UTF-8 byte sequences and improves embedded CLIXML sequences. diff --git a/changelogs/fragments/ssh_agent_misc.yml b/changelogs/fragments/ssh_agent_misc.yml deleted file mode 100644 index d9f746eb362..00000000000 --- a/changelogs/fragments/ssh_agent_misc.yml +++ /dev/null @@ -1,4 +0,0 @@ -bugfixes: - - ssh agent - Fixed several potential startup hangs for badly-behaved or overloaded ssh agents. -minor_changes: - - ssh agent - Added ``SSH_AGENT_EXECUTABLE`` config to allow override of ssh-agent. diff --git a/changelogs/fragments/ssh_raise_exception.yml b/changelogs/fragments/ssh_raise_exception.yml deleted file mode 100644 index f27235f268d..00000000000 --- a/changelogs/fragments/ssh_raise_exception.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -bugfixes: - - ssh - Raise exception when sshpass returns error code (https://github.com/ansible/ansible/issues/58133). diff --git a/changelogs/fragments/ssh_verbosity.yml b/changelogs/fragments/ssh_verbosity.yml deleted file mode 100644 index 09ab28817cb..00000000000 --- a/changelogs/fragments/ssh_verbosity.yml +++ /dev/null @@ -1,4 +0,0 @@ -minor_changes: - - ssh connection plugin - Added ``verbosity`` config to decouple SSH debug output verbosity from Ansible verbosity. - Previously, the Ansible verbosity value was always applied to the SSH client command-line, leading to excessively verbose output. - Set the ``ANSIBLE_SSH_VERBOSITY`` envvar or ``ansible_ssh_verbosity`` Ansible variable to a positive integer to increase SSH client verbosity. diff --git a/changelogs/fragments/string_conversion.yml b/changelogs/fragments/string_conversion.yml deleted file mode 100644 index 58032896171..00000000000 --- a/changelogs/fragments/string_conversion.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -removed_features: - - Removed deprecated STRING_CONVERSION_ACTION (https://github.com/ansible/ansible/issues/84220). diff --git a/changelogs/fragments/sunos_virtinfo.yml b/changelogs/fragments/sunos_virtinfo.yml deleted file mode 100644 index 14528099eae..00000000000 --- a/changelogs/fragments/sunos_virtinfo.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -bugfixes: - - sunos - remove hard coding of virtinfo command in facts gathering code (https://github.com/ansible/ansible/pull/84357). diff --git a/changelogs/fragments/task-error-and-timeout.yml b/changelogs/fragments/task-error-and-timeout.yml deleted file mode 100644 index 8d26593f111..00000000000 --- a/changelogs/fragments/task-error-and-timeout.yml +++ /dev/null @@ -1,7 +0,0 @@ -bugfixes: - - task timeout - Specifying a negative task timeout now results in an error. - - error handling - Error details and tracebacks from connection and built-in action exceptions are preserved. - Previously, much of the detail was lost or mixed into the error message. - -minor_changes: - - task timeout - Specifying a timeout greater than 100,000,000 now results in an error. diff --git a/changelogs/fragments/task_esoterica_deprecation.yml b/changelogs/fragments/task_esoterica_deprecation.yml deleted file mode 100644 index 44bcc94366f..00000000000 --- a/changelogs/fragments/task_esoterica_deprecation.yml +++ /dev/null @@ -1,4 +0,0 @@ -deprecated_features: - - playbook syntax - Using a mapping with the ``action`` keyword is deprecated. (https://github.com/ansible/ansible/issues/84101) - - playbook syntax - Using ``key=value`` args and the task ``args`` keyword on the same task is deprecated. - - playbook syntax - Specifying the task ``args`` keyword without a value is deprecated. diff --git a/changelogs/fragments/template-none.yml b/changelogs/fragments/template-none.yml deleted file mode 100644 index bbe61d6ec73..00000000000 --- a/changelogs/fragments/template-none.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - template action - Template files where the entire file's output renders as ``None`` are no longer emitted as the string "None", but instead render to an empty file as in previous releases. diff --git a/changelogs/fragments/template-sandbox.yml b/changelogs/fragments/template-sandbox.yml deleted file mode 100644 index c1336db7506..00000000000 --- a/changelogs/fragments/template-sandbox.yml +++ /dev/null @@ -1,5 +0,0 @@ -minor_changes: - - templating - Switched from the Jinja immutable sandbox to the standard sandbox. - This restores the ability to use mutation methods such as ``list.append`` and ``dict.update``. - - templating - Relaxed the Jinja sandbox to allow specific bitwise operations which have no filter equivalent. - The allowed methods are ``__and__``, ``__lshift__``, ``__or__``, ``__rshift__``, ``__xor__``. diff --git a/changelogs/fragments/template-tags-on-play-roles.yml b/changelogs/fragments/template-tags-on-play-roles.yml deleted file mode 100644 index 8954ba1b14b..00000000000 --- a/changelogs/fragments/template-tags-on-play-roles.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - Fix templating ``tags`` on plays and roles. (https://github.com/ansible/ansible/issues/69903) diff --git a/changelogs/fragments/templates_types_datatagging.yml b/changelogs/fragments/templates_types_datatagging.yml deleted file mode 100644 index 23f320ac06d..00000000000 --- a/changelogs/fragments/templates_types_datatagging.yml +++ /dev/null @@ -1,161 +0,0 @@ -major_changes: - - variables - The type system underlying Ansible's variable storage has been significantly overhauled and formalized. - Attempts to store unsupported Python object types in variables now more consistently yields early warnings or errors. - - variables - To support new Ansible features, many variable objects are now represented by subclasses of their respective native Python types. - In most cases, they behave indistinguishably from their original types, but some Python libraries do not handle builtin object subclasses properly. - Custom plugins that interact with such libraries may require changes to convert and pass the native types. - - ansible-test - Packages beneath ``module_utils`` can now contain ``__init__.py`` files. - - Jinja plugins - Jinja builtin filter and test plugins are now accessible via their fully-qualified names ``ansible.builtin.{name}``. - -minor_changes: - - templating - Templating errors now provide more information about both the location and context of the error, especially for deeply-nested and/or indirected templating scenarios. - - templating - Handling of omitted values is now a first-class feature of the template engine, and is usable in all Ansible Jinja template contexts. - Any template that resolves to ``omit`` is automatically removed from its parent container during templating. - - templating - Unified ``omit`` behavior now requires that plugins calling ``Templar.template()`` handle cases where the entire template result is omitted, - by catching the ``AnsibleValueOmittedError`` that is raised. - Previously, this condition caused a randomly-generated string marker to appear in the template result. - - templating - Template evaluation is lazier than in previous versions. - Template expressions which resolve only portions of a data structure no longer result in the entire structure being templated. - - handlers - Templated handler names with syntax errors, or that resolve to ``omit`` are now skipped like handlers with undefined variables in their name. - - env lookup - The error message generated for a missing environment variable when ``default`` is an undefined value (e.g. ``undef('something')``) will contain the hint from that undefined value, - except when the undefined value is the default of ``undef()`` with no arguments. Previously, any existing undefined hint would be ignored. - - templating - Embedding ``range()`` values in containers such as lists will result in an error on use. - Previously the value would be converted to a string representing the range parameters, such as ``range(0, 3)``. - - Jinja plugins - Plugins can declare support for undefined values. # DTFIX5: examples, porting guide entry - - templating - Variables of type ``set`` and ``tuple`` are now converted to ``list`` when exiting the final pass of templating. - - templating - Access to an undefined variable from inside a lookup, filter, or test (which raises MarkerError) no longer ends processing of the current template. - The triggering undefined value is returned as the result of the offending plugin invocation, and the template continues to execute. # DTFIX5: porting guide entry, samples needed - - plugin error handling - When raising exceptions in an exception handler, be sure to use ``raise ... from`` as appropriate. - This supersedes the use of the ``AnsibleError`` arg ``orig_exc`` to represent the cause. - Specifying ``orig_exc`` as the cause is still permitted. - Failure to use ``raise ... from`` when ``orig_exc`` is set will result in a warning. - Additionally, if the two cause exceptions do not match, a warning will be issued. - - ansible-test - The ``yamllint`` sanity test now enforces string values for the ``!vault`` tag. - - warnings - All warnings (including deprecation warnings) issued during a task's execution are now accessible via the ``warnings`` and ``deprecations`` keys on the task result. - - troubleshooting - Tracebacks can be collected and displayed for most errors, warnings, and deprecation warnings (including those generated by modules). - Tracebacks are no longer enabled with ``-vvv``; the behavior is directly configurable via the ``DISPLAY_TRACEBACK`` config option. - Module tracebacks passed to ``fail_json`` via the ``exception`` kwarg will not be included in the task result unless error tracebacks are configured. - - display - Deduplication of warning and error messages considers the full content of the message (including source and traceback contexts, if enabled). - This may result in fewer messages being omitted. - - modules - Unhandled exceptions during Python module execution are now returned as structured data from the target. - This allows the new traceback handling to be applied to exceptions raised on targets. - - modules - PowerShell modules can now receive ``datetime.date``, ``datetime.time`` and ``datetime.datetime`` values as ISO 8601 strings. - - modules - PowerShell modules can now receive strings sourced from inline vault-encrypted strings. - - from_json filter - The filter accepts a ``profile`` argument, which defaults to ``tagless``. - - to_json / to_nice_json filters - The filters accept a ``profile`` argument, which defaults to ``tagless``. - - undef jinja function - The ``undef`` jinja function now raises an error if a non-string hint is given. - Attempting to use an undefined hint also results in an error, ensuring incorrect use of the function can be distinguished from the function's normal behavior. - -breaking_changes: - - loops - Omit placeholders no longer leak between loop item templating and task templating. - Previously, ``omit`` placeholders could remain embedded in loop items after templating and be used as an ``omit`` for task templating. - Now, values resolving to ``omit`` are dropped immediately when loop items are templated. - To turn missing values into an ``omit`` for task templating, use ``| default(omit)``. - This solution is backward-compatible with previous versions of ansible-core. - - serialization of ``omit`` sentinel - Serialization of variables containing ``omit`` sentinels (e.g., by the ``to_json`` and ``to_yaml`` filters or ``ansible-inventory``) will fail if the variable has not completed templating. - Previously, serialization succeeded with placeholder strings emitted in the serialized output. - - conditionals - Conditional expressions that result in non-boolean values are now an error by default. - Such results often indicate unintentional use of templates where they are not supported, resulting in a conditional that is always true. - When this option is enabled, conditional expressions which are a literal ``None`` or empty string will evaluate as true, for backwards compatibility. - The error can be temporarily changed to a deprecation warning by enabling the ``ALLOW_BROKEN_CONDITIONALS`` config option. - - templating - Templates are always rendered in Jinja2 native mode. - As a result, non-string values are no longer automatically converted to strings. - - templating - Templates with embedded inline templates that were not contained within a Jinja string constant now result in an error, as support for multi-pass templating was removed for security reasons. - In most cases, such templates can be easily rewritten to avoid the use of embedded inline templates. - - templating - Conditionals and lookups which use embedded inline templates in Jinja string constants now display a warning. - These templates should be converted to their expression equivalent. - - templating - Templates resulting in ``None`` are no longer automatically converted to an empty string. - - template lookup - The ``convert_data`` option is deprecated and no longer has any effect. - Use the ``from_json`` filter on the lookup result instead. - - templating - ``#jinja2:`` overrides in templates with invalid override names or types are now templating errors. - - set_fact - The string values "yes", "no", "true" and "false" were previously converted (ignoring case) to boolean values when not using Jinja2 native mode. - Since Jinja2 native mode is always used, this conversion no longer occurs. - When boolean values are required, native boolean syntax should be used where variables are defined, such as in YAML. - When native boolean syntax is not an option, the ``bool`` filter can be used to parse string values into booleans. - - templating - The ``allow_unsafe_lookups`` option no longer has any effect. - Lookup plugins are responsible for tagging strings containing templates to allow evaluation as a template. - - assert - The ``quiet`` argument must be a commonly-accepted boolean value. - Previously, unrecognized values were silently treated as False. - - plugins - Any plugin that sources or creates templates must properly tag them as trusted. # DTFIX5: porting guide entry for "how?" Don't forget to mention inventory plugin ``trusted_by_default`` config. - - first_found lookup - When specifying ``files`` or ``paths`` as a templated list containing undefined values, the undefined list elements will be discarded with a warning. - Previously, the entire list would be discarded without any warning. - - templating - The result of the ``range()`` global function cannot be returned from a template- it should always be passed to a filter (e.g., ``random``). - Previously, range objects returned from an intermediate template were always converted to a list, which is inconsistent with inline consumption of range objects. - - plugins - Custom Jinja plugins that accept undefined top-level arguments must opt in to receiving them. # DTFIX5: porting guide entry + backcompat behavior description - - plugins - Custom Jinja plugins that use ``environment.getitem`` to retrieve undefined values will now trigger a ``MarkerError`` exception. - This exception must be handled to allow the plugin to return a ``Marker``, or the plugin must opt-in to accepting ``Marker`` values. # DTFIX5: mention the decorator - - templating - Many Jinja plugins (filters, lookups, tests) and methods previously silently ignored undefined inputs, which often masked subtle errors. - Passing an undefined argument to a Jinja plugin or method that does not declare undefined support now results in an undefined value. # DTFIX5: common examples, porting guide, `is defined`, `is undefined`, etc; porting guide should also mention that overly-broad exception handling may mask Undefined errors; also that lazy handling of Undefined can invoke a plugin and bomb out in the middle where it was previously never invoked (plugins with side effects, just don't) - - lookup plugins - Lookup plugins called as `with_(lookup)` will no longer have the `_subdir` attribute set. - - lookup plugins - ``terms`` will always be passed to ``run`` as the first positional arg, where previously it was sometimes passed as a keyword arg when using ``with_`` syntax. - - modules - Ansible modules using ``sys.excepthook`` must use a standard ``try/except`` instead. - - templating - Access to ``_`` prefixed attributes and methods, and methods with known side effects, is no longer permitted. - In cases where a matching mapping key is present, the associated value will be returned instead of an error. - This increases template environment isolation and ensures more consistent behavior between the ``.`` and ``[]`` operators. - - internals - The ``ansible.utils.native_jinja`` Python module has been removed. - - internals - The ``AnsibleLoader`` and ``AnsibleDumper`` classes for working with YAML are now factory functions and cannot be extended. - - public API - The ``ansible.vars.fact_cache.FactCache`` wrapper has been removed. - -security_fixes: - - templating - Ansible's template engine no longer processes Jinja templates in strings unless they are marked as coming from a trusted source. - Untrusted strings containing Jinja template markers are ignored with a warning. - Examples of trusted sources include playbooks, vars files, and many inventory sources. - Examples of untrusted sources include module results and facts. - Plugins which have not been updated to preserve trust while manipulating strings may inadvertently cause them to lose their trusted status. - - templating - Changes to conditional expression handling removed numerous instances of insecure multi-pass templating (which could result in execution of untrusted template expressions). - -bugfixes: - - module defaults - Module defaults are no longer templated unless they are used by a task that does not override them. - Previously, all module defaults for all modules were templated for every task. - - regex_search filter - Corrected return value documentation to reflect None (not empty string) for no match. - - first_found lookup - Corrected return value documentation to reflect None (not empty string) for no files found. - - vars lookup - The ``default`` substitution only applies when trying to look up a variable which is not defined. - If the variable is defined, but templates to an undefined value, the ``default`` substitution will not apply. - Use the ``default`` filter to coerce those values instead. - - to_yaml/to_nice_yaml filters - Eliminated possibility of keyword arg collisions with internally-set defaults. - - Jinja plugins - Errors raised will always be derived from ``AnsibleTemplatePluginError``. - - ansible-test - Fixed traceback when handling certain YAML errors in the ``yamllint`` sanity test. - - YAML parsing - The `!unsafe` tag no longer coerces non-string scalars to strings. - - default callback - Error context is now shown for failing tasks that use the ``debug`` action. - - module arg templating - When using a templated raw task arg and a templated ``args`` keyword, args are now merged. - Previously use of templated raw task args silently ignored all values from the templated ``args`` keyword. - - action plugins - Action plugins that raise unhandled exceptions no longer terminate playbook loops. Previously, exceptions raised by an action plugin caused abnormal loop termination and loss of loop iteration results. - - display - The ``Display.deprecated`` method once again properly handles the ``removed=True`` argument (https://github.com/ansible/ansible/issues/82358). - - stability - Fixed silent process failure on unhandled IOError/OSError under ``linear`` strategy. - - lookup plugins - The ``terms`` arg to the ``run`` method is now always a list. - Previously, there were cases where a non-list could be received. - -deprecated_features: - - display - The ``Display.get_deprecation_message`` method has been deprecated. - Call ``Display.deprecated`` to display a deprecation message, or call it with ``removed=True`` to raise an ``AnsibleError``. - - config - The ``DEFAULT_JINJA2_NATIVE`` option has no effect. - Jinja2 native mode is now the default and only option. - - config - The ``DEFAULT_NULL_REPRESENTATION`` option has no effect. - Null values are no longer automatically converted to another value during templating of single variable references. - - template lookup - The jinja2_native option is no longer used in the Ansible Core code base. - Jinja2 native mode is now the default and only option. - - conditionals - Conditionals using Jinja templating delimiters (e.g., ``{{``, ``{%``) should be rewritten as expressions without delimiters, unless the entire conditional value is a single template that resolves to a trusted string expression. - This is useful for dynamic indirection of conditional expressions, but is limited to trusted literal string expressions. - - templating - The ``disable_lookups`` option has no effect, since plugins must be updated to apply trust before any templating can be performed. - - plugin error handling - The ``AnsibleError`` constructor arg ``suppress_extended_error`` is deprecated. - Using ``suppress_extended_error=True`` has the same effect as ``show_content=False``. - - config - The ``ACTION_WARNINGS`` config has no effect. It previously disabled command warnings, which have since been removed. - - templating - Support for enabling Jinja2 extensions (not plugins) has been deprecated. - - playbook variables - The ``play_hosts`` variable has been deprecated, use ``ansible_play_batch`` instead. - - bool filter - Support for coercing unrecognized input values (including None) has been deprecated. Consult the filter documentation for acceptable values, or consider use of the ``truthy`` and ``falsy`` tests. - - oneline callback - The ``oneline`` callback and its associated ad-hoc CLI args (``-o``, ``--one-line``) are deprecated. - - tree callback - The ``tree`` callback and its associated ad-hoc CLI args (``-t``, ``--tree``) are deprecated. - - CLI - The ``--inventory-file`` option alias is deprecated. Use the ``-i`` or ``--inventory`` option instead. - - first_found lookup - Splitting of file paths on ``,;:`` is deprecated. Pass a list of paths instead. - The ``split`` method on strings can be used to split variables into a list as needed. - - cache plugins - The `ansible.plugins.cache.base` Python module is deprecated. Use `ansible.plugins.cache` instead. - - file loading - Loading text files with ``DataLoader`` containing data that cannot be decoded under the expected encoding is deprecated. - In most cases the encoding must be UTF-8, although some plugins allow choosing a different encoding. - Previously, invalid data was silently wrapped in Unicode surrogate escape sequences, often resulting in later errors or other data corruption. - - callback plugins - The v1 callback API (callback methods not prefixed with `v2_`) is deprecated. - Use `v2_` prefixed methods instead. - - callback plugins - The `v2_on_any` callback method is deprecated. Use specific callback methods instead. - -removed_features: - - modules - Modules returning non-UTF8 strings now result in an error. - The ``MODULE_STRICT_UTF8_RESPONSE`` setting can be used to disable this check. diff --git a/changelogs/fragments/toml-library-support-dropped.yml b/changelogs/fragments/toml-library-support-dropped.yml deleted file mode 100644 index e31ec432699..00000000000 --- a/changelogs/fragments/toml-library-support-dropped.yml +++ /dev/null @@ -1,4 +0,0 @@ -breaking_changes: - - Support for the ``toml`` library has been removed from TOML inventory parsing and dumping. - Use ``tomli`` for parsing on Python 3.10. Python 3.11 and later have built-in support for parsing. - Use ``tomli-w`` to support outputting inventory in TOML format. diff --git a/changelogs/fragments/trim_blocks.yml b/changelogs/fragments/trim_blocks.yml deleted file mode 100644 index 80eba8a1a7c..00000000000 --- a/changelogs/fragments/trim_blocks.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -minor_changes: - - lookup_template - add an option to trim blocks while templating (https://github.com/ansible/ansible/issues/75962). diff --git a/changelogs/fragments/truthy_tests.yml b/changelogs/fragments/truthy_tests.yml deleted file mode 100644 index bd613e32bca..00000000000 --- a/changelogs/fragments/truthy_tests.yml +++ /dev/null @@ -1,3 +0,0 @@ -bugfixes: - - Core Jinja test plugins - Builtin test plugins now always return ``bool`` to avoid spurious deprecation warnings for - some malformed inputs. diff --git a/changelogs/fragments/unarchive_timestamp_t32.yaml b/changelogs/fragments/unarchive_timestamp_t32.yaml deleted file mode 100644 index 969c85de05c..00000000000 --- a/changelogs/fragments/unarchive_timestamp_t32.yaml +++ /dev/null @@ -1,3 +0,0 @@ ---- -bugfixes: - - unarchive - Clamp timestamps from beyond y2038 to representible values when unpacking zip files on platforms that use 32-bit time_t (e.g. Debian i386). diff --git a/changelogs/fragments/unmask_ansible_managed.yml b/changelogs/fragments/unmask_ansible_managed.yml deleted file mode 100644 index 80f79ea1663..00000000000 --- a/changelogs/fragments/unmask_ansible_managed.yml +++ /dev/null @@ -1,3 +0,0 @@ -minor_changes: -- template action and lookup plugin - The value of the ``ansible_managed`` variable (if set) will not be masked by the ``template`` action and lookup. - Previously, the value calculated by the ``DEFAULT_MANAGED_STR`` configuration option always masked the variable value during plugin execution, preventing runtime customization. diff --git a/changelogs/fragments/unnecessary-shebang.yml b/changelogs/fragments/unnecessary-shebang.yml deleted file mode 100644 index a8a8c9df8ac..00000000000 --- a/changelogs/fragments/unnecessary-shebang.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - Remove unnecessary shebang from the ``hostname`` module. diff --git a/changelogs/fragments/update-resolvelib-lt-2_0_0.yml b/changelogs/fragments/update-resolvelib-lt-2_0_0.yml deleted file mode 100644 index 10c4f1a0838..00000000000 --- a/changelogs/fragments/update-resolvelib-lt-2_0_0.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - ansible-galaxy - support ``resolvelib >= 0.5.3, < 2.0.0`` (https://github.com/ansible/ansible/issues/84217). diff --git a/changelogs/fragments/uri_httpexception.yml b/changelogs/fragments/uri_httpexception.yml deleted file mode 100644 index d2b339cf3b8..00000000000 --- a/changelogs/fragments/uri_httpexception.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -bugfixes: - - uri - Handle HTTP exceptions raised while reading the content (https://github.com/ansible/ansible/issues/83794). diff --git a/changelogs/fragments/url_safe_b64_encode_decode.yml b/changelogs/fragments/url_safe_b64_encode_decode.yml deleted file mode 100644 index ebdeeda485c..00000000000 --- a/changelogs/fragments/url_safe_b64_encode_decode.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -minor_changes: - - filter - add support for URL-safe encoding and decoding in b64encode and b64decode (https://github.com/ansible/ansible/issues/84147). diff --git a/changelogs/fragments/user_action_fix.yml b/changelogs/fragments/user_action_fix.yml deleted file mode 100644 index 64ee997d688..00000000000 --- a/changelogs/fragments/user_action_fix.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - user module now avoids changing ownership of files symlinked in provided home dir skeleton diff --git a/changelogs/fragments/user_module.yml b/changelogs/fragments/user_module.yml deleted file mode 100644 index e192234f5f9..00000000000 --- a/changelogs/fragments/user_module.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -bugfixes: - - user - Use higher precedence HOME_MODE as UMASK for path provided (https://github.com/ansible/ansible/pull/84482). diff --git a/changelogs/fragments/user_passphrase.yml b/changelogs/fragments/user_passphrase.yml deleted file mode 100644 index edbb4fa3777..00000000000 --- a/changelogs/fragments/user_passphrase.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -bugfixes: - - user - Update prompt for SSH key passphrase (https://github.com/ansible/ansible/issues/84484). - - user - Set timeout for passphrase interaction. diff --git a/changelogs/fragments/user_ssh_fix.yml b/changelogs/fragments/user_ssh_fix.yml deleted file mode 100644 index b2c47d60e3a..00000000000 --- a/changelogs/fragments/user_ssh_fix.yml +++ /dev/null @@ -1,4 +0,0 @@ -bugfixes: - - user action will now require O(force) to overwrite the public part of an ssh key when generating ssh keys, as was already the case for the private part. -security_fixes: - - user action won't allow ssh-keygen, chown and chmod to run on existing ssh public key file, avoiding traversal on existing symlinks (CVE-2024-9902). diff --git a/changelogs/fragments/v2.19.0-initial-commit.yaml b/changelogs/fragments/v2.19.0-initial-commit.yaml deleted file mode 100644 index 0967ef424bc..00000000000 --- a/changelogs/fragments/v2.19.0-initial-commit.yaml +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/changelogs/fragments/variable_names.yml b/changelogs/fragments/variable_names.yml deleted file mode 100644 index 6e7c01d0730..00000000000 --- a/changelogs/fragments/variable_names.yml +++ /dev/null @@ -1,7 +0,0 @@ -bugfixes: - - variables - Added Jinja scalar singletons (``true``, ``false``, ``none``) to invalid Ansible variable name detection. - Previously, variables with these names could be assigned without error, but could not be resolved. -minor_changes: - - variables - Removed restriction on usage of most Python keywords as Ansible variable names. -deprecated_features: - - inventory plugins - Setting invalid Ansible variable names in inventory plugins is deprecated. diff --git a/changelogs/fragments/vault_cli_fix.yml b/changelogs/fragments/vault_cli_fix.yml deleted file mode 100644 index 424204f4e50..00000000000 --- a/changelogs/fragments/vault_cli_fix.yml +++ /dev/null @@ -1,2 +0,0 @@ -bugfixes: - - ansible-vault will now correctly handle `--prompt`, previously it would issue an error about stdin if no 2nd argument was passed diff --git a/changelogs/fragments/vault_docs_fix.yaml b/changelogs/fragments/vault_docs_fix.yaml deleted file mode 100644 index 584f505b844..00000000000 --- a/changelogs/fragments/vault_docs_fix.yaml +++ /dev/null @@ -1,3 +0,0 @@ ---- -minor_changes: - - vault - improved vault filter documentation by adding missing example content for dump_template_data.j2, refining examples for clarity, and ensuring variable consistency (https://github.com/ansible/ansible/issues/83583). diff --git a/changelogs/fragments/warn-on-reserved.yml b/changelogs/fragments/warn-on-reserved.yml deleted file mode 100644 index a197f7cf121..00000000000 --- a/changelogs/fragments/warn-on-reserved.yml +++ /dev/null @@ -1,2 +0,0 @@ -minor_changes: - - variables - Warnings about reserved variable names now show context where the variable was defined. diff --git a/changelogs/fragments/win-async-refactor.yml b/changelogs/fragments/win-async-refactor.yml deleted file mode 100644 index f86e5cf4b29..00000000000 --- a/changelogs/fragments/win-async-refactor.yml +++ /dev/null @@ -1,3 +0,0 @@ -minor_changes: - - >- - Windows - refactor the async implementation to better handle errors during bootstrapping and avoid WMI when possible. diff --git a/changelogs/fragments/win-wdac-audit.yml b/changelogs/fragments/win-wdac-audit.yml deleted file mode 100644 index d4e6f4b8bd9..00000000000 --- a/changelogs/fragments/win-wdac-audit.yml +++ /dev/null @@ -1,4 +0,0 @@ -bugfixes: - - >- - Windows - add support for running on system where WDAC is in audit mode with - ``Dynamic Code Security`` enabled. diff --git a/changelogs/fragments/windows-app-control.yml b/changelogs/fragments/windows-app-control.yml deleted file mode 100644 index 6f0d803b5f8..00000000000 --- a/changelogs/fragments/windows-app-control.yml +++ /dev/null @@ -1,9 +0,0 @@ -minor_changes: - - >- - windows - Added support for ``#AnsibleRequires -Wrapper`` to request a PowerShell module be run through the - execution wrapper scripts without any module utils specified. - - >- - windows - Added support for running signed modules and scripts with a Windows host protected by Windows App - Control/WDAC. This is a tech preview and the interface may be subject to change. - - >- - windows - Script modules will preserve UTF-8 encoding when executing the script. diff --git a/changelogs/fragments/windows-exec.yml b/changelogs/fragments/windows-exec.yml deleted file mode 100644 index 399eb102bdb..00000000000 --- a/changelogs/fragments/windows-exec.yml +++ /dev/null @@ -1,11 +0,0 @@ -bugfixes: - - psrp - Improve stderr parsing when running raw commands that emit error records or stderr lines. - -minor_changes: - - windows - add hard minimum limit for PowerShell to 5.1. Ansible dropped support for older versions of PowerShell - in the 2.16 release but this requirement is now enforced at runtime. - - windows - refactor windows exec runner to improve efficiency and add better error reporting on failures. - -removed_features: - - windows - removed common module functions ``ConvertFrom-AnsibleJson``, ``Format-AnsibleException`` from Windows - modules as they are not used and add unneeded complexity to the code. diff --git a/changelogs/fragments/winrm-kinit-pexpect.yml b/changelogs/fragments/winrm-kinit-pexpect.yml deleted file mode 100644 index 004987f6751..00000000000 --- a/changelogs/fragments/winrm-kinit-pexpect.yml +++ /dev/null @@ -1,5 +0,0 @@ -minor_changes: - - >- - winrm - Remove need for pexpect on macOS hosts when using ``kinit`` to retrieve the Kerberos TGT. - By default the code will now only use the builtin ``subprocess`` library which should handle issues - with select and a high fd count and also simplify the code. diff --git a/lib/ansible/config/base.yml b/lib/ansible/config/base.yml index 81c1b7c56e7..3eeb0250f6c 100644 --- a/lib/ansible/config/base.yml +++ b/lib/ansible/config/base.yml @@ -1844,29 +1844,6 @@ PAGER: - name: ANSIBLE_PAGER version_added: '2.15' - name: PAGER -PARAMIKO_HOST_KEY_AUTO_ADD: - default: False - description: 'TODO: write it' - env: [{name: ANSIBLE_PARAMIKO_HOST_KEY_AUTO_ADD}] - ini: - - {key: host_key_auto_add, section: paramiko_connection} - type: boolean - deprecated: - why: This option was moved to the plugin itself - version: "2.20" - alternatives: Use the option from the plugin itself. -PARAMIKO_LOOK_FOR_KEYS: - name: look for keys - default: True - description: 'TODO: write it' - env: [{name: ANSIBLE_PARAMIKO_LOOK_FOR_KEYS}] - ini: - - {key: look_for_keys, section: paramiko_connection} - type: boolean - deprecated: - why: This option was moved to the plugin itself - version: "2.20" - alternatives: Use the option from the plugin itself. PERSISTENT_CONTROL_PATH_DIR: name: Persistence socket path default: '{{ ANSIBLE_HOME ~ "/pc" }}' diff --git a/lib/ansible/release.py b/lib/ansible/release.py index 26fa7c6fcc3..cf36273f08e 100644 --- a/lib/ansible/release.py +++ b/lib/ansible/release.py @@ -17,6 +17,6 @@ from __future__ import annotations -__version__ = '2.19.0.dev0' +__version__ = '2.20.0.dev0' __author__ = 'Ansible, Inc.' -__codename__ = "What Is and What Should Never Be" +__codename__ = "TBD" diff --git a/test/sanity/code-smell/deprecated-config.py b/test/sanity/code-smell/deprecated-config.py index 828f12e3136..3077e1c2e68 100644 --- a/test/sanity/code-smell/deprecated-config.py +++ b/test/sanity/code-smell/deprecated-config.py @@ -81,7 +81,7 @@ def main(): for plugin in plugins: data = {} - data['doc'], data['examples'], data['return'], data['metadata'] = get_docstring(plugin, fragment_loader) + data['doc'], data['examples'], data['return'], data['metadata'] = get_docstring(os.path.abspath(plugin), fragment_loader) for result in find_deprecations(data['doc']): print('%s: %s is scheduled for removal in %s' % (plugin, '.'.join(str(i) for i in result[0][:-2]), result[1])) diff --git a/test/sanity/code-smell/update-bundled.py b/test/sanity/code-smell/update-bundled.py index c079bc9dbb9..ef41163e3ea 100644 --- a/test/sanity/code-smell/update-bundled.py +++ b/test/sanity/code-smell/update-bundled.py @@ -51,6 +51,7 @@ def get_bundled_libs(paths): bundled_libs.add('lib/ansible/module_utils/distro/__init__.py') bundled_libs.add('lib/ansible/module_utils/six/__init__.py') + bundled_libs.add('lib/ansible/_internal/_wrapt.py') return bundled_libs diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index 94c4b8c35e0..994e031da82 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -233,3 +233,13 @@ test/integration/targets/ansible-test-sanity-pylint/deprecated_thing.py pylint:a test/integration/targets/ansible-test-sanity-pylint/deprecated_thing.py pylint:ansible-deprecated-date-not-permitted # required to verify plugin against core test/integration/targets/ansible-test-sanity-pylint/deprecated_thing.py pylint:ansible-deprecated-unnecessary-collection-name # required to verify plugin against core test/integration/targets/ansible-test-sanity-pylint/deprecated_thing.py pylint:ansible-deprecated-collection-name-not-permitted # required to verify plugin against core +lib/ansible/cli/doc.py pylint:ansible-deprecated-version # TODO: 2.20 +lib/ansible/galaxy/api.py pylint:ansible-deprecated-version # TODO: 2.20 +lib/ansible/plugins/filter/encryption.py pylint:ansible-deprecated-version # TODO: 2.20 +lib/ansible/utils/encrypt.py pylint:ansible-deprecated-version # TODO: 2.20 +lib/ansible/utils/py3compat.py pylint:ansible-deprecated-version # TODO: 2.20 +lib/ansible/utils/ssh_functions.py pylint:ansible-deprecated-version # TODO: 2.20 +lib/ansible/vars/manager.py pylint:ansible-deprecated-version-comment # TODO: 2.20 +lib/ansible/vars/plugins.py pylint:ansible-deprecated-version # TODO: 2.20 +lib/ansible/modules/dnf.py validate-modules:ansible-deprecated-version # TODO: 2.20 +lib/ansible/modules/dnf5.py validate-modules:ansible-deprecated-version # TODO: 2.20 From d6efb7db8a4cfa930f5271e82930b2b48844479c Mon Sep 17 00:00:00 2001 From: Matt Davis <6775756+nitzmahone@users.noreply.github.com> Date: Mon, 30 Jun 2025 16:33:28 -0700 Subject: [PATCH 07/68] Short-circuit legacy network module prefix->action mapping (#85406) * Short-circuit legacy network module prefix->action mapping * Modified a non-short-circuit compound conditional in a legacy networking path that attempted to resolve an action for any module name containing `_`. The bug was always present, but the typical presentation (an ImportError) was ignored prior to 2.19. * The legacy networking path should be deprecated and removed in 2.20- a module could still be run under the wrong action if one with a matching prefix is found. * unit test fix --- lib/ansible/executor/task_executor.py | 2 +- test/units/executor/test_task_executor.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/ansible/executor/task_executor.py b/lib/ansible/executor/task_executor.py index 1809b1252f1..60c6b392cbc 100644 --- a/lib/ansible/executor/task_executor.py +++ b/lib/ansible/executor/task_executor.py @@ -1129,7 +1129,7 @@ class TaskExecutor: # let action plugin override module, fallback to 'normal' action plugin otherwise elif self._shared_loader_obj.action_loader.has_plugin(self._task.action, collection_list=collections): handler_name = self._task.action - elif all((module_prefix in C.NETWORK_GROUP_MODULES, self._shared_loader_obj.action_loader.has_plugin(network_action, collection_list=collections))): + elif module_prefix in C.NETWORK_GROUP_MODULES and self._shared_loader_obj.action_loader.has_plugin(network_action, collection_list=collections): handler_name = network_action display.vvvv("Using network group action {handler} for {action}".format(handler=handler_name, action=self._task.action), diff --git a/test/units/executor/test_task_executor.py b/test/units/executor/test_task_executor.py index ba919baa28f..695230b93e9 100644 --- a/test/units/executor/test_task_executor.py +++ b/test/units/executor/test_task_executor.py @@ -273,8 +273,7 @@ class TestTaskExecutor(unittest.TestCase): self.assertIs(mock.sentinel.handler, handler) - action_loader.has_plugin.assert_has_calls([mock.call(action, collection_list=te._task.collections), - mock.call(module_prefix, collection_list=te._task.collections)]) + action_loader.has_plugin.assert_has_calls([mock.call(action, collection_list=te._task.collections)]) action_loader.get.assert_called_with( 'ansible.legacy.normal', task=te._task, connection=te._connection, From 6ff63391917cb62f0ebec338c2183d79143e9070 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Mon, 30 Jun 2025 16:33:37 -0700 Subject: [PATCH 08/68] Ensure config env/ini values are tagged (#85404) * Ensure config env/ini values are tagged Config env and ini values now have origin and trust tags applied. * Remove unused import --- lib/ansible/config/manager.py | 43 +++++++++++++------ .../lookup_template/tasks/ansible_managed.yml | 1 - test/units/config/test_manager.py | 28 ++++++++++++ 3 files changed, 58 insertions(+), 14 deletions(-) diff --git a/lib/ansible/config/manager.py b/lib/ansible/config/manager.py index f050ced4d75..c4b0ffbc362 100644 --- a/lib/ansible/config/manager.py +++ b/lib/ansible/config/manager.py @@ -6,6 +6,7 @@ from __future__ import annotations import atexit import decimal import configparser +import functools import os import os.path import sys @@ -248,18 +249,6 @@ def get_config_type(cfile): return ftype -# FIXME: can move to module_utils for use for ini plugins also? -def get_ini_config_value(p, entry): - """ returns the value of last ini entry found """ - value = None - if p is not None: - try: - value = p.get(entry.get('section', 'defaults'), entry.get('key', ''), raw=True) - except Exception: # FIXME: actually report issues here - pass - return value - - def find_ini_config_file(warnings=None): """ Load INI Config File order(first found is used): ENV, CWD, HOME, /etc/ansible """ # FIXME: eventually deprecate ini configs @@ -345,6 +334,7 @@ class ConfigManager: _errors: list[tuple[str, Exception]] def __init__(self, conf_file=None, defs_file=None): + self._get_ini_config_value = functools.cache(self._get_ini_config_value) self._base_defs = {} self._plugins = {} @@ -628,6 +618,7 @@ class ConfigManager: # env vars are next precedence if value is None and defs[config].get('env'): value, origin = self._loop_entries(os.environ, defs[config]['env']) + value = _tags.TrustedAsTemplate().tag(value) origin = 'env: %s' % origin # try config file entries next, if we have one @@ -642,7 +633,7 @@ class ConfigManager: for entry in defs[config][ftype]: # load from config if ftype == 'ini': - temp_value = get_ini_config_value(self._parsers[cfile], entry) + temp_value = self._get_ini_config_value(cfile, entry.get('section', 'defaults'), entry['key']) elif ftype == 'yaml': raise AnsibleError('YAML configuration type has not been implemented yet') else: @@ -724,6 +715,32 @@ class ConfigManager: self._plugins[plugin_type][name] = defs + def _get_ini_config_value(self, config_file: str, section: str, option: str) -> t.Any: + """ + Fetch `option` from the specified `section`. + Returns `None` if the specified `section` or `option` are not present. + Origin and TrustedAsTemplate tags are applied to returned values. + + CAUTION: Although INI sourced configuration values are trusted for templating, that does not automatically mean they will be templated. + It is up to the code consuming configuration values to apply templating if required. + """ + parser = self._parsers[config_file] + value = parser.get(section, option, raw=True, fallback=None) + + if value is not None: + value = self._apply_tags(value, section, option) + + return value + + def _apply_tags(self, value: str, section: str, option: str) -> t.Any: + """Apply origin and trust to the given `value` sourced from the stated `section` and `option`.""" + description = f'section {section!r} option {option!r}' + origin = _tags.Origin(path=self._config_file, description=description) + tags = [origin, _tags.TrustedAsTemplate()] + value = AnsibleTagHelper.tag(value, tags) + + return value + @staticmethod def get_deprecated_msg_from_config(dep_docs, include_removal=False, collection_name=None): diff --git a/test/integration/targets/lookup_template/tasks/ansible_managed.yml b/test/integration/targets/lookup_template/tasks/ansible_managed.yml index 21c0f740956..922a9a0ee07 100644 --- a/test/integration/targets/lookup_template/tasks/ansible_managed.yml +++ b/test/integration/targets/lookup_template/tasks/ansible_managed.yml @@ -1,4 +1,3 @@ -# deprecated: description='ansible_managed has been removed' core_version='2.23' - name: invoke template lookup with content using default injected `ansible_managed` debug: msg: "{{ lookup('template', 'uses_ansible_managed.j2') }}" diff --git a/test/units/config/test_manager.py b/test/units/config/test_manager.py index 65ec9c0c9a1..3bd8f05a3d2 100644 --- a/test/units/config/test_manager.py +++ b/test/units/config/test_manager.py @@ -16,6 +16,7 @@ from ansible.config.manager import ConfigManager, ensure_type, resolve_path, get from ansible.errors import AnsibleOptionsError, AnsibleError from ansible._internal._datatag._tags import Origin, VaultedValue from ansible.module_utils._internal._datatag import AnsibleTagHelper +from ansible.template import is_trusted_as_template from units.mock.vault_helper import VaultTestHelper curdir = os.path.dirname(__file__) @@ -272,3 +273,30 @@ def test_256color_support(key, expected_value): actual_value = manager.get_config_value(key) # THEN: no error assert actual_value == expected_value + + +def test_config_trust_from_env(monkeypatch: pytest.MonkeyPatch) -> None: + expected = "from test" + monkeypatch.setenv("ANSIBLE_TEST_ENTRY", expected) + result = ConfigManager().get_config_value("_Z_TEST_ENTRY") + origin = Origin.get_tag(result) + + assert result == expected + assert is_trusted_as_template(result) + assert origin and origin.description == '' + + +def test_config_trust_from_file(tmp_path: pathlib.Path) -> None: + expected = "from test" + cfg_path = tmp_path / 'test.cfg' + + cfg_path.write_text(f"[testing]\nvalid={expected}") + + result = ConfigManager(str(cfg_path)).get_config_value("_Z_TEST_ENTRY") + origin = Origin.get_tag(result) + + assert result == expected + assert is_trusted_as_template(result) + assert origin + assert origin.path == str(cfg_path) + assert origin.description == "section 'testing' option 'valid'" From a1d25cca00e204438c8cd73decdc9e2b79b11e24 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Mon, 30 Jun 2025 17:41:10 -0700 Subject: [PATCH 09/68] ansible-test - Upgrade coverage to 7.9.1 (#85411) --- changelogs/fragments/ansible-test-coverage-upgrade.yml | 2 ++ test/lib/ansible_test/_data/requirements/ansible-test.txt | 3 ++- test/lib/ansible_test/_internal/coverage_util.py | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 changelogs/fragments/ansible-test-coverage-upgrade.yml diff --git a/changelogs/fragments/ansible-test-coverage-upgrade.yml b/changelogs/fragments/ansible-test-coverage-upgrade.yml new file mode 100644 index 00000000000..bd11de61b37 --- /dev/null +++ b/changelogs/fragments/ansible-test-coverage-upgrade.yml @@ -0,0 +1,2 @@ +minor_changes: + - ansible-test - Upgrade to ``coverage`` version 7.9.1 for Python 3.9 and later. diff --git a/test/lib/ansible_test/_data/requirements/ansible-test.txt b/test/lib/ansible_test/_data/requirements/ansible-test.txt index 50f951c845b..2ff3c71d169 100644 --- a/test/lib/ansible_test/_data/requirements/ansible-test.txt +++ b/test/lib/ansible_test/_data/requirements/ansible-test.txt @@ -1,2 +1,3 @@ # The test-constraints sanity test verifies this file, but changes must be made manually to keep it in up-to-date. -coverage == 7.6.1 ; python_version >= '3.8' and python_version <= '3.13' +coverage == 7.9.1 ; python_version >= '3.9' and python_version <= '3.14' +coverage == 7.6.1 ; python_version >= '3.8' and python_version <= '3.8' diff --git a/test/lib/ansible_test/_internal/coverage_util.py b/test/lib/ansible_test/_internal/coverage_util.py index 07134419be1..3b2c7d19732 100644 --- a/test/lib/ansible_test/_internal/coverage_util.py +++ b/test/lib/ansible_test/_internal/coverage_util.py @@ -70,7 +70,8 @@ class CoverageVersion: COVERAGE_VERSIONS = ( # IMPORTANT: Keep this in sync with the ansible-test.txt requirements file. - CoverageVersion('7.6.1', 7, (3, 8), (3, 13)), + CoverageVersion('7.9.1', 7, (3, 9), (3, 14)), + CoverageVersion('7.6.1', 7, (3, 8), (3, 8)), ) """ This tuple specifies the coverage version to use for Python version ranges. From a4e357507774ef72b025f073158a385569c3f112 Mon Sep 17 00:00:00 2001 From: Xiao Wenyang Date: Wed, 2 Jul 2025 22:27:54 +0800 Subject: [PATCH 10/68] stat module, add option to return SELinux Context Added get_selinux_context option --- .../85217-stat-add-selinux-context.yml | 2 ++ lib/ansible/modules/stat.py | 20 ++++++++++++++++ test/integration/targets/stat/tasks/main.yml | 24 +++++++++++++++++++ 3 files changed, 46 insertions(+) create mode 100644 changelogs/fragments/85217-stat-add-selinux-context.yml diff --git a/changelogs/fragments/85217-stat-add-selinux-context.yml b/changelogs/fragments/85217-stat-add-selinux-context.yml new file mode 100644 index 00000000000..44e32a5d90f --- /dev/null +++ b/changelogs/fragments/85217-stat-add-selinux-context.yml @@ -0,0 +1,2 @@ +minor_changes: + - stat module - add SELinux context as a return value, and add a new option to trigger this return, which is False by default. (https://github.com/ansible/ansible/issues/85217). diff --git a/lib/ansible/modules/stat.py b/lib/ansible/modules/stat.py index 79927d70c91..a1eb14e4030 100644 --- a/lib/ansible/modules/stat.py +++ b/lib/ansible/modules/stat.py @@ -44,6 +44,14 @@ options: version_added: "2.3" get_checksum: version_added: "1.8" + get_selinux_context: + description: + - Get file SELinux context in a list V([user, role, type, range]), + and will get V([None, None, None, None]) if it is not possible to retrieve the context, + either because it does not exist or some other issue. + type: bool + default: no + version_added: '2.20' extends_documentation_fragment: - action_common_attributes - checksum_common @@ -346,6 +354,12 @@ stat: type: list sample: [ immutable, extent ] version_added: 2.3 + selinux_context: + description: The SELinux context of a path + returned: success, path exists and user can execute the path + type: list + sample: [ user, role, type, range ] + version_added: '2.20' version: description: The version/generation attribute of a file according to the filesystem returned: success, path exists, user can execute the path, lsattr is available and filesystem supports @@ -434,6 +448,7 @@ def main(): get_checksum=dict(type='bool', default=True), get_mime=dict(type='bool', default=True, aliases=['mime', 'mime_type', 'mime-type']), get_attributes=dict(type='bool', default=True, aliases=['attr', 'attributes']), + get_selinux_context=dict(type='bool', default=False), checksum_algorithm=dict(type='str', default='sha1', choices=['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'], aliases=['checksum', 'checksum_algo']), @@ -448,6 +463,7 @@ def main(): get_attr = module.params.get('get_attributes') get_checksum = module.params.get('get_checksum') checksum_algorithm = module.params.get('checksum_algorithm') + get_selinux_context = module.params.get('get_selinux_context') # main stat data try: @@ -515,6 +531,10 @@ def main(): if x in out: output[x] = out[x] + # try to get SELinux context + if get_selinux_context: + output['selinux_context'] = module.selinux_context(b_path) + module.exit_json(changed=False, stat=output) diff --git a/test/integration/targets/stat/tasks/main.yml b/test/integration/targets/stat/tasks/main.yml index 374cb2fd0eb..a79ca9195e5 100644 --- a/test/integration/targets/stat/tasks/main.yml +++ b/test/integration/targets/stat/tasks/main.yml @@ -177,3 +177,27 @@ - "stat_result.changed == false" - "stat_result.stat.mimetype == 'text/plain'" - "stat_result.stat.charset == 'us-ascii'" + +- name: check stat a file with get_selinux_context on + stat: + path: "{{ remote_tmp_dir }}/foo.txt" + get_selinux_context: True + register: stat_result + +- debug: var=stat_result + +- assert: + that: + - "'selinux_context' in stat_result.stat" + +- name: check stat a file with get_selinux_context off + stat: + path: "{{ remote_tmp_dir }}/foo.txt" + get_selinux_context: False + register: stat_result + +- debug: var=stat_result + +- assert: + that: + - "'selinux_context' not in stat_result.stat" From 557733abb112fafcccc082cb6f3266ee0ff509d0 Mon Sep 17 00:00:00 2001 From: Abhijeet Kasurde Date: Thu, 3 Jul 2025 08:02:17 -0700 Subject: [PATCH 11/68] yaml filter: update docs (#85261) Signed-off-by: Abhijeet Kasurde --- lib/ansible/plugins/filter/to_nice_yaml.yml | 34 +++++++++++++++-- lib/ansible/plugins/filter/to_yaml.yml | 41 +++++++++++++++------ 2 files changed, 60 insertions(+), 15 deletions(-) diff --git a/lib/ansible/plugins/filter/to_nice_yaml.yml b/lib/ansible/plugins/filter/to_nice_yaml.yml index 664d7ce58c0..85e512198a6 100644 --- a/lib/ansible/plugins/filter/to_nice_yaml.yml +++ b/lib/ansible/plugins/filter/to_nice_yaml.yml @@ -1,5 +1,5 @@ DOCUMENTATION: - name: to_yaml + name: to_nice_yaml author: core team version_added: 'historical' short_description: Convert variable to YAML string @@ -20,10 +20,38 @@ DOCUMENTATION: description: Affects sorting of dictionary keys. default: True type: bool - #default_style=None, canonical=None, width=None, line_break=None, encoding=None, explicit_start=None, explicit_end=None, version=None, tags=None + default_style: + description: + - Indicates the style of the scalar. + choices: + - '' + - "'" + - '"' + - '|' + - '>' + type: string + canonical: + description: + - If set to V(True), export tag type to the output. + type: bool + width: + description: Set the preferred line width. + type: int + line_break: + description: Specify the line break. + type: string + encoding: + description: Specify the output encoding. + type: string + explicit_start: + description: If set to V(True), adds an explicit start using "---". + type: bool + explicit_end: + description: If set to V(True), adds an explicit end using "...". + type: bool notes: - More options may be available, see L(PyYAML documentation, https://pyyaml.org/wiki/PyYAMLDocumentation) for details. - - 'These parameters to C(yaml.dump) will be ignored, as they are overridden internally: I(default_flow_style)' + - 'These parameters to C(yaml.dump) will be ignored, as they are overridden internally: I(default_flow_style), I(allow_unicode).' EXAMPLES: | # dump variable in a template to create a YAML document diff --git a/lib/ansible/plugins/filter/to_yaml.yml b/lib/ansible/plugins/filter/to_yaml.yml index ba71f7ae9c3..40086b1d8a5 100644 --- a/lib/ansible/plugins/filter/to_yaml.yml +++ b/lib/ansible/plugins/filter/to_yaml.yml @@ -20,21 +20,38 @@ DOCUMENTATION: description: Affects sorting of dictionary keys. default: True type: bool + default_style: + description: + - Indicates the style of the scalar. + choices: + - '' + - "'" + - '"' + - '|' + - '>' + type: string + canonical: + description: + - If set to V(True), export tag type to the output. + type: bool + width: + description: Set the preferred line width. + type: integer + line_break: + description: Specify the line break. + type: string + encoding: + description: Specify the output encoding. + type: string + explicit_start: + description: If set to V(True), adds an explicit start using "---". + type: bool + explicit_end: + description: If set to V(True), adds an explicit end using "...". + type: bool notes: - More options may be available, see L(PyYAML documentation, https://pyyaml.org/wiki/PyYAMLDocumentation) for details. - # TODO: find docs for these - #default_flow_style - #default_style - #canonical=None, - #width=None, - #line_break=None, - #encoding=None, - #explicit_start=None, - #explicit_end=None, - #version=None, - #tags=None - EXAMPLES: | # dump variable in a template to create a YAML document {{ github_workflow | to_yaml }} From 205ca648bf4e80736a4324c0521488e09c49c62e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Sj=C3=B6gren?= Date: Thu, 3 Jul 2025 17:26:09 +0200 Subject: [PATCH 12/68] ansible-doc: print where the description is missing (#85320) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ansible-doc: print where the description is missing Signed-off-by: Thomas Sjögren Co-authored-by: Sloane Hertel <19572925+s-hertel@users.noreply.github.com> --- .../ansible-doc-description-verbosity.yml | 2 ++ lib/ansible/cli/doc.py | 2 +- .../roles/test_role4/meta/argument_specs.yml | 34 +++++++++++++++++++ .../roles/test_role4/meta/main.yml | 10 ++++++ test/integration/targets/ansible-doc/runme.sh | 7 ++-- 5 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 changelogs/fragments/ansible-doc-description-verbosity.yml create mode 100644 test/integration/targets/ansible-doc/roles/test_role4/meta/argument_specs.yml create mode 100644 test/integration/targets/ansible-doc/roles/test_role4/meta/main.yml diff --git a/changelogs/fragments/ansible-doc-description-verbosity.yml b/changelogs/fragments/ansible-doc-description-verbosity.yml new file mode 100644 index 00000000000..7c000d995db --- /dev/null +++ b/changelogs/fragments/ansible-doc-description-verbosity.yml @@ -0,0 +1,2 @@ +minor_changes: + - ansible-doc - Return a more verbose error message when the ``description`` field is missing. diff --git a/lib/ansible/cli/doc.py b/lib/ansible/cli/doc.py index 0319e51d09e..6b3c27e3408 100755 --- a/lib/ansible/cli/doc.py +++ b/lib/ansible/cli/doc.py @@ -1265,7 +1265,7 @@ class DocCLI(CLI, RoleMixin): # description is specifically formatted and can either be string or list of strings if 'description' not in opt: - raise AnsibleError("All (sub-)options and return values must have a 'description' field") + raise AnsibleError("All (sub-)options and return values must have a 'description' field", obj=o) text.append('') # TODO: push this to top of for and sort by size, create indent on largest key? diff --git a/test/integration/targets/ansible-doc/roles/test_role4/meta/argument_specs.yml b/test/integration/targets/ansible-doc/roles/test_role4/meta/argument_specs.yml new file mode 100644 index 00000000000..5908f30aba2 --- /dev/null +++ b/test/integration/targets/ansible-doc/roles/test_role4/meta/argument_specs.yml @@ -0,0 +1,34 @@ +--- +argument_specs: + main: + short_description: test_role4 from roles subdir + description: + - In to am attended desirous raptures B(declared) diverted confined at. Collected instantly remaining + up certainly to C(necessary) as. Over walk dull into son boy door went new. + - At or happiness commanded daughters as. Is I(handsome) an declared at received in extended vicinity + subjects. Into miss on he over been late pain an. Only week bore boy what fat case left use. Match round + scale now style far times. Your me past an much. + author: + - John Doe (@john) + - Jane Doe (@jane) + attributes: + diff_mode: + description: Will return details on what has changed (or possibly needs changing in check_mode), when in diff mode + support: partial + details: Not all modules used support this + options: + myopt1: + description: + - First option. + type: "str" + required: true + + myopt2: + type: "int" + default: 8000 + + myopt3: + description: + - Third option. + type: "bool" + default: false diff --git a/test/integration/targets/ansible-doc/roles/test_role4/meta/main.yml b/test/integration/targets/ansible-doc/roles/test_role4/meta/main.yml new file mode 100644 index 00000000000..26d1c2fc038 --- /dev/null +++ b/test/integration/targets/ansible-doc/roles/test_role4/meta/main.yml @@ -0,0 +1,10 @@ +# This meta/main.yml exists to test that it is NOT read, with preference being +# given to the meta/argument_specs.yml file. This spec contains an additional +# entry point that the argument_specs.yml does not. If this file # were read, +# the additional entrypoints would show up in --list output, breaking # tests. +--- +argument_specs: + main: + short_description: test_role4 from roles subdir + holden: + short_description: Rocinante pilot diff --git a/test/integration/targets/ansible-doc/runme.sh b/test/integration/targets/ansible-doc/runme.sh index adeb7a30ade..701d9086795 100755 --- a/test/integration/targets/ansible-doc/runme.sh +++ b/test/integration/targets/ansible-doc/runme.sh @@ -138,7 +138,7 @@ test "$output" -eq 4 echo "testing standalone roles" # Include normal roles (no collection filter) output=$(ansible-doc -t role -l --playbook-dir . | wc -l) -test "$output" -eq 8 +test "$output" -eq 11 echo "testing role precedence" # Test that a role in the playbook dir with the same name as a role in the @@ -288,5 +288,8 @@ echo "test 'sidecar' for no extension module with .py doc" echo "test 'sidecar' for no extension module with .yml doc" [ "$(ansible-doc -M ./library -l ansible.legacy |grep -v 'UNDOCUMENTED' |grep -c facts_one)" == "1" ] -echo "Test j2 plugins get jinja2 instead of path" +echo "test j2 plugins get jinja2 instead of path" ansible-doc -t filter map 2>&1 |grep "${GREP_OPTS[@]}" '(Jinja2)' + +echo "test missing description in test_role4 argument spec" +ansible-doc -t role -r ./roles test_role4 2>&1 >/dev/null | grep -q 'Error extracting role docs from '\''test_role4'\'': All (sub-)options and return values must have a '\''description'\'' field' From e162eda1e6512790f93da860c857b2c243d8d75d Mon Sep 17 00:00:00 2001 From: Abhijeet Kasurde Date: Thu, 3 Jul 2025 09:25:11 -0700 Subject: [PATCH 13/68] apt: Mark dependencies as auto (#85170) * Mark dependent packages as auto which are installed as part of deb file installation Fixes: #78123 Co-authored-by: Matt Clay Signed-off-by: Abhijeet Kasurde --- changelogs/fragments/apt_deb_install.yml | 3 ++ lib/ansible/modules/apt.py | 28 +++++++++------- .../targets/apt/tasks/apt_deb_depend.yml | 33 +++++++++++++++++++ test/integration/targets/apt/tasks/main.yml | 2 ++ .../package_specs/stable/packagetwo-1.0.0 | 10 ++++++ 5 files changed, 65 insertions(+), 11 deletions(-) create mode 100644 changelogs/fragments/apt_deb_install.yml create mode 100644 test/integration/targets/apt/tasks/apt_deb_depend.yml create mode 100644 test/integration/targets/setup_deb_repo/files/package_specs/stable/packagetwo-1.0.0 diff --git a/changelogs/fragments/apt_deb_install.yml b/changelogs/fragments/apt_deb_install.yml new file mode 100644 index 00000000000..4f96af6c7d3 --- /dev/null +++ b/changelogs/fragments/apt_deb_install.yml @@ -0,0 +1,3 @@ +--- +bugfixes: + - apt - mark dependencies installed as part of deb file installation as auto (https://github.com/ansible/ansible/issues/78123). diff --git a/lib/ansible/modules/apt.py b/lib/ansible/modules/apt.py index 5efc3653fcf..53c403133a2 100644 --- a/lib/ansible/modules/apt.py +++ b/lib/ansible/modules/apt.py @@ -372,6 +372,7 @@ import locale as locale_module import os import re import secrets +import shlex import shutil import sys import tempfile @@ -390,8 +391,6 @@ APT_GET_ZERO = "\n0 upgraded, 0 newly installed, 0 to remove" APTITUDE_ZERO = "\n0 packages upgraded, 0 newly installed, 0 to remove" APT_LISTS_PATH = "/var/lib/apt/lists" APT_UPDATE_SUCCESS_STAMP_PATH = "/var/lib/apt/periodic/update-success-stamp" -APT_MARK_INVALID_OP = 'Invalid operation' -APT_MARK_INVALID_OP_DEB6 = 'Usage: apt-mark [options] {markauto|unmarkauto} packages' CLEAN_OP_CHANGED_STR = dict( autoremove='The following packages will be REMOVED', @@ -690,26 +689,30 @@ def parse_diff(output): return {'prepared': '\n'.join(diff[diff_start:diff_end])} -def mark_installed_manually(m, packages): +def mark_installed(m: AnsibleModule, packages: list[str], manual: bool) -> None: + """Mark packages as manually or automatically installed.""" if not packages: return + if manual: + mark_msg = "manually" + mark_op = "manual" + else: + mark_msg = "auto" + mark_op = "auto" + apt_mark_cmd_path = m.get_bin_path("apt-mark") # https://github.com/ansible/ansible/issues/40531 if apt_mark_cmd_path is None: - m.warn("Could not find apt-mark binary, not marking package(s) as manually installed.") + m.warn(f"Could not find apt-mark binary, not marking package(s) as {mark_msg} installed.") return - cmd = "%s manual %s" % (apt_mark_cmd_path, ' '.join(packages)) + cmd = [apt_mark_cmd_path, mark_op] + packages rc, out, err = m.run_command(cmd) - if APT_MARK_INVALID_OP in err or APT_MARK_INVALID_OP_DEB6 in err: - cmd = "%s unmarkauto %s" % (apt_mark_cmd_path, ' '.join(packages)) - rc, out, err = m.run_command(cmd) - if rc != 0: - m.fail_json(msg="'%s' failed: %s" % (cmd, err), stdout=out, stderr=err, rc=rc) + m.fail_json(msg=f"Command {shlex.join(cmd)!r} failed.", stdout=out, stderr=err, rc=rc) def install(m, pkgspec, cache, upgrade=False, default_release=None, @@ -835,7 +838,7 @@ def install(m, pkgspec, cache, upgrade=False, default_release=None, data = dict(changed=False) if not build_dep and not m.check_mode: - mark_installed_manually(m, package_names) + mark_installed(m, package_names, manual=True) return (status, data) @@ -914,6 +917,9 @@ def install_deb( dpkg_options=install_dpkg_options) if not success: m.fail_json(**retvals) + # Mark the dependencies as auto installed + # https://github.com/ansible/ansible/issues/78123 + mark_installed(m, deps_to_install, manual=False) changed = retvals.get('changed', False) if pkgs_to_install: diff --git a/test/integration/targets/apt/tasks/apt_deb_depend.yml b/test/integration/targets/apt/tasks/apt_deb_depend.yml new file mode 100644 index 00000000000..0cb26b6e6d0 --- /dev/null +++ b/test/integration/targets/apt/tasks/apt_deb_depend.yml @@ -0,0 +1,33 @@ +- block: + - name: Clean up before running tests + apt: + name: + - packageone + - packagetwo + state: absent + + - name: Install packageone from deb URL + apt: + deb: https://ci-files.testing.ansible.com/test/integration/targets/apt/packageone_1.0_all.deb + register: packageone_installed + + - name: Check if packagetwo is installed as part of packageone installation + shell: dpkg -s packagetwo + + - name: Check if packageone and packagetwo are marked as auto + shell: apt-mark showauto packageone packagetwo + register: auto_installed + + - name: Make sure packageone is installed manually and packagetwo is marked as auto + assert: + that: + - packageone_installed.changed + - auto_installed.stdout_lines == ["packagetwo"] + + always: + - name: Clean up after tests + apt: + name: + - packageone + - packagetwo + state: absent diff --git a/test/integration/targets/apt/tasks/main.yml b/test/integration/targets/apt/tasks/main.yml index f9c185eba0f..4ae93219d69 100644 --- a/test/integration/targets/apt/tasks/main.yml +++ b/test/integration/targets/apt/tasks/main.yml @@ -22,6 +22,8 @@ - block: - import_tasks: 'apt.yml' + - import_tasks: 'apt_deb_depend.yml' + - import_tasks: 'url-with-deps.yml' - import_tasks: 'apt-multiarch.yml' diff --git a/test/integration/targets/setup_deb_repo/files/package_specs/stable/packagetwo-1.0.0 b/test/integration/targets/setup_deb_repo/files/package_specs/stable/packagetwo-1.0.0 new file mode 100644 index 00000000000..c02d6845524 --- /dev/null +++ b/test/integration/targets/setup_deb_repo/files/package_specs/stable/packagetwo-1.0.0 @@ -0,0 +1,10 @@ +Section: misc +Priority: optional +Standards-Version: 2.3.3 + +Package: packagetwo +Version: 1.0 +Section: system +Maintainer: John Doe +Architecture: all +Description: Dummy package upon which packageone depends From a032f96222670adab880aaedadd683c6fcd6e59d Mon Sep 17 00:00:00 2001 From: Patrick Kingston <66141901+pkingstonxyz@users.noreply.github.com> Date: Thu, 3 Jul 2025 15:59:51 -0400 Subject: [PATCH 14/68] Expand expect module test suite (#85422) * Test expect module for missing pexpect * Test expect module for incorrect timeout type * Test expect module for out of date pexpect failures * Test pexpect.ExceptionPexpect Also fixed task naming to be inline with the rest of the file. * Change pexpect management to venv * Normalize b_out to b'' when falsy for consistent handling * Move test venv from /tmp/ to output_dir * Update tests according to corrections * Remove break_system_packages when installing pexpect 3.3 * Remove unnecessary if statement pexpect.run(...) shouldn't ever return None, so it's safe to remove the check entirely. Maintains 100% coverage --- lib/ansible/modules/expect.py | 3 - .../integration/targets/expect/tasks/main.yml | 92 +++++++++++++++++++ 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/lib/ansible/modules/expect.py b/lib/ansible/modules/expect.py index e9ac2bc13e4..d2d537f1a7d 100644 --- a/lib/ansible/modules/expect.py +++ b/lib/ansible/modules/expect.py @@ -249,9 +249,6 @@ def main(): end_date = datetime.datetime.now() delta = end_date - start_date - if b_out is None: - b_out = b'' - result = dict( cmd=args, stdout=to_native(b_out).rstrip('\r\n'), diff --git a/test/integration/targets/expect/tasks/main.yml b/test/integration/targets/expect/tasks/main.yml index ddb78f8e5da..cd053ddd87a 100644 --- a/test/integration/targets/expect/tasks/main.yml +++ b/test/integration/targets/expect/tasks/main.yml @@ -241,3 +241,95 @@ that: - "non_zero_result.failed" - "non_zero_result.msg == 'non-zero return code'" + +- name: run tests in venv + vars: + venv_dir: "{{ output_dir }}/pexpected_venv" + venv_python: "{{ venv_dir }}/bin/python" + block: + - name: ensure venv exists + pip: + name: coverage + virtualenv: "{{ venv_dir }}" + virtualenv_command: "{{ ansible_python_interpreter }} -m venv" + + - name: trigger pexpect dependency absence + expect: + command: "echo expectThePexpected" + responses: + foo: bar + vars: + ansible_python_interpreter: "{{ venv_python }}" + register: pexpect_dependency_absence + ignore_errors: true + + - name: assert pexpect dependency absence + assert: + that: + - pexpect_dependency_absence is failed + - pexpect_dependency_absence.msg is contains 'pexpect' + + - name: Install pexpect3.3 + pip: + name: pexpect + version: 3.3 + virtualenv: "{{ venv_dir }}" + + - name: trigger fallback to pexpect3.3 methods + expect: + command: "echo pexpectIsABitOutOfDate" + responses: + foo: bar + vars: + ansible_python_interpreter: "{{ venv_python }}" + register: pexpect_version_three_dot_three + + - name: install pexpect3.0 + pip: + name: pexpect + version: 3.0 + virtualenv: "{{ venv_dir }}" + + - name: trigger failure with out of date pexpect version + expect: + command: "echo pexpectIsTooOutOfDate" + responses: + foo: bar + vars: + ansible_python_interpreter: "{{ venv_python }}" + register: pexpect_version_out_of_date + ignore_errors: true + + - name: assert failure with out of date pexpect + assert: + that: + - pexpect_version_out_of_date is failed + - pexpect_version_out_of_date.msg is contains 'Insufficient version of pexpect installed' + +- name: trigger TypeError on timeout value + expect: + command: "echo piIsNotAnInt" + responses: + foo: bar + timeout: pi + register: pexpect_timeout_typeerror + ignore_errors: true + +- name: assert timeout triggers TypeError when not an int + assert: + that: + - pexpect_timeout_typeerror is failed + - pexpect_timeout_typeerror.msg is contains 'cannot be converted to an int' + +- name: trigger pexpect.ExceptionPexpect + expect: + command: commandthatdoesntexist + responses: + foo: bar + register: pexpect_exception_pexpect + ignore_errors: true + +- name: assert failure for a nonexistent command + assert: + that: + - pexpect_exception_pexpect is failed \ No newline at end of file From 6b347bfbf7bd2c0c38201eedf66ed40a5dcb3040 Mon Sep 17 00:00:00 2001 From: Matt Davis <6775756+nitzmahone@users.noreply.github.com> Date: Thu, 3 Jul 2025 13:21:28 -0700 Subject: [PATCH 15/68] default _ansible_tracebacks_for to [] when not specified (#85427) * eases module unit tests that aren't using our fixture --- .../module_utils/_internal/_traceback.py | 2 +- .../module_utils/_internal/test_traceback.py | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 test/units/module_utils/_internal/test_traceback.py diff --git a/lib/ansible/module_utils/_internal/_traceback.py b/lib/ansible/module_utils/_internal/_traceback.py index 473765eff74..a5fdc45afe0 100644 --- a/lib/ansible/module_utils/_internal/_traceback.py +++ b/lib/ansible/module_utils/_internal/_traceback.py @@ -80,7 +80,7 @@ def _is_module_traceback_enabled(event: TracebackEvent) -> bool: from ..basic import _PARSED_MODULE_ARGS _module_tracebacks_enabled_events = frozenset( - TracebackEvent[value.upper()] for value in _PARSED_MODULE_ARGS.get('_ansible_tracebacks_for') + TracebackEvent[value.upper()] for value in _PARSED_MODULE_ARGS.get('_ansible_tracebacks_for', []) ) # type: ignore[union-attr] except BaseException: return True # if things failed early enough that we can't figure this out, assume we want a traceback for troubleshooting diff --git a/test/units/module_utils/_internal/test_traceback.py b/test/units/module_utils/_internal/test_traceback.py new file mode 100644 index 00000000000..01513788b57 --- /dev/null +++ b/test/units/module_utils/_internal/test_traceback.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import pytest +import pytest_mock + +from ansible.module_utils._internal import _traceback + + +@pytest.mark.parametrize("patched_parsed_args, event, expected", ( + (dict(_ansible_tracebacks_for=["error", "warning"]), _traceback.TracebackEvent.ERROR, True), # included value + (dict(_ansible_tracebacks_for=["error", "warning"]), _traceback.TracebackEvent.WARNING, True), # included value + (dict(_ansible_tracebacks_for=["error", "warning"]), _traceback.TracebackEvent.DEPRECATED, False), # excluded value + ({}, _traceback.TracebackEvent.ERROR, False), # unspecified defaults to no tracebacks + (dict(_ansible_tracebacks_for="bogus,values"), _traceback.TracebackEvent.ERROR, True), # parse failure defaults to always enabled + (None, _traceback.TracebackEvent.ERROR, True), # fetch failure defaults to always enabled +), ids=str) +def test_default_module_traceback_config( + patched_parsed_args: dict | None, + event: _traceback.TracebackEvent, + expected: bool, + mocker: pytest_mock.MockerFixture +) -> None: + """Validate MU traceback config behavior (including unconfigured/broken config fallbacks).""" + from ansible.module_utils import basic + + mocker.patch.object(basic, '_PARSED_MODULE_ARGS', patched_parsed_args) + + # this should just be an importlib.reload() on _traceback, but that redeclares the enum type and breaks the world + mocker.patch.object(_traceback, '_module_tracebacks_enabled_events', None) + + assert _traceback._is_module_traceback_enabled(event=event) is expected From 35252fd96cd10c99f441188ac9ac28c092a3a4bc Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Thu, 3 Jul 2025 14:12:38 -0700 Subject: [PATCH 16/68] Drop incidental network tests from CI (#85425) --- .azure-pipelines/azure-pipelines.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml index 056dfbeb68b..3c73f3f2590 100644 --- a/.azure-pipelines/azure-pipelines.yml +++ b/.azure-pipelines/azure-pipelines.yml @@ -220,15 +220,6 @@ stages: test: 2025/psrp/http - name: 2025 SSH Key test: 2025/ssh/key - - stage: Incidental - dependsOn: [] - jobs: - - template: templates/matrix.yml - parameters: - testFormat: i/{0}/1 - targets: - - name: IOS Python - test: ios/csr1000v/ - stage: Summary condition: succeededOrFailed() dependsOn: @@ -240,6 +231,5 @@ stages: - Galaxy - Generic - Incidental_Windows - - Incidental jobs: - template: templates/coverage.yml From d19366331fa1689e13148a0e1c41de8f9af8c1ea Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Thu, 3 Jul 2025 16:25:33 -0700 Subject: [PATCH 17/68] ansible-test - Remove obsolete network provisioning (#85433) --- .azure-pipelines/commands/incidental/ios.sh | 1 - .../commands/incidental/network.sh | 40 - changelogs/fragments/ansible-test-ios.yml | 2 + test/integration/network-integration.cfg | 14 - .../network-integration.requirements.txt | 1 - .../ansible_test/_data/completion/network.txt | 1 - test/sanity/ignore.txt | 1 - .../netcommon/plugins/action/cli_config.py | 39 - .../netcommon/plugins/action/net_get.py | 198 -- .../netcommon/plugins/action/net_put.py | 234 -- .../netcommon/plugins/action/network.py | 206 -- .../plugins/connection/network_cli.py | 1386 --------- .../plugins/connection/persistent.py | 96 - .../doc_fragments/connection_persistent.py | 75 - .../plugins/module_utils/compat/ipaddress.py | 2578 ----------------- .../module_utils/network/common/cfg/base.py | 28 - .../module_utils/network/common/config.py | 475 --- .../network/common/facts/facts.py | 164 -- .../module_utils/network/common/netconf.py | 181 -- .../module_utils/network/common/network.py | 276 -- .../module_utils/network/common/parsing.py | 317 -- .../module_utils/network/common/utils.py | 673 ----- .../netcommon/plugins/modules/cli_config.py | 442 --- .../plugins/plugin_utils/connection_base.py | 184 -- .../cisco/ios/plugins/action/ios.py | 132 - .../cisco/ios/plugins/cliconf/ios.py | 465 --- .../cisco/ios/plugins/doc_fragments/ios.py | 82 - .../plugins/module_utils/network/ios/ios.py | 199 -- .../cisco/ios/plugins/modules/ios_command.py | 230 -- .../cisco/ios/plugins/modules/ios_config.py | 599 ---- .../cisco/ios/plugins/terminal/ios.py | 114 - 31 files changed, 2 insertions(+), 9431 deletions(-) delete mode 120000 .azure-pipelines/commands/incidental/ios.sh delete mode 100755 .azure-pipelines/commands/incidental/network.sh create mode 100644 changelogs/fragments/ansible-test-ios.yml delete mode 100644 test/integration/network-integration.cfg delete mode 100644 test/integration/network-integration.requirements.txt delete mode 100644 test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/action/cli_config.py delete mode 100644 test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/action/net_get.py delete mode 100644 test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/action/net_put.py delete mode 100644 test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/action/network.py delete mode 100644 test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/connection/network_cli.py delete mode 100644 test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/connection/persistent.py delete mode 100644 test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/doc_fragments/connection_persistent.py delete mode 100644 test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/compat/ipaddress.py delete mode 100644 test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/cfg/base.py delete mode 100644 test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/config.py delete mode 100644 test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/facts/facts.py delete mode 100644 test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/netconf.py delete mode 100644 test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/network.py delete mode 100644 test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/parsing.py delete mode 100644 test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/utils.py delete mode 100644 test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/modules/cli_config.py delete mode 100644 test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/plugin_utils/connection_base.py delete mode 100644 test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/action/ios.py delete mode 100644 test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/cliconf/ios.py delete mode 100644 test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/doc_fragments/ios.py delete mode 100644 test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/module_utils/network/ios/ios.py delete mode 100644 test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/modules/ios_command.py delete mode 100644 test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/modules/ios_config.py delete mode 100644 test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/terminal/ios.py diff --git a/.azure-pipelines/commands/incidental/ios.sh b/.azure-pipelines/commands/incidental/ios.sh deleted file mode 120000 index cad3e41b707..00000000000 --- a/.azure-pipelines/commands/incidental/ios.sh +++ /dev/null @@ -1 +0,0 @@ -network.sh \ No newline at end of file diff --git a/.azure-pipelines/commands/incidental/network.sh b/.azure-pipelines/commands/incidental/network.sh deleted file mode 100755 index 1c489f9e31c..00000000000 --- a/.azure-pipelines/commands/incidental/network.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -set -o pipefail -eux - -declare -a args -IFS='/:' read -ra args <<< "$1" - -platform="${args[0]}" -version="${args[1]}" -python_version="${args[2]}" - -target="shippable/${platform}/incidental/" - -stage="${S:-prod}" -provider="${P:-default}" - -# python versions to test in order -# all versions run full tests -IFS=' ' read -r -a python_versions <<< \ - "$(PYTHONPATH="${PWD}/test/lib" python -c 'from ansible_test._internal import constants; print(" ".join(constants.CONTROLLER_PYTHON_VERSIONS))')" - -if [ "${python_version}" ]; then - # limit tests to a single python version - python_versions=("${python_version}") -fi - -for python_version in "${python_versions[@]}"; do - # terminate remote instances on the final python version tested - if [ "${python_version}" = "${python_versions[-1]}" ]; then - terminate="always" - else - terminate="never" - fi - - # shellcheck disable=SC2086 - ansible-test network-integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \ - --platform "${platform}/${version}" \ - --docker default --python "${python_version}" \ - --remote-terminate "${terminate}" --remote-stage "${stage}" --remote-provider "${provider}" -done diff --git a/changelogs/fragments/ansible-test-ios.yml b/changelogs/fragments/ansible-test-ios.yml new file mode 100644 index 00000000000..671758596b2 --- /dev/null +++ b/changelogs/fragments/ansible-test-ios.yml @@ -0,0 +1,2 @@ +minor_changes: + - ansible-test - Removed support for automatic provisioning of obsolete instances for network-integration tests. diff --git a/test/integration/network-integration.cfg b/test/integration/network-integration.cfg deleted file mode 100644 index 00764bcdadb..00000000000 --- a/test/integration/network-integration.cfg +++ /dev/null @@ -1,14 +0,0 @@ -# NOTE: This file is used by ansible-test to override specific Ansible constants -# This file is used by `ansible-test network-integration` - -[defaults] -host_key_checking = False -timeout = 90 - -[ssh_connection] -ssh_args = '-o UserKnownHostsFile=/dev/null' - -[persistent_connection] -command_timeout = 100 -connect_timeout = 100 -connect_retry_timeout = 100 diff --git a/test/integration/network-integration.requirements.txt b/test/integration/network-integration.requirements.txt deleted file mode 100644 index 9c4d78d6f4f..00000000000 --- a/test/integration/network-integration.requirements.txt +++ /dev/null @@ -1 +0,0 @@ -scp # needed by incidental_ios_file diff --git a/test/lib/ansible_test/_data/completion/network.txt b/test/lib/ansible_test/_data/completion/network.txt index cb523d1e4b8..e69de29bb2d 100644 --- a/test/lib/ansible_test/_data/completion/network.txt +++ b/test/lib/ansible_test/_data/completion/network.txt @@ -1 +0,0 @@ -ios/csr1000v collection=cisco.ios connection=ansible.netcommon.network_cli provider=aws arch=x86_64 diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index 994e031da82..c75cef48855 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -115,7 +115,6 @@ test/integration/targets/win_script/files/test_script_with_args.ps1 pslint:PSAvo test/integration/targets/win_script/files/test_script_with_splatting.ps1 pslint:PSAvoidUsingWriteHost # Keep test/integration/targets/ssh_agent/fake_agents/ssh-agent-bad-shebang shebang # required for test test/lib/ansible_test/_data/requirements/sanity.pslint.ps1 pslint:PSCustomUseLiteralPath # Uses wildcards on purpose -test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/cliconf/ios.py pylint:arguments-renamed test/support/windows-integration/collections/ansible_collections/ansible/windows/plugins/module_utils/WebRequest.psm1 pslint!skip test/support/windows-integration/collections/ansible_collections/ansible/windows/plugins/modules/win_uri.ps1 pslint!skip test/support/windows-integration/plugins/modules/async_status.ps1 pslint!skip diff --git a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/action/cli_config.py b/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/action/cli_config.py deleted file mode 100644 index 1dbf890eaf3..00000000000 --- a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/action/cli_config.py +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright 2018 Red Hat Inc. -# -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . -# -from __future__ import annotations - - -from ansible_collections.ansible.netcommon.plugins.action.network import ( - ActionModule as ActionNetworkModule, -) - - -class ActionModule(ActionNetworkModule): - def run(self, tmp=None, task_vars=None): - del tmp # tmp no longer has any effect - - self._config_module = True - if self._play_context.connection.split(".")[-1] != "network_cli": - return { - "failed": True, - "msg": "Connection type %s is not valid for cli_config module" - % self._play_context.connection, - } - - return super(ActionModule, self).run(task_vars=task_vars) diff --git a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/action/net_get.py b/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/action/net_get.py deleted file mode 100644 index 448b970a46a..00000000000 --- a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/action/net_get.py +++ /dev/null @@ -1,198 +0,0 @@ -# (c) 2018, Ansible Inc, -# -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . -from __future__ import annotations - - -import os -import re -import uuid -import hashlib - -from ansible.errors import AnsibleError -from ansible.module_utils.common.text.converters import to_text, to_bytes -from ansible.module_utils.connection import Connection, ConnectionError -from ansible.plugins.action import ActionBase -from ansible.module_utils.six.moves.urllib.parse import urlsplit -from ansible.utils.display import Display - -display = Display() - - -class ActionModule(ActionBase): - def run(self, tmp=None, task_vars=None): - socket_path = None - self._get_network_os(task_vars) - persistent_connection = self._play_context.connection.split(".")[-1] - - result = super(ActionModule, self).run(task_vars=task_vars) - - if persistent_connection != "network_cli": - # It is supported only with network_cli - result["failed"] = True - result["msg"] = ( - "connection type %s is not valid for net_get module," - " please use fully qualified name of network_cli connection type" - % self._play_context.connection - ) - return result - - try: - src = self._task.args["src"] - except KeyError as exc: - return { - "failed": True, - "msg": "missing required argument: %s" % exc, - } - - # Get destination file if specified - dest = self._task.args.get("dest") - - if dest is None: - dest = self._get_default_dest(src) - else: - dest = self._handle_dest_path(dest) - - # Get proto - proto = self._task.args.get("protocol") - if proto is None: - proto = "scp" - - if socket_path is None: - socket_path = self._connection.socket_path - - conn = Connection(socket_path) - sock_timeout = conn.get_option("persistent_command_timeout") - - try: - changed = self._handle_existing_file( - conn, src, dest, proto, sock_timeout - ) - if changed is False: - result["changed"] = changed - result["destination"] = dest - return result - except Exception as exc: - result["msg"] = ( - "Warning: %s idempotency check failed. Check dest" % exc - ) - - try: - conn.get_file( - source=src, destination=dest, proto=proto, timeout=sock_timeout - ) - except Exception as exc: - result["failed"] = True - result["msg"] = "Exception received: %s" % exc - - result["changed"] = changed - result["destination"] = dest - return result - - def _handle_dest_path(self, dest): - working_path = self._get_working_path() - - if os.path.isabs(dest) or urlsplit("dest").scheme: - dst = dest - else: - dst = self._loader.path_dwim_relative(working_path, "", dest) - - return dst - - def _get_src_filename_from_path(self, src_path): - filename_list = re.split("/|:", src_path) - return filename_list[-1] - - def _get_default_dest(self, src_path): - dest_path = self._get_working_path() - src_fname = self._get_src_filename_from_path(src_path) - filename = "%s/%s" % (dest_path, src_fname) - return filename - - def _handle_existing_file(self, conn, source, dest, proto, timeout): - """ - Determines whether the source and destination file match. - - :return: False if source and dest both exist and have matching sha1 sums, True otherwise. - """ - if not os.path.exists(dest): - return True - - cwd = self._loader.get_basedir() - filename = str(uuid.uuid4()) - tmp_dest_file = os.path.join(cwd, filename) - try: - conn.get_file( - source=source, - destination=tmp_dest_file, - proto=proto, - timeout=timeout, - ) - except ConnectionError as exc: - error = to_text(exc) - if error.endswith("No such file or directory"): - if os.path.exists(tmp_dest_file): - os.remove(tmp_dest_file) - return True - - try: - with open(tmp_dest_file, "r") as f: - new_content = f.read() - with open(dest, "r") as f: - old_content = f.read() - except (IOError, OSError): - os.remove(tmp_dest_file) - raise - - sha1 = hashlib.sha1() - old_content_b = to_bytes(old_content, errors="surrogate_or_strict") - sha1.update(old_content_b) - checksum_old = sha1.digest() - - sha1 = hashlib.sha1() - new_content_b = to_bytes(new_content, errors="surrogate_or_strict") - sha1.update(new_content_b) - checksum_new = sha1.digest() - os.remove(tmp_dest_file) - if checksum_old == checksum_new: - return False - return True - - def _get_working_path(self): - cwd = self._loader.get_basedir() - if self._task._role is not None: - cwd = self._task._role._role_path - return cwd - - def _get_network_os(self, task_vars): - if "network_os" in self._task.args and self._task.args["network_os"]: - display.vvvv("Getting network OS from task argument") - network_os = self._task.args["network_os"] - elif self._play_context.network_os: - display.vvvv("Getting network OS from inventory") - network_os = self._play_context.network_os - elif ( - "network_os" in task_vars.get("ansible_facts", {}) - and task_vars["ansible_facts"]["network_os"] - ): - display.vvvv("Getting network OS from fact") - network_os = task_vars["ansible_facts"]["network_os"] - else: - raise AnsibleError( - "ansible_network_os must be specified on this host" - ) - - return network_os diff --git a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/action/net_put.py b/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/action/net_put.py deleted file mode 100644 index 6b769e9060f..00000000000 --- a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/action/net_put.py +++ /dev/null @@ -1,234 +0,0 @@ -# (c) 2018, Ansible Inc, -# -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . -from __future__ import annotations - - -import os -import uuid -import hashlib - -from ansible.errors import AnsibleError -from ansible.module_utils.common.text.converters import to_text, to_bytes -from ansible.module_utils.connection import Connection, ConnectionError -from ansible.plugins.action import ActionBase -from ansible.module_utils.six.moves.urllib.parse import urlsplit -from ansible.utils.display import Display - -display = Display() - - -class ActionModule(ActionBase): - def run(self, tmp=None, task_vars=None): - socket_path = None - network_os = self._get_network_os(task_vars).split(".")[-1] - persistent_connection = self._play_context.connection.split(".")[-1] - - result = super(ActionModule, self).run(task_vars=task_vars) - - if persistent_connection != "network_cli": - # It is supported only with network_cli - result["failed"] = True - result["msg"] = ( - "connection type %s is not valid for net_put module," - " please use fully qualified name of network_cli connection type" - % self._play_context.connection - ) - return result - - try: - src = self._task.args["src"] - except KeyError as exc: - return { - "failed": True, - "msg": "missing required argument: %s" % exc, - } - - src_file_path_name = src - - # Get destination file if specified - dest = self._task.args.get("dest") - - # Get proto - proto = self._task.args.get("protocol") - if proto is None: - proto = "scp" - - # Get mode if set - mode = self._task.args.get("mode") - if mode is None: - mode = "binary" - - if mode == "text": - try: - self._handle_template(convert_data=False) - except ValueError as exc: - return dict(failed=True, msg=to_text(exc)) - - # Now src has resolved file write to disk in current directory for scp - src = self._task.args.get("src") - filename = str(uuid.uuid4()) - cwd = self._loader.get_basedir() - output_file = os.path.join(cwd, filename) - try: - with open(output_file, "wb") as f: - f.write(to_bytes(src, encoding="utf-8")) - except Exception: - os.remove(output_file) - raise - else: - try: - output_file = self._get_binary_src_file(src) - except ValueError as exc: - return dict(failed=True, msg=to_text(exc)) - - if socket_path is None: - socket_path = self._connection.socket_path - - conn = Connection(socket_path) - sock_timeout = conn.get_option("persistent_command_timeout") - - if dest is None: - dest = src_file_path_name - - try: - changed = self._handle_existing_file( - conn, output_file, dest, proto, sock_timeout - ) - if changed is False: - result["changed"] = changed - result["destination"] = dest - return result - except Exception as exc: - result["msg"] = ( - "Warning: %s idempotency check failed. Check dest" % exc - ) - - try: - conn.copy_file( - source=output_file, - destination=dest, - proto=proto, - timeout=sock_timeout, - ) - except Exception as exc: - if to_text(exc) == "No response from server": - if network_os == "iosxr": - # IOSXR sometimes closes socket prematurely after completion - # of file transfer - result[ - "msg" - ] = "Warning: iosxr scp server pre close issue. Please check dest" - else: - result["failed"] = True - result["msg"] = "Exception received: %s" % exc - - if mode == "text": - # Cleanup tmp file expanded with ansible vars - os.remove(output_file) - - result["changed"] = changed - result["destination"] = dest - return result - - def _handle_existing_file(self, conn, source, dest, proto, timeout): - """ - Determines whether the source and destination file match. - - :return: False if source and dest both exist and have matching sha1 sums, True otherwise. - """ - cwd = self._loader.get_basedir() - filename = str(uuid.uuid4()) - tmp_source_file = os.path.join(cwd, filename) - try: - conn.get_file( - source=dest, - destination=tmp_source_file, - proto=proto, - timeout=timeout, - ) - except ConnectionError as exc: - error = to_text(exc) - if error.endswith("No such file or directory"): - if os.path.exists(tmp_source_file): - os.remove(tmp_source_file) - return True - - try: - with open(source, "r") as f: - new_content = f.read() - with open(tmp_source_file, "r") as f: - old_content = f.read() - except (IOError, OSError): - os.remove(tmp_source_file) - raise - - sha1 = hashlib.sha1() - old_content_b = to_bytes(old_content, errors="surrogate_or_strict") - sha1.update(old_content_b) - checksum_old = sha1.digest() - - sha1 = hashlib.sha1() - new_content_b = to_bytes(new_content, errors="surrogate_or_strict") - sha1.update(new_content_b) - checksum_new = sha1.digest() - os.remove(tmp_source_file) - if checksum_old == checksum_new: - return False - return True - - def _get_binary_src_file(self, src): - working_path = self._get_working_path() - - if os.path.isabs(src) or urlsplit("src").scheme: - source = src - else: - source = self._loader.path_dwim_relative( - working_path, "templates", src - ) - if not source: - source = self._loader.path_dwim_relative(working_path, src) - - if not os.path.exists(source): - raise ValueError("path specified in src not found") - - return source - - def _get_working_path(self): - cwd = self._loader.get_basedir() - if self._task._role is not None: - cwd = self._task._role._role_path - return cwd - - def _get_network_os(self, task_vars): - if "network_os" in self._task.args and self._task.args["network_os"]: - display.vvvv("Getting network OS from task argument") - network_os = self._task.args["network_os"] - elif self._play_context.network_os: - display.vvvv("Getting network OS from inventory") - network_os = self._play_context.network_os - elif ( - "network_os" in task_vars.get("ansible_facts", {}) - and task_vars["ansible_facts"]["network_os"] - ): - display.vvvv("Getting network OS from fact") - network_os = task_vars["ansible_facts"]["network_os"] - else: - raise AnsibleError( - "ansible_network_os must be specified on this host" - ) - - return network_os diff --git a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/action/network.py b/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/action/network.py deleted file mode 100644 index ca7a100a251..00000000000 --- a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/action/network.py +++ /dev/null @@ -1,206 +0,0 @@ -# -# (c) 2018 Red Hat Inc. -# -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . -# -from __future__ import annotations - - -import os -import time -import re - -from ansible.errors import AnsibleError -from ansible.module_utils.common.text.converters import to_text, to_bytes -from ansible.module_utils.six.moves.urllib.parse import urlsplit -from ansible.plugins.action.normal import ActionModule as _ActionModule -from ansible.utils.display import Display - -display = Display() - -PRIVATE_KEYS_RE = re.compile("__.+__") - - -class ActionModule(_ActionModule): - def run(self, task_vars=None): - config_module = hasattr(self, "_config_module") and self._config_module - if config_module and self._task.args.get("src"): - try: - self._handle_src_option() - except AnsibleError as e: - return {"failed": True, "msg": e.message, "changed": False} - - result = super(ActionModule, self).run(task_vars=task_vars) - - if ( - config_module - and self._task.args.get("backup") - and not result.get("failed") - ): - self._handle_backup_option(result, task_vars) - - return result - - def _handle_backup_option(self, result, task_vars): - - filename = None - backup_path = None - try: - content = result["__backup__"] - except KeyError: - raise AnsibleError("Failed while reading configuration backup") - - backup_options = self._task.args.get("backup_options") - if backup_options: - filename = backup_options.get("filename") - backup_path = backup_options.get("dir_path") - - if not backup_path: - cwd = self._get_working_path() - backup_path = os.path.join(cwd, "backup") - if not filename: - tstamp = time.strftime( - "%Y-%m-%d@%H:%M:%S", time.localtime(time.time()) - ) - filename = "%s_config.%s" % ( - task_vars["inventory_hostname"], - tstamp, - ) - - dest = os.path.join(backup_path, filename) - backup_path = os.path.expanduser( - os.path.expandvars( - to_bytes(backup_path, errors="surrogate_or_strict") - ) - ) - - if not os.path.exists(backup_path): - os.makedirs(backup_path) - - new_task = self._task.copy() - for item in self._task.args: - if not item.startswith("_"): - new_task.args.pop(item, None) - - new_task.args.update(dict(content=content, dest=dest)) - copy_action = self._shared_loader_obj.action_loader.get( - "copy", - task=new_task, - connection=self._connection, - play_context=self._play_context, - loader=self._loader, - templar=self._templar, - shared_loader_obj=self._shared_loader_obj, - ) - copy_result = copy_action.run(task_vars=task_vars) - if copy_result.get("failed"): - result["failed"] = copy_result["failed"] - result["msg"] = copy_result.get("msg") - return - - result["backup_path"] = dest - if copy_result.get("changed", False): - result["changed"] = copy_result["changed"] - - if backup_options and backup_options.get("filename"): - result["date"] = time.strftime( - "%Y-%m-%d", - time.gmtime(os.stat(result["backup_path"]).st_ctime), - ) - result["time"] = time.strftime( - "%H:%M:%S", - time.gmtime(os.stat(result["backup_path"]).st_ctime), - ) - - else: - result["date"] = tstamp.split("@")[0] - result["time"] = tstamp.split("@")[1] - result["shortname"] = result["backup_path"][::-1].split(".", 1)[1][ - ::-1 - ] - result["filename"] = result["backup_path"].split("/")[-1] - - # strip out any keys that have two leading and two trailing - # underscore characters - for key in list(result.keys()): - if PRIVATE_KEYS_RE.match(key): - del result[key] - - def _get_working_path(self): - cwd = self._loader.get_basedir() - if self._task._role is not None: - cwd = self._task._role._role_path - return cwd - - def _handle_src_option(self, convert_data=True): - src = self._task.args.get("src") - working_path = self._get_working_path() - - if os.path.isabs(src) or urlsplit("src").scheme: - source = src - else: - source = self._loader.path_dwim_relative( - working_path, "templates", src - ) - if not source: - source = self._loader.path_dwim_relative(working_path, src) - - if not os.path.exists(source): - raise AnsibleError("path specified in src not found") - - try: - with open(source, "r") as f: - template_data = to_text(f.read()) - except IOError as e: - raise AnsibleError( - "unable to load src file {0}, I/O error({1}): {2}".format( - source, e.errno, e.strerror - ) - ) - - # Create a template search path in the following order: - # [working_path, self_role_path, dependent_role_paths, dirname(source)] - searchpath = [working_path] - if self._task._role is not None: - searchpath.append(self._task._role._role_path) - if hasattr(self._task, "_block:"): - dep_chain = self._task._block.get_dep_chain() - if dep_chain is not None: - for role in dep_chain: - searchpath.append(role._role_path) - searchpath.append(os.path.dirname(source)) - templar = self._templar.copy_with_new_env(searchpath=searchpath) - self._task.args["src"] = templar.template(template_data) - - def _get_network_os(self, task_vars): - if "network_os" in self._task.args and self._task.args["network_os"]: - display.vvvv("Getting network OS from task argument") - network_os = self._task.args["network_os"] - elif self._play_context.network_os: - display.vvvv("Getting network OS from inventory") - network_os = self._play_context.network_os - elif ( - "network_os" in task_vars.get("ansible_facts", {}) - and task_vars["ansible_facts"]["network_os"] - ): - display.vvvv("Getting network OS from fact") - network_os = task_vars["ansible_facts"]["network_os"] - else: - raise AnsibleError( - "ansible_network_os must be specified on this host" - ) - - return network_os diff --git a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/connection/network_cli.py b/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/connection/network_cli.py deleted file mode 100644 index 5ebd1924541..00000000000 --- a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/connection/network_cli.py +++ /dev/null @@ -1,1386 +0,0 @@ -# (c) 2016 Red Hat Inc. -# (c) 2017 Ansible Project -# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) - -from __future__ import annotations - - -DOCUMENTATION = """ -author: - - Ansible Networking Team (@ansible-network) -name: network_cli -short_description: Use network_cli to run command on network appliances -description: -- This connection plugin provides a connection to remote devices over the SSH and - implements a CLI shell. This connection plugin is typically used by network devices - for sending and receiving CLi commands to network devices. -version_added: 1.0.0 -requirements: -- ansible-pylibssh if using I(ssh_type=libssh) -extends_documentation_fragment: -- ansible.netcommon.connection_persistent -options: - host: - description: - - Specifies the remote device FQDN or IP address to establish the SSH connection - to. - default: inventory_hostname - vars: - - name: inventory_hostname - - name: ansible_host - port: - type: int - description: - - Specifies the port on the remote device that listens for connections when establishing - the SSH connection. - default: 22 - ini: - - section: defaults - key: remote_port - env: - - name: ANSIBLE_REMOTE_PORT - vars: - - name: ansible_port - network_os: - description: - - Configures the device platform network operating system. This value is used - to load the correct terminal and cliconf plugins to communicate with the remote - device. - vars: - - name: ansible_network_os - remote_user: - description: - - The username used to authenticate to the remote device when the SSH connection - is first established. If the remote_user is not specified, the connection will - use the username of the logged in user. - - Can be configured from the CLI via the C(--user) or C(-u) options. - ini: - - section: defaults - key: remote_user - env: - - name: ANSIBLE_REMOTE_USER - vars: - - name: ansible_user - password: - description: - - Configures the user password used to authenticate to the remote device when - first establishing the SSH connection. - vars: - - name: ansible_password - - name: ansible_ssh_pass - - name: ansible_ssh_password - private_key_file: - description: - - The private SSH key or certificate file used to authenticate to the remote device - when first establishing the SSH connection. - ini: - - section: defaults - key: private_key_file - env: - - name: ANSIBLE_PRIVATE_KEY_FILE - vars: - - name: ansible_private_key_file - become: - type: boolean - description: - - The become option will instruct the CLI session to attempt privilege escalation - on platforms that support it. Normally this means transitioning from user mode - to C(enable) mode in the CLI session. If become is set to True and the remote - device does not support privilege escalation or the privilege has already been - elevated, then this option is silently ignored. - - Can be configured from the CLI via the C(--become) or C(-b) options. - default: false - ini: - - section: privilege_escalation - key: become - env: - - name: ANSIBLE_BECOME - vars: - - name: ansible_become - become_errors: - type: str - description: - - This option determines how privilege escalation failures are handled when - I(become) is enabled. - - When set to C(ignore), the errors are silently ignored. - When set to C(warn), a warning message is displayed. - The default option C(fail), triggers a failure and halts execution. - vars: - - name: ansible_network_become_errors - default: fail - choices: ["ignore", "warn", "fail"] - terminal_errors: - type: str - description: - - This option determines how failures while setting terminal parameters - are handled. - - When set to C(ignore), the errors are silently ignored. - When set to C(warn), a warning message is displayed. - The default option C(fail), triggers a failure and halts execution. - vars: - - name: ansible_network_terminal_errors - default: fail - choices: ["ignore", "warn", "fail"] - version_added: 3.1.0 - become_method: - description: - - This option allows the become method to be specified in for handling privilege - escalation. Typically the become_method value is set to C(enable) but could - be defined as other values. - default: sudo - ini: - - section: privilege_escalation - key: become_method - env: - - name: ANSIBLE_BECOME_METHOD - vars: - - name: ansible_become_method - host_key_auto_add: - type: boolean - description: - - By default, Ansible will prompt the user before adding SSH keys to the known - hosts file. Since persistent connections such as network_cli run in background - processes, the user will never be prompted. By enabling this option, unknown - host keys will automatically be added to the known hosts file. - - Be sure to fully understand the security implications of enabling this option - on production systems as it could create a security vulnerability. - default: false - ini: - - section: paramiko_connection - key: host_key_auto_add - env: - - name: ANSIBLE_HOST_KEY_AUTO_ADD - persistent_buffer_read_timeout: - type: float - description: - - Configures, in seconds, the amount of time to wait for the data to be read from - Paramiko channel after the command prompt is matched. This timeout value ensures - that command prompt matched is correct and there is no more data left to be - received from remote host. - default: 0.1 - ini: - - section: persistent_connection - key: buffer_read_timeout - env: - - name: ANSIBLE_PERSISTENT_BUFFER_READ_TIMEOUT - vars: - - name: ansible_buffer_read_timeout - terminal_stdout_re: - type: list - elements: dict - description: - - A single regex pattern or a sequence of patterns along with optional flags to - match the command prompt from the received response chunk. This option accepts - C(pattern) and C(flags) keys. The value of C(pattern) is a python regex pattern - to match the response and the value of C(flags) is the value accepted by I(flags) - argument of I(re.compile) python method to control the way regex is matched - with the response, for example I('re.I'). - vars: - - name: ansible_terminal_stdout_re - terminal_stderr_re: - type: list - elements: dict - description: - - This option provides the regex pattern and optional flags to match the error - string from the received response chunk. This option accepts C(pattern) and - C(flags) keys. The value of C(pattern) is a python regex pattern to match the - response and the value of C(flags) is the value accepted by I(flags) argument - of I(re.compile) python method to control the way regex is matched with the - response, for example I('re.I'). - vars: - - name: ansible_terminal_stderr_re - terminal_initial_prompt: - type: list - elements: string - description: - - A single regex pattern or a sequence of patterns to evaluate the expected prompt - at the time of initial login to the remote host. - vars: - - name: ansible_terminal_initial_prompt - terminal_initial_answer: - type: list - elements: string - description: - - The answer to reply with if the C(terminal_initial_prompt) is matched. The value - can be a single answer or a list of answers for multiple terminal_initial_prompt. - In case the login menu has multiple prompts the sequence of the prompt and excepted - answer should be in same order and the value of I(terminal_prompt_checkall) - should be set to I(True) if all the values in C(terminal_initial_prompt) are - expected to be matched and set to I(False) if any one login prompt is to be - matched. - vars: - - name: ansible_terminal_initial_answer - terminal_initial_prompt_checkall: - type: boolean - description: - - By default the value is set to I(False) and any one of the prompts mentioned - in C(terminal_initial_prompt) option is matched it won't check for other prompts. - When set to I(True) it will check for all the prompts mentioned in C(terminal_initial_prompt) - option in the given order and all the prompts should be received from remote - host if not it will result in timeout. - default: false - vars: - - name: ansible_terminal_initial_prompt_checkall - terminal_inital_prompt_newline: - type: boolean - description: - - This boolean flag, that when set to I(True) will send newline in the response - if any of values in I(terminal_initial_prompt) is matched. - default: true - vars: - - name: ansible_terminal_initial_prompt_newline - network_cli_retries: - description: - - Number of attempts to connect to remote host. The delay time between the retires - increases after every attempt by power of 2 in seconds till either the maximum - attempts are exhausted or any of the C(persistent_command_timeout) or C(persistent_connect_timeout) - timers are triggered. - default: 3 - type: integer - env: - - name: ANSIBLE_NETWORK_CLI_RETRIES - ini: - - section: persistent_connection - key: network_cli_retries - vars: - - name: ansible_network_cli_retries - ssh_type: - description: - - The python package that will be used by the C(network_cli) connection plugin to create a SSH connection to remote host. - - I(libssh) will use the ansible-pylibssh package, which needs to be installed in order to work. - - I(paramiko) will instead use the paramiko package to manage the SSH connection. - - I(auto) will use ansible-pylibssh if that package is installed, otherwise will fallback to paramiko. - default: auto - choices: ["libssh", "paramiko", "auto"] - env: - - name: ANSIBLE_NETWORK_CLI_SSH_TYPE - ini: - - section: persistent_connection - key: ssh_type - vars: - - name: ansible_network_cli_ssh_type - host_key_checking: - description: 'Set this to "False" if you want to avoid host key checking by the underlying tools Ansible uses to connect to the host' - type: boolean - default: True - env: - - name: ANSIBLE_HOST_KEY_CHECKING - - name: ANSIBLE_SSH_HOST_KEY_CHECKING - ini: - - section: defaults - key: host_key_checking - - section: persistent_connection - key: host_key_checking - vars: - - name: ansible_host_key_checking - - name: ansible_ssh_host_key_checking - single_user_mode: - type: boolean - default: false - version_added: 2.0.0 - description: - - This option enables caching of data fetched from the target for re-use. - The cache is invalidated when the target device enters configuration mode. - - Applicable only for platforms where this has been implemented. - env: - - name: ANSIBLE_NETWORK_SINGLE_USER_MODE - vars: - - name: ansible_network_single_user_mode -""" - -import getpass -import json -import logging -import os -import re -import signal -import socket -import time -import traceback -from functools import wraps -from io import BytesIO - -from ansible.errors import AnsibleConnectionFailure, AnsibleError -from ansible.module_utils.common.text.converters import to_bytes, to_text -from ansible.module_utils.basic import missing_required_lib -from ansible.module_utils.six import PY3 -from ansible.module_utils.six.moves import cPickle -from ansible.playbook.play_context import PlayContext -from ansible.plugins.loader import ( - cache_loader, - cliconf_loader, - connection_loader, - terminal_loader, -) -from ansible.utils.display import Display -from ansible_collections.ansible.netcommon.plugins.module_utils.network.common.utils import ( - to_list, -) -from ansible_collections.ansible.netcommon.plugins.plugin_utils.connection_base import ( - NetworkConnectionBase, -) - -try: - from scp import SCPClient - - HAS_SCP = True -except ImportError: - HAS_SCP = False - -HAS_PYLIBSSH = False -display = Display() - - -def ensure_connect(func): - @wraps(func) - def wrapped(self, *args, **kwargs): - if not self._connected: - self._connect() - self.update_cli_prompt_context() - return func(self, *args, **kwargs) - - return wrapped - - -class AnsibleCmdRespRecv(Exception): - pass - - -class Connection(NetworkConnectionBase): - """CLI (shell) SSH connections on Paramiko""" - - transport = "ansible.netcommon.network_cli" - has_pipelining = True - - def __init__(self, play_context, new_stdin, *args, **kwargs): - super(Connection, self).__init__( - play_context, new_stdin, *args, **kwargs - ) - self._ssh_shell = None - - self._matched_prompt = None - self._matched_cmd_prompt = None - self._matched_pattern = None - self._last_response = None - self._history = list() - self._command_response = None - self._last_recv_window = None - self._cache = None - - self._terminal = None - self.cliconf = None - - # Managing prompt context - self._check_prompt = False - - self._task_uuid = to_text(kwargs.get("task_uuid", "")) - self._ssh_type_conn = None - self._ssh_type = None - - self._single_user_mode = False - - if self._network_os: - self._terminal = terminal_loader.get(self._network_os, self) - if not self._terminal: - raise AnsibleConnectionFailure( - "network os %s is not supported" % self._network_os - ) - - self.cliconf = cliconf_loader.get(self._network_os, self) - if self.cliconf: - self._sub_plugin = { - "type": "cliconf", - "name": self.cliconf._load_name, - "obj": self.cliconf, - } - self.queue_message( - "vvvv", - "loaded cliconf plugin %s from path %s for network_os %s" - % ( - self.cliconf._load_name, - self.cliconf._original_path, - self._network_os, - ), - ) - else: - self.queue_message( - "vvvv", - "unable to load cliconf for network_os %s" - % self._network_os, - ) - else: - raise AnsibleConnectionFailure( - "Unable to automatically determine host network os. Please " - "manually configure ansible_network_os value for this host" - ) - self.queue_message("log", "network_os is set to %s" % self._network_os) - - @property - def ssh_type(self): - if self._ssh_type is None: - self._ssh_type = self.get_option("ssh_type") - self.queue_message( - "vvvv", "ssh type is set to %s" % self._ssh_type - ) - # Support autodetection of supported library - if self._ssh_type == "auto": - self.queue_message("vvvv", "autodetecting ssh_type") - if HAS_PYLIBSSH: - self._ssh_type = "libssh" - else: - self.queue_message( - "warning", - "ansible-pylibssh not installed, falling back to paramiko", - ) - self._ssh_type = "paramiko" - self.queue_message( - "vvvv", "ssh type is now set to %s" % self._ssh_type - ) - - if self._ssh_type not in ["paramiko", "libssh"]: - raise AnsibleConnectionFailure( - "Invalid value '%s' set for ssh_type option." - " Expected value is either 'libssh' or 'paramiko'" - % self._ssh_type - ) - - return self._ssh_type - - @property - def ssh_type_conn(self): - if self._ssh_type_conn is None: - if self.ssh_type == "libssh": - connection_plugin = "ansible.netcommon.libssh" - elif self.ssh_type == "paramiko": - # NOTE: This MUST be paramiko or things will break - connection_plugin = "paramiko" - else: - raise AnsibleConnectionFailure( - "Invalid value '%s' set for ssh_type option." - " Expected value is either 'libssh' or 'paramiko'" - % self._ssh_type - ) - - self._ssh_type_conn = connection_loader.get( - connection_plugin, self._play_context, "/dev/null" - ) - - return self._ssh_type_conn - - # To maintain backward compatibility - @property - def paramiko_conn(self): - return self.ssh_type_conn - - def _get_log_channel(self): - name = "p=%s u=%s | " % (os.getpid(), getpass.getuser()) - name += "%s [%s]" % (self.ssh_type, self._play_context.remote_addr) - return name - - @ensure_connect - def get_prompt(self): - """Returns the current prompt from the device""" - return self._matched_prompt - - def exec_command(self, cmd, in_data=None, sudoable=True): - # this try..except block is just to handle the transition to supporting - # network_cli as a toplevel connection. Once connection=local is gone, - # this block can be removed as well and all calls passed directly to - # the local connection - if self._ssh_shell: - try: - cmd = json.loads(to_text(cmd, errors="surrogate_or_strict")) - kwargs = { - "command": to_bytes( - cmd["command"], errors="surrogate_or_strict" - ) - } - for key in ( - "prompt", - "answer", - "sendonly", - "newline", - "prompt_retry_check", - ): - if cmd.get(key) is True or cmd.get(key) is False: - kwargs[key] = cmd[key] - elif cmd.get(key) is not None: - kwargs[key] = to_bytes( - cmd[key], errors="surrogate_or_strict" - ) - return self.send(**kwargs) - except ValueError: - cmd = to_bytes(cmd, errors="surrogate_or_strict") - return self.send(command=cmd) - - else: - return super(Connection, self).exec_command(cmd, in_data, sudoable) - - def get_options(self, hostvars=None): - options = super(Connection, self).get_options(hostvars=hostvars) - options.update(self.ssh_type_conn.get_options(hostvars=hostvars)) - return options - - def set_options(self, task_keys=None, var_options=None, direct=None): - super(Connection, self).set_options( - task_keys=task_keys, var_options=var_options, direct=direct - ) - self.ssh_type_conn.set_options( - task_keys=task_keys, var_options=var_options, direct=direct - ) - # Retain old look_for_keys behaviour, but only if not set - if not any( - [ - task_keys and ("look_for_keys" in task_keys), - var_options and ("look_for_keys" in var_options), - direct and ("look_for_keys" in direct), - ] - ): - look_for_keys = not bool( - self.get_option("password") - and not self.get_option("private_key_file") - ) - if not look_for_keys: - # This actually can't be overridden yet without changes in ansible-core - # TODO: Uncomment when appropriate - # self.queue_message( - # "warning", - # "Option look_for_keys has been implicitly set to {0} because " - # "it was not set explicitly. This is retained to maintain " - # "backwards compatibility with the old behavior. This behavior " - # "will be removed in some release after 2024-01-01".format( - # look_for_keys - # ), - # ) - self.ssh_type_conn.set_option("look_for_keys", look_for_keys) - - def update_play_context(self, pc_data): - """Updates the play context information for the connection""" - pc_data = to_bytes(pc_data) - if PY3: - pc_data = cPickle.loads(pc_data, encoding="bytes") - else: - pc_data = cPickle.loads(pc_data) - play_context = PlayContext() - play_context.deserialize(pc_data) - - self.queue_message("vvvv", "updating play_context for connection") - if self._play_context.become ^ play_context.become: - if play_context.become is True: - auth_pass = play_context.become_pass - self._on_become(become_pass=auth_pass) - self.queue_message("vvvv", "authorizing connection") - else: - self._terminal.on_unbecome() - self.queue_message("vvvv", "deauthorizing connection") - - self._play_context = play_context - if self._ssh_type_conn is not None: - # TODO: This works, but is not really ideal. We would rather use - # set_options, but then we need more custom handling in that - # method. - self._ssh_type_conn._play_context = play_context - - if hasattr(self, "reset_history"): - self.reset_history() - if hasattr(self, "disable_response_logging"): - self.disable_response_logging() - - self._single_user_mode = self.get_option("single_user_mode") - - def set_check_prompt(self, task_uuid): - self._check_prompt = task_uuid - - def update_cli_prompt_context(self): - # set cli prompt context at the start of new task run only - if self._check_prompt and self._task_uuid != self._check_prompt: - self._task_uuid, self._check_prompt = self._check_prompt, False - self.set_cli_prompt_context() - - def _connect(self): - """ - Connects to the remote device and starts the terminal - """ - if display.verbosity > 3: - logging.getLogger(self.ssh_type).setLevel(logging.DEBUG) - - self.queue_message( - "vvvv", "invoked shell using ssh_type: %s" % self.ssh_type - ) - - self._single_user_mode = self.get_option("single_user_mode") - - if not self.connected: - self.ssh_type_conn._set_log_channel(self._get_log_channel()) - self.ssh_type_conn.force_persistence = self.force_persistence - - command_timeout = self.get_option("persistent_command_timeout") - max_pause = min( - [ - self.get_option("persistent_connect_timeout"), - command_timeout, - ] - ) - retries = self.get_option("network_cli_retries") - total_pause = 0 - - for attempt in range(retries + 1): - try: - ssh = self.ssh_type_conn._connect() - break - except AnsibleError: - raise - except Exception as e: - pause = 2 ** (attempt + 1) - if attempt == retries or total_pause >= max_pause: - raise AnsibleConnectionFailure( - to_text(e, errors="surrogate_or_strict") - ) - else: - msg = ( - "network_cli_retry: attempt: %d, caught exception(%s), " - "pausing for %d seconds" - % ( - attempt + 1, - to_text(e, errors="surrogate_or_strict"), - pause, - ) - ) - - self.queue_message("vv", msg) - time.sleep(pause) - total_pause += pause - continue - - self.queue_message("vvvv", "ssh connection done, setting terminal") - self._connected = True - - self._ssh_shell = ssh.ssh.invoke_shell() - if self.ssh_type == "paramiko": - self._ssh_shell.settimeout(command_timeout) - - self.queue_message( - "vvvv", - "loaded terminal plugin for network_os %s" % self._network_os, - ) - - terminal_initial_prompt = ( - self.get_option("terminal_initial_prompt") - or self._terminal.terminal_initial_prompt - ) - terminal_initial_answer = ( - self.get_option("terminal_initial_answer") - or self._terminal.terminal_initial_answer - ) - newline = ( - self.get_option("terminal_inital_prompt_newline") - or self._terminal.terminal_inital_prompt_newline - ) - check_all = ( - self.get_option("terminal_initial_prompt_checkall") or False - ) - - self.receive( - prompts=terminal_initial_prompt, - answer=terminal_initial_answer, - newline=newline, - check_all=check_all, - ) - - if self._play_context.become: - self.queue_message("vvvv", "firing event: on_become") - auth_pass = self._play_context.become_pass - self._on_become(become_pass=auth_pass) - - self.queue_message("vvvv", "firing event: on_open_shell()") - self._on_open_shell() - - self.queue_message( - "vvvv", "ssh connection has completed successfully" - ) - - return self - - def _on_become(self, become_pass=None): - """ - Wraps terminal.on_become() to handle - privilege escalation failures based on user preference - """ - on_become_error = self.get_option("become_errors") - try: - self._terminal.on_become(passwd=become_pass) - except AnsibleConnectionFailure: - if on_become_error == "ignore": - pass - elif on_become_error == "warn": - self.queue_message( - "warning", "on_become: privilege escalation failed" - ) - else: - raise - - def _on_open_shell(self): - """ - Wraps terminal.on_open_shell() to handle - terminal setting failures based on user preference - """ - on_terminal_error = self.get_option("terminal_errors") - try: - self._terminal.on_open_shell() - except AnsibleConnectionFailure: - if on_terminal_error == "ignore": - pass - elif on_terminal_error == "warn": - self.queue_message( - "warning", - "on_open_shell: failed to set terminal parameters", - ) - else: - raise - - def close(self): - """ - Close the active connection to the device - """ - # only close the connection if its connected. - if self._connected: - self.queue_message("debug", "closing ssh connection to device") - if self._ssh_shell: - self.queue_message("debug", "firing event: on_close_shell()") - self._terminal.on_close_shell() - self._ssh_shell.close() - self._ssh_shell = None - self.queue_message("debug", "cli session is now closed") - - self.ssh_type_conn.close() - self._ssh_type_conn = None - self.queue_message( - "debug", "ssh connection has been closed successfully" - ) - super(Connection, self).close() - - def _read_post_command_prompt_match(self): - time.sleep(self.get_option("persistent_buffer_read_timeout")) - data = self._ssh_shell.read_bulk_response() - return data if data else None - - def receive_paramiko( - self, - command=None, - prompts=None, - answer=None, - newline=True, - prompt_retry_check=False, - check_all=False, - strip_prompt=True, - ): - - recv = BytesIO() - cache_socket_timeout = self.get_option("persistent_command_timeout") - self._ssh_shell.settimeout(cache_socket_timeout) - command_prompt_matched = False - handled = False - errored_response = None - - while True: - if command_prompt_matched: - try: - signal.signal( - signal.SIGALRM, self._handle_buffer_read_timeout - ) - signal.setitimer( - signal.ITIMER_REAL, self._buffer_read_timeout - ) - data = self._ssh_shell.recv(256) - signal.alarm(0) - self._log_messages( - "response-%s: %s" % (self._window_count + 1, data) - ) - # if data is still received on channel it indicates the prompt string - # is wrongly matched in between response chunks, continue to read - # remaining response. - command_prompt_matched = False - - # restart command_timeout timer - signal.signal(signal.SIGALRM, self._handle_command_timeout) - signal.alarm(self._command_timeout) - - except AnsibleCmdRespRecv: - # reset socket timeout to global timeout - return self._command_response - else: - data = self._ssh_shell.recv(256) - self._log_messages( - "response-%s: %s" % (self._window_count + 1, data) - ) - # when a channel stream is closed, received data will be empty - if not data: - break - - recv.write(data) - offset = recv.tell() - 256 if recv.tell() > 256 else 0 - recv.seek(offset) - - window = self._strip(recv.read()) - self._last_recv_window = window - self._window_count += 1 - - if prompts and not handled: - handled = self._handle_prompt( - window, prompts, answer, newline, False, check_all - ) - self._matched_prompt_window = self._window_count - elif ( - prompts - and handled - and prompt_retry_check - and self._matched_prompt_window + 1 == self._window_count - ): - # check again even when handled, if same prompt repeats in next window - # (like in the case of a wrong enable password, etc) indicates - # value of answer is wrong, report this as error. - if self._handle_prompt( - window, - prompts, - answer, - newline, - prompt_retry_check, - check_all, - ): - raise AnsibleConnectionFailure( - "For matched prompt '%s', answer is not valid" - % self._matched_cmd_prompt - ) - - if self._find_error(window): - # We can't exit here, as we need to drain the buffer in case - # the error isn't fatal, and will be using the buffer again - errored_response = window - - if self._find_prompt(window): - if errored_response: - raise AnsibleConnectionFailure(errored_response) - self._last_response = recv.getvalue() - resp = self._strip(self._last_response) - self._command_response = self._sanitize( - resp, command, strip_prompt - ) - if self._buffer_read_timeout == 0.0: - # reset socket timeout to global timeout - return self._command_response - else: - command_prompt_matched = True - - def receive_libssh( - self, - command=None, - prompts=None, - answer=None, - newline=True, - prompt_retry_check=False, - check_all=False, - strip_prompt=True, - ): - self._command_response = resp = b"" - command_prompt_matched = False - handled = False - errored_response = None - - while True: - - if command_prompt_matched: - data = self._read_post_command_prompt_match() - if data: - command_prompt_matched = False - else: - return self._command_response - else: - try: - data = self._ssh_shell.read_bulk_response() - # TODO: Should be ConnectionError when pylibssh drops Python 2 support - except OSError: - # Socket has closed - break - - if not data: - continue - self._last_recv_window = self._strip(data) - resp += self._last_recv_window - self._window_count += 1 - - self._log_messages("response-%s: %s" % (self._window_count, data)) - - if prompts and not handled: - handled = self._handle_prompt( - resp, prompts, answer, newline, False, check_all - ) - self._matched_prompt_window = self._window_count - elif ( - prompts - and handled - and prompt_retry_check - and self._matched_prompt_window + 1 == self._window_count - ): - # check again even when handled, if same prompt repeats in next window - # (like in the case of a wrong enable password, etc) indicates - # value of answer is wrong, report this as error. - if self._handle_prompt( - resp, - prompts, - answer, - newline, - prompt_retry_check, - check_all, - ): - raise AnsibleConnectionFailure( - "For matched prompt '%s', answer is not valid" - % self._matched_cmd_prompt - ) - - if self._find_error(resp): - # We can't exit here, as we need to drain the buffer in case - # the error isn't fatal, and will be using the buffer again - errored_response = resp - - if self._find_prompt(resp): - if errored_response: - raise AnsibleConnectionFailure(errored_response) - self._last_response = data - self._command_response += self._sanitize( - resp, command, strip_prompt - ) - command_prompt_matched = True - - def receive( - self, - command=None, - prompts=None, - answer=None, - newline=True, - prompt_retry_check=False, - check_all=False, - strip_prompt=True, - ): - """ - Handles receiving of output from command - """ - self._matched_prompt = None - self._matched_cmd_prompt = None - self._matched_prompt_window = 0 - self._window_count = 0 - - # set terminal regex values for command prompt and errors in response - self._terminal_stderr_re = self._get_terminal_std_re( - "terminal_stderr_re" - ) - self._terminal_stdout_re = self._get_terminal_std_re( - "terminal_stdout_re" - ) - - self._command_timeout = self.get_option("persistent_command_timeout") - self._validate_timeout_value( - self._command_timeout, "persistent_command_timeout" - ) - - self._buffer_read_timeout = self.get_option( - "persistent_buffer_read_timeout" - ) - self._validate_timeout_value( - self._buffer_read_timeout, "persistent_buffer_read_timeout" - ) - - self._log_messages("command: %s" % command) - if self.ssh_type == "libssh": - response = self.receive_libssh( - command, - prompts, - answer, - newline, - prompt_retry_check, - check_all, - strip_prompt, - ) - elif self.ssh_type == "paramiko": - response = self.receive_paramiko( - command, - prompts, - answer, - newline, - prompt_retry_check, - check_all, - strip_prompt, - ) - - return response - - @ensure_connect - def send( - self, - command, - prompt=None, - answer=None, - newline=True, - sendonly=False, - prompt_retry_check=False, - check_all=False, - strip_prompt=True, - ): - """ - Sends the command to the device in the opened shell - """ - # try cache first - if (not prompt) and (self._single_user_mode): - out = self.get_cache().lookup(command) - if out: - self.queue_message( - "vvvv", "cache hit for command: %s" % command - ) - return out - - if check_all: - prompt_len = len(to_list(prompt)) - answer_len = len(to_list(answer)) - if prompt_len != answer_len: - raise AnsibleConnectionFailure( - "Number of prompts (%s) is not same as that of answers (%s)" - % (prompt_len, answer_len) - ) - try: - cmd = b"%s\r" % command - self._history.append(cmd) - self._ssh_shell.sendall(cmd) - self._log_messages("send command: %s" % cmd) - if sendonly: - return - response = self.receive( - command, - prompt, - answer, - newline, - prompt_retry_check, - check_all, - strip_prompt, - ) - response = to_text(response, errors="surrogate_then_replace") - - if (not prompt) and (self._single_user_mode): - if self._needs_cache_invalidation(command): - # invalidate the existing cache - if self.get_cache().keys(): - self.queue_message( - "vvvv", "invalidating existing cache" - ) - self.get_cache().invalidate() - else: - # populate cache - self.queue_message( - "vvvv", "populating cache for command: %s" % command - ) - self.get_cache().populate(command, response) - - return response - except (socket.timeout, AttributeError): - self.queue_message("error", traceback.format_exc()) - raise AnsibleConnectionFailure( - "timeout value %s seconds reached while trying to send command: %s" - % (self._ssh_shell.gettimeout(), command.strip()) - ) - - def _handle_buffer_read_timeout(self, signum, frame): - self.queue_message( - "vvvv", - "Response received, triggered 'persistent_buffer_read_timeout' timer of %s seconds" - % self.get_option("persistent_buffer_read_timeout"), - ) - raise AnsibleCmdRespRecv() - - def _handle_command_timeout(self, signum, frame): - msg = ( - "command timeout triggered, timeout value is %s secs.\nSee the timeout setting options in the Network Debug and Troubleshooting Guide." - % self.get_option("persistent_command_timeout") - ) - self.queue_message("log", msg) - raise AnsibleConnectionFailure(msg) - - def _strip(self, data): - """ - Removes ANSI codes from device response - """ - for regex in self._terminal.ansi_re: - data = regex.sub(b"", data) - return data - - def _handle_prompt( - self, - resp, - prompts, - answer, - newline, - prompt_retry_check=False, - check_all=False, - ): - """ - Matches the command prompt and responds - - :arg resp: Byte string containing the raw response from the remote - :arg prompts: Sequence of byte strings that we consider prompts for input - :arg answer: Sequence of Byte string to send back to the remote if we find a prompt. - A carriage return is automatically appended to this string. - :param prompt_retry_check: Bool value for trying to detect more prompts - :param check_all: Bool value to indicate if all the values in prompt sequence should be matched or any one of - given prompt. - :returns: True if a prompt was found in ``resp``. If check_all is True - will True only after all the prompt in the prompts list are matched. False otherwise. - """ - single_prompt = False - if not isinstance(prompts, list): - prompts = [prompts] - single_prompt = True - if not isinstance(answer, list): - answer = [answer] - try: - prompts_regex = [re.compile(to_bytes(r), re.I) for r in prompts] - except re.error as exc: - raise ConnectionError( - "Failed to compile one or more terminal prompt regexes: %s.\n" - "Prompts provided: %s" % (to_text(exc), prompts) - ) - for index, regex in enumerate(prompts_regex): - match = regex.search(resp) - if match: - self._matched_cmd_prompt = match.group() - self._log_messages( - "matched command prompt: %s" % self._matched_cmd_prompt - ) - - # if prompt_retry_check is enabled to check if same prompt is - # repeated don't send answer again. - if not prompt_retry_check: - prompt_answer = to_bytes( - answer[index] if len(answer) > index else answer[0] - ) - if newline: - prompt_answer += b"\r" - self._ssh_shell.sendall(prompt_answer) - self._log_messages( - "matched command prompt answer: %s" % prompt_answer - ) - if check_all and prompts and not single_prompt: - prompts.pop(0) - answer.pop(0) - return False - return True - return False - - def _sanitize(self, resp, command=None, strip_prompt=True): - """ - Removes elements from the response before returning to the caller - """ - cleaned = [] - for line in resp.splitlines(): - if command and line.strip() == command.strip(): - continue - - for prompt in self._matched_prompt.strip().splitlines(): - if prompt.strip() in line and strip_prompt: - break - else: - cleaned.append(line) - - return b"\n".join(cleaned).strip() - - def _find_error(self, response): - """Searches the buffered response for a matching error condition""" - for stderr_regex in self._terminal_stderr_re: - if stderr_regex.search(response): - self._log_messages( - "matched error regex (terminal_stderr_re) '%s' from response '%s'" - % (stderr_regex.pattern, response) - ) - - self._log_messages( - "matched stdout regex (terminal_stdout_re) '%s' from error response '%s'" - % (self._matched_pattern, response) - ) - return True - - return False - - def _find_prompt(self, response): - """Searches the buffered response for a matching command prompt""" - for stdout_regex in self._terminal_stdout_re: - match = stdout_regex.search(response) - if match: - self._matched_pattern = stdout_regex.pattern - self._matched_prompt = match.group() - self._log_messages( - "matched cli prompt '%s' with regex '%s' from response '%s'" - % (self._matched_prompt, self._matched_pattern, response) - ) - return True - - return False - - def _validate_timeout_value(self, timeout, timer_name): - if timeout < 0: - raise AnsibleConnectionFailure( - "'%s' timer value '%s' is invalid, value should be greater than or equal to zero." - % (timer_name, timeout) - ) - - def transport_test(self, connect_timeout): - """This method enables wait_for_connection to work. - - As it is used by wait_for_connection, it is called by that module's action plugin, - which is on the controller process, which means that nothing done on this instance - should impact the actual persistent connection... this check is for informational - purposes only and should be properly cleaned up. - """ - - # Force a fresh connect if for some reason we have connected before. - self.close() - self._connect() - self.close() - - def _get_terminal_std_re(self, option): - terminal_std_option = self.get_option(option) - terminal_std_re = [] - - if terminal_std_option: - for item in terminal_std_option: - if "pattern" not in item: - raise AnsibleConnectionFailure( - "'pattern' is a required key for option '%s'," - " received option value is %s" % (option, item) - ) - pattern = rb"%s" % to_bytes(item["pattern"]) - flag = item.get("flags", 0) - if flag: - flag = getattr(re, flag.split(".")[1]) - terminal_std_re.append(re.compile(pattern, flag)) - else: - # To maintain backward compatibility - terminal_std_re = getattr(self._terminal, option) - - return terminal_std_re - - def copy_file( - self, source=None, destination=None, proto="scp", timeout=30 - ): - """Copies file over scp/sftp to remote device - - :param source: Source file path - :param destination: Destination file path on remote device - :param proto: Protocol to be used for file transfer, - supported protocol: scp and sftp - :param timeout: Specifies the wait time to receive response from - remote host before triggering timeout exception - :return: None - """ - ssh = self.ssh_type_conn._connect_uncached() - if self.ssh_type == "libssh": - self.ssh_type_conn.put_file(source, destination, proto=proto) - elif self.ssh_type == "paramiko": - if proto == "scp": - if not HAS_SCP: - raise AnsibleError(missing_required_lib("scp")) - with SCPClient( - ssh.get_transport(), socket_timeout=timeout - ) as scp: - scp.put(source, destination) - elif proto == "sftp": - with ssh.open_sftp() as sftp: - sftp.put(source, destination) - else: - raise AnsibleError( - "Do not know how to do transfer file over protocol %s" - % proto - ) - else: - raise AnsibleError( - "Do not know how to do SCP with ssh_type %s" % self.ssh_type - ) - - def get_file(self, source=None, destination=None, proto="scp", timeout=30): - """Fetch file over scp/sftp from remote device - :param source: Source file path - :param destination: Destination file path - :param proto: Protocol to be used for file transfer, - supported protocol: scp and sftp - :param timeout: Specifies the wait time to receive response from - remote host before triggering timeout exception - :return: None - """ - ssh = self.ssh_type_conn._connect_uncached() - if self.ssh_type == "libssh": - self.ssh_type_conn.fetch_file(source, destination, proto=proto) - elif self.ssh_type == "paramiko": - if proto == "scp": - if not HAS_SCP: - raise AnsibleError(missing_required_lib("scp")) - try: - with SCPClient( - ssh.get_transport(), socket_timeout=timeout - ) as scp: - scp.get(source, destination) - except EOFError: - # This appears to be benign. - pass - elif proto == "sftp": - with ssh.open_sftp() as sftp: - sftp.get(source, destination) - else: - raise AnsibleError( - "Do not know how to do transfer file over protocol %s" - % proto - ) - else: - raise AnsibleError( - "Do not know how to do SCP with ssh_type %s" % self.ssh_type - ) - - def get_cache(self): - if not self._cache: - # TO-DO: support jsonfile or other modes of caching with - # a configurable option - self._cache = cache_loader.get("ansible.netcommon.memory") - return self._cache - - def _is_in_config_mode(self): - """ - Check if the target device is in config mode by comparing - the current prompt with the platform's `terminal_config_prompt`. - Returns False if `terminal_config_prompt` is not defined. - - :returns: A boolean indicating if the device is in config mode or not. - """ - cfg_mode = False - cur_prompt = to_text( - self.get_prompt(), errors="surrogate_then_replace" - ).strip() - cfg_prompt = getattr(self._terminal, "terminal_config_prompt", None) - if cfg_prompt and cfg_prompt.match(cur_prompt): - cfg_mode = True - return cfg_mode - - def _needs_cache_invalidation(self, command): - """ - This method determines if it is necessary to invalidate - the existing cache based on whether the device has entered - configuration mode or if the last command sent to the device - is potentially capable of making configuration changes. - - :param command: The last command sent to the target device. - :returns: A boolean indicating if cache invalidation is required or not. - """ - invalidate = False - cfg_cmds = [] - try: - # AnsiblePlugin base class in Ansible 2.9 does not have has_option() method. - # TO-DO: use has_option() when we drop 2.9 support. - cfg_cmds = self.cliconf.get_option("config_commands") - except AttributeError: - cfg_cmds = [] - if (self._is_in_config_mode()) or (to_text(command) in cfg_cmds): - invalidate = True - return invalidate diff --git a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/connection/persistent.py b/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/connection/persistent.py deleted file mode 100644 index 6866688736c..00000000000 --- a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/connection/persistent.py +++ /dev/null @@ -1,96 +0,0 @@ -# 2017 Red Hat Inc. -# (c) 2017 Ansible Project -# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) - -from __future__ import annotations - - -DOCUMENTATION = """author: Ansible Core Team -connection: persistent -short_description: Use a persistent unix socket for connection -description: -- This is a helper plugin to allow making other connections persistent. -options: - persistent_command_timeout: - type: int - description: - - Configures, in seconds, the amount of time to wait for a command to return from - the remote device. If this timer is exceeded before the command returns, the - connection plugin will raise an exception and close - default: 10 - ini: - - section: persistent_connection - key: command_timeout - env: - - name: ANSIBLE_PERSISTENT_COMMAND_TIMEOUT - vars: - - name: ansible_command_timeout -""" -from ansible.executor.task_executor import start_connection -from ansible.plugins.connection import ConnectionBase -from ansible.module_utils.common.text.converters import to_text -from ansible.module_utils.connection import Connection as SocketConnection -from ansible.utils.display import Display - -display = Display() - - -class Connection(ConnectionBase): - """ Local based connections """ - - transport = "ansible.netcommon.persistent" - has_pipelining = False - - def __init__(self, play_context, new_stdin, *args, **kwargs): - super(Connection, self).__init__( - play_context, new_stdin, *args, **kwargs - ) - self._task_uuid = to_text(kwargs.get("task_uuid", "")) - - def _connect(self): - self._connected = True - return self - - def exec_command(self, cmd, in_data=None, sudoable=True): - display.vvvv( - "exec_command(), socket_path=%s" % self.socket_path, - host=self._play_context.remote_addr, - ) - connection = SocketConnection(self.socket_path) - out = connection.exec_command(cmd, in_data=in_data, sudoable=sudoable) - return 0, out, "" - - def put_file(self, in_path, out_path): - pass - - def fetch_file(self, in_path, out_path): - pass - - def close(self): - self._connected = False - - def run(self): - """Returns the path of the persistent connection socket. - - Attempts to ensure (within playcontext.timeout seconds) that the - socket path exists. If the path exists (or the timeout has expired), - returns the socket path. - """ - display.vvvv( - "starting connection from persistent connection plugin", - host=self._play_context.remote_addr, - ) - variables = { - "ansible_command_timeout": self.get_option( - "persistent_command_timeout" - ) - } - socket_path = start_connection( - self._play_context, variables, self._task_uuid - ) - display.vvvv( - "local domain socket path is %s" % socket_path, - host=self._play_context.remote_addr, - ) - setattr(self, "_socket_path", socket_path) - return socket_path diff --git a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/doc_fragments/connection_persistent.py b/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/doc_fragments/connection_persistent.py deleted file mode 100644 index 506b559f03c..00000000000 --- a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/doc_fragments/connection_persistent.py +++ /dev/null @@ -1,75 +0,0 @@ -# -*- coding: utf-8 -*- -# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) - -from __future__ import annotations - - -class ModuleDocFragment(object): - - # Standard files documentation fragment - DOCUMENTATION = r""" -options: - import_modules: - type: boolean - description: - - Reduce CPU usage and network module execution time - by enabling direct execution. Instead of the module being packaged - and executed by the shell, it will be directly executed by the Ansible - control node using the same python interpreter as the Ansible process. - Note- Incompatible with C(asynchronous mode). - Note- Python 3 and Ansible 2.9.16 or greater required. - Note- With Ansible 2.9.x fully qualified modules names are required in tasks. - default: true - ini: - - section: ansible_network - key: import_modules - env: - - name: ANSIBLE_NETWORK_IMPORT_MODULES - vars: - - name: ansible_network_import_modules - persistent_connect_timeout: - type: int - description: - - Configures, in seconds, the amount of time to wait when trying to initially - establish a persistent connection. If this value expires before the connection - to the remote device is completed, the connection will fail. - default: 30 - ini: - - section: persistent_connection - key: connect_timeout - env: - - name: ANSIBLE_PERSISTENT_CONNECT_TIMEOUT - vars: - - name: ansible_connect_timeout - persistent_command_timeout: - type: int - description: - - Configures, in seconds, the amount of time to wait for a command to - return from the remote device. If this timer is exceeded before the - command returns, the connection plugin will raise an exception and - close. - default: 30 - ini: - - section: persistent_connection - key: command_timeout - env: - - name: ANSIBLE_PERSISTENT_COMMAND_TIMEOUT - vars: - - name: ansible_command_timeout - persistent_log_messages: - type: boolean - description: - - This flag will enable logging the command executed and response received from - target device in the ansible log file. For this option to work 'log_path' ansible - configuration option is required to be set to a file path with write access. - - Be sure to fully understand the security implications of enabling this - option as it could create a security vulnerability by logging sensitive information in log file. - default: False - ini: - - section: persistent_connection - key: log_messages - env: - - name: ANSIBLE_PERSISTENT_LOG_MESSAGES - vars: - - name: ansible_persistent_log_messages -""" diff --git a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/compat/ipaddress.py b/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/compat/ipaddress.py deleted file mode 100644 index ce74a1aae00..00000000000 --- a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/compat/ipaddress.py +++ /dev/null @@ -1,2578 +0,0 @@ -# -*- coding: utf-8 -*- - -# This code is part of Ansible, but is an independent component. -# This particular file, and this file only, is based on -# Lib/ipaddress.py of cpython -# It is licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 -# -# 1. This LICENSE AGREEMENT is between the Python Software Foundation -# ("PSF"), and the Individual or Organization ("Licensee") accessing and -# otherwise using this software ("Python") in source or binary form and -# its associated documentation. -# -# 2. Subject to the terms and conditions of this License Agreement, PSF hereby -# grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, -# analyze, test, perform and/or display publicly, prepare derivative works, -# distribute, and otherwise use Python alone or in any derivative version, -# provided, however, that PSF's License Agreement and PSF's notice of copyright, -# i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -# 2011, 2012, 2013, 2014, 2015 Python Software Foundation; All Rights Reserved" -# are retained in Python alone or in any derivative version prepared by Licensee. -# -# 3. In the event Licensee prepares a derivative work that is based on -# or incorporates Python or any part thereof, and wants to make -# the derivative work available to others as provided herein, then -# Licensee hereby agrees to include in any such work a brief summary of -# the changes made to Python. -# -# 4. PSF is making Python available to Licensee on an "AS IS" -# basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -# IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND -# DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -# FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT -# INFRINGE ANY THIRD PARTY RIGHTS. -# -# 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -# FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -# A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, -# OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. -# -# 6. This License Agreement will automatically terminate upon a material -# breach of its terms and conditions. -# -# 7. Nothing in this License Agreement shall be deemed to create any -# relationship of agency, partnership, or joint venture between PSF and -# Licensee. This License Agreement does not grant permission to use PSF -# trademarks or trade name in a trademark sense to endorse or promote -# products or services of Licensee, or any third party. -# -# 8. By copying, installing or otherwise using Python, Licensee -# agrees to be bound by the terms and conditions of this License -# Agreement. - -# Copyright 2007 Google Inc. -# Licensed to PSF under a Contributor Agreement. - -"""A fast, lightweight IPv4/IPv6 manipulation library in Python. - -This library is used to create/poke/manipulate IPv4 and IPv6 addresses -and networks. - -""" - -from __future__ import annotations - - -import itertools -import struct - - -# The following makes it easier for us to script updates of the bundled code and is not part of -# upstream -_BUNDLED_METADATA = {"pypi_name": "ipaddress", "version": "1.0.22"} - -__version__ = "1.0.22" - -# Compatibility functions -_compat_int_types = (int,) -try: - _compat_int_types = (int, long) -except NameError: - pass -try: - _compat_str = unicode -except NameError: - _compat_str = str - assert bytes != str -if b"\0"[0] == 0: # Python 3 semantics - - def _compat_bytes_to_byte_vals(byt): - return byt - - -else: - - def _compat_bytes_to_byte_vals(byt): - return [struct.unpack(b"!B", b)[0] for b in byt] - - -try: - _compat_int_from_byte_vals = int.from_bytes -except AttributeError: - - def _compat_int_from_byte_vals(bytvals, endianess): - assert endianess == "big" - res = 0 - for bv in bytvals: - assert isinstance(bv, _compat_int_types) - res = (res << 8) + bv - return res - - -def _compat_to_bytes(intval, length, endianess): - assert isinstance(intval, _compat_int_types) - assert endianess == "big" - if length == 4: - if intval < 0 or intval >= 2 ** 32: - raise struct.error("integer out of range for 'I' format code") - return struct.pack(b"!I", intval) - elif length == 16: - if intval < 0 or intval >= 2 ** 128: - raise struct.error("integer out of range for 'QQ' format code") - return struct.pack(b"!QQ", intval >> 64, intval & 0xFFFFFFFFFFFFFFFF) - else: - raise NotImplementedError() - - -if hasattr(int, "bit_length"): - # Not int.bit_length , since that won't work in 2.7 where long exists - def _compat_bit_length(i): - return i.bit_length() - - -else: - - def _compat_bit_length(i): - for res in itertools.count(): - if i >> res == 0: - return res - - -def _compat_range(start, end, step=1): - assert step > 0 - i = start - while i < end: - yield i - i += step - - -class _TotalOrderingMixin(object): - __slots__ = () - - # Helper that derives the other comparison operations from - # __lt__ and __eq__ - # We avoid functools.total_ordering because it doesn't handle - # NotImplemented correctly yet (http://bugs.python.org/issue10042) - def __eq__(self, other): - raise NotImplementedError - - def __ne__(self, other): - equal = self.__eq__(other) - if equal is NotImplemented: - return NotImplemented - return not equal - - def __lt__(self, other): - raise NotImplementedError - - def __le__(self, other): - less = self.__lt__(other) - if less is NotImplemented or not less: - return self.__eq__(other) - return less - - def __gt__(self, other): - less = self.__lt__(other) - if less is NotImplemented: - return NotImplemented - equal = self.__eq__(other) - if equal is NotImplemented: - return NotImplemented - return not (less or equal) - - def __ge__(self, other): - less = self.__lt__(other) - if less is NotImplemented: - return NotImplemented - return not less - - -IPV4LENGTH = 32 -IPV6LENGTH = 128 - - -class AddressValueError(ValueError): - """A Value Error related to the address.""" - - -class NetmaskValueError(ValueError): - """A Value Error related to the netmask.""" - - -def ip_address(address): - """Take an IP string/int and return an object of the correct type. - - Args: - address: A string or integer, the IP address. Either IPv4 or - IPv6 addresses may be supplied; integers less than 2**32 will - be considered to be IPv4 by default. - - Returns: - An IPv4Address or IPv6Address object. - - Raises: - ValueError: if the *address* passed isn't either a v4 or a v6 - address - - """ - try: - return IPv4Address(address) - except (AddressValueError, NetmaskValueError): - pass - - try: - return IPv6Address(address) - except (AddressValueError, NetmaskValueError): - pass - - if isinstance(address, bytes): - raise AddressValueError( - "%r does not appear to be an IPv4 or IPv6 address. " - "Did you pass in a bytes (str in Python 2) instead of" - " a unicode object?" % address - ) - - raise ValueError( - "%r does not appear to be an IPv4 or IPv6 address" % address - ) - - -def ip_network(address, strict=True): - """Take an IP string/int and return an object of the correct type. - - Args: - address: A string or integer, the IP network. Either IPv4 or - IPv6 networks may be supplied; integers less than 2**32 will - be considered to be IPv4 by default. - - Returns: - An IPv4Network or IPv6Network object. - - Raises: - ValueError: if the string passed isn't either a v4 or a v6 - address. Or if the network has host bits set. - - """ - try: - return IPv4Network(address, strict) - except (AddressValueError, NetmaskValueError): - pass - - try: - return IPv6Network(address, strict) - except (AddressValueError, NetmaskValueError): - pass - - if isinstance(address, bytes): - raise AddressValueError( - "%r does not appear to be an IPv4 or IPv6 network. " - "Did you pass in a bytes (str in Python 2) instead of" - " a unicode object?" % address - ) - - raise ValueError( - "%r does not appear to be an IPv4 or IPv6 network" % address - ) - - -def ip_interface(address): - """Take an IP string/int and return an object of the correct type. - - Args: - address: A string or integer, the IP address. Either IPv4 or - IPv6 addresses may be supplied; integers less than 2**32 will - be considered to be IPv4 by default. - - Returns: - An IPv4Interface or IPv6Interface object. - - Raises: - ValueError: if the string passed isn't either a v4 or a v6 - address. - - Notes: - The IPv?Interface classes describe an Address on a particular - Network, so they're basically a combination of both the Address - and Network classes. - - """ - try: - return IPv4Interface(address) - except (AddressValueError, NetmaskValueError): - pass - - try: - return IPv6Interface(address) - except (AddressValueError, NetmaskValueError): - pass - - raise ValueError( - "%r does not appear to be an IPv4 or IPv6 interface" % address - ) - - -def v4_int_to_packed(address): - """Represent an address as 4 packed bytes in network (big-endian) order. - - Args: - address: An integer representation of an IPv4 IP address. - - Returns: - The integer address packed as 4 bytes in network (big-endian) order. - - Raises: - ValueError: If the integer is negative or too large to be an - IPv4 IP address. - - """ - try: - return _compat_to_bytes(address, 4, "big") - except (struct.error, OverflowError): - raise ValueError("Address negative or too large for IPv4") - - -def v6_int_to_packed(address): - """Represent an address as 16 packed bytes in network (big-endian) order. - - Args: - address: An integer representation of an IPv6 IP address. - - Returns: - The integer address packed as 16 bytes in network (big-endian) order. - - """ - try: - return _compat_to_bytes(address, 16, "big") - except (struct.error, OverflowError): - raise ValueError("Address negative or too large for IPv6") - - -def _split_optional_netmask(address): - """Helper to split the netmask and raise AddressValueError if needed""" - addr = _compat_str(address).split("/") - if len(addr) > 2: - raise AddressValueError("Only one '/' permitted in %r" % address) - return addr - - -def _find_address_range(addresses): - """Find a sequence of sorted deduplicated IPv#Address. - - Args: - addresses: a list of IPv#Address objects. - - Yields: - A tuple containing the first and last IP addresses in the sequence. - - """ - it = iter(addresses) - first = last = next(it) # pylint: disable=stop-iteration-return - for ip in it: - if ip._ip != last._ip + 1: - yield first, last - first = ip - last = ip - yield first, last - - -def _count_righthand_zero_bits(number, bits): - """Count the number of zero bits on the right hand side. - - Args: - number: an integer. - bits: maximum number of bits to count. - - Returns: - The number of zero bits on the right hand side of the number. - - """ - if number == 0: - return bits - return min(bits, _compat_bit_length(~number & (number - 1))) - - -def summarize_address_range(first, last): - """Summarize a network range given the first and last IP addresses. - - Example: - >>> list(summarize_address_range(IPv4Address('192.0.2.0'), - ... IPv4Address('192.0.2.130'))) - ... #doctest: +NORMALIZE_WHITESPACE - [IPv4Network('192.0.2.0/25'), IPv4Network('192.0.2.128/31'), - IPv4Network('192.0.2.130/32')] - - Args: - first: the first IPv4Address or IPv6Address in the range. - last: the last IPv4Address or IPv6Address in the range. - - Returns: - An iterator of the summarized IPv(4|6) network objects. - - Raise: - TypeError: - If the first and last objects are not IP addresses. - If the first and last objects are not the same version. - ValueError: - If the last object is not greater than the first. - If the version of the first address is not 4 or 6. - - """ - if not ( - isinstance(first, _BaseAddress) and isinstance(last, _BaseAddress) - ): - raise TypeError("first and last must be IP addresses, not networks") - if first.version != last.version: - raise TypeError( - "%s and %s are not of the same version" % (first, last) - ) - if first > last: - raise ValueError("last IP address must be greater than first") - - if first.version == 4: - ip = IPv4Network - elif first.version == 6: - ip = IPv6Network - else: - raise ValueError("unknown IP version") - - ip_bits = first._max_prefixlen - first_int = first._ip - last_int = last._ip - while first_int <= last_int: - nbits = min( - _count_righthand_zero_bits(first_int, ip_bits), - _compat_bit_length(last_int - first_int + 1) - 1, - ) - net = ip((first_int, ip_bits - nbits)) - yield net - first_int += 1 << nbits - if first_int - 1 == ip._ALL_ONES: - break - - -def _collapse_addresses_internal(addresses): - """Loops through the addresses, collapsing concurrent netblocks. - - Example: - - ip1 = IPv4Network('192.0.2.0/26') - ip2 = IPv4Network('192.0.2.64/26') - ip3 = IPv4Network('192.0.2.128/26') - ip4 = IPv4Network('192.0.2.192/26') - - _collapse_addresses_internal([ip1, ip2, ip3, ip4]) -> - [IPv4Network('192.0.2.0/24')] - - This shouldn't be called directly; it is called via - collapse_addresses([]). - - Args: - addresses: A list of IPv4Network's or IPv6Network's - - Returns: - A list of IPv4Network's or IPv6Network's depending on what we were - passed. - - """ - # First merge - to_merge = list(addresses) - subnets = {} - while to_merge: - net = to_merge.pop() - supernet = net.supernet() - existing = subnets.get(supernet) - if existing is None: - subnets[supernet] = net - elif existing != net: - # Merge consecutive subnets - del subnets[supernet] - to_merge.append(supernet) - # Then iterate over resulting networks, skipping subsumed subnets - last = None - for net in sorted(subnets.values()): - if last is not None: - # Since they are sorted, - # last.network_address <= net.network_address is a given. - if last.broadcast_address >= net.broadcast_address: - continue - yield net - last = net - - -def collapse_addresses(addresses): - """Collapse a list of IP objects. - - Example: - collapse_addresses([IPv4Network('192.0.2.0/25'), - IPv4Network('192.0.2.128/25')]) -> - [IPv4Network('192.0.2.0/24')] - - Args: - addresses: An iterator of IPv4Network or IPv6Network objects. - - Returns: - An iterator of the collapsed IPv(4|6)Network objects. - - Raises: - TypeError: If passed a list of mixed version objects. - - """ - addrs = [] - ips = [] - nets = [] - - # split IP addresses and networks - for ip in addresses: - if isinstance(ip, _BaseAddress): - if ips and ips[-1]._version != ip._version: - raise TypeError( - "%s and %s are not of the same version" % (ip, ips[-1]) - ) - ips.append(ip) - elif ip._prefixlen == ip._max_prefixlen: - if ips and ips[-1]._version != ip._version: - raise TypeError( - "%s and %s are not of the same version" % (ip, ips[-1]) - ) - try: - ips.append(ip.ip) - except AttributeError: - ips.append(ip.network_address) - else: - if nets and nets[-1]._version != ip._version: - raise TypeError( - "%s and %s are not of the same version" % (ip, nets[-1]) - ) - nets.append(ip) - - # sort and dedup - ips = sorted(set(ips)) - - # find consecutive address ranges in the sorted sequence and summarize them - if ips: - for first, last in _find_address_range(ips): - addrs.extend(summarize_address_range(first, last)) - - return _collapse_addresses_internal(addrs + nets) - - -def get_mixed_type_key(obj): - """Return a key suitable for sorting between networks and addresses. - - Address and Network objects are not sortable by default; they're - fundamentally different so the expression - - IPv4Address('192.0.2.0') <= IPv4Network('192.0.2.0/24') - - doesn't make any sense. There are some times however, where you may wish - to have ipaddress sort these for you anyway. If you need to do this, you - can use this function as the key= argument to sorted(). - - Args: - obj: either a Network or Address object. - Returns: - appropriate key. - - """ - if isinstance(obj, _BaseNetwork): - return obj._get_networks_key() - elif isinstance(obj, _BaseAddress): - return obj._get_address_key() - return NotImplemented - - -class _IPAddressBase(_TotalOrderingMixin): - - """The mother class.""" - - __slots__ = () - - @property - def exploded(self): - """Return the longhand version of the IP address as a string.""" - return self._explode_shorthand_ip_string() - - @property - def compressed(self): - """Return the shorthand version of the IP address as a string.""" - return _compat_str(self) - - @property - def reverse_pointer(self): - """The name of the reverse DNS pointer for the IP address, e.g.: - >>> ipaddress.ip_address("127.0.0.1").reverse_pointer - '1.0.0.127.in-addr.arpa' - >>> ipaddress.ip_address("2001:db8::1").reverse_pointer - '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa' - - """ - return self._reverse_pointer() - - @property - def version(self): - msg = "%200s has no version specified" % (type(self),) - raise NotImplementedError(msg) - - def _check_int_address(self, address): - if address < 0: - msg = "%d (< 0) is not permitted as an IPv%d address" - raise AddressValueError(msg % (address, self._version)) - if address > self._ALL_ONES: - msg = "%d (>= 2**%d) is not permitted as an IPv%d address" - raise AddressValueError( - msg % (address, self._max_prefixlen, self._version) - ) - - def _check_packed_address(self, address, expected_len): - address_len = len(address) - if address_len != expected_len: - msg = ( - "%r (len %d != %d) is not permitted as an IPv%d address. " - "Did you pass in a bytes (str in Python 2) instead of" - " a unicode object?" - ) - raise AddressValueError( - msg % (address, address_len, expected_len, self._version) - ) - - @classmethod - def _ip_int_from_prefix(cls, prefixlen): - """Turn the prefix length into a bitwise netmask - - Args: - prefixlen: An integer, the prefix length. - - Returns: - An integer. - - """ - return cls._ALL_ONES ^ (cls._ALL_ONES >> prefixlen) - - @classmethod - def _prefix_from_ip_int(cls, ip_int): - """Return prefix length from the bitwise netmask. - - Args: - ip_int: An integer, the netmask in expanded bitwise format - - Returns: - An integer, the prefix length. - - Raises: - ValueError: If the input intermingles zeroes & ones - """ - trailing_zeroes = _count_righthand_zero_bits( - ip_int, cls._max_prefixlen - ) - prefixlen = cls._max_prefixlen - trailing_zeroes - leading_ones = ip_int >> trailing_zeroes - all_ones = (1 << prefixlen) - 1 - if leading_ones != all_ones: - byteslen = cls._max_prefixlen // 8 - details = _compat_to_bytes(ip_int, byteslen, "big") - msg = "Netmask pattern %r mixes zeroes & ones" - raise ValueError(msg % details) - return prefixlen - - @classmethod - def _report_invalid_netmask(cls, netmask_str): - msg = "%r is not a valid netmask" % netmask_str - raise NetmaskValueError(msg) - - @classmethod - def _prefix_from_prefix_string(cls, prefixlen_str): - """Return prefix length from a numeric string - - Args: - prefixlen_str: The string to be converted - - Returns: - An integer, the prefix length. - - Raises: - NetmaskValueError: If the input is not a valid netmask - """ - # int allows a leading +/- as well as surrounding whitespace, - # so we ensure that isn't the case - if not _BaseV4._DECIMAL_DIGITS.issuperset(prefixlen_str): - cls._report_invalid_netmask(prefixlen_str) - try: - prefixlen = int(prefixlen_str) - except ValueError: - cls._report_invalid_netmask(prefixlen_str) - if not (0 <= prefixlen <= cls._max_prefixlen): - cls._report_invalid_netmask(prefixlen_str) - return prefixlen - - @classmethod - def _prefix_from_ip_string(cls, ip_str): - """Turn a netmask/hostmask string into a prefix length - - Args: - ip_str: The netmask/hostmask to be converted - - Returns: - An integer, the prefix length. - - Raises: - NetmaskValueError: If the input is not a valid netmask/hostmask - """ - # Parse the netmask/hostmask like an IP address. - try: - ip_int = cls._ip_int_from_string(ip_str) - except AddressValueError: - cls._report_invalid_netmask(ip_str) - - # Try matching a netmask (this would be /1*0*/ as a bitwise regexp). - # Note that the two ambiguous cases (all-ones and all-zeroes) are - # treated as netmasks. - try: - return cls._prefix_from_ip_int(ip_int) - except ValueError: - pass - - # Invert the bits, and try matching a /0+1+/ hostmask instead. - ip_int ^= cls._ALL_ONES - try: - return cls._prefix_from_ip_int(ip_int) - except ValueError: - cls._report_invalid_netmask(ip_str) - - def __reduce__(self): - return self.__class__, (_compat_str(self),) - - -class _BaseAddress(_IPAddressBase): - - """A generic IP object. - - This IP class contains the version independent methods which are - used by single IP addresses. - """ - - __slots__ = () - - def __int__(self): - return self._ip - - def __eq__(self, other): - try: - return self._ip == other._ip and self._version == other._version - except AttributeError: - return NotImplemented - - def __lt__(self, other): - if not isinstance(other, _IPAddressBase): - return NotImplemented - if not isinstance(other, _BaseAddress): - raise TypeError( - "%s and %s are not of the same type" % (self, other) - ) - if self._version != other._version: - raise TypeError( - "%s and %s are not of the same version" % (self, other) - ) - if self._ip != other._ip: - return self._ip < other._ip - return False - - # Shorthand for Integer addition and subtraction. This is not - # meant to ever support addition/subtraction of addresses. - def __add__(self, other): - if not isinstance(other, _compat_int_types): - return NotImplemented - return self.__class__(int(self) + other) - - def __sub__(self, other): - if not isinstance(other, _compat_int_types): - return NotImplemented - return self.__class__(int(self) - other) - - def __repr__(self): - return "%s(%r)" % (self.__class__.__name__, _compat_str(self)) - - def __str__(self): - return _compat_str(self._string_from_ip_int(self._ip)) - - def __hash__(self): - return hash(hex(int(self._ip))) - - def _get_address_key(self): - return (self._version, self) - - def __reduce__(self): - return self.__class__, (self._ip,) - - -class _BaseNetwork(_IPAddressBase): - - """A generic IP network object. - - This IP class contains the version independent methods which are - used by networks. - - """ - - def __init__(self, address): - self._cache = {} - - def __repr__(self): - return "%s(%r)" % (self.__class__.__name__, _compat_str(self)) - - def __str__(self): - return "%s/%d" % (self.network_address, self.prefixlen) - - def hosts(self): - """Generate Iterator over usable hosts in a network. - - This is like __iter__ except it doesn't return the network - or broadcast addresses. - - """ - network = int(self.network_address) - broadcast = int(self.broadcast_address) - for x in _compat_range(network + 1, broadcast): - yield self._address_class(x) - - def __iter__(self): - network = int(self.network_address) - broadcast = int(self.broadcast_address) - for x in _compat_range(network, broadcast + 1): - yield self._address_class(x) - - def __getitem__(self, n): - network = int(self.network_address) - broadcast = int(self.broadcast_address) - if n >= 0: - if network + n > broadcast: - raise IndexError("address out of range") - return self._address_class(network + n) - else: - n += 1 - if broadcast + n < network: - raise IndexError("address out of range") - return self._address_class(broadcast + n) - - def __lt__(self, other): - if not isinstance(other, _IPAddressBase): - return NotImplemented - if not isinstance(other, _BaseNetwork): - raise TypeError( - "%s and %s are not of the same type" % (self, other) - ) - if self._version != other._version: - raise TypeError( - "%s and %s are not of the same version" % (self, other) - ) - if self.network_address != other.network_address: - return self.network_address < other.network_address - if self.netmask != other.netmask: - return self.netmask < other.netmask - return False - - def __eq__(self, other): - try: - return ( - self._version == other._version - and self.network_address == other.network_address - and int(self.netmask) == int(other.netmask) - ) - except AttributeError: - return NotImplemented - - def __hash__(self): - return hash(int(self.network_address) ^ int(self.netmask)) - - def __contains__(self, other): - # always false if one is v4 and the other is v6. - if self._version != other._version: - return False - # dealing with another network. - if isinstance(other, _BaseNetwork): - return False - # dealing with another address - else: - # address - return ( - int(self.network_address) - <= int(other._ip) - <= int(self.broadcast_address) - ) - - def overlaps(self, other): - """Tell if self is partly contained in other.""" - return self.network_address in other or ( - self.broadcast_address in other - or ( - other.network_address in self - or (other.broadcast_address in self) - ) - ) - - @property - def broadcast_address(self): - x = self._cache.get("broadcast_address") - if x is None: - x = self._address_class( - int(self.network_address) | int(self.hostmask) - ) - self._cache["broadcast_address"] = x - return x - - @property - def hostmask(self): - x = self._cache.get("hostmask") - if x is None: - x = self._address_class(int(self.netmask) ^ self._ALL_ONES) - self._cache["hostmask"] = x - return x - - @property - def with_prefixlen(self): - return "%s/%d" % (self.network_address, self._prefixlen) - - @property - def with_netmask(self): - return "%s/%s" % (self.network_address, self.netmask) - - @property - def with_hostmask(self): - return "%s/%s" % (self.network_address, self.hostmask) - - @property - def num_addresses(self): - """Number of hosts in the current subnet.""" - return int(self.broadcast_address) - int(self.network_address) + 1 - - @property - def _address_class(self): - # Returning bare address objects (rather than interfaces) allows for - # more consistent behaviour across the network address, broadcast - # address and individual host addresses. - msg = "%200s has no associated address class" % (type(self),) - raise NotImplementedError(msg) - - @property - def prefixlen(self): - return self._prefixlen - - def address_exclude(self, other): - """Remove an address from a larger block. - - For example: - - addr1 = ip_network('192.0.2.0/28') - addr2 = ip_network('192.0.2.1/32') - list(addr1.address_exclude(addr2)) = - [IPv4Network('192.0.2.0/32'), IPv4Network('192.0.2.2/31'), - IPv4Network('192.0.2.4/30'), IPv4Network('192.0.2.8/29')] - - or IPv6: - - addr1 = ip_network('2001:db8::1/32') - addr2 = ip_network('2001:db8::1/128') - list(addr1.address_exclude(addr2)) = - [ip_network('2001:db8::1/128'), - ip_network('2001:db8::2/127'), - ip_network('2001:db8::4/126'), - ip_network('2001:db8::8/125'), - ... - ip_network('2001:db8:8000::/33')] - - Args: - other: An IPv4Network or IPv6Network object of the same type. - - Returns: - An iterator of the IPv(4|6)Network objects which is self - minus other. - - Raises: - TypeError: If self and other are of differing address - versions, or if other is not a network object. - ValueError: If other is not completely contained by self. - - """ - if not self._version == other._version: - raise TypeError( - "%s and %s are not of the same version" % (self, other) - ) - - if not isinstance(other, _BaseNetwork): - raise TypeError("%s is not a network object" % other) - - if not other.subnet_of(self): - raise ValueError("%s not contained in %s" % (other, self)) - if other == self: - return - - # Make sure we're comparing the network of other. - other = other.__class__( - "%s/%s" % (other.network_address, other.prefixlen) - ) - - s1, s2 = self.subnets() - while s1 != other and s2 != other: - if other.subnet_of(s1): - yield s2 - s1, s2 = s1.subnets() - elif other.subnet_of(s2): - yield s1 - s1, s2 = s2.subnets() - else: - # If we got here, there's a bug somewhere. - raise AssertionError( - "Error performing exclusion: " - "s1: %s s2: %s other: %s" % (s1, s2, other) - ) - if s1 == other: - yield s2 - elif s2 == other: - yield s1 - else: - # If we got here, there's a bug somewhere. - raise AssertionError( - "Error performing exclusion: " - "s1: %s s2: %s other: %s" % (s1, s2, other) - ) - - def compare_networks(self, other): - """Compare two IP objects. - - This is only concerned about the comparison of the integer - representation of the network addresses. This means that the - host bits aren't considered at all in this method. If you want - to compare host bits, you can easily enough do a - 'HostA._ip < HostB._ip' - - Args: - other: An IP object. - - Returns: - If the IP versions of self and other are the same, returns: - - -1 if self < other: - eg: IPv4Network('192.0.2.0/25') < IPv4Network('192.0.2.128/25') - IPv6Network('2001:db8::1000/124') < - IPv6Network('2001:db8::2000/124') - 0 if self == other - eg: IPv4Network('192.0.2.0/24') == IPv4Network('192.0.2.0/24') - IPv6Network('2001:db8::1000/124') == - IPv6Network('2001:db8::1000/124') - 1 if self > other - eg: IPv4Network('192.0.2.128/25') > IPv4Network('192.0.2.0/25') - IPv6Network('2001:db8::2000/124') > - IPv6Network('2001:db8::1000/124') - - Raises: - TypeError if the IP versions are different. - - """ - # does this need to raise a ValueError? - if self._version != other._version: - raise TypeError( - "%s and %s are not of the same type" % (self, other) - ) - # self._version == other._version below here: - if self.network_address < other.network_address: - return -1 - if self.network_address > other.network_address: - return 1 - # self.network_address == other.network_address below here: - if self.netmask < other.netmask: - return -1 - if self.netmask > other.netmask: - return 1 - return 0 - - def _get_networks_key(self): - """Network-only key function. - - Returns an object that identifies this address' network and - netmask. This function is a suitable "key" argument for sorted() - and list.sort(). - - """ - return (self._version, self.network_address, self.netmask) - - def subnets(self, prefixlen_diff=1, new_prefix=None): - """The subnets which join to make the current subnet. - - In the case that self contains only one IP - (self._prefixlen == 32 for IPv4 or self._prefixlen == 128 - for IPv6), yield an iterator with just ourself. - - Args: - prefixlen_diff: An integer, the amount the prefix length - should be increased by. This should not be set if - new_prefix is also set. - new_prefix: The desired new prefix length. This must be a - larger number (smaller prefix) than the existing prefix. - This should not be set if prefixlen_diff is also set. - - Returns: - An iterator of IPv(4|6) objects. - - Raises: - ValueError: The prefixlen_diff is too small or too large. - OR - prefixlen_diff and new_prefix are both set or new_prefix - is a smaller number than the current prefix (smaller - number means a larger network) - - """ - if self._prefixlen == self._max_prefixlen: - yield self - return - - if new_prefix is not None: - if new_prefix < self._prefixlen: - raise ValueError("new prefix must be longer") - if prefixlen_diff != 1: - raise ValueError("cannot set prefixlen_diff and new_prefix") - prefixlen_diff = new_prefix - self._prefixlen - - if prefixlen_diff < 0: - raise ValueError("prefix length diff must be > 0") - new_prefixlen = self._prefixlen + prefixlen_diff - - if new_prefixlen > self._max_prefixlen: - raise ValueError( - "prefix length diff %d is invalid for netblock %s" - % (new_prefixlen, self) - ) - - start = int(self.network_address) - end = int(self.broadcast_address) + 1 - step = (int(self.hostmask) + 1) >> prefixlen_diff - for new_addr in _compat_range(start, end, step): - current = self.__class__((new_addr, new_prefixlen)) - yield current - - def supernet(self, prefixlen_diff=1, new_prefix=None): - """The supernet containing the current network. - - Args: - prefixlen_diff: An integer, the amount the prefix length of - the network should be decreased by. For example, given a - /24 network and a prefixlen_diff of 3, a supernet with a - /21 netmask is returned. - - Returns: - An IPv4 network object. - - Raises: - ValueError: If self.prefixlen - prefixlen_diff < 0. I.e., you have - a negative prefix length. - OR - If prefixlen_diff and new_prefix are both set or new_prefix is a - larger number than the current prefix (larger number means a - smaller network) - - """ - if self._prefixlen == 0: - return self - - if new_prefix is not None: - if new_prefix > self._prefixlen: - raise ValueError("new prefix must be shorter") - if prefixlen_diff != 1: - raise ValueError("cannot set prefixlen_diff and new_prefix") - prefixlen_diff = self._prefixlen - new_prefix - - new_prefixlen = self.prefixlen - prefixlen_diff - if new_prefixlen < 0: - raise ValueError( - "current prefixlen is %d, cannot have a prefixlen_diff of %d" - % (self.prefixlen, prefixlen_diff) - ) - return self.__class__( - ( - int(self.network_address) - & (int(self.netmask) << prefixlen_diff), - new_prefixlen, - ) - ) - - @property - def is_multicast(self): - """Test if the address is reserved for multicast use. - - Returns: - A boolean, True if the address is a multicast address. - See RFC 2373 2.7 for details. - - """ - return ( - self.network_address.is_multicast - and self.broadcast_address.is_multicast - ) - - @staticmethod - def _is_subnet_of(a, b): - try: - # Always false if one is v4 and the other is v6. - if a._version != b._version: - raise TypeError( - "%s and %s are not of the same version" % (a, b) - ) - return ( - b.network_address <= a.network_address - and b.broadcast_address >= a.broadcast_address - ) - except AttributeError: - raise TypeError( - "Unable to test subnet containment " - "between %s and %s" % (a, b) - ) - - def subnet_of(self, other): - """Return True if this network is a subnet of other.""" - return self._is_subnet_of(self, other) - - def supernet_of(self, other): - """Return True if this network is a supernet of other.""" - return self._is_subnet_of(other, self) - - @property - def is_reserved(self): - """Test if the address is otherwise IETF reserved. - - Returns: - A boolean, True if the address is within one of the - reserved IPv6 Network ranges. - - """ - return ( - self.network_address.is_reserved - and self.broadcast_address.is_reserved - ) - - @property - def is_link_local(self): - """Test if the address is reserved for link-local. - - Returns: - A boolean, True if the address is reserved per RFC 4291. - - """ - return ( - self.network_address.is_link_local - and self.broadcast_address.is_link_local - ) - - @property - def is_private(self): - """Test if this address is allocated for private networks. - - Returns: - A boolean, True if the address is reserved per - iana-ipv4-special-registry or iana-ipv6-special-registry. - - """ - return ( - self.network_address.is_private - and self.broadcast_address.is_private - ) - - @property - def is_global(self): - """Test if this address is allocated for public networks. - - Returns: - A boolean, True if the address is not reserved per - iana-ipv4-special-registry or iana-ipv6-special-registry. - - """ - return not self.is_private - - @property - def is_unspecified(self): - """Test if the address is unspecified. - - Returns: - A boolean, True if this is the unspecified address as defined in - RFC 2373 2.5.2. - - """ - return ( - self.network_address.is_unspecified - and self.broadcast_address.is_unspecified - ) - - @property - def is_loopback(self): - """Test if the address is a loopback address. - - Returns: - A boolean, True if the address is a loopback address as defined in - RFC 2373 2.5.3. - - """ - return ( - self.network_address.is_loopback - and self.broadcast_address.is_loopback - ) - - -class _BaseV4(object): - - """Base IPv4 object. - - The following methods are used by IPv4 objects in both single IP - addresses and networks. - - """ - - __slots__ = () - _version = 4 - # Equivalent to 255.255.255.255 or 32 bits of 1's. - _ALL_ONES = (2 ** IPV4LENGTH) - 1 - _DECIMAL_DIGITS = frozenset("0123456789") - - # the valid octets for host and netmasks. only useful for IPv4. - _valid_mask_octets = frozenset([255, 254, 252, 248, 240, 224, 192, 128, 0]) - - _max_prefixlen = IPV4LENGTH - # There are only a handful of valid v4 netmasks, so we cache them all - # when constructed (see _make_netmask()). - _netmask_cache = {} - - def _explode_shorthand_ip_string(self): - return _compat_str(self) - - @classmethod - def _make_netmask(cls, arg): - """Make a (netmask, prefix_len) tuple from the given argument. - - Argument can be: - - an integer (the prefix length) - - a string representing the prefix length (e.g. "24") - - a string representing the prefix netmask (e.g. "255.255.255.0") - """ - if arg not in cls._netmask_cache: - if isinstance(arg, _compat_int_types): - prefixlen = arg - else: - try: - # Check for a netmask in prefix length form - prefixlen = cls._prefix_from_prefix_string(arg) - except NetmaskValueError: - # Check for a netmask or hostmask in dotted-quad form. - # This may raise NetmaskValueError. - prefixlen = cls._prefix_from_ip_string(arg) - netmask = IPv4Address(cls._ip_int_from_prefix(prefixlen)) - cls._netmask_cache[arg] = netmask, prefixlen - return cls._netmask_cache[arg] - - @classmethod - def _ip_int_from_string(cls, ip_str): - """Turn the given IP string into an integer for comparison. - - Args: - ip_str: A string, the IP ip_str. - - Returns: - The IP ip_str as an integer. - - Raises: - AddressValueError: if ip_str isn't a valid IPv4 Address. - - """ - if not ip_str: - raise AddressValueError("Address cannot be empty") - - octets = ip_str.split(".") - if len(octets) != 4: - raise AddressValueError("Expected 4 octets in %r" % ip_str) - - try: - return _compat_int_from_byte_vals( - map(cls._parse_octet, octets), "big" - ) - except ValueError as exc: - raise AddressValueError("%s in %r" % (exc, ip_str)) - - @classmethod - def _parse_octet(cls, octet_str): - """Convert a decimal octet into an integer. - - Args: - octet_str: A string, the number to parse. - - Returns: - The octet as an integer. - - Raises: - ValueError: if the octet isn't strictly a decimal from [0..255]. - - """ - if not octet_str: - raise ValueError("Empty octet not permitted") - # Whitelist the characters, since int() allows a lot of bizarre stuff. - if not cls._DECIMAL_DIGITS.issuperset(octet_str): - msg = "Only decimal digits permitted in %r" - raise ValueError(msg % octet_str) - # We do the length check second, since the invalid character error - # is likely to be more informative for the user - if len(octet_str) > 3: - msg = "At most 3 characters permitted in %r" - raise ValueError(msg % octet_str) - # Convert to integer (we know digits are legal) - octet_int = int(octet_str, 10) - # Any octets that look like they *might* be written in octal, - # and which don't look exactly the same in both octal and - # decimal are rejected as ambiguous - if octet_int > 7 and octet_str[0] == "0": - msg = "Ambiguous (octal/decimal) value in %r not permitted" - raise ValueError(msg % octet_str) - if octet_int > 255: - raise ValueError("Octet %d (> 255) not permitted" % octet_int) - return octet_int - - @classmethod - def _string_from_ip_int(cls, ip_int): - """Turns a 32-bit integer into dotted decimal notation. - - Args: - ip_int: An integer, the IP address. - - Returns: - The IP address as a string in dotted decimal notation. - - """ - return ".".join( - _compat_str( - struct.unpack(b"!B", b)[0] if isinstance(b, bytes) else b - ) - for b in _compat_to_bytes(ip_int, 4, "big") - ) - - def _is_hostmask(self, ip_str): - """Test if the IP string is a hostmask (rather than a netmask). - - Args: - ip_str: A string, the potential hostmask. - - Returns: - A boolean, True if the IP string is a hostmask. - - """ - bits = ip_str.split(".") - try: - parts = [x for x in map(int, bits) if x in self._valid_mask_octets] - except ValueError: - return False - if len(parts) != len(bits): - return False - if parts[0] < parts[-1]: - return True - return False - - def _reverse_pointer(self): - """Return the reverse DNS pointer name for the IPv4 address. - - This implements the method described in RFC1035 3.5. - - """ - reverse_octets = _compat_str(self).split(".")[::-1] - return ".".join(reverse_octets) + ".in-addr.arpa" - - @property - def max_prefixlen(self): - return self._max_prefixlen - - @property - def version(self): - return self._version - - -class IPv4Address(_BaseV4, _BaseAddress): - - """Represent and manipulate single IPv4 Addresses.""" - - __slots__ = ("_ip", "__weakref__") - - def __init__(self, address): - - """ - Args: - address: A string or integer representing the IP - - Additionally, an integer can be passed, so - IPv4Address('192.0.2.1') == IPv4Address(3221225985). - or, more generally - IPv4Address(int(IPv4Address('192.0.2.1'))) == - IPv4Address('192.0.2.1') - - Raises: - AddressValueError: If ipaddress isn't a valid IPv4 address. - - """ - # Efficient constructor from integer. - if isinstance(address, _compat_int_types): - self._check_int_address(address) - self._ip = address - return - - # Constructing from a packed address - if isinstance(address, bytes): - self._check_packed_address(address, 4) - bvs = _compat_bytes_to_byte_vals(address) - self._ip = _compat_int_from_byte_vals(bvs, "big") - return - - # Assume input argument to be string or any object representation - # which converts into a formatted IP string. - addr_str = _compat_str(address) - if "/" in addr_str: - raise AddressValueError("Unexpected '/' in %r" % address) - self._ip = self._ip_int_from_string(addr_str) - - @property - def packed(self): - """The binary representation of this address.""" - return v4_int_to_packed(self._ip) - - @property - def is_reserved(self): - """Test if the address is otherwise IETF reserved. - - Returns: - A boolean, True if the address is within the - reserved IPv4 Network range. - - """ - return self in self._constants._reserved_network - - @property - def is_private(self): - """Test if this address is allocated for private networks. - - Returns: - A boolean, True if the address is reserved per - iana-ipv4-special-registry. - - """ - return any(self in net for net in self._constants._private_networks) - - @property - def is_global(self): - return ( - self not in self._constants._public_network and not self.is_private - ) - - @property - def is_multicast(self): - """Test if the address is reserved for multicast use. - - Returns: - A boolean, True if the address is multicast. - See RFC 3171 for details. - - """ - return self in self._constants._multicast_network - - @property - def is_unspecified(self): - """Test if the address is unspecified. - - Returns: - A boolean, True if this is the unspecified address as defined in - RFC 5735 3. - - """ - return self == self._constants._unspecified_address - - @property - def is_loopback(self): - """Test if the address is a loopback address. - - Returns: - A boolean, True if the address is a loopback per RFC 3330. - - """ - return self in self._constants._loopback_network - - @property - def is_link_local(self): - """Test if the address is reserved for link-local. - - Returns: - A boolean, True if the address is link-local per RFC 3927. - - """ - return self in self._constants._linklocal_network - - -class IPv4Interface(IPv4Address): - def __init__(self, address): - if isinstance(address, (bytes, _compat_int_types)): - IPv4Address.__init__(self, address) - self.network = IPv4Network(self._ip) - self._prefixlen = self._max_prefixlen - return - - if isinstance(address, tuple): - IPv4Address.__init__(self, address[0]) - if len(address) > 1: - self._prefixlen = int(address[1]) - else: - self._prefixlen = self._max_prefixlen - - self.network = IPv4Network(address, strict=False) - self.netmask = self.network.netmask - self.hostmask = self.network.hostmask - return - - addr = _split_optional_netmask(address) - IPv4Address.__init__(self, addr[0]) - - self.network = IPv4Network(address, strict=False) - self._prefixlen = self.network._prefixlen - - self.netmask = self.network.netmask - self.hostmask = self.network.hostmask - - def __str__(self): - return "%s/%d" % ( - self._string_from_ip_int(self._ip), - self.network.prefixlen, - ) - - def __eq__(self, other): - address_equal = IPv4Address.__eq__(self, other) - if not address_equal or address_equal is NotImplemented: - return address_equal - try: - return self.network == other.network - except AttributeError: - # An interface with an associated network is NOT the - # same as an unassociated address. That's why the hash - # takes the extra info into account. - return False - - def __lt__(self, other): - address_less = IPv4Address.__lt__(self, other) - if address_less is NotImplemented: - return NotImplemented - try: - return ( - self.network < other.network - or self.network == other.network - and address_less - ) - except AttributeError: - # We *do* allow addresses and interfaces to be sorted. The - # unassociated address is considered less than all interfaces. - return False - - def __hash__(self): - return self._ip ^ self._prefixlen ^ int(self.network.network_address) - - __reduce__ = _IPAddressBase.__reduce__ - - @property - def ip(self): - return IPv4Address(self._ip) - - @property - def with_prefixlen(self): - return "%s/%s" % (self._string_from_ip_int(self._ip), self._prefixlen) - - @property - def with_netmask(self): - return "%s/%s" % (self._string_from_ip_int(self._ip), self.netmask) - - @property - def with_hostmask(self): - return "%s/%s" % (self._string_from_ip_int(self._ip), self.hostmask) - - -class IPv4Network(_BaseV4, _BaseNetwork): - - """This class represents and manipulates 32-bit IPv4 network + addresses.. - - Attributes: [examples for IPv4Network('192.0.2.0/27')] - .network_address: IPv4Address('192.0.2.0') - .hostmask: IPv4Address('0.0.0.31') - .broadcast_address: IPv4Address('192.0.2.32') - .netmask: IPv4Address('255.255.255.224') - .prefixlen: 27 - - """ - - # Class to use when creating address objects - _address_class = IPv4Address - - def __init__(self, address, strict=True): - - """Instantiate a new IPv4 network object. - - Args: - address: A string or integer representing the IP [& network]. - '192.0.2.0/24' - '192.0.2.0/255.255.255.0' - '192.0.0.2/0.0.0.255' - are all functionally the same in IPv4. Similarly, - '192.0.2.1' - '192.0.2.1/255.255.255.255' - '192.0.2.1/32' - are also functionally equivalent. That is to say, failing to - provide a subnetmask will create an object with a mask of /32. - - If the mask (portion after the / in the argument) is given in - dotted quad form, it is treated as a netmask if it starts with a - non-zero field (e.g. /255.0.0.0 == /8) and as a hostmask if it - starts with a zero field (e.g. 0.255.255.255 == /8), with the - single exception of an all-zero mask which is treated as a - netmask == /0. If no mask is given, a default of /32 is used. - - Additionally, an integer can be passed, so - IPv4Network('192.0.2.1') == IPv4Network(3221225985) - or, more generally - IPv4Interface(int(IPv4Interface('192.0.2.1'))) == - IPv4Interface('192.0.2.1') - - Raises: - AddressValueError: If ipaddress isn't a valid IPv4 address. - NetmaskValueError: If the netmask isn't valid for - an IPv4 address. - ValueError: If strict is True and a network address is not - supplied. - - """ - _BaseNetwork.__init__(self, address) - - # Constructing from a packed address or integer - if isinstance(address, (_compat_int_types, bytes)): - self.network_address = IPv4Address(address) - self.netmask, self._prefixlen = self._make_netmask( - self._max_prefixlen - ) - # FIXME: address/network test here. - return - - if isinstance(address, tuple): - if len(address) > 1: - arg = address[1] - else: - # We weren't given an address[1] - arg = self._max_prefixlen - self.network_address = IPv4Address(address[0]) - self.netmask, self._prefixlen = self._make_netmask(arg) - packed = int(self.network_address) - if packed & int(self.netmask) != packed: - if strict: - raise ValueError("%s has host bits set" % self) - else: - self.network_address = IPv4Address( - packed & int(self.netmask) - ) - return - - # Assume input argument to be string or any object representation - # which converts into a formatted IP prefix string. - addr = _split_optional_netmask(address) - self.network_address = IPv4Address(self._ip_int_from_string(addr[0])) - - if len(addr) == 2: - arg = addr[1] - else: - arg = self._max_prefixlen - self.netmask, self._prefixlen = self._make_netmask(arg) - - if strict: - if ( - IPv4Address(int(self.network_address) & int(self.netmask)) - != self.network_address - ): - raise ValueError("%s has host bits set" % self) - self.network_address = IPv4Address( - int(self.network_address) & int(self.netmask) - ) - - if self._prefixlen == (self._max_prefixlen - 1): - self.hosts = self.__iter__ - - @property - def is_global(self): - """Test if this address is allocated for public networks. - - Returns: - A boolean, True if the address is not reserved per - iana-ipv4-special-registry. - - """ - return ( - not ( - self.network_address in IPv4Network("100.64.0.0/10") - and self.broadcast_address in IPv4Network("100.64.0.0/10") - ) - and not self.is_private - ) - - -class _IPv4Constants(object): - - _linklocal_network = IPv4Network("169.254.0.0/16") - - _loopback_network = IPv4Network("127.0.0.0/8") - - _multicast_network = IPv4Network("224.0.0.0/4") - - _public_network = IPv4Network("100.64.0.0/10") - - _private_networks = [ - IPv4Network("0.0.0.0/8"), - IPv4Network("10.0.0.0/8"), - IPv4Network("127.0.0.0/8"), - IPv4Network("169.254.0.0/16"), - IPv4Network("172.16.0.0/12"), - IPv4Network("192.0.0.0/29"), - IPv4Network("192.0.0.170/31"), - IPv4Network("192.0.2.0/24"), - IPv4Network("192.168.0.0/16"), - IPv4Network("198.18.0.0/15"), - IPv4Network("198.51.100.0/24"), - IPv4Network("203.0.113.0/24"), - IPv4Network("240.0.0.0/4"), - IPv4Network("255.255.255.255/32"), - ] - - _reserved_network = IPv4Network("240.0.0.0/4") - - _unspecified_address = IPv4Address("0.0.0.0") - - -IPv4Address._constants = _IPv4Constants - - -class _BaseV6(object): - - """Base IPv6 object. - - The following methods are used by IPv6 objects in both single IP - addresses and networks. - - """ - - __slots__ = () - _version = 6 - _ALL_ONES = (2 ** IPV6LENGTH) - 1 - _HEXTET_COUNT = 8 - _HEX_DIGITS = frozenset("0123456789ABCDEFabcdef") - _max_prefixlen = IPV6LENGTH - - # There are only a bunch of valid v6 netmasks, so we cache them all - # when constructed (see _make_netmask()). - _netmask_cache = {} - - @classmethod - def _make_netmask(cls, arg): - """Make a (netmask, prefix_len) tuple from the given argument. - - Argument can be: - - an integer (the prefix length) - - a string representing the prefix length (e.g. "24") - - a string representing the prefix netmask (e.g. "255.255.255.0") - """ - if arg not in cls._netmask_cache: - if isinstance(arg, _compat_int_types): - prefixlen = arg - else: - prefixlen = cls._prefix_from_prefix_string(arg) - netmask = IPv6Address(cls._ip_int_from_prefix(prefixlen)) - cls._netmask_cache[arg] = netmask, prefixlen - return cls._netmask_cache[arg] - - @classmethod - def _ip_int_from_string(cls, ip_str): - """Turn an IPv6 ip_str into an integer. - - Args: - ip_str: A string, the IPv6 ip_str. - - Returns: - An int, the IPv6 address - - Raises: - AddressValueError: if ip_str isn't a valid IPv6 Address. - - """ - if not ip_str: - raise AddressValueError("Address cannot be empty") - - parts = ip_str.split(":") - - # An IPv6 address needs at least 2 colons (3 parts). - _min_parts = 3 - if len(parts) < _min_parts: - msg = "At least %d parts expected in %r" % (_min_parts, ip_str) - raise AddressValueError(msg) - - # If the address has an IPv4-style suffix, convert it to hexadecimal. - if "." in parts[-1]: - try: - ipv4_int = IPv4Address(parts.pop())._ip - except AddressValueError as exc: - raise AddressValueError("%s in %r" % (exc, ip_str)) - parts.append("%x" % ((ipv4_int >> 16) & 0xFFFF)) - parts.append("%x" % (ipv4_int & 0xFFFF)) - - # An IPv6 address can't have more than 8 colons (9 parts). - # The extra colon comes from using the "::" notation for a single - # leading or trailing zero part. - _max_parts = cls._HEXTET_COUNT + 1 - if len(parts) > _max_parts: - msg = "At most %d colons permitted in %r" % ( - _max_parts - 1, - ip_str, - ) - raise AddressValueError(msg) - - # Disregarding the endpoints, find '::' with nothing in between. - # This indicates that a run of zeroes has been skipped. - skip_index = None - for i in _compat_range(1, len(parts) - 1): - if not parts[i]: - if skip_index is not None: - # Can't have more than one '::' - msg = "At most one '::' permitted in %r" % ip_str - raise AddressValueError(msg) - skip_index = i - - # parts_hi is the number of parts to copy from above/before the '::' - # parts_lo is the number of parts to copy from below/after the '::' - if skip_index is not None: - # If we found a '::', then check if it also covers the endpoints. - parts_hi = skip_index - parts_lo = len(parts) - skip_index - 1 - if not parts[0]: - parts_hi -= 1 - if parts_hi: - msg = "Leading ':' only permitted as part of '::' in %r" - raise AddressValueError(msg % ip_str) # ^: requires ^:: - if not parts[-1]: - parts_lo -= 1 - if parts_lo: - msg = "Trailing ':' only permitted as part of '::' in %r" - raise AddressValueError(msg % ip_str) # :$ requires ::$ - parts_skipped = cls._HEXTET_COUNT - (parts_hi + parts_lo) - if parts_skipped < 1: - msg = "Expected at most %d other parts with '::' in %r" - raise AddressValueError(msg % (cls._HEXTET_COUNT - 1, ip_str)) - else: - # Otherwise, allocate the entire address to parts_hi. The - # endpoints could still be empty, but _parse_hextet() will check - # for that. - if len(parts) != cls._HEXTET_COUNT: - msg = "Exactly %d parts expected without '::' in %r" - raise AddressValueError(msg % (cls._HEXTET_COUNT, ip_str)) - if not parts[0]: - msg = "Leading ':' only permitted as part of '::' in %r" - raise AddressValueError(msg % ip_str) # ^: requires ^:: - if not parts[-1]: - msg = "Trailing ':' only permitted as part of '::' in %r" - raise AddressValueError(msg % ip_str) # :$ requires ::$ - parts_hi = len(parts) - parts_lo = 0 - parts_skipped = 0 - - try: - # Now, parse the hextets into a 128-bit integer. - ip_int = 0 - for i in range(parts_hi): - ip_int <<= 16 - ip_int |= cls._parse_hextet(parts[i]) - ip_int <<= 16 * parts_skipped - for i in range(-parts_lo, 0): - ip_int <<= 16 - ip_int |= cls._parse_hextet(parts[i]) - return ip_int - except ValueError as exc: - raise AddressValueError("%s in %r" % (exc, ip_str)) - - @classmethod - def _parse_hextet(cls, hextet_str): - """Convert an IPv6 hextet string into an integer. - - Args: - hextet_str: A string, the number to parse. - - Returns: - The hextet as an integer. - - Raises: - ValueError: if the input isn't strictly a hex number from - [0..FFFF]. - - """ - # Whitelist the characters, since int() allows a lot of bizarre stuff. - if not cls._HEX_DIGITS.issuperset(hextet_str): - raise ValueError("Only hex digits permitted in %r" % hextet_str) - # We do the length check second, since the invalid character error - # is likely to be more informative for the user - if len(hextet_str) > 4: - msg = "At most 4 characters permitted in %r" - raise ValueError(msg % hextet_str) - # Length check means we can skip checking the integer value - return int(hextet_str, 16) - - @classmethod - def _compress_hextets(cls, hextets): - """Compresses a list of hextets. - - Compresses a list of strings, replacing the longest continuous - sequence of "0" in the list with "" and adding empty strings at - the beginning or at the end of the string such that subsequently - calling ":".join(hextets) will produce the compressed version of - the IPv6 address. - - Args: - hextets: A list of strings, the hextets to compress. - - Returns: - A list of strings. - - """ - best_doublecolon_start = -1 - best_doublecolon_len = 0 - doublecolon_start = -1 - doublecolon_len = 0 - for index, hextet in enumerate(hextets): - if hextet == "0": - doublecolon_len += 1 - if doublecolon_start == -1: - # Start of a sequence of zeros. - doublecolon_start = index - if doublecolon_len > best_doublecolon_len: - # This is the longest sequence of zeros so far. - best_doublecolon_len = doublecolon_len - best_doublecolon_start = doublecolon_start - else: - doublecolon_len = 0 - doublecolon_start = -1 - - if best_doublecolon_len > 1: - best_doublecolon_end = ( - best_doublecolon_start + best_doublecolon_len - ) - # For zeros at the end of the address. - if best_doublecolon_end == len(hextets): - hextets += [""] - hextets[best_doublecolon_start:best_doublecolon_end] = [""] - # For zeros at the beginning of the address. - if best_doublecolon_start == 0: - hextets = [""] + hextets - - return hextets - - @classmethod - def _string_from_ip_int(cls, ip_int=None): - """Turns a 128-bit integer into hexadecimal notation. - - Args: - ip_int: An integer, the IP address. - - Returns: - A string, the hexadecimal representation of the address. - - Raises: - ValueError: The address is bigger than 128 bits of all ones. - - """ - if ip_int is None: - ip_int = int(cls._ip) - - if ip_int > cls._ALL_ONES: - raise ValueError("IPv6 address is too large") - - hex_str = "%032x" % ip_int - hextets = ["%x" % int(hex_str[x : x + 4], 16) for x in range(0, 32, 4)] - - hextets = cls._compress_hextets(hextets) - return ":".join(hextets) - - def _explode_shorthand_ip_string(self): - """Expand a shortened IPv6 address. - - Args: - ip_str: A string, the IPv6 address. - - Returns: - A string, the expanded IPv6 address. - - """ - if isinstance(self, IPv6Network): - ip_str = _compat_str(self.network_address) - elif isinstance(self, IPv6Interface): - ip_str = _compat_str(self.ip) - else: - ip_str = _compat_str(self) - - ip_int = self._ip_int_from_string(ip_str) - hex_str = "%032x" % ip_int - parts = [hex_str[x : x + 4] for x in range(0, 32, 4)] - if isinstance(self, (_BaseNetwork, IPv6Interface)): - return "%s/%d" % (":".join(parts), self._prefixlen) - return ":".join(parts) - - def _reverse_pointer(self): - """Return the reverse DNS pointer name for the IPv6 address. - - This implements the method described in RFC3596 2.5. - - """ - reverse_chars = self.exploded[::-1].replace(":", "") - return ".".join(reverse_chars) + ".ip6.arpa" - - @property - def max_prefixlen(self): - return self._max_prefixlen - - @property - def version(self): - return self._version - - -class IPv6Address(_BaseV6, _BaseAddress): - - """Represent and manipulate single IPv6 Addresses.""" - - __slots__ = ("_ip", "__weakref__") - - def __init__(self, address): - """Instantiate a new IPv6 address object. - - Args: - address: A string or integer representing the IP - - Additionally, an integer can be passed, so - IPv6Address('2001:db8::') == - IPv6Address(42540766411282592856903984951653826560) - or, more generally - IPv6Address(int(IPv6Address('2001:db8::'))) == - IPv6Address('2001:db8::') - - Raises: - AddressValueError: If address isn't a valid IPv6 address. - - """ - # Efficient constructor from integer. - if isinstance(address, _compat_int_types): - self._check_int_address(address) - self._ip = address - return - - # Constructing from a packed address - if isinstance(address, bytes): - self._check_packed_address(address, 16) - bvs = _compat_bytes_to_byte_vals(address) - self._ip = _compat_int_from_byte_vals(bvs, "big") - return - - # Assume input argument to be string or any object representation - # which converts into a formatted IP string. - addr_str = _compat_str(address) - if "/" in addr_str: - raise AddressValueError("Unexpected '/' in %r" % address) - self._ip = self._ip_int_from_string(addr_str) - - @property - def packed(self): - """The binary representation of this address.""" - return v6_int_to_packed(self._ip) - - @property - def is_multicast(self): - """Test if the address is reserved for multicast use. - - Returns: - A boolean, True if the address is a multicast address. - See RFC 2373 2.7 for details. - - """ - return self in self._constants._multicast_network - - @property - def is_reserved(self): - """Test if the address is otherwise IETF reserved. - - Returns: - A boolean, True if the address is within one of the - reserved IPv6 Network ranges. - - """ - return any(self in x for x in self._constants._reserved_networks) - - @property - def is_link_local(self): - """Test if the address is reserved for link-local. - - Returns: - A boolean, True if the address is reserved per RFC 4291. - - """ - return self in self._constants._linklocal_network - - @property - def is_site_local(self): - """Test if the address is reserved for site-local. - - Note that the site-local address space has been deprecated by RFC 3879. - Use is_private to test if this address is in the space of unique local - addresses as defined by RFC 4193. - - Returns: - A boolean, True if the address is reserved per RFC 3513 2.5.6. - - """ - return self in self._constants._sitelocal_network - - @property - def is_private(self): - """Test if this address is allocated for private networks. - - Returns: - A boolean, True if the address is reserved per - iana-ipv6-special-registry. - - """ - return any(self in net for net in self._constants._private_networks) - - @property - def is_global(self): - """Test if this address is allocated for public networks. - - Returns: - A boolean, true if the address is not reserved per - iana-ipv6-special-registry. - - """ - return not self.is_private - - @property - def is_unspecified(self): - """Test if the address is unspecified. - - Returns: - A boolean, True if this is the unspecified address as defined in - RFC 2373 2.5.2. - - """ - return self._ip == 0 - - @property - def is_loopback(self): - """Test if the address is a loopback address. - - Returns: - A boolean, True if the address is a loopback address as defined in - RFC 2373 2.5.3. - - """ - return self._ip == 1 - - @property - def ipv4_mapped(self): - """Return the IPv4 mapped address. - - Returns: - If the IPv6 address is a v4 mapped address, return the - IPv4 mapped address. Return None otherwise. - - """ - if (self._ip >> 32) != 0xFFFF: - return None - return IPv4Address(self._ip & 0xFFFFFFFF) - - @property - def teredo(self): - """Tuple of embedded teredo IPs. - - Returns: - Tuple of the (server, client) IPs or None if the address - doesn't appear to be a teredo address (doesn't start with - 2001::/32) - - """ - if (self._ip >> 96) != 0x20010000: - return None - return ( - IPv4Address((self._ip >> 64) & 0xFFFFFFFF), - IPv4Address(~self._ip & 0xFFFFFFFF), - ) - - @property - def sixtofour(self): - """Return the IPv4 6to4 embedded address. - - Returns: - The IPv4 6to4-embedded address if present or None if the - address doesn't appear to contain a 6to4 embedded address. - - """ - if (self._ip >> 112) != 0x2002: - return None - return IPv4Address((self._ip >> 80) & 0xFFFFFFFF) - - -class IPv6Interface(IPv6Address): - def __init__(self, address): - if isinstance(address, (bytes, _compat_int_types)): - IPv6Address.__init__(self, address) - self.network = IPv6Network(self._ip) - self._prefixlen = self._max_prefixlen - return - if isinstance(address, tuple): - IPv6Address.__init__(self, address[0]) - if len(address) > 1: - self._prefixlen = int(address[1]) - else: - self._prefixlen = self._max_prefixlen - self.network = IPv6Network(address, strict=False) - self.netmask = self.network.netmask - self.hostmask = self.network.hostmask - return - - addr = _split_optional_netmask(address) - IPv6Address.__init__(self, addr[0]) - self.network = IPv6Network(address, strict=False) - self.netmask = self.network.netmask - self._prefixlen = self.network._prefixlen - self.hostmask = self.network.hostmask - - def __str__(self): - return "%s/%d" % ( - self._string_from_ip_int(self._ip), - self.network.prefixlen, - ) - - def __eq__(self, other): - address_equal = IPv6Address.__eq__(self, other) - if not address_equal or address_equal is NotImplemented: - return address_equal - try: - return self.network == other.network - except AttributeError: - # An interface with an associated network is NOT the - # same as an unassociated address. That's why the hash - # takes the extra info into account. - return False - - def __lt__(self, other): - address_less = IPv6Address.__lt__(self, other) - if address_less is NotImplemented: - return NotImplemented - try: - return ( - self.network < other.network - or self.network == other.network - and address_less - ) - except AttributeError: - # We *do* allow addresses and interfaces to be sorted. The - # unassociated address is considered less than all interfaces. - return False - - def __hash__(self): - return self._ip ^ self._prefixlen ^ int(self.network.network_address) - - __reduce__ = _IPAddressBase.__reduce__ - - @property - def ip(self): - return IPv6Address(self._ip) - - @property - def with_prefixlen(self): - return "%s/%s" % (self._string_from_ip_int(self._ip), self._prefixlen) - - @property - def with_netmask(self): - return "%s/%s" % (self._string_from_ip_int(self._ip), self.netmask) - - @property - def with_hostmask(self): - return "%s/%s" % (self._string_from_ip_int(self._ip), self.hostmask) - - @property - def is_unspecified(self): - return self._ip == 0 and self.network.is_unspecified - - @property - def is_loopback(self): - return self._ip == 1 and self.network.is_loopback - - -class IPv6Network(_BaseV6, _BaseNetwork): - - """This class represents and manipulates 128-bit IPv6 networks. - - Attributes: [examples for IPv6('2001:db8::1000/124')] - .network_address: IPv6Address('2001:db8::1000') - .hostmask: IPv6Address('::f') - .broadcast_address: IPv6Address('2001:db8::100f') - .netmask: IPv6Address('ffff:ffff:ffff:ffff:ffff:ffff:ffff:fff0') - .prefixlen: 124 - - """ - - # Class to use when creating address objects - _address_class = IPv6Address - - def __init__(self, address, strict=True): - """Instantiate a new IPv6 Network object. - - Args: - address: A string or integer representing the IPv6 network or the - IP and prefix/netmask. - '2001:db8::/128' - '2001:db8:0000:0000:0000:0000:0000:0000/128' - '2001:db8::' - are all functionally the same in IPv6. That is to say, - failing to provide a subnetmask will create an object with - a mask of /128. - - Additionally, an integer can be passed, so - IPv6Network('2001:db8::') == - IPv6Network(42540766411282592856903984951653826560) - or, more generally - IPv6Network(int(IPv6Network('2001:db8::'))) == - IPv6Network('2001:db8::') - - strict: A boolean. If true, ensure that we have been passed - A true network address, eg, 2001:db8::1000/124 and not an - IP address on a network, eg, 2001:db8::1/124. - - Raises: - AddressValueError: If address isn't a valid IPv6 address. - NetmaskValueError: If the netmask isn't valid for - an IPv6 address. - ValueError: If strict was True and a network address was not - supplied. - - """ - _BaseNetwork.__init__(self, address) - - # Efficient constructor from integer or packed address - if isinstance(address, (bytes, _compat_int_types)): - self.network_address = IPv6Address(address) - self.netmask, self._prefixlen = self._make_netmask( - self._max_prefixlen - ) - return - - if isinstance(address, tuple): - if len(address) > 1: - arg = address[1] - else: - arg = self._max_prefixlen - self.netmask, self._prefixlen = self._make_netmask(arg) - self.network_address = IPv6Address(address[0]) - packed = int(self.network_address) - if packed & int(self.netmask) != packed: - if strict: - raise ValueError("%s has host bits set" % self) - else: - self.network_address = IPv6Address( - packed & int(self.netmask) - ) - return - - # Assume input argument to be string or any object representation - # which converts into a formatted IP prefix string. - addr = _split_optional_netmask(address) - - self.network_address = IPv6Address(self._ip_int_from_string(addr[0])) - - if len(addr) == 2: - arg = addr[1] - else: - arg = self._max_prefixlen - self.netmask, self._prefixlen = self._make_netmask(arg) - - if strict: - if ( - IPv6Address(int(self.network_address) & int(self.netmask)) - != self.network_address - ): - raise ValueError("%s has host bits set" % self) - self.network_address = IPv6Address( - int(self.network_address) & int(self.netmask) - ) - - if self._prefixlen == (self._max_prefixlen - 1): - self.hosts = self.__iter__ - - def hosts(self): - """Generate Iterator over usable hosts in a network. - - This is like __iter__ except it doesn't return the - Subnet-Router anycast address. - - """ - network = int(self.network_address) - broadcast = int(self.broadcast_address) - for x in _compat_range(network + 1, broadcast + 1): - yield self._address_class(x) - - @property - def is_site_local(self): - """Test if the address is reserved for site-local. - - Note that the site-local address space has been deprecated by RFC 3879. - Use is_private to test if this address is in the space of unique local - addresses as defined by RFC 4193. - - Returns: - A boolean, True if the address is reserved per RFC 3513 2.5.6. - - """ - return ( - self.network_address.is_site_local - and self.broadcast_address.is_site_local - ) - - -class _IPv6Constants(object): - - _linklocal_network = IPv6Network("fe80::/10") - - _multicast_network = IPv6Network("ff00::/8") - - _private_networks = [ - IPv6Network("::1/128"), - IPv6Network("::/128"), - IPv6Network("::ffff:0:0/96"), - IPv6Network("100::/64"), - IPv6Network("2001::/23"), - IPv6Network("2001:2::/48"), - IPv6Network("2001:db8::/32"), - IPv6Network("2001:10::/28"), - IPv6Network("fc00::/7"), - IPv6Network("fe80::/10"), - ] - - _reserved_networks = [ - IPv6Network("::/8"), - IPv6Network("100::/8"), - IPv6Network("200::/7"), - IPv6Network("400::/6"), - IPv6Network("800::/5"), - IPv6Network("1000::/4"), - IPv6Network("4000::/3"), - IPv6Network("6000::/3"), - IPv6Network("8000::/3"), - IPv6Network("A000::/3"), - IPv6Network("C000::/3"), - IPv6Network("E000::/4"), - IPv6Network("F000::/5"), - IPv6Network("F800::/6"), - IPv6Network("FE00::/9"), - ] - - _sitelocal_network = IPv6Network("fec0::/10") - - -IPv6Address._constants = _IPv6Constants diff --git a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/cfg/base.py b/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/cfg/base.py deleted file mode 100644 index 75f80aa3f88..00000000000 --- a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/cfg/base.py +++ /dev/null @@ -1,28 +0,0 @@ -# -# -*- coding: utf-8 -*- -# Copyright 2019 Red Hat -# GNU General Public License v3.0+ -# (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) -""" -The base class for all resource modules -""" -from __future__ import annotations - -from ansible_collections.ansible.netcommon.plugins.module_utils.network.common.network import ( - get_resource_connection, -) - - -class ConfigBase(object): - """ The base class for all resource modules - """ - - ACTION_STATES = ["merged", "replaced", "overridden", "deleted"] - - def __init__(self, module): - self._module = module - self.state = module.params["state"] - self._connection = None - - if self.state not in ["rendered", "parsed"]: - self._connection = get_resource_connection(module) diff --git a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/config.py b/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/config.py deleted file mode 100644 index d6f278a53fd..00000000000 --- a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/config.py +++ /dev/null @@ -1,475 +0,0 @@ -# This code is part of Ansible, but is an independent component. -# This particular file snippet, and this file snippet only, is BSD licensed. -# Modules you write using this snippet, which is embedded dynamically by Ansible -# still belong to the author of the module, and may assign their own license -# to the complete work. -# -# (c) 2016 Red Hat Inc. -# -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -from __future__ import annotations - -import re -import hashlib - -from ansible.module_utils.six.moves import zip -from ansible.module_utils.common.text.converters import to_bytes, to_native - -DEFAULT_COMMENT_TOKENS = ["#", "!", "/*", "*/", "echo"] - -DEFAULT_IGNORE_LINES_RE = set( - [ - re.compile(r"Using \d+ out of \d+ bytes"), - re.compile(r"Building configuration"), - re.compile(r"Current configuration : \d+ bytes"), - ] -) - - -try: - Pattern = re._pattern_type -except AttributeError: - Pattern = re.Pattern - - -class ConfigLine(object): - def __init__(self, raw): - self.text = str(raw).strip() - self.raw = raw - self._children = list() - self._parents = list() - - def __str__(self): - return self.raw - - def __eq__(self, other): - return self.line == other.line - - def __ne__(self, other): - return not self.__eq__(other) - - def __getitem__(self, key): - for item in self._children: - if item.text == key: - return item - raise KeyError(key) - - @property - def line(self): - line = self.parents - line.append(self.text) - return " ".join(line) - - @property - def children(self): - return _obj_to_text(self._children) - - @property - def child_objs(self): - return self._children - - @property - def parents(self): - return _obj_to_text(self._parents) - - @property - def path(self): - config = _obj_to_raw(self._parents) - config.append(self.raw) - return "\n".join(config) - - @property - def has_children(self): - return len(self._children) > 0 - - @property - def has_parents(self): - return len(self._parents) > 0 - - def add_child(self, obj): - if not isinstance(obj, ConfigLine): - raise AssertionError("child must be of type `ConfigLine`") - self._children.append(obj) - - -def ignore_line(text, tokens=None): - for item in tokens or DEFAULT_COMMENT_TOKENS: - if text.startswith(item): - return True - for regex in DEFAULT_IGNORE_LINES_RE: - if regex.match(text): - return True - - -def _obj_to_text(x): - return [o.text for o in x] - - -def _obj_to_raw(x): - return [o.raw for o in x] - - -def _obj_to_block(objects, visited=None): - items = list() - for o in objects: - if o not in items: - items.append(o) - for child in o._children: - if child not in items: - items.append(child) - return _obj_to_raw(items) - - -def dumps(objects, output="block", comments=False): - if output == "block": - items = _obj_to_block(objects) - elif output == "commands": - items = _obj_to_text(objects) - elif output == "raw": - items = _obj_to_raw(objects) - else: - raise TypeError("unknown value supplied for keyword output") - - if output == "block": - if comments: - for index, item in enumerate(items): - nextitem = index + 1 - if ( - nextitem < len(items) - and not item.startswith(" ") - and items[nextitem].startswith(" ") - ): - item = "!\n%s" % item - items[index] = item - items.append("!") - items.append("end") - - return "\n".join(items) - - -class NetworkConfig(object): - def __init__(self, indent=1, contents=None, ignore_lines=None): - self._indent = indent - self._items = list() - self._config_text = None - - if ignore_lines: - for item in ignore_lines: - if not isinstance(item, Pattern): - item = re.compile(item) - DEFAULT_IGNORE_LINES_RE.add(item) - - if contents: - self.load(contents) - - @property - def items(self): - return self._items - - @property - def config_text(self): - return self._config_text - - @property - def sha1(self): - sha1 = hashlib.sha1() - sha1.update(to_bytes(str(self), errors="surrogate_or_strict")) - return sha1.digest() - - def __getitem__(self, key): - for line in self: - if line.text == key: - return line - raise KeyError(key) - - def __iter__(self): - return iter(self._items) - - def __str__(self): - return "\n".join([c.raw for c in self.items]) - - def __len__(self): - return len(self._items) - - def load(self, s): - self._config_text = s - self._items = self.parse(s) - - def loadfp(self, fp): - with open(fp) as f: - return self.load(f.read()) - - def parse(self, lines, comment_tokens=None): - toplevel = re.compile(r"\S") - childline = re.compile(r"^\s*(.+)$") - entry_reg = re.compile(r"([{};])") - - ancestors = list() - config = list() - - indents = [0] - - for linenum, line in enumerate( - to_native(lines, errors="surrogate_or_strict").split("\n") - ): - text = entry_reg.sub("", line).strip() - - cfg = ConfigLine(line) - - if not text or ignore_line(text, comment_tokens): - continue - - # handle top level commands - if toplevel.match(line): - ancestors = [cfg] - indents = [0] - - # handle sub level commands - else: - match = childline.match(line) - line_indent = match.start(1) - - if line_indent < indents[-1]: - while indents[-1] > line_indent: - indents.pop() - - if line_indent > indents[-1]: - indents.append(line_indent) - - curlevel = len(indents) - 1 - parent_level = curlevel - 1 - - cfg._parents = ancestors[:curlevel] - - if curlevel > len(ancestors): - config.append(cfg) - continue - - for i in range(curlevel, len(ancestors)): - ancestors.pop() - - ancestors.append(cfg) - ancestors[parent_level].add_child(cfg) - - config.append(cfg) - - return config - - def get_object(self, path): - for item in self.items: - if item.text == path[-1]: - if item.parents == path[:-1]: - return item - - def get_block(self, path): - if not isinstance(path, list): - raise AssertionError("path argument must be a list object") - obj = self.get_object(path) - if not obj: - raise ValueError("path does not exist in config") - return self._expand_block(obj) - - def get_block_config(self, path): - block = self.get_block(path) - return dumps(block, "block") - - def _expand_block(self, configobj, S=None): - if S is None: - S = list() - S.append(configobj) - for child in configobj._children: - if child in S: - continue - self._expand_block(child, S) - return S - - def _diff_line(self, other): - updates = list() - for item in self.items: - if item not in other: - updates.append(item) - return updates - - def _diff_strict(self, other): - updates = list() - # block extracted from other does not have all parents - # but the last one. In case of multiple parents we need - # to add additional parents. - if other and isinstance(other, list) and len(other) > 0: - start_other = other[0] - if start_other.parents: - for parent in start_other.parents: - other.insert(0, ConfigLine(parent)) - for index, line in enumerate(self.items): - try: - if str(line).strip() != str(other[index]).strip(): - updates.append(line) - except (AttributeError, IndexError): - updates.append(line) - return updates - - def _diff_exact(self, other): - updates = list() - if len(other) != len(self.items): - updates.extend(self.items) - else: - for ours, theirs in zip(self.items, other): - if ours != theirs: - updates.extend(self.items) - break - return updates - - def difference(self, other, match="line", path=None, replace=None): - """Perform a config diff against the another network config - - :param other: instance of NetworkConfig to diff against - :param match: type of diff to perform. valid values are 'line', - 'strict', 'exact' - :param path: context in the network config to filter the diff - :param replace: the method used to generate the replacement lines. - valid values are 'block', 'line' - - :returns: a string of lines that are different - """ - if path and match != "line": - try: - other = other.get_block(path) - except ValueError: - other = list() - else: - other = other.items - - # generate a list of ConfigLines that aren't in other - meth = getattr(self, "_diff_%s" % match) - updates = meth(other) - - if replace == "block": - parents = list() - for item in updates: - if not item.has_parents: - parents.append(item) - else: - for p in item._parents: - if p not in parents: - parents.append(p) - - updates = list() - for item in parents: - updates.extend(self._expand_block(item)) - - visited = set() - expanded = list() - - for item in updates: - for p in item._parents: - if p.line not in visited: - visited.add(p.line) - expanded.append(p) - expanded.append(item) - visited.add(item.line) - - return expanded - - def add(self, lines, parents=None): - ancestors = list() - offset = 0 - obj = None - - # global config command - if not parents: - for line in lines: - # handle ignore lines - if ignore_line(line): - continue - - item = ConfigLine(line) - item.raw = line - if item not in self.items: - self.items.append(item) - - else: - for index, p in enumerate(parents): - try: - i = index + 1 - obj = self.get_block(parents[:i])[0] - ancestors.append(obj) - - except ValueError: - # add parent to config - offset = index * self._indent - obj = ConfigLine(p) - obj.raw = p.rjust(len(p) + offset) - if ancestors: - obj._parents = list(ancestors) - ancestors[-1]._children.append(obj) - self.items.append(obj) - ancestors.append(obj) - - # add child objects - for line in lines: - # handle ignore lines - if ignore_line(line): - continue - - # check if child already exists - for child in ancestors[-1]._children: - if child.text == line: - break - else: - offset = len(parents) * self._indent - item = ConfigLine(line) - item.raw = line.rjust(len(line) + offset) - item._parents = ancestors - ancestors[-1]._children.append(item) - self.items.append(item) - - -class CustomNetworkConfig(NetworkConfig): - def items_text(self): - return [item.text for item in self.items] - - def expand_section(self, configobj, S=None): - if S is None: - S = list() - S.append(configobj) - for child in configobj.child_objs: - if child in S: - continue - self.expand_section(child, S) - return S - - def to_block(self, section): - return "\n".join([item.raw for item in section]) - - def get_section(self, path): - try: - section = self.get_section_objects(path) - return self.to_block(section) - except ValueError: - return list() - - def get_section_objects(self, path): - if not isinstance(path, list): - path = [path] - obj = self.get_object(path) - if not obj: - raise ValueError("path does not exist in config") - return self.expand_section(obj) diff --git a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/facts/facts.py b/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/facts/facts.py deleted file mode 100644 index 0a484d0607b..00000000000 --- a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/facts/facts.py +++ /dev/null @@ -1,164 +0,0 @@ -# -# -*- coding: utf-8 -*- -# Copyright 2019 Red Hat -# GNU General Public License v3.0+ -# (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) -""" -The facts base class -this contains methods common to all facts subsets -""" -from __future__ import annotations - -from ansible_collections.ansible.netcommon.plugins.module_utils.network.common.network import ( - get_resource_connection, -) -from ansible.module_utils.six import iteritems - - -class FactsBase(object): - """ - The facts base class - """ - - def __init__(self, module): - self._module = module - self._warnings = [] - self._gather_subset = module.params.get("gather_subset") - self._gather_network_resources = module.params.get( - "gather_network_resources" - ) - self._connection = None - if module.params.get("state") not in ["rendered", "parsed"]: - self._connection = get_resource_connection(module) - - self.ansible_facts = {"ansible_network_resources": {}} - self.ansible_facts["ansible_net_gather_network_resources"] = list() - self.ansible_facts["ansible_net_gather_subset"] = list() - - if not self._gather_subset: - self._gather_subset = ["!config"] - if not self._gather_network_resources: - self._gather_network_resources = ["!all"] - - def gen_runable(self, subsets, valid_subsets, resource_facts=False): - """ Generate the runable subset - - :param module: The module instance - :param subsets: The provided subsets - :param valid_subsets: The valid subsets - :param resource_facts: A boolean flag - :rtype: list - :returns: The runable subsets - """ - runable_subsets = set() - exclude_subsets = set() - minimal_gather_subset = set() - if not resource_facts: - minimal_gather_subset = frozenset(["default"]) - - for subset in subsets: - if subset == "all": - runable_subsets.update(valid_subsets) - continue - if subset == "min" and minimal_gather_subset: - runable_subsets.update(minimal_gather_subset) - continue - if subset.startswith("!"): - subset = subset[1:] - if subset == "min": - exclude_subsets.update(minimal_gather_subset) - continue - if subset == "all": - exclude_subsets.update( - valid_subsets - minimal_gather_subset - ) - continue - exclude = True - else: - exclude = False - - if subset not in valid_subsets: - self._module.fail_json( - msg="Subset must be one of [%s], got %s" - % ( - ", ".join(sorted(list(valid_subsets))), - subset, - ) - ) - - if exclude: - exclude_subsets.add(subset) - else: - runable_subsets.add(subset) - - if not runable_subsets: - runable_subsets.update(valid_subsets) - runable_subsets.difference_update(exclude_subsets) - return runable_subsets - - def get_network_resources_facts( - self, facts_resource_obj_map, resource_facts_type=None, data=None - ): - """ - :param fact_resource_subsets: - :param data: previously collected configuration - :return: - """ - if not resource_facts_type: - resource_facts_type = self._gather_network_resources - - restorun_subsets = self.gen_runable( - resource_facts_type, - frozenset(facts_resource_obj_map.keys()), - resource_facts=True, - ) - if restorun_subsets: - self.ansible_facts["ansible_net_gather_network_resources"] = list( - restorun_subsets - ) - instances = list() - for key in restorun_subsets: - fact_cls_obj = facts_resource_obj_map.get(key) - if fact_cls_obj: - instances.append(fact_cls_obj(self._module)) - else: - self._warnings.extend( - [ - "network resource fact gathering for '%s' is not supported" - % key - ] - ) - - for inst in instances: - inst.populate_facts(self._connection, self.ansible_facts, data) - - def get_network_legacy_facts( - self, fact_legacy_obj_map, legacy_facts_type=None - ): - if not legacy_facts_type: - legacy_facts_type = self._gather_subset - - runable_subsets = self.gen_runable( - legacy_facts_type, frozenset(fact_legacy_obj_map.keys()) - ) - if runable_subsets: - facts = dict() - # default subset should always returned be with legacy facts subsets - if "default" not in runable_subsets: - runable_subsets.add("default") - self.ansible_facts["ansible_net_gather_subset"] = list( - runable_subsets - ) - - instances = list() - for key in runable_subsets: - instances.append(fact_legacy_obj_map[key](self._module)) - - for inst in instances: - inst.populate() - facts.update(inst.facts) - self._warnings.extend(inst.warnings) - - for key, value in iteritems(facts): - key = "ansible_net_%s" % key - self.ansible_facts[key] = value diff --git a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/netconf.py b/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/netconf.py deleted file mode 100644 index 4979dac9e22..00000000000 --- a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/netconf.py +++ /dev/null @@ -1,181 +0,0 @@ -# This code is part of Ansible, but is an independent component. -# This particular file snippet, and this file snippet only, is BSD licensed. -# Modules you write using this snippet, which is embedded dynamically by Ansible -# still belong to the author of the module, and may assign their own license -# to the complete work. -# -# (c) 2017 Red Hat Inc. -# -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -from __future__ import annotations - -import sys - -from ansible.module_utils.common.text.converters import to_text, to_bytes -from ansible.module_utils.connection import Connection, ConnectionError - -try: - from ncclient.xml_ import NCElement, new_ele, sub_ele - - HAS_NCCLIENT = True -except (ImportError, AttributeError): - HAS_NCCLIENT = False - -try: - from lxml.etree import Element, fromstring, XMLSyntaxError -except ImportError: - from xml.etree.ElementTree import Element, fromstring - - if sys.version_info < (2, 7): - from xml.parsers.expat import ExpatError as XMLSyntaxError - else: - from xml.etree.ElementTree import ParseError as XMLSyntaxError - -NS_MAP = {"nc": "urn:ietf:params:xml:ns:netconf:base:1.0"} - - -def exec_rpc(module, *args, **kwargs): - connection = NetconfConnection(module._socket_path) - return connection.execute_rpc(*args, **kwargs) - - -class NetconfConnection(Connection): - def __init__(self, socket_path): - super(NetconfConnection, self).__init__(socket_path) - - def __rpc__(self, name, *args, **kwargs): - """Executes the json-rpc and returns the output received - from remote device. - :name: rpc method to be executed over connection plugin that implements jsonrpc 2.0 - :args: Ordered list of params passed as arguments to rpc method - :kwargs: Dict of valid key, value pairs passed as arguments to rpc method - - For usage refer the respective connection plugin docs. - """ - self.check_rc = kwargs.pop("check_rc", True) - self.ignore_warning = kwargs.pop("ignore_warning", True) - - response = self._exec_jsonrpc(name, *args, **kwargs) - if "error" in response: - rpc_error = response["error"].get("data") - return self.parse_rpc_error( - to_bytes(rpc_error, errors="surrogate_then_replace") - ) - - return fromstring( - to_bytes(response["result"], errors="surrogate_then_replace") - ) - - def parse_rpc_error(self, rpc_error): - if self.check_rc: - try: - error_root = fromstring(rpc_error) - root = Element("root") - root.append(error_root) - - error_list = root.findall(".//nc:rpc-error", NS_MAP) - if not error_list: - raise ConnectionError( - to_text(rpc_error, errors="surrogate_then_replace") - ) - - warnings = [] - for error in error_list: - message_ele = error.find("./nc:error-message", NS_MAP) - - if message_ele is None: - message_ele = error.find("./nc:error-info", NS_MAP) - - message = ( - message_ele.text if message_ele is not None else None - ) - - severity = error.find("./nc:error-severity", NS_MAP).text - - if ( - severity == "warning" - and self.ignore_warning - and message is not None - ): - warnings.append(message) - else: - raise ConnectionError( - to_text(rpc_error, errors="surrogate_then_replace") - ) - return warnings - except XMLSyntaxError: - raise ConnectionError(rpc_error) - - -def transform_reply(): - return b""" - - - - - - - - - - - - - - - - - - - - - """ - - -# Note: Workaround for ncclient 0.5.3 -def remove_namespaces(data): - if not HAS_NCCLIENT: - raise ImportError( - "ncclient is required but does not appear to be installed. " - "It can be installed using `pip install ncclient`" - ) - return NCElement(data, transform_reply()).data_xml - - -def build_root_xml_node(tag): - return new_ele(tag) - - -def build_child_xml_node(parent, tag, text=None, attrib=None): - element = sub_ele(parent, tag) - if text: - element.text = to_text(text) - if attrib: - element.attrib.update(attrib) - return element - - -def build_subtree(parent, path): - element = parent - for field in path.split("/"): - sub_element = build_child_xml_node(element, field) - element = sub_element - return element diff --git a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/network.py b/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/network.py deleted file mode 100644 index c1f1d7b5b3f..00000000000 --- a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/network.py +++ /dev/null @@ -1,276 +0,0 @@ -# This code is part of Ansible, but is an independent component. -# This particular file snippet, and this file snippet only, is BSD licensed. -# Modules you write using this snippet, which is embedded dynamically by Ansible -# still belong to the author of the module, and may assign their own license -# to the complete work. -# -# Copyright (c) 2015 Peter Sprygada, -# -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import annotations - -import traceback -import json - -from ansible.module_utils.common.text.converters import to_text, to_native -from ansible.module_utils.basic import AnsibleModule -from ansible.module_utils.basic import env_fallback -from ansible.module_utils.connection import Connection, ConnectionError -from ansible_collections.ansible.netcommon.plugins.module_utils.network.common.netconf import ( - NetconfConnection, -) -from ansible_collections.ansible.netcommon.plugins.module_utils.network.common.parsing import ( - Cli, -) -from ansible.module_utils.six import iteritems - - -NET_TRANSPORT_ARGS = dict( - host=dict(required=True), - port=dict(type="int"), - username=dict(fallback=(env_fallback, ["ANSIBLE_NET_USERNAME"])), - password=dict( - no_log=True, fallback=(env_fallback, ["ANSIBLE_NET_PASSWORD"]) - ), - ssh_keyfile=dict( - fallback=(env_fallback, ["ANSIBLE_NET_SSH_KEYFILE"]), type="path" - ), - authorize=dict( - default=False, - fallback=(env_fallback, ["ANSIBLE_NET_AUTHORIZE"]), - type="bool", - ), - auth_pass=dict( - no_log=True, fallback=(env_fallback, ["ANSIBLE_NET_AUTH_PASS"]) - ), - provider=dict(type="dict", no_log=True), - transport=dict(choices=list()), - timeout=dict(default=10, type="int"), -) - -NET_CONNECTION_ARGS = dict() - -NET_CONNECTIONS = dict() - - -def _transitional_argument_spec(): - argument_spec = {} - for key, value in iteritems(NET_TRANSPORT_ARGS): - value["required"] = False - argument_spec[key] = value - return argument_spec - - -def to_list(val): - if isinstance(val, (list, tuple)): - return list(val) - elif val is not None: - return [val] - else: - return list() - - -class ModuleStub(object): - def __init__(self, argument_spec, fail_json): - self.params = dict() - for key, value in argument_spec.items(): - self.params[key] = value.get("default") - self.fail_json = fail_json - - -class NetworkError(Exception): - def __init__(self, msg, **kwargs): - super(NetworkError, self).__init__(msg) - self.kwargs = kwargs - - -class Config(object): - def __init__(self, connection): - self.connection = connection - - def __call__(self, commands, **kwargs): - lines = to_list(commands) - return self.connection.configure(lines, **kwargs) - - def load_config(self, commands, **kwargs): - commands = to_list(commands) - return self.connection.load_config(commands, **kwargs) - - def get_config(self, **kwargs): - return self.connection.get_config(**kwargs) - - def save_config(self): - return self.connection.save_config() - - -class NetworkModule(AnsibleModule): - def __init__(self, *args, **kwargs): - connect_on_load = kwargs.pop("connect_on_load", True) - - argument_spec = NET_TRANSPORT_ARGS.copy() - argument_spec["transport"]["choices"] = NET_CONNECTIONS.keys() - argument_spec.update(NET_CONNECTION_ARGS.copy()) - - if kwargs.get("argument_spec"): - argument_spec.update(kwargs["argument_spec"]) - kwargs["argument_spec"] = argument_spec - - super(NetworkModule, self).__init__(*args, **kwargs) - - self.connection = None - self._cli = None - self._config = None - - try: - transport = self.params["transport"] or "__default__" - cls = NET_CONNECTIONS[transport] - self.connection = cls() - except KeyError: - self.fail_json( - msg="Unknown transport or no default transport specified" - ) - except (TypeError, NetworkError) as exc: - self.fail_json( - msg=to_native(exc), exception=traceback.format_exc() - ) - - if connect_on_load: - self.connect() - - @property - def cli(self): - if not self.connected: - self.connect() - if self._cli: - return self._cli - self._cli = Cli(self.connection) - return self._cli - - @property - def config(self): - if not self.connected: - self.connect() - if self._config: - return self._config - self._config = Config(self.connection) - return self._config - - @property - def connected(self): - return self.connection._connected - - def _load_params(self): - super(NetworkModule, self)._load_params() - provider = self.params.get("provider") or dict() - for key, value in provider.items(): - for args in [NET_TRANSPORT_ARGS, NET_CONNECTION_ARGS]: - if key in args: - if self.params.get(key) is None and value is not None: - self.params[key] = value - - def connect(self): - try: - if not self.connected: - self.connection.connect(self.params) - if self.params["authorize"]: - self.connection.authorize(self.params) - self.log( - "connected to %s:%s using %s" - % ( - self.params["host"], - self.params["port"], - self.params["transport"], - ) - ) - except NetworkError as exc: - self.fail_json( - msg=to_native(exc), exception=traceback.format_exc() - ) - - def disconnect(self): - try: - if self.connected: - self.connection.disconnect() - self.log("disconnected from %s" % self.params["host"]) - except NetworkError as exc: - self.fail_json( - msg=to_native(exc), exception=traceback.format_exc() - ) - - -def register_transport(transport, default=False): - def register(cls): - NET_CONNECTIONS[transport] = cls - if default: - NET_CONNECTIONS["__default__"] = cls - return cls - - return register - - -def add_argument(key, value): - NET_CONNECTION_ARGS[key] = value - - -def get_resource_connection(module): - if hasattr(module, "_connection"): - return module._connection - - capabilities = get_capabilities(module) - network_api = capabilities.get("network_api") - if network_api in ("cliconf", "nxapi", "eapi", "exosapi"): - module._connection = Connection(module._socket_path) - elif network_api == "netconf": - module._connection = NetconfConnection(module._socket_path) - elif network_api == "local": - # This isn't supported, but we shouldn't fail here. - # Set the connection to a fake connection so it fails sensibly. - module._connection = LocalResourceConnection(module) - else: - module.fail_json( - msg="Invalid connection type {0!s}".format(network_api) - ) - - return module._connection - - -def get_capabilities(module): - if hasattr(module, "capabilities"): - return module._capabilities - try: - capabilities = Connection(module._socket_path).get_capabilities() - except ConnectionError as exc: - module.fail_json(msg=to_text(exc, errors="surrogate_then_replace")) - except AssertionError: - # No socket_path, connection most likely local. - return dict(network_api="local") - module._capabilities = json.loads(capabilities) - - return module._capabilities - - -class LocalResourceConnection: - def __init__(self, module): - self.module = module - - def get(self, *args, **kwargs): - self.module.fail_json( - msg="Network resource modules not supported over local connection." - ) diff --git a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/parsing.py b/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/parsing.py deleted file mode 100644 index 2e8e174e551..00000000000 --- a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/parsing.py +++ /dev/null @@ -1,317 +0,0 @@ -# This code is part of Ansible, but is an independent component. -# This particular file snippet, and this file snippet only, is BSD licensed. -# Modules you write using this snippet, which is embedded dynamically by Ansible -# still belong to the author of the module, and may assign their own license -# to the complete work. -# -# Copyright (c) 2015 Peter Sprygada, -# -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import annotations - -import re -import shlex -import time - -from ansible.module_utils.parsing.convert_bool import ( - BOOLEANS_TRUE, - BOOLEANS_FALSE, -) -from ansible.module_utils.six import string_types, text_type -from ansible.module_utils.six.moves import zip - - -def to_list(val): - if isinstance(val, (list, tuple)): - return list(val) - elif val is not None: - return [val] - else: - return list() - - -class FailedConditionsError(Exception): - def __init__(self, msg, failed_conditions): - super(FailedConditionsError, self).__init__(msg) - self.failed_conditions = failed_conditions - - -class FailedConditionalError(Exception): - def __init__(self, msg, failed_conditional): - super(FailedConditionalError, self).__init__(msg) - self.failed_conditional = failed_conditional - - -class AddCommandError(Exception): - def __init__(self, msg, command): - super(AddCommandError, self).__init__(msg) - self.command = command - - -class AddConditionError(Exception): - def __init__(self, msg, condition): - super(AddConditionError, self).__init__(msg) - self.condition = condition - - -class Cli(object): - def __init__(self, connection): - self.connection = connection - self.default_output = connection.default_output or "text" - self._commands = list() - - @property - def commands(self): - return [str(c) for c in self._commands] - - def __call__(self, commands, output=None): - objects = list() - for cmd in to_list(commands): - objects.append(self.to_command(cmd, output)) - return self.connection.run_commands(objects) - - def to_command( - self, command, output=None, prompt=None, response=None, **kwargs - ): - output = output or self.default_output - if isinstance(command, Command): - return command - if isinstance(prompt, string_types): - prompt = re.compile(re.escape(prompt)) - return Command( - command, output, prompt=prompt, response=response, **kwargs - ) - - def add_commands(self, commands, output=None, **kwargs): - for cmd in commands: - self._commands.append(self.to_command(cmd, output, **kwargs)) - - def run_commands(self): - responses = self.connection.run_commands(self._commands) - for resp, cmd in zip(responses, self._commands): - cmd.response = resp - - # wipe out the commands list to avoid issues if additional - # commands are executed later - self._commands = list() - - return responses - - -class Command(object): - def __init__( - self, command, output=None, prompt=None, response=None, **kwargs - ): - - self.command = command - self.output = output - self.command_string = command - - self.prompt = prompt - self.response = response - - self.args = kwargs - - def __str__(self): - return self.command_string - - -class CommandRunner(object): - def __init__(self, module): - self.module = module - - self.items = list() - self.conditionals = set() - - self.commands = list() - - self.retries = 10 - self.interval = 1 - - self.match = "all" - - self._default_output = module.connection.default_output - - def add_command( - self, command, output=None, prompt=None, response=None, **kwargs - ): - if command in [str(c) for c in self.commands]: - raise AddCommandError( - "duplicated command detected", command=command - ) - cmd = self.module.cli.to_command( - command, output=output, prompt=prompt, response=response, **kwargs - ) - self.commands.append(cmd) - - def get_command(self, command, output=None): - for cmd in self.commands: - if cmd.command == command: - return cmd.response - raise ValueError("command '%s' not found" % command) - - def get_responses(self): - return [cmd.response for cmd in self.commands] - - def add_conditional(self, condition): - try: - self.conditionals.add(Conditional(condition)) - except AttributeError as exc: - raise AddConditionError(msg=str(exc), condition=condition) - - def run(self): - while self.retries > 0: - self.module.cli.add_commands(self.commands) - responses = self.module.cli.run_commands() - - for item in list(self.conditionals): - if item(responses): - if self.match == "any": - return item - self.conditionals.remove(item) - - if not self.conditionals: - break - - time.sleep(self.interval) - self.retries -= 1 - else: - failed_conditions = [item.raw for item in self.conditionals] - errmsg = ( - "One or more conditional statements have not been satisfied" - ) - raise FailedConditionsError(errmsg, failed_conditions) - - -class Conditional(object): - """Used in command modules to evaluate waitfor conditions - """ - - OPERATORS = { - "eq": ["eq", "=="], - "neq": ["neq", "ne", "!="], - "gt": ["gt", ">"], - "ge": ["ge", ">="], - "lt": ["lt", "<"], - "le": ["le", "<="], - "contains": ["contains"], - "matches": ["matches"], - } - - def __init__(self, conditional, encoding=None): - self.raw = conditional - self.negate = False - try: - components = shlex.split(conditional) - key, val = components[0], components[-1] - op_components = components[1:-1] - if "not" in op_components: - self.negate = True - op_components.pop(op_components.index("not")) - op = op_components[0] - - except ValueError: - raise ValueError("failed to parse conditional") - - self.key = key - self.func = self._func(op) - self.value = self._cast_value(val) - - def __call__(self, data): - value = self.get_value(dict(result=data)) - if not self.negate: - return self.func(value) - else: - return not self.func(value) - - def _cast_value(self, value): - if value in BOOLEANS_TRUE: - return True - elif value in BOOLEANS_FALSE: - return False - elif re.match(r"^\d+\.d+$", value): - return float(value) - elif re.match(r"^\d+$", value): - return int(value) - else: - return text_type(value) - - def _func(self, oper): - for func, operators in self.OPERATORS.items(): - if oper in operators: - return getattr(self, func) - raise AttributeError("unknown operator: %s" % oper) - - def get_value(self, result): - try: - return self.get_json(result) - except (IndexError, TypeError, AttributeError): - msg = "unable to apply conditional to result" - raise FailedConditionalError(msg, self.raw) - - def get_json(self, result): - string = re.sub(r"\[[\'|\"]", ".", self.key) - string = re.sub(r"[\'|\"]\]", ".", string) - parts = re.split(r"\.(?=[^\]]*(?:\[|$))", string) - for part in parts: - match = re.findall(r"\[(\S+?)\]", part) - if match: - key = part[: part.find("[")] - result = result[key] - for m in match: - try: - m = int(m) - except ValueError: - m = str(m) - result = result[m] - else: - result = result.get(part) - return result - - def number(self, value): - if "." in str(value): - return float(value) - else: - return int(value) - - def eq(self, value): - return value == self.value - - def neq(self, value): - return value != self.value - - def gt(self, value): - return self.number(value) > self.value - - def ge(self, value): - return self.number(value) >= self.value - - def lt(self, value): - return self.number(value) < self.value - - def le(self, value): - return self.number(value) <= self.value - - def contains(self, value): - return str(self.value) in value - - def matches(self, value): - match = re.search(self.value, value, re.M) - return match is not None diff --git a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/utils.py b/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/utils.py deleted file mode 100644 index 0b594ad8c72..00000000000 --- a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/module_utils/network/common/utils.py +++ /dev/null @@ -1,673 +0,0 @@ -# This code is part of Ansible, but is an independent component. -# This particular file snippet, and this file snippet only, is BSD licensed. -# Modules you write using this snippet, which is embedded dynamically by Ansible -# still belong to the author of the module, and may assign their own license -# to the complete work. -# -# (c) 2016 Red Hat Inc. -# -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# - -# Networking tools for network modules only -from __future__ import annotations - -import re -import ast -import operator -import socket -import json - -from itertools import chain - -from ansible.module_utils.common.text.converters import to_text, to_bytes -from ansible.module_utils.six.moves.collections_abc import Mapping -from ansible.module_utils.six import iteritems, string_types -from ansible.module_utils import basic -from ansible.module_utils.parsing.convert_bool import boolean - -try: - from jinja2 import Environment, StrictUndefined - from jinja2.exceptions import UndefinedError - - HAS_JINJA2 = True -except ImportError: - HAS_JINJA2 = False - - -OPERATORS = frozenset(["ge", "gt", "eq", "neq", "lt", "le"]) -ALIASES = frozenset( - [("min", "ge"), ("max", "le"), ("exactly", "eq"), ("neq", "ne")] -) - - -def to_list(val): - if isinstance(val, (list, tuple, set)): - return list(val) - elif val is not None: - return [val] - else: - return list() - - -def to_lines(stdout): - for item in stdout: - if isinstance(item, string_types): - item = to_text(item).split("\n") - yield item - - -def transform_commands(module): - transform = ComplexList( - dict( - command=dict(key=True), - output=dict(), - prompt=dict(type="list"), - answer=dict(type="list"), - newline=dict(type="bool", default=True), - sendonly=dict(type="bool", default=False), - check_all=dict(type="bool", default=False), - ), - module, - ) - - return transform(module.params["commands"]) - - -def sort_list(val): - if isinstance(val, list): - return sorted(val) - return val - - -class Entity(object): - """Transforms a dict to with an argument spec - - This class will take a dict and apply an Ansible argument spec to the - values. The resulting dict will contain all of the keys in the param - with appropriate values set. - - Example:: - - argument_spec = dict( - command=dict(key=True), - display=dict(default='text', choices=['text', 'json']), - validate=dict(type='bool') - ) - transform = Entity(module, argument_spec) - value = dict(command='foo') - result = transform(value) - print result - {'command': 'foo', 'display': 'text', 'validate': None} - - Supported argument spec: - * key - specifies how to map a single value to a dict - * read_from - read and apply the argument_spec from the module - * required - a value is required - * type - type of value (uses AnsibleModule type checker) - * fallback - implements fallback function - * choices - set of valid options - * default - default value - """ - - def __init__( - self, module, attrs=None, args=None, keys=None, from_argspec=False - ): - args = [] if args is None else args - - self._attributes = attrs or {} - self._module = module - - for arg in args: - self._attributes[arg] = dict() - if from_argspec: - self._attributes[arg]["read_from"] = arg - if keys and arg in keys: - self._attributes[arg]["key"] = True - - self.attr_names = frozenset(self._attributes.keys()) - - _has_key = False - - for name, attr in iteritems(self._attributes): - if attr.get("read_from"): - if attr["read_from"] not in self._module.argument_spec: - module.fail_json( - msg="argument %s does not exist" % attr["read_from"] - ) - spec = self._module.argument_spec.get(attr["read_from"]) - for key, value in iteritems(spec): - if key not in attr: - attr[key] = value - - if attr.get("key"): - if _has_key: - module.fail_json(msg="only one key value can be specified") - _has_key = True - attr["required"] = True - - def serialize(self): - return self._attributes - - def to_dict(self, value): - obj = {} - for name, attr in iteritems(self._attributes): - if attr.get("key"): - obj[name] = value - else: - obj[name] = attr.get("default") - return obj - - def __call__(self, value, strict=True): - if not isinstance(value, dict): - value = self.to_dict(value) - - if strict: - unknown = set(value).difference(self.attr_names) - if unknown: - self._module.fail_json( - msg="invalid keys: %s" % ",".join(unknown) - ) - - for name, attr in iteritems(self._attributes): - if value.get(name) is None: - value[name] = attr.get("default") - - if attr.get("fallback") and not value.get(name): - fallback = attr.get("fallback", (None,)) - fallback_strategy = fallback[0] - fallback_args = [] - fallback_kwargs = {} - if fallback_strategy is not None: - for item in fallback[1:]: - if isinstance(item, dict): - fallback_kwargs = item - else: - fallback_args = item - try: - value[name] = fallback_strategy( - *fallback_args, **fallback_kwargs - ) - except basic.AnsibleFallbackNotFound: - continue - - if attr.get("required") and value.get(name) is None: - self._module.fail_json( - msg="missing required attribute %s" % name - ) - - if "choices" in attr: - if value[name] not in attr["choices"]: - self._module.fail_json( - msg="%s must be one of %s, got %s" - % (name, ", ".join(attr["choices"]), value[name]) - ) - - if value[name] is not None: - value_type = attr.get("type", "str") - type_checker = self._module._CHECK_ARGUMENT_TYPES_DISPATCHER[ - value_type - ] - type_checker(value[name]) - elif value.get(name): - value[name] = self._module.params[name] - - return value - - -class EntityCollection(Entity): - """Extends ```Entity``` to handle a list of dicts """ - - def __call__(self, iterable, strict=True): - if iterable is None: - iterable = [ - super(EntityCollection, self).__call__( - self._module.params, strict - ) - ] - - if not isinstance(iterable, (list, tuple)): - self._module.fail_json(msg="value must be an iterable") - - return [ - (super(EntityCollection, self).__call__(i, strict)) - for i in iterable - ] - - -# these two are for backwards compatibility and can be removed once all of the -# modules that use them are updated -class ComplexDict(Entity): - def __init__(self, attrs, module, *args, **kwargs): - super(ComplexDict, self).__init__(module, attrs, *args, **kwargs) - - -class ComplexList(EntityCollection): - def __init__(self, attrs, module, *args, **kwargs): - super(ComplexList, self).__init__(module, attrs, *args, **kwargs) - - -def dict_diff(base, comparable): - """ Generate a dict object of differences - - This function will compare two dict objects and return the difference - between them as a dict object. For scalar values, the key will reflect - the updated value. If the key does not exist in `comparable`, then then no - key will be returned. For lists, the value in comparable will wholly replace - the value in base for the key. For dicts, the returned value will only - return keys that are different. - - :param base: dict object to base the diff on - :param comparable: dict object to compare against base - - :returns: new dict object with differences - """ - if not isinstance(base, dict): - raise AssertionError("`base` must be of type ") - if not isinstance(comparable, dict): - if comparable is None: - comparable = dict() - else: - raise AssertionError("`comparable` must be of type ") - - updates = dict() - - for key, value in iteritems(base): - if isinstance(value, dict): - item = comparable.get(key) - if item is not None: - sub_diff = dict_diff(value, comparable[key]) - if sub_diff: - updates[key] = sub_diff - else: - comparable_value = comparable.get(key) - if comparable_value is not None: - if sort_list(base[key]) != sort_list(comparable_value): - updates[key] = comparable_value - - for key in set(comparable.keys()).difference(base.keys()): - updates[key] = comparable.get(key) - - return updates - - -def dict_merge(base, other): - """ Return a new dict object that combines base and other - - This will create a new dict object that is a combination of the key/value - pairs from base and other. When both keys exist, the value will be - selected from other. If the value is a list object, the two lists will - be combined and duplicate entries removed. - - :param base: dict object to serve as base - :param other: dict object to combine with base - - :returns: new combined dict object - """ - if not isinstance(base, dict): - raise AssertionError("`base` must be of type ") - if not isinstance(other, dict): - raise AssertionError("`other` must be of type ") - - combined = dict() - - for key, value in iteritems(base): - if isinstance(value, dict): - if key in other: - item = other.get(key) - if item is not None: - if isinstance(other[key], Mapping): - combined[key] = dict_merge(value, other[key]) - else: - combined[key] = other[key] - else: - combined[key] = item - else: - combined[key] = value - elif isinstance(value, list): - if key in other: - item = other.get(key) - if item is not None: - try: - combined[key] = list(set(chain(value, item))) - except TypeError: - value.extend([i for i in item if i not in value]) - combined[key] = value - else: - combined[key] = item - else: - combined[key] = value - else: - if key in other: - other_value = other.get(key) - if other_value is not None: - if sort_list(base[key]) != sort_list(other_value): - combined[key] = other_value - else: - combined[key] = value - else: - combined[key] = other_value - else: - combined[key] = value - - for key in set(other.keys()).difference(base.keys()): - combined[key] = other.get(key) - - return combined - - -def param_list_to_dict(param_list, unique_key="name", remove_key=True): - """Rotates a list of dictionaries to be a dictionary of dictionaries. - - :param param_list: The aforementioned list of dictionaries - :param unique_key: The name of a key which is present and unique in all of param_list's dictionaries. The value - behind this key will be the key each dictionary can be found at in the new root dictionary - :param remove_key: If True, remove unique_key from the individual dictionaries before returning. - """ - param_dict = {} - for params in param_list: - params = params.copy() - if remove_key: - name = params.pop(unique_key) - else: - name = params.get(unique_key) - param_dict[name] = params - - return param_dict - - -def conditional(expr, val, cast=None): - match = re.match(r"^(.+)\((.+)\)$", str(expr), re.I) - if match: - op, arg = match.groups() - else: - op = "eq" - if " " in str(expr): - raise AssertionError("invalid expression: cannot contain spaces") - arg = expr - - if cast is None and val is not None: - arg = type(val)(arg) - elif callable(cast): - arg = cast(arg) - val = cast(val) - - op = next((oper for alias, oper in ALIASES if op == alias), op) - - if not hasattr(operator, op) and op not in OPERATORS: - raise ValueError("unknown operator: %s" % op) - - func = getattr(operator, op) - return func(val, arg) - - -def ternary(value, true_val, false_val): - """ value ? true_val : false_val """ - if value: - return true_val - else: - return false_val - - -def remove_default_spec(spec): - for item in spec: - if "default" in spec[item]: - del spec[item]["default"] - - -def validate_ip_address(address): - try: - socket.inet_aton(address) - except socket.error: - return False - return address.count(".") == 3 - - -def validate_ip_v6_address(address): - try: - socket.inet_pton(socket.AF_INET6, address) - except socket.error: - return False - return True - - -def validate_prefix(prefix): - if prefix and not 0 <= int(prefix) <= 32: - return False - return True - - -def load_provider(spec, args): - provider = args.get("provider") or {} - for key, value in iteritems(spec): - if key not in provider: - if "fallback" in value: - provider[key] = _fallback(value["fallback"]) - elif "default" in value: - provider[key] = value["default"] - else: - provider[key] = None - if "authorize" in provider: - # Coerce authorize to provider if a string has somehow snuck in. - provider["authorize"] = boolean(provider["authorize"] or False) - args["provider"] = provider - return provider - - -def _fallback(fallback): - strategy = fallback[0] - args = [] - kwargs = {} - - for item in fallback[1:]: - if isinstance(item, dict): - kwargs = item - else: - args = item - try: - return strategy(*args, **kwargs) - except basic.AnsibleFallbackNotFound: - pass - - -def generate_dict(spec): - """ - Generate dictionary which is in sync with argspec - - :param spec: A dictionary that is the argspec of the module - :rtype: A dictionary - :returns: A dictionary in sync with argspec with default value - """ - obj = {} - if not spec: - return obj - - for key, val in iteritems(spec): - if "default" in val: - dct = {key: val["default"]} - elif "type" in val and val["type"] == "dict": - dct = {key: generate_dict(val["options"])} - else: - dct = {key: None} - obj.update(dct) - return obj - - -def parse_conf_arg(cfg, arg): - """ - Parse config based on argument - - :param cfg: A text string which is a line of configuration. - :param arg: A text string which is to be matched. - :rtype: A text string - :returns: A text string if match is found - """ - match = re.search(r"%s (.+)(\n|$)" % arg, cfg, re.M) - if match: - result = match.group(1).strip() - else: - result = None - return result - - -def parse_conf_cmd_arg(cfg, cmd, res1, res2=None, delete_str="no"): - """ - Parse config based on command - - :param cfg: A text string which is a line of configuration. - :param cmd: A text string which is the command to be matched - :param res1: A text string to be returned if the command is present - :param res2: A text string to be returned if the negate command - is present - :param delete_str: A text string to identify the start of the - negate command - :rtype: A text string - :returns: A text string if match is found - """ - match = re.search(r"\n\s+%s(\n|$)" % cmd, cfg) - if match: - return res1 - if res2 is not None: - match = re.search(r"\n\s+%s %s(\n|$)" % (delete_str, cmd), cfg) - if match: - return res2 - return None - - -def get_xml_conf_arg(cfg, path, data="text"): - """ - :param cfg: The top level configuration lxml Element tree object - :param path: The relative xpath w.r.t to top level element (cfg) - to be searched in the xml hierarchy - :param data: The type of data to be returned for the matched xml node. - Valid values are text, tag, attrib, with default as text. - :return: Returns the required type for the matched xml node or else None - """ - match = cfg.xpath(path) - if len(match): - if data == "tag": - result = getattr(match[0], "tag") - elif data == "attrib": - result = getattr(match[0], "attrib") - else: - result = getattr(match[0], "text") - else: - result = None - return result - - -def remove_empties(cfg_dict): - """ - Generate final config dictionary - - :param cfg_dict: A dictionary parsed in the facts system - :rtype: A dictionary - :returns: A dictionary by eliminating keys that have null values - """ - final_cfg = {} - if not cfg_dict: - return final_cfg - - for key, val in iteritems(cfg_dict): - dct = None - if isinstance(val, dict): - child_val = remove_empties(val) - if child_val: - dct = {key: child_val} - elif ( - isinstance(val, list) - and val - and all(isinstance(x, dict) for x in val) - ): - child_val = [remove_empties(x) for x in val] - if child_val: - dct = {key: child_val} - elif val not in [None, [], {}, (), ""]: - dct = {key: val} - if dct: - final_cfg.update(dct) - return final_cfg - - -def validate_config(spec, data): - """ - Validate if the input data against the AnsibleModule spec format - :param spec: Ansible argument spec - :param data: Data to be validated - :return: - """ - params = basic._ANSIBLE_ARGS - basic._ANSIBLE_ARGS = to_bytes(json.dumps({"ANSIBLE_MODULE_ARGS": data})) - validated_data = basic.AnsibleModule(spec).params - basic._ANSIBLE_ARGS = params - return validated_data - - -def search_obj_in_list(name, lst, key="name"): - if not lst: - return None - else: - for item in lst: - if item.get(key) == name: - return item - - -class Template: - def __init__(self): - if not HAS_JINJA2: - raise ImportError( - "jinja2 is required but does not appear to be installed. " - "It can be installed using `pip install jinja2`" - ) - - self.env = Environment(undefined=StrictUndefined) - self.env.filters.update({"ternary": ternary}) - - def __call__(self, value, variables=None, fail_on_undefined=True): - variables = variables or {} - - if not self.contains_vars(value): - return value - - try: - value = self.env.from_string(value).render(variables) - except UndefinedError: - if not fail_on_undefined: - return None - raise - - if value: - try: - return ast.literal_eval(value) - except Exception: - return str(value) - else: - return None - - def contains_vars(self, data): - if isinstance(data, string_types): - for marker in ( - self.env.block_start_string, - self.env.variable_start_string, - self.env.comment_start_string, - ): - if marker in data: - return True - return False diff --git a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/modules/cli_config.py b/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/modules/cli_config.py deleted file mode 100644 index ef6f102c02d..00000000000 --- a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/modules/cli_config.py +++ /dev/null @@ -1,442 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -# (c) 2018, Ansible by Red Hat, inc -# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) - -from __future__ import annotations - - -ANSIBLE_METADATA = { - "metadata_version": "1.1", - "status": ["preview"], - "supported_by": "network", -} - - -DOCUMENTATION = """module: cli_config -author: Trishna Guha (@trishnaguha) -notes: -- The commands will be returned only for platforms that do not support onbox diff. - The C(--diff) option with the playbook will return the difference in configuration - for devices that has support for onbox diff -short_description: Push text based configuration to network devices over network_cli -description: -- This module provides platform agnostic way of pushing text based configuration to - network devices over network_cli connection plugin. -extends_documentation_fragment: -- ansible.netcommon.network_agnostic -options: - config: - description: - - The config to be pushed to the network device. This argument is mutually exclusive - with C(rollback) and either one of the option should be given as input. The - config should have indentation that the device uses. - type: str - commit: - description: - - The C(commit) argument instructs the module to push the configuration to the - device. This is mapped to module check mode. - type: bool - replace: - description: - - If the C(replace) argument is set to C(yes), it will replace the entire running-config - of the device with the C(config) argument value. For devices that support replacing - running configuration from file on device like NXOS/JUNOS, the C(replace) argument - takes path to the file on the device that will be used for replacing the entire - running-config. The value of C(config) option should be I(None) for such devices. - Nexus 9K devices only support replace. Use I(net_put) or I(nxos_file_copy) in - case of NXOS module to copy the flat file to remote device and then use set - the fullpath to this argument. - type: str - backup: - description: - - This argument will cause the module to create a full backup of the current running - config from the remote device before any changes are made. If the C(backup_options) - value is not given, the backup file is written to the C(backup) folder in the - playbook root directory or role root directory, if playbook is part of an ansible - role. If the directory does not exist, it is created. - type: bool - default: 'no' - rollback: - description: - - The C(rollback) argument instructs the module to rollback the current configuration - to the identifier specified in the argument. If the specified rollback identifier - does not exist on the remote device, the module will fail. To rollback to the - most recent commit, set the C(rollback) argument to 0. This option is mutually - exclusive with C(config). - commit_comment: - description: - - The C(commit_comment) argument specifies a text string to be used when committing - the configuration. If the C(commit) argument is set to False, this argument - is silently ignored. This argument is only valid for the platforms that support - commit operation with comment. - type: str - defaults: - description: - - The I(defaults) argument will influence how the running-config is collected - from the device. When the value is set to true, the command used to collect - the running-config is append with the all keyword. When the value is set to - false, the command is issued without the all keyword. - default: 'no' - type: bool - multiline_delimiter: - description: - - This argument is used when pushing a multiline configuration element to the - device. It specifies the character to use as the delimiting character. This - only applies to the configuration action. - type: str - diff_replace: - description: - - Instructs the module on the way to perform the configuration on the device. - If the C(diff_replace) argument is set to I(line) then the modified lines are - pushed to the device in configuration mode. If the argument is set to I(block) - then the entire command block is pushed to the device in configuration mode - if any line is not correct. Note that this parameter will be ignored if the - platform has onbox diff support. - choices: - - line - - block - - config - diff_match: - description: - - Instructs the module on the way to perform the matching of the set of commands - against the current device config. If C(diff_match) is set to I(line), commands - are matched line by line. If C(diff_match) is set to I(strict), command lines - are matched with respect to position. If C(diff_match) is set to I(exact), command - lines must be an equal match. Finally, if C(diff_match) is set to I(none), the - module will not attempt to compare the source configuration with the running - configuration on the remote device. Note that this parameter will be ignored - if the platform has onbox diff support. - choices: - - line - - strict - - exact - - none - diff_ignore_lines: - description: - - Use this argument to specify one or more lines that should be ignored during - the diff. This is used for lines in the configuration that are automatically - updated by the system. This argument takes a list of regular expressions or - exact line matches. Note that this parameter will be ignored if the platform - has onbox diff support. - backup_options: - description: - - This is a dict object containing configurable options related to backup file - path. The value of this option is read only when C(backup) is set to I(yes), - if C(backup) is set to I(no) this option will be silently ignored. - suboptions: - filename: - description: - - The filename to be used to store the backup configuration. If the filename - is not given it will be generated based on the hostname, current time and - date in format defined by _config.@ - dir_path: - description: - - This option provides the path ending with directory name in which the backup - configuration file will be stored. If the directory does not exist it will - be first created and the filename is either the value of C(filename) or - default filename as described in C(filename) options description. If the - path value is not given in that case a I(backup) directory will be created - in the current working directory and backup configuration will be copied - in C(filename) within I(backup) directory. - type: path - type: dict -""" - -EXAMPLES = """ -- name: configure device with config - cli_config: - config: "{{ lookup('template', 'basic/config.j2') }}" - -- name: multiline config - cli_config: - config: | - hostname foo - feature nxapi - -- name: configure device with config with defaults enabled - cli_config: - config: "{{ lookup('template', 'basic/config.j2') }}" - defaults: yes - -- name: Use diff_match - cli_config: - config: "{{ lookup('file', 'interface_config') }}" - diff_match: none - -- name: nxos replace config - cli_config: - replace: 'bootflash:nxoscfg' - -- name: junos replace config - cli_config: - replace: '/var/home/ansible/junos01.cfg' - -- name: commit with comment - cli_config: - config: set system host-name foo - commit_comment: this is a test - -- name: configurable backup path - cli_config: - config: "{{ lookup('template', 'basic/config.j2') }}" - backup: yes - backup_options: - filename: backup.cfg - dir_path: /home/user -""" - -RETURN = """ -commands: - description: The set of commands that will be pushed to the remote device - returned: always - type: list - sample: ['interface Loopback999', 'no shutdown'] -backup_path: - description: The full path to the backup file - returned: when backup is yes - type: str - sample: /playbooks/ansible/backup/hostname_config.2016-07-16@22:28:34 -""" - -import json - -from ansible.module_utils.basic import AnsibleModule -from ansible.module_utils.connection import Connection -from ansible.module_utils.common.text.converters import to_text - - -def validate_args(module, device_operations): - """validate param if it is supported on the platform - """ - feature_list = [ - "replace", - "rollback", - "commit_comment", - "defaults", - "multiline_delimiter", - "diff_replace", - "diff_match", - "diff_ignore_lines", - ] - - for feature in feature_list: - if module.params[feature]: - supports_feature = device_operations.get("supports_%s" % feature) - if supports_feature is None: - module.fail_json( - "This platform does not specify whether %s is supported or not. " - "Please report an issue against this platform's cliconf plugin." - % feature - ) - elif not supports_feature: - module.fail_json( - msg="Option %s is not supported on this platform" % feature - ) - - -def run( - module, device_operations, connection, candidate, running, rollback_id -): - result = {} - resp = {} - config_diff = [] - banner_diff = {} - - replace = module.params["replace"] - commit_comment = module.params["commit_comment"] - multiline_delimiter = module.params["multiline_delimiter"] - diff_replace = module.params["diff_replace"] - diff_match = module.params["diff_match"] - diff_ignore_lines = module.params["diff_ignore_lines"] - - commit = not module.check_mode - - if replace in ("yes", "true", "True"): - replace = True - elif replace in ("no", "false", "False"): - replace = False - - if ( - replace is not None - and replace not in [True, False] - and candidate is not None - ): - module.fail_json( - msg="Replace value '%s' is a configuration file path already" - " present on the device. Hence 'replace' and 'config' options" - " are mutually exclusive" % replace - ) - - if rollback_id is not None: - resp = connection.rollback(rollback_id, commit) - if "diff" in resp: - result["changed"] = True - - elif device_operations.get("supports_onbox_diff"): - if diff_replace: - module.warn( - "diff_replace is ignored as the device supports onbox diff" - ) - if diff_match: - module.warn( - "diff_mattch is ignored as the device supports onbox diff" - ) - if diff_ignore_lines: - module.warn( - "diff_ignore_lines is ignored as the device supports onbox diff" - ) - - if candidate and not isinstance(candidate, list): - candidate = candidate.strip("\n").splitlines() - - kwargs = { - "candidate": candidate, - "commit": commit, - "replace": replace, - "comment": commit_comment, - } - resp = connection.edit_config(**kwargs) - - if "diff" in resp: - result["changed"] = True - - elif device_operations.get("supports_generate_diff"): - kwargs = {"candidate": candidate, "running": running} - if diff_match: - kwargs.update({"diff_match": diff_match}) - if diff_replace: - kwargs.update({"diff_replace": diff_replace}) - if diff_ignore_lines: - kwargs.update({"diff_ignore_lines": diff_ignore_lines}) - - diff_response = connection.get_diff(**kwargs) - - config_diff = diff_response.get("config_diff") - banner_diff = diff_response.get("banner_diff") - - if config_diff: - if isinstance(config_diff, list): - candidate = config_diff - else: - candidate = config_diff.splitlines() - - kwargs = { - "candidate": candidate, - "commit": commit, - "replace": replace, - "comment": commit_comment, - } - if commit: - connection.edit_config(**kwargs) - result["changed"] = True - result["commands"] = config_diff.split("\n") - - if banner_diff: - candidate = json.dumps(banner_diff) - - kwargs = {"candidate": candidate, "commit": commit} - if multiline_delimiter: - kwargs.update({"multiline_delimiter": multiline_delimiter}) - if commit: - connection.edit_banner(**kwargs) - result["changed"] = True - - if module._diff: - if "diff" in resp: - result["diff"] = {"prepared": resp["diff"]} - else: - diff = "" - if config_diff: - if isinstance(config_diff, list): - diff += "\n".join(config_diff) - else: - diff += config_diff - if banner_diff: - diff += json.dumps(banner_diff) - result["diff"] = {"prepared": diff} - - return result - - -def main(): - """main entry point for execution - """ - backup_spec = dict(filename=dict(), dir_path=dict(type="path")) - argument_spec = dict( - backup=dict(default=False, type="bool"), - backup_options=dict(type="dict", options=backup_spec), - config=dict(type="str"), - commit=dict(type="bool"), - replace=dict(type="str"), - rollback=dict(type="int"), - commit_comment=dict(type="str"), - defaults=dict(default=False, type="bool"), - multiline_delimiter=dict(type="str"), - diff_replace=dict(choices=["line", "block", "config"]), - diff_match=dict(choices=["line", "strict", "exact", "none"]), - diff_ignore_lines=dict(type="list"), - ) - - mutually_exclusive = [("config", "rollback")] - required_one_of = [["backup", "config", "rollback"]] - - module = AnsibleModule( - argument_spec=argument_spec, - mutually_exclusive=mutually_exclusive, - required_one_of=required_one_of, - supports_check_mode=True, - ) - - result = {"changed": False} - - connection = Connection(module._socket_path) - capabilities = module.from_json(connection.get_capabilities()) - - if capabilities: - device_operations = capabilities.get("device_operations", dict()) - validate_args(module, device_operations) - else: - device_operations = dict() - - if module.params["defaults"]: - if "get_default_flag" in capabilities.get("rpc"): - flags = connection.get_default_flag() - else: - flags = "all" - else: - flags = [] - - candidate = module.params["config"] - candidate = ( - to_text(candidate, errors="surrogate_then_replace") - if candidate - else None - ) - running = connection.get_config(flags=flags) - rollback_id = module.params["rollback"] - - if module.params["backup"]: - result["__backup__"] = running - - if candidate or rollback_id or module.params["replace"]: - try: - result.update( - run( - module, - device_operations, - connection, - candidate, - running, - rollback_id, - ) - ) - except Exception as exc: - module.fail_json(msg=to_text(exc)) - - module.exit_json(**result) - - -if __name__ == "__main__": - main() diff --git a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/plugin_utils/connection_base.py b/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/plugin_utils/connection_base.py deleted file mode 100644 index 29faa8b6b23..00000000000 --- a/test/support/network-integration/collections/ansible_collections/ansible/netcommon/plugins/plugin_utils/connection_base.py +++ /dev/null @@ -1,184 +0,0 @@ -# (c) 2012-2014, Michael DeHaan -# (c) 2015 Toshio Kuratomi -# (c) 2017, Peter Sprygada -# (c) 2017 Ansible Project -from __future__ import annotations - - -import os - -from ansible import constants as C -from ansible.plugins.connection import ConnectionBase -from ansible.plugins.loader import connection_loader -from ansible.utils.display import Display -from ansible.utils.path import unfrackpath - -display = Display() - - -__all__ = ["NetworkConnectionBase"] - -BUFSIZE = 65536 - - -class NetworkConnectionBase(ConnectionBase): - """ - A base class for network-style connections. - """ - - force_persistence = True - # Do not use _remote_is_local in other connections - _remote_is_local = True - - def __init__(self, play_context, new_stdin, *args, **kwargs): - super(NetworkConnectionBase, self).__init__( - play_context, new_stdin, *args, **kwargs - ) - self._messages = [] - self._conn_closed = False - - self._network_os = self._play_context.network_os - - self._local = connection_loader.get("local", play_context, "/dev/null") - self._local.set_options() - - self._sub_plugin = {} - self._cached_variables = (None, None, None) - - # reconstruct the socket_path and set instance values accordingly - self._ansible_playbook_pid = kwargs.get("ansible_playbook_pid") - self._update_connection_state() - - def __getattr__(self, name): - try: - return self.__dict__[name] - except KeyError: - if not name.startswith("_"): - plugin = self._sub_plugin.get("obj") - if plugin: - method = getattr(plugin, name, None) - if method is not None: - return method - raise AttributeError( - "'%s' object has no attribute '%s'" - % (self.__class__.__name__, name) - ) - - def exec_command(self, cmd, in_data=None, sudoable=True): - return self._local.exec_command(cmd, in_data, sudoable) - - def queue_message(self, level, message): - """ - Adds a message to the queue of messages waiting to be pushed back to the controller process. - - :arg level: A string which can either be the name of a method in display, or 'log'. When - the messages are returned to task_executor, a value of log will correspond to - ``display.display(message, log_only=True)``, while another value will call ``display.[level](message)`` - """ - self._messages.append((level, message)) - - def pop_messages(self): - messages, self._messages = self._messages, [] - return messages - - def put_file(self, in_path, out_path): - """Transfer a file from local to remote""" - return self._local.put_file(in_path, out_path) - - def fetch_file(self, in_path, out_path): - """Fetch a file from remote to local""" - return self._local.fetch_file(in_path, out_path) - - def reset(self): - """ - Reset the connection - """ - if self._socket_path: - self.queue_message( - "vvvv", - "resetting persistent connection for socket_path %s" - % self._socket_path, - ) - self.close() - self.queue_message("vvvv", "reset call on connection instance") - - def close(self): - self._conn_closed = True - if self._connected: - self._connected = False - - def get_options(self, hostvars=None): - options = super(NetworkConnectionBase, self).get_options( - hostvars=hostvars - ) - - if ( - self._sub_plugin.get("obj") - and self._sub_plugin.get("type") != "external" - ): - try: - options.update( - self._sub_plugin["obj"].get_options(hostvars=hostvars) - ) - except AttributeError: - pass - - return options - - def set_options(self, task_keys=None, var_options=None, direct=None): - super(NetworkConnectionBase, self).set_options( - task_keys=task_keys, var_options=var_options, direct=direct - ) - if self.get_option("persistent_log_messages"): - warning = ( - "Persistent connection logging is enabled for %s. This will log ALL interactions" - % self._play_context.remote_addr - ) - logpath = getattr(C, "DEFAULT_LOG_PATH") - if logpath is not None: - warning += " to %s" % logpath - self.queue_message( - "warning", - "%s and WILL NOT redact sensitive configuration like passwords. USE WITH CAUTION!" - % warning, - ) - - if ( - self._sub_plugin.get("obj") - and self._sub_plugin.get("type") != "external" - ): - try: - self._sub_plugin["obj"].set_options( - task_keys=task_keys, var_options=var_options, direct=direct - ) - except AttributeError: - pass - - def _update_connection_state(self): - """ - Reconstruct the connection socket_path and check if it exists - - If the socket path exists then the connection is active and set - both the _socket_path value to the path and the _connected value - to True. If the socket path doesn't exist, leave the socket path - value to None and the _connected value to False - """ - ssh = connection_loader.get("ssh", class_only=True) - control_path = ssh._create_control_path( - self._play_context.remote_addr, - self._play_context.port, - self._play_context.remote_user, - self._play_context.connection, - self._ansible_playbook_pid, - ) - - tmp_path = unfrackpath(C.PERSISTENT_CONTROL_PATH_DIR) - socket_path = unfrackpath(control_path % dict(directory=tmp_path)) - - if os.path.exists(socket_path): - self._connected = True - self._socket_path = socket_path - - def _log_messages(self, message): - if self.get_option("persistent_log_messages"): - self.queue_message("log", message) diff --git a/test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/action/ios.py b/test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/action/ios.py deleted file mode 100644 index be708957b79..00000000000 --- a/test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/action/ios.py +++ /dev/null @@ -1,132 +0,0 @@ -# -# (c) 2016 Red Hat Inc. -# -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . -# -from __future__ import annotations - - -import sys -import copy - -from ansible_collections.ansible.netcommon.plugins.action.network import ( - ActionModule as ActionNetworkModule, -) -from ansible_collections.ansible.netcommon.plugins.module_utils.network.common.utils import ( - load_provider, -) -from ansible_collections.cisco.ios.plugins.module_utils.network.ios.ios import ( - ios_provider_spec, -) -from ansible.utils.display import Display - -display = Display() - - -class ActionModule(ActionNetworkModule): - def run(self, tmp=None, task_vars=None): - del tmp # tmp no longer has any effect - - module_name = self._task.action.split(".")[-1] - self._config_module = True if module_name == "ios_config" else False - persistent_connection = self._play_context.connection.split(".")[-1] - warnings = [] - - if persistent_connection == "network_cli": - provider = self._task.args.get("provider", {}) - if any(provider.values()): - display.warning( - "provider is unnecessary when using network_cli and will be ignored" - ) - del self._task.args["provider"] - elif self._play_context.connection == "local": - provider = load_provider(ios_provider_spec, self._task.args) - pc = copy.deepcopy(self._play_context) - pc.connection = "ansible.netcommon.network_cli" - pc.network_os = "cisco.ios.ios" - pc.remote_addr = provider["host"] or self._play_context.remote_addr - pc.port = int(provider["port"] or self._play_context.port or 22) - pc.remote_user = ( - provider["username"] or self._play_context.connection_user - ) - pc.password = provider["password"] or self._play_context.password - pc.private_key_file = ( - provider["ssh_keyfile"] or self._play_context.private_key_file - ) - pc.become = provider["authorize"] or False - if pc.become: - pc.become_method = "enable" - pc.become_pass = provider["auth_pass"] - - connection = self._shared_loader_obj.connection_loader.get( - "ansible.netcommon.persistent", - pc, - sys.stdin, - task_uuid=self._task._uuid, - ) - - # TODO: Remove below code after ansible minimal is cut out - if connection is None: - pc.connection = "network_cli" - pc.network_os = "ios" - connection = self._shared_loader_obj.connection_loader.get( - "persistent", pc, sys.stdin, task_uuid=self._task._uuid - ) - - display.vvv( - "using connection plugin %s (was local)" % pc.connection, - pc.remote_addr, - ) - - command_timeout = ( - int(provider["timeout"]) - if provider["timeout"] - else connection.get_option("persistent_command_timeout") - ) - connection.set_options( - direct={"persistent_command_timeout": command_timeout} - ) - - socket_path = connection.run() - display.vvvv("socket_path: %s" % socket_path, pc.remote_addr) - if not socket_path: - return { - "failed": True, - "msg": "unable to open shell. Please see: " - + "https://docs.ansible.com/ansible/latest/network/user_guide/network_debug_troubleshooting.html#category-unable-to-open-shell", - } - - task_vars["ansible_socket"] = socket_path - warnings.append( - [ - "connection local support for this module is deprecated and will be removed in version 2.14, use connection %s" - % pc.connection - ] - ) - else: - return { - "failed": True, - "msg": "Connection type %s is not valid for this module" - % self._play_context.connection, - } - - result = super(ActionModule, self).run(task_vars=task_vars) - if warnings: - if "warnings" in result: - result["warnings"].extend(warnings) - else: - result["warnings"] = warnings - return result diff --git a/test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/cliconf/ios.py b/test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/cliconf/ios.py deleted file mode 100644 index 1b5cb459430..00000000000 --- a/test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/cliconf/ios.py +++ /dev/null @@ -1,465 +0,0 @@ -# -# (c) 2017 Red Hat Inc. -# -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . -# -from __future__ import annotations - - -DOCUMENTATION = """ ---- -author: Ansible Networking Team -cliconf: ios -short_description: Use ios cliconf to run command on Cisco IOS platform -description: - - This ios plugin provides low level abstraction apis for - sending and receiving CLI commands from Cisco IOS network devices. -version_added: "2.4" -""" - -import re -import time -import json - -from collections.abc import Mapping - -from ansible.errors import AnsibleConnectionFailure -from ansible.module_utils.common.text.converters import to_text -from ansible.module_utils.six import iteritems -from ansible_collections.ansible.netcommon.plugins.module_utils.network.common.config import ( - NetworkConfig, - dumps, -) -from ansible_collections.ansible.netcommon.plugins.module_utils.network.common.utils import ( - to_list, -) -from ansible.plugins.cliconf import CliconfBase, enable_mode - - -class Cliconf(CliconfBase): - @enable_mode - def get_config(self, source="running", flags=None, format=None): - if source not in ("running", "startup"): - raise ValueError( - "fetching configuration from %s is not supported" % source - ) - - if format: - raise ValueError( - "'format' value %s is not supported for get_config" % format - ) - - if not flags: - flags = [] - if source == "running": - cmd = "show running-config " - else: - cmd = "show startup-config " - - cmd += " ".join(to_list(flags)) - cmd = cmd.strip() - - return self.send_command(cmd) - - def get_diff( - self, - candidate=None, - running=None, - diff_match="line", - diff_ignore_lines=None, - path=None, - diff_replace="line", - ): - """ - Generate diff between candidate and running configuration. If the - remote host supports onbox diff capabilities ie. supports_onbox_diff in that case - candidate and running configurations are not required to be passed as argument. - In case if onbox diff capability is not supported candidate argument is mandatory - and running argument is optional. - :param candidate: The configuration which is expected to be present on remote host. - :param running: The base configuration which is used to generate diff. - :param diff_match: Instructs how to match the candidate configuration with current device configuration - Valid values are 'line', 'strict', 'exact', 'none'. - 'line' - commands are matched line by line - 'strict' - command lines are matched with respect to position - 'exact' - command lines must be an equal match - 'none' - will not compare the candidate configuration with the running configuration - :param diff_ignore_lines: Use this argument to specify one or more lines that should be - ignored during the diff. This is used for lines in the configuration - that are automatically updated by the system. This argument takes - a list of regular expressions or exact line matches. - :param path: The ordered set of parents that uniquely identify the section or hierarchy - the commands should be checked against. If the parents argument - is omitted, the commands are checked against the set of top - level or global commands. - :param diff_replace: Instructs on the way to perform the configuration on the device. - If the replace argument is set to I(line) then the modified lines are - pushed to the device in configuration mode. If the replace argument is - set to I(block) then the entire command block is pushed to the device in - configuration mode if any line is not correct. - :return: Configuration diff in json format. - { - 'config_diff': '', - 'banner_diff': {} - } - - """ - diff = {} - device_operations = self.get_device_operations() - option_values = self.get_option_values() - - if candidate is None and device_operations["supports_generate_diff"]: - raise ValueError( - "candidate configuration is required to generate diff" - ) - - if diff_match not in option_values["diff_match"]: - raise ValueError( - "'match' value %s in invalid, valid values are %s" - % (diff_match, ", ".join(option_values["diff_match"])) - ) - - if diff_replace not in option_values["diff_replace"]: - raise ValueError( - "'replace' value %s in invalid, valid values are %s" - % (diff_replace, ", ".join(option_values["diff_replace"])) - ) - - # prepare candidate configuration - candidate_obj = NetworkConfig(indent=1) - want_src, want_banners = self._extract_banners(candidate) - candidate_obj.load(want_src) - - if running and diff_match != "none": - # running configuration - have_src, have_banners = self._extract_banners(running) - running_obj = NetworkConfig( - indent=1, contents=have_src, ignore_lines=diff_ignore_lines - ) - configdiffobjs = candidate_obj.difference( - running_obj, path=path, match=diff_match, replace=diff_replace - ) - - else: - configdiffobjs = candidate_obj.items - have_banners = {} - - diff["config_diff"] = ( - dumps(configdiffobjs, "commands") if configdiffobjs else "" - ) - banners = self._diff_banners(want_banners, have_banners) - diff["banner_diff"] = banners if banners else {} - return diff - - @enable_mode - def edit_config( - self, candidate=None, commit=True, replace=None, comment=None - ): - resp = {} - operations = self.get_device_operations() - self.check_edit_config_capability( - operations, candidate, commit, replace, comment - ) - - results = [] - requests = [] - if commit: - self.send_command("configure terminal") - for line in to_list(candidate): - if not isinstance(line, Mapping): - line = {"command": line} - - cmd = line["command"] - if cmd != "end" and cmd[0] != "!": - results.append(self.send_command(**line)) - requests.append(cmd) - - self.send_command("end") - else: - raise ValueError("check mode is not supported") - - resp["request"] = requests - resp["response"] = results - return resp - - def edit_macro( - self, candidate=None, commit=True, replace=None, comment=None - ): - """ - ios_config: - lines: "{{ macro_lines }}" - parents: "macro name {{ macro_name }}" - after: '@' - match: line - replace: block - """ - resp = {} - operations = self.get_device_operations() - self.check_edit_config_capability( - operations, candidate, commit, replace, comment - ) - - results = [] - requests = [] - if commit: - commands = "" - self.send_command("config terminal") - time.sleep(0.1) - # first item: macro command - commands += candidate.pop(0) + "\n" - multiline_delimiter = candidate.pop(-1) - for line in candidate: - commands += " " + line + "\n" - commands += multiline_delimiter + "\n" - obj = {"command": commands, "sendonly": True} - results.append(self.send_command(**obj)) - requests.append(commands) - - time.sleep(0.1) - self.send_command("end", sendonly=True) - time.sleep(0.1) - results.append(self.send_command("\n")) - requests.append("\n") - - resp["request"] = requests - resp["response"] = results - return resp - - def get( - self, - command=None, - prompt=None, - answer=None, - sendonly=False, - output=None, - newline=True, - check_all=False, - ): - if not command: - raise ValueError("must provide value of command to execute") - if output: - raise ValueError( - "'output' value %s is not supported for get" % output - ) - - return self.send_command( - command=command, - prompt=prompt, - answer=answer, - sendonly=sendonly, - newline=newline, - check_all=check_all, - ) - - def get_device_info(self): - device_info = {} - - device_info["network_os"] = "ios" - reply = self.get(command="show version") - data = to_text(reply, errors="surrogate_or_strict").strip() - - match = re.search(r"Version (\S+)", data) - if match: - device_info["network_os_version"] = match.group(1).strip(",") - - model_search_strs = [ - r"^[Cc]isco (.+) \(revision", - r"^[Cc]isco (\S+).+bytes of .*memory", - ] - for item in model_search_strs: - match = re.search(item, data, re.M) - if match: - version = match.group(1).split(" ") - device_info["network_os_model"] = version[0] - break - - match = re.search(r"^(.+) uptime", data, re.M) - if match: - device_info["network_os_hostname"] = match.group(1) - - match = re.search(r'image file is "(.+)"', data) - if match: - device_info["network_os_image"] = match.group(1) - - return device_info - - def get_device_operations(self): - return { - "supports_diff_replace": True, - "supports_commit": False, - "supports_rollback": False, - "supports_defaults": True, - "supports_onbox_diff": False, - "supports_commit_comment": False, - "supports_multiline_delimiter": True, - "supports_diff_match": True, - "supports_diff_ignore_lines": True, - "supports_generate_diff": True, - "supports_replace": False, - } - - def get_option_values(self): - return { - "format": ["text"], - "diff_match": ["line", "strict", "exact", "none"], - "diff_replace": ["line", "block"], - "output": [], - } - - def get_capabilities(self): - result = super(Cliconf, self).get_capabilities() - result["rpc"] += [ - "edit_banner", - "get_diff", - "run_commands", - "get_defaults_flag", - ] - result["device_operations"] = self.get_device_operations() - result.update(self.get_option_values()) - return json.dumps(result) - - def edit_banner( - self, candidate=None, multiline_delimiter="@", commit=True - ): - """ - Edit banner on remote device - :param banners: Banners to be loaded in json format - :param multiline_delimiter: Line delimiter for banner - :param commit: Boolean value that indicates if the device candidate - configuration should be pushed in the running configuration or discarded. - :param diff: Boolean flag to indicate if configuration that is applied on remote host should - generated and returned in response or not - :return: Returns response of executing the configuration command received - from remote host - """ - resp = {} - banners_obj = json.loads(candidate) - results = [] - requests = [] - if commit: - for key, value in iteritems(banners_obj): - key += " %s" % multiline_delimiter - self.send_command("config terminal", sendonly=True) - for cmd in [key, value, multiline_delimiter]: - obj = {"command": cmd, "sendonly": True} - results.append(self.send_command(**obj)) - requests.append(cmd) - - self.send_command("end", sendonly=True) - time.sleep(0.1) - results.append(self.send_command("\n")) - requests.append("\n") - - resp["request"] = requests - resp["response"] = results - - return resp - - def run_commands(self, commands=None, check_rc=True): - if commands is None: - raise ValueError("'commands' value is required") - - responses = list() - for cmd in to_list(commands): - if not isinstance(cmd, Mapping): - cmd = {"command": cmd} - - output = cmd.pop("output", None) - if output: - raise ValueError( - "'output' value %s is not supported for run_commands" - % output - ) - - try: - out = self.send_command(**cmd) - except AnsibleConnectionFailure as e: - if check_rc: - raise - out = getattr(e, "err", to_text(e)) - - responses.append(out) - - return responses - - def get_defaults_flag(self): - """ - The method identifies the filter that should be used to fetch running-configuration - with defaults. - :return: valid default filter - """ - out = self.get("show running-config ?") - out = to_text(out, errors="surrogate_then_replace") - - commands = set() - for line in out.splitlines(): - if line.strip(): - commands.add(line.strip().split()[0]) - - if "all" in commands: - return "all" - else: - return "full" - - def set_cli_prompt_context(self): - """ - Make sure we are in the operational cli mode - :return: None - """ - if self._connection.connected: - out = self._connection.get_prompt() - - if out is None: - raise AnsibleConnectionFailure( - message=u"cli prompt is not identified from the last received" - u" response window: %s" - % self._connection._last_recv_window - ) - - if re.search( - r"config.*\)#", - to_text(out, errors="surrogate_then_replace").strip(), - ): - self._connection.queue_message( - "vvvv", "wrong context, sending end to device" - ) - self._connection.send_command("end") - - def _extract_banners(self, config): - banners = {} - banner_cmds = re.findall(r"^banner (\w+)", config, re.M) - for cmd in banner_cmds: - regex = r"banner %s \^C(.+?)(?=\^C)" % cmd - match = re.search(regex, config, re.S) - if match: - key = "banner %s" % cmd - banners[key] = match.group(1).strip() - - for cmd in banner_cmds: - regex = r"banner %s \^C(.+?)(?=\^C)" % cmd - match = re.search(regex, config, re.S) - if match: - config = config.replace(str(match.group(1)), "") - - config = re.sub(r"banner \w+ \^C\^C", "!! banner removed", config) - return config, banners - - def _diff_banners(self, want, have): - candidate = {} - for key, value in iteritems(want): - if value != have.get(key): - candidate[key] = value - return candidate diff --git a/test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/doc_fragments/ios.py b/test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/doc_fragments/ios.py deleted file mode 100644 index 1b4ede27ec8..00000000000 --- a/test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/doc_fragments/ios.py +++ /dev/null @@ -1,82 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright: (c) 2015, Peter Sprygada -# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) -from __future__ import annotations - - -class ModuleDocFragment(object): - - # Standard files documentation fragment - DOCUMENTATION = r"""options: - provider: - description: - - B(Deprecated) - - 'Starting with Ansible 2.5 we recommend using C(connection: network_cli).' - - For more information please see the L(IOS Platform Options guide, ../network/user_guide/platform_ios.html). - - HORIZONTALLINE - - A dict object containing connection details. - type: dict - suboptions: - host: - description: - - Specifies the DNS host name or address for connecting to the remote device - over the specified transport. The value of host is used as the destination - address for the transport. - type: str - required: true - port: - description: - - Specifies the port to use when building the connection to the remote device. - type: int - default: 22 - username: - description: - - Configures the username to use to authenticate the connection to the remote - device. This value is used to authenticate the SSH session. If the value - is not specified in the task, the value of environment variable C(ANSIBLE_NET_USERNAME) - will be used instead. - type: str - password: - description: - - Specifies the password to use to authenticate the connection to the remote - device. This value is used to authenticate the SSH session. If the value - is not specified in the task, the value of environment variable C(ANSIBLE_NET_PASSWORD) - will be used instead. - type: str - timeout: - description: - - Specifies the timeout in seconds for communicating with the network device - for either connecting or sending commands. If the timeout is exceeded before - the operation is completed, the module will error. - type: int - default: 10 - ssh_keyfile: - description: - - Specifies the SSH key to use to authenticate the connection to the remote - device. This value is the path to the key used to authenticate the SSH - session. If the value is not specified in the task, the value of environment - variable C(ANSIBLE_NET_SSH_KEYFILE) will be used instead. - type: path - authorize: - description: - - Instructs the module to enter privileged mode on the remote device before - sending any commands. If not specified, the device will attempt to execute - all commands in non-privileged mode. If the value is not specified in the - task, the value of environment variable C(ANSIBLE_NET_AUTHORIZE) will be - used instead. - type: bool - default: false - auth_pass: - description: - - Specifies the password to use if required to enter privileged mode on the - remote device. If I(authorize) is false, then this argument does nothing. - If the value is not specified in the task, the value of environment variable - C(ANSIBLE_NET_AUTH_PASS) will be used instead. - type: str -notes: -- For more information on using Ansible to manage network devices see the :ref:`Ansible - Network Guide ` -- For more information on using Ansible to manage Cisco devices see the `Cisco integration - page `_. -""" diff --git a/test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/module_utils/network/ios/ios.py b/test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/module_utils/network/ios/ios.py deleted file mode 100644 index a21e047547b..00000000000 --- a/test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/module_utils/network/ios/ios.py +++ /dev/null @@ -1,199 +0,0 @@ -# This code is part of Ansible, but is an independent component. -# This particular file snippet, and this file snippet only, is BSD licensed. -# Modules you write using this snippet, which is embedded dynamically by Ansible -# still belong to the author of the module, and may assign their own license -# to the complete work. -# -# (c) 2016 Red Hat Inc. -# -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -from __future__ import annotations - -import json - -from ansible.module_utils.common.text.converters import to_text -from ansible.module_utils.basic import env_fallback -from ansible_collections.ansible.netcommon.plugins.module_utils.network.common.utils import ( - to_list, -) -from ansible.module_utils.connection import Connection, ConnectionError - -_DEVICE_CONFIGS = {} - -ios_provider_spec = { - "host": dict(), - "port": dict(type="int"), - "username": dict(fallback=(env_fallback, ["ANSIBLE_NET_USERNAME"])), - "password": dict( - fallback=(env_fallback, ["ANSIBLE_NET_PASSWORD"]), no_log=True - ), - "ssh_keyfile": dict( - fallback=(env_fallback, ["ANSIBLE_NET_SSH_KEYFILE"]), type="path" - ), - "authorize": dict( - fallback=(env_fallback, ["ANSIBLE_NET_AUTHORIZE"]), type="bool" - ), - "auth_pass": dict( - fallback=(env_fallback, ["ANSIBLE_NET_AUTH_PASS"]), no_log=True - ), - "timeout": dict(type="int"), -} -ios_argument_spec = { - "provider": dict( - type="dict", options=ios_provider_spec, removed_in_version=2.14 - ) -} - - -def get_provider_argspec(): - return ios_provider_spec - - -def get_connection(module): - if hasattr(module, "_ios_connection"): - return module._ios_connection - - capabilities = get_capabilities(module) - network_api = capabilities.get("network_api") - if network_api == "cliconf": - module._ios_connection = Connection(module._socket_path) - else: - module.fail_json(msg="Invalid connection type %s" % network_api) - - return module._ios_connection - - -def get_capabilities(module): - if hasattr(module, "_ios_capabilities"): - return module._ios_capabilities - try: - capabilities = Connection(module._socket_path).get_capabilities() - except ConnectionError as exc: - module.fail_json(msg=to_text(exc, errors="surrogate_then_replace")) - module._ios_capabilities = json.loads(capabilities) - return module._ios_capabilities - - -def get_defaults_flag(module): - connection = get_connection(module) - try: - out = connection.get_defaults_flag() - except ConnectionError as exc: - module.fail_json(msg=to_text(exc, errors="surrogate_then_replace")) - return to_text(out, errors="surrogate_then_replace").strip() - - -def get_config(module, flags=None): - flags = to_list(flags) - - section_filter = False - if flags and "section" in flags[-1]: - section_filter = True - - flag_str = " ".join(flags) - - try: - return _DEVICE_CONFIGS[flag_str] - except KeyError: - connection = get_connection(module) - try: - out = connection.get_config(flags=flags) - except ConnectionError as exc: - if section_filter: - # Some ios devices don't understand `| section foo` - out = get_config(module, flags=flags[:-1]) - else: - module.fail_json( - msg=to_text(exc, errors="surrogate_then_replace") - ) - cfg = to_text(out, errors="surrogate_then_replace").strip() - _DEVICE_CONFIGS[flag_str] = cfg - return cfg - - -def run_commands(module, commands, check_rc=True): - connection = get_connection(module) - try: - return connection.run_commands(commands=commands, check_rc=check_rc) - except ConnectionError as exc: - module.fail_json(msg=to_text(exc)) - - -def load_config(module, commands): - connection = get_connection(module) - - try: - resp = connection.edit_config(commands) - return resp.get("response") - except ConnectionError as exc: - module.fail_json(msg=to_text(exc)) - - -def normalize_interface(name): - """Return the normalized interface name - """ - if not name: - return - - def _get_number(name): - digits = "" - for char in name: - if char.isdigit() or char in "/.": - digits += char - return digits - - if name.lower().startswith("gi"): - if_type = "GigabitEthernet" - elif name.lower().startswith("te"): - if_type = "TenGigabitEthernet" - elif name.lower().startswith("fa"): - if_type = "FastEthernet" - elif name.lower().startswith("fo"): - if_type = "FortyGigabitEthernet" - elif name.lower().startswith("et"): - if_type = "Ethernet" - elif name.lower().startswith("vl"): - if_type = "Vlan" - elif name.lower().startswith("lo"): - if_type = "loopback" - elif name.lower().startswith("po"): - if_type = "port-channel" - elif name.lower().startswith("nv"): - if_type = "nve" - elif name.lower().startswith("twe"): - if_type = "TwentyFiveGigE" - elif name.lower().startswith("hu"): - if_type = "HundredGigE" - else: - if_type = None - - number_list = name.split(" ") - if len(number_list) == 2: - if_number = number_list[-1].strip() - else: - if_number = _get_number(name) - - if if_type: - proper_interface = if_type + if_number - else: - proper_interface = name - - return proper_interface diff --git a/test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/modules/ios_command.py b/test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/modules/ios_command.py deleted file mode 100644 index 9486a03c91f..00000000000 --- a/test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/modules/ios_command.py +++ /dev/null @@ -1,230 +0,0 @@ -#!/usr/bin/python -# -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . -# -from __future__ import annotations - -ANSIBLE_METADATA = { - "metadata_version": "1.1", - "status": ["preview"], - "supported_by": "network", -} - - -DOCUMENTATION = """module: ios_command -author: Peter Sprygada (@privateip) -short_description: Run commands on remote devices running Cisco IOS -description: -- Sends arbitrary commands to an ios node and returns the results read from the device. - This module includes an argument that will cause the module to wait for a specific - condition before returning or timing out if the condition is not met. -- This module does not support running commands in configuration mode. Please use - M(ios_config) to configure IOS devices. -extends_documentation_fragment: -- cisco.ios.ios -notes: -- Tested against IOS 15.6 -options: - commands: - description: - - List of commands to send to the remote ios device over the configured provider. - The resulting output from the command is returned. If the I(wait_for) argument - is provided, the module is not returned until the condition is satisfied or - the number of retries has expired. If a command sent to the device requires - answering a prompt, it is possible to pass a dict containing I(command), I(answer) - and I(prompt). Common answers are 'y' or "\r" (carriage return, must be double - quotes). See examples. - required: true - wait_for: - description: - - List of conditions to evaluate against the output of the command. The task will - wait for each condition to be true before moving forward. If the conditional - is not true within the configured number of retries, the task fails. See examples. - aliases: - - waitfor - match: - description: - - The I(match) argument is used in conjunction with the I(wait_for) argument to - specify the match policy. Valid values are C(all) or C(any). If the value - is set to C(all) then all conditionals in the wait_for must be satisfied. If - the value is set to C(any) then only one of the values must be satisfied. - default: all - choices: - - any - - all - retries: - description: - - Specifies the number of retries a command should by tried before it is considered - failed. The command is run on the target device every retry and evaluated against - the I(wait_for) conditions. - default: 10 - interval: - description: - - Configures the interval in seconds to wait between retries of the command. If - the command does not pass the specified conditions, the interval indicates how - long to wait before trying the command again. - default: 1 -""" - -EXAMPLES = r""" -tasks: - - name: run show version on remote devices - ios_command: - commands: show version - - - name: run show version and check to see if output contains IOS - ios_command: - commands: show version - wait_for: result[0] contains IOS - - - name: run multiple commands on remote nodes - ios_command: - commands: - - show version - - show interfaces - - - name: run multiple commands and evaluate the output - ios_command: - commands: - - show version - - show interfaces - wait_for: - - result[0] contains IOS - - result[1] contains Loopback0 - - - name: run commands that require answering a prompt - ios_command: - commands: - - command: 'clear counters GigabitEthernet0/1' - prompt: 'Clear "show interface" counters on this interface \[confirm\]' - answer: 'y' - - command: 'clear counters GigabitEthernet0/2' - prompt: '[confirm]' - answer: "\r" -""" - -RETURN = """ -stdout: - description: The set of responses from the commands - returned: always apart from low level errors (such as action plugin) - type: list - sample: ['...', '...'] -stdout_lines: - description: The value of stdout split into a list - returned: always apart from low level errors (such as action plugin) - type: list - sample: [['...', '...'], ['...'], ['...']] -failed_conditions: - description: The list of conditionals that have failed - returned: failed - type: list - sample: ['...', '...'] -""" -import time - -from ansible.module_utils.common.text.converters import to_text -from ansible.module_utils.basic import AnsibleModule -from ansible_collections.ansible.netcommon.plugins.module_utils.network.common.parsing import ( - Conditional, -) -from ansible_collections.ansible.netcommon.plugins.module_utils.network.common.utils import ( - transform_commands, - to_lines, -) -from ansible_collections.cisco.ios.plugins.module_utils.network.ios.ios import ( - run_commands, -) -from ansible_collections.cisco.ios.plugins.module_utils.network.ios.ios import ( - ios_argument_spec, -) - - -def parse_commands(module, warnings): - commands = transform_commands(module) - - if module.check_mode: - for item in list(commands): - if not item["command"].startswith("show"): - warnings.append( - "Only show commands are supported when using check mode, not " - "executing %s" % item["command"] - ) - commands.remove(item) - - return commands - - -def main(): - """main entry point for module execution - """ - argument_spec = dict( - commands=dict(type="list", required=True), - wait_for=dict(type="list", aliases=["waitfor"]), - match=dict(default="all", choices=["all", "any"]), - retries=dict(default=10, type="int"), - interval=dict(default=1, type="int"), - ) - - argument_spec.update(ios_argument_spec) - - module = AnsibleModule( - argument_spec=argument_spec, supports_check_mode=True - ) - - warnings = list() - result = {"changed": False, "warnings": warnings} - commands = parse_commands(module, warnings) - wait_for = module.params["wait_for"] or list() - - try: - conditionals = [Conditional(c) for c in wait_for] - except AttributeError as exc: - module.fail_json(msg=to_text(exc)) - - retries = module.params["retries"] - interval = module.params["interval"] - match = module.params["match"] - - while retries > 0: - responses = run_commands(module, commands) - - for item in list(conditionals): - if item(responses): - if match == "any": - conditionals = list() - break - conditionals.remove(item) - - if not conditionals: - break - - time.sleep(interval) - retries -= 1 - - if conditionals: - failed_conditions = [item.raw for item in conditionals] - msg = "One or more conditional statements have not been satisfied" - module.fail_json(msg=msg, failed_conditions=failed_conditions) - - result.update( - {"stdout": responses, "stdout_lines": list(to_lines(responses))} - ) - - module.exit_json(**result) - - -if __name__ == "__main__": - main() diff --git a/test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/modules/ios_config.py b/test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/modules/ios_config.py deleted file mode 100644 index f9b49a17b8f..00000000000 --- a/test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/modules/ios_config.py +++ /dev/null @@ -1,599 +0,0 @@ -#!/usr/bin/python -# -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . -# -from __future__ import annotations - -ANSIBLE_METADATA = { - "metadata_version": "1.1", - "status": ["preview"], - "supported_by": "network", -} - - -DOCUMENTATION = """module: ios_config -author: Peter Sprygada (@privateip) -short_description: Manage Cisco IOS configuration sections -description: -- Cisco IOS configurations use a simple block indent file syntax for segmenting configuration - into sections. This module provides an implementation for working with IOS configuration - sections in a deterministic way. -extends_documentation_fragment: -- cisco.ios.ios -notes: -- Tested against IOS 15.6 -- Abbreviated commands are NOT idempotent, - see L(Network FAQ,../network/user_guide/faq.html#why-do-the-config-modules-always-return-changed-true-with-abbreviated-commands). -options: - lines: - description: - - The ordered set of commands that should be configured in the section. The commands - must be the exact same commands as found in the device running-config. Be sure - to note the configuration command syntax as some commands are automatically - modified by the device config parser. - aliases: - - commands - parents: - description: - - The ordered set of parents that uniquely identify the section or hierarchy the - commands should be checked against. If the parents argument is omitted, the - commands are checked against the set of top level or global commands. - src: - description: - - Specifies the source path to the file that contains the configuration or configuration - template to load. The path to the source file can either be the full path on - the Ansible control host or a relative path from the playbook or role root directory. This - argument is mutually exclusive with I(lines), I(parents). - before: - description: - - The ordered set of commands to push on to the command stack if a change needs - to be made. This allows the playbook designer the opportunity to perform configuration - commands prior to pushing any changes without affecting how the set of commands - are matched against the system. - after: - description: - - The ordered set of commands to append to the end of the command stack if a change - needs to be made. Just like with I(before) this allows the playbook designer - to append a set of commands to be executed after the command set. - match: - description: - - Instructs the module on the way to perform the matching of the set of commands - against the current device config. If match is set to I(line), commands are - matched line by line. If match is set to I(strict), command lines are matched - with respect to position. If match is set to I(exact), command lines must be - an equal match. Finally, if match is set to I(none), the module will not attempt - to compare the source configuration with the running configuration on the remote - device. - choices: - - line - - strict - - exact - - none - default: line - replace: - description: - - Instructs the module on the way to perform the configuration on the device. - If the replace argument is set to I(line) then the modified lines are pushed - to the device in configuration mode. If the replace argument is set to I(block) - then the entire command block is pushed to the device in configuration mode - if any line is not correct. - default: line - choices: - - line - - block - multiline_delimiter: - description: - - This argument is used when pushing a multiline configuration element to the - IOS device. It specifies the character to use as the delimiting character. This - only applies to the configuration action. - default: '@' - backup: - description: - - This argument will cause the module to create a full backup of the current C(running-config) - from the remote device before any changes are made. If the C(backup_options) - value is not given, the backup file is written to the C(backup) folder in the - playbook root directory or role root directory, if playbook is part of an ansible - role. If the directory does not exist, it is created. - type: bool - default: 'no' - running_config: - description: - - The module, by default, will connect to the remote device and retrieve the current - running-config to use as a base for comparing against the contents of source. - There are times when it is not desirable to have the task get the current running-config - for every task in a playbook. The I(running_config) argument allows the implementer - to pass in the configuration to use as the base config for comparison. - aliases: - - config - defaults: - description: - - This argument specifies whether or not to collect all defaults when getting - the remote device running config. When enabled, the module will get the current - config by issuing the command C(show running-config all). - type: bool - default: 'no' - save_when: - description: - - When changes are made to the device running-configuration, the changes are not - copied to non-volatile storage by default. Using this argument will change - that before. If the argument is set to I(always), then the running-config will - always be copied to the startup-config and the I(modified) flag will always - be set to True. If the argument is set to I(modified), then the running-config - will only be copied to the startup-config if it has changed since the last save - to startup-config. If the argument is set to I(never), the running-config will - never be copied to the startup-config. If the argument is set to I(changed), - then the running-config will only be copied to the startup-config if the task - has made a change. I(changed) was added in Ansible 2.5. - default: never - choices: - - always - - never - - modified - - changed - diff_against: - description: - - When using the C(ansible-playbook --diff) command line argument the module can - generate diffs against different sources. - - When this option is configure as I(startup), the module will return the diff - of the running-config against the startup-config. - - When this option is configured as I(intended), the module will return the diff - of the running-config against the configuration provided in the C(intended_config) - argument. - - When this option is configured as I(running), the module will return the before - and after diff of the running-config with respect to any changes made to the - device configuration. - choices: - - running - - startup - - intended - diff_ignore_lines: - description: - - Use this argument to specify one or more lines that should be ignored during - the diff. This is used for lines in the configuration that are automatically - updated by the system. This argument takes a list of regular expressions or - exact line matches. - intended_config: - description: - - The C(intended_config) provides the master configuration that the node should - conform to and is used to check the final running-config against. This argument - will not modify any settings on the remote device and is strictly used to check - the compliance of the current device's configuration against. When specifying - this argument, the task should also modify the C(diff_against) value and set - it to I(intended). - backup_options: - description: - - This is a dict object containing configurable options related to backup file - path. The value of this option is read only when C(backup) is set to I(yes), - if C(backup) is set to I(no) this option will be silently ignored. - suboptions: - filename: - description: - - The filename to be used to store the backup configuration. If the filename - is not given it will be generated based on the hostname, current time and - date in format defined by _config.@ - dir_path: - description: - - This option provides the path ending with directory name in which the backup - configuration file will be stored. If the directory does not exist it will - be first created and the filename is either the value of C(filename) or - default filename as described in C(filename) options description. If the - path value is not given in that case a I(backup) directory will be created - in the current working directory and backup configuration will be copied - in C(filename) within I(backup) directory. - type: path - type: dict -""" - -EXAMPLES = """ -- name: configure top level configuration - ios_config: - lines: hostname {{ inventory_hostname }} - -- name: configure interface settings - ios_config: - lines: - - description test interface - - ip address 172.31.1.1 255.255.255.0 - parents: interface Ethernet1 - -- name: configure ip helpers on multiple interfaces - ios_config: - lines: - - ip helper-address 172.26.1.10 - - ip helper-address 172.26.3.8 - parents: "{{ item }}" - with_items: - - interface Ethernet1 - - interface Ethernet2 - - interface GigabitEthernet1 - -- name: configure policer in Scavenger class - ios_config: - lines: - - conform-action transmit - - exceed-action drop - parents: - - policy-map Foo - - class Scavenger - - police cir 64000 - -- name: load new acl into device - ios_config: - lines: - - 10 permit ip host 192.0.2.1 any log - - 20 permit ip host 192.0.2.2 any log - - 30 permit ip host 192.0.2.3 any log - - 40 permit ip host 192.0.2.4 any log - - 50 permit ip host 192.0.2.5 any log - parents: ip access-list extended test - before: no ip access-list extended test - match: exact - -- name: check the running-config against master config - ios_config: - diff_against: intended - intended_config: "{{ lookup('file', 'master.cfg') }}" - -- name: check the startup-config against the running-config - ios_config: - diff_against: startup - diff_ignore_lines: - - ntp clock .* - -- name: save running to startup when modified - ios_config: - save_when: modified - -- name: for idempotency, use full-form commands - ios_config: - lines: - # - shut - - shutdown - # parents: int gig1/0/11 - parents: interface GigabitEthernet1/0/11 - -# Set boot image based on comparison to a group_var (version) and the version -# that is returned from the `ios_facts` module -- name: SETTING BOOT IMAGE - ios_config: - lines: - - no boot system - - boot system flash bootflash:{{new_image}} - host: "{{ inventory_hostname }}" - when: ansible_net_version != version - -- name: render a Jinja2 template onto an IOS device - ios_config: - backup: yes - src: ios_template.j2 - -- name: configurable backup path - ios_config: - src: ios_template.j2 - backup: yes - backup_options: - filename: backup.cfg - dir_path: /home/user -""" - -RETURN = """ -updates: - description: The set of commands that will be pushed to the remote device - returned: always - type: list - sample: ['hostname foo', 'router ospf 1', 'router-id 192.0.2.1'] -commands: - description: The set of commands that will be pushed to the remote device - returned: always - type: list - sample: ['hostname foo', 'router ospf 1', 'router-id 192.0.2.1'] -backup_path: - description: The full path to the backup file - returned: when backup is yes - type: str - sample: /playbooks/ansible/backup/ios_config.2016-07-16@22:28:34 -filename: - description: The name of the backup file - returned: when backup is yes and filename is not specified in backup options - type: str - sample: ios_config.2016-07-16@22:28:34 -shortname: - description: The full path to the backup file excluding the timestamp - returned: when backup is yes and filename is not specified in backup options - type: str - sample: /playbooks/ansible/backup/ios_config -date: - description: The date extracted from the backup file name - returned: when backup is yes - type: str - sample: "2016-07-16" -time: - description: The time extracted from the backup file name - returned: when backup is yes - type: str - sample: "22:28:34" -""" -import json - -from ansible.module_utils.common.text.converters import to_text -from ansible.module_utils.connection import ConnectionError -from ansible_collections.cisco.ios.plugins.module_utils.network.ios.ios import ( - run_commands, - get_config, -) -from ansible_collections.cisco.ios.plugins.module_utils.network.ios.ios import ( - get_defaults_flag, - get_connection, -) -from ansible_collections.cisco.ios.plugins.module_utils.network.ios.ios import ( - ios_argument_spec, -) -from ansible.module_utils.basic import AnsibleModule -from ansible_collections.ansible.netcommon.plugins.module_utils.network.common.config import ( - NetworkConfig, - dumps, -) - - -def check_args(module, warnings): - if module.params["multiline_delimiter"]: - if len(module.params["multiline_delimiter"]) != 1: - module.fail_json( - msg="multiline_delimiter value can only be a " - "single character" - ) - - -def edit_config_or_macro(connection, commands): - # only catch the macro configuration command, - # not negated 'no' variation. - if commands[0].startswith("macro name"): - connection.edit_macro(candidate=commands) - else: - connection.edit_config(candidate=commands) - - -def get_candidate_config(module): - candidate = "" - if module.params["src"]: - candidate = module.params["src"] - - elif module.params["lines"]: - candidate_obj = NetworkConfig(indent=1) - parents = module.params["parents"] or list() - candidate_obj.add(module.params["lines"], parents=parents) - candidate = dumps(candidate_obj, "raw") - - return candidate - - -def get_running_config(module, current_config=None, flags=None): - running = module.params["running_config"] - if not running: - if not module.params["defaults"] and current_config: - running = current_config - else: - running = get_config(module, flags=flags) - - return running - - -def save_config(module, result): - result["changed"] = True - if not module.check_mode: - run_commands(module, "copy running-config startup-config\r") - else: - module.warn( - "Skipping command `copy running-config startup-config` " - "due to check_mode. Configuration not copied to " - "non-volatile storage" - ) - - -def main(): - """ main entry point for module execution - """ - backup_spec = dict(filename=dict(), dir_path=dict(type="path")) - argument_spec = dict( - src=dict(type="path"), - lines=dict(aliases=["commands"], type="list"), - parents=dict(type="list"), - before=dict(type="list"), - after=dict(type="list"), - match=dict( - default="line", choices=["line", "strict", "exact", "none"] - ), - replace=dict(default="line", choices=["line", "block"]), - multiline_delimiter=dict(default="@"), - running_config=dict(aliases=["config"]), - intended_config=dict(), - defaults=dict(type="bool", default=False), - backup=dict(type="bool", default=False), - backup_options=dict(type="dict", options=backup_spec), - save_when=dict( - choices=["always", "never", "modified", "changed"], default="never" - ), - diff_against=dict(choices=["startup", "intended", "running"]), - diff_ignore_lines=dict(type="list"), - ) - - argument_spec.update(ios_argument_spec) - - mutually_exclusive = [("lines", "src"), ("parents", "src")] - - required_if = [ - ("match", "strict", ["lines"]), - ("match", "exact", ["lines"]), - ("replace", "block", ["lines"]), - ("diff_against", "intended", ["intended_config"]), - ] - - module = AnsibleModule( - argument_spec=argument_spec, - mutually_exclusive=mutually_exclusive, - required_if=required_if, - supports_check_mode=True, - ) - - result = {"changed": False} - - warnings = list() - check_args(module, warnings) - result["warnings"] = warnings - - diff_ignore_lines = module.params["diff_ignore_lines"] - config = None - contents = None - flags = get_defaults_flag(module) if module.params["defaults"] else [] - connection = get_connection(module) - - if module.params["backup"] or ( - module._diff and module.params["diff_against"] == "running" - ): - contents = get_config(module, flags=flags) - config = NetworkConfig(indent=1, contents=contents) - if module.params["backup"]: - result["__backup__"] = contents - - if any((module.params["lines"], module.params["src"])): - match = module.params["match"] - replace = module.params["replace"] - path = module.params["parents"] - - candidate = get_candidate_config(module) - running = get_running_config(module, contents, flags=flags) - try: - response = connection.get_diff( - candidate=candidate, - running=running, - diff_match=match, - diff_ignore_lines=diff_ignore_lines, - path=path, - diff_replace=replace, - ) - except ConnectionError as exc: - module.fail_json(msg=to_text(exc, errors="surrogate_then_replace")) - - config_diff = response["config_diff"] - banner_diff = response["banner_diff"] - - if config_diff or banner_diff: - commands = config_diff.split("\n") - - if module.params["before"]: - commands[:0] = module.params["before"] - - if module.params["after"]: - commands.extend(module.params["after"]) - - result["commands"] = commands - result["updates"] = commands - result["banners"] = banner_diff - - # send the configuration commands to the device and merge - # them with the current running config - if not module.check_mode: - if commands: - edit_config_or_macro(connection, commands) - if banner_diff: - connection.edit_banner( - candidate=json.dumps(banner_diff), - multiline_delimiter=module.params[ - "multiline_delimiter" - ], - ) - - result["changed"] = True - - running_config = module.params["running_config"] - startup_config = None - - if module.params["save_when"] == "always": - save_config(module, result) - elif module.params["save_when"] == "modified": - output = run_commands( - module, ["show running-config", "show startup-config"] - ) - - running_config = NetworkConfig( - indent=1, contents=output[0], ignore_lines=diff_ignore_lines - ) - startup_config = NetworkConfig( - indent=1, contents=output[1], ignore_lines=diff_ignore_lines - ) - - if running_config.sha1 != startup_config.sha1: - save_config(module, result) - elif module.params["save_when"] == "changed" and result["changed"]: - save_config(module, result) - - if module._diff: - if not running_config: - output = run_commands(module, "show running-config") - contents = output[0] - else: - contents = running_config - - # recreate the object in order to process diff_ignore_lines - running_config = NetworkConfig( - indent=1, contents=contents, ignore_lines=diff_ignore_lines - ) - - if module.params["diff_against"] == "running": - if module.check_mode: - module.warn( - "unable to perform diff against running-config due to check mode" - ) - contents = None - else: - contents = config.config_text - - elif module.params["diff_against"] == "startup": - if not startup_config: - output = run_commands(module, "show startup-config") - contents = output[0] - else: - contents = startup_config.config_text - - elif module.params["diff_against"] == "intended": - contents = module.params["intended_config"] - - if contents is not None: - base_config = NetworkConfig( - indent=1, contents=contents, ignore_lines=diff_ignore_lines - ) - - if running_config.sha1 != base_config.sha1: - before, after = "", "" - if module.params["diff_against"] == "intended": - before = running_config - after = base_config - elif module.params["diff_against"] in ("startup", "running"): - before = base_config - after = running_config - - result.update( - { - "changed": True, - "diff": {"before": str(before), "after": str(after)}, - } - ) - - module.exit_json(**result) - - -if __name__ == "__main__": - main() diff --git a/test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/terminal/ios.py b/test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/terminal/ios.py deleted file mode 100644 index 7194eb459e1..00000000000 --- a/test/support/network-integration/collections/ansible_collections/cisco/ios/plugins/terminal/ios.py +++ /dev/null @@ -1,114 +0,0 @@ -# -# (c) 2016 Red Hat Inc. -# -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . -# -from __future__ import annotations - - -import json -import re - -from ansible.errors import AnsibleConnectionFailure -from ansible.module_utils.common.text.converters import to_text, to_bytes -from ansible.plugins.terminal import TerminalBase -from ansible.utils.display import Display - -display = Display() - - -class TerminalModule(TerminalBase): - - terminal_stdout_re = [ - re.compile(br"[\r\n]?[\w\+\-\.:\/\[\]]+(?:\([^\)]+\)){0,3}(?:[>#]) ?$") - ] - - terminal_stderr_re = [ - re.compile(br"% ?Error"), - # re.compile(br"^% \w+", re.M), - re.compile(br"% ?Bad secret"), - re.compile(br"[\r\n%] Bad passwords"), - re.compile(br"invalid input", re.I), - re.compile(br"(?:incomplete|ambiguous) command", re.I), - re.compile(br"connection timed out", re.I), - re.compile(br"[^\r\n]+ not found"), - re.compile(br"'[^']' +returned error code: ?\d+"), - re.compile(br"Bad mask", re.I), - re.compile(br"% ?(\S+) ?overlaps with ?(\S+)", re.I), - re.compile(br"[%\S] ?Error: ?[\s]+", re.I), - re.compile(br"[%\S] ?Informational: ?[\s]+", re.I), - re.compile(br"Command authorization failed"), - ] - - def on_open_shell(self): - try: - self._exec_cli_command(b"terminal length 0") - except AnsibleConnectionFailure: - raise AnsibleConnectionFailure("unable to set terminal parameters") - - try: - self._exec_cli_command(b"terminal width 512") - try: - self._exec_cli_command(b"terminal width 0") - except AnsibleConnectionFailure: - pass - except AnsibleConnectionFailure: - display.display( - "WARNING: Unable to set terminal width, command responses may be truncated" - ) - - def on_become(self, passwd=None): - if self._get_prompt().endswith(b"#"): - return - - cmd = {u"command": u"enable"} - if passwd: - # Note: python-3.5 cannot combine u"" and r"" together. Thus make - # an r string and use to_text to ensure it's text on both py2 and py3. - cmd[u"prompt"] = to_text( - r"[\r\n]?(?:.*)?[Pp]assword: ?$", errors="surrogate_or_strict" - ) - cmd[u"answer"] = passwd - cmd[u"prompt_retry_check"] = True - try: - self._exec_cli_command( - to_bytes(json.dumps(cmd), errors="surrogate_or_strict") - ) - prompt = self._get_prompt() - if prompt is None or not prompt.endswith(b"#"): - raise AnsibleConnectionFailure( - "failed to elevate privilege to enable mode still at prompt [%s]" - % prompt - ) - except AnsibleConnectionFailure as e: - prompt = self._get_prompt() - raise AnsibleConnectionFailure( - "unable to elevate privilege to enable mode, at prompt [%s] with error: %s" - % (prompt, e.message) - ) - - def on_unbecome(self): - prompt = self._get_prompt() - if prompt is None: - # if prompt is None most likely the terminal is hung up at a prompt - return - - if b"(config" in prompt: - self._exec_cli_command(b"end") - self._exec_cli_command(b"disable") - - elif prompt.endswith(b"#"): - self._exec_cli_command(b"disable") From 34f2f39abe5e7b7f6ad96b889300553d0160532c Mon Sep 17 00:00:00 2001 From: Abhijeet Kasurde Date: Mon, 7 Jul 2025 10:06:08 -0700 Subject: [PATCH 18/68] json filter: update docs (#84989) Signed-off-by: Abhijeet Kasurde Co-authored-by: Brian Coca Co-authored-by: Matt Clay --- lib/ansible/plugins/filter/to_json.yml | 12 ++++++++---- lib/ansible/plugins/filter/to_nice_json.yml | 5 +++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/ansible/plugins/filter/to_json.yml b/lib/ansible/plugins/filter/to_json.yml index 003e5a19eb9..55b7607cab6 100644 --- a/lib/ansible/plugins/filter/to_json.yml +++ b/lib/ansible/plugins/filter/to_json.yml @@ -23,8 +23,9 @@ DOCUMENTATION: default: True version_added: '2.9' allow_nan: - description: When V(False), strict adherence to float value limits of the JSON specifications, so C(nan), C(inf) and C(-inf) values will produce errors. - When V(True), JavaScript equivalents will be used (C(NaN), C(Infinity), C(-Infinity)). + description: + - When V(False), out-of-range float values C(nan), C(inf) and C(-inf) will result in an error. + - When V(True), out-of-range float values will be represented using their JavaScript equivalents, C(NaN), C(Infinity) and C(-Infinity). default: True type: bool check_circular: @@ -42,8 +43,11 @@ DOCUMENTATION: separators: description: The C(item) and C(key) separator to be used in the serialized output, default may change depending on O(indent) and Python version. - default: "(', ', ': ')" - type: tuple + default: + - ', ' + - ': ' + type: list + elements: str skipkeys: description: If V(True), keys that are not basic Python types will be skipped. default: False diff --git a/lib/ansible/plugins/filter/to_nice_json.yml b/lib/ansible/plugins/filter/to_nice_json.yml index abaeee0c071..2c87f3e6d67 100644 --- a/lib/ansible/plugins/filter/to_nice_json.yml +++ b/lib/ansible/plugins/filter/to_nice_json.yml @@ -23,8 +23,9 @@ DOCUMENTATION: default: True version_added: '2.9' allow_nan: - description: When V(False), strict adherence to float value limits of the JSON specification, so C(nan), C(inf) and C(-inf) values will produce errors. - When V(True), JavaScript equivalents will be used (C(NaN), C(Infinity), C(-Infinity)). + description: + - When V(False), out-of-range float values C(nan), C(inf) and C(-inf) will result in an error. + - When V(True), out-of-range float values will be represented using their JavaScript equivalents, C(NaN), C(Infinity) and C(-Infinity). default: True type: bool check_circular: From 8207406306782657c8a74f825745e6d1592828f6 Mon Sep 17 00:00:00 2001 From: Abhijeet Kasurde Date: Mon, 7 Jul 2025 11:44:19 -0700 Subject: [PATCH 19/68] Fixed string type in tags fieldattribute (#84655) Signed-off-by: Abhijeet Kasurde --- lib/ansible/playbook/taggable.py | 3 +-- test/units/playbook/test_base.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/ansible/playbook/taggable.py b/lib/ansible/playbook/taggable.py index 64d8494b945..5823b775947 100644 --- a/lib/ansible/playbook/taggable.py +++ b/lib/ansible/playbook/taggable.py @@ -20,7 +20,6 @@ from __future__ import annotations import typing as t from ansible.errors import AnsibleError -from ansible.module_utils.six import string_types from ansible.module_utils.common.sentinel import Sentinel from ansible.module_utils._internal._datatag import AnsibleTagHelper from ansible.playbook.attribute import FieldAttribute @@ -40,7 +39,7 @@ def _flatten_tags(tags: list[str | int]) -> list[str | int]: class Taggable: untagged = frozenset(['untagged']) - tags = FieldAttribute(isa='list', default=list, listof=(string_types, int), extend=True) + tags = FieldAttribute(isa='list', default=list, listof=(str, int), extend=True) def _load_tags(self, attr, ds): if isinstance(ds, list): diff --git a/test/units/playbook/test_base.py b/test/units/playbook/test_base.py index 85fafcec56e..135c31b8c08 100644 --- a/test/units/playbook/test_base.py +++ b/test/units/playbook/test_base.py @@ -332,6 +332,7 @@ class BaseSubClass(base.Base): test_attr_int = FieldAttribute(isa='int', always_post_validate=True) test_attr_float = FieldAttribute(isa='float', default=3.14159, always_post_validate=True) test_attr_list = FieldAttribute(isa='list', listof=(str,), always_post_validate=True) + test_attr_mixed_list = FieldAttribute(isa='list', listof=(str, int), always_post_validate=True) test_attr_list_no_listof = FieldAttribute(isa='list', always_post_validate=True) test_attr_list_required = FieldAttribute(isa='list', listof=(str,), required=True, default=list, always_post_validate=True) @@ -518,6 +519,16 @@ class TestBaseSubClass(TestBase): bsc = self._base_validate(ds) self.assertEqual(string_list, bsc._test_attr_list) + def test_attr_mixed_list(self): + mixed_list = ['foo', 1] + ds = {'test_attr_mixed_list': mixed_list} + bsc = self._base_validate(ds) + self.assertEqual(mixed_list, bsc._test_attr_mixed_list) + + def test_attr_mixed_list_invalid(self): + ds = {'test_attr_mixed_list': [['foo'], 1]} + self.assertRaises(AnsibleParserError, self._base_validate, ds) + def test_attr_list_none(self): ds = {'test_attr_list': None} bsc = self._base_validate(ds) From e4135b6f1939987af8a498e1e91d71b1dc260c9c Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Tue, 8 Jul 2025 10:57:35 +0200 Subject: [PATCH 20/68] yum_repository: remove deprecated keepcache option (#85441) --- .../yum_repository-remove-keepcache.yml | 2 ++ lib/ansible/modules/yum_repository.py | 17 +---------------- 2 files changed, 3 insertions(+), 16 deletions(-) create mode 100644 changelogs/fragments/yum_repository-remove-keepcache.yml diff --git a/changelogs/fragments/yum_repository-remove-keepcache.yml b/changelogs/fragments/yum_repository-remove-keepcache.yml new file mode 100644 index 00000000000..6c6222549c7 --- /dev/null +++ b/changelogs/fragments/yum_repository-remove-keepcache.yml @@ -0,0 +1,2 @@ +removed_features: + - yum_repository - remove deprecated ``keepcache`` option. diff --git a/lib/ansible/modules/yum_repository.py b/lib/ansible/modules/yum_repository.py index c43feee736d..013e85d2d8e 100644 --- a/lib/ansible/modules/yum_repository.py +++ b/lib/ansible/modules/yum_repository.py @@ -183,14 +183,6 @@ options: - This parameter is deprecated as it has no effect with dnf as an underlying package manager and will be removed in ansible-core 2.22. type: bool - keepcache: - description: - - Either V(1) or V(0). Determines whether or not yum keeps the cache of - headers and packages after successful installation. - - This parameter is deprecated as it is only valid in the main configuration - and will be removed in ansible-core 2.20. - choices: ['0', '1'] - type: str metadata_expire: description: - Time (in seconds) after which the metadata will expire. @@ -466,13 +458,7 @@ class YumRepo: for key, value in sorted(self.params.items()): if value is None: continue - if key == 'keepcache': - self.module.deprecate( - "'keepcache' parameter is deprecated as it is only valid in " - "the main configuration.", - version='2.20' - ) - elif key == 'async': + if key == 'async': self.module.deprecate( "'async' parameter is deprecated as it has been removed on systems supported by ansible-core", version='2.22', @@ -557,7 +543,6 @@ def main(): includepkgs=dict(type='list', elements='str'), ip_resolve=dict(choices=['4', '6', 'IPv4', 'IPv6', 'whatever']), keepalive=dict(type='bool'), - keepcache=dict(choices=['0', '1']), metadata_expire=dict(), metadata_expire_filter=dict( choices=[ From c24b187f888ecf32e84a9409465be33ae8953ab1 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Tue, 8 Jul 2025 10:58:19 +0200 Subject: [PATCH 21/68] dnf/dnf5: remove deprecated install_repoquery option (#85440) https://github.com/ansible/ansible/issues/85410 --- changelogs/fragments/dnf-remove-install_repoquery.yml | 2 ++ lib/ansible/module_utils/yumdnf.py | 5 ----- lib/ansible/modules/dnf.py | 7 ------- lib/ansible/modules/dnf5.py | 6 ------ test/sanity/ignore.txt | 2 -- 5 files changed, 2 insertions(+), 20 deletions(-) create mode 100644 changelogs/fragments/dnf-remove-install_repoquery.yml diff --git a/changelogs/fragments/dnf-remove-install_repoquery.yml b/changelogs/fragments/dnf-remove-install_repoquery.yml new file mode 100644 index 00000000000..b804acc3e0d --- /dev/null +++ b/changelogs/fragments/dnf-remove-install_repoquery.yml @@ -0,0 +1,2 @@ +removed_features: + - dnf/dnf5 - remove deprecated ``install_repoquery`` option. diff --git a/lib/ansible/module_utils/yumdnf.py b/lib/ansible/module_utils/yumdnf.py index b2cbba3fde2..bdcf5ad7f72 100644 --- a/lib/ansible/module_utils/yumdnf.py +++ b/lib/ansible/module_utils/yumdnf.py @@ -32,10 +32,6 @@ yumdnf_argument_spec = dict( enablerepo=dict(type='list', elements='str', default=[]), exclude=dict(type='list', elements='str', default=[]), installroot=dict(type='str', default="/"), - install_repoquery=dict( - type='bool', default=True, - removed_in_version='2.20', removed_from_collection='ansible.builtin', - ), install_weak_deps=dict(type='bool', default=True), list=dict(type='str'), name=dict(type='list', elements='str', aliases=['pkg'], default=[]), @@ -85,7 +81,6 @@ class YumDnf(metaclass=ABCMeta): self.enablerepo = self.module.params.get('enablerepo', []) self.exclude = self.module.params['exclude'] self.installroot = self.module.params['installroot'] - self.install_repoquery = self.module.params['install_repoquery'] self.install_weak_deps = self.module.params['install_weak_deps'] self.list = self.module.params['list'] self.names = [p.strip() for p in self.module.params['name']] diff --git a/lib/ansible/modules/dnf.py b/lib/ansible/modules/dnf.py index 07f0384b5c9..1922ba85e79 100644 --- a/lib/ansible/modules/dnf.py +++ b/lib/ansible/modules/dnf.py @@ -211,13 +211,6 @@ options: type: bool default: "no" version_added: "2.7" - install_repoquery: - description: - - This is effectively a no-op in DNF as it is not needed with DNF. - - This option is deprecated and will be removed in ansible-core 2.20. - type: bool - default: "yes" - version_added: "2.7" download_only: description: - Only download the packages, do not install them. diff --git a/lib/ansible/modules/dnf5.py b/lib/ansible/modules/dnf5.py index a0e4a4ef5aa..cd9bf6e3f2e 100644 --- a/lib/ansible/modules/dnf5.py +++ b/lib/ansible/modules/dnf5.py @@ -180,12 +180,6 @@ options: in the earlier transaction). type: bool default: "no" - install_repoquery: - description: - - This is effectively a no-op in DNF as it is not needed with DNF. - - This option is deprecated and will be removed in ansible-core 2.20. - type: bool - default: "yes" download_only: description: - Only download the packages, do not install them. diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index c75cef48855..0751ca5b155 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -240,5 +240,3 @@ lib/ansible/utils/py3compat.py pylint:ansible-deprecated-version # TODO: 2.20 lib/ansible/utils/ssh_functions.py pylint:ansible-deprecated-version # TODO: 2.20 lib/ansible/vars/manager.py pylint:ansible-deprecated-version-comment # TODO: 2.20 lib/ansible/vars/plugins.py pylint:ansible-deprecated-version # TODO: 2.20 -lib/ansible/modules/dnf.py validate-modules:ansible-deprecated-version # TODO: 2.20 -lib/ansible/modules/dnf5.py validate-modules:ansible-deprecated-version # TODO: 2.20 From da6735160db41b7b31d34b5f46f17952592fac7f Mon Sep 17 00:00:00 2001 From: Sloane Hertel <19572925+s-hertel@users.noreply.github.com> Date: Tue, 8 Jul 2025 10:12:42 -0400 Subject: [PATCH 22/68] Fix passing callbacks the delegated connection, host, port, and user (#85397) Fix ssh plugin host variables. Variable names should conform to Python variable naming rules. , and not contain characters like "[" or "]". Update unit test --- ...fix-displaying-delegate_to-ansible_host.yml | 3 +++ lib/ansible/executor/task_executor.py | 18 +++++++++++------- lib/ansible/plugins/connection/ssh.py | 2 -- .../delegate_to/delegate_vars_handling.yml | 8 -------- .../delegate_to/delegate_vars_inventory | 2 ++ test/integration/targets/delegate_to/runme.sh | 6 +++++- test/units/plugins/connection/test_ssh.py | 4 ++-- 7 files changed, 23 insertions(+), 20 deletions(-) create mode 100644 changelogs/fragments/fix-displaying-delegate_to-ansible_host.yml create mode 100644 test/integration/targets/delegate_to/delegate_vars_inventory diff --git a/changelogs/fragments/fix-displaying-delegate_to-ansible_host.yml b/changelogs/fragments/fix-displaying-delegate_to-ansible_host.yml new file mode 100644 index 00000000000..741b7379487 --- /dev/null +++ b/changelogs/fragments/fix-displaying-delegate_to-ansible_host.yml @@ -0,0 +1,3 @@ +bugfixes: + - callback plugins - fix displaying the rendered ``ansible_host`` variable with ``delegate_to`` (https://github.com/ansible/ansible/issues/84922). + - ssh connection - fix documented variables for the ``host`` option. Connection options can be configured with delegated variables in general. diff --git a/lib/ansible/executor/task_executor.py b/lib/ansible/executor/task_executor.py index 60c6b392cbc..fa328f396cf 100644 --- a/lib/ansible/executor/task_executor.py +++ b/lib/ansible/executor/task_executor.py @@ -19,7 +19,7 @@ from ansible.errors import ( AnsibleError, AnsibleParserError, AnsibleUndefinedVariable, AnsibleTaskError, AnsibleValueOmittedError, ) -from ansible.executor.task_result import _RawTaskResult +from ansible.executor.task_result import _RawTaskResult, _SUB_PRESERVE from ansible._internal._datatag import _utils from ansible.module_utils._internal import _messages from ansible.module_utils.datatag import native_type_name, deprecator_from_collection_name @@ -771,14 +771,18 @@ class TaskExecutor: # on the results side without having to do any further templating # also now add connection vars results when delegating if self._task.delegate_to: - result["_ansible_delegated_vars"] = {'ansible_delegated_host': self._task.delegate_to} - for k in plugin_vars: - result["_ansible_delegated_vars"][k] = cvars.get(k) + result["_ansible_delegated_vars"] = { + "ansible_delegated_host": self._task.delegate_to, + "ansible_connection": current_connection, + } # note: here for callbacks that rely on this info to display delegation - for requireshed in ('ansible_host', 'ansible_port', 'ansible_user', 'ansible_connection'): - if requireshed not in result["_ansible_delegated_vars"] and requireshed in cvars: - result["_ansible_delegated_vars"][requireshed] = cvars.get(requireshed) + for k in plugin_vars: + if k not in _SUB_PRESERVE["_ansible_delegated_vars"]: + continue + + for o in C.config.get_plugin_options_from_var("connection", current_connection, k): + result["_ansible_delegated_vars"][k] = self._connection.get_option(o) # and return display.debug("attempt loop complete, returning result") diff --git a/lib/ansible/plugins/connection/ssh.py b/lib/ansible/plugins/connection/ssh.py index 08ff188cf6c..a4f7fedcd1c 100644 --- a/lib/ansible/plugins/connection/ssh.py +++ b/lib/ansible/plugins/connection/ssh.py @@ -34,8 +34,6 @@ DOCUMENTATION = """ - name: inventory_hostname - name: ansible_host - name: ansible_ssh_host - - name: delegated_vars['ansible_host'] - - name: delegated_vars['ansible_ssh_host'] host_key_checking: description: Determines if SSH should reject or not a connection after checking host keys. default: True diff --git a/test/integration/targets/delegate_to/delegate_vars_handling.yml b/test/integration/targets/delegate_to/delegate_vars_handling.yml index 13860a910c9..be3ed72ad6e 100644 --- a/test/integration/targets/delegate_to/delegate_vars_handling.yml +++ b/test/integration/targets/delegate_to/delegate_vars_handling.yml @@ -1,11 +1,3 @@ -- name: setup delegated host - hosts: localhost - gather_facts: false - tasks: - - add_host: - name: delegatetome - ansible_host: 127.0.0.4 - - name: ensure we dont use orig host vars if delegated one does not define them hosts: testhost gather_facts: false diff --git a/test/integration/targets/delegate_to/delegate_vars_inventory b/test/integration/targets/delegate_to/delegate_vars_inventory new file mode 100644 index 00000000000..7d32753e0bf --- /dev/null +++ b/test/integration/targets/delegate_to/delegate_vars_inventory @@ -0,0 +1,2 @@ +[all] +delegatetome ansible_host="{{ hostip }}" hostip="127.0.0.4" diff --git a/test/integration/targets/delegate_to/runme.sh b/test/integration/targets/delegate_to/runme.sh index ce5a607d25f..1fed90bfe17 100755 --- a/test/integration/targets/delegate_to/runme.sh +++ b/test/integration/targets/delegate_to/runme.sh @@ -57,7 +57,11 @@ ansible-playbook delegate_facts_block.yml -i inventory -v "$@" ansible-playbook test_delegate_to_loop_caching.yml -i inventory -v "$@" # ensure we are using correct settings when delegating -ANSIBLE_TIMEOUT=3 ansible-playbook delegate_vars_handling.yml -i inventory -v "$@" +ANSIBLE_TIMEOUT=3 ansible-playbook delegate_vars_handling.yml -i inventory -i delegate_vars_inventory -v | tee out +if grep '{{ hostip }}' out; then + echo 'Callback displayed the ansible_host template instead of the rendered value.' + exit 1 +fi ansible-playbook has_hostvars.yml -i inventory -v "$@" diff --git a/test/units/plugins/connection/test_ssh.py b/test/units/plugins/connection/test_ssh.py index e818d27ef86..12011afe5be 100644 --- a/test/units/plugins/connection/test_ssh.py +++ b/test/units/plugins/connection/test_ssh.py @@ -210,8 +210,8 @@ class TestConnectionBaseClass(unittest.TestCase): mock_ospe.return_value = True conn._build_command.return_value = 'some command to run' conn._bare_run.return_value = (0, '', '') - conn.host = "some_host" + conn.set_option("host", "some_host") conn.set_option('reconnection_retries', 9) conn.set_option('ssh_transfer_method', None) # default is smart @@ -261,8 +261,8 @@ class TestConnectionBaseClass(unittest.TestCase): conn._build_command.return_value = 'some command to run' conn._bare_run.return_value = (0, '', '') - conn.host = "some_host" + conn.set_option("host", "some_host") conn.set_option('reconnection_retries', 9) conn.set_option('ssh_transfer_method', None) # default is smart From 319dca2ea8b7ca3d3dbf003ee592cbdba308dfc6 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Thu, 10 Jul 2025 15:05:22 -0400 Subject: [PATCH 23/68] meta: clear_host_errors, clarify what this applies to (#85463) * meta: clear_host_errors, clarify what this applies to It clears host for selection, but does not change state in the current iterator. * also update example --- lib/ansible/modules/meta.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/ansible/modules/meta.py b/lib/ansible/modules/meta.py index 64d9c1c9f61..90a0b0045b2 100644 --- a/lib/ansible/modules/meta.py +++ b/lib/ansible/modules/meta.py @@ -33,6 +33,7 @@ options: - V(clear_facts) (added in Ansible 2.1) causes the gathered facts for the hosts specified in the play's list of hosts to be cleared, including the fact cache. - V(clear_host_errors) (added in Ansible 2.1) clears the failed state (if any) from hosts specified in the play's list of hosts. + This will make them available for targetting in subsequent plays, but not continue execution in the current play. - V(end_play) (added in Ansible 2.2) causes the play to end without failing the host(s). Note that this affects all hosts. - V(reset_connection) (added in Ansible 2.3) interrupts a persistent connection (i.e. ssh + control persist) - V(end_host) (added in Ansible 2.8) is a per-host variation of V(end_play). Causes the play to end for the current host without failing it. @@ -108,7 +109,7 @@ EXAMPLES = r""" - name: Clear gathered facts from all currently targeted hosts ansible.builtin.meta: clear_facts -# Example showing how to continue using a failed target +# Example showing how to continue using a failed target, for the next play - name: Bring host back to play after failure ansible.builtin.copy: src: file From ee96f8e912f6ee5c3c3ce49f88836554500acecb Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Tue, 15 Jul 2025 12:50:56 -0500 Subject: [PATCH 24/68] Use the API task URI directly for polling collection imports (#85419) --- .../fragments/galaxy-use-import-task-url.yml | 2 + lib/ansible/galaxy/api.py | 22 ++++------ lib/ansible/galaxy/collection/__init__.py | 15 +------ test/units/galaxy/test_api.py | 40 +++++++------------ test/units/galaxy/test_collection.py | 2 +- 5 files changed, 26 insertions(+), 55 deletions(-) create mode 100644 changelogs/fragments/galaxy-use-import-task-url.yml diff --git a/changelogs/fragments/galaxy-use-import-task-url.yml b/changelogs/fragments/galaxy-use-import-task-url.yml new file mode 100644 index 00000000000..f557e67e8ce --- /dev/null +++ b/changelogs/fragments/galaxy-use-import-task-url.yml @@ -0,0 +1,2 @@ +bugfixes: +- ansible-galaxy - Use the provided import task url, instead of parsing to get the task id and reconstructing the URL diff --git a/lib/ansible/galaxy/api.py b/lib/ansible/galaxy/api.py index 0a737c46ffb..60689f44670 100644 --- a/lib/ansible/galaxy/api.py +++ b/lib/ansible/galaxy/api.py @@ -689,10 +689,10 @@ class GalaxyAPI: error_context_msg='Error when publishing collection to %s (%s)' % (self.name, self.api_server)) - return resp['task'] + return urljoin(self.api_server, resp['task']) @g_connect(['v2', 'v3']) - def wait_import_task(self, task_id, timeout=0): + def wait_import_task(self, task_url, timeout=0): """ Waits until the import process on the Galaxy server has completed or the timeout is reached. @@ -703,22 +703,14 @@ class GalaxyAPI: state = 'waiting' data = None - # Construct the appropriate URL per version - if 'v3' in self.available_api_versions: - full_url = _urljoin(self.api_server, self.available_api_versions['v3'], - 'imports/collections', task_id, '/') - else: - full_url = _urljoin(self.api_server, self.available_api_versions['v2'], - 'collection-imports', task_id, '/') - - display.display("Waiting until Galaxy import task %s has completed" % full_url) + display.display("Waiting until Galaxy import task %s has completed" % task_url) start = time.time() wait = C.GALAXY_COLLECTION_IMPORT_POLL_INTERVAL while timeout == 0 or (time.time() - start) < timeout: try: - data = self._call_galaxy(full_url, method='GET', auth_required=True, - error_context_msg='Error when getting import task results at %s' % full_url) + data = self._call_galaxy(task_url, method='GET', auth_required=True, + error_context_msg='Error when getting import task results at %s' % task_url) except GalaxyError as e: if e.http_code != 404: raise @@ -740,7 +732,7 @@ class GalaxyAPI: wait = min(30, wait * C.GALAXY_COLLECTION_IMPORT_POLL_FACTOR) if state == 'waiting': raise AnsibleError("Timeout while waiting for the Galaxy import process to finish, check progress at '%s'" - % to_native(full_url)) + % to_native(task_url)) for message in data.get('messages', []): level = message['level'] @@ -754,7 +746,7 @@ class GalaxyAPI: if state == 'failed': code = to_native(data['error'].get('code', 'UNKNOWN')) description = to_native( - data['error'].get('description', "Unknown error, see %s for more details" % full_url)) + data['error'].get('description', "Unknown error, see %s for more details" % task_url)) raise AnsibleError("Galaxy import process failed: %s (Code: %s)" % (description, code)) @g_connect(['v2', 'v3']) diff --git a/lib/ansible/galaxy/collection/__init__.py b/lib/ansible/galaxy/collection/__init__.py index 3bc0f27ee57..38737468dcd 100644 --- a/lib/ansible/galaxy/collection/__init__.py +++ b/lib/ansible/galaxy/collection/__init__.py @@ -623,24 +623,11 @@ def publish_collection(collection_path, api, wait, timeout): import_uri = api.publish_collection(collection_path) if wait: - # Galaxy returns a url fragment which differs between v2 and v3. The second to last entry is - # always the task_id, though. - # v2: {"task": "https://galaxy-dev.ansible.com/api/v2/collection-imports/35573/"} - # v3: {"task": "/api/automation-hub/v3/imports/collections/838d1308-a8f4-402c-95cb-7823f3806cd8/"} - task_id = None - for path_segment in reversed(import_uri.split('/')): - if path_segment: - task_id = path_segment - break - - if not task_id: - raise AnsibleError("Publishing the collection did not return valid task info. Cannot wait for task status. Returned task info: '%s'" % import_uri) - with _display_progress( "Collection has been published to the Galaxy server " "{api.name!s} {api.api_server!s}".format(api=api), ): - api.wait_import_task(task_id, timeout) + api.wait_import_task(import_uri, timeout) display.display("Collection has been successfully published and imported to the Galaxy server %s %s" % (api.name, api.api_server)) else: diff --git a/test/units/galaxy/test_api.py b/test/units/galaxy/test_api.py index 22d83e43b3c..58985f1bb0e 100644 --- a/test/units/galaxy/test_api.py +++ b/test/units/galaxy/test_api.py @@ -451,15 +451,13 @@ def test_publish_failure(api_version, collection_url, response, expected, collec api.publish_collection(collection_artifact) -@pytest.mark.parametrize('server_url, api_version, token_type, token_ins, import_uri, full_import_uri', [ +@pytest.mark.parametrize('server_url, api_version, token_type, token_ins, full_import_uri', [ ('https://galaxy.server.com/api', 'v2', 'Token', GalaxyToken('my token'), - '1234', 'https://galaxy.server.com/api/v2/collection-imports/1234/'), ('https://galaxy.server.com/api/automation-hub/', 'v3', 'Bearer', KeycloakToken(auth_url='https://api.test/'), - '1234', 'https://galaxy.server.com/api/automation-hub/v3/imports/collections/1234/'), ]) -def test_wait_import_task(server_url, api_version, token_type, token_ins, import_uri, full_import_uri, monkeypatch): +def test_wait_import_task(server_url, api_version, token_type, token_ins, full_import_uri, monkeypatch): api = get_test_galaxy_api(server_url, api_version, token_ins=token_ins) mock_token_get = MagicMock() @@ -473,7 +471,7 @@ def test_wait_import_task(server_url, api_version, token_type, token_ins, import mock_display = MagicMock() monkeypatch.setattr(Display, 'display', mock_display) - api.wait_import_task(import_uri) + api.wait_import_task(full_import_uri) assert mock_open.call_count == 1 assert mock_open.mock_calls[0][1][0] == full_import_uri @@ -483,15 +481,13 @@ def test_wait_import_task(server_url, api_version, token_type, token_ins, import assert mock_display.mock_calls[0][1][0] == 'Waiting until Galaxy import task %s has completed' % full_import_uri -@pytest.mark.parametrize('server_url, api_version, token_type, token_ins, import_uri, full_import_uri', [ +@pytest.mark.parametrize('server_url, api_version, token_type, token_ins, full_import_uri', [ ('https://galaxy.server.com/api/', 'v2', 'Token', GalaxyToken('my token'), - '1234', 'https://galaxy.server.com/api/v2/collection-imports/1234/'), ('https://galaxy.server.com/api/automation-hub', 'v3', 'Bearer', KeycloakToken(auth_url='https://api.test/'), - '1234', 'https://galaxy.server.com/api/automation-hub/v3/imports/collections/1234/'), ]) -def test_wait_import_task_multiple_requests(server_url, api_version, token_type, token_ins, import_uri, full_import_uri, monkeypatch): +def test_wait_import_task_multiple_requests(server_url, api_version, token_type, token_ins, full_import_uri, monkeypatch): api = get_test_galaxy_api(server_url, api_version, token_ins=token_ins) mock_token_get = MagicMock() @@ -513,7 +509,7 @@ def test_wait_import_task_multiple_requests(server_url, api_version, token_type, monkeypatch.setattr(time, 'sleep', MagicMock()) - api.wait_import_task(import_uri) + api.wait_import_task(full_import_uri) assert mock_open.call_count == 2 assert mock_open.mock_calls[0][1][0] == full_import_uri @@ -529,15 +525,13 @@ def test_wait_import_task_multiple_requests(server_url, api_version, token_type, 'Galaxy import process has a status of test, wait 2 seconds before trying again' -@pytest.mark.parametrize('server_url, api_version, token_type, token_ins, import_uri, full_import_uri,', [ +@pytest.mark.parametrize('server_url, api_version, token_type, token_ins, full_import_uri,', [ ('https://galaxy.server.com/api/', 'v2', 'Token', GalaxyToken('my token'), - '1234', 'https://galaxy.server.com/api/v2/collection-imports/1234/'), ('https://galaxy.server.com/api/automation-hub/', 'v3', 'Bearer', KeycloakToken(auth_url='https://api.test/'), - '1234', 'https://galaxy.server.com/api/automation-hub/v3/imports/collections/1234/'), ]) -def test_wait_import_task_with_failure(server_url, api_version, token_type, token_ins, import_uri, full_import_uri, monkeypatch): +def test_wait_import_task_with_failure(server_url, api_version, token_type, token_ins, full_import_uri, monkeypatch): api = get_test_galaxy_api(server_url, api_version, token_ins=token_ins) mock_token_get = MagicMock() @@ -586,7 +580,7 @@ def test_wait_import_task_with_failure(server_url, api_version, token_type, toke expected = to_native(u'Galaxy import process failed: Becäuse I said so! (Code: GW001)') with pytest.raises(AnsibleError, match=re.escape(expected)): - api.wait_import_task(import_uri) + api.wait_import_task(full_import_uri) assert mock_open.call_count == 1 assert mock_open.mock_calls[0][1][0] == full_import_uri @@ -605,15 +599,13 @@ def test_wait_import_task_with_failure(server_url, api_version, token_type, toke assert mock_err.mock_calls[0][1][0] == u'Galaxy import error message: Somé error' -@pytest.mark.parametrize('server_url, api_version, token_type, token_ins, import_uri, full_import_uri', [ +@pytest.mark.parametrize('server_url, api_version, token_type, token_ins, full_import_uri', [ ('https://galaxy.server.com/api/', 'v2', 'Token', GalaxyToken('my_token'), - '1234', 'https://galaxy.server.com/api/v2/collection-imports/1234/'), ('https://galaxy.server.com/api/automation-hub/', 'v3', 'Bearer', KeycloakToken(auth_url='https://api.test/'), - '1234', 'https://galaxy.server.com/api/automation-hub/v3/imports/collections/1234/'), ]) -def test_wait_import_task_with_failure_no_error(server_url, api_version, token_type, token_ins, import_uri, full_import_uri, monkeypatch): +def test_wait_import_task_with_failure_no_error(server_url, api_version, token_type, token_ins, full_import_uri, monkeypatch): api = get_test_galaxy_api(server_url, api_version, token_ins=token_ins) mock_token_get = MagicMock() @@ -658,7 +650,7 @@ def test_wait_import_task_with_failure_no_error(server_url, api_version, token_t expected = 'Galaxy import process failed: Unknown error, see %s for more details \\(Code: UNKNOWN\\)' % full_import_uri with pytest.raises(AnsibleError, match=expected): - api.wait_import_task(import_uri) + api.wait_import_task(full_import_uri) assert mock_open.call_count == 1 assert mock_open.mock_calls[0][1][0] == full_import_uri @@ -677,15 +669,13 @@ def test_wait_import_task_with_failure_no_error(server_url, api_version, token_t assert mock_err.mock_calls[0][1][0] == u'Galaxy import error message: Somé error' -@pytest.mark.parametrize('server_url, api_version, token_type, token_ins, import_uri, full_import_uri', [ +@pytest.mark.parametrize('server_url, api_version, token_type, token_ins, full_import_uri', [ ('https://galaxy.server.com/api', 'v2', 'Token', GalaxyToken('my token'), - '1234', 'https://galaxy.server.com/api/v2/collection-imports/1234/'), ('https://galaxy.server.com/api/automation-hub', 'v3', 'Bearer', KeycloakToken(auth_url='https://api.test/'), - '1234', 'https://galaxy.server.com/api/automation-hub/v3/imports/collections/1234/'), ]) -def test_wait_import_task_timeout(server_url, api_version, token_type, token_ins, import_uri, full_import_uri, monkeypatch): +def test_wait_import_task_timeout(server_url, api_version, token_type, token_ins, full_import_uri, monkeypatch): api = get_test_galaxy_api(server_url, api_version, token_ins=token_ins) mock_token_get = MagicMock() @@ -709,7 +699,7 @@ def test_wait_import_task_timeout(server_url, api_version, token_type, token_ins expected = "Timeout while waiting for the Galaxy import process to finish, check progress at '%s'" % full_import_uri with pytest.raises(AnsibleError, match=expected): - api.wait_import_task(import_uri, 1) + api.wait_import_task(full_import_uri, 1) assert mock_open.call_count > 1 assert mock_open.mock_calls[0][1][0] == full_import_uri diff --git a/test/units/galaxy/test_collection.py b/test/units/galaxy/test_collection.py index 0c00c1884ba..7115a1da44f 100644 --- a/test/units/galaxy/test_collection.py +++ b/test/units/galaxy/test_collection.py @@ -884,7 +884,7 @@ def test_publish_with_wait(galaxy_server, collection_artifact, monkeypatch): assert mock_publish.mock_calls[0][1][0] == artifact_path assert mock_wait.call_count == 1 - assert mock_wait.mock_calls[0][1][0] == '1234' + assert mock_wait.mock_calls[0][1][0] == fake_import_uri assert mock_display.mock_calls[0][1][0] == "Collection has been published to the Galaxy server test_server %s" \ % galaxy_server.api_server From eb5e57a8f08479c623729e3c17cc5ce7b88dd43c Mon Sep 17 00:00:00 2001 From: Abhijeet Kasurde Date: Tue, 15 Jul 2025 21:21:17 -0700 Subject: [PATCH 25/68] six: bump version from 1.16 to 1.17 (#85478) Fixes: #85408 Signed-off-by: Abhijeet Kasurde Co-authored-by: Sloane Hertel <19572925+s-hertel@users.noreply.github.com> --- changelogs/fragments/six_1.7.0.yml | 3 +++ lib/ansible/module_utils/six/__init__.py | 17 +++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) create mode 100644 changelogs/fragments/six_1.7.0.yml diff --git a/changelogs/fragments/six_1.7.0.yml b/changelogs/fragments/six_1.7.0.yml new file mode 100644 index 00000000000..88793085b2a --- /dev/null +++ b/changelogs/fragments/six_1.7.0.yml @@ -0,0 +1,3 @@ +--- +minor_changes: + - six - bump six version from 1.16.0 to 1.17.0 (https://github.com/ansible/ansible/issues/85408). diff --git a/lib/ansible/module_utils/six/__init__.py b/lib/ansible/module_utils/six/__init__.py index 4e74af7c00e..20504cbeddb 100644 --- a/lib/ansible/module_utils/six/__init__.py +++ b/lib/ansible/module_utils/six/__init__.py @@ -3,7 +3,7 @@ # upstream vendored file that we're not going to modify on our own # pylint: disable=undefined-variable # -# Copyright (c) 2010-2020 Benjamin Peterson +# Copyright (c) 2010-2024 Benjamin Peterson # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -35,10 +35,10 @@ import types # The following makes it easier for us to script updates of the bundled code. It is not part of # upstream six -_BUNDLED_METADATA = {"pypi_name": "six", "version": "1.16.0"} +_BUNDLED_METADATA = {"pypi_name": "six", "version": "1.17.0"} __author__ = "Benjamin Peterson " -__version__ = "1.16.0" +__version__ = "1.17.0" # Useful for very coarse version differentiation. @@ -273,7 +273,7 @@ _moved_attributes = [ MovedAttribute("reduce", "__builtin__", "functools"), MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), MovedAttribute("StringIO", "StringIO", "io"), - MovedAttribute("UserDict", "UserDict", "collections"), + MovedAttribute("UserDict", "UserDict", "collections", "IterableUserDict", "UserDict"), MovedAttribute("UserList", "UserList", "collections"), MovedAttribute("UserString", "UserString", "collections"), MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), @@ -445,12 +445,17 @@ _urllib_request_moved_attributes = [ MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), MovedAttribute("urlretrieve", "urllib", "urllib.request"), MovedAttribute("urlcleanup", "urllib", "urllib.request"), - MovedAttribute("URLopener", "urllib", "urllib.request"), - MovedAttribute("FancyURLopener", "urllib", "urllib.request"), MovedAttribute("proxy_bypass", "urllib", "urllib.request"), MovedAttribute("parse_http_list", "urllib2", "urllib.request"), MovedAttribute("parse_keqv_list", "urllib2", "urllib.request"), ] +if sys.version_info[:2] < (3, 14): + _urllib_request_moved_attributes.extend( + [ + MovedAttribute("URLopener", "urllib", "urllib.request"), + MovedAttribute("FancyURLopener", "urllib", "urllib.request"), + ] + ) for attr in _urllib_request_moved_attributes: setattr(Module_six_moves_urllib_request, attr.name, attr) del attr From 38823665858c57ffd1fbc08c6acbead20d339c51 Mon Sep 17 00:00:00 2001 From: Jordan Borean Date: Fri, 18 Jul 2025 14:51:40 +1000 Subject: [PATCH 26/68] Add AnsiballZ debugging support with debugpy (#85476) * Add AnsiballZ debugging support with debugpy Adds support for debugging AnsiballZ modules with debugpy which is used by VSCode as its Python debugger DAP. Debugging can either be done through a manual Debugpy listening server through a launch.json configuration or through the new ansible-test --dev-debug-on-deman argument. * Fix up integration test * Simplify config option and move mypy ignore * Use new API if available and fix typo * Guard the import of debugpy * Fix sanity import issue * Minor cosmetic adjustments * Simplify debugger setup * ansible-test - Refactor debugging interface * Add ansible-test debug integration tests * Fix ansible-test shell when in unsupported dir --------- Co-authored-by: Matt Clay --- lib/ansible/_internal/_ansiballz/_builder.py | 39 +- lib/ansible/config/base.yml | 22 +- .../_ansiballz/_extensions/_debugpy.py | 97 +++++ .../_ansiballz/_extensions/_pydevd.py | 6 +- .../ansiballz_debugging/tasks/main.yml | 2 +- .../ansible-test-debugging-env/aliases | 2 + .../ansible-test-debugging-env/runme.sh | 4 + .../ansible-test-debugging-inventory/aliases | 2 + .../ansible-test-debugging-inventory/runme.sh | 4 + .../targets/ansible-test-debugging/aliases | 5 + .../ansible-test-debugging/tasks/main.yml | 98 +++++ .../_internal/commands/shell/__init__.py | 95 +++-- test/lib/ansible_test/_internal/debugging.py | 386 +++++++++++++++--- .../ansible_test/_internal/host_profiles.py | 86 ++-- test/lib/ansible_test/_internal/metadata.py | 49 +-- .../_internal/python_requirements.py | 4 +- .../sanity/pylint/config/ansible-test.cfg | 1 + test/sanity/code-smell/mypy/ansible-core.ini | 3 + test/sanity/code-smell/mypy/ansible-test.ini | 6 + 19 files changed, 722 insertions(+), 189 deletions(-) create mode 100644 lib/ansible/module_utils/_internal/_ansiballz/_extensions/_debugpy.py create mode 100644 test/integration/targets/ansible-test-debugging-env/aliases create mode 100755 test/integration/targets/ansible-test-debugging-env/runme.sh create mode 100644 test/integration/targets/ansible-test-debugging-inventory/aliases create mode 100755 test/integration/targets/ansible-test-debugging-inventory/runme.sh create mode 100644 test/integration/targets/ansible-test-debugging/aliases create mode 100644 test/integration/targets/ansible-test-debugging/tasks/main.yml diff --git a/lib/ansible/_internal/_ansiballz/_builder.py b/lib/ansible/_internal/_ansiballz/_builder.py index eff6392904c..76c756fe195 100644 --- a/lib/ansible/_internal/_ansiballz/_builder.py +++ b/lib/ansible/_internal/_ansiballz/_builder.py @@ -6,7 +6,7 @@ import json import typing as t from ansible.module_utils._internal._ansiballz import _extensions -from ansible.module_utils._internal._ansiballz._extensions import _pydevd, _coverage +from ansible.module_utils._internal._ansiballz._extensions import _debugpy, _pydevd, _coverage from ansible.constants import config _T = t.TypeVar('_T') @@ -17,15 +17,18 @@ class ExtensionManager: def __init__( self, - debugger: _pydevd.Options | None = None, + pydevd: _pydevd.Options | None = None, + debugpy: _debugpy.Options | None = None, coverage: _coverage.Options | None = None, ) -> None: options = dict( - _pydevd=debugger, + _pydevd=pydevd, + _debugpy=debugpy, _coverage=coverage, ) - self._debugger = debugger + self._pydevd = pydevd + self._debugpy = debugpy self._coverage = coverage self._extension_names = tuple(name for name, option in options.items() if option) self._module_names = tuple(f'{_extensions.__name__}.{name}' for name in self._extension_names) @@ -35,7 +38,7 @@ class ExtensionManager: @property def debugger_enabled(self) -> bool: """Returns True if the debugger extension is enabled, otherwise False.""" - return bool(self._debugger) + return bool(self._pydevd or self._debugpy) @property def extension_names(self) -> tuple[str, ...]: @@ -51,10 +54,16 @@ class ExtensionManager: """Return the configured extensions and their options.""" extension_options: dict[str, t.Any] = {} - if self._debugger: + if self._debugpy: + extension_options['_debugpy'] = dataclasses.replace( + self._debugpy, + source_mapping=self._get_source_mapping(self._debugpy.source_mapping), + ) + + if self._pydevd: extension_options['_pydevd'] = dataclasses.replace( - self._debugger, - source_mapping=self._get_source_mapping(), + self._pydevd, + source_mapping=self._get_source_mapping(self._pydevd.source_mapping), ) if self._coverage: @@ -64,18 +73,19 @@ class ExtensionManager: return extensions - def _get_source_mapping(self) -> dict[str, str]: + def _get_source_mapping(self, debugger_mapping: dict[str, str]) -> dict[str, str]: """Get the source mapping, adjusting the source root as needed.""" - if self._debugger.source_mapping: - source_mapping = {self._translate_path(key): value for key, value in self.source_mapping.items()} + if debugger_mapping: + source_mapping = {self._translate_path(key, debugger_mapping): value for key, value in self.source_mapping.items()} else: source_mapping = self.source_mapping return source_mapping - def _translate_path(self, path: str) -> str: + @staticmethod + def _translate_path(path: str, debugger_mapping: dict[str, str]) -> str: """Translate a local path to a foreign path.""" - for replace, match in self._debugger.source_mapping.items(): + for replace, match in debugger_mapping.items(): if path.startswith(match): return replace + path[len(match) :] @@ -85,7 +95,8 @@ class ExtensionManager: def create(cls, task_vars: dict[str, object]) -> t.Self: """Create an instance using the provided task vars.""" return cls( - debugger=cls._get_options('_ANSIBALLZ_DEBUGGER_CONFIG', _pydevd.Options, task_vars), + pydevd=cls._get_options('_ANSIBALLZ_PYDEVD_CONFIG', _pydevd.Options, task_vars), + debugpy=cls._get_options('_ANSIBALLZ_DEBUGPY_CONFIG', _debugpy.Options, task_vars), coverage=cls._get_options('_ANSIBALLZ_COVERAGE_CONFIG', _coverage.Options, task_vars), ) diff --git a/lib/ansible/config/base.yml b/lib/ansible/config/base.yml index 3eeb0250f6c..ad28844b8c2 100644 --- a/lib/ansible/config/base.yml +++ b/lib/ansible/config/base.yml @@ -11,16 +11,26 @@ _ANSIBALLZ_COVERAGE_CONFIG: vars: - {name: _ansible_ansiballz_coverage_config} version_added: '2.19' -_ANSIBALLZ_DEBUGGER_CONFIG: - name: Configure the AnsiballZ remote debugging extension +_ANSIBALLZ_DEBUGPY_CONFIG: + name: Configure the AnsiballZ remote debugging extension for debugpy description: - - Enables and configures the AnsiballZ remote debugging extension. + - Enables and configures the AnsiballZ remote debugging extension for debugpy. - This is for internal use only. env: - - {name: _ANSIBLE_ANSIBALLZ_DEBUGGER_CONFIG} + - {name: _ANSIBLE_ANSIBALLZ_DEBUGPY_CONFIG} vars: - - {name: _ansible_ansiballz_debugger_config} - version_added: '2.19' + - {name: _ansible_ansiballz_debugpy_config} + version_added: '2.20' +_ANSIBALLZ_PYDEVD_CONFIG: + name: Configure the AnsiballZ remote debugging extension for pydevd + description: + - Enables and configures the AnsiballZ remote debugging extension for pydevd. + - This is for internal use only. + env: + - {name: _ANSIBLE_ANSIBALLZ_PYDEVD_CONFIG} + vars: + - {name: _ansible_ansiballz_pydevd_config} + version_added: '2.20' _ANSIBLE_CONNECTION_PATH: env: - name: _ANSIBLE_CONNECTION_PATH diff --git a/lib/ansible/module_utils/_internal/_ansiballz/_extensions/_debugpy.py b/lib/ansible/module_utils/_internal/_ansiballz/_extensions/_debugpy.py new file mode 100644 index 00000000000..df411962e2c --- /dev/null +++ b/lib/ansible/module_utils/_internal/_ansiballz/_extensions/_debugpy.py @@ -0,0 +1,97 @@ +""" +Remote debugging support for AnsiballZ modules with debugpy. + +To use with VS Code: + +1) Choose an available port for VS Code to listen on (e.g. 5678). +2) Ensure `debugpy` is installed for the interpreter(s) which will run the code being debugged. +3) Create the following launch.json configuration + + { + "version": "0.2.0", + "configurations": [ + { + "name": "Python Debug Server", + "type": "debugpy", + "request": "attach", + "listen": { + "host": "localhost", + "port": 5678, + }, + }, + { + "name": "ansible-playbook main.yml", + "type": "debugpy", + "request": "launch", + "module": "ansible", + "args": [ + "playbook", + "main.yml" + ], + "env": { + "_ANSIBLE_ANSIBALLZ_DEBUGPY_CONFIG": "{\"host\": \"localhost\", \"port\": 5678}" + }, + "console": "integratedTerminal", + } + ], + "compounds": [ + { + "name": "Test Module Debugging", + "configurations": [ + "Python Debug Server", + "ansible-playbook main.yml" + ], + "stopAll": true + } + ] + } + +4) Set any desired breakpoints. +5) Configure the Run and Debug view to use the "Test Module Debugging" compound configuration. +6) Press F5 to start debugging. +""" + +from __future__ import annotations + +import dataclasses +import json +import os +import pathlib + +import typing as t + + +@dataclasses.dataclass(frozen=True) +class Options: + """Debugger options for debugpy.""" + + host: str = 'localhost' + """The host to connect to for remote debugging.""" + port: int = 5678 + """The port to connect to for remote debugging.""" + connect: dict[str, object] = dataclasses.field(default_factory=dict) + """The options to pass to the `debugpy.connect` method.""" + source_mapping: dict[str, str] = dataclasses.field(default_factory=dict) + """ + A mapping of source paths to provide to debugpy. + This setting is used internally by AnsiballZ and is not required unless Ansible CLI commands are run from a different system than your IDE. + In that scenario, use this setting instead of configuring source mapping in your IDE. + The key is a path known to the IDE. + The value is the same path as known to the Ansible CLI. + Both file paths and directories are supported. + """ + + +def run(args: dict[str, t.Any]) -> None: # pragma: nocover + """Enable remote debugging.""" + import debugpy + + options = Options(**args) + temp_dir = pathlib.Path(__file__).parent.parent.parent.parent.parent.parent + path_mapping = [[key, str(temp_dir / value)] for key, value in options.source_mapping.items()] + + os.environ['PATHS_FROM_ECLIPSE_TO_PYTHON'] = json.dumps(path_mapping) + + debugpy.connect((options.host, options.port), **options.connect) + + pass # A convenient place to put a breakpoint diff --git a/lib/ansible/module_utils/_internal/_ansiballz/_extensions/_pydevd.py b/lib/ansible/module_utils/_internal/_ansiballz/_extensions/_pydevd.py index eec234d9d10..c82232d91a4 100644 --- a/lib/ansible/module_utils/_internal/_ansiballz/_extensions/_pydevd.py +++ b/lib/ansible/module_utils/_internal/_ansiballz/_extensions/_pydevd.py @@ -7,14 +7,12 @@ To use with PyCharm: 2) Create a Python Debug Server using that port. 3) Start the Python Debug Server. 4) Ensure the correct version of `pydevd-pycharm` is installed for the interpreter(s) which will run the code being debugged. -5) Configure Ansible with the `_ANSIBALLZ_DEBUGGER_CONFIG` option. +5) Configure Ansible with the `_ANSIBALLZ_PYDEVD_CONFIG` option. See `Options` below for the structure of the debugger configuration. Example configuration using an environment variable: - export _ANSIBLE_ANSIBALLZ_DEBUGGER_CONFIG='{"module": "pydevd_pycharm", "settrace": {"host": "localhost", "port": 5678, "suspend": false}}' + export _ANSIBLE_ANSIBALLZ_PYDEVD_CONFIG='{"module": "pydevd_pycharm", "settrace": {"host": "localhost", "port": 5678, "suspend": false}}' 6) Set any desired breakpoints. 7) Run Ansible commands. - -A similar process should work for other pydevd based debuggers, such as Visual Studio Code, but they have not been tested. """ from __future__ import annotations diff --git a/test/integration/targets/ansiballz_debugging/tasks/main.yml b/test/integration/targets/ansiballz_debugging/tasks/main.yml index 5d052ed967c..9d8dd7e248b 100644 --- a/test/integration/targets/ansiballz_debugging/tasks/main.yml +++ b/test/integration/targets/ansiballz_debugging/tasks/main.yml @@ -1,7 +1,7 @@ - name: Run a module with remote debugging configured to use a bogus debugger module ping: vars: - _ansible_ansiballz_debugger_config: + _ansible_ansiballz_pydevd_config: module: not_a_valid_debugger_module register: result ignore_errors: yes diff --git a/test/integration/targets/ansible-test-debugging-env/aliases b/test/integration/targets/ansible-test-debugging-env/aliases new file mode 100644 index 00000000000..bf60838a88a --- /dev/null +++ b/test/integration/targets/ansible-test-debugging-env/aliases @@ -0,0 +1,2 @@ +shippable/generic/group1 # runs in the default test container +context/controller diff --git a/test/integration/targets/ansible-test-debugging-env/runme.sh b/test/integration/targets/ansible-test-debugging-env/runme.sh new file mode 100755 index 00000000000..1ec9edd7ad8 --- /dev/null +++ b/test/integration/targets/ansible-test-debugging-env/runme.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Used to support the ansible-test-debugging integration test. + +env diff --git a/test/integration/targets/ansible-test-debugging-inventory/aliases b/test/integration/targets/ansible-test-debugging-inventory/aliases new file mode 100644 index 00000000000..bf60838a88a --- /dev/null +++ b/test/integration/targets/ansible-test-debugging-inventory/aliases @@ -0,0 +1,2 @@ +shippable/generic/group1 # runs in the default test container +context/controller diff --git a/test/integration/targets/ansible-test-debugging-inventory/runme.sh b/test/integration/targets/ansible-test-debugging-inventory/runme.sh new file mode 100755 index 00000000000..48d4c626190 --- /dev/null +++ b/test/integration/targets/ansible-test-debugging-inventory/runme.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Used to support the ansible-test-debugging integration test. + +cat "${INVENTORY_PATH}" diff --git a/test/integration/targets/ansible-test-debugging/aliases b/test/integration/targets/ansible-test-debugging/aliases new file mode 100644 index 00000000000..69cffa10864 --- /dev/null +++ b/test/integration/targets/ansible-test-debugging/aliases @@ -0,0 +1,5 @@ +shippable/generic/group1 # runs in the default test container +needs/target/ansible-test-debugging-env # indirectly used by ansible-test, included here for change detection +needs/target/ansible-test-debugging-inventory # indirectly used by ansible-test, included here for change detection +context/controller +gather_facts/no diff --git a/test/integration/targets/ansible-test-debugging/tasks/main.yml b/test/integration/targets/ansible-test-debugging/tasks/main.yml new file mode 100644 index 00000000000..f91159441b5 --- /dev/null +++ b/test/integration/targets/ansible-test-debugging/tasks/main.yml @@ -0,0 +1,98 @@ +- name: Define supported debuggers + set_fact: + debuggers: + - pydevd + - debugpy + +- name: Run without debugging features enabled + command: ansible-test shell -v -- env + register: result + +- assert: + that: + - result.stderr_lines is not contains 'Debugging' + +- name: Run a command which does not support debugging + command: ansible-test env -v + register: result + +- assert: + that: + - result.stderr_lines is not contains 'Debugging' + +- name: Verify on-demand debugging gracefully handles not running under a debugger + command: ansible-test shell -v --dev-debug-on-demand -- env + register: result + +- assert: + that: + - result.stderr_lines is contains 'Debugging disabled because no debugger was detected.' + +- name: Verify manual debugging gracefully handles lack of configuration + command: ansible-test shell -v --dev-debug-cli -- env + register: result + +- assert: + that: + - result.stderr_lines is contains 'Debugging disabled because no debugger configuration was provided.' + +- name: Verify invalid debugger configuration is handled + command: ansible-test shell --dev-debug-cli -- env + environment: > + {% set key = "ANSIBLE_TEST_REMOTE_DEBUGGER_" + item.upper() %}{{ ('{"' + key + '": "{\"invalid_key\": true}"}') | from_json }} + register: result + loop: "{{ debuggers }}" + ignore_errors: yes + +- assert: + that: + - item.stderr is search("Invalid " + item.item + " settings.*invalid_key") + loop: "{{ result.results }}" + +- name: Verify CLI debugger can be manually enabled (shell) + command: ansible-test shell --dev-debug-cli -- env + environment: > + {% set key = "ANSIBLE_TEST_REMOTE_DEBUGGER_" + item.upper() %}{{ ('{"' + key + '": ""}') | from_json }} + register: result + loop: "{{ debuggers }}" + +- assert: + that: + - item.stdout is contains "ANSIBLE_TEST_DEBUGGER_CONFIG" + loop: "{{ result.results }}" + +- name: Verify CLI debugger can be manually enabled (integration) + command: ansible-test integration ansible-test-debugging-env --dev-debug-cli + environment: > + {% set key = "ANSIBLE_TEST_REMOTE_DEBUGGER_" + item.upper() %}{{ ('{"' + key + '": ""}') | from_json }} + register: result + loop: "{{ debuggers }}" + +- assert: + that: + - item.stdout is contains "ANSIBLE_TEST_DEBUGGER_CONFIG" + loop: "{{ result.results }}" + +- name: Verify AnsiballZ debugger can be manually enabled (shell) + command: ansible-test shell --dev-debug-ansiballz -- env + environment: > + {% set key = "ANSIBLE_TEST_REMOTE_DEBUGGER_" + item.upper() %}{{ ('{"' + key + '": ""}') | from_json }} + register: result + loop: "{{ debuggers }}" + +- assert: + that: + - item.stdout is contains("_ANSIBLE_ANSIBALLZ_" + item.item.upper() + "_CONFIG") + loop: "{{ result.results }}" + +- name: Verify AnsiballZ debugger can be manually enabled (integration) + command: ansible-test integration ansible-test-debugging-inventory --dev-debug-ansiballz + environment: > + {% set key = "ANSIBLE_TEST_REMOTE_DEBUGGER_" + item.upper() %}{{ ('{"' + key + '": ""}') | from_json }} + register: result + loop: "{{ debuggers }}" + +- assert: + that: + - item.stdout is contains("_ansible_ansiballz_" + item.item + "_config") + loop: "{{ result.results }}" diff --git a/test/lib/ansible_test/_internal/commands/shell/__init__.py b/test/lib/ansible_test/_internal/commands/shell/__init__.py index dc14b1659b7..89fbd8d72c8 100644 --- a/test/lib/ansible_test/_internal/commands/shell/__init__.py +++ b/test/lib/ansible_test/_internal/commands/shell/__init__.py @@ -2,11 +2,16 @@ from __future__ import annotations +import contextlib import dataclasses import os import sys import typing as t +from ...data import ( + data_context, +) + from ...util import ( ApplicationError, OutputStream, @@ -101,19 +106,33 @@ def command_shell(args: ShellConfig) -> None: if args.export: return - if args.cmd: - # Running a command is assumed to be non-interactive. Only a shell (no command) is interactive. - # If we want to support interactive commands in the future, we'll need an `--interactive` command line option. - # Command stderr output is allowed to mix with our own output, which is all sent to stderr. - con.run(args.cmd, capture=False, interactive=False, output_stream=OutputStream.ORIGINAL) - return - if isinstance(con, LocalConnection) and isinstance(target_profile, DebuggableProfile) and target_profile.debugging_enabled: - # HACK: ensure the pydevd port visible in the shell is the forwarded port, not the original - args.metadata.debugger_settings = dataclasses.replace(args.metadata.debugger_settings, port=target_profile.pydevd_port) + # HACK: ensure the debugger port visible in the shell is the forwarded port, not the original + args.metadata.debugger_settings = dataclasses.replace(args.metadata.debugger_settings, port=target_profile.debugger_port) - with metadata_context(args): - interactive_shell(args, target_profile, con) + with contextlib.nullcontext() if data_context().content.unsupported else metadata_context(args): + if args.cmd: + non_interactive_shell(args, target_profile, con) + else: + interactive_shell(args, target_profile, con) + + +def non_interactive_shell( + args: ShellConfig, + target_profile: SshTargetHostProfile, + con: Connection, +) -> None: + """Run a non-interactive shell command.""" + if isinstance(target_profile, PosixProfile): + env = get_environment_variables(args, target_profile, con) + cmd = get_env_command(env) + args.cmd + else: + cmd = args.cmd + + # Running a command is assumed to be non-interactive. Only a shell (no command) is interactive. + # If we want to support interactive commands in the future, we'll need an `--interactive` command line option. + # Command stderr output is allowed to mix with our own output, which is all sent to stderr. + con.run(cmd, capture=False, interactive=False, output_stream=OutputStream.ORIGINAL) def interactive_shell( @@ -135,23 +154,8 @@ def interactive_shell( python = target_profile.python # make sure the python interpreter has been initialized before opening a shell display.info(f'Target Python {python.version} is at: {python.path}') - optional_vars = ( - 'TERM', # keep backspace working - ) - - env = {name: os.environ[name] for name in optional_vars if name in os.environ} - - if isinstance(con, LocalConnection): # configure the controller environment - env.update(ansible_environment(args)) - env.update(get_injector_env(target_profile.python, env)) - env.update(ANSIBLE_TEST_METADATA_PATH=os.path.abspath(args.metadata_path)) - - if isinstance(target_profile, DebuggableProfile): - env.update(target_profile.get_ansiballz_environment_variables()) - env.update(target_profile.get_ansible_cli_environment_variables()) - - if env: - cmd = ['/usr/bin/env'] + [f'{name}={value}' for name, value in env.items()] + env = get_environment_variables(args, target_profile, con) + cmd = get_env_command(env) cmd += [shell, '-i'] else: @@ -175,3 +179,38 @@ def interactive_shell( raise HostConnectionError(f'SSH shell connection failed for host {target_profile.config}: {ex}', callback) from ex raise + + +def get_env_command(env: dict[str, str]) -> list[str]: + """Get an `env` command to set the given environment variables, if any.""" + if not env: + return [] + + return ['/usr/bin/env'] + [f'{name}={value}' for name, value in env.items()] + + +def get_environment_variables( + args: ShellConfig, + target_profile: PosixProfile, + con: Connection, +) -> dict[str, str]: + """Get the environment variables to expose to the shell.""" + if data_context().content.unsupported: + return {} + + optional_vars = ( + 'TERM', # keep backspace working + ) + + env = {name: os.environ[name] for name in optional_vars if name in os.environ} + + if isinstance(con, LocalConnection): # configure the controller environment + env.update(ansible_environment(args)) + env.update(get_injector_env(target_profile.python, env)) + env.update(ANSIBLE_TEST_METADATA_PATH=os.path.abspath(args.metadata_path)) + + if isinstance(target_profile, DebuggableProfile): + env.update(target_profile.get_ansiballz_environment_variables()) + env.update(target_profile.get_ansible_cli_environment_variables()) + + return env diff --git a/test/lib/ansible_test/_internal/debugging.py b/test/lib/ansible_test/_internal/debugging.py index bd5fc452ad9..b3c4a605ec8 100644 --- a/test/lib/ansible_test/_internal/debugging.py +++ b/test/lib/ansible_test/_internal/debugging.py @@ -2,16 +2,25 @@ from __future__ import annotations +import abc import dataclasses +import importlib import json import os import re +import sys +import typing as t from .util import ( cache, display, raw_command, ApplicationError, + get_subclasses, +) + +from .util_common import ( + CommonConfig, ) from .processes import ( @@ -24,76 +33,308 @@ from .config import ( ) from .metadata import ( - DebuggerSettings, DebuggerFlags, ) -from . import ( +from .data import ( data_context, - CommonConfig, ) -def initialize_debugger(args: CommonConfig) -> None: - """Initialize the debugger settings before delegation.""" - if not isinstance(args, EnvironmentConfig): - return +class DebuggerProfile(t.Protocol): + """Protocol for debugger profiles.""" - if args.metadata.loaded: - return # after delegation + @property + def debugger_host(self) -> str: + """The hostname to expose to the debugger.""" - if collection := data_context().content.collection: - args.metadata.collection_root = collection.root - - load_debugger_settings(args) + @property + def debugger_port(self) -> int: + """The port to expose to the debugger.""" + def get_source_mapping(self) -> dict[str, str]: + """The source mapping to expose to the debugger.""" + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class DebuggerSettings(metaclass=abc.ABCMeta): + """Common debugger settings.""" + + port: int = 5678 + """ + The port on the origin host which is listening for incoming connections from the debugger. + SSH port forwarding will be automatically configured for non-local hosts to connect to this port as needed. + """ + + def as_dict(self) -> dict[str, object]: + """Convert this instance to a dict.""" + data = dataclasses.asdict(self) + data.update(__type__=self.__class__.__name__) + + return data + + @classmethod + def from_dict(cls, value: dict[str, t.Any]) -> t.Self: + """Load an instance from a dict.""" + debug_cls = globals()[value.pop('__type__')] + + return debug_cls(**value) + + @classmethod + def get_debug_type(cls) -> str: + """Return the name for this debugger.""" + return cls.__name__.removesuffix('Settings').lower() + + @classmethod + def get_config_env_var_name(cls) -> str: + """Return the name of the environment variable used to customize settings for this debugger.""" + return f'ANSIBLE_TEST_REMOTE_DEBUGGER_{cls.get_debug_type().upper()}' + + @classmethod + def parse(cls, value: str) -> t.Self: + """Parse debugger settings from the given JSON and apply defaults.""" + try: + settings = cls(**json.loads(value)) + except Exception as ex: + raise ApplicationError(f"Invalid {cls.get_debug_type()} settings: {ex}") from ex + + return cls.apply_defaults(settings) + + @classmethod + @abc.abstractmethod + def is_active(cls) -> bool: + """Detect if the debugger is active.""" + + @classmethod + @abc.abstractmethod + def apply_defaults(cls, settings: t.Self) -> t.Self: + """Apply defaults to the given settings.""" + + @abc.abstractmethod + def get_python_package(self) -> str: + """The Python package to install for debugging.""" + + @abc.abstractmethod + def activate_debugger(self, profile: DebuggerProfile) -> None: + """Activate the debugger in ansible-test after delegation.""" + + @abc.abstractmethod + def get_ansiballz_config(self, profile: DebuggerProfile) -> dict[str, object]: + """Gets the extra configuration data for the AnsiballZ extension module.""" + + @abc.abstractmethod + def get_cli_arguments(self, profile: DebuggerProfile) -> list[str]: + """Get command line arguments for the debugger when running Ansible CLI programs.""" + + @abc.abstractmethod + def get_environment_variables(self, profile: DebuggerProfile) -> dict[str, str]: + """Get environment variables needed to configure the debugger for debugging.""" + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class PydevdSettings(DebuggerSettings): + """Settings for the pydevd debugger.""" + + package: str | None = None + """ + The Python package to install for debugging. + If `None` then the package will be auto-detected. + If an empty string, then no package will be installed. + """ + + module: str | None = None + """ + The Python module to import for debugging. + This should be pydevd or a derivative. + If not provided it will be auto-detected. + """ + + settrace: dict[str, object] = dataclasses.field(default_factory=dict) + """ + Options to pass to the `{module}.settrace` method. + Used for running AnsiballZ modules only. + The `host` and `port` options will be provided by ansible-test. + The `suspend` option defaults to `False`. + """ + + args: list[str] = dataclasses.field(default_factory=list) + """ + Arguments to pass to `pydevd` on the command line. + Used for running Ansible CLI programs only. + The `--client` and `--port` options will be provided by ansible-test. + """ + + @classmethod + def is_active(cls) -> bool: + return detect_pydevd_port() is not None + + @classmethod + def apply_defaults(cls, settings: t.Self) -> t.Self: + if not settings.module: + if not settings.package or 'pydevd-pycharm' in settings.package: + module = 'pydevd_pycharm' + else: + module = 'pydevd' -def parse_debugger_settings(value: str) -> DebuggerSettings: - """Parse remote debugger settings and apply defaults.""" - try: - settings = DebuggerSettings(**json.loads(value)) - except Exception as ex: - raise ApplicationError(f"Invalid debugger settings: {ex}") from ex + settings = dataclasses.replace(settings, module=module) - if not settings.module: - if not settings.package or 'pydevd-pycharm' in settings.package: - module = 'pydevd_pycharm' + if settings.package is None: + if settings.module == 'pydevd_pycharm': + if pycharm_version := detect_pycharm_version(): + package = f'pydevd-pycharm~={pycharm_version}' + else: + package = None + else: + package = 'pydevd' + + settings = dataclasses.replace(settings, package=package) + + settings.settrace.setdefault('suspend', False) + + if port := detect_pydevd_port(): + settings = dataclasses.replace(settings, port=port) + + if detect_pycharm_process(): + # This only works with the default PyCharm debugger. + # Using it with PyCharm's "Python Debug Server" results in hangs in Ansible workers. + # Further investigation is required to understand the cause. + settings = dataclasses.replace(settings, args=settings.args + ['--multiprocess']) + + return settings + + def get_python_package(self) -> str: + if self.package is None and self.module == 'pydevd_pycharm': + display.warning('Skipping installation of `pydevd-pycharm` since the running PyCharm version was not detected.') + + return self.package + + def activate_debugger(self, profile: DebuggerProfile) -> None: + debugging_module = importlib.import_module(self.module) + debugging_module.settrace(**self._get_settrace_arguments(profile)) + + def get_ansiballz_config(self, profile: DebuggerProfile) -> dict[str, object]: + return dict( + module=self.module, + settrace=self._get_settrace_arguments(profile), + source_mapping=profile.get_source_mapping(), + ) + + def get_cli_arguments(self, profile: DebuggerProfile) -> list[str]: + # Although `pydevd_pycharm` can be used to invoke `settrace`, it cannot be used to run the debugger on the command line. + return ['-m', 'pydevd', '--client', profile.debugger_host, '--port', str(profile.debugger_port)] + self.args + ['--file'] + + def get_environment_variables(self, profile: DebuggerProfile) -> dict[str, str]: + return dict( + PATHS_FROM_ECLIPSE_TO_PYTHON=json.dumps(list(profile.get_source_mapping().items())), + PYDEVD_DISABLE_FILE_VALIDATION="1", + ) + + def _get_settrace_arguments(self, profile: DebuggerProfile) -> dict[str, object]: + """Get settrace arguments for pydevd.""" + return self.settrace | dict( + host=profile.debugger_host, + port=profile.debugger_port, + ) + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class DebugpySettings(DebuggerSettings): + """Settings for the debugpy debugger.""" + + connect: dict[str, object] = dataclasses.field(default_factory=dict) + """ + Options to pass to the `debugpy.connect` method. + Used for running AnsiballZ modules and ansible-test after delegation. + The endpoint addr, `access_token`, and `parent_session_pid` options will be provided by ansible-test. + """ + + args: list[str] = dataclasses.field(default_factory=list) + """ + Arguments to pass to `debugpy` on the command line. + Used for running Ansible CLI programs only. + The `--connect`, `--adapter-access-token`, and `--parent-session-pid` options will be provided by ansible-test. + """ + + @classmethod + def is_active(cls) -> bool: + return detect_debugpy_options() is not None + + @classmethod + def apply_defaults(cls, settings: t.Self) -> t.Self: + if options := detect_debugpy_options(): + settings = dataclasses.replace(settings, port=options.port) + settings.connect.update( + access_token=options.adapter_access_token, + parent_session_pid=os.getpid(), + ) else: - module = 'pydevd' + display.warning('Debugging will be limited to the first connection. Run ansible-test under debugpy to support multiple connections.') - settings = dataclasses.replace(settings, module=module) + return settings - if settings.package is None: - if settings.module == 'pydevd_pycharm': - if pycharm_version := detect_pycharm_version(): - package = f'pydevd-pycharm~={pycharm_version}' - else: - package = None - else: - package = 'pydevd' + def get_python_package(self) -> str: + return 'debugpy' + + def activate_debugger(self, profile: DebuggerProfile) -> None: + import debugpy # pylint: disable=import-error + + debugpy.connect((profile.debugger_host, profile.debugger_port), **self.connect) + + def get_ansiballz_config(self, profile: DebuggerProfile) -> dict[str, object]: + return dict( + host=profile.debugger_host, + port=profile.debugger_port, + connect=self.connect, + source_mapping=profile.get_source_mapping(), + ) + + def get_cli_arguments(self, profile: DebuggerProfile) -> list[str]: + cli_args = ['-m', 'debugpy', '--connect', f"{profile.debugger_host}:{profile.debugger_port}"] + + if access_token := self.connect.get('access_token'): + cli_args += ['--adapter-access-token', str(access_token)] + + if session_pid := self.connect.get('parent_session_pid'): + cli_args += ['--parent-session-pid', str(session_pid)] + + if self.args: + cli_args += self.args - settings = dataclasses.replace(settings, package=package) + return cli_args - settings.settrace.setdefault('suspend', False) + def get_environment_variables(self, profile: DebuggerProfile) -> dict[str, str]: + return dict( + PATHS_FROM_ECLIPSE_TO_PYTHON=json.dumps(list(profile.get_source_mapping().items())), + PYDEVD_DISABLE_FILE_VALIDATION="1", + ) - if port := detect_pydevd_port(): - settings = dataclasses.replace(settings, port=port) - if detect_pycharm_process(): - # This only works with the default PyCharm debugger. - # Using it with PyCharm's "Python Debug Server" results in hangs in Ansible workers. - # Further investigation is required to understand the cause. - settings = dataclasses.replace(settings, args=settings.args + ['--multiprocess']) +def initialize_debugger(args: CommonConfig) -> None: + """Initialize the debugger settings before delegation.""" + if not isinstance(args, EnvironmentConfig): + return - return settings + if args.metadata.loaded: + return # after delegation + + if collection := data_context().content.collection: + args.metadata.collection_root = collection.root + + load_debugger_settings(args) def load_debugger_settings(args: EnvironmentConfig) -> None: """Load the remote debugger settings.""" + use_debugger: type[DebuggerSettings] | None = None + if args.metadata.debugger_flags.on_demand: # On-demand debugging only enables debugging if we're running under a debugger, otherwise it's a no-op. - if not detect_pydevd_port(): + for candidate_debugger in get_subclasses(DebuggerSettings): + if candidate_debugger.is_active(): + use_debugger = candidate_debugger + break + else: display.info('Debugging disabled because no debugger was detected.', verbosity=1) args.metadata.debugger_flags = DebuggerFlags.all(False) return @@ -107,13 +348,22 @@ def load_debugger_settings(args: EnvironmentConfig) -> None: if not args.metadata.debugger_flags.enable: return - value = os.environ.get('ANSIBLE_TEST_REMOTE_DEBUGGER') or '{}' - settings = parse_debugger_settings(value) - - display.info(f'>>> Debugger Settings\n{json.dumps(dataclasses.asdict(settings), indent=4)}', verbosity=3) + if not use_debugger: # detect debug type based on env var + for candidate_debugger in get_subclasses(DebuggerSettings): + if candidate_debugger.get_config_env_var_name() in os.environ: + use_debugger = candidate_debugger + break + else: + display.info('Debugging disabled because no debugger configuration was provided.', verbosity=1) + args.metadata.debugger_flags = DebuggerFlags.all(False) + return + config = os.environ.get(use_debugger.get_config_env_var_name()) or '{}' + settings = use_debugger.parse(config) args.metadata.debugger_settings = settings + display.info(f'>>> Debugger Settings ({use_debugger.get_debug_type()})\n{json.dumps(dataclasses.asdict(settings), indent=4)}', verbosity=3) + @cache def detect_pydevd_port() -> int | None: @@ -140,8 +390,6 @@ def detect_pycharm_version() -> str | None: display.info(f'Detected PyCharm version {version}.', verbosity=1) return version - display.warning('Skipping installation of `pydevd-pycharm` since the running PyCharm version could not be detected.') - return None @@ -164,3 +412,43 @@ def detect_pycharm_process() -> Process | None: def get_current_process_cached() -> Process: """Return the current process. The result is cached.""" return get_current_process() + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class DebugpyOptions: + """Options detected from the debugpy instance hosting this process.""" + + port: int + adapter_access_token: str | None + + +@cache +def detect_debugpy_options() -> DebugpyOptions | None: + """Return the options for the debugpy instance hosting this process, or `None` if not detected.""" + if "debugpy" not in sys.modules: + return None + + import debugpy # pylint: disable=import-error + + # get_cli_options is the new public API introduced after debugpy 1.8.15. + # We should remove the debugpy.server cli fallback once the new version is + # released. + if hasattr(debugpy, 'get_cli_options'): + opts = debugpy.get_cli_options() + else: + from debugpy.server import cli # pylint: disable=import-error + opts = cli.options + + # address can be None if the debugger is not configured through the CLI as + # we expected. + if not opts.address: + return None + + port = opts.address[1] + + display.info(f'Detected debugpy debugger port {port}.', verbosity=1) + + return DebugpyOptions( + port=port, + adapter_access_token=opts.adapter_access_token, + ) diff --git a/test/lib/ansible_test/_internal/host_profiles.py b/test/lib/ansible_test/_internal/host_profiles.py index dc28b66c17a..f8d5fbf1e19 100644 --- a/test/lib/ansible_test/_internal/host_profiles.py +++ b/test/lib/ansible_test/_internal/host_profiles.py @@ -4,7 +4,6 @@ from __future__ import annotations import abc import dataclasses -import importlib import json import os import pathlib @@ -140,6 +139,11 @@ from .dev.container_probe import ( check_container_cgroup_status, ) +from .debugging import ( + DebuggerProfile, + DebuggerSettings, +) + TControllerHostConfig = t.TypeVar('TControllerHostConfig', bound=ControllerHostConfig) THostConfig = t.TypeVar('THostConfig', bound=HostConfig) TPosixConfig = t.TypeVar('TPosixConfig', bound=PosixConfig) @@ -292,12 +296,17 @@ class HostProfile(t.Generic[THostConfig], metaclass=abc.ABCMeta): return f'{self.__class__.__name__}: {self.name}' -class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta): +class DebuggableProfile(HostProfile[THostConfig], DebuggerProfile, metaclass=abc.ABCMeta): """Base class for profiles remote debugging.""" - __PYDEVD_PORT_KEY = 'pydevd_port' + __DEBUGGING_PORT_KEY = 'debugging_port' __DEBUGGING_FORWARDER_KEY = 'debugging_forwarder' + @property + def debugger(self) -> DebuggerSettings | None: + """The debugger settings for this host if present and enabled, otherwise None.""" + return self.args.metadata.debugger_settings + @property def debugging_enabled(self) -> bool: """Returns `True` if debugging is enabled for this profile, otherwise `False`.""" @@ -307,9 +316,14 @@ class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta): return self.args.metadata.debugger_flags.ansiballz @property - def pydevd_port(self) -> int: - """The pydevd port to use.""" - return self.state.get(self.__PYDEVD_PORT_KEY) or self.origin_pydev_port + def debugger_host(self) -> str: + """The debugger host to use.""" + return 'localhost' + + @property + def debugger_port(self) -> int: + """The debugger port to use.""" + return self.state.get(self.__DEBUGGING_PORT_KEY) or self.origin_debugger_port @property def debugging_forwarder(self) -> SshProcess | None: @@ -322,23 +336,23 @@ class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta): self.cache[self.__DEBUGGING_FORWARDER_KEY] = value @property - def origin_pydev_port(self) -> int: - """The pydevd port on the origin.""" - return self.args.metadata.debugger_settings.port + def origin_debugger_port(self) -> int: + """The debugger port on the origin.""" + return self.debugger.port def enable_debugger_forwarding(self, ssh: SshConnectionDetail) -> None: - """Enable pydevd port forwarding from the origin.""" + """Enable debugger port forwarding from the origin.""" if not self.debugging_enabled: return - endpoint = ('localhost', self.origin_pydev_port) + endpoint = ('localhost', self.origin_debugger_port) forwards = [endpoint] self.debugging_forwarder = create_ssh_port_forwards(self.args, ssh, forwards) port_forwards = self.debugging_forwarder.collect_port_forwards() - self.state[self.__PYDEVD_PORT_KEY] = port = port_forwards[endpoint] + self.state[self.__DEBUGGING_PORT_KEY] = port = port_forwards[endpoint] display.info(f'Remote debugging of {self.name!r} is available on port {port}.', verbosity=1) @@ -355,19 +369,6 @@ class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta): self.debugging_forwarder.wait() - def get_pydevd_settrace_arguments(self) -> dict[str, object]: - """Get settrace arguments for pydevd.""" - return self.args.metadata.debugger_settings.settrace | dict( - host="localhost", - port=self.pydevd_port, - ) - - def get_pydevd_environment_variables(self) -> dict[str, str]: - """Get environment variables needed to configure pydevd for debugging.""" - return dict( - PATHS_FROM_ECLIPSE_TO_PYTHON=json.dumps(list(self.get_source_mapping().items())), - ) - def get_source_mapping(self) -> dict[str, str]: """Get the source mapping from the given metadata.""" from . import data_context @@ -396,10 +397,9 @@ class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta): display.info('Activating remote debugging of ansible-test.', verbosity=1) - os.environ.update(self.get_pydevd_environment_variables()) + os.environ.update(self.debugger.get_environment_variables(self)) - debugging_module = importlib.import_module(self.args.metadata.debugger_settings.module) - debugging_module.settrace(**self.get_pydevd_settrace_arguments()) + self.debugger.activate_debugger(self) pass # pylint: disable=unnecessary-pass # when suspend is True, execution pauses here -- it's also a convenient place to put a breakpoint @@ -411,9 +411,11 @@ class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta): if not self.args.metadata.debugger_flags.ansiballz: return {} - return dict( - _ansible_ansiballz_debugger_config=json.dumps(self.get_ansiballz_debugger_config()), - ) + debug_type = self.debugger.get_debug_type() + + return { + f"_ansible_ansiballz_{debug_type}_config": json.dumps(self.get_ansiballz_debugger_config()), + } def get_ansiballz_environment_variables(self) -> dict[str, t.Any]: """ @@ -423,20 +425,18 @@ class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta): if not self.args.metadata.debugger_flags.ansiballz: return {} - return dict( - _ANSIBLE_ANSIBALLZ_DEBUGGER_CONFIG=json.dumps(self.get_ansiballz_debugger_config()), - ) + debug_type = self.debugger.get_debug_type().upper() + + return { + f"_ANSIBLE_ANSIBALLZ_{debug_type}_CONFIG": json.dumps(self.get_ansiballz_debugger_config()), + } def get_ansiballz_debugger_config(self) -> dict[str, t.Any]: """ Return config for remote debugging of AnsiballZ modules. When delegating, this function must be called after delegation. """ - debugger_config = dict( - module=self.args.metadata.debugger_settings.module, - settrace=self.get_pydevd_settrace_arguments(), - source_mapping=self.get_source_mapping(), - ) + debugger_config = self.debugger.get_ansiballz_config(self) display.info(f'>>> Debugger Config ({self.name} AnsiballZ)\n{json.dumps(debugger_config, indent=4)}', verbosity=3) @@ -451,8 +451,8 @@ class DebuggableProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta): return {} debugger_config = dict( - args=['-m', 'pydevd', '--client', 'localhost', '--port', str(self.pydevd_port)] + self.args.metadata.debugger_settings.args + ['--file'], - env=self.get_pydevd_environment_variables(), + args=self.debugger.get_cli_arguments(self), + env=self.debugger.get_environment_variables(self), ) display.info(f'>>> Debugger Config ({self.name} Ansible CLI)\n{json.dumps(debugger_config, indent=4)}', verbosity=3) @@ -597,9 +597,9 @@ class ControllerProfile(SshTargetHostProfile[ControllerConfig], PosixProfile[Con return self.controller_profile.name @property - def pydevd_port(self) -> int: + def debugger_port(self) -> int: """The pydevd port to use.""" - return self.controller_profile.pydevd_port + return self.controller_profile.debugger_port def get_controller_target_connections(self) -> list[SshConnection]: """Return SSH connection(s) for accessing the host as a target from the controller.""" diff --git a/test/lib/ansible_test/_internal/metadata.py b/test/lib/ansible_test/_internal/metadata.py index 5aaceecd31c..7f9f6e0bc30 100644 --- a/test/lib/ansible_test/_internal/metadata.py +++ b/test/lib/ansible_test/_internal/metadata.py @@ -22,6 +22,9 @@ from .diff import ( FileDiff, ) +if t.TYPE_CHECKING: + from .debugging import DebuggerSettings + class Metadata: """Metadata object for passing data to delegated tests.""" @@ -71,7 +74,7 @@ class Metadata: ansible_test_root=self.ansible_test_root, collection_root=self.collection_root, debugger_flags=dataclasses.asdict(self.debugger_flags), - debugger_settings=dataclasses.asdict(self.debugger_settings) if self.debugger_settings else None, + debugger_settings=self.debugger_settings.as_dict() if self.debugger_settings else None, ) def to_file(self, path: str) -> None: @@ -91,6 +94,8 @@ class Metadata: @staticmethod def from_dict(data: dict[str, t.Any]) -> Metadata: """Return metadata loaded from the specified dictionary.""" + from .debugging import DebuggerSettings + metadata = Metadata( debugger_flags=DebuggerFlags(**data['debugger_flags']), ) @@ -103,7 +108,7 @@ class Metadata: metadata.ansible_lib_root = data['ansible_lib_root'] metadata.ansible_test_root = data['ansible_test_root'] metadata.collection_root = data['collection_root'] - metadata.debugger_settings = DebuggerSettings(**data['debugger_settings']) if data['debugger_settings'] else None + metadata.debugger_settings = DebuggerSettings.from_dict(data['debugger_settings']) if data['debugger_settings'] else None metadata.loaded = True return metadata @@ -155,46 +160,6 @@ class ChangeDescription: return changes -@dataclasses.dataclass(frozen=True, kw_only=True) -class DebuggerSettings: - """Settings for remote debugging.""" - - module: str | None = None - """ - The Python module to import. - This should be pydevd or a derivative. - If not provided it will be auto-detected. - """ - - package: str | None = None - """ - The Python package to install for debugging. - If `None` then the package will be auto-detected. - If an empty string, then no package will be installed. - """ - - settrace: dict[str, object] = dataclasses.field(default_factory=dict) - """ - Options to pass to the `{module}.settrace` method. - Used for running AnsiballZ modules only. - The `host` and `port` options will be provided by ansible-test. - The `suspend` option defaults to `False`. - """ - - args: list[str] = dataclasses.field(default_factory=list) - """ - Arguments to pass to `pydevd` on the command line. - Used for running Ansible CLI programs only. - The `--client` and `--port` options will be provided by ansible-test. - """ - - port: int = 5678 - """ - The port on the origin host which is listening for incoming connections from pydevd. - SSH port forwarding will be automatically configured for non-local hosts to connect to this port as needed. - """ - - @dataclasses.dataclass(frozen=True, kw_only=True) class DebuggerFlags: """Flags for enabling specific debugging features.""" diff --git a/test/lib/ansible_test/_internal/python_requirements.py b/test/lib/ansible_test/_internal/python_requirements.py index 68077542eaf..32041555900 100644 --- a/test/lib/ansible_test/_internal/python_requirements.py +++ b/test/lib/ansible_test/_internal/python_requirements.py @@ -170,11 +170,11 @@ def install_requirements( from .host_profiles import DebuggableProfile - if isinstance(host_profile, DebuggableProfile) and host_profile.debugging_enabled and args.metadata.debugger_settings.package: + if isinstance(host_profile, DebuggableProfile) and host_profile.debugger and host_profile.debugger.get_python_package(): commands.append(PipInstall( requirements=[], constraints=[], - packages=[args.metadata.debugger_settings.package], + packages=[host_profile.debugger.get_python_package()], )) if not commands: diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg index d9219b5d4ae..42aba85c7db 100644 --- a/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg @@ -21,6 +21,7 @@ disable= broad-exception-raised, # many exceptions with no need for a custom type too-few-public-methods, too-many-public-methods, + too-many-ancestors, too-many-arguments, too-many-branches, too-many-instance-attributes, diff --git a/test/sanity/code-smell/mypy/ansible-core.ini b/test/sanity/code-smell/mypy/ansible-core.ini index 94c3596ae50..ae51059471b 100644 --- a/test/sanity/code-smell/mypy/ansible-core.ini +++ b/test/sanity/code-smell/mypy/ansible-core.ini @@ -133,3 +133,6 @@ ignore_missing_imports = True [mypy-jinja2.nativetypes] ignore_missing_imports = True + +[mypy-debugpy] +ignore_missing_imports = True diff --git a/test/sanity/code-smell/mypy/ansible-test.ini b/test/sanity/code-smell/mypy/ansible-test.ini index 81a5d64e4fc..2fd72dd9a95 100644 --- a/test/sanity/code-smell/mypy/ansible-test.ini +++ b/test/sanity/code-smell/mypy/ansible-test.ini @@ -77,3 +77,9 @@ ignore_missing_imports = True [mypy-py._path.local] ignore_missing_imports = True + +[mypy-debugpy] +ignore_missing_imports = True + +[mypy-debugpy.server] +ignore_missing_imports = True From 8c3d23513fb3eff4fae53fda8ee674f1ceca587f Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Fri, 18 Jul 2025 14:15:19 -0400 Subject: [PATCH 27/68] removed removed acceleration keywords (#85507) --- lib/ansible/keyword_desc.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/ansible/keyword_desc.yml b/lib/ansible/keyword_desc.yml index 39d703a3a3f..2d4ec9aaa2a 100644 --- a/lib/ansible/keyword_desc.yml +++ b/lib/ansible/keyword_desc.yml @@ -1,6 +1,3 @@ -accelerate: "*DEPRECATED*, set to True to use accelerate connection plugin." -accelerate_ipv6: "*DEPRECATED*, set to True to force accelerate plugin to use ipv6 for its connection." -accelerate_port: "*DEPRECATED*, set to override default port use for accelerate connection." action: "The 'action' to execute for a task, it normally translates into a C(module) or action plugin." args: "A secondary way to add arguments into a task. Takes a dictionary in which keys map to options and values." always: List of tasks, in a block, that execute no matter if there is an error in the block or not. From 19c7ec6ed24443d91389dbef7ead33fa712f8cbd Mon Sep 17 00:00:00 2001 From: Abhijeet Kasurde Date: Tue, 22 Jul 2025 08:16:52 -0700 Subject: [PATCH 28/68] Distribution: Added UCS in Debian OS Family (#85496) * Identify Univention Corporate Server as Debian OS Family. Fixes: #85490 Signed-off-by: Abhijeet Kasurde --- changelogs/fragments/ucs.yml | 3 ++ .../module_utils/facts/system/distribution.py | 3 +- .../univention_corporate_server_5.2.json | 51 +++++++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 changelogs/fragments/ucs.yml create mode 100644 test/units/module_utils/facts/system/distribution/fixtures/univention_corporate_server_5.2.json diff --git a/changelogs/fragments/ucs.yml b/changelogs/fragments/ucs.yml new file mode 100644 index 00000000000..d1a9baf83e6 --- /dev/null +++ b/changelogs/fragments/ucs.yml @@ -0,0 +1,3 @@ +--- +minor_changes: + - Added Univention Corporate Server as a part of Debian OS distribution family (https://github.com/ansible/ansible/issues/85490). diff --git a/lib/ansible/module_utils/facts/system/distribution.py b/lib/ansible/module_utils/facts/system/distribution.py index 9f81beab64d..aeb78d789bf 100644 --- a/lib/ansible/module_utils/facts/system/distribution.py +++ b/lib/ansible/module_utils/facts/system/distribution.py @@ -529,7 +529,8 @@ class Distribution(object): 'EuroLinux', 'Kylin Linux Advanced Server', 'MIRACLE'], 'Debian': ['Debian', 'Ubuntu', 'Raspbian', 'Neon', 'KDE neon', 'Linux Mint', 'SteamOS', 'Devuan', 'Kali', 'Cumulus Linux', - 'Pop!_OS', 'Parrot', 'Pardus GNU/Linux', 'Uos', 'Deepin', 'OSMC', 'Linux Mint Debian Edition'], + 'Pop!_OS', 'Parrot', 'Pardus GNU/Linux', 'Uos', 'Deepin', 'OSMC', + 'Linux Mint Debian Edition', 'Univention Corporate Server'], 'Suse': ['SuSE', 'SLES', 'SLED', 'openSUSE', 'openSUSE Tumbleweed', 'SLES_SAP', 'SUSE_LINUX', 'openSUSE Leap', 'ALP-Dolomite', 'SL-Micro', 'openSUSE MicroOS'], diff --git a/test/units/module_utils/facts/system/distribution/fixtures/univention_corporate_server_5.2.json b/test/units/module_utils/facts/system/distribution/fixtures/univention_corporate_server_5.2.json new file mode 100644 index 00000000000..5a40c61f399 --- /dev/null +++ b/test/units/module_utils/facts/system/distribution/fixtures/univention_corporate_server_5.2.json @@ -0,0 +1,51 @@ +{ + "name": "Univention Corporate Server 5.2", + "distro": { + "codename": "bookworm", + "id": "ucs", + "name": "Univention Corporate Server", + "version": "5.2", + "version_best": "5.2", + "lsb_release_info": { + "distributor_id": "Ucs", + "description": "Univention Corporate Server 5.2", + "release": "5.2", + "codename": "bookworm" + }, + "os_release_info": { + "pretty_name": "Univention Corporate Server 5.2", + "name": "Univention Corporate Server", + "version": "5.2", + "id": "ucs", + "version_id": "5.2", + "id_like": "debian", + "version_codename": "bookworm", + "home_url": "https://www.univention.com/", + "documentation_url": "https://docs.software-univention.de/", + "support_url": "https://www.univention.com/products/support/", + "bug_report_url": "https://forge.univention.org/", + "privacy_policy_url": "https://www.univention.com/privacy-statement/privacy-statement-univention-corporate-server/", + "ansi_color": "0;38;2;221;4;45", + "cpe_name": "cpe:/o:univention:univention_corporate_server:5.2", + "codename": "bookworm" + } + }, + "input": { + "/etc/os-release": "PRETTY_NAME=\"Univention Corporate Server 5.2\"\nNAME=\"Univention Corporate Server\"\nVERSION=\"5.2\"\nID=\"ucs\"\nVERSION_ID=\"5.2\"\nID_LIKE=\"debian\"\nVERSION_CODENAME=bookworm\nHOME_URL=\"https://www.univention.com/\"\nDOCUMENTATION_URL=\"https://docs.software-univention.de/\"\nSUPPORT_URL=\"https://www.univention.com/products/support/\"\nBUG_REPORT_URL=\"https://forge.univention.org/\"\nPRIVACY_POLICY_URL=\"https://www.univention.com/privacy-statement/privacy-statement-univention-corporate-server/\"\nANSI_COLOR=\"0;38;2;221;4;45\"\nCPE_NAME=\"cpe:/o:univention:univention_corporate_server:5.2\"\n", + "/etc/lsb-release": "# Warning: This file is auto-generated and might be overwritten by\n# univention-config-registry.\n# Please edit the following file(s) instead:\n# Warnung: Diese Datei wurde automatisch generiert und kann durch\n# univention-config-registry ueberschrieben werden.\n# Bitte bearbeiten Sie an Stelle dessen die folgende(n) Datei(en):\n# \n# \t/etc/univention/templates/files/etc/lsb-release\n# \n\nDISTRIB_ID=Univention\nDISTRIB_RELEASE=\"5.2-2 errata148\"\nDISTRIB_DESCRIPTION=\"Univention Corporate Server 5.2-2 errata148\"\n", + "/usr/lib/os-release": "PRETTY_NAME=\"Univention Corporate Server 5.2\"\nNAME=\"Univention Corporate Server\"\nVERSION=\"5.2\"\nID=\"ucs\"\nVERSION_ID=\"5.2\"\nID_LIKE=\"debian\"\nVERSION_CODENAME=bookworm\nHOME_URL=\"https://www.univention.com/\"\nDOCUMENTATION_URL=\"https://docs.software-univention.de/\"\nSUPPORT_URL=\"https://www.univention.com/products/support/\"\nBUG_REPORT_URL=\"https://forge.univention.org/\"\nPRIVACY_POLICY_URL=\"https://www.univention.com/privacy-statement/privacy-statement-univention-corporate-server/\"\nANSI_COLOR=\"0;38;2;221;4;45\"\nCPE_NAME=\"cpe:/o:univention:univention_corporate_server:5.2\"\n" + }, + "platform.dist": [ + "ucs", + "5.2", + "bookworm" + ], + "result": { + "distribution": "Univention Corporate Server", + "distribution_version": "5.2", + "distribution_release": "bookworm", + "distribution_major_version": "5", + "os_family": "Debian" + }, + "platform.release": "4.19.0-20-amd64" +} From a2427e45df903aa247346e51447adba7848c706f Mon Sep 17 00:00:00 2001 From: Abhijeet Kasurde Date: Tue, 22 Jul 2025 08:18:38 -0700 Subject: [PATCH 29/68] docs: use correct URL in docs (#85491) * wait_for: use correct URL in docs Signed-off-by: Abhijeet Kasurde --- lib/ansible/modules/user.py | 3 +-- lib/ansible/modules/wait_for.py | 4 ++-- lib/ansible/plugins/lookup/url.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/ansible/modules/user.py b/lib/ansible/modules/user.py index 28eef5c31c0..b81258153fd 100644 --- a/lib/ansible/modules/user.py +++ b/lib/ansible/modules/user.py @@ -2504,11 +2504,10 @@ class DarwinUser(User): Please note that password must be cleartext. """ # some documentation on how is stored passwords on OSX: - # http://blog.lostpassword.com/2012/07/cracking-mac-os-x-lion-accounts-passwords/ # http://null-byte.wonderhowto.com/how-to/hack-mac-os-x-lion-passwords-0130036/ # http://pastebin.com/RYqxi7Ca # on OSX 10.8+ hash is SALTED-SHA512-PBKDF2 - # https://pythonhosted.org/passlib/lib/passlib.hash.pbkdf2_digest.html + # https://passlib.readthedocs.io/en/stable/lib/passlib.hash.pbkdf2_digest.html # https://gist.github.com/nueh/8252572 cmd = self._get_dscl() if self.password: diff --git a/lib/ansible/modules/wait_for.py b/lib/ansible/modules/wait_for.py index a076bb14b65..31165e78f55 100644 --- a/lib/ansible/modules/wait_for.py +++ b/lib/ansible/modules/wait_for.py @@ -216,13 +216,13 @@ elapsed: type: int sample: 23 match_groups: - description: Tuple containing all the subgroups of the match as returned by U(https://docs.python.org/3/library/re.html#re.MatchObject.groups) + description: Tuple containing all the subgroups of the match as returned by U(https://docs.python.org/3/library/re.html#re.Match.groups) returned: always type: list sample: ['match 1', 'match 2'] match_groupdict: description: Dictionary containing all the named subgroups of the match, keyed by the subgroup name, - as returned by U(https://docs.python.org/3/library/re.html#re.MatchObject.groupdict) + as returned by U(https://docs.python.org/3/library/re.html#re.Match.groupdict) returned: always type: dict sample: diff --git a/lib/ansible/plugins/lookup/url.py b/lib/ansible/plugins/lookup/url.py index d3d3a4db37d..26c7c09d278 100644 --- a/lib/ansible/plugins/lookup/url.py +++ b/lib/ansible/plugins/lookup/url.py @@ -164,7 +164,7 @@ options: description: - SSL/TLS Ciphers to use for the request - 'When a list is provided, all ciphers are joined in order with C(:)' - - See the L(OpenSSL Cipher List Format,https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT) + - See the L(OpenSSL Cipher List Format,https://docs.openssl.org/master/man1/openssl-ciphers/#cipher-list-format) for more details. - The available ciphers is dependent on the Python and OpenSSL/LibreSSL versions type: list From d2be8684b3aec27adc2d5eb00630a1bfcb44454b Mon Sep 17 00:00:00 2001 From: Abhijeet Kasurde Date: Tue, 22 Jul 2025 08:27:50 -0700 Subject: [PATCH 30/68] Remove deprecated py3compat.environ call (#85501) Signed-off-by: Abhijeet Kasurde --- changelogs/fragments/remove_py3compat.yml | 3 +++ lib/ansible/utils/py3compat.py | 27 ----------------------- test/sanity/ignore.txt | 1 - 3 files changed, 3 insertions(+), 28 deletions(-) create mode 100644 changelogs/fragments/remove_py3compat.yml delete mode 100644 lib/ansible/utils/py3compat.py diff --git a/changelogs/fragments/remove_py3compat.yml b/changelogs/fragments/remove_py3compat.yml new file mode 100644 index 00000000000..2e2ad0739eb --- /dev/null +++ b/changelogs/fragments/remove_py3compat.yml @@ -0,0 +1,3 @@ +--- +removed_features: + - py3compat - remove deprecated ``py3compat.environ`` call. diff --git a/lib/ansible/utils/py3compat.py b/lib/ansible/utils/py3compat.py deleted file mode 100644 index 374a3dbcef2..00000000000 --- a/lib/ansible/utils/py3compat.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*- -# -# (c) 2018, Toshio Kuratomi -# Copyright: Contributors to the Ansible project -# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) - -from __future__ import annotations - -import os - -from ansible.utils.display import Display - - -display = Display() - - -def __getattr__(name): - if name != 'environ': - raise AttributeError(name) - - display.deprecated( - msg='`ansible.utils.py3compat.environ` is deprecated.', - version='2.20', - help_text='Use `os.environ` from the Python standard library instead.', - ) - - return os.environ diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index 0751ca5b155..94f0ee69336 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -236,7 +236,6 @@ lib/ansible/cli/doc.py pylint:ansible-deprecated-version # TODO: 2.20 lib/ansible/galaxy/api.py pylint:ansible-deprecated-version # TODO: 2.20 lib/ansible/plugins/filter/encryption.py pylint:ansible-deprecated-version # TODO: 2.20 lib/ansible/utils/encrypt.py pylint:ansible-deprecated-version # TODO: 2.20 -lib/ansible/utils/py3compat.py pylint:ansible-deprecated-version # TODO: 2.20 lib/ansible/utils/ssh_functions.py pylint:ansible-deprecated-version # TODO: 2.20 lib/ansible/vars/manager.py pylint:ansible-deprecated-version-comment # TODO: 2.20 lib/ansible/vars/plugins.py pylint:ansible-deprecated-version # TODO: 2.20 From 737c68ace3d0f6c748f465ef561856e343293d7e Mon Sep 17 00:00:00 2001 From: Patrick Kingston <66141901+pkingstonxyz@users.noreply.github.com> Date: Tue, 22 Jul 2025 11:48:33 -0400 Subject: [PATCH 31/68] Add install_python_debian parameter to deb822_repository (#85487) * Add python3-debian dependency installation in module * Add tests to exercise automatic dependency installation * Fix broken tests to include new module parameter Co-authored-by: Abhijeet Kasurde --- ...ency-installation-to-deb822_repository.yml | 2 + lib/ansible/modules/deb822_repository.py | 81 ++++++++++++++++++- .../targets/deb822_repository/tasks/main.yml | 49 +++++++++++ .../targets/deb822_repository/tasks/test.yml | 2 + 4 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 changelogs/fragments/85487-add-dependency-installation-to-deb822_repository.yml diff --git a/changelogs/fragments/85487-add-dependency-installation-to-deb822_repository.yml b/changelogs/fragments/85487-add-dependency-installation-to-deb822_repository.yml new file mode 100644 index 00000000000..3731b75f0b9 --- /dev/null +++ b/changelogs/fragments/85487-add-dependency-installation-to-deb822_repository.yml @@ -0,0 +1,2 @@ +minor_changes: + - deb822_repository - Add automatic installation of the ``python3-debian`` package if it is missing by adding the parameter ``install_python_debian`` \ No newline at end of file diff --git a/lib/ansible/modules/deb822_repository.py b/lib/ansible/modules/deb822_repository.py index d4d6205511e..00278fb0342 100644 --- a/lib/ansible/modules/deb822_repository.py +++ b/lib/ansible/modules/deb822_repository.py @@ -67,6 +67,17 @@ options: - Determines the path to the C(InRelease) file, relative to the normal position of an C(InRelease) file. type: str + install_python_debian: + description: + - Whether to automatically try to install the Python C(debian) library or not, if it is not already installed. + Without this library, the module does not work. + - Runs C(apt install python3-debian). + - Only works with the system Python. If you are using a Python on the remote that is not + the system Python, set O(install_python_debian=false) and ensure that the Python C(debian) library + for your Python version is installed some other way. + type: bool + default: false + version_added: '2.20' languages: description: - Defines which languages information such as translated @@ -228,6 +239,7 @@ key_filename: import os import re +import sys import tempfile import textwrap @@ -235,6 +247,7 @@ from ansible.module_utils.basic import AnsibleModule from ansible.module_utils.basic import missing_required_lib from ansible.module_utils.common.collections import is_sequence from ansible.module_utils.common.file import S_IRWXU_RXG_RXO, S_IRWU_RG_RO +from ansible.module_utils.common.respawn import has_respawned, probe_interpreters_for_module, respawn_module from ansible.module_utils.common.text.converters import to_bytes from ansible.module_utils.common.text.converters import to_native from ansible.module_utils.six import raise_from # type: ignore[attr-defined] @@ -357,6 +370,21 @@ def write_signed_by_key(module, v, slug): return changed, filename, None +def install_python_debian(module, deb_pkg_name): + + if not module.check_mode: + apt_path = module.get_bin_path('apt', required=True) + if apt_path: + rc, so, se = module.run_command([apt_path, 'update']) + if rc != 0: + module.fail_json(msg=f"Failed update while auto installing {deb_pkg_name} due to '{se.strip()}'") + rc, so, se = module.run_command([apt_path, 'install', deb_pkg_name, '-y', '-q']) + if rc != 0: + module.fail_json(msg=f"Failed to auto-install {deb_pkg_name} due to : '{se.strip()}'") + else: + module.fail_json(msg=f"{deb_pkg_name} must be installed to use check mode") + + def main(): module = AnsibleModule( argument_spec={ @@ -395,6 +423,10 @@ def main(): 'inrelease_path': { 'type': 'str', }, + 'install_python_debian': { + 'type': 'bool', + 'default': False, + }, 'languages': { 'elements': 'str', 'type': 'list', @@ -453,8 +485,53 @@ def main(): ) if not HAS_DEBIAN: - module.fail_json(msg=missing_required_lib("python3-debian"), - exception=DEBIAN_IMP_ERR) + deb_pkg_name = 'python3-debian' + # This interpreter can't see the debian Python library- we'll do the following to try and fix that as per + # the apt_repository module: + # 1) look in common locations for system-owned interpreters that can see it; if we find one, respawn under it + # 2) finding none, try to install a matching python-debian package for the current interpreter version; + # we limit to the current interpreter version to try and avoid installing a whole other Python just + # for deb support + # 3) if we installed a support package, try to respawn under what we think is the right interpreter (could be + # the current interpreter again, but we'll let it respawn anyway for simplicity) + # 4) if still not working, return an error and give up (some corner cases not covered, but this shouldn't be + # made any more complex than it already is to try and cover more, eg, custom interpreters taking over + # system locations) + + if has_respawned(): + # this shouldn't be possible; short-circuit early if it happens... + module.fail_json(msg=f"{deb_pkg_name} must be installed and visible from {sys.executable}.") + + interpreters = ['/usr/bin/python3', '/usr/bin/python'] + + interpreter = probe_interpreters_for_module(interpreters, 'debian') + + if interpreter: + # found the Python bindings; respawn this module under the interpreter where we found them + respawn_module(interpreter) + # this is the end of the line for this process, it will exit here once the respawned module has completed + + # don't make changes if we're in check_mode + if module.check_mode: + module.fail_json(msg=f"{deb_pkg_name} must be installed to use check mode. If run with install_python_debian, this module can auto-install it.") + + if module.params['install_python_debian']: + install_python_debian(module, deb_pkg_name) + else: + module.fail_json(msg=f'{deb_pkg_name} is not installed, and install_python_debian is False') + + # try again to find the bindings in common places + interpreter = probe_interpreters_for_module(interpreters, 'debian') + + if interpreter: + # found the Python bindings; respawn this module under the interpreter where we found them + # NB: respawn is somewhat wasteful if it's this interpreter, but simplifies the code + respawn_module(interpreter) + # this is the end of the line for this process, it will exit here once the respawned module has completed + else: + # we've done all we can do; just tell the user it's busted and get out + module.fail_json(msg=missing_required_lib(deb_pkg_name), + exception=DEBIAN_IMP_ERR) check_mode = module.check_mode diff --git a/test/integration/targets/deb822_repository/tasks/main.yml b/test/integration/targets/deb822_repository/tasks/main.yml index 561ef2a6f50..585792f73a2 100644 --- a/test/integration/targets/deb822_repository/tasks/main.yml +++ b/test/integration/targets/deb822_repository/tasks/main.yml @@ -1,13 +1,62 @@ - meta: end_play when: ansible_os_family != 'Debian' +- set_fact: + python_debian: python3-debian + - block: + # UNINSTALL 'python3-debian' + # The `deb822_repository` module has the smarts to auto-install `python3-debian`. To + # test, we will first uninstall `python3-debian`. + - name: check {{ python_debian }} with dpkg + shell: dpkg -s {{ python_debian }} + register: dpkg_result + ignore_errors: true + + - name: uninstall {{ python_debian }} with apt + apt: pkg={{ python_debian }} state=absent purge=yes + register: apt_result + when: dpkg_result is successful + + - name: check failure when python3-debian is absent + deb822_repository: + name: myrepo + types: deb + uris: "http://example.com/debian/" + suites: stable + components: main + register: no_python3_debian + ignore_errors: true + + - name: Assert failure with absent python3-debian + assert: + that: no_python3_debian.msg is contains 'python3-debian is not installed' + + - name: run deb822 to check for python3-debian installation + deb822_repository: + name: myrepo + types: deb + uris: "http://example.com/debian/" + suites: stable + components: main + install_python_debian: true + + - name: Clean up the previously added basic Debian repository + deb822_repository: + name: myrepo + state: absent + - name: install python3-debian apt: name: python3-debian state: present register: py3_deb_install + - name: assert python3-debian already installed + assert: + that: + - not py3_deb_install.changed + - import_tasks: test.yml - import_tasks: install.yml diff --git a/test/integration/targets/deb822_repository/tasks/test.yml b/test/integration/targets/deb822_repository/tasks/test.yml index 345ccb7ad8d..6f8a5204e1c 100644 --- a/test/integration/targets/deb822_repository/tasks/test.yml +++ b/test/integration/targets/deb822_repository/tasks/test.yml @@ -119,6 +119,7 @@ focal_archive_expected: |- Components: main restricted Date-Max-Future: 10 + Install-Python-Debian: no X-Repolib-Name: ansible-test focal archive Suites: focal focal-updates Types: deb @@ -192,6 +193,7 @@ vars: signed_by_inline_expected: |- Components: main contrib non-free + Install-Python-Debian: no X-Repolib-Name: ansible-test Signed-By: -----BEGIN PGP PUBLIC KEY BLOCK----- From 2fbd7c114efe7dfc6795d5553f866e6a2b002a3c Mon Sep 17 00:00:00 2001 From: Matt Davis <6775756+nitzmahone@users.noreply.github.com> Date: Tue, 22 Jul 2025 10:39:54 -0700 Subject: [PATCH 32/68] Rewrite exception key on failed_when: false (#85516) * Prevents callback handle_exception from displaying the captured exception when the task is not failed. * Added tests. Co-authored-by: Matt Clay --- changelogs/fragments/failed-when-exception.yml | 4 ++++ lib/ansible/executor/task_executor.py | 5 ++++- test/integration/targets/failed_when/aliases | 1 + test/integration/targets/failed_when/tasks/main.yml | 10 ++++++++++ 4 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 changelogs/fragments/failed-when-exception.yml diff --git a/changelogs/fragments/failed-when-exception.yml b/changelogs/fragments/failed-when-exception.yml new file mode 100644 index 00000000000..9affb86fc61 --- /dev/null +++ b/changelogs/fragments/failed-when-exception.yml @@ -0,0 +1,4 @@ +bugfixes: + - failed_when - When using ``failed_when`` to suppress an error, the ``exception`` key in the result is renamed to ``failed_when_suppressed_exception``. + This prevents the error from being displayed by callbacks after being suppressed. + (https://github.com/ansible/ansible/issues/85505) diff --git a/lib/ansible/executor/task_executor.py b/lib/ansible/executor/task_executor.py index fa328f396cf..223abfc3ac8 100644 --- a/lib/ansible/executor/task_executor.py +++ b/lib/ansible/executor/task_executor.py @@ -712,7 +712,10 @@ class TaskExecutor: condname = 'failed' if self._task.failed_when: - result['failed_when_result'] = result['failed'] = self._task._resolve_conditional(self._task.failed_when, vars_copy) + is_failed = result['failed_when_result'] = result['failed'] = self._task._resolve_conditional(self._task.failed_when, vars_copy) + + if not is_failed and (suppressed_exception := result.pop('exception', None)): + result['failed_when_suppressed_exception'] = suppressed_exception except AnsibleError as e: result['failed'] = True diff --git a/test/integration/targets/failed_when/aliases b/test/integration/targets/failed_when/aliases index 498fedd558e..e4fc6744ad2 100644 --- a/test/integration/targets/failed_when/aliases +++ b/test/integration/targets/failed_when/aliases @@ -1,2 +1,3 @@ +gather_facts/no shippable/posix/group4 context/controller diff --git a/test/integration/targets/failed_when/tasks/main.yml b/test/integration/targets/failed_when/tasks/main.yml index c87c7f81da8..c3745176b06 100644 --- a/test/integration/targets/failed_when/tasks/main.yml +++ b/test/integration/targets/failed_when/tasks/main.yml @@ -24,6 +24,8 @@ - assert: that: - "'failed' in result and not result.failed" + - result.exception is undefined + - result.failed_when_suppressed_exception is undefined - name: command rc 0 failed_when_result False shell: exit 0 @@ -35,6 +37,8 @@ that: - "'failed' in result and not result.failed" - "'failed_when_result' in result and not result.failed_when_result" + - result.exception is undefined + - result.failed_when_suppressed_exception is undefined - name: command rc 1 failed_when_result True shell: exit 1 @@ -46,6 +50,8 @@ that: - "'failed' in result and result.failed" - "'failed_when_result' in result and result.failed_when_result" + - result.exception is defined + - result.failed_when_suppressed_exception is undefined - name: command rc 1 failed_when_result undef shell: exit 1 @@ -55,6 +61,8 @@ - assert: that: - "'failed' in result and result.failed" + - result.exception is defined + - result.failed_when_suppressed_exception is undefined - name: command rc 1 failed_when_result False shell: exit 1 @@ -66,6 +74,8 @@ that: - "'failed' in result and not result.failed" - "'failed_when_result' in result and not result.failed_when_result" + - result.exception is undefined + - result.failed_when_suppressed_exception is defined - name: invalid conditional command: echo foo From a0d56d2f4f32f174bd6c463118a3a38eeb70921b Mon Sep 17 00:00:00 2001 From: Aditya Putta Date: Tue, 22 Jul 2025 14:51:20 -0500 Subject: [PATCH 33/68] Usage of 'sleep' parameter in ansible.builtin.wait_for_connection (#85435) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Aditya Putta Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) Co-authored-by: Abhijeet Kasurde --- lib/ansible/modules/wait_for_connection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/ansible/modules/wait_for_connection.py b/lib/ansible/modules/wait_for_connection.py index 8007b10ee2b..ee2b507a4dd 100644 --- a/lib/ansible/modules/wait_for_connection.py +++ b/lib/ansible/modules/wait_for_connection.py @@ -104,9 +104,10 @@ EXAMPLES = r""" - cmd.exe /c winrm.cmd quickconfig -quiet -force delegate_to: localhost - - name: Wait for system to become reachable over WinRM + - name: Wait for system to become reachable over WinRM, polling every 10 seconds ansible.builtin.wait_for_connection: timeout: 900 + sleep: 10 - name: Gather facts for first time ansible.builtin.setup: From 19f9c66004c04c0479ee00bc770057ae0aae572b Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Wed, 23 Jul 2025 11:49:14 -0400 Subject: [PATCH 34/68] plugin config options are now correctly fetched with origin (#85488) * also update callbcacks, since they override these functions due to backwards compat _options being taken for CLI --- changelogs/fragments/plugins_fix_origin.yml | 2 + lib/ansible/config/manager.py | 10 +++- lib/ansible/plugins/__init__.py | 28 +++++---- lib/ansible/plugins/callback/__init__.py | 7 ++- .../targets/config/lookup_plugins/broken.py | 57 +++++++++++++++++++ .../targets/config/match_option_methods.yml | 37 ++++++++++++ test/integration/targets/config/runme.sh | 3 + 7 files changed, 130 insertions(+), 14 deletions(-) create mode 100644 changelogs/fragments/plugins_fix_origin.yml create mode 100644 test/integration/targets/config/lookup_plugins/broken.py create mode 100644 test/integration/targets/config/match_option_methods.yml diff --git a/changelogs/fragments/plugins_fix_origin.yml b/changelogs/fragments/plugins_fix_origin.yml new file mode 100644 index 00000000000..55d5b23228c --- /dev/null +++ b/changelogs/fragments/plugins_fix_origin.yml @@ -0,0 +1,2 @@ +bugfixes: + - plugins config, get_option_and_origin now correctly displays the value and origin of the option. diff --git a/lib/ansible/config/manager.py b/lib/ansible/config/manager.py index c4b0ffbc362..33f398e199b 100644 --- a/lib/ansible/config/manager.py +++ b/lib/ansible/config/manager.py @@ -450,13 +450,17 @@ class ConfigManager: pass def get_plugin_options(self, plugin_type, name, keys=None, variables=None, direct=None): + options, dummy = self.get_plugin_options_and_origins(plugin_type, name, keys=keys, variables=variables, direct=direct) + return options + def get_plugin_options_and_origins(self, plugin_type, name, keys=None, variables=None, direct=None): options = {} + origins = {} defs = self.get_configuration_definitions(plugin_type=plugin_type, name=name) for option in defs: - options[option] = self.get_config_value(option, plugin_type=plugin_type, plugin_name=name, keys=keys, variables=variables, direct=direct) - - return options + options[option], origins[option] = self.get_config_value_and_origin(option, plugin_type=plugin_type, plugin_name=name, keys=keys, + variables=variables, direct=direct) + return options, origins def get_plugin_vars(self, plugin_type, name): diff --git a/lib/ansible/plugins/__init__.py b/lib/ansible/plugins/__init__.py index 833f18e34e6..e259960299e 100644 --- a/lib/ansible/plugins/__init__.py +++ b/lib/ansible/plugins/__init__.py @@ -79,6 +79,7 @@ class AnsiblePlugin(_AnsiblePluginInfoMixin, _ConfigurablePlugin, metaclass=abc. def __init__(self): self._options = {} + self._origins = {} self._defs = None @property @@ -98,11 +99,16 @@ class AnsiblePlugin(_AnsiblePluginInfoMixin, _ConfigurablePlugin, metaclass=abc. return bool(possible_fqcns.intersection(set(self.ansible_aliases))) def get_option_and_origin(self, option, hostvars=None): - try: - option_value, origin = C.config.get_config_value_and_origin(option, plugin_type=self.plugin_type, plugin_name=self._load_name, variables=hostvars) - except AnsibleError as e: - raise KeyError(str(e)) - return option_value, origin + if option not in self._options: + try: + # some plugins don't use set_option(s) and cannot use direct settings, so this populates the local copy for them + self._options[option], self._origins[option] = C.config.get_config_value_and_origin(option, plugin_type=self.plugin_type, + plugin_name=self._load_name, variables=hostvars) + except AnsibleError as e: + # callers expect key error on missing + raise KeyError() from e + + return self._options[option], self._origins[option] @functools.cached_property def __plugin_info(self): @@ -113,11 +119,10 @@ class AnsiblePlugin(_AnsiblePluginInfoMixin, _ConfigurablePlugin, metaclass=abc. return _plugin_info.get_plugin_info(self) def get_option(self, option, hostvars=None): - if option not in self._options: - option_value, dummy = self.get_option_and_origin(option, hostvars=hostvars) - self.set_option(option, option_value) - return self._options.get(option) + # let it populate _options + self.get_option_and_origin(option, hostvars=hostvars) + return self._options[option] def get_options(self, hostvars=None): options = {} @@ -127,6 +132,7 @@ class AnsiblePlugin(_AnsiblePluginInfoMixin, _ConfigurablePlugin, metaclass=abc. def set_option(self, option, value): self._options[option] = C.config.get_config_value(option, plugin_type=self.plugin_type, plugin_name=self._load_name, direct={option: value}) + self._origins[option] = 'Direct' _display._report_config_warnings(self.__plugin_info) def set_options(self, task_keys=None, var_options=None, direct=None): @@ -137,12 +143,14 @@ class AnsiblePlugin(_AnsiblePluginInfoMixin, _ConfigurablePlugin, metaclass=abc. :arg var_options: Dict with either 'connection variables' :arg direct: Dict with 'direct assignment' """ - self._options = C.config.get_plugin_options(self.plugin_type, self._load_name, keys=task_keys, variables=var_options, direct=direct) + self._options, self._origins = C.config.get_plugin_options_and_origins(self.plugin_type, self._load_name, keys=task_keys, + variables=var_options, direct=direct) # allow extras/wildcards from vars that are not directly consumed in configuration # this is needed to support things like winrm that can have extended protocol options we don't directly handle if self.allow_extras and var_options and '_extras' in var_options: # these are largely unvalidated passthroughs, either plugin or underlying API will validate + # TODO: deprecate and remove, most plugins that needed this don't use this facility anymore self._options['_extras'] = var_options['_extras'] _display._report_config_warnings(self.__plugin_info) diff --git a/lib/ansible/plugins/callback/__init__.py b/lib/ansible/plugins/callback/__init__.py index 2fc52c45c74..aa8beaea290 100644 --- a/lib/ansible/plugins/callback/__init__.py +++ b/lib/ansible/plugins/callback/__init__.py @@ -228,10 +228,14 @@ class CallbackBase(AnsiblePlugin): def set_option(self, k, v): self._plugin_options[k] = C.config.get_config_value(k, plugin_type=self.plugin_type, plugin_name=self._load_name, direct={k: v}) + self._origins[k] = 'direct' def get_option(self, k, hostvars=None): return self._plugin_options[k] + def get_option_and_origin(self, k, hostvars=None): + return self._plugin_options[k], self._origins[k] + def has_option(self, option): return (option in self._plugin_options) @@ -241,7 +245,8 @@ class CallbackBase(AnsiblePlugin): """ # load from config - self._plugin_options = C.config.get_plugin_options(self.plugin_type, self._load_name, keys=task_keys, variables=var_options, direct=direct) + self._plugin_options, self._origins = C.config.get_plugin_options_and_origins(self.plugin_type, self._load_name, + keys=task_keys, variables=var_options, direct=direct) @staticmethod def host_label(result: CallbackTaskResult) -> str: diff --git a/test/integration/targets/config/lookup_plugins/broken.py b/test/integration/targets/config/lookup_plugins/broken.py new file mode 100644 index 00000000000..ee37b030d24 --- /dev/null +++ b/test/integration/targets/config/lookup_plugins/broken.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025, Felix Fontein , The Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +from __future__ import annotations + + +DOCUMENTATION = r""" +name: broken +short_description: Test input precedence +author: Felix Fontein (@felixfontein) +description: + - Test input precedence. +options: + _terms: + description: + - Ignored. + type: list + elements: str + required: true + some_option: + description: + - The interesting part. + type: str + default: default value + env: + - name: PLAYGROUND_TEST_1 + - name: PLAYGROUND_TEST_2 + vars: + - name: playground_test_1 + - name: playground_test_2 + ini: + - key: playground_test_1 + section: playground + - key: playground_test_2 + section: playground +""" + +EXAMPLES = r"""#""" + +RETURN = r""" +_list: + description: + - The value of O(some_option). + type: list + elements: str +""" + +from ansible.plugins.lookup import LookupBase + + +class LookupModule(LookupBase): + def run(self, terms, variables=None, **kwargs): + """Generate list.""" + self.set_options(var_options=variables, direct=kwargs) + + return [self.get_option("some_option"), *self.get_option_and_origin("some_option")] diff --git a/test/integration/targets/config/match_option_methods.yml b/test/integration/targets/config/match_option_methods.yml new file mode 100644 index 00000000000..c3f1412a78f --- /dev/null +++ b/test/integration/targets/config/match_option_methods.yml @@ -0,0 +1,37 @@ +- hosts: localhost + gather_facts: false + vars: + direct: "{{ query('broken', some_option='foo') }}" + default: "{{ query('broken') }}" + tasks: + - name: Set directly but also have vars + set_fact: + direct_with_vars: "{{ query('broken', some_option='foo') }}" + vars: + playground_test_1: var 1 + playground_test_2: var 2 + - name: Set via vars only + set_fact: + vars_only: "{{ query('broken') }}" + vars: + playground_test_1: var 1 + playground_test_2: var 2 + + - debug: msg={{q('vars', item)}} + loop: + - direct + - default + - direct_with_vars + - vars_only + + - name: now ensure it all worked as expected (simple value, origin value, origin) + assert: + that: + - direct[0] == direct[1] + - direct[2] == 'Direct' + - default[0] == default[1] + - default[2] == 'default' + - direct_with_vars[0] == direct_with_vars[1] + - direct_with_vars[2] == 'Direct' + - vars_only[0] == vars_only[1] + - vars_only[2].startswith('var:') diff --git a/test/integration/targets/config/runme.sh b/test/integration/targets/config/runme.sh index 0ed659d1b74..4f8d4eb396c 100755 --- a/test/integration/targets/config/runme.sh +++ b/test/integration/targets/config/runme.sh @@ -43,3 +43,6 @@ done # ensure we don't show default templates, but templated defaults [ "$(ansible-config init |grep '={{' -c )" -eq 0 ] + +# test seldom used '_and_origin' api +ansible-playbook match_option_methods.yml "$@" From cdb7af094bcd293b49ac2ff7cc03bc867b2e07d0 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Wed, 23 Jul 2025 14:42:45 -0700 Subject: [PATCH 35/68] ansible-test - Use OS packages on FreeBSD 13.5 (#85530) --- changelogs/fragments/ansible-test-freebsd-bootstrap.yml | 2 ++ test/lib/ansible_test/_util/target/setup/bootstrap.sh | 3 +++ 2 files changed, 5 insertions(+) create mode 100644 changelogs/fragments/ansible-test-freebsd-bootstrap.yml diff --git a/changelogs/fragments/ansible-test-freebsd-bootstrap.yml b/changelogs/fragments/ansible-test-freebsd-bootstrap.yml new file mode 100644 index 00000000000..39a6cc61d7d --- /dev/null +++ b/changelogs/fragments/ansible-test-freebsd-bootstrap.yml @@ -0,0 +1,2 @@ +minor_changes: + - ansible-test - Use OS packages to satisfy controller requirements on FreeBSD 13.5 during managed instance bootstrapping. diff --git a/test/lib/ansible_test/_util/target/setup/bootstrap.sh b/test/lib/ansible_test/_util/target/setup/bootstrap.sh index b2093e79523..4f3bce8d6e3 100644 --- a/test/lib/ansible_test/_util/target/setup/bootstrap.sh +++ b/test/lib/ansible_test/_util/target/setup/bootstrap.sh @@ -169,6 +169,9 @@ bootstrap_remote_freebsd() # Declare platform/python version combinations which do not have supporting OS packages available. # For these combinations ansible-test will use pip to install the requirements instead. case "${platform_version}/${python_version}" in + 13.5/3.11) + # defaults available + ;; 14.2/3.11) # defaults available ;; From 181dbdbf272c177e147eca9ea0edda9a6ee37863 Mon Sep 17 00:00:00 2001 From: Abhijeet Kasurde Date: Thu, 24 Jul 2025 08:37:15 -0700 Subject: [PATCH 36/68] filter: update integration tests (#84990) Signed-off-by: Abhijeet Kasurde --- .../fragments/password_hash_encrypt.yml | 3 ++ lib/ansible/utils/encrypt.py | 2 + .../targets/filter_core/tasks/main.yml | 48 +++++++++++++++---- .../targets/filter_core/tasks/to_json.yml | 28 +++++++++++ .../targets/filter_core/tasks/to_yaml.yml | 27 +++++++++++ 5 files changed, 98 insertions(+), 10 deletions(-) create mode 100644 changelogs/fragments/password_hash_encrypt.yml create mode 100644 test/integration/targets/filter_core/tasks/to_json.yml create mode 100644 test/integration/targets/filter_core/tasks/to_yaml.yml diff --git a/changelogs/fragments/password_hash_encrypt.yml b/changelogs/fragments/password_hash_encrypt.yml new file mode 100644 index 00000000000..5cbdfa7c053 --- /dev/null +++ b/changelogs/fragments/password_hash_encrypt.yml @@ -0,0 +1,3 @@ +--- +minor_changes: + - encrypt - check datatype of salt_size in password_hash filter. diff --git a/lib/ansible/utils/encrypt.py b/lib/ansible/utils/encrypt.py index 9f44394403a..016ab76a466 100644 --- a/lib/ansible/utils/encrypt.py +++ b/lib/ansible/utils/encrypt.py @@ -99,6 +99,8 @@ class PasslibHash(BaseHash): salt = self._clean_salt(salt) rounds = self._clean_rounds(rounds) ident = self._clean_ident(ident) + if salt_size is not None and not isinstance(salt_size, int): + raise TypeError("salt_size must be an integer") return self._hash(secret, salt=salt, salt_size=salt_size, rounds=rounds, ident=ident) def _clean_ident(self, ident): diff --git a/test/integration/targets/filter_core/tasks/main.yml b/test/integration/targets/filter_core/tasks/main.yml index e812eec6ac6..c11b21c40c6 100644 --- a/test/integration/targets/filter_core/tasks/main.yml +++ b/test/integration/targets/filter_core/tasks/main.yml @@ -384,6 +384,7 @@ that: - '"%Y-%m-%d"|strftime(1585247522) == "2020-03-26"' - '"%Y-%m-%d"|strftime("1585247522.0") == "2020-03-26"' + - '"%Y-%m-%d"|strftime("1585247522.0", utc=True) == "2020-03-26"' - '("%Y"|strftime(None)).startswith("20")' # Current date, can't check much there. - strftime_fail is failed - '"Invalid value for epoch value" in strftime_fail.msg' @@ -538,15 +539,11 @@ ignore_errors: yes register: password_hash_2 -- name: Verify password_hash - assert: - that: - - "'what in the WORLD is up?'|password_hash|length in (120, 106)" - # This throws a vastly different error on py2 vs py3, so we just check - # that it's a failure, not a substring of the exception. - - password_hash_1 is failed - - password_hash_2 is failed - - "'is not in the list of supported passlib algorithms' in password_hash_2.msg" +- name: Verify password_hash throws on weird rounds + set_fact: + foo: '{{ "hey" | password_hash(rounds=1) }}' + ignore_errors: yes + register: password_hash_3 - name: test using passlib with an unsupported hash type set_fact: @@ -554,8 +551,16 @@ ignore_errors: yes register: unsupported_hash_type -- assert: +- name: Verify password_hash + assert: that: + - "'what in the WORLD is up?'|password_hash|length in (120, 106)" + - password_hash_1 is failed + - "'salt_size must be an integer' in password_hash_1.msg" + - password_hash_2 is failed + - "'is not in the list of supported passlib algorithms' in password_hash_2.msg" + - password_hash_3 is failed + - "'Could not hash the secret' in password_hash_3.msg" - "'msdcc is not in the list of supported passlib algorithms' in unsupported_hash_type.msg" - name: Verify to_uuid throws on weird namespace @@ -832,3 +837,26 @@ splitty: - "1,2,3" - "4,5,6" + +- name: test to_yaml and to_nice_yaml + include_tasks: to_yaml.yml + +- name: test to_json and to_nice_json + include_tasks: to_json.yml + +- name: commonpath filter + set_fact: + msg: "{{ ['/foo/bar/foobar','/foo/bar'] | commonpath }}" + register: commonpath_01 + +- name: commonpath filter raises exception + set_fact: + msg: "{{ '/foo/bar/foobar' | commonpath }}" + register: commonpath_02 + ignore_errors: yes + +- name: Check if commonpath works + assert: + that: + - '"/foo/bar" in commonpath_01.ansible_facts.msg' + - "'|commonpath expects' in commonpath_02.msg" diff --git a/test/integration/targets/filter_core/tasks/to_json.yml b/test/integration/targets/filter_core/tasks/to_json.yml new file mode 100644 index 00000000000..b902a7a8853 --- /dev/null +++ b/test/integration/targets/filter_core/tasks/to_json.yml @@ -0,0 +1,28 @@ +# Copyright: Contributors to the Ansible project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +- name: Check if to_json works + set_fact: + msg: "{{ list_one | to_json }}" + vars: + list_one: + - one + - two + register: json_01 + +- name: Check if to_nice_json works + set_fact: + msg: "{{ list_one | to_json }}" + vars: + list_one: + - one + - two + register: json_02 + +- name: Assert + assert: + that: + - not json_01.failed + - json_01.ansible_facts.msg == "[\"one\", \"two\"]" + - not json_02.failed + - json_02.ansible_facts.msg == "[\"one\", \"two\"]" diff --git a/test/integration/targets/filter_core/tasks/to_yaml.yml b/test/integration/targets/filter_core/tasks/to_yaml.yml new file mode 100644 index 00000000000..f26f80c4639 --- /dev/null +++ b/test/integration/targets/filter_core/tasks/to_yaml.yml @@ -0,0 +1,27 @@ +# Copyright: Contributors to the Ansible project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +- name: Check to_yaml exception handling + set_fact: + msg: "{{ 'thing' | to_yaml(nonexistent_argument=True) }}" + register: yaml_test_01 + ignore_errors: yes + +- name: Check default_style + set_fact: + msg: "{{ 'thing' | to_yaml(default_style='|') }}" + register: yaml_test_02 + ignore_errors: yes + +- name: Check canonical + set_fact: + msg: "{{ 'thing' | to_yaml(canonical=True) }}" + register: yaml_test_03 + ignore_errors: yes + +- name: Test to_yaml + assert: + that: + - yaml_test_01.failed + - yaml_test_02.ansible_facts.msg == "|-\n thing\n" + - yaml_test_03.ansible_facts.msg == "---\n!!str \"thing\"\n" From 853752a3eb83691f61acc05c6ae322ad752d9c30 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Thu, 24 Jul 2025 21:09:06 -0700 Subject: [PATCH 37/68] ansible-test - Upgrade to coverage 7.10.0 (#85541) --- changelogs/fragments/ansible-test-coverage-upgrade.yml | 2 +- test/lib/ansible_test/_data/requirements/ansible-test.txt | 2 +- test/lib/ansible_test/_internal/coverage_util.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/changelogs/fragments/ansible-test-coverage-upgrade.yml b/changelogs/fragments/ansible-test-coverage-upgrade.yml index bd11de61b37..6f4b2062dd6 100644 --- a/changelogs/fragments/ansible-test-coverage-upgrade.yml +++ b/changelogs/fragments/ansible-test-coverage-upgrade.yml @@ -1,2 +1,2 @@ minor_changes: - - ansible-test - Upgrade to ``coverage`` version 7.9.1 for Python 3.9 and later. + - ansible-test - Upgrade to ``coverage`` version 7.10.0 for Python 3.9 and later. diff --git a/test/lib/ansible_test/_data/requirements/ansible-test.txt b/test/lib/ansible_test/_data/requirements/ansible-test.txt index 2ff3c71d169..5ae4ded86ca 100644 --- a/test/lib/ansible_test/_data/requirements/ansible-test.txt +++ b/test/lib/ansible_test/_data/requirements/ansible-test.txt @@ -1,3 +1,3 @@ # The test-constraints sanity test verifies this file, but changes must be made manually to keep it in up-to-date. -coverage == 7.9.1 ; python_version >= '3.9' and python_version <= '3.14' +coverage == 7.10.0 ; python_version >= '3.9' and python_version <= '3.14' coverage == 7.6.1 ; python_version >= '3.8' and python_version <= '3.8' diff --git a/test/lib/ansible_test/_internal/coverage_util.py b/test/lib/ansible_test/_internal/coverage_util.py index 3b2c7d19732..4d4824674bf 100644 --- a/test/lib/ansible_test/_internal/coverage_util.py +++ b/test/lib/ansible_test/_internal/coverage_util.py @@ -70,7 +70,7 @@ class CoverageVersion: COVERAGE_VERSIONS = ( # IMPORTANT: Keep this in sync with the ansible-test.txt requirements file. - CoverageVersion('7.9.1', 7, (3, 9), (3, 14)), + CoverageVersion('7.10.0', 7, (3, 9), (3, 14)), CoverageVersion('7.6.1', 7, (3, 8), (3, 8)), ) """ From ee297bb7cad2c6cf2f890a97a1c717af223c29fb Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Thu, 24 Jul 2025 21:09:30 -0700 Subject: [PATCH 38/68] ansible-test - Limit bootstrap package install retries (#85544) --- .../ansible-test-bootstrap-retry.yml | 2 + .../_util/target/setup/bootstrap.sh | 50 +++++++++++++------ 2 files changed, 36 insertions(+), 16 deletions(-) create mode 100644 changelogs/fragments/ansible-test-bootstrap-retry.yml diff --git a/changelogs/fragments/ansible-test-bootstrap-retry.yml b/changelogs/fragments/ansible-test-bootstrap-retry.yml new file mode 100644 index 00000000000..ce86a6f3f7c --- /dev/null +++ b/changelogs/fragments/ansible-test-bootstrap-retry.yml @@ -0,0 +1,2 @@ +bugfixes: + - ansible-test - Limit package install retries during managed remote instance bootstrapping. diff --git a/test/lib/ansible_test/_util/target/setup/bootstrap.sh b/test/lib/ansible_test/_util/target/setup/bootstrap.sh index 4f3bce8d6e3..b7c373380d4 100644 --- a/test/lib/ansible_test/_util/target/setup/bootstrap.sh +++ b/test/lib/ansible_test/_util/target/setup/bootstrap.sh @@ -2,6 +2,24 @@ set -eu +retry_init() +{ + attempt=0 +} + +retry_or_fail() +{ + attempt=$((attempt + 1)) + + if [ $attempt -gt 5 ]; then + echo "Failed to install packages. Giving up." + exit 1 + fi + + echo "Failed to install packages. Sleeping before trying again..." + sleep 10 +} + remove_externally_managed_marker() { "${python_interpreter}" -c ' @@ -64,13 +82,13 @@ install_pip() { ;; esac + retry_init while true; do curl --silent --show-error "${pip_bootstrap_url}" -o /tmp/get-pip.py && \ "${python_interpreter}" /tmp/get-pip.py --disable-pip-version-check --quiet && \ rm /tmp/get-pip.py \ && break - echo "Failed to install packages. Sleeping before trying again..." - sleep 10 + retry_or_fail done fi } @@ -99,21 +117,21 @@ bootstrap_remote_alpine() " fi + retry_init while true; do # shellcheck disable=SC2086 apk add -q ${packages} \ && break - echo "Failed to install packages. Sleeping before trying again..." - sleep 10 + retry_or_fail done # Upgrade the `libexpat` package to ensure that an upgraded Python (`pyexpat`) continues to work. + retry_init while true; do # shellcheck disable=SC2086 apk upgrade -q libexpat \ && break - echo "Failed to upgrade libexpat. Sleeping before trying again..." - sleep 10 + retry_or_fail done } @@ -138,12 +156,12 @@ bootstrap_remote_fedora() " fi + retry_init while true; do # shellcheck disable=SC2086 dnf install -q -y ${packages} \ && break - echo "Failed to install packages. Sleeping before trying again..." - sleep 10 + retry_or_fail done } @@ -194,13 +212,13 @@ bootstrap_remote_freebsd() " fi + retry_init while true; do # shellcheck disable=SC2086 env ASSUME_ALWAYS_YES=YES pkg bootstrap && \ pkg install -q -y ${packages} \ && break - echo "Failed to install packages. Sleeping before trying again..." - sleep 10 + retry_or_fail done install_pip @@ -275,12 +293,12 @@ bootstrap_remote_rhel_9() " fi + retry_init while true; do # shellcheck disable=SC2086 dnf install -q -y ${packages} \ && break - echo "Failed to install packages. Sleeping before trying again..." - sleep 10 + retry_or_fail done } @@ -305,12 +323,12 @@ bootstrap_remote_rhel_10() " fi + retry_init while true; do # shellcheck disable=SC2086 dnf install -q -y ${packages} \ && break - echo "Failed to install packages. Sleeping before trying again..." - sleep 10 + retry_or_fail done } @@ -357,13 +375,13 @@ bootstrap_remote_ubuntu() " fi + retry_init while true; do # shellcheck disable=SC2086 apt-get update -qq -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -qq -y --no-install-recommends ${packages} \ && break - echo "Failed to install packages. Sleeping before trying again..." - sleep 10 + retry_or_fail done } From ea238c9eb5e46d63b99f9bd81209eaa3d5e6ab9e Mon Sep 17 00:00:00 2001 From: Abhijeet Kasurde Date: Thu, 24 Jul 2025 21:35:30 -0700 Subject: [PATCH 39/68] tqm: Use correct warning methods (#85434) * Use display.error_as_warning instead of self.warning * Use display.error_as_warning instead of display.warning_as_error. Signed-off-by: Abhijeet Kasurde --- changelogs/fragments/tqm.yml | 4 ++++ lib/ansible/executor/task_queue_manager.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 changelogs/fragments/tqm.yml diff --git a/changelogs/fragments/tqm.yml b/changelogs/fragments/tqm.yml new file mode 100644 index 00000000000..d73157a8d51 --- /dev/null +++ b/changelogs/fragments/tqm.yml @@ -0,0 +1,4 @@ +--- +bugfixes: + - tqm - use display.error_as_warning instead of self.warning. + - tqm - use display.error_as_warning instead of display.warning_as_error. diff --git a/lib/ansible/executor/task_queue_manager.py b/lib/ansible/executor/task_queue_manager.py index c02f2b3a4f9..c48f3b9bb00 100644 --- a/lib/ansible/executor/task_queue_manager.py +++ b/lib/ansible/executor/task_queue_manager.py @@ -179,7 +179,7 @@ class TaskQueueManager: for fd in (STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO): os.set_inheritable(fd, False) except Exception as ex: - self.warning(f"failed to set stdio as non inheritable: {ex}") + display.error_as_warning("failed to set stdio as non inheritable", exception=ex) self._callback_lock = threading.Lock() @@ -269,7 +269,7 @@ class TaskQueueManager: display.warning("Skipping callback '%s', as it does not create a valid plugin instance." % callback_name) continue except Exception as ex: - display.warning_as_error(f"Failed to load callback plugin {callback_name!r}.", exception=ex) + display.error_as_warning(f"Failed to load callback plugin {callback_name!r}.", exception=ex) continue def run(self, play): From 35830cb05413bd3a2c8298e086b9a9ab23f9b9b1 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Fri, 25 Jul 2025 14:39:03 -0700 Subject: [PATCH 40/68] ansible-test - Fix coverage config (#85555) --- .../ansible-test-coverage-config.yml | 7 +++ .../commands/integration/coverage.py | 4 +- .../ansible_test/_internal/coverage_util.py | 53 ++++++++++--------- 3 files changed, 37 insertions(+), 27 deletions(-) create mode 100644 changelogs/fragments/ansible-test-coverage-config.yml diff --git a/changelogs/fragments/ansible-test-coverage-config.yml b/changelogs/fragments/ansible-test-coverage-config.yml new file mode 100644 index 00000000000..66a23c84f16 --- /dev/null +++ b/changelogs/fragments/ansible-test-coverage-config.yml @@ -0,0 +1,7 @@ +bugfixes: + - ansible-test - Use a consistent coverage config for all collection testing. + - ansible-test - Always exclude the ``tests/output/`` directory from a collection's code coverage. + (https://github.com/ansible/ansible/issues/84244) + +minor_changes: + - ansible-test - Improve formatting of generated coverage config file. diff --git a/test/lib/ansible_test/_internal/commands/integration/coverage.py b/test/lib/ansible_test/_internal/commands/integration/coverage.py index f518b5bd372..21cf0537e0f 100644 --- a/test/lib/ansible_test/_internal/commands/integration/coverage.py +++ b/test/lib/ansible_test/_internal/commands/integration/coverage.py @@ -167,7 +167,7 @@ class PosixCoverageHandler(CoverageHandler[PosixConfig]): coverage_config_path = os.path.join(self.common_temp_path, COVERAGE_CONFIG_NAME) coverage_output_path = os.path.join(self.common_temp_path, ResultType.COVERAGE.name) - coverage_config = generate_coverage_config(self.args) + coverage_config = generate_coverage_config() write_text_file(coverage_config_path, coverage_config, create_directories=True) @@ -260,7 +260,7 @@ class PosixCoverageHandler(CoverageHandler[PosixConfig]): """Return a dictionary of variables for setup and teardown of POSIX coverage.""" return dict( common_temp_dir=self.common_temp_path, - coverage_config=generate_coverage_config(self.args), + coverage_config=generate_coverage_config(), coverage_config_path=os.path.join(self.common_temp_path, COVERAGE_CONFIG_NAME), coverage_output_path=os.path.join(self.common_temp_path, ResultType.COVERAGE.name), mode_directory=f'{MODE_DIRECTORY:04o}', diff --git a/test/lib/ansible_test/_internal/coverage_util.py b/test/lib/ansible_test/_internal/coverage_util.py index 4d4824674bf..ae136ea842a 100644 --- a/test/lib/ansible_test/_internal/coverage_util.py +++ b/test/lib/ansible_test/_internal/coverage_util.py @@ -6,11 +6,10 @@ import dataclasses import os import sqlite3 import tempfile +import textwrap import typing as t from .config import ( - IntegrationConfig, - SanityConfig, TestConfig, ) @@ -218,7 +217,7 @@ def get_coverage_config(args: TestConfig) -> str: except AttributeError: pass - coverage_config = generate_coverage_config(args) + coverage_config = generate_coverage_config() if args.explain: temp_dir = '/tmp/coverage-temp-dir' @@ -236,10 +235,10 @@ def get_coverage_config(args: TestConfig) -> str: return path -def generate_coverage_config(args: TestConfig) -> str: +def generate_coverage_config() -> str: """Generate code coverage configuration for tests.""" if data_context().content.collection: - coverage_config = generate_collection_coverage_config(args) + coverage_config = generate_collection_coverage_config() else: coverage_config = generate_ansible_coverage_config() @@ -266,12 +265,29 @@ omit = */test/results/* """ + coverage_config = coverage_config.lstrip() + return coverage_config -def generate_collection_coverage_config(args: TestConfig) -> str: +def generate_collection_coverage_config() -> str: """Generate code coverage configuration for Ansible Collection tests.""" - coverage_config = """ + include_patterns = [ + # {base}/ansible_collections/{ns}/{col}/* + os.path.join(data_context().content.root, '*'), + # */ansible_collections/{ns}/{col}/* (required to pick up AnsiballZ coverage) + os.path.join('*', data_context().content.collection.directory, '*'), + ] + + omit_patterns = [ + # {base}/ansible_collections/{ns}/{col}/tests/output/* + os.path.join(data_context().content.root, data_context().content.results_path, '*'), + ] + + include = textwrap.indent('\n'.join(include_patterns), ' ' * 4) + omit = textwrap.indent('\n'.join(omit_patterns), ' ' * 4) + + coverage_config = f""" [run] branch = True concurrency = @@ -280,28 +296,15 @@ concurrency = parallel = True disable_warnings = no-data-collected -""" - if isinstance(args, IntegrationConfig): - coverage_config += """ -include = - %s/* - */%s/* -""" % (data_context().content.root, data_context().content.collection.directory) - elif isinstance(args, SanityConfig): - # temporary work-around for import sanity test - coverage_config += """ include = - %s/* +{include} omit = - %s/* -""" % (data_context().content.root, os.path.join(data_context().content.root, data_context().content.results_path)) - else: - coverage_config += """ -include = - %s/* -""" % data_context().content.root +{omit} +""" + + coverage_config = coverage_config.lstrip() return coverage_config From 2e71e5aeb915c3a90ef4b4d6a50cdb5822fa27b7 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Fri, 25 Jul 2025 15:11:24 -0700 Subject: [PATCH 41/68] ansible-test - Replace RHEL 9.5 with 9.6 (#85558) --- .azure-pipelines/azure-pipelines.yml | 16 ++++++++-------- changelogs/fragments/ansible-test-rhel-9.6.yml | 2 ++ .../lib/ansible_test/_data/completion/remote.txt | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 changelogs/fragments/ansible-test-rhel-9.6.yml diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml index 3c73f3f2590..2e814a157af 100644 --- a/.azure-pipelines/azure-pipelines.yml +++ b/.azure-pipelines/azure-pipelines.yml @@ -90,10 +90,10 @@ stages: targets: - name: macOS 15.3 test: macos/15.3 - - name: RHEL 9.5 py39 - test: rhel/9.5@3.9 - - name: RHEL 9.5 py312 - test: rhel/9.5@3.12 + - name: RHEL 9.6 py39 + test: rhel/9.6@3.9 + - name: RHEL 9.6 py312 + test: rhel/9.6@3.12 - name: RHEL 10.0 test: rhel/10.0 - name: FreeBSD 13.5 @@ -108,8 +108,8 @@ stages: targets: - name: macOS 15.3 test: macos/15.3 - - name: RHEL 9.5 - test: rhel/9.5 + - name: RHEL 9.6 + test: rhel/9.6 - name: RHEL 10.0 test: rhel/10.0 - name: FreeBSD 13.5 @@ -127,8 +127,8 @@ stages: test: alpine/3.21 - name: Fedora 41 test: fedora/41 - - name: RHEL 9.5 - test: rhel/9.5 + - name: RHEL 9.6 + test: rhel/9.6 - name: RHEL 10.0 test: rhel/10.0 - name: Ubuntu 24.04 diff --git a/changelogs/fragments/ansible-test-rhel-9.6.yml b/changelogs/fragments/ansible-test-rhel-9.6.yml new file mode 100644 index 00000000000..72aae77b96c --- /dev/null +++ b/changelogs/fragments/ansible-test-rhel-9.6.yml @@ -0,0 +1,2 @@ +minor_changes: + - ansible-test - Replace RHEL 9.5 with 9.6. diff --git a/test/lib/ansible_test/_data/completion/remote.txt b/test/lib/ansible_test/_data/completion/remote.txt index 71727f56d35..59e72bc823b 100644 --- a/test/lib/ansible_test/_data/completion/remote.txt +++ b/test/lib/ansible_test/_data/completion/remote.txt @@ -7,7 +7,7 @@ freebsd/14.2 python=3.11 python_dir=/usr/local/bin become=su_sudo provider=aws a freebsd python_dir=/usr/local/bin become=su_sudo provider=aws arch=x86_64 macos/15.3 python=3.13 python_dir=/usr/local/bin become=sudo provider=parallels arch=x86_64 macos python_dir=/usr/local/bin become=sudo provider=parallels arch=x86_64 -rhel/9.5 python=3.9,3.12 become=sudo provider=aws arch=x86_64 +rhel/9.6 python=3.9,3.12 become=sudo provider=aws arch=x86_64 rhel/10.0 python=3.12 become=sudo provider=aws arch=x86_64 rhel become=sudo provider=aws arch=x86_64 ubuntu/22.04 python=3.10 become=sudo provider=aws arch=x86_64 From 58c9f480291060a98a49c18e86b10b200b0a009b Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Fri, 25 Jul 2025 18:09:37 -0700 Subject: [PATCH 42/68] ansible-test - Replace FreeBSD 14.2 with 14.3 (#85561) * ansible-test - Replace FreeBSD 14.2 with 14.3 * Fix CA bundle search priority --- .azure-pipelines/azure-pipelines.yml | 8 ++++---- changelogs/fragments/ansible-test-freebsd-14.3.yml | 2 ++ test/integration/targets/uri/tasks/main.yml | 2 +- test/lib/ansible_test/_data/completion/remote.txt | 2 +- test/lib/ansible_test/_util/target/setup/bootstrap.sh | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) create mode 100644 changelogs/fragments/ansible-test-freebsd-14.3.yml diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml index 2e814a157af..cba19751b1a 100644 --- a/.azure-pipelines/azure-pipelines.yml +++ b/.azure-pipelines/azure-pipelines.yml @@ -98,8 +98,8 @@ stages: test: rhel/10.0 - name: FreeBSD 13.5 test: freebsd/13.5 - - name: FreeBSD 14.2 - test: freebsd/14.2 + - name: FreeBSD 14.3 + test: freebsd/14.3 groups: - 1 - 2 @@ -114,8 +114,8 @@ stages: test: rhel/10.0 - name: FreeBSD 13.5 test: freebsd/13.5 - - name: FreeBSD 14.2 - test: freebsd/14.2 + - name: FreeBSD 14.3 + test: freebsd/14.3 groups: - 3 - 4 diff --git a/changelogs/fragments/ansible-test-freebsd-14.3.yml b/changelogs/fragments/ansible-test-freebsd-14.3.yml new file mode 100644 index 00000000000..26bed1f894a --- /dev/null +++ b/changelogs/fragments/ansible-test-freebsd-14.3.yml @@ -0,0 +1,2 @@ +minor_changes: + - ansible-test - Replace FreeBSD 14.2 with 14.3. diff --git a/test/integration/targets/uri/tasks/main.yml b/test/integration/targets/uri/tasks/main.yml index fdf14b80431..4ab625f9db9 100644 --- a/test/integration/targets/uri/tasks/main.yml +++ b/test/integration/targets/uri/tasks/main.yml @@ -202,11 +202,11 @@ stat: path: '{{ item }}' loop: + - '{{ cafile_path.stdout_lines|default(["/_i_dont_exist_ca.pem"])|first }}' - /etc/ssl/certs/ca-bundle.crt - /etc/ssl/certs/ca-certificates.crt - /var/lib/ca-certificates/ca-bundle.pem - /usr/local/share/certs/ca-root-nss.crt - - '{{ cafile_path.stdout_lines|default(["/_i_dont_exist_ca.pem"])|first }}' - /etc/ssl/cert.pem register: ca_bundle_candidates diff --git a/test/lib/ansible_test/_data/completion/remote.txt b/test/lib/ansible_test/_data/completion/remote.txt index 59e72bc823b..8281bfac5e7 100644 --- a/test/lib/ansible_test/_data/completion/remote.txt +++ b/test/lib/ansible_test/_data/completion/remote.txt @@ -3,7 +3,7 @@ alpine become=doas_sudo provider=aws arch=x86_64 fedora/41 python=3.13 become=sudo provider=aws arch=x86_64 fedora become=sudo provider=aws arch=x86_64 freebsd/13.5 python=3.11 python_dir=/usr/local/bin become=su_sudo provider=aws arch=x86_64 -freebsd/14.2 python=3.11 python_dir=/usr/local/bin become=su_sudo provider=aws arch=x86_64 +freebsd/14.3 python=3.11 python_dir=/usr/local/bin become=su_sudo provider=aws arch=x86_64 freebsd python_dir=/usr/local/bin become=su_sudo provider=aws arch=x86_64 macos/15.3 python=3.13 python_dir=/usr/local/bin become=sudo provider=parallels arch=x86_64 macos python_dir=/usr/local/bin become=sudo provider=parallels arch=x86_64 diff --git a/test/lib/ansible_test/_util/target/setup/bootstrap.sh b/test/lib/ansible_test/_util/target/setup/bootstrap.sh index b7c373380d4..6947c34f21a 100644 --- a/test/lib/ansible_test/_util/target/setup/bootstrap.sh +++ b/test/lib/ansible_test/_util/target/setup/bootstrap.sh @@ -190,7 +190,7 @@ bootstrap_remote_freebsd() 13.5/3.11) # defaults available ;; - 14.2/3.11) + 14.3/3.11) # defaults available ;; *) From 9b266da14f0a941a2bb8deb5c43b57aad9ccea38 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Fri, 25 Jul 2025 18:43:40 -0700 Subject: [PATCH 43/68] ansible-test - Update containers and remotes (#85562) --- .azure-pipelines/azure-pipelines.yml | 28 +++++++++---------- changelogs/fragments/ansible-test-remotes.yml | 4 +++ .../ansible_test/_data/completion/docker.txt | 8 +++--- .../ansible_test/_data/completion/remote.txt | 4 +-- 4 files changed, 24 insertions(+), 20 deletions(-) create mode 100644 changelogs/fragments/ansible-test-remotes.yml diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml index cba19751b1a..9ec53512fbf 100644 --- a/.azure-pipelines/azure-pipelines.yml +++ b/.azure-pipelines/azure-pipelines.yml @@ -123,10 +123,10 @@ stages: - template: templates/matrix.yml # context/controller (ansible-test container management) parameters: targets: - - name: Alpine 3.21 - test: alpine/3.21 - - name: Fedora 41 - test: fedora/41 + - name: Alpine 3.22 + test: alpine/3.22 + - name: Fedora 42 + test: fedora/42 - name: RHEL 9.6 test: rhel/9.6 - name: RHEL 10.0 @@ -142,10 +142,10 @@ stages: parameters: testFormat: linux/{0} targets: - - name: Alpine 3.21 - test: alpine321 - - name: Fedora 41 - test: fedora41 + - name: Alpine 3.22 + test: alpine322 + - name: Fedora 42 + test: fedora42 - name: Ubuntu 22.04 test: ubuntu2204 - name: Ubuntu 24.04 @@ -157,10 +157,10 @@ stages: parameters: testFormat: linux/{0} targets: - - name: Alpine 3.21 - test: alpine321 - - name: Fedora 41 - test: fedora41 + - name: Alpine 3.22 + test: alpine322 + - name: Fedora 42 + test: fedora42 - name: Ubuntu 24.04 test: ubuntu2404 groups: @@ -171,8 +171,8 @@ stages: parameters: testFormat: linux/{0} targets: - - name: Fedora 41 - test: fedora41 + - name: Fedora 42 + test: fedora42 groups: - 7 - stage: Galaxy diff --git a/changelogs/fragments/ansible-test-remotes.yml b/changelogs/fragments/ansible-test-remotes.yml new file mode 100644 index 00000000000..75ed942912c --- /dev/null +++ b/changelogs/fragments/ansible-test-remotes.yml @@ -0,0 +1,4 @@ +minor_changes: + - ansible-test - Upgrade Fedora 41 to Fedora 42. + - ansible-test - Upgrade Alpine 3.21 to 3.22. + - ansible-test - Update Ubuntu containers. diff --git a/test/lib/ansible_test/_data/completion/docker.txt b/test/lib/ansible_test/_data/completion/docker.txt index ae4669343f0..6a473949062 100644 --- a/test/lib/ansible_test/_data/completion/docker.txt +++ b/test/lib/ansible_test/_data/completion/docker.txt @@ -1,7 +1,7 @@ base image=quay.io/ansible/base-test-container:8.2.0 python=3.13,3.8,3.9,3.10,3.11,3.12 default image=quay.io/ansible/default-test-container:11.6.0 python=3.13,3.8,3.9,3.10,3.11,3.12 context=collection default image=quay.io/ansible/ansible-core-test-container:11.6.0 python=3.13,3.8,3.9,3.10,3.11,3.12 context=ansible-core -alpine321 image=quay.io/ansible/alpine321-test-container:9.1.0 python=3.12 cgroup=none audit=none -fedora41 image=quay.io/ansible/fedora41-test-container:9.0.0 python=3.13 cgroup=v2-only -ubuntu2204 image=quay.io/ansible/ubuntu2204-test-container:9.0.0 python=3.10 -ubuntu2404 image=quay.io/ansible/ubuntu2404-test-container:9.0.0 python=3.12 +alpine322 image=quay.io/ansible/alpine322-test-container:10.0.0 python=3.12 cgroup=none audit=none +fedora42 image=quay.io/ansible/fedora42-test-container:10.0.0 python=3.13 cgroup=v2-only +ubuntu2204 image=quay.io/ansible/ubuntu2204-test-container:10.0.0 python=3.10 +ubuntu2404 image=quay.io/ansible/ubuntu2404-test-container:10.0.0 python=3.12 diff --git a/test/lib/ansible_test/_data/completion/remote.txt b/test/lib/ansible_test/_data/completion/remote.txt index 8281bfac5e7..7405c369889 100644 --- a/test/lib/ansible_test/_data/completion/remote.txt +++ b/test/lib/ansible_test/_data/completion/remote.txt @@ -1,6 +1,6 @@ -alpine/3.21 python=3.12 become=doas_sudo provider=aws arch=x86_64 +alpine/3.22 python=3.12 become=doas_sudo provider=aws arch=x86_64 alpine become=doas_sudo provider=aws arch=x86_64 -fedora/41 python=3.13 become=sudo provider=aws arch=x86_64 +fedora/42 python=3.13 become=sudo provider=aws arch=x86_64 fedora become=sudo provider=aws arch=x86_64 freebsd/13.5 python=3.11 python_dir=/usr/local/bin become=su_sudo provider=aws arch=x86_64 freebsd/14.3 python=3.11 python_dir=/usr/local/bin become=su_sudo provider=aws arch=x86_64 From 43a20c9f68d9feb6284462dde7527fa33f333cb3 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Mon, 28 Jul 2025 13:47:14 -0700 Subject: [PATCH 44/68] Fix paramiko deprecation unit test (#85573) The unit test previously assumed paramiko was installed, and would fail if it was not. --- .../plugins/connection/test_paramiko_ssh.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/test/units/plugins/connection/test_paramiko_ssh.py b/test/units/plugins/connection/test_paramiko_ssh.py index 94419055b18..68b77a1f01b 100644 --- a/test/units/plugins/connection/test_paramiko_ssh.py +++ b/test/units/plugins/connection/test_paramiko_ssh.py @@ -18,8 +18,10 @@ from __future__ import annotations +import sys from io import StringIO import pytest +import typing as t from ansible.module_utils import _internal from ansible.plugins.connection import paramiko_ssh as paramiko_ssh_module @@ -59,17 +61,25 @@ def test_paramiko_connect(play_context, in_stream, mocker): assert connection._connected is True -def test_deprecation_warning_controller(): +def test_deprecation_warning_controller(monkeypatch: pytest.MonkeyPatch) -> None: """Ensures deprecation warnings are generated for external paramiko imports.""" assert _internal.is_controller + sentinel: t.Any = object() + + monkeypatch.delitem(sys.modules, 'ansible') + monkeypatch.delitem(sys.modules, 'ansible.module_utils') + monkeypatch.delitem(sys.modules, 'ansible.module_utils.compat') + monkeypatch.delitem(sys.modules, 'ansible.module_utils.compat.paramiko') + monkeypatch.setitem(sys.modules, 'paramiko', sentinel) + # ensure direct access to `_` prefixed attrs does not warn with emits_warnings(deprecation_pattern=[], warning_pattern=[]): from ansible.module_utils.compat import paramiko - assert paramiko._paramiko is not None + assert paramiko._paramiko is sentinel assert isinstance(paramiko._PARAMIKO_IMPORT_ERR, (Exception, type(None))) with emits_warnings(deprecation_pattern=["The 'paramiko' compat import is deprecated", "The 'PARAMIKO_IMPORT_ERR' compat import is deprecated"]): from ansible.module_utils.compat import paramiko - assert paramiko.paramiko is not None + assert paramiko.paramiko is sentinel assert isinstance(paramiko.PARAMIKO_IMPORT_ERR, (Exception, type(None))) From 647e7409eb02af837b5c2856b802f328704931db Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Mon, 28 Jul 2025 15:04:36 -0700 Subject: [PATCH 45/68] Avoid mocker fixture in copy unit tests (#85575) The mocker fixture does not clean up mocks early enough, which can cause problems for pytest infrastructure. --- test/units/modules/test_copy.py | 54 +++++++++++++++++---------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/test/units/modules/test_copy.py b/test/units/modules/test_copy.py index 6f15bed1223..685f79fd0e2 100644 --- a/test/units/modules/test_copy.py +++ b/test/units/modules/test_copy.py @@ -5,6 +5,8 @@ from __future__ import annotations +import unittest.mock + import pytest from ansible.modules.copy import AnsibleModuleError, split_pre_existing_dir @@ -97,38 +99,38 @@ ONE_DIR_DATA += tuple(item[:3] for item in TWO_DIRS_DATA) @pytest.mark.parametrize('directory, expected', ((d[0], d[4]) for d in THREE_DIRS_DATA)) @pytest.mark.xfail(reason='broken test and/or code, original test missing assert', strict=False) -def test_split_pre_existing_dir_three_levels_exist(directory, expected, mocker): - mocker.patch('os.path.exists', side_effect=[True, True, True]) - assert split_pre_existing_dir(directory) == expected +def test_split_pre_existing_dir_three_levels_exist(directory, expected): + with unittest.mock.patch('os.path.exists', side_effect=[True, True, True]): + assert split_pre_existing_dir(directory) == expected @pytest.mark.parametrize('directory, expected', ((d[0], d[3]) for d in TWO_DIRS_DATA)) @pytest.mark.xfail(reason='broken test and/or code, original test missing assert', strict=False) -def test_split_pre_existing_dir_two_levels_exist(directory, expected, mocker): - mocker.patch('os.path.exists', side_effect=[True, True, False]) - assert split_pre_existing_dir(directory) == expected +def test_split_pre_existing_dir_two_levels_exist(directory, expected): + with unittest.mock.patch('os.path.exists', side_effect=[True, True, False]): + assert split_pre_existing_dir(directory) == expected @pytest.mark.parametrize('directory, expected', ((d[0], d[2]) for d in ONE_DIR_DATA)) @pytest.mark.xfail(reason='broken test and/or code, original test missing assert', strict=False) -def test_split_pre_existing_dir_one_level_exists(directory, expected, mocker): - mocker.patch('os.path.exists', side_effect=[True, False, False]) - assert split_pre_existing_dir(directory) == expected +def test_split_pre_existing_dir_one_level_exists(directory, expected): + with unittest.mock.patch('os.path.exists', side_effect=[True, False, False]): + assert split_pre_existing_dir(directory) == expected @pytest.mark.parametrize('directory', (d[0] for d in ONE_DIR_DATA if d[1] is None)) -def test_split_pre_existing_dir_root_does_not_exist(directory, mocker): - mocker.patch('os.path.exists', return_value=False) - with pytest.raises(AnsibleModuleError) as excinfo: - split_pre_existing_dir(directory) +def test_split_pre_existing_dir_root_does_not_exist(directory): + with unittest.mock.patch('os.path.exists', return_value=False): + with pytest.raises(AnsibleModuleError) as excinfo: + split_pre_existing_dir(directory) assert excinfo.value.results['msg'].startswith("The '/' directory doesn't exist on this machine.") @pytest.mark.parametrize('directory, expected', ((d[0], d[1]) for d in ONE_DIR_DATA if not d[0].startswith('/'))) @pytest.mark.xfail(reason='broken test and/or code, original test missing assert', strict=False) -def test_split_pre_existing_dir_working_dir_exists(directory, expected, mocker): - mocker.patch('os.path.exists', return_value=False) - assert split_pre_existing_dir(directory) == expected +def test_split_pre_existing_dir_working_dir_exists(directory, expected): + with unittest.mock.patch('os.path.exists', return_value=False): + assert split_pre_existing_dir(directory) == expected # @@ -206,26 +208,26 @@ INVALID_DATA = ( @pytest.mark.parametrize('stat_info, mode_string, expected', DATA) -def test_good_symbolic_modes(mocker, stat_info, mode_string, expected): - mock_stat = mocker.MagicMock() +def test_good_symbolic_modes(stat_info, mode_string, expected): + mock_stat = unittest.mock.MagicMock() mock_stat.st_mode = stat_info assert AnsibleModule._symbolic_mode_to_octal(mock_stat, mode_string) == expected @pytest.mark.parametrize('stat_info, mode_string, expected', UMASK_DATA) -def test_umask_with_symbolic_modes(mocker, stat_info, mode_string, expected): - mock_umask = mocker.patch('os.umask') - mock_umask.return_value = 0o7 +def test_umask_with_symbolic_modes(stat_info, mode_string, expected): + with unittest.mock.patch('os.umask') as mock_umask: + mock_umask.return_value = 0o7 - mock_stat = mocker.MagicMock() - mock_stat.st_mode = stat_info + mock_stat = unittest.mock.MagicMock() + mock_stat.st_mode = stat_info - assert AnsibleModule._symbolic_mode_to_octal(mock_stat, mode_string) == expected + assert AnsibleModule._symbolic_mode_to_octal(mock_stat, mode_string) == expected @pytest.mark.parametrize('stat_info, mode_string, expected', INVALID_DATA) -def test_invalid_symbolic_modes(mocker, stat_info, mode_string, expected): - mock_stat = mocker.MagicMock() +def test_invalid_symbolic_modes(stat_info, mode_string, expected): + mock_stat = unittest.mock.MagicMock() mock_stat.st_mode = stat_info with pytest.raises(ValueError) as exc: assert AnsibleModule._symbolic_mode_to_octal(mock_stat, mode_string) == 'blah' From 6b2b665ef7b3ea35676e2cda648d2adc919a92f2 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Mon, 28 Jul 2025 16:16:14 -0700 Subject: [PATCH 46/68] Add support for Python 3.14 and drop Python 3.8 (#85576) --- .azure-pipelines/azure-pipelines.yml | 2 +- changelogs/fragments/python-support.yml | 3 +++ lib/ansible/cli/__init__.py | 8 ++++--- .../_internal/_datatag/__init__.py | 12 ++-------- lib/ansible/module_utils/basic.py | 5 ++-- .../collection_loader/_collection_finder.py | 21 +++++++--------- .../ansible_test/_data/completion/docker.txt | 6 ++--- .../_data/pytest/config/legacy.ini | 4 ---- .../_data/requirements/ansible-test.txt | 1 - test/lib/ansible_test/_internal/cli/compat.py | 2 +- .../_internal/commands/units/__init__.py | 14 +---------- .../ansible_test/_internal/coverage_util.py | 1 - .../_util/target/common/constants.py | 2 +- test/sanity/code-smell/black.py | 2 ++ test/sanity/ignore.txt | 24 ++++++++++--------- .../_internal/_patches/test_patches.py | 2 ++ .../module_utils/common/test_collections.py | 2 +- .../test_check_required_arguments.py | 5 +++- 18 files changed, 50 insertions(+), 66 deletions(-) create mode 100644 changelogs/fragments/python-support.yml delete mode 100644 test/lib/ansible_test/_data/pytest/config/legacy.ini diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml index 9ec53512fbf..92bf8b8455a 100644 --- a/.azure-pipelines/azure-pipelines.yml +++ b/.azure-pipelines/azure-pipelines.yml @@ -54,12 +54,12 @@ stages: nameFormat: Python {0} testFormat: units/{0} targets: - - test: 3.8 - test: 3.9 - test: '3.10' - test: 3.11 - test: 3.12 - test: 3.13 + - test: 3.14 - stage: Windows dependsOn: [] jobs: diff --git a/changelogs/fragments/python-support.yml b/changelogs/fragments/python-support.yml new file mode 100644 index 00000000000..8b86c3245fb --- /dev/null +++ b/changelogs/fragments/python-support.yml @@ -0,0 +1,3 @@ +major_changes: + - ansible - Add support for Python 3.14. + - ansible - Drop support for Python 3.8 on targets. diff --git a/lib/ansible/cli/__init__.py b/lib/ansible/cli/__init__.py index 0adc00c9bc9..da5cacc13bf 100644 --- a/lib/ansible/cli/__init__.py +++ b/lib/ansible/cli/__init__.py @@ -23,10 +23,12 @@ if 1 <= len(sys.argv) <= 2 and os.path.basename(sys.argv[0]) == "ansible" and os # Used for determining if the system is running a new enough python version # and should only restrict on our documented minimum versions -if sys.version_info < (3, 11): +_PY_MIN = (3, 11) + +if sys.version_info < _PY_MIN: raise SystemExit( - 'ERROR: Ansible requires Python 3.11 or newer on the controller. ' - 'Current version: %s' % ''.join(sys.version.splitlines()) + f"ERROR: Ansible requires Python {'.'.join(map(str, _PY_MIN))} or newer on the controller. " + f"Current version: {''.join(sys.version.splitlines())}" ) diff --git a/lib/ansible/module_utils/_internal/_datatag/__init__.py b/lib/ansible/module_utils/_internal/_datatag/__init__.py index 479a0278d0a..a00214e2a02 100644 --- a/lib/ansible/module_utils/_internal/_datatag/__init__.py +++ b/lib/ansible/module_utils/_internal/_datatag/__init__.py @@ -500,17 +500,9 @@ class AnsibleDatatagBase(AnsibleSerializableDataclass, metaclass=abc.ABCMeta): _known_tag_type_map: t.Dict[str, t.Type[AnsibleDatatagBase]] = {} _known_tag_types: t.Set[t.Type[AnsibleDatatagBase]] = set() -if sys.version_info >= (3, 9): - # Include the key and value types in the type hints on Python 3.9 and later. - # Earlier versions do not support subscriptable dict. - # deprecated: description='always use subscriptable dict' python_version='3.8' - class _AnsibleTagsMapping(dict[type[_TAnsibleDatatagBase], _TAnsibleDatatagBase]): - __slots__ = _NO_INSTANCE_STORAGE -else: - - class _AnsibleTagsMapping(dict): - __slots__ = _NO_INSTANCE_STORAGE +class _AnsibleTagsMapping(dict[type[_TAnsibleDatatagBase], _TAnsibleDatatagBase]): + __slots__ = _NO_INSTANCE_STORAGE class _EmptyROInternalTagsMapping(dict): diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index b6104396ded..f361e2c84cf 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -11,12 +11,13 @@ import typing as t # Used for determining if the system is running a new enough python version # and should only restrict on our documented minimum versions -_PY_MIN = (3, 8) +_PY_MIN = (3, 9) if sys.version_info < _PY_MIN: print(json.dumps(dict( failed=True, - msg=f"ansible-core requires a minimum of Python version {'.'.join(map(str, _PY_MIN))}. Current version: {''.join(sys.version.splitlines())}", + msg=f"Ansible requires Python {'.'.join(map(str, _PY_MIN))} or newer on the target. " + f"Current version: {''.join(sys.version.splitlines())}", ))) sys.exit(1) diff --git a/lib/ansible/utils/collection_loader/_collection_finder.py b/lib/ansible/utils/collection_loader/_collection_finder.py index c654667d978..961fcef546f 100644 --- a/lib/ansible/utils/collection_loader/_collection_finder.py +++ b/lib/ansible/utils/collection_loader/_collection_finder.py @@ -26,20 +26,15 @@ from . import _to_bytes, _to_text from ._collection_config import AnsibleCollectionConfig try: - try: - # Available on Python >= 3.11 - # We ignore the import error that will trigger when running mypy with - # older Python versions. - from importlib.resources.abc import TraversableResources # type: ignore[import] - except ImportError: - # Used with Python 3.9 and 3.10 only - # This member is still available as an alias up until Python 3.14 but - # is deprecated as of Python 3.12. - from importlib.abc import TraversableResources # deprecated: description='TraversableResources move' python_version='3.10' + # Available on Python >= 3.11 + # We ignore the import error that will trigger when running mypy with + # older Python versions. + from importlib.resources.abc import TraversableResources # type: ignore[import] except ImportError: - # Python < 3.9 - # deprecated: description='TraversableResources fallback' python_version='3.8' - TraversableResources = object # type: ignore[assignment,misc] + # Used with Python 3.9 and 3.10 only + # This member is still available as an alias up until Python 3.14 but + # is deprecated as of Python 3.12. + from importlib.abc import TraversableResources # deprecated: description='TraversableResources move' python_version='3.10' # NB: this supports import sanity test providing a different impl try: diff --git a/test/lib/ansible_test/_data/completion/docker.txt b/test/lib/ansible_test/_data/completion/docker.txt index 6a473949062..7d1823c5046 100644 --- a/test/lib/ansible_test/_data/completion/docker.txt +++ b/test/lib/ansible_test/_data/completion/docker.txt @@ -1,6 +1,6 @@ -base image=quay.io/ansible/base-test-container:8.2.0 python=3.13,3.8,3.9,3.10,3.11,3.12 -default image=quay.io/ansible/default-test-container:11.6.0 python=3.13,3.8,3.9,3.10,3.11,3.12 context=collection -default image=quay.io/ansible/ansible-core-test-container:11.6.0 python=3.13,3.8,3.9,3.10,3.11,3.12 context=ansible-core +base image=quay.io/ansible/base-test-container:9.1.0 python=3.13,3.9,3.10,3.11,3.12,3.14 +default image=quay.io/ansible/default-test-container:12.0.0 python=3.13,3.9,3.10,3.11,3.12,3.14 context=collection +default image=quay.io/ansible/ansible-core-test-container:12.0.0 python=3.13,3.9,3.10,3.11,3.12,3.14 context=ansible-core alpine322 image=quay.io/ansible/alpine322-test-container:10.0.0 python=3.12 cgroup=none audit=none fedora42 image=quay.io/ansible/fedora42-test-container:10.0.0 python=3.13 cgroup=v2-only ubuntu2204 image=quay.io/ansible/ubuntu2204-test-container:10.0.0 python=3.10 diff --git a/test/lib/ansible_test/_data/pytest/config/legacy.ini b/test/lib/ansible_test/_data/pytest/config/legacy.ini deleted file mode 100644 index b2668dc2871..00000000000 --- a/test/lib/ansible_test/_data/pytest/config/legacy.ini +++ /dev/null @@ -1,4 +0,0 @@ -[pytest] -xfail_strict = true -mock_use_standalone_module = true -junit_family = xunit1 diff --git a/test/lib/ansible_test/_data/requirements/ansible-test.txt b/test/lib/ansible_test/_data/requirements/ansible-test.txt index 5ae4ded86ca..404cc49b2c0 100644 --- a/test/lib/ansible_test/_data/requirements/ansible-test.txt +++ b/test/lib/ansible_test/_data/requirements/ansible-test.txt @@ -1,3 +1,2 @@ # The test-constraints sanity test verifies this file, but changes must be made manually to keep it in up-to-date. coverage == 7.10.0 ; python_version >= '3.9' and python_version <= '3.14' -coverage == 7.6.1 ; python_version >= '3.8' and python_version <= '3.8' diff --git a/test/lib/ansible_test/_internal/cli/compat.py b/test/lib/ansible_test/_internal/cli/compat.py index 43645695c5a..c5363673dcd 100644 --- a/test/lib/ansible_test/_internal/cli/compat.py +++ b/test/lib/ansible_test/_internal/cli/compat.py @@ -69,7 +69,7 @@ def controller_python(version: t.Optional[str]) -> t.Optional[str]: def get_fallback_remote_controller() -> str: """Return the remote fallback platform for the controller.""" - platform = 'freebsd' # lower cost than RHEL and macOS + platform = 'fedora' # Fedora is lower cost than other remotes and always supports a recent Python version candidates = [item for item in filter_completion(remote_completion()).values() if item.controller_supported and item.platform == platform] fallback = sorted(candidates, key=lambda value: str_to_version(value.version), reverse=True)[0] return fallback.name diff --git a/test/lib/ansible_test/_internal/commands/units/__init__.py b/test/lib/ansible_test/_internal/commands/units/__init__.py index 1e2f84b38be..91562da1c11 100644 --- a/test/lib/ansible_test/_internal/commands/units/__init__.py +++ b/test/lib/ansible_test/_internal/commands/units/__init__.py @@ -241,19 +241,7 @@ def command_units(args: UnitsConfig) -> None: sys.exit() for test_context, python, paths, env in test_sets: - # When using pytest-mock, make sure that features introduced in Python 3.8 are available to older Python versions. - # This is done by enabling the mock_use_standalone_module feature, which forces use of mock even when unittest.mock is available. - # Later Python versions have not introduced additional unittest.mock features, so use of mock is not needed as of Python 3.8. - # If future Python versions introduce new unittest.mock features, they will not be available to older Python versions. - # Having the cutoff at Python 3.8 also eases packaging of ansible-core since no supported controller version requires the use of mock. - # - # NOTE: This only affects use of pytest-mock. - # Collection unit tests may directly import mock, which will be provided by ansible-test when it installs requirements using pip. - # Although mock is available for ansible-core unit tests, they should import unittest.mock instead. - if str_to_version(python.version) < (3, 8): - config_name = 'legacy.ini' - else: - config_name = 'default.ini' + config_name = 'default.ini' cmd = [ 'pytest', diff --git a/test/lib/ansible_test/_internal/coverage_util.py b/test/lib/ansible_test/_internal/coverage_util.py index ae136ea842a..95ec08a483c 100644 --- a/test/lib/ansible_test/_internal/coverage_util.py +++ b/test/lib/ansible_test/_internal/coverage_util.py @@ -70,7 +70,6 @@ class CoverageVersion: COVERAGE_VERSIONS = ( # IMPORTANT: Keep this in sync with the ansible-test.txt requirements file. CoverageVersion('7.10.0', 7, (3, 9), (3, 14)), - CoverageVersion('7.6.1', 7, (3, 8), (3, 8)), ) """ This tuple specifies the coverage version to use for Python version ranges. diff --git a/test/lib/ansible_test/_util/target/common/constants.py b/test/lib/ansible_test/_util/target/common/constants.py index 31f56adcdae..ad412aa23df 100644 --- a/test/lib/ansible_test/_util/target/common/constants.py +++ b/test/lib/ansible_test/_util/target/common/constants.py @@ -5,7 +5,6 @@ from __future__ import annotations REMOTE_ONLY_PYTHON_VERSIONS = ( - '3.8', '3.9', '3.10', ) @@ -14,4 +13,5 @@ CONTROLLER_PYTHON_VERSIONS = ( '3.11', '3.12', '3.13', + '3.14', ) diff --git a/test/sanity/code-smell/black.py b/test/sanity/code-smell/black.py index f066eeb59b4..c1dbea01920 100644 --- a/test/sanity/code-smell/black.py +++ b/test/sanity/code-smell/black.py @@ -21,6 +21,8 @@ def main() -> None: remote_only_python_versions = os.environ['ANSIBLE_TEST_REMOTE_ONLY_PYTHON_VERSIONS'].split(',') fix_mode = bool(int(os.environ['ANSIBLE_TEST_FIX_MODE'])) + controller_python_versions.remove('3.14') # black does not yet support formatting for Python 3.14 + target_python_versions = remote_only_python_versions + controller_python_versions black(controller_paths, controller_python_versions, fix_mode) diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index 94f0ee69336..dae96e59468 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -27,12 +27,12 @@ lib/ansible/modules/systemd_service.py validate-modules:parameter-invalid lib/ansible/modules/user.py validate-modules:doc-default-does-not-match-spec lib/ansible/modules/user.py validate-modules:use-run-command-not-popen lib/ansible/module_utils/basic.py pylint:unused-import # deferring resolution to allow enabling the rule now -lib/ansible/module_utils/compat/selinux.py import-3.8!skip # pass/fail depends on presence of libselinux.so lib/ansible/module_utils/compat/selinux.py import-3.9!skip # pass/fail depends on presence of libselinux.so lib/ansible/module_utils/compat/selinux.py import-3.10!skip # pass/fail depends on presence of libselinux.so lib/ansible/module_utils/compat/selinux.py import-3.11!skip # pass/fail depends on presence of libselinux.so lib/ansible/module_utils/compat/selinux.py import-3.12!skip # pass/fail depends on presence of libselinux.so lib/ansible/module_utils/compat/selinux.py import-3.13!skip # pass/fail depends on presence of libselinux.so +lib/ansible/module_utils/compat/selinux.py import-3.14!skip # pass/fail depends on presence of libselinux.so lib/ansible/module_utils/compat/selinux.py pylint:unidiomatic-typecheck lib/ansible/module_utils/distro/_distro.py no-assert lib/ansible/module_utils/powershell/Ansible.ModuleUtils.ArgvParser.psm1 pslint:PSUseApprovedVerbs @@ -57,6 +57,7 @@ lib/ansible/plugins/inventory/host_list.py pylint:arguments-renamed lib/ansible/_internal/_wrapt.py mypy-3.11!skip # vendored code lib/ansible/_internal/_wrapt.py mypy-3.12!skip # vendored code lib/ansible/_internal/_wrapt.py mypy-3.13!skip # vendored code +lib/ansible/_internal/_wrapt.py mypy-3.14!skip # vendored code lib/ansible/_internal/_wrapt.py pep8!skip # vendored code lib/ansible/_internal/_wrapt.py pylint!skip # vendored code lib/ansible/_internal/ansible_collections/ansible/_protomatter/README.md no-unwanted-files @@ -211,16 +212,17 @@ test/units/module_utils/facts/test_facts.py mypy-3.9:assignment test/units/modules/mount_facts_data.py mypy-3.9:arg-type test/units/modules/test_apt.py mypy-3.9:name-match test/units/modules/test_mount_facts.py mypy-3.9:index -test/units/module_utils/basic/test_exit_json.py mypy-3.8:assignment -test/units/module_utils/basic/test_exit_json.py mypy-3.8:misc -test/units/module_utils/facts/other/test_facter.py mypy-3.8:assignment -test/units/module_utils/facts/other/test_ohai.py mypy-3.8:assignment -test/units/module_utils/facts/system/test_lsb.py mypy-3.8:assignment -test/units/module_utils/facts/test_collectors.py mypy-3.8:assignment -test/units/module_utils/facts/test_facts.py mypy-3.8:assignment -test/units/modules/mount_facts_data.py mypy-3.8:arg-type -test/units/modules/test_apt.py mypy-3.8:name-match -test/units/modules/test_mount_facts.py mypy-3.8:index +test/units/module_utils/basic/test_exit_json.py mypy-3.14:assignment +test/units/module_utils/basic/test_exit_json.py mypy-3.14:misc +test/units/module_utils/facts/other/test_facter.py mypy-3.14:assignment +test/units/module_utils/facts/other/test_ohai.py mypy-3.14:assignment +test/units/module_utils/facts/system/test_lsb.py mypy-3.14:assignment +test/units/module_utils/facts/test_collectors.py mypy-3.14:assignment +test/units/module_utils/facts/test_facts.py mypy-3.14:assignment +test/units/modules/mount_facts_data.py mypy-3.14:arg-type +test/units/modules/test_apt.py mypy-3.14:name-match +test/units/modules/test_mount_facts.py mypy-3.14:index +test/units/playbook/test_base.py mypy-3.14:assignment test/integration/targets/interpreter_discovery_python/library/test_non_python_interpreter.py shebang # test needs non-standard shebang test/integration/targets/inventory_script/bad_shebang shebang # test needs an invalid shebang test/integration/targets/ansible-test-sanity-pylint/ansible_collections/ns/col/plugins/lookup/deprecated.py pylint!skip # validated as a collection diff --git a/test/units/module_utils/_internal/_patches/test_patches.py b/test/units/module_utils/_internal/_patches/test_patches.py index 3710e6f3990..08c84972868 100644 --- a/test/units/module_utils/_internal/_patches/test_patches.py +++ b/test/units/module_utils/_internal/_patches/test_patches.py @@ -10,6 +10,7 @@ import typing as t import pytest from ansible.module_utils._internal import _patches +from ansible.module_utils._internal._patches import _socket_patch from ansible.module_utils.common._utils import get_all_subclasses module_to_patch = sys.modules[__name__] @@ -36,6 +37,7 @@ def get_patch_required_test_cases() -> list: xfail_patch_when: dict[type[_patches.CallablePatch], bool] = { # Example: # _patches._some_patch_module.SomePatchClass: sys.version_info >= (3, 13), + _socket_patch.GetAddrInfoPatch: sys.version_info >= (3, 14), } patches = sorted(get_all_subclasses(_patches.CallablePatch), key=lambda item: item.__name__) diff --git a/test/units/module_utils/common/test_collections.py b/test/units/module_utils/common/test_collections.py index 7a95c515171..2e31b6b4787 100644 --- a/test/units/module_utils/common/test_collections.py +++ b/test/units/module_utils/common/test_collections.py @@ -130,7 +130,7 @@ class TestImmutableDict: # ImmutableDict is unhashable when one of its values is unhashable imdict = ImmutableDict({u'café': u'くらとみ', 1: [1, 2]}) - expected_reason = r"^unhashable type: 'list'$" + expected_reason = r"unhashable type: 'list'" with pytest.raises(TypeError, match=expected_reason): hash(imdict) diff --git a/test/units/module_utils/common/validation/test_check_required_arguments.py b/test/units/module_utils/common/validation/test_check_required_arguments.py index 16e79fe7dc2..7f8ab2b7728 100644 --- a/test/units/module_utils/common/validation/test_check_required_arguments.py +++ b/test/units/module_utils/common/validation/test_check_required_arguments.py @@ -4,6 +4,8 @@ from __future__ import annotations +import sys + import pytest from ansible.module_utils.common.text.converters import to_native @@ -84,4 +86,5 @@ def test_check_required_arguments_missing_none(): def test_check_required_arguments_no_params(arguments_terms): with pytest.raises(TypeError) as te: check_required_arguments(arguments_terms, None) - assert "'NoneType' is not iterable" in to_native(te.value) + expected = "argument of type 'NoneType' is not a container or iterable" if sys.version_info >= (3, 14) else "'NoneType' is not iterable" + assert expected in to_native(te.value) From 5fbce46dcf12804b2786d1eda2c2f809a8d9bc7f Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Mon, 28 Jul 2025 18:19:34 -0700 Subject: [PATCH 47/68] ansible-test - Update pinned sanity test requirements (#85577) --- .../ansible-test-sanity-requirements.yml | 2 ++ lib/ansible/_internal/_templating/_engine.py | 2 +- .../collection_loader/_collection_finder.py | 3 ++- .../_data/requirements/sanity.changelog.txt | 2 +- .../_data/requirements/sanity.pep8.txt | 2 +- .../_data/requirements/sanity.pylint.txt | 4 ++-- .../commands/integration/__init__.py | 2 +- test/sanity/code-smell/black.requirements.txt | 2 +- test/sanity/code-smell/mypy.requirements.txt | 24 ++++++++++--------- test/sanity/code-smell/mypy/ansible-core.ini | 3 +++ .../code-smell/pymarkdown.requirements.txt | 7 +++--- test/sanity/ignore.txt | 4 ---- 12 files changed, 31 insertions(+), 26 deletions(-) create mode 100644 changelogs/fragments/ansible-test-sanity-requirements.yml diff --git a/changelogs/fragments/ansible-test-sanity-requirements.yml b/changelogs/fragments/ansible-test-sanity-requirements.yml new file mode 100644 index 00000000000..9ed05f794c7 --- /dev/null +++ b/changelogs/fragments/ansible-test-sanity-requirements.yml @@ -0,0 +1,2 @@ +minor_changes: + - ansible-test - Update pinned sanity test requirements. diff --git a/lib/ansible/_internal/_templating/_engine.py b/lib/ansible/_internal/_templating/_engine.py index de3d70e38d1..4beb1806291 100644 --- a/lib/ansible/_internal/_templating/_engine.py +++ b/lib/ansible/_internal/_templating/_engine.py @@ -194,7 +194,7 @@ class TemplateEngine: return self._variables @available_variables.setter - def available_variables(self, variables: dict[str, t.Any]) -> None: + def available_variables(self, variables: dict[str, t.Any] | ChainMap[str, t.Any]) -> None: self._variables = variables def resolve_variable_expression( diff --git a/lib/ansible/utils/collection_loader/_collection_finder.py b/lib/ansible/utils/collection_loader/_collection_finder.py index 961fcef546f..27320806957 100644 --- a/lib/ansible/utils/collection_loader/_collection_finder.py +++ b/lib/ansible/utils/collection_loader/_collection_finder.py @@ -34,7 +34,8 @@ except ImportError: # Used with Python 3.9 and 3.10 only # This member is still available as an alias up until Python 3.14 but # is deprecated as of Python 3.12. - from importlib.abc import TraversableResources # deprecated: description='TraversableResources move' python_version='3.10' + # deprecated: description='TraversableResources move' python_version='3.10' + from importlib.abc import TraversableResources # type: ignore[assignment,no-redef] # NB: this supports import sanity test providing a different impl try: diff --git a/test/lib/ansible_test/_data/requirements/sanity.changelog.txt b/test/lib/ansible_test/_data/requirements/sanity.changelog.txt index 368734b8940..8a7226bee8f 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.changelog.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.changelog.txt @@ -6,4 +6,4 @@ PyYAML==6.0.2 rstcheck==5.0.0 semantic-version==2.10.0 types-docutils==0.18.3 -typing_extensions==4.13.2 +typing_extensions==4.14.1 diff --git a/test/lib/ansible_test/_data/requirements/sanity.pep8.txt b/test/lib/ansible_test/_data/requirements/sanity.pep8.txt index 0d62b8f0610..b950209d6ab 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.pep8.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.pep8.txt @@ -1,2 +1,2 @@ # edit "sanity.pep8.in" and generate with: hacking/update-sanity-requirements.py --test pep8 -pycodestyle==2.13.0 +pycodestyle==2.14.0 diff --git a/test/lib/ansible_test/_data/requirements/sanity.pylint.txt b/test/lib/ansible_test/_data/requirements/sanity.pylint.txt index f8daebf3804..bf28bb62b59 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.pylint.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.pylint.txt @@ -1,9 +1,9 @@ # edit "sanity.pylint.in" and generate with: hacking/update-sanity-requirements.py --test pylint -astroid==3.3.10 +astroid==3.3.11 dill==0.4.0 isort==6.0.1 mccabe==0.7.0 platformdirs==4.3.8 pylint==3.3.7 PyYAML==6.0.2 -tomlkit==0.13.2 +tomlkit==0.13.3 diff --git a/test/lib/ansible_test/_internal/commands/integration/__init__.py b/test/lib/ansible_test/_internal/commands/integration/__init__.py index 19d513c9da1..029164d65c9 100644 --- a/test/lib/ansible_test/_internal/commands/integration/__init__.py +++ b/test/lib/ansible_test/_internal/commands/integration/__init__.py @@ -331,7 +331,7 @@ def integration_test_environment( display.info('Copying %s/ to %s/' % (dir_src, dir_dst), verbosity=2) if not args.explain: - shutil.copytree(to_bytes(dir_src), to_bytes(dir_dst), symlinks=True) # type: ignore[arg-type] # incorrect type stub omits bytes path support + shutil.copytree(to_bytes(dir_src), to_bytes(dir_dst), symlinks=True) # type: ignore[type-var,arg-type] # type stub omits bytes path support for file_src, file_dst in file_copies: display.info('Copying %s to %s' % (file_src, file_dst), verbosity=2) diff --git a/test/sanity/code-smell/black.requirements.txt b/test/sanity/code-smell/black.requirements.txt index ad9b228f2f2..ffb59dd6b3a 100644 --- a/test/sanity/code-smell/black.requirements.txt +++ b/test/sanity/code-smell/black.requirements.txt @@ -1,6 +1,6 @@ # edit "black.requirements.in" and generate with: hacking/update-sanity-requirements.py --test black black==25.1.0 -click==8.2.0 +click==8.2.1 mypy_extensions==1.1.0 packaging==25.0 pathspec==0.12.1 diff --git a/test/sanity/code-smell/mypy.requirements.txt b/test/sanity/code-smell/mypy.requirements.txt index 22d69830fbc..fd91a7b5352 100644 --- a/test/sanity/code-smell/mypy.requirements.txt +++ b/test/sanity/code-smell/mypy.requirements.txt @@ -1,22 +1,24 @@ # edit "mypy.requirements.in" and generate with: hacking/update-sanity-requirements.py --test mypy cffi==1.17.1 -cryptography==44.0.3 +cryptography==45.0.5 iniconfig==2.1.0 Jinja2==3.1.6 MarkupSafe==3.0.2 -mypy==1.15.0 +mypy==1.17.0 mypy_extensions==1.1.0 packaging==25.0 -pluggy==1.5.0 +pathspec==0.12.1 +pluggy==1.6.0 pycparser==2.22 -pytest==8.3.5 -pytest-mock==3.14.0 +Pygments==2.19.2 +pytest==8.4.1 +pytest-mock==3.14.1 tomli==2.2.1 types-backports==0.1.3 -types-paramiko==3.5.0.20240928 -types-PyYAML==6.0.12.20250402 -types-requests==2.32.0.20250328 -types-setuptools==80.4.0.20250511 +types-paramiko==3.5.0.20250708 +types-PyYAML==6.0.12.20250516 +types-requests==2.32.4.20250611 +types-setuptools==80.9.0.20250529 types-toml==0.10.8.20240310 -typing_extensions==4.13.2 -urllib3==2.4.0 +typing_extensions==4.14.1 +urllib3==2.5.0 diff --git a/test/sanity/code-smell/mypy/ansible-core.ini b/test/sanity/code-smell/mypy/ansible-core.ini index ae51059471b..7703e800387 100644 --- a/test/sanity/code-smell/mypy/ansible-core.ini +++ b/test/sanity/code-smell/mypy/ansible-core.ini @@ -26,6 +26,9 @@ ignore_missing_imports = True [mypy-ansible.module_utils.six.moves.*] ignore_missing_imports = True +[mypy-pkg_resources.*] +ignore_missing_imports = True + [mypy-coverage.*] ignore_missing_imports = True diff --git a/test/sanity/code-smell/pymarkdown.requirements.txt b/test/sanity/code-smell/pymarkdown.requirements.txt index 88497f3d0fa..3062b71ad06 100644 --- a/test/sanity/code-smell/pymarkdown.requirements.txt +++ b/test/sanity/code-smell/pymarkdown.requirements.txt @@ -1,9 +1,10 @@ # edit "pymarkdown.requirements.in" and generate with: hacking/update-sanity-requirements.py --test pymarkdown -application_properties==0.8.2 +application_properties==0.9.0 Columnar==1.4.1 -pymarkdownlnt==0.9.29 +pyjson5==1.6.9 +pymarkdownlnt==0.9.31 PyYAML==6.0.2 tomli==2.2.1 toolz==1.0.0 -typing_extensions==4.13.2 +typing_extensions==4.14.1 wcwidth==0.2.13 diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index dae96e59468..f2dd0a3f405 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -169,7 +169,6 @@ test/units/module_utils/facts/test_facts.py mypy-3.13:assignment test/units/modules/mount_facts_data.py mypy-3.13:arg-type test/units/modules/test_apt.py mypy-3.13:name-match test/units/modules/test_mount_facts.py mypy-3.13:index -test/units/playbook/test_base.py mypy-3.13:assignment test/units/module_utils/basic/test_exit_json.py mypy-3.12:assignment test/units/module_utils/basic/test_exit_json.py mypy-3.12:misc test/units/module_utils/facts/other/test_facter.py mypy-3.12:assignment @@ -180,7 +179,6 @@ test/units/module_utils/facts/test_facts.py mypy-3.12:assignment test/units/modules/mount_facts_data.py mypy-3.12:arg-type test/units/modules/test_apt.py mypy-3.12:name-match test/units/modules/test_mount_facts.py mypy-3.12:index -test/units/playbook/test_base.py mypy-3.12:assignment test/units/module_utils/basic/test_exit_json.py mypy-3.11:assignment test/units/module_utils/basic/test_exit_json.py mypy-3.11:misc test/units/module_utils/facts/other/test_facter.py mypy-3.11:assignment @@ -191,7 +189,6 @@ test/units/module_utils/facts/test_facts.py mypy-3.11:assignment test/units/modules/mount_facts_data.py mypy-3.11:arg-type test/units/modules/test_apt.py mypy-3.11:name-match test/units/modules/test_mount_facts.py mypy-3.11:index -test/units/playbook/test_base.py mypy-3.11:assignment test/units/module_utils/basic/test_exit_json.py mypy-3.10:assignment test/units/module_utils/basic/test_exit_json.py mypy-3.10:misc test/units/module_utils/facts/other/test_facter.py mypy-3.10:assignment @@ -222,7 +219,6 @@ test/units/module_utils/facts/test_facts.py mypy-3.14:assignment test/units/modules/mount_facts_data.py mypy-3.14:arg-type test/units/modules/test_apt.py mypy-3.14:name-match test/units/modules/test_mount_facts.py mypy-3.14:index -test/units/playbook/test_base.py mypy-3.14:assignment test/integration/targets/interpreter_discovery_python/library/test_non_python_interpreter.py shebang # test needs non-standard shebang test/integration/targets/inventory_script/bad_shebang shebang # test needs an invalid shebang test/integration/targets/ansible-test-sanity-pylint/ansible_collections/ns/col/plugins/lookup/deprecated.py pylint!skip # validated as a collection From 62dc40ce1c0fbfa2c8595426604fd3c06b929650 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Mon, 28 Jul 2025 18:29:00 -0700 Subject: [PATCH 48/68] Update azure-pipelines-test-container (#85578) --- .azure-pipelines/azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml index 92bf8b8455a..9d618ca0394 100644 --- a/.azure-pipelines/azure-pipelines.yml +++ b/.azure-pipelines/azure-pipelines.yml @@ -31,7 +31,7 @@ variables: - name: fetchDepth value: 500 - name: defaultContainer - value: quay.io/ansible/azure-pipelines-test-container:6.0.0 + value: quay.io/ansible/azure-pipelines-test-container:7.0.0 pool: Standard From e62cfa2c022691a911ad93a5f6b8603059c87876 Mon Sep 17 00:00:00 2001 From: Abhijeet Kasurde Date: Tue, 29 Jul 2025 08:08:29 -0700 Subject: [PATCH 49/68] test: correct variable in debug (#85529) Signed-off-by: Abhijeet Kasurde --- test/integration/targets/user/tasks/test_create_user.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/targets/user/tasks/test_create_user.yml b/test/integration/targets/user/tasks/test_create_user.yml index 44707dc7fbe..1e2bc51b0f6 100644 --- a/test/integration/targets/user/tasks/test_create_user.yml +++ b/test/integration/targets/user/tasks/test_create_user.yml @@ -16,7 +16,7 @@ register: user_test0_1 - debug: - var: user_test0 + var: user_test0_0 verbosity: 2 - name: make a list of users From ac5eb232e9aa2be82687e016ae05e3883b6a8c33 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Tue, 29 Jul 2025 18:11:23 +0200 Subject: [PATCH 50/68] Do not re-add tags on blocks from within import_tasks (#85471) --- changelogs/fragments/import_tasks-dont-readd-tags.yml | 2 ++ lib/ansible/playbook/helpers.py | 7 ------- 2 files changed, 2 insertions(+), 7 deletions(-) create mode 100644 changelogs/fragments/import_tasks-dont-readd-tags.yml diff --git a/changelogs/fragments/import_tasks-dont-readd-tags.yml b/changelogs/fragments/import_tasks-dont-readd-tags.yml new file mode 100644 index 00000000000..506e56a00fc --- /dev/null +++ b/changelogs/fragments/import_tasks-dont-readd-tags.yml @@ -0,0 +1,2 @@ +bugfixes: + - Do not re-add ``tags`` on blocks from within ``import_tasks``. diff --git a/lib/ansible/playbook/helpers.py b/lib/ansible/playbook/helpers.py index f4d7a82a8ec..eb96213e216 100644 --- a/lib/ansible/playbook/helpers.py +++ b/lib/ansible/playbook/helpers.py @@ -229,13 +229,6 @@ def load_list_of_tasks(ds, play, block=None, role=None, task_include=None, use_h variable_manager=variable_manager, ) - tags = ti_copy.tags[:] - - # now we extend the tags on each of the included blocks - for b in included_blocks: - b.tags = list(set(b.tags).union(tags)) - # FIXME - END - # FIXME: handlers shouldn't need this special handling, but do # right now because they don't iterate blocks correctly if use_handlers: From fcb829f7240d6eae86cc60dba4197dc5f2fc71c5 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Tue, 29 Jul 2025 20:36:36 +0200 Subject: [PATCH 51/68] Fix incorrect return value type of _copy_included_file (#85472) --- lib/ansible/plugins/strategy/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/ansible/plugins/strategy/__init__.py b/lib/ansible/plugins/strategy/__init__.py index e24aa132309..848aa366137 100644 --- a/lib/ansible/plugins/strategy/__init__.py +++ b/lib/ansible/plugins/strategy/__init__.py @@ -53,6 +53,9 @@ from ansible.utils.sentinel import Sentinel from ansible.utils.vars import combine_vars from ansible.vars.clean import strip_internal_keys, module_response_deepcopy +if t.TYPE_CHECKING: + from ansible.playbook.role_include import IncludeRole + display = Display() __all__ = ['StrategyBase'] @@ -799,7 +802,7 @@ class StrategyBase: return ret_results - def _copy_included_file(self, included_file: IncludedFile) -> IncludedFile: + def _copy_included_file(self, included_file: IncludedFile) -> TaskInclude | IncludeRole: """ A proven safe and performant way to create a copy of an included file """ From 340ca1b4934348863393afab5d1c8090dd7a3bed Mon Sep 17 00:00:00 2001 From: Maxime <12089026+mcbloch@users.noreply.github.com> Date: Tue, 29 Jul 2025 23:59:47 +0200 Subject: [PATCH 52/68] Fix code typo in docstring of DataLoader (#85570) --- lib/ansible/parsing/dataloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ansible/parsing/dataloader.py b/lib/ansible/parsing/dataloader.py index 58dd2c7e414..d8ce2a95ef9 100644 --- a/lib/ansible/parsing/dataloader.py +++ b/lib/ansible/parsing/dataloader.py @@ -49,7 +49,7 @@ class DataLoader: Usage: dl = DataLoader() - # optionally: dl.set_vault_secrets([('default', ansible.parsing.vault.PrompVaultSecret(...),)]) + # optionally: dl.set_vault_secrets([('default', ansible.parsing.vault.PromptVaultSecret(...),)]) ds = dl.load('...') ds = dl.load_from_file('/path/to/file') """ From 5faa256178173d7e4cbbd66d08df518631b2515b Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Wed, 30 Jul 2025 11:08:12 -0400 Subject: [PATCH 53/68] ansible-galaxy, dont display internals (#85571) Also note broken test for invalid collection paths Co-authored-by: Sloane Hertel <19572925+s-hertel@users.noreply.github.com> --- changelogs/fragments/hide_proto.yml | 2 ++ lib/ansible/galaxy/collection/__init__.py | 10 ++++++++-- .../targets/ansible-galaxy-collection/tasks/list.yml | 7 +++++-- 3 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 changelogs/fragments/hide_proto.yml diff --git a/changelogs/fragments/hide_proto.yml b/changelogs/fragments/hide_proto.yml new file mode 100644 index 00000000000..6159156d047 --- /dev/null +++ b/changelogs/fragments/hide_proto.yml @@ -0,0 +1,2 @@ +bugfixes: + - ansible-galaxy no longer shows the internal protomatter collection when listing. diff --git a/lib/ansible/galaxy/collection/__init__.py b/lib/ansible/galaxy/collection/__init__.py index 38737468dcd..77e3870bdd2 100644 --- a/lib/ansible/galaxy/collection/__init__.py +++ b/lib/ansible/galaxy/collection/__init__.py @@ -84,6 +84,7 @@ if t.TYPE_CHECKING: FileManifestEntryType = t.Dict[FileMetaKeysType, t.Union[str, int, None]] FilesManifestType = t.Dict[t.Literal['files', 'format'], t.Union[t.List[FileManifestEntryType], int]] +import ansible import ansible.constants as C from ansible.errors import AnsibleError from ansible.galaxy.api import GalaxyAPI @@ -143,6 +144,8 @@ ModifiedContent = namedtuple('ModifiedContent', ['filename', 'expected', 'instal SIGNATURE_COUNT_RE = r"^(?P\+)?(?:(?P\d+)|(?Pall))$" +_ANSIBLE_PACKAGE_PATH = pathlib.Path(ansible.__file__).parent + @dataclass class ManifestControl: @@ -1429,9 +1432,12 @@ def find_existing_collections(path_filter, artifacts_manager, namespace_filter=N paths = set() for path in files('ansible_collections').glob('*/*/'): path = _normalize_collection_path(path) - if not path.is_dir(): + if path.is_relative_to(_ANSIBLE_PACKAGE_PATH): + # skip internal path, those collections are not for galaxy use + continue + elif not path.is_dir(): continue - if path_filter: + elif path_filter: for pf in path_filter: try: path.relative_to(_normalize_collection_path(pf)) diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/list.yml b/test/integration/targets/ansible-galaxy-collection/tasks/list.yml index 4e2569f7215..c0eeca8349c 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/list.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/list.yml @@ -155,9 +155,12 @@ environment: ANSIBLE_COLLECTIONS_PATH: "i_dont_exist" -- assert: +- name: Ensure we get the expected error + assert: that: - - "'{}' not in list_result_error.stdout" + # FIXME: This test is currently incorrect, but not a fix to do in this PR, proper test is the commented out one. + - "'{}' in list_result_error.stdout" + #- "'None of the provided paths were usable' in list_result_error.stderr" - name: install an artifact to the second collections path command: ansible-galaxy collection install namespace1.name1 -s galaxy_ng {{ galaxy_verbosity }} -p "{{ galaxy_dir }}/prod" From f51e626c329e4fc79a82f2e5287f91c221e8867c Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Wed, 30 Jul 2025 12:25:13 -0500 Subject: [PATCH 54/68] Move the shell target to group4 controller (#85594) * Move the shell target to group4 controller * rename shell target to shell-plugins --- .../{shell => shell-plugins}/action_plugins/test_shell.py | 0 test/integration/targets/shell-plugins/aliases | 2 ++ .../connection_plugins/test_connection_default.py | 0 .../connection_plugins/test_connection_override.py | 0 test/integration/targets/{shell => shell-plugins}/meta/main.yml | 0 .../targets/{shell => shell-plugins}/tasks/command-building.yml | 0 .../integration/targets/{shell => shell-plugins}/tasks/main.yml | 0 .../{shell => shell-plugins}/test-command-building-playbook.yml | 0 test/integration/targets/shell/aliases | 1 - 9 files changed, 2 insertions(+), 1 deletion(-) rename test/integration/targets/{shell => shell-plugins}/action_plugins/test_shell.py (100%) create mode 100644 test/integration/targets/shell-plugins/aliases rename test/integration/targets/{shell => shell-plugins}/connection_plugins/test_connection_default.py (100%) rename test/integration/targets/{shell => shell-plugins}/connection_plugins/test_connection_override.py (100%) rename test/integration/targets/{shell => shell-plugins}/meta/main.yml (100%) rename test/integration/targets/{shell => shell-plugins}/tasks/command-building.yml (100%) rename test/integration/targets/{shell => shell-plugins}/tasks/main.yml (100%) rename test/integration/targets/{shell => shell-plugins}/test-command-building-playbook.yml (100%) delete mode 100644 test/integration/targets/shell/aliases diff --git a/test/integration/targets/shell/action_plugins/test_shell.py b/test/integration/targets/shell-plugins/action_plugins/test_shell.py similarity index 100% rename from test/integration/targets/shell/action_plugins/test_shell.py rename to test/integration/targets/shell-plugins/action_plugins/test_shell.py diff --git a/test/integration/targets/shell-plugins/aliases b/test/integration/targets/shell-plugins/aliases new file mode 100644 index 00000000000..498fedd558e --- /dev/null +++ b/test/integration/targets/shell-plugins/aliases @@ -0,0 +1,2 @@ +shippable/posix/group4 +context/controller diff --git a/test/integration/targets/shell/connection_plugins/test_connection_default.py b/test/integration/targets/shell-plugins/connection_plugins/test_connection_default.py similarity index 100% rename from test/integration/targets/shell/connection_plugins/test_connection_default.py rename to test/integration/targets/shell-plugins/connection_plugins/test_connection_default.py diff --git a/test/integration/targets/shell/connection_plugins/test_connection_override.py b/test/integration/targets/shell-plugins/connection_plugins/test_connection_override.py similarity index 100% rename from test/integration/targets/shell/connection_plugins/test_connection_override.py rename to test/integration/targets/shell-plugins/connection_plugins/test_connection_override.py diff --git a/test/integration/targets/shell/meta/main.yml b/test/integration/targets/shell-plugins/meta/main.yml similarity index 100% rename from test/integration/targets/shell/meta/main.yml rename to test/integration/targets/shell-plugins/meta/main.yml diff --git a/test/integration/targets/shell/tasks/command-building.yml b/test/integration/targets/shell-plugins/tasks/command-building.yml similarity index 100% rename from test/integration/targets/shell/tasks/command-building.yml rename to test/integration/targets/shell-plugins/tasks/command-building.yml diff --git a/test/integration/targets/shell/tasks/main.yml b/test/integration/targets/shell-plugins/tasks/main.yml similarity index 100% rename from test/integration/targets/shell/tasks/main.yml rename to test/integration/targets/shell-plugins/tasks/main.yml diff --git a/test/integration/targets/shell/test-command-building-playbook.yml b/test/integration/targets/shell-plugins/test-command-building-playbook.yml similarity index 100% rename from test/integration/targets/shell/test-command-building-playbook.yml rename to test/integration/targets/shell-plugins/test-command-building-playbook.yml diff --git a/test/integration/targets/shell/aliases b/test/integration/targets/shell/aliases deleted file mode 100644 index a6dafcf8cd8..00000000000 --- a/test/integration/targets/shell/aliases +++ /dev/null @@ -1 +0,0 @@ -shippable/posix/group1 From 29a298a6172b0f7dd66ca06b3aaaaf103788b84e Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Wed, 30 Jul 2025 15:54:44 -0400 Subject: [PATCH 55/68] test fix to avoid permissions issue (#85598) --- test/integration/targets/copy/tasks/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/targets/copy/tasks/tests.yml b/test/integration/targets/copy/tasks/tests.yml index 2b6b5c6ff60..44067ae319f 100644 --- a/test/integration/targets/copy/tasks/tests.yml +++ b/test/integration/targets/copy/tasks/tests.yml @@ -2450,8 +2450,8 @@ - name: Verify atime and mtime update on content change (same partition) vars: - remote_file: "{{ remote_tmp_dir }}/foo.txt" - ansible_remote_tmp: "{{ remote_tmp_dir }}" + remote_file: "/tmp/foo.txt" + ansible_remote_tmp: "/tmp" block: - name: Create a dest file shell: "echo Test content > {{ remote_file }}" From 8aad1418f6069d90de963d5187204419f62ad893 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Wed, 30 Jul 2025 15:42:38 -0700 Subject: [PATCH 56/68] Ignore failing unarchive test on FreeBSD (#85601) --- test/integration/targets/unarchive/tasks/test_zip.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/integration/targets/unarchive/tasks/test_zip.yml b/test/integration/targets/unarchive/tasks/test_zip.yml index 52de9cd2847..b3b5e5f984a 100644 --- a/test/integration/targets/unarchive/tasks/test_zip.yml +++ b/test/integration/targets/unarchive/tasks/test_zip.yml @@ -43,6 +43,12 @@ assert: that: - "unarchive03b.changed == false" + # This test is broken on FreeBSD when running split, but not when running on the controller (2.19 and earlier). + # On the controller, 'foo-unarchive-777.txt' is extracted as 0755, and the test never verifies it should be 0777. + # The unarchive module expects the permissions to be 0755 in this case, so does not report it as changed. + # When split, it is extracted as 0777, which causes the test to fail because it is reported as changed. + # This appears to be due to faulty logic in unarchive around the permissions expected on BSD. + when: ansible_distribution != 'FreeBSD' - name: nuke zip destination file: From f2612fbe3ae3c9d5aafab612ee3f04dc2eb24378 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Wed, 30 Jul 2025 16:39:38 -0700 Subject: [PATCH 57/68] Drop Python 3.11 controller support (#85590) --- .azure-pipelines/azure-pipelines.yml | 8 ++------ changelogs/fragments/python-support.yml | 1 + hacking/README.md | 2 +- lib/ansible/_internal/__init__.py | 5 +---- lib/ansible/_internal/_ansiballz/_builder.py | 4 +--- lib/ansible/_internal/_collection_proxy.py | 16 +++++++--------- lib/ansible/_internal/_json/__init__.py | 7 +++---- .../_internal/_templating/_jinja_plugins.py | 3 +-- lib/ansible/cli/__init__.py | 2 +- lib/ansible/config/base.yml | 2 +- packaging/release.py | 6 +----- pyproject.toml | 4 ++-- .../commands/sanity/validate_modules.py | 6 +----- .../_util/target/common/constants.py | 2 +- .../ansible_test/_util/target/setup/bootstrap.sh | 6 ------ test/sanity/ignore.txt | 2 +- test/units/requirements.txt | 8 ++++---- 17 files changed, 29 insertions(+), 55 deletions(-) diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml index 9d618ca0394..228389367e8 100644 --- a/.azure-pipelines/azure-pipelines.yml +++ b/.azure-pipelines/azure-pipelines.yml @@ -112,10 +112,6 @@ stages: test: rhel/9.6 - name: RHEL 10.0 test: rhel/10.0 - - name: FreeBSD 13.5 - test: freebsd/13.5 - - name: FreeBSD 14.3 - test: freebsd/14.3 groups: - 3 - 4 @@ -183,9 +179,9 @@ stages: nameFormat: Python {0} testFormat: galaxy/{0}/1 targets: - - test: 3.11 - test: 3.12 - test: 3.13 + - test: 3.14 - stage: Generic dependsOn: [] jobs: @@ -194,9 +190,9 @@ stages: nameFormat: Python {0} testFormat: generic/{0}/1 targets: - - test: 3.11 - test: 3.12 - test: 3.13 + - test: 3.14 - stage: Incidental_Windows displayName: Incidental Windows dependsOn: [] diff --git a/changelogs/fragments/python-support.yml b/changelogs/fragments/python-support.yml index 8b86c3245fb..437bfbb66e6 100644 --- a/changelogs/fragments/python-support.yml +++ b/changelogs/fragments/python-support.yml @@ -1,3 +1,4 @@ major_changes: - ansible - Add support for Python 3.14. - ansible - Drop support for Python 3.8 on targets. + - ansible - Drop support for Python 3.11 on the controller. diff --git a/hacking/README.md b/hacking/README.md index 534a7e4db0e..6eddf2e9e94 100644 --- a/hacking/README.md +++ b/hacking/README.md @@ -5,7 +5,7 @@ env-setup --------- The 'env-setup' script modifies your environment to allow you to run -ansible from a git checkout using python >= 3.11. +ansible from a git checkout using a supported Python version. First, set up your environment to run from the checkout: diff --git a/lib/ansible/_internal/__init__.py b/lib/ansible/_internal/__init__.py index 2975a528b6a..35d883b0181 100644 --- a/lib/ansible/_internal/__init__.py +++ b/lib/ansible/_internal/__init__.py @@ -30,10 +30,7 @@ def import_controller_module(module_name: str, /) -> t.Any: return importlib.import_module(module_name) -_T = t.TypeVar('_T') - - -def experimental(obj: _T) -> _T: +def experimental[T](obj: T) -> T: """ Decorator for experimental types and methods outside the `_internal` package which accept or expose internal types. As with internal APIs, these are subject to change at any time without notice. diff --git a/lib/ansible/_internal/_ansiballz/_builder.py b/lib/ansible/_internal/_ansiballz/_builder.py index 76c756fe195..5c8671fd080 100644 --- a/lib/ansible/_internal/_ansiballz/_builder.py +++ b/lib/ansible/_internal/_ansiballz/_builder.py @@ -9,8 +9,6 @@ from ansible.module_utils._internal._ansiballz import _extensions from ansible.module_utils._internal._ansiballz._extensions import _debugpy, _pydevd, _coverage from ansible.constants import config -_T = t.TypeVar('_T') - class ExtensionManager: """AnsiballZ extension manager.""" @@ -101,7 +99,7 @@ class ExtensionManager: ) @classmethod - def _get_options(cls, name: str, config_type: type[_T], task_vars: dict[str, object]) -> _T | None: + def _get_options[T](cls, name: str, config_type: type[T], task_vars: dict[str, object]) -> T | None: """Parse configuration from the named environment variable as the specified type, or None if not configured.""" if (value := config.get_config_value(name, variables=task_vars)) is None: return None diff --git a/lib/ansible/_internal/_collection_proxy.py b/lib/ansible/_internal/_collection_proxy.py index b14dcf386fa..ea3f10e26a9 100644 --- a/lib/ansible/_internal/_collection_proxy.py +++ b/lib/ansible/_internal/_collection_proxy.py @@ -3,26 +3,24 @@ from __future__ import annotations as _annotations import collections.abc as _c import typing as _t -_T_co = _t.TypeVar('_T_co', covariant=True) - -class SequenceProxy(_c.Sequence[_T_co]): +class SequenceProxy[T](_c.Sequence[T]): """A read-only sequence proxy.""" # DTFIX5: needs unit test coverage __slots__ = ('__value',) - def __init__(self, value: _c.Sequence[_T_co]) -> None: + def __init__(self, value: _c.Sequence[T]) -> None: self.__value = value @_t.overload - def __getitem__(self, index: int) -> _T_co: ... + def __getitem__(self, index: int) -> T: ... @_t.overload - def __getitem__(self, index: slice) -> _c.Sequence[_T_co]: ... + def __getitem__(self, index: slice) -> _c.Sequence[T]: ... - def __getitem__(self, index: int | slice) -> _T_co | _c.Sequence[_T_co]: + def __getitem__(self, index: int | slice) -> T | _c.Sequence[T]: if isinstance(index, slice): return self.__class__(self.__value[index]) @@ -34,10 +32,10 @@ class SequenceProxy(_c.Sequence[_T_co]): def __contains__(self, item: object) -> bool: return item in self.__value - def __iter__(self) -> _t.Iterator[_T_co]: + def __iter__(self) -> _t.Iterator[T]: yield from self.__value - def __reversed__(self) -> _c.Iterator[_T_co]: + def __reversed__(self) -> _c.Iterator[T]: return reversed(self.__value) def index(self, *args) -> int: diff --git a/lib/ansible/_internal/_json/__init__.py b/lib/ansible/_internal/_json/__init__.py index 94b53fcc8fa..fd827e68e8a 100644 --- a/lib/ansible/_internal/_json/__init__.py +++ b/lib/ansible/_internal/_json/__init__.py @@ -24,7 +24,6 @@ from ansible._internal._templating import _transform from ansible.module_utils import _internal from ansible.module_utils._internal import _datatag -_T = t.TypeVar('_T') _sentinel = object() @@ -115,7 +114,7 @@ class AnsibleVariableVisitor: if func := getattr(super(), '__exit__', None): func(*args, **kwargs) - def visit(self, value: _T) -> _T: + def visit[T](self, value: T) -> T: """ Enforces Ansible's variable type system restrictions before a var is accepted in inventory. Also, conditionally implements template trust compatibility, depending on the plugin's declared understanding (or lack thereof). This always recursively copies inputs to fully isolate @@ -143,7 +142,7 @@ class AnsibleVariableVisitor: return self._visit(None, key) # key=None prevents state tracking from seeing the key as value - def _visit(self, key: t.Any, value: _T) -> _T: + def _visit[T](self, key: t.Any, value: T) -> T: """Internal implementation to recursively visit a data structure's contents.""" self._current = key # supports StateTrackingMixIn @@ -168,7 +167,7 @@ class AnsibleVariableVisitor: value = value._native_copy() value_type = type(value) - result: _T + result: T # DTFIX-FUTURE: Visitor generally ignores dict/mapping keys by default except for debugging and schema-aware checking. # It could be checking keys destined for variable storage to apply more strict rules about key shape and type. diff --git a/lib/ansible/_internal/_templating/_jinja_plugins.py b/lib/ansible/_internal/_templating/_jinja_plugins.py index 482dabfbb01..a79d9b18067 100644 --- a/lib/ansible/_internal/_templating/_jinja_plugins.py +++ b/lib/ansible/_internal/_templating/_jinja_plugins.py @@ -29,7 +29,6 @@ from ._utils import LazyOptions, TemplateContext _display = Display() -_TCallable = t.TypeVar("_TCallable", bound=t.Callable) _ITERATOR_TYPES: t.Final = (c.Iterator, c.ItemsView, c.KeysView, c.ValuesView, range) @@ -169,7 +168,7 @@ class _DirectCall: _marker_attr: t.Final[str] = "_directcall" @classmethod - def mark(cls, src: _TCallable) -> _TCallable: + def mark[T: t.Callable](cls, src: T) -> T: setattr(src, cls._marker_attr, True) return src diff --git a/lib/ansible/cli/__init__.py b/lib/ansible/cli/__init__.py index da5cacc13bf..2a4ca0f3a71 100644 --- a/lib/ansible/cli/__init__.py +++ b/lib/ansible/cli/__init__.py @@ -23,7 +23,7 @@ if 1 <= len(sys.argv) <= 2 and os.path.basename(sys.argv[0]) == "ansible" and os # Used for determining if the system is running a new enough python version # and should only restrict on our documented minimum versions -_PY_MIN = (3, 11) +_PY_MIN = (3, 12) if sys.version_info < _PY_MIN: raise SystemExit( diff --git a/lib/ansible/config/base.yml b/lib/ansible/config/base.yml index ad28844b8c2..56dca21bbc0 100644 --- a/lib/ansible/config/base.yml +++ b/lib/ansible/config/base.yml @@ -1689,12 +1689,12 @@ INTERPRETER_PYTHON: INTERPRETER_PYTHON_FALLBACK: name: Ordered list of Python interpreters to check for in discovery default: + - python3.14 - python3.13 - python3.12 - python3.11 - python3.10 - python3.9 - - python3.8 - /usr/bin/python3 - python3 vars: diff --git a/packaging/release.py b/packaging/release.py index c16b21f1f22..59077ec5eb9 100755 --- a/packaging/release.py +++ b/packaging/release.py @@ -1271,11 +1271,7 @@ def test_sdist() -> None: except FileNotFoundError: raise ApplicationError(f"Missing sdist: {sdist_file.relative_to(CHECKOUT_DIR)}") from None - # deprecated: description='extractall fallback without filter' python_version='3.11' - if hasattr(tarfile, 'data_filter'): - sdist.extractall(temp_dir, filter='data') # type: ignore[call-arg] - else: - sdist.extractall(temp_dir) + sdist.extractall(temp_dir, filter='data') pyc_glob = "*.pyc*" pyc_files = sorted(path.relative_to(temp_dir) for path in temp_dir.rglob(pyc_glob)) diff --git a/pyproject.toml b/pyproject.toml index 36035530920..b652ac7dd5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools >= 66.1.0, <= 80.3.1", "wheel == 0.45.1"] # lower bound build-backend = "setuptools.build_meta" [project] -requires-python = ">=3.11" +requires-python = ">=3.12" name = "ansible-core" authors = [ {name = "Ansible Project"}, @@ -20,9 +20,9 @@ classifiers = [ "Natural Language :: English", "Operating System :: POSIX", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: 3 :: Only", "Topic :: System :: Installation/Setup", "Topic :: System :: Systems Administration", diff --git a/test/lib/ansible_test/_internal/commands/sanity/validate_modules.py b/test/lib/ansible_test/_internal/commands/sanity/validate_modules.py index 29f271afa81..b51582e4e90 100644 --- a/test/lib/ansible_test/_internal/commands/sanity/validate_modules.py +++ b/test/lib/ansible_test/_internal/commands/sanity/validate_modules.py @@ -160,11 +160,7 @@ class ValidateModulesTest(SanitySingleVersion): temp_dir = process_scoped_temporary_directory(args) with tarfile.open(path) as file: - # deprecated: description='extractall fallback without filter' python_version='3.11' - if hasattr(tarfile, 'data_filter'): - file.extractall(temp_dir, filter='data') # type: ignore[call-arg] - else: - file.extractall(temp_dir) + file.extractall(temp_dir, filter='data') cmd.extend([ '--original-plugins', temp_dir, diff --git a/test/lib/ansible_test/_util/target/common/constants.py b/test/lib/ansible_test/_util/target/common/constants.py index ad412aa23df..4d55c8286ee 100644 --- a/test/lib/ansible_test/_util/target/common/constants.py +++ b/test/lib/ansible_test/_util/target/common/constants.py @@ -7,10 +7,10 @@ from __future__ import annotations REMOTE_ONLY_PYTHON_VERSIONS = ( '3.9', '3.10', + '3.11', ) CONTROLLER_PYTHON_VERSIONS = ( - '3.11', '3.12', '3.13', '3.14', diff --git a/test/lib/ansible_test/_util/target/setup/bootstrap.sh b/test/lib/ansible_test/_util/target/setup/bootstrap.sh index 6947c34f21a..7193bfd16bb 100644 --- a/test/lib/ansible_test/_util/target/setup/bootstrap.sh +++ b/test/lib/ansible_test/_util/target/setup/bootstrap.sh @@ -187,12 +187,6 @@ bootstrap_remote_freebsd() # Declare platform/python version combinations which do not have supporting OS packages available. # For these combinations ansible-test will use pip to install the requirements instead. case "${platform_version}/${python_version}" in - 13.5/3.11) - # defaults available - ;; - 14.3/3.11) - # defaults available - ;; *) # just assume nothing is available jinja2_pkg="" # not available diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index f2dd0a3f405..07b4474dc64 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -54,7 +54,6 @@ lib/ansible/plugins/cache/base.py ansible-doc!skip # not a plugin, but a stub f lib/ansible/plugins/callback/__init__.py pylint:arguments-renamed lib/ansible/plugins/inventory/advanced_host_list.py pylint:arguments-renamed lib/ansible/plugins/inventory/host_list.py pylint:arguments-renamed -lib/ansible/_internal/_wrapt.py mypy-3.11!skip # vendored code lib/ansible/_internal/_wrapt.py mypy-3.12!skip # vendored code lib/ansible/_internal/_wrapt.py mypy-3.13!skip # vendored code lib/ansible/_internal/_wrapt.py mypy-3.14!skip # vendored code @@ -237,3 +236,4 @@ lib/ansible/utils/encrypt.py pylint:ansible-deprecated-version # TODO: 2.20 lib/ansible/utils/ssh_functions.py pylint:ansible-deprecated-version # TODO: 2.20 lib/ansible/vars/manager.py pylint:ansible-deprecated-version-comment # TODO: 2.20 lib/ansible/vars/plugins.py pylint:ansible-deprecated-version # TODO: 2.20 +lib/ansible/galaxy/role.py pylint:ansible-deprecated-python-version-comment # TODO: 2.20 diff --git a/test/units/requirements.txt b/test/units/requirements.txt index fa461030387..97d7b779c2e 100644 --- a/test/units/requirements.txt +++ b/test/units/requirements.txt @@ -1,5 +1,5 @@ -bcrypt ; python_version >= '3.11' # controller only -passlib ; python_version >= '3.11' # controller only -pexpect ; python_version >= '3.11' # controller only -pywinrm ; python_version >= '3.11' # controller only +bcrypt ; python_version >= '3.12' # controller only +passlib ; python_version >= '3.12' # controller only +pexpect ; python_version >= '3.12' # controller only +pywinrm ; python_version >= '3.12' # controller only typing_extensions; python_version < '3.11' # some unit tests need Annotated and get_type_hints(include_extras=True) From 5083eaffc62bc6742db2442870dae2deb9ea92dd Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Wed, 30 Jul 2025 17:51:55 -0700 Subject: [PATCH 58/68] Eliminate more TypeVar and ParamSpec usage (#85602) --- lib/ansible/executor/task_queue_manager.py | 4 +--- lib/ansible/plugins/callback/__init__.py | 3 --- lib/ansible/plugins/connection/__init__.py | 10 +++------- lib/ansible/plugins/connection/ssh.py | 4 +--- lib/ansible/template/__init__.py | 10 +++------- lib/ansible/utils/display.py | 4 +--- lib/ansible/utils/unsafe_proxy.py | 4 ---- packaging/release.py | 5 +---- 8 files changed, 10 insertions(+), 34 deletions(-) diff --git a/lib/ansible/executor/task_queue_manager.py b/lib/ansible/executor/task_queue_manager.py index c48f3b9bb00..84450b60e69 100644 --- a/lib/ansible/executor/task_queue_manager.py +++ b/lib/ansible/executor/task_queue_manager.py @@ -58,8 +58,6 @@ STDERR_FILENO = 2 display = Display() -_T = t.TypeVar('_T') - @dataclasses.dataclass(frozen=True, kw_only=True, slots=True) class CallbackSend: @@ -413,7 +411,7 @@ class TaskQueueManager: return defunct @staticmethod - def _first_arg_of_type(value_type: t.Type[_T], args: t.Sequence) -> _T | None: + def _first_arg_of_type[T](value_type: t.Type[T], args: t.Sequence) -> T | None: return next((arg for arg in args if isinstance(arg, value_type)), None) @lock_decorator(attr='_callback_lock') diff --git a/lib/ansible/plugins/callback/__init__.py b/lib/ansible/plugins/callback/__init__.py index aa8beaea290..ea675ee444f 100644 --- a/lib/ansible/plugins/callback/__init__.py +++ b/lib/ansible/plugins/callback/__init__.py @@ -60,9 +60,6 @@ _YAML_BREAK_CHARS = '\n\x85\u2028\u2029' # NL, NEL, LS, PS _SPACE_BREAK_RE = re.compile(fr' +([{_YAML_BREAK_CHARS}])') -_T_callable = t.TypeVar("_T_callable", bound=t.Callable) - - class _AnsibleCallbackDumper(_dumper.AnsibleDumper): def __init__(self, *args, lossy: bool = False, **kwargs): super().__init__(*args, **kwargs) diff --git a/lib/ansible/plugins/connection/__init__.py b/lib/ansible/plugins/connection/__init__.py index 553235884fd..0078cd9c52e 100644 --- a/lib/ansible/plugins/connection/__init__.py +++ b/lib/ansible/plugins/connection/__init__.py @@ -6,7 +6,6 @@ from __future__ import annotations import collections.abc as c import fcntl -import io import os import shlex import typing as t @@ -16,7 +15,7 @@ from functools import wraps from ansible import constants as C from ansible.errors import AnsibleValueOmittedError -from ansible.module_utils.common.text.converters import to_bytes, to_text +from ansible.module_utils.common.text.converters import to_text from ansible.playbook.play_context import PlayContext from ansible.plugins import AnsiblePlugin from ansible.plugins.become import BecomeBase @@ -32,9 +31,6 @@ __all__ = ['ConnectionBase', 'ensure_connect'] BUFSIZE = 65536 -P = t.ParamSpec('P') -T = t.TypeVar('T') - class ConnectionKwargs(t.TypedDict): task_uuid: str @@ -42,7 +38,7 @@ class ConnectionKwargs(t.TypedDict): shell: t.NotRequired[ShellBase] -def ensure_connect( +def ensure_connect[T, **P]( func: c.Callable[t.Concatenate[ConnectionBase, P], T], ) -> c.Callable[t.Concatenate[ConnectionBase, P], T]: @wraps(func) @@ -135,7 +131,7 @@ class ConnectionBase(AnsiblePlugin): pass @abstractmethod - def _connect(self: T) -> T: + def _connect[T](self: T) -> T: """Connect to the host we've been initialized with""" @ensure_connect diff --git a/lib/ansible/plugins/connection/ssh.py b/lib/ansible/plugins/connection/ssh.py index a4f7fedcd1c..b9b6c1356a1 100644 --- a/lib/ansible/plugins/connection/ssh.py +++ b/lib/ansible/plugins/connection/ssh.py @@ -459,8 +459,6 @@ else: display = Display() -P = t.ParamSpec('P') - # error messages that indicate 255 return code is not from ssh itself. b_NOT_SSH_ERRORS = (b'Traceback (most recent call last):', # Python-2.6 when there's an exception # while invoking a script via -m @@ -547,7 +545,7 @@ def _handle_error( display.vvv(msg, host=host) -def _ssh_retry( +def _ssh_retry[**P]( func: c.Callable[t.Concatenate[Connection, P], tuple[int, bytes, bytes]], ) -> c.Callable[t.Concatenate[Connection, P], tuple[int, bytes, bytes]]: """ diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py index 7e3a93bf3ba..c2ecb92313b 100644 --- a/lib/ansible/template/__init__.py +++ b/lib/ansible/template/__init__.py @@ -25,7 +25,6 @@ if _t.TYPE_CHECKING: # pragma: nocover _display: _t.Final[_Display] = _Display() _UNSET = _t.cast(_t.Any, object()) -_TTrustable = _t.TypeVar('_TTrustable', bound=str | _io.IOBase | _t.TextIO | _t.BinaryIO) _TRUSTABLE_TYPES = (str, _io.IOBase) AnsibleUndefined = _jinja_common.UndefinedMarker @@ -361,7 +360,7 @@ def generate_ansible_template_vars( return _template_vars.generate_ansible_template_vars(path=path, fullpath=fullpath, dest_path=dest_path, include_ansible_managed=True) -def trust_as_template(value: _TTrustable) -> _TTrustable: +def trust_as_template[T: str | _io.IOBase | _t.TextIO | _t.BinaryIO](value: T) -> T: """ Returns `value` tagged as trusted for templating. Raises a `TypeError` if `value` is not a supported type. @@ -385,10 +384,7 @@ def is_trusted_as_template(value: object) -> bool: return isinstance(value, _TRUSTABLE_TYPES) and _tags.TrustedAsTemplate.is_tagged_on(value) -_TCallable = _t.TypeVar('_TCallable', bound=_t.Callable) - - -def accept_args_markers(plugin: _TCallable) -> _TCallable: +def accept_args_markers[T: _t.Callable](plugin: T) -> T: """ A decorator to mark a Jinja plugin as capable of handling `Marker` values for its top-level arguments. Non-decorated plugin invocation is skipped when a top-level argument is a `Marker`, with the first such value substituted as the plugin result. @@ -399,7 +395,7 @@ def accept_args_markers(plugin: _TCallable) -> _TCallable: return plugin -def accept_lazy_markers(plugin: _TCallable) -> _TCallable: +def accept_lazy_markers[T: _t.Callable](plugin: T) -> T: """ A decorator to mark a Jinja plugin as capable of handling `Marker` values retrieved from lazy containers. Non-decorated plugins will trigger a `MarkerError` exception when attempting to retrieve a `Marker` from a lazy container. diff --git a/lib/ansible/utils/display.py b/lib/ansible/utils/display.py index 8650dcb91af..a0abd5dccd5 100644 --- a/lib/ansible/utils/display.py +++ b/lib/ansible/utils/display.py @@ -67,8 +67,6 @@ if t.TYPE_CHECKING: # avoid circular import at runtime from ansible.executor.task_queue_manager import FinalQueue -P = t.ParamSpec('P') - _LIBC = ctypes.cdll.LoadLibrary(ctypes.util.find_library('c')) # Set argtypes, to avoid segfault if the wrong type is provided, # restype is assumed to be c_int @@ -388,7 +386,7 @@ class Display(metaclass=Singleton): self.b_cowsay = b_cow_path @staticmethod - def _proxy( + def _proxy[**P]( func: c.Callable[t.Concatenate[Display, P], None] ) -> c.Callable[..., None]: @wraps(func) diff --git a/lib/ansible/utils/unsafe_proxy.py b/lib/ansible/utils/unsafe_proxy.py index 9ecd6aa5999..1a2c6d04b24 100644 --- a/lib/ansible/utils/unsafe_proxy.py +++ b/lib/ansible/utils/unsafe_proxy.py @@ -12,12 +12,8 @@ from ansible.module_utils.common.collections import is_sequence from ansible._internal._datatag._tags import TrustedAsTemplate from ansible.module_utils.six import binary_type, text_type -import typing as t - __all__ = ['AnsibleUnsafe', 'wrap_var'] -T = t.TypeVar('T') - class AnsibleUnsafe: def __new__(cls, value): diff --git a/packaging/release.py b/packaging/release.py index 59077ec5eb9..e4c89bd84e1 100755 --- a/packaging/release.py +++ b/packaging/release.py @@ -42,9 +42,6 @@ from packaging.version import Version, InvalidVersion # region CLI Framework -C = t.TypeVar("C", bound=t.Callable[..., None]) - - def path_to_str(value: t.Any) -> str: """Return the given value converted to a string suitable for use as a command line argument.""" return f"{value}/" if isinstance(value, pathlib.Path) and value.is_dir() else str(value) @@ -188,7 +185,7 @@ class CommandFramework: self.arguments = kwargs self.parsed_arguments: argparse.Namespace | None = None - def __call__(self, func: C) -> C: + def __call__[T: t.Callable[..., None]](self, func: T) -> T: """Register the decorated function as a CLI command.""" self.commands.append(func) return func From dc5209a3fdd80b354028fc47787e2fd3585f5b5f Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Wed, 30 Jul 2025 19:04:47 -0700 Subject: [PATCH 59/68] ansible-test - Replace TypeVar usage (#85603) --- test/lib/ansible_test/_internal/cache.py | 7 ++---- .../_internal/commands/coverage/combine.py | 4 +--- .../commands/integration/__init__.py | 8 ++----- .../commands/integration/coverage.py | 4 +--- .../_internal/commands/integration/filters.py | 15 ++++-------- test/lib/ansible_test/_internal/completion.py | 7 ++---- test/lib/ansible_test/_internal/config.py | 9 ++------ .../ansible_test/_internal/host_profiles.py | 17 +++++--------- .../_internal/provider/__init__.py | 7 ++---- .../ansible_test/_internal/provisioning.py | 7 ++---- test/lib/ansible_test/_internal/target.py | 8 ++----- test/lib/ansible_test/_internal/thread.py | 5 +--- test/lib/ansible_test/_internal/util.py | 23 ++++++++----------- .../_util/target/cli/ansible_test_cli_stub.py | 6 +++-- 14 files changed, 41 insertions(+), 86 deletions(-) diff --git a/test/lib/ansible_test/_internal/cache.py b/test/lib/ansible_test/_internal/cache.py index 38ed1a806ef..fa554f8baa7 100644 --- a/test/lib/ansible_test/_internal/cache.py +++ b/test/lib/ansible_test/_internal/cache.py @@ -3,14 +3,11 @@ from __future__ import annotations import collections.abc as c -import typing as t from .config import ( CommonConfig, ) -TValue = t.TypeVar('TValue') - class CommonCache: """Common cache.""" @@ -18,14 +15,14 @@ class CommonCache: def __init__(self, args: CommonConfig) -> None: self.args = args - def get(self, key: str, factory: c.Callable[[], TValue]) -> TValue: + def get[TValue](self, key: str, factory: c.Callable[[], TValue]) -> TValue: """Return the value from the cache identified by the given key, using the specified factory method if it is not found.""" if key not in self.args.cache: self.args.cache[key] = factory() return self.args.cache[key] - def get_with_args(self, key: str, factory: c.Callable[[CommonConfig], TValue]) -> TValue: + def get_with_args[TValue](self, key: str, factory: c.Callable[[CommonConfig], TValue]) -> TValue: """Return the value from the cache identified by the given key, using the specified factory method (which accepts args) if it is not found.""" if key not in self.args.cache: self.args.cache[key] = factory(self.args) diff --git a/test/lib/ansible_test/_internal/commands/coverage/combine.py b/test/lib/ansible_test/_internal/commands/coverage/combine.py index b67ae7372b5..2cd402c2c4d 100644 --- a/test/lib/ansible_test/_internal/commands/coverage/combine.py +++ b/test/lib/ansible_test/_internal/commands/coverage/combine.py @@ -63,8 +63,6 @@ from . import ( PathChecker, ) -TValue = t.TypeVar('TValue') - def command_coverage_combine(args: CoverageCombineConfig) -> None: """Patch paths in coverage files and merge into a single file.""" @@ -287,7 +285,7 @@ def _get_coverage_targets(args: CoverageCombineConfig, walk_func: c.Callable) -> return sources -def _build_stub_groups( +def _build_stub_groups[TValue]( args: CoverageCombineConfig, sources: list[tuple[str, int]], default_stub_value: c.Callable[[list[str]], dict[str, TValue]], diff --git a/test/lib/ansible_test/_internal/commands/integration/__init__.py b/test/lib/ansible_test/_internal/commands/integration/__init__.py index 029164d65c9..421c3d4a3d4 100644 --- a/test/lib/ansible_test/_internal/commands/integration/__init__.py +++ b/test/lib/ansible_test/_internal/commands/integration/__init__.py @@ -41,7 +41,6 @@ from ...target import ( walk_integration_targets, IntegrationTarget, walk_internal_targets, - TIntegrationTarget, IntegrationTargetType, ) @@ -50,7 +49,6 @@ from ...config import ( NetworkIntegrationConfig, PosixIntegrationConfig, WindowsIntegrationConfig, - TIntegrationConfig, ) from ...io import ( @@ -132,8 +130,6 @@ from .coverage import ( CoverageManager, ) -THostProfile = t.TypeVar('THostProfile', bound=HostProfile) - def generate_dependency_map(integration_targets: list[IntegrationTarget]) -> dict[str, set[IntegrationTarget]]: """Analyze the given list of integration test targets and return a dictionary expressing target names and the targets on which they depend.""" @@ -856,7 +852,7 @@ class IntegrationCache(CommonCache): return self.get('dependency_map', lambda: generate_dependency_map(self.integration_targets)) -def filter_profiles_for_target(args: IntegrationConfig, profiles: list[THostProfile], target: IntegrationTarget) -> list[THostProfile]: +def filter_profiles_for_target[T: HostProfile](args: IntegrationConfig, profiles: list[T], target: IntegrationTarget) -> list[T]: """Return a list of profiles after applying target filters.""" if target.target_type == IntegrationTargetType.CONTROLLER: profile_filter = get_target_filter(args, [args.controller], True) @@ -912,7 +908,7 @@ If necessary, context can be controlled by adding entries to the "aliases" file return exclude -def command_integration_filter( +def command_integration_filter[TIntegrationTarget: IntegrationTarget, TIntegrationConfig: IntegrationConfig]( args: TIntegrationConfig, targets: c.Iterable[TIntegrationTarget], ) -> tuple[HostState, tuple[TIntegrationTarget, ...]]: diff --git a/test/lib/ansible_test/_internal/commands/integration/coverage.py b/test/lib/ansible_test/_internal/commands/integration/coverage.py index 21cf0537e0f..f2bcc101042 100644 --- a/test/lib/ansible_test/_internal/commands/integration/coverage.py +++ b/test/lib/ansible_test/_internal/commands/integration/coverage.py @@ -79,10 +79,8 @@ from ...inventory import ( create_posix_inventory, ) -THostConfig = t.TypeVar('THostConfig', bound=HostConfig) - -class CoverageHandler(t.Generic[THostConfig], metaclass=abc.ABCMeta): +class CoverageHandler[THostConfig: HostConfig](metaclass=abc.ABCMeta): """Base class for configuring hosts for integration test code coverage.""" def __init__(self, args: IntegrationConfig, host_state: HostState, inventory_path: str) -> None: diff --git a/test/lib/ansible_test/_internal/commands/integration/filters.py b/test/lib/ansible_test/_internal/commands/integration/filters.py index 2ce65811fe5..2be67dc0adc 100644 --- a/test/lib/ansible_test/_internal/commands/integration/filters.py +++ b/test/lib/ansible_test/_internal/commands/integration/filters.py @@ -40,13 +40,8 @@ from ...host_profiles import ( HostProfile, ) -THostConfig = t.TypeVar('THostConfig', bound=HostConfig) -TPosixConfig = t.TypeVar('TPosixConfig', bound=PosixConfig) -TRemoteConfig = t.TypeVar('TRemoteConfig', bound=RemoteConfig) -THostProfile = t.TypeVar('THostProfile', bound=HostProfile) - -class TargetFilter(t.Generic[THostConfig], metaclass=abc.ABCMeta): +class TargetFilter[THostConfig: HostConfig](metaclass=abc.ABCMeta): """Base class for target filters.""" def __init__(self, args: IntegrationConfig, configs: list[THostConfig], controller: bool) -> None: @@ -92,7 +87,7 @@ class TargetFilter(t.Generic[THostConfig], metaclass=abc.ABCMeta): exclude.update(skipped) display.warning(f'Excluding {self.host_type} tests marked {marked} {reason}: {", ".join(skipped)}') - def filter_profiles(self, profiles: list[THostProfile], target: IntegrationTarget) -> list[THostProfile]: + def filter_profiles[THostProfile: HostProfile](self, profiles: list[THostProfile], target: IntegrationTarget) -> list[THostProfile]: """Filter the list of profiles, returning only those which are not skipped for the given target.""" del target return profiles @@ -138,7 +133,7 @@ class TargetFilter(t.Generic[THostConfig], metaclass=abc.ABCMeta): self.skip('unstable', 'which require --allow-unstable or prefixing with "unstable/"', targets, exclude, override) -class PosixTargetFilter(TargetFilter[TPosixConfig]): +class PosixTargetFilter[TPosixConfig: PosixConfig](TargetFilter[TPosixConfig]): """Target filter for POSIX hosts.""" def filter_targets(self, targets: list[IntegrationTarget], exclude: set[str]) -> None: @@ -169,10 +164,10 @@ class PosixSshTargetFilter(PosixTargetFilter[PosixSshConfig]): """Target filter for POSIX SSH hosts.""" -class RemoteTargetFilter(TargetFilter[TRemoteConfig]): +class RemoteTargetFilter[TRemoteConfig: RemoteConfig](TargetFilter[TRemoteConfig]): """Target filter for remote Ansible Core CI managed hosts.""" - def filter_profiles(self, profiles: list[THostProfile], target: IntegrationTarget) -> list[THostProfile]: + def filter_profiles[THostProfile: HostProfile](self, profiles: list[THostProfile], target: IntegrationTarget) -> list[THostProfile]: """Filter the list of profiles, returning only those which are not skipped for the given target.""" profiles = super().filter_profiles(profiles, target) diff --git a/test/lib/ansible_test/_internal/completion.py b/test/lib/ansible_test/_internal/completion.py index e371c3d01db..874ee2bd20b 100644 --- a/test/lib/ansible_test/_internal/completion.py +++ b/test/lib/ansible_test/_internal/completion.py @@ -250,10 +250,7 @@ class WindowsRemoteCompletionConfig(RemoteCompletionConfig): connection: str = '' -TCompletionConfig = t.TypeVar('TCompletionConfig', bound=CompletionConfig) - - -def load_completion(name: str, completion_type: t.Type[TCompletionConfig]) -> dict[str, TCompletionConfig]: +def load_completion[TCompletionConfig: CompletionConfig](name: str, completion_type: t.Type[TCompletionConfig]) -> dict[str, TCompletionConfig]: """Load the named completion entries, returning them in dictionary form using the specified completion type.""" lines = read_lines_without_comments(os.path.join(ANSIBLE_TEST_DATA_ROOT, 'completion', '%s.txt' % name), remove_blank_lines=True) @@ -283,7 +280,7 @@ def parse_completion_entry(value: str) -> tuple[str, dict[str, str]]: return name, data -def filter_completion( +def filter_completion[TCompletionConfig: CompletionConfig]( completion: dict[str, TCompletionConfig], controller_only: bool = False, include_defaults: bool = False, diff --git a/test/lib/ansible_test/_internal/config.py b/test/lib/ansible_test/_internal/config.py index d559871a243..a9abc3c875f 100644 --- a/test/lib/ansible_test/_internal/config.py +++ b/test/lib/ansible_test/_internal/config.py @@ -38,8 +38,6 @@ from .host_configs import ( VirtualPythonConfig, ) -THostConfig = t.TypeVar('THostConfig', bound=HostConfig) - class TerminateMode(enum.Enum): """When to terminate instances.""" @@ -166,7 +164,7 @@ class EnvironmentConfig(CommonConfig): """Host configuration for the targets.""" return self.host_settings.targets - def only_target(self, target_type: t.Type[THostConfig]) -> THostConfig: + def only_target[THostConfig: HostConfig](self, target_type: t.Type[THostConfig]) -> THostConfig: """ Return the host configuration for the target. Requires that there is exactly one target of the specified type. @@ -183,7 +181,7 @@ class EnvironmentConfig(CommonConfig): return target - def only_targets(self, target_type: t.Type[THostConfig]) -> list[THostConfig]: + def only_targets[THostConfig: HostConfig](self, target_type: t.Type[THostConfig]) -> list[THostConfig]: """ Return a list of target host configurations. Requires that there are one or more targets, all the specified type. @@ -318,9 +316,6 @@ class IntegrationConfig(TestConfig): return ansible_config_path -TIntegrationConfig = t.TypeVar('TIntegrationConfig', bound=IntegrationConfig) - - class PosixIntegrationConfig(IntegrationConfig): """Configuration for the posix integration command.""" diff --git a/test/lib/ansible_test/_internal/host_profiles.py b/test/lib/ansible_test/_internal/host_profiles.py index f8d5fbf1e19..d926ec38454 100644 --- a/test/lib/ansible_test/_internal/host_profiles.py +++ b/test/lib/ansible_test/_internal/host_profiles.py @@ -144,11 +144,6 @@ from .debugging import ( DebuggerSettings, ) -TControllerHostConfig = t.TypeVar('TControllerHostConfig', bound=ControllerHostConfig) -THostConfig = t.TypeVar('THostConfig', bound=HostConfig) -TPosixConfig = t.TypeVar('TPosixConfig', bound=PosixConfig) -TRemoteConfig = t.TypeVar('TRemoteConfig', bound=RemoteConfig) - class ControlGroupError(ApplicationError): """Raised when the container host does not have the necessary cgroup support to run a container.""" @@ -239,7 +234,7 @@ class Inventory: display.info(f'>>> Inventory\n{inventory_text}', verbosity=3) -class HostProfile(t.Generic[THostConfig], metaclass=abc.ABCMeta): +class HostProfile[THostConfig: HostConfig](metaclass=abc.ABCMeta): """Base class for host profiles.""" def __init__( @@ -296,7 +291,7 @@ class HostProfile(t.Generic[THostConfig], metaclass=abc.ABCMeta): return f'{self.__class__.__name__}: {self.name}' -class DebuggableProfile(HostProfile[THostConfig], DebuggerProfile, metaclass=abc.ABCMeta): +class DebuggableProfile[THostConfig: HostConfig](HostProfile[THostConfig], DebuggerProfile, metaclass=abc.ABCMeta): """Base class for profiles remote debugging.""" __DEBUGGING_PORT_KEY = 'debugging_port' @@ -462,7 +457,7 @@ class DebuggableProfile(HostProfile[THostConfig], DebuggerProfile, metaclass=abc ) -class PosixProfile(HostProfile[TPosixConfig], metaclass=abc.ABCMeta): +class PosixProfile[TPosixConfig: PosixConfig](HostProfile[TPosixConfig], metaclass=abc.ABCMeta): """Base class for POSIX host profiles.""" @property @@ -484,7 +479,7 @@ class PosixProfile(HostProfile[TPosixConfig], metaclass=abc.ABCMeta): return python -class ControllerHostProfile(PosixProfile[TControllerHostConfig], DebuggableProfile[TControllerHostConfig], metaclass=abc.ABCMeta): +class ControllerHostProfile[T: ControllerHostConfig](PosixProfile[T], DebuggableProfile[T], metaclass=abc.ABCMeta): """Base class for profiles usable as a controller.""" @abc.abstractmethod @@ -496,7 +491,7 @@ class ControllerHostProfile(PosixProfile[TControllerHostConfig], DebuggableProfi """Return the working directory for the host.""" -class SshTargetHostProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta): +class SshTargetHostProfile[THostConfig: HostConfig](HostProfile[THostConfig], metaclass=abc.ABCMeta): """Base class for profiles offering SSH connectivity.""" @abc.abstractmethod @@ -504,7 +499,7 @@ class SshTargetHostProfile(HostProfile[THostConfig], metaclass=abc.ABCMeta): """Return SSH connection(s) for accessing the host as a target from the controller.""" -class RemoteProfile(SshTargetHostProfile[TRemoteConfig], metaclass=abc.ABCMeta): +class RemoteProfile[TRemoteConfig: RemoteConfig](SshTargetHostProfile[TRemoteConfig], metaclass=abc.ABCMeta): """Base class for remote instance profiles.""" @property diff --git a/test/lib/ansible_test/_internal/provider/__init__.py b/test/lib/ansible_test/_internal/provider/__init__.py index 1b3863c0a35..56f8b96f9eb 100644 --- a/test/lib/ansible_test/_internal/provider/__init__.py +++ b/test/lib/ansible_test/_internal/provider/__init__.py @@ -12,12 +12,12 @@ from ..util import ( ) -def get_path_provider_classes(provider_type: t.Type[TPathProvider]) -> list[t.Type[TPathProvider]]: +def get_path_provider_classes[TPathProvider: PathProvider](provider_type: t.Type[TPathProvider]) -> list[t.Type[TPathProvider]]: """Return a list of path provider classes of the given type.""" return sorted(get_subclasses(provider_type), key=lambda subclass: (subclass.priority, subclass.__name__)) -def find_path_provider( +def find_path_provider[TPathProvider: PathProvider]( provider_type: t.Type[TPathProvider], provider_classes: list[t.Type[TPathProvider]], path: str, @@ -71,6 +71,3 @@ class PathProvider(metaclass=abc.ABCMeta): @abc.abstractmethod def is_content_root(path: str) -> bool: """Return True if the given path is a content root for this provider.""" - - -TPathProvider = t.TypeVar('TPathProvider', bound=PathProvider) diff --git a/test/lib/ansible_test/_internal/provisioning.py b/test/lib/ansible_test/_internal/provisioning.py index e7ff02f2ce6..66679d263c5 100644 --- a/test/lib/ansible_test/_internal/provisioning.py +++ b/test/lib/ansible_test/_internal/provisioning.py @@ -48,9 +48,6 @@ from .pypi_proxy import ( run_pypi_proxy, ) -THostProfile = t.TypeVar('THostProfile', bound=HostProfile) -TEnvironmentConfig = t.TypeVar('TEnvironmentConfig', bound=EnvironmentConfig) - class PrimeContainers(ApplicationError): """Exception raised to end execution early after priming containers.""" @@ -91,7 +88,7 @@ class HostState: return list(itertools.chain.from_iterable([target.get_controller_target_connections() for target in self.target_profiles if isinstance(target, SshTargetHostProfile)])) - def targets(self, profile_type: t.Type[THostProfile]) -> list[THostProfile]: + def targets[THostProfile: HostProfile](self, profile_type: t.Type[THostProfile]) -> list[THostProfile]: """The list of target(s), verified to be of the specified type.""" if not self.target_profiles: raise Exception('No target profiles found.') @@ -101,7 +98,7 @@ class HostState: return t.cast(list[THostProfile], self.target_profiles) -def prepare_profiles( +def prepare_profiles[TEnvironmentConfig: EnvironmentConfig]( args: TEnvironmentConfig, targets_use_pypi: bool = False, skip_setup: bool = False, diff --git a/test/lib/ansible_test/_internal/target.py b/test/lib/ansible_test/_internal/target.py index c27e917f34f..5b78dd0d894 100644 --- a/test/lib/ansible_test/_internal/target.py +++ b/test/lib/ansible_test/_internal/target.py @@ -65,7 +65,7 @@ def walk_completion_targets(targets: c.Iterable[CompletionTarget], prefix: str, return tuple(sorted(matches)) -def walk_internal_targets( +def walk_internal_targets[TCompletionTarget: CompletionTarget]( targets: c.Iterable[TCompletionTarget], includes: t.Optional[list[str]] = None, excludes: t.Optional[list[str]] = None, @@ -87,7 +87,7 @@ def walk_internal_targets( return tuple(sorted(internal_targets, key=lambda sort_target: sort_target.name)) -def filter_targets( +def filter_targets[TCompletionTarget: CompletionTarget]( targets: c.Iterable[TCompletionTarget], patterns: list[str], include: bool = True, @@ -711,7 +711,3 @@ class TargetPatternsNotMatched(ApplicationError): message = 'Target pattern not matched: %s' % self.patterns[0] super().__init__(message) - - -TCompletionTarget = t.TypeVar('TCompletionTarget', bound=CompletionTarget) -TIntegrationTarget = t.TypeVar('TIntegrationTarget', bound=IntegrationTarget) diff --git a/test/lib/ansible_test/_internal/thread.py b/test/lib/ansible_test/_internal/thread.py index 515d2c73daa..49ee3f8bf17 100644 --- a/test/lib/ansible_test/_internal/thread.py +++ b/test/lib/ansible_test/_internal/thread.py @@ -11,9 +11,6 @@ import queue import typing as t -TCallable = t.TypeVar('TCallable', bound=t.Callable[..., t.Any]) - - class WrappedThread(threading.Thread): """Wrapper around Thread which captures results and exceptions.""" @@ -50,7 +47,7 @@ class WrappedThread(threading.Thread): return result -def mutex(func: TCallable) -> TCallable: +def mutex[TCallable: t.Callable[..., t.Any]](func: TCallable) -> TCallable: """Enforce exclusive access on a decorated function.""" lock = threading.Lock() diff --git a/test/lib/ansible_test/_internal/util.py b/test/lib/ansible_test/_internal/util.py index 1da631a0b5f..1811f04def6 100644 --- a/test/lib/ansible_test/_internal/util.py +++ b/test/lib/ansible_test/_internal/util.py @@ -57,11 +57,6 @@ from .constants import ( SUPPORTED_PYTHON_VERSIONS, ) -C = t.TypeVar('C') -TBase = t.TypeVar('TBase') -TKey = t.TypeVar('TKey') -TValue = t.TypeVar('TValue') - PYTHON_PATHS: dict[str, str] = {} COVERAGE_CONFIG_NAME = 'coveragerc' @@ -180,7 +175,7 @@ def is_valid_identifier(value: str) -> bool: return value.isidentifier() and not keyword.iskeyword(value) -def cache(func: c.Callable[[], TValue]) -> c.Callable[[], TValue]: +def cache[TValue](func: c.Callable[[], TValue]) -> c.Callable[[], TValue]: """Enforce exclusive access on a decorated function and cache the result.""" storage: dict[None, TValue] = {} sentinel = object() @@ -313,7 +308,7 @@ def read_lines_without_comments(path: str, remove_blank_lines: bool = False, opt return lines -def exclude_none_values(data: dict[TKey, t.Optional[TValue]]) -> dict[TKey, TValue]: +def exclude_none_values[TKey, TValue](data: dict[TKey, t.Optional[TValue]]) -> dict[TKey, TValue]: """Return the provided dictionary with any None values excluded.""" return dict((key, value) for key, value in data.items() if value is not None) @@ -1058,7 +1053,7 @@ def format_command_output(stdout: str | None, stderr: str | None) -> str: return message -def retry(func: t.Callable[..., TValue], ex_type: t.Type[BaseException] = SubprocessError, sleep: int = 10, attempts: int = 10, warn: bool = True) -> TValue: +def retry[T](func: t.Callable[..., T], ex_type: t.Type[BaseException] = SubprocessError, sleep: int = 10, attempts: int = 10, warn: bool = True) -> T: """Retry the specified function on failure.""" for dummy in range(1, attempts): try: @@ -1091,7 +1086,7 @@ def parse_to_list_of_dict(pattern: str, value: str) -> list[dict[str, str]]: return matched -def get_subclasses(class_type: t.Type[C]) -> list[t.Type[C]]: +def get_subclasses[C](class_type: t.Type[C]) -> list[t.Type[C]]: """Returns a list of types that are concrete subclasses of the given type.""" subclasses: set[t.Type[C]] = set() queue: list[t.Type[C]] = [class_type] @@ -1167,7 +1162,7 @@ def import_plugins(directory: str, root: t.Optional[str] = None) -> None: load_module(module_path, name) -def load_plugins(base_type: t.Type[C], database: dict[str, t.Type[C]]) -> None: +def load_plugins[C](base_type: t.Type[C], database: dict[str, t.Type[C]]) -> None: """ Load plugins of the specified type and track them in the specified database. Only plugins which have already been imported will be loaded. @@ -1194,19 +1189,19 @@ def sanitize_host_name(name: str) -> str: return re.sub('[^A-Za-z0-9]+', '-', name)[:63].strip('-') -def get_generic_type(base_type: t.Type, generic_base_type: t.Type[TValue]) -> t.Optional[t.Type[TValue]]: +def get_generic_type[TValue](base_type: t.Type, generic_base_type: t.Type[TValue]) -> t.Optional[t.Type[TValue]]: """Return the generic type arg derived from the generic_base_type type that is associated with the base_type type, if any, otherwise return None.""" # noinspection PyUnresolvedReferences type_arg = t.get_args(base_type.__orig_bases__[0])[0] return None if isinstance(type_arg, generic_base_type) else type_arg -def get_type_associations(base_type: t.Type[TBase], generic_base_type: t.Type[TValue]) -> list[tuple[t.Type[TValue], t.Type[TBase]]]: +def get_type_associations[TBase, TValue](base_type: t.Type[TBase], generic_base_type: t.Type[TValue]) -> list[tuple[t.Type[TValue], t.Type[TBase]]]: """Create and return a list of tuples associating generic_base_type derived types with a corresponding base_type derived type.""" return [item for item in [(get_generic_type(sc_type, generic_base_type), sc_type) for sc_type in get_subclasses(base_type)] if item[1]] -def get_type_map(base_type: t.Type[TBase], generic_base_type: t.Type[TValue]) -> dict[t.Type[TValue], t.Type[TBase]]: +def get_type_map[TBase, TValue](base_type: t.Type[TBase], generic_base_type: t.Type[TValue]) -> dict[t.Type[TValue], t.Type[TBase]]: """Create and return a mapping of generic_base_type derived types to base_type derived types.""" return {item[0]: item[1] for item in get_type_associations(base_type, generic_base_type)} @@ -1227,7 +1222,7 @@ def verify_sys_executable(path: str) -> t.Optional[str]: return expected_executable -def type_guard(sequence: c.Sequence[t.Any], guard_type: t.Type[C]) -> t.TypeGuard[c.Sequence[C]]: +def type_guard[C](sequence: c.Sequence[t.Any], guard_type: t.Type[C]) -> t.TypeGuard[c.Sequence[C]]: """ Raises an exception if any item in the given sequence does not match the specified guard type. Use with assert so that type checkers are aware of the type guard. diff --git a/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py b/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py index 9cb5d04ae0c..e8057fae19c 100755 --- a/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py +++ b/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py @@ -6,6 +6,7 @@ from __future__ import annotations +import importlib import os import sys @@ -29,8 +30,9 @@ def main(args=None): if any(not os.get_blocking(handle.fileno()) for handle in (sys.stdin, sys.stdout, sys.stderr)): raise SystemExit('Standard input, output and error file handles must be blocking to run ansible-test.') - # noinspection PyProtectedMember - from ansible_test._internal import main as cli_main + # avoid using import to hide it from mypy + internal = importlib.import_module('ansible_test._internal') + cli_main = getattr(internal, 'main') cli_main(args) From c0256d6edfd984543f2a1d9391e7c5f5e3747294 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Thu, 31 Jul 2025 15:08:30 +0200 Subject: [PATCH 60/68] dnf/dnf5: address issues discovered in 2.16 yum tests (#83659) Fixes #85554 --- lib/ansible/modules/dnf.py | 86 ++++++++----------- lib/ansible/modules/dnf5.py | 65 +++++++------- test/integration/targets/dnf/tasks/repo.yml | 93 +++++++++++++++++++++ 3 files changed, 165 insertions(+), 79 deletions(-) diff --git a/lib/ansible/modules/dnf.py b/lib/ansible/modules/dnf.py index 1922ba85e79..9d14a90dc15 100644 --- a/lib/ansible/modules/dnf.py +++ b/lib/ansible/modules/dnf.py @@ -208,6 +208,8 @@ options: packages to install (because dependencies between the downgraded package and others can cause changes to the packages which were in the earlier transaction). + - Since this feature is not provided by C(dnf) itself but by M(ansible.builtin.dnf) module, + using this in combination with wildcard characters in O(name) may result in an unexpected results. type: bool default: "no" version_added: "2.7" @@ -701,72 +703,56 @@ class DnfModule(YumDnf): self.module.exit_json(msg="", results=results) def _is_installed(self, pkg): - installed_query = dnf.subject.Subject(pkg).get_best_query(sack=self.base.sack).installed() - if dnf.util.is_glob_pattern(pkg): - available_query = dnf.subject.Subject(pkg).get_best_query(sack=self.base.sack).available() - return not ( - {p.name for p in available_query} - {p.name for p in installed_query} - ) - else: - return bool(installed_query) + return bool(dnf.subject.Subject(pkg).get_best_query(sack=self.base.sack).installed()) def _is_newer_version_installed(self, pkg_spec): + # expects a versioned package spec try: if isinstance(pkg_spec, dnf.package.Package): installed = sorted(self.base.sack.query().installed().filter(name=pkg_spec.name, arch=pkg_spec.arch))[-1] return installed.evr_gt(pkg_spec) else: - available = dnf.subject.Subject(pkg_spec).get_best_query(sack=self.base.sack).available() - installed = self.base.sack.query().installed().filter(name=available[0].name) - for arch in sorted(set(p.arch for p in installed)): # select only from already-installed arches for this case - installed_pkg = sorted(installed.filter(arch=arch))[-1] - try: - available_pkg = sorted(available.filter(arch=arch))[-1] - except IndexError: - continue # nothing currently available for this arch; keep going - if installed_pkg.evr_gt(available_pkg): - return True - return False + solution = dnf.subject.Subject(pkg_spec).get_best_solution(self.base.sack) + q = solution["query"] + if not q or not solution['nevra'] or solution['nevra'].has_just_name(): + return False + installed = self.base.sack.query().installed().filter(name=solution['nevra'].name) + if not installed: + return False + return installed[0].evr_gt(q[0]) except IndexError: return False def _mark_package_install(self, pkg_spec, upgrade=False): """Mark the package for install.""" - is_newer_version_installed = self._is_newer_version_installed(pkg_spec) - is_installed = self._is_installed(pkg_spec) msg = '' try: - if is_newer_version_installed: + if dnf.util.is_glob_pattern(pkg_spec): + # Special case for package specs that contain glob characters. + # For these we skip `is_installed` and `is_newer_version_installed` tests that allow for the + # allow_downgrade feature and pass the package specs to dnf. + # Since allow_downgrade is not available in dnf and while it is relatively easy to implement it for + # package specs that evaluate to a single package, trying to mimic what would the dnf machinery do + # for glob package specs and then filtering those for allow_downgrade appears to always + # result in naive/inferior solution. + # NOTE this has historically never worked even before https://github.com/ansible/ansible/pull/82725 + # where our (buggy) custom code ignored wildcards for the installed checks. + # TODO reasearch how feasible it is to implement the above + if upgrade: + # for upgrade we pass the spec to both upgrade and install, to satisfy both available and installed + # packages evaluated from the glob spec + try: + self.base.upgrade(pkg_spec) + except dnf.exceptions.PackagesNotInstalledError: + pass + self.base.install(pkg_spec, strict=self.base.conf.strict) + elif self._is_newer_version_installed(pkg_spec): if self.allow_downgrade: - # dnf only does allow_downgrade, we have to handle this ourselves - # because it allows a possibility for non-idempotent transactions - # on a system's package set (pending the yum repo has many old - # NVRs indexed) - if upgrade: - if is_installed: # Case 1 - # TODO: Is this case reachable? - # - # _is_installed() demands a name (*not* NVR) or else is always False - # (wildcards are treated literally). - # - # Meanwhile, _is_newer_version_installed() demands something versioned - # or else is always false. - # - # I fail to see how they can both be true at the same time for any - # given pkg_spec. -re - self.base.upgrade(pkg_spec) - else: # Case 2 - self.base.install(pkg_spec, strict=self.base.conf.strict) - else: # Case 3 - self.base.install(pkg_spec, strict=self.base.conf.strict) - else: # Case 4, Nothing to do, report back - pass - elif is_installed: # A potentially older (or same) version is installed - if upgrade: # Case 5 + self.base.install(pkg_spec, strict=self.base.conf.strict) + elif self._is_installed(pkg_spec): + if upgrade: self.base.upgrade(pkg_spec) - else: # Case 6, Nothing to do, report back - pass - else: # Case 7, The package is not installed, simply install it + else: self.base.install(pkg_spec, strict=self.base.conf.strict) except dnf.exceptions.MarkingError as e: msg = "No package {0} available.".format(pkg_spec) diff --git a/lib/ansible/modules/dnf5.py b/lib/ansible/modules/dnf5.py index cd9bf6e3f2e..dc2d0eb5e13 100644 --- a/lib/ansible/modules/dnf5.py +++ b/lib/ansible/modules/dnf5.py @@ -178,6 +178,8 @@ options: packages to install (because dependencies between the downgraded package and others can cause changes to the packages which were in the earlier transaction). + - Since this feature is not provided by C(dnf5) itself but by M(ansible.builtin.dnf5) module, + using this in combination with wildcard characters in O(name) may result in an unexpected results. type: bool default: "no" download_only: @@ -362,7 +364,7 @@ libdnf5 = None LIBDNF5_ERRORS = RuntimeError -def is_installed(base, spec): +def get_resolve_spec_settings(): settings = libdnf5.base.ResolveSpecSettings() try: settings.set_group_with_name(True) @@ -388,47 +390,34 @@ def is_installed(base, spec): settings.group_with_name = True settings.with_binaries = False settings.with_provides = False + return settings + + +def is_installed(base, spec): + settings = get_resolve_spec_settings() installed_query = libdnf5.rpm.PackageQuery(base) installed_query.filter_installed() match, nevra = installed_query.resolve_pkg_spec(spec, settings, True) - - # FIXME use `is_glob_pattern` function when available: - # https://github.com/rpm-software-management/dnf5/issues/1563 - glob_patterns = set("*[?") - if any(set(char) & glob_patterns for char in spec): - available_query = libdnf5.rpm.PackageQuery(base) - available_query.filter_available() - available_query.resolve_pkg_spec(spec, settings, True) - - return not ( - {p.get_name() for p in available_query} - {p.get_name() for p in installed_query} - ) - else: - return match + return match def is_newer_version_installed(base, spec): - # FIXME investigate whether this function can be replaced by dnf5's allow_downgrade option + # expects a versioned package spec if "/" in spec: spec = spec.split("/")[-1] if spec.endswith(".rpm"): spec = spec[:-4] - try: - spec_nevra = next(iter(libdnf5.rpm.Nevra.parse(spec))) - except LIBDNF5_ERRORS: - return False - except StopIteration: - return False - - spec_version = spec_nevra.get_version() - if not spec_version: + settings = get_resolve_spec_settings() + match, spec_nevra = libdnf5.rpm.PackageQuery(base).resolve_pkg_spec(spec, settings, True) + if not match or spec_nevra.has_just_name(): return False + spec_name = spec_nevra.get_name() installed = libdnf5.rpm.PackageQuery(base) installed.filter_installed() - installed.filter_name([spec_nevra.get_name()]) + installed.filter_name([spec_name]) installed.filter_latest_evr() try: installed_package = list(installed)[-1] @@ -436,8 +425,8 @@ def is_newer_version_installed(base, spec): return False target = libdnf5.rpm.PackageQuery(base) - target.filter_name([spec_nevra.get_name()]) - target.filter_version([spec_version]) + target.filter_name([spec_name]) + target.filter_version([spec_nevra.get_version()]) spec_release = spec_nevra.get_release() if spec_release: target.filter_release([spec_release]) @@ -719,8 +708,26 @@ class Dnf5Module(YumDnf): goal.add_rpm_upgrade(settings) elif self.state in {"installed", "present", "latest"}: upgrade = self.state == "latest" + # FIXME use `is_glob_pattern` function when available: + # https://github.com/rpm-software-management/dnf5/issues/1563 + glob_patterns = set("*[?") for spec in self.names: - if is_newer_version_installed(base, spec): + if any(set(char) & glob_patterns for char in spec): + # Special case for package specs that contain glob characters. + # For these we skip `is_installed` and `is_newer_version_installed` tests that allow for the + # allow_downgrade feature and pass the package specs to dnf. + # Since allow_downgrade is not available in dnf and while it is relatively easy to implement it for + # package specs that evaluate to a single package, trying to mimic what would the dnf machinery do + # for glob package specs and then filtering those for allow_downgrade appears to always + # result in naive/inferior solution. + # TODO reasearch how feasible it is to implement the above + if upgrade: + # for upgrade we pass the spec to both upgrade and install, to satisfy both available and installed + # packages evaluated from the glob spec + goal.add_upgrade(spec, settings) + if not self.update_only: + goal.add_install(spec, settings) + elif is_newer_version_installed(base, spec): if self.allow_downgrade: goal.add_install(spec, settings) elif is_installed(base, spec): diff --git a/test/integration/targets/dnf/tasks/repo.yml b/test/integration/targets/dnf/tasks/repo.yml index 8240b580f38..00034169b09 100644 --- a/test/integration/targets/dnf/tasks/repo.yml +++ b/test/integration/targets/dnf/tasks/repo.yml @@ -630,6 +630,87 @@ - provides-package - provided-package +# https://github.com/ansible/ansible/issues/45250 +- block: + - name: Install dinginessentail-1.0, dinginessentail-olive-1.0, landsidescalping-1.0 + dnf: + name: "dinginessentail-1.0,dinginessentail-olive-1.0,landsidescalping-1.0" + state: present + + - name: Upgrade dinginessentail* + dnf: + name: dinginessentail* + state: latest + register: dnf_result + + - name: Check dinginessentail with rpm + shell: rpm -q dinginessentail + register: rpm_result + + - name: Verify update of dinginessentail + assert: + that: + - "rpm_result.stdout.startswith('dinginessentail-1.1-1')" + + - name: Check dinginessentail-olive with rpm + shell: rpm -q dinginessentail-olive + register: rpm_result + + - name: Verify update of dinginessentail-olive + assert: + that: + - "rpm_result.stdout.startswith('dinginessentail-olive-1.1-1')" + + - name: Check landsidescalping with rpm + shell: rpm -q landsidescalping + register: rpm_result + + - name: Verify landsidescalping did NOT get updated + assert: + that: + - "rpm_result.stdout.startswith('landsidescalping-1.0-1')" + + - name: Verify yum module outputs + assert: + that: + - "dnf_result is changed" + - "'msg' in dnf_result" + - "'rc' in dnf_result" + - "'results' in dnf_result" + always: + - name: Clean up + dnf: + name: dinginessentail,dinginessentail-olive,landsidescalping + state: absent + +- name: test allow_downgrade + block: + - dnf: + name: dinginessentail-1.1 + state: present + + - dnf: + name: dinginessentail-1.0 + state: present + allow_downgrade: true + - dnf: + name: dinginessentail-1.1 + state: present + + - dnf: + name: dinginessentail-1.0 + state: present + allow_downgrade: false + register: r + + - assert: + that: + - r is not changed + always: + - dnf: + name: dinginessentail + state: absent + - name: Test failures occured during loading repositories are properly handled vars: repo_name: test-non-existing-gpgkey-file @@ -657,3 +738,15 @@ - file: name: /etc/yum.repos.d/{{ repo_name }}.repo state: absent + + +- name: Attempt to install a package with invalid name + dnf: + name: invalid[package_name] + register: r + ignore_errors: true + +- assert: + that: + - r is failed + - r.msg is contains("Failed to install some of the specified packages") From 817f0af2391388a8b9381a3fec951bf3cdc40569 Mon Sep 17 00:00:00 2001 From: Patrick Kingston <66141901+pkingstonxyz@users.noreply.github.com> Date: Thu, 31 Jul 2025 10:11:04 -0400 Subject: [PATCH 61/68] pip - fix reporting changed for empty venv creation (#85556) * Add correct empty venv creation handling Makes the module exit with changed=True when an empty venv is created with correct module output * Add tests for creating an empty venv --- .../fragments/85556-fix-pip-changed.yml | 2 ++ lib/ansible/modules/pip.py | 10 +++++-- test/integration/targets/pip/tasks/pip.yml | 29 +++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 changelogs/fragments/85556-fix-pip-changed.yml diff --git a/changelogs/fragments/85556-fix-pip-changed.yml b/changelogs/fragments/85556-fix-pip-changed.yml new file mode 100644 index 00000000000..8d09a0db0b5 --- /dev/null +++ b/changelogs/fragments/85556-fix-pip-changed.yml @@ -0,0 +1,2 @@ +bugfixes: + - pip - Fix pip module output so that it returns changed when the only operation is initializing a venv. \ No newline at end of file diff --git a/lib/ansible/modules/pip.py b/lib/ansible/modules/pip.py index 20e6249b759..6af7e7afdeb 100644 --- a/lib/ansible/modules/pip.py +++ b/lib/ansible/modules/pip.py @@ -608,7 +608,7 @@ def setup_virtualenv(module, env, chdir, out, err): err += err_venv if rc != 0: _fail(module, cmd, out, err) - return out, err + return out, err, cmd class Package: @@ -741,11 +741,12 @@ def main(): err = '' out = '' + venv_cmd = '' if env: if not os.path.exists(os.path.join(env, 'bin', 'activate')): venv_created = True - out, err = setup_virtualenv(module, env, chdir, out, err) + out, err, venv_cmd = setup_virtualenv(module, env, chdir, out, err) py_bin = os.path.join(env, 'bin', 'python') else: py_bin = module.params['executable'] or sys.executable @@ -811,6 +812,11 @@ def main(): cmd.extend(to_native(p) for p in packages) elif requirements: cmd.extend(['-r', requirements]) + elif venv_created and not name and not requirements: + # ONLY creating an empty venv + module.exit_json(changed=venv_created, cmd=venv_cmd, name=name, version=version, + state=state, requirements=requirements, virtualenv=env, + stdout=out, stderr=err) else: module.warn("No valid name or requirements file found.") module.exit_json(changed=False) diff --git a/test/integration/targets/pip/tasks/pip.yml b/test/integration/targets/pip/tasks/pip.yml index 9bdb2f79485..a4c9b8d4f40 100644 --- a/test/integration/targets/pip/tasks/pip.yml +++ b/test/integration/targets/pip/tasks/pip.yml @@ -213,6 +213,35 @@ that: - "diff_case_check_mode is not changed" +- name: ensure is a fresh venv + file: + state: absent + name: "{{ remote_tmp_dir }}/emptyvenv" + +- name: create an empty venv + pip: + name: [] + virtualenv: "{{ remote_tmp_dir }}/emptyvenv" + register: empty_venv + +- name: check venv was created + file: + name: "{{ remote_tmp_dir }}/emptyvenv" + register: check_empty_venv_exists + +- name: try creating an empty venv so it isn't changed + pip: + name: [] + virtualenv: "{{ remote_tmp_dir }}/emptyvenv" + register: second_empty_venv + +- name: assert that the venv was created and has no packages + assert: + that: + - empty_venv.changed + - not check_empty_venv_exists.changed + - not second_empty_venv.changed + # ansible#23204 - name: ensure is a fresh virtualenv file: From 97b2242b78f253b5d6b61c44871e51cc90457ea2 Mon Sep 17 00:00:00 2001 From: Abhijeet Kasurde Date: Thu, 31 Jul 2025 08:35:16 -0700 Subject: [PATCH 62/68] wrapt: Bump version from 1.15.0 to 1.17.2 (#85500) Fixes: #85407 Signed-off-by: Abhijeet Kasurde --- changelogs/fragments/wrapt_1.17.2.yml | 3 + lib/ansible/_internal/_wrapt.py | 406 +++++++------------------- 2 files changed, 108 insertions(+), 301 deletions(-) create mode 100644 changelogs/fragments/wrapt_1.17.2.yml diff --git a/changelogs/fragments/wrapt_1.17.2.yml b/changelogs/fragments/wrapt_1.17.2.yml new file mode 100644 index 00000000000..749a5ad63c6 --- /dev/null +++ b/changelogs/fragments/wrapt_1.17.2.yml @@ -0,0 +1,3 @@ +--- +minor_changes: + - wrapt - bump version from 1.15.0 to 1.17.2 (https://github.com/ansible/ansible/issues/85407). diff --git a/lib/ansible/_internal/_wrapt.py b/lib/ansible/_internal/_wrapt.py index d493baaa717..b2fcefef016 100644 --- a/lib/ansible/_internal/_wrapt.py +++ b/lib/ansible/_internal/_wrapt.py @@ -23,7 +23,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -# copied from https://github.com/GrahamDumpleton/wrapt/blob/1.15.0/src/wrapt/wrappers.py +# copied from https://github.com/GrahamDumpleton/wrapt/blob/1.17.2/src/wrapt/wrappers.py # LOCAL PATCHES: # - disabled optional relative import of the _wrappers C extension; we shouldn't need it @@ -31,13 +31,10 @@ from __future__ import annotations # The following makes it easier for us to script updates of the bundled code -_BUNDLED_METADATA = {"pypi_name": "wrapt", "version": "1.15.0"} +_BUNDLED_METADATA = {"pypi_name": "wrapt", "version": "1.17.2"} -import os import sys -import functools import operator -import weakref import inspect PY2 = sys.version_info[0] == 2 @@ -129,6 +126,9 @@ class ObjectProxy(with_metaclass(_ObjectProxyMetaType)): except AttributeError: pass + def __self_setattr__(self, name, value): + object.__setattr__(self, name, value) + @property def __name__(self): return self.__wrapped__.__name__ @@ -161,12 +161,15 @@ class ObjectProxy(with_metaclass(_ObjectProxyMetaType)): type(self.__wrapped__).__name__, id(self.__wrapped__)) + def __format__(self, format_spec): + return format(self.__wrapped__, format_spec) + def __reversed__(self): return reversed(self.__wrapped__) if not PY2: - def __round__(self): - return round(self.__wrapped__) + def __round__(self, ndigits=None): + return round(self.__wrapped__, ndigits) if sys.hexversion >= 0x03070000: def __mro_entries__(self, bases): @@ -472,7 +475,7 @@ class ObjectProxy(with_metaclass(_ObjectProxyMetaType)): def __reduce__(self): raise NotImplementedError( - 'object proxy must define __reduce_ex__()') + 'object proxy must define __reduce__()') def __reduce_ex__(self, protocol): raise NotImplementedError( @@ -525,10 +528,10 @@ class PartialCallableObjectProxy(ObjectProxy): class _FunctionWrapperBase(ObjectProxy): __slots__ = ('_self_instance', '_self_wrapper', '_self_enabled', - '_self_binding', '_self_parent') + '_self_binding', '_self_parent', '_self_owner') def __init__(self, wrapped, instance, wrapper, enabled=None, - binding='function', parent=None): + binding='callable', parent=None, owner=None): super(_FunctionWrapperBase, self).__init__(wrapped) @@ -537,60 +540,68 @@ class _FunctionWrapperBase(ObjectProxy): object.__setattr__(self, '_self_enabled', enabled) object.__setattr__(self, '_self_binding', binding) object.__setattr__(self, '_self_parent', parent) + object.__setattr__(self, '_self_owner', owner) def __get__(self, instance, owner): - # This method is actually doing double duty for both unbound and - # bound derived wrapper classes. It should possibly be broken up - # and the distinct functionality moved into the derived classes. - # Can't do that straight away due to some legacy code which is - # relying on it being here in this base class. + # This method is actually doing double duty for both unbound and bound + # derived wrapper classes. It should possibly be broken up and the + # distinct functionality moved into the derived classes. Can't do that + # straight away due to some legacy code which is relying on it being + # here in this base class. # - # The distinguishing attribute which determines whether we are - # being called in an unbound or bound wrapper is the parent - # attribute. If binding has never occurred, then the parent will - # be None. + # The distinguishing attribute which determines whether we are being + # called in an unbound or bound wrapper is the parent attribute. If + # binding has never occurred, then the parent will be None. # - # First therefore, is if we are called in an unbound wrapper. In - # this case we perform the binding. + # First therefore, is if we are called in an unbound wrapper. In this + # case we perform the binding. # - # We have one special case to worry about here. This is where we - # are decorating a nested class. In this case the wrapped class - # would not have a __get__() method to call. In that case we - # simply return self. + # We have two special cases to worry about here. These are where we are + # decorating a class or builtin function as neither provide a __get__() + # method to call. In this case we simply return self. # - # Note that we otherwise still do binding even if instance is - # None and accessing an unbound instance method from a class. - # This is because we need to be able to later detect that - # specific case as we will need to extract the instance from the - # first argument of those passed in. + # Note that we otherwise still do binding even if instance is None and + # accessing an unbound instance method from a class. This is because we + # need to be able to later detect that specific case as we will need to + # extract the instance from the first argument of those passed in. if self._self_parent is None: - if not inspect.isclass(self.__wrapped__): - descriptor = self.__wrapped__.__get__(instance, owner) + # Technically can probably just check for existence of __get__ on + # the wrapped object, but this is more explicit. + + if self._self_binding == 'builtin': + return self - return self.__bound_function_wrapper__(descriptor, instance, - self._self_wrapper, self._self_enabled, - self._self_binding, self) + if self._self_binding == "class": + return self - return self + binder = getattr(self.__wrapped__, '__get__', None) - # Now we have the case of binding occurring a second time on what - # was already a bound function. In this case we would usually - # return ourselves again. This mirrors what Python does. + if binder is None: + return self + + descriptor = binder(instance, owner) + + return self.__bound_function_wrapper__(descriptor, instance, + self._self_wrapper, self._self_enabled, + self._self_binding, self, owner) + + # Now we have the case of binding occurring a second time on what was + # already a bound function. In this case we would usually return + # ourselves again. This mirrors what Python does. # - # The special case this time is where we were originally bound - # with an instance of None and we were likely an instance - # method. In that case we rebind against the original wrapped - # function from the parent again. + # The special case this time is where we were originally bound with an + # instance of None and we were likely an instance method. In that case + # we rebind against the original wrapped function from the parent again. - if self._self_instance is None and self._self_binding == 'function': + if self._self_instance is None and self._self_binding in ('function', 'instancemethod', 'callable'): descriptor = self._self_parent.__wrapped__.__get__( instance, owner) return self._self_parent.__bound_function_wrapper__( descriptor, instance, self._self_wrapper, self._self_enabled, self._self_binding, - self._self_parent) + self._self_parent, owner) return self @@ -617,7 +628,7 @@ class _FunctionWrapperBase(ObjectProxy): # a function that was already bound to an instance. In that case # we want to extract the instance from the function and use it. - if self._self_binding in ('function', 'classmethod'): + if self._self_binding in ('function', 'instancemethod', 'classmethod', 'callable'): if self._self_instance is None: instance = getattr(self.__wrapped__, '__self__', None) if instance is not None: @@ -668,11 +679,11 @@ class BoundFunctionWrapper(_FunctionWrapperBase): self, args = _unpack_self(*args) - # If enabled has been specified, then evaluate it at this point - # and if the wrapper is not to be executed, then simply return - # the bound function rather than a bound wrapper for the bound - # function. When evaluating enabled, if it is callable we call - # it, otherwise we evaluate it as a boolean. + # If enabled has been specified, then evaluate it at this point and if + # the wrapper is not to be executed, then simply return the bound + # function rather than a bound wrapper for the bound function. When + # evaluating enabled, if it is callable we call it, otherwise we + # evaluate it as a boolean. if self._self_enabled is not None: if callable(self._self_enabled): @@ -681,18 +692,27 @@ class BoundFunctionWrapper(_FunctionWrapperBase): elif not self._self_enabled: return self.__wrapped__(*args, **kwargs) - # We need to do things different depending on whether we are - # likely wrapping an instance method vs a static method or class - # method. + # We need to do things different depending on whether we are likely + # wrapping an instance method vs a static method or class method. if self._self_binding == 'function': + if self._self_instance is None and args: + instance, newargs = args[0], args[1:] + if isinstance(instance, self._self_owner): + wrapped = PartialCallableObjectProxy(self.__wrapped__, instance) + return self._self_wrapper(wrapped, instance, newargs, kwargs) + + return self._self_wrapper(self.__wrapped__, self._self_instance, + args, kwargs) + + elif self._self_binding == 'callable': if self._self_instance is None: # This situation can occur where someone is calling the - # instancemethod via the class type and passing the instance - # as the first argument. We need to shift the args before - # making the call to the wrapper and effectively bind the - # instance to the wrapped function using a partial so the - # wrapper doesn't see anything as being different. + # instancemethod via the class type and passing the instance as + # the first argument. We need to shift the args before making + # the call to the wrapper and effectively bind the instance to + # the wrapped function using a partial so the wrapper doesn't + # see anything as being different. if not args: raise TypeError('missing 1 required positional argument') @@ -794,259 +814,43 @@ class FunctionWrapper(_FunctionWrapperBase): # or patch it in the __dict__ of the class type. # # So to get the best outcome we can, whenever we aren't sure what - # it is, we label it as a 'function'. If it was already bound and + # it is, we label it as a 'callable'. If it was already bound and # that is rebound later, we assume that it will be an instance - # method and try an cope with the possibility that the 'self' + # method and try and cope with the possibility that the 'self' # argument it being passed as an explicit argument and shuffle # the arguments around to extract 'self' for use as the instance. - if isinstance(wrapped, classmethod): - binding = 'classmethod' - - elif isinstance(wrapped, staticmethod): - binding = 'staticmethod' - - elif hasattr(wrapped, '__self__'): - if inspect.isclass(wrapped.__self__): - binding = 'classmethod' - else: - binding = 'function' - - else: - binding = 'function' - - super(FunctionWrapper, self).__init__(wrapped, None, wrapper, - enabled, binding) - -# disabled support for native extension; we likely don't need it -# try: -# if not os.environ.get('WRAPT_DISABLE_EXTENSIONS'): -# from ._wrappers import (ObjectProxy, CallableObjectProxy, -# PartialCallableObjectProxy, FunctionWrapper, -# BoundFunctionWrapper, _FunctionWrapperBase) -# except ImportError: -# pass - -# Helper functions for applying wrappers to existing functions. - -def resolve_path(module, name): - if isinstance(module, string_types): - __import__(module) - module = sys.modules[module] - - parent = module - - path = name.split('.') - attribute = path[0] - - # We can't just always use getattr() because in doing - # that on a class it will cause binding to occur which - # will complicate things later and cause some things not - # to work. For the case of a class we therefore access - # the __dict__ directly. To cope though with the wrong - # class being given to us, or a method being moved into - # a base class, we need to walk the class hierarchy to - # work out exactly which __dict__ the method was defined - # in, as accessing it from __dict__ will fail if it was - # not actually on the class given. Fallback to using - # getattr() if we can't find it. If it truly doesn't - # exist, then that will fail. - - def lookup_attribute(parent, attribute): - if inspect.isclass(parent): - for cls in inspect.getmro(parent): - if attribute in vars(cls): - return vars(cls)[attribute] - else: - return getattr(parent, attribute) - else: - return getattr(parent, attribute) - - original = lookup_attribute(parent, attribute) - - for attribute in path[1:]: - parent = original - original = lookup_attribute(parent, attribute) - - return (parent, attribute, original) - -def apply_patch(parent, attribute, replacement): - setattr(parent, attribute, replacement) - -def wrap_object(module, name, factory, args=(), kwargs={}): - (parent, attribute, original) = resolve_path(module, name) - wrapper = factory(original, *args, **kwargs) - apply_patch(parent, attribute, wrapper) - return wrapper - -# Function for applying a proxy object to an attribute of a class -# instance. The wrapper works by defining an attribute of the same name -# on the class which is a descriptor and which intercepts access to the -# instance attribute. Note that this cannot be used on attributes which -# are themselves defined by a property object. - -class AttributeWrapper(object): - - def __init__(self, attribute, factory, args, kwargs): - self.attribute = attribute - self.factory = factory - self.args = args - self.kwargs = kwargs - - def __get__(self, instance, owner): - value = instance.__dict__[self.attribute] - return self.factory(value, *self.args, **self.kwargs) - - def __set__(self, instance, value): - instance.__dict__[self.attribute] = value - - def __delete__(self, instance): - del instance.__dict__[self.attribute] - -def wrap_object_attribute(module, name, factory, args=(), kwargs={}): - path, attribute = name.rsplit('.', 1) - parent = resolve_path(module, path)[2] - wrapper = AttributeWrapper(attribute, factory, args, kwargs) - apply_patch(parent, attribute, wrapper) - return wrapper - -# Functions for creating a simple decorator using a FunctionWrapper, -# plus short cut functions for applying wrappers to functions. These are -# for use when doing monkey patching. For a more featured way of -# creating decorators see the decorator decorator instead. - -def function_wrapper(wrapper): - def _wrapper(wrapped, instance, args, kwargs): - target_wrapped = args[0] - if instance is None: - target_wrapper = wrapper - elif inspect.isclass(instance): - target_wrapper = wrapper.__get__(None, instance) - else: - target_wrapper = wrapper.__get__(instance, type(instance)) - return FunctionWrapper(target_wrapped, target_wrapper) - return FunctionWrapper(wrapper, _wrapper) - -def wrap_function_wrapper(module, name, wrapper): - return wrap_object(module, name, FunctionWrapper, (wrapper,)) - -def patch_function_wrapper(module, name): - def _wrapper(wrapper): - return wrap_object(module, name, FunctionWrapper, (wrapper,)) - return _wrapper - -def transient_function_wrapper(module, name): - def _decorator(wrapper): - def _wrapper(wrapped, instance, args, kwargs): - target_wrapped = args[0] - if instance is None: - target_wrapper = wrapper - elif inspect.isclass(instance): - target_wrapper = wrapper.__get__(None, instance) - else: - target_wrapper = wrapper.__get__(instance, type(instance)) - def _execute(wrapped, instance, args, kwargs): - (parent, attribute, original) = resolve_path(module, name) - replacement = FunctionWrapper(original, target_wrapper) - setattr(parent, attribute, replacement) - try: - return wrapped(*args, **kwargs) - finally: - setattr(parent, attribute, original) - return FunctionWrapper(target_wrapped, _execute) - return FunctionWrapper(wrapper, _wrapper) - return _decorator - -# A weak function proxy. This will work on instance methods, class -# methods, static methods and regular functions. Special treatment is -# needed for the method types because the bound method is effectively a -# transient object and applying a weak reference to one will immediately -# result in it being destroyed and the weakref callback called. The weak -# reference is therefore applied to the instance the method is bound to -# and the original function. The function is then rebound at the point -# of a call via the weak function proxy. - -def _weak_function_proxy_callback(ref, proxy, callback): - if proxy._self_expired: - return - - proxy._self_expired = True - - # This could raise an exception. We let it propagate back and let - # the weakref.proxy() deal with it, at which point it generally - # prints out a short error message direct to stderr and keeps going. - - if callback is not None: - callback(proxy) - -class WeakFunctionProxy(ObjectProxy): - - __slots__ = ('_self_expired', '_self_instance') - - def __init__(self, wrapped, callback=None): - # We need to determine if the wrapped function is actually a - # bound method. In the case of a bound method, we need to keep a - # reference to the original unbound function and the instance. - # This is necessary because if we hold a reference to the bound - # function, it will be the only reference and given it is a - # temporary object, it will almost immediately expire and - # the weakref callback triggered. So what is done is that we - # hold a reference to the instance and unbound function and - # when called bind the function to the instance once again and - # then call it. Note that we avoid using a nested function for - # the callback here so as not to cause any odd reference cycles. - - _callback = callback and functools.partial( - _weak_function_proxy_callback, proxy=self, - callback=callback) - - self._self_expired = False + binding = None if isinstance(wrapped, _FunctionWrapperBase): - self._self_instance = weakref.ref(wrapped._self_instance, - _callback) + binding = wrapped._self_binding - if wrapped._self_parent is not None: - super(WeakFunctionProxy, self).__init__( - weakref.proxy(wrapped._self_parent, _callback)) + if not binding: + if inspect.isbuiltin(wrapped): + binding = 'builtin' - else: - super(WeakFunctionProxy, self).__init__( - weakref.proxy(wrapped, _callback)) - - return - - try: - self._self_instance = weakref.ref(wrapped.__self__, _callback) - - super(WeakFunctionProxy, self).__init__( - weakref.proxy(wrapped.__func__, _callback)) - - except AttributeError: - self._self_instance = None - - super(WeakFunctionProxy, self).__init__( - weakref.proxy(wrapped, _callback)) - - def __call__(*args, **kwargs): - def _unpack_self(self, *args): - return self, args + elif inspect.isfunction(wrapped): + binding = 'function' - self, args = _unpack_self(*args) + elif inspect.isclass(wrapped): + binding = 'class' - # We perform a boolean check here on the instance and wrapped - # function as that will trigger the reference error prior to - # calling if the reference had expired. + elif isinstance(wrapped, classmethod): + binding = 'classmethod' - instance = self._self_instance and self._self_instance() - function = self.__wrapped__ and self.__wrapped__ + elif isinstance(wrapped, staticmethod): + binding = 'staticmethod' - # If the wrapped function was originally a bound function, for - # which we retained a reference to the instance and the unbound - # function we need to rebind the function and then call it. If - # not just called the wrapped function. + elif hasattr(wrapped, '__self__'): + if inspect.isclass(wrapped.__self__): + binding = 'classmethod' + elif inspect.ismethod(wrapped): + binding = 'instancemethod' + else: + binding = 'callable' - if instance is None: - return self.__wrapped__(*args, **kwargs) + else: + binding = 'callable' - return function.__get__(instance, type(instance))(*args, **kwargs) \ No newline at end of file + super(FunctionWrapper, self).__init__(wrapped, None, wrapper, + enabled, binding) From 945516c209268a91cc1f7169bf7d20517e398561 Mon Sep 17 00:00:00 2001 From: Sloane Hertel <19572925+s-hertel@users.noreply.github.com> Date: Mon, 4 Aug 2025 13:45:26 -0400 Subject: [PATCH 63/68] ansible-galaxy - remove internal path when using AnsibleCollectionConfig.collection_paths (#85596) * remove internal collections earlier to ignore consistently for different sub-commands * remove internal collection handling from the dependency resolver * add a test to ensure ansible._protomatter is not in the output of ansible-galaxy collection list * fix existing test to ensure an error is given if no valid collection path is configured * changelog --- changelogs/fragments/85596-hide-proto.yml | 3 +++ lib/ansible/cli/galaxy.py | 18 +++++++++++++++--- lib/ansible/galaxy/collection/__init__.py | 10 ++-------- .../dependency_resolution/dataclasses.py | 10 ---------- lib/ansible/plugins/loader.py | 2 +- .../collection_loader/_collection_config.py | 5 +++++ .../collection_loader/_collection_finder.py | 3 ++- .../ansible-galaxy-collection/tasks/list.yml | 6 +++--- 8 files changed, 31 insertions(+), 26 deletions(-) create mode 100644 changelogs/fragments/85596-hide-proto.yml diff --git a/changelogs/fragments/85596-hide-proto.yml b/changelogs/fragments/85596-hide-proto.yml new file mode 100644 index 00000000000..b6fb2127cf3 --- /dev/null +++ b/changelogs/fragments/85596-hide-proto.yml @@ -0,0 +1,3 @@ +bugfixes: +- >- + ``ansible-galaxy collection list`` - fail when none of the configured collection paths exist. diff --git a/lib/ansible/cli/galaxy.py b/lib/ansible/cli/galaxy.py index 6c8c749f9b4..981b02c7128 100755 --- a/lib/ansible/cli/galaxy.py +++ b/lib/ansible/cli/galaxy.py @@ -213,6 +213,18 @@ class GalaxyCLI(CLI): self.lazy_role_api = None super(GalaxyCLI, self).__init__(args) + @property + def collection_paths(self): + """ + Exclude lib/ansible/_internal/ansible_collections/. + """ + # exclude bundled collections, e.g. ansible._protomatter + return [ + path + for path in AnsibleCollectionConfig.collection_paths + if path != AnsibleCollectionConfig._internal_collections + ] + def init_parser(self): """ create an options parser for bin/ansible """ @@ -1281,7 +1293,7 @@ class GalaxyCLI(CLI): """Compare checksums with the collection(s) found on the server and the installed copy. This does not verify dependencies.""" collections = context.CLIARGS['args'] - search_paths = AnsibleCollectionConfig.collection_paths + search_paths = self.collection_paths ignore_errors = context.CLIARGS['ignore_errors'] local_verify_only = context.CLIARGS['offline'] requirements_file = context.CLIARGS['requirements'] @@ -1423,7 +1435,7 @@ class GalaxyCLI(CLI): collections_path = C.COLLECTIONS_PATHS managed_paths = set(validate_collection_path(p) for p in C.COLLECTIONS_PATHS) - read_req_paths = set(validate_collection_path(p) for p in AnsibleCollectionConfig.collection_paths) + read_req_paths = set(validate_collection_path(p) for p in self.collection_paths) unexpected_path = C.GALAXY_COLLECTIONS_PATH_WARNING and not any(p.startswith(path) for p in managed_paths) if unexpected_path and any(p.startswith(path) for p in read_req_paths): @@ -1639,7 +1651,7 @@ class GalaxyCLI(CLI): collection_name = context.CLIARGS['collection'] default_collections_path = set(C.COLLECTIONS_PATHS) collections_search_paths = ( - set(context.CLIARGS['collections_path'] or []) | default_collections_path | set(AnsibleCollectionConfig.collection_paths) + set(context.CLIARGS['collections_path'] or []) | default_collections_path | set(self.collection_paths) ) collections_in_paths = {} diff --git a/lib/ansible/galaxy/collection/__init__.py b/lib/ansible/galaxy/collection/__init__.py index 77e3870bdd2..38737468dcd 100644 --- a/lib/ansible/galaxy/collection/__init__.py +++ b/lib/ansible/galaxy/collection/__init__.py @@ -84,7 +84,6 @@ if t.TYPE_CHECKING: FileManifestEntryType = t.Dict[FileMetaKeysType, t.Union[str, int, None]] FilesManifestType = t.Dict[t.Literal['files', 'format'], t.Union[t.List[FileManifestEntryType], int]] -import ansible import ansible.constants as C from ansible.errors import AnsibleError from ansible.galaxy.api import GalaxyAPI @@ -144,8 +143,6 @@ ModifiedContent = namedtuple('ModifiedContent', ['filename', 'expected', 'instal SIGNATURE_COUNT_RE = r"^(?P\+)?(?:(?P\d+)|(?Pall))$" -_ANSIBLE_PACKAGE_PATH = pathlib.Path(ansible.__file__).parent - @dataclass class ManifestControl: @@ -1432,12 +1429,9 @@ def find_existing_collections(path_filter, artifacts_manager, namespace_filter=N paths = set() for path in files('ansible_collections').glob('*/*/'): path = _normalize_collection_path(path) - if path.is_relative_to(_ANSIBLE_PACKAGE_PATH): - # skip internal path, those collections are not for galaxy use - continue - elif not path.is_dir(): + if not path.is_dir(): continue - elif path_filter: + if path_filter: for pf in path_filter: try: path.relative_to(_normalize_collection_path(pf)) diff --git a/lib/ansible/galaxy/dependency_resolution/dataclasses.py b/lib/ansible/galaxy/dependency_resolution/dataclasses.py index 9877efdfc38..5fe66c16f1c 100644 --- a/lib/ansible/galaxy/dependency_resolution/dataclasses.py +++ b/lib/ansible/galaxy/dependency_resolution/dataclasses.py @@ -26,9 +26,6 @@ if t.TYPE_CHECKING: '_ComputedReqKindsMixin', ) -import ansible -import ansible.release - from ansible.errors import AnsibleError, AnsibleAssertionError from ansible.galaxy.api import GalaxyAPI from ansible.galaxy.collection import HAS_PACKAGING, PkgReq @@ -42,7 +39,6 @@ _ALLOW_CONCRETE_POINTER_IN_SOURCE = False # NOTE: This is a feature flag _GALAXY_YAML = b'galaxy.yml' _MANIFEST_JSON = b'MANIFEST.json' _SOURCE_METADATA_FILE = b'GALAXY.yml' -_ANSIBLE_PACKAGE_PATH = pathlib.Path(ansible.__file__).parent display = Display() @@ -229,12 +225,6 @@ class _ComputedReqKindsMixin: dir_path = dir_path.rstrip(to_bytes(os.path.sep)) if not _is_collection_dir(dir_path): dir_pathlib = pathlib.Path(to_text(dir_path)) - - # special handling for bundled collections without manifests, e.g., ansible._protomatter - if dir_pathlib.is_relative_to(_ANSIBLE_PACKAGE_PATH): - req_name = f'{dir_pathlib.parent.name}.{dir_pathlib.name}' - return cls(req_name, ansible.release.__version__, dir_path, 'dir', None) - display.warning( u"Collection at '{path!s}' does not have a {manifest_json!s} " u'file, nor has it {galaxy_yml!s}: cannot detect version.'. diff --git a/lib/ansible/plugins/loader.py b/lib/ansible/plugins/loader.py index b1f380558eb..b650bd82ed1 100644 --- a/lib/ansible/plugins/loader.py +++ b/lib/ansible/plugins/loader.py @@ -1674,7 +1674,7 @@ def _configure_collection_loader(prefix_collections_path=None): # insert the internal ansible._protomatter collection up front paths = [os.path.dirname(_internal.__file__)] + list(prefix_collections_path) + C.COLLECTIONS_PATHS - finder = _AnsibleCollectionFinder(paths, C.COLLECTIONS_SCAN_SYS_PATH) + finder = _AnsibleCollectionFinder(paths, C.COLLECTIONS_SCAN_SYS_PATH, internal_collections=paths[0]) finder._install() # this should succeed now diff --git a/lib/ansible/utils/collection_loader/_collection_config.py b/lib/ansible/utils/collection_loader/_collection_config.py index 6b305787ff1..234ec2b038e 100644 --- a/lib/ansible/utils/collection_loader/_collection_config.py +++ b/lib/ansible/utils/collection_loader/_collection_config.py @@ -62,6 +62,11 @@ class _AnsibleCollectionConfig(type): cls._require_finder() return [_to_text(p) for p in cls._collection_finder._n_collection_paths] + @property + def _internal_collections(cls): + cls._require_finder() + return cls._collection_finder._internal_collections + @property def default_collection(cls): return cls._default_collection diff --git a/lib/ansible/utils/collection_loader/_collection_finder.py b/lib/ansible/utils/collection_loader/_collection_finder.py index 27320806957..7e788808fb0 100644 --- a/lib/ansible/utils/collection_loader/_collection_finder.py +++ b/lib/ansible/utils/collection_loader/_collection_finder.py @@ -182,7 +182,7 @@ class _AnsibleTraversableResources(TraversableResources): class _AnsibleCollectionFinder: - def __init__(self, paths=None, scan_sys_paths=True): + def __init__(self, paths=None, scan_sys_paths=True, internal_collections=None): # TODO: accept metadata loader override self._ansible_pkg_path = _to_text(os.path.dirname(_to_bytes(sys.modules['ansible'].__file__))) @@ -209,6 +209,7 @@ class _AnsibleCollectionFinder: if p not in good_paths and os.path.isdir(_to_bytes(os.path.join(p, 'ansible_collections'))): good_paths.append(p) + self._internal_collections = internal_collections self._n_configured_paths = good_paths self._n_cached_collection_paths = None self._n_cached_collection_qualified_paths = None diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/list.yml b/test/integration/targets/ansible-galaxy-collection/tasks/list.yml index c0eeca8349c..159d139adf0 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/list.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/list.yml @@ -67,6 +67,7 @@ - 'list_result.stdout is regex "dev.collection4\s+\*"' - 'list_result.stdout is regex "dev.collection5\s+\*"' - 'list_result.stdout is regex "dev.collection6\s+\*"' + - 'list_result.stdout is not regex "ansible._protomatter\s+.*"' - name: list collections in human format command: ansible-galaxy collection list --format human @@ -158,9 +159,8 @@ - name: Ensure we get the expected error assert: that: - # FIXME: This test is currently incorrect, but not a fix to do in this PR, proper test is the commented out one. - - "'{}' in list_result_error.stdout" - #- "'None of the provided paths were usable' in list_result_error.stderr" + - "'{}' not in list_result_error.stdout" + - "'None of the provided paths were usable' in list_result_error.stderr" - name: install an artifact to the second collections path command: ansible-galaxy collection install namespace1.name1 -s galaxy_ng {{ galaxy_verbosity }} -p "{{ galaxy_dir }}/prod" From ca5871f2569669c61f0685961f39fff8ab966404 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Mon, 4 Aug 2025 16:58:44 -0400 Subject: [PATCH 64/68] add fragments to return (#72635) * ansible-doc add ability to use doc_fragments for RETURN --------- Co-authored-by: Felix Fontein --- changelogs/fragments/return_fragments.yml | 2 + lib/ansible/plugins/loader.py | 2 +- lib/ansible/utils/plugin_docs.py | 71 ++++++++++--------- .../doc_fragments/test_return_frag.py | 17 +++++ .../library/test_docs_return_fragments.py | 56 +++++++++++++++ test/integration/targets/ansible-doc/test.yml | 31 +++++--- .../test_docs_return_fragments.output | 34 +++++++++ 7 files changed, 170 insertions(+), 43 deletions(-) create mode 100644 changelogs/fragments/return_fragments.yml create mode 100644 test/integration/targets/ansible-doc/doc_fragments/test_return_frag.py create mode 100644 test/integration/targets/ansible-doc/library/test_docs_return_fragments.py create mode 100644 test/integration/targets/ansible-doc/test_docs_return_fragments.output diff --git a/changelogs/fragments/return_fragments.yml b/changelogs/fragments/return_fragments.yml new file mode 100644 index 00000000000..89d8ddff158 --- /dev/null +++ b/changelogs/fragments/return_fragments.yml @@ -0,0 +1,2 @@ +minor_changes: + - ansible-doc adds support for RETURN documentation to support doc fragment plugins diff --git a/lib/ansible/plugins/loader.py b/lib/ansible/plugins/loader.py index b650bd82ed1..bad7f88991b 100644 --- a/lib/ansible/plugins/loader.py +++ b/lib/ansible/plugins/loader.py @@ -517,7 +517,7 @@ class PluginLoader: # filename, cn = find_plugin_docfile( name, type_name, self, [os.path.dirname(path)], C.YAML_DOC_EXTENSIONS) if dstring: - add_fragments(dstring, path, fragment_loader=fragment_loader, is_module=(type_name == 'module')) + add_fragments(dstring, path, fragment_loader=fragment_loader, is_module=(type_name == 'module'), section='DOCUMENTATION') if 'options' in dstring and isinstance(dstring['options'], dict): C.config.initialize_plugin_configuration_definitions(type_name, name, dstring['options']) diff --git a/lib/ansible/utils/plugin_docs.py b/lib/ansible/utils/plugin_docs.py index 13db9a0f2e4..8c1d9b46658 100644 --- a/lib/ansible/utils/plugin_docs.py +++ b/lib/ansible/utils/plugin_docs.py @@ -18,6 +18,7 @@ from ansible.parsing.yaml.loader import AnsibleLoader from ansible.utils.display import Display from ansible._internal._datatag import _tags +_FRAGMENTABLE = ('DOCUMENTATION', 'RETURN') display = Display() @@ -125,7 +126,10 @@ def remove_current_collection_from_versions_and_dates(fragment, collection_name, _process_versions_and_dates(fragment, is_module, return_docs, remove) -def add_fragments(doc, filename, fragment_loader, is_module=False): +def add_fragments(doc, filename, fragment_loader, is_module=False, section='DOCUMENTATION'): + + if section not in _FRAGMENTABLE: + raise AnsibleError(f"Invalid fragment section ({section}) passed to render {filename}, it can only be one of {_FRAGMENTABLE!r}") fragments = doc.pop('extends_documentation_fragment', []) @@ -134,14 +138,14 @@ def add_fragments(doc, filename, fragment_loader, is_module=False): unknown_fragments = [] - # doc_fragments are allowed to specify a fragment var other than DOCUMENTATION + # doc_fragments are allowed to specify a fragment var other than DOCUMENTATION or RETURN # with a . separator; this is complicated by collections-hosted doc_fragments that # use the same separator. Assume it's collection-hosted normally first, try to load # as-specified. If failure, assume the right-most component is a var, split it off, # and retry the load. for fragment_slug in fragments: fragment_name = fragment_slug.strip() - fragment_var = 'DOCUMENTATION' + fragment_var = section fragment_class = fragment_loader.get(fragment_name) if fragment_class is None and '.' in fragment_slug: @@ -157,7 +161,7 @@ def add_fragments(doc, filename, fragment_loader, is_module=False): # trust-tagged source propagates to loaded values; expressions and templates in config require trust fragment_yaml = _tags.TrustedAsTemplate().tag(getattr(fragment_class, fragment_var, None)) if fragment_yaml is None: - if fragment_var != 'DOCUMENTATION': + if fragment_var not in _FRAGMENTABLE: # if it's asking for something specific that's missing, that's an error unknown_fragments.append(fragment_slug) continue @@ -168,35 +172,31 @@ def add_fragments(doc, filename, fragment_loader, is_module=False): real_fragment_name = getattr(fragment_class, 'ansible_name') real_collection_name = '.'.join(real_fragment_name.split('.')[0:2]) if '.' in real_fragment_name else '' - add_collection_to_versions_and_dates(fragment, real_collection_name, is_module=is_module) - - if 'notes' in fragment: - notes = fragment.pop('notes') - if notes: - if 'notes' not in doc: - doc['notes'] = [] - doc['notes'].extend(notes) - - if 'seealso' in fragment: - seealso = fragment.pop('seealso') - if seealso: - if 'seealso' not in doc: - doc['seealso'] = [] - doc['seealso'].extend(seealso) - - if 'options' not in fragment and 'attributes' not in fragment: - raise Exception("missing options or attributes in fragment (%s), possibly misformatted?: %s" % (fragment_name, filename)) - - # ensure options themselves are directly merged - for doc_key in ['options', 'attributes']: - if doc_key in fragment: - if doc_key in doc: - try: - merge_fragment(doc[doc_key], fragment.pop(doc_key)) - except Exception as e: - raise AnsibleError("%s %s (%s) of unknown type: %s" % (to_native(e), doc_key, fragment_name, filename)) - else: - doc[doc_key] = fragment.pop(doc_key) + add_collection_to_versions_and_dates(fragment, real_collection_name, is_module=is_module, return_docs=(section == 'RETURN')) + + if section == 'DOCUMENTATION': + # notes, seealso, options and attributes entries are specificly merged, but only occur in documentation section + for doc_key in ['notes', 'seealso']: + if doc_key in fragment: + entries = fragment.pop(doc_key) + if entries: + if doc_key not in doc: + doc[doc_key] = [] + doc[doc_key].extend(entries) + + if 'options' not in fragment and 'attributes' not in fragment: + raise Exception("missing options or attributes in fragment (%s), possibly misformatted?: %s" % (fragment_name, filename)) + + # ensure options themselves are directly merged + for doc_key in ['options', 'attributes']: + if doc_key in fragment: + if doc_key in doc: + try: + merge_fragment(doc[doc_key], fragment.pop(doc_key)) + except Exception as e: + raise AnsibleError("%s %s (%s) of unknown type: %s" % (to_native(e), doc_key, fragment_name, filename)) + else: + doc[doc_key] = fragment.pop(doc_key) # merge rest of the sections try: @@ -230,13 +230,16 @@ def get_docstring(filename, fragment_loader, verbose=False, ignore_errors=False, add_collection_to_versions_and_dates(data['doc'], collection_name, is_module=is_module) # add fragments to documentation - add_fragments(data['doc'], filename, fragment_loader=fragment_loader, is_module=is_module) + add_fragments(data['doc'], filename, fragment_loader=fragment_loader, is_module=is_module, section='DOCUMENTATION') if data.get('returndocs', False): # add collection name to versions and dates if collection_name is not None: add_collection_to_versions_and_dates(data['returndocs'], collection_name, is_module=is_module, return_docs=True) + # add fragments to return + add_fragments(data['returndocs'], filename, fragment_loader=fragment_loader, is_module=is_module, section='RETURN') + return data['doc'], data['plainexamples'], data['returndocs'], data['metadata'] diff --git a/test/integration/targets/ansible-doc/doc_fragments/test_return_frag.py b/test/integration/targets/ansible-doc/doc_fragments/test_return_frag.py new file mode 100644 index 00000000000..528d44c1aa9 --- /dev/null +++ b/test/integration/targets/ansible-doc/doc_fragments/test_return_frag.py @@ -0,0 +1,17 @@ +from __future__ import annotations + + +class ModuleDocFragment(object): + + # Standard documentation fragment + RETURN = r''' +y_notlast: + description: A return from fragment + type: str + returned: it depends TM + +z_last: + description: A a return from fragment with merge. + type: str + returned: success +''' diff --git a/test/integration/targets/ansible-doc/library/test_docs_return_fragments.py b/test/integration/targets/ansible-doc/library/test_docs_return_fragments.py new file mode 100644 index 00000000000..76e9b24d19e --- /dev/null +++ b/test/integration/targets/ansible-doc/library/test_docs_return_fragments.py @@ -0,0 +1,56 @@ +#!/usr/bin/python + +from __future__ import annotations + + +DOCUMENTATION = ''' +--- +module: test_docs_returns +short_description: Test module +description: + - Test module +author: + - Ansible Core Team +''' + +EXAMPLES = ''' +''' + +RETURN = ''' +m_middle: + description: + - This should be in the middle. + - Has some more data + type: dict + returned: success and 1st of month + contains: + suboption: + description: A suboption. + type: str + choices: [ARF, BARN, c_without_capital_first_letter] + +a_first: + description: A first result. + type: str + returned: success + +z_last: + example: this is a merge +extends_documentation_fragment: + - test_return_frag +''' + + +from ansible.module_utils.basic import AnsibleModule + + +def main(): + module = AnsibleModule( + argument_spec=dict(), + ) + + module.exit_json() + + +if __name__ == '__main__': + main() diff --git a/test/integration/targets/ansible-doc/test.yml b/test/integration/targets/ansible-doc/test.yml index 1ea77869f75..fbf36ae2593 100644 --- a/test/integration/targets/ansible-doc/test.yml +++ b/test/integration/targets/ansible-doc/test.yml @@ -4,6 +4,7 @@ ANSIBLE_LIBRARY: "{{ playbook_dir }}/library" ANSIBLE_NOCOLOR: 1 ANSIBLE_DEPRECATION_WARNINGS: 1 + ANSIBLE_DOC_FRAGMENT_PLUGINS: "{{ playbook_dir }}/doc_fragments" vars: # avoid header that has full path and won't work across random paths for tests actual_output_clean: '{{ actual_output.splitlines()[1:] }}' @@ -26,28 +27,26 @@ register: result ignore_errors: true - - set_fact: - actual_output: "{{ result.stdout }}" - expected_output: "{{ lookup('file', 'test_docs_suboptions.output') }}" - - assert: that: - result is succeeded - actual_output_clean == expected_output_clean + vars: + actual_output: "{{ result.stdout }}" + expected_output: "{{ lookup('file', 'test_docs_suboptions.output') }}" - name: module with return docs shell: ansible-doc test_docs_returns| tail -n +3 register: result ignore_errors: true - - set_fact: - actual_output: "{{ result.stdout }}" - expected_output: "{{ lookup('file', 'test_docs_returns.output') }}" - - assert: that: - result is succeeded - actual_output_clean == expected_output_clean + vars: + actual_output: "{{ result.stdout }}" + expected_output: "{{ lookup('file', 'test_docs_returns.output') }}" - name: module with broken return docs command: ansible-doc test_docs_returns_broken @@ -59,6 +58,22 @@ - result is failed - '"module test_docs_returns_broken Missing documentation (or could not parse documentation)" in result.stderr' + - name: module with return docs with fragments + command: ansible-doc test_docs_return_fragments + register: result + ignore_errors: true + + - name: test fragments work + assert: + that: + - result is success + - "'z_last' in actual_output" + - "'y_notlast' in actual_output" + - actual_output_clean == expected_output_clean + vars: + actual_output: "{{ result.stdout }}" + expected_output: "{{ lookup('file', 'test_docs_return_fragments.output') }}" + - name: non-existent module command: ansible-doc test_does_not_exist register: result diff --git a/test/integration/targets/ansible-doc/test_docs_return_fragments.output b/test/integration/targets/ansible-doc/test_docs_return_fragments.output new file mode 100644 index 00000000000..68f3d9da403 --- /dev/null +++ b/test/integration/targets/ansible-doc/test_docs_return_fragments.output @@ -0,0 +1,34 @@ +> MODULE test_docs_return_fragments + + Test module + +AUTHOR: Ansible Core Team + +EXAMPLES: + + +RETURN VALUES: + +- a_first A first result. + returned: success + type: str + +- m_middle This should be in the middle. + Has some more data + returned: success and 1st of month + type: dict + contains: + + - suboption A suboption. + choices: [ARF, BARN, c_without_capital_first_letter] + type: str + +- y_notlast A return from fragment + returned: it depends TM + type: str + +- z_last A a return from fragment with merge. + example: this is a merge + returned: success + type: str + From faf86ca2b3e7b06660528d69a9839bb8d1409f70 Mon Sep 17 00:00:00 2001 From: Matt Davis <6775756+nitzmahone@users.noreply.github.com> Date: Mon, 4 Aug 2025 16:43:59 -0700 Subject: [PATCH 65/68] Add temporary module result serialization hook (#85609) * Add temporary module result serialization hook * Sanity test fix --------- Co-authored-by: Matt Clay --- changelogs/fragments/module_direct_exec.yml | 2 ++ lib/ansible/module_utils/basic.py | 12 +++++++-- .../module_utils/basic/test_exit_json.py | 25 ++++++++++++++++++- 3 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 changelogs/fragments/module_direct_exec.yml diff --git a/changelogs/fragments/module_direct_exec.yml b/changelogs/fragments/module_direct_exec.yml new file mode 100644 index 00000000000..edc407023d8 --- /dev/null +++ b/changelogs/fragments/module_direct_exec.yml @@ -0,0 +1,2 @@ +minor_changes: + - AnsibleModule - Add temporary internal monkeypatch-able hook to alter module result serialization by splitting serialization from ``_return_formatted`` into ``_record_module_result``. diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index f361e2c84cf..ba87c7b3850 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -1513,11 +1513,19 @@ class AnsibleModule(object): # strip no_log collisions kwargs = remove_values(kwargs, self.no_log_values) - # return preserved + # graft preserved values back on kwargs.update(preserved) + self._record_module_result(kwargs) + + def _record_module_result(self, o: dict[str, t.Any]) -> None: + """ + Temporary internal hook to enable modification/bypass of module result serialization. + + Monkeypatched by ansible.netcommon for direct in-worker module execution. + """ encoder = _json.get_module_encoder(_ANSIBLE_PROFILE, _json.Direction.MODULE_TO_CONTROLLER) - print('\n%s' % json.dumps(kwargs, cls=encoder)) + print('\n%s' % json.dumps(o, cls=encoder)) def exit_json(self, **kwargs) -> t.NoReturn: """ return from the module, without error """ diff --git a/test/units/module_utils/basic/test_exit_json.py b/test/units/module_utils/basic/test_exit_json.py index 47d96d3c1d6..f010e56f630 100644 --- a/test/units/module_utils/basic/test_exit_json.py +++ b/test/units/module_utils/basic/test_exit_json.py @@ -10,7 +10,7 @@ import datetime import typing as t import pytest - +import pytest_mock EMPTY_INVOCATION: dict[str, dict[str, t.Any]] = {u'module_args': {}} DATETIME = datetime.datetime.strptime('2020-07-13 12:50:00', '%Y-%m-%d %H:%M:%S') @@ -153,3 +153,26 @@ class TestAnsibleModuleExitValuesRemoved: out, err = capfd.readouterr() assert json.loads(out) == expected + + def test_record_module_result(self, mocker: pytest_mock.MockerFixture, stdin) -> None: + """Ensure that the temporary _record_module_result hook is called correctly.""" + recorded_result = None + + expected_result = dict(changed=False, worked="yay") + + def _record_module_result(_self, o: object) -> None: + assert isinstance(o, dict) + + nonlocal recorded_result + recorded_result = o + + from ansible.module_utils.basic import AnsibleModule + + mocker.patch.object(AnsibleModule, '_record_module_result', _record_module_result) + + am = AnsibleModule(argument_spec=dict()) + + with pytest.raises(SystemExit): + am.exit_json(**expected_result) + + assert expected_result.items() <= recorded_result.items() From 9ed7164ed6041593bc65056d5e53f098174aeaf4 Mon Sep 17 00:00:00 2001 From: Abhijeet Kasurde Date: Thu, 7 Aug 2025 12:16:52 -0700 Subject: [PATCH 66/68] service_facts: Handle KeyError while processing service name (#85572) * service_facts: Handle KeyError while processing service name As a part of follow up review, * Handle KeyError with exception handling * Warn user about the missing service name in the given service details Signed-off-by: Abhijeet Kasurde --- changelogs/fragments/openrc.yml | 4 ++++ lib/ansible/modules/service_facts.py | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 changelogs/fragments/openrc.yml diff --git a/changelogs/fragments/openrc.yml b/changelogs/fragments/openrc.yml new file mode 100644 index 00000000000..c6d7f4a4ccf --- /dev/null +++ b/changelogs/fragments/openrc.yml @@ -0,0 +1,4 @@ +--- +minor_changes: + - service_facts - warn user about missing service details instead of ignoring. + - service_facts - handle keyerror exceptions with warning. diff --git a/lib/ansible/modules/service_facts.py b/lib/ansible/modules/service_facts.py index 4b67b61e42c..35b2866caee 100644 --- a/lib/ansible/modules/service_facts.py +++ b/lib/ansible/modules/service_facts.py @@ -232,7 +232,11 @@ class ServiceScanService(BaseService): if service_name == "*": continue service_state = line_data[1] - service_runlevels = all_services_runlevels[service_name] + try: + service_runlevels = all_services_runlevels[service_name] + except KeyError: + self.module.warn(f"Service {service_name} not found in the service list") + continue service_data = {"name": service_name, "runlevels": service_runlevels, "state": service_state, "source": "openrc"} services[service_name] = service_data From 4c04b8c7c3f43b9c50831e97e4479e428cef9bd6 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Thu, 7 Aug 2025 21:52:47 +0200 Subject: [PATCH 67/68] IncludedFile: store _from args for proper deduplication (#85628) * IncludedFile: store _from args for proper deduplication Fixes #66497 Co-authored-by: Matt Martz --- .../66497-include_role-_from-dedup.yml | 2 ++ lib/ansible/playbook/included_file.py | 2 +- .../targets/includes/includes_from_dedup.yml | 18 ++++++++++++++++++ .../roles/role_from_args/tasks/localhost.yml | 2 ++ .../roles/role_from_args/tasks/testhost.yml | 2 ++ .../roles/role_from_args/vars/localhost.yml | 1 + .../roles/role_from_args/vars/testhost.yml | 1 + test/integration/targets/includes/runme.sh | 2 ++ test/units/playbook/test_included_file.py | 4 ++-- 9 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 changelogs/fragments/66497-include_role-_from-dedup.yml create mode 100644 test/integration/targets/includes/includes_from_dedup.yml create mode 100644 test/integration/targets/includes/roles/role_from_args/tasks/localhost.yml create mode 100644 test/integration/targets/includes/roles/role_from_args/tasks/testhost.yml create mode 100644 test/integration/targets/includes/roles/role_from_args/vars/localhost.yml create mode 100644 test/integration/targets/includes/roles/role_from_args/vars/testhost.yml diff --git a/changelogs/fragments/66497-include_role-_from-dedup.yml b/changelogs/fragments/66497-include_role-_from-dedup.yml new file mode 100644 index 00000000000..95497230160 --- /dev/null +++ b/changelogs/fragments/66497-include_role-_from-dedup.yml @@ -0,0 +1,2 @@ +bugfixes: + - include_role - allow host specific values in all ``*_from`` arguments (https://github.com/ansible/ansible/issues/66497) diff --git a/lib/ansible/playbook/included_file.py b/lib/ansible/playbook/included_file.py index ace611d86f2..51cdc7281a7 100644 --- a/lib/ansible/playbook/included_file.py +++ b/lib/ansible/playbook/included_file.py @@ -203,7 +203,7 @@ class IncludedFile: for from_arg in new_task.FROM_ARGS: if from_arg in include_args: from_key = from_arg.removesuffix('_from') - new_task._from_files[from_key] = include_args.pop(from_arg) + new_task._from_files[from_key] = include_args.get(from_arg) inc_file = IncludedFile(role_name, include_args, special_vars, new_task, is_role=True) diff --git a/test/integration/targets/includes/includes_from_dedup.yml b/test/integration/targets/includes/includes_from_dedup.yml new file mode 100644 index 00000000000..0902e359cab --- /dev/null +++ b/test/integration/targets/includes/includes_from_dedup.yml @@ -0,0 +1,18 @@ +- hosts: testhost,localhost + tasks: + - include_role: + name: role_from_args + vars_from: "{{ inventory_hostname }}.yml" + tasks_from: "{{ inventory_hostname }}.yml" + + - assert: + that: + - f1 is defined + - f1 == inventory_hostname + when: inventory_hostname == "testhost" + + - assert: + that: + - f2 is defined + - f2 == inventory_hostname + when: inventory_hostname == "localhost" diff --git a/test/integration/targets/includes/roles/role_from_args/tasks/localhost.yml b/test/integration/targets/includes/roles/role_from_args/tasks/localhost.yml new file mode 100644 index 00000000000..bd32273d800 --- /dev/null +++ b/test/integration/targets/includes/roles/role_from_args/tasks/localhost.yml @@ -0,0 +1,2 @@ +- set_fact: + f2: "{{ v }}" diff --git a/test/integration/targets/includes/roles/role_from_args/tasks/testhost.yml b/test/integration/targets/includes/roles/role_from_args/tasks/testhost.yml new file mode 100644 index 00000000000..33efa34d2f2 --- /dev/null +++ b/test/integration/targets/includes/roles/role_from_args/tasks/testhost.yml @@ -0,0 +1,2 @@ +- set_fact: + f1: "{{ v }}" diff --git a/test/integration/targets/includes/roles/role_from_args/vars/localhost.yml b/test/integration/targets/includes/roles/role_from_args/vars/localhost.yml new file mode 100644 index 00000000000..be240abd4c4 --- /dev/null +++ b/test/integration/targets/includes/roles/role_from_args/vars/localhost.yml @@ -0,0 +1 @@ +v: localhost diff --git a/test/integration/targets/includes/roles/role_from_args/vars/testhost.yml b/test/integration/targets/includes/roles/role_from_args/vars/testhost.yml new file mode 100644 index 00000000000..1d1e5d31fa2 --- /dev/null +++ b/test/integration/targets/includes/roles/role_from_args/vars/testhost.yml @@ -0,0 +1 @@ +v: testhost diff --git a/test/integration/targets/includes/runme.sh b/test/integration/targets/includes/runme.sh index 97effac9cd8..f87e4048b6d 100755 --- a/test/integration/targets/includes/runme.sh +++ b/test/integration/targets/includes/runme.sh @@ -16,3 +16,5 @@ grep -q "'include_tasks' is not a valid attribute for a Play" <<< "$result" ansible-playbook includes_loop_rescue.yml --extra-vars strategy=linear "$@" ansible-playbook includes_loop_rescue.yml --extra-vars strategy=free "$@" + +ansible-playbook includes_from_dedup.yml -i ../../inventory "$@" diff --git a/test/units/playbook/test_included_file.py b/test/units/playbook/test_included_file.py index 94976d82d83..8c7b39cd67b 100644 --- a/test/units/playbook/test_included_file.py +++ b/test/units/playbook/test_included_file.py @@ -312,8 +312,8 @@ def test_process_include_simulate_free_block_role_tasks(mock_iterator, mock_vari assert res[0]._hosts == [host1] assert res[1]._hosts == [host2] - assert res[0]._args == {} - assert res[1]._args == {} + assert res[0]._args == {'tasks_from': 'task1.yml'} + assert res[1]._args == {'tasks_from': 'task2.yml'} assert res[0]._vars == {} assert res[1]._vars == {} From 9a6420e1d55d0947aa96c0521542cbc336ed9356 Mon Sep 17 00:00:00 2001 From: Sloane Hertel <19572925+s-hertel@users.noreply.github.com> Date: Fri, 8 Aug 2025 15:51:00 -0400 Subject: [PATCH 68/68] Fix nested import_tasks when the parent import contains a template (#85599) * Test nested import_tasks when the parent import contains a template * Fix templating the parent_include.args --- .../85599-fix-templating-import_tasks-parent-include.yml | 2 ++ lib/ansible/playbook/helpers.py | 1 + .../targets/include_import_tasks_nested/tasks/main.yml | 2 ++ .../targets/include_import_tasks_nested/tests/main.yml | 1 + .../include_import_tasks_nested/tests/tests_relative.yml | 0 5 files changed, 6 insertions(+) create mode 100644 changelogs/fragments/85599-fix-templating-import_tasks-parent-include.yml create mode 100644 test/integration/targets/include_import_tasks_nested/tests/main.yml create mode 100644 test/integration/targets/include_import_tasks_nested/tests/tests_relative.yml diff --git a/changelogs/fragments/85599-fix-templating-import_tasks-parent-include.yml b/changelogs/fragments/85599-fix-templating-import_tasks-parent-include.yml new file mode 100644 index 00000000000..c24c8531dc0 --- /dev/null +++ b/changelogs/fragments/85599-fix-templating-import_tasks-parent-include.yml @@ -0,0 +1,2 @@ +bugfixes: + - import_tasks - fix templating parent include arguments. diff --git a/lib/ansible/playbook/helpers.py b/lib/ansible/playbook/helpers.py index eb96213e216..805f46145d3 100644 --- a/lib/ansible/playbook/helpers.py +++ b/lib/ansible/playbook/helpers.py @@ -169,6 +169,7 @@ def load_list_of_tasks(ds, play, block=None, role=None, task_include=None, use_h if not isinstance(parent_include, TaskInclude): parent_include = parent_include._parent continue + parent_include.post_validate(templar=templar) parent_include_dir = os.path.dirname(parent_include.args.get('_raw_params')) if cumulative_path is None: cumulative_path = parent_include_dir diff --git a/test/integration/targets/include_import_tasks_nested/tasks/main.yml b/test/integration/targets/include_import_tasks_nested/tasks/main.yml index 5d6726772f4..671eb2f1c5d 100644 --- a/test/integration/targets/include_import_tasks_nested/tasks/main.yml +++ b/test/integration/targets/include_import_tasks_nested/tasks/main.yml @@ -9,3 +9,5 @@ - assert: that: - nested_adjacent_count|int == 2 + +- import_tasks: "{{ role_path }}/tests/main.yml" diff --git a/test/integration/targets/include_import_tasks_nested/tests/main.yml b/test/integration/targets/include_import_tasks_nested/tests/main.yml new file mode 100644 index 00000000000..d862b1fb2cf --- /dev/null +++ b/test/integration/targets/include_import_tasks_nested/tests/main.yml @@ -0,0 +1 @@ +- import_tasks: tests_relative.yml diff --git a/test/integration/targets/include_import_tasks_nested/tests/tests_relative.yml b/test/integration/targets/include_import_tasks_nested/tests/tests_relative.yml new file mode 100644 index 00000000000..e69de29bb2d