From ff6998f2b90967fb7efbdd4eb0caf8a5d21e434c Mon Sep 17 00:00:00 2001 From: Matt Davis <6775756+nitzmahone@users.noreply.github.com> Date: Mon, 5 May 2025 18:00:02 -0700 Subject: [PATCH] Switch to stackwalk caller ID (#85095) * See changelog fragment for most changes. * Defer early config warnings until display is functioning, eliminating related fallback display logic. * Added more type annotations and docstrings. * ansible-test - pylint sanity for deprecations improved. * Refactored inline legacy resolutions in PluginLoader. Co-authored-by: Matt Clay --- changelogs/fragments/deprecator.yml | 17 + .../fragments/templates_types_datatagging.yml | 4 - hacking/test-module.py | 11 +- lib/ansible/_internal/_ansiballz.py | 5 +- lib/ansible/_internal/_templating/_datatag.py | 7 +- lib/ansible/_internal/_templating/_engine.py | 7 +- .../_internal/_templating/_jinja_plugins.py | 8 +- lib/ansible/cli/__init__.py | 5 +- lib/ansible/cli/arguments/option_helpers.py | 5 +- lib/ansible/cli/doc.py | 1 - lib/ansible/constants.py | 62 --- lib/ansible/errors/__init__.py | 8 +- lib/ansible/executor/module_common.py | 18 +- lib/ansible/executor/task_executor.py | 14 +- lib/ansible/galaxy/api.py | 2 +- lib/ansible/galaxy/collection/__init__.py | 6 +- .../module_utils/_internal/_ansiballz.py | 34 +- .../_internal/_dataclass_annotation_patch.py | 64 --- .../module_utils/_internal/_datatag/_tags.py | 28 +- .../module_utils/_internal/_deprecator.py | 134 +++++ .../_internal/_plugin_exec_context.py | 49 -- .../module_utils/_internal/_plugin_info.py | 25 + .../module_utils/_internal/_validation.py | 14 + lib/ansible/module_utils/basic.py | 81 ++- lib/ansible/module_utils/common/arg_spec.py | 11 +- lib/ansible/module_utils/common/messages.py | 63 ++- lib/ansible/module_utils/common/process.py | 1 - lib/ansible/module_utils/common/respawn.py | 7 - lib/ansible/module_utils/common/warnings.py | 26 +- lib/ansible/module_utils/datatag.py | 26 +- lib/ansible/playbook/task.py | 2 - lib/ansible/plugins/__init__.py | 26 +- lib/ansible/plugins/action/__init__.py | 8 - lib/ansible/plugins/action/gather_facts.py | 6 +- lib/ansible/plugins/callback/oneline.py | 8 +- lib/ansible/plugins/callback/tree.py | 8 +- lib/ansible/plugins/connection/local.py | 2 +- .../plugins/connection/paramiko_ssh.py | 11 +- lib/ansible/plugins/filter/core.py | 5 +- lib/ansible/plugins/inventory/__init__.py | 4 +- lib/ansible/plugins/loader.py | 324 +++++++----- lib/ansible/plugins/lookup/url.py | 4 +- lib/ansible/plugins/strategy/__init__.py | 12 +- lib/ansible/template/__init__.py | 2 +- .../collection_loader/_collection_meta.py | 8 +- lib/ansible/utils/display.py | 204 +++++--- lib/ansible/utils/py3compat.py | 8 +- lib/ansible/utils/ssh_functions.py | 5 +- lib/ansible/vars/manager.py | 28 +- lib/ansible/vars/plugins.py | 8 +- .../col/plugins/action/do_deprecated_stuff.py | 36 ++ .../ns/col/plugins/lookup/deprecated.py | 64 +++ .../plugins/module_utils/deprecated_utils.py | 34 ++ .../deprecated_thing.py | 28 ++ .../ansible-test-sanity-pylint/expected.txt | 37 +- .../ansible-test-sanity-pylint/runme.sh | 11 +- .../plugins/inventory/statichost.py | 2 + .../expected_stderr.txt | 6 +- .../library/datatag_module.py | 24 - .../library/tagging_sample.py | 4 +- .../action_plugins/action_with_dep.py | 14 - .../foo/bar/plugins/__init__.py | 0 .../foo/bar/plugins/action/__init__.py | 0 .../foo/bar/plugins/action/noisy_action.py | 18 + .../module_utils/shared_deprecation.py | 26 + .../foo/bar/plugins/modules/__init__.py | 0 .../foo/bar/plugins/modules/noisy.py | 18 + .../targets/deprecations/deprecated.yml | 39 +- .../targets/deprecations/disabled.yml | 14 - .../targets/deprecations/library/noisy.py | 14 - .../integration/targets/deprecations/runme.sh | 6 +- .../module_utils/module_utils_test.yml | 2 +- .../targets/plugin_namespace/tasks/main.yml | 4 +- .../test_plugins/test_test.py | 4 +- .../emit_deprecation_warning.py | 2 +- .../lookup_plugins/synthetic_plugin_info.py | 1 - .../targets/protomatter/tasks/main.yml | 7 +- .../_internal/commands/sanity/pylint.py | 1 + .../sanity/pylint/plugins/deprecated.py | 399 --------------- .../sanity/pylint/plugins/deprecated_calls.py | 475 ++++++++++++++++++ .../pylint/plugins/deprecated_comment.py | 137 +++++ test/sanity/code-smell/mypy/ansible-test.ini | 9 + test/sanity/ignore.txt | 9 + test/units/conftest.py | 10 + .../module_common/test_recursive_finder.py | 4 +- .../module_utils/_internal/test_deprecator.py | 73 +++ .../module_utils/basic/test_deprecate_warn.py | 91 ---- .../common/warnings/test_deprecate.py | 4 +- .../module_utils/datatag/test_datatag.py | 16 +- test/units/plugins/test_plugins.py | 4 +- test/units/utils/test_display.py | 88 +++- test/units/utils/test_listify.py | 5 +- .../utils/test_serialization_profiles.py | 3 + 93 files changed, 1934 insertions(+), 1235 deletions(-) create mode 100644 changelogs/fragments/deprecator.yml delete mode 100644 lib/ansible/module_utils/_internal/_dataclass_annotation_patch.py create mode 100644 lib/ansible/module_utils/_internal/_deprecator.py delete mode 100644 lib/ansible/module_utils/_internal/_plugin_exec_context.py create mode 100644 lib/ansible/module_utils/_internal/_plugin_info.py create mode 100644 lib/ansible/module_utils/_internal/_validation.py create mode 100644 test/integration/targets/ansible-test-sanity-pylint/ansible_collections/ns/col/plugins/action/do_deprecated_stuff.py create mode 100644 test/integration/targets/ansible-test-sanity-pylint/ansible_collections/ns/col/plugins/module_utils/deprecated_utils.py create mode 100644 test/integration/targets/ansible-test-sanity-pylint/deprecated_thing.py delete mode 100644 test/integration/targets/data_tagging_controller/library/datatag_module.py delete mode 100644 test/integration/targets/deprecations/action_plugins/action_with_dep.py create mode 100644 test/integration/targets/deprecations/collections/ansible_collections/foo/bar/plugins/__init__.py create mode 100644 test/integration/targets/deprecations/collections/ansible_collections/foo/bar/plugins/action/__init__.py create mode 100644 test/integration/targets/deprecations/collections/ansible_collections/foo/bar/plugins/action/noisy_action.py create mode 100644 test/integration/targets/deprecations/collections/ansible_collections/foo/bar/plugins/module_utils/shared_deprecation.py create mode 100644 test/integration/targets/deprecations/collections/ansible_collections/foo/bar/plugins/modules/__init__.py create mode 100644 test/integration/targets/deprecations/collections/ansible_collections/foo/bar/plugins/modules/noisy.py delete mode 100644 test/integration/targets/deprecations/disabled.yml delete mode 100644 test/integration/targets/deprecations/library/noisy.py delete mode 100644 test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated.py create mode 100644 test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated_calls.py create mode 100644 test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated_comment.py create mode 100644 test/units/module_utils/_internal/test_deprecator.py delete mode 100644 test/units/module_utils/basic/test_deprecate_warn.py diff --git a/changelogs/fragments/deprecator.yml b/changelogs/fragments/deprecator.yml new file mode 100644 index 00000000000..2a5ed82c2d4 --- /dev/null +++ b/changelogs/fragments/deprecator.yml @@ -0,0 +1,17 @@ +minor_changes: + - modules - The ``AnsibleModule.deprecate`` function no longer sends deprecation messages to the target host's logging system. + - ansible-test - Improved ``pylint`` checks for Ansible-specific deprecation functions. + - deprecations - Removed support for specifying deprecation dates as a ``datetime.date``, which was included in an earlier 2.19 pre-release. + - deprecations - Some argument names to ``deprecate_value`` for consistency with existing APIs. + An earlier 2.19 pre-release included a ``removal_`` prefix on the ``date`` and ``version`` arguments. + - deprecations - Collection name strings not of the form ``ns.coll`` passed to deprecation API functions will result in an error. + - collection metadata - The collection loader now parses scalar values from ``meta/runtime.yml`` as strings. + This avoids issues caused by unquoted values such as versions or dates being parsed as types other than strings. + - deprecation warnings - Deprecation warning APIs automatically capture the identity of the deprecating plugin. + The ``collection_name`` argument is only required to correctly attribute deprecations that occur in module_utils or other non-plugin code. + - deprecation warnings - Improved deprecation messages to more clearly indicate the affected content, including plugin name when available. + +deprecated_features: + - plugins - Accessing plugins with ``_``-prefixed filenames without the ``_`` prefix is deprecated. + - Passing a ``warnings` or ``deprecations`` key to ``exit_json`` or ``fail_json`` is deprecated. + Use ``AnsibleModule.warn`` or ``AnsibleModule.deprecate`` instead. diff --git a/changelogs/fragments/templates_types_datatagging.yml b/changelogs/fragments/templates_types_datatagging.yml index ac65f6c668b..84875d2d6a4 100644 --- a/changelogs/fragments/templates_types_datatagging.yml +++ b/changelogs/fragments/templates_types_datatagging.yml @@ -47,10 +47,6 @@ minor_changes: - to_json / to_nice_json filters - The filters accept a ``profile`` argument, which defaults to ``tagless``. - undef jinja function - The ``undef`` jinja function now raises an error if a non-string hint is given. Attempting to use an undefined hint also results in an error, ensuring incorrect use of the function can be distinguished from the function's normal behavior. - - display - The ``collection_name`` arg to ``Display.deprecated`` no longer has any effect. - Information about the calling plugin is automatically captured by the display infrastructure, included in the displayed messages, and made available to callbacks. - - modules - The ``collection_name`` arg to Python module-side ``deprecate`` methods no longer has any effect. - Information about the calling module is automatically captured by the warning infrastructure and included in the module result. breaking_changes: - loops - Omit placeholders no longer leak between loop item templating and task templating. diff --git a/hacking/test-module.py b/hacking/test-module.py index ca0e1ab425d..a5cf7862311 100755 --- a/hacking/test-module.py +++ b/hacking/test-module.py @@ -40,7 +40,6 @@ import shutil from pathlib import Path -from ansible.module_utils.common.messages import PluginInfo from ansible.release import __version__ import ansible.utils.vars as utils_vars from ansible.parsing.dataloader import DataLoader @@ -172,15 +171,8 @@ def boilerplate_module(modfile, args, interpreters, check, destfile): modname = os.path.basename(modfile) modname = os.path.splitext(modname)[0] - plugin = PluginInfo( - requested_name=modname, - resolved_name=modname, - type='module', - ) - built_module = module_common.modify_module( module_name=modname, - plugin=plugin, module_path=modfile, module_args=complex_args, templar=Templar(loader=loader), @@ -225,10 +217,11 @@ def ansiballz_setup(modfile, modname, interpreters): # All the directories in an AnsiBallZ that modules can live core_dirs = glob.glob(os.path.join(debug_dir, 'ansible/modules')) + non_core_dirs = glob.glob(os.path.join(debug_dir, 'ansible/legacy')) collection_dirs = glob.glob(os.path.join(debug_dir, 'ansible_collections/*/*/plugins/modules')) # There's only one module in an AnsiBallZ payload so look for the first module and then exit - for module_dir in core_dirs + collection_dirs: + for module_dir in core_dirs + collection_dirs + non_core_dirs: for dirname, directories, filenames in os.walk(module_dir): for filename in filenames: if filename == modname + '.py': diff --git a/lib/ansible/_internal/_ansiballz.py b/lib/ansible/_internal/_ansiballz.py index b60d02de1b1..eda09288fb7 100644 --- a/lib/ansible/_internal/_ansiballz.py +++ b/lib/ansible/_internal/_ansiballz.py @@ -42,7 +42,6 @@ def _ansiballz_main( module_fqn: str, params: str, profile: str, - plugin_info_dict: dict[str, object], date_time: datetime.datetime, coverage_config: str | None, coverage_output: str | None, @@ -142,7 +141,6 @@ def _ansiballz_main( run_module( json_params=json_params, profile=profile, - plugin_info_dict=plugin_info_dict, module_fqn=module_fqn, modlib_path=modlib_path, coverage_config=coverage_config, @@ -230,13 +228,12 @@ def _ansiballz_main( run_module( json_params=json_params, profile=profile, - plugin_info_dict=plugin_info_dict, module_fqn=module_fqn, modlib_path=modlib_path, ) else: - print('WARNING: Unknown debug command. Doing nothing.') + print(f'FATAL: Unknown debug command {command!r}. Doing nothing.') # # See comments in the debug() method for information on debugging diff --git a/lib/ansible/_internal/_templating/_datatag.py b/lib/ansible/_internal/_templating/_datatag.py index a7696f8ba41..44e491826c5 100644 --- a/lib/ansible/_internal/_templating/_datatag.py +++ b/lib/ansible/_internal/_templating/_datatag.py @@ -12,7 +12,6 @@ from ansible.utils.display import Display from ._access import NotifiableAccessContextBase from ._utils import TemplateContext - display = Display() @@ -57,10 +56,10 @@ class DeprecatedAccessAuditContext(NotifiableAccessContextBase): display._deprecated_with_plugin_info( msg=msg, help_text=item.deprecated.help_text, - version=item.deprecated.removal_version, - date=item.deprecated.removal_date, + version=item.deprecated.version, + date=item.deprecated.date, obj=item.template, - plugin=item.deprecated.plugin, + deprecator=item.deprecated.deprecator, ) return result diff --git a/lib/ansible/_internal/_templating/_engine.py b/lib/ansible/_internal/_templating/_engine.py index b15c64e791c..a5690219a4a 100644 --- a/lib/ansible/_internal/_templating/_engine.py +++ b/lib/ansible/_internal/_templating/_engine.py @@ -566,7 +566,12 @@ class TemplateEngine: ) if _TemplateConfig.allow_broken_conditionals: - _display.deprecated(msg=msg, obj=conditional, help_text=self._BROKEN_CONDITIONAL_ALLOWED_FRAGMENT, version='2.23') + _display.deprecated( + msg=msg, + obj=conditional, + help_text=self._BROKEN_CONDITIONAL_ALLOWED_FRAGMENT, + version='2.23', + ) return bool_result diff --git a/lib/ansible/_internal/_templating/_jinja_plugins.py b/lib/ansible/_internal/_templating/_jinja_plugins.py index 1b623184560..a988a0e1d11 100644 --- a/lib/ansible/_internal/_templating/_jinja_plugins.py +++ b/lib/ansible/_internal/_templating/_jinja_plugins.py @@ -9,7 +9,6 @@ import functools import typing as t from ansible.module_utils._internal._ambient_context import AmbientContextBase -from ansible.module_utils._internal._plugin_exec_context import PluginExecContext from ansible.module_utils.common.collections import is_sequence from ansible.module_utils._internal._datatag import AnsibleTagHelper from ansible._internal._datatag._tags import TrustedAsTemplate @@ -111,7 +110,7 @@ class JinjaPluginIntercept(c.MutableMapping): return first_marker try: - with JinjaCallContext(accept_lazy_markers=instance.accept_lazy_markers), PluginExecContext(executing_plugin=instance): + with JinjaCallContext(accept_lazy_markers=instance.accept_lazy_markers): return instance.j2_function(*lazify_container_args(args), **lazify_container_kwargs(kwargs)) except MarkerError as ex: return ex.source @@ -212,10 +211,7 @@ def _invoke_lookup(*, plugin_name: str, lookup_terms: list, lookup_kwargs: dict[ wantlist = lookup_kwargs.pop('wantlist', False) errors = lookup_kwargs.pop('errors', 'strict') - with ( - JinjaCallContext(accept_lazy_markers=instance.accept_lazy_markers), - PluginExecContext(executing_plugin=instance), - ): + with JinjaCallContext(accept_lazy_markers=instance.accept_lazy_markers): try: if _TemplateConfig.allow_embedded_templates: # for backwards compat, only trust constant templates in lookup terms diff --git a/lib/ansible/cli/__init__.py b/lib/ansible/cli/__init__.py index 723106d0315..abe8c7cb058 100644 --- a/lib/ansible/cli/__init__.py +++ b/lib/ansible/cli/__init__.py @@ -10,7 +10,6 @@ import os import signal import sys - # We overload the ``ansible`` adhoc command to provide the functionality for # ``SSH_ASKPASS``. This code is here, and not in ``adhoc.py`` to bypass # unnecessary code. The program provided to ``SSH_ASKPASS`` can only be invoked @@ -106,6 +105,7 @@ except Exception as ex: from ansible import context +from ansible.utils import display as _display from ansible.cli.arguments import option_helpers as opt_help from ansible.inventory.manager import InventoryManager from ansible.module_utils.six import string_types @@ -122,6 +122,7 @@ from ansible.utils.collection_loader import AnsibleCollectionConfig from ansible.utils.collection_loader._collection_finder import _get_collection_name_from_path from ansible.utils.path import unfrackpath from ansible.vars.manager import VariableManager +from ansible.module_utils._internal import _deprecator try: import argcomplete @@ -257,7 +258,7 @@ class CLI(ABC): else: display.v(u"No config file found; using defaults") - C.handle_config_noise(display) + _display._report_config_warnings(_deprecator.ANSIBLE_CORE_DEPRECATOR) @staticmethod def split_vault_id(vault_id): diff --git a/lib/ansible/cli/arguments/option_helpers.py b/lib/ansible/cli/arguments/option_helpers.py index f43d62adb75..131ec8caeb2 100644 --- a/lib/ansible/cli/arguments/option_helpers.py +++ b/lib/ansible/cli/arguments/option_helpers.py @@ -56,7 +56,10 @@ class DeprecatedArgument: from ansible.utils.display import Display - Display().deprecated(f'The {option!r} argument is deprecated.', version=self.version) + Display().deprecated( # pylint: disable=ansible-invalid-deprecated-version + msg=f'The {option!r} argument is deprecated.', + version=self.version, + ) class ArgumentParser(argparse.ArgumentParser): diff --git a/lib/ansible/cli/doc.py b/lib/ansible/cli/doc.py index be982af0e38..550ac9de9a9 100755 --- a/lib/ansible/cli/doc.py +++ b/lib/ansible/cli/doc.py @@ -1335,7 +1335,6 @@ class DocCLI(CLI, RoleMixin): 'This was unintentionally allowed when plugin attributes were added, ' 'but the feature does not map well to role argument specs.', version='2.20', - collection_name='ansible.builtin', ) text.append("") text.append(_format("ATTRIBUTES:", 'bold')) diff --git a/lib/ansible/constants.py b/lib/ansible/constants.py index baa6bf6f8d6..7648feebbfe 100644 --- a/lib/ansible/constants.py +++ b/lib/ansible/constants.py @@ -10,9 +10,7 @@ from string import ascii_letters, digits from ansible.config.manager import ConfigManager from ansible.module_utils.common.text.converters import to_text -from ansible.module_utils.common.collections import Sequence from ansible.module_utils.parsing.convert_bool import BOOLEANS_TRUE -from ansible.release import __version__ from ansible.utils.fqcn import add_internal_fqcns # initialize config manager/config data to read/store global settings @@ -20,68 +18,11 @@ from ansible.utils.fqcn import add_internal_fqcns config = ConfigManager() -def _warning(msg): - """ display is not guaranteed here, nor it being the full class, but try anyways, fallback to sys.stderr.write """ - try: - from ansible.utils.display import Display - Display().warning(msg) - except Exception: - import sys - sys.stderr.write(' [WARNING] %s\n' % (msg)) - - -def _deprecated(msg, version): - """ display is not guaranteed here, nor it being the full class, but try anyways, fallback to sys.stderr.write """ - try: - from ansible.utils.display import Display - Display().deprecated(msg, version=version) - except Exception: - import sys - sys.stderr.write(' [DEPRECATED] %s, to be removed in %s\n' % (msg, version)) - - -def handle_config_noise(display=None): - - if display is not None: - w = display.warning - d = display.deprecated - else: - w = _warning - d = _deprecated - - while config.WARNINGS: - warn = config.WARNINGS.pop() - w(warn) - - while config.DEPRECATED: - # tuple with name and options - dep = config.DEPRECATED.pop(0) - msg = config.get_deprecated_msg_from_config(dep[1]) - # use tabs only for ansible-doc? - msg = msg.replace("\t", "") - d(f"{dep[0]} option. {msg}", version=dep[1]['version']) - - def set_constant(name, value, export=vars()): """ sets constants and returns resolved options dict """ export[name] = value -class _DeprecatedSequenceConstant(Sequence): - def __init__(self, value, msg, version): - self._value = value - self._msg = msg - self._version = version - - def __len__(self): - _deprecated(self._msg, self._version) - return len(self._value) - - def __getitem__(self, y): - _deprecated(self._msg, self._version) - return self._value[y] - - # CONSTANTS ### yes, actual ones # The following are hard-coded action names @@ -245,6 +186,3 @@ MAGIC_VARIABLE_MAPPING = dict( # POPULATE SETTINGS FROM CONFIG ### for setting in config.get_configuration_definitions(): set_constant(setting, config.get_config_value(setting, variables=vars())) - -# emit any warnings or deprecations -handle_config_noise() diff --git a/lib/ansible/errors/__init__.py b/lib/ansible/errors/__init__.py index d3536459cfb..a345f5d04af 100644 --- a/lib/ansible/errors/__init__.py +++ b/lib/ansible/errors/__init__.py @@ -18,6 +18,9 @@ from ..module_utils.datatag import native_type_name from ansible._internal._datatag import _tags from .._internal._errors import _utils +if t.TYPE_CHECKING: + from ansible.plugins import loader as _t_loader + class ExitCode(enum.IntEnum): SUCCESS = 0 # used by TQM, must be bit-flag safe @@ -374,8 +377,9 @@ class _AnsibleActionDone(AnsibleAction): class AnsiblePluginError(AnsibleError): """Base class for Ansible plugin-related errors that do not need AnsibleError contextual data.""" - def __init__(self, message=None, plugin_load_context=None): - super(AnsiblePluginError, self).__init__(message) + def __init__(self, message: str | None = None, plugin_load_context: _t_loader.PluginLoadContext | None = None, help_text: str | None = None) -> None: + super(AnsiblePluginError, self).__init__(message, help_text=help_text) + self.plugin_load_context = plugin_load_context diff --git a/lib/ansible/executor/module_common.py b/lib/ansible/executor/module_common.py index d98c70ee598..3ccc24af1a4 100644 --- a/lib/ansible/executor/module_common.py +++ b/lib/ansible/executor/module_common.py @@ -39,7 +39,6 @@ from io import BytesIO from ansible._internal import _locking from ansible._internal._datatag import _utils from ansible.module_utils._internal import _dataclass_validation -from ansible.module_utils.common.messages import PluginInfo from ansible.module_utils.common.yaml import yaml_load from ansible._internal._datatag._tags import Origin from ansible.module_utils.common.json import Direction, get_module_encoder @@ -56,6 +55,7 @@ from ansible.template import Templar from ansible.utils.collection_loader._collection_finder import _get_collection_metadata, _nested_dict_get from ansible.module_utils._internal import _json, _ansiballz from ansible.module_utils import basic as _basic +from ansible.module_utils.common import messages as _messages if t.TYPE_CHECKING: from ansible import template as _template @@ -434,7 +434,13 @@ class ModuleUtilLocatorBase: else: msg += '.' - display.deprecated(msg, removal_version, removed, removal_date, self._collection_name) + display.deprecated( # pylint: disable=ansible-deprecated-date-not-permitted,ansible-deprecated-unnecessary-collection-name + msg=msg, + version=removal_version, + removed=removed, + date=removal_date, + deprecator=_messages.PluginInfo._from_collection_name(self._collection_name), + ) if 'redirect' in routing_entry: self.redirected = True source_pkg = '.'.join(name_parts) @@ -944,7 +950,6 @@ class _CachedModule: def _find_module_utils( *, module_name: str, - plugin: PluginInfo, b_module_data: bytes, module_path: str, module_args: dict[object, object], @@ -1020,7 +1025,9 @@ def _find_module_utils( # People should start writing collections instead of modules in roles so we # may never fix this display.debug('ANSIBALLZ: Could not determine module FQN') - remote_module_fqn = 'ansible.modules.%s' % module_name + # FIXME: add integration test to validate that builtins and legacy modules with the same name are tracked separately by the caching mechanism + # FIXME: surrogate FQN should be unique per source path- role-packaged modules with name collisions can still be aliased + remote_module_fqn = 'ansible.legacy.%s' % module_name if module_substyle == 'python': date_time = datetime.datetime.now(datetime.timezone.utc) @@ -1126,7 +1133,6 @@ def _find_module_utils( module_fqn=remote_module_fqn, params=encoded_params, profile=module_metadata.serialization_profile, - plugin_info_dict=dataclasses.asdict(plugin), date_time=date_time, coverage_config=coverage_config, coverage_output=coverage_output, @@ -1236,7 +1242,6 @@ def _extract_interpreter(b_module_data): def modify_module( *, module_name: str, - plugin: PluginInfo, module_path, module_args, templar, @@ -1277,7 +1282,6 @@ def modify_module( module_bits = _find_module_utils( module_name=module_name, - plugin=plugin, b_module_data=b_module_data, module_path=module_path, module_args=module_args, diff --git a/lib/ansible/executor/task_executor.py b/lib/ansible/executor/task_executor.py index 5e80c3d7296..8c99f199fab 100644 --- a/lib/ansible/executor/task_executor.py +++ b/lib/ansible/executor/task_executor.py @@ -22,8 +22,7 @@ from ansible.errors import ( ) from ansible.executor.task_result import _RawTaskResult from ansible._internal._datatag import _utils -from ansible.module_utils._internal._plugin_exec_context import PluginExecContext -from ansible.module_utils.common.messages import Detail, WarningSummary, DeprecationSummary +from ansible.module_utils.common.messages import Detail, WarningSummary, DeprecationSummary, PluginInfo from ansible.module_utils.datatag import native_type_name from ansible._internal._datatag._tags import TrustedAsTemplate from ansible.module_utils.parsing.convert_bool import boolean @@ -640,8 +639,8 @@ class TaskExecutor: if self._task.timeout: old_sig = signal.signal(signal.SIGALRM, task_timeout) signal.alarm(self._task.timeout) - with PluginExecContext(self._handler): - result = self._handler.run(task_vars=vars_copy) + + result = self._handler.run(task_vars=vars_copy) # DTFIX-RELEASE: nuke this, it hides a lot of error detail- remove the active exception propagation hack from AnsibleActionFail at the same time except (AnsibleActionFail, AnsibleActionSkip) as e: @@ -844,13 +843,12 @@ class TaskExecutor: if not isinstance(deprecation, DeprecationSummary): # translate non-DeprecationMessageDetail message dicts try: - if deprecation.pop('collection_name', ...) is not ...: + if (collection_name := deprecation.pop('collection_name', ...)) is not ...: # deprecated: description='enable the deprecation message for collection_name' core_version='2.23' + # CAUTION: This deprecation cannot be enabled until the replacement (deprecator) has been documented, and the schema finalized. # self.deprecated('The `collection_name` key in the `deprecations` dictionary is deprecated.', version='2.27') - pass + deprecation.update(deprecator=PluginInfo._from_collection_name(collection_name)) - # DTFIX-RELEASE: when plugin isn't set, do it at the boundary where we receive the module/action results - # that may even allow us to never set it in modules/actions directly and to populate it at the boundary deprecation = DeprecationSummary( details=( Detail(msg=deprecation.pop('msg')), diff --git a/lib/ansible/galaxy/api.py b/lib/ansible/galaxy/api.py index eb3ddb51663..021d56f0254 100644 --- a/lib/ansible/galaxy/api.py +++ b/lib/ansible/galaxy/api.py @@ -138,7 +138,7 @@ def g_connect(versions): 'The v2 Ansible Galaxy API is deprecated and no longer supported. ' 'Ensure that you have configured the ansible-galaxy CLI to utilize an ' 'updated and supported version of Ansible Galaxy.', - version='2.20' + version='2.20', ) return method(self, *args, **kwargs) diff --git a/lib/ansible/galaxy/collection/__init__.py b/lib/ansible/galaxy/collection/__init__.py index 829f7aa19d2..844fa6397b2 100644 --- a/lib/ansible/galaxy/collection/__init__.py +++ b/lib/ansible/galaxy/collection/__init__.py @@ -201,9 +201,9 @@ class CollectionSignatureError(Exception): # FUTURE: expose actual verify result details for a collection on this object, maybe reimplement as dataclass on py3.8+ class CollectionVerifyResult: - def __init__(self, collection_name): # type: (str) -> None - self.collection_name = collection_name # type: str - self.success = True # type: bool + def __init__(self, collection_name: str) -> None: + self.collection_name = collection_name + self.success = True def verify_local_collection(local_collection, remote_collection, artifacts_manager): diff --git a/lib/ansible/module_utils/_internal/_ansiballz.py b/lib/ansible/module_utils/_internal/_ansiballz.py index d728663409e..65d0781a62f 100644 --- a/lib/ansible/module_utils/_internal/_ansiballz.py +++ b/lib/ansible/module_utils/_internal/_ansiballz.py @@ -6,7 +6,6 @@ from __future__ import annotations import atexit -import dataclasses import importlib.util import json import os @@ -15,17 +14,14 @@ import sys import typing as t from . import _errors -from ._plugin_exec_context import PluginExecContext, HasPluginInfo from .. import basic from ..common.json import get_module_encoder, Direction -from ..common.messages import PluginInfo def run_module( *, json_params: bytes, profile: str, - plugin_info_dict: dict[str, object], module_fqn: str, modlib_path: str, init_globals: dict[str, t.Any] | None = None, @@ -38,7 +34,6 @@ def run_module( _run_module( json_params=json_params, profile=profile, - plugin_info_dict=plugin_info_dict, module_fqn=module_fqn, modlib_path=modlib_path, init_globals=init_globals, @@ -80,7 +75,6 @@ def _run_module( *, json_params: bytes, profile: str, - plugin_info_dict: dict[str, object], module_fqn: str, modlib_path: str, init_globals: dict[str, t.Any] | None = None, @@ -92,12 +86,11 @@ def _run_module( init_globals = init_globals or {} init_globals.update(_module_fqn=module_fqn, _modlib_path=modlib_path) - with PluginExecContext(_ModulePluginWrapper(PluginInfo._from_dict(plugin_info_dict))): - # Run the module. By importing it as '__main__', it executes as a script. - runpy.run_module(mod_name=module_fqn, init_globals=init_globals, run_name='__main__', alter_sys=True) + # Run the module. By importing it as '__main__', it executes as a script. + runpy.run_module(mod_name=module_fqn, init_globals=init_globals, run_name='__main__', alter_sys=True) - # An Ansible module must print its own results and exit. If execution reaches this point, that did not happen. - raise RuntimeError('New-style module did not handle its own exit.') + # An Ansible module must print its own results and exit. If execution reaches this point, that did not happen. + raise RuntimeError('New-style module did not handle its own exit.') def _handle_exception(exception: BaseException, profile: str) -> t.NoReturn: @@ -112,22 +105,3 @@ def _handle_exception(exception: BaseException, profile: str) -> t.NoReturn: print(json.dumps(result, cls=encoder)) # pylint: disable=ansible-bad-function sys.exit(1) # pylint: disable=ansible-bad-function - - -@dataclasses.dataclass(frozen=True) -class _ModulePluginWrapper(HasPluginInfo): - """Modules aren't plugin instances; this adapter implements the `HasPluginInfo` protocol to allow `PluginExecContext` infra to work with modules.""" - - plugin: PluginInfo - - @property - def _load_name(self) -> str: - return self.plugin.requested_name - - @property - def ansible_name(self) -> str: - return self.plugin.resolved_name - - @property - def plugin_type(self) -> str: - return self.plugin.type diff --git a/lib/ansible/module_utils/_internal/_dataclass_annotation_patch.py b/lib/ansible/module_utils/_internal/_dataclass_annotation_patch.py deleted file mode 100644 index 1d1f913908c..00000000000 --- a/lib/ansible/module_utils/_internal/_dataclass_annotation_patch.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Patch broken ClassVar support in dataclasses when ClassVar is accessed via a module other than `typing`.""" - -# deprecated: description='verify ClassVar support in dataclasses has been fixed in Python before removing this patching code', python_version='3.12' - -from __future__ import annotations - -import dataclasses -import sys -import typing as t - -# trigger the bug by exposing typing.ClassVar via a module reference that is not `typing` -_ts = sys.modules[__name__] -ClassVar = t.ClassVar - - -def patch_dataclasses_is_type() -> None: - if not _is_patch_needed(): - return # pragma: nocover - - try: - real_is_type = dataclasses._is_type # type: ignore[attr-defined] - except AttributeError: # pragma: nocover - raise RuntimeError("unable to patch broken dataclasses ClassVar support") from None - - # patch dataclasses._is_type - impl from https://github.com/python/cpython/blob/4c6d4f5cb33e48519922d635894eef356faddba2/Lib/dataclasses.py#L709-L765 - def _is_type(annotation, cls, a_module, a_type, is_type_predicate): - match = dataclasses._MODULE_IDENTIFIER_RE.match(annotation) # type: ignore[attr-defined] - if match: - ns = None - module_name = match.group(1) - if not module_name: - # No module name, assume the class's module did - # "from dataclasses import InitVar". - ns = sys.modules.get(cls.__module__).__dict__ - else: - # Look up module_name in the class's module. - module = sys.modules.get(cls.__module__) - if module and module.__dict__.get(module_name): # this is the patched line; removed `is a_module` - ns = sys.modules.get(a_type.__module__).__dict__ - if ns and is_type_predicate(ns.get(match.group(2)), a_module): - return True - return False - - _is_type._orig_impl = real_is_type # type: ignore[attr-defined] # stash this away to allow unit tests to undo the patch - - dataclasses._is_type = _is_type # type: ignore[attr-defined] - - try: - if _is_patch_needed(): - raise RuntimeError("patching had no effect") # pragma: nocover - except Exception as ex: # pragma: nocover - dataclasses._is_type = real_is_type # type: ignore[attr-defined] - raise RuntimeError("dataclasses ClassVar support is still broken after patching") from ex - - -def _is_patch_needed() -> bool: - @dataclasses.dataclass - class CheckClassVar: - # this is the broken case requiring patching: ClassVar dot-referenced from a module that is not `typing` is treated as an instance field - # DTFIX-RELEASE: add link to CPython bug report to-be-filed (or update associated deprecation comments if we don't) - a_classvar: _ts.ClassVar[int] # type: ignore[name-defined] - a_field: int - - return len(dataclasses.fields(CheckClassVar)) != 1 diff --git a/lib/ansible/module_utils/_internal/_datatag/_tags.py b/lib/ansible/module_utils/_internal/_datatag/_tags.py index b50e08ee9c3..9b271e142ee 100644 --- a/lib/ansible/module_utils/_internal/_datatag/_tags.py +++ b/lib/ansible/module_utils/_internal/_datatag/_tags.py @@ -1,7 +1,6 @@ from __future__ import annotations import dataclasses -import datetime import typing as t from ansible.module_utils.common import messages as _messages @@ -12,27 +11,6 @@ from ansible.module_utils._internal import _datatag class Deprecated(_datatag.AnsibleDatatagBase): msg: str help_text: t.Optional[str] = None - removal_date: t.Optional[datetime.date] = None - removal_version: t.Optional[str] = None - plugin: t.Optional[_messages.PluginInfo] = None - - @classmethod - def _from_dict(cls, d: t.Dict[str, t.Any]) -> Deprecated: - source = d - removal_date = source.get('removal_date') - - if removal_date is not None: - source = source.copy() - source['removal_date'] = datetime.date.fromisoformat(removal_date) - - return cls(**source) - - def _as_dict(self) -> t.Dict[str, t.Any]: - # deprecated: description='no-args super() with slotted dataclass requires 3.14+' python_version='3.13' - # see: https://github.com/python/cpython/pull/124455 - value = super(Deprecated, self)._as_dict() - - if self.removal_date is not None: - value['removal_date'] = self.removal_date.isoformat() - - return value + date: t.Optional[str] = None + version: t.Optional[str] = None + deprecator: t.Optional[_messages.PluginInfo] = None diff --git a/lib/ansible/module_utils/_internal/_deprecator.py b/lib/ansible/module_utils/_internal/_deprecator.py new file mode 100644 index 00000000000..64ab40da0e3 --- /dev/null +++ b/lib/ansible/module_utils/_internal/_deprecator.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import inspect +import re +import pathlib +import sys +import typing as t + +from ansible.module_utils.common.messages import PluginInfo + +_ansible_module_base_path: t.Final = pathlib.Path(sys.modules['ansible'].__file__).parent +"""Runtime-detected base path of the `ansible` Python package to distinguish between Ansible-owned and external code.""" + +ANSIBLE_CORE_DEPRECATOR: t.Final = PluginInfo._from_collection_name('ansible.builtin') +"""Singleton `PluginInfo` instance for ansible-core callers where the plugin can/should not be identified in messages.""" + +INDETERMINATE_DEPRECATOR: t.Final = PluginInfo(resolved_name='indeterminate', type='indeterminate') +"""Singleton `PluginInfo` instance for indeterminate deprecator.""" + +_DEPRECATOR_PLUGIN_TYPES = frozenset( + { + 'action', + 'become', + 'cache', + 'callback', + 'cliconf', + 'connection', + # doc_fragments - no code execution + # filter - basename inadequate to identify plugin + 'httpapi', + 'inventory', + 'lookup', + 'module', # only for collections + 'netconf', + 'shell', + 'strategy', + 'terminal', + # test - basename inadequate to identify plugin + 'vars', + } +) +"""Plugin types which are valid for identifying a deprecator for deprecation purposes.""" + +_AMBIGUOUS_DEPRECATOR_PLUGIN_TYPES = frozenset( + { + 'filter', + 'test', + } +) +"""Plugin types for which basename cannot be used to identify the plugin name.""" + + +def get_best_deprecator(*, deprecator: PluginInfo | None = None, collection_name: str | None = None) -> PluginInfo: + """Return the best-available `PluginInfo` for the caller of this method.""" + _skip_stackwalk = True + + if deprecator and collection_name: + raise ValueError('Specify only one of `deprecator` or `collection_name`.') + + return deprecator or PluginInfo._from_collection_name(collection_name) or get_caller_plugin_info() or INDETERMINATE_DEPRECATOR + + +def get_caller_plugin_info() -> PluginInfo | None: + """Try to get `PluginInfo` for the caller of this method, ignoring marked infrastructure stack frames.""" + _skip_stackwalk = True + + if frame_info := next((frame_info for frame_info in inspect.stack() if '_skip_stackwalk' not in frame_info.frame.f_locals), None): + return _path_as_core_plugininfo(frame_info.filename) or _path_as_collection_plugininfo(frame_info.filename) + + return None # pragma: nocover + + +def _path_as_core_plugininfo(path: str) -> PluginInfo | None: + """Return a `PluginInfo` instance if the provided `path` refers to a core plugin.""" + try: + relpath = str(pathlib.Path(path).relative_to(_ansible_module_base_path)) + except ValueError: + return None # not ansible-core + + namespace = 'ansible.builtin' + + if match := re.match(r'plugins/(?P\w+)/(?P\w+)', relpath): + plugin_name = match.group("plugin_name") + plugin_type = match.group("plugin_type") + + if plugin_type not in _DEPRECATOR_PLUGIN_TYPES: + # The plugin type isn't a known deprecator type, so we have to assume the caller is intermediate code. + # We have no way of knowing if the intermediate code is deprecating its own feature, or acting on behalf of another plugin. + # Callers in this case need to identify the deprecating plugin name, otherwise only ansible-core will be reported. + # Reporting ansible-core is never wrong, it just may be missing an additional detail (plugin name) in the "on behalf of" case. + return ANSIBLE_CORE_DEPRECATOR + elif match := re.match(r'modules/(?P\w+)', relpath): + # AnsiballZ Python package for core modules + plugin_name = match.group("module_name") + plugin_type = "module" + elif match := re.match(r'legacy/(?P\w+)', relpath): + # AnsiballZ Python package for non-core library/role modules + namespace = 'ansible.legacy' + + plugin_name = match.group("module_name") + plugin_type = "module" + else: + return ANSIBLE_CORE_DEPRECATOR # non-plugin core path, safe to use ansible-core for the same reason as the non-deprecator plugin type case above + + name = f'{namespace}.{plugin_name}' + + return PluginInfo(resolved_name=name, type=plugin_type) + + +def _path_as_collection_plugininfo(path: str) -> PluginInfo | None: + """Return a `PluginInfo` instance if the provided `path` refers to a collection plugin.""" + if not (match := re.search(r'/ansible_collections/(?P\w+)/(?P\w+)/plugins/(?P\w+)/(?P\w+)', path)): + return None + + plugin_type = match.group('plugin_type') + + if plugin_type in _AMBIGUOUS_DEPRECATOR_PLUGIN_TYPES: + # We're able to detect the namespace, collection and plugin type -- but we have no way to identify the plugin name currently. + # To keep things simple we'll fall back to just identifying the namespace and collection. + # In the future we could improve the detection and/or make it easier for a caller to identify the plugin name. + return PluginInfo._from_collection_name('.'.join((match.group('ns'), match.group('coll')))) + + if plugin_type == 'modules': + plugin_type = 'module' + + if plugin_type not in _DEPRECATOR_PLUGIN_TYPES: + # The plugin type isn't a known deprecator type, so we have to assume the caller is intermediate code. + # We have no way of knowing if the intermediate code is deprecating its own feature, or acting on behalf of another plugin. + # Callers in this case need to identify the deprecator to avoid ambiguity, since it could be the same collection or another collection. + return INDETERMINATE_DEPRECATOR + + name = '.'.join((match.group('ns'), match.group('coll'), match.group('plugin_name'))) + + return PluginInfo(resolved_name=name, type=plugin_type) diff --git a/lib/ansible/module_utils/_internal/_plugin_exec_context.py b/lib/ansible/module_utils/_internal/_plugin_exec_context.py deleted file mode 100644 index 332badc29c9..00000000000 --- a/lib/ansible/module_utils/_internal/_plugin_exec_context.py +++ /dev/null @@ -1,49 +0,0 @@ -from __future__ import annotations - -import typing as t - -from ._ambient_context import AmbientContextBase -from ..common.messages import PluginInfo - - -class HasPluginInfo(t.Protocol): - """Protocol to type-annotate and expose PluginLoader-set values.""" - - @property - def _load_name(self) -> str: - """The requested name used to load the plugin.""" - - @property - def ansible_name(self) -> str: - """Fully resolved plugin name.""" - - @property - def plugin_type(self) -> str: - """Plugin type name.""" - - -class PluginExecContext(AmbientContextBase): - """Execution context that wraps all plugin invocations to allow infrastructure introspection of the currently-executing plugin instance.""" - - def __init__(self, executing_plugin: HasPluginInfo) -> None: - self._executing_plugin = executing_plugin - - @property - def executing_plugin(self) -> HasPluginInfo: - return self._executing_plugin - - @property - def plugin_info(self) -> PluginInfo: - return PluginInfo( - requested_name=self._executing_plugin._load_name, - resolved_name=self._executing_plugin.ansible_name, - type=self._executing_plugin.plugin_type, - ) - - @classmethod - def get_current_plugin_info(cls) -> PluginInfo | None: - """Utility method to extract a PluginInfo for the currently executing plugin (or None if no plugin is executing).""" - if ctx := cls.current(optional=True): - return ctx.plugin_info - - return None diff --git a/lib/ansible/module_utils/_internal/_plugin_info.py b/lib/ansible/module_utils/_internal/_plugin_info.py new file mode 100644 index 00000000000..12ae2998d6b --- /dev/null +++ b/lib/ansible/module_utils/_internal/_plugin_info.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import typing as t + +from ..common import messages as _messages + + +class HasPluginInfo(t.Protocol): + """Protocol to type-annotate and expose PluginLoader-set values.""" + + @property + def ansible_name(self) -> str | None: + """Fully resolved plugin name.""" + + @property + def plugin_type(self) -> str: + """Plugin type name.""" + + +def get_plugin_info(value: HasPluginInfo) -> _messages.PluginInfo: + """Utility method that returns a `PluginInfo` from an object implementing the `HasPluginInfo` protocol.""" + return _messages.PluginInfo( + resolved_name=value.ansible_name, + type=value.plugin_type, + ) diff --git a/lib/ansible/module_utils/_internal/_validation.py b/lib/ansible/module_utils/_internal/_validation.py new file mode 100644 index 00000000000..d6f65052fb1 --- /dev/null +++ b/lib/ansible/module_utils/_internal/_validation.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +import keyword + + +def validate_collection_name(collection_name: object, name: str = 'collection_name') -> None: + """Validate a collection name.""" + if not isinstance(collection_name, str): + raise TypeError(f"{name} must be {str} instead of {type(collection_name)}") + + parts = collection_name.split('.') + + if len(parts) != 2 or not all(part.isidentifier() and not keyword.iskeyword(part) for part in parts): + raise ValueError(f"{name} must consist of two non-keyword identifiers separated by '.'") diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index 4c406501db7..980a880bba5 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -75,7 +75,7 @@ except ImportError: # Python2 & 3 way to get NoneType NoneType = type(None) -from ._internal import _traceback, _errors, _debugging +from ._internal import _traceback, _errors, _debugging, _deprecator from .common.text.converters import ( to_native, @@ -509,16 +509,31 @@ class AnsibleModule(object): warn(warning) self.log('[WARNING] %s' % warning) - def deprecate(self, msg, version=None, date=None, collection_name=None): - if version is not None and date is not None: - raise AssertionError("implementation error -- version and date must not both be set") - deprecate(msg, version=version, date=date) - # For compatibility, we accept that neither version nor date is set, - # and treat that the same as if version would not have been set - if date is not None: - self.log('[DEPRECATION WARNING] %s %s' % (msg, date)) - else: - self.log('[DEPRECATION WARNING] %s %s' % (msg, version)) + def deprecate( + self, + msg: str, + version: str | None = None, + date: str | None = None, + collection_name: str | None = None, + *, + deprecator: _messages.PluginInfo | None = None, + help_text: str | None = None, + ) -> None: + """ + Record a deprecation warning to be returned with the module result. + Most callers do not need to provide `collection_name` or `deprecator` -- but provide only one if needed. + Specify `version` or `date`, but not both. + If `date` is a string, it must be in the form `YYYY-MM-DD`. + """ + _skip_stackwalk = True + + deprecate( # pylint: disable=ansible-deprecated-date-not-permitted,ansible-deprecated-unnecessary-collection-name + msg=msg, + version=version, + date=date, + deprecator=_deprecator.get_best_deprecator(deprecator=deprecator, collection_name=collection_name), + help_text=help_text, + ) def load_file_common_arguments(self, params, path=None): """ @@ -1404,6 +1419,7 @@ class AnsibleModule(object): self.cleanup(path) def _return_formatted(self, kwargs): + _skip_stackwalk = True self.add_path_info(kwargs) @@ -1411,6 +1427,13 @@ class AnsibleModule(object): kwargs['invocation'] = {'module_args': self.params} if 'warnings' in kwargs: + self.deprecate( # pylint: disable=ansible-deprecated-unnecessary-collection-name + msg='Passing `warnings` to `exit_json` or `fail_json` is deprecated.', + version='2.23', + help_text='Use `AnsibleModule.warn` instead.', + deprecator=_deprecator.ANSIBLE_CORE_DEPRECATOR, + ) + if isinstance(kwargs['warnings'], list): for w in kwargs['warnings']: self.warn(w) @@ -1422,17 +1445,38 @@ class AnsibleModule(object): kwargs['warnings'] = warnings if 'deprecations' in kwargs: + self.deprecate( # pylint: disable=ansible-deprecated-unnecessary-collection-name + msg='Passing `deprecations` to `exit_json` or `fail_json` is deprecated.', + version='2.23', + help_text='Use `AnsibleModule.deprecate` instead.', + deprecator=_deprecator.ANSIBLE_CORE_DEPRECATOR, + ) + if isinstance(kwargs['deprecations'], list): for d in kwargs['deprecations']: - if isinstance(d, SEQUENCETYPE) and len(d) == 2: - self.deprecate(d[0], version=d[1]) + if isinstance(d, (KeysView, Sequence)) and len(d) == 2: + self.deprecate( # pylint: disable=ansible-deprecated-unnecessary-collection-name,ansible-invalid-deprecated-version + msg=d[0], + version=d[1], + deprecator=_deprecator.get_best_deprecator(), + ) elif isinstance(d, Mapping): - self.deprecate(d['msg'], version=d.get('version'), date=d.get('date'), - collection_name=d.get('collection_name')) + self.deprecate( # pylint: disable=ansible-deprecated-date-not-permitted,ansible-deprecated-unnecessary-collection-name + msg=d['msg'], + version=d.get('version'), + date=d.get('date'), + deprecator=_deprecator.get_best_deprecator(collection_name=d.get('collection_name')), + ) else: - self.deprecate(d) # pylint: disable=ansible-deprecated-no-version + self.deprecate( # pylint: disable=ansible-deprecated-unnecessary-collection-name,ansible-deprecated-no-version + msg=d, + deprecator=_deprecator.get_best_deprecator(), + ) else: - self.deprecate(kwargs['deprecations']) # pylint: disable=ansible-deprecated-no-version + self.deprecate( # pylint: disable=ansible-deprecated-unnecessary-collection-name,ansible-deprecated-no-version + msg=kwargs['deprecations'], + deprecator=_deprecator.get_best_deprecator(), + ) deprecations = get_deprecations() if deprecations: @@ -1452,6 +1496,7 @@ class AnsibleModule(object): def exit_json(self, **kwargs) -> t.NoReturn: """ return from the module, without error """ + _skip_stackwalk = True self.do_cleanup_files() self._return_formatted(kwargs) @@ -1473,6 +1518,8 @@ class AnsibleModule(object): When `exception` is not specified, a formatted traceback will be retrieved from the current exception. If no exception is pending, the current call stack will be used instead. """ + _skip_stackwalk = True + msg = str(msg) # coerce to str instead of raising an error due to an invalid type kwargs.update( diff --git a/lib/ansible/module_utils/common/arg_spec.py b/lib/ansible/module_utils/common/arg_spec.py index 37019e7df33..f4883778ec4 100644 --- a/lib/ansible/module_utils/common/arg_spec.py +++ b/lib/ansible/module_utils/common/arg_spec.py @@ -22,6 +22,7 @@ from ansible.module_utils.common.parameters import ( from ansible.module_utils.common.text.converters import to_native from ansible.module_utils.common.warnings import deprecate, warn +from ansible.module_utils.common import messages as _messages from ansible.module_utils.common.validation import ( check_mutually_exclusive, @@ -300,9 +301,13 @@ class ModuleArgumentSpecValidator(ArgumentSpecValidator): result = super(ModuleArgumentSpecValidator, self).validate(parameters) for d in result._deprecations: - deprecate(d['msg'], - version=d.get('version'), date=d.get('date'), - collection_name=d.get('collection_name')) + # DTFIX-FUTURE: pass an actual deprecator instead of one derived from collection_name + deprecate( # pylint: disable=ansible-deprecated-date-not-permitted,ansible-deprecated-unnecessary-collection-name + msg=d['msg'], + version=d.get('version'), + date=d.get('date'), + deprecator=_messages.PluginInfo._from_collection_name(d.get('collection_name')), + ) for w in result._warnings: warn('Both option {option} and its alias {alias} are set.'.format(option=w['option'], alias=w['alias'])) diff --git a/lib/ansible/module_utils/common/messages.py b/lib/ansible/module_utils/common/messages.py index a4ec12f8494..41b87a0ac01 100644 --- a/lib/ansible/module_utils/common/messages.py +++ b/lib/ansible/module_utils/common/messages.py @@ -13,7 +13,7 @@ import dataclasses as _dataclasses # deprecated: description='typing.Self exists in Python 3.11+' python_version='3.10' from ..compat import typing as _t -from ansible.module_utils._internal import _datatag +from ansible.module_utils._internal import _datatag, _validation if _sys.version_info >= (3, 10): # Using slots for reduced memory usage and improved performance. @@ -27,13 +27,27 @@ else: class PluginInfo(_datatag.AnsibleSerializableDataclass): """Information about a loaded plugin.""" - requested_name: str - """The plugin name as requested, before resolving, which may be partially or fully qualified.""" resolved_name: str """The resolved canonical plugin name; always fully-qualified for collection plugins.""" type: str """The plugin type.""" + _COLLECTION_ONLY_TYPE: _t.ClassVar[str] = 'collection' + """This is not a real plugin type. It's a placeholder for use by a `PluginInfo` instance which references a collection without a plugin.""" + + @classmethod + def _from_collection_name(cls, collection_name: str | None) -> _t.Self | None: + """Returns an instance with the special `collection` type to refer to a non-plugin or ambiguous caller within a collection.""" + if not collection_name: + return None + + _validation.validate_collection_name(collection_name) + + return cls( + resolved_name=collection_name, + type=cls._COLLECTION_ONLY_TYPE, + ) + @_dataclasses.dataclass(**_dataclass_kwargs) class Detail(_datatag.AnsibleSerializableDataclass): @@ -75,34 +89,37 @@ class WarningSummary(SummaryBase): class DeprecationSummary(WarningSummary): """Deprecation summary with details (possibly derived from an exception __cause__ chain) and an optional traceback.""" - version: _t.Optional[str] = None - date: _t.Optional[str] = None - plugin: _t.Optional[PluginInfo] = None - - @property - def collection_name(self) -> _t.Optional[str]: - if not self.plugin: - return None - - parts = self.plugin.resolved_name.split('.') - - if len(parts) < 2: - return None - - collection_name = '.'.join(parts[:2]) + deprecator: _t.Optional[PluginInfo] = None + """ + The identifier for the content which is being deprecated. + """ - # deprecated: description='enable the deprecation message for collection_name' core_version='2.23' - # from ansible.module_utils.datatag import deprecate_value - # collection_name = deprecate_value(collection_name, 'The `collection_name` property is deprecated.', removal_version='2.27') + date: _t.Optional[str] = None + """ + The date after which a new release of `deprecator` will remove the feature described by `msg`. + Ignored if `deprecator` is not provided. + """ - return collection_name + version: _t.Optional[str] = None + """ + The version of `deprecator` which will remove the feature described by `msg`. + Ignored if `deprecator` is not provided. + Ignored if `date` is provided. + """ def _as_simple_dict(self) -> _t.Dict[str, _t.Any]: """Returns a dictionary representation of the deprecation object in the format exposed to playbooks.""" + from ansible.module_utils._internal._deprecator import INDETERMINATE_DEPRECATOR # circular import from messages + + if self.deprecator and self.deprecator != INDETERMINATE_DEPRECATOR: + collection_name = '.'.join(self.deprecator.resolved_name.split('.')[:2]) + else: + collection_name = None + result = self._as_dict() result.update( msg=self._format(), - collection_name=self.collection_name, + collection_name=collection_name, ) return result diff --git a/lib/ansible/module_utils/common/process.py b/lib/ansible/module_utils/common/process.py index eb11f8e44d1..5c546ec6c43 100644 --- a/lib/ansible/module_utils/common/process.py +++ b/lib/ansible/module_utils/common/process.py @@ -29,7 +29,6 @@ def get_bin_path(arg, opt_dirs=None, required=None): deprecate( msg="The `required` parameter in `get_bin_path` API is deprecated.", version="2.21", - collection_name="ansible.builtin", ) paths = [] diff --git a/lib/ansible/module_utils/common/respawn.py b/lib/ansible/module_utils/common/respawn.py index c0874fb2911..a294f01ec01 100644 --- a/lib/ansible/module_utils/common/respawn.py +++ b/lib/ansible/module_utils/common/respawn.py @@ -3,14 +3,12 @@ from __future__ import annotations -import dataclasses import os import pathlib import subprocess import sys import typing as t -from ansible.module_utils._internal import _plugin_exec_context from ansible.module_utils.common.text.converters import to_bytes _ANSIBLE_PARENT_PATH = pathlib.Path(__file__).parents[3] @@ -99,7 +97,6 @@ if __name__ == '__main__': json_params = {json_params!r} profile = {profile!r} - plugin_info_dict = {plugin_info_dict!r} module_fqn = {module_fqn!r} modlib_path = {modlib_path!r} @@ -110,19 +107,15 @@ if __name__ == '__main__': _ansiballz.run_module( json_params=json_params, profile=profile, - plugin_info_dict=plugin_info_dict, module_fqn=module_fqn, modlib_path=modlib_path, init_globals=dict(_respawned=True), ) """ - plugin_info = _plugin_exec_context.PluginExecContext.get_current_plugin_info() - respawn_code = respawn_code_template.format( json_params=basic._ANSIBLE_ARGS, profile=basic._ANSIBLE_PROFILE, - plugin_info_dict=dataclasses.asdict(plugin_info), module_fqn=module_fqn, modlib_path=modlib_path, ) diff --git a/lib/ansible/module_utils/common/warnings.py b/lib/ansible/module_utils/common/warnings.py index 432e3be3ad5..4d405590939 100644 --- a/lib/ansible/module_utils/common/warnings.py +++ b/lib/ansible/module_utils/common/warnings.py @@ -4,15 +4,12 @@ from __future__ import annotations as _annotations -import datetime as _datetime import typing as _t -from ansible.module_utils._internal import _traceback, _plugin_exec_context +from ansible.module_utils._internal import _traceback, _deprecator from ansible.module_utils.common import messages as _messages from ansible.module_utils import _internal -_UNSET = _t.cast(_t.Any, object()) - def warn(warning: str) -> None: """Record a warning to be returned with the module result.""" @@ -28,22 +25,23 @@ def warn(warning: str) -> None: def deprecate( msg: str, version: str | None = None, - date: str | _datetime.date | None = None, - collection_name: str | None = _UNSET, + date: str | None = None, + collection_name: str | None = None, *, + deprecator: _messages.PluginInfo | None = None, help_text: str | None = None, obj: object | None = None, ) -> None: """ - Record a deprecation warning to be returned with the module result. + Record a deprecation warning. The `obj` argument is only useful in a controller context; it is ignored for target-side callers. + Most callers do not need to provide `collection_name` or `deprecator` -- but provide only one if needed. + Specify `version` or `date`, but not both. + If `date` is a string, it must be in the form `YYYY-MM-DD`. """ - if isinstance(date, _datetime.date): - date = str(date) + _skip_stackwalk = True - # deprecated: description='enable the deprecation message for collection_name' core_version='2.23' - # if collection_name is not _UNSET: - # deprecate('The `collection_name` argument to `deprecate` is deprecated.', version='2.27') + deprecator = _deprecator.get_best_deprecator(deprecator=deprecator, collection_name=collection_name) if _internal.is_controller: _display = _internal.import_controller_module('ansible.utils.display').Display() @@ -53,6 +51,8 @@ def deprecate( date=date, help_text=help_text, obj=obj, + # skip passing collection_name; get_best_deprecator already accounted for it when present + deprecator=deprecator, ) return @@ -64,7 +64,7 @@ def deprecate( formatted_traceback=_traceback.maybe_capture_traceback(_traceback.TracebackEvent.DEPRECATED), version=version, date=date, - plugin=_plugin_exec_context.PluginExecContext.get_current_plugin_info(), + deprecator=deprecator, )] = None diff --git a/lib/ansible/module_utils/datatag.py b/lib/ansible/module_utils/datatag.py index 0e182e3d042..e32b0ec7538 100644 --- a/lib/ansible/module_utils/datatag.py +++ b/lib/ansible/module_utils/datatag.py @@ -1,11 +1,11 @@ """Public API for data tagging.""" from __future__ import annotations as _annotations -import datetime as _datetime import typing as _t -from ._internal import _plugin_exec_context, _datatag +from ._internal import _datatag, _deprecator from ._internal._datatag import _tags +from .common import messages as _messages _T = _t.TypeVar('_T') @@ -14,28 +14,28 @@ def deprecate_value( value: _T, msg: str, *, + version: str | None = None, + date: str | None = None, + collection_name: str | None = None, + deprecator: _messages.PluginInfo | None = None, help_text: str | None = None, - removal_date: str | _datetime.date | None = None, - removal_version: str | None = None, ) -> _T: """ Return `value` tagged with the given deprecation details. The types `None` and `bool` cannot be deprecated and are returned unmodified. Raises a `TypeError` if `value` is not a supported type. - If `removal_date` is a string, it must be in the form `YYYY-MM-DD`. - This function is only supported in contexts where an Ansible plugin/module is executing. + Most callers do not need to provide `collection_name` or `deprecator` -- but provide only one if needed. + Specify `version` or `date`, but not both. + If `date` is provided, it should be in the form `YYYY-MM-DD`. """ - if isinstance(removal_date, str): - # The `fromisoformat` method accepts other ISO 8601 formats than `YYYY-MM-DD` starting with Python 3.11. - # That should be considered undocumented behavior of `deprecate_value` rather than an intentional feature. - removal_date = _datetime.date.fromisoformat(removal_date) + _skip_stackwalk = True deprecated = _tags.Deprecated( msg=msg, help_text=help_text, - removal_date=removal_date, - removal_version=removal_version, - plugin=_plugin_exec_context.PluginExecContext.get_current_plugin_info(), + date=date, + version=version, + deprecator=_deprecator.get_best_deprecator(deprecator=deprecator, collection_name=collection_name), ) return deprecated.tag(value) diff --git a/lib/ansible/playbook/task.py b/lib/ansible/playbook/task.py index 6579922624e..304e9d3c752 100644 --- a/lib/ansible/playbook/task.py +++ b/lib/ansible/playbook/task.py @@ -227,8 +227,6 @@ class Task(Base, Conditional, Taggable, CollectionSearch, Notifiable, Delegatabl raise AnsibleError("you must specify a value when using %s" % k, obj=ds) new_ds['loop_with'] = loop_name new_ds['loop'] = v - # display.deprecated("with_ type loops are being phased out, use the 'loop' keyword instead", - # version="2.10", collection_name='ansible.builtin') def preprocess_data(self, ds): """ diff --git a/lib/ansible/plugins/__init__.py b/lib/ansible/plugins/__init__.py index 5e597da2f9e..8b6bb1630ff 100644 --- a/lib/ansible/plugins/__init__.py +++ b/lib/ansible/plugins/__init__.py @@ -20,24 +20,26 @@ from __future__ import annotations import abc +import functools import types import typing as t from ansible import constants as C from ansible.errors import AnsibleError from ansible.utils.display import Display +from ansible.utils import display as _display -from ansible.module_utils._internal import _plugin_exec_context +from ansible.module_utils._internal import _plugin_info display = Display() if t.TYPE_CHECKING: - from .loader import PluginPathContext + from . import loader as _t_loader # Global so that all instances of a PluginLoader will share the caches MODULE_CACHE = {} # type: dict[str, dict[str, types.ModuleType]] -PATH_CACHE = {} # type: dict[str, list[PluginPathContext] | None] -PLUGIN_PATH_CACHE = {} # type: dict[str, dict[str, dict[str, PluginPathContext]]] +PATH_CACHE = {} # type: dict[str, list[_t_loader.PluginPathContext] | None] +PLUGIN_PATH_CACHE = {} # type: dict[str, dict[str, dict[str, _t_loader.PluginPathContext]]] def get_plugin_class(obj): @@ -50,10 +52,10 @@ def get_plugin_class(obj): class _ConfigurablePlugin(t.Protocol): """Protocol to provide type-safe access to config for plugin-related mixins.""" - def get_option(self, option: str, hostvars: dict[str, object] | None = None) -> object: ... + def get_option(self, option: str, hostvars: dict[str, object] | None = None) -> t.Any: ... -class _AnsiblePluginInfoMixin(_plugin_exec_context.HasPluginInfo): +class _AnsiblePluginInfoMixin(_plugin_info.HasPluginInfo): """Mixin to provide type annotations and default values for existing PluginLoader-set load-time attrs.""" _original_path: str | None = None _load_name: str | None = None @@ -102,6 +104,14 @@ class AnsiblePlugin(_AnsiblePluginInfoMixin, _ConfigurablePlugin, metaclass=abc. raise KeyError(str(e)) return option_value, origin + @functools.cached_property + def __plugin_info(self): + """ + Internal cached property to retrieve `PluginInfo` for this plugin instance. + Only for use by the `AnsiblePlugin` base class. + """ + return _plugin_info.get_plugin_info(self) + def get_option(self, option, hostvars=None): if option not in self._options: @@ -117,7 +127,7 @@ class AnsiblePlugin(_AnsiblePluginInfoMixin, _ConfigurablePlugin, metaclass=abc. def set_option(self, option, value): self._options[option] = C.config.get_config_value(option, plugin_type=self.plugin_type, plugin_name=self._load_name, direct={option: value}) - C.handle_config_noise(display) + _display._report_config_warnings(self.__plugin_info) def set_options(self, task_keys=None, var_options=None, direct=None): """ @@ -134,7 +144,7 @@ class AnsiblePlugin(_AnsiblePluginInfoMixin, _ConfigurablePlugin, metaclass=abc. if self.allow_extras and var_options and '_extras' in var_options: # these are largely unvalidated passthroughs, either plugin or underlying API will validate self._options['_extras'] = var_options['_extras'] - C.handle_config_noise(display) + _display._report_config_warnings(self.__plugin_info) def has_option(self, option): if not self._options: diff --git a/lib/ansible/plugins/action/__init__.py b/lib/ansible/plugins/action/__init__.py index a69ec36b597..63095bee382 100644 --- a/lib/ansible/plugins/action/__init__.py +++ b/lib/ansible/plugins/action/__init__.py @@ -318,13 +318,6 @@ class ActionBase(ABC, _AnsiblePluginInfoMixin): final_environment: dict[str, t.Any] = {} self._compute_environment_string(final_environment) - # `modify_module` adapts PluginInfo to allow target-side use of `PluginExecContext` since modules aren't plugins - plugin = PluginInfo( - requested_name=module_name, - resolved_name=result.resolved_fqcn, - type='module', - ) - # modify_module will exit early if interpreter discovery is required; re-run after if necessary for _dummy in (1, 2): try: @@ -338,7 +331,6 @@ class ActionBase(ABC, _AnsiblePluginInfoMixin): async_timeout=self._task.async_val, environment=final_environment, remote_is_local=bool(getattr(self._connection, '_remote_is_local', False)), - plugin=plugin, become_plugin=self._connection.become, ) diff --git a/lib/ansible/plugins/action/gather_facts.py b/lib/ansible/plugins/action/gather_facts.py index 11ef07c2380..64dc457cfb7 100644 --- a/lib/ansible/plugins/action/gather_facts.py +++ b/lib/ansible/plugins/action/gather_facts.py @@ -28,10 +28,8 @@ class ActionModule(ActionBase): # TODO: remove in favor of controller side argspec detecting valid arguments # network facts modules must support gather_subset - try: - name = self._connection.ansible_name.removeprefix('ansible.netcommon.') - except AttributeError: - name = self._connection._load_name.split('.')[-1] + name = self._connection.ansible_name.removeprefix('ansible.netcommon.') + if name not in ('network_cli', 'httpapi', 'netconf'): subset = mod_args.pop('gather_subset', None) if subset not in ('all', ['all'], None): diff --git a/lib/ansible/plugins/callback/oneline.py b/lib/ansible/plugins/callback/oneline.py index 7320e45288a..38c2cc5cfd3 100644 --- a/lib/ansible/plugins/callback/oneline.py +++ b/lib/ansible/plugins/callback/oneline.py @@ -17,6 +17,7 @@ from ansible import constants as C from ansible.plugins.callback import CallbackBase from ansible.template import Templar from ansible.executor.task_result import CallbackTaskResult +from ansible.module_utils._internal import _deprecator class CallbackModule(CallbackBase): @@ -32,7 +33,12 @@ class CallbackModule(CallbackBase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._display.deprecated('The oneline callback plugin is deprecated.', version='2.23') + + self._display.deprecated( # pylint: disable=ansible-deprecated-unnecessary-collection-name + msg='The oneline callback plugin is deprecated.', + version='2.23', + deprecator=_deprecator.ANSIBLE_CORE_DEPRECATOR, # entire plugin being removed; this improves the messaging + ) def _command_generic_msg(self, hostname, result, caption): stdout = result.get('stdout', '').replace('\n', '\\n').replace('\r', '\\r') diff --git a/lib/ansible/plugins/callback/tree.py b/lib/ansible/plugins/callback/tree.py index 3d69e8ac96f..ac6fbee7b44 100644 --- a/lib/ansible/plugins/callback/tree.py +++ b/lib/ansible/plugins/callback/tree.py @@ -34,6 +34,7 @@ from ansible.executor.task_result import CallbackTaskResult from ansible.module_utils.common.text.converters import to_bytes, to_text from ansible.plugins.callback import CallbackBase from ansible.utils.path import makedirs_safe, unfrackpath +from ansible.module_utils._internal import _deprecator class CallbackModule(CallbackBase): @@ -48,7 +49,12 @@ class CallbackModule(CallbackBase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._display.deprecated('The tree callback plugin is deprecated.', version='2.23') + + self._display.deprecated( # pylint: disable=ansible-deprecated-unnecessary-collection-name + msg='The tree callback plugin is deprecated.', + version='2.23', + deprecator=_deprecator.ANSIBLE_CORE_DEPRECATOR, # entire plugin being removed; this improves the messaging + ) def set_options(self, task_keys=None, var_options=None, direct=None): """ override to set self.tree """ diff --git a/lib/ansible/plugins/connection/local.py b/lib/ansible/plugins/connection/local.py index ac5c1d8fa45..934ef52738c 100644 --- a/lib/ansible/plugins/connection/local.py +++ b/lib/ansible/plugins/connection/local.py @@ -252,7 +252,7 @@ class Connection(ConnectionBase): def _become_success_timeout(self) -> int: """Timeout value for become success in seconds.""" if (timeout := self.get_option('become_success_timeout')) < 1: - timeout = C.config.get_configuration_definitions('connection', 'local')['become_success_timeout']['default'] + timeout = C.config.get_config_default('become_success_timeout', plugin_type='connection', plugin_name='local') return timeout diff --git a/lib/ansible/plugins/connection/paramiko_ssh.py b/lib/ansible/plugins/connection/paramiko_ssh.py index 04117d7de7d..1078a853842 100644 --- a/lib/ansible/plugins/connection/paramiko_ssh.py +++ b/lib/ansible/plugins/connection/paramiko_ssh.py @@ -248,11 +248,13 @@ from ansible.errors import ( AnsibleError, AnsibleFileNotFound, ) + +from ansible.module_utils.common.text.converters import to_bytes, to_native, to_text from ansible.module_utils.compat.paramiko import _PARAMIKO_IMPORT_ERR as PARAMIKO_IMPORT_ERR, _paramiko as paramiko from ansible.plugins.connection import ConnectionBase from ansible.utils.display import Display from ansible.utils.path import makedirs_safe -from ansible.module_utils.common.text.converters import to_bytes, to_native, to_text +from ansible.module_utils._internal import _deprecator display = Display() @@ -327,7 +329,12 @@ class Connection(ConnectionBase): _log_channel: str | None = None def __init__(self, *args, **kwargs): - display.deprecated('The paramiko connection plugin is deprecated.', version='2.21') + display.deprecated( # pylint: disable=ansible-deprecated-unnecessary-collection-name + msg='The paramiko connection plugin is deprecated.', + version='2.21', + deprecator=_deprecator.ANSIBLE_CORE_DEPRECATOR, # entire plugin being removed; this improves the messaging + ) + super().__init__(*args, **kwargs) def _cache_key(self) -> str: diff --git a/lib/ansible/plugins/filter/core.py b/lib/ansible/plugins/filter/core.py index 7dbbb0e0a57..cfe2022c806 100644 --- a/lib/ansible/plugins/filter/core.py +++ b/lib/ansible/plugins/filter/core.py @@ -115,7 +115,10 @@ def to_bool(value: object) -> bool: result = value_to_check == 1 # backwards compatibility with the old code which checked: value in ('yes', 'on', '1', 'true', 1) # NB: update the doc string to reflect reality once this fallback is removed - display.deprecated(f'The `bool` filter coerced invalid value {value!r} ({native_type_name(value)}) to {result!r}.', version='2.23') + display.deprecated( + msg=f'The `bool` filter coerced invalid value {value!r} ({native_type_name(value)}) to {result!r}.', + version='2.23', + ) return result diff --git a/lib/ansible/plugins/inventory/__init__.py b/lib/ansible/plugins/inventory/__init__.py index cdf1eb608be..bd2c0f7bdc5 100644 --- a/lib/ansible/plugins/inventory/__init__.py +++ b/lib/ansible/plugins/inventory/__init__.py @@ -28,7 +28,7 @@ from collections.abc import Mapping from ansible import template as _template from ansible.errors import AnsibleError, AnsibleParserError, AnsibleValueOmittedError from ansible.inventory.group import to_safe_group_name as original_safe -from ansible.module_utils._internal import _plugin_exec_context +from ansible.module_utils._internal import _plugin_info from ansible.parsing.utils.addresses import parse_address from ansible.parsing.dataloader import DataLoader from ansible.plugins import AnsiblePlugin, _ConfigurablePlugin @@ -314,7 +314,7 @@ class BaseFileInventoryPlugin(_BaseInventoryPlugin): super(BaseFileInventoryPlugin, self).__init__() -class Cacheable(_plugin_exec_context.HasPluginInfo, _ConfigurablePlugin): +class Cacheable(_plugin_info.HasPluginInfo, _ConfigurablePlugin): """Mixin for inventory plugins which support caching.""" _cache: CachePluginAdjudicator diff --git a/lib/ansible/plugins/loader.py b/lib/ansible/plugins/loader.py index 77ee7d63fbf..b29dda8a766 100644 --- a/lib/ansible/plugins/loader.py +++ b/lib/ansible/plugins/loader.py @@ -29,7 +29,7 @@ from ansible.module_utils.common.text.converters import to_bytes, to_text, to_na from ansible.module_utils.six import string_types from ansible.parsing.yaml.loader import AnsibleLoader from ansible._internal._yaml._loader import AnsibleInstrumentedLoader -from ansible.plugins import get_plugin_class, MODULE_CACHE, PATH_CACHE, PLUGIN_PATH_CACHE +from ansible.plugins import get_plugin_class, MODULE_CACHE, PATH_CACHE, PLUGIN_PATH_CACHE, AnsibleJinja2Plugin from ansible.utils.collection_loader import AnsibleCollectionConfig, AnsibleCollectionRef from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder, _get_collection_metadata from ansible.utils.display import Display @@ -135,29 +135,44 @@ class PluginPathContext(object): class PluginLoadContext(object): - def __init__(self): - self.original_name = None - self.redirect_list = [] - self.error_list = [] - self.import_error_list = [] - self.load_attempts = [] - self.pending_redirect = None - self.exit_reason = None - self.plugin_resolved_path = None - self.plugin_resolved_name = None - self.plugin_resolved_collection = None # empty string for resolved plugins from user-supplied paths - self.deprecated = False - self.removal_date = None - self.removal_version = None - self.deprecation_warnings = [] - self.resolved = False - self._resolved_fqcn = None - self.action_plugin = None + def __init__(self, plugin_type: str, legacy_package_name: str) -> None: + self.original_name: str | None = None + self.redirect_list: list[str] = [] + self.raw_error_list: list[Exception] = [] + """All exception instances encountered during the plugin load.""" + self.error_list: list[str] = [] + """Stringified exceptions, excluding import errors.""" + self.import_error_list: list[Exception] = [] + """All ImportError exception instances encountered during the plugin load.""" + self.load_attempts: list[str] = [] + self.pending_redirect: str | None = None + self.exit_reason: str | None = None + self.plugin_resolved_path: str | None = None + self.plugin_resolved_name: str | None = None + """For collection plugins, the resolved Python module FQ __name__; for non-collections, the short name.""" + self.plugin_resolved_collection: str | None = None # empty string for resolved plugins from user-supplied paths + """For collection plugins, the resolved collection {ns}.{col}; empty string for non-collection plugins.""" + self.deprecated: bool = False + self.removal_date: str | None = None + self.removal_version: str | None = None + self.deprecation_warnings: list[str] = [] + self.resolved: bool = False + self._resolved_fqcn: str | None = None + self.action_plugin: str | None = None + self._plugin_type: str = plugin_type + """The type of the plugin.""" + self._legacy_package_name = legacy_package_name + """The legacy sys.modules package name from the plugin loader instance; stored to prevent potentially incorrect manual computation.""" + self._python_module_name: str | None = None + """ + The fully qualified Python module name for the plugin (accessible via `sys.modules`). + For non-collection non-core plugins, this may include a non-existent synthetic package element with a hash of the file path to avoid collisions. + """ @property - def resolved_fqcn(self): + def resolved_fqcn(self) -> str | None: if not self.resolved: - return + return None if not self._resolved_fqcn: final_plugin = self.redirect_list[-1] @@ -169,7 +184,7 @@ class PluginLoadContext(object): return self._resolved_fqcn - def record_deprecation(self, name, deprecation, collection_name): + def record_deprecation(self, name: str, deprecation: dict[str, t.Any] | None, collection_name: str) -> t.Self: if not deprecation: return self @@ -183,7 +198,12 @@ class PluginLoadContext(object): removal_version = None warning_text = '{0} has been deprecated.{1}{2}'.format(name, ' ' if warning_text else '', warning_text) - display.deprecated(warning_text, date=removal_date, version=removal_version, collection_name=collection_name) + display.deprecated( # pylint: disable=ansible-deprecated-date-not-permitted,ansible-deprecated-unnecessary-collection-name + msg=warning_text, + date=removal_date, + version=removal_version, + deprecator=PluginInfo._from_collection_name(collection_name), + ) self.deprecated = True if removal_date: @@ -193,28 +213,79 @@ class PluginLoadContext(object): self.deprecation_warnings.append(warning_text) return self - def resolve(self, resolved_name, resolved_path, resolved_collection, exit_reason, action_plugin): + def resolve(self, resolved_name: str, resolved_path: str, resolved_collection: str, exit_reason: str, action_plugin: str) -> t.Self: + """Record a resolved collection plugin.""" self.pending_redirect = None self.plugin_resolved_name = resolved_name self.plugin_resolved_path = resolved_path self.plugin_resolved_collection = resolved_collection self.exit_reason = exit_reason + self._python_module_name = resolved_name self.resolved = True self.action_plugin = action_plugin + + return self + + def resolve_legacy(self, name: str, pull_cache: dict[str, PluginPathContext]) -> t.Self: + """Record a resolved legacy plugin.""" + plugin_path_context = pull_cache[name] + + self.plugin_resolved_name = name + self.plugin_resolved_path = plugin_path_context.path + self.plugin_resolved_collection = 'ansible.builtin' if plugin_path_context.internal else '' + self._resolved_fqcn = 'ansible.builtin.' + name if plugin_path_context.internal else name + self._python_module_name = self._make_legacy_python_module_name() + self.resolved = True + + return self + + def resolve_legacy_jinja_plugin(self, name: str, known_plugin: AnsibleJinja2Plugin) -> t.Self: + """Record a resolved legacy Jinja plugin.""" + internal = known_plugin.ansible_name.startswith('ansible.builtin.') + + self.plugin_resolved_name = name + self.plugin_resolved_path = known_plugin._original_path + self.plugin_resolved_collection = 'ansible.builtin' if internal else '' + self._resolved_fqcn = known_plugin.ansible_name + self._python_module_name = self._make_legacy_python_module_name() + self.resolved = True + return self - def redirect(self, redirect_name): + def redirect(self, redirect_name: str) -> t.Self: self.pending_redirect = redirect_name self.exit_reason = 'pending redirect resolution from {0} to {1}'.format(self.original_name, redirect_name) self.resolved = False + return self - def nope(self, exit_reason): + def nope(self, exit_reason: str) -> t.Self: self.pending_redirect = None self.exit_reason = exit_reason self.resolved = False + return self + def _make_legacy_python_module_name(self) -> str: + """ + Generate a fully-qualified Python module name for a legacy/builtin plugin. + + The same package namespace is shared for builtin and legacy plugins. + Explicit requests for builtins via `ansible.builtin` are handled elsewhere with an aliased collection package resolved by the collection loader. + Only unqualified and `ansible.legacy`-qualified requests land here; whichever plugin is visible at the time will end up in sys.modules. + Filter and test plugin host modules receive special name suffixes to avoid collisions unrelated to the actual plugin name. + """ + name = os.path.splitext(self.plugin_resolved_path)[0] + basename = os.path.basename(name) + + if self._plugin_type in ('filter', 'test'): + # Unlike other plugin types, filter and test plugin names are independent of the file where they are defined. + # As a result, the Python module name must be derived from the full path of the plugin. + # This prevents accidental shadowing of unrelated plugins of the same type. + basename += f'_{abs(hash(self.plugin_resolved_path))}' + + return f'{self._legacy_package_name}.{basename}' + class PluginLoader: """ @@ -224,7 +295,15 @@ class PluginLoader: paths, and the python path. The first match is used. """ - def __init__(self, class_name, package, config, subdir, aliases=None, required_base_class=None): + def __init__( + self, + class_name: str, + package: str, + config: str | list[str], + subdir: str, + aliases: dict[str, str] | None = None, + required_base_class: str | None = None, + ) -> None: aliases = {} if aliases is None else aliases self.class_name = class_name @@ -250,15 +329,15 @@ class PluginLoader: PLUGIN_PATH_CACHE[class_name] = defaultdict(dict) # hold dirs added at runtime outside of config - self._extra_dirs = [] + self._extra_dirs: list[str] = [] # caches self._module_cache = MODULE_CACHE[class_name] self._paths = PATH_CACHE[class_name] self._plugin_path_cache = PLUGIN_PATH_CACHE[class_name] - self._plugin_instance_cache = {} if self.subdir == 'vars_plugins' else None + self._plugin_instance_cache: dict[str, tuple[object, PluginLoadContext]] | None = {} if self.subdir == 'vars_plugins' else None - self._searched_paths = set() + self._searched_paths: set[str] = set() @property def type(self): @@ -488,7 +567,13 @@ class PluginLoader: entry = collection_meta.get('plugin_routing', {}).get(plugin_type, {}).get(subdir_qualified_resource, None) return entry - def _find_fq_plugin(self, fq_name, extension, plugin_load_context, ignore_deprecated=False): + def _find_fq_plugin( + self, + fq_name: str, + extension: str | None, + plugin_load_context: PluginLoadContext, + ignore_deprecated: bool = False, + ) -> PluginLoadContext: """Search builtin paths to find a plugin. No external paths are searched, meaning plugins inside roles inside collections will be ignored. """ @@ -525,17 +610,13 @@ class PluginLoader: version=removal_version, date=removal_date, removed=True, - plugin=PluginInfo( - requested_name=acr.collection, - resolved_name=acr.collection, - type='collection', - ), + deprecator=PluginInfo._from_collection_name(acr.collection), ) - plugin_load_context.removal_date = removal_date - plugin_load_context.removal_version = removal_version + plugin_load_context.date = removal_date + plugin_load_context.version = removal_version plugin_load_context.resolved = True plugin_load_context.exit_reason = removed_msg - raise AnsiblePluginRemovedError(removed_msg, plugin_load_context=plugin_load_context) + raise AnsiblePluginRemovedError(message=removed_msg, plugin_load_context=plugin_load_context) redirect = routing_metadata.get('redirect', None) @@ -623,7 +704,7 @@ class PluginLoader: collection_list: list[str] | None = None, ) -> PluginLoadContext: """ Find a plugin named name, returning contextual info about the load, recursively resolving redirection """ - plugin_load_context = PluginLoadContext() + plugin_load_context = PluginLoadContext(self.type, self.package) plugin_load_context.original_name = name while True: result = self._resolve_plugin_step(name, mod_type, ignore_deprecated, check_aliases, collection_list, plugin_load_context=plugin_load_context) @@ -636,11 +717,8 @@ class PluginLoader: else: break - # TODO: smuggle these to the controller when we're in a worker, reduce noise from normal things like missing plugin packages during collection search - if plugin_load_context.error_list: - display.warning("errors were encountered during the plugin load for {0}:\n{1}".format(name, plugin_load_context.error_list)) - - # TODO: display/return import_error_list? Only useful for forensics... + for ex in plugin_load_context.raw_error_list: + display.error_as_warning(f"Error loading plugin {name!r}.", ex) # FIXME: store structured deprecation data in PluginLoadContext and use display.deprecate # if plugin_load_context.deprecated and C.config.get_config_value('DEPRECATION_WARNINGS'): @@ -650,9 +728,15 @@ class PluginLoader: return plugin_load_context - # FIXME: name bikeshed - def _resolve_plugin_step(self, name, mod_type='', ignore_deprecated=False, - check_aliases=False, collection_list=None, plugin_load_context=PluginLoadContext()): + def _resolve_plugin_step( + self, + name: str, + mod_type: str = '', + ignore_deprecated: bool = False, + check_aliases: bool = False, + collection_list: list[str] | None = None, + plugin_load_context: PluginLoadContext | None = None, + ) -> PluginLoadContext: if not plugin_load_context: raise ValueError('A PluginLoadContext is required') @@ -707,11 +791,14 @@ class PluginLoader: except (AnsiblePluginRemovedError, AnsiblePluginCircularRedirect, AnsibleCollectionUnsupportedVersionError): # these are generally fatal, let them fly raise - except ImportError as ie: - plugin_load_context.import_error_list.append(ie) except Exception as ex: - # FIXME: keep actual errors, not just assembled messages - plugin_load_context.error_list.append(to_native(ex)) + plugin_load_context.raw_error_list.append(ex) + + # DTFIX-RELEASE: can we deprecate/remove these stringified versions? + if isinstance(ex, ImportError): + plugin_load_context.import_error_list.append(ex) + else: + plugin_load_context.error_list.append(str(ex)) if plugin_load_context.error_list: display.debug(msg='plugin lookup for {0} failed; errors: {1}'.format(name, '; '.join(plugin_load_context.error_list))) @@ -737,13 +824,7 @@ class PluginLoader: # requested mod_type pull_cache = self._plugin_path_cache[suffix] try: - path_with_context = pull_cache[name] - plugin_load_context.plugin_resolved_path = path_with_context.path - plugin_load_context.plugin_resolved_name = name - plugin_load_context.plugin_resolved_collection = 'ansible.builtin' if path_with_context.internal else '' - plugin_load_context._resolved_fqcn = ('ansible.builtin.' + name if path_with_context.internal else name) - plugin_load_context.resolved = True - return plugin_load_context + return plugin_load_context.resolve_legacy(name=name, pull_cache=pull_cache) except KeyError: # Cache miss. Now let's find the plugin pass @@ -796,13 +877,7 @@ class PluginLoader: self._searched_paths.add(path) try: - path_with_context = pull_cache[name] - plugin_load_context.plugin_resolved_path = path_with_context.path - plugin_load_context.plugin_resolved_name = name - plugin_load_context.plugin_resolved_collection = 'ansible.builtin' if path_with_context.internal else '' - plugin_load_context._resolved_fqcn = 'ansible.builtin.' + name if path_with_context.internal else name - plugin_load_context.resolved = True - return plugin_load_context + return plugin_load_context.resolve_legacy(name=name, pull_cache=pull_cache) except KeyError: # Didn't find the plugin in this directory. Load modules from the next one pass @@ -810,18 +885,18 @@ class PluginLoader: # if nothing is found, try finding alias/deprecated if not name.startswith('_'): alias_name = '_' + name - # We've already cached all the paths at this point - if alias_name in pull_cache: - path_with_context = pull_cache[alias_name] - if not ignore_deprecated and not os.path.islink(path_with_context.path): - # FIXME: this is not always the case, some are just aliases - display.deprecated('%s is kept for backwards compatibility but usage is discouraged. ' # pylint: disable=ansible-deprecated-no-version - 'The module documentation details page may explain more about this rationale.' % name.lstrip('_')) - plugin_load_context.plugin_resolved_path = path_with_context.path - plugin_load_context.plugin_resolved_name = alias_name - plugin_load_context.plugin_resolved_collection = 'ansible.builtin' if path_with_context.internal else '' - plugin_load_context._resolved_fqcn = 'ansible.builtin.' + alias_name if path_with_context.internal else alias_name - plugin_load_context.resolved = True + + try: + plugin_load_context.resolve_legacy(name=alias_name, pull_cache=pull_cache) + except KeyError: + pass + else: + display.deprecated( + msg=f'Plugin {name!r} automatically redirected to {alias_name!r}.', + help_text=f'Use {alias_name!r} instead of {name!r} to refer to the plugin.', + version='2.23', + ) + return plugin_load_context # last ditch, if it's something that can be redirected, look for a builtin redirect before giving up @@ -831,7 +906,7 @@ class PluginLoader: return plugin_load_context.nope('{0} is not eligible for last-chance resolution'.format(name)) - def has_plugin(self, name, collection_list=None): + def has_plugin(self, name: str, collection_list: list[str] | None = None) -> bool: """ Checks if a plugin named name exists """ try: @@ -842,41 +917,37 @@ class PluginLoader: # log and continue, likely an innocuous type/package loading failure in collections import display.debug('has_plugin error: {0}'.format(to_text(ex))) - __contains__ = has_plugin - - def _load_module_source(self, name, path): + return False - # avoid collisions across plugins - if name.startswith('ansible_collections.'): - full_name = name - else: - full_name = '.'.join([self.package, name]) + __contains__ = has_plugin - if full_name in sys.modules: + def _load_module_source(self, *, python_module_name: str, path: str) -> types.ModuleType: + if python_module_name in sys.modules: # Avoids double loading, See https://github.com/ansible/ansible/issues/13110 - return sys.modules[full_name] + return sys.modules[python_module_name] with warnings.catch_warnings(): # FIXME: this still has issues if the module was previously imported but not "cached", # we should bypass this entire codepath for things that are directly importable warnings.simplefilter("ignore", RuntimeWarning) - spec = importlib.util.spec_from_file_location(to_native(full_name), to_native(path)) + spec = importlib.util.spec_from_file_location(to_native(python_module_name), to_native(path)) module = importlib.util.module_from_spec(spec) # mimic import machinery; make the module-being-loaded available in sys.modules during import # and remove if there's a failure... - sys.modules[full_name] = module + sys.modules[python_module_name] = module try: spec.loader.exec_module(module) except Exception: - del sys.modules[full_name] + del sys.modules[python_module_name] raise return module def _update_object( self, + *, obj: _AnsiblePluginInfoMixin, name: str, path: str, @@ -907,9 +978,9 @@ class PluginLoader: is_core_plugin = ctx.plugin_load_context.plugin_resolved_collection == 'ansible.builtin' if self.class_name == 'StrategyModule' and not is_core_plugin: display.deprecated( # pylint: disable=ansible-deprecated-no-version - 'Use of strategy plugins not included in ansible.builtin are deprecated and do not carry ' - 'any backwards compatibility guarantees. No alternative for third party strategy plugins ' - 'is currently planned.' + msg='Use of strategy plugins not included in ansible.builtin are deprecated and do not carry ' + 'any backwards compatibility guarantees. No alternative for third party strategy plugins ' + 'is currently planned.', ) return ctx.object @@ -936,8 +1007,6 @@ class PluginLoader: return get_with_context_result(None, plugin_load_context) fq_name = plugin_load_context.resolved_fqcn - if '.' not in fq_name and plugin_load_context.plugin_resolved_collection: - fq_name = '.'.join((plugin_load_context.plugin_resolved_collection, fq_name)) resolved_type_name = plugin_load_context.plugin_resolved_name path = plugin_load_context.plugin_resolved_path if (cached_result := (self._plugin_instance_cache or {}).get(fq_name)) and cached_result[1].resolved: @@ -947,7 +1016,7 @@ class PluginLoader: redirected_names = plugin_load_context.redirect_list or [] if path not in self._module_cache: - self._module_cache[path] = self._load_module_source(resolved_type_name, path) + self._module_cache[path] = self._load_module_source(python_module_name=plugin_load_context._python_module_name, path=path) found_in_cache = False self._load_config_defs(resolved_type_name, self._module_cache[path], path) @@ -974,7 +1043,7 @@ class PluginLoader: # A plugin may need to use its _load_name in __init__ (for example, to set # or get options from config), so update the object before using the constructor instance = object.__new__(obj) - self._update_object(instance, resolved_type_name, path, redirected_names, fq_name) + self._update_object(obj=instance, name=resolved_type_name, path=path, redirected_names=redirected_names, resolved=fq_name) obj.__init__(instance, *args, **kwargs) # pylint: disable=unnecessary-dunder-call obj = instance except TypeError as e: @@ -984,12 +1053,12 @@ class PluginLoader: return get_with_context_result(None, plugin_load_context) raise - self._update_object(obj, resolved_type_name, path, redirected_names, fq_name) + self._update_object(obj=obj, name=resolved_type_name, path=path, redirected_names=redirected_names, resolved=fq_name) if self._plugin_instance_cache is not None and getattr(obj, 'is_stateless', False): self._plugin_instance_cache[fq_name] = (obj, plugin_load_context) elif self._plugin_instance_cache is not None: # The cache doubles as the load order, so record the FQCN even if the plugin hasn't set is_stateless = True - self._plugin_instance_cache[fq_name] = (None, PluginLoadContext()) + self._plugin_instance_cache[fq_name] = (None, PluginLoadContext(self.type, self.package)) return get_with_context_result(obj, plugin_load_context) def _display_plugin_load(self, class_name, name, searched_paths, path, found_in_cache=None, class_only=None): @@ -1064,10 +1133,15 @@ class PluginLoader: basename = os.path.basename(name) is_j2 = isinstance(self, Jinja2Loader) + if path in legacy_excluding_builtin: + fqcn = basename + else: + fqcn = f"ansible.builtin.{basename}" + if is_j2: ref_name = path else: - ref_name = basename + ref_name = fqcn if not is_j2 and basename in _PLUGIN_FILTERS[self.package]: # j2 plugins get processed in own class, here they would just be container files @@ -1090,26 +1164,18 @@ class PluginLoader: yield path continue - if path in legacy_excluding_builtin: - fqcn = basename - else: - fqcn = f"ansible.builtin.{basename}" - if (cached_result := (self._plugin_instance_cache or {}).get(fqcn)) and cached_result[1].resolved: # Here just in case, but we don't call all() multiple times for vars plugins, so this should not be used. yield cached_result[0] continue if path not in self._module_cache: - if self.type in ('filter', 'test'): - # filter and test plugin files can contain multiple plugins - # they must have a unique python module name to prevent them from shadowing each other - full_name = '{0}_{1}'.format(abs(hash(path)), basename) - else: - full_name = basename + path_context = PluginPathContext(path, path not in legacy_excluding_builtin) + load_context = PluginLoadContext(self.type, self.package) + load_context.resolve_legacy(basename, {basename: path_context}) try: - module = self._load_module_source(full_name, path) + module = self._load_module_source(python_module_name=load_context._python_module_name, path=path) except Exception as e: display.warning("Skipping plugin (%s), cannot load: %s" % (path, to_text(e))) continue @@ -1147,7 +1213,7 @@ class PluginLoader: except TypeError as e: display.warning("Skipping plugin (%s) as it seems to be incomplete: %s" % (path, to_text(e))) - self._update_object(obj, basename, path, resolved=fqcn) + self._update_object(obj=obj, name=basename, path=path, resolved=fqcn) if self._plugin_instance_cache is not None: needs_enabled = False @@ -1239,7 +1305,7 @@ class Jinja2Loader(PluginLoader): try: # use 'parent' loader class to find files, but cannot return this as it can contain multiple plugins per file if plugin_path not in self._module_cache: - self._module_cache[plugin_path] = self._load_module_source(full_name, plugin_path) + self._module_cache[plugin_path] = self._load_module_source(python_module_name=full_name, path=plugin_path) module = self._module_cache[plugin_path] obj = getattr(module, self.class_name) except Exception as e: @@ -1262,7 +1328,7 @@ class Jinja2Loader(PluginLoader): plugin = self._plugin_wrapper_type(func) if plugin in plugins: continue - self._update_object(plugin, full, plugin_path, resolved=fq_name) + self._update_object(obj=plugin, name=full, path=plugin_path, resolved=fq_name) plugins.append(plugin) return plugins @@ -1276,7 +1342,7 @@ class Jinja2Loader(PluginLoader): requested_name = name - context = PluginLoadContext() + context = PluginLoadContext(self.type, self.package) # avoid collection path for legacy name = name.removeprefix('ansible.legacy.') @@ -1288,11 +1354,8 @@ class Jinja2Loader(PluginLoader): if isinstance(known_plugin, _DeferredPluginLoadFailure): raise known_plugin.ex - context.resolved = True - context.plugin_resolved_name = name - context.plugin_resolved_path = known_plugin._original_path - context.plugin_resolved_collection = 'ansible.builtin' if known_plugin.ansible_name.startswith('ansible.builtin.') else '' - context._resolved_fqcn = known_plugin.ansible_name + context.resolve_legacy_jinja_plugin(name, known_plugin) + return get_with_context_result(known_plugin, context) plugin = None @@ -1328,7 +1391,12 @@ class Jinja2Loader(PluginLoader): warning_text = f'{self.type.title()} "{key}" has been deprecated.{" " if warning_text else ""}{warning_text}' - display.deprecated(warning_text, version=removal_version, date=removal_date, collection_name=acr.collection) + display.deprecated( # pylint: disable=ansible-deprecated-date-not-permitted,ansible-deprecated-unnecessary-collection-name + msg=warning_text, + version=removal_version, + date=removal_date, + deprecator=PluginInfo._from_collection_name(acr.collection), + ) # check removal tombstone_entry = routing_entry.get('tombstone') @@ -1343,11 +1411,7 @@ class Jinja2Loader(PluginLoader): version=removal_version, date=removal_date, removed=True, - plugin=PluginInfo( - requested_name=acr.collection, - resolved_name=acr.collection, - type='collection', - ), + deprecator=PluginInfo._from_collection_name(acr.collection), ) raise AnsiblePluginRemovedError(exc_msg) @@ -1400,7 +1464,7 @@ class Jinja2Loader(PluginLoader): plugin = self._plugin_wrapper_type(func) if plugin: context = plugin_impl.plugin_load_context - self._update_object(plugin, requested_name, plugin_impl.object._original_path, resolved=fq_name) + self._update_object(obj=plugin, name=requested_name, path=plugin_impl.object._original_path, resolved=fq_name) # context will have filename, which for tests/filters might not be correct context._resolved_fqcn = plugin.ansible_name # FIXME: once we start caching these results, we'll be missing functions that would have loaded later diff --git a/lib/ansible/plugins/lookup/url.py b/lib/ansible/plugins/lookup/url.py index 7c15cba3e01..d3d3a4db37d 100644 --- a/lib/ansible/plugins/lookup/url.py +++ b/lib/ansible/plugins/lookup/url.py @@ -230,8 +230,8 @@ class LookupModule(LookupBase): display.vvvv("url lookup connecting to %s" % term) if self.get_option('follow_redirects') in ('yes', 'no'): display.deprecated( - "Using 'yes' or 'no' for 'follow_redirects' parameter is deprecated.", - version='2.22' + msg="Using 'yes' or 'no' for 'follow_redirects' parameter is deprecated.", + version='2.22', ) try: response = open_url( diff --git a/lib/ansible/plugins/strategy/__init__.py b/lib/ansible/plugins/strategy/__init__.py index b117604d7d8..d8d76d23e0a 100644 --- a/lib/ansible/plugins/strategy/__init__.py +++ b/lib/ansible/plugins/strategy/__init__.py @@ -822,12 +822,12 @@ class StrategyBase: """ if handle_stats_and_callbacks: display.deprecated( - "Reporting play recap stats and running callbacks functionality for " - "``include_tasks`` in ``StrategyBase._load_included_file`` is deprecated. " - "See ``https://github.com/ansible/ansible/pull/79260`` for guidance on how to " - "move the reporting into specific strategy plugins to account for " - "``include_role`` tasks as well.", - version="2.21" + msg="Reporting play recap stats and running callbacks functionality for " + "``include_tasks`` in ``StrategyBase._load_included_file`` is deprecated. " + "See ``https://github.com/ansible/ansible/pull/79260`` for guidance on how to " + "move the reporting into specific strategy plugins to account for " + "``include_role`` tasks as well.", + version="2.21", ) display.debug("loading included file: %s" % included_file._filename) try: diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py index 2bb6fd76014..e74772295ae 100644 --- a/lib/ansible/template/__init__.py +++ b/lib/ansible/template/__init__.py @@ -388,7 +388,7 @@ def generate_ansible_template_vars(path: str, fullpath: str | None = None, dest_ value=ansible_managed, msg="The `ansible_managed` variable is deprecated.", help_text="Define and use a custom variable instead.", - removal_version='2.23', + version='2.23', ) temp_vars = dict( diff --git a/lib/ansible/utils/collection_loader/_collection_meta.py b/lib/ansible/utils/collection_loader/_collection_meta.py index 84147b12ad8..c193f3f8ba9 100644 --- a/lib/ansible/utils/collection_loader/_collection_meta.py +++ b/lib/ansible/utils/collection_loader/_collection_meta.py @@ -24,11 +24,13 @@ def _meta_yml_to_dict(yaml_string_data: bytes | str, content_id): import yaml try: - from yaml import CSafeLoader as SafeLoader + from yaml import CBaseLoader as BaseLoader except (ImportError, AttributeError): - from yaml import SafeLoader # type: ignore[assignment] + from yaml import BaseLoader # type: ignore[assignment] - routing_dict = yaml.load(yaml_string_data, Loader=SafeLoader) + # Using BaseLoader ensures that all scalars are strings. + # Doing so avoids parsing unquoted versions as floats, dates as datetime.date, etc. + routing_dict = yaml.load(yaml_string_data, Loader=BaseLoader) if not routing_dict: routing_dict = {} if not isinstance(routing_dict, Mapping): diff --git a/lib/ansible/utils/display.py b/lib/ansible/utils/display.py index 46d8c94b713..e4bd71ef623 100644 --- a/lib/ansible/utils/display.py +++ b/lib/ansible/utils/display.py @@ -18,7 +18,6 @@ from __future__ import annotations import dataclasses -import datetime try: import curses @@ -50,9 +49,10 @@ from functools import wraps from struct import unpack, pack from ansible import constants as C +from ansible.constants import config from ansible.errors import AnsibleAssertionError, AnsiblePromptInterrupt, AnsiblePromptNoninteractive, AnsibleError from ansible._internal._errors import _utils -from ansible.module_utils._internal import _ambient_context, _plugin_exec_context +from ansible.module_utils._internal import _ambient_context, _deprecator from ansible.module_utils.common.text.converters import to_bytes, to_text from ansible._internal._datatag._tags import TrustedAsTemplate from ansible.module_utils.common.messages import ErrorSummary, WarningSummary, DeprecationSummary, Detail, SummaryBase, PluginInfo @@ -76,8 +76,6 @@ _LIBC.wcswidth.argtypes = (ctypes.c_wchar_p, ctypes.c_int) # Max for c_int _MAX_INT = 2 ** (ctypes.sizeof(ctypes.c_int) * 8 - 1) - 1 -_UNSET = t.cast(t.Any, object()) - MOVE_TO_BOL = b'\r' CLEAR_TO_EOL = b'\x1b[K' @@ -555,7 +553,7 @@ class Display(metaclass=Singleton): msg: str, version: str | None = None, removed: bool = False, - date: str | datetime.date | None = None, + date: str | None = None, collection_name: str | None = None, ) -> str: """Return a deprecation message and help text for non-display purposes (e.g., exception messages).""" @@ -570,7 +568,7 @@ class Display(metaclass=Singleton): version=version, removed=removed, date=date, - plugin=_plugin_exec_context.PluginExecContext.get_current_plugin_info(), + deprecator=PluginInfo._from_collection_name(collection_name), ) if removed: @@ -582,57 +580,63 @@ class Display(metaclass=Singleton): def _get_deprecation_message_with_plugin_info( self, + *, msg: str, - version: str | None = None, + version: str | None, removed: bool = False, - date: str | datetime.date | None = None, - plugin: PluginInfo | None = None, + date: str | None, + deprecator: PluginInfo | None, ) -> str: """Internal use only. Return a deprecation message and help text for display.""" - msg = msg.strip() - - if msg and msg[-1] not in ['!', '?', '.']: - msg += '.' + # DTFIX-RELEASE: the logic for omitting date/version doesn't apply to the payload, so it shows up in vars in some cases when it should not if removed: removal_fragment = 'This feature was removed' - help_text = 'Please update your playbooks.' else: removal_fragment = 'This feature will be removed' - help_text = '' - if plugin: - from_fragment = f'from the {self._describe_plugin_info(plugin)}' + if not deprecator or deprecator.type == _deprecator.INDETERMINATE_DEPRECATOR.type: + collection = None + plugin_fragment = '' + elif deprecator.type == _deprecator.PluginInfo._COLLECTION_ONLY_TYPE: + collection = deprecator.resolved_name + plugin_fragment = '' else: - from_fragment = '' + parts = deprecator.resolved_name.split('.') + plugin_name = parts[-1] + # DTFIX-RELEASE: normalize 'modules' -> 'module' before storing it so we can eliminate the normalization here + plugin_type = "module" if deprecator.type in ("module", "modules") else f'{deprecator.type} plugin' - if date: - when = 'in a release after {0}.'.format(date) - elif version: - when = 'in version {0}.'.format(version) - else: - when = 'in a future release.' + collection = '.'.join(parts[:2]) if len(parts) > 2 else None + plugin_fragment = f'{plugin_type} {plugin_name!r}' - message_text = ' '.join(f for f in [msg, removal_fragment, from_fragment, when, help_text] if f) + if collection and plugin_fragment: + plugin_fragment += ' in' - return message_text + if collection == 'ansible.builtin': + collection_fragment = 'ansible-core' + elif collection: + collection_fragment = f'collection {collection!r}' + else: + collection_fragment = '' - @staticmethod - def _describe_plugin_info(plugin_info: PluginInfo) -> str: - """Return a brief description of the plugin info, including name(s) and type.""" - name = repr(plugin_info.resolved_name) - clarification = f' (requested as {plugin_info.requested_name!r})' if plugin_info.requested_name != plugin_info.resolved_name else '' - - if plugin_info.type in ("module", "modules"): - # DTFIX-RELEASE: pluginloader or AnsiblePlugin needs a "type desc" property that doesn't suffer from legacy "inconsistencies" like this - plugin_type = "module" - elif plugin_info.type == "collection": - # not a real plugin type, but used for tombstone errors generated by plugin loader - plugin_type = plugin_info.type + if not collection: + when_fragment = 'in the future' if not removed else '' + elif date: + when_fragment = f'in a release after {date}' + elif version: + when_fragment = f'version {version}' else: - plugin_type = f'{plugin_info.type} plugin' + when_fragment = 'in a future release' if not removed else '' - return f'{name} {plugin_type}{clarification}' + if plugin_fragment or collection_fragment: + from_fragment = 'from' + else: + from_fragment = '' + + deprecation_msg = ' '.join(f for f in [removal_fragment, from_fragment, plugin_fragment, collection_fragment, when_fragment] if f) + '.' + + return _join_sentences(msg, deprecation_msg) def _wrap_message(self, msg: str, wrap_text: bool) -> str: if wrap_text and self._wrap_stderr: @@ -661,20 +665,24 @@ class Display(metaclass=Singleton): msg: str, version: str | None = None, removed: bool = False, - date: str | datetime.date | None = None, - collection_name: str | None = _UNSET, + date: str | None = None, + collection_name: str | None = None, *, + deprecator: PluginInfo | None = None, help_text: str | None = None, obj: t.Any = None, ) -> None: - """Display a deprecation warning message, if enabled.""" - # deprecated: description='enable the deprecation message for collection_name' core_version='2.23' - # if collection_name is not _UNSET: - # self.deprecated('The `collection_name` argument to `deprecated` is deprecated.', version='2.27') - + """ + Display a deprecation warning message, if enabled. + Most callers do not need to provide `collection_name` or `deprecator` -- but provide only one if needed. + Specify `version` or `date`, but not both. + If `date` is a string, it must be in the form `YYYY-MM-DD`. + """ # DTFIX-RELEASE: are there any deprecation calls where the feature is switching from enabled to disabled, rather than being removed entirely? # DTFIX-RELEASE: are there deprecated features which should going through deferred deprecation instead? + _skip_stackwalk = True + self._deprecated_with_plugin_info( msg=msg, version=version, @@ -682,32 +690,36 @@ class Display(metaclass=Singleton): date=date, help_text=help_text, obj=obj, - plugin=_plugin_exec_context.PluginExecContext.get_current_plugin_info(), + deprecator=_deprecator.get_best_deprecator(deprecator=deprecator, collection_name=collection_name), ) def _deprecated_with_plugin_info( self, + *, msg: str, - version: str | None = None, + version: str | None, removed: bool = False, - date: str | datetime.date | None = None, - *, - help_text: str | None = None, - obj: t.Any = None, - plugin: PluginInfo | None = None, + date: str | None, + help_text: str | None, + obj: t.Any, + deprecator: PluginInfo | None, ) -> None: """ This is the internal pre-proxy half of the `deprecated` implementation. Any logic that must occur on workers needs to be implemented here. """ + _skip_stackwalk = True + if removed: - raise AnsibleError(self._get_deprecation_message_with_plugin_info( + formatted_msg = self._get_deprecation_message_with_plugin_info( msg=msg, version=version, removed=removed, date=date, - plugin=plugin, - )) + deprecator=deprecator, + ) + + raise AnsibleError(formatted_msg) if source_context := _utils.SourceContext.from_value(obj): formatted_source_context = str(source_context) @@ -723,8 +735,8 @@ class Display(metaclass=Singleton): ), ), version=version, - date=str(date) if isinstance(date, datetime.date) else date, - plugin=plugin, + date=date, + deprecator=deprecator, formatted_traceback=_traceback.maybe_capture_traceback(_traceback.TracebackEvent.DEPRECATED), ) @@ -1225,20 +1237,70 @@ def _get_message_lines(message: str, help_text: str | None, formatted_source_con return message_lines +def _join_sentences(first: str | None, second: str | None) -> str: + """Join two sentences together.""" + first = (first or '').strip() + second = (second or '').strip() + + if first and first[-1] not in ('!', '?', '.'): + first += '.' + + if second and second[-1] not in ('!', '?', '.'): + second += '.' + + if first and not second: + return first + + if not first and second: + return second + + return ' '.join((first, second)) + + def format_message(summary: SummaryBase) -> str: - details: t.Sequence[Detail] + details: c.Sequence[Detail] = summary.details + + if isinstance(summary, DeprecationSummary) and details: + # augment the first detail element for deprecations to include additional diagnostic info and help text + detail_list = list(details) + detail = detail_list[0] + + deprecation_msg = _display._get_deprecation_message_with_plugin_info( + msg=detail.msg, + version=summary.version, + date=summary.date, + deprecator=summary.deprecator, + ) - if isinstance(summary, DeprecationSummary): - details = [detail if idx else dataclasses.replace( + detail_list[0] = dataclasses.replace( detail, - msg=_display._get_deprecation_message_with_plugin_info( - msg=detail.msg, - version=summary.version, - date=summary.date, - plugin=summary.plugin, - ), - ) for idx, detail in enumerate(summary.details)] - else: - details = summary.details + msg=deprecation_msg, + help_text=detail.help_text, + ) + + details = detail_list return _format_error_details(details, summary.formatted_traceback) + + +def _report_config_warnings(deprecator: PluginInfo) -> None: + """Called by config to report warnings/deprecations collected during a config parse.""" + while config.WARNINGS: + warn = config.WARNINGS.pop() + _display.warning(warn) + + while config.DEPRECATED: + # tuple with name and options + dep = config.DEPRECATED.pop(0) + msg = config.get_deprecated_msg_from_config(dep[1]).replace("\t", "") + + _display.deprecated( # pylint: disable=ansible-deprecated-unnecessary-collection-name,ansible-invalid-deprecated-version + msg=f"{dep[0]} option. {msg}", + version=dep[1]['version'], + deprecator=deprecator, + ) + + +# emit any warnings or deprecations +# in the event config fails before display is up, we'll lose warnings -- but that's OK, since everything is broken anyway +_report_config_warnings(_deprecator.ANSIBLE_CORE_DEPRECATOR) diff --git a/lib/ansible/utils/py3compat.py b/lib/ansible/utils/py3compat.py index 53f06ff9128..ae3b0702da4 100644 --- a/lib/ansible/utils/py3compat.py +++ b/lib/ansible/utils/py3compat.py @@ -6,7 +6,6 @@ from __future__ import annotations -import inspect import os from ansible.utils.display import Display @@ -19,13 +18,8 @@ def __getattr__(name): if name != 'environ': raise AttributeError(name) - caller = inspect.stack()[1] - display.deprecated( - ( - 'ansible.utils.py3compat.environ is deprecated in favor of os.environ. ' - f'Accessed by {caller.filename} line number {caller.lineno}' - ), + msg='ansible.utils.py3compat.environ is deprecated in favor of os.environ.', version='2.20', ) diff --git a/lib/ansible/utils/ssh_functions.py b/lib/ansible/utils/ssh_functions.py index 58c241fef19..6a29f1a272f 100644 --- a/lib/ansible/utils/ssh_functions.py +++ b/lib/ansible/utils/ssh_functions.py @@ -56,7 +56,10 @@ def set_default_transport(): # deal with 'smart' connection .. one time .. if C.DEFAULT_TRANSPORT == 'smart': - display.deprecated("The 'smart' option for connections is deprecated. Set the connection plugin directly instead.", version='2.20') + display.deprecated( + msg="The 'smart' option for connections is deprecated. Set the connection plugin directly instead.", + version='2.20', + ) # see if SSH can support ControlPersist if not use paramiko if not check_for_controlpersist('ssh') and paramiko is not None: diff --git a/lib/ansible/vars/manager.py b/lib/ansible/vars/manager.py index f23d0ec62fb..3bd401c763d 100644 --- a/lib/ansible/vars/manager.py +++ b/lib/ansible/vars/manager.py @@ -25,6 +25,8 @@ from collections import defaultdict from collections.abc import Mapping, MutableMapping from ansible import constants as C +from ansible.module_utils._internal import _deprecator +from ansible.module_utils._internal._datatag import _tags from ansible.errors import (AnsibleError, AnsibleParserError, AnsibleUndefinedVariable, AnsibleFileNotFound, AnsibleAssertionError, AnsibleValueOmittedError) from ansible.inventory.host import Host @@ -32,7 +34,6 @@ from ansible.inventory.helpers import sort_groups, get_group_vars from ansible.inventory.manager import InventoryManager from ansible.module_utils.datatag import native_type_name from ansible.module_utils.six import text_type -from ansible.module_utils.datatag import deprecate_value from ansible.parsing.dataloader import DataLoader from ansible._internal._templating._engine import TemplateEngine from ansible.plugins.loader import cache_loader @@ -50,8 +51,12 @@ if t.TYPE_CHECKING: display = Display() # deprecated: description='enable top-level facts deprecation' core_version='2.20' -# _DEPRECATE_TOP_LEVEL_FACT_MSG = sys.intern('Top-level facts are deprecated, use `ansible_facts` instead.') -# _DEPRECATE_TOP_LEVEL_FACT_REMOVAL_VERSION = sys.intern('2.22') +# _DEPRECATE_TOP_LEVEL_FACT_TAG = _tags.Deprecated( +# msg='Top-level facts are deprecated.', +# version='2.24', +# deprecator=_deprecator.ANSIBLE_CORE_DEPRECATOR, +# help_text='Use `ansible_facts` instead.', +# ) def _deprecate_top_level_fact(value: t.Any) -> t.Any: @@ -61,7 +66,7 @@ def _deprecate_top_level_fact(value: t.Any) -> t.Any: Unique tag instances are required to achieve the correct de-duplication within a top-level templating operation. """ # deprecated: description='enable top-level facts deprecation' core_version='2.20' - # return deprecate_value(value, _DEPRECATE_TOP_LEVEL_FACT_MSG, removal_version=_DEPRECATE_TOP_LEVEL_FACT_REMOVAL_VERSION) + # return _DEPRECATE_TOP_LEVEL_FACT_TAG.tag(value) return value @@ -96,6 +101,13 @@ class VariableManager: _ALLOWED = frozenset(['plugins_by_group', 'groups_plugins_play', 'groups_plugins_inventory', 'groups_inventory', 'all_plugins_play', 'all_plugins_inventory', 'all_inventory']) + _PLAY_HOSTS_DEPRECATED_TAG = _tags.Deprecated( + msg='The `play_hosts` magic variable is deprecated.', + version='2.23', + deprecator=_deprecator.ANSIBLE_CORE_DEPRECATOR, + help_text='Use `ansible_play_batch` instead.', + ) + def __init__(self, loader: DataLoader | None = None, inventory: InventoryManager | None = None, version_info: dict[str, str] | None = None) -> None: self._nonpersistent_fact_cache: defaultdict[str, dict] = defaultdict(dict) self._vars_cache: defaultdict[str, dict] = defaultdict(dict) @@ -477,12 +489,8 @@ class VariableManager: variables['ansible_play_hosts'] = [x for x in variables['ansible_play_hosts_all'] if x not in play._removed_hosts] variables['ansible_play_batch'] = [x for x in _hosts if x not in play._removed_hosts] - variables['play_hosts'] = deprecate_value( - value=variables['ansible_play_batch'], - msg='The `play_hosts` magic variable is deprecated.', - removal_version='2.23', - help_text='Use `ansible_play_batch` instead.', - ) + # use a static tag instead of `deprecate_value` to avoid stackwalk in a hot code path + variables['play_hosts'] = self._PLAY_HOSTS_DEPRECATED_TAG.tag(variables['ansible_play_batch']) # Set options vars for option, option_value in self._options_vars.items(): diff --git a/lib/ansible/vars/plugins.py b/lib/ansible/vars/plugins.py index a5fcd7bc602..63361da66f2 100644 --- a/lib/ansible/vars/plugins.py +++ b/lib/ansible/vars/plugins.py @@ -34,10 +34,10 @@ def get_plugin_vars(loader, plugin, path, entities): except AttributeError: if hasattr(plugin, 'get_host_vars') or hasattr(plugin, 'get_group_vars'): display.deprecated( - f"The vars plugin {plugin.ansible_name} from {plugin._original_path} is relying " - "on the deprecated entrypoints 'get_host_vars' and 'get_group_vars'. " - "This plugin should be updated to inherit from BaseVarsPlugin and define " - "a 'get_vars' method as the main entrypoint instead.", + msg=f"The vars plugin {plugin.ansible_name} from {plugin._original_path} is relying " + "on the deprecated entrypoints 'get_host_vars' and 'get_group_vars'. " + "This plugin should be updated to inherit from BaseVarsPlugin and define " + "a 'get_vars' method as the main entrypoint instead.", version="2.20", ) try: diff --git a/test/integration/targets/ansible-test-sanity-pylint/ansible_collections/ns/col/plugins/action/do_deprecated_stuff.py b/test/integration/targets/ansible-test-sanity-pylint/ansible_collections/ns/col/plugins/action/do_deprecated_stuff.py new file mode 100644 index 00000000000..f7dc448c66f --- /dev/null +++ b/test/integration/targets/ansible-test-sanity-pylint/ansible_collections/ns/col/plugins/action/do_deprecated_stuff.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from ansible.plugins.action import ActionBase +from ansible.utils.display import _display +from ansible.module_utils.common.messages import PluginInfo + +# extra lines below to allow for adding more imports without shifting the line numbers of the code that follows +# +# +# +# +# + + +class ActionModule(ActionBase): + def run(self, tmp=None, task_vars=None): + result = super(ActionModule, self).run(tmp, task_vars) + deprecator = PluginInfo._from_collection_name('ns.col') + + # ansible-deprecated-version - only ansible-core can encounter this + _display.deprecated(msg='ansible-deprecated-no-version') + # ansible-invalid-deprecated-version - only ansible-core can encounter this + _display.deprecated(msg='collection-deprecated-version', version='1.0.0') + _display.deprecated(msg='collection-invalid-deprecated-version', version='not-a-version') + # ansible-deprecated-no-collection-name - only a module_utils can encounter this + _display.deprecated(msg='wrong-collection-deprecated', collection_name='ns.wrong', version='3.0.0') + _display.deprecated(msg='ansible-expired-deprecated-date', date='2000-01-01') + _display.deprecated(msg='ansible-invalid-deprecated-date', date='not-a-date') + _display.deprecated(msg='ansible-deprecated-both-version-and-date', version='3.0.0', date='2099-01-01') + _display.deprecated(msg='removal-version-must-be-major', version='3.1.0') + # ansible-deprecated-date-not-permitted - only ansible-core can encounter this + _display.deprecated(msg='ansible-deprecated-unnecessary-collection-name', deprecator=deprecator, version='3.0.0') + # ansible-deprecated-collection-name-not-permitted - only ansible-core can encounter this + _display.deprecated(msg='ansible-deprecated-both-collection-name-and-deprecator', collection_name='ns.col', deprecator=deprecator, version='3.0.0') + + return result diff --git a/test/integration/targets/ansible-test-sanity-pylint/ansible_collections/ns/col/plugins/lookup/deprecated.py b/test/integration/targets/ansible-test-sanity-pylint/ansible_collections/ns/col/plugins/lookup/deprecated.py index 4fd6ae40417..afb9edb2e14 100644 --- a/test/integration/targets/ansible-test-sanity-pylint/ansible_collections/ns/col/plugins/lookup/deprecated.py +++ b/test/integration/targets/ansible-test-sanity-pylint/ansible_collections/ns/col/plugins/lookup/deprecated.py @@ -1,3 +1,4 @@ +"""This Python module calls deprecation functions in a variety of ways to validate call inference is supported in all common scenarios.""" # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) from __future__ import annotations @@ -13,9 +14,72 @@ author: EXAMPLES = """#""" RETURN = """#""" +import ansible.utils.display +import ansible.module_utils.common.warnings + +from ansible.module_utils import datatag +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.basic import deprecate +from ansible.module_utils.common import warnings +from ansible.module_utils.common.warnings import deprecate as basic_deprecate +from ansible.module_utils.datatag import deprecate_value from ansible.plugins.lookup import LookupBase +from ansible.utils import display as x_display +from ansible.utils.display import Display as XDisplay +from ansible.utils.display import _display + +global_display = XDisplay() +other_global_display = x_display.Display() +foreign_global_display = x_display._display + +# extra lines below to allow for adding more imports without shifting the line numbers of the code that follows +# +# +# +# +# +# +# class LookupModule(LookupBase): def run(self, **kwargs): return [] + + +class MyModule(AnsibleModule): + """A class.""" + + do_deprecated = global_display.deprecated + + def my_method(self) -> None: + """A method.""" + + self.deprecate('', version='2.0.0', collection_name='ns.col') + + +def give_me_a_func(): + return global_display.deprecated + + +def do_stuff() -> None: + """A function.""" + d1 = x_display.Display() + d2 = XDisplay() + + MyModule.do_deprecated('', version='2.0.0', collection_name='ns.col') + basic_deprecate('', version='2.0.0', collection_name='ns.col') + ansible.utils.display._display.deprecated('', version='2.0.0', collection_name='ns.col') + d1.deprecated('', version='2.0.0', collection_name='ns.col') + d2.deprecated('', version='2.0.0', collection_name='ns.col') + x_display.Display().deprecated('', version='2.0.0', collection_name='ns.col') + XDisplay().deprecated('', version='2.0.0', collection_name='ns.col') + warnings.deprecate('', version='2.0.0', collection_name='ns.col') + deprecate('', version='2.0.0', collection_name='ns.col') + datatag.deprecate_value("thing", '', collection_name='ns.col', version='2.0.0') + deprecate_value("thing", '', collection_name='ns.col', version='2.0.0') + global_display.deprecated('', version='2.0.0', collection_name='ns.col') + other_global_display.deprecated('', version='2.0.0', collection_name='ns.col') + foreign_global_display.deprecated('', version='2.0.0', collection_name='ns.col') + _display.deprecated('', version='2.0.0', collection_name='ns.col') + give_me_a_func()("hello") # not detected diff --git a/test/integration/targets/ansible-test-sanity-pylint/ansible_collections/ns/col/plugins/module_utils/deprecated_utils.py b/test/integration/targets/ansible-test-sanity-pylint/ansible_collections/ns/col/plugins/module_utils/deprecated_utils.py new file mode 100644 index 00000000000..5e1d183d730 --- /dev/null +++ b/test/integration/targets/ansible-test-sanity-pylint/ansible_collections/ns/col/plugins/module_utils/deprecated_utils.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from ansible.module_utils.common.messages import PluginInfo +from ansible.module_utils.common.warnings import deprecate + +# extra lines below to allow for adding more imports without shifting the line numbers of the code that follows +# +# +# +# +# +# +# +# + + +def do_stuff() -> None: + deprecator = PluginInfo._from_collection_name('ns.col') + + # ansible-deprecated-version - only ansible-core can encounter this + deprecate(msg='ansible-deprecated-no-version', collection_name='ns.col') + # ansible-invalid-deprecated-version - only ansible-core can encounter this + deprecate(msg='collection-deprecated-version', collection_name='ns.col', version='1.0.0') + deprecate(msg='collection-invalid-deprecated-version', collection_name='ns.col', version='not-a-version') + # ansible-deprecated-no-collection-name - module_utils cannot encounter this + deprecate(msg='wrong-collection-deprecated', collection_name='ns.wrong', version='3.0.0') + deprecate(msg='ansible-expired-deprecated-date', collection_name='ns.col', date='2000-01-01') + deprecate(msg='ansible-invalid-deprecated-date', collection_name='ns.col', date='not-a-date') + deprecate(msg='ansible-deprecated-both-version-and-date', collection_name='ns.col', version='3.0.0', date='2099-01-01') + deprecate(msg='removal-version-must-be-major', collection_name='ns.col', version='3.1.0') + # ansible-deprecated-date-not-permitted - only ansible-core can encounter this + # ansible-deprecated-unnecessary-collection-name - module_utils cannot encounter this + # ansible-deprecated-collection-name-not-permitted - only ansible-core can encounter this + deprecate(msg='ansible-deprecated-both-collection-name-and-deprecator', collection_name='ns.col', deprecator=deprecator, version='3.0.0') diff --git a/test/integration/targets/ansible-test-sanity-pylint/deprecated_thing.py b/test/integration/targets/ansible-test-sanity-pylint/deprecated_thing.py new file mode 100644 index 00000000000..b37db5337b1 --- /dev/null +++ b/test/integration/targets/ansible-test-sanity-pylint/deprecated_thing.py @@ -0,0 +1,28 @@ +""" +This file is not used by the integration test, but serves a related purpose. +It triggers sanity test failures that can only occur for ansible-core, which need to be ignored to ensure the pylint plugin is functioning properly. +""" +from __future__ import annotations + +from ansible.module_utils.common.messages import PluginInfo +from ansible.module_utils.common.warnings import deprecate + + +def do_stuff() -> None: + deprecator = PluginInfo._from_collection_name('ansible.builtin') + + deprecate(msg='ansible-deprecated-version', version='2.18') + deprecate(msg='ansible-deprecated-no-version') + deprecate(msg='ansible-invalid-deprecated-version', version='not-a-version') + # collection-deprecated-version - ansible-core cannot encounter this + # collection-invalid-deprecated-version - ansible-core cannot encounter this + # ansible-deprecated-no-collection-name - ansible-core cannot encounter this + # wrong-collection-deprecated - ansible-core cannot encounter this + # ansible-expired-deprecated-date - ansible-core cannot encounter this + # ansible-invalid-deprecated-date - ansible-core cannot encounter this + # ansible-deprecated-both-version-and-date - ansible-core cannot encounter this + # removal-version-must-be-major - ansible-core cannot encounter this + deprecate(msg='ansible-deprecated-date-not-permitted', date='2099-01-01') + deprecate(msg='ansible-deprecated-unnecessary-collection-name', deprecator=deprecator, version='2.99') + deprecate(msg='ansible-deprecated-collection-name-not-permitted', collection_name='ansible.builtin', version='2.99') + # ansible-deprecated-both-collection-name-and-deprecator - ansible-core cannot encounter this diff --git a/test/integration/targets/ansible-test-sanity-pylint/expected.txt b/test/integration/targets/ansible-test-sanity-pylint/expected.txt index 9c53a308e69..60419e786d0 100644 --- a/test/integration/targets/ansible-test-sanity-pylint/expected.txt +++ b/test/integration/targets/ansible-test-sanity-pylint/expected.txt @@ -1 +1,36 @@ -plugins/lookup/deprecated.py:26:0: collection-deprecated-version: Deprecated version ('2.0.0') found in call to Display.deprecated or AnsibleModule.deprecate +plugins/action/do_deprecated_stuff.py:21:8: ansible-deprecated-no-version: Found 'ansible.utils.display.Display.deprecated' call without a version or date +plugins/action/do_deprecated_stuff.py:23:8: collection-deprecated-version: Deprecated version '1.0.0' found in call to 'ansible.utils.display.Display.deprecated' +plugins/action/do_deprecated_stuff.py:24:8: collection-invalid-deprecated-version: Invalid deprecated version 'not-a-version' found in call to 'ansible.utils.display.Display.deprecated' +plugins/action/do_deprecated_stuff.py:26:8: wrong-collection-deprecated: Wrong collection_name 'ns.wrong' found in call to 'ansible.utils.display.Display.deprecated' +plugins/action/do_deprecated_stuff.py:27:8: ansible-expired-deprecated-date: Expired date '2000-01-01' found in call to 'ansible.utils.display.Display.deprecated' +plugins/action/do_deprecated_stuff.py:28:8: ansible-invalid-deprecated-date: Invalid date 'not-a-date' found in call to 'ansible.utils.display.Display.deprecated' +plugins/action/do_deprecated_stuff.py:29:8: ansible-deprecated-both-version-and-date: Both version and date found in call to 'ansible.utils.display.Display.deprecated' +plugins/action/do_deprecated_stuff.py:30:8: removal-version-must-be-major: Removal version '3.1.0' must be a major release, not a minor or patch release, see https://semver.org/ +plugins/action/do_deprecated_stuff.py:32:8: ansible-deprecated-unnecessary-collection-name: Unnecessary 'deprecator' found in call to 'ansible.utils.display.Display.deprecated' +plugins/action/do_deprecated_stuff.py:34:8: ansible-deprecated-both-collection-name-and-deprecator: Both collection_name and deprecator found in call to 'ansible.utils.display.Display.deprecated' +plugins/action/do_deprecated_stuff.py:34:8: ansible-deprecated-unnecessary-collection-name: Unnecessary 'deprecator' found in call to 'ansible.utils.display.Display.deprecated' +plugins/lookup/deprecated.py:58:8: collection-deprecated-version: Deprecated version '2.0.0' found in call to 'ansible.module_utils.basic.AnsibleModule.deprecate' +plugins/lookup/deprecated.py:70:4: collection-deprecated-version: Deprecated version '2.0.0' found in call to 'ansible.utils.display.Display.deprecated' +plugins/lookup/deprecated.py:71:4: collection-deprecated-version: Deprecated version '2.0.0' found in call to 'ansible.module_utils.common.warnings.deprecate' +plugins/lookup/deprecated.py:72:4: collection-deprecated-version: Deprecated version '2.0.0' found in call to 'ansible.utils.display.Display.deprecated' +plugins/lookup/deprecated.py:73:4: collection-deprecated-version: Deprecated version '2.0.0' found in call to 'ansible.utils.display.Display.deprecated' +plugins/lookup/deprecated.py:74:4: collection-deprecated-version: Deprecated version '2.0.0' found in call to 'ansible.utils.display.Display.deprecated' +plugins/lookup/deprecated.py:75:4: collection-deprecated-version: Deprecated version '2.0.0' found in call to 'ansible.utils.display.Display.deprecated' +plugins/lookup/deprecated.py:76:4: collection-deprecated-version: Deprecated version '2.0.0' found in call to 'ansible.utils.display.Display.deprecated' +plugins/lookup/deprecated.py:77:4: collection-deprecated-version: Deprecated version '2.0.0' found in call to 'ansible.module_utils.common.warnings.deprecate' +plugins/lookup/deprecated.py:78:4: collection-deprecated-version: Deprecated version '2.0.0' found in call to 'ansible.module_utils.common.warnings.deprecate' +plugins/lookup/deprecated.py:79:4: collection-deprecated-version: Deprecated version '2.0.0' found in call to 'ansible.module_utils.datatag.deprecate_value' +plugins/lookup/deprecated.py:80:4: collection-deprecated-version: Deprecated version '2.0.0' found in call to 'ansible.module_utils.datatag.deprecate_value' +plugins/lookup/deprecated.py:81:4: collection-deprecated-version: Deprecated version '2.0.0' found in call to 'ansible.utils.display.Display.deprecated' +plugins/lookup/deprecated.py:82:4: collection-deprecated-version: Deprecated version '2.0.0' found in call to 'ansible.utils.display.Display.deprecated' +plugins/lookup/deprecated.py:83:4: collection-deprecated-version: Deprecated version '2.0.0' found in call to 'ansible.utils.display.Display.deprecated' +plugins/lookup/deprecated.py:84:4: collection-deprecated-version: Deprecated version '2.0.0' found in call to 'ansible.utils.display.Display.deprecated' +plugins/module_utils/deprecated_utils.py:21:4: ansible-deprecated-no-version: Found 'ansible.module_utils.common.warnings.deprecate' call without a version or date +plugins/module_utils/deprecated_utils.py:23:4: collection-deprecated-version: Deprecated version '1.0.0' found in call to 'ansible.module_utils.common.warnings.deprecate' +plugins/module_utils/deprecated_utils.py:24:4: collection-invalid-deprecated-version: Invalid deprecated version 'not-a-version' found in call to 'ansible.module_utils.common.warnings.deprecate' +plugins/module_utils/deprecated_utils.py:26:4: wrong-collection-deprecated: Wrong collection_name 'ns.wrong' found in call to 'ansible.module_utils.common.warnings.deprecate' +plugins/module_utils/deprecated_utils.py:27:4: ansible-expired-deprecated-date: Expired date '2000-01-01' found in call to 'ansible.module_utils.common.warnings.deprecate' +plugins/module_utils/deprecated_utils.py:28:4: ansible-invalid-deprecated-date: Invalid date 'not-a-date' found in call to 'ansible.module_utils.common.warnings.deprecate' +plugins/module_utils/deprecated_utils.py:29:4: ansible-deprecated-both-version-and-date: Both version and date found in call to 'ansible.module_utils.common.warnings.deprecate' +plugins/module_utils/deprecated_utils.py:30:4: removal-version-must-be-major: Removal version '3.1.0' must be a major release, not a minor or patch release, see https://semver.org/ +plugins/module_utils/deprecated_utils.py:34:4: ansible-deprecated-both-collection-name-and-deprecator: Both collection_name and deprecator found in call to 'ansible.module_utils.common.warnings.deprecate' diff --git a/test/integration/targets/ansible-test-sanity-pylint/runme.sh b/test/integration/targets/ansible-test-sanity-pylint/runme.sh index 72190bfaa0f..788ca7389b0 100755 --- a/test/integration/targets/ansible-test-sanity-pylint/runme.sh +++ b/test/integration/targets/ansible-test-sanity-pylint/runme.sh @@ -4,15 +4,6 @@ set -eu source ../collection/setup.sh -# Create test scenarios at runtime that do not pass sanity tests. -# This avoids the need to create ignore entries for the tests. - -echo " -from ansible.utils.display import Display - -display = Display() -display.deprecated('', version='2.0.0', collection_name='ns.col')" >> plugins/lookup/deprecated.py - # Verify deprecation checking works for normal releases and pre-releases. for version in 2.0.0 2.0.0-dev0; do @@ -23,3 +14,5 @@ for version in 2.0.0 2.0.0-dev0; do diff -u "${TEST_DIR}/expected.txt" actual-stdout.txt grep -f "${TEST_DIR}/expected.txt" actual-stderr.txt done + +echo "PASS" diff --git a/test/integration/targets/collections/collections/ansible_collections/testns/content_adj/plugins/inventory/statichost.py b/test/integration/targets/collections/collections/ansible_collections/testns/content_adj/plugins/inventory/statichost.py index aa0d179e494..bb7699320ee 100644 --- a/test/integration/targets/collections/collections/ansible_collections/testns/content_adj/plugins/inventory/statichost.py +++ b/test/integration/targets/collections/collections/ansible_collections/testns/content_adj/plugins/inventory/statichost.py @@ -45,6 +45,8 @@ class InventoryModule(BaseInventoryPlugin, Cacheable): # Initialize and validate options self._read_config_data(path) + self.load_cache_plugin() + # Exercise cache cache_key = self.get_cache_key(path) attempt_to_read_cache = self.get_option('cache') and cache diff --git a/test/integration/targets/data_tagging_controller/expected_stderr.txt b/test/integration/targets/data_tagging_controller/expected_stderr.txt index 69bc89d4d2c..3b6c5cd517f 100644 --- a/test/integration/targets/data_tagging_controller/expected_stderr.txt +++ b/test/integration/targets/data_tagging_controller/expected_stderr.txt @@ -1,5 +1,5 @@ [WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg. -[DEPRECATION WARNING]: `something_old` is deprecated, don't use it! This feature will be removed in version 1.2.3. -[DEPRECATION WARNING]: `something_old` is deprecated, don't use it! This feature will be removed in version 1.2.3. +[DEPRECATION WARNING]: `something_old` is deprecated, don't use it! This feature will be removed from module 'tagging_sample' in collection 'ansible.legacy' version 9.9999. +[DEPRECATION WARNING]: `something_old` is deprecated, don't use it! This feature will be removed from module 'tagging_sample' in collection 'ansible.legacy' version 9.9999. [WARNING]: Encountered untrusted template or expression. -[DEPRECATION WARNING]: `something_old` is deprecated, don't use it! This feature will be removed in version 1.2.3. +[DEPRECATION WARNING]: `something_old` is deprecated, don't use it! This feature will be removed from module 'tagging_sample' in collection 'ansible.legacy' version 9.9999. diff --git a/test/integration/targets/data_tagging_controller/library/datatag_module.py b/test/integration/targets/data_tagging_controller/library/datatag_module.py deleted file mode 100644 index 985ee6e42b9..00000000000 --- a/test/integration/targets/data_tagging_controller/library/datatag_module.py +++ /dev/null @@ -1,24 +0,0 @@ -from __future__ import annotations - -from ansible.module_utils.basic import AnsibleModule -from ansible.module_utils._internal._datatag._tags import Deprecated - - -def main(): - mod = AnsibleModule(argument_spec={ - 'sensitive_module_arg': dict(default='NO DISPLAY, sensitive arg to module', type='str', no_log=True), - }) - - result = { - 'good_key': 'good value', - 'deprecated_key': Deprecated(msg="`deprecated_key` is deprecated, don't use it!", removal_version='1.2.3').tag('deprecated value'), - 'sensitive_module_arg': mod.params['sensitive_module_arg'], - 'unmarked_template': '{{ ["me", "see", "not", "should"] | sort(reverse=true) | join(" ") }}', - 'changed': False - } - - mod.exit_json(**result) - - -if __name__ == '__main__': - main() diff --git a/test/integration/targets/data_tagging_controller/library/tagging_sample.py b/test/integration/targets/data_tagging_controller/library/tagging_sample.py index 2d6fb0a10ec..c745772e204 100644 --- a/test/integration/targets/data_tagging_controller/library/tagging_sample.py +++ b/test/integration/targets/data_tagging_controller/library/tagging_sample.py @@ -1,7 +1,7 @@ from __future__ import annotations from ansible.module_utils.basic import AnsibleModule -from ansible.module_utils._internal._datatag._tags import Deprecated +from ansible.module_utils.datatag import deprecate_value METADATA = """ schema_version: 1 @@ -16,7 +16,7 @@ def main(): something_old_value = 'an old thing' # Deprecated needs args; tag the value and store it - something_old_value = Deprecated(msg="`something_old` is deprecated, don't use it!", removal_version='1.2.3').tag(something_old_value) + something_old_value = deprecate_value(something_old_value, "`something_old` is deprecated, don't use it!", version='9.9999') result = { 'something_old': something_old_value, diff --git a/test/integration/targets/deprecations/action_plugins/action_with_dep.py b/test/integration/targets/deprecations/action_plugins/action_with_dep.py deleted file mode 100644 index f3010695a32..00000000000 --- a/test/integration/targets/deprecations/action_plugins/action_with_dep.py +++ /dev/null @@ -1,14 +0,0 @@ -from __future__ import annotations - -from ansible.module_utils.datatag import deprecate_value -from ansible.plugins.action import ActionBase - - -class ActionModule(ActionBase): - def run(self, tmp=None, task_vars=None): - result = super().run(tmp, task_vars) - result.update(deprecated_thing=deprecate_value("deprecated thing", msg="Deprecated thing is deprecated.", removal_version='999.999')) - - self._display.deprecated("did a deprecated thing", version="999.999") - - return result diff --git a/test/integration/targets/deprecations/collections/ansible_collections/foo/bar/plugins/__init__.py b/test/integration/targets/deprecations/collections/ansible_collections/foo/bar/plugins/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/integration/targets/deprecations/collections/ansible_collections/foo/bar/plugins/action/__init__.py b/test/integration/targets/deprecations/collections/ansible_collections/foo/bar/plugins/action/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/integration/targets/deprecations/collections/ansible_collections/foo/bar/plugins/action/noisy_action.py b/test/integration/targets/deprecations/collections/ansible_collections/foo/bar/plugins/action/noisy_action.py new file mode 100644 index 00000000000..7b9b11a7fa4 --- /dev/null +++ b/test/integration/targets/deprecations/collections/ansible_collections/foo/bar/plugins/action/noisy_action.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from ansible.plugins.action import ActionBase +from ansible.module_utils.common.warnings import deprecate + +from ..module_utils.shared_deprecation import get_deprecation_kwargs, get_deprecated_value + + +class ActionModule(ActionBase): + def run(self, tmp=None, task_vars=None): + result = super().run(tmp, task_vars) + + for deprecate_kw in get_deprecation_kwargs(): + deprecate(**deprecate_kw) # pylint: disable=ansible-deprecated-no-version + + result.update(deprecated_result=get_deprecated_value()) + + return result diff --git a/test/integration/targets/deprecations/collections/ansible_collections/foo/bar/plugins/module_utils/shared_deprecation.py b/test/integration/targets/deprecations/collections/ansible_collections/foo/bar/plugins/module_utils/shared_deprecation.py new file mode 100644 index 00000000000..9442309ee48 --- /dev/null +++ b/test/integration/targets/deprecations/collections/ansible_collections/foo/bar/plugins/module_utils/shared_deprecation.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from ansible.module_utils.common.messages import PluginInfo +from ansible.module_utils.datatag import deprecate_value + + +def get_deprecation_kwargs() -> list[dict[str, object]]: + return [ + dict(msg="Deprecation that passes collection_name, version, and help_text.", version='9999.9', collection_name='bla.bla', help_text="Help text."), + dict( + msg="Deprecation that passes deprecator and datetime.date.", + date='2034-01-02', + deprecator=PluginInfo._from_collection_name('bla.bla'), + ), + dict(msg="Deprecation that passes deprecator and string date.", date='2034-01-02', deprecator=PluginInfo._from_collection_name('bla.bla')), + dict(msg="Deprecation that passes no deprecator, collection name, or date/version."), + ] + + +def get_deprecated_value() -> str: + return deprecate_value( # pylint: disable=ansible-deprecated-unnecessary-collection-name,ansible-deprecated-collection-name-not-permitted + value='a deprecated value', + msg="value is deprecated", + collection_name='foo.bar', + version='9999.9', + ) diff --git a/test/integration/targets/deprecations/collections/ansible_collections/foo/bar/plugins/modules/__init__.py b/test/integration/targets/deprecations/collections/ansible_collections/foo/bar/plugins/modules/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/integration/targets/deprecations/collections/ansible_collections/foo/bar/plugins/modules/noisy.py b/test/integration/targets/deprecations/collections/ansible_collections/foo/bar/plugins/modules/noisy.py new file mode 100644 index 00000000000..8f6e25acce6 --- /dev/null +++ b/test/integration/targets/deprecations/collections/ansible_collections/foo/bar/plugins/modules/noisy.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from ansible.module_utils.basic import AnsibleModule +from ..module_utils.shared_deprecation import get_deprecation_kwargs, get_deprecated_value + + +def main() -> None: + m = AnsibleModule({}) + m.warn("This is a warning.") + + for deprecate_kw in get_deprecation_kwargs(): + m.deprecate(**deprecate_kw) # pylint: disable=ansible-deprecated-no-version + + m.exit_json(deprecated_result=get_deprecated_value()) + + +if __name__ == '__main__': + main() diff --git a/test/integration/targets/deprecations/deprecated.yml b/test/integration/targets/deprecations/deprecated.yml index 2f927569afb..66467358e9f 100644 --- a/test/integration/targets/deprecations/deprecated.yml +++ b/test/integration/targets/deprecations/deprecated.yml @@ -1,19 +1,42 @@ - hosts: testhost gather_facts: no tasks: - - name: invoke an action that fires a deprecation and returns a deprecated value - action_with_dep: - register: action_result + - name: invoke a module that fires deprecations and returns a deprecated value + foo.bar.noisy: + register: noisy_module_result + + - name: invoke an action that fires deprecations and returns a deprecated value + foo.bar.noisy_action: + register: noisy_action_result + + - name: validate deprecation warnings fired by action/module + assert: + that: + - item.deprecations | length == 4 + - item.deprecations[0].msg is contains "passes collection_name, version, and help_text" + - item.deprecations[0].collection_name == 'bla.bla' + - item.deprecations[0].version == '9999.9' + - item.deprecations[1].msg is contains "passes deprecator and date" + - item.deprecations[1].collection_name == 'bla.bla' + - item.deprecations[1].date == '2034-01-02' + - item.deprecations[2].msg is contains "passes deprecator and string date" + - item.deprecations[2].collection_name == 'bla.bla' + - item.deprecations[2].date == '2034-01-02' + - item.deprecations[3].msg is contains "passes no deprecator, collection name, or date/version" + - item.deprecations[3].collection_name == 'foo.bar' + - item.deprecations[3].date is not defined + loop: '{{ [noisy_module_result, noisy_action_result] }}' - name: touch the deprecated value debug: - var: action_result.deprecated_thing + var: noisy_module_result.deprecated_result register: debug_result - - name: validate the presence of the deprecation warnings + - name: validate deprecation warnings from tagged result assert: that: - - action_result.deprecations | length == 1 - - action_result.deprecations[0].msg is contains "did a deprecated thing" - debug_result.deprecations | length == 1 - - debug_result.deprecations[0].msg is contains "Deprecated thing is deprecated." + - debug_result.deprecations[0].msg is contains "value is deprecated" + - debug_result.deprecations[0].date is not defined + - debug_result.deprecations[0].version is defined + - debug_result.deprecations[0].collection_name == 'foo.bar' diff --git a/test/integration/targets/deprecations/disabled.yml b/test/integration/targets/deprecations/disabled.yml deleted file mode 100644 index 818d320767a..00000000000 --- a/test/integration/targets/deprecations/disabled.yml +++ /dev/null @@ -1,14 +0,0 @@ -- hosts: testhost - gather_facts: no - tasks: - - name: invoke a module that returns a warning and deprecation warning - noisy: - register: result - - - name: verify the warning and deprecation are visible in templating - assert: - that: - - result.warnings | length == 1 - - result.warnings[0] == "This is a warning." - - result.deprecations | length == 1 - - result.deprecations[0].msg == "This is a deprecation." diff --git a/test/integration/targets/deprecations/library/noisy.py b/test/integration/targets/deprecations/library/noisy.py deleted file mode 100644 index d402a6db036..00000000000 --- a/test/integration/targets/deprecations/library/noisy.py +++ /dev/null @@ -1,14 +0,0 @@ -from __future__ import annotations - -from ansible.module_utils.basic import AnsibleModule - - -def main() -> None: - m = AnsibleModule({}) - m.warn("This is a warning.") - m.deprecate("This is a deprecation.", version='9999.9') - m.exit_json() - - -if __name__ == '__main__': - main() diff --git a/test/integration/targets/deprecations/runme.sh b/test/integration/targets/deprecations/runme.sh index 370778801e9..1d291ce8873 100755 --- a/test/integration/targets/deprecations/runme.sh +++ b/test/integration/targets/deprecations/runme.sh @@ -4,12 +4,12 @@ set -eux -o pipefail export ANSIBLE_DEPRECATION_WARNINGS=False -ansible-playbook disabled.yml -i ../../inventory "${@}" 2>&1 | tee disabled.txt +ansible-playbook deprecated.yml -i ../../inventory "${@}" 2>&1 | tee disabled.txt grep "This is a warning" disabled.txt # should be visible -if grep "This is a deprecation" disabled.txt; then - echo "ERROR: deprecation should not be visible" +if grep "DEPRECATION" disabled.txt; then + echo "ERROR: deprecation warnings should not be visible" exit 1 fi diff --git a/test/integration/targets/module_utils/module_utils_test.yml b/test/integration/targets/module_utils/module_utils_test.yml index 93f94d107b4..07291d66fc5 100644 --- a/test/integration/targets/module_utils/module_utils_test.yml +++ b/test/integration/targets/module_utils/module_utils_test.yml @@ -47,7 +47,7 @@ assert: that: - result is failed - - result['msg'].endswith("Could not find imported module support code for ansible.modules.test_failure. Looked for (['ansible.module_utils.zebra.foo4', 'ansible.module_utils.zebra'])") + - result.msg is contains "Could not find imported module support code for ansible.legacy.test_failure. Looked for (['ansible.module_utils.zebra.foo4', 'ansible.module_utils.zebra'])" - name: Test that alias deprecation works test_alias_deprecation: diff --git a/test/integration/targets/plugin_namespace/tasks/main.yml b/test/integration/targets/plugin_namespace/tasks/main.yml index c671dc02d36..197531b92de 100644 --- a/test/integration/targets/plugin_namespace/tasks/main.yml +++ b/test/integration/targets/plugin_namespace/tasks/main.yml @@ -5,7 +5,7 @@ - assert: that: - # filter names are prefixed with a unique hash value to prevent shadowing of other plugins - - filter_name | regex_search('^ansible\.plugins\.filter\.[0-9]+_test_filter$') is truthy + # filter names include a unique hash value to prevent shadowing of other plugins + - filter_name | regex_search('^ansible\.plugins\.filter\.test_filter_[0-9]+$') is truthy - lookup_name == 'ansible.plugins.lookup.lookup_name' - test_name_ok diff --git a/test/integration/targets/plugin_namespace/test_plugins/test_test.py b/test/integration/targets/plugin_namespace/test_plugins/test_test.py index fd949e49ca8..f6faa4d05da 100644 --- a/test/integration/targets/plugin_namespace/test_plugins/test_test.py +++ b/test/integration/targets/plugin_namespace/test_plugins/test_test.py @@ -4,8 +4,8 @@ import re def test_name_ok(value): - # test names are prefixed with a unique hash value to prevent shadowing of other plugins - return bool(re.match(r'^ansible\.plugins\.test\.[0-9]+_test_test$', __name__)) + # test names include a unique hash value to prevent shadowing of other plugins + return bool(re.match(r'^ansible\.plugins\.test\.test_test_[0-9]+$', __name__)) class TestModule: diff --git a/test/integration/targets/protomatter/lookup_plugins/emit_deprecation_warning.py b/test/integration/targets/protomatter/lookup_plugins/emit_deprecation_warning.py index 14b18845650..f7acb9180cf 100644 --- a/test/integration/targets/protomatter/lookup_plugins/emit_deprecation_warning.py +++ b/test/integration/targets/protomatter/lookup_plugins/emit_deprecation_warning.py @@ -6,6 +6,6 @@ from ansible.utils.display import Display class LookupModule(LookupBase): def run(self, terms, variables=None, **kwargs): - Display().deprecated("Hello World!") + Display().deprecated("Hello World!", version='2.9999') return [] diff --git a/test/integration/targets/protomatter/lookup_plugins/synthetic_plugin_info.py b/test/integration/targets/protomatter/lookup_plugins/synthetic_plugin_info.py index 97020711023..c5968f217cd 100644 --- a/test/integration/targets/protomatter/lookup_plugins/synthetic_plugin_info.py +++ b/test/integration/targets/protomatter/lookup_plugins/synthetic_plugin_info.py @@ -7,7 +7,6 @@ from ansible.plugins.lookup import LookupBase class LookupModule(LookupBase): def run(self, terms, variables=None, **kwargs): return [PluginInfo( - requested_name='requested_name', resolved_name='resolved_name', type='type', )] diff --git a/test/integration/targets/protomatter/tasks/main.yml b/test/integration/targets/protomatter/tasks/main.yml index 51b41badfdb..766b3882647 100644 --- a/test/integration/targets/protomatter/tasks/main.yml +++ b/test/integration/targets/protomatter/tasks/main.yml @@ -43,7 +43,6 @@ vars: some_var: Hello expected_plugin_info: - requested_name: requested_name resolved_name: resolved_name type: type @@ -101,9 +100,9 @@ - name: test the unmask filter assert: that: - - deprecation_warning.deprecations[0].plugin | type_debug == 'dict' - - (deprecation_warning.deprecations[0] | ansible._protomatter.unmask("PluginInfo")).plugin | type_debug == "PluginInfo" - - (deprecation_warning.deprecations[0] | ansible._protomatter.unmask(["PluginInfo"])).plugin | type_debug == "PluginInfo" + - deprecation_warning.deprecations[0].deprecator | type_debug == 'dict' + - (deprecation_warning.deprecations[0] | ansible._protomatter.unmask("PluginInfo")).deprecator | type_debug == "PluginInfo" + - (deprecation_warning.deprecations[0] | ansible._protomatter.unmask(["PluginInfo"])).deprecator | type_debug == "PluginInfo" - 1 | ansible._protomatter.unmask("PluginInfo") == 1 - missing_var | ansible._protomatter.unmask("PluginInfo") is undefined diff --git a/test/lib/ansible_test/_internal/commands/sanity/pylint.py b/test/lib/ansible_test/_internal/commands/sanity/pylint.py index e9ca8820238..5665f72aabc 100644 --- a/test/lib/ansible_test/_internal/commands/sanity/pylint.py +++ b/test/lib/ansible_test/_internal/commands/sanity/pylint.py @@ -240,6 +240,7 @@ class PylintTest(SanitySingleVersion): # plugin: deprecated (ansible-test) if data_context().content.collection: plugin_options.update({'--collection-name': data_context().content.collection.full_name}) + plugin_options.update({'--collection-path': os.path.join(data_context().content.collection.root, data_context().content.collection.directory)}) if collection_detail and collection_detail.version: plugin_options.update({'--collection-version': collection_detail.version}) diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated.py b/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated.py deleted file mode 100644 index d03dafae8c9..00000000000 --- a/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated.py +++ /dev/null @@ -1,399 +0,0 @@ -"""Ansible specific plyint plugin for checking deprecations.""" - -# (c) 2018, Matt Martz -# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) -# -*- coding: utf-8 -*- -from __future__ import annotations - -import datetime -import re -import shlex -import typing as t -from tokenize import COMMENT, TokenInfo - -import astroid - -try: - from pylint.checkers.utils import check_messages -except ImportError: - from pylint.checkers.utils import only_required_for_messages as check_messages - -from pylint.checkers import BaseChecker, BaseTokenChecker - -from ansible.module_utils.compat.version import LooseVersion -from ansible.release import __version__ as ansible_version_raw -from ansible.utils.version import SemanticVersion - -MSGS = { - 'E9501': ("Deprecated version (%r) found in call to Display.deprecated " - "or AnsibleModule.deprecate", - "ansible-deprecated-version", - "Used when a call to Display.deprecated specifies a version " - "less than or equal to the current version of Ansible", - {'minversion': (2, 6)}), - 'E9502': ("Display.deprecated call without a version or date", - "ansible-deprecated-no-version", - "Used when a call to Display.deprecated does not specify a " - "version or date", - {'minversion': (2, 6)}), - 'E9503': ("Invalid deprecated version (%r) found in call to " - "Display.deprecated or AnsibleModule.deprecate", - "ansible-invalid-deprecated-version", - "Used when a call to Display.deprecated specifies an invalid " - "Ansible version number", - {'minversion': (2, 6)}), - 'E9504': ("Deprecated version (%r) found in call to Display.deprecated " - "or AnsibleModule.deprecate", - "collection-deprecated-version", - "Used when a call to Display.deprecated specifies a collection " - "version less than or equal to the current version of this " - "collection", - {'minversion': (2, 6)}), - 'E9505': ("Invalid deprecated version (%r) found in call to " - "Display.deprecated or AnsibleModule.deprecate", - "collection-invalid-deprecated-version", - "Used when a call to Display.deprecated specifies an invalid " - "collection version number", - {'minversion': (2, 6)}), - 'E9506': ("No collection name found in call to Display.deprecated or " - "AnsibleModule.deprecate", - "ansible-deprecated-no-collection-name", - "The current collection name in format `namespace.name` must " - "be provided as collection_name when calling Display.deprecated " - "or AnsibleModule.deprecate (`ansible.builtin` for ansible-core)", - {'minversion': (2, 6)}), - 'E9507': ("Wrong collection name (%r) found in call to " - "Display.deprecated or AnsibleModule.deprecate", - "wrong-collection-deprecated", - "The name of the current collection must be passed to the " - "Display.deprecated resp. AnsibleModule.deprecate calls " - "(`ansible.builtin` for ansible-core)", - {'minversion': (2, 6)}), - 'E9508': ("Expired date (%r) found in call to Display.deprecated " - "or AnsibleModule.deprecate", - "ansible-deprecated-date", - "Used when a call to Display.deprecated specifies a date " - "before today", - {'minversion': (2, 6)}), - 'E9509': ("Invalid deprecated date (%r) found in call to " - "Display.deprecated or AnsibleModule.deprecate", - "ansible-invalid-deprecated-date", - "Used when a call to Display.deprecated specifies an invalid " - "date. It must be a string in format `YYYY-MM-DD` (ISO 8601)", - {'minversion': (2, 6)}), - 'E9510': ("Both version and date found in call to " - "Display.deprecated or AnsibleModule.deprecate", - "ansible-deprecated-both-version-and-date", - "Only one of version and date must be specified", - {'minversion': (2, 6)}), - 'E9511': ("Removal version (%r) must be a major release, not a minor or " - "patch release (see the specification at https://semver.org/)", - "removal-version-must-be-major", - "Used when a call to Display.deprecated or " - "AnsibleModule.deprecate for a collection specifies a version " - "which is not of the form x.0.0", - {'minversion': (2, 6)}), -} - - -ANSIBLE_VERSION = LooseVersion('.'.join(ansible_version_raw.split('.')[:3])) - - -def _get_expr_name(node): - """Function to get either ``attrname`` or ``name`` from ``node.func.expr`` - - Created specifically for the case of ``display.deprecated`` or ``self._display.deprecated`` - """ - try: - return node.func.expr.attrname - except AttributeError: - # If this fails too, we'll let it raise, the caller should catch it - return node.func.expr.name - - -def _get_func_name(node): - """Function to get either ``attrname`` or ``name`` from ``node.func`` - - Created specifically for the case of ``from ansible.module_utils.common.warnings import deprecate`` - """ - try: - return node.func.attrname - except AttributeError: - return node.func.name - - -def parse_isodate(value): - """Parse an ISO 8601 date string.""" - msg = 'Expected ISO 8601 date string (YYYY-MM-DD)' - if not isinstance(value, str): - raise ValueError(msg) - # From Python 3.7 in, there is datetime.date.fromisoformat(). For older versions, - # we have to do things manually. - if not re.match('^[0-9]{4}-[0-9]{2}-[0-9]{2}$', value): - raise ValueError(msg) - try: - return datetime.datetime.strptime(value, '%Y-%m-%d').date() - except ValueError: - raise ValueError(msg) from None - - -class AnsibleDeprecatedChecker(BaseChecker): - """Checks for Display.deprecated calls to ensure that the ``version`` - has not passed or met the time for removal - """ - - name = 'deprecated' - msgs = MSGS - - options = ( - ('collection-name', { - 'default': None, - 'type': 'string', - 'metavar': '', - 'help': 'The collection\'s name used to check collection names in deprecations.', - }), - ('collection-version', { - 'default': None, - 'type': 'string', - 'metavar': '', - 'help': 'The collection\'s version number used to check deprecations.', - }), - ) - - def _check_date(self, node, date): - if not isinstance(date, str): - self.add_message('ansible-invalid-deprecated-date', node=node, args=(date,)) - return - - try: - date_parsed = parse_isodate(date) - except ValueError: - self.add_message('ansible-invalid-deprecated-date', node=node, args=(date,)) - return - - if date_parsed < datetime.date.today(): - self.add_message('ansible-deprecated-date', node=node, args=(date,)) - - def _check_version(self, node, version, collection_name): - if collection_name is None: - collection_name = 'ansible.builtin' - if not isinstance(version, (str, float)): - if collection_name == 'ansible.builtin': - symbol = 'ansible-invalid-deprecated-version' - else: - symbol = 'collection-invalid-deprecated-version' - self.add_message(symbol, node=node, args=(version,)) - return - - version_no = str(version) - - if collection_name == 'ansible.builtin': - # Ansible-base - try: - if not version_no: - raise ValueError('Version string should not be empty') - loose_version = LooseVersion(str(version_no)) - if ANSIBLE_VERSION >= loose_version: - self.add_message('ansible-deprecated-version', node=node, args=(version,)) - except ValueError: - self.add_message('ansible-invalid-deprecated-version', node=node, args=(version,)) - elif collection_name: - # Collections - try: - if not version_no: - raise ValueError('Version string should not be empty') - semantic_version = SemanticVersion(version_no) - if collection_name == self.collection_name and self.collection_version is not None: - if self.collection_version >= semantic_version: - self.add_message('collection-deprecated-version', node=node, args=(version,)) - if semantic_version.major != 0 and (semantic_version.minor != 0 or semantic_version.patch != 0): - self.add_message('removal-version-must-be-major', node=node, args=(version,)) - except ValueError: - self.add_message('collection-invalid-deprecated-version', node=node, args=(version,)) - - @property - def collection_name(self) -> t.Optional[str]: - """Return the collection name, or None if ansible-core is being tested.""" - return self.linter.config.collection_name - - @property - def collection_version(self) -> t.Optional[SemanticVersion]: - """Return the collection version, or None if ansible-core is being tested.""" - if self.linter.config.collection_version is None: - return None - sem_ver = SemanticVersion(self.linter.config.collection_version) - # Ignore pre-release for version comparison to catch issues before the final release is cut. - sem_ver.prerelease = () - return sem_ver - - @check_messages(*(MSGS.keys())) - def visit_call(self, node): - """Visit a call node.""" - version = None - date = None - collection_name = None - try: - funcname = _get_func_name(node) - if (funcname == 'deprecated' and 'display' in _get_expr_name(node) or - funcname == 'deprecate'): - if node.keywords: - for keyword in node.keywords: - if len(node.keywords) == 1 and keyword.arg is None: - # This is likely a **kwargs splat - return - if keyword.arg == 'version': - if isinstance(keyword.value.value, astroid.Name): - # This is likely a variable - return - version = keyword.value.value - if keyword.arg == 'date': - if isinstance(keyword.value.value, astroid.Name): - # This is likely a variable - return - date = keyword.value.value - if keyword.arg == 'collection_name': - if isinstance(keyword.value.value, astroid.Name): - # This is likely a variable - return - collection_name = keyword.value.value - if not version and not date: - try: - version = node.args[1].value - except IndexError: - self.add_message('ansible-deprecated-no-version', node=node) - return - if version and date: - self.add_message('ansible-deprecated-both-version-and-date', node=node) - - if collection_name: - this_collection = collection_name == (self.collection_name or 'ansible.builtin') - if not this_collection: - self.add_message('wrong-collection-deprecated', node=node, args=(collection_name,)) - elif self.collection_name is not None: - self.add_message('ansible-deprecated-no-collection-name', node=node) - - if date: - self._check_date(node, date) - elif version: - self._check_version(node, version, collection_name) - except AttributeError: - # Not the type of node we are interested in - pass - - -class AnsibleDeprecatedCommentChecker(BaseTokenChecker): - """Checks for ``# deprecated:`` comments to ensure that the ``version`` - has not passed or met the time for removal - """ - - name = 'deprecated-comment' - msgs = { - 'E9601': ("Deprecated core version (%r) found: %s", - "ansible-deprecated-version-comment", - "Used when a '# deprecated:' comment specifies a version " - "less than or equal to the current version of Ansible", - {'minversion': (2, 6)}), - 'E9602': ("Deprecated comment contains invalid keys %r", - "ansible-deprecated-version-comment-invalid-key", - "Used when a '#deprecated:' comment specifies invalid data", - {'minversion': (2, 6)}), - 'E9603': ("Deprecated comment missing version", - "ansible-deprecated-version-comment-missing-version", - "Used when a '#deprecated:' comment specifies invalid data", - {'minversion': (2, 6)}), - 'E9604': ("Deprecated python version (%r) found: %s", - "ansible-deprecated-python-version-comment", - "Used when a '#deprecated:' comment specifies a python version " - "less than or equal to the minimum python version", - {'minversion': (2, 6)}), - 'E9605': ("Deprecated comment contains invalid version %r: %s", - "ansible-deprecated-version-comment-invalid-version", - "Used when a '#deprecated:' comment specifies an invalid version", - {'minversion': (2, 6)}), - } - - def process_tokens(self, tokens: list[TokenInfo]) -> None: - for token in tokens: - if token.type == COMMENT: - self._process_comment(token) - - def _deprecated_string_to_dict(self, token: TokenInfo, string: str) -> dict[str, str]: - valid_keys = {'description', 'core_version', 'python_version'} - data = dict.fromkeys(valid_keys) - for opt in shlex.split(string): - if '=' not in opt: - data[opt] = None - continue - key, _sep, value = opt.partition('=') - data[key] = value - if not any((data['core_version'], data['python_version'])): - self.add_message( - 'ansible-deprecated-version-comment-missing-version', - line=token.start[0], - col_offset=token.start[1], - ) - bad = set(data).difference(valid_keys) - if bad: - self.add_message( - 'ansible-deprecated-version-comment-invalid-key', - line=token.start[0], - col_offset=token.start[1], - args=(','.join(bad),) - ) - return data - - def _process_python_version(self, token: TokenInfo, data: dict[str, str]) -> None: - check_version = '.'.join(map(str, self.linter.config.py_version)) - - try: - if LooseVersion(data['python_version']) < LooseVersion(check_version): - self.add_message( - 'ansible-deprecated-python-version-comment', - line=token.start[0], - col_offset=token.start[1], - args=( - data['python_version'], - data['description'] or 'description not provided', - ), - ) - except (ValueError, TypeError) as exc: - self.add_message( - 'ansible-deprecated-version-comment-invalid-version', - line=token.start[0], - col_offset=token.start[1], - args=(data['python_version'], exc) - ) - - def _process_core_version(self, token: TokenInfo, data: dict[str, str]) -> None: - try: - if ANSIBLE_VERSION >= LooseVersion(data['core_version']): - self.add_message( - 'ansible-deprecated-version-comment', - line=token.start[0], - col_offset=token.start[1], - args=( - data['core_version'], - data['description'] or 'description not provided', - ) - ) - except (ValueError, TypeError) as exc: - self.add_message( - 'ansible-deprecated-version-comment-invalid-version', - line=token.start[0], - col_offset=token.start[1], - args=(data['core_version'], exc) - ) - - def _process_comment(self, token: TokenInfo) -> None: - if token.string.startswith('# deprecated:'): - data = self._deprecated_string_to_dict(token, token.string[13:].strip()) - if data['core_version']: - self._process_core_version(token, data) - if data['python_version']: - self._process_python_version(token, data) - - -def register(linter): - """required method to auto register this checker """ - linter.register_checker(AnsibleDeprecatedChecker(linter)) - linter.register_checker(AnsibleDeprecatedCommentChecker(linter)) diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated_calls.py b/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated_calls.py new file mode 100644 index 00000000000..3447e857614 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated_calls.py @@ -0,0 +1,475 @@ +"""Ansible-specific pylint plugin for checking deprecation calls.""" + +# (c) 2018, Matt Martz +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import annotations + +import dataclasses +import datetime +import functools +import pathlib + +import astroid +import astroid.context +import astroid.typing + +import pylint.lint +import pylint.checkers +import pylint.checkers.utils + +import ansible.release + +from ansible.module_utils._internal._deprecator import INDETERMINATE_DEPRECATOR, _path_as_collection_plugininfo +from ansible.module_utils.compat.version import StrictVersion +from ansible.utils.version import SemanticVersion + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class DeprecationCallArgs: + """Arguments passed to a deprecation function.""" + + msg: object = None + version: object = None + date: object = None + collection_name: object = None + deprecator: object = None + help_text: object = None # only on Display.deprecated, warnings.deprecate and deprecate_value + obj: object = None # only on Display.deprecated and warnings.deprecate + removed: object = None # only on Display.deprecated + value: object = None # only on deprecate_value + + +class AnsibleDeprecatedChecker(pylint.checkers.BaseChecker): + """Checks for deprecated calls to ensure proper usage.""" + + name = 'deprecated-calls' + msgs = { + 'E9501': ( + "Deprecated version %r found in call to %r", + "ansible-deprecated-version", + None, + ), + 'E9502': ( + "Found %r call without a version or date", + "ansible-deprecated-no-version", + None, + ), + 'E9503': ( + "Invalid deprecated version %r found in call to %r", + "ansible-invalid-deprecated-version", + None, + ), + 'E9504': ( + "Deprecated version %r found in call to %r", + "collection-deprecated-version", + None, + ), + 'E9505': ( + "Invalid deprecated version %r found in call to %r", + "collection-invalid-deprecated-version", + None, + ), + 'E9506': ( + "No collection_name or deprecator found in call to %r", + "ansible-deprecated-no-collection-name", + None, + ), + 'E9507': ( + "Wrong collection_name %r found in call to %r", + "wrong-collection-deprecated", + None, + ), + 'E9508': ( + "Expired date %r found in call to %r", + "ansible-expired-deprecated-date", + None, + ), + 'E9509': ( + "Invalid date %r found in call to %r", + "ansible-invalid-deprecated-date", + None, + ), + 'E9510': ( + "Both version and date found in call to %r", + "ansible-deprecated-both-version-and-date", + None, + ), + 'E9511': ( + "Removal version %r must be a major release, not a minor or patch release, see https://semver.org/", + "removal-version-must-be-major", + None, + ), + 'E9512': ( + "Passing date is not permitted in call to %r for ansible-core, use a version instead", + "ansible-deprecated-date-not-permitted", + None, + ), + 'E9513': ( + "Unnecessary %r found in call to %r", + "ansible-deprecated-unnecessary-collection-name", + None, + ), + 'E9514': ( + "Passing collection_name not permitted in call to %r for ansible-core, use deprecator instead", + "ansible-deprecated-collection-name-not-permitted", + None, + ), + 'E9515': ( + "Both collection_name and deprecator found in call to %r", + "ansible-deprecated-both-collection-name-and-deprecator", + None, + ), + } + + options = ( + ( + 'collection-name', + dict( + default=None, + type='string', + metavar='', + help="The name of the collection to check.", + ), + ), + ( + 'collection-version', + dict( + default=None, + type='string', + metavar='', + help="The version of the collection to check.", + ), + ), + ( + 'collection-path', + dict( + default=None, + type='string', + metavar='', + help="The path of the collection to check.", + ), + ), + ) + + ANSIBLE_VERSION = StrictVersion('.'.join(ansible.release.__version__.split('.')[:3])) + """The current ansible-core X.Y.Z version.""" + + DEPRECATION_MODULE_FUNCTIONS: dict[tuple[str, str], tuple[str, ...]] = { + ('ansible.module_utils.common.warnings', 'deprecate'): ('msg', 'version', 'date', 'collection_name'), + ('ansible.module_utils.datatag', 'deprecate_value'): ('value', 'msg'), + ('ansible.module_utils.basic', 'AnsibleModule.deprecate'): ('msg', 'version', 'date', 'collection_name'), + ('ansible.utils.display', 'Display.deprecated'): ('msg', 'version', 'removed', 'date', 'collection_name'), + } + """Mapping of deprecation module+function and their positional arguments.""" + + DEPRECATION_MODULES = frozenset(key[0] for key in DEPRECATION_MODULE_FUNCTIONS) + """Modules which contain deprecation functions.""" + + DEPRECATION_FUNCTIONS = {'.'.join(key): value for key, value in DEPRECATION_MODULE_FUNCTIONS.items()} + """Mapping of deprecation functions and their positional arguments.""" + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + self.inference_context = astroid.context.InferenceContext() + self.module_cache: dict[str, astroid.Module] = {} + + @functools.cached_property + def collection_name(self) -> str | None: + """Return the collection name, or None if ansible-core is being tested.""" + return self.linter.config.collection_name or None + + @functools.cached_property + def collection_path(self) -> pathlib.Path: + """Return the collection path. Not valid when ansible-core is being tested.""" + return pathlib.Path(self.linter.config.collection_path) + + @functools.cached_property + def collection_version(self) -> SemanticVersion | None: + """Return the collection version, or None if ansible-core is being tested.""" + if not self.linter.config.collection_version: + return None + + sem_ver = SemanticVersion(self.linter.config.collection_version) + sem_ver.prerelease = () # ignore pre-release for version comparison to catch issues before the final release is cut + + return sem_ver + + @functools.cached_property + def is_ansible_core(self) -> bool: + """True if ansible-core is being tested.""" + return not self.collection_name + + @functools.cached_property + def today_utc(self) -> datetime.date: + """Today's date in UTC.""" + return datetime.datetime.now(tz=datetime.timezone.utc).date() + + def is_deprecator_required(self) -> bool | None: + """Determine is a `collection_name` or `deprecator` is required (True), unnecessary (False) or optional (None).""" + if self.is_ansible_core: + return False # in ansible-core, never provide the deprecator -- if it really is needed, disable the sanity test inline for that line of code + + plugin_info = _path_as_collection_plugininfo(self.linter.current_file) + + if plugin_info is INDETERMINATE_DEPRECATOR: + return True # deprecator cannot be detected, caller must provide deprecator + + # deprecation: description='deprecate collection_name/deprecator now that detection is widely available' core_version='2.23' + # When this deprecation triggers, change the return type here to False. + # At that point, callers should be able to omit the collection_name/deprecator in all but a few cases (inline ignores can be used for those cases) + return None + + @pylint.checkers.utils.only_required_for_messages(*(msgs.keys())) + def visit_call(self, node: astroid.Call) -> None: + """Visit a call node.""" + if inferred := self.infer(node.func): + name = self.get_fully_qualified_name(inferred) + + if args := self.DEPRECATION_FUNCTIONS.get(name): + self.check_call(node, name, args) + + def infer(self, node: astroid.NodeNG) -> astroid.NodeNG | None: + """Return the inferred node from the given node, or `None` if it cannot be unambiguously inferred.""" + names: list[str] = [] + target: astroid.NodeNG | None = node + inferred: astroid.typing.InferenceResult | None = None + + while target: + if inferred := astroid.util.safe_infer(target, self.inference_context): + break + + if isinstance(target, astroid.Call): + inferred = self.infer(target.func) + break + + if isinstance(target, astroid.FunctionDef): + inferred = target + break + + if isinstance(target, astroid.Name): + target = self.infer_name(target) + elif isinstance(target, astroid.AssignName) and isinstance(target.parent, astroid.Assign): + target = target.parent.value + elif isinstance(target, astroid.Attribute): + names.append(target.attrname) + target = target.expr + else: + break + + for name in reversed(names): + if not isinstance(inferred, (astroid.Module, astroid.ClassDef)): + inferred = None + break + + try: + inferred = inferred[name] + except KeyError: + inferred = None + else: + inferred = self.infer(inferred) + + if isinstance(inferred, astroid.FunctionDef) and isinstance(inferred.parent, astroid.ClassDef): + inferred = astroid.BoundMethod(inferred, inferred.parent) + + return inferred + + def infer_name(self, node: astroid.Name) -> astroid.NodeNG | None: + """Infer the node referenced by the given name, or `None` if it cannot be unambiguously inferred.""" + scope = node.scope() + name = None + + while scope: + try: + assignment = scope[node.name] + except KeyError: + scope = scope.parent.scope() if scope.parent else None + continue + + if isinstance(assignment, astroid.AssignName) and isinstance(assignment.parent, astroid.Assign): + name = assignment.parent.value + elif isinstance(assignment, astroid.ImportFrom): + if module := self.get_module(assignment): + scope = module.scope() + continue + + break + + return name + + def get_module(self, node: astroid.ImportFrom) -> astroid.Module | None: + """Import the requested module if possible and cache the result.""" + module_name = pylint.checkers.utils.get_import_name(node, node.modname) + + if module_name not in self.DEPRECATION_MODULES: + return None # avoid unnecessary import overhead + + if module := self.module_cache.get(module_name): + return module + + module = node.do_import_module() + + if module.name != module_name: + raise RuntimeError(f'Attempted to import {module_name!r} but found {module.name!r} instead.') + + self.module_cache[module_name] = module + + return module + + @staticmethod + def get_fully_qualified_name(node: astroid.NodeNG) -> str | None: + """Return the fully qualified name of the given inferred node.""" + parent = node.parent + parts: tuple[str, ...] | None + + if isinstance(node, astroid.FunctionDef) and isinstance(parent, astroid.Module): + parts = (parent.name, node.name) + elif isinstance(node, astroid.BoundMethod) and isinstance(parent, astroid.ClassDef) and isinstance(parent.parent, astroid.Module): + parts = (parent.parent.name, parent.name, node.name) + else: + parts = None + + return '.'.join(parts) if parts else None + + def check_call(self, node: astroid.Call, name: str, args: tuple[str, ...]) -> None: + """Check the given deprecation call node for valid arguments.""" + call_args = self.get_deprecation_call_args(node, args) + + self.check_collection_name(node, name, call_args) + + if not call_args.version and not call_args.date: + self.add_message('ansible-deprecated-no-version', node=node, args=(name,)) + return + + if call_args.date and self.is_ansible_core: + self.add_message('ansible-deprecated-date-not-permitted', node=node, args=(name,)) + return + + if call_args.version and call_args.date: + self.add_message('ansible-deprecated-both-version-and-date', node=node, args=(name,)) + return + + if call_args.date: + self.check_date(node, name, call_args) + + if call_args.version: + self.check_version(node, name, call_args) + + @staticmethod + def get_deprecation_call_args(node: astroid.Call, args: tuple[str, ...]) -> DeprecationCallArgs: + """Get the deprecation call arguments from the given node.""" + fields: dict[str, object] = {} + + for idx, arg in enumerate(node.args): + field = args[idx] + fields[field] = arg + + for keyword in node.keywords: + if keyword.arg is not None: + fields[keyword.arg] = keyword.value + + for key, value in fields.items(): + if isinstance(value, astroid.Const): + fields[key] = value.value + + return DeprecationCallArgs(**fields) + + def check_collection_name(self, node: astroid.Call, name: str, args: DeprecationCallArgs) -> None: + """Check the collection name provided to the given call node.""" + deprecator_requirement = self.is_deprecator_required() + + if self.is_ansible_core and args.collection_name: + self.add_message('ansible-deprecated-collection-name-not-permitted', node=node, args=(name,)) + return + + if args.collection_name and args.deprecator: + self.add_message('ansible-deprecated-both-collection-name-and-deprecator', node=node, args=(name,)) + + if deprecator_requirement is True: + if not args.collection_name and not args.deprecator: + self.add_message('ansible-deprecated-no-collection-name', node=node, args=(name,)) + return + elif deprecator_requirement is False: + if args.collection_name: + self.add_message('ansible-deprecated-unnecessary-collection-name', node=node, args=('collection_name', name,)) + return + + if args.deprecator: + self.add_message('ansible-deprecated-unnecessary-collection-name', node=node, args=('deprecator', name,)) + return + else: + # collection_name may be needed for backward compat with 2.18 and earlier, since it is only detected in 2.19 and later + + if args.deprecator: + # Unlike collection_name, which is needed for backward compat, deprecator is generally not needed by collections. + # For the very rare cases where this is needed by collections, an inline pylint ignore can be used to silence it. + self.add_message('ansible-deprecated-unnecessary-collection-name', node=node, args=('deprecator', name,)) + return + + expected_collection_name = 'ansible.builtin' if self.is_ansible_core else self.collection_name + + if args.collection_name and args.collection_name != expected_collection_name: + # if collection_name is provided and a constant, report when it does not match the expected name + self.add_message('wrong-collection-deprecated', node=node, args=(args.collection_name, name)) + + def check_version(self, node: astroid.Call, name: str, args: DeprecationCallArgs) -> None: + """Check the version provided to the given call node.""" + if self.collection_name: + self.check_collection_version(node, name, args) + else: + self.check_core_version(node, name, args) + + def check_core_version(self, node: astroid.Call, name: str, args: DeprecationCallArgs) -> None: + """Check the core version provided to the given call node.""" + try: + if not isinstance(args.version, str) or not args.version: + raise ValueError() + + strict_version = StrictVersion(args.version) + except ValueError: + self.add_message('ansible-invalid-deprecated-version', node=node, args=(args.version, name)) + return + + if self.ANSIBLE_VERSION >= strict_version: + self.add_message('ansible-deprecated-version', node=node, args=(args.version, name)) + + def check_collection_version(self, node: astroid.Call, name: str, args: DeprecationCallArgs) -> None: + """Check the collection version provided to the given call node.""" + try: + if not isinstance(args.version, str) or not args.version: + raise ValueError() + + semantic_version = SemanticVersion(args.version) + except ValueError: + self.add_message('collection-invalid-deprecated-version', node=node, args=(args.version, name)) + return + + if self.collection_version >= semantic_version: + self.add_message('collection-deprecated-version', node=node, args=(args.version, name)) + + if semantic_version.major != 0 and (semantic_version.minor != 0 or semantic_version.patch != 0): + self.add_message('removal-version-must-be-major', node=node, args=(args.version,)) + + def check_date(self, node: astroid.Call, name: str, args: DeprecationCallArgs) -> None: + """Check the date provided to the given call node.""" + try: + date_parsed = self.parse_isodate(args.date) + except (ValueError, TypeError): + self.add_message('ansible-invalid-deprecated-date', node=node, args=(args.date, name)) + else: + if date_parsed < self.today_utc: + self.add_message('ansible-expired-deprecated-date', node=node, args=(args.date, name)) + + @staticmethod + def parse_isodate(value: object) -> datetime.date: + """Parse an ISO 8601 date string.""" + if isinstance(value, str): + return datetime.date.fromisoformat(value) + + raise TypeError(type(value)) + + +def register(linter: pylint.lint.PyLinter) -> None: + """Required method to auto-register this checker.""" + linter.register_checker(AnsibleDeprecatedChecker(linter)) diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated_comment.py b/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated_comment.py new file mode 100644 index 00000000000..bc0df193872 --- /dev/null +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated_comment.py @@ -0,0 +1,137 @@ +"""Ansible-specific pylint plugin for checking deprecation comments.""" + +# (c) 2018, Matt Martz +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import annotations + +import shlex +import tokenize + +import pylint.checkers +import pylint.lint + +import ansible.release + +from ansible.module_utils.compat.version import LooseVersion + + +class AnsibleDeprecatedCommentChecker(pylint.checkers.BaseTokenChecker): + """Checks for ``# deprecated:`` comments to ensure that the ``version`` has not passed or met the time for removal.""" + + name = 'deprecated-comment' + msgs = { + 'E9601': ( + "Deprecated core version (%r) found: %s", + "ansible-deprecated-version-comment", + None, + ), + 'E9602': ( + "Deprecated comment contains invalid keys %r", + "ansible-deprecated-version-comment-invalid-key", + None, + ), + 'E9603': ( + "Deprecated comment missing version", + "ansible-deprecated-version-comment-missing-version", + None, + ), + 'E9604': ( + "Deprecated python version (%r) found: %s", + "ansible-deprecated-python-version-comment", + None, + ), + 'E9605': ( + "Deprecated comment contains invalid version %r: %s", + "ansible-deprecated-version-comment-invalid-version", + None, + ), + } + + ANSIBLE_VERSION = LooseVersion('.'.join(ansible.release.__version__.split('.')[:3])) + """The current ansible-core X.Y.Z version.""" + + def process_tokens(self, tokens: list[tokenize.TokenInfo]) -> None: + for token in tokens: + if token.type == tokenize.COMMENT: + self._process_comment(token) + + def _deprecated_string_to_dict(self, token: tokenize.TokenInfo, string: str) -> dict[str, str]: + valid_keys = {'description', 'core_version', 'python_version'} + data = dict.fromkeys(valid_keys) + for opt in shlex.split(string): + if '=' not in opt: + data[opt] = None + continue + key, _sep, value = opt.partition('=') + data[key] = value + if not any((data['core_version'], data['python_version'])): + self.add_message( + 'ansible-deprecated-version-comment-missing-version', + line=token.start[0], + col_offset=token.start[1], + ) + bad = set(data).difference(valid_keys) + if bad: + self.add_message( + 'ansible-deprecated-version-comment-invalid-key', + line=token.start[0], + col_offset=token.start[1], + args=(','.join(bad),), + ) + return data + + def _process_python_version(self, token: tokenize.TokenInfo, data: dict[str, str]) -> None: + check_version = '.'.join(map(str, self.linter.config.py_version)) # minimum supported Python version provided by ansible-test + + try: + if LooseVersion(check_version) > LooseVersion(data['python_version']): + self.add_message( + 'ansible-deprecated-python-version-comment', + line=token.start[0], + col_offset=token.start[1], + args=( + data['python_version'], + data['description'] or 'description not provided', + ), + ) + except (ValueError, TypeError) as exc: + self.add_message( + 'ansible-deprecated-version-comment-invalid-version', + line=token.start[0], + col_offset=token.start[1], + args=(data['python_version'], exc), + ) + + def _process_core_version(self, token: tokenize.TokenInfo, data: dict[str, str]) -> None: + try: + if self.ANSIBLE_VERSION >= LooseVersion(data['core_version']): + self.add_message( + 'ansible-deprecated-version-comment', + line=token.start[0], + col_offset=token.start[1], + args=( + data['core_version'], + data['description'] or 'description not provided', + ), + ) + except (ValueError, TypeError) as exc: + self.add_message( + 'ansible-deprecated-version-comment-invalid-version', + line=token.start[0], + col_offset=token.start[1], + args=(data['core_version'], exc), + ) + + def _process_comment(self, token: tokenize.TokenInfo) -> None: + if token.string.startswith('# deprecated:'): + data = self._deprecated_string_to_dict(token, token.string[13:].strip()) + if data['core_version']: + self._process_core_version(token, data) + if data['python_version']: + self._process_python_version(token, data) + + +def register(linter: pylint.lint.PyLinter) -> None: + """Required method to auto-register this checker.""" + linter.register_checker(AnsibleDeprecatedCommentChecker(linter)) diff --git a/test/sanity/code-smell/mypy/ansible-test.ini b/test/sanity/code-smell/mypy/ansible-test.ini index db7bb21af81..81a5d64e4fc 100644 --- a/test/sanity/code-smell/mypy/ansible-test.ini +++ b/test/sanity/code-smell/mypy/ansible-test.ini @@ -36,6 +36,15 @@ ignore_missing_imports = True [mypy-astroid] ignore_missing_imports = True +[mypy-astroid.typing] +ignore_missing_imports = True + +[mypy-astroid.context] +ignore_missing_imports = True + +[mypy-pylint] +ignore_missing_imports = True + [mypy-pylint.interfaces] ignore_missing_imports = True diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index 3cb789f9ed9..64937d1cc3c 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -218,3 +218,12 @@ test/units/modules/test_apt.py mypy-3.8:name-match test/units/modules/test_mount_facts.py mypy-3.8:index test/integration/targets/interpreter_discovery_python/library/test_non_python_interpreter.py shebang # test needs non-standard shebang test/integration/targets/inventory_script/bad_shebang shebang # test needs an invalid shebang +test/integration/targets/ansible-test-sanity-pylint/ansible_collections/ns/col/plugins/lookup/deprecated.py pylint!skip # validated as a collection +test/integration/targets/ansible-test-sanity-pylint/ansible_collections/ns/col/plugins/action/do_deprecated_stuff.py pylint!skip # validated as a collection +test/integration/targets/ansible-test-sanity-pylint/ansible_collections/ns/col/plugins/module_utils/deprecated_utils.py pylint!skip # validated as a collection +test/integration/targets/ansible-test-sanity-pylint/deprecated_thing.py pylint:ansible-deprecated-version # required to verify plugin against core +test/integration/targets/ansible-test-sanity-pylint/deprecated_thing.py pylint:ansible-deprecated-no-version # required to verify plugin against core +test/integration/targets/ansible-test-sanity-pylint/deprecated_thing.py pylint:ansible-invalid-deprecated-version # required to verify plugin against core +test/integration/targets/ansible-test-sanity-pylint/deprecated_thing.py pylint:ansible-deprecated-date-not-permitted # required to verify plugin against core +test/integration/targets/ansible-test-sanity-pylint/deprecated_thing.py pylint:ansible-deprecated-unnecessary-collection-name # required to verify plugin against core +test/integration/targets/ansible-test-sanity-pylint/deprecated_thing.py pylint:ansible-deprecated-collection-name-not-permitted # required to verify plugin against core diff --git a/test/units/conftest.py b/test/units/conftest.py index 36b5a5e92d9..fa07c76f2a5 100644 --- a/test/units/conftest.py +++ b/test/units/conftest.py @@ -6,6 +6,8 @@ import pytest import sys import typing as t +import pytest_mock + try: from ansible import _internal # sets is_controller=True in controller context from ansible.module_utils._internal import is_controller # allow checking is_controller @@ -24,6 +26,8 @@ else: from .controller_only_conftest import * # pylint: disable=wildcard-import,unused-wildcard-import +from ansible.module_utils import _internal as _module_utils_internal + def pytest_configure(config: pytest.Config): config.addinivalue_line("markers", "autoparam(value): metadata-driven parametrization") @@ -73,3 +77,9 @@ def pytest_collection_finish(session: pytest.Session): for finder in sys.meta_path: if "_AnsibleCollectionFinder" in type(finder).__name__: assert False, "a collection loader was active after collection" + + +@pytest.fixture +def as_target(mocker: pytest_mock.MockerFixture) -> None: + """Force execution in the context of a target host instead of the controller.""" + mocker.patch.object(_module_utils_internal, 'is_controller', False) diff --git a/test/units/executor/module_common/test_recursive_finder.py b/test/units/executor/module_common/test_recursive_finder.py index 3c11984df76..57b75ac90e3 100644 --- a/test/units/executor/module_common/test_recursive_finder.py +++ b/test/units/executor/module_common/test_recursive_finder.py @@ -27,12 +27,12 @@ MODULE_UTILS_BASIC_FILES = frozenset(('ansible/__init__.py', 'ansible/module_utils/basic.py', 'ansible/module_utils/six/__init__.py', 'ansible/module_utils/_internal/__init__.py', - 'ansible/module_utils/_internal/_ambient_context.py', 'ansible/module_utils/_internal/_ansiballz.py', 'ansible/module_utils/_internal/_dataclass_validation.py', 'ansible/module_utils/_internal/_datatag/__init__.py', 'ansible/module_utils/_internal/_datatag/_tags.py', 'ansible/module_utils/_internal/_debugging.py', + 'ansible/module_utils/_internal/_deprecator.py', 'ansible/module_utils/_internal/_errors.py', 'ansible/module_utils/_internal/_json/__init__.py', 'ansible/module_utils/_internal/_json/_legacy_encoder.py', @@ -41,10 +41,10 @@ MODULE_UTILS_BASIC_FILES = frozenset(('ansible/__init__.py', 'ansible/module_utils/_internal/_json/_profiles/_module_legacy_m2c.py', 'ansible/module_utils/_internal/_json/_profiles/_tagless.py', 'ansible/module_utils/_internal/_traceback.py', + 'ansible/module_utils/_internal/_validation.py', 'ansible/module_utils/_internal/_patches/_dataclass_annotation_patch.py', 'ansible/module_utils/_internal/_patches/_socket_patch.py', 'ansible/module_utils/_internal/_patches/_sys_intern_patch.py', - 'ansible/module_utils/_internal/_plugin_exec_context.py', 'ansible/module_utils/_internal/_patches/__init__.py', 'ansible/module_utils/common/collections.py', 'ansible/module_utils/common/parameters.py', diff --git a/test/units/module_utils/_internal/test_deprecator.py b/test/units/module_utils/_internal/test_deprecator.py new file mode 100644 index 00000000000..6c873c37040 --- /dev/null +++ b/test/units/module_utils/_internal/test_deprecator.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import importlib.abc +import importlib.util + +import ansible +import pathlib +import pytest + +from ansible.module_utils.common.messages import PluginInfo +from ansible.module_utils._internal import _deprecator + + +class FakePathLoader(importlib.abc.SourceLoader): + """A test loader that can fake out the code/frame paths to simulate callers of various types without relying on actual files on disk.""" + def get_filename(self, fullname): + if fullname.startswith('ansible.'): + basepath = pathlib.Path(ansible.__file__).parent.parent + else: + basepath = '/x/y' + + return f'{basepath}/{fullname.replace(".", "/")}' + + def get_data(self, path): + return b''' +from ansible.module_utils._internal import _deprecator + +def do_stuff(): + return _deprecator.get_caller_plugin_info() +''' + + def exec_module(self, module): + return super().exec_module(module) + + +@pytest.mark.parametrize("python_fq_name,expected_resolved_name,expected_plugin_type", ( + # legacy module callers + ('ansible.legacy.blah', 'ansible.legacy.blah', 'module'), + # core callers + ('ansible.modules.ping', 'ansible.builtin.ping', 'module'), + ('ansible.plugins.filters.core', _deprecator.ANSIBLE_CORE_DEPRECATOR.resolved_name, _deprecator.ANSIBLE_CORE_DEPRECATOR.type), + ('ansible.plugins.tests.core', _deprecator.ANSIBLE_CORE_DEPRECATOR.resolved_name, _deprecator.ANSIBLE_CORE_DEPRECATOR.type), + ('ansible.nonplugin_something', _deprecator.ANSIBLE_CORE_DEPRECATOR.resolved_name, _deprecator.ANSIBLE_CORE_DEPRECATOR.type), + # collections plugin callers + ('ansible_collections.foo.bar.plugins.modules.module_thing', 'foo.bar.module_thing', 'module'), + ('ansible_collections.foo.bar.plugins.filter.somefilter', 'foo.bar', PluginInfo._COLLECTION_ONLY_TYPE), + ('ansible_collections.foo.bar.plugins.test.sometest', 'foo.bar', PluginInfo._COLLECTION_ONLY_TYPE), + # indeterminate callers (e.g. collection module_utils- must specify since they might be calling on behalf of another + ('ansible_collections.foo.bar.plugins.module_utils.something', + _deprecator.INDETERMINATE_DEPRECATOR.resolved_name, _deprecator.INDETERMINATE_DEPRECATOR.type), + # other callers + ('something.else', None, None), + ('ansible_collections.foo.bar.nonplugin_something', None, None), +)) +def test_get_caller_plugin_info(python_fq_name: str, expected_resolved_name: str, expected_plugin_type: str): + """Validates the expected `PluginInfo` values received from various types of core/non-core/collection callers.""" + # invoke a standalone fake loader that generates a Python module with the specified FQ python name (converted to a corresponding __file__ entry) that + # pretends as if it called `get_caller_plugin_info()` and returns its result + loader = FakePathLoader() + spec = importlib.util.spec_from_loader(name=python_fq_name, loader=loader) + mod = importlib.util.module_from_spec(spec) + + loader.exec_module(mod) + + pi: PluginInfo = mod.do_stuff() + + if not expected_resolved_name and not expected_plugin_type: + assert pi is None + return + + assert pi is not None + assert pi.resolved_name == expected_resolved_name + assert pi.type == expected_plugin_type diff --git a/test/units/module_utils/basic/test_deprecate_warn.py b/test/units/module_utils/basic/test_deprecate_warn.py deleted file mode 100644 index c46176d1fe0..00000000000 --- a/test/units/module_utils/basic/test_deprecate_warn.py +++ /dev/null @@ -1,91 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (c) 2017 Ansible Project -# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) - -from __future__ import annotations - -import json -import typing as t - -import pytest - -from ansible.module_utils._internal._ansiballz import _ModulePluginWrapper -from ansible.module_utils._internal._plugin_exec_context import PluginExecContext -from ansible.module_utils.basic import AnsibleModule -from ansible.module_utils.common.json import Direction, get_module_decoder -from ansible.module_utils.common import warnings -from ansible.module_utils.common.messages import Detail, DeprecationSummary, WarningSummary, PluginInfo - -from units.mock.messages import make_summary - -pytestmark = pytest.mark.usefixtures("module_env_mocker") - - -@pytest.mark.parametrize('stdin', [{}], indirect=['stdin']) -def test_warn(am, capfd): - - am.warn('warning1') - - with pytest.raises(SystemExit): - am.exit_json(warnings=['warning2']) - out, err = capfd.readouterr() - actual = json.loads(out, cls=get_module_decoder('legacy', Direction.MODULE_TO_CONTROLLER))['warnings'] - expected = [make_summary(WarningSummary, Detail(msg=msg)) for msg in ['warning1', 'warning2']] - assert actual == expected - - -@pytest.mark.parametrize('kwargs,plugin_name,stdin', ( - (dict(msg='deprecation1'), None, {}), - (dict(msg='deprecation3', version='2.4'), None, {}), - (dict(msg='deprecation4', date='2020-03-10'), None, {}), - (dict(msg='deprecation5'), 'ansible.builtin.ping', {}), - (dict(msg='deprecation7', version='2.4'), 'ansible.builtin.ping', {}), - (dict(msg='deprecation8', date='2020-03-10'), 'ansible.builtin.ping', {}), -), indirect=['stdin']) -def test_deprecate(am: AnsibleModule, capfd, kwargs: dict[str, t.Any], plugin_name: str | None) -> None: - plugin_info = PluginInfo(requested_name=plugin_name, resolved_name=plugin_name, type='module') if plugin_name else None - executing_plugin = _ModulePluginWrapper(plugin_info) if plugin_info else None - collection_name = plugin_name.rpartition('.')[0] if plugin_name else None - - with PluginExecContext.when(bool(executing_plugin), executing_plugin=executing_plugin): - am.deprecate(**kwargs) - - assert warnings.get_deprecation_messages() == (dict(collection_name=collection_name, **kwargs),) - - with pytest.raises(SystemExit): - am.exit_json(deprecations=['deprecation9', ('deprecation10', '2.4')]) - - out, err = capfd.readouterr() - - output = json.loads(out, cls=get_module_decoder('legacy', Direction.MODULE_TO_CONTROLLER)) - - assert ('warnings' not in output or output['warnings'] == []) - - msg = kwargs.pop('msg') - - assert output['deprecations'] == [ - make_summary(DeprecationSummary, Detail(msg=msg), **kwargs, plugin=plugin_info), - make_summary(DeprecationSummary, Detail(msg='deprecation9'), plugin=plugin_info), - make_summary(DeprecationSummary, Detail(msg='deprecation10'), version='2.4', plugin=plugin_info), - ] - - -@pytest.mark.parametrize('stdin', [{}], indirect=['stdin']) -def test_deprecate_without_list(am, capfd): - with pytest.raises(SystemExit): - am.exit_json(deprecations='Simple deprecation warning') - - out, err = capfd.readouterr() - output = json.loads(out, cls=get_module_decoder('legacy', Direction.MODULE_TO_CONTROLLER)) - assert ('warnings' not in output or output['warnings'] == []) - assert output['deprecations'] == [ - make_summary(DeprecationSummary, Detail(msg='Simple deprecation warning')), - ] - - -@pytest.mark.parametrize('stdin', [{}], indirect=['stdin']) -def test_deprecate_without_list_version_date_not_set(am, capfd): - with pytest.raises(AssertionError) as ctx: - am.deprecate('Simple deprecation warning', date='', version='') # pylint: disable=ansible-deprecated-no-version - assert ctx.value.args[0] == "implementation error -- version and date must not both be set" diff --git a/test/units/module_utils/common/warnings/test_deprecate.py b/test/units/module_utils/common/warnings/test_deprecate.py index 417a97506c9..8566bec7880 100644 --- a/test/units/module_utils/common/warnings/test_deprecate.py +++ b/test/units/module_utils/common/warnings/test_deprecate.py @@ -6,6 +6,8 @@ from __future__ import annotations +import typing as t + import pytest from ansible.module_utils._internal import _traceback @@ -18,7 +20,7 @@ pytestmark = pytest.mark.usefixtures("module_env_mocker") def test_dedupe_with_traceback(module_env_mocker: ModuleEnvMocker) -> None: module_env_mocker.set_traceback_config([_traceback.TracebackEvent.DEPRECATED]) - deprecate_args = dict(msg="same", version="1.2.3", collection_name="blar.blar") + deprecate_args: dict[str, t.Any] = dict(msg="same", version="1.2.3", collection_name="blar.blar") # DeprecationMessageDetail dataclass object hash is the dedupe key; presence of differing tracebacks or SourceContexts affects de-dupe diff --git a/test/units/module_utils/datatag/test_datatag.py b/test/units/module_utils/datatag/test_datatag.py index bbf86332440..487d7a805a7 100644 --- a/test/units/module_utils/datatag/test_datatag.py +++ b/test/units/module_utils/datatag/test_datatag.py @@ -83,7 +83,7 @@ message_instances = [ make_summary(ErrorSummary, Detail(msg="bla"), formatted_traceback="tb"), make_summary(WarningSummary, Detail(msg="bla", formatted_source_context="sc"), formatted_traceback="tb"), make_summary(DeprecationSummary, Detail(msg="bla", formatted_source_context="sc"), formatted_traceback="tb", version="1.2.3"), - PluginInfo(requested_name='a.b.c', resolved_name='a.b.c', type='module'), + PluginInfo(resolved_name='a.b.c', type='module'), ] @@ -215,7 +215,7 @@ def test_tag_types() -> None: def test_deprecated_invalid_date_type() -> None: with pytest.raises(TypeError): - Deprecated(msg="test", removal_date="wrong") # type: ignore + Deprecated(msg="test", date=42) # type: ignore def test_tag_with_invalid_tag_type() -> None: @@ -356,8 +356,8 @@ class TestDatatagTarget(AutoParamSupport): later = t.cast(t.Self, Later(locals())) tag_instances_with_reprs: t.Annotated[t.List[t.Tuple[AnsibleDatatagBase, str]], ParamDesc(["value", "expected_repr"])] = [ - (Deprecated(msg="hi mom, I am deprecated", removal_date=datetime.date(2023, 1, 2), removal_version="42.42"), - "Deprecated(msg='hi mom, I am deprecated', removal_date='2023-01-02', removal_version='42.42')"), + (Deprecated(msg="hi mom, I am deprecated", date='2023-01-02', version="42.42"), + "Deprecated(msg='hi mom, I am deprecated', date='2023-01-02', version='42.42')"), (Deprecated(msg="minimal"), "Deprecated(msg='minimal')") ] @@ -573,8 +573,8 @@ class TestDatatagTarget(AutoParamSupport): t.List[t.Tuple[t.Type[AnsibleDatatagBase], t.Dict[str, object]]], ParamDesc(["tag_type", "init_kwargs"]) ] = [ (Deprecated, dict(msg=ExampleSingletonTag().tag(''))), - (Deprecated, dict(removal_date=ExampleSingletonTag().tag(''), msg='')), - (Deprecated, dict(removal_version=ExampleSingletonTag().tag(''), msg='')), + (Deprecated, dict(date=ExampleSingletonTag().tag(''), msg='')), + (Deprecated, dict(version=ExampleSingletonTag().tag(''), msg='')), ] @pytest.mark.autoparam(later.test_dataclass_tag_base_field_validation_fail_instances) @@ -589,8 +589,8 @@ class TestDatatagTarget(AutoParamSupport): t.List[t.Tuple[t.Type[AnsibleDatatagBase], t.Dict[str, object]]], ParamDesc(["tag_type", "init_kwargs"]) ] = [ (Deprecated, dict(msg='')), - (Deprecated, dict(msg='', removal_date=datetime.date.today())), - (Deprecated, dict(msg='', removal_version='')), + (Deprecated, dict(msg='', date='2025-01-01')), + (Deprecated, dict(msg='', version='')), ] @pytest.mark.autoparam(later.test_dataclass_tag_base_field_validation_pass_instances) diff --git a/test/units/plugins/test_plugins.py b/test/units/plugins/test_plugins.py index 1bd9aef3a2d..b86e1c13f84 100644 --- a/test/units/plugins/test_plugins.py +++ b/test/units/plugins/test_plugins.py @@ -95,9 +95,9 @@ class TestErrors(unittest.TestCase): fixture_path = os.path.join(os.path.dirname(__file__), 'loader_fixtures') pl = PluginLoader('test', '', 'test', 'test_plugin') - one = pl._load_module_source('import_fixture', os.path.join(fixture_path, 'import_fixture.py')) + one = pl._load_module_source(python_module_name='import_fixture', path=os.path.join(fixture_path, 'import_fixture.py')) # This line wouldn't even succeed if we didn't short circuit on finding a duplicate name - two = pl._load_module_source('import_fixture', '/path/to/import_fixture.py') + two = pl._load_module_source(python_module_name='import_fixture', path='/path/to/import_fixture.py') self.assertEqual(one, two) diff --git a/test/units/utils/test_display.py b/test/units/utils/test_display.py index 76384e66a07..7acedd18f44 100644 --- a/test/units/utils/test_display.py +++ b/test/units/utils/test_display.py @@ -4,14 +4,18 @@ from __future__ import annotations +import datetime import locale import sys +import typing as t import unicodedata + from unittest.mock import MagicMock import pytest -from ansible.module_utils.common.messages import Detail, WarningSummary, DeprecationSummary +from ansible.module_utils._internal import _deprecator +from ansible.module_utils.common.messages import Detail, WarningSummary, DeprecationSummary, PluginInfo from ansible.utils.display import _LIBC, _MAX_INT, Display, get_text_width, format_message from ansible.utils.multiprocessing import context as multiprocessing_context @@ -156,9 +160,9 @@ def test_format_message_deprecation_with_multiple_details() -> None: ), )) - assert result == '''Ignoring ExceptionX. This feature will be removed in a future release: Something went wrong. + assert result == '''Ignoring ExceptionX. This feature will be removed in the future: Something went wrong. -Ignoring ExceptionX. This feature will be removed in a future release. Plugins must handle it internally. +Ignoring ExceptionX. This feature will be removed in the future. Plugins must handle it internally. <<< caused by >>> @@ -168,3 +172,81 @@ Origin: /some/path ... ''' + + +A_DATE = datetime.date(2025, 1, 1) +CORE = PluginInfo._from_collection_name('ansible.builtin') +CORE_MODULE = PluginInfo(resolved_name='ansible.builtin.ping', type='module') +CORE_PLUGIN = PluginInfo(resolved_name='ansible.builtin.debug', type='action') +COLL = PluginInfo._from_collection_name('ns.col') +COLL_MODULE = PluginInfo(resolved_name='ns.col.ping', type='module') +COLL_PLUGIN = PluginInfo(resolved_name='ns.col.debug', type='action') +INDETERMINATE = _deprecator.INDETERMINATE_DEPRECATOR +LEGACY_MODULE = PluginInfo(resolved_name='ping', type='module') +LEGACY_PLUGIN = PluginInfo(resolved_name='debug', type='action') + + +@pytest.mark.parametrize('kwargs, expected', ( + # removed + (dict(msg="Hi", removed=True), "Hi. This feature was removed."), + (dict(msg="Hi", version="2.99", deprecator=CORE, removed=True), "Hi. This feature was removed from ansible-core version 2.99."), + (dict(msg="Hi", date=A_DATE, deprecator=COLL_MODULE, removed=True), + "Hi. This feature was removed from module 'ping' in collection 'ns.col' in a release after 2025-01-01."), + # no deprecator or indeterminate + (dict(msg="Hi"), "Hi. This feature will be removed in the future."), + (dict(msg="Hi", version="2.99"), "Hi. This feature will be removed in the future."), + (dict(msg="Hi", date=A_DATE), "Hi. This feature will be removed in the future."), + (dict(msg="Hi", version="2.99", deprecator=INDETERMINATE), "Hi. This feature will be removed in the future."), + (dict(msg="Hi", date=A_DATE, deprecator=INDETERMINATE), "Hi. This feature will be removed in the future."), + # deprecator without plugin + (dict(msg="Hi", deprecator=CORE), "Hi. This feature will be removed from ansible-core in a future release."), + (dict(msg="Hi", deprecator=COLL), "Hi. This feature will be removed from collection 'ns.col' in a future release."), + (dict(msg="Hi", version="2.99", deprecator=CORE), "Hi. This feature will be removed from ansible-core version 2.99."), + (dict(msg="Hi", version="2.99", deprecator=COLL), "Hi. This feature will be removed from collection 'ns.col' version 2.99."), + (dict(msg="Hi", date=A_DATE, deprecator=COLL), "Hi. This feature will be removed from collection 'ns.col' in a release after 2025-01-01."), + # deprecator with module + (dict(msg="Hi", deprecator=CORE_MODULE), "Hi. This feature will be removed from module 'ping' in ansible-core in a future release."), + (dict(msg="Hi", deprecator=COLL_MODULE), "Hi. This feature will be removed from module 'ping' in collection 'ns.col' in a future release."), + (dict(msg="Hi", deprecator=LEGACY_MODULE), "Hi. This feature will be removed from module 'ping' in the future."), + (dict(msg="Hi", version="2.99", deprecator=CORE_MODULE), "Hi. This feature will be removed from module 'ping' in ansible-core version 2.99."), + (dict(msg="Hi", version="2.99", deprecator=COLL_MODULE), "Hi. This feature will be removed from module 'ping' in collection 'ns.col' version 2.99."), + (dict(msg="Hi", version="2.99", deprecator=LEGACY_MODULE), "Hi. This feature will be removed from module 'ping' in the future."), + (dict(msg="Hi", date=A_DATE, deprecator=COLL_MODULE), + "Hi. This feature will be removed from module 'ping' in collection 'ns.col' in a release after 2025-01-01."), + (dict(msg="Hi", date=A_DATE, deprecator=LEGACY_MODULE), "Hi. This feature will be removed from module 'ping' in the future."), + # deprecator with plugin + (dict(msg="Hi", deprecator=CORE_PLUGIN), "Hi. This feature will be removed from action plugin 'debug' in ansible-core in a future release."), + (dict(msg="Hi", deprecator=COLL_PLUGIN), "Hi. This feature will be removed from action plugin 'debug' in collection 'ns.col' in a future release."), + (dict(msg="Hi", deprecator=LEGACY_PLUGIN), "Hi. This feature will be removed from action plugin 'debug' in the future."), + (dict(msg="Hi", version="2.99", deprecator=CORE_PLUGIN), "Hi. This feature will be removed from action plugin 'debug' in ansible-core version 2.99."), + (dict(msg="Hi", version="2.99", deprecator=COLL_PLUGIN), + "Hi. This feature will be removed from action plugin 'debug' in collection 'ns.col' version 2.99."), + (dict(msg="Hi", version="2.99", deprecator=LEGACY_PLUGIN), "Hi. This feature will be removed from action plugin 'debug' in the future."), + (dict(msg="Hi", date=A_DATE, deprecator=COLL_PLUGIN), + "Hi. This feature will be removed from action plugin 'debug' in collection 'ns.col' in a release after 2025-01-01."), + (dict(msg="Hi", date=A_DATE, deprecator=LEGACY_PLUGIN), "Hi. This feature will be removed from action plugin 'debug' in the future."), +)) +def test_get_deprecation_message_with_plugin_info(kwargs: dict[str, t.Any], expected: str) -> None: + for kwarg in ('version', 'date', 'deprecator'): + kwargs.setdefault(kwarg, None) + + msg = Display()._get_deprecation_message_with_plugin_info(**kwargs) + + assert msg == expected + + +@pytest.mark.parametrize("kw,expected", ( + (dict(msg="hi"), "[DEPRECATION WARNING]: hi. This feature will be removed in the future."), + (dict(msg="hi", removed=True), "[DEPRECATED]: hi. This feature was removed."), + (dict(msg="hi", version="1.23"), "[DEPRECATION WARNING]: hi. This feature will be removed in the future."), + (dict(msg="hi", date="2025-01-01"), "[DEPRECATION WARNING]: hi. This feature will be removed in the future."), + (dict(msg="hi", collection_name="foo.bar"), "[DEPRECATION WARNING]: hi. This feature will be removed from collection 'foo.bar' in a future release."), + (dict(msg="hi", version="1.23", collection_name="foo.bar"), + "[DEPRECATION WARNING]: hi. This feature will be removed from collection 'foo.bar' version 1.23."), + (dict(msg="hi", date="2025-01-01", collection_name="foo.bar"), + "[DEPRECATION WARNING]: hi. This feature will be removed from collection 'foo.bar' in a release after 2025-01-01."), +)) +def test_get_deprecation_message(kw: dict[str, t.Any], expected: str) -> None: + """Validate the deprecated public version of this function.""" + + assert Display().get_deprecation_message(**kw) == expected diff --git a/test/units/utils/test_listify.py b/test/units/utils/test_listify.py index 4f892ed43f0..61db43fa777 100644 --- a/test/units/utils/test_listify.py +++ b/test/units/utils/test_listify.py @@ -22,4 +22,7 @@ def test_listify_lookup_plugin_terms(test_input: t.Any, expected: t.Any, mocker: assert listify_lookup_plugin_terms(test_input) == expected - deprecated.assert_called_once_with(msg='"listify_lookup_plugin_terms" is obsolete and in most cases unnecessary', version='2.23') + deprecated.assert_called_once_with( + msg='"listify_lookup_plugin_terms" is obsolete and in most cases unnecessary', + version='2.23', + ) diff --git a/test/units/utils/test_serialization_profiles.py b/test/units/utils/test_serialization_profiles.py index 28f6f7aa5be..0c60c9b4f61 100644 --- a/test/units/utils/test_serialization_profiles.py +++ b/test/units/utils/test_serialization_profiles.py @@ -79,6 +79,9 @@ def test_cache_persistence_schema() -> None: """ # DTFIX-RELEASE: update tests to ensure new fields on contracts will fail this test if they have defaults which are omitted from serialization # one possibility: monkeypatch the default field value omission away so that any new field will invalidate the schema + + # DTFIX-RELEASE: ensure all types/attrs included in _profiles._common_module_response_types are represented here, since they can appear in cached responses + expected_schema_id = 1 expected_schema_hash = "bf52e60cf1d25a3f8b6bfdf734781ee07cfe46e94189d2f538815c5000b617c6"