diff --git a/changelogs/fragments/ansible-doc-jinja-builtins.yml b/changelogs/fragments/ansible-doc-jinja-builtins.yml new file mode 100644 index 00000000000..49df43b56cb --- /dev/null +++ b/changelogs/fragments/ansible-doc-jinja-builtins.yml @@ -0,0 +1,2 @@ +minor_changes: + - ansible-doc - Return dynamic stub when reporting on Jinja filters and tests not explicitly documented in Ansible diff --git a/lib/ansible/_internal/_templating/_jinja_plugins.py b/lib/ansible/_internal/_templating/_jinja_plugins.py index 7303881dd21..6ce9fe8e47e 100644 --- a/lib/ansible/_internal/_templating/_jinja_plugins.py +++ b/lib/ansible/_internal/_templating/_jinja_plugins.py @@ -6,8 +6,12 @@ import collections.abc as c import dataclasses import datetime import functools +import inspect +import re import typing as t +from jinja2 import defaults + from ansible.module_utils._internal._ambient_context import AmbientContextBase from ansible.module_utils.common.collections import is_sequence from ansible.module_utils._internal._datatag import AnsibleTagHelper @@ -338,3 +342,28 @@ def _wrap_plugin_output(o: t.Any) -> t.Any: o = list(o) return _AnsibleLazyTemplateMixin._try_create(o, LazyOptions.SKIP_TEMPLATES) + + +_PLUGIN_SOURCES = dict( + filter=defaults.DEFAULT_FILTERS, + test=defaults.DEFAULT_TESTS, +) + + +def _get_builtin_short_description(plugin: object) -> str: + """ + Make a reasonable effort to break a function docstring down to a single sentence. + We can't use the full docstring due to embedded formatting, particularly RST. + This isn't intended to be perfect, just good enough until we can write our own docs for these. + """ + value = re.split(r'(\.|!|\s\(|:\s)', inspect.getdoc(plugin), 1)[0].replace('\n', ' ') + + if value: + value += '.' + + return value + + +def get_jinja_builtin_plugin_descriptions(plugin_type: str) -> dict[str, str]: + """Returns a dictionary of Jinja builtin plugin names and their short descriptions.""" + return {f'ansible.builtin.{name}': _get_builtin_short_description(plugin) for name, plugin in _PLUGIN_SOURCES[plugin_type].items() if name.isidentifier()} diff --git a/lib/ansible/cli/doc.py b/lib/ansible/cli/doc.py index c5c23120607..8f0018d009a 100755 --- a/lib/ansible/cli/doc.py +++ b/lib/ansible/cli/doc.py @@ -9,12 +9,14 @@ from __future__ import annotations # ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first from ansible.cli import CLI +import collections.abc import importlib import pkgutil import os import os.path import re import textwrap +import typing as t import yaml @@ -35,7 +37,7 @@ from ansible.parsing.plugin_docs import read_docstub from ansible.parsing.yaml.dumper import AnsibleDumper from ansible.parsing.yaml.loader import AnsibleLoader from ansible._internal._yaml._loader import AnsibleInstrumentedLoader -from ansible.plugins.list import list_plugins +from ansible.plugins.list import _list_plugins_with_info, _PluginDocMetadata from ansible.plugins.loader import action_loader, fragment_loader from ansible.utils.collection_loader import AnsibleCollectionConfig, AnsibleCollectionRef from ansible.utils.collection_loader._collection_finder import _get_collection_name_from_path @@ -44,6 +46,7 @@ from ansible.utils.display import Display from ansible.utils.plugin_docs import get_plugin_docs, get_docstring, get_versioned_doclink from ansible.template import trust_as_template from ansible._internal import _json +from ansible._internal._templating import _jinja_plugins display = Display() @@ -788,41 +791,47 @@ class DocCLI(CLI, RoleMixin): return coll_filter def _list_plugins(self, plugin_type, content): - - results = {} - self.plugins = {} - loader = DocCLI._prep_loader(plugin_type) + DocCLI._prep_loader(plugin_type) coll_filter = self._get_collection_filter() - plugin_list = list_plugins(plugin_type, coll_filter) + plugins = _list_plugins_with_info(plugin_type, coll_filter) # Remove the internal ansible._protomatter plugins if getting all plugins if not coll_filter: - plugin_list = {k: v for k, v in plugin_list.items() if not k.startswith('ansible._protomatter.')} - - self.plugins.update(plugin_list) + plugins = {k: v for k, v in plugins.items() if not k.startswith('ansible._protomatter.')} # get appropriate content depending on option if content == 'dir': - results = self._get_plugin_list_descriptions(loader) + results = self._get_plugin_list_descriptions(plugins) elif content == 'files': - results = {k: self.plugins[k][0] for k in self.plugins.keys()} + results = {k: v.path for k, v in plugins.items()} else: - results = {k: {} for k in self.plugins.keys()} + results = {k: {} for k in plugins.keys()} self.plugin_list = set() # reset for next iteration return results - def _get_plugins_docs(self, plugin_type, names, fail_ok=False, fail_on_errors=True): - + def _get_plugins_docs(self, plugin_type: str, names: collections.abc.Iterable[str], fail_ok: bool = False, fail_on_errors: bool = True) -> dict[str, dict]: loader = DocCLI._prep_loader(plugin_type) + if plugin_type in ('filter', 'test'): + jinja2_builtins = _jinja_plugins.get_jinja_builtin_plugin_descriptions(plugin_type) + jinja2_builtins.update({name.split('.')[-1]: value for name, value in jinja2_builtins.items()}) # add short-named versions for lookup + else: + jinja2_builtins = {} + # get the docs for plugins in the command line list plugin_docs = {} for plugin in names: - doc = {} + doc: dict[str, t.Any] = {} try: - doc, plainexamples, returndocs, metadata = get_plugin_docs(plugin, plugin_type, loader, fragment_loader, (context.CLIARGS['verbosity'] > 0)) + doc, plainexamples, returndocs, metadata = self._get_plugin_docs_with_jinja2_builtins( + plugin, + plugin_type, + loader, + fragment_loader, + jinja2_builtins, + ) except AnsiblePluginNotFound as e: display.warning(to_native(e)) continue @@ -859,6 +868,39 @@ class DocCLI(CLI, RoleMixin): return plugin_docs + def _get_plugin_docs_with_jinja2_builtins( + self, + plugin_name: str, + plugin_type: str, + loader: t.Any, + fragment_loader: t.Any, + jinja_builtins: dict[str, str], + ) -> tuple[dict, str | None, dict | None, dict | None]: + try: + return get_plugin_docs(plugin_name, plugin_type, loader, fragment_loader, (context.CLIARGS['verbosity'] > 0)) + except Exception: + if (desc := jinja_builtins.get(plugin_name, ...)) is not ...: + short_name = plugin_name.split('.')[-1] + long_name = f'ansible.builtin.{short_name}' + # Dynamically build a doc stub for any Jinja2 builtin plugin we haven't + # explicitly documented. + doc = dict( + collection='ansible.builtin', + plugin_name=long_name, + filename='', + short_description=desc, + description=[ + desc, + '', + f"This is the Jinja builtin {plugin_type} plugin {short_name!r}.", + f"See: U(https://jinja.palletsprojects.com/en/stable/templates/#jinja-{plugin_type}s.{short_name})", + ], + ) + + return doc, None, None, None + + raise + def _get_roles_path(self): """ Add any 'roles' subdir in playbook dir to the roles search path. @@ -1007,10 +1049,10 @@ class DocCLI(CLI, RoleMixin): def get_all_plugins_of_type(plugin_type): loader = getattr(plugin_loader, '%s_loader' % plugin_type) paths = loader._get_paths_with_context() - plugins = {} + plugins = [] for path_context in paths: - plugins.update(list_plugins(plugin_type)) - return sorted(plugins.keys()) + plugins += _list_plugins_with_info(plugin_type).keys() + return sorted(plugins) @staticmethod def get_plugin_metadata(plugin_type, plugin_name): @@ -1107,18 +1149,20 @@ class DocCLI(CLI, RoleMixin): return text - def _get_plugin_list_descriptions(self, loader): + def _get_plugin_list_descriptions(self, plugins: dict[str, _PluginDocMetadata]) -> dict[str, str]: descs = {} - for plugin in self.plugins.keys(): + for plugin, plugin_info in plugins.items(): # TODO: move to plugin itself i.e: plugin.get_desc() doc = None - filename = Path(to_native(self.plugins[plugin][0])) + docerror = None - try: - doc = read_docstub(filename) - except Exception as e: - docerror = e + if plugin_info.path: + filename = Path(to_native(plugin_info.path)) + try: + doc = read_docstub(filename) + except Exception as e: + docerror = e # plugin file was empty or had error, lets try other options if doc is None: @@ -1133,9 +1177,15 @@ class DocCLI(CLI, RoleMixin): except Exception as e: docerror = e - if docerror: - display.warning("%s has a documentation formatting error: %s" % (plugin, docerror)) - continue + # Do a final fallback to see if the plugin is a shadowed Jinja2 plugin + # without any explicit documentation. + if doc is None and plugin_info.jinja_builtin_short_description: + descs[plugin] = plugin_info.jinja_builtin_short_description + continue + + if docerror: + display.error_as_warning(f"{plugin} has a documentation formatting error.", exception=docerror) + continue if not doc or not isinstance(doc, dict): desc = 'UNDOCUMENTED' diff --git a/lib/ansible/plugins/filter/core.py b/lib/ansible/plugins/filter/core.py index 6afd3045fd7..3fddb7f413d 100644 --- a/lib/ansible/plugins/filter/core.py +++ b/lib/ansible/plugins/filter/core.py @@ -816,7 +816,6 @@ class FilterModule(object): 'groupby': _cleansed_groupby, # Jinja builtins that need special arg handling - # DTFIX1: document these now that they're overridden, or hide them so they don't show up as undocumented 'd': ansible_default, # replaces the implementation instead of wrapping it 'default': ansible_default, # replaces the implementation instead of wrapping it 'map': wrapped_map, diff --git a/lib/ansible/plugins/list.py b/lib/ansible/plugins/list.py index a5a877ec99b..b040bbd4004 100644 --- a/lib/ansible/plugins/list.py +++ b/lib/ansible/plugins/list.py @@ -4,6 +4,7 @@ from __future__ import annotations +import dataclasses import os from ansible import context @@ -14,6 +15,7 @@ from ansible.module_utils.common.text.converters import to_native, to_bytes from ansible.plugins import loader from ansible.utils.display import Display from ansible.utils.collection_loader._collection_finder import _get_collection_path +from ansible._internal._templating._jinja_plugins import get_jinja_builtin_plugin_descriptions display = Display() @@ -25,6 +27,20 @@ IGNORE = { } +@dataclasses.dataclass(kw_only=True, frozen=True, slots=True) +class _PluginDocMetadata: + """Information about a plugin.""" + + name: str + """The fully qualified name of the plugin.""" + path: bytes | None = None + """The path to the plugin file, or None if not available.""" + plugin_obj: object | None = None + """The loaded plugin object, or None if not loaded.""" + jinja_builtin_short_description: str | None = None + """The short description of the plugin if it is a Jinja builtin, otherwise None.""" + + def get_composite_name(collection, name, path, depth): resolved_collection = collection if '.' not in name: @@ -116,21 +132,37 @@ def _list_j2_plugins_from_file(collection, plugin_path, ptype, plugin_name): return file_plugins -def list_collection_plugins(ptype, collections, search_paths=None): +def list_collection_plugins(ptype: str, collections: dict[str, bytes], search_paths: list[str] | None = None) -> dict[str, tuple[bytes, object | None]]: + # Kept for backwards compatibility. + return { + name: (info.path, info.plugin_obj) + for name, info in _list_collection_plugins_with_info(ptype, collections).items() + } + + +def _list_collection_plugins_with_info( + ptype: str, + collections: dict[str, bytes], +) -> dict[str, _PluginDocMetadata]: # TODO: update to use importlib.resources - # starts at {plugin_name: filepath, ...}, but changes at the end - plugins = {} try: ploader = getattr(loader, '{0}_loader'.format(ptype)) except AttributeError: raise AnsibleError(f"Cannot list plugins, incorrect plugin type {ptype!r} supplied.") from None + builtin_jinja_plugins = {} + plugin_paths = {} + # get plugins for each collection - for collection in collections.keys(): + for collection, path in collections.items(): if collection == 'ansible.builtin': # dirs from ansible install, but not configured paths dirs = [d.path for d in ploader._get_paths_with_context() if d.internal] + + if ptype in ('filter', 'test'): + builtin_jinja_plugins = get_jinja_builtin_plugin_descriptions(ptype) + elif collection == 'ansible.legacy': # configured paths + search paths (should include basedirs/-M) dirs = [d.path for d in ploader._get_paths_with_context() if not d.internal] @@ -139,7 +171,7 @@ def list_collection_plugins(ptype, collections, search_paths=None): else: # search path in this case is for locating collection itselfA b_ptype = to_bytes(C.COLLECTION_PTYPE_COMPAT.get(ptype, ptype)) - dirs = [to_native(os.path.join(collections[collection], b'plugins', b_ptype))] + dirs = [to_native(os.path.join(path, b'plugins', b_ptype))] # acr = AnsibleCollectionRef.try_parse_fqcr(collection, ptype) # if acr: # dirs = acr.subdirs @@ -147,30 +179,51 @@ def list_collection_plugins(ptype, collections, search_paths=None): # raise Exception('bad acr for %s, %s' % (collection, ptype)) - plugins.update(_list_plugins_from_paths(ptype, dirs, collection)) + plugin_paths.update(_list_plugins_from_paths(ptype, dirs, collection)) - # return plugin and it's class object, None for those not verifiable or failing + plugins = {} if ptype in ('module',): # no 'invalid' tests for modules - for plugin in plugins.keys(): - plugins[plugin] = (plugins[plugin], None) + for plugin, plugin_path in plugin_paths.items(): + plugins[plugin] = _PluginDocMetadata(name=plugin, path=plugin_path) else: # detect invalid plugin candidates AND add loaded object to return data - for plugin in list(plugins.keys()): + for plugin, plugin_path in plugin_paths.items(): pobj = None try: pobj = ploader.get(plugin, class_only=True) except Exception as e: - display.vvv("The '{0}' {1} plugin could not be loaded from '{2}': {3}".format(plugin, ptype, plugins[plugin], to_native(e))) + display.vvv("The '{0}' {1} plugin could not be loaded from '{2}': {3}".format(plugin, ptype, plugin_path, to_native(e))) - # sets final {plugin_name: (filepath, class|NONE if not loaded), ...} - plugins[plugin] = (plugins[plugin], pobj) + plugins[plugin] = _PluginDocMetadata( + name=plugin, + path=plugin_path, + plugin_obj=pobj, + jinja_builtin_short_description=builtin_jinja_plugins.get(plugin), + ) + + # Add in any builtin Jinja2 plugins that have not been shadowed in Ansible. + plugins.update( + (plugin_name, _PluginDocMetadata(name=plugin_name, jinja_builtin_short_description=plugin_description)) + for plugin_name, plugin_description in builtin_jinja_plugins.items() if plugin_name not in plugins + ) - # {plugin_name: (filepath, class), ...} return plugins -def list_plugins(ptype, collections=None, search_paths=None): +def list_plugins(ptype: str, collections: list[str] | None = None, search_paths: list[str] | None = None) -> dict[str, tuple[bytes, object | None]]: + # Kept for backwards compatibility. + return { + name: (info.path, info.plugin_obj) + for name, info in _list_plugins_with_info(ptype, collections, search_paths).items() + } + + +def _list_plugins_with_info( + ptype: str, + collections: list[str] = None, + search_paths: list[str] | None = None, +) -> dict[str, _PluginDocMetadata]: if isinstance(collections, str): collections = [collections] @@ -195,7 +248,7 @@ def list_plugins(ptype, collections=None, search_paths=None): raise AnsibleError(f"Cannot use supplied collection {collection!r}.") from ex if plugin_collections: - plugins.update(list_collection_plugins(ptype, plugin_collections)) + plugins.update(_list_collection_plugins_with_info(ptype, plugin_collections)) return plugins diff --git a/test/integration/targets/ansible-doc/test.yml b/test/integration/targets/ansible-doc/test.yml index 685a09b0df9..1ea77869f75 100644 --- a/test/integration/targets/ansible-doc/test.yml +++ b/test/integration/targets/ansible-doc/test.yml @@ -207,3 +207,120 @@ - assert: that: - result.stdout_lines | select("match", "^ansible\\._protomatter\\.") | length == result.stdout_lines | length + + - name: List all filter plugins without any filter + command: ansible-doc -t filter --list --json + register: filter_list_all_raw + + - set_fact: + filter_list_all: "{{ filter_list_all_raw.stdout | from_json }}" + + - name: Check Jinja2 filters have a short description on list without any filter + assert: + that: + - filter_list_all['ansible.builtin.default'] is contains("the passed default value") + - filter_list_all['ansible.builtin.sum'] is contains("sum of a sequence") + + - name: List only ansible.builtin filter plugins + command: ansible-doc ansible.builtin -t filter --list --json + register: filter_list_builtin_raw + + - set_fact: + filter_list_builtin: "{{ filter_list_builtin_raw.stdout | from_json }}" + + - name: Check Jinja2 filters have a short description on list with builtin filter + assert: + that: + - filter_list_builtin['ansible.builtin.default'] is contains("the passed default value") + - filter_list_builtin['ansible.builtin.sum'] is contains("sum of a sequence") + + - name: List only ansible.builtin filter plugin files + command: ansible-doc ansible.builtin -t filter --list_files + register: filter_list_builtin_files + + - name: Verify files for Jinja2 filters + assert: + that: + - filter_list_builtin_files.stdout_lines | select("match", "^ansible\\.builtin\\.default\\s+.*/core\\.py") | length == 1 + - filter_list_builtin_files.stdout_lines | select("match", "^ansible\\.builtin\\.sum\\s+None") | length == 1 + + - name: Get jinja2 filter docs shadowed by Ansible + command: ansible-doc ansible.builtin.default -t filter + register: filter_jinja2_default + + - name: Assert jinja2 filter docs shadowed by Ansible + assert: + that: + - filter_jinja2_default.stdout is search("\\s+FILTER ansible.builtin.default\\s+") + - filter_jinja2_default.stdout is search("the Jinja builtin filter plugin 'default'") + - filter_jinja2_default.stdout is search("the passed default value") + - filter_jinja2_default.stdout is search("https://jinja\\.palletsprojects\\.com/.*") + + - name: Get jinja2 filter docs not shadowed by Ansible + command: ansible-doc ansible.builtin.sum -t filter + register: filter_jinja2_sum + + - name: Assert jinja2 filter docs not shadowed by Ansible + assert: + that: + - filter_jinja2_sum.stdout is search("\\s+FILTER ansible.builtin.sum\\s+") + - filter_jinja2_sum.stdout is search("the Jinja builtin filter plugin 'sum'") + - filter_jinja2_sum.stdout is search("sum of a sequence") + - filter_jinja2_sum.stdout is search("https://jinja\\.palletsprojects\\.com/.*") + + - name: List all test plugins without any filter + command: ansible-doc -t test --list --json + register: test_list_all_raw + + - set_fact: + test_list_all: "{{ test_list_all_raw.stdout | from_json }}" + + - name: Check Jinja2 tests have a short description on list without any filter + assert: + that: + - test_list_all['ansible.builtin.number'] is contains("is a number") + + - name: List only ansible.builtin test plugins + command: ansible-doc ansible.builtin -t test --list --json + register: test_list_builtin_raw + + - set_fact: + test_list_builtin: "{{ test_list_builtin_raw.stdout | from_json }}" + + - name: Check Jinja2 tests have a short description on list with builtin tests + assert: + that: + - test_list_builtin['ansible.builtin.number'] is contains("is a number") + + - name: List only ansible.builtin test plugin files + command: ansible-doc ansible.builtin -t test --list_files + register: test_list_builtin_files + + - name: Verify files for Jinja2 filters + assert: + that: + - test_list_builtin_files.stdout_lines | select("match", "^ansible\\.builtin\\.number\\s+None") | length == 1 + + - name: Get jinja2 test docs + command: ansible-doc ansible.builtin.number -t test + register: test_jinja2_number + + - name: Assert jinja2 test docs + assert: + that: + - test_jinja2_number.stdout is search("\\s+TEST ansible.builtin.number\\s+") + - test_jinja2_number.stdout is search("the Jinja builtin test plugin 'number'") + - test_jinja2_number.stdout is contains("is a number") + - test_jinja2_number.stdout is search("https://jinja\\.palletsprojects\\.com/.*") + + - name: Get jinja2 test docs for an unqualified builtin + command: ansible-doc number -t test + register: test_jinja2_number_short + + - name: Assert jinja2 test docs + assert: + that: + - test_jinja2_number_short.stdout is search("\\s+TEST ansible.builtin.number\\s+") + - test_jinja2_number_short.stdout is search("the Jinja builtin test plugin 'number'") + - test_jinja2_number_short.stdout is contains("is a number") + - test_jinja2_number_short.stdout is search("https://jinja\\.palletsprojects\\.com/.*")