doc - Dynamically document jinja builtins (#85215)

* doc - Dynamically document jinja builtins

This change has `ansible-doc` dynamically generate the documentation for
any Jinja builtin filter and test plugins. These dynamic stubs will
point to the official Jinja documentation pages for more information.

* Fix sanity issues

* Add tests

* Update Jinja builtin doc gen

Co-authored-by: Matt Clay <matt@mystile.com>

---------

Co-authored-by: Matt Davis <nitzmahone@redhat.com>
Co-authored-by: Matt Clay <matt@mystile.com>
(cherry picked from commit 8f2622c39f)

changelogs/fragments/ansible-doc-jinja-builtins.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - ansible-doc - Return dynamic stub when reporting on Jinja filters and tests not explicitly documented in Ansible

lib/ansible/_internal/_templating/_jinja_plugins.py

@@ -6,8 +6,12 @@ import collections.abc as c
 import dataclasses
 import datetime
 import functools
+import inspect
+import re
 import typing as t
 
+from jinja2 import defaults
+
 from ansible.module_utils._internal._ambient_context import AmbientContextBase
 from ansible.module_utils.common.collections import is_sequence
 from ansible.module_utils._internal._datatag import AnsibleTagHelper
@@ -338,3 +342,28 @@ def _wrap_plugin_output(o: t.Any) -> t.Any:
         o = list(o)
 
     return _AnsibleLazyTemplateMixin._try_create(o, LazyOptions.SKIP_TEMPLATES)
+
+
+_PLUGIN_SOURCES = dict(
+    filter=defaults.DEFAULT_FILTERS,
+    test=defaults.DEFAULT_TESTS,
+)
+
+
+def _get_builtin_short_description(plugin: object) -> str:
+    """
+    Make a reasonable effort to break a function docstring down to a single sentence.
+    We can't use the full docstring due to embedded formatting, particularly RST.
+    This isn't intended to be perfect, just good enough until we can write our own docs for these.
+    """
+    value = re.split(r'(\.|!|\s\(|:\s)', inspect.getdoc(plugin), 1)[0].replace('\n', ' ')
+
+    if value:
+        value += '.'
+
+    return value
+
+
+def get_jinja_builtin_plugin_descriptions(plugin_type: str) -> dict[str, str]:
+    """Returns a dictionary of Jinja builtin plugin names and their short descriptions."""
+    return {f'ansible.builtin.{name}': _get_builtin_short_description(plugin) for name, plugin in _PLUGIN_SOURCES[plugin_type].items() if name.isidentifier()}

lib/ansible/cli/doc.py

@@ -9,12 +9,14 @@ from __future__ import annotations
 # ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first
 from ansible.cli import CLI
 
+import collections.abc
 import importlib
 import pkgutil
 import os
 import os.path
 import re
 import textwrap
+import typing as t
 
 import yaml
 
@@ -35,7 +37,7 @@ from ansible.parsing.plugin_docs import read_docstub
 from ansible.parsing.yaml.dumper import AnsibleDumper
 from ansible.parsing.yaml.loader import AnsibleLoader
 from ansible._internal._yaml._loader import AnsibleInstrumentedLoader
-from ansible.plugins.list import list_plugins
+from ansible.plugins.list import _list_plugins_with_info, _PluginDocMetadata
 from ansible.plugins.loader import action_loader, fragment_loader
 from ansible.utils.collection_loader import AnsibleCollectionConfig, AnsibleCollectionRef
 from ansible.utils.collection_loader._collection_finder import _get_collection_name_from_path
@@ -44,6 +46,7 @@ from ansible.utils.display import Display
 from ansible.utils.plugin_docs import get_plugin_docs, get_docstring, get_versioned_doclink
 from ansible.template import trust_as_template
 from ansible._internal import _json
+from ansible._internal._templating import _jinja_plugins
 
 display = Display()
 
@@ -788,41 +791,47 @@ class DocCLI(CLI, RoleMixin):
         return coll_filter
 
     def _list_plugins(self, plugin_type, content):
-
+        DocCLI._prep_loader(plugin_type)
-        results = {}
-        self.plugins = {}
-        loader = DocCLI._prep_loader(plugin_type)
 
         coll_filter = self._get_collection_filter()
-        plugin_list = list_plugins(plugin_type, coll_filter)
+        plugins = _list_plugins_with_info(plugin_type, coll_filter)
 
         # Remove the internal ansible._protomatter plugins if getting all plugins
         if not coll_filter:
-            plugin_list = {k: v for k, v in plugin_list.items() if not k.startswith('ansible._protomatter.')}
+            plugins = {k: v for k, v in plugins.items() if not k.startswith('ansible._protomatter.')}
-
-        self.plugins.update(plugin_list)
 
         # get appropriate content depending on option
         if content == 'dir':
-            results = self._get_plugin_list_descriptions(loader)
+            results = self._get_plugin_list_descriptions(plugins)
         elif content == 'files':
-            results = {k: self.plugins[k][0] for k in self.plugins.keys()}
+            results = {k: v.path for k, v in plugins.items()}
         else:
-            results = {k: {} for k in self.plugins.keys()}
+            results = {k: {} for k in plugins.keys()}
             self.plugin_list = set()  # reset for next iteration
 
         return results
 
-    def _get_plugins_docs(self, plugin_type, names, fail_ok=False, fail_on_errors=True):
+    def _get_plugins_docs(self, plugin_type: str, names: collections.abc.Iterable[str], fail_ok: bool = False, fail_on_errors: bool = True) -> dict[str, dict]:
-
         loader = DocCLI._prep_loader(plugin_type)
 
+        if plugin_type in ('filter', 'test'):
+            jinja2_builtins = _jinja_plugins.get_jinja_builtin_plugin_descriptions(plugin_type)
+            jinja2_builtins.update({name.split('.')[-1]: value for name, value in jinja2_builtins.items()})  # add short-named versions for lookup
+        else:
+            jinja2_builtins = {}
+
         # get the docs for plugins in the command line list
         plugin_docs = {}
         for plugin in names:
-            doc = {}
+            doc: dict[str, t.Any] = {}
             try:
-                doc, plainexamples, returndocs, metadata = get_plugin_docs(plugin, plugin_type, loader, fragment_loader, (context.CLIARGS['verbosity'] > 0))
+                doc, plainexamples, returndocs, metadata = self._get_plugin_docs_with_jinja2_builtins(
+                    plugin,
+                    plugin_type,
+                    loader,
+                    fragment_loader,
+                    jinja2_builtins,
+                )
             except AnsiblePluginNotFound as e:
                 display.warning(to_native(e))
                 continue
@@ -859,6 +868,39 @@ class DocCLI(CLI, RoleMixin):
 
         return plugin_docs
 
+    def _get_plugin_docs_with_jinja2_builtins(
+        self,
+        plugin_name: str,
+        plugin_type: str,
+        loader: t.Any,
+        fragment_loader: t.Any,
+        jinja_builtins: dict[str, str],
+    ) -> tuple[dict, str | None, dict | None, dict | None]:
+        try:
+            return get_plugin_docs(plugin_name, plugin_type, loader, fragment_loader, (context.CLIARGS['verbosity'] > 0))
+        except Exception:
+            if (desc := jinja_builtins.get(plugin_name, ...)) is not ...:
+                short_name = plugin_name.split('.')[-1]
+                long_name = f'ansible.builtin.{short_name}'
+                # Dynamically build a doc stub for any Jinja2 builtin plugin we haven't
+                # explicitly documented.
+                doc = dict(
+                    collection='ansible.builtin',
+                    plugin_name=long_name,
+                    filename='',
+                    short_description=desc,
+                    description=[
+                        desc,
+                        '',
+                        f"This is the Jinja builtin {plugin_type} plugin {short_name!r}.",
+                        f"See: U(https://jinja.palletsprojects.com/en/stable/templates/#jinja-{plugin_type}s.{short_name})",
+                    ],
+                )
+
+                return doc, None, None, None
+
+            raise
+
     def _get_roles_path(self):
         """
          Add any 'roles' subdir in playbook dir to the roles search path.
@@ -1007,10 +1049,10 @@ class DocCLI(CLI, RoleMixin):
     def get_all_plugins_of_type(plugin_type):
         loader = getattr(plugin_loader, '%s_loader' % plugin_type)
         paths = loader._get_paths_with_context()
-        plugins = {}
+        plugins = []
         for path_context in paths:
-            plugins.update(list_plugins(plugin_type))
+            plugins += _list_plugins_with_info(plugin_type).keys()
-        return sorted(plugins.keys())
+        return sorted(plugins)
 
     @staticmethod
     def get_plugin_metadata(plugin_type, plugin_name):
@@ -1107,14 +1149,16 @@ class DocCLI(CLI, RoleMixin):
 
         return text
 
-    def _get_plugin_list_descriptions(self, loader):
+    def _get_plugin_list_descriptions(self, plugins: dict[str, _PluginDocMetadata]) -> dict[str, str]:
 
         descs = {}
-        for plugin in self.plugins.keys():
+        for plugin, plugin_info in plugins.items():
             # TODO: move to plugin itself i.e: plugin.get_desc()
             doc = None
-            filename = Path(to_native(self.plugins[plugin][0]))
+
             docerror = None
+            if plugin_info.path:
+                filename = Path(to_native(plugin_info.path))
                 try:
                     doc = read_docstub(filename)
                 except Exception as e:
@@ -1133,8 +1177,14 @@ class DocCLI(CLI, RoleMixin):
                     except Exception as e:
                         docerror = e
 
+                # Do a final fallback to see if the plugin is a shadowed Jinja2 plugin
+                # without any explicit documentation.
+                if doc is None and plugin_info.jinja_builtin_short_description:
+                    descs[plugin] = plugin_info.jinja_builtin_short_description
+                    continue
+
                 if docerror:
-                display.warning("%s has a documentation formatting error: %s" % (plugin, docerror))
+                    display.error_as_warning(f"{plugin} has a documentation formatting error.", exception=docerror)
                     continue
 
             if not doc or not isinstance(doc, dict):

lib/ansible/plugins/filter/core.py

@@ -816,7 +816,6 @@ class FilterModule(object):
             'groupby': _cleansed_groupby,
 
             # Jinja builtins that need special arg handling
-            # DTFIX1: document these now that they're overridden, or hide them so they don't show up as undocumented
             'd': ansible_default,  # replaces the implementation instead of wrapping it
             'default': ansible_default,  # replaces the implementation instead of wrapping it
             'map': wrapped_map,

lib/ansible/plugins/list.py

@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 
+import dataclasses
 import os
 
 from ansible import context
@@ -14,6 +15,7 @@ from ansible.module_utils.common.text.converters import to_native, to_bytes
 from ansible.plugins import loader
 from ansible.utils.display import Display
 from ansible.utils.collection_loader._collection_finder import _get_collection_path
+from ansible._internal._templating._jinja_plugins import get_jinja_builtin_plugin_descriptions
 
 display = Display()
 
@@ -25,6 +27,20 @@ IGNORE = {
 }
 
 
+@dataclasses.dataclass(kw_only=True, frozen=True, slots=True)
+class _PluginDocMetadata:
+    """Information about a plugin."""
+
+    name: str
+    """The fully qualified name of the plugin."""
+    path: bytes | None = None
+    """The path to the plugin file, or None if not available."""
+    plugin_obj: object | None = None
+    """The loaded plugin object, or None if not loaded."""
+    jinja_builtin_short_description: str | None = None
+    """The short description of the plugin if it is a Jinja builtin, otherwise None."""
+
+
 def get_composite_name(collection, name, path, depth):
     resolved_collection = collection
     if '.' not in name:
@@ -116,21 +132,37 @@ def _list_j2_plugins_from_file(collection, plugin_path, ptype, plugin_name):
     return file_plugins
 
 
-def list_collection_plugins(ptype, collections, search_paths=None):
+def list_collection_plugins(ptype: str, collections: dict[str, bytes], search_paths: list[str] | None = None) -> dict[str, tuple[bytes, object | None]]:
+    # Kept for backwards compatibility.
+    return {
+        name: (info.path, info.plugin_obj)
+        for name, info in _list_collection_plugins_with_info(ptype, collections).items()
+    }
+
+
+def _list_collection_plugins_with_info(
+    ptype: str,
+    collections: dict[str, bytes],
+) -> dict[str, _PluginDocMetadata]:
     # TODO: update to use importlib.resources
 
-    # starts at  {plugin_name: filepath, ...}, but changes at the end
-    plugins = {}
     try:
         ploader = getattr(loader, '{0}_loader'.format(ptype))
     except AttributeError:
         raise AnsibleError(f"Cannot list plugins, incorrect plugin type {ptype!r} supplied.") from None
 
+    builtin_jinja_plugins = {}
+    plugin_paths = {}
+
     # get plugins for each collection
-    for collection in collections.keys():
+    for collection, path in collections.items():
         if collection == 'ansible.builtin':
             # dirs from ansible install, but not configured paths
             dirs = [d.path for d in ploader._get_paths_with_context() if d.internal]
+
+            if ptype in ('filter', 'test'):
+                builtin_jinja_plugins = get_jinja_builtin_plugin_descriptions(ptype)
+
         elif collection == 'ansible.legacy':
             # configured paths + search paths (should include basedirs/-M)
             dirs = [d.path for d in ploader._get_paths_with_context() if not d.internal]
@@ -139,7 +171,7 @@ def list_collection_plugins(ptype, collections, search_paths=None):
         else:
             # search path in this case is for locating collection itselfA
             b_ptype = to_bytes(C.COLLECTION_PTYPE_COMPAT.get(ptype, ptype))
-            dirs = [to_native(os.path.join(collections[collection], b'plugins', b_ptype))]
+            dirs = [to_native(os.path.join(path, b'plugins', b_ptype))]
             # acr = AnsibleCollectionRef.try_parse_fqcr(collection, ptype)
             # if acr:
             #     dirs = acr.subdirs
@@ -147,30 +179,51 @@ def list_collection_plugins(ptype, collections, search_paths=None):
 
             #     raise Exception('bad acr for %s, %s' % (collection, ptype))
 
-        plugins.update(_list_plugins_from_paths(ptype, dirs, collection))
+        plugin_paths.update(_list_plugins_from_paths(ptype, dirs, collection))
 
-    #  return plugin and it's class object, None for those not verifiable or failing
+    plugins = {}
     if ptype in ('module',):
         # no 'invalid' tests for modules
-        for plugin in plugins.keys():
+        for plugin, plugin_path in plugin_paths.items():
-            plugins[plugin] = (plugins[plugin], None)
+            plugins[plugin] = _PluginDocMetadata(name=plugin, path=plugin_path)
     else:
         # detect invalid plugin candidates AND add loaded object to return data
-        for plugin in list(plugins.keys()):
+        for plugin, plugin_path in plugin_paths.items():
             pobj = None
             try:
                 pobj = ploader.get(plugin, class_only=True)
             except Exception as e:
-                display.vvv("The '{0}' {1} plugin could not be loaded from '{2}': {3}".format(plugin, ptype, plugins[plugin], to_native(e)))
+                display.vvv("The '{0}' {1} plugin could not be loaded from '{2}': {3}".format(plugin, ptype, plugin_path, to_native(e)))
 
-            # sets final {plugin_name: (filepath, class|NONE if not loaded), ...}
+            plugins[plugin] = _PluginDocMetadata(
-            plugins[plugin] = (plugins[plugin], pobj)
+                name=plugin,
+                path=plugin_path,
+                plugin_obj=pobj,
+                jinja_builtin_short_description=builtin_jinja_plugins.get(plugin),
+            )
+
+        # Add in any builtin Jinja2 plugins that have not been shadowed in Ansible.
+        plugins.update(
+            (plugin_name, _PluginDocMetadata(name=plugin_name, jinja_builtin_short_description=plugin_description))
+            for plugin_name, plugin_description in builtin_jinja_plugins.items() if plugin_name not in plugins
+        )
 
-    # {plugin_name: (filepath, class), ...}
     return plugins
 
 
-def list_plugins(ptype, collections=None, search_paths=None):
+def list_plugins(ptype: str, collections: list[str] | None = None, search_paths: list[str] | None = None) -> dict[str, tuple[bytes, object | None]]:
+    # Kept for backwards compatibility.
+    return {
+        name: (info.path, info.plugin_obj)
+        for name, info in _list_plugins_with_info(ptype, collections, search_paths).items()
+    }
+
+
+def _list_plugins_with_info(
+    ptype: str,
+    collections: list[str] = None,
+    search_paths: list[str] | None = None,
+) -> dict[str, _PluginDocMetadata]:
     if isinstance(collections, str):
         collections = [collections]
 
@@ -195,7 +248,7 @@ def list_plugins(ptype, collections=None, search_paths=None):
                     raise AnsibleError(f"Cannot use supplied collection {collection!r}.") from ex
 
     if plugin_collections:
-        plugins.update(list_collection_plugins(ptype, plugin_collections))
+        plugins.update(_list_collection_plugins_with_info(ptype, plugin_collections))
 
     return plugins
 

test/integration/targets/ansible-doc/test.yml

@@ -207,3 +207,120 @@
     - assert:
         that:
           - result.stdout_lines | select("match", "^ansible\\._protomatter\\.") | length == result.stdout_lines | length
+
+    - name: List all filter plugins without any filter
+      command: ansible-doc -t filter --list --json
+      register: filter_list_all_raw
+
+    - set_fact:
+        filter_list_all: "{{ filter_list_all_raw.stdout | from_json }}"
+
+    - name: Check Jinja2 filters have a short description on list without any filter
+      assert:
+        that:
+        - filter_list_all['ansible.builtin.default'] is contains("the passed default value")
+        - filter_list_all['ansible.builtin.sum'] is contains("sum of a sequence")
+
+    - name: List only ansible.builtin filter plugins
+      command: ansible-doc ansible.builtin -t filter --list --json
+      register: filter_list_builtin_raw
+
+    - set_fact:
+        filter_list_builtin: "{{ filter_list_builtin_raw.stdout | from_json }}"
+
+    - name: Check Jinja2 filters have a short description on list with builtin filter
+      assert:
+        that:
+        - filter_list_builtin['ansible.builtin.default'] is contains("the passed default value")
+        - filter_list_builtin['ansible.builtin.sum'] is contains("sum of a sequence")
+
+    - name: List only ansible.builtin filter plugin files
+      command: ansible-doc ansible.builtin -t filter --list_files
+      register: filter_list_builtin_files
+
+    - name: Verify files for Jinja2 filters
+      assert:
+        that:
+        - filter_list_builtin_files.stdout_lines | select("match", "^ansible\\.builtin\\.default\\s+.*/core\\.py") | length == 1
+        - filter_list_builtin_files.stdout_lines | select("match", "^ansible\\.builtin\\.sum\\s+None") | length == 1
+
+    - name: Get jinja2 filter docs shadowed by Ansible
+      command: ansible-doc ansible.builtin.default -t filter
+      register: filter_jinja2_default
+
+    - name: Assert jinja2 filter docs shadowed by Ansible
+      assert:
+        that:
+        - filter_jinja2_default.stdout is search("\\s+FILTER ansible.builtin.default\\s+")
+        - filter_jinja2_default.stdout is search("the Jinja builtin filter plugin 'default'")
+        - filter_jinja2_default.stdout is search("the passed default value")
+        - filter_jinja2_default.stdout is search("https://jinja\\.palletsprojects\\.com/.*")
+
+    - name: Get jinja2 filter docs not shadowed by Ansible
+      command: ansible-doc ansible.builtin.sum -t filter
+      register: filter_jinja2_sum
+
+    - name: Assert jinja2 filter docs not shadowed by Ansible
+      assert:
+        that:
+        - filter_jinja2_sum.stdout is search("\\s+FILTER ansible.builtin.sum\\s+")
+        - filter_jinja2_sum.stdout is search("the Jinja builtin filter plugin 'sum'")
+        - filter_jinja2_sum.stdout is search("sum of a sequence")
+        - filter_jinja2_sum.stdout is search("https://jinja\\.palletsprojects\\.com/.*")
+
+    - name: List all test plugins without any filter
+      command: ansible-doc -t test --list --json
+      register: test_list_all_raw
+
+    - set_fact:
+        test_list_all: "{{ test_list_all_raw.stdout | from_json }}"
+
+    - name: Check Jinja2 tests have a short description on list without any filter
+      assert:
+        that:
+        - test_list_all['ansible.builtin.number'] is contains("is a number")
+
+    - name: List only ansible.builtin test plugins
+      command: ansible-doc ansible.builtin -t test --list --json
+      register: test_list_builtin_raw
+
+    - set_fact:
+        test_list_builtin: "{{ test_list_builtin_raw.stdout | from_json }}"
+
+    - name: Check Jinja2 tests have a short description on list with builtin tests
+      assert:
+        that:
+        - test_list_builtin['ansible.builtin.number'] is contains("is a number")
+
+    - name: List only ansible.builtin test plugin files
+      command: ansible-doc ansible.builtin -t test --list_files
+      register: test_list_builtin_files
+
+    - name: Verify files for Jinja2 filters
+      assert:
+        that:
+        - test_list_builtin_files.stdout_lines | select("match", "^ansible\\.builtin\\.number\\s+None") | length == 1
+
+    - name: Get jinja2 test docs
+      command: ansible-doc ansible.builtin.number -t test
+      register: test_jinja2_number
+
+    - name: Assert jinja2 test docs
+      assert:
+        that:
+        - test_jinja2_number.stdout is search("\\s+TEST ansible.builtin.number\\s+")
+        - test_jinja2_number.stdout is search("the Jinja builtin test plugin 'number'")
+        - test_jinja2_number.stdout is contains("is a number")
+        - test_jinja2_number.stdout is search("https://jinja\\.palletsprojects\\.com/.*")
+
+    - name: Get jinja2 test docs for an unqualified builtin
+      command: ansible-doc number -t test
+      register: test_jinja2_number_short
+
+    - name: Assert jinja2 test docs
+      assert:
+        that:
+        - test_jinja2_number_short.stdout is search("\\s+TEST ansible.builtin.number\\s+")
+        - test_jinja2_number_short.stdout is search("the Jinja builtin test plugin 'number'")
+        - test_jinja2_number_short.stdout is contains("is a number")
+        - test_jinja2_number_short.stdout is search("https://jinja\\.palletsprojects\\.com/.*")