add fragments to return (#72635)

* ansible-doc add ability to use doc_fragments for RETURN


---------

Co-authored-by: Felix Fontein <felix@fontein.de>

changelogs/fragments/return_fragments.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - ansible-doc adds support for RETURN documentation to support doc fragment plugins

lib/ansible/plugins/loader.py

@@ -517,7 +517,7 @@ class PluginLoader:
                 #     filename, cn = find_plugin_docfile( name, type_name, self, [os.path.dirname(path)], C.YAML_DOC_EXTENSIONS)
 
                 if dstring:
-                    add_fragments(dstring, path, fragment_loader=fragment_loader, is_module=(type_name == 'module'))
+                    add_fragments(dstring, path, fragment_loader=fragment_loader, is_module=(type_name == 'module'), section='DOCUMENTATION')
 
                     if 'options' in dstring and isinstance(dstring['options'], dict):
                         C.config.initialize_plugin_configuration_definitions(type_name, name, dstring['options'])

lib/ansible/utils/plugin_docs.py

@@ -18,6 +18,7 @@ from ansible.parsing.yaml.loader import AnsibleLoader
 from ansible.utils.display import Display
 from ansible._internal._datatag import _tags
 
+_FRAGMENTABLE = ('DOCUMENTATION', 'RETURN')
 display = Display()
 
 
@@ -125,7 +126,10 @@ def remove_current_collection_from_versions_and_dates(fragment, collection_name,
     _process_versions_and_dates(fragment, is_module, return_docs, remove)
 
 
-def add_fragments(doc, filename, fragment_loader, is_module=False):
+def add_fragments(doc, filename, fragment_loader, is_module=False, section='DOCUMENTATION'):
+
+    if section not in _FRAGMENTABLE:
+        raise AnsibleError(f"Invalid fragment section ({section}) passed to render {filename}, it can only be one of {_FRAGMENTABLE!r}")
 
     fragments = doc.pop('extends_documentation_fragment', [])
 
@@ -134,14 +138,14 @@ def add_fragments(doc, filename, fragment_loader, is_module=False):
 
     unknown_fragments = []
 
-    # doc_fragments are allowed to specify a fragment var other than DOCUMENTATION
+    # doc_fragments are allowed to specify a fragment var other than DOCUMENTATION or RETURN
     # with a . separator; this is complicated by collections-hosted doc_fragments that
     # use the same separator. Assume it's collection-hosted normally first, try to load
     # as-specified. If failure, assume the right-most component is a var, split it off,
     # and retry the load.
     for fragment_slug in fragments:
         fragment_name = fragment_slug.strip()
-        fragment_var = 'DOCUMENTATION'
+        fragment_var = section
 
         fragment_class = fragment_loader.get(fragment_name)
         if fragment_class is None and '.' in fragment_slug:
@@ -157,7 +161,7 @@ def add_fragments(doc, filename, fragment_loader, is_module=False):
         # trust-tagged source propagates to loaded values; expressions and templates in config require trust
         fragment_yaml = _tags.TrustedAsTemplate().tag(getattr(fragment_class, fragment_var, None))
         if fragment_yaml is None:
-            if fragment_var != 'DOCUMENTATION':
+            if fragment_var not in _FRAGMENTABLE:
                 # if it's asking for something specific that's missing, that's an error
                 unknown_fragments.append(fragment_slug)
                 continue
@@ -168,35 +172,31 @@ def add_fragments(doc, filename, fragment_loader, is_module=False):
 
         real_fragment_name = getattr(fragment_class, 'ansible_name')
         real_collection_name = '.'.join(real_fragment_name.split('.')[0:2]) if '.' in real_fragment_name else ''
-        add_collection_to_versions_and_dates(fragment, real_collection_name, is_module=is_module)
-
-        if 'notes' in fragment:
-            notes = fragment.pop('notes')
-            if notes:
-                if 'notes' not in doc:
-                    doc['notes'] = []
-                doc['notes'].extend(notes)
-
-        if 'seealso' in fragment:
-            seealso = fragment.pop('seealso')
-            if seealso:
-                if 'seealso' not in doc:
-                    doc['seealso'] = []
-                doc['seealso'].extend(seealso)
-
-        if 'options' not in fragment and 'attributes' not in fragment:
-            raise Exception("missing options or attributes in fragment (%s), possibly misformatted?: %s" % (fragment_name, filename))
-
-        # ensure options themselves are directly merged
-        for doc_key in ['options', 'attributes']:
-            if doc_key in fragment:
-                if doc_key in doc:
-                    try:
-                        merge_fragment(doc[doc_key], fragment.pop(doc_key))
-                    except Exception as e:
-                        raise AnsibleError("%s %s (%s) of unknown type: %s" % (to_native(e), doc_key, fragment_name, filename))
-                else:
-                    doc[doc_key] = fragment.pop(doc_key)
+        add_collection_to_versions_and_dates(fragment, real_collection_name, is_module=is_module, return_docs=(section == 'RETURN'))
+
+        if section == 'DOCUMENTATION':
+            # notes, seealso, options and attributes entries are specificly merged, but only occur in documentation section
+            for doc_key in ['notes', 'seealso']:
+                if doc_key in fragment:
+                    entries = fragment.pop(doc_key)
+                    if entries:
+                        if doc_key not in doc:
+                            doc[doc_key] = []
+                        doc[doc_key].extend(entries)
+
+            if 'options' not in fragment and 'attributes' not in fragment:
+                raise Exception("missing options or attributes in fragment (%s), possibly misformatted?: %s" % (fragment_name, filename))
+
+            # ensure options themselves are directly merged
+            for doc_key in ['options', 'attributes']:
+                if doc_key in fragment:
+                    if doc_key in doc:
+                        try:
+                            merge_fragment(doc[doc_key], fragment.pop(doc_key))
+                        except Exception as e:
+                            raise AnsibleError("%s %s (%s) of unknown type: %s" % (to_native(e), doc_key, fragment_name, filename))
+                    else:
+                        doc[doc_key] = fragment.pop(doc_key)
 
         # merge rest of the sections
         try:
@@ -230,13 +230,16 @@ def get_docstring(filename, fragment_loader, verbose=False, ignore_errors=False,
             add_collection_to_versions_and_dates(data['doc'], collection_name, is_module=is_module)
 
         # add fragments to documentation
-        add_fragments(data['doc'], filename, fragment_loader=fragment_loader, is_module=is_module)
+        add_fragments(data['doc'], filename, fragment_loader=fragment_loader, is_module=is_module, section='DOCUMENTATION')
 
     if data.get('returndocs', False):
         # add collection name to versions and dates
         if collection_name is not None:
             add_collection_to_versions_and_dates(data['returndocs'], collection_name, is_module=is_module, return_docs=True)
 
+        # add fragments to return
+        add_fragments(data['returndocs'], filename, fragment_loader=fragment_loader, is_module=is_module, section='RETURN')
+
     return data['doc'], data['plainexamples'], data['returndocs'], data['metadata']
 
 

test/integration/targets/ansible-doc/doc_fragments/test_return_frag.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+
+class ModuleDocFragment(object):
+
+    # Standard documentation fragment
+    RETURN = r'''
+y_notlast:
+    description: A return from fragment
+    type: str
+    returned: it depends TM
+
+z_last:
+    description: A a return from fragment with merge.
+    type: str
+    returned: success
+'''

test/integration/targets/ansible-doc/library/test_docs_return_fragments.py

@@ -0,0 +1,56 @@
+#!/usr/bin/python
+
+from __future__ import annotations
+
+
+DOCUMENTATION = '''
+---
+module: test_docs_returns
+short_description: Test module
+description:
+    - Test module
+author:
+    - Ansible Core Team
+'''
+
+EXAMPLES = '''
+'''
+
+RETURN = '''
+m_middle:
+    description:
+        - This should be in the middle.
+        - Has some more data
+    type: dict
+    returned: success and 1st of month
+    contains:
+        suboption:
+            description: A suboption.
+            type: str
+            choices: [ARF, BARN, c_without_capital_first_letter]
+
+a_first:
+    description: A first result.
+    type: str
+    returned: success
+
+z_last:
+    example: this is a merge
+extends_documentation_fragment:
+    - test_return_frag
+'''
+
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def main():
+    module = AnsibleModule(
+        argument_spec=dict(),
+    )
+
+    module.exit_json()
+
+
+if __name__ == '__main__':
+    main()

test/integration/targets/ansible-doc/test.yml

@@ -4,6 +4,7 @@
     ANSIBLE_LIBRARY: "{{ playbook_dir }}/library"
     ANSIBLE_NOCOLOR: 1
     ANSIBLE_DEPRECATION_WARNINGS: 1
+    ANSIBLE_DOC_FRAGMENT_PLUGINS: "{{ playbook_dir }}/doc_fragments"
   vars:
     # avoid header that has full path and won't work across random paths for tests
     actual_output_clean: '{{ actual_output.splitlines()[1:] }}'
@@ -26,28 +27,26 @@
       register: result
       ignore_errors: true
 
-    - set_fact:
-        actual_output: "{{ result.stdout }}"
-        expected_output: "{{ lookup('file', 'test_docs_suboptions.output') }}"
-
     - assert:
         that:
           - result is succeeded
           - actual_output_clean == expected_output_clean
+      vars:
+        actual_output: "{{ result.stdout }}"
+        expected_output: "{{ lookup('file', 'test_docs_suboptions.output') }}"
 
     - name: module with return docs
       shell: ansible-doc test_docs_returns| tail -n +3
       register: result
       ignore_errors: true
 
-    - set_fact:
-        actual_output: "{{ result.stdout }}"
-        expected_output: "{{ lookup('file', 'test_docs_returns.output') }}"
-
     - assert:
         that:
           - result is succeeded
           - actual_output_clean == expected_output_clean
+      vars:
+        actual_output: "{{ result.stdout }}"
+        expected_output: "{{ lookup('file', 'test_docs_returns.output') }}"
 
     - name: module with broken return docs
       command: ansible-doc test_docs_returns_broken
@@ -59,6 +58,22 @@
           - result is failed
           - '"module test_docs_returns_broken Missing documentation (or could not parse documentation)" in result.stderr'
 
+    - name: module with return docs with fragments
+      command: ansible-doc test_docs_return_fragments
+      register: result
+      ignore_errors: true
+
+    - name: test fragments work
+      assert:
+        that:
+          - result is success
+          - "'z_last' in actual_output"
+          - "'y_notlast' in actual_output"
+          - actual_output_clean == expected_output_clean
+      vars:
+        actual_output: "{{ result.stdout }}"
+        expected_output: "{{ lookup('file', 'test_docs_return_fragments.output') }}"
+
     - name: non-existent module
       command: ansible-doc test_does_not_exist
       register: result

test/integration/targets/ansible-doc/test_docs_return_fragments.output

@@ -0,0 +1,34 @@
+> MODULE test_docs_return_fragments
+
+  Test module
+
+AUTHOR: Ansible Core Team
+
+EXAMPLES:
+
+
+RETURN VALUES:
+
+- a_first  A first result.
+        returned: success
+        type: str
+
+- m_middle  This should be in the middle.
+             Has some more data
+        returned: success and 1st of month
+        type: dict
+        contains:
+
+        - suboption  A suboption.
+          choices: [ARF, BARN, c_without_capital_first_letter]
+          type: str
+
+- y_notlast  A return from fragment
+        returned: it depends TM
+        type: str
+
+- z_last  A a return from fragment with merge.
+        example: this is a merge
+        returned: success
+        type: str
+