validate-modules: check_mode attribute and compare with supports_check_mode parameter (#80090)

Co-authored-by: Brian Scholer <1260690+briantist@users.noreply.github.com>

changelogs/fragments/80090-validate-modules-check_mode-attribute.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - "validate-modules sanity test - if the ``check_mode`` attribute is present, check that it coincides with the ``support_check_mode`` parameter of ``AnsibleModule`` (https://github.com/ansible/ansible/pull/80090)."

docs/docsite/rst/dev_guide/testing/sanity/validate-modules.rst

@@ -56,6 +56,8 @@ Codes
   documentation-syntax-error                                   Documentation        Error                  Invalid ``DOCUMENTATION`` schema
   import-before-documentation                                  Imports              Error                  Import found before documentation variables. All imports must appear below ``DOCUMENTATION``/``EXAMPLES``/``RETURN``
   import-error                                                 Documentation        Error                  ``Exception`` attempting to import module for ``argument_spec`` introspection
+  attributes-check-mode                                        Documentation        Error                  If a module documents the ``check_mode`` attribute, its ``support`` value must be compatible with the ``supports_check_mode`` parameter of ``AnsibleModule``
+  attributes-check-mode-details                                Documentation        Error                  If a module documents the ``check_mode`` attribute with support values ``partial`` or ``N/A``, it must provide ``details``
   import-placement                                             Locations            Warning                Imports should be directly below ``DOCUMENTATION``/``EXAMPLES``/``RETURN``
   imports-improper-location                                    Imports              Error                  Imports should be directly below ``DOCUMENTATION``/``EXAMPLES``/``RETURN``
   incompatible-choices                                         Documentation        Error                  Choices value from the argument_spec is not compatible with type defined in the argument_spec

lib/ansible/modules/wait_for.py

@@ -100,7 +100,7 @@ options:
 extends_documentation_fragment: action_common_attributes
 attributes:
     check_mode:
-        support: full
+        support: none
     diff_mode:
         support: none
     platform:

test/integration/targets/ansible-test-sanity-validate-modules/ansible_collections/ns/col/plugins/modules/check_mode_attribute_1.py

@@ -0,0 +1,33 @@
+#!/usr/bin/python
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+DOCUMENTATION = '''
+module: check_mode_attribute_1
+short_description: Test for check mode attribute 1
+description: Test for check mode attribute 1.
+author:
+  - Ansible Core Team
+extends_documentation_fragment:
+  - ansible.builtin.action_common_attributes
+attributes:
+  check_mode:
+    # doc says full support, code says none
+    support: full
+  diff_mode:
+    support: none
+  platform:
+    platforms: all
+'''
+
+EXAMPLES = '''#'''
+RETURN = ''''''
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+if __name__ == '__main__':
+    module = AnsibleModule(argument_spec=dict(), supports_check_mode=False)
+    module.exit_json()

test/integration/targets/ansible-test-sanity-validate-modules/ansible_collections/ns/col/plugins/modules/check_mode_attribute_2.py

@@ -0,0 +1,34 @@
+#!/usr/bin/python
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+DOCUMENTATION = '''
+module: check_mode_attribute_2
+short_description: Test for check mode attribute 2
+description: Test for check mode attribute 2.
+author:
+  - Ansible Core Team
+extends_documentation_fragment:
+  - ansible.builtin.action_common_attributes
+attributes:
+  check_mode:
+    # doc says partial support, code says none
+    support: partial
+    details: Whatever this means.
+  diff_mode:
+    support: none
+  platform:
+    platforms: all
+'''
+
+EXAMPLES = '''#'''
+RETURN = ''''''
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+if __name__ == '__main__':
+    module = AnsibleModule(argument_spec=dict(), supports_check_mode=False)
+    module.exit_json()

test/integration/targets/ansible-test-sanity-validate-modules/ansible_collections/ns/col/plugins/modules/check_mode_attribute_3.py

@@ -0,0 +1,33 @@
+#!/usr/bin/python
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+DOCUMENTATION = '''
+module: check_mode_attribute_3
+short_description: Test for check mode attribute 3
+description: Test for check mode attribute 3.
+author:
+  - Ansible Core Team
+extends_documentation_fragment:
+  - ansible.builtin.action_common_attributes
+attributes:
+  check_mode:
+    # doc says no support, code says some
+    support: none
+  diff_mode:
+    support: none
+  platform:
+    platforms: all
+'''
+
+EXAMPLES = '''#'''
+RETURN = ''''''
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+if __name__ == '__main__':
+    module = AnsibleModule(argument_spec=dict(), supports_check_mode=True)
+    module.exit_json()

test/integration/targets/ansible-test-sanity-validate-modules/ansible_collections/ns/col/plugins/modules/check_mode_attribute_4.py

@@ -0,0 +1,33 @@
+#!/usr/bin/python
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+DOCUMENTATION = '''
+module: check_mode_attribute_4
+short_description: Test for check mode attribute 4
+description: Test for check mode attribute 4.
+author:
+  - Ansible Core Team
+extends_documentation_fragment:
+  - ansible.builtin.action_common_attributes
+attributes:
+  check_mode:
+    # documentation says some support, but no details
+    support: partial
+  diff_mode:
+    support: none
+  platform:
+    platforms: all
+'''
+
+EXAMPLES = '''#'''
+RETURN = ''''''
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+if __name__ == '__main__':
+    module = AnsibleModule(argument_spec=dict(), supports_check_mode=True)
+    module.exit_json()

test/integration/targets/ansible-test-sanity-validate-modules/ansible_collections/ns/col/plugins/modules/check_mode_attribute_5.py

@@ -0,0 +1,33 @@
+#!/usr/bin/python
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+DOCUMENTATION = '''
+module: check_mode_attribute_5
+short_description: Test for check mode attribute 5
+description: Test for check mode attribute 5.
+author:
+  - Ansible Core Team
+extends_documentation_fragment:
+  - ansible.builtin.action_common_attributes
+attributes:
+  check_mode:
+    # Everything is correct: both docs and code claim no support
+    support: none
+  diff_mode:
+    support: none
+  platform:
+    platforms: all
+'''
+
+EXAMPLES = '''#'''
+RETURN = ''''''
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+if __name__ == '__main__':
+    module = AnsibleModule(argument_spec=dict(), supports_check_mode=False)
+    module.exit_json()

test/integration/targets/ansible-test-sanity-validate-modules/ansible_collections/ns/col/plugins/modules/check_mode_attribute_6.py

@@ -0,0 +1,34 @@
+#!/usr/bin/python
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+DOCUMENTATION = '''
+module: check_mode_attribute_6
+short_description: Test for check mode attribute 6
+description: Test for check mode attribute 6.
+author:
+  - Ansible Core Team
+extends_documentation_fragment:
+  - ansible.builtin.action_common_attributes
+attributes:
+  check_mode:
+    # Everything is correct: docs says partial support *with details*, code claims (at least some) support
+    support: partial
+    details: Some details.
+  diff_mode:
+    support: none
+  platform:
+    platforms: all
+'''
+
+EXAMPLES = '''#'''
+RETURN = ''''''
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+if __name__ == '__main__':
+    module = AnsibleModule(argument_spec=dict(), supports_check_mode=True)
+    module.exit_json()

test/integration/targets/ansible-test-sanity-validate-modules/ansible_collections/ns/col/plugins/modules/check_mode_attribute_7.py

@@ -0,0 +1,33 @@
+#!/usr/bin/python
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+DOCUMENTATION = '''
+module: check_mode_attribute_7
+short_description: Test for check mode attribute 7
+description: Test for check mode attribute 7.
+author:
+  - Ansible Core Team
+extends_documentation_fragment:
+  - ansible.builtin.action_common_attributes
+attributes:
+  check_mode:
+    # Everything is correct: docs says full support, code claims (at least some) support
+    support: full
+  diff_mode:
+    support: none
+  platform:
+    platforms: all
+'''
+
+EXAMPLES = '''#'''
+RETURN = ''''''
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+if __name__ == '__main__':
+    module = AnsibleModule(argument_spec=dict(), supports_check_mode=True)
+    module.exit_json()

test/integration/targets/ansible-test-sanity-validate-modules/expected.txt

@@ -1,4 +1,8 @@
 plugins/lookup/import_order_lookup.py:5:0: import-before-documentation: Import found before documentation variables. All imports must appear below DOCUMENTATION/EXAMPLES/RETURN.
+plugins/modules/check_mode_attribute_1.py:0:0: attributes-check-mode: The module does not declare support for check mode, but the check_mode attribute's support value is 'full' and not 'none'
+plugins/modules/check_mode_attribute_2.py:0:0: attributes-check-mode: The module does not declare support for check mode, but the check_mode attribute's support value is 'partial' and not 'none'
+plugins/modules/check_mode_attribute_3.py:0:0: attributes-check-mode: The module does declare support for check mode, but the check_mode attribute's support value is 'none'
+plugins/modules/check_mode_attribute_4.py:0:0: attributes-check-mode-details: The module declares it does not fully support check mode, but has no details on what exactly that means
 plugins/modules/import_order.py:8:0: import-before-documentation: Import found before documentation variables. All imports must appear below DOCUMENTATION/EXAMPLES/RETURN.
 plugins/modules/invalid_yaml_syntax.py:0:0: deprecation-mismatch: "meta/runtime.yml" and DOCUMENTATION.deprecation do not agree.
 plugins/modules/invalid_yaml_syntax.py:0:0: missing-documentation: No DOCUMENTATION provided

test/integration/targets/ansible-test-sanity-validate-modules/runme.sh

@@ -6,7 +6,7 @@ set -eux
 
 ansible-test sanity --test validate-modules --color --truncate 0 --failure-ok --lint "${@}" 1> actual-stdout.txt 2> actual-stderr.txt
 diff -u "${TEST_DIR}/expected.txt" actual-stdout.txt
-grep -f "${TEST_DIR}/expected.txt" actual-stderr.txt
+grep -F -f "${TEST_DIR}/expected.txt" actual-stderr.txt
 
 cd ../ps_only
 

test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py

@@ -1220,6 +1220,31 @@ class ModuleValidator(Validator):
 
         self._validate_argument_spec(docs, spec, kwargs)
 
+        if isinstance(docs, Mapping) and isinstance(docs.get('attributes'), Mapping):
+            if isinstance(docs['attributes'].get('check_mode'), Mapping):
+                support_value = docs['attributes']['check_mode'].get('support')
+                if not kwargs.get('supports_check_mode', False):
+                    if support_value != 'none':
+                        self.reporter.error(
+                            path=self.object_path,
+                            code='attributes-check-mode',
+                            msg="The module does not declare support for check mode, but the check_mode attribute's"
+                                " support value is '%s' and not 'none'" % support_value
+                        )
+                else:
+                    if support_value not in ('full', 'partial', 'N/A'):
+                        self.reporter.error(
+                            path=self.object_path,
+                            code='attributes-check-mode',
+                            msg="The module does declare support for check mode, but the check_mode attribute's support value is '%s'" % support_value
+                        )
+                if support_value in ('partial', 'N/A') and docs['attributes']['check_mode'].get('details') in (None, '', []):
+                    self.reporter.error(
+                        path=self.object_path,
+                        code='attributes-check-mode-details',
+                        msg="The module declares it does not fully support check mode, but has no details on what exactly that means"
+                    )
+
     def _validate_list_of_module_args(self, name, terms, spec, context):
         if terms is None:
             return