Merge 52bb792159 into 7b4d4ed672

lib/ansible/module_utils/basic.py

@@ -204,6 +204,25 @@ FILE_COMMON_ARGUMENTS = dict(
     unsafe_writes=dict(type='bool', default=False, fallback=(env_fallback, ['ANSIBLE_UNSAFE_WRITES'])),  # should be available to any module using atomic_move
 )
 
+DECRYPT_COMMON_ARGUMENT = dict(
+    decrypt=dict(
+        type='bool',
+        default=True,
+        ),
+)
+
+VALIDATE_COMMON_ARGUMENT = dict(
+    validate=dict(
+        type='str',
+    ),
+)
+
+COMMON_ARGUMENTS_TO_FRAGMENTS = {
+    'FILE_COMMON_ARGUMENTS':   'files',
+    'DECRYPT_COMMON_ARGUMENT': 'decrypt',
+    'VALIDATE_COMMON_ARGUMENT': 'validate',
+}
+
 PASSWD_ARG_RE = re.compile(r'^[-]{0,2}pass[-]?(word|wd)?')
 
 # Used for parsing symbolic file perms
@@ -366,7 +385,7 @@ def missing_required_lib(library, reason=None, url=None):
 class AnsibleModule(object):
     def __init__(self, argument_spec, bypass_checks=False, no_log=False,
                  mutually_exclusive=None, required_together=None,
-                 required_one_of=None, add_file_common_args=False,
+                 required_one_of=None, extends_common_args=(),
                  supports_check_mode=False, required_if=None, required_by=None):
 
         """
@@ -406,8 +425,8 @@ class AnsibleModule(object):
         self._options_context = list()
         self._tmpdir = None
 
-        if add_file_common_args:
+        for shared_arguments in extends_common_args:
-            for k, v in FILE_COMMON_ARGUMENTS.items():
+            for k, v in shared_arguments.items():
                 if k not in self.argument_spec:
                     self.argument_spec[k] = v
 

lib/ansible/modules/assemble.py

@@ -100,8 +100,6 @@ extends_documentation_fragment:
     - action_common_attributes
     - action_common_attributes.flow
     - action_common_attributes.files
-    - decrypt
-    - files
 """
 
 EXAMPLES = r"""
@@ -130,7 +128,7 @@ import os
 import re
 import tempfile
 
-from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.basic import AnsibleModule, FILE_COMMON_ARGUMENTS, DECRYPT_COMMON_ARGUMENT
 from ansible.module_utils.common.text.converters import to_native
 
 
@@ -201,13 +199,14 @@ def main():
             regexp=dict(type='str'),
             ignore_hidden=dict(type='bool', default=False),
             validate=dict(type='str'),
-
+        ),
+        extends_common_args=(
+            FILE_COMMON_ARGUMENTS,
             # Options that are for the action plugin, but ignored by the module itself.
             # We have them here so that the tests pass without ignores, which
             # reduces the likelihood of further bugs added.
-            decrypt=dict(type='bool', default=True),
+            DECRYPT_COMMON_ARGUMENT,
         ),
-        add_file_common_args=True,
         supports_check_mode=True,
     )
 

lib/ansible/modules/blockinfile.py

@@ -118,8 +118,6 @@ notes:
 extends_documentation_fragment:
     - action_common_attributes
     - action_common_attributes.files
-    - files
-    - validate
 attributes:
     check_mode:
         support: full
@@ -200,7 +198,7 @@ import re
 import os
 import tempfile
 
-from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.basic import AnsibleModule, FILE_COMMON_ARGUMENTS, VALIDATE_COMMON_ARGUMENT
 from ansible.module_utils.common.text.converters import to_native
 
 
@@ -249,7 +247,6 @@ def main():
             insertbefore=dict(type='str'),
             create=dict(type='bool', default=False),
             backup=dict(type='bool', default=False),
-            validate=dict(type='str'),
             marker_begin=dict(type='str', default='BEGIN'),
             marker_end=dict(type='str', default='END'),
             append_newline=dict(type='bool', default=False),
@@ -257,7 +254,7 @@ def main():
             encoding=dict(type='str', default='utf-8'),
         ),
         mutually_exclusive=[['insertbefore', 'insertafter']],
-        add_file_common_args=True,
+        extends_common_args=(FILE_COMMON_ARGUMENTS, VALIDATE_COMMON_ARGUMENT),
         supports_check_mode=True
     )
     params = module.params

lib/ansible/modules/copy.py

@@ -118,9 +118,6 @@ options:
     type: str
     version_added: '2.5'
 extends_documentation_fragment:
-    - decrypt
-    - files
-    - validate
     - action_common_attributes
     - action_common_attributes.files
     - action_common_attributes.flow
@@ -293,7 +290,7 @@ import stat
 import tempfile
 
 from ansible.module_utils.common.text.converters import to_bytes, to_native
-from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.basic import AnsibleModule, FILE_COMMON_ARGUMENTS, VALIDATE_COMMON_ARGUMENT, DECRYPT_COMMON_ARGUMENT
 
 
 class AnsibleModuleError(Exception):
@@ -472,14 +469,13 @@ def main():
             dest=dict(type='path', required=True),
             backup=dict(type='bool', default=False),
             force=dict(type='bool', default=True),
-            validate=dict(type='str'),
             directory_mode=dict(type='raw'),
             remote_src=dict(type='bool', default=False),
             local_follow=dict(type='bool'),
             checksum=dict(type='str'),
             follow=dict(type='bool', default=False),
         ),
-        add_file_common_args=True,
+        extends_common_args=(FILE_COMMON_ARGUMENTS, DECRYPT_COMMON_ARGUMENT, VALIDATE_COMMON_ARGUMENT),
         supports_check_mode=True,
     )
 

lib/ansible/modules/file.py

@@ -12,7 +12,7 @@ DOCUMENTATION = r"""
 module: file
 version_added: historical
 short_description: Manage files and file properties
-extends_documentation_fragment: [files, action_common_attributes]
+extends_documentation_fragment: [action_common_attributes]
 description:
 - Set attributes of files, directories, or symlinks and their targets.
 - Alternatively, remove files, symlinks or directories.
@@ -236,7 +236,7 @@ import time
 from pwd import getpwnam, getpwuid
 from grp import getgrnam, getgrgid
 
-from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.basic import AnsibleModule, FILE_COMMON_ARGUMENTS
 from ansible.module_utils.common.text.converters import to_bytes, to_native
 from ansible.module_utils.common.sentinel import Sentinel
 
@@ -963,16 +963,16 @@ def main():
             path=dict(type='path', required=True, aliases=['dest', 'name']),
             _original_basename=dict(type='str'),  # Internal use only, for recursive ops
             recurse=dict(type='bool', default=False),
-            force=dict(type='bool', default=False),  # Note: Should not be in file_common_args in future
+            force=dict(type='bool', default=False),
-            follow=dict(type='bool', default=True),  # Note: Different default than file_common_args
+            follow=dict(type='bool', default=True),
             _diff_peek=dict(type='bool'),  # Internal use only, for internal checks in the action plugins
-            src=dict(type='path'),  # Note: Should not be in file_common_args in future
+            src=dict(type='path'),
             modification_time=dict(type='str'),
             modification_time_format=dict(type='str', default='%Y%m%d%H%M.%S'),
             access_time=dict(type='str'),
             access_time_format=dict(type='str', default='%Y%m%d%H%M.%S'),
         ),
-        add_file_common_args=True,
+        extends_common_args=(FILE_COMMON_ARGUMENTS,),
         supports_check_mode=True,
     )
 

lib/ansible/modules/get_url.py

@@ -202,7 +202,6 @@ options:
     version_added: '2.14'
 # informational: requirements for nodes
 extends_documentation_fragment:
-    - files
     - action_common_attributes
 attributes:
     check_mode:
@@ -376,7 +375,7 @@ import tempfile
 from datetime import datetime, timezone
 from urllib.parse import urlsplit
 
-from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.basic import AnsibleModule, FILE_COMMON_ARGUMENTS
 from ansible.module_utils.common.text.converters import to_native
 from ansible.module_utils.urls import fetch_url, url_argument_spec
 
@@ -531,7 +530,7 @@ def main():
     module = AnsibleModule(
         # not checking because of daisy chain to file module
         argument_spec=argument_spec,
-        add_file_common_args=True,
+        extends_common_args=(FILE_COMMON_ARGUMENTS,),
         supports_check_mode=True,
     )
 

lib/ansible/modules/lineinfile.py

@@ -133,8 +133,6 @@ options:
 extends_documentation_fragment:
     - action_common_attributes
     - action_common_attributes.files
-    - files
-    - validate
 attributes:
     check_mode:
         support: full
@@ -253,7 +251,7 @@ import re
 import tempfile
 
 # import module snippets
-from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.basic import AnsibleModule, FILE_COMMON_ARGUMENTS, VALIDATE_COMMON_ARGUMENT
 from ansible.module_utils.common.text.converters import to_bytes, to_native, to_text
 
 
@@ -581,11 +579,10 @@ def main():
             create=dict(type='bool', default=False),
             backup=dict(type='bool', default=False),
             firstmatch=dict(type='bool', default=False),
-            validate=dict(type='str'),
         ),
         mutually_exclusive=[
             ['insertbefore', 'insertafter'], ['regexp', 'search_string'], ['backrefs', 'search_string']],
-        add_file_common_args=True,
+        extends_common_args=(FILE_COMMON_ARGUMENTS, VALIDATE_COMMON_ARGUMENT),
         supports_check_mode=True,
     )
 

lib/ansible/modules/replace.py

@@ -14,8 +14,6 @@ author: Evan Kaufman (@EvanK)
 extends_documentation_fragment:
     - action_common_attributes
     - action_common_attributes.files
-    - files
-    - validate
 attributes:
     check_mode:
         support: full
@@ -184,7 +182,7 @@ import re
 import tempfile
 
 from ansible.module_utils.common.text.converters import to_text
-from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.basic import AnsibleModule, FILE_COMMON_ARGUMENTS, VALIDATE_COMMON_ARGUMENT
 
 
 def write_changes(module, contents, path, encoding='utf-8'):
@@ -229,10 +227,9 @@ def main():
             after=dict(type='str'),
             before=dict(type='str'),
             backup=dict(type='bool', default=False),
-            validate=dict(type='str'),
             encoding=dict(type='str', default='utf-8'),
         ),
-        add_file_common_args=True,
+        extends_common_args=(FILE_COMMON_ARGUMENTS, VALIDATE_COMMON_ARGUMENT),
         supports_check_mode=True,
     )
 

lib/ansible/modules/unarchive.py

@@ -254,7 +254,7 @@ from functools import partial
 from zipfile import ZipFile
 
 from ansible.module_utils.common.text.converters import to_bytes, to_native, to_text
-from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.basic import AnsibleModule, FILE_COMMON_ARGUMENTS, DECRYPT_COMMON_ARGUMENT
 from ansible.module_utils.common.process import get_bin_path
 from ansible.module_utils.common.locale import get_best_parsable_locale
 from ansible.module_utils.urls import fetch_file
@@ -1067,9 +1067,11 @@ def main():
             # We have them here so that the sanity tests pass without ignores, which
             # reduces the likelihood of further bugs added.
             copy=dict(type='bool', default=True),
-            decrypt=dict(type='bool', default=True),
         ),
-        add_file_common_args=True,
+        extends_common_args=(
+            FILE_COMMON_ARGUMENTS,
+            DECRYPT_COMMON_ARGUMENT,  # Also here for the action plugin
+        ),
         # check-mode only works for zip files, we cover that later
         supports_check_mode=True,
         mutually_exclusive=[('include', 'exclude')],

lib/ansible/modules/uri.py

@@ -442,7 +442,7 @@ from collections.abc import Mapping, Sequence
 from datetime import datetime, timezone
 from urllib.parse import urlencode, urljoin
 
-from ansible.module_utils.basic import AnsibleModule, sanitize_keys
+from ansible.module_utils.basic import AnsibleModule, sanitize_keys, FILE_COMMON_ARGUMENTS
 from ansible.module_utils.common.text.converters import to_native, to_text
 from ansible.module_utils.urls import (
     fetch_url,
@@ -614,7 +614,7 @@ def main():
 
     module = AnsibleModule(
         argument_spec=argument_spec,
-        add_file_common_args=True,
+        extends_common_args=(FILE_COMMON_ARGUMENTS,),
         mutually_exclusive=[['body', 'src']],
     )
 

lib/ansible/modules/yum_repository.py

@@ -359,7 +359,6 @@ options:
 
 extends_documentation_fragment:
     - action_common_attributes
-    - files
 attributes:
     check_mode:
         support: full
@@ -586,7 +585,7 @@ def main():
             ["state", "present", ["description"]],
         ],
         argument_spec=argument_spec,
-        add_file_common_args=True,
+        extends_common_args=(FILE_COMMON_ARGUMENTS,),
         supports_check_mode=True,
     )
 

lib/ansible/parsing/plugin_docs.py

@@ -9,6 +9,7 @@ import yaml
 
 from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleParserError
+from ansible.module_utils.basic import COMMON_ARGUMENTS_TO_FRAGMENTS
 from ansible.module_utils.common.text.converters import to_text, to_native
 from ansible.parsing.yaml.loader import AnsibleLoader
 from ansible.utils.display import Display
@@ -65,31 +66,44 @@ def read_docstring_from_python_file(filename, verbose=True, ignore_errors=True):
         with open(filename, 'rb') as b_module_data:
             M = ast.parse(b_module_data.read())
 
-        for child in M.body:
+        for child in ast.walk(M):
             if isinstance(child, ast.Assign):
-                for t in child.targets:
+                # Find the AnsibleModule(extend_common_args=...) and extend fragments
-                    try:
+                if isinstance(child.value, ast.Call) and getattr(child.value.func, 'id', None) == 'AnsibleModule':  # Magic string
-                        theid = t.id
+                    for kw in child.value.keywords:
-                    except AttributeError:
+                        if kw.arg == 'extends_common_args' and isinstance(kw.value, ast.Tuple):  # Magic string
-                        # skip errors can happen when trying to use the normal code
+                            if data['doc'] is None:
-                        display.warning("Building documentation, failed to assign id for %s on %s, skipping" % (t, filename))
+                                if 'extends_documentation_fragment' not in data['doc']:
-                        continue
+                                    data['doc']['extends_documentation_fragment'] = []
-
+                            for arg_name_node in kw.value.elts:
-                    if theid in string_to_vars:
+                                if isinstance(arg_name_node, ast.Name):
-                        varkey = string_to_vars[theid]
+                                    fragment_name = COMMON_ARGUMENTS_TO_FRAGMENTS[arg_name_node.id]
-                        if isinstance(child.value, ast.Dict):
+                                    data['doc']['extends_documentation_fragment'].append(fragment_name)
-                            data[varkey] = ast.literal_eval(child.value)
+                # If it's top level, find the specified docs
-                        else:
+                if child in M.body:
-                            if theid == 'EXAMPLES':
+                    for t in child.targets:
-                                # examples 'can' be yaml, but even if so, we dont want to parse as such here
+                        try:
-                                # as it can create undesired 'objects' that don't display well as docs.
+                            theid = t.id
-                                data[varkey] = to_text(child.value.value)
+                        except AttributeError:
+                            # skip errors can happen when trying to use the normal code
+                            display.warning("Building documentation, failed to assign id for %s on %s, skipping" % (t, filename))
+                            continue
+
+                        if theid in string_to_vars:
+                            varkey = string_to_vars[theid]
+                            if isinstance(child.value, ast.Dict):
+                                data[varkey] = ast.literal_eval(child.value)
                             else:
-                                # string should be yaml if already not a dict
+                                if theid == 'EXAMPLES':
-                                child_value = _tags.Origin(path=filename, line_num=child.value.lineno).tag(child.value.value)
+                                    # examples 'can' be yaml, but even if so, we dont want to parse as such here
-                                data[varkey] = yaml.load(child_value, Loader=AnsibleLoader)
+                                    # as it can create undesired 'objects' that don't display well as docs.
-
+                                    data[varkey] = to_text(child.value.value)
-                        display.debug('Documentation assigned: %s' % varkey)
+                                else:
+                                    # string should be yaml if already not a dict
+                                    child_value = _tags.Origin(path=filename, line_num=child.value.lineno).tag(child.value.value)
+                                    data[varkey] = yaml.load(child_value, Loader=AnsibleLoader)
+
+                            display.debug('Documentation assigned: %s' % varkey)
 
     except Exception as ex:
         msg = f"Unable to parse documentation in python file {filename!r}"

test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/module_args.py

@@ -163,13 +163,14 @@ def get_py_argument_spec(filename, collection):
             fake.kwargs[arg_name] = arg
         # for ping kwargs == {'argument_spec':{'data':{'type':'str','default':'pong'}}, 'supports_check_mode':True}
         argument_spec = fake.kwargs.get('argument_spec') or {}
-        # If add_file_common_args is truish, add options from FILE_COMMON_ARGUMENTS when not present.
+        # Add the common arguments from the extends_common_args
         # This is the only modification to argument_spec done by AnsibleModule itself, and which is
         # not caught by setup_env's AnsibleModule replacement
-        if fake.kwargs.get('add_file_common_args'):
+        if args := fake.kwargs.get('extends_common_arguments'):
-            for k, v in FILE_COMMON_ARGUMENTS.items():
+            for common in args:
-                if k not in argument_spec:
+                for k, v in common.items():
-                    argument_spec[k] = v
+                    if k not in argument_spec:
+                        argument_spec[k] = v
         return argument_spec, fake.kwargs
     except (TypeError, IndexError):
         return {}, {}

test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/schema.py

@@ -367,7 +367,7 @@ def ansible_module_kwargs_schema(module_name, for_collection):
         'bypass_checks': bool,
         'no_log': bool,
         'check_invalid_arguments': Any(None, bool),
-        'add_file_common_args': bool,
+        'extends_common_args': tuple[dict],
         'supports_check_mode': bool,
     }
     if module_name.endswith(('_info', '_facts')):

test/units/module_utils/basic/test_argument_spec.py

@@ -15,6 +15,7 @@ import pytest
 from unittest.mock import MagicMock
 from ansible.module_utils import basic
 from ansible.module_utils.api import basic_auth_argument_spec, rate_limit_argument_spec, retry_argument_spec
+from ansible.module_utils.basic import FILE_COMMON_ARGUMENTS
 from ansible.module_utils.common.warnings import get_deprecation_messages, get_warning_messages
 import builtins
 
@@ -138,7 +139,7 @@ def complex_argspec():
         mutually_exclusive=mut_ex,
         required_together=req_to,
         no_log=True,
-        add_file_common_args=True,
+        extends_common_args=FILE_COMMON_ARGUMENTS,
         supports_check_mode=True,
     )
     return kwargs
@@ -188,7 +189,7 @@ def options_argspec_list():
     kwargs = dict(
         argument_spec=arg_spec,
         no_log=True,
-        add_file_common_args=True,
+        extends_common_args=FILE_COMMON_ARGUMENTS,
         supports_check_mode=True
     )
     return kwargs