[stable-2.9] Fix ansible-doc traceback and sanity test. (#62040)

* Fix ansible-doc traceback for removed modules.

This avoids tracebacks with errors like the following when a module has been removed:

module module_name missing documentation (or could not parse documentation): 'NoneType' object does not support item assignment

* Fix ansible-doc sanity test warning handling.

Warnings about removed modules/plugins on stderr are now properly ignored.

Previously an ansible-doc error could result in unrelated errors going undetected because tests were stopped early and the underlying error was ignored.
(cherry picked from commit 064e8e1ef4)

Co-authored-by: Matt Clay <matt@mystile.com>

changelogs/fragments/ansible-doc-removed-traceback.yml

@@ -0,0 +1,2 @@
+bugfixes:
+    - ansible-doc now properly handles removed modules/plugins

changelogs/fragments/ansible-test-ansible-doc.yml

@@ -0,0 +1,2 @@
+bugfixes:
+    - ansible-test now properly handles warnings for removed modules/plugins

lib/ansible/cli/doc.py

@@ -273,7 +273,10 @@ class DocCLI(CLI):
 
             if not any(filename.endswith(x) for x in C.BLACKLIST_EXTS):
                 doc, plainexamples, returndocs, metadata = get_docstring(filename, fragment_loader, verbose=(context.CLIARGS['verbosity'] > 0))
-                doc['filename'] = filename
+
+                if doc:
+                    # doc may be None, such as when the module has been removed
+                    doc['filename'] = filename
 
         except Exception as e:
             display.vvv(traceback.format_exc())

test/lib/ansible_test/_internal/sanity/ansible_doc.py

@@ -116,14 +116,6 @@ class AnsibleDocTest(SanitySingleVersion):
                 stderr = ex.stderr
                 status = ex.status
 
-            if stderr:
-                errors = stderr.strip().splitlines()
-                messages = [self.parse_error(e, target_paths) for e in errors]
-
-                if messages and all(messages):
-                    error_messages += messages
-                    continue
-
             if status:
                 summary = u'%s' % SubprocessError(cmd=cmd, status=status, stderr=stderr)
                 return SanityFailure(self.name, summary=summary)
@@ -131,6 +123,10 @@ class AnsibleDocTest(SanitySingleVersion):
             if stdout:
                 display.info(stdout.strip(), verbosity=3)
 
+            if stderr:
+                # ignore removed module/plugin warnings
+                stderr = re.sub(r'\[WARNING\]: [^ ]+ [^ ]+ has been removed\n', '', stderr).strip()
+
             if stderr:
                 summary = u'Output on stderr from ansible-doc is considered an error.\n\n%s' % SubprocessError(cmd, stderr=stderr)
                 return SanityFailure(self.name, summary=summary)
@@ -144,30 +140,3 @@ class AnsibleDocTest(SanitySingleVersion):
             return SanityFailure(self.name, messages=error_messages)
 
         return SanitySuccess(self.name)
-
-    @staticmethod
-    def parse_error(error, target_paths):
-        """
-        :type error: str
-        :type target_paths: dict[str, dict[str, str]]
-        :rtype: SanityMessage | None
-        """
-        # example error messages from lib/ansible/cli/doc.py:
-        #   ERROR! module ping missing documentation (or could not parse documentation): expected string or buffer
-        #   [ERROR]: module ping has a documentation error formatting or is missing documentation.
-        match = re.search(r'^[^ ]*ERROR[^ ]* (?P<type>[^ ]+) (?P<name>[^ ]+) (?P<text>.*)$', error)
-
-        if match:
-            groups = match.groupdict()
-
-            error_type = groups['type']
-            error_name = groups['name']
-            error_text = groups['text']
-
-            if error_name in target_paths.get(error_type, {}):
-                return SanityMessage(
-                    message=error_text,
-                    path=target_paths[error_type][error_name],
-                )
-
-        return None

test/sanity/ignore.txt

@@ -5511,9 +5511,9 @@ lib/ansible/modules/system/user.py validate-modules:use-run-command-not-popen
 lib/ansible/modules/system/user.py validate-modules:doc-default-does-not-match-spec
 lib/ansible/modules/system/user.py validate-modules:doc-default-incompatible-type
 lib/ansible/modules/system/xfconf.py validate-modules:parameter-type-not-in-doc
-lib/ansible/modules/utilities/helper/_accelerate.py ansible-doc
 lib/ansible/modules/utilities/logic/async_status.py use-argspec-type-path
 lib/ansible/modules/utilities/logic/async_status.py validate-modules!skip
+lib/ansible/modules/utilities/logic/async_wrapper.py ansible-doc!skip  # not an actual module
 lib/ansible/modules/utilities/logic/async_wrapper.py use-argspec-type-path
 lib/ansible/modules/utilities/logic/wait_for.py pylint:blacklisted-name
 lib/ansible/modules/web_infrastructure/ansible_tower/tower_credential.py validate-modules:doc-choices-do-not-match-spec
@@ -5709,7 +5709,7 @@ lib/ansible/plugins/action/normal.py action-plugin-docs # default action plugin
 lib/ansible/plugins/action/nxos.py action-plugin-docs # base class for deprecated network platform modules using `connection: local`
 lib/ansible/plugins/action/sros.py action-plugin-docs # base class for deprecated network platform modules using `connection: local`
 lib/ansible/plugins/action/vyos.py action-plugin-docs # base class for deprecated network platform modules using `connection: local`
-lib/ansible/plugins/cache/base.py ansible-doc  # not a plugin, but a stub for backwards compatibility
+lib/ansible/plugins/cache/base.py ansible-doc!skip  # not a plugin, but a stub for backwards compatibility
 lib/ansible/plugins/callback/hipchat.py pylint:blacklisted-name
 lib/ansible/plugins/connection/lxc.py pylint:blacklisted-name
 lib/ansible/plugins/doc_fragments/a10.py future-import-boilerplate