Increase scope of mypy sanity test (#84288)

* Increase scope of mypy sanity test

* Fix issues reported by mypy

test/lib/ansible_test/_util/controller/sanity/integration-aliases/yaml_to_json.py

@@ -9,6 +9,6 @@ from yaml import load
 try:
     from yaml import CSafeLoader as SafeLoader
 except ImportError:
-    from yaml import SafeLoader
+    from yaml import SafeLoader  # type: ignore[assignment]
 
 json.dump(load(sys.stdin, Loader=SafeLoader), sys.stdout)

test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated.py

@@ -12,16 +12,6 @@ from tokenize import COMMENT, TokenInfo
 
 import astroid
 
-# support pylint 2.x and 3.x -- remove when supporting only 3.x
-try:
-    from pylint.interfaces import IAstroidChecker, ITokenChecker
-except ImportError:
-    class IAstroidChecker:
-        """Backwards compatibility for 2.x / 3.x support."""
-
-    class ITokenChecker:
-        """Backwards compatibility for 2.x / 3.x support."""
-
 try:
     from pylint.checkers.utils import check_messages
 except ImportError:
@@ -151,7 +141,6 @@ class AnsibleDeprecatedChecker(BaseChecker):
     has not passed or met the time for removal
     """
 
-    __implements__ = (IAstroidChecker,)
     name = 'deprecated'
     msgs = MSGS
 
@@ -296,8 +285,6 @@ class AnsibleDeprecatedCommentChecker(BaseTokenChecker):
     has not passed or met the time for removal
     """
 
-    __implements__ = (ITokenChecker,)
-
     name = 'deprecated-comment'
     msgs = {
         'E9601': ("Deprecated core version (%r) found: %s",

test/lib/ansible_test/_util/controller/sanity/pylint/plugins/string_format.py

@@ -6,13 +6,6 @@ from __future__ import annotations
 
 import astroid
 
-# support pylint 2.x and 3.x -- remove when supporting only 3.x
-try:
-    from pylint.interfaces import IAstroidChecker
-except ImportError:
-    class IAstroidChecker:
-        """Backwards compatibility for 2.x / 3.x support."""
-
 try:
     from pylint.checkers.utils import check_messages
 except ImportError:
@@ -38,7 +31,6 @@ class AnsibleStringFormatChecker(BaseChecker):
     is valid and the arguments match the format string.
     """
 
-    __implements__ = (IAstroidChecker,)
     name = 'string'
     msgs = MSGS
 

test/lib/ansible_test/_util/controller/sanity/pylint/plugins/unwanted.py

@@ -6,13 +6,6 @@ import typing as t
 
 import astroid
 
-# support pylint 2.x and 3.x -- remove when supporting only 3.x
-try:
-    from pylint.interfaces import IAstroidChecker
-except ImportError:
-    class IAstroidChecker:
-        """Backwards compatibility for 2.x / 3.x support."""
-
 from pylint.checkers import BaseChecker
 
 ANSIBLE_TEST_MODULES_PATH = os.environ['ANSIBLE_TEST_MODULES_PATH']
@@ -63,7 +56,6 @@ def is_module_path(path):  # type: (str) -> bool
 
 class AnsibleUnwantedChecker(BaseChecker):
     """Checker for unwanted imports and functions."""
-    __implements__ = (IAstroidChecker,)
 
     name = 'unwanted'
 

test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py

@@ -65,7 +65,7 @@ def setup_collection_loader():
 setup_collection_loader()
 
 from ansible import __version__ as ansible_version
-from ansible.executor.module_common import REPLACER_WINDOWS, NEW_STYLE_PYTHON_MODULE_RE
+from ansible.executor.module_common import REPLACER_WINDOWS as _REPLACER_WINDOWS, NEW_STYLE_PYTHON_MODULE_RE
 from ansible.module_utils.common.collections import is_iterable
 from ansible.module_utils.common.parameters import DEFAULT_TYPE_VALIDATORS
 from ansible.module_utils.compat.version import StrictVersion, LooseVersion
@@ -90,7 +90,7 @@ from .utils import CaptureStd, NoArgsAnsibleModule, compare_unordered_lists, par
 TRY_EXCEPT = ast.Try
 # REPLACER_WINDOWS from ansible.executor.module_common is byte
 # string but we need unicode for Python 3
-REPLACER_WINDOWS = REPLACER_WINDOWS.decode('utf-8')
+REPLACER_WINDOWS = _REPLACER_WINDOWS.decode('utf-8')
 
 REJECTLIST_DIRS = frozenset(('.git', 'test', '.github', '.idea'))
 INDENT_REGEX = re.compile(r'([\t]*)')
@@ -311,8 +311,8 @@ class ModuleValidator(Validator):
 
         self.analyze_arg_spec = analyze_arg_spec and plugin_type == 'module'
 
-        self._Version = LooseVersion
+        self._Version: type[LooseVersion | SemanticVersion] = LooseVersion
-        self._StrictVersion = StrictVersion
+        self._StrictVersion: type[StrictVersion | SemanticVersion] = StrictVersion
 
         self.collection = collection
         self.collection_name = 'ansible.builtin'

test/lib/ansible_test/_util/controller/sanity/yamllint/yamllinter.py

@@ -12,7 +12,7 @@ import yaml
 from yaml.resolver import Resolver
 from yaml.constructor import SafeConstructor
 from yaml.error import MarkedYAMLError
-from yaml.cyaml import CParser
+from yaml.cyaml import CParser  # type: ignore[attr-defined]
 
 from yamllint import linter
 from yamllint.config import YamlLintConfig
@@ -45,17 +45,17 @@ class TestConstructor(SafeConstructor):
 
 TestConstructor.add_constructor(
     '!unsafe',
-    TestConstructor.construct_yaml_unsafe)
+    TestConstructor.construct_yaml_unsafe)  # type: ignore[type-var]
 
 
 TestConstructor.add_constructor(
     '!vault',
-    TestConstructor.construct_yaml_str)
+    TestConstructor.construct_yaml_str)  # type: ignore[type-var]
 
 
 TestConstructor.add_constructor(
     '!vault-encrypted',
-    TestConstructor.construct_yaml_str)
+    TestConstructor.construct_yaml_str)  # type: ignore[type-var]
 
 
 class TestLoader(CParser, TestConstructor, Resolver):
@@ -135,11 +135,11 @@ class YamlChecker:
 
             self.messages += [self.result_to_message(r, path, lineno - 1, key) for r in messages]
 
-    def check_parsable(self, path, contents, lineno=1, allow_multiple=False, prefix=""):  # type: (str, str, int, bool) -> None
+    def check_parsable(self, path: str, contents: str, lineno: int = 1, allow_multiple: bool = False, prefix: str = "") -> None:
         """Check the given contents to verify they can be parsed as YAML."""
         prefix = f"{prefix}: " if prefix else ""
         try:
-            documents = len(list(yaml.load_all(contents, Loader=TestLoader)))
+            documents = len(list(yaml.load_all(contents, Loader=TestLoader)))  # type: ignore[arg-type]
             if documents > 1 and not allow_multiple:
                 self.messages += [{'code': 'multiple-yaml-documents',
                                    'message': f'{prefix}expected a single document in the stream',

test/lib/ansible_test/_util/controller/tools/yaml_to_json.py

@@ -10,7 +10,7 @@ from yaml import load
 try:
     from yaml import CSafeLoader as SafeLoader
 except ImportError:
-    from yaml import SafeLoader
+    from yaml import SafeLoader  # type: ignore[assignment]
 
 # unique ISO date marker matching the one present in importer.py
 ISO_DATE_MARKER = 'isodate:f23983df-f3df-453c-9904-bcd08af468cc:'

test/lib/ansible_test/_util/target/pytest/plugins/ansible_forked.py

@@ -63,6 +63,8 @@ def run_parent(item, pid, result_path):  # type: (Item, int, str) -> list[TestRe
     """Wait for the child process to exit and return the test reports. Called in the parent process."""
     exit_code = waitstatus_to_exitcode(os.waitpid(pid, 0)[1])
 
+    reports: list[TestReport]
+
     if exit_code:
         reason = "Test CRASHED with exit code {}.".format(exit_code)
         report = TestReport(item.nodeid, item.location, {x: 1 for x in item.keywords}, "failed", reason, "call", user_properties=item.user_properties)
@@ -73,8 +75,10 @@ def run_parent(item, pid, result_path):  # type: (Item, int, str) -> list[TestRe
 
         reports = [report]
     else:
+        captured_warnings: list[warnings.WarningMessage]
+
         with open(result_path, "rb") as result_file:
-            reports, captured_warnings = pickle.load(result_file)  # type: list[TestReport], list[warnings.WarningMessage]
+            reports, captured_warnings = pickle.load(result_file)
 
         for warning in captured_warnings:
             warnings.warn_explicit(warning.message, warning.category, warning.filename, warning.lineno)

test/lib/ansible_test/_util/target/pytest/plugins/ansible_pytest_collections.py

@@ -69,7 +69,7 @@ def enable_assertion_rewriting_hook():  # type: () -> None
     # noinspection PyProtectedMember
     from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionPkgLoaderBase
 
-    _AnsibleCollectionPkgLoaderBase.exec_module = exec_module
+    _AnsibleCollectionPkgLoaderBase.exec_module = exec_module  # type: ignore[method-assign]
 
 
 def pytest_configure():

test/lib/ansible_test/_util/target/setup/requirements.py

@@ -62,13 +62,15 @@ def main():  # type: () -> None
 
 
 # noinspection PyUnusedLocal
-def bootstrap(pip, options):  # type: (str, t.Dict[str, t.Any]) -> None
+def bootstrap(pip: str, options: dict[str, t.Any]) -> None:
     """Bootstrap pip and related packages in an empty virtual environment."""
     pip_version = options['pip_version']
     packages = options['packages']
     setuptools = options['setuptools']
     wheel = options['wheel']
 
+    del options
+
     url = 'https://ci-files.testing.ansible.com/ansible-test/get-pip-%s.py' % pip_version
     cache_path = os.path.expanduser('~/.ansible/test/cache/get_pip_%s.py' % pip_version.replace(".", "_"))
     temp_path = cache_path + '.download'
@@ -100,31 +102,33 @@ https://github.com/ansible/ansible/issues/77304
     env = common_pip_environment()
     env.update(GET_PIP=cache_path)
 
-    options = common_pip_options()
+    pip_options = common_pip_options()
-    options.extend(packages)
+    pip_options.extend(packages)
 
     if not setuptools:
-        options.append('--no-setuptools')
+        pip_options.append('--no-setuptools')
 
     if not wheel:
-        options.append('--no-wheel')
+        pip_options.append('--no-wheel')
 
-    command = [sys.executable, pip] + options
+    command = [sys.executable, pip] + pip_options
 
     execute_command(command, env=env)
 
 
-def install(pip, options):  # type: (str, t.Dict[str, t.Any]) -> None
+def install(pip: str, options: dict[str, t.Any]) -> None:
     """Perform a pip install."""
     requirements = options['requirements']
     constraints = options['constraints']
     packages = options['packages']
 
+    del options
+
     tempdir = tempfile.mkdtemp(prefix='ansible-test-', suffix='-requirements')
 
     try:
-        options = common_pip_options()
+        pip_options = common_pip_options()
-        options.extend(packages)
+        pip_options.extend(packages)
 
         for path, content in requirements:
             if path.split(os.sep)[0] in ('test', 'requirements'):
@@ -136,13 +140,13 @@ def install(pip, options):  # type: (str, t.Dict[str, t.Any]) -> None
                     pre_build.execute(pip)
 
             write_text_file(os.path.join(tempdir, path), content, True)
-            options.extend(['-r', path])
+            pip_options.extend(['-r', path])
 
         for path, content in constraints:
             write_text_file(os.path.join(tempdir, path), content, True)
-            options.extend(['-c', path])
+            pip_options.extend(['-c', path])
 
-        command = [sys.executable, pip, 'install'] + options
+        command = [sys.executable, pip, 'install'] + pip_options
 
         env = common_pip_environment()
 
@@ -163,8 +167,8 @@ class PreBuild:
         tempdir = tempfile.mkdtemp(prefix='ansible-test-', suffix='-pre-build')
 
         try:
-            options = common_pip_options()
+            pip_options = common_pip_options()
-            options.append(self.requirement)
+            pip_options.append(self.requirement)
 
             constraints = '\n'.join(self.constraints) + '\n'
             constraints_path = os.path.join(tempdir, 'constraints.txt')
@@ -174,7 +178,7 @@ class PreBuild:
             env = common_pip_environment()
             env.update(PIP_CONSTRAINT=constraints_path)
 
-            command = [sys.executable, pip, 'wheel'] + options
+            command = [sys.executable, pip, 'wheel'] + pip_options
 
             execute_command(command, env=env, cwd=tempdir)
         finally:
@@ -206,15 +210,17 @@ def parse_pre_build_instructions(requirements):  # type: (str) -> list[PreBuild]
     return instructions
 
 
-def uninstall(pip, options):  # type: (str, t.Dict[str, t.Any]) -> None
+def uninstall(pip: str, options: dict[str, t.Any]) -> None:
     """Perform a pip uninstall."""
     packages = options['packages']
     ignore_errors = options['ignore_errors']
 
-    options = common_pip_options()
+    del options
-    options.extend(packages)
+
+    pip_options = common_pip_options()
+    pip_options.extend(packages)
 
-    command = [sys.executable, pip, 'uninstall', '-y'] + options
+    command = [sys.executable, pip, 'uninstall', '-y'] + pip_options
 
     env = common_pip_environment()
 
@@ -226,13 +232,13 @@ def uninstall(pip, options):  # type: (str, t.Dict[str, t.Any]) -> None
 
 
 # noinspection PyUnusedLocal
-def version(pip, options):  # type: (str, t.Dict[str, t.Any]) -> None
+def version(pip: str, options: dict[str, t.Any]) -> None:
     """Report the pip version."""
     del options
 
-    options = common_pip_options()
+    pip_options = common_pip_options()
 
-    command = [sys.executable, pip, '-V'] + options
+    command = [sys.executable, pip, '-V'] + pip_options
 
     env = common_pip_environment()
 
@@ -264,11 +270,11 @@ def common_pip_options():  # type: () -> t.List[str]
 def devnull():  # type: () -> t.IO[bytes]
     """Return a file object that references devnull."""
     try:
-        return devnull.file
+        return devnull.file  # type: ignore[attr-defined]
     except AttributeError:
-        devnull.file = open(os.devnull, 'w+b')  # pylint: disable=consider-using-with
+        devnull.file = open(os.devnull, 'w+b')  # type: ignore[attr-defined]  # pylint: disable=consider-using-with
 
-    return devnull.file
+    return devnull.file  # type: ignore[attr-defined]
 
 
 def download_file(url, path):  # type: (str, str) -> None

test/lib/ansible_test/_util/target/tools/virtualenvcheck.py

@@ -5,7 +5,7 @@ import json
 
 try:
     # virtualenv <20
-    from sys import real_prefix
+    from sys import real_prefix  # type: ignore[attr-defined]
 except ImportError:
     real_prefix = None
 

test/lib/ansible_test/_util/target/tools/yamlcheck.py

@@ -11,7 +11,7 @@ except ImportError:
 try:
     from yaml import CLoader
 except ImportError:
-    CLoader = None
+    CLoader = None  # type: ignore[misc]
 
 print(json.dumps(dict(
     yaml=bool(yaml),

test/sanity/code-smell/mypy.json

@@ -1,10 +1,10 @@
 {
     "prefixes": [
         "lib/ansible/",
-        "test/lib/ansible_test/_internal/",
         "packaging/",
-        "test/units",
+        "test/lib/ansible_test/",
-        "test/lib/ansible_test/_util/target/sanity/import/"
+        "test/sanity/",
+        "test/units/"
     ],
     "extensions": [
         ".py"

test/sanity/code-smell/mypy.py

@@ -31,13 +31,16 @@ def main() -> None:
     remote_only_python_versions = os.environ['ANSIBLE_TEST_REMOTE_ONLY_PYTHON_VERSIONS'].split(',')
 
     contexts = (
-        MyPyContext('ansible-test', ['test/lib/ansible_test/_util/target/sanity/import/'], controller_python_versions),
+        MyPyContext('ansible-test', ['test/lib/ansible_test/'], controller_python_versions),
-        MyPyContext('ansible-test', ['test/lib/ansible_test/_internal/'], controller_python_versions),
+        MyPyContext('ansible-test', ['test/lib/ansible_test/_util/target/'], remote_only_python_versions),
+
         MyPyContext('ansible-core', ['lib/ansible/'], controller_python_versions),
-        MyPyContext('modules', ['lib/ansible/modules/', 'lib/ansible/module_utils/'], remote_only_python_versions),
+        MyPyContext('ansible-core', ['lib/ansible/modules/', 'lib/ansible/module_utils/'], remote_only_python_versions),
-        MyPyContext('packaging', ['packaging/'], controller_python_versions),
+
-        MyPyContext('modules', ['test/units/modules/', 'test/units/module_utils/'], remote_only_python_versions),
         MyPyContext('ansible-core', ['test/units/'], controller_python_versions),
+        MyPyContext('ansible-core', ['test/units/modules/', 'test/units/module_utils/'], remote_only_python_versions),
+
+        MyPyContext('packaging', ['packaging/'], controller_python_versions),
     )
 
     unfiltered_messages: list[SanityMessage] = []

test/sanity/code-smell/mypy/ansible-test.ini

@@ -26,3 +26,45 @@ ignore_missing_imports = True
 
 [mypy-StringIO]
 ignore_missing_imports = True
+
+[mypy-voluptuous]
+ignore_missing_imports = True
+
+[mypy-voluptuous.humanize]
+ignore_missing_imports = True
+
+[mypy-astroid]
+ignore_missing_imports = True
+
+[mypy-pylint.interfaces]
+ignore_missing_imports = True
+
+[mypy-pylint.checkers]
+ignore_missing_imports = True
+
+[mypy-pylint.checkers.utils]
+ignore_missing_imports = True
+
+[mypy-pylint.lint]
+ignore_missing_imports = True
+
+[mypy-antsibull_docs_parser]
+ignore_missing_imports = True
+
+[mypy-antsibull_docs_parser.parser]
+ignore_missing_imports = True
+
+[mypy-yamllint]
+ignore_missing_imports = True
+
+[mypy-yamllint.config]
+ignore_missing_imports = True
+
+[mypy-py]
+ignore_missing_imports = True
+
+[mypy-py._path]
+ignore_missing_imports = True
+
+[mypy-py._path.local]
+ignore_missing_imports = True

test/sanity/code-smell/mypy/modules.ini

@@ -1,92 +0,0 @@
-# IMPORTANT
-# Set "ignore_missing_imports" per package below, rather than globally.
-# That will help identify missing type stubs that should be added to the sanity test environment.
-
-[mypy]
-
-[mypy-ansible.module_utils.six.moves.*]
-ignore_missing_imports = True
-
-[mypy-pexpect.*]
-ignore_missing_imports = True
-
-[mypy-md5.*]
-ignore_missing_imports = True
-
-[mypy-rpmUtils.*]
-ignore_missing_imports = True
-
-[mypy-rpm.*]
-ignore_missing_imports = True
-
-[mypy-psutil.*]
-ignore_missing_imports = True
-
-[mypy-dnf.*]
-ignore_missing_imports = True
-
-[mypy-apt.*]
-ignore_missing_imports = True
-
-[mypy-apt_pkg.*]
-ignore_missing_imports = True
-
-[mypy-gssapi.*]
-ignore_missing_imports = True
-
-[mypy-_ssl.*]
-ignore_missing_imports = True
-
-[mypy-urllib_gssapi.*]
-ignore_missing_imports = True
-
-[mypy-systemd.*]
-ignore_missing_imports = True
-
-[mypy-sha.*]
-ignore_missing_imports = True
-
-[mypy-distro.*]
-ignore_missing_imports = True
-
-[mypy-selinux.*]
-ignore_missing_imports = True
-
-[mypy-urllib2.*]
-ignore_missing_imports = True
-
-[mypy-httplib.*]
-ignore_missing_imports = True
-
-[mypy-compiler.*]
-ignore_missing_imports = True
-
-[mypy-aptsources.*]
-ignore_missing_imports = True
-
-[mypy-urllib3.*]
-ignore_missing_imports = True
-
-[mypy-requests.*]
-ignore_missing_imports = True
-
-[mypy-pkg_resources.*]
-ignore_missing_imports = True
-
-[mypy-urllib.*]
-ignore_missing_imports = True
-
-[mypy-email.*]
-ignore_missing_imports = True
-
-[mypy-selectors.*]
-ignore_missing_imports = True
-
-[mypy-importlib.*]
-ignore_missing_imports = True
-
-[mypy-collections.*]
-ignore_missing_imports = True
-
-[mypy-http.*]
-ignore_missing_imports = True