Update mypy and its dependencies. (#79812)

* Update mypy and its dependencies.

* Fix requirements.

* Remove unnecessary matching constraints.

lib/ansible/module_utils/common/yaml.py

@@ -24,12 +24,12 @@ if HAS_YAML:
     try:
         from yaml import CSafeLoader as SafeLoader
         from yaml import CSafeDumper as SafeDumper
-        from yaml.cyaml import CParser as Parser
+        from yaml.cyaml import CParser as Parser  # type: ignore[attr-defined]
 
         HAS_LIBYAML = True
     except (ImportError, AttributeError):
-        from yaml import SafeLoader  # type: ignore[misc]
-        from yaml import SafeDumper  # type: ignore[misc]
+        from yaml import SafeLoader  # type: ignore[assignment]
+        from yaml import SafeDumper  # type: ignore[assignment]
         from yaml.parser import Parser  # type: ignore[misc]
 
     yaml_load = _partial(_yaml.load, Loader=SafeLoader)

lib/ansible/module_utils/compat/typing.py

@@ -13,7 +13,7 @@ except Exception:  # pylint: disable=broad-except
     pass
 
 try:
-    from typing import *  # type: ignore[misc]
+    from typing import *  # type: ignore[assignment]
 except Exception:  # pylint: disable=broad-except
     pass
 

lib/ansible/module_utils/urls.py

@@ -99,7 +99,7 @@ except ImportError:
     import urllib2 as urllib_request  # type: ignore[no-redef]
     from urllib2 import AbstractHTTPHandler, BaseHandler  # type: ignore[no-redef]
 
-urllib_request.HTTPRedirectHandler.http_error_308 = urllib_request.HTTPRedirectHandler.http_error_307  # type: ignore[attr-defined]
+urllib_request.HTTPRedirectHandler.http_error_308 = urllib_request.HTTPRedirectHandler.http_error_307  # type: ignore[attr-defined,assignment]
 
 try:
     from ansible.module_utils.six.moves.urllib.parse import urlparse, urlunparse, unquote
@@ -181,7 +181,7 @@ try:
     from ssl import match_hostname, CertificateError
 except ImportError:
     try:
-        from backports.ssl_match_hostname import match_hostname, CertificateError  # type: ignore[misc]
+        from backports.ssl_match_hostname import match_hostname, CertificateError  # type: ignore[assignment]
     except ImportError:
         HAS_MATCH_HOSTNAME = False
 
@@ -1461,7 +1461,7 @@ class Request:
                 url = urlunparse(parsed_list)
 
             if use_gssapi:
-                if HTTPGSSAPIAuthHandler:
+                if HTTPGSSAPIAuthHandler:  # type: ignore[truthy-function]
                     handlers.append(HTTPGSSAPIAuthHandler(username, password))
                 else:
                     imp_err_msg = missing_required_lib('gssapi', reason='for use_gssapi=True',

lib/ansible/modules/pip.py

@@ -459,7 +459,7 @@ def _have_pip_module():  # type: () -> bool
     except ImportError:
         find_spec = None  # type: ignore[assignment] # type: ignore[no-redef]
 
-    if find_spec:
+    if find_spec:  # type: ignore[truthy-function]
         # noinspection PyBroadException
         try:
             # noinspection PyUnresolvedReferences

lib/ansible/playbook/base.py

@@ -72,7 +72,7 @@ def _validate_action_group_metadata(action, found_group_metadata, fq_group_name)
 
 class FieldAttributeBase:
 
-    @classmethod
+    @classmethod  # type: ignore[misc]
     @property
     def fattributes(cls):
         return cls._fattributes()

lib/ansible/template/__init__.py

@@ -518,7 +518,7 @@ class AnsibleEnvironment(NativeEnvironment):
     '''
     context_class = AnsibleContext
     template_class = AnsibleJ2Template
-    concat = staticmethod(ansible_eval_concat)
+    concat = staticmethod(ansible_eval_concat)  # type: ignore[assignment]
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -533,7 +533,7 @@ class AnsibleEnvironment(NativeEnvironment):
 
 
 class AnsibleNativeEnvironment(AnsibleEnvironment):
-    concat = staticmethod(ansible_native_concat)
+    concat = staticmethod(ansible_native_concat)  # type: ignore[assignment]
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)

test/lib/ansible_test/_data/requirements/sanity.mypy.in

@@ -1,10 +1,10 @@
-mypy[python2] != 0.971  # regression in 0.971 (see https://github.com/python/mypy/pull/13223)
+mypy
+cryptography  # type stubs not published separately
+jinja2  # type stubs not published separately
 packaging  # type stubs not published separately
 types-backports
-types-jinja2
-types-paramiko < 2.8.14  # newer versions drop support for Python 2.7
-types-pyyaml < 6  # PyYAML 6+ stubs do not support Python 2.7
-types-cryptography < 3.3.16  # newer versions drop support for Python 2.7
+types-paramiko
+types-pyyaml
 types-requests
 types-setuptools
 types-toml

test/lib/ansible_test/_data/requirements/sanity.mypy.txt

@@ -1,20 +1,19 @@
 # edit "sanity.mypy.in" and generate with: hacking/update-sanity-requirements.py --test mypy
-mypy==0.961
+cffi==1.15.1
+cryptography==39.0.0
+Jinja2==3.1.2
+MarkupSafe==2.1.2
+mypy==0.991
 mypy-extensions==0.4.3
-packaging==21.3
-pyparsing==3.0.9
+packaging==23.0
+pycparser==2.21
 tomli==2.0.1
-typed-ast==1.5.4
 types-backports==0.1.3
-types-cryptography==3.3.15
-types-enum34==1.1.8
-types-ipaddress==1.0.8
-types-Jinja2==2.11.9
-types-MarkupSafe==1.1.10
-types-paramiko==2.8.13
-types-PyYAML==5.4.12
-types-requests==2.28.10
-types-setuptools==65.3.0
-types-toml==0.10.8
-types-urllib3==1.26.24
-typing_extensions==4.3.0
+types-docutils==0.19.1.2
+types-paramiko==2.12.0.3
+types-PyYAML==6.0.12.3
+types-requests==2.28.11.8
+types-setuptools==65.7.0.3
+types-toml==0.10.8.1
+types-urllib3==1.26.25.4
+typing_extensions==4.4.0

test/lib/ansible_test/_internal/commands/sanity/mypy.py

@@ -19,6 +19,7 @@ from . import (
 from ...constants import (
     CONTROLLER_PYTHON_VERSIONS,
     REMOTE_ONLY_PYTHON_VERSIONS,
+    SUPPORTED_PYTHON_VERSIONS,
 )
 
 from ...test import (
@@ -36,6 +37,7 @@ from ...util import (
     ANSIBLE_TEST_CONTROLLER_ROOT,
     ApplicationError,
     is_subdir,
+    str_to_version,
 )
 
 from ...util_common import (
@@ -72,6 +74,15 @@ class MypyTest(SanityMultipleVersion):
                 target.path.startswith('lib/ansible/') or target.path.startswith('test/lib/ansible_test/_internal/')
                 or target.path.startswith('test/lib/ansible_test/_util/target/sanity/import/'))]
 
+    @property
+    def supported_python_versions(self) -> t.Optional[tuple[str, ...]]:
+        """A tuple of supported Python versions or None if the test does not depend on specific Python versions."""
+        # mypy 0.981 dropped support for Python 2
+        # see: https://mypy-lang.blogspot.com/2022/09/mypy-0981-released.html
+        # cryptography dropped support for Python 3.5 in version 3.3
+        # see: https://cryptography.io/en/latest/changelog/#v3-3
+        return tuple(version for version in SUPPORTED_PYTHON_VERSIONS if str_to_version(version) >= (3, 6))
+
     @property
     def error_code(self) -> t.Optional[str]:
         """Error code for ansible-test matching the format used by the underlying test program, or None if the program does not use error codes."""

test/lib/ansible_test/_util/controller/sanity/mypy/ansible-test.ini

@@ -6,10 +6,10 @@
 # There are ~350 errors reported in ansible-test when strict optional checking is enabled.
 # Until the number of occurrences are greatly reduced, it's better to disable strict checking.
 strict_optional = False
-# There are ~25 errors reported in ansible-test under the 'misc' code.
-# The majority of those errors are "Only concrete class can be given", which is due to a limitation of mypy.
-# See: https://github.com/python/mypy/issues/5374
-disable_error_code = misc
+# There are ~13 type-abstract errors reported in ansible-test.
+# This is due to assumptions mypy makes about Type and abstract types.
+# See: https://discuss.python.org/t/add-abstracttype-to-the-typing-module/21996/13
+disable_error_code = type-abstract
 
 [mypy-argcomplete]
 ignore_missing_imports = True

test/sanity/ignore.txt

@@ -11,6 +11,12 @@ lib/ansible/executor/powershell/async_watchdog.ps1 pslint:PSCustomUseLiteralPath
 lib/ansible/executor/powershell/async_wrapper.ps1 pslint:PSCustomUseLiteralPath
 lib/ansible/executor/powershell/exec_wrapper.ps1 pslint:PSCustomUseLiteralPath
 lib/ansible/executor/task_queue_manager.py pylint:disallowed-name
+lib/ansible/galaxy/collection/__init__.py mypy-3.9:attr-defined  # inline ignore has no effect
+lib/ansible/galaxy/collection/__init__.py mypy-3.10:attr-defined  # inline ignore has no effect
+lib/ansible/galaxy/collection/__init__.py mypy-3.11:attr-defined  # inline ignore has no effect
+lib/ansible/parsing/yaml/constructor.py mypy-3.9:type-var  # too many occurrences to ignore inline
+lib/ansible/parsing/yaml/constructor.py mypy-3.10:type-var  # too many occurrences to ignore inline
+lib/ansible/parsing/yaml/constructor.py mypy-3.11:type-var  # too many occurrences to ignore inline
 lib/ansible/keyword_desc.yml no-unwanted-files
 lib/ansible/modules/apt.py validate-modules:parameter-invalid
 lib/ansible/modules/apt_repository.py validate-modules:parameter-invalid