From 27cd5e77d839527b418ba5978bf0f5ca10ae976a Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Wed, 25 Jan 2023 12:46:09 -0800 Subject: [PATCH] Update mypy and its dependencies. (#79812) * Update mypy and its dependencies. * Fix requirements. * Remove unnecessary matching constraints. --- lib/ansible/module_utils/common/yaml.py | 6 ++-- lib/ansible/module_utils/compat/typing.py | 2 +- lib/ansible/module_utils/urls.py | 6 ++-- lib/ansible/modules/pip.py | 2 +- lib/ansible/playbook/base.py | 2 +- lib/ansible/template/__init__.py | 4 +-- .../_data/requirements/sanity.mypy.in | 10 +++--- .../_data/requirements/sanity.mypy.txt | 31 +++++++++---------- .../_internal/commands/sanity/mypy.py | 11 +++++++ .../controller/sanity/mypy/ansible-test.ini | 8 ++--- test/sanity/ignore.txt | 6 ++++ 11 files changed, 52 insertions(+), 36 deletions(-) diff --git a/lib/ansible/module_utils/common/yaml.py b/lib/ansible/module_utils/common/yaml.py index e79cc096420..770d41d5338 100644 --- a/lib/ansible/module_utils/common/yaml.py +++ b/lib/ansible/module_utils/common/yaml.py @@ -24,12 +24,12 @@ if HAS_YAML: try: from yaml import CSafeLoader as SafeLoader from yaml import CSafeDumper as SafeDumper - from yaml.cyaml import CParser as Parser + from yaml.cyaml import CParser as Parser # type: ignore[attr-defined] HAS_LIBYAML = True except (ImportError, AttributeError): - from yaml import SafeLoader # type: ignore[misc] - from yaml import SafeDumper # type: ignore[misc] + from yaml import SafeLoader # type: ignore[assignment] + from yaml import SafeDumper # type: ignore[assignment] from yaml.parser import Parser # type: ignore[misc] yaml_load = _partial(_yaml.load, Loader=SafeLoader) diff --git a/lib/ansible/module_utils/compat/typing.py b/lib/ansible/module_utils/compat/typing.py index 27b25f7738b..d3760bf2d67 100644 --- a/lib/ansible/module_utils/compat/typing.py +++ b/lib/ansible/module_utils/compat/typing.py @@ -13,7 +13,7 @@ except Exception: # pylint: disable=broad-except pass try: - from typing import * # type: ignore[misc] + from typing import * # type: ignore[assignment] except Exception: # pylint: disable=broad-except pass diff --git a/lib/ansible/module_utils/urls.py b/lib/ansible/module_utils/urls.py index 542f89b08e3..1fbba99ae83 100644 --- a/lib/ansible/module_utils/urls.py +++ b/lib/ansible/module_utils/urls.py @@ -99,7 +99,7 @@ except ImportError: import urllib2 as urllib_request # type: ignore[no-redef] from urllib2 import AbstractHTTPHandler, BaseHandler # type: ignore[no-redef] -urllib_request.HTTPRedirectHandler.http_error_308 = urllib_request.HTTPRedirectHandler.http_error_307 # type: ignore[attr-defined] +urllib_request.HTTPRedirectHandler.http_error_308 = urllib_request.HTTPRedirectHandler.http_error_307 # type: ignore[attr-defined,assignment] try: from ansible.module_utils.six.moves.urllib.parse import urlparse, urlunparse, unquote @@ -181,7 +181,7 @@ try: from ssl import match_hostname, CertificateError except ImportError: try: - from backports.ssl_match_hostname import match_hostname, CertificateError # type: ignore[misc] + from backports.ssl_match_hostname import match_hostname, CertificateError # type: ignore[assignment] except ImportError: HAS_MATCH_HOSTNAME = False @@ -1461,7 +1461,7 @@ class Request: url = urlunparse(parsed_list) if use_gssapi: - if HTTPGSSAPIAuthHandler: + if HTTPGSSAPIAuthHandler: # type: ignore[truthy-function] handlers.append(HTTPGSSAPIAuthHandler(username, password)) else: imp_err_msg = missing_required_lib('gssapi', reason='for use_gssapi=True', diff --git a/lib/ansible/modules/pip.py b/lib/ansible/modules/pip.py index a9930ccdbf0..353c800a6e6 100644 --- a/lib/ansible/modules/pip.py +++ b/lib/ansible/modules/pip.py @@ -459,7 +459,7 @@ def _have_pip_module(): # type: () -> bool except ImportError: find_spec = None # type: ignore[assignment] # type: ignore[no-redef] - if find_spec: + if find_spec: # type: ignore[truthy-function] # noinspection PyBroadException try: # noinspection PyUnresolvedReferences diff --git a/lib/ansible/playbook/base.py b/lib/ansible/playbook/base.py index bf007581308..0f95c5fea9a 100644 --- a/lib/ansible/playbook/base.py +++ b/lib/ansible/playbook/base.py @@ -72,7 +72,7 @@ def _validate_action_group_metadata(action, found_group_metadata, fq_group_name) class FieldAttributeBase: - @classmethod + @classmethod # type: ignore[misc] @property def fattributes(cls): return cls._fattributes() diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py index 1498d3f8134..f6513042022 100644 --- a/lib/ansible/template/__init__.py +++ b/lib/ansible/template/__init__.py @@ -518,7 +518,7 @@ class AnsibleEnvironment(NativeEnvironment): ''' context_class = AnsibleContext template_class = AnsibleJ2Template - concat = staticmethod(ansible_eval_concat) + concat = staticmethod(ansible_eval_concat) # type: ignore[assignment] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -533,7 +533,7 @@ class AnsibleEnvironment(NativeEnvironment): class AnsibleNativeEnvironment(AnsibleEnvironment): - concat = staticmethod(ansible_native_concat) + concat = staticmethod(ansible_native_concat) # type: ignore[assignment] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/test/lib/ansible_test/_data/requirements/sanity.mypy.in b/test/lib/ansible_test/_data/requirements/sanity.mypy.in index 98dead6cac9..f01ae948d89 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.mypy.in +++ b/test/lib/ansible_test/_data/requirements/sanity.mypy.in @@ -1,10 +1,10 @@ -mypy[python2] != 0.971 # regression in 0.971 (see https://github.com/python/mypy/pull/13223) +mypy +cryptography # type stubs not published separately +jinja2 # type stubs not published separately packaging # type stubs not published separately types-backports -types-jinja2 -types-paramiko < 2.8.14 # newer versions drop support for Python 2.7 -types-pyyaml < 6 # PyYAML 6+ stubs do not support Python 2.7 -types-cryptography < 3.3.16 # newer versions drop support for Python 2.7 +types-paramiko +types-pyyaml types-requests types-setuptools types-toml diff --git a/test/lib/ansible_test/_data/requirements/sanity.mypy.txt b/test/lib/ansible_test/_data/requirements/sanity.mypy.txt index 9dffc8fbcfe..ba5699337d5 100644 --- a/test/lib/ansible_test/_data/requirements/sanity.mypy.txt +++ b/test/lib/ansible_test/_data/requirements/sanity.mypy.txt @@ -1,20 +1,19 @@ # edit "sanity.mypy.in" and generate with: hacking/update-sanity-requirements.py --test mypy -mypy==0.961 +cffi==1.15.1 +cryptography==39.0.0 +Jinja2==3.1.2 +MarkupSafe==2.1.2 +mypy==0.991 mypy-extensions==0.4.3 -packaging==21.3 -pyparsing==3.0.9 +packaging==23.0 +pycparser==2.21 tomli==2.0.1 -typed-ast==1.5.4 types-backports==0.1.3 -types-cryptography==3.3.15 -types-enum34==1.1.8 -types-ipaddress==1.0.8 -types-Jinja2==2.11.9 -types-MarkupSafe==1.1.10 -types-paramiko==2.8.13 -types-PyYAML==5.4.12 -types-requests==2.28.10 -types-setuptools==65.3.0 -types-toml==0.10.8 -types-urllib3==1.26.24 -typing_extensions==4.3.0 +types-docutils==0.19.1.2 +types-paramiko==2.12.0.3 +types-PyYAML==6.0.12.3 +types-requests==2.28.11.8 +types-setuptools==65.7.0.3 +types-toml==0.10.8.1 +types-urllib3==1.26.25.4 +typing_extensions==4.4.0 diff --git a/test/lib/ansible_test/_internal/commands/sanity/mypy.py b/test/lib/ansible_test/_internal/commands/sanity/mypy.py index c8497139782..24938c63519 100644 --- a/test/lib/ansible_test/_internal/commands/sanity/mypy.py +++ b/test/lib/ansible_test/_internal/commands/sanity/mypy.py @@ -19,6 +19,7 @@ from . import ( from ...constants import ( CONTROLLER_PYTHON_VERSIONS, REMOTE_ONLY_PYTHON_VERSIONS, + SUPPORTED_PYTHON_VERSIONS, ) from ...test import ( @@ -36,6 +37,7 @@ from ...util import ( ANSIBLE_TEST_CONTROLLER_ROOT, ApplicationError, is_subdir, + str_to_version, ) from ...util_common import ( @@ -72,6 +74,15 @@ class MypyTest(SanityMultipleVersion): target.path.startswith('lib/ansible/') or target.path.startswith('test/lib/ansible_test/_internal/') or target.path.startswith('test/lib/ansible_test/_util/target/sanity/import/'))] + @property + def supported_python_versions(self) -> t.Optional[tuple[str, ...]]: + """A tuple of supported Python versions or None if the test does not depend on specific Python versions.""" + # mypy 0.981 dropped support for Python 2 + # see: https://mypy-lang.blogspot.com/2022/09/mypy-0981-released.html + # cryptography dropped support for Python 3.5 in version 3.3 + # see: https://cryptography.io/en/latest/changelog/#v3-3 + return tuple(version for version in SUPPORTED_PYTHON_VERSIONS if str_to_version(version) >= (3, 6)) + @property def error_code(self) -> t.Optional[str]: """Error code for ansible-test matching the format used by the underlying test program, or None if the program does not use error codes.""" diff --git a/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-test.ini b/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-test.ini index 190e9529a0a..5e319794981 100644 --- a/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-test.ini +++ b/test/lib/ansible_test/_util/controller/sanity/mypy/ansible-test.ini @@ -6,10 +6,10 @@ # There are ~350 errors reported in ansible-test when strict optional checking is enabled. # Until the number of occurrences are greatly reduced, it's better to disable strict checking. strict_optional = False -# There are ~25 errors reported in ansible-test under the 'misc' code. -# The majority of those errors are "Only concrete class can be given", which is due to a limitation of mypy. -# See: https://github.com/python/mypy/issues/5374 -disable_error_code = misc +# There are ~13 type-abstract errors reported in ansible-test. +# This is due to assumptions mypy makes about Type and abstract types. +# See: https://discuss.python.org/t/add-abstracttype-to-the-typing-module/21996/13 +disable_error_code = type-abstract [mypy-argcomplete] ignore_missing_imports = True diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index 58a44d1ebd1..d4c908877b0 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -11,6 +11,12 @@ lib/ansible/executor/powershell/async_watchdog.ps1 pslint:PSCustomUseLiteralPath lib/ansible/executor/powershell/async_wrapper.ps1 pslint:PSCustomUseLiteralPath lib/ansible/executor/powershell/exec_wrapper.ps1 pslint:PSCustomUseLiteralPath lib/ansible/executor/task_queue_manager.py pylint:disallowed-name +lib/ansible/galaxy/collection/__init__.py mypy-3.9:attr-defined # inline ignore has no effect +lib/ansible/galaxy/collection/__init__.py mypy-3.10:attr-defined # inline ignore has no effect +lib/ansible/galaxy/collection/__init__.py mypy-3.11:attr-defined # inline ignore has no effect +lib/ansible/parsing/yaml/constructor.py mypy-3.9:type-var # too many occurrences to ignore inline +lib/ansible/parsing/yaml/constructor.py mypy-3.10:type-var # too many occurrences to ignore inline +lib/ansible/parsing/yaml/constructor.py mypy-3.11:type-var # too many occurrences to ignore inline lib/ansible/keyword_desc.yml no-unwanted-files lib/ansible/modules/apt.py validate-modules:parameter-invalid lib/ansible/modules/apt_repository.py validate-modules:parameter-invalid