From f5c742cdfd800b42a8f9ac2e88b652e02903e848 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Wed, 11 Oct 2023 19:04:30 -0700 Subject: [PATCH] ansible-test - Clean up old Python 2 compat code (#81962) * Update generated code __future__ import * Remove Python 2.x compat from compile sanity test * Remove __metaclass__ from pylint good names list * Remove Python 2.x compat from ansible-test injector * Remove Python 2.x compat from ansible-test importer * Remove Python 2.x compat from ansible-test units * Remove Python 2.x compat from validate-modules * Remove Python 2.x compat from pylint plugins * Remove more Python 2.x compat from ansible-test * Remove Python 2.x openssl detection in ansible-test * Remove obsolete Python 2.x comment * Remove obsolete ansible-test Python 2.x support * Remove unused bootstrap script functions * Keep mypy happy * Remove unused imports * Keep pylint happy --- test/lib/ansible_test/__init__.py | 3 +- test/lib/ansible_test/_internal/config.py | 1 - .../ansible_test/_internal/content_config.py | 5 --- .../_internal/python_requirements.py | 33 +++---------------- .../lib/ansible_test/_internal/util_common.py | 2 +- test/lib/ansible_test/_internal/venv.py | 2 +- test/lib/ansible_test/_util/__init__.py | 3 +- .../pylint/config/ansible-test-target.cfg | 1 - .../sanity/pylint/config/ansible-test.cfg | 1 - .../sanity/pylint/config/code-smell.cfg | 1 - .../sanity/pylint/plugins/deprecated.py | 3 +- .../validate-modules/validate_modules/main.py | 26 +++++++-------- .../lib/ansible_test/_util/target/__init__.py | 3 +- .../_util/target/common/__init__.py | 2 -- .../_util/target/injector/python.py | 20 ++--------- .../plugins/ansible_pytest_collections.py | 3 -- .../_util/target/sanity/compile/compile.py | 12 ++----- .../_util/target/sanity/import/importer.py | 10 ------ .../_util/target/setup/bootstrap.sh | 23 ------------- 19 files changed, 26 insertions(+), 128 deletions(-) delete mode 100644 test/lib/ansible_test/_util/target/common/__init__.py diff --git a/test/lib/ansible_test/__init__.py b/test/lib/ansible_test/__init__.py index 527d413a98d..223f9b4bcfe 100644 --- a/test/lib/ansible_test/__init__.py +++ b/test/lib/ansible_test/__init__.py @@ -1,2 +1 @@ -# Empty __init__.py to allow importing of `ansible_test._util.target.common` under Python 2.x. -# This allows the ansible-test entry point to report supported Python versions before exiting. +# Empty __init__.py to allow relative imports to work under mypy. diff --git a/test/lib/ansible_test/_internal/config.py b/test/lib/ansible_test/_internal/config.py index dbc137b5a29..18d815c8bc3 100644 --- a/test/lib/ansible_test/_internal/config.py +++ b/test/lib/ansible_test/_internal/config.py @@ -65,7 +65,6 @@ class ContentConfig: modules: ModulesConfig python_versions: tuple[str, ...] - py2_support: bool class EnvironmentConfig(CommonConfig): diff --git a/test/lib/ansible_test/_internal/content_config.py b/test/lib/ansible_test/_internal/content_config.py index 7ac1876c09e..c9c37df646b 100644 --- a/test/lib/ansible_test/_internal/content_config.py +++ b/test/lib/ansible_test/_internal/content_config.py @@ -29,7 +29,6 @@ from .io import ( from .util import ( ApplicationError, display, - str_to_version, ) from .data import ( @@ -75,13 +74,9 @@ def parse_content_config(data: t.Any) -> ContentConfig: python_versions = tuple(version for version in SUPPORTED_PYTHON_VERSIONS if version in CONTROLLER_PYTHON_VERSIONS or version in modules.python_versions) - # True if Python 2.x is supported. - py2_support = any(version for version in python_versions if str_to_version(version)[0] == 2) - return ContentConfig( modules=modules, python_versions=python_versions, - py2_support=py2_support, ) diff --git a/test/lib/ansible_test/_internal/python_requirements.py b/test/lib/ansible_test/_internal/python_requirements.py index b5c2fbdaede..80e0c20fc6a 100644 --- a/test/lib/ansible_test/_internal/python_requirements.py +++ b/test/lib/ansible_test/_internal/python_requirements.py @@ -5,7 +5,6 @@ import base64 import dataclasses import json import os -import re import typing as t from .encoding import ( @@ -24,9 +23,7 @@ from .util import ( ApplicationError, SubprocessError, display, - find_executable, raw_command, - str_to_version, version_to_str, ) @@ -536,34 +533,12 @@ def get_cryptography_requirements(python: PythonConfig) -> list[str]: def get_openssl_version(python: PythonConfig) -> t.Optional[tuple[int, ...]]: """Return the openssl version.""" - if not python.version.startswith('2.'): - # OpenSSL version checking only works on Python 3.x. - # This should be the most accurate, since it is the Python we will be using. - version = json.loads(raw_command([python.path, os.path.join(ANSIBLE_TEST_TOOLS_ROOT, 'sslcheck.py')], capture=True)[0])['version'] + version = json.loads(raw_command([python.path, os.path.join(ANSIBLE_TEST_TOOLS_ROOT, 'sslcheck.py')], capture=True)[0])['version'] - if version: - display.info(f'Detected OpenSSL version {version_to_str(version)} under Python {python.version}.', verbosity=1) + if version: + display.info(f'Detected OpenSSL version {version_to_str(version)} under Python {python.version}.', verbosity=1) - return tuple(version) - - # Fall back to detecting the OpenSSL version from the CLI. - # This should provide an adequate solution on Python 2.x. - openssl_path = find_executable('openssl', required=False) - - if openssl_path: - try: - result = raw_command([openssl_path, 'version'], capture=True)[0] - except SubprocessError: - result = '' - - match = re.search(r'^OpenSSL (?P[0-9]+\.[0-9]+\.[0-9]+)', result) - - if match: - version = str_to_version(match.group('version')) - - display.info(f'Detected OpenSSL version {version_to_str(version)} using the openssl CLI.', verbosity=1) - - return version + return tuple(version) display.info('Unable to detect OpenSSL version.', verbosity=1) diff --git a/test/lib/ansible_test/_internal/util_common.py b/test/lib/ansible_test/_internal/util_common.py index 77a6165c8e7..a6904971b8a 100644 --- a/test/lib/ansible_test/_internal/util_common.py +++ b/test/lib/ansible_test/_internal/util_common.py @@ -409,7 +409,7 @@ def create_interpreter_wrapper(interpreter: str, injected_interpreter: str) -> N code = textwrap.dedent(''' #!%s - from __future__ import absolute_import + from __future__ import annotations from os import execv from sys import argv diff --git a/test/lib/ansible_test/_internal/venv.py b/test/lib/ansible_test/_internal/venv.py index 68541fee21b..cdd73b0adb6 100644 --- a/test/lib/ansible_test/_internal/venv.py +++ b/test/lib/ansible_test/_internal/venv.py @@ -159,7 +159,7 @@ def run_venv( pip: bool, path: str, ) -> bool: - """Create a virtual environment using the 'venv' module. Not available on Python 2.x.""" + """Create a virtual environment using the 'venv' module.""" cmd = [run_python, '-m', 'venv'] if system_site_packages: diff --git a/test/lib/ansible_test/_util/__init__.py b/test/lib/ansible_test/_util/__init__.py index 527d413a98d..8f58623f0ac 100644 --- a/test/lib/ansible_test/_util/__init__.py +++ b/test/lib/ansible_test/_util/__init__.py @@ -1,2 +1 @@ -# Empty __init__.py to allow importing of `ansible_test._util.target.common` under Python 2.x. -# This allows the ansible-test entry point to report supported Python versions before exiting. +# Empty __init__.py to keep pylint happy. diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test-target.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test-target.cfg index f8a0a8af3ff..51d0bb01d38 100644 --- a/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test-target.cfg +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test-target.cfg @@ -35,7 +35,6 @@ bad-names= tutu, good-names= - __metaclass__, C, ex, i, diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg index 5bec36fdebb..801adbe145c 100644 --- a/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/ansible-test.cfg @@ -34,7 +34,6 @@ bad-names= tutu, good-names= - __metaclass__, C, ex, i, diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/config/code-smell.cfg b/test/lib/ansible_test/_util/controller/sanity/pylint/config/code-smell.cfg index c30eb37a749..adc0b95e947 100644 --- a/test/lib/ansible_test/_util/controller/sanity/pylint/config/code-smell.cfg +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/config/code-smell.cfg @@ -33,7 +33,6 @@ bad-names= tutu, good-names= - __metaclass__, C, ex, i, diff --git a/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated.py b/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated.py index f6c833738d8..be4dba57bc3 100644 --- a/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated.py +++ b/test/lib/ansible_test/_util/controller/sanity/pylint/plugins/deprecated.py @@ -32,7 +32,6 @@ except ImportError: from pylint.checkers import BaseChecker, BaseTokenChecker from ansible.module_utils.compat.version import LooseVersion -from ansible.module_utils.six import string_types from ansible.release import __version__ as ansible_version_raw from ansible.utils.version import SemanticVersion @@ -137,7 +136,7 @@ def _get_func_name(node): def parse_isodate(value): """Parse an ISO 8601 date string.""" msg = 'Expected ISO 8601 date string (YYYY-MM-DD)' - if not isinstance(value, string_types): + if not isinstance(value, str): raise ValueError(msg) # From Python 3.7 in, there is datetime.date.fromisoformat(). For older versions, # we have to do things manually. diff --git a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py index 2b92a56c205..29107910453 100644 --- a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py +++ b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py @@ -70,7 +70,6 @@ from ansible.module_utils.common.collections import is_iterable from ansible.module_utils.common.parameters import DEFAULT_TYPE_VALIDATORS from ansible.module_utils.compat.version import StrictVersion, LooseVersion from ansible.module_utils.basic import to_bytes -from ansible.module_utils.six import PY3, with_metaclass, string_types from ansible.plugins.loader import fragment_loader from ansible.plugins.list import IGNORE as REJECTLIST from ansible.utils.plugin_docs import add_collection_to_versions_and_dates, add_fragments, get_docstring @@ -87,14 +86,11 @@ from .schema import ( from .utils import CaptureStd, NoArgsAnsibleModule, compare_unordered_lists, parse_yaml, parse_isodate -if PY3: - # Because there is no ast.TryExcept in Python 3 ast module - TRY_EXCEPT = ast.Try - # REPLACER_WINDOWS from ansible.executor.module_common is byte - # string but we need unicode for Python 3 - REPLACER_WINDOWS = REPLACER_WINDOWS.decode('utf-8') -else: - TRY_EXCEPT = ast.TryExcept +# Because there is no ast.TryExcept in Python 3 ast module +TRY_EXCEPT = ast.Try +# REPLACER_WINDOWS from ansible.executor.module_common is byte +# string but we need unicode for Python 3 +REPLACER_WINDOWS = REPLACER_WINDOWS.decode('utf-8') REJECTLIST_DIRS = frozenset(('.git', 'test', '.github', '.idea')) INDENT_REGEX = re.compile(r'([\t]*)') @@ -269,7 +265,7 @@ class Reporter: return 3 if sum(ret) else 0 -class Validator(with_metaclass(abc.ABCMeta, object)): +class Validator(metaclass=abc.ABCMeta): """Validator instances are intended to be run on a single object. if you are scanning multiple objects for problems, you'll want to have a separate Validator for each one.""" @@ -1193,7 +1189,7 @@ class ModuleValidator(Validator): for entry in object: self._validate_semantic_markup(entry) return - if not isinstance(object, string_types): + if not isinstance(object, str): return if self.collection: @@ -1374,7 +1370,7 @@ class ModuleValidator(Validator): continue bad_term = False for term in check: - if not isinstance(term, string_types): + if not isinstance(term, str): msg = name if context: msg += " found in %s" % " -> ".join(context) @@ -1442,7 +1438,7 @@ class ModuleValidator(Validator): continue bad_term = False for term in requirements: - if not isinstance(term, string_types): + if not isinstance(term, str): msg = "required_if" if context: msg += " found in %s" % " -> ".join(context) @@ -1525,13 +1521,13 @@ class ModuleValidator(Validator): # This is already reported by schema checking return for key, value in terms.items(): - if isinstance(value, string_types): + if isinstance(value, str): value = [value] if not isinstance(value, (list, tuple)): # This is already reported by schema checking continue for term in value: - if not isinstance(term, string_types): + if not isinstance(term, str): # This is already reported by schema checking continue if len(set(value)) != len(value) or key in value: diff --git a/test/lib/ansible_test/_util/target/__init__.py b/test/lib/ansible_test/_util/target/__init__.py index 527d413a98d..8f58623f0ac 100644 --- a/test/lib/ansible_test/_util/target/__init__.py +++ b/test/lib/ansible_test/_util/target/__init__.py @@ -1,2 +1 @@ -# Empty __init__.py to allow importing of `ansible_test._util.target.common` under Python 2.x. -# This allows the ansible-test entry point to report supported Python versions before exiting. +# Empty __init__.py to keep pylint happy. diff --git a/test/lib/ansible_test/_util/target/common/__init__.py b/test/lib/ansible_test/_util/target/common/__init__.py deleted file mode 100644 index 527d413a98d..00000000000 --- a/test/lib/ansible_test/_util/target/common/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Empty __init__.py to allow importing of `ansible_test._util.target.common` under Python 2.x. -# This allows the ansible-test entry point to report supported Python versions before exiting. diff --git a/test/lib/ansible_test/_util/target/injector/python.py b/test/lib/ansible_test/_util/target/injector/python.py index eef61da514e..82f59b1d191 100644 --- a/test/lib/ansible_test/_util/target/injector/python.py +++ b/test/lib/ansible_test/_util/target/injector/python.py @@ -2,6 +2,7 @@ """Provides an entry point for python scripts and python modules on the controller with the current python interpreter and optional code coverage collection.""" from __future__ import annotations +import importlib.util import os import sys @@ -18,22 +19,7 @@ def main(): if coverage_output: args += ['-m', 'coverage.__main__', 'run', '--rcfile', coverage_config] else: - if sys.version_info >= (3, 4): - # noinspection PyUnresolvedReferences - import importlib.util - - # noinspection PyUnresolvedReferences - found = bool(importlib.util.find_spec('coverage')) - else: - # noinspection PyDeprecation - import imp - - try: - # noinspection PyDeprecation - imp.find_module('coverage') - found = True - except ImportError: - found = False + found = bool(importlib.util.find_spec('coverage')) if not found: sys.exit('ERROR: Could not find `coverage` module. ' @@ -61,7 +47,7 @@ def find_program(name, executable): # type: (str, bool) -> str Raises an exception if the program is not found. """ path = os.environ.get('PATH', os.path.defpath) - seen = set([os.path.abspath(__file__)]) + seen = {os.path.abspath(__file__)} mode = os.F_OK | os.X_OK if executable else os.F_OK for base in path.split(os.path.pathsep): diff --git a/test/lib/ansible_test/_util/target/pytest/plugins/ansible_pytest_collections.py b/test/lib/ansible_test/_util/target/pytest/plugins/ansible_pytest_collections.py index 060a954b9b9..3aa2e129623 100644 --- a/test/lib/ansible_test/_util/target/pytest/plugins/ansible_pytest_collections.py +++ b/test/lib/ansible_test/_util/target/pytest/plugins/ansible_pytest_collections.py @@ -38,9 +38,6 @@ def enable_assertion_rewriting_hook(): # type: () -> None """ import sys - if sys.version_info[0] == 2: - return # Python 2.x is not supported - hook_name = '_pytest.assertion.rewrite.AssertionRewritingHook' hooks = [hook for hook in sys.meta_path if hook.__class__.__module__ + '.' + hook.__class__.__qualname__ == hook_name] diff --git a/test/lib/ansible_test/_util/target/sanity/compile/compile.py b/test/lib/ansible_test/_util/target/sanity/compile/compile.py index 4e11f7cd7de..3dfec3910fe 100644 --- a/test/lib/ansible_test/_util/target/sanity/compile/compile.py +++ b/test/lib/ansible_test/_util/target/sanity/compile/compile.py @@ -5,7 +5,6 @@ import sys ENCODING = 'utf-8' ERRORS = 'replace' -Text = type(u'') def main(): @@ -28,21 +27,14 @@ def compile_source(path): else: return - # In some situations offset can be None. This can happen for syntax errors on Python 2.6 - # (__future__ import following after a regular import). - offset = offset or 0 - result = "%s:%d:%d: %s: %s" % (path, lineno, offset, extype.__name__, safe_message(message)) - if sys.version_info <= (3,): - result = result.encode(ENCODING, ERRORS) - print(result) def safe_message(value): - """Given an input value as text or bytes, return the first non-empty line as text, ensuring it can be round-tripped as UTF-8.""" - if isinstance(value, Text): + """Given an input value as str or bytes, return the first non-empty line as str, ensuring it can be round-tripped as UTF-8.""" + if isinstance(value, str): value = value.encode(ENCODING, ERRORS) value = value.decode(ENCODING, ERRORS) diff --git a/test/lib/ansible_test/_util/target/sanity/import/importer.py b/test/lib/ansible_test/_util/target/sanity/import/importer.py index 4211bc4e604..32eb424e226 100644 --- a/test/lib/ansible_test/_util/target/sanity/import/importer.py +++ b/test/lib/ansible_test/_util/target/sanity/import/importer.py @@ -541,16 +541,6 @@ def main(): "ignore", "AnsibleCollectionFinder has already been configured") - if sys.version_info[0] == 2: - warnings.filterwarnings( - "ignore", - "Python 2 is no longer supported by the Python core team. Support for it is now deprecated in cryptography," - " and will be removed in a future release.") - warnings.filterwarnings( - "ignore", - "Python 2 is no longer supported by the Python core team. Support for it is now deprecated in cryptography," - " and will be removed in the next release.") - try: yield finally: diff --git a/test/lib/ansible_test/_util/target/setup/bootstrap.sh b/test/lib/ansible_test/_util/target/setup/bootstrap.sh index 571991c0b4f..e38b585b191 100644 --- a/test/lib/ansible_test/_util/target/setup/bootstrap.sh +++ b/test/lib/ansible_test/_util/target/setup/bootstrap.sh @@ -65,18 +65,6 @@ install_pip() { fi } -pip_install() { - pip_packages="$1" - - while true; do - # shellcheck disable=SC2086 - "${python_interpreter}" -m pip install --disable-pip-version-check ${pip_packages} \ - && break - echo "Failed to install packages. Sleeping before trying again..." - sleep 10 - done -} - bootstrap_remote_alpine() { py_pkg_prefix="py3" @@ -279,17 +267,6 @@ bootstrap_remote_rhel() esac } -bootstrap_remote_rhel_pinned_pip_packages() -{ - # pin packaging and pyparsing to match the downstream vendored versions - pip_packages=" - packaging==20.4 - pyparsing==2.4.7 - " - - pip_install "${pip_packages}" -} - bootstrap_remote_ubuntu() { py_pkg_prefix="python3"