ansible-test - Remove more Python 2.7 / 3.6 support (#81898)

pull/81903/head
Matt Clay 12 months ago committed by GitHub
parent dfc62589f6
commit 921efce43a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -0,0 +1,2 @@
minor_changes:
- ansible-test - Removed fallback to ``virtualenv`` when ``-m venv`` is non-functional.

@ -1,5 +1,3 @@
# The test-constraints sanity test verifies this file, but changes must be made manually to keep it in up-to-date.
virtualenv == 16.7.12 ; python_version < '3'
coverage == 7.3.2 ; python_version >= '3.8' and python_version <= '3.12'
coverage == 6.5.0 ; python_version >= '3.7' and python_version <= '3.7'
coverage == 4.5.4 ; python_version >= '2.6' and python_version <= '3.6'

@ -1,15 +1,10 @@
# do not add a cryptography or pyopenssl constraint to this file, they require special handling, see get_cryptography_requirements in python_requirements.py
# do not add a coverage constraint to this file, it is handled internally by ansible-test
packaging < 21.0 ; python_version < '3.6' # packaging 21.0 requires Python 3.6 or newer
pywinrm >= 0.3.0 ; python_version < '3.11' # message encryption support
pywinrm >= 0.4.3 ; python_version >= '3.11' # support for Python 3.11
pytest < 5.0.0, >= 4.5.0 ; python_version == '2.7' # pytest 5.0.0 and later will no longer support python 2.7
pytest >= 4.5.0 ; python_version > '2.7' # pytest 4.5.0 added support for --strict-markers
pytest >= 4.5.0 # pytest 4.5.0 added support for --strict-markers
ntlm-auth >= 1.3.0 # message encryption support using cryptography
requests-ntlm >= 1.1.0 # message encryption support
requests-credssp >= 0.1.0 # message encryption support
pyparsing < 3.0.0 ; python_version < '3.5' # pyparsing 3 and later require python 3.5 or later
mock >= 2.0.0 # needed for features backported from Python 3.6 unittest.mock (assert_called, assert_called_once...)
pytest-mock >= 1.4.0 # needed for mock_use_standalone_module pytest option
setuptools < 45 ; python_version == '2.7' # setuptools 45 and later require python 3.5 or later
wheel < 0.38.0 ; python_version < '3.7' # wheel 0.38.0 and later require python 3.7 or later

@ -1117,7 +1117,6 @@ def create_sanity_virtualenv(
commands = collect_requirements( # create_sanity_virtualenv()
python=python,
controller=True,
virtualenv=False,
command=None,
ansible=False,
cryptography=False,

@ -47,11 +47,6 @@ from ...ansible_util import (
ansible_environment,
)
from ...python_requirements import (
PipUnavailableError,
install_requirements,
)
from ...config import (
SanityConfig,
)
@ -68,10 +63,6 @@ from ...host_configs import (
PythonConfig,
)
from ...venv import (
get_virtualenv_version,
)
def _get_module_test(module_restrictions: bool) -> c.Callable[[str], bool]:
"""Create a predicate which tests whether a path can be used by modules or not."""
@ -109,15 +100,6 @@ class ImportTest(SanityMultipleVersion):
paths = [target.path for target in targets.include]
if python.version.startswith('2.') and (get_virtualenv_version(args, python.path) or (0,)) < (13,):
# hack to make sure that virtualenv is available under Python 2.x
# on Python 3.x we can use the built-in venv
# version 13+ is required to use the `--no-wheel` option
try:
install_requirements(args, python, virtualenv=True, controller=False) # sanity (import)
except PipUnavailableError as ex:
display.warning(str(ex))
temp_root = os.path.join(ResultType.TMP.path, 'sanity', 'import')
messages = []

@ -19,7 +19,6 @@ from . import (
from ...constants import (
CONTROLLER_PYTHON_VERSIONS,
REMOTE_ONLY_PYTHON_VERSIONS,
SUPPORTED_PYTHON_VERSIONS,
)
from ...test import (
@ -37,7 +36,6 @@ from ...util import (
ANSIBLE_TEST_CONTROLLER_ROOT,
ApplicationError,
is_subdir,
str_to_version,
)
from ...util_common import (
@ -76,15 +74,6 @@ class MypyTest(SanityMultipleVersion):
or target.path.startswith('packaging/')
or target.path.startswith('test/lib/ansible_test/_util/target/sanity/import/'))]
@property
def supported_python_versions(self) -> t.Optional[tuple[str, ...]]:
"""A tuple of supported Python versions or None if the test does not depend on specific Python versions."""
# mypy 0.981 dropped support for Python 2
# see: https://mypy-lang.blogspot.com/2022/09/mypy-0981-released.html
# cryptography dropped support for Python 3.5 in version 3.3
# see: https://cryptography.io/en/latest/changelog/#v3-3
return tuple(version for version in SUPPORTED_PYTHON_VERSIONS if str_to_version(version) >= (3, 6))
@property
def error_code(self) -> t.Optional[str]:
"""Error code for ansible-test matching the format used by the underlying test program, or None if the program does not use error codes."""

@ -71,7 +71,6 @@ COVERAGE_VERSIONS = (
# IMPORTANT: Keep this in sync with the ansible-test.txt requirements file.
CoverageVersion('7.3.2', 7, (3, 8), (3, 12)),
CoverageVersion('6.5.0', 7, (3, 7), (3, 7)),
CoverageVersion('4.5.4', 0, (2, 6), (3, 6)),
)
"""
This tuple specifies the coverage version to use for Python version ranges.

@ -63,9 +63,6 @@ from .coverage_util import (
QUIET_PIP_SCRIPT_PATH = os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'setup', 'quiet_pip.py')
REQUIREMENTS_SCRIPT_PATH = os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'setup', 'requirements.py')
# IMPORTANT: Keep this in sync with the ansible-test.txt requirements file.
VIRTUALENV_VERSION = '16.7.12'
# Pip Abstraction
@ -132,7 +129,6 @@ def install_requirements(
ansible: bool = False,
command: bool = False,
coverage: bool = False,
virtualenv: bool = False,
controller: bool = True,
connection: t.Optional[Connection] = None,
) -> None:
@ -172,7 +168,6 @@ def install_requirements(
cryptography=cryptography,
command=args.command if command else None,
coverage=coverage,
virtualenv=virtualenv,
minimize=False,
sanity=None,
)
@ -207,7 +202,6 @@ def collect_requirements(
ansible: bool,
cryptography: bool,
coverage: bool,
virtualenv: bool,
minimize: bool,
command: t.Optional[str],
sanity: t.Optional[str],
@ -215,11 +209,6 @@ def collect_requirements(
"""Collect requirements for the given Python using the specified arguments."""
commands: list[PipCommand] = []
if virtualenv:
# sanity tests on Python 2.x install virtualenv when it is too old or is not already installed and the `--requirements` option is given
# the last version of virtualenv with no dependencies is used to minimize the changes made outside a virtual environment
commands.extend(collect_package_install(packages=[f'virtualenv=={VIRTUALENV_VERSION}'], constraints=False))
if coverage:
commands.extend(collect_package_install(packages=[f'coverage=={get_coverage_version(python.version).coverage_version}'], constraints=False))

@ -15,7 +15,6 @@ from .config import (
from .util import (
find_python,
SubprocessError,
get_available_python_versions,
ANSIBLE_TEST_TARGET_TOOLS_ROOT,
display,
remove_tree,
@ -85,49 +84,21 @@ def create_virtual_environment(
system_site_packages: bool = False,
pip: bool = False,
) -> bool:
"""Create a virtual environment using venv or virtualenv for the requested Python version."""
"""Create a virtual environment using venv for the requested Python version."""
if not os.path.exists(python.path):
# the requested python version could not be found
return False
if str_to_version(python.version) >= (3, 0):
# use the built-in 'venv' module on Python 3.x
# creating a virtual environment using 'venv' when running in a virtual environment created by 'virtualenv' results
# in a copy of the original virtual environment instead of creation of a new one
# avoid this issue by only using "real" python interpreters to invoke 'venv'
for real_python in iterate_real_pythons(python.version):
if run_venv(args, real_python, system_site_packages, pip, path):
display.info('Created Python %s virtual environment using "venv": %s' % (python.version, path), verbosity=1)
return True
# something went wrong, most likely the package maintainer for the Python installation removed ensurepip
# which will prevent creation of a virtual environment without installation of other OS packages
# use the installed 'virtualenv' module on the Python requested version
if run_virtualenv(args, python.path, python.path, system_site_packages, pip, path):
display.info('Created Python %s virtual environment using "virtualenv": %s' % (python.version, path), verbosity=1)
return True
available_pythons = get_available_python_versions()
for available_python_version, available_python_interpreter in sorted(available_pythons.items()):
if available_python_interpreter == python.path:
# already attempted to use this interpreter
continue
virtualenv_version = get_virtualenv_version(args, available_python_interpreter)
if not virtualenv_version:
# virtualenv not available for this Python or we were unable to detect the version
continue
# try using 'virtualenv' from another Python to setup the desired version
if run_virtualenv(args, available_python_interpreter, python.path, system_site_packages, pip, path):
display.info('Created Python %s virtual environment using "virtualenv" on Python %s: %s' % (python.version, available_python_version, path),
verbosity=1)
# creating a virtual environment using 'venv' when running in a virtual environment created by 'virtualenv' results
# in a copy of the original virtual environment instead of creation of a new one
# avoid this issue by only using "real" python interpreters to invoke 'venv'
for real_python in iterate_real_pythons(python.version):
if run_venv(args, real_python, system_site_packages, pip, path):
display.info('Created Python %s virtual environment using "venv": %s' % (python.version, path), verbosity=1)
return True
# no suitable 'virtualenv' available
# something went wrong, most likely the package maintainer for the Python installation removed ensurepip
# which will prevent creation of a virtual environment without installation of other OS packages
return False
@ -210,72 +181,3 @@ def run_venv(
return False
return True
def run_virtualenv(
args: EnvironmentConfig,
run_python: str,
env_python: str,
system_site_packages: bool,
pip: bool,
path: str,
) -> bool:
"""Create a virtual environment using the 'virtualenv' module."""
# always specify which interpreter to use to guarantee the desired interpreter is provided
# otherwise virtualenv may select a different interpreter than the one running virtualenv
cmd = [run_python, '-m', 'virtualenv', '--python', env_python]
if system_site_packages:
cmd.append('--system-site-packages')
if not pip:
cmd.append('--no-pip')
# these options provide consistency with venv, which does not install them without pip
cmd.append('--no-setuptools')
cmd.append('--no-wheel')
cmd.append(path)
try:
run_command(args, cmd, capture=True)
except SubprocessError as ex:
remove_tree(path)
if args.verbosity > 1:
display.error(ex.message)
return False
return True
def get_virtualenv_version(args: EnvironmentConfig, python: str) -> t.Optional[tuple[int, ...]]:
"""Get the virtualenv version for the given python interpreter, if available, otherwise return None."""
try:
cache = get_virtualenv_version.cache # type: ignore[attr-defined]
except AttributeError:
cache = get_virtualenv_version.cache = {} # type: ignore[attr-defined]
if python not in cache:
try:
stdout = run_command(args, [python, '-m', 'virtualenv', '--version'], capture=True)[0]
except SubprocessError as ex:
stdout = ''
if args.verbosity > 1:
display.error(ex.message)
version = None
if stdout:
# noinspection PyBroadException
try:
version = str_to_version(stdout.strip())
except Exception: # pylint: disable=broad-except
pass
cache[python] = version
version = cache[python]
return version

@ -11,12 +11,7 @@ import pickle
import tempfile
import warnings
from pytest import Item, hookimpl
try:
from pytest import TestReport
except ImportError:
from _pytest.runner import TestReport # Backwards compatibility with pytest < 7. Remove once Python 2.7 is not supported.
from pytest import Item, hookimpl, TestReport
from _pytest.runner import runtestprotocol

@ -49,9 +49,6 @@ customize_bashrc()
install_pip() {
if ! "${python_interpreter}" -m pip.__main__ --version --disable-pip-version-check 2>/dev/null; then
case "${python_version}" in
"2.7")
pip_bootstrap_url="https://ci-files.testing.ansible.com/ansible-test/get-pip-20.3.4.py"
;;
*)
pip_bootstrap_url="https://ci-files.testing.ansible.com/ansible-test/get-pip-23.1.2.py"
;;

@ -6,14 +6,12 @@ import os
import re
import runpy
import sys
import warnings
BUILTIN_FILTERER_FILTER = logging.Filterer.filter
LOGGING_MESSAGE_FILTER = re.compile("^("
".*Running pip install with root privileges is generally not a good idea.*|" # custom Fedora patch [1]
".*Running pip as the 'root' user can result in broken permissions .*|" # pip 21.1
"DEPRECATION: Python 2.7 will reach the end of its life .*|" # pip 19.2.3
"Ignoring .*: markers .* don't match your environment|"
"Looking in indexes: .*|" # pypi-test-container
"Requirement already satisfied.*"
@ -21,13 +19,6 @@ LOGGING_MESSAGE_FILTER = re.compile("^("
# [1] https://src.fedoraproject.org/rpms/python-pip/blob/f34/f/emit-a-warning-when-running-with-root-privileges.patch
WARNING_MESSAGE_FILTERS = (
# DEPRECATION: Python 2.7 reached the end of its life on January 1st, 2020. Please upgrade your Python as Python 2.7 is no longer maintained.
# pip 21.0 will drop support for Python 2.7 in January 2021.
# More details about Python 2 support in pip, can be found at https://pip.pypa.io/en/latest/development/release-process/#python-2-support
'DEPRECATION: Python 2.7 reached the end of its life ',
)
def custom_filterer_filter(self, record):
"""Globally omit logging of unwanted messages."""
@ -43,11 +34,6 @@ def main():
# It also avoids problems with loss of color output and mixing up the order of stdout/stderr messages.
logging.Filterer.filter = custom_filterer_filter
for message_filter in WARNING_MESSAGE_FILTERS:
# Setting filterwarnings in code is necessary because of the following:
# Python 2.7 cannot use the -W option to match warning text after a colon. This makes it impossible to match specific warning messages.
warnings.filterwarnings('ignore', message_filter)
get_pip = os.environ.get('GET_PIP')
try:

@ -69,16 +69,13 @@ def main():
def check_ansible_test(path: str, requirements: list[tuple[int, str, re.Match]]) -> None:
sys.path.insert(0, str(pathlib.Path(__file__).parent.parent.parent.joinpath('lib')))
from ansible_test._internal.python_requirements import VIRTUALENV_VERSION
from ansible_test._internal.coverage_util import COVERAGE_VERSIONS
from ansible_test._internal.util import version_to_str
expected_lines = set([
f"virtualenv == {VIRTUALENV_VERSION} ; python_version < '3'",
] + [
expected_lines = set((
f"coverage == {item.coverage_version} ; python_version >= '{version_to_str(item.min_python)}' and python_version <= '{version_to_str(item.max_python)}'"
for item in COVERAGE_VERSIONS
])
))
for idx, requirement in enumerate(requirements):
lineno, line, match = requirement

Loading…
Cancel
Save