diff --git a/changelogs/fragments/ansible-test-requirements-install.yml b/changelogs/fragments/ansible-test-requirements-install.yml new file mode 100644 index 00000000000..7189c5247f9 --- /dev/null +++ b/changelogs/fragments/ansible-test-requirements-install.yml @@ -0,0 +1,5 @@ +bugfixes: + - ansible-test can now install argparse with ``--requirements`` or delegation when the pip version in use is older than version 7.1 + - ansible-test now upgrades ``pip`` with `--requirements`` or delegation as needed when the pip version in use is older than version 7.1 + - ansible-test now installs the correct version of ``cryptography`` with ``--requirements`` or delegation when setuptools is older than version 18.5 + - ansible-test now limits the version of ``setuptools`` on Python 2.6 to versions older than 37 diff --git a/test/lib/ansible_test/_data/cryptography-constraints.txt b/test/lib/ansible_test/_data/cryptography-constraints.txt new file mode 100644 index 00000000000..3ea40ba116d --- /dev/null +++ b/test/lib/ansible_test/_data/cryptography-constraints.txt @@ -0,0 +1 @@ +idna < 2.8 ; python_version < '2.7' # idna 2.8+ requires python 2.7+ diff --git a/test/lib/ansible_test/_data/requirements/ansible-test.txt b/test/lib/ansible_test/_data/requirements/ansible-test.txt index d08d921a32a..7b596e1b6ca 100644 --- a/test/lib/ansible_test/_data/requirements/ansible-test.txt +++ b/test/lib/ansible_test/_data/requirements/ansible-test.txt @@ -1 +1,6 @@ argparse ; python_version < '2.7' + +# pip 7.1 added support for constraints, which are required by ansible-test to install most python requirements +# see https://github.com/pypa/pip/blame/e648e00dc0226ade30ade99591b245b0c98e86c9/NEWS.rst#L1258 +pip >= 7.1, < 10 ; python_version < '2.7' # pip 10+ drops support for python 2.6 (sanity_ok) +pip >= 7.1 ; python_version >= '2.7' # sanity_ok diff --git a/test/lib/ansible_test/_data/requirements/constraints.txt b/test/lib/ansible_test/_data/requirements/constraints.txt index b96d623cd1b..409f529ac91 100644 --- a/test/lib/ansible_test/_data/requirements/constraints.txt +++ b/test/lib/ansible_test/_data/requirements/constraints.txt @@ -39,7 +39,8 @@ pyone == 1.1.9 # newer versions do not pass current integration tests boto3 < 1.11 ; python_version < '2.7' # boto3 1.11 drops Python 2.6 support botocore >= 1.10.0, < 1.14 ; python_version < '2.7' # adds support for the following AWS services: secretsmanager, fms, and acm-pca; botocore 1.14 drops Python 2.6 support botocore >= 1.10.0 ; python_version >= '2.7' # adds support for the following AWS services: secretsmanager, fms, and acm-pca -setuptools < 45 ; python_version <= '2.7' # setuptools 45 and later require python 3.5 or later +setuptools < 37 ; python_version == '2.6' # setuptools 37 and later require python 2.7 or later +setuptools < 45 ; python_version == '2.7' # setuptools 45 and later require python 3.5 or later # freeze pylint and its requirements for consistent test results astroid == 2.2.5 diff --git a/test/lib/ansible_test/_internal/cli.py b/test/lib/ansible_test/_internal/cli.py index cc31f1b97fa..c0aa37e5a81 100644 --- a/test/lib/ansible_test/_internal/cli.py +++ b/test/lib/ansible_test/_internal/cli.py @@ -202,7 +202,10 @@ def parse_args(): except ImportError: if '--requirements' not in sys.argv: raise - raw_command(generate_pip_install(generate_pip_command(sys.executable), 'ansible-test')) + # install argparse without using constraints since pip may be too old to support them + # not using the ansible-test requirements file since this install is for sys.executable rather than the delegated python (which may be different) + # argparse has no special requirements, so upgrading pip is not required here + raw_command(generate_pip_install(generate_pip_command(sys.executable), 'argparse', packages=['argparse'], use_constraints=False)) import argparse try: diff --git a/test/lib/ansible_test/_internal/executor.py b/test/lib/ansible_test/_internal/executor.py index 3d1a1ab3c0e..787b9667957 100644 --- a/test/lib/ansible_test/_internal/executor.py +++ b/test/lib/ansible_test/_internal/executor.py @@ -72,6 +72,7 @@ from .util import ( tempdir, open_zipfile, SUPPORTED_PYTHON_VERSIONS, + str_to_version, ) from .util_common import ( @@ -188,6 +189,40 @@ def create_shell_command(command): return cmd +def get_setuptools_version(args, python): # type: (EnvironmentConfig, str) -> t.Tuple[int] + """Return the setuptools version for the given python.""" + try: + return str_to_version(raw_command([python, '-c', 'import setuptools; print(setuptools.__version__)'], capture=True)[0]) + except SubprocessError: + if args.explain: + return tuple() # ignore errors in explain mode in case setuptools is not aleady installed + + raise + + +def get_cryptography_requirement(args, python_version): # type: (EnvironmentConfig, str) -> str + """ + Return the correct cryptography requirement for the given python version. + The version of cryptograpy installed depends on the python version and setuptools version. + """ + python = find_python(python_version) + setuptools_version = get_setuptools_version(args, python) + + if setuptools_version >= (18, 5): + if python_version == '2.6': + # cryptography 2.2+ requires python 2.7+ + # see https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst#22---2018-03-19 + cryptography = 'cryptography < 2.2' + else: + cryptography = 'cryptography' + else: + # cryptography 2.1+ requires setuptools 18.5+ + # see https://github.com/pyca/cryptography/blob/62287ae18383447585606b9d0765c0f1b8a9777c/setup.py#L26 + cryptography = 'cryptography < 2.1' + + return cryptography + + def install_command_requirements(args, python_version=None): """ :type args: EnvironmentConfig @@ -222,6 +257,22 @@ def install_command_requirements(args, python_version=None): pip = generate_pip_command(find_python(python_version)) + # make sure basic ansible-test requirements are met, including making sure that pip is recent enough to support constraints + # virtualenvs created by older distributions may include very old pip versions, such as those created in the centos6 test container (pip 6.0.8) + run_command(args, generate_pip_install(pip, 'ansible-test', use_constraints=False)) + + # make sure setuptools is available before trying to install cryptography + # the installed version of setuptools affects the version of cryptography to install + run_command(args, generate_pip_install(pip, 'setuptools', packages=['setuptools'])) + + # install the latest cryptography version that the current requirements can support + # use a custom constraints file to avoid the normal constraints file overriding the chosen version of cryptography + # if not installed here later install commands may try to install an unsupported version due to the presence of older setuptools + # this is done instead of upgrading setuptools to allow tests to function with older distribution provided versions of setuptools + run_command(args, generate_pip_install(pip, 'cryptography', + packages=[get_cryptography_requirement(args, python_version)], + constraints=os.path.join(ANSIBLE_TEST_DATA_ROOT, 'cryptography-constraints.txt'))) + commands = [generate_pip_install(pip, args.command, packages=packages)] if isinstance(args, IntegrationConfig): @@ -333,14 +384,16 @@ License: GPLv3+ write_text_file(pkg_info_path, pkg_info.lstrip(), create_directories=True) -def generate_pip_install(pip, command, packages=None): +def generate_pip_install(pip, command, packages=None, constraints=None, use_constraints=True): """ :type pip: list[str] :type command: str :type packages: list[str] | None + :type constraints: str | None + :type use_constraints: bool :rtype: list[str] | None """ - constraints = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'requirements', 'constraints.txt') + constraints = constraints or os.path.join(ANSIBLE_TEST_DATA_ROOT, 'requirements', 'constraints.txt') requirements = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'requirements', '%s.txt' % command) options = [] @@ -375,7 +428,10 @@ def generate_pip_install(pip, command, packages=None): if not options: return None - return pip + ['install', '--disable-pip-version-check', '-c', constraints] + options + if use_constraints: + options.extend(['-c', constraints]) + + return pip + ['install', '--disable-pip-version-check'] + options def command_shell(args): diff --git a/test/lib/ansible_test/_internal/util.py b/test/lib/ansible_test/_internal/util.py index 8341aefc9c0..3f70f5ef402 100644 --- a/test/lib/ansible_test/_internal/util.py +++ b/test/lib/ansible_test/_internal/util.py @@ -865,6 +865,16 @@ def paths_to_dirs(paths): # type: (t.List[str]) -> t.List[str] return sorted(dir_names) +def str_to_version(version): # type: (str) -> t.Tuple[int] + """Return a version tuple from a version string.""" + return tuple(int(n) for n in version.split('.')) + + +def version_to_str(version): # type: (t.Tuple[int]) -> str + """Return a version string from a version tuple.""" + return '.'.join(str(n) for n in version) + + def import_plugins(directory, root=None): # type: (str, t.Optional[str]) -> None """ Import plugins from the given directory relative to the given root.