Fix ansible-test requirements installation.

changelogs/fragments/ansible-test-requirements-install.yml

@@ -0,0 +1,5 @@
+bugfixes:
+    - ansible-test can now install argparse with ``--requirements`` or delegation when the pip version in use is older than version 7.1
+    - ansible-test now upgrades ``pip`` with `--requirements`` or delegation as needed when the pip version in use is older than version 7.1
+    - ansible-test now installs the correct version of ``cryptography`` with ``--requirements`` or delegation when setuptools is older than version 18.5
+    - ansible-test now limits the version of ``setuptools`` on Python 2.6 to versions older than 37

test/lib/ansible_test/_data/cryptography-constraints.txt

@@ -0,0 +1 @@
+idna < 2.8 ; python_version < '2.7' # idna 2.8+ requires python 2.7+

test/lib/ansible_test/_data/requirements/ansible-test.txt

@@ -1 +1,6 @@
 argparse ; python_version < '2.7'
+
+# pip 7.1 added support for constraints, which are required by ansible-test to install most python requirements
+# see https://github.com/pypa/pip/blame/e648e00dc0226ade30ade99591b245b0c98e86c9/NEWS.rst#L1258
+pip >= 7.1, < 10 ; python_version < '2.7' # pip 10+ drops support for python 2.6 (sanity_ok)
+pip >= 7.1 ; python_version >= '2.7' # sanity_ok

test/lib/ansible_test/_data/requirements/constraints.txt

@@ -39,7 +39,8 @@ pyone == 1.1.9 # newer versions do not pass current integration tests
 boto3 < 1.11 ; python_version < '2.7' # boto3 1.11 drops Python 2.6 support
 botocore >= 1.10.0, < 1.14 ; python_version < '2.7' # adds support for the following AWS services: secretsmanager, fms, and acm-pca; botocore 1.14 drops Python 2.6 support
 botocore >= 1.10.0 ; python_version >= '2.7' # adds support for the following AWS services: secretsmanager, fms, and acm-pca
-setuptools < 45 ; python_version <= '2.7' # setuptools 45 and later require python 3.5 or later
+setuptools < 37 ; python_version == '2.6' # setuptools 37 and later require python 2.7 or later
+setuptools < 45 ; python_version == '2.7' # setuptools 45 and later require python 3.5 or later
 
 # freeze pylint and its requirements for consistent test results
 astroid == 2.2.5

test/lib/ansible_test/_internal/cli.py

@@ -202,7 +202,10 @@ def parse_args():
     except ImportError:
         if '--requirements' not in sys.argv:
             raise
-        raw_command(generate_pip_install(generate_pip_command(sys.executable), 'ansible-test'))
+        # install argparse without using constraints since pip may be too old to support them
+        # not using the ansible-test requirements file since this install is for sys.executable rather than the delegated python (which may be different)
+        # argparse has no special requirements, so upgrading pip is not required here
+        raw_command(generate_pip_install(generate_pip_command(sys.executable), 'argparse', packages=['argparse'], use_constraints=False))
         import argparse
 
     try:

test/lib/ansible_test/_internal/executor.py

@@ -72,6 +72,7 @@ from .util import (
     tempdir,
     open_zipfile,
     SUPPORTED_PYTHON_VERSIONS,
+    str_to_version,
 )
 
 from .util_common import (
@@ -188,6 +189,40 @@ def create_shell_command(command):
     return cmd
 
 
+def get_setuptools_version(args, python):  # type: (EnvironmentConfig, str) -> t.Tuple[int]
+    """Return the setuptools version for the given python."""
+    try:
+        return str_to_version(raw_command([python, '-c', 'import setuptools; print(setuptools.__version__)'], capture=True)[0])
+    except SubprocessError:
+        if args.explain:
+            return tuple()  # ignore errors in explain mode in case setuptools is not aleady installed
+
+        raise
+
+
+def get_cryptography_requirement(args, python_version):  # type: (EnvironmentConfig, str) -> str
+    """
+    Return the correct cryptography requirement for the given python version.
+    The version of cryptograpy installed depends on the python version and setuptools version.
+    """
+    python = find_python(python_version)
+    setuptools_version = get_setuptools_version(args, python)
+
+    if setuptools_version >= (18, 5):
+        if python_version == '2.6':
+            # cryptography 2.2+ requires python 2.7+
+            # see https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst#22---2018-03-19
+            cryptography = 'cryptography < 2.2'
+        else:
+            cryptography = 'cryptography'
+    else:
+        # cryptography 2.1+ requires setuptools 18.5+
+        # see https://github.com/pyca/cryptography/blob/62287ae18383447585606b9d0765c0f1b8a9777c/setup.py#L26
+        cryptography = 'cryptography < 2.1'
+
+    return cryptography
+
+
 def install_command_requirements(args, python_version=None):
     """
     :type args: EnvironmentConfig
@@ -222,6 +257,22 @@ def install_command_requirements(args, python_version=None):
 
     pip = generate_pip_command(find_python(python_version))
 
+    # make sure basic ansible-test requirements are met, including making sure that pip is recent enough to support constraints
+    # virtualenvs created by older distributions may include very old pip versions, such as those created in the centos6 test container (pip 6.0.8)
+    run_command(args, generate_pip_install(pip, 'ansible-test', use_constraints=False))
+
+    # make sure setuptools is available before trying to install cryptography
+    # the installed version of setuptools affects the version of cryptography to install
+    run_command(args, generate_pip_install(pip, 'setuptools', packages=['setuptools']))
+
+    # install the latest cryptography version that the current requirements can support
+    # use a custom constraints file to avoid the normal constraints file overriding the chosen version of cryptography
+    # if not installed here later install commands may try to install an unsupported version due to the presence of older setuptools
+    # this is done instead of upgrading setuptools to allow tests to function with older distribution provided versions of setuptools
+    run_command(args, generate_pip_install(pip, 'cryptography',
+                                           packages=[get_cryptography_requirement(args, python_version)],
+                                           constraints=os.path.join(ANSIBLE_TEST_DATA_ROOT, 'cryptography-constraints.txt')))
+
     commands = [generate_pip_install(pip, args.command, packages=packages)]
 
     if isinstance(args, IntegrationConfig):
@@ -333,14 +384,16 @@ License: GPLv3+
     write_text_file(pkg_info_path, pkg_info.lstrip(), create_directories=True)
 
 
-def generate_pip_install(pip, command, packages=None):
+def generate_pip_install(pip, command, packages=None, constraints=None, use_constraints=True):
     """
     :type pip: list[str]
     :type command: str
     :type packages: list[str] | None
+    :type constraints: str | None
+    :type use_constraints: bool
     :rtype: list[str] | None
     """
-    constraints = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'requirements', 'constraints.txt')
+    constraints = constraints or os.path.join(ANSIBLE_TEST_DATA_ROOT, 'requirements', 'constraints.txt')
     requirements = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'requirements', '%s.txt' % command)
 
     options = []
@@ -375,7 +428,10 @@ def generate_pip_install(pip, command, packages=None):
     if not options:
         return None
 
-    return pip + ['install', '--disable-pip-version-check', '-c', constraints] + options
+    if use_constraints:
+        options.extend(['-c', constraints])
+
+    return pip + ['install', '--disable-pip-version-check'] + options
 
 
 def command_shell(args):

test/lib/ansible_test/_internal/util.py

@@ -865,6 +865,16 @@ def paths_to_dirs(paths):  # type: (t.List[str]) -> t.List[str]
     return sorted(dir_names)
 
 
+def str_to_version(version):  # type: (str) -> t.Tuple[int]
+    """Return a version tuple from a version string."""
+    return tuple(int(n) for n in version.split('.'))
+
+
+def version_to_str(version):  # type: (t.Tuple[int]) -> str
+    """Return a version string from a version tuple."""
+    return '.'.join(str(n) for n in version)
+
+
 def import_plugins(directory, root=None):  # type: (str, t.Optional[str]) -> None
     """
     Import plugins from the given directory relative to the given root.