diff --git a/test/lib/ansible_test/_internal/cli.py b/test/lib/ansible_test/_internal/cli.py index 39077e30f3c..6b2dd72b17d 100644 --- a/test/lib/ansible_test/_internal/cli.py +++ b/test/lib/ansible_test/_internal/cli.py @@ -643,6 +643,10 @@ def add_environments(parser, tox_version=False, tox_only=False): action='store_true', help='run from the local environment') + environments.add_argument('--venv', + action='store_true', + help='run from ansible-test managed virtual environments') + if data_context().content.is_ansible: if tox_version: environments.add_argument('--tox', diff --git a/test/lib/ansible_test/_internal/config.py b/test/lib/ansible_test/_internal/config.py index 6609648ce73..c1add1c6320 100644 --- a/test/lib/ansible_test/_internal/config.py +++ b/test/lib/ansible_test/_internal/config.py @@ -44,6 +44,7 @@ class EnvironmentConfig(CommonConfig): super(EnvironmentConfig, self).__init__(args, command) self.local = args.local is True + self.venv = args.venv if args.tox is True or args.tox is False or args.tox is None: self.tox = args.tox is True @@ -87,7 +88,7 @@ class EnvironmentConfig(CommonConfig): self.python_version = self.python or actual_major_minor self.python_interpreter = args.python_interpreter - self.delegate = self.tox or self.docker or self.remote + self.delegate = self.tox or self.docker or self.remote or self.venv self.delegate_args = [] # type: t.List[str] if self.delegate: diff --git a/test/lib/ansible_test/_internal/delegation.py b/test/lib/ansible_test/_internal/delegation.py index a45c136c7d0..8e2236295e4 100644 --- a/test/lib/ansible_test/_internal/delegation.py +++ b/test/lib/ansible_test/_internal/delegation.py @@ -7,6 +7,8 @@ import re import sys import tempfile +from . import types as t + from .executor import ( SUPPORTED_PYTHON_VERSIONS, HTTPTESTER_HOSTS, @@ -46,6 +48,7 @@ from .util import ( display, ANSIBLE_BIN_PATH, ANSIBLE_TEST_DATA_ROOT, + tempdir, ) from .util_common import ( @@ -81,6 +84,10 @@ from .payload import ( create_payload, ) +from .venv import ( + create_virtual_environment, +) + def check_delegation_args(args): """ @@ -124,6 +131,10 @@ def delegate_command(args, exclude, require, integration_targets): :type integration_targets: tuple[IntegrationTarget] :rtype: bool """ + if args.venv: + delegate_venv(args, exclude, require, integration_targets) + return True + if args.tox: delegate_tox(args, exclude, require, integration_targets) return True @@ -204,6 +215,53 @@ def delegate_tox(args, exclude, require, integration_targets): run_command(args, tox + cmd, env=env) +def delegate_venv(args, # type: EnvironmentConfig + exclude, # type: t.List[str] + require, # type: t.List[str] + integration_targets, # type: t.Tuple[IntegrationTarget, ...] + ): # type: (...) -> None + """Delegate ansible-test execution to a virtual environment using venv or virtualenv.""" + if args.python: + versions = (args.python_version,) + else: + versions = SUPPORTED_PYTHON_VERSIONS + + if args.httptester: + needs_httptester = sorted(target.name for target in integration_targets if 'needs/httptester/' in target.aliases) + + if needs_httptester: + display.warning('Use --docker or --remote to enable httptester for tests marked "needs/httptester": %s' % ', '.join(needs_httptester)) + + venvs = dict((version, os.path.join(ResultType.TMP.path, 'delegation', 'python%s' % version)) for version in versions) + venvs = dict((version, path) for version, path in venvs.items() if create_virtual_environment(args, version, path)) + + if not venvs: + raise ApplicationError('No usable virtual environment support found.') + + options = { + '--venv': 0, + } + + with tempdir() as inject_path: + for version, path in venvs.items(): + os.symlink(os.path.join(path, 'bin', 'python'), os.path.join(inject_path, 'python%s' % version)) + + python_interpreter = os.path.join(inject_path, 'python%s' % args.python_version) + + cmd = generate_command(args, python_interpreter, ANSIBLE_BIN_PATH, data_context().content.root, options, exclude, require) + + if isinstance(args, TestConfig): + if args.coverage and not args.coverage_label: + cmd += ['--coverage-label', 'venv'] + + env = common_environment() + env.update( + PATH=inject_path + os.pathsep + env['PATH'], + ) + + run_command(args, cmd, env=env) + + def delegate_docker(args, exclude, require, integration_targets): """ :type args: EnvironmentConfig diff --git a/test/lib/ansible_test/_internal/executor.py b/test/lib/ansible_test/_internal/executor.py index af0b9805ae5..d02b30f92c1 100644 --- a/test/lib/ansible_test/_internal/executor.py +++ b/test/lib/ansible_test/_internal/executor.py @@ -63,6 +63,7 @@ from .util import ( get_ansible_version, tempdir, open_zipfile, + SUPPORTED_PYTHON_VERSIONS, ) from .util_common import ( @@ -139,19 +140,6 @@ from .data import ( data_context, ) -REMOTE_ONLY_PYTHON_VERSIONS = ( - '2.6', -) - -SUPPORTED_PYTHON_VERSIONS = ( - '2.6', - '2.7', - '3.5', - '3.6', - '3.7', - '3.8', -) - HTTPTESTER_HOSTS = ( 'ansible.http.tests', 'sni1.ansible.http.tests', diff --git a/test/lib/ansible_test/_internal/sanity/import.py b/test/lib/ansible_test/_internal/sanity/import.py index e146be8408c..34a49c59c2b 100644 --- a/test/lib/ansible_test/_internal/sanity/import.py +++ b/test/lib/ansible_test/_internal/sanity/import.py @@ -11,6 +11,7 @@ from ..sanity import ( SanityMessage, SanityFailure, SanitySuccess, + SanitySkipped, SANITY_ROOT, ) @@ -22,10 +23,11 @@ from ..util import ( SubprocessError, remove_tree, display, - find_python, parse_to_list_of_dict, is_subdir, ANSIBLE_LIB_ROOT, + generate_pip_command, + find_python, ) from ..util_common import ( @@ -51,6 +53,10 @@ from ..coverage_util import ( coverage_context, ) +from ..venv import ( + create_virtual_environment, +) + from ..data import ( data_context, ) @@ -70,6 +76,14 @@ class ImportTest(SanityMultipleVersion): :type python_version: str :rtype: TestResult """ + capture_pip = args.verbosity < 2 + + if python_version.startswith('2.') and args.requirements: + # hack to make sure that virtualenv is available under Python 2.x + # on Python 3.x we can use the built-in venv + pip = generate_pip_command(find_python(python_version)) + run_command(args, generate_pip_install(pip, 'sanity.import', packages=['virtualenv']), capture=capture_pip) + settings = self.load_processor(args, python_version) paths = [target.path for target in targets.include] @@ -84,14 +98,9 @@ class ImportTest(SanityMultipleVersion): remove_tree(virtual_environment_path) - python = find_python(python_version) - - cmd = [python, '-m', 'virtualenv', virtual_environment_path, '--python', python, '--no-setuptools', '--no-wheel'] - - if not args.coverage: - cmd.append('--no-pip') - - run_command(args, cmd, capture=True) + if not create_virtual_environment(args, python_version, virtual_environment_path): + display.warning("Skipping sanity test '%s' on Python %s due to missing virtual environment support." % (self.name, python_version)) + return SanitySkipped(self.name, python_version) # add the importer to our virtual environment so it can be accessed through the coverage injector importer_path = os.path.join(virtual_environment_bin, 'importer.py') @@ -132,12 +141,16 @@ class ImportTest(SanityMultipleVersion): SANITY_MINIMAL_DIR=os.path.relpath(virtual_environment_path, data_context().content.root) + os.path.sep, ) + virtualenv_python = os.path.join(virtual_environment_bin, 'python') + virtualenv_pip = generate_pip_command(virtualenv_python) + # make sure coverage is available in the virtual environment if needed if args.coverage: - run_command(args, generate_pip_install(['pip'], 'sanity.import', packages=['setuptools']), env=env) - run_command(args, generate_pip_install(['pip'], 'sanity.import', packages=['coverage']), env=env) - run_command(args, ['pip', 'uninstall', '--disable-pip-version-check', '-y', 'setuptools'], env=env) - run_command(args, ['pip', 'uninstall', '--disable-pip-version-check', '-y', 'pip'], env=env) + run_command(args, generate_pip_install(virtualenv_pip, 'sanity.import', packages=['setuptools']), env=env, capture=capture_pip) + run_command(args, generate_pip_install(virtualenv_pip, 'sanity.import', packages=['coverage']), env=env, capture=capture_pip) + + run_command(args, virtualenv_pip + ['uninstall', '--disable-pip-version-check', '-y', 'setuptools'], env=env, capture=capture_pip) + run_command(args, virtualenv_pip + ['uninstall', '--disable-pip-version-check', '-y', 'pip'], env=env, capture=capture_pip) cmd = ['importer.py'] @@ -147,8 +160,6 @@ class ImportTest(SanityMultipleVersion): results = [] - virtualenv_python = os.path.join(virtual_environment_bin, 'python') - try: with coverage_context(args): stdout, stderr = intercept_command(args, cmd, self.name, env, capture=True, data=data, python_version=python_version, diff --git a/test/lib/ansible_test/_internal/sanity/validate_modules.py b/test/lib/ansible_test/_internal/sanity/validate_modules.py index b7d777b6087..d760077138a 100644 --- a/test/lib/ansible_test/_internal/sanity/validate_modules.py +++ b/test/lib/ansible_test/_internal/sanity/validate_modules.py @@ -82,13 +82,13 @@ class ValidateModulesTest(SanitySingleVersion): if data_context().content.collection: cmd.extend(['--collection', data_context().content.collection.directory]) - - if args.base_branch: - cmd.extend([ - '--base-branch', args.base_branch, - ]) else: - display.warning('Cannot perform module comparison against the base branch. Base branch not detected when running locally.') + if args.base_branch: + cmd.extend([ + '--base-branch', args.base_branch, + ]) + else: + display.warning('Cannot perform module comparison against the base branch. Base branch not detected when running locally.') try: stdout, stderr = run_command(args, cmd, env=env, capture=True) diff --git a/test/lib/ansible_test/_internal/units/__init__.py b/test/lib/ansible_test/_internal/units/__init__.py index f4221a0d849..2ca68f3b322 100644 --- a/test/lib/ansible_test/_internal/units/__init__.py +++ b/test/lib/ansible_test/_internal/units/__init__.py @@ -11,6 +11,7 @@ from ..util import ( get_available_python_versions, is_subdir, SubprocessError, + REMOTE_ONLY_PYTHON_VERSIONS, ) from ..util_common import ( @@ -45,7 +46,6 @@ from ..executor import ( Delegate, get_changes_filter, install_command_requirements, - REMOTE_ONLY_PYTHON_VERSIONS, SUPPORTED_PYTHON_VERSIONS, ) diff --git a/test/lib/ansible_test/_internal/util.py b/test/lib/ansible_test/_internal/util.py index 24e5038b8da..fce04ad0af1 100644 --- a/test/lib/ansible_test/_internal/util.py +++ b/test/lib/ansible_test/_internal/util.py @@ -99,6 +99,19 @@ ENCODING = 'utf-8' Text = type(u'') +REMOTE_ONLY_PYTHON_VERSIONS = ( + '2.6', +) + +SUPPORTED_PYTHON_VERSIONS = ( + '2.6', + '2.7', + '3.5', + '3.6', + '3.7', + '3.8', +) + def to_optional_bytes(value, errors='strict'): # type: (t.Optional[t.AnyStr], str) -> t.Optional[bytes] """Return the given value as bytes encoded using UTF-8 if not already bytes, or None if the value is None.""" @@ -301,7 +314,15 @@ def get_ansible_version(): # type: () -> str def get_available_python_versions(versions): # type: (t.List[str]) -> t.Dict[str, str] """Return a dictionary indicating which of the requested Python versions are available.""" - return dict((version, path) for version, path in ((version, find_python(version, required=False)) for version in versions) if path) + try: + return get_available_python_versions.result + except AttributeError: + pass + + get_available_python_versions.result = dict((version, path) for version, path in + ((version, find_python(version, required=False)) for version in versions) if path) + + return get_available_python_versions.result def generate_pip_command(python): @@ -893,7 +914,7 @@ def load_module(path, name): # type: (str, str) -> None @contextlib.contextmanager -def tempdir(): +def tempdir(): # type: () -> str """Creates a temporary directory that is deleted outside the context scope.""" temp_path = tempfile.mkdtemp() yield temp_path diff --git a/test/lib/ansible_test/_internal/venv.py b/test/lib/ansible_test/_internal/venv.py new file mode 100644 index 00000000000..036ec517ab2 --- /dev/null +++ b/test/lib/ansible_test/_internal/venv.py @@ -0,0 +1,154 @@ +"""Virtual environment management.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os + +from . import types as t + +from .config import ( + EnvironmentConfig, +) + +from .util import ( + find_python, + SubprocessError, + get_available_python_versions, + SUPPORTED_PYTHON_VERSIONS, + display, +) + +from .util_common import ( + run_command, +) + + +def create_virtual_environment(args, # type: EnvironmentConfig + version, # type: str + path, # type: str + system_site_packages=False, # type: bool + pip=True, # type: bool + ): # type: (...) -> bool + """Create a virtual environment using venv or virtualenv for the requested Python version.""" + if os.path.isdir(path): + display.info('Using existing Python %s virtual environment: %s' % (version, path), verbosity=1) + return True + + python = find_python(version, required=False) + python_version = tuple(int(v) for v in version.split('.')) + + if not python: + # the requested python version could not be found + return False + + if python_version >= (3, 0): + # use the built-in 'venv' module on Python 3.x + if run_venv(args, python, system_site_packages, pip, path): + display.info('Created Python %s virtual environment using "venv": %s' % (version, path), verbosity=1) + return True + + # something went wrong, this shouldn't happen + return False + + # use the installed 'virtualenv' module on the Python requested version + if run_virtualenv(args, python, python, system_site_packages, pip, path): + display.info('Created Python %s virtual environment using "virtualenv": %s' % (version, path), verbosity=1) + return True + + available_pythons = get_available_python_versions(SUPPORTED_PYTHON_VERSIONS) + + for available_python_version, available_python_interpreter in sorted(available_pythons.items()): + virtualenv_version = get_virtualenv_version(args, available_python_interpreter) + + if not virtualenv_version: + # virtualenv not available for this Python or we were unable to detect the version + continue + + if python_version == (2, 6) and virtualenv_version >= (16, 0, 0): + # virtualenv 16.0.0 dropped python 2.6 support: https://virtualenv.pypa.io/en/latest/changes/#v16-0-0-2018-05-16 + continue + + # try using 'virtualenv' from another Python to setup the desired version + if run_virtualenv(args, available_python_interpreter, python, system_site_packages, pip, path): + display.info('Created Python %s virtual environment using "virtualenv" on Python %s: %s' % (version, available_python_version, path), verbosity=1) + return True + + # no suitable 'virtualenv' available + return False + + +def run_venv(args, # type: EnvironmentConfig + run_python, # type: str + system_site_packages, # type: bool + pip, # type: bool + path, # type: str + ): # type: (...) -> bool + """Create a virtual environment using the 'venv' module. Not available on Python 2.x.""" + cmd = [run_python, '-m', 'venv'] + + if system_site_packages: + cmd.append('--system-site-packages') + + if not pip: + cmd.append('--without-pip') + + cmd.append(path) + + try: + run_command(args, cmd, capture=True) + except SubprocessError: + return False + + return True + + +def run_virtualenv(args, # type: EnvironmentConfig + run_python, # type: str + env_python, # type: str + system_site_packages, # type: bool + pip, # type: bool + path, # type: str + ): # type: (...) -> bool + """Create a virtual environment using the 'virtualenv' module.""" + cmd = [run_python, '-m', 'virtualenv', '--python', env_python] + + if system_site_packages: + cmd.append('--system-site-packages') + + if not pip: + cmd.append('--no-pip') + + cmd.append(path) + + try: + run_command(args, cmd, capture=True) + except SubprocessError: + return False + + return True + + +def get_virtualenv_version(args, python): # type: (EnvironmentConfig, str) -> t.Optional[t.Tuple[int, ...]] + """Get the virtualenv version for the given python intepreter, if available.""" + try: + return get_virtualenv_version.result + except AttributeError: + pass + + get_virtualenv_version.result = None + + cmd = [python, '-m', 'virtualenv', '--version'] + + try: + stdout = run_command(args, cmd, capture=True)[0] + except SubprocessError: + stdout = '' + + if stdout: + # noinspection PyBroadException + try: + get_virtualenv_version.result = tuple(int(v) for v in stdout.strip().split('.')) + except Exception: # pylint: disable=broad-except + pass + + return get_virtualenv_version.result