Overhaul ansible-test integration tests. (#76111)

* Overhaul ansible-test integration tests.
* ansible-test - Fix import test pyyaml usage.
* ansible-test - Remove unused import.
* ansible-test - Fix traceback when pip is unavailable.
* ansible-test - Fix typo in port forwarding message.
* ansible-test - Fix controller logic in requirements install.
* Fix unit tests in ansible-test integration test.

Unit tests are now run for available Python versions which
provide `virtualenv` (Python 2.x) or `venv` (Python 3.x).
pull/76137/head
Matt Clay 3 years ago committed by GitHub
parent b9694ce4fb
commit cae7d2a671
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -0,0 +1,3 @@
bugfixes:
- ansible-test - Target integration test requirements are now correctly installed for target environments running on the controller.
- ansible-test - Automatic target requirements installation is now based on the target environment instead of the controller environment.

@ -0,0 +1,2 @@
bugfixes:
- ansible-test - Fix traceback in ``import`` sanity test on Python 2.7 when ``pip`` is not available.

@ -0,0 +1,2 @@
bugfixes:
- ansible-test - Fix installation and usage of ``pyyaml`` requirement for the ``import`` sanity test for collections.

@ -1,2 +1,3 @@
shippable/posix/group1
shippable/posix/group1 # runs in the distro test containers
shippable/generic/group1 # runs in the default test container
context/controller

@ -1,20 +0,0 @@
#!/usr/bin/env bash
set -eux -o pipefail
cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
cd "${WORK_DIR}/ansible_collections/ns/col_constraints"
# common args for all tests
# each test will be run in a separate venv to verify that requirements have been properly specified
common=(--venv --python "${ANSIBLE_TEST_PYTHON_VERSION}" --color --truncate 0 "${@}")
# unit tests
rm -rf "tests/output"
ansible-test units "${common[@]}"
# integration tests
rm -rf "tests/output"
ansible-test integration "${common[@]}"

@ -5,22 +5,13 @@ set -eux -o pipefail
cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
cd "${WORK_DIR}/ansible_collections/ns/col"
# rename the sanity ignore file to match the current ansible version and update import ignores with the python version
ansible_version="$(python -c 'import ansible.release; print(".".join(ansible.release.__version__.split(".")[:2]))')"
if [[ "${ANSIBLE_TEST_PYTHON_VERSION}" =~ ^2\. ]] || [[ "${ANSIBLE_TEST_PYTHON_VERSION}" =~ ^3\.[567] ]]; then
# Non-module/module_utils plugins are not checked on these remote-only Python versions
sed "s/ import$/ import-${ANSIBLE_TEST_PYTHON_VERSION}/;" < "tests/sanity/ignore.txt" | grep -v 'plugins/[^m].* import' > "tests/sanity/ignore-${ansible_version}.txt"
else
sed "s/ import$/ import-${ANSIBLE_TEST_PYTHON_VERSION}/;" < "tests/sanity/ignore.txt" > "tests/sanity/ignore-${ansible_version}.txt"
fi
cat "tests/sanity/ignore-${ansible_version}.txt"
"${TEST_DIR}/collection-tests/update-ignore.py"
# common args for all tests
common=(--venv --color --truncate 0 "${@}")
test_common=("${common[@]}" --python "${ANSIBLE_TEST_PYTHON_VERSION}")
# run a lightweight test that generates code coverge output
ansible-test sanity --test import "${test_common[@]}" --coverage
ansible-test sanity --test import "${common[@]}" --coverage
# report on code coverage in all supported formats
ansible-test coverage report "${common[@]}"

@ -0,0 +1,8 @@
#!/usr/bin/env bash
set -eux -o pipefail
cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
cd "${WORK_DIR}/ansible_collections/ns/col_constraints"
ansible-test integration --venv --color --truncate 0 "${@}"

@ -0,0 +1,8 @@
#!/usr/bin/env bash
set -eux -o pipefail
cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
cd "${WORK_DIR}/ansible_collections/ns/col"
ansible-test integration --venv --color --truncate 0 "${@}"

@ -0,0 +1,10 @@
#!/usr/bin/env bash
set -eux -o pipefail
cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
cd "${WORK_DIR}/ansible_collections/ns/col"
"${TEST_DIR}/collection-tests/update-ignore.py"
ansible-test sanity --color --truncate 0 "${@}"

@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -eux -o pipefail
cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
cd "${WORK_DIR}/ansible_collections/ns/col_constraints"
options=$("${TEST_DIR}"/collection-tests/venv-pythons.py)
IFS=', ' read -r -a pythons <<< "${options}"
ansible-test units --color --truncate 0 "${pythons[@]}" "${@}"

@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -eux -o pipefail
cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
cd "${WORK_DIR}/ansible_collections/ns/col"
options=$("${TEST_DIR}"/collection-tests/venv-pythons.py)
IFS=', ' read -r -a pythons <<< "${options}"
ansible-test units --color --truncate 0 "${pythons[@]}" "${@}"

@ -0,0 +1,51 @@
#!/usr/bin/env python
"""Rewrite a sanity ignore file to expand Python versions for import ignores and write the file out with the correct Ansible version in the name."""
import os
import sys
from ansible import release
def main():
ansible_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(release.__file__))))
source_root = os.path.join(ansible_root, 'test', 'lib')
sys.path.insert(0, source_root)
from ansible_test._internal import constants
src_path = 'tests/sanity/ignore.txt'
directory = os.path.dirname(src_path)
name, ext = os.path.splitext(os.path.basename(src_path))
major_minor = '.'.join(release.__version__.split('.')[:2])
dst_path = os.path.join(directory, f'{name}-{major_minor}{ext}')
with open(src_path) as src_file:
src_lines = src_file.read().splitlines()
dst_lines = []
for line in src_lines:
path, rule = line.split(' ')
if rule != 'import':
dst_lines.append(line)
continue
if path.startswith('plugins/module'):
python_versions = constants.SUPPORTED_PYTHON_VERSIONS
else:
python_versions = constants.CONTROLLER_PYTHON_VERSIONS
for python_version in python_versions:
dst_lines.append(f'{line}-{python_version}')
ignores = '\n'.join(dst_lines) + '\n'
with open(dst_path, 'w') as dst_file:
dst_file.write(ignores)
if __name__ == '__main__':
main()

@ -0,0 +1,42 @@
#!/usr/bin/env python
"""Return target Python options for use with ansible-test."""
import os
import shutil
import subprocess
import sys
from ansible import release
def main():
ansible_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(release.__file__))))
source_root = os.path.join(ansible_root, 'test', 'lib')
sys.path.insert(0, source_root)
from ansible_test._internal import constants
args = []
for python_version in constants.SUPPORTED_PYTHON_VERSIONS:
executable = shutil.which(f'python{python_version}')
if executable:
if python_version.startswith('2.'):
cmd = [executable, '-m', 'virtualenv', '--version']
else:
cmd = [executable, '-m', 'venv', '--help']
process = subprocess.run(cmd, capture_output=True, check=False)
print(f'{executable} - {"fail" if process.returncode else "pass"}', file=sys.stderr)
if not process.returncode:
args.extend(['--target-python', f'venv/{python_version}'])
print(' '.join(args))
if __name__ == '__main__':
main()

@ -1,49 +0,0 @@
#!/usr/bin/env bash
set -eux -o pipefail
cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
cd "${WORK_DIR}/ansible_collections/ns/col"
# rename the sanity ignore file to match the current ansible version and update import ignores with the python version
ansible_version="$(python -c 'import ansible.release; print(".".join(ansible.release.__version__.split(".")[:2]))')"
if [[ "${ANSIBLE_TEST_PYTHON_VERSION}" =~ ^2\. ]] || [[ "${ANSIBLE_TEST_PYTHON_VERSION}" =~ ^3\.[567] ]]; then
# Non-module/module_utils plugins are not checked on these remote-only Python versions
sed "s/ import$/ import-${ANSIBLE_TEST_PYTHON_VERSION}/;" < "tests/sanity/ignore.txt" | grep -v 'plugins/[^m].* import' > "tests/sanity/ignore-${ansible_version}.txt"
else
sed "s/ import$/ import-${ANSIBLE_TEST_PYTHON_VERSION}/;" < "tests/sanity/ignore.txt" > "tests/sanity/ignore-${ansible_version}.txt"
fi
cat "tests/sanity/ignore-${ansible_version}.txt"
# common args for all tests
# each test will be run in a separate venv to verify that requirements have been properly specified
common=(--venv --python "${ANSIBLE_TEST_PYTHON_VERSION}" --color --truncate 0 "${@}")
# sanity tests
tests=()
set +x
while IFS='' read -r line; do
tests+=("$line");
done < <(
ansible-test sanity --list-tests
)
set -x
for test in "${tests[@]}"; do
rm -rf "tests/output"
ansible-test sanity "${common[@]}" --test "${test}"
done
# unit tests
rm -rf "tests/output"
ansible-test units "${common[@]}"
# integration tests
rm -rf "tests/output"
ansible-test integration "${common[@]}"

@ -0,0 +1 @@
pyyaml == 5.4.1 # needed for yaml_to_json.py

@ -440,9 +440,10 @@ def command_integration_filtered(
if isinstance(target_profile, ControllerProfile):
if host_state.controller_profile.python.path != target_profile.python.path:
install_requirements(args, target_python, command=True) # integration
install_requirements(args, target_python, command=True, controller=False) # integration
elif isinstance(target_profile, SshTargetHostProfile):
install_requirements(args, target_python, command=True, connection=target_profile.get_controller_target_connections()[0]) # integration
connection = target_profile.get_controller_target_connections()[0]
install_requirements(args, target_python, command=True, controller=False, connection=connection) # integration
coverage_manager = CoverageManager(args, host_state, inventory_path)
coverage_manager.setup()

@ -2,7 +2,6 @@
from __future__ import annotations
import os
import tempfile
import typing as t
from . import (
@ -34,6 +33,7 @@ from ...util import (
display,
parse_to_list_of_dict,
is_subdir,
ANSIBLE_TEST_TOOLS_ROOT,
)
from ...util_common import (
@ -45,6 +45,7 @@ from ...ansible_util import (
)
from ...python_requirements import (
PipUnavailableError,
install_requirements,
)
@ -94,7 +95,10 @@ class ImportTest(SanityMultipleVersion):
if python.version.startswith('2.'):
# hack to make sure that virtualenv is available under Python 2.x
# on Python 3.x we can use the built-in venv
install_requirements(args, python, virtualenv=True) # sanity (import)
try:
install_requirements(args, python, virtualenv=True, controller=False) # sanity (import)
except PipUnavailableError as ex:
display.warning(ex)
temp_root = os.path.join(ResultType.TMP.path, 'sanity', 'import')
@ -134,24 +138,24 @@ class ImportTest(SanityMultipleVersion):
)
if data_context().content.collection:
external_python = create_sanity_virtualenv(args, args.controller_python, self.name, context=self.name)
env.update(
SANITY_COLLECTION_FULL_NAME=data_context().content.collection.full_name,
SANITY_EXTERNAL_PYTHON=python.path,
SANITY_EXTERNAL_PYTHON=external_python.path,
SANITY_YAML_TO_JSON=os.path.join(ANSIBLE_TEST_TOOLS_ROOT, 'yaml_to_json.py'),
)
display.info(import_type + ': ' + data, verbosity=4)
cmd = ['importer.py']
try:
with tempfile.TemporaryDirectory(prefix='ansible-test', suffix='-import') as temp_dir:
# make the importer available in the temporary directory
os.symlink(os.path.abspath(os.path.join(TARGET_SANITY_ROOT, 'import', 'importer.py')), os.path.join(temp_dir, 'importer.py'))
os.symlink(os.path.abspath(os.path.join(TARGET_SANITY_ROOT, 'import', 'yaml_to_json.py')), os.path.join(temp_dir, 'yaml_to_json.py'))
# add the importer to the path so it can be accessed through the coverage injector
env['PATH'] = os.pathsep.join([temp_dir, env['PATH']])
env.update(
PATH=os.pathsep.join([os.path.join(TARGET_SANITY_ROOT, 'import'), env['PATH']]),
)
try:
stdout, stderr = cover_python(args, virtualenv_python, cmd, self.name, env, capture=True, data=data)
if stdout or stderr:

@ -227,7 +227,7 @@ def command_units(args): # type: (UnitsConfig) -> None
controller = any(test_context == TestContext.controller for test_context, python, paths, env in final_candidates)
if args.requirements_mode != 'skip':
install_requirements(args, target_profile.python, ansible=controller, command=True) # units
install_requirements(args, target_profile.python, ansible=controller, command=True, controller=False) # units
test_sets.extend(final_candidates)

@ -829,7 +829,7 @@ def cleanup_ssh_ports(
for process in ssh_processes:
process.terminate()
display.info('Waiting for the %s host SSH port forwarding processs(es) to terminate.' % host_type, verbosity=1)
display.info('Waiting for the %s host SSH port forwarding process(es) to terminate.' % host_type, verbosity=1)
for process in ssh_processes:
process.wait()

@ -25,6 +25,7 @@ from .util import (
ANSIBLE_TEST_DATA_ROOT,
ANSIBLE_TEST_TARGET_ROOT,
ANSIBLE_TEST_TOOLS_ROOT,
ApplicationError,
SubprocessError,
display,
find_executable,
@ -65,6 +66,12 @@ REQUIREMENTS_SCRIPT_PATH = os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'setup', 'requ
# Pip Abstraction
class PipUnavailableError(ApplicationError):
"""Exception raised when pip is not available."""
def __init__(self, python): # type: (PythonConfig) -> None
super().__init__(f'Python {python.version} at "{python.path}" does not have pip available.')
@dataclasses.dataclass(frozen=True)
class PipCommand:
"""Base class for pip commands."""""
@ -97,6 +104,11 @@ class PipUninstall(PipCommand):
ignore_errors: bool
@dataclasses.dataclass(frozen=True)
class PipVersion(PipCommand):
"""Details required to get the pip version."""
# Entry Points
@ -107,13 +119,12 @@ def install_requirements(
command=False, # type: bool
coverage=False, # type: bool
virtualenv=False, # type: bool
controller=True, # type: bool
connection=None, # type: t.Optional[Connection]
): # type: (...) -> None
"""Install requirements for the given Python using the specified arguments."""
create_result_directories(args)
controller = not connection
if not requirements_allowed(args, controller):
return
@ -220,7 +231,18 @@ def run_pip(
script = prepare_pip_script(commands)
if not args.explain:
try:
connection.run([python.path], data=script)
except SubprocessError:
script = prepare_pip_script([PipVersion()])
try:
connection.run([python.path], data=script, capture=True)
except SubprocessError as ex:
if 'pip is unavailable:' in ex.stdout + ex.stderr:
raise PipUnavailableError(python)
raise
# Collect

@ -23,7 +23,8 @@ def main():
ansible_path = os.path.dirname(os.path.dirname(ansible.__file__))
temp_path = os.environ['SANITY_TEMP_PATH'] + os.path.sep
external_python = os.environ.get('SANITY_EXTERNAL_PYTHON') or sys.executable
external_python = os.environ.get('SANITY_EXTERNAL_PYTHON')
yaml_to_json_path = os.environ.get('SANITY_YAML_TO_JSON')
collection_full_name = os.environ.get('SANITY_COLLECTION_FULL_NAME')
collection_root = os.environ.get('ANSIBLE_COLLECTIONS_PATH')
import_type = os.environ.get('SANITY_IMPORTER_TYPE')
@ -48,7 +49,6 @@ def main():
from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder
from ansible.utils.collection_loader import _collection_finder
yaml_to_json_path = os.path.join(os.path.dirname(__file__), 'yaml_to_json.py')
yaml_to_dict_cache = {}
# unique ISO date marker matching the one present in yaml_to_json.py

@ -5,6 +5,7 @@ __metaclass__ = type
import logging
import re
import runpy
import sys
import warnings
BUILTIN_FILTERER_FILTER = logging.Filterer.filter
@ -51,7 +52,11 @@ def main():
# Python 2.7 cannot use the -W option to match warning text after a colon. This makes it impossible to match specific warning messages.
warnings.filterwarnings('ignore', message_filter)
try:
runpy.run_module('pip.__main__', run_name='__main__', alter_sys=True)
except ImportError as ex:
print('pip is unavailable: %s' % ex)
sys.exit(1)
if __name__ == '__main__':

@ -114,6 +114,18 @@ def uninstall(pip, options): # type: (str, t.Dict[str, t.Any]) -> None
raise
# noinspection PyUnusedLocal
def version(pip, options): # type: (str, t.Dict[str, t.Any]) -> None
"""Report the pip version."""
del options
options = common_pip_options()
command = [sys.executable, pip, '-V'] + options
execute_command(command, capture=True)
def common_pip_options(): # type: () -> t.List[str]
"""Return a list of common pip options."""
return [

Loading…
Cancel
Save