From 4255ee8e5a467bba54241306674d23e674d4ad3e Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Mon, 15 Jul 2019 16:20:03 -0700 Subject: [PATCH] Prepare ansible-test for collections support. Another round of changes to prepare ansible-test for supporting collections to help keep later PRs a manageable size. --- test/runner/lib/delegation.py | 61 ++++++++++++++++----------- test/runner/lib/executor.py | 8 ++-- test/runner/lib/import_analysis.py | 26 +++++++----- test/runner/lib/manage_ci.py | 12 +++--- test/runner/lib/payload.py | 27 ++++++++++++ test/runner/lib/sanity/import.py | 6 ++- test/runner/lib/sanity/pslint.py | 5 ++- test/runner/lib/sanity/pylint.py | 10 +++-- test/runner/lib/sanity/sanity_docs.py | 6 ++- test/runner/lib/test.py | 3 ++ test/runner/lib/types.py | 1 + 11 files changed, 113 insertions(+), 52 deletions(-) create mode 100644 test/runner/lib/payload.py diff --git a/test/runner/lib/delegation.py b/test/runner/lib/delegation.py index 94709ae4ae8..f71e6068cec 100644 --- a/test/runner/lib/delegation.py +++ b/test/runner/lib/delegation.py @@ -7,9 +7,6 @@ import re import sys import tempfile -import lib.pytar -import lib.thread - from lib.executor import ( SUPPORTED_PYTHON_VERSIONS, HTTPTESTER_HOSTS, @@ -49,6 +46,7 @@ from lib.util import ( from lib.util_common import ( run_command, + INSTALL_ROOT, ) from lib.docker_util import ( @@ -71,6 +69,10 @@ from lib.target import ( IntegrationTarget, ) +from lib.payload import ( + create_payload, +) + def check_delegation_args(args): """ @@ -163,7 +165,7 @@ def delegate_tox(args, exclude, require, integration_targets): tox.append('--') - cmd = generate_command(args, None, os.path.abspath('bin/ansible-test'), options, exclude, require) + cmd = generate_command(args, None, INSTALL_ROOT, INSTALL_ROOT, options, exclude, require) if not args.python: cmd += ['--python', version] @@ -224,7 +226,11 @@ def delegate_docker(args, exclude, require, integration_targets): } python_interpreter = get_python_interpreter(args, get_docker_completion(), args.docker_raw) - cmd = generate_command(args, python_interpreter, '/root/ansible/bin/ansible-test', options, exclude, require) + + install_root = '/root/ansible' + content_root = install_root + + cmd = generate_command(args, python_interpreter, install_root, content_root, options, exclude, require) if isinstance(args, TestConfig): if args.coverage and not args.coverage_label: @@ -243,13 +249,7 @@ def delegate_docker(args, exclude, require, integration_targets): with tempfile.NamedTemporaryFile(prefix='ansible-source-', suffix='.tgz') as local_source_fd: try: - if not args.explain: - if args.docker_keep_git: - tar_filter = lib.pytar.AllowGitTarFilter() - else: - tar_filter = lib.pytar.DefaultTarFilter() - - lib.pytar.create_tarfile(local_source_fd.name, '.', tar_filter) + create_payload(args, local_source_fd.name) if use_httptester: httptester_id = run_httptester(args) @@ -296,7 +296,7 @@ def delegate_docker(args, exclude, require, integration_targets): test_id = test_id.strip() # write temporary files to /root since /tmp isn't ready immediately on container start - docker_put(args, test_id, 'test/runner/setup/docker.sh', '/root/docker.sh') + docker_put(args, test_id, os.path.join(INSTALL_ROOT, 'test/runner/setup/docker.sh'), '/root/docker.sh') docker_exec(args, test_id, ['/bin/bash', '/root/docker.sh']) docker_put(args, test_id, local_source_fd.name, '/root/ansible.tgz') docker_exec(args, test_id, ['mkdir', '/root/ansible']) @@ -310,16 +310,16 @@ def delegate_docker(args, exclude, require, integration_targets): # also disconnect from the network once requirements have been installed if isinstance(args, UnitsConfig): writable_dirs = [ - '/root/ansible/.pytest_cache', + os.path.join(content_root, '.pytest_cache'), ] docker_exec(args, test_id, ['mkdir', '-p'] + writable_dirs) docker_exec(args, test_id, ['chmod', '777'] + writable_dirs) - docker_exec(args, test_id, ['find', '/root/ansible/test/results/', '-type', 'd', '-exec', 'chmod', '777', '{}', '+']) + docker_exec(args, test_id, ['find', os.path.join(content_root, 'test/results/'), '-type', 'd', '-exec', 'chmod', '777', '{}', '+']) docker_exec(args, test_id, ['chmod', '755', '/root']) - docker_exec(args, test_id, ['chmod', '644', '/root/ansible/%s' % args.metadata_path]) + docker_exec(args, test_id, ['chmod', '644', os.path.join(content_root, args.metadata_path)]) docker_exec(args, test_id, ['useradd', 'pytest', '--create-home']) @@ -338,7 +338,7 @@ def delegate_docker(args, exclude, require, integration_targets): docker_exec(args, test_id, cmd, options=cmd_options) finally: with tempfile.NamedTemporaryFile(prefix='ansible-result-', suffix='.tgz') as local_result_fd: - docker_exec(args, test_id, ['tar', 'czf', '/root/results.tgz', '-C', '/root/ansible/test', 'results']) + docker_exec(args, test_id, ['tar', 'czf', '/root/results.tgz', '-C', os.path.join(content_root, 'test'), 'results']) docker_get(args, test_id, '/root/results.tgz', local_result_fd.name) run_command(args, ['tar', 'oxzf', local_result_fd.name, '-C', 'test']) finally: @@ -377,6 +377,7 @@ def delegate_remote(args, exclude, require, integration_targets): httptester_id = None ssh_options = [] + content_root = None try: core_ci.start() @@ -399,7 +400,11 @@ def delegate_remote(args, exclude, require, integration_targets): } python_interpreter = get_python_interpreter(args, get_remote_completion(), args.remote) - cmd = generate_command(args, python_interpreter, 'ansible/bin/ansible-test', options, exclude, require) + + install_root = 'ansible' + content_root = install_root + + cmd = generate_command(args, python_interpreter, install_root, content_root, options, exclude, require) if httptester_id: cmd += ['--inject-httptester'] @@ -440,8 +445,8 @@ def delegate_remote(args, exclude, require, integration_targets): if args.raw: download = False - if download: - manage.ssh('rm -rf /tmp/results && cp -a ansible/test/results /tmp/results && chmod -R a+r /tmp/results') + if download and content_root: + manage.ssh('rm -rf /tmp/results && cp -a %s/test/results /tmp/results && chmod -R a+r /tmp/results' % content_root) manage.download('/tmp/results', 'test') finally: if args.remote_terminate == 'always' or (args.remote_terminate == 'success' and success): @@ -451,11 +456,12 @@ def delegate_remote(args, exclude, require, integration_targets): docker_rm(args, httptester_id) -def generate_command(args, python_interpreter, path, options, exclude, require): +def generate_command(args, python_interpreter, install_root, content_root, options, exclude, require): """ :type args: EnvironmentConfig :type python_interpreter: str | None - :type path: str + :type install_root: str + :type content_root: str :type options: dict[str, int] :type exclude: list[str] :type require: list[str] @@ -463,7 +469,7 @@ def generate_command(args, python_interpreter, path, options, exclude, require): """ options['--color'] = 1 - cmd = [path] + cmd = [os.path.join(install_root, 'bin/ansible-test')] if python_interpreter: cmd = [python_interpreter] + cmd @@ -472,7 +478,14 @@ def generate_command(args, python_interpreter, path, options, exclude, require): # This is only needed because ansible-test relies on Python's file system encoding. # Environments that do not have the locale configured are thus unable to work with unicode file paths. # Examples include FreeBSD and some Linux containers. - cmd = ['/usr/bin/env', 'LC_ALL=en_US.UTF-8'] + cmd + env_vars = dict( + LC_ALL='en_US.UTF-8', + ANSIBLE_TEST_CONTENT_ROOT=content_root, + ) + + env_args = ['%s=%s' % (key, env_vars[key]) for key in sorted(env_vars)] + + cmd = ['/usr/bin/env'] + env_args + cmd cmd += list(filter_options(args, sys.argv[1:], options, exclude, require)) cmd += ['--color', 'yes' if args.color else 'no'] diff --git a/test/runner/lib/executor.py b/test/runner/lib/executor.py index 956e60f2581..95779f891c9 100644 --- a/test/runner/lib/executor.py +++ b/test/runner/lib/executor.py @@ -20,7 +20,6 @@ import shutil import lib.types as t -import lib.pytar import lib.thread from lib.core_ci import ( @@ -279,6 +278,9 @@ def generate_egg_info(args): """ :type args: EnvironmentConfig """ + if not os.path.exists(os.path.join(INSTALL_ROOT, 'setup.py')): + return + if os.path.isdir(os.path.join(INSTALL_ROOT, 'lib/ansible.egg-info')): return @@ -292,8 +294,8 @@ def generate_pip_install(pip, command, packages=None): :type packages: list[str] | None :rtype: list[str] | None """ - constraints = 'test/runner/requirements/constraints.txt' - requirements = 'test/runner/requirements/%s.txt' % command + constraints = os.path.join(INSTALL_ROOT, 'test/runner/requirements/constraints.txt') + requirements = os.path.join(INSTALL_ROOT, 'test/runner/requirements/%s.txt' % command) options = [] diff --git a/test/runner/lib/import_analysis.py b/test/runner/lib/import_analysis.py index f9e452ff437..f7c388eca2a 100644 --- a/test/runner/lib/import_analysis.py +++ b/test/runner/lib/import_analysis.py @@ -122,23 +122,27 @@ def enumerate_module_utils(): module_utils = [] base_path = 'lib/ansible/module_utils' + paths = [] + for root, _dir_names, file_names in os.walk(base_path): for file_name in file_names: - path = os.path.join(root, file_name) - name, ext = os.path.splitext(file_name) + paths.append(os.path.join(root, file_name)) - if path == 'lib/ansible/module_utils/__init__.py': - continue + for path in paths: + name, ext = os.path.splitext(path) - if ext != '.py': - continue + if path == 'lib/ansible/module_utils/__init__.py': + continue + + if ext != '.py': + continue - if name == '__init__': - module_util = root - else: - module_util = os.path.join(root, name) + if name.endswith('/__init__'): + module_util = os.path.dirname(name) + else: + module_util = name - module_utils.append(module_util[4:].replace('/', '.')) + module_utils.append(module_util[4:].replace('/', '.')) return set(module_utils) diff --git a/test/runner/lib/manage_ci.py b/test/runner/lib/manage_ci.py index c2a14cfe3ed..33d7edb0b4f 100644 --- a/test/runner/lib/manage_ci.py +++ b/test/runner/lib/manage_ci.py @@ -6,8 +6,6 @@ import os import tempfile import time -import lib.pytar - from lib.util import ( SubprocessError, ApplicationError, @@ -17,6 +15,7 @@ from lib.util import ( from lib.util_common import ( intercept_command, run_command, + INSTALL_ROOT, ) from lib.core_ci import ( @@ -31,6 +30,10 @@ from lib.config import ( ShellConfig, ) +from lib.payload import ( + create_payload, +) + class ManageWindowsCI: """Manage access to a Windows instance provided by Ansible Core CI.""" @@ -237,7 +240,7 @@ class ManagePosixCI: """Configure remote host for testing. :type python_version: str """ - self.upload('test/runner/setup/remote.sh', '/tmp') + self.upload(os.path.join(INSTALL_ROOT, 'test/runner/setup/remote.sh'), '/tmp') self.ssh('chmod +x /tmp/remote.sh && /tmp/remote.sh %s %s' % (self.core_ci.platform, python_version)) def upload_source(self): @@ -246,8 +249,7 @@ class ManagePosixCI: remote_source_dir = '/tmp' remote_source_path = os.path.join(remote_source_dir, os.path.basename(local_source_fd.name)) - if not self.core_ci.args.explain: - lib.pytar.create_tarfile(local_source_fd.name, '.', lib.pytar.DefaultTarFilter()) + create_payload(self.core_ci.args, local_source_fd.name) self.upload(local_source_fd.name, remote_source_dir) self.ssh('rm -rf ~/ansible && mkdir ~/ansible && cd ~/ansible && tar oxzf %s' % remote_source_path) diff --git a/test/runner/lib/payload.py b/test/runner/lib/payload.py new file mode 100644 index 00000000000..52ffeb52f38 --- /dev/null +++ b/test/runner/lib/payload.py @@ -0,0 +1,27 @@ +"""Payload management for sending Ansible files and test content to other systems (VMs, containers).""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from lib.config import ( + CommonConfig, + EnvironmentConfig, +) + +from lib.pytar import ( + AllowGitTarFilter, + create_tarfile, + DefaultTarFilter, +) + + +def create_payload(args, dst_path): # type: (CommonConfig, str) -> None + """Create a payload for delegation.""" + if args.explain: + return + + if isinstance(args, EnvironmentConfig) and args.docker_keep_git: + tar_filter = AllowGitTarFilter() + else: + tar_filter = DefaultTarFilter() + + create_tarfile(dst_path, '.', tar_filter) diff --git a/test/runner/lib/sanity/import.py b/test/runner/lib/sanity/import.py index 30ca88261a5..d6c4d708fca 100644 --- a/test/runner/lib/sanity/import.py +++ b/test/runner/lib/sanity/import.py @@ -20,11 +20,13 @@ from lib.util import ( read_lines_without_comments, parse_to_list_of_dict, make_dirs, + is_subdir, ) from lib.util_common import ( intercept_command, run_command, + INSTALL_ROOT, ) from lib.ansible_util import ( @@ -58,7 +60,7 @@ class ImportTest(SanityMultipleVersion): i.path for i in targets.include if os.path.splitext(i.path)[1] == '.py' and - (i.path.startswith('lib/ansible/modules/') or i.path.startswith('lib/ansible/module_utils/')) and + (is_subdir(i.path, 'lib/ansible/modules/') or is_subdir(i.path, 'lib/ansible/module_utils/')) and i.path not in skip_paths_set ) @@ -85,7 +87,7 @@ class ImportTest(SanityMultipleVersion): # add the importer to our virtual environment so it can be accessed through the coverage injector importer_path = os.path.join(virtual_environment_bin, 'importer.py') if not args.explain: - os.symlink(os.path.abspath('test/sanity/import/importer.py'), importer_path) + os.symlink(os.path.abspath(os.path.join(INSTALL_ROOT, 'test/sanity/import/importer.py')), importer_path) # create a minimal python library python_path = os.path.abspath('test/runner/.tox/import/lib') diff --git a/test/runner/lib/sanity/pslint.py b/test/runner/lib/sanity/pslint.py index d9495789280..0174ccdaf23 100644 --- a/test/runner/lib/sanity/pslint.py +++ b/test/runner/lib/sanity/pslint.py @@ -25,6 +25,7 @@ from lib.util import ( from lib.util_common import ( run_command, + INSTALL_ROOT, ) from lib.config import ( @@ -84,8 +85,8 @@ class PslintTest(SanitySingleVersion): # Make sure requirements are installed before running sanity checks cmds = [ - ['test/runner/requirements/sanity.ps1'], - ['test/sanity/pslint/pslint.ps1'] + paths + [os.path.join(INSTALL_ROOT, 'test/runner/requirements/sanity.ps1')], + [os.path.join(INSTALL_ROOT, 'test/sanity/pslint/pslint.ps1')] + paths ] stdout = '' diff --git a/test/runner/lib/sanity/pylint.py b/test/runner/lib/sanity/pylint.py index cd6cdd5b8bf..b1595f321ab 100644 --- a/test/runner/lib/sanity/pylint.py +++ b/test/runner/lib/sanity/pylint.py @@ -24,6 +24,7 @@ from lib.util import ( read_lines_without_comments, ConfigParser, INSTALL_ROOT, + is_subdir, ) from lib.util_common import ( @@ -113,16 +114,17 @@ class PylintTest(SanitySingleVersion): skip_paths_set = set(skip_paths) - paths = sorted(i.path for i in targets.include if (os.path.splitext(i.path)[1] == '.py' or i.path.startswith('bin/')) and i.path not in skip_paths_set) + paths = sorted(i.path for i in targets.include if (os.path.splitext(i.path)[1] == '.py' or is_subdir(i.path, 'bin/')) and i.path not in skip_paths_set) - module_paths = [p.split(os.path.sep) for p in paths if p.startswith('lib/ansible/modules/')] + module_paths = [p.split(os.path.sep) for p in paths if is_subdir(p, 'lib/ansible/modules/')] module_dirs = sorted(set([p[3] for p in module_paths if len(p) > 4])) large_module_group_threshold = 500 large_module_groups = [key for key, value in itertools.groupby(module_paths, lambda p: p[3] if len(p) > 4 else '') if len(list(value)) > large_module_group_threshold] - large_module_group_paths = [p.split(os.path.sep) for p in paths if any(p.startswith('lib/ansible/modules/%s/' % g) for g in large_module_groups)] + large_module_group_paths = [p.split(os.path.sep) for p in paths + if any(is_subdir(p, os.path.join('lib/ansible/modules/', g)) for g in large_module_groups)] large_module_group_dirs = sorted(set([os.path.sep.join(p[3:5]) for p in large_module_group_paths if len(p) > 5])) contexts = [] @@ -148,7 +150,7 @@ class PylintTest(SanitySingleVersion): :type path_to_filter: str :rtype: bool """ - return path_to_filter.startswith(path_filter) + return is_subdir(path_to_filter, path_filter) return context_filter diff --git a/test/runner/lib/sanity/sanity_docs.py b/test/runner/lib/sanity/sanity_docs.py index 3527fe31978..782d9b3c748 100644 --- a/test/runner/lib/sanity/sanity_docs.py +++ b/test/runner/lib/sanity/sanity_docs.py @@ -4,6 +4,10 @@ __metaclass__ = type import os +from lib.util import ( + INSTALL_ROOT, +) + from lib.sanity import ( SanitySingleVersion, SanityMessage, @@ -26,7 +30,7 @@ class SanityDocsTest(SanitySingleVersion): :type targets: SanityTargets :rtype: TestResult """ - sanity_dir = 'docs/docsite/rst/dev_guide/testing/sanity' + sanity_dir = os.path.join(INSTALL_ROOT, 'docs/docsite/rst/dev_guide/testing/sanity') sanity_docs = set(part[0] for part in (os.path.splitext(name) for name in os.listdir(sanity_dir)) if part[1] == '.rst') sanity_tests = set(sanity_test.name for sanity_test in sanity_get_tests()) diff --git a/test/runner/lib/test.py b/test/runner/lib/test.py index 68ed9280058..eeaac39889c 100644 --- a/test/runner/lib/test.py +++ b/test/runner/lib/test.py @@ -8,6 +8,7 @@ import os from lib.util import ( display, + make_dirs, ) from lib.config import ( @@ -332,6 +333,8 @@ class TestFailure(TestResult): if args.explain: return + make_dirs(os.path.dirname(path)) + with open(path, 'w') as bot_fd: json.dump(bot_data, bot_fd, indent=4, sort_keys=True) bot_fd.write('\n') diff --git a/test/runner/lib/types.py b/test/runner/lib/types.py index c1c278c2295..1606a4f8d38 100644 --- a/test/runner/lib/types.py +++ b/test/runner/lib/types.py @@ -5,6 +5,7 @@ __metaclass__ = type try: from typing import ( Any, + Callable, Dict, FrozenSet, Iterable,