From 57dc7ec265bbc741126fa46e44ff3bb6adae5624 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Thu, 8 Aug 2019 16:14:19 -0700 Subject: [PATCH] Prepare ansible-test for inclusion in setup.py (#60294) * Cache ansible version lookup. * Fix method of determining ANSIBLE_ROOT. * Clean up based on PyCharm inspections. * Generate minimal PKG-INFO without setup.py. * Use ANSIBLE_LIB_ROOT where possible. * Use import instead of subprocess to get version. * Fix install layout type. * Correct required paths message for installs. * Update list of files copied during delegation. * Fix ansible-test entry point. * Fix pylint issue. * Fix version lookup on Python 2.x. * Fix pylint issue. * Remove unwanted print statement. --- .../_data/cli/ansible_test_cli_stub.py | 5 ++- .../_data/sanity/import/importer.py | 3 ++ .../ansible_test/_internal/ansible_util.py | 3 +- test/lib/ansible_test/_internal/data.py | 17 ++++++--- test/lib/ansible_test/_internal/env.py | 37 +++++++++---------- test/lib/ansible_test/_internal/executor.py | 33 +++++++++++++++-- test/lib/ansible_test/_internal/payload.py | 10 ++--- .../_internal/provider/layout/ansible.py | 5 --- .../_internal/provider/layout/collection.py | 1 - .../ansible_test/_internal/sanity/__init__.py | 2 +- .../ansible_test/_internal/sanity/import.py | 8 ++-- test/lib/ansible_test/_internal/util.py | 15 +++++++- .../lib/ansible_test/_internal/util_common.py | 1 - 13 files changed, 87 insertions(+), 53 deletions(-) diff --git a/test/lib/ansible_test/_data/cli/ansible_test_cli_stub.py b/test/lib/ansible_test/_data/cli/ansible_test_cli_stub.py index 4c075075f5f..d12b6334ef5 100755 --- a/test/lib/ansible_test/_data/cli/ansible_test_cli_stub.py +++ b/test/lib/ansible_test/_data/cli/ansible_test_cli_stub.py @@ -11,13 +11,14 @@ import sys def main(): """Main program entry point.""" - ansible_root = os.path.abspath(os.path.join(os.path.dirname(os.path.dirname(__file__)))) + ansible_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) source_root = os.path.join(ansible_root, 'test', 'lib') - if os.path.exists(os.path.join(ansible_root, 'setup.py')) and os.path.exists(os.path.join(source_root, 'ansible_test', '_internal', 'cli.py')): + if os.path.exists(os.path.join(source_root, 'ansible_test', '_internal', 'cli.py')): # running from source, use that version of ansible-test instead of any version that may already be installed sys.path.insert(0, source_root) + # noinspection PyProtectedMember from ansible_test._internal.cli import main as cli_main cli_main() diff --git a/test/lib/ansible_test/_data/sanity/import/importer.py b/test/lib/ansible_test/_data/sanity/import/importer.py index e0606950b6b..561ebf9b593 100755 --- a/test/lib/ansible_test/_data/sanity/import/importer.py +++ b/test/lib/ansible_test/_data/sanity/import/importer.py @@ -24,6 +24,7 @@ def main(): import imp try: + # noinspection PyCompatibility from StringIO import StringIO except ImportError: from io import StringIO @@ -34,6 +35,7 @@ def main(): try: from ansible.utils.collection_loader import AnsibleCollectionLoader except ImportError: + # noinspection PyPep8Naming AnsibleCollectionLoader = None class ImporterAnsibleModuleException(Exception): @@ -58,6 +60,7 @@ def main(): if AnsibleCollectionLoader: # allow importing code from collections + # noinspection PyCallingNonCallable sys.meta_path.insert(0, AnsibleCollectionLoader()) for path in sys.argv[1:] or sys.stdin.read().splitlines(): diff --git a/test/lib/ansible_test/_internal/ansible_util.py b/test/lib/ansible_test/_internal/ansible_util.py index e36829e1dc6..75be193152c 100644 --- a/test/lib/ansible_test/_internal/ansible_util.py +++ b/test/lib/ansible_test/_internal/ansible_util.py @@ -15,6 +15,7 @@ from .util import ( find_python, ApplicationError, ANSIBLE_ROOT, + ANSIBLE_LIB_ROOT, ANSIBLE_TEST_DATA_ROOT, ) @@ -67,7 +68,7 @@ def ansible_environment(args, color=True, ansible_config=None): ANSIBLE_RETRY_FILES_ENABLED='false', ANSIBLE_CONFIG=os.path.abspath(ansible_config), ANSIBLE_LIBRARY='/dev/null', - PYTHONPATH=os.path.join(ANSIBLE_ROOT, 'lib'), + PYTHONPATH=os.path.dirname(ANSIBLE_LIB_ROOT), PAGER='/bin/cat', PATH=path, ) diff --git a/test/lib/ansible_test/_internal/data.py b/test/lib/ansible_test/_internal/data.py index a99b573abc1..0b777e4f854 100644 --- a/test/lib/ansible_test/_internal/data.py +++ b/test/lib/ansible_test/_internal/data.py @@ -11,6 +11,7 @@ from .util import ( import_plugins, ANSIBLE_ROOT, is_subdir, + ANSIBLE_IS_INSTALLED, ) from .provider import ( @@ -57,7 +58,7 @@ class DataContext: content = self.create_content_layout(self.__layout_providers, self.__source_providers, content_path, False) if content.is_ansible: - install = content + install = InstallLayout(ANSIBLE_ROOT, content.all_files()) else: install = None elif is_subdir(current_path, ANSIBLE_ROOT): @@ -129,12 +130,18 @@ def data_init(): # type: () -> DataContext try: context = DataContext() except ProviderNotFoundForPath: - raise ApplicationError('''The current working directory must be at or below one of: + options = [ + ' - an Ansible collection: {...}/ansible_collections/{namespace}/{collection}/', + ] - - Ansible source: %s/ - - Ansible collection: {...}/ansible_collections/{namespace}/{collection}/ + if not ANSIBLE_IS_INSTALLED: + options.insert(0, ' - the Ansible source: %s/' % ANSIBLE_ROOT) -Current working directory: %s''' % (ANSIBLE_ROOT, os.getcwd())) + raise ApplicationError('''The current working directory must be at or below: + +%s + +Current working directory: %s''' % ('\n'.join(options), os.getcwd())) return context diff --git a/test/lib/ansible_test/_internal/env.py b/test/lib/ansible_test/_internal/env.py index 2a2e0aa880d..ce7a65876aa 100644 --- a/test/lib/ansible_test/_internal/env.py +++ b/test/lib/ansible_test/_internal/env.py @@ -20,13 +20,10 @@ from .config import ( from .util import ( display, find_executable, - raw_command, SubprocessError, ApplicationError, -) - -from .ansible_util import ( - ansible_environment, + load_module, + ANSIBLE_LIB_ROOT, ) from .git import ( @@ -81,7 +78,7 @@ def show_dump_env(args): data = dict( ansible=dict( - version=get_ansible_version(args), + version=get_ansible_version(), ), docker=get_docker_details(args), environ=os.environ.copy(), @@ -238,21 +235,21 @@ def show_dict(data, verbose, root_verbosity=0, path=None): display.info(indent + '%s: %s' % (key, value), verbosity=verbosity) -def get_ansible_version(args): - """ - :type args: CommonConfig - :rtype: str | None - """ - code = 'from __future__ import (print_function); from ansible.release import __version__; print(__version__)' - cmd = [sys.executable, '-c', code] - env = ansible_environment(args) - +def get_ansible_version(): # type: () -> str + """Return the Ansible version.""" try: - ansible_version, _dummy = raw_command(cmd, env=env, capture=True) - ansible_version = ansible_version.strip() - except SubprocessError as ex: - display.warning('Unable to get Ansible version:\n%s' % ex) - ansible_version = None + return get_ansible_version.version + except AttributeError: + pass + + # ansible may not be in our sys.path + # avoids a symlink to release.py since ansible placement relative to ansible-test may change during delegation + load_module(os.path.join(ANSIBLE_LIB_ROOT, 'release.py'), 'ansible_release') + + # noinspection PyUnresolvedReferences + from ansible_release import __version__ as ansible_version # pylint: disable=import-error + + get_ansible_version.version = ansible_version return ansible_version diff --git a/test/lib/ansible_test/_internal/executor.py b/test/lib/ansible_test/_internal/executor.py index 0323d756b16..ce278543beb 100644 --- a/test/lib/ansible_test/_internal/executor.py +++ b/test/lib/ansible_test/_internal/executor.py @@ -60,7 +60,7 @@ from .util import ( get_remote_completion, COVERAGE_OUTPUT_NAME, cmd_quote, - ANSIBLE_ROOT, + ANSIBLE_LIB_ROOT, ANSIBLE_TEST_DATA_ROOT, get_available_python_versions, is_subdir, @@ -87,6 +87,10 @@ from .ansible_util import ( check_pyyaml, ) +from .env import ( + get_ansible_version, +) + from .target import ( IntegrationTarget, walk_internal_targets, @@ -299,13 +303,34 @@ def generate_egg_info(args): """ :type args: EnvironmentConfig """ - if not os.path.exists(os.path.join(ANSIBLE_ROOT, 'setup.py')): + if args.explain: return - if os.path.isdir(os.path.join(ANSIBLE_ROOT, 'lib/ansible.egg-info')): + egg_info_path = ANSIBLE_LIB_ROOT + '.egg-info' + + if os.path.exists(egg_info_path): return - run_command(args, [args.python_executable, 'setup.py', 'egg_info'], cwd=ANSIBLE_ROOT, capture=args.verbosity < 3) + # minimal PKG-INFO stub following the format defined in PEP 241 + # required for older setuptools versions to avoid a traceback when importing pkg_resources from packages like cryptography + # newer setuptools versions are happy with an empty directory + # including a stub here means we don't need to locate the existing file or have setup.py generate it when running from source + pkg_info = ''' +Metadata-Version: 1.0 +Name: ansible +Version: %s +Platform: UNKNOWN +Summary: Radically simple IT automation +Author-email: info@ansible.com +License: GPLv3+ +''' % get_ansible_version() + + os.mkdir(egg_info_path) + + pkg_info_path = os.path.join(egg_info_path, 'PKG-INFO') + + with open(pkg_info_path, 'w') as pkg_info_fd: + pkg_info_fd.write(pkg_info.lstrip()) def generate_pip_install(pip, command, packages=None): diff --git a/test/lib/ansible_test/_internal/payload.py b/test/lib/ansible_test/_internal/payload.py index 12127e75d05..f3c59ec30a7 100644 --- a/test/lib/ansible_test/_internal/payload.py +++ b/test/lib/ansible_test/_internal/payload.py @@ -41,17 +41,13 @@ def create_payload(args, dst_path): # type: (CommonConfig, str) -> None f[1].startswith('bin/') or f[1].startswith('lib/') or f[1].startswith('test/lib/') or - f[1].startswith('packaging/requirements/') or f[1] in ( - 'setup.py', - 'README.rst', - 'requirements.txt', - # units only - 'test/units/ansible.cfg', - # integration only 'test/integration/integration.cfg', 'test/integration/integration_config.yml', 'test/integration/inventory', + 'test/integration/network-integration.cfg', + 'test/integration/target-prefixes.network', + 'test/integration/windows-integration.cfg', )] if not isinstance(args, (ShellConfig, IntegrationConfig)): diff --git a/test/lib/ansible_test/_internal/provider/layout/ansible.py b/test/lib/ansible_test/_internal/provider/layout/ansible.py index 4ed6e835a40..49bbe601f69 100644 --- a/test/lib/ansible_test/_internal/provider/layout/ansible.py +++ b/test/lib/ansible_test/_internal/provider/layout/ansible.py @@ -3,14 +3,9 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type import os -import re from ... import types as t -from ...util import ( - ANSIBLE_TEST_ROOT, -) - from . import ( ContentLayout, LayoutProvider, diff --git a/test/lib/ansible_test/_internal/provider/layout/collection.py b/test/lib/ansible_test/_internal/provider/layout/collection.py index a09f0bfd031..a0633f4d514 100644 --- a/test/lib/ansible_test/_internal/provider/layout/collection.py +++ b/test/lib/ansible_test/_internal/provider/layout/collection.py @@ -3,7 +3,6 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type import os -import re from ... import types as t diff --git a/test/lib/ansible_test/_internal/sanity/__init__.py b/test/lib/ansible_test/_internal/sanity/__init__.py index 4dcde04f07d..d4e4814b79a 100644 --- a/test/lib/ansible_test/_internal/sanity/__init__.py +++ b/test/lib/ansible_test/_internal/sanity/__init__.py @@ -238,7 +238,7 @@ class SanityIgnoreParser: def __init__(self, args): # type: (SanityConfig) -> None if data_context().content.collection: - ansible_version = '%s.%s' % tuple(get_ansible_version(args).split('.')[:2]) + ansible_version = '%s.%s' % tuple(get_ansible_version().split('.')[:2]) ansible_label = 'Ansible %s' % ansible_version file_name = 'ignore-%s.txt' % ansible_version diff --git a/test/lib/ansible_test/_internal/sanity/import.py b/test/lib/ansible_test/_internal/sanity/import.py index 921565e5495..2e9215bf54e 100644 --- a/test/lib/ansible_test/_internal/sanity/import.py +++ b/test/lib/ansible_test/_internal/sanity/import.py @@ -26,6 +26,7 @@ from ..util import ( parse_to_list_of_dict, make_dirs, is_subdir, + ANSIBLE_LIB_ROOT, ) from ..util_common import ( @@ -51,7 +52,6 @@ from ..coverage_util import ( from ..data import ( data_context, - ANSIBLE_ROOT, ) @@ -109,7 +109,7 @@ class ImportTest(SanityMultipleVersion): with open(ansible_init, 'w'): pass - os.symlink(os.path.join(ANSIBLE_ROOT, 'lib/ansible/module_utils'), ansible_link) + os.symlink(os.path.join(ANSIBLE_LIB_ROOT, 'module_utils'), ansible_link) if data_context().content.collection: # inject just enough Ansible code for the collections loader to work on all supported Python versions @@ -120,8 +120,8 @@ class ImportTest(SanityMultipleVersion): with open(os.path.join(ansible_path, 'utils/__init__.py'), 'w'): pass - os.symlink(os.path.join(ANSIBLE_ROOT, 'lib/ansible/utils/collection_loader.py'), os.path.join(ansible_path, 'utils/collection_loader.py')) - os.symlink(os.path.join(ANSIBLE_ROOT, 'lib/ansible/utils/singleton.py'), os.path.join(ansible_path, 'utils/singleton.py')) + os.symlink(os.path.join(ANSIBLE_LIB_ROOT, 'utils', 'collection_loader.py'), os.path.join(ansible_path, 'utils', 'collection_loader.py')) + os.symlink(os.path.join(ANSIBLE_LIB_ROOT, 'utils', 'singleton.py'), os.path.join(ansible_path, 'utils', 'singleton.py')) make_dirs(os.path.join(ansible_path, 'modules')) with open(os.path.join(ansible_path, 'modules/__init__.py'), 'w'): diff --git a/test/lib/ansible_test/_internal/util.py b/test/lib/ansible_test/_internal/util.py index 3b830b5629f..ccad8e20a0f 100644 --- a/test/lib/ansible_test/_internal/util.py +++ b/test/lib/ansible_test/_internal/util.py @@ -62,8 +62,19 @@ except AttributeError: COVERAGE_CONFIG_NAME = 'coveragerc' COVERAGE_OUTPUT_NAME = 'coverage' -ANSIBLE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) -ANSIBLE_TEST_ROOT = os.path.join(ANSIBLE_ROOT, 'test', 'lib', 'ansible_test') +ANSIBLE_TEST_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# assume running from install +ANSIBLE_ROOT = os.path.dirname(ANSIBLE_TEST_ROOT) +ANSIBLE_LIB_ROOT = os.path.join(ANSIBLE_ROOT, 'ansible') +ANSIBLE_IS_INSTALLED = True + +if not os.path.exists(ANSIBLE_LIB_ROOT): + # running from source + ANSIBLE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(ANSIBLE_TEST_ROOT))) + ANSIBLE_LIB_ROOT = os.path.join(ANSIBLE_ROOT, 'lib', 'ansible') + ANSIBLE_IS_INSTALLED = False + ANSIBLE_TEST_DATA_ROOT = os.path.join(ANSIBLE_TEST_ROOT, '_data') ANSIBLE_TEST_CONFIG_ROOT = os.path.join(ANSIBLE_TEST_ROOT, 'config') diff --git a/test/lib/ansible_test/_internal/util_common.py b/test/lib/ansible_test/_internal/util_common.py index ed323552d6d..5649234d6b3 100644 --- a/test/lib/ansible_test/_internal/util_common.py +++ b/test/lib/ansible_test/_internal/util_common.py @@ -15,7 +15,6 @@ from .util import ( COVERAGE_OUTPUT_NAME, display, find_python, - ANSIBLE_ROOT, is_shippable, MODE_DIRECTORY, MODE_FILE_EXECUTE,