Prepare ansible-test for inclusion in setup.py (#60294)

* Cache ansible version lookup.
* Fix method of determining ANSIBLE_ROOT.
* Clean up based on PyCharm inspections.
* Generate minimal PKG-INFO without setup.py.
* Use ANSIBLE_LIB_ROOT where possible.
* Use import instead of subprocess to get version.
* Fix install layout type.
* Correct required paths message for installs.
* Update list of files copied during delegation.
* Fix ansible-test entry point.
* Fix pylint issue.
* Fix version lookup on Python 2.x.
* Fix pylint issue.
* Remove unwanted print statement.
pull/60297/head
Matt Clay 5 years ago committed by GitHub
parent b38cb37728
commit 57dc7ec265
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -11,13 +11,14 @@ import sys
def main(): def main():
"""Main program entry point.""" """Main program entry point."""
ansible_root = os.path.abspath(os.path.join(os.path.dirname(os.path.dirname(__file__)))) ansible_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
source_root = os.path.join(ansible_root, 'test', 'lib') source_root = os.path.join(ansible_root, 'test', 'lib')
if os.path.exists(os.path.join(ansible_root, 'setup.py')) and os.path.exists(os.path.join(source_root, 'ansible_test', '_internal', 'cli.py')): if os.path.exists(os.path.join(source_root, 'ansible_test', '_internal', 'cli.py')):
# running from source, use that version of ansible-test instead of any version that may already be installed # running from source, use that version of ansible-test instead of any version that may already be installed
sys.path.insert(0, source_root) sys.path.insert(0, source_root)
# noinspection PyProtectedMember
from ansible_test._internal.cli import main as cli_main from ansible_test._internal.cli import main as cli_main
cli_main() cli_main()

@ -24,6 +24,7 @@ def main():
import imp import imp
try: try:
# noinspection PyCompatibility
from StringIO import StringIO from StringIO import StringIO
except ImportError: except ImportError:
from io import StringIO from io import StringIO
@ -34,6 +35,7 @@ def main():
try: try:
from ansible.utils.collection_loader import AnsibleCollectionLoader from ansible.utils.collection_loader import AnsibleCollectionLoader
except ImportError: except ImportError:
# noinspection PyPep8Naming
AnsibleCollectionLoader = None AnsibleCollectionLoader = None
class ImporterAnsibleModuleException(Exception): class ImporterAnsibleModuleException(Exception):
@ -58,6 +60,7 @@ def main():
if AnsibleCollectionLoader: if AnsibleCollectionLoader:
# allow importing code from collections # allow importing code from collections
# noinspection PyCallingNonCallable
sys.meta_path.insert(0, AnsibleCollectionLoader()) sys.meta_path.insert(0, AnsibleCollectionLoader())
for path in sys.argv[1:] or sys.stdin.read().splitlines(): for path in sys.argv[1:] or sys.stdin.read().splitlines():

@ -15,6 +15,7 @@ from .util import (
find_python, find_python,
ApplicationError, ApplicationError,
ANSIBLE_ROOT, ANSIBLE_ROOT,
ANSIBLE_LIB_ROOT,
ANSIBLE_TEST_DATA_ROOT, ANSIBLE_TEST_DATA_ROOT,
) )
@ -67,7 +68,7 @@ def ansible_environment(args, color=True, ansible_config=None):
ANSIBLE_RETRY_FILES_ENABLED='false', ANSIBLE_RETRY_FILES_ENABLED='false',
ANSIBLE_CONFIG=os.path.abspath(ansible_config), ANSIBLE_CONFIG=os.path.abspath(ansible_config),
ANSIBLE_LIBRARY='/dev/null', ANSIBLE_LIBRARY='/dev/null',
PYTHONPATH=os.path.join(ANSIBLE_ROOT, 'lib'), PYTHONPATH=os.path.dirname(ANSIBLE_LIB_ROOT),
PAGER='/bin/cat', PAGER='/bin/cat',
PATH=path, PATH=path,
) )

@ -11,6 +11,7 @@ from .util import (
import_plugins, import_plugins,
ANSIBLE_ROOT, ANSIBLE_ROOT,
is_subdir, is_subdir,
ANSIBLE_IS_INSTALLED,
) )
from .provider import ( from .provider import (
@ -57,7 +58,7 @@ class DataContext:
content = self.create_content_layout(self.__layout_providers, self.__source_providers, content_path, False) content = self.create_content_layout(self.__layout_providers, self.__source_providers, content_path, False)
if content.is_ansible: if content.is_ansible:
install = content install = InstallLayout(ANSIBLE_ROOT, content.all_files())
else: else:
install = None install = None
elif is_subdir(current_path, ANSIBLE_ROOT): elif is_subdir(current_path, ANSIBLE_ROOT):
@ -129,12 +130,18 @@ def data_init(): # type: () -> DataContext
try: try:
context = DataContext() context = DataContext()
except ProviderNotFoundForPath: except ProviderNotFoundForPath:
raise ApplicationError('''The current working directory must be at or below one of: options = [
' - an Ansible collection: {...}/ansible_collections/{namespace}/{collection}/',
]
- Ansible source: %s/ if not ANSIBLE_IS_INSTALLED:
- Ansible collection: {...}/ansible_collections/{namespace}/{collection}/ options.insert(0, ' - the Ansible source: %s/' % ANSIBLE_ROOT)
Current working directory: %s''' % (ANSIBLE_ROOT, os.getcwd())) raise ApplicationError('''The current working directory must be at or below:
%s
Current working directory: %s''' % ('\n'.join(options), os.getcwd()))
return context return context

@ -20,13 +20,10 @@ from .config import (
from .util import ( from .util import (
display, display,
find_executable, find_executable,
raw_command,
SubprocessError, SubprocessError,
ApplicationError, ApplicationError,
) load_module,
ANSIBLE_LIB_ROOT,
from .ansible_util import (
ansible_environment,
) )
from .git import ( from .git import (
@ -81,7 +78,7 @@ def show_dump_env(args):
data = dict( data = dict(
ansible=dict( ansible=dict(
version=get_ansible_version(args), version=get_ansible_version(),
), ),
docker=get_docker_details(args), docker=get_docker_details(args),
environ=os.environ.copy(), environ=os.environ.copy(),
@ -238,21 +235,21 @@ def show_dict(data, verbose, root_verbosity=0, path=None):
display.info(indent + '%s: %s' % (key, value), verbosity=verbosity) display.info(indent + '%s: %s' % (key, value), verbosity=verbosity)
def get_ansible_version(args): def get_ansible_version(): # type: () -> str
""" """Return the Ansible version."""
:type args: CommonConfig
:rtype: str | None
"""
code = 'from __future__ import (print_function); from ansible.release import __version__; print(__version__)'
cmd = [sys.executable, '-c', code]
env = ansible_environment(args)
try: try:
ansible_version, _dummy = raw_command(cmd, env=env, capture=True) return get_ansible_version.version
ansible_version = ansible_version.strip() except AttributeError:
except SubprocessError as ex: pass
display.warning('Unable to get Ansible version:\n%s' % ex)
ansible_version = None # ansible may not be in our sys.path
# avoids a symlink to release.py since ansible placement relative to ansible-test may change during delegation
load_module(os.path.join(ANSIBLE_LIB_ROOT, 'release.py'), 'ansible_release')
# noinspection PyUnresolvedReferences
from ansible_release import __version__ as ansible_version # pylint: disable=import-error
get_ansible_version.version = ansible_version
return ansible_version return ansible_version

@ -60,7 +60,7 @@ from .util import (
get_remote_completion, get_remote_completion,
COVERAGE_OUTPUT_NAME, COVERAGE_OUTPUT_NAME,
cmd_quote, cmd_quote,
ANSIBLE_ROOT, ANSIBLE_LIB_ROOT,
ANSIBLE_TEST_DATA_ROOT, ANSIBLE_TEST_DATA_ROOT,
get_available_python_versions, get_available_python_versions,
is_subdir, is_subdir,
@ -87,6 +87,10 @@ from .ansible_util import (
check_pyyaml, check_pyyaml,
) )
from .env import (
get_ansible_version,
)
from .target import ( from .target import (
IntegrationTarget, IntegrationTarget,
walk_internal_targets, walk_internal_targets,
@ -299,13 +303,34 @@ def generate_egg_info(args):
""" """
:type args: EnvironmentConfig :type args: EnvironmentConfig
""" """
if not os.path.exists(os.path.join(ANSIBLE_ROOT, 'setup.py')): if args.explain:
return return
if os.path.isdir(os.path.join(ANSIBLE_ROOT, 'lib/ansible.egg-info')): egg_info_path = ANSIBLE_LIB_ROOT + '.egg-info'
if os.path.exists(egg_info_path):
return return
run_command(args, [args.python_executable, 'setup.py', 'egg_info'], cwd=ANSIBLE_ROOT, capture=args.verbosity < 3) # minimal PKG-INFO stub following the format defined in PEP 241
# required for older setuptools versions to avoid a traceback when importing pkg_resources from packages like cryptography
# newer setuptools versions are happy with an empty directory
# including a stub here means we don't need to locate the existing file or have setup.py generate it when running from source
pkg_info = '''
Metadata-Version: 1.0
Name: ansible
Version: %s
Platform: UNKNOWN
Summary: Radically simple IT automation
Author-email: info@ansible.com
License: GPLv3+
''' % get_ansible_version()
os.mkdir(egg_info_path)
pkg_info_path = os.path.join(egg_info_path, 'PKG-INFO')
with open(pkg_info_path, 'w') as pkg_info_fd:
pkg_info_fd.write(pkg_info.lstrip())
def generate_pip_install(pip, command, packages=None): def generate_pip_install(pip, command, packages=None):

@ -41,17 +41,13 @@ def create_payload(args, dst_path): # type: (CommonConfig, str) -> None
f[1].startswith('bin/') or f[1].startswith('bin/') or
f[1].startswith('lib/') or f[1].startswith('lib/') or
f[1].startswith('test/lib/') or f[1].startswith('test/lib/') or
f[1].startswith('packaging/requirements/') or
f[1] in ( f[1] in (
'setup.py',
'README.rst',
'requirements.txt',
# units only
'test/units/ansible.cfg',
# integration only
'test/integration/integration.cfg', 'test/integration/integration.cfg',
'test/integration/integration_config.yml', 'test/integration/integration_config.yml',
'test/integration/inventory', 'test/integration/inventory',
'test/integration/network-integration.cfg',
'test/integration/target-prefixes.network',
'test/integration/windows-integration.cfg',
)] )]
if not isinstance(args, (ShellConfig, IntegrationConfig)): if not isinstance(args, (ShellConfig, IntegrationConfig)):

@ -3,14 +3,9 @@ from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
import os import os
import re
from ... import types as t from ... import types as t
from ...util import (
ANSIBLE_TEST_ROOT,
)
from . import ( from . import (
ContentLayout, ContentLayout,
LayoutProvider, LayoutProvider,

@ -3,7 +3,6 @@ from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
import os import os
import re
from ... import types as t from ... import types as t

@ -238,7 +238,7 @@ class SanityIgnoreParser:
def __init__(self, args): # type: (SanityConfig) -> None def __init__(self, args): # type: (SanityConfig) -> None
if data_context().content.collection: if data_context().content.collection:
ansible_version = '%s.%s' % tuple(get_ansible_version(args).split('.')[:2]) ansible_version = '%s.%s' % tuple(get_ansible_version().split('.')[:2])
ansible_label = 'Ansible %s' % ansible_version ansible_label = 'Ansible %s' % ansible_version
file_name = 'ignore-%s.txt' % ansible_version file_name = 'ignore-%s.txt' % ansible_version

@ -26,6 +26,7 @@ from ..util import (
parse_to_list_of_dict, parse_to_list_of_dict,
make_dirs, make_dirs,
is_subdir, is_subdir,
ANSIBLE_LIB_ROOT,
) )
from ..util_common import ( from ..util_common import (
@ -51,7 +52,6 @@ from ..coverage_util import (
from ..data import ( from ..data import (
data_context, data_context,
ANSIBLE_ROOT,
) )
@ -109,7 +109,7 @@ class ImportTest(SanityMultipleVersion):
with open(ansible_init, 'w'): with open(ansible_init, 'w'):
pass pass
os.symlink(os.path.join(ANSIBLE_ROOT, 'lib/ansible/module_utils'), ansible_link) os.symlink(os.path.join(ANSIBLE_LIB_ROOT, 'module_utils'), ansible_link)
if data_context().content.collection: if data_context().content.collection:
# inject just enough Ansible code for the collections loader to work on all supported Python versions # inject just enough Ansible code for the collections loader to work on all supported Python versions
@ -120,8 +120,8 @@ class ImportTest(SanityMultipleVersion):
with open(os.path.join(ansible_path, 'utils/__init__.py'), 'w'): with open(os.path.join(ansible_path, 'utils/__init__.py'), 'w'):
pass pass
os.symlink(os.path.join(ANSIBLE_ROOT, 'lib/ansible/utils/collection_loader.py'), os.path.join(ansible_path, 'utils/collection_loader.py')) os.symlink(os.path.join(ANSIBLE_LIB_ROOT, 'utils', 'collection_loader.py'), os.path.join(ansible_path, 'utils', 'collection_loader.py'))
os.symlink(os.path.join(ANSIBLE_ROOT, 'lib/ansible/utils/singleton.py'), os.path.join(ansible_path, 'utils/singleton.py')) os.symlink(os.path.join(ANSIBLE_LIB_ROOT, 'utils', 'singleton.py'), os.path.join(ansible_path, 'utils', 'singleton.py'))
make_dirs(os.path.join(ansible_path, 'modules')) make_dirs(os.path.join(ansible_path, 'modules'))
with open(os.path.join(ansible_path, 'modules/__init__.py'), 'w'): with open(os.path.join(ansible_path, 'modules/__init__.py'), 'w'):

@ -62,8 +62,19 @@ except AttributeError:
COVERAGE_CONFIG_NAME = 'coveragerc' COVERAGE_CONFIG_NAME = 'coveragerc'
COVERAGE_OUTPUT_NAME = 'coverage' COVERAGE_OUTPUT_NAME = 'coverage'
ANSIBLE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) ANSIBLE_TEST_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
ANSIBLE_TEST_ROOT = os.path.join(ANSIBLE_ROOT, 'test', 'lib', 'ansible_test')
# assume running from install
ANSIBLE_ROOT = os.path.dirname(ANSIBLE_TEST_ROOT)
ANSIBLE_LIB_ROOT = os.path.join(ANSIBLE_ROOT, 'ansible')
ANSIBLE_IS_INSTALLED = True
if not os.path.exists(ANSIBLE_LIB_ROOT):
# running from source
ANSIBLE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(ANSIBLE_TEST_ROOT)))
ANSIBLE_LIB_ROOT = os.path.join(ANSIBLE_ROOT, 'lib', 'ansible')
ANSIBLE_IS_INSTALLED = False
ANSIBLE_TEST_DATA_ROOT = os.path.join(ANSIBLE_TEST_ROOT, '_data') ANSIBLE_TEST_DATA_ROOT = os.path.join(ANSIBLE_TEST_ROOT, '_data')
ANSIBLE_TEST_CONFIG_ROOT = os.path.join(ANSIBLE_TEST_ROOT, 'config') ANSIBLE_TEST_CONFIG_ROOT = os.path.join(ANSIBLE_TEST_ROOT, 'config')

@ -15,7 +15,6 @@ from .util import (
COVERAGE_OUTPUT_NAME, COVERAGE_OUTPUT_NAME,
display, display,
find_python, find_python,
ANSIBLE_ROOT,
is_shippable, is_shippable,
MODE_DIRECTORY, MODE_DIRECTORY,
MODE_FILE_EXECUTE, MODE_FILE_EXECUTE,

Loading…
Cancel
Save