mirror of https://github.com/ansible/ansible.git
Initial ansible-test support for collections. (#59197)
* Initial ansible-test support for collections. * Include cloud config in delegation payload. * Add missing types import and fix `t` shadowing. * Fix plugin traceback when config_path not set. * Fix encoding issues. * Remove unused imports. * More encoding fixes. * Handle delegation outside exception handler. * Inject ssh keys only if not already in place. * More defensive approach to getting remote pwd. * Add missing string format var. * Correct PowerShell require regex. * Rename `is_install` and `INSTALL_ROOT`.pull/38583/head
Matt Clay
7 years ago
committed by
GitHub
parent
67c69f3540
commit
79eca9c8fb
@ -0,0 +1,89 @@
|
||||
"""Utility code for facilitating collection of code coverage when running tests."""
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
from lib.config import (
|
||||
IntegrationConfig,
|
||||
SanityConfig,
|
||||
TestConfig,
|
||||
)
|
||||
|
||||
from lib.util import (
|
||||
COVERAGE_CONFIG_PATH,
|
||||
remove_tree,
|
||||
)
|
||||
|
||||
from lib.data import (
|
||||
data_context,
|
||||
)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def coverage_context(args): # type: (TestConfig) -> None
|
||||
"""Content to set up and clean up code coverage configuration for tests."""
|
||||
coverage_setup(args)
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
coverage_cleanup(args)
|
||||
|
||||
|
||||
def coverage_setup(args): # type: (TestConfig) -> None
|
||||
"""Set up code coverage configuration before running tests."""
|
||||
if args.coverage and data_context().content.collection:
|
||||
coverage_config = generate_collection_coverage_config(args)
|
||||
|
||||
if args.explain:
|
||||
args.coverage_config_base_path = '/tmp/coverage-temp-dir'
|
||||
else:
|
||||
args.coverage_config_base_path = tempfile.mkdtemp()
|
||||
|
||||
with open(os.path.join(args.coverage_config_base_path, COVERAGE_CONFIG_PATH), 'w') as coverage_config_path_fd:
|
||||
coverage_config_path_fd.write(coverage_config)
|
||||
|
||||
|
||||
def coverage_cleanup(args): # type: (TestConfig) -> None
|
||||
"""Clean up code coverage configuration after tests have finished."""
|
||||
if args.coverage_config_base_path and not args.explain:
|
||||
remove_tree(args.coverage_config_base_path)
|
||||
args.coverage_config_base_path = None
|
||||
|
||||
|
||||
def generate_collection_coverage_config(args): # type: (TestConfig) -> str
|
||||
"""Generate code coverage configuration for tests."""
|
||||
coverage_config = '''
|
||||
[run]
|
||||
branch = True
|
||||
concurrency = multiprocessing
|
||||
parallel = True
|
||||
disable_warnings =
|
||||
no-data-collected
|
||||
'''
|
||||
|
||||
if isinstance(args, IntegrationConfig):
|
||||
coverage_config += '''
|
||||
include =
|
||||
%s/*
|
||||
*/%s/*
|
||||
''' % (data_context().content.root, data_context().content.collection.directory)
|
||||
elif isinstance(args, SanityConfig):
|
||||
# temporary work-around for import sanity test
|
||||
coverage_config += '''
|
||||
include =
|
||||
%s/*
|
||||
|
||||
omit =
|
||||
*/test/runner/.tox/*
|
||||
''' % data_context().content.root
|
||||
else:
|
||||
coverage_config += '''
|
||||
include =
|
||||
%s/*
|
||||
''' % data_context().content.root
|
||||
|
||||
return coverage_config
|
||||
@ -0,0 +1,148 @@
|
||||
"""Context information for the current invocation of ansible-test."""
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
|
||||
import lib.types as t
|
||||
|
||||
from lib.util import (
|
||||
ApplicationError,
|
||||
import_plugins,
|
||||
ANSIBLE_ROOT,
|
||||
is_subdir,
|
||||
)
|
||||
|
||||
from lib.provider import (
|
||||
find_path_provider,
|
||||
get_path_provider_classes,
|
||||
ProviderNotFoundForPath,
|
||||
)
|
||||
|
||||
from lib.provider.source import (
|
||||
SourceProvider,
|
||||
)
|
||||
|
||||
from lib.provider.source.unversioned import (
|
||||
UnversionedSource,
|
||||
)
|
||||
|
||||
from lib.provider.layout import (
|
||||
ContentLayout,
|
||||
InstallLayout,
|
||||
LayoutProvider,
|
||||
)
|
||||
|
||||
|
||||
class UnexpectedSourceRoot(ApplicationError):
|
||||
"""Exception generated when a source root is found below a layout root."""
|
||||
def __init__(self, source_root, layout_root): # type: (str, str) -> None
|
||||
super(UnexpectedSourceRoot, self).__init__('Source root "%s" cannot be below layout root "%s".' % (source_root, layout_root))
|
||||
|
||||
self.source_root = source_root
|
||||
self.layout_root = layout_root
|
||||
|
||||
|
||||
class DataContext:
|
||||
"""Data context providing details about the current execution environment for ansible-test."""
|
||||
def __init__(self):
|
||||
content_path = os.environ.get('ANSIBLE_TEST_CONTENT_ROOT')
|
||||
current_path = os.getcwd()
|
||||
|
||||
self.__layout_providers = get_path_provider_classes(LayoutProvider)
|
||||
self.__source_providers = get_path_provider_classes(SourceProvider)
|
||||
self.payload_callbacks = [] # type: t.List[t.Callable[t.List[t.Tuple[str, str]], None]]
|
||||
|
||||
if content_path:
|
||||
content = self.create_content_layout(self.__layout_providers, self.__source_providers, content_path, False)
|
||||
|
||||
if content.is_ansible:
|
||||
install = content
|
||||
else:
|
||||
install = None
|
||||
elif is_subdir(current_path, ANSIBLE_ROOT):
|
||||
content = self.create_content_layout(self.__layout_providers, self.__source_providers, ANSIBLE_ROOT, False)
|
||||
install = InstallLayout(ANSIBLE_ROOT, content.all_files())
|
||||
else:
|
||||
content = self.create_content_layout(self.__layout_providers, self.__source_providers, current_path, True)
|
||||
install = None
|
||||
|
||||
self.__install = install # type: t.Optional[InstallLayout]
|
||||
self.content = content # type: ContentLayout
|
||||
|
||||
@staticmethod
|
||||
def create_content_layout(layout_providers, # type: t.List[t.Type[LayoutProvider]]
|
||||
source_providers, # type: t.List[t.Type[SourceProvider]]
|
||||
root, # type: str
|
||||
walk, # type: bool
|
||||
): # type: (...) -> ContentLayout
|
||||
"""Create a content layout using the given providers and root path."""
|
||||
layout_provider = find_path_provider(LayoutProvider, layout_providers, root, walk)
|
||||
|
||||
try:
|
||||
source_provider = find_path_provider(SourceProvider, source_providers, root, walk)
|
||||
except ProviderNotFoundForPath:
|
||||
source_provider = UnversionedSource(layout_provider.root)
|
||||
|
||||
if source_provider.root != layout_provider.root and is_subdir(source_provider.root, layout_provider.root):
|
||||
raise UnexpectedSourceRoot(source_provider.root, layout_provider.root)
|
||||
|
||||
layout = layout_provider.create(layout_provider.root, source_provider.get_paths(layout_provider.root))
|
||||
|
||||
return layout
|
||||
|
||||
@staticmethod
|
||||
def create_install_layout(source_providers): # type: (t.List[t.Type[SourceProvider]]) -> InstallLayout
|
||||
"""Create an install layout using the given source provider."""
|
||||
try:
|
||||
source_provider = find_path_provider(SourceProvider, source_providers, ANSIBLE_ROOT, False)
|
||||
except ProviderNotFoundForPath:
|
||||
source_provider = UnversionedSource(ANSIBLE_ROOT)
|
||||
|
||||
paths = source_provider.get_paths(ANSIBLE_ROOT)
|
||||
|
||||
return InstallLayout(ANSIBLE_ROOT, paths)
|
||||
|
||||
@property
|
||||
def install(self): # type: () -> InstallLayout
|
||||
"""Return the install context, loaded on demand."""
|
||||
if not self.__install:
|
||||
self.__install = self.create_install_layout(self.__source_providers)
|
||||
|
||||
return self.__install
|
||||
|
||||
def register_payload_callback(self, callback): # type: (t.Callable[t.List[t.Tuple[str, str]], None]) -> None
|
||||
"""Register the given payload callback."""
|
||||
self.payload_callbacks.append(callback)
|
||||
|
||||
|
||||
def data_init(): # type: () -> DataContext
|
||||
"""Initialize provider plugins."""
|
||||
provider_types = (
|
||||
'layout',
|
||||
'source',
|
||||
)
|
||||
|
||||
for provider_type in provider_types:
|
||||
import_plugins('provider/%s' % provider_type)
|
||||
|
||||
try:
|
||||
context = DataContext()
|
||||
except ProviderNotFoundForPath:
|
||||
raise ApplicationError('''The current working directory must be at or below one of:
|
||||
|
||||
- Ansible source: %s/
|
||||
- Ansible collection: {...}/ansible_collections/{namespace}/{collection}/
|
||||
|
||||
Current working directory: %s''' % (ANSIBLE_ROOT, os.getcwd()))
|
||||
|
||||
return context
|
||||
|
||||
|
||||
def data_context(): # type: () -> DataContext
|
||||
"""Return the current data context."""
|
||||
try:
|
||||
return data_context.instance
|
||||
except AttributeError:
|
||||
data_context.instance = data_init()
|
||||
return data_context.instance
|
||||
@ -0,0 +1,74 @@
|
||||
"""Provider (plugin) infrastructure for ansible-test."""
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import abc
|
||||
import os
|
||||
|
||||
import lib.types as t
|
||||
|
||||
from lib.util import (
|
||||
ABC,
|
||||
ApplicationError,
|
||||
get_subclasses,
|
||||
)
|
||||
|
||||
|
||||
try:
|
||||
C = t.TypeVar('C', 'PathProvider', 'PathProvider')
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
|
||||
def get_path_provider_classes(provider_type): # type: (t.Type[C]) -> t.List[t.Type[C]]
|
||||
"""Return a list of path provider classes of the given type."""
|
||||
return sorted(get_subclasses(provider_type), key=lambda c: (c.priority, c.__name__))
|
||||
|
||||
|
||||
def find_path_provider(provider_type, provider_classes, path, walk): # type: (t.Type[C], t.List[t.Type[C]], str, bool) -> C
|
||||
"""Return the first found path provider of the given type for the given path."""
|
||||
sequences = sorted(set(pc.sequence for pc in provider_classes if pc.sequence > 0))
|
||||
|
||||
for sequence in sequences:
|
||||
candidate_path = path
|
||||
tier_classes = [pc for pc in provider_classes if pc.sequence == sequence]
|
||||
|
||||
while True:
|
||||
for provider_class in tier_classes:
|
||||
if provider_class.is_content_root(candidate_path):
|
||||
return provider_class(candidate_path)
|
||||
|
||||
if not walk:
|
||||
break
|
||||
|
||||
parent_path = os.path.dirname(candidate_path)
|
||||
|
||||
if parent_path == candidate_path:
|
||||
break
|
||||
|
||||
candidate_path = parent_path
|
||||
|
||||
raise ProviderNotFoundForPath(provider_type, path)
|
||||
|
||||
|
||||
class ProviderNotFoundForPath(ApplicationError):
|
||||
"""Exception generated when a path based provider cannot be found for a given path."""
|
||||
def __init__(self, provider_type, path): # type: (t.Type, str) -> None
|
||||
super(ProviderNotFoundForPath, self).__init__('No %s found for path: %s' % (provider_type.__name__, path))
|
||||
|
||||
self.provider_type = provider_type
|
||||
self.path = path
|
||||
|
||||
|
||||
class PathProvider(ABC):
|
||||
"""Base class for provider plugins that are path based."""
|
||||
sequence = 500
|
||||
priority = 500
|
||||
|
||||
def __init__(self, root): # type: (str) -> None
|
||||
self.root = root
|
||||
|
||||
@staticmethod
|
||||
@abc.abstractmethod
|
||||
def is_content_root(path): # type: (str) -> bool
|
||||
"""Return True if the given path is a content root for this provider."""
|
||||
@ -0,0 +1,183 @@
|
||||
"""Code for finding content."""
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import abc
|
||||
import collections
|
||||
import os
|
||||
|
||||
import lib.types as t
|
||||
|
||||
from lib.util import (
|
||||
ANSIBLE_ROOT,
|
||||
)
|
||||
|
||||
from .. import (
|
||||
PathProvider,
|
||||
)
|
||||
|
||||
|
||||
class Layout:
|
||||
"""Description of content locations and helper methods to access content."""
|
||||
def __init__(self,
|
||||
root, # type: str
|
||||
paths, # type: t.List[str]
|
||||
): # type: (...) -> None
|
||||
self.root = root
|
||||
|
||||
self.__paths = paths
|
||||
self.__tree = paths_to_tree(paths)
|
||||
|
||||
def all_files(self): # type: () -> t.List[str]
|
||||
"""Return a list of all file paths."""
|
||||
return self.__paths
|
||||
|
||||
def walk_files(self, directory): # type: (str) -> t.List[str]
|
||||
"""Return a list of file paths found recursively under the given directory."""
|
||||
parts = directory.rstrip(os.sep).split(os.sep)
|
||||
item = get_tree_item(self.__tree, parts)
|
||||
|
||||
if not item:
|
||||
return []
|
||||
|
||||
directories = collections.deque(item[0].values())
|
||||
|
||||
files = list(item[1])
|
||||
|
||||
while directories:
|
||||
item = directories.pop()
|
||||
directories.extend(item[0].values())
|
||||
files.extend(item[1])
|
||||
|
||||
return files
|
||||
|
||||
def get_dirs(self, directory): # type: (str) -> t.List[str]
|
||||
"""Return a list directory paths found directly under the given directory."""
|
||||
parts = directory.rstrip(os.sep).split(os.sep)
|
||||
item = get_tree_item(self.__tree, parts)
|
||||
return [os.path.join(directory, key) for key in item[0].keys()] if item else []
|
||||
|
||||
def get_files(self, directory): # type: (str) -> t.List[str]
|
||||
"""Return a list of file paths found directly under the given directory."""
|
||||
parts = directory.rstrip(os.sep).split(os.sep)
|
||||
item = get_tree_item(self.__tree, parts)
|
||||
return item[1] if item else []
|
||||
|
||||
|
||||
class InstallLayout(Layout):
|
||||
"""Information about the current Ansible install."""
|
||||
|
||||
|
||||
class ContentLayout(Layout):
|
||||
"""Information about the current Ansible content being tested."""
|
||||
def __init__(self,
|
||||
root, # type: str
|
||||
paths, # type: t.List[str]
|
||||
plugin_paths, # type: t.Dict[str, str]
|
||||
provider_paths, # type: t.Dict[str, str]
|
||||
code_path=None, # type: t.Optional[str]
|
||||
collection=None, # type: t.Optional[CollectionDetail]
|
||||
util_path=None, # type: t.Optional[str]
|
||||
unit_path=None, # type: t.Optional[str]
|
||||
unit_module_path=None, # type: t.Optional[str]
|
||||
integration_path=None, # type: t.Optional[str]
|
||||
): # type: (...) -> None
|
||||
super(ContentLayout, self).__init__(root, paths)
|
||||
|
||||
self.plugin_paths = plugin_paths
|
||||
self.provider_paths = provider_paths
|
||||
self.code_path = code_path
|
||||
self.collection = collection
|
||||
self.util_path = util_path
|
||||
self.unit_path = unit_path
|
||||
self.unit_module_path = unit_module_path
|
||||
self.integration_path = integration_path
|
||||
self.is_ansible = root == ANSIBLE_ROOT
|
||||
|
||||
@property
|
||||
def prefix(self): # type: () -> str
|
||||
"""Return the collection prefix or an empty string if not a collection."""
|
||||
if self.collection:
|
||||
return self.collection.prefix
|
||||
|
||||
return ''
|
||||
|
||||
@property
|
||||
def module_path(self): # type: () -> t.Optional[str]
|
||||
"""Return the path where modules are found, if any."""
|
||||
return self.plugin_paths.get('modules')
|
||||
|
||||
@property
|
||||
def module_utils_path(self): # type: () -> t.Optional[str]
|
||||
"""Return the path where module_utils are found, if any."""
|
||||
return self.plugin_paths.get('module_utils')
|
||||
|
||||
@property
|
||||
def module_utils_powershell_path(self): # type: () -> t.Optional[str]
|
||||
"""Return the path where powershell module_utils are found, if any."""
|
||||
if self.is_ansible:
|
||||
return os.path.join(self.plugin_paths['module_utils'], 'powershell')
|
||||
|
||||
return self.plugin_paths.get('module_utils')
|
||||
|
||||
@property
|
||||
def module_utils_csharp_path(self): # type: () -> t.Optional[str]
|
||||
"""Return the path where csharp module_utils are found, if any."""
|
||||
if self.is_ansible:
|
||||
return os.path.join(self.plugin_paths['module_utils'], 'csharp')
|
||||
|
||||
return self.plugin_paths.get('module_utils')
|
||||
|
||||
|
||||
class CollectionDetail:
|
||||
"""Details about the layout of the current collection."""
|
||||
def __init__(self,
|
||||
name, # type: str
|
||||
namespace, # type: str
|
||||
root, # type: str
|
||||
prefix, # type: str
|
||||
): # type: (...) -> None
|
||||
self.name = name
|
||||
self.namespace = namespace
|
||||
self.root = root
|
||||
self.prefix = prefix
|
||||
self.directory = os.path.join('ansible_collections', namespace, name)
|
||||
|
||||
|
||||
class LayoutProvider(PathProvider):
|
||||
"""Base class for layout providers."""
|
||||
@abc.abstractmethod
|
||||
def create(self, root, paths): # type: (str, t.List[str]) -> ContentLayout
|
||||
"""Create a layout using the given root and paths."""
|
||||
|
||||
|
||||
def paths_to_tree(paths): # type: (t.List[str]) -> t.Tuple(t.Dict[str, t.Any], t.List[str])
|
||||
"""Return a filesystem tree from the given list of paths."""
|
||||
tree = {}, []
|
||||
|
||||
for path in paths:
|
||||
parts = path.split(os.sep)
|
||||
root = tree
|
||||
|
||||
for part in parts[:-1]:
|
||||
if part not in root[0]:
|
||||
root[0][part] = {}, []
|
||||
|
||||
root = root[0][part]
|
||||
|
||||
root[1].append(path)
|
||||
|
||||
return tree
|
||||
|
||||
|
||||
def get_tree_item(tree, parts): # type: (t.Tuple(t.Dict[str, t.Any], t.List[str]), t.List[str]) -> t.Optional[t.Tuple(t.Dict[str, t.Any], t.List[str])]
|
||||
"""Return the portion of the tree found under the path given by parts, or None if it does not exist."""
|
||||
root = tree
|
||||
|
||||
for part in parts:
|
||||
root = root[0].get(part)
|
||||
|
||||
if not root:
|
||||
return None
|
||||
|
||||
return root
|
||||
@ -0,0 +1,45 @@
|
||||
"""Layout provider for Ansible source."""
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
import lib.types as t
|
||||
|
||||
from . import (
|
||||
ContentLayout,
|
||||
LayoutProvider,
|
||||
)
|
||||
|
||||
|
||||
class AnsibleLayout(LayoutProvider):
|
||||
"""Layout provider for Ansible source."""
|
||||
@staticmethod
|
||||
def is_content_root(path): # type: (str) -> bool
|
||||
"""Return True if the given path is a content root for this provider."""
|
||||
return os.path.exists(os.path.join(path, 'setup.py')) and os.path.exists(os.path.join(path, 'bin/ansible-test'))
|
||||
|
||||
def create(self, root, paths): # type: (str, t.List[str]) -> ContentLayout
|
||||
"""Create a Layout using the given root and paths."""
|
||||
plugin_types = sorted(set(p.split('/')[3] for p in paths if re.search(r'^lib/ansible/plugins/[^/]+/', p)))
|
||||
provider_types = sorted(set(p.split('/')[4] for p in paths if re.search(r'^test/runner/lib/provider/[^/]+/', p)))
|
||||
|
||||
plugin_paths = dict((p, os.path.join('lib/ansible/plugins', p)) for p in plugin_types)
|
||||
provider_paths = dict((p, os.path.join('test/runner/lib/provider', p)) for p in provider_types)
|
||||
|
||||
plugin_paths.update(dict(
|
||||
modules='lib/ansible/modules',
|
||||
module_utils='lib/ansible/module_utils',
|
||||
))
|
||||
|
||||
return ContentLayout(root,
|
||||
paths,
|
||||
plugin_paths=plugin_paths,
|
||||
provider_paths=provider_paths,
|
||||
code_path='lib/ansible',
|
||||
util_path='test/utils',
|
||||
unit_path='test/units',
|
||||
unit_module_path='test/units/modules',
|
||||
integration_path='test/integration',
|
||||
)
|
||||
@ -0,0 +1,60 @@
|
||||
"""Layout provider for Ansible collections."""
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
import lib.types as t
|
||||
|
||||
from . import (
|
||||
ContentLayout,
|
||||
LayoutProvider,
|
||||
CollectionDetail,
|
||||
)
|
||||
|
||||
|
||||
class CollectionLayout(LayoutProvider):
|
||||
"""Layout provider for Ansible collections."""
|
||||
__module_path = 'plugins/modules'
|
||||
__unit_path = 'test/unit'
|
||||
|
||||
@staticmethod
|
||||
def is_content_root(path): # type: (str) -> bool
|
||||
"""Return True if the given path is a content root for this provider."""
|
||||
if os.path.basename(os.path.dirname(os.path.dirname(path))) == 'ansible_collections':
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def create(self, root, paths): # type: (str, t.List[str]) -> ContentLayout
|
||||
"""Create a Layout using the given root and paths."""
|
||||
plugin_types = sorted(set(p.split('/')[1] for p in paths if re.search(r'^plugins/[^/]+/', p)))
|
||||
provider_types = sorted(set(p.split('/')[2] for p in paths if re.search(r'^test/provider/[^/]+/', p)))
|
||||
|
||||
plugin_paths = dict((p, os.path.join('plugins', p)) for p in plugin_types)
|
||||
provider_paths = dict((p, os.path.join('test/provider', p)) for p in provider_types)
|
||||
|
||||
collection_root = os.path.dirname(os.path.dirname(root))
|
||||
collection_dir = os.path.relpath(root, collection_root)
|
||||
collection_namespace, collection_name = collection_dir.split(os.sep)
|
||||
|
||||
collection_prefix = '%s.%s.' % (collection_namespace, collection_name)
|
||||
collection_root = os.path.dirname(collection_root)
|
||||
|
||||
return ContentLayout(root,
|
||||
paths,
|
||||
plugin_paths=plugin_paths,
|
||||
provider_paths=provider_paths,
|
||||
code_path='',
|
||||
collection=CollectionDetail(
|
||||
name=collection_name,
|
||||
namespace=collection_namespace,
|
||||
root=collection_root,
|
||||
prefix=collection_prefix,
|
||||
),
|
||||
util_path='test/util',
|
||||
unit_path='test/unit',
|
||||
unit_module_path='test/units/plugins/modules',
|
||||
integration_path='test/integration',
|
||||
)
|
||||
@ -0,0 +1,18 @@
|
||||
"""Common code for source providers."""
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import abc
|
||||
|
||||
import lib.types as t
|
||||
|
||||
from .. import (
|
||||
PathProvider,
|
||||
)
|
||||
|
||||
|
||||
class SourceProvider(PathProvider):
|
||||
"""Base class for source providers."""
|
||||
@abc.abstractmethod
|
||||
def get_paths(self, path): # type: (str) -> t.List[str]
|
||||
"""Return the list of available content paths under the given path."""
|
||||
@ -0,0 +1,31 @@
|
||||
"""Source provider for a content root managed by git version control."""
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
|
||||
import lib.types as t
|
||||
|
||||
from lib.git import (
|
||||
Git,
|
||||
)
|
||||
|
||||
from . import (
|
||||
SourceProvider,
|
||||
)
|
||||
|
||||
|
||||
class GitSource(SourceProvider):
|
||||
"""Source provider for a content root managed by git version control."""
|
||||
@staticmethod
|
||||
def is_content_root(path): # type: (str) -> bool
|
||||
"""Return True if the given path is a content root for this provider."""
|
||||
return os.path.exists(os.path.join(path, '.git'))
|
||||
|
||||
def get_paths(self, path): # type: (str) -> t.List[str]
|
||||
"""Return the list of available content paths under the given path."""
|
||||
git = Git(path)
|
||||
|
||||
paths = git.get_file_names(['--cached', '--others', '--exclude-standard'])
|
||||
|
||||
return paths
|
||||
@ -0,0 +1,77 @@
|
||||
"""Fallback source provider when no other provider matches the content root."""
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import os
|
||||
|
||||
import lib.types as t
|
||||
|
||||
from lib.constants import (
|
||||
TIMEOUT_PATH,
|
||||
)
|
||||
|
||||
from . import (
|
||||
SourceProvider,
|
||||
)
|
||||
|
||||
|
||||
class UnversionedSource(SourceProvider):
|
||||
"""Fallback source provider when no other provider matches the content root."""
|
||||
sequence = 0 # disable automatic detection
|
||||
|
||||
@staticmethod
|
||||
def is_content_root(path): # type: (str) -> bool
|
||||
"""Return True if the given path is a content root for this provider."""
|
||||
return False
|
||||
|
||||
def get_paths(self, path): # type: (str) -> t.List[str]
|
||||
"""Return the list of available content paths under the given path."""
|
||||
paths = []
|
||||
|
||||
kill_any_dir = (
|
||||
'.idea',
|
||||
'.pytest_cache',
|
||||
'__pycache__',
|
||||
'ansible.egg-info',
|
||||
)
|
||||
|
||||
kill_sub_dir = {
|
||||
'test/runner': (
|
||||
'.tox',
|
||||
),
|
||||
'test': (
|
||||
'results',
|
||||
'cache',
|
||||
),
|
||||
'docs/docsite': (
|
||||
'_build',
|
||||
),
|
||||
}
|
||||
|
||||
kill_sub_file = {
|
||||
'': (
|
||||
TIMEOUT_PATH,
|
||||
),
|
||||
}
|
||||
|
||||
kill_extensions = (
|
||||
'.pyc',
|
||||
'.retry',
|
||||
)
|
||||
|
||||
for root, dir_names, file_names in os.walk(path):
|
||||
rel_root = os.path.relpath(root, path)
|
||||
|
||||
if rel_root == '.':
|
||||
rel_root = ''
|
||||
|
||||
for kill in kill_any_dir + kill_sub_dir.get(rel_root, ()):
|
||||
if kill in dir_names:
|
||||
dir_names.remove(kill)
|
||||
|
||||
kill_files = kill_sub_file.get(rel_root, ())
|
||||
|
||||
paths.extend([os.path.join(rel_root, file_name) for file_name in file_names
|
||||
if not os.path.splitext(file_name)[1] in kill_extensions and file_name not in kill_files])
|
||||
|
||||
return paths
|
||||
@ -1,109 +0,0 @@
|
||||
"""Python native TGZ creation."""
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import abc
|
||||
import tarfile
|
||||
import os
|
||||
|
||||
from lib.util import (
|
||||
display,
|
||||
ABC,
|
||||
)
|
||||
|
||||
from lib.constants import (
|
||||
TIMEOUT_PATH,
|
||||
)
|
||||
|
||||
# improve performance by disabling uid/gid lookups
|
||||
tarfile.pwd = None
|
||||
tarfile.grp = None
|
||||
|
||||
|
||||
class TarFilter(ABC):
|
||||
"""Filter to use when creating a tar file."""
|
||||
@abc.abstractmethod
|
||||
def ignore(self, item):
|
||||
"""
|
||||
:type item: tarfile.TarInfo
|
||||
:rtype: tarfile.TarInfo | None
|
||||
"""
|
||||
|
||||
|
||||
class DefaultTarFilter(TarFilter):
|
||||
"""
|
||||
To reduce archive time and size, ignore non-versioned files which are large or numerous.
|
||||
Also ignore miscellaneous git related files since the .git directory is ignored.
|
||||
"""
|
||||
def __init__(self):
|
||||
self.ignore_dirs = (
|
||||
'.tox',
|
||||
'.git',
|
||||
'.idea',
|
||||
'.pytest_cache',
|
||||
'__pycache__',
|
||||
'ansible.egg-info',
|
||||
)
|
||||
|
||||
self.ignore_files = (
|
||||
'.gitignore',
|
||||
'.gitdir',
|
||||
TIMEOUT_PATH,
|
||||
)
|
||||
|
||||
self.ignore_extensions = (
|
||||
'.pyc',
|
||||
'.retry',
|
||||
)
|
||||
|
||||
def ignore(self, item):
|
||||
"""
|
||||
:type item: tarfile.TarInfo
|
||||
:rtype: tarfile.TarInfo | None
|
||||
"""
|
||||
filename = os.path.basename(item.path)
|
||||
ext = os.path.splitext(filename)[1]
|
||||
dirs = os.path.split(item.path)
|
||||
|
||||
if not item.isdir():
|
||||
if item.path.startswith('./test/results/'):
|
||||
return None
|
||||
|
||||
if item.path.startswith('./docs/docsite/_build/'):
|
||||
return None
|
||||
|
||||
if filename in self.ignore_files:
|
||||
return None
|
||||
|
||||
if ext in self.ignore_extensions:
|
||||
return None
|
||||
|
||||
if any(d in self.ignore_dirs for d in dirs):
|
||||
return None
|
||||
|
||||
return item
|
||||
|
||||
|
||||
class AllowGitTarFilter(DefaultTarFilter):
|
||||
"""
|
||||
Filter that allows git related files normally excluded by the default tar filter.
|
||||
"""
|
||||
def __init__(self):
|
||||
super(AllowGitTarFilter, self).__init__()
|
||||
|
||||
self.ignore_dirs = tuple(d for d in self.ignore_dirs if not d.startswith('.git'))
|
||||
self.ignore_files = tuple(f for f in self.ignore_files if not f.startswith('.git'))
|
||||
|
||||
|
||||
def create_tarfile(dst_path, src_path, tar_filter):
|
||||
"""
|
||||
:type dst_path: str
|
||||
:type src_path: str
|
||||
:type tar_filter: TarFilter
|
||||
"""
|
||||
display.info('Creating a compressed tar archive of path: %s' % src_path, verbosity=1)
|
||||
|
||||
with tarfile.TarFile.gzopen(dst_path, mode='w', compresslevel=4) as tar:
|
||||
tar.add(src_path, filter=tar_filter.ignore)
|
||||
|
||||
display.info('Resulting archive is %d bytes.' % os.path.getsize(dst_path), verbosity=1)
|
||||
@ -0,0 +1,8 @@
|
||||
azure-requirements.py
|
||||
botmeta.py
|
||||
changelog.py
|
||||
configure-remoting-ps1.py
|
||||
deprecated-config.py
|
||||
docs-build.py
|
||||
test-constraints.py
|
||||
update-bundled.py
|
||||
@ -0,0 +1,133 @@
|
||||
[MESSAGES CONTROL]
|
||||
|
||||
disable=
|
||||
abstract-method,
|
||||
access-member-before-definition,
|
||||
ansible-deprecated-version,
|
||||
arguments-differ,
|
||||
assignment-from-no-return,
|
||||
assignment-from-none,
|
||||
attribute-defined-outside-init,
|
||||
bad-continuation,
|
||||
bad-indentation,
|
||||
bad-mcs-classmethod-argument,
|
||||
broad-except,
|
||||
c-extension-no-member,
|
||||
cell-var-from-loop,
|
||||
chained-comparison,
|
||||
comparison-with-callable,
|
||||
consider-iterating-dictionary,
|
||||
consider-merging-isinstance,
|
||||
consider-using-dict-comprehension,
|
||||
consider-using-enumerate,
|
||||
consider-using-get,
|
||||
consider-using-in,
|
||||
consider-using-set-comprehension,
|
||||
consider-using-ternary,
|
||||
deprecated-lambda,
|
||||
deprecated-method,
|
||||
deprecated-module,
|
||||
eval-used,
|
||||
exec-used,
|
||||
expression-not-assigned,
|
||||
fixme,
|
||||
function-redefined,
|
||||
global-statement,
|
||||
global-variable-undefined,
|
||||
import-self,
|
||||
inconsistent-return-statements,
|
||||
invalid-envvar-default,
|
||||
invalid-name,
|
||||
invalid-sequence-index,
|
||||
keyword-arg-before-vararg,
|
||||
len-as-condition,
|
||||
line-too-long,
|
||||
literal-comparison,
|
||||
locally-disabled,
|
||||
method-hidden,
|
||||
misplaced-comparison-constant,
|
||||
missing-docstring,
|
||||
no-else-raise,
|
||||
no-else-return,
|
||||
no-init,
|
||||
no-member,
|
||||
no-name-in-module,
|
||||
no-self-use,
|
||||
no-value-for-parameter,
|
||||
non-iterator-returned,
|
||||
not-a-mapping,
|
||||
not-an-iterable,
|
||||
not-callable,
|
||||
old-style-class,
|
||||
pointless-statement,
|
||||
pointless-string-statement,
|
||||
possibly-unused-variable,
|
||||
protected-access,
|
||||
redefined-argument-from-local,
|
||||
redefined-builtin,
|
||||
redefined-outer-name,
|
||||
redefined-variable-type,
|
||||
reimported,
|
||||
relative-beyond-top-level, # https://github.com/PyCQA/pylint/issues/2967
|
||||
signature-differs,
|
||||
simplifiable-if-expression,
|
||||
simplifiable-if-statement,
|
||||
subprocess-popen-preexec-fn,
|
||||
super-init-not-called,
|
||||
superfluous-parens,
|
||||
too-few-public-methods,
|
||||
too-many-ancestors,
|
||||
too-many-arguments,
|
||||
too-many-boolean-expressions,
|
||||
too-many-branches,
|
||||
too-many-function-args,
|
||||
too-many-instance-attributes,
|
||||
too-many-lines,
|
||||
too-many-locals,
|
||||
too-many-nested-blocks,
|
||||
too-many-public-methods,
|
||||
too-many-return-statements,
|
||||
too-many-statements,
|
||||
trailing-comma-tuple,
|
||||
trailing-comma-tuple,
|
||||
try-except-raise,
|
||||
unbalanced-tuple-unpacking,
|
||||
undefined-loop-variable,
|
||||
unexpected-keyword-arg,
|
||||
ungrouped-imports,
|
||||
unidiomatic-typecheck,
|
||||
unnecessary-pass,
|
||||
unsubscriptable-object,
|
||||
unsupported-assignment-operation,
|
||||
unsupported-delete-operation,
|
||||
unsupported-membership-test,
|
||||
unused-argument,
|
||||
unused-import,
|
||||
unused-variable,
|
||||
used-before-assignment,
|
||||
useless-object-inheritance,
|
||||
useless-return,
|
||||
useless-super-delegation,
|
||||
wrong-import-order,
|
||||
wrong-import-position,
|
||||
|
||||
[BASIC]
|
||||
|
||||
bad-names=foo,
|
||||
bar,
|
||||
baz,
|
||||
toto,
|
||||
tutu,
|
||||
tata,
|
||||
_,
|
||||
|
||||
good-names=i,
|
||||
j,
|
||||
k,
|
||||
ex,
|
||||
Run,
|
||||
|
||||
[TYPECHECK]
|
||||
|
||||
ignored-modules=
|
||||
_MovedItems,
|
||||
@ -0,0 +1,30 @@
|
||||
"""Enable unit testing of Ansible collections."""
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
# set by ansible-test to a single directory, rather than a list of directories as supported by Ansible itself
|
||||
ANSIBLE_COLLECTIONS_PATH = os.path.join(os.environ['ANSIBLE_COLLECTIONS_PATHS'], 'ansible_collections')
|
||||
|
||||
|
||||
def collection_pypkgpath(self):
|
||||
for parent in self.parts(reverse=True):
|
||||
if str(parent) == ANSIBLE_COLLECTIONS_PATH:
|
||||
return parent
|
||||
|
||||
raise Exception('File "%s" not found in collection path "%s".' % (self.strpath, ANSIBLE_COLLECTIONS_PATH))
|
||||
|
||||
|
||||
def pytest_configure():
|
||||
from ansible.utils.collection_loader import AnsibleCollectionLoader
|
||||
|
||||
# allow unit tests to import code from collections
|
||||
sys.meta_path.insert(0, AnsibleCollectionLoader())
|
||||
|
||||
import py._path.local
|
||||
|
||||
# force collections unit tests to be loaded with the ansible_collections namespace
|
||||
# original idea from https://stackoverflow.com/questions/50174130/how-do-i-pytest-a-project-using-pep-420-namespace-packages/50175552#50175552
|
||||
py._path.local.LocalPath.pypkgpath = collection_pypkgpath
|
||||
Loading…
Reference in New Issue