mirror of https://github.com/ansible/ansible.git
collection routing (#67684)
* `meta/` directory in collections * runtime metadata for redirection/deprecation/removal of plugin loads * a compatibility layer to keep existing content working on ansible-base + collections * a Python import redirection layer to keep collections-hosted (and otherwise moved) content importable by things that don't know better * supported Ansible version validation on collection loadspull/69714/head
Matt Davis
5 years ago
committed by
GitHub
parent
fdfa6fec75
commit
f7dfa817ae
@ -0,0 +1,10 @@
|
|||||||
|
major_changes:
|
||||||
|
- Plugin routing allows collections to declare deprecation, redirection targets, and removals for all plugin types.
|
||||||
|
- Routing data built into Ansible 2.10 ensures that 2.9 content should work unmodified on 2.10. Formerly included
|
||||||
|
modules and plugins that were moved to collections are still accessible by their original unqualified names, so long
|
||||||
|
as their destination collections are installed.
|
||||||
|
- Collections may declare a list of supported/tested Ansible versions for the collection. A warning is issued if a
|
||||||
|
collection does not support the Ansible version that loads it (can also be configured as silent or a fatal error).
|
||||||
|
Collections that do not declare supported Ansible versions do not issue a warning/error.
|
||||||
|
- Plugins that import module_utils and other ansible namespaces that have moved to collections should
|
||||||
|
continue to work unmodified.
|
||||||
@ -1,2 +1,2 @@
|
|||||||
minor_changes:
|
minor_changes:
|
||||||
- validate-modules checks for deprecated in collections against meta/routing.yml
|
- validate-modules checks for deprecated in collections against meta/runtime.yml
|
||||||
|
|||||||
@ -1,603 +0,0 @@
|
|||||||
# (c) 2019 Ansible Project
|
|
||||||
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
|
||||||
|
|
||||||
from __future__ import (absolute_import, division, print_function)
|
|
||||||
__metaclass__ = type
|
|
||||||
|
|
||||||
import os
|
|
||||||
import os.path
|
|
||||||
import re
|
|
||||||
import sys
|
|
||||||
|
|
||||||
from types import ModuleType
|
|
||||||
|
|
||||||
from ansible.module_utils._text import to_bytes, to_native, to_text
|
|
||||||
from ansible.module_utils.compat.importlib import import_module
|
|
||||||
from ansible.module_utils.six import iteritems, string_types, with_metaclass
|
|
||||||
from ansible.utils.singleton import Singleton
|
|
||||||
|
|
||||||
_SYNTHETIC_PACKAGES = {
|
|
||||||
# these provide fallback package definitions when there are no on-disk paths
|
|
||||||
'ansible_collections': dict(type='pkg_only', allow_external_subpackages=True),
|
|
||||||
'ansible_collections.ansible': dict(type='pkg_only', allow_external_subpackages=True),
|
|
||||||
# these implement the ansible.builtin synthetic collection mapped to the packages inside the ansible distribution
|
|
||||||
'ansible_collections.ansible.builtin': dict(type='pkg_only'),
|
|
||||||
'ansible_collections.ansible.builtin.plugins': dict(type='map', map='ansible.plugins'),
|
|
||||||
'ansible_collections.ansible.builtin.plugins.module_utils': dict(type='map', map='ansible.module_utils', graft=True),
|
|
||||||
'ansible_collections.ansible.builtin.plugins.modules': dict(type='flatmap', flatmap='ansible.modules', graft=True),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# FIXME: exception handling/error logging
|
|
||||||
class AnsibleCollectionLoader(with_metaclass(Singleton, object)):
|
|
||||||
def __init__(self, config=None):
|
|
||||||
if config:
|
|
||||||
paths = config.get_config_value('COLLECTIONS_PATHS')
|
|
||||||
else:
|
|
||||||
paths = os.environ.get('ANSIBLE_COLLECTIONS_PATHS', '').split(os.pathsep)
|
|
||||||
|
|
||||||
if isinstance(paths, string_types):
|
|
||||||
paths = [paths]
|
|
||||||
elif paths is None:
|
|
||||||
paths = []
|
|
||||||
|
|
||||||
# expand any placeholders in configured paths
|
|
||||||
paths = [
|
|
||||||
to_native(os.path.expanduser(p), errors='surrogate_or_strict')
|
|
||||||
for p in paths
|
|
||||||
]
|
|
||||||
|
|
||||||
# Append all ``ansible_collections`` dirs from sys.path to the end
|
|
||||||
for path in sys.path:
|
|
||||||
if (
|
|
||||||
path not in paths and
|
|
||||||
os.path.isdir(to_bytes(
|
|
||||||
os.path.join(path, 'ansible_collections'),
|
|
||||||
errors='surrogate_or_strict',
|
|
||||||
))
|
|
||||||
):
|
|
||||||
paths.append(path)
|
|
||||||
|
|
||||||
self._n_configured_paths = paths
|
|
||||||
|
|
||||||
self._n_playbook_paths = []
|
|
||||||
self._default_collection = None
|
|
||||||
# pre-inject grafted package maps so we can force them to use the right loader instead of potentially delegating to a "normal" loader
|
|
||||||
for syn_pkg_def in (p for p in iteritems(_SYNTHETIC_PACKAGES) if p[1].get('graft')):
|
|
||||||
pkg_name = syn_pkg_def[0]
|
|
||||||
pkg_def = syn_pkg_def[1]
|
|
||||||
|
|
||||||
newmod = ModuleType(pkg_name)
|
|
||||||
newmod.__package__ = pkg_name
|
|
||||||
newmod.__file__ = '<ansible_synthetic_collection_package>'
|
|
||||||
pkg_type = pkg_def.get('type')
|
|
||||||
|
|
||||||
# TODO: need to rethink map style so we can just delegate all the loading
|
|
||||||
|
|
||||||
if pkg_type == 'flatmap':
|
|
||||||
newmod.__loader__ = AnsibleFlatMapLoader(import_module(pkg_def['flatmap']))
|
|
||||||
newmod.__path__ = []
|
|
||||||
|
|
||||||
sys.modules[pkg_name] = newmod
|
|
||||||
|
|
||||||
@property
|
|
||||||
def n_collection_paths(self):
|
|
||||||
return self._n_playbook_paths + self._n_configured_paths
|
|
||||||
|
|
||||||
def get_collection_path(self, collection_name):
|
|
||||||
if not AnsibleCollectionRef.is_valid_collection_name(collection_name):
|
|
||||||
raise ValueError('{0} is not a valid collection name'.format(to_native(collection_name)))
|
|
||||||
|
|
||||||
m = import_module('ansible_collections.{0}'.format(collection_name))
|
|
||||||
|
|
||||||
return m.__file__
|
|
||||||
|
|
||||||
def set_playbook_paths(self, b_playbook_paths):
|
|
||||||
if isinstance(b_playbook_paths, string_types):
|
|
||||||
b_playbook_paths = [b_playbook_paths]
|
|
||||||
|
|
||||||
# track visited paths; we have to preserve the dir order as-passed in case there are duplicate collections (first one wins)
|
|
||||||
added_paths = set()
|
|
||||||
|
|
||||||
# de-dupe and ensure the paths are native strings (Python seems to do this for package paths etc, so assume it's safe)
|
|
||||||
self._n_playbook_paths = [os.path.join(to_native(p), 'collections') for p in b_playbook_paths if not (p in added_paths or added_paths.add(p))]
|
|
||||||
# FIXME: only allow setting this once, or handle any necessary cache/package path invalidations internally?
|
|
||||||
|
|
||||||
# FIXME: is there a better place to store this?
|
|
||||||
# FIXME: only allow setting this once
|
|
||||||
def set_default_collection(self, collection_name):
|
|
||||||
self._default_collection = collection_name
|
|
||||||
|
|
||||||
@property
|
|
||||||
def default_collection(self):
|
|
||||||
return self._default_collection
|
|
||||||
|
|
||||||
def find_module(self, fullname, path=None):
|
|
||||||
if self._find_module(fullname, path, load=False)[0]:
|
|
||||||
return self
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
def load_module(self, fullname):
|
|
||||||
mod = self._find_module(fullname, None, load=True)[1]
|
|
||||||
|
|
||||||
if not mod:
|
|
||||||
raise ImportError('module {0} not found'.format(fullname))
|
|
||||||
|
|
||||||
return mod
|
|
||||||
|
|
||||||
def _find_module(self, fullname, path, load):
|
|
||||||
# this loader is only concerned with items under the Ansible Collections namespace hierarchy, ignore others
|
|
||||||
if not fullname.startswith('ansible_collections.') and fullname != 'ansible_collections':
|
|
||||||
return False, None
|
|
||||||
|
|
||||||
if sys.modules.get(fullname):
|
|
||||||
if not load:
|
|
||||||
return True, None
|
|
||||||
|
|
||||||
return True, sys.modules[fullname]
|
|
||||||
|
|
||||||
newmod = None
|
|
||||||
|
|
||||||
# this loader implements key functionality for Ansible collections
|
|
||||||
# * implicit distributed namespace packages for the root Ansible namespace (no pkgutil.extend_path hackery reqd)
|
|
||||||
# * implicit package support for Python 2.7 (no need for __init__.py in collections, except to use standard Py2.7 tooling)
|
|
||||||
# * preventing controller-side code injection during collection loading
|
|
||||||
# * (default loader would execute arbitrary package code from all __init__.py's)
|
|
||||||
|
|
||||||
parent_pkg_name = '.'.join(fullname.split('.')[:-1])
|
|
||||||
|
|
||||||
parent_pkg = sys.modules.get(parent_pkg_name)
|
|
||||||
|
|
||||||
if parent_pkg_name and not parent_pkg:
|
|
||||||
raise ImportError('parent package {0} not found'.format(parent_pkg_name))
|
|
||||||
|
|
||||||
# are we at or below the collection level? eg a.mynamespace.mycollection.something.else
|
|
||||||
# if so, we don't want distributed namespace behavior; first mynamespace.mycollection on the path is where
|
|
||||||
# we'll load everything from (ie, don't fall back to another mynamespace.mycollection lower on the path)
|
|
||||||
sub_collection = fullname.count('.') > 1
|
|
||||||
|
|
||||||
synpkg_def = _SYNTHETIC_PACKAGES.get(fullname)
|
|
||||||
synpkg_remainder = ''
|
|
||||||
|
|
||||||
if not synpkg_def:
|
|
||||||
# if the parent is a grafted package, we have some special work to do, otherwise just look for stuff on disk
|
|
||||||
parent_synpkg_def = _SYNTHETIC_PACKAGES.get(parent_pkg_name)
|
|
||||||
if parent_synpkg_def and parent_synpkg_def.get('graft'):
|
|
||||||
synpkg_def = parent_synpkg_def
|
|
||||||
synpkg_remainder = '.' + fullname.rpartition('.')[2]
|
|
||||||
|
|
||||||
# FUTURE: collapse as much of this back to on-demand as possible (maybe stub packages that get replaced when actually loaded?)
|
|
||||||
if synpkg_def:
|
|
||||||
pkg_type = synpkg_def.get('type')
|
|
||||||
if not pkg_type:
|
|
||||||
raise KeyError('invalid synthetic package type (no package "type" specified)')
|
|
||||||
if pkg_type == 'map':
|
|
||||||
map_package = synpkg_def.get('map')
|
|
||||||
|
|
||||||
if not map_package:
|
|
||||||
raise KeyError('invalid synthetic map package definition (no target "map" defined)')
|
|
||||||
|
|
||||||
if not load:
|
|
||||||
return True, None
|
|
||||||
|
|
||||||
mod = import_module(map_package + synpkg_remainder)
|
|
||||||
|
|
||||||
sys.modules[fullname] = mod
|
|
||||||
|
|
||||||
return True, mod
|
|
||||||
elif pkg_type == 'flatmap':
|
|
||||||
raise NotImplementedError()
|
|
||||||
elif pkg_type == 'pkg_only':
|
|
||||||
if not load:
|
|
||||||
return True, None
|
|
||||||
|
|
||||||
newmod = ModuleType(fullname)
|
|
||||||
newmod.__package__ = fullname
|
|
||||||
newmod.__file__ = '<ansible_synthetic_collection_package>'
|
|
||||||
newmod.__loader__ = self
|
|
||||||
newmod.__path__ = []
|
|
||||||
|
|
||||||
if not synpkg_def.get('allow_external_subpackages'):
|
|
||||||
# if external subpackages are NOT allowed, we're done
|
|
||||||
sys.modules[fullname] = newmod
|
|
||||||
return True, newmod
|
|
||||||
|
|
||||||
# if external subpackages ARE allowed, check for on-disk implementations and return a normal
|
|
||||||
# package if we find one, otherwise return the one we created here
|
|
||||||
|
|
||||||
if not parent_pkg: # top-level package, look for NS subpackages on all collection paths
|
|
||||||
package_paths = [self._extend_path_with_ns(p, fullname) for p in self.n_collection_paths]
|
|
||||||
else: # subpackage; search in all subpaths (we'll limit later inside a collection)
|
|
||||||
package_paths = [self._extend_path_with_ns(p, fullname) for p in parent_pkg.__path__]
|
|
||||||
|
|
||||||
for candidate_child_path in package_paths:
|
|
||||||
code_object = None
|
|
||||||
is_package = True
|
|
||||||
location = None
|
|
||||||
# check for implicit sub-package first
|
|
||||||
if os.path.isdir(to_bytes(candidate_child_path)):
|
|
||||||
# Py3.x implicit namespace packages don't have a file location, so they don't support get_data
|
|
||||||
# (which assumes the parent dir or that the loader has an internal mapping); so we have to provide
|
|
||||||
# a bogus leaf file on the __file__ attribute for pkgutil.get_data to strip off
|
|
||||||
location = os.path.join(candidate_child_path, '__synthetic__')
|
|
||||||
else:
|
|
||||||
for source_path in [os.path.join(candidate_child_path, '__init__.py'),
|
|
||||||
candidate_child_path + '.py']:
|
|
||||||
if not os.path.isfile(to_bytes(source_path)):
|
|
||||||
continue
|
|
||||||
|
|
||||||
if not load:
|
|
||||||
return True, None
|
|
||||||
|
|
||||||
with open(to_bytes(source_path), 'rb') as fd:
|
|
||||||
source = fd.read()
|
|
||||||
|
|
||||||
code_object = compile(source=source, filename=source_path, mode='exec', flags=0, dont_inherit=True)
|
|
||||||
location = source_path
|
|
||||||
is_package = source_path.endswith('__init__.py')
|
|
||||||
break
|
|
||||||
|
|
||||||
if not location:
|
|
||||||
continue
|
|
||||||
|
|
||||||
newmod = ModuleType(fullname)
|
|
||||||
newmod.__file__ = location
|
|
||||||
newmod.__loader__ = self
|
|
||||||
|
|
||||||
if is_package:
|
|
||||||
if sub_collection: # we never want to search multiple instances of the same collection; use first found
|
|
||||||
newmod.__path__ = [candidate_child_path]
|
|
||||||
else:
|
|
||||||
newmod.__path__ = package_paths
|
|
||||||
|
|
||||||
newmod.__package__ = fullname
|
|
||||||
else:
|
|
||||||
newmod.__package__ = parent_pkg_name
|
|
||||||
|
|
||||||
sys.modules[fullname] = newmod
|
|
||||||
|
|
||||||
if code_object:
|
|
||||||
# FIXME: decide cases where we don't actually want to exec the code?
|
|
||||||
exec(code_object, newmod.__dict__)
|
|
||||||
|
|
||||||
return True, newmod
|
|
||||||
|
|
||||||
# even if we didn't find one on disk, fall back to a synthetic package if we have one...
|
|
||||||
if newmod:
|
|
||||||
sys.modules[fullname] = newmod
|
|
||||||
return True, newmod
|
|
||||||
|
|
||||||
# FIXME: need to handle the "no dirs present" case for at least the root and synthetic internal collections like ansible.builtin
|
|
||||||
|
|
||||||
return False, None
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _extend_path_with_ns(path, ns):
|
|
||||||
ns_path_add = ns.rsplit('.', 1)[-1]
|
|
||||||
|
|
||||||
return os.path.join(path, ns_path_add)
|
|
||||||
|
|
||||||
def get_data(self, filename):
|
|
||||||
with open(filename, 'rb') as fd:
|
|
||||||
return fd.read()
|
|
||||||
|
|
||||||
|
|
||||||
class AnsibleFlatMapLoader(object):
|
|
||||||
_extension_blacklist = ['.pyc', '.pyo']
|
|
||||||
|
|
||||||
def __init__(self, root_package):
|
|
||||||
self._root_package = root_package
|
|
||||||
self._dirtree = None
|
|
||||||
|
|
||||||
def _init_dirtree(self):
|
|
||||||
# FIXME: thread safety
|
|
||||||
root_path = os.path.dirname(self._root_package.__file__)
|
|
||||||
flat_files = []
|
|
||||||
# FIXME: make this a dict of filename->dir for faster direct lookup?
|
|
||||||
# FIXME: deal with _ prefixed deprecated files (or require another method for collections?)
|
|
||||||
# FIXME: fix overloaded filenames (eg, rename Windows setup to win_setup)
|
|
||||||
for root, dirs, files in os.walk(root_path):
|
|
||||||
# add all files in this dir that don't have a blacklisted extension
|
|
||||||
flat_files.extend(((root, f) for f in files if not any((f.endswith(ext) for ext in self._extension_blacklist))))
|
|
||||||
|
|
||||||
# HACK: Put Windows modules at the end of the list. This makes collection_loader behave
|
|
||||||
# the same way as plugin loader, preventing '.ps1' from modules being selected before '.py'
|
|
||||||
# modules simply because '.ps1' files may be above '.py' files in the flat_files list.
|
|
||||||
#
|
|
||||||
# The expected sort order is paths in the order they were in 'flat_files'
|
|
||||||
# with paths ending in '/windows' at the end, also in the original order they were
|
|
||||||
# in 'flat_files'. The .sort() method is guaranteed to be stable, so original order is preserved.
|
|
||||||
flat_files.sort(key=lambda p: p[0].endswith('/windows'))
|
|
||||||
self._dirtree = flat_files
|
|
||||||
|
|
||||||
def find_file(self, filename):
|
|
||||||
# FIXME: thread safety
|
|
||||||
if not self._dirtree:
|
|
||||||
self._init_dirtree()
|
|
||||||
|
|
||||||
if '.' not in filename: # no extension specified, use extension regex to filter
|
|
||||||
extensionless_re = re.compile(r'^{0}(\..+)?$'.format(re.escape(filename)))
|
|
||||||
# why doesn't Python have first()?
|
|
||||||
try:
|
|
||||||
# FIXME: store extensionless in a separate direct lookup?
|
|
||||||
filepath = next(os.path.join(r, f) for r, f in self._dirtree if extensionless_re.match(f))
|
|
||||||
except StopIteration:
|
|
||||||
raise IOError("couldn't find {0}".format(filename))
|
|
||||||
else: # actual filename, just look it up
|
|
||||||
# FIXME: this case sucks; make it a lookup
|
|
||||||
try:
|
|
||||||
filepath = next(os.path.join(r, f) for r, f in self._dirtree if f == filename)
|
|
||||||
except StopIteration:
|
|
||||||
raise IOError("couldn't find {0}".format(filename))
|
|
||||||
|
|
||||||
return filepath
|
|
||||||
|
|
||||||
def get_data(self, filename):
|
|
||||||
found_file = self.find_file(filename)
|
|
||||||
|
|
||||||
with open(found_file, 'rb') as fd:
|
|
||||||
return fd.read()
|
|
||||||
|
|
||||||
|
|
||||||
# TODO: implement these for easier inline debugging?
|
|
||||||
# def get_source(self, fullname):
|
|
||||||
# def get_code(self, fullname):
|
|
||||||
# def is_package(self, fullname):
|
|
||||||
|
|
||||||
|
|
||||||
class AnsibleCollectionRef:
|
|
||||||
# FUTURE: introspect plugin loaders to get these dynamically?
|
|
||||||
VALID_REF_TYPES = frozenset(to_text(r) for r in ['action', 'become', 'cache', 'callback', 'cliconf', 'connection',
|
|
||||||
'doc_fragments', 'filter', 'httpapi', 'inventory', 'lookup',
|
|
||||||
'module_utils', 'modules', 'netconf', 'role', 'shell', 'strategy',
|
|
||||||
'terminal', 'test', 'vars'])
|
|
||||||
|
|
||||||
# FIXME: tighten this up to match Python identifier reqs, etc
|
|
||||||
VALID_COLLECTION_NAME_RE = re.compile(to_text(r'^(\w+)\.(\w+)$'))
|
|
||||||
VALID_SUBDIRS_RE = re.compile(to_text(r'^\w+(\.\w+)*$'))
|
|
||||||
VALID_FQCR_RE = re.compile(to_text(r'^\w+\.\w+\.\w+(\.\w+)*$')) # can have 0-N included subdirs as well
|
|
||||||
|
|
||||||
def __init__(self, collection_name, subdirs, resource, ref_type):
|
|
||||||
"""
|
|
||||||
Create an AnsibleCollectionRef from components
|
|
||||||
:param collection_name: a collection name of the form 'namespace.collectionname'
|
|
||||||
:param subdirs: optional subdir segments to be appended below the plugin type (eg, 'subdir1.subdir2')
|
|
||||||
:param resource: the name of the resource being references (eg, 'mymodule', 'someaction', 'a_role')
|
|
||||||
:param ref_type: the type of the reference, eg 'module', 'role', 'doc_fragment'
|
|
||||||
"""
|
|
||||||
collection_name = to_text(collection_name, errors='strict')
|
|
||||||
if subdirs is not None:
|
|
||||||
subdirs = to_text(subdirs, errors='strict')
|
|
||||||
resource = to_text(resource, errors='strict')
|
|
||||||
ref_type = to_text(ref_type, errors='strict')
|
|
||||||
|
|
||||||
if not self.is_valid_collection_name(collection_name):
|
|
||||||
raise ValueError('invalid collection name (must be of the form namespace.collection): {0}'.format(to_native(collection_name)))
|
|
||||||
|
|
||||||
if ref_type not in self.VALID_REF_TYPES:
|
|
||||||
raise ValueError('invalid collection ref_type: {0}'.format(ref_type))
|
|
||||||
|
|
||||||
self.collection = collection_name
|
|
||||||
if subdirs:
|
|
||||||
if not re.match(self.VALID_SUBDIRS_RE, subdirs):
|
|
||||||
raise ValueError('invalid subdirs entry: {0} (must be empty/None or of the form subdir1.subdir2)'.format(to_native(subdirs)))
|
|
||||||
self.subdirs = subdirs
|
|
||||||
else:
|
|
||||||
self.subdirs = u''
|
|
||||||
|
|
||||||
self.resource = resource
|
|
||||||
self.ref_type = ref_type
|
|
||||||
|
|
||||||
package_components = [u'ansible_collections', self.collection]
|
|
||||||
|
|
||||||
if self.ref_type == u'role':
|
|
||||||
package_components.append(u'roles')
|
|
||||||
else:
|
|
||||||
# we assume it's a plugin
|
|
||||||
package_components += [u'plugins', self.ref_type]
|
|
||||||
|
|
||||||
if self.subdirs:
|
|
||||||
package_components.append(self.subdirs)
|
|
||||||
|
|
||||||
if self.ref_type == u'role':
|
|
||||||
# roles are their own resource
|
|
||||||
package_components.append(self.resource)
|
|
||||||
|
|
||||||
self.n_python_package_name = to_native('.'.join(package_components))
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def from_fqcr(ref, ref_type):
|
|
||||||
"""
|
|
||||||
Parse a string as a fully-qualified collection reference, raises ValueError if invalid
|
|
||||||
:param ref: collection reference to parse (a valid ref is of the form 'ns.coll.resource' or 'ns.coll.subdir1.subdir2.resource')
|
|
||||||
:param ref_type: the type of the reference, eg 'module', 'role', 'doc_fragment'
|
|
||||||
:return: a populated AnsibleCollectionRef object
|
|
||||||
"""
|
|
||||||
# assuming the fq_name is of the form (ns).(coll).(optional_subdir_N).(resource_name),
|
|
||||||
# we split the resource name off the right, split ns and coll off the left, and we're left with any optional
|
|
||||||
# subdirs that need to be added back below the plugin-specific subdir we'll add. So:
|
|
||||||
# ns.coll.resource -> ansible_collections.ns.coll.plugins.(plugintype).resource
|
|
||||||
# ns.coll.subdir1.resource -> ansible_collections.ns.coll.plugins.subdir1.(plugintype).resource
|
|
||||||
# ns.coll.rolename -> ansible_collections.ns.coll.roles.rolename
|
|
||||||
if not AnsibleCollectionRef.is_valid_fqcr(ref):
|
|
||||||
raise ValueError('{0} is not a valid collection reference'.format(to_native(ref)))
|
|
||||||
|
|
||||||
ref = to_text(ref, errors='strict')
|
|
||||||
ref_type = to_text(ref_type, errors='strict')
|
|
||||||
|
|
||||||
resource_splitname = ref.rsplit(u'.', 1)
|
|
||||||
package_remnant = resource_splitname[0]
|
|
||||||
resource = resource_splitname[1]
|
|
||||||
|
|
||||||
# split the left two components of the collection package name off, anything remaining is plugin-type
|
|
||||||
# specific subdirs to be added back on below the plugin type
|
|
||||||
package_splitname = package_remnant.split(u'.', 2)
|
|
||||||
if len(package_splitname) == 3:
|
|
||||||
subdirs = package_splitname[2]
|
|
||||||
else:
|
|
||||||
subdirs = u''
|
|
||||||
|
|
||||||
collection_name = u'.'.join(package_splitname[0:2])
|
|
||||||
|
|
||||||
return AnsibleCollectionRef(collection_name, subdirs, resource, ref_type)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def try_parse_fqcr(ref, ref_type):
|
|
||||||
"""
|
|
||||||
Attempt to parse a string as a fully-qualified collection reference, returning None on failure (instead of raising an error)
|
|
||||||
:param ref: collection reference to parse (a valid ref is of the form 'ns.coll.resource' or 'ns.coll.subdir1.subdir2.resource')
|
|
||||||
:param ref_type: the type of the reference, eg 'module', 'role', 'doc_fragment'
|
|
||||||
:return: a populated AnsibleCollectionRef object on successful parsing, else None
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
return AnsibleCollectionRef.from_fqcr(ref, ref_type)
|
|
||||||
except ValueError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def legacy_plugin_dir_to_plugin_type(legacy_plugin_dir_name):
|
|
||||||
"""
|
|
||||||
Utility method to convert from a PluginLoader dir name to a plugin ref_type
|
|
||||||
:param legacy_plugin_dir_name: PluginLoader dir name (eg, 'action_plugins', 'library')
|
|
||||||
:return: the corresponding plugin ref_type (eg, 'action', 'role')
|
|
||||||
"""
|
|
||||||
legacy_plugin_dir_name = to_text(legacy_plugin_dir_name)
|
|
||||||
|
|
||||||
plugin_type = legacy_plugin_dir_name.replace(u'_plugins', u'')
|
|
||||||
|
|
||||||
if plugin_type == u'library':
|
|
||||||
plugin_type = u'modules'
|
|
||||||
|
|
||||||
if plugin_type not in AnsibleCollectionRef.VALID_REF_TYPES:
|
|
||||||
raise ValueError('{0} cannot be mapped to a valid collection ref type'.format(to_native(legacy_plugin_dir_name)))
|
|
||||||
|
|
||||||
return plugin_type
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def is_valid_fqcr(ref, ref_type=None):
|
|
||||||
"""
|
|
||||||
Validates if is string is a well-formed fully-qualified collection reference (does not look up the collection itself)
|
|
||||||
:param ref: candidate collection reference to validate (a valid ref is of the form 'ns.coll.resource' or 'ns.coll.subdir1.subdir2.resource')
|
|
||||||
:param ref_type: optional reference type to enable deeper validation, eg 'module', 'role', 'doc_fragment'
|
|
||||||
:return: True if the collection ref passed is well-formed, False otherwise
|
|
||||||
"""
|
|
||||||
|
|
||||||
ref = to_text(ref)
|
|
||||||
|
|
||||||
if not ref_type:
|
|
||||||
return bool(re.match(AnsibleCollectionRef.VALID_FQCR_RE, ref))
|
|
||||||
|
|
||||||
return bool(AnsibleCollectionRef.try_parse_fqcr(ref, ref_type))
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def is_valid_collection_name(collection_name):
|
|
||||||
"""
|
|
||||||
Validates if the given string is a well-formed collection name (does not look up the collection itself)
|
|
||||||
:param collection_name: candidate collection name to validate (a valid name is of the form 'ns.collname')
|
|
||||||
:return: True if the collection name passed is well-formed, False otherwise
|
|
||||||
"""
|
|
||||||
|
|
||||||
collection_name = to_text(collection_name)
|
|
||||||
|
|
||||||
return bool(re.match(AnsibleCollectionRef.VALID_COLLECTION_NAME_RE, collection_name))
|
|
||||||
|
|
||||||
|
|
||||||
def get_collection_role_path(role_name, collection_list=None):
|
|
||||||
acr = AnsibleCollectionRef.try_parse_fqcr(role_name, 'role')
|
|
||||||
|
|
||||||
if acr:
|
|
||||||
# looks like a valid qualified collection ref; skip the collection_list
|
|
||||||
collection_list = [acr.collection]
|
|
||||||
subdirs = acr.subdirs
|
|
||||||
resource = acr.resource
|
|
||||||
elif not collection_list:
|
|
||||||
return None # not a FQ role and no collection search list spec'd, nothing to do
|
|
||||||
else:
|
|
||||||
resource = role_name # treat as unqualified, loop through the collection search list to try and resolve
|
|
||||||
subdirs = ''
|
|
||||||
|
|
||||||
for collection_name in collection_list:
|
|
||||||
try:
|
|
||||||
acr = AnsibleCollectionRef(collection_name=collection_name, subdirs=subdirs, resource=resource, ref_type='role')
|
|
||||||
# FIXME: error handling/logging; need to catch any import failures and move along
|
|
||||||
|
|
||||||
# FIXME: this line shouldn't be necessary, but py2 pkgutil.get_data is delegating back to built-in loader when it shouldn't
|
|
||||||
pkg = import_module(acr.n_python_package_name)
|
|
||||||
|
|
||||||
if pkg is not None:
|
|
||||||
# the package is now loaded, get the collection's package and ask where it lives
|
|
||||||
path = os.path.dirname(to_bytes(sys.modules[acr.n_python_package_name].__file__, errors='surrogate_or_strict'))
|
|
||||||
return resource, to_text(path, errors='surrogate_or_strict'), collection_name
|
|
||||||
|
|
||||||
except IOError:
|
|
||||||
continue
|
|
||||||
except Exception as ex:
|
|
||||||
# FIXME: pick out typical import errors first, then error logging
|
|
||||||
continue
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
_N_COLLECTION_PATH_RE = re.compile(r'/ansible_collections/([^/]+)/([^/]+)')
|
|
||||||
|
|
||||||
|
|
||||||
def get_collection_name_from_path(path):
|
|
||||||
"""
|
|
||||||
Return the containing collection name for a given path, or None if the path is not below a configured collection, or
|
|
||||||
the collection cannot be loaded (eg, the collection is masked by another of the same name higher in the configured
|
|
||||||
collection roots).
|
|
||||||
:param n_path: native-string path to evaluate for collection containment
|
|
||||||
:return: collection name or None
|
|
||||||
"""
|
|
||||||
n_collection_paths = [to_native(os.path.abspath(to_bytes(p))) for p in AnsibleCollectionLoader().n_collection_paths]
|
|
||||||
|
|
||||||
b_path = os.path.abspath(to_bytes(path))
|
|
||||||
n_path = to_native(b_path)
|
|
||||||
|
|
||||||
for coll_path in n_collection_paths:
|
|
||||||
common_prefix = to_native(os.path.commonprefix([b_path, to_bytes(coll_path)]))
|
|
||||||
if common_prefix == coll_path:
|
|
||||||
# strip off the common prefix (handle weird testing cases of nested collection roots, eg)
|
|
||||||
collection_remnant = n_path[len(coll_path):]
|
|
||||||
# commonprefix may include the trailing /, prepend to the remnant if necessary (eg trailing / on root)
|
|
||||||
if collection_remnant and collection_remnant[0] != '/':
|
|
||||||
collection_remnant = '/' + collection_remnant
|
|
||||||
# the path lives under this collection root, see if it maps to a collection
|
|
||||||
found_collection = _N_COLLECTION_PATH_RE.search(collection_remnant)
|
|
||||||
if not found_collection:
|
|
||||||
continue
|
|
||||||
n_collection_name = '{0}.{1}'.format(*found_collection.groups())
|
|
||||||
|
|
||||||
loaded_collection_path = AnsibleCollectionLoader().get_collection_path(n_collection_name)
|
|
||||||
|
|
||||||
if not loaded_collection_path:
|
|
||||||
return None
|
|
||||||
|
|
||||||
# ensure we're using the canonical real path, with the bogus __synthetic__ stripped off
|
|
||||||
b_loaded_collection_path = os.path.dirname(os.path.abspath(to_bytes(loaded_collection_path)))
|
|
||||||
|
|
||||||
# if the collection path prefix matches the path prefix we were passed, it's the same collection that's loaded
|
|
||||||
if os.path.commonprefix([b_path, b_loaded_collection_path]) == b_loaded_collection_path:
|
|
||||||
return n_collection_name
|
|
||||||
|
|
||||||
return None # if not, it's a collection, but not the same collection the loader sees, so ignore it
|
|
||||||
|
|
||||||
|
|
||||||
def set_collection_playbook_paths(b_playbook_paths):
|
|
||||||
AnsibleCollectionLoader().set_playbook_paths(b_playbook_paths)
|
|
||||||
|
|
||||||
|
|
||||||
def resource_from_fqcr(ref):
|
|
||||||
"""
|
|
||||||
Return resource from a fully-qualified collection reference,
|
|
||||||
or from a simple resource name.
|
|
||||||
|
|
||||||
For fully-qualified collection references, this is equivalent to
|
|
||||||
``AnsibleCollectionRef.from_fqcr(ref).resource``.
|
|
||||||
|
|
||||||
:param ref: collection reference to parse
|
|
||||||
:return: the resource as a unicode string
|
|
||||||
"""
|
|
||||||
ref = to_text(ref, errors='strict')
|
|
||||||
return ref.split(u'.')[-1]
|
|
||||||
@ -0,0 +1,23 @@
|
|||||||
|
# (c) 2019 Ansible Project
|
||||||
|
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||||
|
|
||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
# FIXME: decide what of this we want to actually be public/toplevel, put other stuff on a utility class?
|
||||||
|
from ._collection_config import AnsibleCollectionConfig
|
||||||
|
from ._collection_finder import AnsibleCollectionRef
|
||||||
|
from ansible.module_utils.common.text.converters import to_text
|
||||||
|
|
||||||
|
|
||||||
|
def resource_from_fqcr(ref):
|
||||||
|
"""
|
||||||
|
Return resource from a fully-qualified collection reference,
|
||||||
|
or from a simple resource name.
|
||||||
|
For fully-qualified collection references, this is equivalent to
|
||||||
|
``AnsibleCollectionRef.from_fqcr(ref).resource``.
|
||||||
|
:param ref: collection reference to parse
|
||||||
|
:return: the resource as a unicode string
|
||||||
|
"""
|
||||||
|
ref = to_text(ref, errors='strict')
|
||||||
|
return ref.split(u'.')[-1]
|
||||||
@ -0,0 +1,101 @@
|
|||||||
|
# (c) 2019 Ansible Project
|
||||||
|
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||||
|
|
||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
from ansible.module_utils.common.text.converters import to_text
|
||||||
|
from ansible.module_utils.six import with_metaclass
|
||||||
|
|
||||||
|
|
||||||
|
class _EventSource:
|
||||||
|
def __init__(self):
|
||||||
|
self._handlers = set()
|
||||||
|
|
||||||
|
def __iadd__(self, handler):
|
||||||
|
if not callable(handler):
|
||||||
|
raise ValueError('handler must be callable')
|
||||||
|
self._handlers.add(handler)
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __isub__(self, handler):
|
||||||
|
try:
|
||||||
|
self._handlers.remove(handler)
|
||||||
|
except KeyError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return self
|
||||||
|
|
||||||
|
def _on_exception(self, handler, exc, *args, **kwargs):
|
||||||
|
# if we return True, we want the caller to re-raise
|
||||||
|
return True
|
||||||
|
|
||||||
|
def fire(self, *args, **kwargs):
|
||||||
|
for h in self._handlers:
|
||||||
|
try:
|
||||||
|
h(*args, **kwargs)
|
||||||
|
except Exception as ex:
|
||||||
|
if self._on_exception(h, ex, *args, **kwargs):
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
class _AnsibleCollectionConfig(type):
|
||||||
|
def __init__(cls, meta, name, bases):
|
||||||
|
cls._collection_finder = None
|
||||||
|
cls._default_collection = None
|
||||||
|
cls._on_collection_load = _EventSource()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def collection_finder(cls):
|
||||||
|
return cls._collection_finder
|
||||||
|
|
||||||
|
@collection_finder.setter
|
||||||
|
def collection_finder(cls, value):
|
||||||
|
if cls._collection_finder:
|
||||||
|
raise ValueError('an AnsibleCollectionFinder has already been configured')
|
||||||
|
|
||||||
|
cls._collection_finder = value
|
||||||
|
|
||||||
|
@property
|
||||||
|
def collection_paths(cls):
|
||||||
|
cls._require_finder()
|
||||||
|
return [to_text(p) for p in cls._collection_finder._n_collection_paths]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def default_collection(cls):
|
||||||
|
return cls._default_collection
|
||||||
|
|
||||||
|
@default_collection.setter
|
||||||
|
def default_collection(cls, value):
|
||||||
|
if cls._default_collection:
|
||||||
|
raise ValueError('default collection {0} has already been configured'.format(value))
|
||||||
|
|
||||||
|
cls._default_collection = value
|
||||||
|
|
||||||
|
@property
|
||||||
|
def on_collection_load(cls):
|
||||||
|
return cls._on_collection_load
|
||||||
|
|
||||||
|
@on_collection_load.setter
|
||||||
|
def on_collection_load(cls, value):
|
||||||
|
if value is not cls._on_collection_load:
|
||||||
|
raise ValueError('on_collection_load is not directly settable (use +=)')
|
||||||
|
|
||||||
|
@property
|
||||||
|
def playbook_paths(cls):
|
||||||
|
cls._require_finder()
|
||||||
|
return [to_text(p) for p in cls._collection_finder._n_playbook_paths]
|
||||||
|
|
||||||
|
@playbook_paths.setter
|
||||||
|
def playbook_paths(cls, value):
|
||||||
|
cls._require_finder()
|
||||||
|
cls._collection_finder.set_playbook_paths(value)
|
||||||
|
|
||||||
|
def _require_finder(cls):
|
||||||
|
if not cls._collection_finder:
|
||||||
|
raise NotImplementedError('an AnsibleCollectionFinder has not been installed in this process')
|
||||||
|
|
||||||
|
|
||||||
|
# concrete class of our metaclass type that defines the class properties we want
|
||||||
|
class AnsibleCollectionConfig(with_metaclass(_AnsibleCollectionConfig)):
|
||||||
|
pass
|
||||||
@ -0,0 +1,953 @@
|
|||||||
|
# (c) 2019 Ansible Project
|
||||||
|
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||||
|
|
||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
import os
|
||||||
|
import os.path
|
||||||
|
import pkgutil
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
# DO NOT add new non-stdlib import deps here, this loader is used by external tools (eg ansible-test import sanity)
|
||||||
|
# that only allow stdlib and module_utils
|
||||||
|
from ansible.module_utils.common.text.converters import to_native, to_text, to_bytes
|
||||||
|
from ansible.module_utils.six import string_types, PY3
|
||||||
|
from ._collection_config import AnsibleCollectionConfig
|
||||||
|
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from types import ModuleType
|
||||||
|
|
||||||
|
try:
|
||||||
|
from importlib import import_module
|
||||||
|
except ImportError:
|
||||||
|
def import_module(name):
|
||||||
|
__import__(name)
|
||||||
|
return sys.modules[name]
|
||||||
|
|
||||||
|
try:
|
||||||
|
from importlib import reload as reload_module
|
||||||
|
except ImportError:
|
||||||
|
# 2.7 has a global reload function instead...
|
||||||
|
reload_module = reload # pylint:disable=undefined-variable
|
||||||
|
|
||||||
|
# NB: this supports import sanity test providing a different impl
|
||||||
|
try:
|
||||||
|
from ._collection_meta import _meta_yml_to_dict
|
||||||
|
except ImportError:
|
||||||
|
_meta_yml_to_dict = None
|
||||||
|
|
||||||
|
|
||||||
|
class _AnsibleCollectionFinder:
|
||||||
|
def __init__(self, paths=None, scan_sys_paths=True):
|
||||||
|
# TODO: accept metadata loader override
|
||||||
|
self._ansible_pkg_path = to_native(os.path.dirname(to_bytes(sys.modules['ansible'].__file__)))
|
||||||
|
|
||||||
|
if isinstance(paths, string_types):
|
||||||
|
paths = [paths]
|
||||||
|
elif paths is None:
|
||||||
|
paths = []
|
||||||
|
|
||||||
|
# expand any placeholders in configured paths
|
||||||
|
paths = [os.path.expanduser(to_native(p, errors='surrogate_or_strict')) for p in paths]
|
||||||
|
|
||||||
|
if scan_sys_paths:
|
||||||
|
# append all sys.path entries with an ansible_collections package
|
||||||
|
for path in sys.path:
|
||||||
|
if (
|
||||||
|
path not in paths and
|
||||||
|
os.path.isdir(to_bytes(
|
||||||
|
os.path.join(path, 'ansible_collections'),
|
||||||
|
errors='surrogate_or_strict',
|
||||||
|
))
|
||||||
|
):
|
||||||
|
paths.append(path)
|
||||||
|
|
||||||
|
self._n_configured_paths = paths
|
||||||
|
self._n_cached_collection_paths = None
|
||||||
|
self._n_cached_collection_qualified_paths = None
|
||||||
|
|
||||||
|
self._n_playbook_paths = []
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _remove(cls):
|
||||||
|
for mps in sys.meta_path:
|
||||||
|
if isinstance(mps, _AnsibleCollectionFinder):
|
||||||
|
sys.meta_path.remove(mps)
|
||||||
|
|
||||||
|
# remove any path hooks that look like ours
|
||||||
|
for ph in sys.path_hooks:
|
||||||
|
if hasattr(ph, '__self__') and isinstance(ph.__self__, _AnsibleCollectionFinder):
|
||||||
|
sys.path_hooks.remove(ph)
|
||||||
|
|
||||||
|
# zap any cached path importer cache entries that might refer to us
|
||||||
|
sys.path_importer_cache.clear()
|
||||||
|
|
||||||
|
AnsibleCollectionConfig._collection_finder = None
|
||||||
|
|
||||||
|
# validate via the public property that we really killed it
|
||||||
|
if AnsibleCollectionConfig.collection_finder is not None:
|
||||||
|
raise AssertionError('_AnsibleCollectionFinder remove did not reset AnsibleCollectionConfig.collection_finder')
|
||||||
|
|
||||||
|
def _install(self):
|
||||||
|
self._remove()
|
||||||
|
sys.meta_path.insert(0, self)
|
||||||
|
|
||||||
|
sys.path_hooks.insert(0, self._ansible_collection_path_hook)
|
||||||
|
|
||||||
|
AnsibleCollectionConfig.collection_finder = self
|
||||||
|
|
||||||
|
def _ansible_collection_path_hook(self, path):
|
||||||
|
path = to_native(path)
|
||||||
|
interesting_paths = self._n_cached_collection_qualified_paths
|
||||||
|
if not interesting_paths:
|
||||||
|
interesting_paths = [os.path.join(p, 'ansible_collections') for p in
|
||||||
|
self._n_collection_paths]
|
||||||
|
interesting_paths.insert(0, self._ansible_pkg_path)
|
||||||
|
self._n_cached_collection_qualified_paths = interesting_paths
|
||||||
|
|
||||||
|
if any(path.startswith(p) for p in interesting_paths):
|
||||||
|
return _AnsiblePathHookFinder(self, path)
|
||||||
|
|
||||||
|
raise ImportError('not interested')
|
||||||
|
|
||||||
|
@property
|
||||||
|
def _n_collection_paths(self):
|
||||||
|
paths = self._n_cached_collection_paths
|
||||||
|
if not paths:
|
||||||
|
self._n_cached_collection_paths = paths = self._n_playbook_paths + self._n_configured_paths
|
||||||
|
return paths
|
||||||
|
|
||||||
|
def set_playbook_paths(self, playbook_paths):
|
||||||
|
if isinstance(playbook_paths, string_types):
|
||||||
|
playbook_paths = [playbook_paths]
|
||||||
|
|
||||||
|
# track visited paths; we have to preserve the dir order as-passed in case there are duplicate collections (first one wins)
|
||||||
|
added_paths = set()
|
||||||
|
|
||||||
|
# de-dupe
|
||||||
|
self._n_playbook_paths = [os.path.join(to_native(p), 'collections') for p in playbook_paths if not (p in added_paths or added_paths.add(p))]
|
||||||
|
self._n_cached_collection_paths = None
|
||||||
|
# HACK: playbook CLI sets this relatively late, so we've already loaded some packages whose paths might depend on this. Fix those up.
|
||||||
|
# NB: this should NOT be used for late additions; ideally we'd fix the playbook dir setup earlier in Ansible init
|
||||||
|
# to prevent this from occurring
|
||||||
|
for pkg in ['ansible_collections', 'ansible_collections.ansible']:
|
||||||
|
self._reload_hack(pkg)
|
||||||
|
|
||||||
|
def _reload_hack(self, fullname):
|
||||||
|
m = sys.modules.get(fullname)
|
||||||
|
if not m:
|
||||||
|
return
|
||||||
|
reload_module(m)
|
||||||
|
|
||||||
|
def find_module(self, fullname, path=None):
|
||||||
|
# Figure out what's being asked for, and delegate to a special-purpose loader
|
||||||
|
|
||||||
|
split_name = fullname.split('.')
|
||||||
|
toplevel_pkg = split_name[0]
|
||||||
|
module_to_find = split_name[-1]
|
||||||
|
part_count = len(split_name)
|
||||||
|
|
||||||
|
if toplevel_pkg not in ['ansible', 'ansible_collections']:
|
||||||
|
# not interested in anything other than ansible_collections (and limited cases under ansible)
|
||||||
|
return None
|
||||||
|
|
||||||
|
# sanity check what we're getting from import, canonicalize path values
|
||||||
|
if part_count == 1:
|
||||||
|
if path:
|
||||||
|
raise ValueError('path should not be specified for top-level packages (trying to find {0})'.format(fullname))
|
||||||
|
else:
|
||||||
|
# seed the path to the configured collection roots
|
||||||
|
path = self._n_collection_paths
|
||||||
|
|
||||||
|
if part_count > 1 and path is None:
|
||||||
|
raise ValueError('path must be specified for subpackages (trying to find {0})'.format(fullname))
|
||||||
|
|
||||||
|
# NB: actual "find"ing is delegated to the constructors on the various loaders; they'll ImportError if not found
|
||||||
|
try:
|
||||||
|
if toplevel_pkg == 'ansible':
|
||||||
|
# something under the ansible package, delegate to our internal loader in case of redirections
|
||||||
|
return _AnsibleInternalRedirectLoader(fullname=fullname, path_list=path)
|
||||||
|
if part_count == 1:
|
||||||
|
return _AnsibleCollectionRootPkgLoader(fullname=fullname, path_list=path)
|
||||||
|
if part_count == 2: # ns pkg eg, ansible_collections, ansible_collections.somens
|
||||||
|
return _AnsibleCollectionNSPkgLoader(fullname=fullname, path_list=path)
|
||||||
|
elif part_count == 3: # collection pkg eg, ansible_collections.somens.somecoll
|
||||||
|
return _AnsibleCollectionPkgLoader(fullname=fullname, path_list=path)
|
||||||
|
# anything below the collection
|
||||||
|
return _AnsibleCollectionLoader(fullname=fullname, path_list=path)
|
||||||
|
except ImportError:
|
||||||
|
# TODO: log attempt to load context
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# Implements a path_hook finder for iter_modules (since it's only path based). This finder does not need to actually
|
||||||
|
# function as a finder in most cases, since our meta_path finder is consulted first for *almost* everything, except
|
||||||
|
# pkgutil.iter_modules, and under py2, pkgutil.get_data if the parent package passed has not been loaded yet.
|
||||||
|
class _AnsiblePathHookFinder:
|
||||||
|
def __init__(self, collection_finder, pathctx):
|
||||||
|
# when called from a path_hook, find_module doesn't usually get the path arg, so this provides our context
|
||||||
|
self._pathctx = to_native(pathctx)
|
||||||
|
self._collection_finder = collection_finder
|
||||||
|
if PY3:
|
||||||
|
# cache the native FileFinder (take advantage of its filesystem cache for future find/load requests)
|
||||||
|
self._file_finder = None
|
||||||
|
|
||||||
|
# class init is fun- this method has a self arg that won't get used
|
||||||
|
def _get_filefinder_path_hook(self=None):
|
||||||
|
_file_finder_hook = None
|
||||||
|
if PY3:
|
||||||
|
# try to find the FileFinder hook to call for fallback path-based imports in Py3
|
||||||
|
_file_finder_hook = [ph for ph in sys.path_hooks if 'FileFinder' in repr(ph)]
|
||||||
|
if len(_file_finder_hook) != 1:
|
||||||
|
raise Exception('need exactly one FileFinder import hook (found {0})'.format(len(_file_finder_hook)))
|
||||||
|
_file_finder_hook = _file_finder_hook[0]
|
||||||
|
|
||||||
|
return _file_finder_hook
|
||||||
|
|
||||||
|
_filefinder_path_hook = _get_filefinder_path_hook()
|
||||||
|
|
||||||
|
def find_module(self, fullname, path=None):
|
||||||
|
# we ignore the passed in path here- use what we got from the path hook init
|
||||||
|
split_name = fullname.split('.')
|
||||||
|
toplevel_pkg = split_name[0]
|
||||||
|
|
||||||
|
if toplevel_pkg == 'ansible_collections':
|
||||||
|
# collections content? delegate to the collection finder
|
||||||
|
return self._collection_finder.find_module(fullname, path=[self._pathctx])
|
||||||
|
else:
|
||||||
|
# Something else; we'd normally restrict this to `ansible` descendent modules so that any weird loader
|
||||||
|
# behavior that arbitrary Python modules have can be serviced by those loaders. In some dev/test
|
||||||
|
# scenarios (eg a venv under a collection) our path_hook signs us up to load non-Ansible things, and
|
||||||
|
# it's too late by the time we've reached this point, but also too expensive for the path_hook to figure
|
||||||
|
# out what we *shouldn't* be loading with the limited info it has. So we'll just delegate to the
|
||||||
|
# normal path-based loader as best we can to service it. This also allows us to take advantage of Python's
|
||||||
|
# built-in FS caching and byte-compilation for most things.
|
||||||
|
if PY3:
|
||||||
|
# create or consult our cached file finder for this path
|
||||||
|
if not self._file_finder:
|
||||||
|
self._file_finder = _AnsiblePathHookFinder._filefinder_path_hook(self._pathctx)
|
||||||
|
|
||||||
|
spec = self._file_finder.find_spec(fullname)
|
||||||
|
if not spec:
|
||||||
|
return None
|
||||||
|
return spec.loader
|
||||||
|
else:
|
||||||
|
# call py2's internal loader
|
||||||
|
return pkgutil.ImpImporter(self._pathctx).find_module(fullname)
|
||||||
|
|
||||||
|
def iter_modules(self, prefix):
|
||||||
|
# NB: this currently represents only what's on disk, and does not handle package redirection
|
||||||
|
return _iter_modules_impl([self._pathctx], prefix)
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return "{0}(path='{1}')".format(self.__class__.__name__, self._pathctx)
|
||||||
|
|
||||||
|
|
||||||
|
class _AnsibleCollectionPkgLoaderBase:
|
||||||
|
_allows_package_code = False
|
||||||
|
|
||||||
|
def __init__(self, fullname, path_list=None):
|
||||||
|
self._fullname = fullname
|
||||||
|
self._redirect_module = None
|
||||||
|
self._split_name = fullname.split('.')
|
||||||
|
self._rpart_name = fullname.rpartition('.')
|
||||||
|
self._parent_package_name = self._rpart_name[0] # eg ansible_collections for ansible_collections.somens, '' for toplevel
|
||||||
|
self._package_to_load = self._rpart_name[2] # eg somens for ansible_collections.somens
|
||||||
|
|
||||||
|
self._source_code_path = None
|
||||||
|
self._decoded_source = None
|
||||||
|
self._compiled_code = None
|
||||||
|
|
||||||
|
self._validate_args()
|
||||||
|
|
||||||
|
self._candidate_paths = self._get_candidate_paths([to_native(p) for p in path_list])
|
||||||
|
self._subpackage_search_paths = self._get_subpackage_search_paths(self._candidate_paths)
|
||||||
|
|
||||||
|
self._validate_final()
|
||||||
|
|
||||||
|
# allow subclasses to validate args and sniff split values before we start digging around
|
||||||
|
def _validate_args(self):
|
||||||
|
if self._split_name[0] != 'ansible_collections':
|
||||||
|
raise ImportError('this loader can only load packages from the ansible_collections package, not {0}'.format(self._fullname))
|
||||||
|
|
||||||
|
# allow subclasses to customize candidate path filtering
|
||||||
|
def _get_candidate_paths(self, path_list):
|
||||||
|
return [os.path.join(p, self._package_to_load) for p in path_list]
|
||||||
|
|
||||||
|
# allow subclasses to customize finding paths
|
||||||
|
def _get_subpackage_search_paths(self, candidate_paths):
|
||||||
|
# filter candidate paths for existence (NB: silently ignoring package init code and same-named modules)
|
||||||
|
return [p for p in candidate_paths if os.path.isdir(to_bytes(p))]
|
||||||
|
|
||||||
|
# allow subclasses to customize state validation/manipulation before we return the loader instance
|
||||||
|
def _validate_final(self):
|
||||||
|
return
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
@contextmanager
|
||||||
|
def _new_or_existing_module(name, **kwargs):
|
||||||
|
# handle all-or-nothing sys.modules creation/use-existing/delete-on-exception-if-created behavior
|
||||||
|
created_module = False
|
||||||
|
module = sys.modules.get(name)
|
||||||
|
try:
|
||||||
|
if not module:
|
||||||
|
module = ModuleType(name)
|
||||||
|
created_module = True
|
||||||
|
sys.modules[name] = module
|
||||||
|
# always override the values passed, except name (allow reference aliasing)
|
||||||
|
for attr, value in kwargs.items():
|
||||||
|
setattr(module, attr, value)
|
||||||
|
yield module
|
||||||
|
except Exception:
|
||||||
|
if created_module:
|
||||||
|
if sys.modules.get(name):
|
||||||
|
sys.modules.pop(name)
|
||||||
|
raise
|
||||||
|
|
||||||
|
# basic module/package location support
|
||||||
|
# NB: this does not support distributed packages!
|
||||||
|
@staticmethod
|
||||||
|
def _module_file_from_path(leaf_name, path):
|
||||||
|
has_code = True
|
||||||
|
package_path = os.path.join(to_native(path), to_native(leaf_name))
|
||||||
|
module_path = None
|
||||||
|
|
||||||
|
# if the submodule is a package, assemble valid submodule paths, but stop looking for a module
|
||||||
|
if os.path.isdir(to_bytes(package_path)):
|
||||||
|
# is there a package init?
|
||||||
|
module_path = os.path.join(package_path, '__init__.py')
|
||||||
|
if not os.path.isfile(to_bytes(module_path)):
|
||||||
|
module_path = os.path.join(package_path, '__synthetic__')
|
||||||
|
has_code = False
|
||||||
|
else:
|
||||||
|
module_path = package_path + '.py'
|
||||||
|
package_path = None
|
||||||
|
if not os.path.isfile(to_bytes(module_path)):
|
||||||
|
raise ImportError('{0} not found at {1}'.format(leaf_name, path))
|
||||||
|
|
||||||
|
return module_path, has_code, package_path
|
||||||
|
|
||||||
|
def load_module(self, fullname):
|
||||||
|
# short-circuit redirect; we've already imported the redirected module, so just alias it and return it
|
||||||
|
if self._redirect_module:
|
||||||
|
sys.modules[self._fullname] = self._redirect_module
|
||||||
|
return self._redirect_module
|
||||||
|
|
||||||
|
# we're actually loading a module/package
|
||||||
|
module_attrs = dict(
|
||||||
|
__loader__=self,
|
||||||
|
__file__=self.get_filename(fullname),
|
||||||
|
__package__=self._parent_package_name # sane default for non-packages
|
||||||
|
)
|
||||||
|
|
||||||
|
# eg, I am a package
|
||||||
|
if self._subpackage_search_paths is not None: # empty is legal
|
||||||
|
module_attrs['__path__'] = self._subpackage_search_paths
|
||||||
|
module_attrs['__package__'] = fullname # per PEP366
|
||||||
|
|
||||||
|
with self._new_or_existing_module(fullname, **module_attrs) as module:
|
||||||
|
# execute the module's code in its namespace
|
||||||
|
exec(self.get_code(fullname), module.__dict__)
|
||||||
|
|
||||||
|
return module
|
||||||
|
|
||||||
|
def is_package(self, fullname):
|
||||||
|
if fullname != self._fullname:
|
||||||
|
raise ValueError('this loader cannot answer is_package for {0}, only {1}'.format(fullname, self._fullname))
|
||||||
|
return self._subpackage_search_paths is not None
|
||||||
|
|
||||||
|
def get_source(self, fullname):
|
||||||
|
if self._decoded_source:
|
||||||
|
return self._decoded_source
|
||||||
|
if fullname != self._fullname:
|
||||||
|
raise ValueError('this loader cannot load source for {0}, only {1}'.format(fullname, self._fullname))
|
||||||
|
if not self._source_code_path:
|
||||||
|
return None
|
||||||
|
# FIXME: what do we want encoding/newline requirements to be?
|
||||||
|
self._decoded_source = self.get_data(self._source_code_path)
|
||||||
|
return self._decoded_source
|
||||||
|
|
||||||
|
def get_data(self, path):
|
||||||
|
if not path:
|
||||||
|
raise ValueError('a path must be specified')
|
||||||
|
|
||||||
|
# TODO: ensure we're being asked for a path below something we own
|
||||||
|
# TODO: try to handle redirects internally?
|
||||||
|
|
||||||
|
if not path[0] == '/':
|
||||||
|
# relative to current package, search package paths if possible (this may not be necessary)
|
||||||
|
# candidate_paths = [os.path.join(ssp, path) for ssp in self._subpackage_search_paths]
|
||||||
|
raise ValueError('relative resource paths not supported')
|
||||||
|
else:
|
||||||
|
candidate_paths = [path]
|
||||||
|
|
||||||
|
for p in candidate_paths:
|
||||||
|
b_path = to_bytes(p)
|
||||||
|
if os.path.isfile(b_path):
|
||||||
|
with open(b_path, 'rb') as fd:
|
||||||
|
return fd.read()
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _synthetic_filename(self, fullname):
|
||||||
|
return '<ansible_synthetic_collection_package>'
|
||||||
|
|
||||||
|
def get_filename(self, fullname):
|
||||||
|
if fullname != self._fullname:
|
||||||
|
raise ValueError('this loader cannot find files for {0}, only {1}'.format(fullname, self._fullname))
|
||||||
|
|
||||||
|
filename = self._source_code_path
|
||||||
|
|
||||||
|
if not filename and self.is_package(fullname):
|
||||||
|
if len(self._subpackage_search_paths) == 1:
|
||||||
|
filename = os.path.join(self._subpackage_search_paths[0], '__synthetic__')
|
||||||
|
else:
|
||||||
|
filename = self._synthetic_filename(fullname)
|
||||||
|
|
||||||
|
return filename
|
||||||
|
|
||||||
|
def get_code(self, fullname):
|
||||||
|
if self._compiled_code:
|
||||||
|
return self._compiled_code
|
||||||
|
|
||||||
|
# this may or may not be an actual filename, but it's the value we'll use for __file__
|
||||||
|
filename = self.get_filename(fullname)
|
||||||
|
if not filename:
|
||||||
|
filename = '<string>'
|
||||||
|
|
||||||
|
source_code = self.get_source(fullname)
|
||||||
|
if not source_code:
|
||||||
|
source_code = ''
|
||||||
|
|
||||||
|
self._compiled_code = compile(source=source_code, filename=filename, mode='exec', flags=0, dont_inherit=True)
|
||||||
|
|
||||||
|
return self._compiled_code
|
||||||
|
|
||||||
|
def iter_modules(self, prefix):
|
||||||
|
return _iter_modules_impl(self._subpackage_search_paths, prefix)
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return '{0}(path={1})'.format(self.__class__.__name__, self._subpackage_search_paths or self._source_code_path)
|
||||||
|
|
||||||
|
|
||||||
|
class _AnsibleCollectionRootPkgLoader(_AnsibleCollectionPkgLoaderBase):
|
||||||
|
def _validate_args(self):
|
||||||
|
super(_AnsibleCollectionRootPkgLoader, self)._validate_args()
|
||||||
|
if len(self._split_name) != 1:
|
||||||
|
raise ImportError('this loader can only load the ansible_collections toplevel package, not {0}'.format(self._fullname))
|
||||||
|
|
||||||
|
|
||||||
|
# Implements Ansible's custom namespace package support.
|
||||||
|
# The ansible_collections package and one level down (collections namespaces) are Python namespace packages
|
||||||
|
# that search across all configured collection roots. The collection package (two levels down) is the first one found
|
||||||
|
# on the configured collection root path, and Python namespace package aggregation is not allowed at or below
|
||||||
|
# the collection. Implements implicit package (package dir) support for both Py2/3. Package init code is ignored
|
||||||
|
# by this loader.
|
||||||
|
class _AnsibleCollectionNSPkgLoader(_AnsibleCollectionPkgLoaderBase):
|
||||||
|
def _validate_args(self):
|
||||||
|
super(_AnsibleCollectionNSPkgLoader, self)._validate_args()
|
||||||
|
if len(self._split_name) != 2:
|
||||||
|
raise ImportError('this loader can only load collections namespace packages, not {0}'.format(self._fullname))
|
||||||
|
|
||||||
|
def _validate_final(self):
|
||||||
|
# special-case the `ansible` namespace, since `ansible.builtin` is magical
|
||||||
|
if not self._subpackage_search_paths and self._package_to_load != 'ansible':
|
||||||
|
raise ImportError('no {0} found in {1}'.format(self._package_to_load, self._candidate_paths))
|
||||||
|
|
||||||
|
|
||||||
|
# handles locating the actual collection package and associated metadata
|
||||||
|
class _AnsibleCollectionPkgLoader(_AnsibleCollectionPkgLoaderBase):
|
||||||
|
def _validate_args(self):
|
||||||
|
super(_AnsibleCollectionPkgLoader, self)._validate_args()
|
||||||
|
if len(self._split_name) != 3:
|
||||||
|
raise ImportError('this loader can only load collection packages, not {0}'.format(self._fullname))
|
||||||
|
|
||||||
|
def _validate_final(self):
|
||||||
|
if self._split_name[1:3] == ['ansible', 'builtin']:
|
||||||
|
# we don't want to allow this one to have on-disk search capability
|
||||||
|
self._subpackage_search_paths = []
|
||||||
|
elif not self._subpackage_search_paths:
|
||||||
|
raise ImportError('no {0} found in {1}'.format(self._package_to_load, self._candidate_paths))
|
||||||
|
else:
|
||||||
|
# only search within the first collection we found
|
||||||
|
self._subpackage_search_paths = [self._subpackage_search_paths[0]]
|
||||||
|
|
||||||
|
def load_module(self, fullname):
|
||||||
|
if not _meta_yml_to_dict:
|
||||||
|
raise ValueError('ansible.utils.collection_loader._meta_yml_to_dict is not set')
|
||||||
|
|
||||||
|
module = super(_AnsibleCollectionPkgLoader, self).load_module(fullname)
|
||||||
|
|
||||||
|
module._collection_meta = {}
|
||||||
|
# TODO: load collection metadata, cache in __loader__ state
|
||||||
|
|
||||||
|
collection_name = '.'.join(self._split_name[1:3])
|
||||||
|
|
||||||
|
if collection_name == 'ansible.builtin':
|
||||||
|
# ansible.builtin is a synthetic collection, get its routing config from the Ansible distro
|
||||||
|
raw_routing = pkgutil.get_data('ansible.config', 'ansible_builtin_runtime.yml')
|
||||||
|
else:
|
||||||
|
b_routing_meta_path = to_bytes(os.path.join(module.__path__[0], 'meta/runtime.yml'))
|
||||||
|
if os.path.isfile(b_routing_meta_path):
|
||||||
|
with open(b_routing_meta_path, 'rb') as fd:
|
||||||
|
raw_routing = fd.read()
|
||||||
|
else:
|
||||||
|
raw_routing = ''
|
||||||
|
try:
|
||||||
|
if raw_routing:
|
||||||
|
routing_dict = _meta_yml_to_dict(raw_routing, (collection_name, 'runtime.yml'))
|
||||||
|
module._collection_meta = self._canonicalize_meta(routing_dict)
|
||||||
|
except Exception as ex:
|
||||||
|
raise ValueError('error parsing collection metadata: {0}'.format(to_native(ex)))
|
||||||
|
|
||||||
|
AnsibleCollectionConfig.on_collection_load.fire(collection_name=collection_name, collection_path=os.path.dirname(module.__file__))
|
||||||
|
|
||||||
|
return module
|
||||||
|
|
||||||
|
def _canonicalize_meta(self, meta_dict):
|
||||||
|
# TODO: rewrite import keys and all redirect targets that start with .. (current namespace) and . (current collection)
|
||||||
|
# OR we could do it all on the fly?
|
||||||
|
# if not meta_dict:
|
||||||
|
# return {}
|
||||||
|
#
|
||||||
|
# ns_name = '.'.join(self._split_name[0:2])
|
||||||
|
# collection_name = '.'.join(self._split_name[0:3])
|
||||||
|
#
|
||||||
|
# #
|
||||||
|
# for routing_type, routing_type_dict in iteritems(meta_dict.get('plugin_routing', {})):
|
||||||
|
# for plugin_key, plugin_dict in iteritems(routing_type_dict):
|
||||||
|
# redirect = plugin_dict.get('redirect', '')
|
||||||
|
# if redirect.startswith('..'):
|
||||||
|
# redirect = redirect[2:]
|
||||||
|
|
||||||
|
return meta_dict
|
||||||
|
|
||||||
|
|
||||||
|
# loads everything under a collection, including handling redirections defined by the collection
|
||||||
|
class _AnsibleCollectionLoader(_AnsibleCollectionPkgLoaderBase):
|
||||||
|
# HACK: stash this in a better place
|
||||||
|
_redirected_package_map = {}
|
||||||
|
_allows_package_code = True
|
||||||
|
|
||||||
|
def _validate_args(self):
|
||||||
|
super(_AnsibleCollectionLoader, self)._validate_args()
|
||||||
|
if len(self._split_name) < 4:
|
||||||
|
raise ValueError('this loader is only for sub-collection modules/packages, not {0}'.format(self._fullname))
|
||||||
|
|
||||||
|
def _get_candidate_paths(self, path_list):
|
||||||
|
if len(path_list) != 1 and self._split_name[1:3] != ['ansible', 'builtin']:
|
||||||
|
raise ValueError('this loader requires exactly one path to search')
|
||||||
|
|
||||||
|
return path_list
|
||||||
|
|
||||||
|
def _get_subpackage_search_paths(self, candidate_paths):
|
||||||
|
collection_meta = _get_collection_metadata('.'.join(self._split_name[1:3]))
|
||||||
|
|
||||||
|
# check for explicit redirection, as well as ancestor package-level redirection (only load the actual code once!)
|
||||||
|
redirect = None
|
||||||
|
explicit_redirect = False
|
||||||
|
|
||||||
|
routing_entry = _nested_dict_get(collection_meta, ['import_redirection', self._fullname])
|
||||||
|
if routing_entry:
|
||||||
|
redirect = routing_entry.get('redirect')
|
||||||
|
|
||||||
|
if redirect:
|
||||||
|
explicit_redirect = True
|
||||||
|
else:
|
||||||
|
redirect = _get_ancestor_redirect(self._redirected_package_map, self._fullname)
|
||||||
|
|
||||||
|
# NB: package level redirection requires hooking all future imports beneath the redirected source package
|
||||||
|
# in order to ensure sanity on future relative imports. We always import everything under its "real" name,
|
||||||
|
# then add a sys.modules entry with the redirected name using the same module instance. If we naively imported
|
||||||
|
# the source for each redirection, most submodules would import OK, but we'd have N runtime copies of the module
|
||||||
|
# (one for each name), and relative imports that ascend above the redirected package would break (since they'd
|
||||||
|
# see the redirected ancestor package contents instead of the package where they actually live).
|
||||||
|
if redirect:
|
||||||
|
# FIXME: wrap this so we can be explicit about a failed redirection
|
||||||
|
self._redirect_module = import_module(redirect)
|
||||||
|
if explicit_redirect and hasattr(self._redirect_module, '__path__') and self._redirect_module.__path__:
|
||||||
|
# if the import target looks like a package, store its name so we can rewrite future descendent loads
|
||||||
|
# FIXME: shouldn't this be in a shared location? This is currently per loader instance, so
|
||||||
|
self._redirected_package_map[self._fullname] = redirect
|
||||||
|
|
||||||
|
# if we redirected, don't do any further custom package logic
|
||||||
|
return None
|
||||||
|
|
||||||
|
# we're not doing a redirect- try to find what we need to actually load a module/package
|
||||||
|
|
||||||
|
# this will raise ImportError if we can't find the requested module/package at all
|
||||||
|
if not candidate_paths:
|
||||||
|
# noplace to look, just ImportError
|
||||||
|
raise ImportError('package has no paths')
|
||||||
|
|
||||||
|
found_path, has_code, package_path = self._module_file_from_path(self._package_to_load, candidate_paths[0])
|
||||||
|
|
||||||
|
# still here? we found something to load...
|
||||||
|
if has_code:
|
||||||
|
self._source_code_path = found_path
|
||||||
|
|
||||||
|
if package_path:
|
||||||
|
return [package_path] # always needs to be a list
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# This loader only answers for intercepted Ansible Python modules. Normal imports will fail here and be picked up later
|
||||||
|
# by our path_hook importer (which proxies the built-in import mechanisms, allowing normal caching etc to occur)
|
||||||
|
class _AnsibleInternalRedirectLoader:
|
||||||
|
def __init__(self, fullname, path_list):
|
||||||
|
self._redirect = None
|
||||||
|
|
||||||
|
split_name = fullname.split('.')
|
||||||
|
toplevel_pkg = split_name[0]
|
||||||
|
module_to_load = split_name[-1]
|
||||||
|
|
||||||
|
if toplevel_pkg != 'ansible':
|
||||||
|
raise ImportError('not interested')
|
||||||
|
|
||||||
|
builtin_meta = _get_collection_metadata('ansible.builtin')
|
||||||
|
|
||||||
|
routing_entry = _nested_dict_get(builtin_meta, ['import_redirection', fullname])
|
||||||
|
if routing_entry:
|
||||||
|
self._redirect = routing_entry.get('redirect')
|
||||||
|
|
||||||
|
if not self._redirect:
|
||||||
|
raise ImportError('not redirected, go ask path_hook')
|
||||||
|
|
||||||
|
def load_module(self, fullname):
|
||||||
|
# since we're delegating to other loaders, this should only be called for internal redirects where we answered
|
||||||
|
# find_module with this loader, in which case we'll just directly import the redirection target, insert it into
|
||||||
|
# sys.modules under the name it was requested by, and return the original module.
|
||||||
|
|
||||||
|
# should never see this
|
||||||
|
if not self._redirect:
|
||||||
|
raise ValueError('no redirect found for {0}'.format(fullname))
|
||||||
|
|
||||||
|
# FIXME: smuggle redirection context, provide warning/error that we tried and failed to redirect
|
||||||
|
mod = import_module(self._redirect)
|
||||||
|
sys.modules[fullname] = mod
|
||||||
|
return mod
|
||||||
|
|
||||||
|
|
||||||
|
class AnsibleCollectionRef:
|
||||||
|
# FUTURE: introspect plugin loaders to get these dynamically?
|
||||||
|
VALID_REF_TYPES = frozenset(to_text(r) for r in ['action', 'become', 'cache', 'callback', 'cliconf', 'connection',
|
||||||
|
'doc_fragments', 'filter', 'httpapi', 'inventory', 'lookup',
|
||||||
|
'module_utils', 'modules', 'netconf', 'role', 'shell', 'strategy',
|
||||||
|
'terminal', 'test', 'vars'])
|
||||||
|
|
||||||
|
# FIXME: tighten this up to match Python identifier reqs, etc
|
||||||
|
VALID_COLLECTION_NAME_RE = re.compile(to_text(r'^(\w+)\.(\w+)$'))
|
||||||
|
VALID_SUBDIRS_RE = re.compile(to_text(r'^\w+(\.\w+)*$'))
|
||||||
|
VALID_FQCR_RE = re.compile(to_text(r'^\w+\.\w+\.\w+(\.\w+)*$')) # can have 0-N included subdirs as well
|
||||||
|
|
||||||
|
def __init__(self, collection_name, subdirs, resource, ref_type):
|
||||||
|
"""
|
||||||
|
Create an AnsibleCollectionRef from components
|
||||||
|
:param collection_name: a collection name of the form 'namespace.collectionname'
|
||||||
|
:param subdirs: optional subdir segments to be appended below the plugin type (eg, 'subdir1.subdir2')
|
||||||
|
:param resource: the name of the resource being references (eg, 'mymodule', 'someaction', 'a_role')
|
||||||
|
:param ref_type: the type of the reference, eg 'module', 'role', 'doc_fragment'
|
||||||
|
"""
|
||||||
|
collection_name = to_text(collection_name, errors='strict')
|
||||||
|
if subdirs is not None:
|
||||||
|
subdirs = to_text(subdirs, errors='strict')
|
||||||
|
resource = to_text(resource, errors='strict')
|
||||||
|
ref_type = to_text(ref_type, errors='strict')
|
||||||
|
|
||||||
|
if not self.is_valid_collection_name(collection_name):
|
||||||
|
raise ValueError('invalid collection name (must be of the form namespace.collection): {0}'.format(to_native(collection_name)))
|
||||||
|
|
||||||
|
if ref_type not in self.VALID_REF_TYPES:
|
||||||
|
raise ValueError('invalid collection ref_type: {0}'.format(ref_type))
|
||||||
|
|
||||||
|
self.collection = collection_name
|
||||||
|
if subdirs:
|
||||||
|
if not re.match(self.VALID_SUBDIRS_RE, subdirs):
|
||||||
|
raise ValueError('invalid subdirs entry: {0} (must be empty/None or of the form subdir1.subdir2)'.format(to_native(subdirs)))
|
||||||
|
self.subdirs = subdirs
|
||||||
|
else:
|
||||||
|
self.subdirs = u''
|
||||||
|
|
||||||
|
self.resource = resource
|
||||||
|
self.ref_type = ref_type
|
||||||
|
|
||||||
|
package_components = [u'ansible_collections', self.collection]
|
||||||
|
fqcr_components = [self.collection]
|
||||||
|
|
||||||
|
self.n_python_collection_package_name = to_native('.'.join(package_components))
|
||||||
|
|
||||||
|
if self.ref_type == u'role':
|
||||||
|
package_components.append(u'roles')
|
||||||
|
else:
|
||||||
|
# we assume it's a plugin
|
||||||
|
package_components += [u'plugins', self.ref_type]
|
||||||
|
|
||||||
|
if self.subdirs:
|
||||||
|
package_components.append(self.subdirs)
|
||||||
|
fqcr_components.append(self.subdirs)
|
||||||
|
|
||||||
|
if self.ref_type == u'role':
|
||||||
|
# roles are their own resource
|
||||||
|
package_components.append(self.resource)
|
||||||
|
|
||||||
|
fqcr_components.append(self.resource)
|
||||||
|
|
||||||
|
self.n_python_package_name = to_native('.'.join(package_components))
|
||||||
|
self._fqcr = u'.'.join(fqcr_components)
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return 'AnsibleCollectionRef(collection={0!r}, subdirs={1!r}, resource={2!r})'.format(self.collection, self.subdirs, self.resource)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def fqcr(self):
|
||||||
|
return self._fqcr
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def from_fqcr(ref, ref_type):
|
||||||
|
"""
|
||||||
|
Parse a string as a fully-qualified collection reference, raises ValueError if invalid
|
||||||
|
:param ref: collection reference to parse (a valid ref is of the form 'ns.coll.resource' or 'ns.coll.subdir1.subdir2.resource')
|
||||||
|
:param ref_type: the type of the reference, eg 'module', 'role', 'doc_fragment'
|
||||||
|
:return: a populated AnsibleCollectionRef object
|
||||||
|
"""
|
||||||
|
# assuming the fq_name is of the form (ns).(coll).(optional_subdir_N).(resource_name),
|
||||||
|
# we split the resource name off the right, split ns and coll off the left, and we're left with any optional
|
||||||
|
# subdirs that need to be added back below the plugin-specific subdir we'll add. So:
|
||||||
|
# ns.coll.resource -> ansible_collections.ns.coll.plugins.(plugintype).resource
|
||||||
|
# ns.coll.subdir1.resource -> ansible_collections.ns.coll.plugins.subdir1.(plugintype).resource
|
||||||
|
# ns.coll.rolename -> ansible_collections.ns.coll.roles.rolename
|
||||||
|
if not AnsibleCollectionRef.is_valid_fqcr(ref):
|
||||||
|
raise ValueError('{0} is not a valid collection reference'.format(to_native(ref)))
|
||||||
|
|
||||||
|
ref = to_text(ref, errors='strict')
|
||||||
|
ref_type = to_text(ref_type, errors='strict')
|
||||||
|
|
||||||
|
resource_splitname = ref.rsplit(u'.', 1)
|
||||||
|
package_remnant = resource_splitname[0]
|
||||||
|
resource = resource_splitname[1]
|
||||||
|
|
||||||
|
# split the left two components of the collection package name off, anything remaining is plugin-type
|
||||||
|
# specific subdirs to be added back on below the plugin type
|
||||||
|
package_splitname = package_remnant.split(u'.', 2)
|
||||||
|
if len(package_splitname) == 3:
|
||||||
|
subdirs = package_splitname[2]
|
||||||
|
else:
|
||||||
|
subdirs = u''
|
||||||
|
|
||||||
|
collection_name = u'.'.join(package_splitname[0:2])
|
||||||
|
|
||||||
|
return AnsibleCollectionRef(collection_name, subdirs, resource, ref_type)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def try_parse_fqcr(ref, ref_type):
|
||||||
|
"""
|
||||||
|
Attempt to parse a string as a fully-qualified collection reference, returning None on failure (instead of raising an error)
|
||||||
|
:param ref: collection reference to parse (a valid ref is of the form 'ns.coll.resource' or 'ns.coll.subdir1.subdir2.resource')
|
||||||
|
:param ref_type: the type of the reference, eg 'module', 'role', 'doc_fragment'
|
||||||
|
:return: a populated AnsibleCollectionRef object on successful parsing, else None
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return AnsibleCollectionRef.from_fqcr(ref, ref_type)
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def legacy_plugin_dir_to_plugin_type(legacy_plugin_dir_name):
|
||||||
|
"""
|
||||||
|
Utility method to convert from a PluginLoader dir name to a plugin ref_type
|
||||||
|
:param legacy_plugin_dir_name: PluginLoader dir name (eg, 'action_plugins', 'library')
|
||||||
|
:return: the corresponding plugin ref_type (eg, 'action', 'role')
|
||||||
|
"""
|
||||||
|
legacy_plugin_dir_name = to_text(legacy_plugin_dir_name)
|
||||||
|
|
||||||
|
plugin_type = legacy_plugin_dir_name.replace(u'_plugins', u'')
|
||||||
|
|
||||||
|
if plugin_type == u'library':
|
||||||
|
plugin_type = u'modules'
|
||||||
|
|
||||||
|
if plugin_type not in AnsibleCollectionRef.VALID_REF_TYPES:
|
||||||
|
raise ValueError('{0} cannot be mapped to a valid collection ref type'.format(to_native(legacy_plugin_dir_name)))
|
||||||
|
|
||||||
|
return plugin_type
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def is_valid_fqcr(ref, ref_type=None):
|
||||||
|
"""
|
||||||
|
Validates if is string is a well-formed fully-qualified collection reference (does not look up the collection itself)
|
||||||
|
:param ref: candidate collection reference to validate (a valid ref is of the form 'ns.coll.resource' or 'ns.coll.subdir1.subdir2.resource')
|
||||||
|
:param ref_type: optional reference type to enable deeper validation, eg 'module', 'role', 'doc_fragment'
|
||||||
|
:return: True if the collection ref passed is well-formed, False otherwise
|
||||||
|
"""
|
||||||
|
|
||||||
|
ref = to_text(ref)
|
||||||
|
|
||||||
|
if not ref_type:
|
||||||
|
return bool(re.match(AnsibleCollectionRef.VALID_FQCR_RE, ref))
|
||||||
|
|
||||||
|
return bool(AnsibleCollectionRef.try_parse_fqcr(ref, ref_type))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def is_valid_collection_name(collection_name):
|
||||||
|
"""
|
||||||
|
Validates if the given string is a well-formed collection name (does not look up the collection itself)
|
||||||
|
:param collection_name: candidate collection name to validate (a valid name is of the form 'ns.collname')
|
||||||
|
:return: True if the collection name passed is well-formed, False otherwise
|
||||||
|
"""
|
||||||
|
|
||||||
|
collection_name = to_text(collection_name)
|
||||||
|
|
||||||
|
return bool(re.match(AnsibleCollectionRef.VALID_COLLECTION_NAME_RE, collection_name))
|
||||||
|
|
||||||
|
|
||||||
|
def _get_collection_role_path(role_name, collection_list=None):
|
||||||
|
acr = AnsibleCollectionRef.try_parse_fqcr(role_name, 'role')
|
||||||
|
|
||||||
|
if acr:
|
||||||
|
# looks like a valid qualified collection ref; skip the collection_list
|
||||||
|
collection_list = [acr.collection]
|
||||||
|
subdirs = acr.subdirs
|
||||||
|
resource = acr.resource
|
||||||
|
elif not collection_list:
|
||||||
|
return None # not a FQ role and no collection search list spec'd, nothing to do
|
||||||
|
else:
|
||||||
|
resource = role_name # treat as unqualified, loop through the collection search list to try and resolve
|
||||||
|
subdirs = ''
|
||||||
|
|
||||||
|
for collection_name in collection_list:
|
||||||
|
try:
|
||||||
|
acr = AnsibleCollectionRef(collection_name=collection_name, subdirs=subdirs, resource=resource, ref_type='role')
|
||||||
|
# FIXME: error handling/logging; need to catch any import failures and move along
|
||||||
|
pkg = import_module(acr.n_python_package_name)
|
||||||
|
|
||||||
|
if pkg is not None:
|
||||||
|
# the package is now loaded, get the collection's package and ask where it lives
|
||||||
|
path = os.path.dirname(to_bytes(sys.modules[acr.n_python_package_name].__file__, errors='surrogate_or_strict'))
|
||||||
|
return resource, to_text(path, errors='surrogate_or_strict'), collection_name
|
||||||
|
|
||||||
|
except IOError:
|
||||||
|
continue
|
||||||
|
except Exception as ex:
|
||||||
|
# FIXME: pick out typical import errors first, then error logging
|
||||||
|
continue
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _get_collection_name_from_path(path):
|
||||||
|
"""
|
||||||
|
Return the containing collection name for a given path, or None if the path is not below a configured collection, or
|
||||||
|
the collection cannot be loaded (eg, the collection is masked by another of the same name higher in the configured
|
||||||
|
collection roots).
|
||||||
|
:param path: path to evaluate for collection containment
|
||||||
|
:return: collection name or None
|
||||||
|
"""
|
||||||
|
|
||||||
|
# FIXME: mess with realpath canonicalization or not?
|
||||||
|
path = to_native(path)
|
||||||
|
|
||||||
|
path_parts = path.split('/')
|
||||||
|
if path_parts.count('ansible_collections') != 1:
|
||||||
|
return None
|
||||||
|
|
||||||
|
ac_pos = path_parts.index('ansible_collections')
|
||||||
|
|
||||||
|
# make sure it's followed by at least a namespace and collection name
|
||||||
|
if len(path_parts) < ac_pos + 3:
|
||||||
|
return None
|
||||||
|
|
||||||
|
candidate_collection_name = '.'.join(path_parts[ac_pos + 1:ac_pos + 3])
|
||||||
|
|
||||||
|
try:
|
||||||
|
# we've got a name for it, now see if the path prefix matches what the loader sees
|
||||||
|
imported_pkg_path = to_native(os.path.dirname(to_bytes(import_module('ansible_collections.' + candidate_collection_name).__file__)))
|
||||||
|
except ImportError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# reassemble the original path prefix up the collection name, and it should match what we just imported. If not
|
||||||
|
# this is probably a collection root that's not configured.
|
||||||
|
|
||||||
|
original_path_prefix = os.path.join('/', *path_parts[0:ac_pos + 3])
|
||||||
|
|
||||||
|
if original_path_prefix != imported_pkg_path:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return candidate_collection_name
|
||||||
|
|
||||||
|
|
||||||
|
def _get_import_redirect(collection_meta_dict, fullname):
|
||||||
|
if not collection_meta_dict:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return _nested_dict_get(collection_meta_dict, ['import_redirection', fullname, 'redirect'])
|
||||||
|
|
||||||
|
|
||||||
|
def _get_ancestor_redirect(redirected_package_map, fullname):
|
||||||
|
# walk the requested module's ancestor packages to see if any have been previously redirected
|
||||||
|
cur_pkg = fullname
|
||||||
|
while cur_pkg:
|
||||||
|
cur_pkg = cur_pkg.rpartition('.')[0]
|
||||||
|
ancestor_redirect = redirected_package_map.get(cur_pkg)
|
||||||
|
if ancestor_redirect:
|
||||||
|
# rewrite the prefix on fullname so we import the target first, then alias it
|
||||||
|
redirect = ancestor_redirect + fullname[len(cur_pkg):]
|
||||||
|
return redirect
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _nested_dict_get(root_dict, key_list):
|
||||||
|
cur_value = root_dict
|
||||||
|
for key in key_list:
|
||||||
|
cur_value = cur_value.get(key)
|
||||||
|
if not cur_value:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return cur_value
|
||||||
|
|
||||||
|
|
||||||
|
def _iter_modules_impl(paths, prefix=''):
|
||||||
|
# NB: this currently only iterates what's on disk- redirected modules are not considered
|
||||||
|
if not prefix:
|
||||||
|
prefix = ''
|
||||||
|
else:
|
||||||
|
prefix = to_native(prefix)
|
||||||
|
# yield (module_loader, name, ispkg) for each module/pkg under path
|
||||||
|
# TODO: implement ignore/silent catch for unreadable?
|
||||||
|
for b_path in map(to_bytes, paths):
|
||||||
|
if not os.path.isdir(b_path):
|
||||||
|
continue
|
||||||
|
for b_basename in sorted(os.listdir(b_path)):
|
||||||
|
b_candidate_module_path = os.path.join(b_path, b_basename)
|
||||||
|
if os.path.isdir(b_candidate_module_path):
|
||||||
|
# exclude things that obviously aren't Python package dirs
|
||||||
|
# FIXME: this dir is adjustable in py3.8+, check for it
|
||||||
|
if b'.' in b_basename or b_basename == b'__pycache__':
|
||||||
|
continue
|
||||||
|
|
||||||
|
# TODO: proper string handling?
|
||||||
|
yield prefix + to_native(b_basename), True
|
||||||
|
else:
|
||||||
|
# FIXME: match builtin ordering for package/dir/file, support compiled?
|
||||||
|
if b_basename.endswith(b'.py') and b_basename != b'__init__.py':
|
||||||
|
yield prefix + to_native(os.path.splitext(b_basename)[0]), False
|
||||||
|
|
||||||
|
|
||||||
|
def _get_collection_metadata(collection_name):
|
||||||
|
collection_name = to_native(collection_name)
|
||||||
|
if not collection_name or not isinstance(collection_name, string_types) or len(collection_name.split('.')) != 2:
|
||||||
|
raise ValueError('collection_name must be a non-empty string of the form namespace.collection')
|
||||||
|
|
||||||
|
try:
|
||||||
|
collection_pkg = import_module('ansible_collections.' + collection_name)
|
||||||
|
except ImportError:
|
||||||
|
raise ValueError('unable to locate collection {0}'.format(collection_name))
|
||||||
|
|
||||||
|
_collection_meta = getattr(collection_pkg, '_collection_meta', None)
|
||||||
|
|
||||||
|
if _collection_meta is None:
|
||||||
|
raise ValueError('collection metadata was not loaded for collection {0}'.format(collection_name))
|
||||||
|
|
||||||
|
return _collection_meta
|
||||||
@ -0,0 +1,17 @@
|
|||||||
|
# (c) 2019 Ansible Project
|
||||||
|
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||||
|
|
||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
from yaml import safe_load
|
||||||
|
|
||||||
|
|
||||||
|
def _meta_yml_to_dict(yaml_string_data, content_id):
|
||||||
|
routing_dict = safe_load(yaml_string_data)
|
||||||
|
if not routing_dict:
|
||||||
|
routing_dict = {}
|
||||||
|
# TODO: change this to Mapping abc?
|
||||||
|
if not isinstance(routing_dict, dict):
|
||||||
|
raise ValueError('collection metadata must be a dictionary')
|
||||||
|
return routing_dict
|
||||||
@ -1 +1,2 @@
|
|||||||
shippable/posix/group1
|
shippable/posix/group1
|
||||||
|
disabled
|
||||||
|
|||||||
@ -0,0 +1,39 @@
|
|||||||
|
plugin_routing:
|
||||||
|
action:
|
||||||
|
uses_redirected_action:
|
||||||
|
redirect: testns.testcoll.subclassed_normal
|
||||||
|
connection:
|
||||||
|
redirected_local:
|
||||||
|
redirect: ansible.builtin.local
|
||||||
|
modules:
|
||||||
|
multilevel1:
|
||||||
|
redirect: testns.testcoll.multilevel2
|
||||||
|
multilevel2:
|
||||||
|
redirect: testns.testcoll.multilevel3
|
||||||
|
multilevel3:
|
||||||
|
redirect: testns.testcoll.ping
|
||||||
|
uses_redirected_action:
|
||||||
|
redirect: ansible.builtin.ping
|
||||||
|
setup.ps1: ansible.windows.setup
|
||||||
|
looped_ping:
|
||||||
|
redirect: testns.testcoll.looped_ping2
|
||||||
|
looped_ping2:
|
||||||
|
redirect: testns.testcoll.looped_ping
|
||||||
|
bogus_redirect:
|
||||||
|
redirect: bogus.collection.shouldbomb
|
||||||
|
deprecated_ping:
|
||||||
|
deprecation:
|
||||||
|
removal_date: 2020-12-31
|
||||||
|
warning_text: old_ping will be removed in a future release of this collection. Use new_ping instead.
|
||||||
|
foobar_facts:
|
||||||
|
redirect: foobar_info
|
||||||
|
aliased_ping:
|
||||||
|
redirect: ansible.builtin.ping
|
||||||
|
dead_ping:
|
||||||
|
tombstone:
|
||||||
|
removal_date: 2019-12-31
|
||||||
|
warning_text: dead_ping has been removed
|
||||||
|
module_utils:
|
||||||
|
moved_out_root:
|
||||||
|
redirect: testns.content_adj.sub1.foomodule
|
||||||
|
requires_ansible: '>=2.11'
|
||||||
@ -0,0 +1,11 @@
|
|||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
from ansible.plugins.action.normal import ActionModule as NormalAction
|
||||||
|
|
||||||
|
|
||||||
|
class ActionModule(NormalAction):
|
||||||
|
def run(self, *args, **kwargs):
|
||||||
|
result = super(ActionModule, self).run(*args, **kwargs)
|
||||||
|
result['hacked'] = 'I got run under a subclassed normal, yay'
|
||||||
|
return result
|
||||||
@ -0,0 +1,20 @@
|
|||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
from ansible.plugins.action import ActionBase
|
||||||
|
from ansible.module_utils.formerly_core import thingtocall
|
||||||
|
|
||||||
|
|
||||||
|
class ActionModule(ActionBase):
|
||||||
|
TRANSFERS_FILES = False
|
||||||
|
_VALID_ARGS = frozenset()
|
||||||
|
|
||||||
|
def run(self, tmp=None, task_vars=None):
|
||||||
|
if task_vars is None:
|
||||||
|
task_vars = dict()
|
||||||
|
|
||||||
|
result = super(ActionModule, self).run(None, task_vars)
|
||||||
|
|
||||||
|
result = dict(changed=False, ttc_res=thingtocall())
|
||||||
|
|
||||||
|
return result
|
||||||
@ -0,0 +1,6 @@
|
|||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
|
||||||
|
def nested_same():
|
||||||
|
return 'hello from nested_same'
|
||||||
@ -0,0 +1,11 @@
|
|||||||
|
# NB: this module should never be loaded, since we'll see the subpkg_with_init package dir first
|
||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
|
||||||
|
def thingtocall():
|
||||||
|
raise Exception('this should never be called (loaded discrete module instead of package module)')
|
||||||
|
|
||||||
|
|
||||||
|
def anotherthingtocall():
|
||||||
|
raise Exception('this should never be called (loaded discrete module instead of package module)')
|
||||||
@ -0,0 +1,13 @@
|
|||||||
|
#!/usr/bin/python
|
||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
print(json.dumps(dict(changed=False, source='user', is_deprecated=True)))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@ -0,0 +1,19 @@
|
|||||||
|
#!/usr/bin/python
|
||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from ansible_collections.testns.testcoll.plugins.module_utils.moved_out_root import importme
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
mu_result = importme()
|
||||||
|
print(json.dumps(dict(changed=False, source='user', mu_result=mu_result)))
|
||||||
|
|
||||||
|
sys.exit()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@ -0,0 +1,19 @@
|
|||||||
|
#!/usr/bin/python
|
||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from ansible.module_utils.formerly_core import thingtocall
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
mu_result = thingtocall()
|
||||||
|
print(json.dumps(dict(changed=False, source='user', mu_result=mu_result)))
|
||||||
|
|
||||||
|
sys.exit()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@ -0,0 +1,19 @@
|
|||||||
|
#!/usr/bin/python
|
||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from ansible_collections.testns.testcoll.plugins.module_utils.nested_same.nested_same.nested_same import nested_same
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
mu_result = nested_same()
|
||||||
|
print(json.dumps(dict(changed=False, source='user', mu_result=mu_result)))
|
||||||
|
|
||||||
|
sys.exit()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@ -0,0 +1,19 @@
|
|||||||
|
#!/usr/bin/python
|
||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from ansible_collections.testns.testcoll.plugins.module_utils.nested_same.nested_same import nested_same
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
mu_result = nested_same.nested_same()
|
||||||
|
print(json.dumps(dict(changed=False, source='user', mu_result=mu_result)))
|
||||||
|
|
||||||
|
sys.exit()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@ -0,0 +1,6 @@
|
|||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
|
||||||
|
def importme():
|
||||||
|
return "hello from {0}".format(__name__)
|
||||||
@ -0,0 +1,13 @@
|
|||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
|
||||||
|
def override_formerly_core_masked_filter(*args, **kwargs):
|
||||||
|
return 'hello from overridden formerly_core_masked_filter'
|
||||||
|
|
||||||
|
|
||||||
|
class FilterModule(object):
|
||||||
|
def filters(self):
|
||||||
|
return {
|
||||||
|
'formerly_core_masked_filter': override_formerly_core_masked_filter
|
||||||
|
}
|
||||||
@ -0,0 +1,26 @@
|
|||||||
|
- name: test a collection-hosted connection plugin against hosts from collection-hosted inventory plugins
|
||||||
|
hosts: dynamic_host_a, dynamic_host_redirected
|
||||||
|
gather_facts: no
|
||||||
|
vars:
|
||||||
|
ansible_connection: testns.testcoll.localconn
|
||||||
|
ansible_localconn_connectionvar: from_play
|
||||||
|
tasks:
|
||||||
|
- raw: echo 'hello world'
|
||||||
|
register: connection_out
|
||||||
|
|
||||||
|
- assert:
|
||||||
|
that:
|
||||||
|
- connection_out.stdout == "localconn ran echo 'hello world'"
|
||||||
|
# ensure that the connection var we overrode above made it into the running config
|
||||||
|
- connection_out.stderr == "connectionvar is from_play"
|
||||||
|
|
||||||
|
|
||||||
|
- hosts: localhost
|
||||||
|
gather_facts: no
|
||||||
|
tasks:
|
||||||
|
- assert:
|
||||||
|
that:
|
||||||
|
- hostvars['dynamic_host_a'] is defined
|
||||||
|
- hostvars['dynamic_host_a'].connection_out.stdout == "localconn ran echo 'hello world'"
|
||||||
|
- hostvars['dynamic_host_redirected'] is defined
|
||||||
|
- hostvars['dynamic_host_redirected'].connection_out.stdout == "localconn ran echo 'hello world'"
|
||||||
@ -0,0 +1,3 @@
|
|||||||
|
# use a plugin redirected by core to a collection to ensure inventory redirection and redirected config names are working
|
||||||
|
plugin: formerly_core_inventory # this is defined in the ansible-base runtime.yml routing to point at testns.content_adj.statichost
|
||||||
|
hostname: dynamic_host_redirected
|
||||||
@ -0,0 +1,46 @@
|
|||||||
|
- hosts: localhost
|
||||||
|
gather_facts: no
|
||||||
|
collections:
|
||||||
|
- testns.testcoll
|
||||||
|
vars:
|
||||||
|
# redirect connection
|
||||||
|
ansible_connection: testns.testcoll.redirected_local
|
||||||
|
tasks:
|
||||||
|
- assert:
|
||||||
|
that: ('data' | testns.testcoll.testfilter) == 'data_via_testfilter_from_userdir'
|
||||||
|
|
||||||
|
# redirect module (multiple levels)
|
||||||
|
- multilevel1:
|
||||||
|
# redirect action
|
||||||
|
- uses_redirected_action:
|
||||||
|
# redirect import (consumed via action)
|
||||||
|
- uses_redirected_import:
|
||||||
|
# redirect lookup
|
||||||
|
- assert:
|
||||||
|
that: lookup('formerly_core_lookup') == 'mylookup_from_user_dir'
|
||||||
|
# redirect filter
|
||||||
|
- assert:
|
||||||
|
that: ('yes' | formerly_core_filter) == True
|
||||||
|
# legacy filter should mask redirected
|
||||||
|
- assert:
|
||||||
|
that: ('' | formerly_core_masked_filter) == 'hello from overridden formerly_core_masked_filter'
|
||||||
|
# redirect test
|
||||||
|
- assert:
|
||||||
|
that:
|
||||||
|
- "'stuff' is formerly_core_test('tuf')"
|
||||||
|
- "'hello override' is formerly_core_masked_test"
|
||||||
|
# redirect module (formerly internal)
|
||||||
|
- formerly_core_ping:
|
||||||
|
# redirect module from collection (with subdir)
|
||||||
|
- testns.testcoll.module_subdir.subdir_ping_module:
|
||||||
|
# redirect module_utils plugin (consumed via module)
|
||||||
|
- uses_core_redirected_mu:
|
||||||
|
# deprecated module (issues warning)
|
||||||
|
- deprecated_ping:
|
||||||
|
# redirect module (internal alias)
|
||||||
|
- aliased_ping:
|
||||||
|
# redirect module (cycle detection, fatal)
|
||||||
|
# - looped_ping:
|
||||||
|
|
||||||
|
# removed module (fatal)
|
||||||
|
# - dead_ping:
|
||||||
@ -0,0 +1,16 @@
|
|||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
|
||||||
|
def override_formerly_core_masked_test(value, *args, **kwargs):
|
||||||
|
if value != 'hello override':
|
||||||
|
raise Exception('expected "hello override" only...')
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
class TestModule(object):
|
||||||
|
def tests(self):
|
||||||
|
return {
|
||||||
|
'formerly_core_masked_test': override_formerly_core_masked_test
|
||||||
|
}
|
||||||
@ -0,0 +1,16 @@
|
|||||||
|
"""Read YAML from stdin and write JSON to stdout."""
|
||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from yaml import load
|
||||||
|
|
||||||
|
try:
|
||||||
|
from yaml import CSafeLoader as SafeLoader
|
||||||
|
except ImportError:
|
||||||
|
from yaml import SafeLoader
|
||||||
|
|
||||||
|
|
||||||
|
json.dump(load(sys.stdin, Loader=SafeLoader), sys.stdout)
|
||||||
@ -0,0 +1,4 @@
|
|||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
raise Exception('this module should never be loaded')
|
||||||
@ -0,0 +1,4 @@
|
|||||||
|
plugin_routing:
|
||||||
|
modules:
|
||||||
|
rerouted_module:
|
||||||
|
redirect: ansible.builtin.ping
|
||||||
@ -0,0 +1,5 @@
|
|||||||
|
from ..module_utils.my_util import question
|
||||||
|
|
||||||
|
|
||||||
|
def action_code():
|
||||||
|
return "hello from my_action.py"
|
||||||
@ -0,0 +1,5 @@
|
|||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
|
||||||
|
raise Exception('this should never run')
|
||||||
@ -0,0 +1,6 @@
|
|||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
|
||||||
|
def module_code():
|
||||||
|
return "hello from amodule.py"
|
||||||
@ -0,0 +1,5 @@
|
|||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
|
||||||
|
raise Exception('this code should never execute')
|
||||||
@ -0,0 +1,5 @@
|
|||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
|
||||||
|
raise Exception('this code should never execute')
|
||||||
@ -0,0 +1,5 @@
|
|||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
|
||||||
|
raise Exception('this code should never execute')
|
||||||
@ -0,0 +1,5 @@
|
|||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
|
||||||
|
raise Exception('this code should never execute')
|
||||||
@ -0,0 +1,809 @@
|
|||||||
|
from __future__ import (absolute_import, division, print_function)
|
||||||
|
__metaclass__ = type
|
||||||
|
|
||||||
|
import os
|
||||||
|
import pkgutil
|
||||||
|
import pytest
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from ansible.module_utils.six import PY3, string_types
|
||||||
|
from ansible.module_utils.compat.importlib import import_module
|
||||||
|
from ansible.utils.collection_loader import AnsibleCollectionConfig, AnsibleCollectionRef
|
||||||
|
from ansible.utils.collection_loader._collection_finder import (
|
||||||
|
_AnsibleCollectionFinder, _AnsibleCollectionLoader, _AnsibleCollectionNSPkgLoader, _AnsibleCollectionPkgLoader,
|
||||||
|
_AnsibleCollectionPkgLoaderBase, _AnsibleCollectionRootPkgLoader, _AnsiblePathHookFinder,
|
||||||
|
_get_collection_name_from_path, _get_collection_role_path, _get_collection_metadata, _iter_modules_impl
|
||||||
|
)
|
||||||
|
from ansible.utils.collection_loader._collection_config import _EventSource
|
||||||
|
from units.compat.mock import MagicMock, NonCallableMagicMock, patch
|
||||||
|
|
||||||
|
|
||||||
|
# fixture to ensure we always clean up the import stuff when we're done
|
||||||
|
@pytest.fixture(autouse=True, scope='function')
|
||||||
|
def teardown(*args, **kwargs):
|
||||||
|
yield
|
||||||
|
reset_collections_loader_state()
|
||||||
|
|
||||||
|
# BEGIN STANDALONE TESTS - these exercise behaviors of the individual components without the import machinery
|
||||||
|
|
||||||
|
|
||||||
|
def test_finder_setup():
|
||||||
|
# ensure scalar path is listified
|
||||||
|
f = _AnsibleCollectionFinder(paths='/bogus/bogus')
|
||||||
|
assert isinstance(f._n_collection_paths, list)
|
||||||
|
|
||||||
|
# ensure sys.path paths that have an ansible_collections dir are added to the end of the collections paths
|
||||||
|
with patch.object(sys, 'path', ['/bogus', default_test_collection_paths[1], '/morebogus', default_test_collection_paths[0]]):
|
||||||
|
f = _AnsibleCollectionFinder(paths=['/explicit', '/other'])
|
||||||
|
assert f._n_collection_paths == ['/explicit', '/other', default_test_collection_paths[1], default_test_collection_paths[0]]
|
||||||
|
|
||||||
|
configured_paths = ['/bogus']
|
||||||
|
playbook_paths = ['/playbookdir']
|
||||||
|
f = _AnsibleCollectionFinder(paths=configured_paths)
|
||||||
|
assert f._n_collection_paths == configured_paths
|
||||||
|
f.set_playbook_paths(playbook_paths)
|
||||||
|
assert f._n_collection_paths == extend_paths(playbook_paths, 'collections') + configured_paths
|
||||||
|
|
||||||
|
# ensure scalar playbook_paths gets listified
|
||||||
|
f.set_playbook_paths(playbook_paths[0])
|
||||||
|
assert f._n_collection_paths == extend_paths(playbook_paths, 'collections') + configured_paths
|
||||||
|
|
||||||
|
|
||||||
|
def test_finder_not_interested():
|
||||||
|
f = get_default_finder()
|
||||||
|
assert f.find_module('nothanks') is None
|
||||||
|
assert f.find_module('nothanks.sub', path=['/bogus/dir']) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_finder_ns():
|
||||||
|
# ensure we can still load ansible_collections and ansible_collections.ansible when they don't exist on disk
|
||||||
|
f = _AnsibleCollectionFinder(paths=['/bogus/bogus'])
|
||||||
|
loader = f.find_module('ansible_collections')
|
||||||
|
assert isinstance(loader, _AnsibleCollectionRootPkgLoader)
|
||||||
|
|
||||||
|
loader = f.find_module('ansible_collections.ansible', path=['/bogus/bogus'])
|
||||||
|
assert isinstance(loader, _AnsibleCollectionNSPkgLoader)
|
||||||
|
|
||||||
|
f = get_default_finder()
|
||||||
|
loader = f.find_module('ansible_collections')
|
||||||
|
assert isinstance(loader, _AnsibleCollectionRootPkgLoader)
|
||||||
|
|
||||||
|
# path is not allowed for top-level
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
f.find_module('ansible_collections', path=['whatever'])
|
||||||
|
|
||||||
|
# path is required for subpackages
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
f.find_module('ansible_collections.whatever', path=None)
|
||||||
|
|
||||||
|
paths = [os.path.join(p, 'ansible_collections/nonexistns') for p in default_test_collection_paths]
|
||||||
|
|
||||||
|
# test missing
|
||||||
|
loader = f.find_module('ansible_collections.nonexistns', paths)
|
||||||
|
assert loader is None
|
||||||
|
|
||||||
|
|
||||||
|
# keep these up top to make sure the loader install/remove are working, since we rely on them heavily in the tests
|
||||||
|
def test_loader_remove():
|
||||||
|
fake_mp = [MagicMock(), _AnsibleCollectionFinder(), MagicMock(), _AnsibleCollectionFinder()]
|
||||||
|
fake_ph = [MagicMock().m1, MagicMock().m2, _AnsibleCollectionFinder()._ansible_collection_path_hook, NonCallableMagicMock]
|
||||||
|
# must nest until 2.6 compilation is totally donezo
|
||||||
|
with patch.object(sys, 'meta_path', fake_mp):
|
||||||
|
with patch.object(sys, 'path_hooks', fake_ph):
|
||||||
|
_AnsibleCollectionFinder()._remove()
|
||||||
|
assert len(sys.meta_path) == 2
|
||||||
|
# no AnsibleCollectionFinders on the meta path after remove is called
|
||||||
|
assert all((not isinstance(mpf, _AnsibleCollectionFinder) for mpf in sys.meta_path))
|
||||||
|
assert len(sys.path_hooks) == 3
|
||||||
|
# none of the remaining path hooks should point at an AnsibleCollectionFinder
|
||||||
|
assert all((not isinstance(ph.__self__, _AnsibleCollectionFinder) for ph in sys.path_hooks if hasattr(ph, '__self__')))
|
||||||
|
assert AnsibleCollectionConfig.collection_finder is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_loader_install():
|
||||||
|
fake_mp = [MagicMock(), _AnsibleCollectionFinder(), MagicMock(), _AnsibleCollectionFinder()]
|
||||||
|
fake_ph = [MagicMock().m1, MagicMock().m2, _AnsibleCollectionFinder()._ansible_collection_path_hook, NonCallableMagicMock]
|
||||||
|
# must nest until 2.6 compilation is totally donezo
|
||||||
|
with patch.object(sys, 'meta_path', fake_mp):
|
||||||
|
with patch.object(sys, 'path_hooks', fake_ph):
|
||||||
|
f = _AnsibleCollectionFinder()
|
||||||
|
f._install()
|
||||||
|
assert len(sys.meta_path) == 3 # should have removed the existing ACFs and installed a new one
|
||||||
|
assert sys.meta_path[0] is f # at the front
|
||||||
|
# the rest of the meta_path should not be AnsibleCollectionFinders
|
||||||
|
assert all((not isinstance(mpf, _AnsibleCollectionFinder) for mpf in sys.meta_path[1:]))
|
||||||
|
assert len(sys.path_hooks) == 4 # should have removed the existing ACF path hooks and installed a new one
|
||||||
|
# the first path hook should be ours, make sure it's pointing at the right instance
|
||||||
|
assert hasattr(sys.path_hooks[0], '__self__') and sys.path_hooks[0].__self__ is f
|
||||||
|
# the rest of the path_hooks should not point at an AnsibleCollectionFinder
|
||||||
|
assert all((not isinstance(ph.__self__, _AnsibleCollectionFinder) for ph in sys.path_hooks[1:] if hasattr(ph, '__self__')))
|
||||||
|
assert AnsibleCollectionConfig.collection_finder is f
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
AnsibleCollectionConfig.collection_finder = f
|
||||||
|
|
||||||
|
|
||||||
|
def test_finder_coll():
|
||||||
|
f = get_default_finder()
|
||||||
|
|
||||||
|
tests = [
|
||||||
|
{'name': 'ansible_collections.testns.testcoll', 'test_paths': [default_test_collection_paths]},
|
||||||
|
{'name': 'ansible_collections.ansible.builtin', 'test_paths': [['/bogus'], default_test_collection_paths]},
|
||||||
|
]
|
||||||
|
# ensure finder works for legit paths and bogus paths
|
||||||
|
for test_dict in tests:
|
||||||
|
# splat the dict values to our locals
|
||||||
|
globals().update(test_dict)
|
||||||
|
parent_pkg = name.rpartition('.')[0]
|
||||||
|
for paths in test_paths:
|
||||||
|
paths = [os.path.join(p, parent_pkg.replace('.', '/')) for p in paths]
|
||||||
|
loader = f.find_module(name, path=paths)
|
||||||
|
assert isinstance(loader, _AnsibleCollectionPkgLoader)
|
||||||
|
|
||||||
|
|
||||||
|
def test_root_loader_not_interested():
|
||||||
|
with pytest.raises(ImportError):
|
||||||
|
_AnsibleCollectionRootPkgLoader('not_ansible_collections_toplevel', path_list=[])
|
||||||
|
|
||||||
|
with pytest.raises(ImportError):
|
||||||
|
_AnsibleCollectionRootPkgLoader('ansible_collections.somens', path_list=['/bogus'])
|
||||||
|
|
||||||
|
|
||||||
|
def test_root_loader():
|
||||||
|
name = 'ansible_collections'
|
||||||
|
# ensure this works even when ansible_collections doesn't exist on disk
|
||||||
|
for paths in [], default_test_collection_paths:
|
||||||
|
if name in sys.modules:
|
||||||
|
del sys.modules[name]
|
||||||
|
loader = _AnsibleCollectionRootPkgLoader(name, paths)
|
||||||
|
assert repr(loader).startswith('_AnsibleCollectionRootPkgLoader(path=')
|
||||||
|
module = loader.load_module(name)
|
||||||
|
assert module.__name__ == name
|
||||||
|
assert module.__path__ == [p for p in extend_paths(paths, name) if os.path.isdir(p)]
|
||||||
|
# even if the dir exists somewhere, this loader doesn't support get_data, so make __file__ a non-file
|
||||||
|
assert module.__file__ == '<ansible_synthetic_collection_package>'
|
||||||
|
assert module.__package__ == name
|
||||||
|
assert sys.modules.get(name) == module
|
||||||
|
|
||||||
|
|
||||||
|
def test_nspkg_loader_not_interested():
|
||||||
|
with pytest.raises(ImportError):
|
||||||
|
_AnsibleCollectionNSPkgLoader('not_ansible_collections_toplevel.something', path_list=[])
|
||||||
|
|
||||||
|
with pytest.raises(ImportError):
|
||||||
|
_AnsibleCollectionNSPkgLoader('ansible_collections.somens.somecoll', path_list=[])
|
||||||
|
|
||||||
|
|
||||||
|
def test_nspkg_loader_load_module():
|
||||||
|
# ensure the loader behaves on the toplevel and ansible packages for both legit and missing/bogus paths
|
||||||
|
for name in ['ansible_collections.ansible', 'ansible_collections.testns']:
|
||||||
|
parent_pkg = name.partition('.')[0]
|
||||||
|
module_to_load = name.rpartition('.')[2]
|
||||||
|
paths = extend_paths(default_test_collection_paths, parent_pkg)
|
||||||
|
existing_child_paths = [p for p in extend_paths(paths, module_to_load) if os.path.exists(p)]
|
||||||
|
if name in sys.modules:
|
||||||
|
del sys.modules[name]
|
||||||
|
loader = _AnsibleCollectionNSPkgLoader(name, path_list=paths)
|
||||||
|
assert repr(loader).startswith('_AnsibleCollectionNSPkgLoader(path=')
|
||||||
|
module = loader.load_module(name)
|
||||||
|
assert module.__name__ == name
|
||||||
|
assert isinstance(module.__loader__, _AnsibleCollectionNSPkgLoader)
|
||||||
|
assert module.__path__ == existing_child_paths
|
||||||
|
assert module.__package__ == name
|
||||||
|
assert module.__file__ == '<ansible_synthetic_collection_package>'
|
||||||
|
assert sys.modules.get(name) == module
|
||||||
|
|
||||||
|
|
||||||
|
def test_collpkg_loader_not_interested():
|
||||||
|
with pytest.raises(ImportError):
|
||||||
|
_AnsibleCollectionPkgLoader('not_ansible_collections', path_list=[])
|
||||||
|
|
||||||
|
with pytest.raises(ImportError):
|
||||||
|
_AnsibleCollectionPkgLoader('ansible_collections.ns', path_list=['/bogus/bogus'])
|
||||||
|
|
||||||
|
|
||||||
|
def test_collpkg_loader_load_module():
|
||||||
|
reset_collections_loader_state()
|
||||||
|
with patch('ansible.utils.collection_loader.AnsibleCollectionConfig') as p:
|
||||||
|
for name in ['ansible_collections.ansible.builtin', 'ansible_collections.testns.testcoll']:
|
||||||
|
parent_pkg = name.rpartition('.')[0]
|
||||||
|
module_to_load = name.rpartition('.')[2]
|
||||||
|
paths = extend_paths(default_test_collection_paths, parent_pkg)
|
||||||
|
existing_child_paths = [p for p in extend_paths(paths, module_to_load) if os.path.exists(p)]
|
||||||
|
is_builtin = 'ansible.builtin' in name
|
||||||
|
if name in sys.modules:
|
||||||
|
del sys.modules[name]
|
||||||
|
loader = _AnsibleCollectionPkgLoader(name, path_list=paths)
|
||||||
|
assert repr(loader).startswith('_AnsibleCollectionPkgLoader(path=')
|
||||||
|
module = loader.load_module(name)
|
||||||
|
assert module.__name__ == name
|
||||||
|
assert isinstance(module.__loader__, _AnsibleCollectionPkgLoader)
|
||||||
|
if is_builtin:
|
||||||
|
assert module.__path__ == []
|
||||||
|
else:
|
||||||
|
assert module.__path__ == [existing_child_paths[0]]
|
||||||
|
|
||||||
|
assert module.__package__ == name
|
||||||
|
if is_builtin:
|
||||||
|
assert module.__file__ == '<ansible_synthetic_collection_package>'
|
||||||
|
else:
|
||||||
|
assert module.__file__.endswith('__synthetic__') and os.path.isdir(os.path.dirname(module.__file__))
|
||||||
|
assert sys.modules.get(name) == module
|
||||||
|
|
||||||
|
assert hasattr(module, '_collection_meta') and isinstance(module._collection_meta, dict)
|
||||||
|
|
||||||
|
# FIXME: validate _collection_meta contents match what's on disk (or not)
|
||||||
|
|
||||||
|
# if the module has metadata, try loading it with busted metadata
|
||||||
|
if module._collection_meta:
|
||||||
|
_collection_finder = import_module('ansible.utils.collection_loader._collection_finder')
|
||||||
|
with patch.object(_collection_finder, '_meta_yml_to_dict', side_effect=Exception('bang')):
|
||||||
|
with pytest.raises(Exception) as ex:
|
||||||
|
_AnsibleCollectionPkgLoader(name, path_list=paths).load_module(name)
|
||||||
|
assert 'error parsing collection metadata' in str(ex.value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_coll_loader():
|
||||||
|
with patch('ansible.utils.collection_loader.AnsibleCollectionConfig'):
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
# not a collection
|
||||||
|
_AnsibleCollectionLoader('ansible_collections')
|
||||||
|
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
# bogus paths
|
||||||
|
_AnsibleCollectionLoader('ansible_collections.testns.testcoll', path_list=[])
|
||||||
|
|
||||||
|
# FIXME: more
|
||||||
|
|
||||||
|
|
||||||
|
def test_path_hook_setup():
|
||||||
|
with patch.object(sys, 'path_hooks', []):
|
||||||
|
found_hook = None
|
||||||
|
pathhook_exc = None
|
||||||
|
try:
|
||||||
|
found_hook = _AnsiblePathHookFinder._get_filefinder_path_hook()
|
||||||
|
except Exception as phe:
|
||||||
|
pathhook_exc = phe
|
||||||
|
|
||||||
|
if PY3:
|
||||||
|
assert str(pathhook_exc) == 'need exactly one FileFinder import hook (found 0)'
|
||||||
|
else:
|
||||||
|
assert found_hook is None
|
||||||
|
|
||||||
|
assert repr(_AnsiblePathHookFinder(object(), '/bogus/path')) == "_AnsiblePathHookFinder(path='/bogus/path')"
|
||||||
|
|
||||||
|
|
||||||
|
def test_new_or_existing_module():
|
||||||
|
module_name = 'blar.test.module'
|
||||||
|
pkg_name = module_name.rpartition('.')[0]
|
||||||
|
|
||||||
|
# create new module case
|
||||||
|
nuke_module_prefix(module_name)
|
||||||
|
with _AnsibleCollectionPkgLoaderBase._new_or_existing_module(module_name, __package__=pkg_name) as new_module:
|
||||||
|
# the module we just created should now exist in sys.modules
|
||||||
|
assert sys.modules.get(module_name) is new_module
|
||||||
|
assert new_module.__name__ == module_name
|
||||||
|
|
||||||
|
# the module should stick since we didn't raise an exception in the contextmgr
|
||||||
|
assert sys.modules.get(module_name) is new_module
|
||||||
|
|
||||||
|
# reuse existing module case
|
||||||
|
with _AnsibleCollectionPkgLoaderBase._new_or_existing_module(module_name, __attr1__=42, blar='yo') as existing_module:
|
||||||
|
assert sys.modules.get(module_name) is new_module # should be the same module we created earlier
|
||||||
|
assert hasattr(existing_module, '__package__') and existing_module.__package__ == pkg_name
|
||||||
|
assert hasattr(existing_module, '__attr1__') and existing_module.__attr1__ == 42
|
||||||
|
assert hasattr(existing_module, 'blar') and existing_module.blar == 'yo'
|
||||||
|
|
||||||
|
# exception during update existing shouldn't zap existing module from sys.modules
|
||||||
|
with pytest.raises(ValueError) as ve:
|
||||||
|
with _AnsibleCollectionPkgLoaderBase._new_or_existing_module(module_name) as existing_module:
|
||||||
|
err_to_raise = ValueError('bang')
|
||||||
|
raise err_to_raise
|
||||||
|
# make sure we got our error
|
||||||
|
assert ve.value is err_to_raise
|
||||||
|
# and that the module still exists
|
||||||
|
assert sys.modules.get(module_name) is existing_module
|
||||||
|
|
||||||
|
# test module removal after exception during creation
|
||||||
|
nuke_module_prefix(module_name)
|
||||||
|
with pytest.raises(ValueError) as ve:
|
||||||
|
with _AnsibleCollectionPkgLoaderBase._new_or_existing_module(module_name) as new_module:
|
||||||
|
err_to_raise = ValueError('bang')
|
||||||
|
raise err_to_raise
|
||||||
|
# make sure we got our error
|
||||||
|
assert ve.value is err_to_raise
|
||||||
|
# and that the module was removed
|
||||||
|
assert sys.modules.get(module_name) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_iter_modules_impl():
|
||||||
|
modules_trailer = 'ansible_collections/testns/testcoll/plugins'
|
||||||
|
modules_pkg_prefix = modules_trailer.replace('/', '.') + '.'
|
||||||
|
modules_path = os.path.join(default_test_collection_paths[0], modules_trailer)
|
||||||
|
modules = list(_iter_modules_impl([modules_path], modules_pkg_prefix))
|
||||||
|
|
||||||
|
assert modules
|
||||||
|
assert set([('ansible_collections.testns.testcoll.plugins.action', True),
|
||||||
|
('ansible_collections.testns.testcoll.plugins.module_utils', True),
|
||||||
|
('ansible_collections.testns.testcoll.plugins.modules', True)]) == set(modules)
|
||||||
|
|
||||||
|
modules_trailer = 'ansible_collections/testns/testcoll/plugins/modules'
|
||||||
|
modules_pkg_prefix = modules_trailer.replace('/', '.') + '.'
|
||||||
|
modules_path = os.path.join(default_test_collection_paths[0], modules_trailer)
|
||||||
|
modules = list(_iter_modules_impl([modules_path], modules_pkg_prefix))
|
||||||
|
|
||||||
|
assert modules
|
||||||
|
assert len(modules) == 1
|
||||||
|
assert modules[0][0] == 'ansible_collections.testns.testcoll.plugins.modules.amodule' # name
|
||||||
|
assert modules[0][1] is False # is_pkg
|
||||||
|
|
||||||
|
# FIXME: more
|
||||||
|
|
||||||
|
|
||||||
|
# BEGIN IN-CIRCUIT TESTS - these exercise behaviors of the loader when wired up to the import machinery
|
||||||
|
|
||||||
|
|
||||||
|
def test_import_from_collection(monkeypatch):
|
||||||
|
collection_root = os.path.join(os.path.dirname(__file__), 'fixtures', 'collections')
|
||||||
|
collection_path = os.path.join(collection_root, 'ansible_collections/testns/testcoll/plugins/module_utils/my_util.py')
|
||||||
|
|
||||||
|
# THIS IS UNSTABLE UNDER A DEBUGGER
|
||||||
|
# the trace we're expecting to be generated when running the code below:
|
||||||
|
# answer = question()
|
||||||
|
expected_trace_log = [
|
||||||
|
(collection_path, 5, 'call'),
|
||||||
|
(collection_path, 6, 'line'),
|
||||||
|
(collection_path, 6, 'return'),
|
||||||
|
]
|
||||||
|
|
||||||
|
# define the collection root before any ansible code has been loaded
|
||||||
|
# otherwise config will have already been loaded and changing the environment will have no effect
|
||||||
|
monkeypatch.setenv('ANSIBLE_COLLECTIONS_PATHS', collection_root)
|
||||||
|
|
||||||
|
finder = _AnsibleCollectionFinder(paths=[collection_root])
|
||||||
|
reset_collections_loader_state(finder)
|
||||||
|
|
||||||
|
from ansible_collections.testns.testcoll.plugins.module_utils.my_util import question
|
||||||
|
|
||||||
|
original_trace_function = sys.gettrace()
|
||||||
|
trace_log = []
|
||||||
|
|
||||||
|
if original_trace_function:
|
||||||
|
# enable tracing while preserving the existing trace function (coverage)
|
||||||
|
def my_trace_function(frame, event, arg):
|
||||||
|
trace_log.append((frame.f_code.co_filename, frame.f_lineno, event))
|
||||||
|
|
||||||
|
# the original trace function expects to have itself set as the trace function
|
||||||
|
sys.settrace(original_trace_function)
|
||||||
|
# call the original trace function
|
||||||
|
original_trace_function(frame, event, arg)
|
||||||
|
# restore our trace function
|
||||||
|
sys.settrace(my_trace_function)
|
||||||
|
|
||||||
|
return my_trace_function
|
||||||
|
else:
|
||||||
|
# no existing trace function, so our trace function is much simpler
|
||||||
|
def my_trace_function(frame, event, arg):
|
||||||
|
trace_log.append((frame.f_code.co_filename, frame.f_lineno, event))
|
||||||
|
|
||||||
|
return my_trace_function
|
||||||
|
|
||||||
|
sys.settrace(my_trace_function)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# run a minimal amount of code while the trace is running
|
||||||
|
# adding more code here, including use of a context manager, will add more to our trace
|
||||||
|
answer = question()
|
||||||
|
finally:
|
||||||
|
sys.settrace(original_trace_function)
|
||||||
|
|
||||||
|
# make sure 'import ... as ...' works on builtin synthetic collections
|
||||||
|
# the following import is not supported (it tries to find module_utils in ansible.plugins)
|
||||||
|
# import ansible_collections.ansible.builtin.plugins.module_utils as c1
|
||||||
|
import ansible_collections.ansible.builtin.plugins.action as c2
|
||||||
|
import ansible_collections.ansible.builtin.plugins as c3
|
||||||
|
import ansible_collections.ansible.builtin as c4
|
||||||
|
import ansible_collections.ansible as c5
|
||||||
|
import ansible_collections as c6
|
||||||
|
|
||||||
|
# make sure 'import ...' works on builtin synthetic collections
|
||||||
|
import ansible_collections.ansible.builtin.plugins.module_utils
|
||||||
|
|
||||||
|
import ansible_collections.ansible.builtin.plugins.action
|
||||||
|
assert ansible_collections.ansible.builtin.plugins.action == c3.action == c2
|
||||||
|
|
||||||
|
import ansible_collections.ansible.builtin.plugins
|
||||||
|
assert ansible_collections.ansible.builtin.plugins == c4.plugins == c3
|
||||||
|
|
||||||
|
import ansible_collections.ansible.builtin
|
||||||
|
assert ansible_collections.ansible.builtin == c5.builtin == c4
|
||||||
|
|
||||||
|
import ansible_collections.ansible
|
||||||
|
assert ansible_collections.ansible == c6.ansible == c5
|
||||||
|
|
||||||
|
import ansible_collections
|
||||||
|
assert ansible_collections == c6
|
||||||
|
|
||||||
|
# make sure 'from ... import ...' works on builtin synthetic collections
|
||||||
|
from ansible_collections.ansible import builtin
|
||||||
|
from ansible_collections.ansible.builtin import plugins
|
||||||
|
assert builtin.plugins == plugins
|
||||||
|
|
||||||
|
from ansible_collections.ansible.builtin.plugins import action
|
||||||
|
from ansible_collections.ansible.builtin.plugins.action import command
|
||||||
|
assert action.command == command
|
||||||
|
|
||||||
|
from ansible_collections.ansible.builtin.plugins.module_utils import basic
|
||||||
|
from ansible_collections.ansible.builtin.plugins.module_utils.basic import AnsibleModule
|
||||||
|
assert basic.AnsibleModule == AnsibleModule
|
||||||
|
|
||||||
|
# make sure relative imports work from collections code
|
||||||
|
# these require __package__ to be set correctly
|
||||||
|
import ansible_collections.testns.testcoll.plugins.module_utils.my_other_util
|
||||||
|
import ansible_collections.testns.testcoll.plugins.action.my_action
|
||||||
|
|
||||||
|
# verify that code loaded from a collection does not inherit __future__ statements from the collection loader
|
||||||
|
if sys.version_info[0] == 2:
|
||||||
|
# if the collection code inherits the division future feature from the collection loader this will fail
|
||||||
|
assert answer == 1
|
||||||
|
else:
|
||||||
|
assert answer == 1.5
|
||||||
|
|
||||||
|
# verify that the filename and line number reported by the trace is correct
|
||||||
|
# this makes sure that collection loading preserves file paths and line numbers
|
||||||
|
assert trace_log == expected_trace_log
|
||||||
|
|
||||||
|
|
||||||
|
def test_eventsource():
|
||||||
|
es = _EventSource()
|
||||||
|
# fire when empty should succeed
|
||||||
|
es.fire(42)
|
||||||
|
handler1 = MagicMock()
|
||||||
|
handler2 = MagicMock()
|
||||||
|
es += handler1
|
||||||
|
es.fire(99, my_kwarg='blah')
|
||||||
|
handler1.assert_called_with(99, my_kwarg='blah')
|
||||||
|
es += handler2
|
||||||
|
es.fire(123, foo='bar')
|
||||||
|
handler1.assert_called_with(123, foo='bar')
|
||||||
|
handler2.assert_called_with(123, foo='bar')
|
||||||
|
es -= handler2
|
||||||
|
handler1.reset_mock()
|
||||||
|
handler2.reset_mock()
|
||||||
|
es.fire(123, foo='bar')
|
||||||
|
handler1.assert_called_with(123, foo='bar')
|
||||||
|
handler2.assert_not_called()
|
||||||
|
es -= handler1
|
||||||
|
handler1.reset_mock()
|
||||||
|
es.fire('blah', kwarg=None)
|
||||||
|
handler1.assert_not_called()
|
||||||
|
handler2.assert_not_called()
|
||||||
|
es -= handler1 # should succeed silently
|
||||||
|
handler_bang = MagicMock(side_effect=Exception('bang'))
|
||||||
|
es += handler_bang
|
||||||
|
with pytest.raises(Exception) as ex:
|
||||||
|
es.fire(123)
|
||||||
|
assert 'bang' in str(ex.value)
|
||||||
|
handler_bang.assert_called_with(123)
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
es += 42
|
||||||
|
|
||||||
|
|
||||||
|
def test_on_collection_load():
|
||||||
|
finder = get_default_finder()
|
||||||
|
reset_collections_loader_state(finder)
|
||||||
|
|
||||||
|
load_handler = MagicMock()
|
||||||
|
AnsibleCollectionConfig.on_collection_load += load_handler
|
||||||
|
|
||||||
|
m = import_module('ansible_collections.testns.testcoll')
|
||||||
|
load_handler.assert_called_once_with(collection_name='testns.testcoll', collection_path=os.path.dirname(m.__file__))
|
||||||
|
|
||||||
|
_meta = _get_collection_metadata('testns.testcoll')
|
||||||
|
assert _meta
|
||||||
|
# FIXME: compare to disk
|
||||||
|
|
||||||
|
finder = get_default_finder()
|
||||||
|
reset_collections_loader_state(finder)
|
||||||
|
|
||||||
|
AnsibleCollectionConfig.on_collection_load += MagicMock(side_effect=Exception('bang'))
|
||||||
|
with pytest.raises(Exception) as ex:
|
||||||
|
import_module('ansible_collections.testns.testcoll')
|
||||||
|
assert 'bang' in str(ex.value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_collection_config():
|
||||||
|
finder = get_default_finder()
|
||||||
|
reset_collections_loader_state(finder)
|
||||||
|
assert AnsibleCollectionConfig.default_collection is None
|
||||||
|
AnsibleCollectionConfig.default_collection = 'foo.bar'
|
||||||
|
assert AnsibleCollectionConfig.default_collection == 'foo.bar'
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
AnsibleCollectionConfig.default_collection = 'bar.baz'
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_collection_detection():
|
||||||
|
finder = get_default_finder()
|
||||||
|
reset_collections_loader_state(finder)
|
||||||
|
|
||||||
|
# we're clearly not under a collection path
|
||||||
|
assert _get_collection_name_from_path('/') is None
|
||||||
|
|
||||||
|
# something that looks like a collection path but isn't importable by our finder
|
||||||
|
assert _get_collection_name_from_path('/foo/ansible_collections/bogusns/boguscoll/bar') is None
|
||||||
|
|
||||||
|
# legit, at the top of the collection
|
||||||
|
live_collection_path = os.path.join(os.path.dirname(__file__), 'fixtures/collections/ansible_collections/testns/testcoll')
|
||||||
|
assert _get_collection_name_from_path(live_collection_path) == 'testns.testcoll'
|
||||||
|
|
||||||
|
# legit, deeper inside the collection
|
||||||
|
live_collection_deep_path = os.path.join(live_collection_path, 'plugins/modules')
|
||||||
|
assert _get_collection_name_from_path(live_collection_deep_path) == 'testns.testcoll'
|
||||||
|
|
||||||
|
# this one should be hidden by the real testns.testcoll, so should not resolve
|
||||||
|
masked_collection_path = os.path.join(os.path.dirname(__file__), 'fixtures/collections_masked/ansible_collections/testns/testcoll')
|
||||||
|
assert _get_collection_name_from_path(masked_collection_path) is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'role_name,collection_list,expected_collection_name,expected_path_suffix',
|
||||||
|
[
|
||||||
|
('some_role', ['testns.testcoll', 'ansible.bogus'], 'testns.testcoll', 'testns/testcoll/roles/some_role'),
|
||||||
|
('testns.testcoll.some_role', ['ansible.bogus', 'testns.testcoll'], 'testns.testcoll', 'testns/testcoll/roles/some_role'),
|
||||||
|
('testns.testcoll.some_role', [], 'testns.testcoll', 'testns/testcoll/roles/some_role'),
|
||||||
|
('testns.testcoll.some_role', None, 'testns.testcoll', 'testns/testcoll/roles/some_role'),
|
||||||
|
('some_role', [], None, None),
|
||||||
|
('some_role', None, None, None),
|
||||||
|
])
|
||||||
|
def test_collection_role_name_location(role_name, collection_list, expected_collection_name, expected_path_suffix):
|
||||||
|
finder = get_default_finder()
|
||||||
|
reset_collections_loader_state(finder)
|
||||||
|
|
||||||
|
expected_path = None
|
||||||
|
if expected_path_suffix:
|
||||||
|
expected_path = os.path.join(os.path.dirname(__file__), 'fixtures/collections/ansible_collections', expected_path_suffix)
|
||||||
|
|
||||||
|
found = _get_collection_role_path(role_name, collection_list)
|
||||||
|
|
||||||
|
if found:
|
||||||
|
assert found[0] == role_name.rpartition('.')[2]
|
||||||
|
assert found[1] == expected_path
|
||||||
|
assert found[2] == expected_collection_name
|
||||||
|
else:
|
||||||
|
assert expected_collection_name is None and expected_path_suffix is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_bogus_imports():
|
||||||
|
finder = get_default_finder()
|
||||||
|
reset_collections_loader_state(finder)
|
||||||
|
|
||||||
|
# ensure ImportError on known-bogus imports
|
||||||
|
bogus_imports = ['bogus_toplevel', 'ansible_collections.bogusns', 'ansible_collections.testns.boguscoll',
|
||||||
|
'ansible_collections.testns.testcoll.bogussub', 'ansible_collections.ansible.builtin.bogussub']
|
||||||
|
for bogus_import in bogus_imports:
|
||||||
|
with pytest.raises(ImportError):
|
||||||
|
import_module(bogus_import)
|
||||||
|
|
||||||
|
|
||||||
|
def test_finder_playbook_paths():
|
||||||
|
finder = get_default_finder()
|
||||||
|
reset_collections_loader_state(finder)
|
||||||
|
|
||||||
|
import ansible_collections
|
||||||
|
import ansible_collections.ansible
|
||||||
|
import ansible_collections.testns
|
||||||
|
|
||||||
|
# ensure the package modules look like we expect
|
||||||
|
assert hasattr(ansible_collections, '__path__') and len(ansible_collections.__path__) > 0
|
||||||
|
assert hasattr(ansible_collections.ansible, '__path__') and len(ansible_collections.ansible.__path__) > 0
|
||||||
|
assert hasattr(ansible_collections.testns, '__path__') and len(ansible_collections.testns.__path__) > 0
|
||||||
|
|
||||||
|
# these shouldn't be visible yet, since we haven't added the playbook dir
|
||||||
|
with pytest.raises(ImportError):
|
||||||
|
import ansible_collections.ansible.playbook_adj_other
|
||||||
|
|
||||||
|
with pytest.raises(ImportError):
|
||||||
|
import ansible_collections.testns.playbook_adj_other
|
||||||
|
|
||||||
|
assert AnsibleCollectionConfig.playbook_paths == []
|
||||||
|
playbook_path_fixture_dir = os.path.join(os.path.dirname(__file__), 'fixtures/playbook_path')
|
||||||
|
|
||||||
|
# configure the playbook paths
|
||||||
|
AnsibleCollectionConfig.playbook_paths = [playbook_path_fixture_dir]
|
||||||
|
|
||||||
|
# playbook paths go to the front of the line
|
||||||
|
assert AnsibleCollectionConfig.collection_paths[0] == os.path.join(playbook_path_fixture_dir, 'collections')
|
||||||
|
|
||||||
|
# playbook paths should be updated on the existing root ansible_collections path, as well as on the 'ansible' namespace (but no others!)
|
||||||
|
assert ansible_collections.__path__[0] == os.path.join(playbook_path_fixture_dir, 'collections/ansible_collections')
|
||||||
|
assert ansible_collections.ansible.__path__[0] == os.path.join(playbook_path_fixture_dir, 'collections/ansible_collections/ansible')
|
||||||
|
assert all('playbook_path' not in p for p in ansible_collections.testns.__path__)
|
||||||
|
|
||||||
|
# should succeed since we fixed up the package path
|
||||||
|
import ansible_collections.ansible.playbook_adj_other
|
||||||
|
# should succeed since we didn't import freshns before hacking in the path
|
||||||
|
import ansible_collections.freshns.playbook_adj_other
|
||||||
|
# should fail since we've already imported something from this path and didn't fix up its package path
|
||||||
|
with pytest.raises(ImportError):
|
||||||
|
import ansible_collections.testns.playbook_adj_other
|
||||||
|
|
||||||
|
|
||||||
|
def test_toplevel_iter_modules():
|
||||||
|
finder = get_default_finder()
|
||||||
|
reset_collections_loader_state(finder)
|
||||||
|
|
||||||
|
modules = list(pkgutil.iter_modules(default_test_collection_paths, ''))
|
||||||
|
assert len(modules) == 1
|
||||||
|
assert modules[0][1] == 'ansible_collections'
|
||||||
|
|
||||||
|
|
||||||
|
def test_iter_modules_namespaces():
|
||||||
|
finder = get_default_finder()
|
||||||
|
reset_collections_loader_state(finder)
|
||||||
|
|
||||||
|
paths = extend_paths(default_test_collection_paths, 'ansible_collections')
|
||||||
|
modules = list(pkgutil.iter_modules(paths, 'ansible_collections.'))
|
||||||
|
assert len(modules) == 2
|
||||||
|
assert all(m[2] is True for m in modules)
|
||||||
|
assert all(isinstance(m[0], _AnsiblePathHookFinder) for m in modules)
|
||||||
|
assert set(['ansible_collections.testns', 'ansible_collections.ansible']) == set(m[1] for m in modules)
|
||||||
|
|
||||||
|
|
||||||
|
def test_collection_get_data():
|
||||||
|
finder = get_default_finder()
|
||||||
|
reset_collections_loader_state(finder)
|
||||||
|
|
||||||
|
# something that's there
|
||||||
|
d = pkgutil.get_data('ansible_collections.testns.testcoll', 'plugins/action/my_action.py')
|
||||||
|
assert b'hello from my_action.py' in d
|
||||||
|
|
||||||
|
# something that's not there
|
||||||
|
d = pkgutil.get_data('ansible_collections.testns.testcoll', 'bogus/bogus')
|
||||||
|
assert d is None
|
||||||
|
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
plugins_pkg = import_module('ansible_collections.ansible.builtin')
|
||||||
|
assert not os.path.exists(os.path.dirname(plugins_pkg.__file__))
|
||||||
|
d = pkgutil.get_data('ansible_collections.ansible.builtin', 'plugins/connection/local.py')
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'ref,ref_type,expected_collection,expected_subdirs,expected_resource,expected_python_pkg_name',
|
||||||
|
[
|
||||||
|
('ns.coll.myaction', 'action', 'ns.coll', '', 'myaction', 'ansible_collections.ns.coll.plugins.action'),
|
||||||
|
('ns.coll.subdir1.subdir2.myaction', 'action', 'ns.coll', 'subdir1.subdir2', 'myaction', 'ansible_collections.ns.coll.plugins.action.subdir1.subdir2'),
|
||||||
|
('ns.coll.myrole', 'role', 'ns.coll', '', 'myrole', 'ansible_collections.ns.coll.roles.myrole'),
|
||||||
|
('ns.coll.subdir1.subdir2.myrole', 'role', 'ns.coll', 'subdir1.subdir2', 'myrole', 'ansible_collections.ns.coll.roles.subdir1.subdir2.myrole'),
|
||||||
|
])
|
||||||
|
def test_fqcr_parsing_valid(ref, ref_type, expected_collection,
|
||||||
|
expected_subdirs, expected_resource, expected_python_pkg_name):
|
||||||
|
assert AnsibleCollectionRef.is_valid_fqcr(ref, ref_type)
|
||||||
|
|
||||||
|
r = AnsibleCollectionRef.from_fqcr(ref, ref_type)
|
||||||
|
assert r.collection == expected_collection
|
||||||
|
assert r.subdirs == expected_subdirs
|
||||||
|
assert r.resource == expected_resource
|
||||||
|
assert r.n_python_package_name == expected_python_pkg_name
|
||||||
|
|
||||||
|
r = AnsibleCollectionRef.try_parse_fqcr(ref, ref_type)
|
||||||
|
assert r.collection == expected_collection
|
||||||
|
assert r.subdirs == expected_subdirs
|
||||||
|
assert r.resource == expected_resource
|
||||||
|
assert r.n_python_package_name == expected_python_pkg_name
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'ref,ref_type,expected_error_type,expected_error_expression',
|
||||||
|
[
|
||||||
|
('no_dots_at_all_action', 'action', ValueError, 'is not a valid collection reference'),
|
||||||
|
('no_nscoll.myaction', 'action', ValueError, 'is not a valid collection reference'),
|
||||||
|
('ns.coll.myaction', 'bogus', ValueError, 'invalid collection ref_type'),
|
||||||
|
])
|
||||||
|
def test_fqcr_parsing_invalid(ref, ref_type, expected_error_type, expected_error_expression):
|
||||||
|
assert not AnsibleCollectionRef.is_valid_fqcr(ref, ref_type)
|
||||||
|
|
||||||
|
with pytest.raises(expected_error_type) as curerr:
|
||||||
|
AnsibleCollectionRef.from_fqcr(ref, ref_type)
|
||||||
|
|
||||||
|
assert re.search(expected_error_expression, str(curerr.value))
|
||||||
|
|
||||||
|
r = AnsibleCollectionRef.try_parse_fqcr(ref, ref_type)
|
||||||
|
assert r is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'name,subdirs,resource,ref_type,python_pkg_name',
|
||||||
|
[
|
||||||
|
('ns.coll', None, 'res', 'doc_fragments', 'ansible_collections.ns.coll.plugins.doc_fragments'),
|
||||||
|
('ns.coll', 'subdir1', 'res', 'doc_fragments', 'ansible_collections.ns.coll.plugins.doc_fragments.subdir1'),
|
||||||
|
('ns.coll', 'subdir1.subdir2', 'res', 'action', 'ansible_collections.ns.coll.plugins.action.subdir1.subdir2'),
|
||||||
|
])
|
||||||
|
def test_collectionref_components_valid(name, subdirs, resource, ref_type, python_pkg_name):
|
||||||
|
x = AnsibleCollectionRef(name, subdirs, resource, ref_type)
|
||||||
|
|
||||||
|
assert x.collection == name
|
||||||
|
if subdirs:
|
||||||
|
assert x.subdirs == subdirs
|
||||||
|
else:
|
||||||
|
assert x.subdirs == ''
|
||||||
|
|
||||||
|
assert x.resource == resource
|
||||||
|
assert x.ref_type == ref_type
|
||||||
|
assert x.n_python_package_name == python_pkg_name
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'dirname,expected_result',
|
||||||
|
[
|
||||||
|
('become_plugins', 'become'),
|
||||||
|
('cache_plugins', 'cache'),
|
||||||
|
('connection_plugins', 'connection'),
|
||||||
|
('library', 'modules'),
|
||||||
|
('filter_plugins', 'filter'),
|
||||||
|
('bogus_plugins', ValueError),
|
||||||
|
(None, ValueError)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
def test_legacy_plugin_dir_to_plugin_type(dirname, expected_result):
|
||||||
|
if isinstance(expected_result, string_types):
|
||||||
|
assert AnsibleCollectionRef.legacy_plugin_dir_to_plugin_type(dirname) == expected_result
|
||||||
|
else:
|
||||||
|
with pytest.raises(expected_result):
|
||||||
|
AnsibleCollectionRef.legacy_plugin_dir_to_plugin_type(dirname)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'name,subdirs,resource,ref_type,expected_error_type,expected_error_expression',
|
||||||
|
[
|
||||||
|
('bad_ns', '', 'resource', 'action', ValueError, 'invalid collection name'),
|
||||||
|
('ns.coll.', '', 'resource', 'action', ValueError, 'invalid collection name'),
|
||||||
|
('ns.coll', 'badsubdir#', 'resource', 'action', ValueError, 'invalid subdirs entry'),
|
||||||
|
('ns.coll', 'badsubdir.', 'resource', 'action', ValueError, 'invalid subdirs entry'),
|
||||||
|
('ns.coll', '.badsubdir', 'resource', 'action', ValueError, 'invalid subdirs entry'),
|
||||||
|
('ns.coll', '', 'resource', 'bogus', ValueError, 'invalid collection ref_type'),
|
||||||
|
])
|
||||||
|
def test_collectionref_components_invalid(name, subdirs, resource, ref_type, expected_error_type, expected_error_expression):
|
||||||
|
with pytest.raises(expected_error_type) as curerr:
|
||||||
|
AnsibleCollectionRef(name, subdirs, resource, ref_type)
|
||||||
|
|
||||||
|
assert re.search(expected_error_expression, str(curerr.value))
|
||||||
|
|
||||||
|
|
||||||
|
# BEGIN TEST SUPPORT
|
||||||
|
|
||||||
|
default_test_collection_paths = [
|
||||||
|
os.path.join(os.path.dirname(__file__), 'fixtures', 'collections'),
|
||||||
|
os.path.join(os.path.dirname(__file__), 'fixtures', 'collections_masked'),
|
||||||
|
'/bogus/bogussub'
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def get_default_finder():
|
||||||
|
return _AnsibleCollectionFinder(paths=default_test_collection_paths)
|
||||||
|
|
||||||
|
|
||||||
|
def extend_paths(path_list, suffix):
|
||||||
|
suffix = suffix.replace('.', '/')
|
||||||
|
return [os.path.join(p, suffix) for p in path_list]
|
||||||
|
|
||||||
|
|
||||||
|
def nuke_module_prefix(prefix):
|
||||||
|
for module_to_nuke in [m for m in sys.modules if m.startswith(prefix)]:
|
||||||
|
sys.modules.pop(module_to_nuke)
|
||||||
|
|
||||||
|
|
||||||
|
def reset_collections_loader_state(metapath_finder=None):
|
||||||
|
_AnsibleCollectionFinder._remove()
|
||||||
|
|
||||||
|
nuke_module_prefix('ansible_collections')
|
||||||
|
nuke_module_prefix('ansible.modules')
|
||||||
|
nuke_module_prefix('ansible.plugins')
|
||||||
|
|
||||||
|
# FIXME: better to move this someplace else that gets cleaned up automatically?
|
||||||
|
_AnsibleCollectionLoader._redirected_package_map = {}
|
||||||
|
|
||||||
|
AnsibleCollectionConfig._default_collection = None
|
||||||
|
AnsibleCollectionConfig._on_collection_load = _EventSource()
|
||||||
|
|
||||||
|
if metapath_finder:
|
||||||
|
metapath_finder._install()
|
||||||
@ -1 +0,0 @@
|
|||||||
from ..module_utils.my_util import question
|
|
||||||
@ -1,215 +0,0 @@
|
|||||||
from __future__ import (absolute_import, division, print_function)
|
|
||||||
__metaclass__ = type
|
|
||||||
|
|
||||||
import os
|
|
||||||
import pytest
|
|
||||||
import re
|
|
||||||
import sys
|
|
||||||
|
|
||||||
from ansible.utils.collection_loader import AnsibleCollectionRef
|
|
||||||
|
|
||||||
|
|
||||||
def test_import_from_collection(monkeypatch):
|
|
||||||
collection_root = os.path.join(os.path.dirname(__file__), 'fixtures', 'collections')
|
|
||||||
collection_path = os.path.join(collection_root, 'ansible_collections/my_namespace/my_collection/plugins/module_utils/my_util.py')
|
|
||||||
|
|
||||||
# the trace we're expecting to be generated when running the code below:
|
|
||||||
# answer = question()
|
|
||||||
expected_trace_log = [
|
|
||||||
(collection_path, 5, 'call'),
|
|
||||||
(collection_path, 6, 'line'),
|
|
||||||
(collection_path, 6, 'return'),
|
|
||||||
]
|
|
||||||
|
|
||||||
# define the collection root before any ansible code has been loaded
|
|
||||||
# otherwise config will have already been loaded and changing the environment will have no effect
|
|
||||||
monkeypatch.setenv('ANSIBLE_COLLECTIONS_PATHS', collection_root)
|
|
||||||
|
|
||||||
from ansible.utils.collection_loader import AnsibleCollectionLoader
|
|
||||||
|
|
||||||
# zap the singleton collection loader instance if it exists
|
|
||||||
AnsibleCollectionLoader._Singleton__instance = None
|
|
||||||
|
|
||||||
for index in [idx for idx, obj in enumerate(sys.meta_path) if isinstance(obj, AnsibleCollectionLoader)]:
|
|
||||||
# replace any existing collection loaders that may exist
|
|
||||||
# since these were loaded during unit test collection
|
|
||||||
# they will not have the correct configuration
|
|
||||||
sys.meta_path[index] = AnsibleCollectionLoader()
|
|
||||||
|
|
||||||
# make sure the collection loader is installed
|
|
||||||
# this will be a no-op if the collection loader is already installed
|
|
||||||
# which will depend on whether or not any tests being run imported ansible.plugins.loader during unit test collection
|
|
||||||
from ansible.plugins.loader import _configure_collection_loader
|
|
||||||
_configure_collection_loader() # currently redundant, the import above already calls this
|
|
||||||
|
|
||||||
from ansible_collections.my_namespace.my_collection.plugins.module_utils.my_util import question
|
|
||||||
|
|
||||||
original_trace_function = sys.gettrace()
|
|
||||||
trace_log = []
|
|
||||||
|
|
||||||
if original_trace_function:
|
|
||||||
# enable tracing while preserving the existing trace function (coverage)
|
|
||||||
def my_trace_function(frame, event, arg):
|
|
||||||
trace_log.append((frame.f_code.co_filename, frame.f_lineno, event))
|
|
||||||
|
|
||||||
# the original trace function expects to have itself set as the trace function
|
|
||||||
sys.settrace(original_trace_function)
|
|
||||||
# call the original trace function
|
|
||||||
original_trace_function(frame, event, arg)
|
|
||||||
# restore our trace function
|
|
||||||
sys.settrace(my_trace_function)
|
|
||||||
|
|
||||||
return my_trace_function
|
|
||||||
else:
|
|
||||||
# no existing trace function, so our trace function is much simpler
|
|
||||||
def my_trace_function(frame, event, arg):
|
|
||||||
trace_log.append((frame.f_code.co_filename, frame.f_lineno, event))
|
|
||||||
|
|
||||||
return my_trace_function
|
|
||||||
|
|
||||||
sys.settrace(my_trace_function)
|
|
||||||
|
|
||||||
try:
|
|
||||||
# run a minimal amount of code while the trace is running
|
|
||||||
# adding more code here, including use of a context manager, will add more to our trace
|
|
||||||
answer = question()
|
|
||||||
finally:
|
|
||||||
sys.settrace(original_trace_function)
|
|
||||||
|
|
||||||
# make sure 'import ... as ...' works on builtin synthetic collections
|
|
||||||
# the following import is not supported (it tries to find module_utils in ansible.plugins)
|
|
||||||
# import ansible_collections.ansible.builtin.plugins.module_utils as c1
|
|
||||||
import ansible_collections.ansible.builtin.plugins.action as c2
|
|
||||||
import ansible_collections.ansible.builtin.plugins as c3
|
|
||||||
import ansible_collections.ansible.builtin as c4
|
|
||||||
import ansible_collections.ansible as c5
|
|
||||||
import ansible_collections as c6
|
|
||||||
|
|
||||||
# make sure 'import ...' works on builtin synthetic collections
|
|
||||||
import ansible_collections.ansible.builtin.plugins.module_utils
|
|
||||||
|
|
||||||
import ansible_collections.ansible.builtin.plugins.action
|
|
||||||
assert ansible_collections.ansible.builtin.plugins.action == c3.action == c2
|
|
||||||
|
|
||||||
import ansible_collections.ansible.builtin.plugins
|
|
||||||
assert ansible_collections.ansible.builtin.plugins == c4.plugins == c3
|
|
||||||
|
|
||||||
import ansible_collections.ansible.builtin
|
|
||||||
assert ansible_collections.ansible.builtin == c5.builtin == c4
|
|
||||||
|
|
||||||
import ansible_collections.ansible
|
|
||||||
assert ansible_collections.ansible == c6.ansible == c5
|
|
||||||
|
|
||||||
import ansible_collections
|
|
||||||
assert ansible_collections == c6
|
|
||||||
|
|
||||||
# make sure 'from ... import ...' works on builtin synthetic collections
|
|
||||||
from ansible_collections.ansible import builtin
|
|
||||||
from ansible_collections.ansible.builtin import plugins
|
|
||||||
assert builtin.plugins == plugins
|
|
||||||
|
|
||||||
from ansible_collections.ansible.builtin.plugins import action
|
|
||||||
from ansible_collections.ansible.builtin.plugins.action import command
|
|
||||||
assert action.command == command
|
|
||||||
|
|
||||||
from ansible_collections.ansible.builtin.plugins.module_utils import basic
|
|
||||||
from ansible_collections.ansible.builtin.plugins.module_utils.basic import AnsibleModule
|
|
||||||
assert basic.AnsibleModule == AnsibleModule
|
|
||||||
|
|
||||||
# make sure relative imports work from collections code
|
|
||||||
# these require __package__ to be set correctly
|
|
||||||
import ansible_collections.my_namespace.my_collection.plugins.module_utils.my_other_util
|
|
||||||
import ansible_collections.my_namespace.my_collection.plugins.action.my_action
|
|
||||||
|
|
||||||
# verify that code loaded from a collection does not inherit __future__ statements from the collection loader
|
|
||||||
if sys.version_info[0] == 2:
|
|
||||||
# if the collection code inherits the division future feature from the collection loader this will fail
|
|
||||||
assert answer == 1
|
|
||||||
else:
|
|
||||||
assert answer == 1.5
|
|
||||||
|
|
||||||
# verify that the filename and line number reported by the trace is correct
|
|
||||||
# this makes sure that collection loading preserves file paths and line numbers
|
|
||||||
assert trace_log == expected_trace_log
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
'ref,ref_type,expected_collection,expected_subdirs,expected_resource,expected_python_pkg_name',
|
|
||||||
[
|
|
||||||
('ns.coll.myaction', 'action', 'ns.coll', '', 'myaction', 'ansible_collections.ns.coll.plugins.action'),
|
|
||||||
('ns.coll.subdir1.subdir2.myaction', 'action', 'ns.coll', 'subdir1.subdir2', 'myaction', 'ansible_collections.ns.coll.plugins.action.subdir1.subdir2'),
|
|
||||||
('ns.coll.myrole', 'role', 'ns.coll', '', 'myrole', 'ansible_collections.ns.coll.roles.myrole'),
|
|
||||||
('ns.coll.subdir1.subdir2.myrole', 'role', 'ns.coll', 'subdir1.subdir2', 'myrole', 'ansible_collections.ns.coll.roles.subdir1.subdir2.myrole'),
|
|
||||||
])
|
|
||||||
def test_fqcr_parsing_valid(ref, ref_type, expected_collection,
|
|
||||||
expected_subdirs, expected_resource, expected_python_pkg_name):
|
|
||||||
assert AnsibleCollectionRef.is_valid_fqcr(ref, ref_type)
|
|
||||||
|
|
||||||
r = AnsibleCollectionRef.from_fqcr(ref, ref_type)
|
|
||||||
assert r.collection == expected_collection
|
|
||||||
assert r.subdirs == expected_subdirs
|
|
||||||
assert r.resource == expected_resource
|
|
||||||
assert r.n_python_package_name == expected_python_pkg_name
|
|
||||||
|
|
||||||
r = AnsibleCollectionRef.try_parse_fqcr(ref, ref_type)
|
|
||||||
assert r.collection == expected_collection
|
|
||||||
assert r.subdirs == expected_subdirs
|
|
||||||
assert r.resource == expected_resource
|
|
||||||
assert r.n_python_package_name == expected_python_pkg_name
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
'ref,ref_type,expected_error_type,expected_error_expression',
|
|
||||||
[
|
|
||||||
('no_dots_at_all_action', 'action', ValueError, 'is not a valid collection reference'),
|
|
||||||
('no_nscoll.myaction', 'action', ValueError, 'is not a valid collection reference'),
|
|
||||||
('ns.coll.myaction', 'bogus', ValueError, 'invalid collection ref_type'),
|
|
||||||
])
|
|
||||||
def test_fqcr_parsing_invalid(ref, ref_type, expected_error_type, expected_error_expression):
|
|
||||||
assert not AnsibleCollectionRef.is_valid_fqcr(ref, ref_type)
|
|
||||||
|
|
||||||
with pytest.raises(expected_error_type) as curerr:
|
|
||||||
AnsibleCollectionRef.from_fqcr(ref, ref_type)
|
|
||||||
|
|
||||||
assert re.search(expected_error_expression, str(curerr.value))
|
|
||||||
|
|
||||||
r = AnsibleCollectionRef.try_parse_fqcr(ref, ref_type)
|
|
||||||
assert r is None
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
'name,subdirs,resource,ref_type,python_pkg_name',
|
|
||||||
[
|
|
||||||
('ns.coll', None, 'res', 'doc_fragments', 'ansible_collections.ns.coll.plugins.doc_fragments'),
|
|
||||||
('ns.coll', 'subdir1', 'res', 'doc_fragments', 'ansible_collections.ns.coll.plugins.doc_fragments.subdir1'),
|
|
||||||
('ns.coll', 'subdir1.subdir2', 'res', 'action', 'ansible_collections.ns.coll.plugins.action.subdir1.subdir2'),
|
|
||||||
])
|
|
||||||
def test_collectionref_components_valid(name, subdirs, resource, ref_type, python_pkg_name):
|
|
||||||
x = AnsibleCollectionRef(name, subdirs, resource, ref_type)
|
|
||||||
|
|
||||||
assert x.collection == name
|
|
||||||
if subdirs:
|
|
||||||
assert x.subdirs == subdirs
|
|
||||||
else:
|
|
||||||
assert x.subdirs == ''
|
|
||||||
|
|
||||||
assert x.resource == resource
|
|
||||||
assert x.ref_type == ref_type
|
|
||||||
assert x.n_python_package_name == python_pkg_name
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
'name,subdirs,resource,ref_type,expected_error_type,expected_error_expression',
|
|
||||||
[
|
|
||||||
('bad_ns', '', 'resource', 'action', ValueError, 'invalid collection name'),
|
|
||||||
('ns.coll.', '', 'resource', 'action', ValueError, 'invalid collection name'),
|
|
||||||
('ns.coll', 'badsubdir#', 'resource', 'action', ValueError, 'invalid subdirs entry'),
|
|
||||||
('ns.coll', 'badsubdir.', 'resource', 'action', ValueError, 'invalid subdirs entry'),
|
|
||||||
('ns.coll', '.badsubdir', 'resource', 'action', ValueError, 'invalid subdirs entry'),
|
|
||||||
('ns.coll', '', 'resource', 'bogus', ValueError, 'invalid collection ref_type'),
|
|
||||||
])
|
|
||||||
def test_collectionref_components_invalid(name, subdirs, resource, ref_type, expected_error_type, expected_error_expression):
|
|
||||||
with pytest.raises(expected_error_type) as curerr:
|
|
||||||
AnsibleCollectionRef(name, subdirs, resource, ref_type)
|
|
||||||
|
|
||||||
assert re.search(expected_error_expression, str(curerr.value))
|
|
||||||
Loading…
Reference in New Issue