Fix plugin names for collection plugins. (#60317)

* Fix plugin names for collection plugins.

Add an integration test to verify plugin __name__ is correct for collection plugins.

* Fix collection loader PEP 302 compliance.

The `find_module` function now returns `None` if the module cannot be found. Previously it would return `self` for modules which did not exist.

Returning a loader from `find_module` which cannot find the module will result in import errors on Python 2.x when using implicit relative imports.

* add changelog

* sanity/units/merge fixes
pull/62566/head
Matt Clay 5 years ago committed by Matt Davis
parent cbe8271745
commit 1c64dba3c9

@ -0,0 +1,2 @@
bugfixes:
- collection loader - fixed relative imports on Python 2.7, ensure pluginloader caches use full name to prevent names from being clobbered (https://github.com/ansible/ansible/pull/60317)

@ -322,6 +322,8 @@ class PluginLoader:
acr = AnsibleCollectionRef.from_fqcr(fq_name, plugin_type) acr = AnsibleCollectionRef.from_fqcr(fq_name, plugin_type)
n_resource = to_native(acr.resource, errors='strict') n_resource = to_native(acr.resource, errors='strict')
# we want this before the extension is added
full_name = '{0}.{1}'.format(acr.n_python_package_name, n_resource)
if extension: if extension:
n_resource += extension n_resource += extension
@ -336,10 +338,10 @@ class PluginLoader:
if hasattr(pkg, '__loader__') and isinstance(pkg.__loader__, AnsibleFlatMapLoader): if hasattr(pkg, '__loader__') and isinstance(pkg.__loader__, AnsibleFlatMapLoader):
try: try:
file_path = pkg.__loader__.find_file(n_resource) file_path = pkg.__loader__.find_file(n_resource)
return to_text(file_path) return full_name, to_text(file_path)
except IOError: except IOError:
# this loader already takes care of extensionless files, so if we didn't find it, just bail # this loader already takes care of extensionless files, so if we didn't find it, just bail
return None return None, None
pkg_path = os.path.dirname(pkg.__file__) pkg_path = os.path.dirname(pkg.__file__)
@ -347,27 +349,31 @@ class PluginLoader:
# FIXME: and is file or file link or ... # FIXME: and is file or file link or ...
if os.path.exists(n_resource_path): if os.path.exists(n_resource_path):
return to_text(n_resource_path) return full_name, to_text(n_resource_path)
# look for any matching extension in the package location (sans filter) # look for any matching extension in the package location (sans filter)
ext_blacklist = ['.pyc', '.pyo'] ext_blacklist = ['.pyc', '.pyo']
found_files = [f for f in glob.iglob(os.path.join(pkg_path, n_resource) + '.*') if os.path.isfile(f) and os.path.splitext(f)[1] not in ext_blacklist] found_files = [f for f in glob.iglob(os.path.join(pkg_path, n_resource) + '.*') if os.path.isfile(f) and os.path.splitext(f)[1] not in ext_blacklist]
if not found_files: if not found_files:
return None return None, None
if len(found_files) > 1: if len(found_files) > 1:
# TODO: warn? # TODO: warn?
pass pass
return to_text(found_files[0]) return full_name, to_text(found_files[0])
def find_plugin(self, name, mod_type='', ignore_deprecated=False, check_aliases=False, collection_list=None): def find_plugin(self, name, mod_type='', ignore_deprecated=False, check_aliases=False, collection_list=None):
''' Find a plugin named name ''' ''' Find a plugin named name '''
return self.find_plugin_with_name(name, mod_type, ignore_deprecated, check_aliases, collection_list)[1]
def find_plugin_with_name(self, name, mod_type='', ignore_deprecated=False, check_aliases=False, collection_list=None):
''' Find a plugin named name '''
global _PLUGIN_FILTERS global _PLUGIN_FILTERS
if name in _PLUGIN_FILTERS[self.package]: if name in _PLUGIN_FILTERS[self.package]:
return None return None, None
if mod_type: if mod_type:
suffix = mod_type suffix = mod_type
@ -392,22 +398,23 @@ class PluginLoader:
# HACK: refactor this properly # HACK: refactor this properly
if candidate_name.startswith('ansible.legacy'): if candidate_name.startswith('ansible.legacy'):
# just pass the raw name to the old lookup function to check in all the usual locations # just pass the raw name to the old lookup function to check in all the usual locations
full_name = name
p = self._find_plugin_legacy(name.replace('ansible.legacy.', '', 1), ignore_deprecated, check_aliases, suffix) p = self._find_plugin_legacy(name.replace('ansible.legacy.', '', 1), ignore_deprecated, check_aliases, suffix)
else: else:
p = self._find_fq_plugin(candidate_name, suffix) full_name, p = self._find_fq_plugin(candidate_name, suffix)
if p: if p:
return p return full_name, p
except Exception as ex: except Exception as ex:
errors.append(to_native(ex)) errors.append(to_native(ex))
if errors: if errors:
display.debug(msg='plugin lookup for {0} failed; errors: {1}'.format(name, '; '.join(errors))) display.debug(msg='plugin lookup for {0} failed; errors: {1}'.format(name, '; '.join(errors)))
return None return None, None
# if we got here, there's no collection list and it's not an FQ name, so do legacy lookup # if we got here, there's no collection list and it's not an FQ name, so do legacy lookup
return self._find_plugin_legacy(name, ignore_deprecated, check_aliases, suffix) return name, self._find_plugin_legacy(name, ignore_deprecated, check_aliases, suffix)
def _find_plugin_legacy(self, name, ignore_deprecated=False, check_aliases=False, suffix=None): def _find_plugin_legacy(self, name, ignore_deprecated=False, check_aliases=False, suffix=None):
@ -501,6 +508,9 @@ class PluginLoader:
def _load_module_source(self, name, path): def _load_module_source(self, name, path):
# avoid collisions across plugins # avoid collisions across plugins
if name.startswith('ansible_collections.'):
full_name = name
else:
full_name = '.'.join([self.package, name]) full_name = '.'.join([self.package, name])
if full_name in sys.modules: if full_name in sys.modules:
@ -534,7 +544,7 @@ class PluginLoader:
collection_list = kwargs.pop('collection_list', None) collection_list = kwargs.pop('collection_list', None)
if name in self.aliases: if name in self.aliases:
name = self.aliases[name] name = self.aliases[name]
path = self.find_plugin(name, collection_list=collection_list) name, path = self.find_plugin_with_name(name, collection_list=collection_list)
if path is None: if path is None:
return None return None

@ -103,15 +103,29 @@ class AnsibleCollectionLoader(with_metaclass(Singleton, object)):
return self._default_collection return self._default_collection
def find_module(self, fullname, path=None): def find_module(self, fullname, path=None):
# this loader is only concerned with items under the Ansible Collections namespace hierarchy, ignore others if self._find_module(fullname, path, load=False)[0]:
if fullname and fullname.split('.', 1)[0] == 'ansible_collections':
return self return self
return None return None
def load_module(self, fullname): def load_module(self, fullname):
mod = self._find_module(fullname, None, load=True)[1]
if not mod:
raise ImportError('module {0} not found'.format(fullname))
return mod
def _find_module(self, fullname, path, load):
# this loader is only concerned with items under the Ansible Collections namespace hierarchy, ignore others
if not fullname.startswith('ansible_collections.') and fullname != 'ansible_collections':
return False, None
if sys.modules.get(fullname): if sys.modules.get(fullname):
return sys.modules[fullname] if not load:
return True, None
return True, sys.modules[fullname]
newmod = None newmod = None
@ -153,14 +167,21 @@ class AnsibleCollectionLoader(with_metaclass(Singleton, object)):
if not map_package: if not map_package:
raise KeyError('invalid synthetic map package definition (no target "map" defined)') raise KeyError('invalid synthetic map package definition (no target "map" defined)')
if not load:
return True, None
mod = import_module(map_package + synpkg_remainder) mod = import_module(map_package + synpkg_remainder)
sys.modules[fullname] = mod sys.modules[fullname] = mod
return mod return True, mod
elif pkg_type == 'flatmap': elif pkg_type == 'flatmap':
raise NotImplementedError() raise NotImplementedError()
elif pkg_type == 'pkg_only': elif pkg_type == 'pkg_only':
if not load:
return True, None
newmod = ModuleType(fullname) newmod = ModuleType(fullname)
newmod.__package__ = fullname newmod.__package__ = fullname
newmod.__file__ = '<ansible_synthetic_collection_package>' newmod.__file__ = '<ansible_synthetic_collection_package>'
@ -170,7 +191,7 @@ class AnsibleCollectionLoader(with_metaclass(Singleton, object)):
if not synpkg_def.get('allow_external_subpackages'): if not synpkg_def.get('allow_external_subpackages'):
# if external subpackages are NOT allowed, we're done # if external subpackages are NOT allowed, we're done
sys.modules[fullname] = newmod sys.modules[fullname] = newmod
return newmod return True, newmod
# if external subpackages ARE allowed, check for on-disk implementations and return a normal # if external subpackages ARE allowed, check for on-disk implementations and return a normal
# package if we find one, otherwise return the one we created here # package if we find one, otherwise return the one we created here
@ -196,6 +217,9 @@ class AnsibleCollectionLoader(with_metaclass(Singleton, object)):
if not os.path.isfile(to_bytes(source_path)): if not os.path.isfile(to_bytes(source_path)):
continue continue
if not load:
return True, None
with open(to_bytes(source_path), 'rb') as fd: with open(to_bytes(source_path), 'rb') as fd:
source = fd.read() source = fd.read()
@ -227,16 +251,16 @@ class AnsibleCollectionLoader(with_metaclass(Singleton, object)):
# FIXME: decide cases where we don't actually want to exec the code? # FIXME: decide cases where we don't actually want to exec the code?
exec(code_object, newmod.__dict__) exec(code_object, newmod.__dict__)
return newmod return True, newmod
# even if we didn't find one on disk, fall back to a synthetic package if we have one... # even if we didn't find one on disk, fall back to a synthetic package if we have one...
if newmod: if newmod:
sys.modules[fullname] = newmod sys.modules[fullname] = newmod
return newmod return True, newmod
# FIXME: need to handle the "no dirs present" case for at least the root and synthetic internal collections like ansible.builtin # FIXME: need to handle the "no dirs present" case for at least the root and synthetic internal collections like ansible.builtin
raise ImportError('module {0} not found'.format(fullname)) return False, None
@staticmethod @staticmethod
def _extend_path_with_ns(path, ns): def _extend_path_with_ns(path, ns):

@ -0,0 +1,2 @@
shippable/posix/group1
skip/python2.6

@ -0,0 +1,15 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
def filter_name(a):
return __name__
class FilterModule(object):
def filters(self):
filters = {
'filter_name': filter_name,
}
return filters

@ -0,0 +1,9 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
from ansible.plugins.lookup import LookupBase
class LookupModule(LookupBase):
def run(self, terms, variables, **kwargs):
return [__name__]

@ -0,0 +1,10 @@
# do not add future boilerplate to this plugin
# specifically, do not add absolute_import, as the purpose of this plugin is to test implicit relative imports on Python 2.x
__metaclass__ = type
from ansible.plugins.lookup import LookupBase
class LookupModule(LookupBase):
def run(self, terms, variables, **kwargs):
return [__name__]

@ -0,0 +1,13 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
def test_name_ok(value):
return __name__ == 'ansible_collections.my_ns.my_col.plugins.test.test_test'
class TestModule:
def tests(self):
return {
'test_name_ok': test_name_ok,
}

@ -0,0 +1,12 @@
- set_fact:
filter_name: "{{ 1 | my_ns.my_col.filter_name }}"
lookup_name: "{{ lookup('my_ns.my_col.lookup_name') }}"
lookup_no_future_boilerplate: "{{ lookup('my_ns.my_col.lookup_no_future_boilerplate') }}"
test_name_ok: "{{ 1 is my_ns.my_col.test_name_ok }}"
- assert:
that:
- filter_name == 'ansible_collections.my_ns.my_col.plugins.filter.test_filter'
- lookup_name == 'ansible_collections.my_ns.my_col.plugins.lookup.lookup_name'
- lookup_no_future_boilerplate == 'ansible_collections.my_ns.my_col.plugins.lookup.lookup_no_future_boilerplate'
- test_name_ok

@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -eux
ANSIBLE_COLLECTIONS_PATHS="${PWD}/collection_root" ansible-playbook test.yml -i ../../inventory "$@"

@ -0,0 +1,3 @@
- hosts: testhost
roles:
- my_ns.my_col.test

@ -5949,6 +5949,7 @@ test/integration/targets/async_fail/library/async_test.py future-import-boilerpl
test/integration/targets/async_fail/library/async_test.py metaclass-boilerplate test/integration/targets/async_fail/library/async_test.py metaclass-boilerplate
test/integration/targets/aws_lambda/files/mini_lambda.py future-import-boilerplate test/integration/targets/aws_lambda/files/mini_lambda.py future-import-boilerplate
test/integration/targets/aws_lambda/files/mini_lambda.py metaclass-boilerplate test/integration/targets/aws_lambda/files/mini_lambda.py metaclass-boilerplate
test/integration/targets/collections_plugin_namespace/collection_root/ansible_collections/my_ns/my_col/plugins/lookup/lookup_no_future_boilerplate.py future-import-boilerplate
test/integration/targets/collections_relative_imports/collection_root/ansible_collections/my_ns/my_col/plugins/module_utils/my_util2.py pylint:relative-beyond-top-level test/integration/targets/collections_relative_imports/collection_root/ansible_collections/my_ns/my_col/plugins/module_utils/my_util2.py pylint:relative-beyond-top-level
test/integration/targets/collections_relative_imports/collection_root/ansible_collections/my_ns/my_col/plugins/module_utils/my_util3.py pylint:relative-beyond-top-level test/integration/targets/collections_relative_imports/collection_root/ansible_collections/my_ns/my_col/plugins/module_utils/my_util3.py pylint:relative-beyond-top-level
test/integration/targets/collections_relative_imports/collection_root/ansible_collections/my_ns/my_col/plugins/modules/my_module.py pylint:relative-beyond-top-level test/integration/targets/collections_relative_imports/collection_root/ansible_collections/my_ns/my_col/plugins/modules/my_module.py pylint:relative-beyond-top-level

Loading…
Cancel
Save