From 780ee45819f7b295c53d839e327908477f4fe502 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Mon, 22 Apr 2019 19:32:50 -0400 Subject: [PATCH] ensure inventory plugin loading rel to play (#51177) Ensure inventory plugin loading rel to play fixes #51033 * clarify paths * now adding dirs funciton in loader * better warnings * each cli should handle adding dirs depending on context --- .../fragments/allow_inv_plugin_loading.yml | 2 + lib/ansible/cli/__init__.py | 1 - lib/ansible/plugins/loader.py | 10 +- .../targets/rel_plugin_loading/aliases | 1 + .../targets/rel_plugin_loading/notyaml.yml | 5 + .../targets/rel_plugin_loading/runme.sh | 5 + .../subdir/inventory_plugins/notyaml.py | 168 ++++++++++++++++++ .../rel_plugin_loading/subdir/play.yml | 6 + 8 files changed, 196 insertions(+), 2 deletions(-) create mode 100644 changelogs/fragments/allow_inv_plugin_loading.yml create mode 100644 test/integration/targets/rel_plugin_loading/aliases create mode 100644 test/integration/targets/rel_plugin_loading/notyaml.yml create mode 100755 test/integration/targets/rel_plugin_loading/runme.sh create mode 100644 test/integration/targets/rel_plugin_loading/subdir/inventory_plugins/notyaml.py create mode 100644 test/integration/targets/rel_plugin_loading/subdir/play.yml diff --git a/changelogs/fragments/allow_inv_plugin_loading.yml b/changelogs/fragments/allow_inv_plugin_loading.yml new file mode 100644 index 00000000000..ee96d273482 --- /dev/null +++ b/changelogs/fragments/allow_inv_plugin_loading.yml @@ -0,0 +1,2 @@ +bugfixes: + - allow loading inventory plugins adjacent to playbooks diff --git a/lib/ansible/cli/__init__.py b/lib/ansible/cli/__init__.py index 0543bc1b33b..fd2deb23409 100644 --- a/lib/ansible/cli/__init__.py +++ b/lib/ansible/cli/__init__.py @@ -9,7 +9,6 @@ __metaclass__ = type import getpass import os -import os.path import re import subprocess import sys diff --git a/lib/ansible/plugins/loader.py b/lib/ansible/plugins/loader.py index 7c6c99a7c3c..f245b7b30f5 100644 --- a/lib/ansible/plugins/loader.py +++ b/lib/ansible/plugins/loader.py @@ -51,7 +51,7 @@ def add_all_plugin_dirs(path): if os.path.isdir(plugin_path): obj.add_directory(to_text(plugin_path)) else: - display.warning("Ignoring invalid path provided to plugin path: %s is not a directory" % to_native(path)) + display.warning("Ignoring invalid path provided to plugin path: '%s' is not a directory" % to_native(path)) def get_shell_plugin(shell_type=None, executable=None): @@ -87,6 +87,13 @@ def get_shell_plugin(shell_type=None, executable=None): return shell +def add_dirs_to_loader(which_loader, paths): + + loader = getattr(sys.modules[__name__], '%s_loader' % which_loader) + for path in paths: + loader.add_directory(path, with_subdir=True) + + class PluginLoader: ''' PluginLoader loads plugins from the configured plugin directories. @@ -435,6 +442,7 @@ class PluginLoader: # looks like _get_paths() never forces a cache refresh so if we expect # additional directories to be added later, it is buggy. for path in (p for p in self._get_paths() if p not in self._searched_paths and os.path.isdir(p)): + display.debug('trying %s' % path) try: full_paths = (os.path.join(path, f) for f in os.listdir(path)) except OSError as e: diff --git a/test/integration/targets/rel_plugin_loading/aliases b/test/integration/targets/rel_plugin_loading/aliases new file mode 100644 index 00000000000..b59832142f2 --- /dev/null +++ b/test/integration/targets/rel_plugin_loading/aliases @@ -0,0 +1 @@ +shippable/posix/group3 diff --git a/test/integration/targets/rel_plugin_loading/notyaml.yml b/test/integration/targets/rel_plugin_loading/notyaml.yml new file mode 100644 index 00000000000..23ab032334f --- /dev/null +++ b/test/integration/targets/rel_plugin_loading/notyaml.yml @@ -0,0 +1,5 @@ +all: + hosts: + testhost: + ansible_connection: local + ansible_python_interpreter: "{{ansible_playbook_python}}" diff --git a/test/integration/targets/rel_plugin_loading/runme.sh b/test/integration/targets/rel_plugin_loading/runme.sh new file mode 100755 index 00000000000..34e70fdd552 --- /dev/null +++ b/test/integration/targets/rel_plugin_loading/runme.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -eux + +ANSIBLE_INVENTORY_ENABLED=notyaml ansible-playbook subdir/play.yml -i notyaml.yml "$@" diff --git a/test/integration/targets/rel_plugin_loading/subdir/inventory_plugins/notyaml.py b/test/integration/targets/rel_plugin_loading/subdir/inventory_plugins/notyaml.py new file mode 100644 index 00000000000..d013fc48374 --- /dev/null +++ b/test/integration/targets/rel_plugin_loading/subdir/inventory_plugins/notyaml.py @@ -0,0 +1,168 @@ +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + inventory: yaml + version_added: "2.4" + short_description: Uses a specific YAML file as an inventory source. + description: + - "YAML-based inventory, should start with the C(all) group and contain hosts/vars/children entries." + - Host entries can have sub-entries defined, which will be treated as variables. + - Vars entries are normal group vars. + - "Children are 'child groups', which can also have their own vars/hosts/children and so on." + - File MUST have a valid extension, defined in configuration. + notes: + - If you want to set vars for the C(all) group inside the inventory file, the C(all) group must be the first entry in the file. + - Whitelisted in configuration by default. + options: + yaml_extensions: + description: list of 'valid' extensions for files containing YAML + type: list + default: ['.yaml', '.yml', '.json'] + env: + - name: ANSIBLE_YAML_FILENAME_EXT + - name: ANSIBLE_INVENTORY_PLUGIN_EXTS + ini: + - key: yaml_valid_extensions + section: defaults + - section: inventory_plugin_yaml + key: yaml_valid_extensions + +''' +EXAMPLES = ''' +all: # keys must be unique, i.e. only one 'hosts' per group + hosts: + test1: + test2: + host_var: value + vars: + group_all_var: value + children: # key order does not matter, indentation does + other_group: + children: + group_x: + hosts: + test5 + vars: + g2_var2: value3 + hosts: + test4: + ansible_host: 127.0.0.1 + last_group: + hosts: + test1 # same host as above, additional group membership + vars: + group_last_var: value +''' + +import os + +from ansible.errors import AnsibleError, AnsibleParserError +from ansible.module_utils.six import string_types +from ansible.module_utils._text import to_native, to_text +from ansible.module_utils.common._collections_compat import MutableMapping +from ansible.plugins.inventory import BaseFileInventoryPlugin + +NoneType = type(None) + + +class InventoryModule(BaseFileInventoryPlugin): + + NAME = 'yaml' + + def __init__(self): + + super(InventoryModule, self).__init__() + + def verify_file(self, path): + + valid = False + if super(InventoryModule, self).verify_file(path): + file_name, ext = os.path.splitext(path) + if not ext or ext in self.get_option('yaml_extensions'): + valid = True + return valid + + def parse(self, inventory, loader, path, cache=True): + ''' parses the inventory file ''' + + super(InventoryModule, self).parse(inventory, loader, path) + self.set_options() + + try: + data = self.loader.load_from_file(path, cache=False) + except Exception as e: + raise AnsibleParserError(e) + + if not data: + raise AnsibleParserError('Parsed empty YAML file') + elif not isinstance(data, MutableMapping): + raise AnsibleParserError('YAML inventory has invalid structure, it should be a dictionary, got: %s' % type(data)) + elif data.get('plugin'): + raise AnsibleParserError('Plugin configuration YAML file, not YAML inventory') + + # We expect top level keys to correspond to groups, iterate over them + # to get host, vars and subgroups (which we iterate over recursivelly) + if isinstance(data, MutableMapping): + for group_name in data: + self._parse_group(group_name, data[group_name]) + else: + raise AnsibleParserError("Invalid data from file, expected dictionary and got:\n\n%s" % to_native(data)) + + def _parse_group(self, group, group_data): + + if isinstance(group_data, (MutableMapping, NoneType)): + + try: + self.inventory.add_group(group) + except AnsibleError as e: + raise AnsibleParserError("Unable to add group %s: %s" % (group, to_text(e))) + + if group_data is not None: + # make sure they are dicts + for section in ['vars', 'children', 'hosts']: + if section in group_data: + # convert strings to dicts as these are allowed + if isinstance(group_data[section], string_types): + group_data[section] = {group_data[section]: None} + + if not isinstance(group_data[section], (MutableMapping, NoneType)): + raise AnsibleParserError('Invalid "%s" entry for "%s" group, requires a dictionary, found "%s" instead.' % + (section, group, type(group_data[section]))) + + for key in group_data: + + if not isinstance(group_data[key], (MutableMapping, NoneType)): + self.display.warning('Skipping key (%s) in group (%s) as it is not a mapping, it is a %s' % (key, group, type(group_data[key]))) + continue + + if isinstance(group_data[key], NoneType): + self.display.vvv('Skipping empty key (%s) in group (%s)' % (key, group)) + elif key == 'vars': + for var in group_data[key]: + self.inventory.set_variable(group, var, group_data[key][var]) + elif key == 'children': + for subgroup in group_data[key]: + self._parse_group(subgroup, group_data[key][subgroup]) + self.inventory.add_child(group, subgroup) + + elif key == 'hosts': + for host_pattern in group_data[key]: + hosts, port = self._parse_host(host_pattern) + self._populate_host_vars(hosts, group_data[key][host_pattern] or {}, group, port) + else: + self.display.warning('Skipping unexpected key (%s) in group (%s), only "vars", "children" and "hosts" are valid' % (key, group)) + + else: + self.display.warning("Skipping '%s' as this is not a valid group definition" % group) + + def _parse_host(self, host_pattern): + ''' + Each host key can be a pattern, try to process it and add variables as needed + ''' + (hostnames, port) = self._expand_hostpattern(host_pattern) + + return hostnames, port diff --git a/test/integration/targets/rel_plugin_loading/subdir/play.yml b/test/integration/targets/rel_plugin_loading/subdir/play.yml new file mode 100644 index 00000000000..2326b14a41e --- /dev/null +++ b/test/integration/targets/rel_plugin_loading/subdir/play.yml @@ -0,0 +1,6 @@ +- hosts: all + gather_facts: false + tasks: + - assert: + that: + - inventory_hostname == 'testhost'