From 9949629e5abef91b6fb3a09651c9fd92e9916dd3 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Tue, 6 Nov 2018 10:02:36 -0600 Subject: [PATCH] Add toml inventory plugin (#41593) * First pass at a toml inventory * Make EXAMPLES yaml * Remove unnecessary comment * Small formatting changes * Add ansible-inventory option to list as TOML * TOML inventory improvements, to allow a more simple inventory, specifically related to children * changelog * Simplify logic * Dedupe _expand_hostpattern, making it available to all inventory plugins * Don't make the TOML inventory dependent on the YAML inventory * Quote IP address values * Add more TOML examples * Further cleanups * Enable the toml inventory to run by default * Create toml specific dumper * 2.8 * Clean up imports * No toml pygments lexer * Don't raise an exception early when toml isn't present, and move toml to the end, since it requires an external dep * Require toml>=0.10.0 * Further clean up of empty data * Don't require toml>=0.10.0, but prefer it, add code for fallback in older versions * Ensure we actually pass an encoder to toml.dumps * Simplify recursive data converter * Appease tests, since we haven't limited controller testing to 2.7+ * Update docstring for convert_yaml_objects_to_native * remove outdated catching of AttributeError * We don't need to catch ImportError when import ansible.plugins.inventory.toml * Add note about what self.dump_funcs.update is doing * Address some things * A little extra comment * Fix toml availability check * Don't create an intermediate list * Require toml file extension * Add metadata * Remove TOML docs from intro_inventory to prevent people from getting the wrong idea * It's in defaults, remove note * core supported, indicate very clearly that this is preview status --- changelogs/fragments/toml-inventory.yaml | 2 + lib/ansible/cli/inventory.py | 53 ++++ lib/ansible/config/base.yml | 2 +- lib/ansible/plugins/inventory/__init__.py | 26 ++ .../plugins/inventory/advanced_host_list.py | 28 +- lib/ansible/plugins/inventory/ini.py | 29 +- lib/ansible/plugins/inventory/toml.py | 268 ++++++++++++++++++ lib/ansible/plugins/inventory/yaml.py | 28 +- 8 files changed, 353 insertions(+), 83 deletions(-) create mode 100644 changelogs/fragments/toml-inventory.yaml create mode 100644 lib/ansible/plugins/inventory/toml.py diff --git a/changelogs/fragments/toml-inventory.yaml b/changelogs/fragments/toml-inventory.yaml new file mode 100644 index 00000000000..5029aa14636 --- /dev/null +++ b/changelogs/fragments/toml-inventory.yaml @@ -0,0 +1,2 @@ +minor_changes: +- inventory - added new TOML inventory plugin (https://github.com/ansible/ansible/pull/41593) diff --git a/lib/ansible/cli/inventory.py b/lib/ansible/cli/inventory.py index 6590c98103c..a39148e7076 100644 --- a/lib/ansible/cli/inventory.py +++ b/lib/ansible/cli/inventory.py @@ -94,6 +94,8 @@ class InventoryCLI(CLI): # graph self.parser.add_option("-y", "--yaml", action="store_true", default=False, dest='yaml', help='Use YAML format instead of default JSON, ignored for --graph') + self.parser.add_option('--toml', action='store_true', default=False, dest='toml', + help='Use TOML format instead of default JSON, ignored for --graph') self.parser.add_option("--vars", action="store_true", default=False, dest='show_vars', help='Add vars to graph display, ignored unless used with --graph') @@ -176,6 +178,8 @@ class InventoryCLI(CLI): top = self._get_group('all') if self.options.yaml: results = self.yaml_inventory(top) + elif self.options.toml: + results = self.toml_inventory(top) else: results = self.json_inventory(top) results = self.dump(results) @@ -193,6 +197,13 @@ class InventoryCLI(CLI): import yaml from ansible.parsing.yaml.dumper import AnsibleDumper results = yaml.dump(stuff, Dumper=AnsibleDumper, default_flow_style=False) + elif self.options.toml: + from ansible.plugins.inventory.toml import toml_dumps, HAS_TOML + if not HAS_TOML: + raise AnsibleError( + 'The python "toml" library is required when using the TOML output format' + ) + results = toml_dumps(stuff) else: import json from ansible.parsing.ajson import AnsibleJSONEncoder @@ -385,3 +396,45 @@ class InventoryCLI(CLI): return results return format_group(top) + + def toml_inventory(self, top): + seen = set() + has_ungrouped = bool(next(g.hosts for g in top.child_groups if g.name == 'ungrouped')) + + def format_group(group): + results = {} + results[group.name] = {} + + results[group.name]['children'] = [] + for subgroup in sorted(group.child_groups, key=attrgetter('name')): + if subgroup.name == 'ungrouped' and not has_ungrouped: + continue + if group.name != 'all': + results[group.name]['children'].append(subgroup.name) + results.update(format_group(subgroup)) + + if group.name != 'all': + for host in sorted(group.hosts, key=attrgetter('name')): + if host.name not in seen: + seen.add(host.name) + host_vars = self._get_host_variables(host=host) + self._remove_internal(host_vars) + else: + host_vars = {} + try: + results[group.name]['hosts'][host.name] = host_vars + except KeyError: + results[group.name]['hosts'] = {host.name: host_vars} + + if self.options.export: + results[group.name]['vars'] = self._get_group_variables(group) + + self._remove_empty(results[group.name]) + if not results[group.name]: + del results[group.name] + + return results + + results = format_group(top) + + return results diff --git a/lib/ansible/config/base.yml b/lib/ansible/config/base.yml index 816496297bc..0cf58dc4805 100644 --- a/lib/ansible/config/base.yml +++ b/lib/ansible/config/base.yml @@ -1408,7 +1408,7 @@ INVENTORY_ANY_UNPARSED_IS_FAILED: version_added: "2.7" INVENTORY_ENABLED: name: Active Inventory plugins - default: ['host_list', 'script', 'yaml', 'ini', 'auto'] + default: ['host_list', 'script', 'yaml', 'ini', 'toml', 'auto'] description: List of enabled inventory plugins, it also determines the order in which they are used. env: [{name: ANSIBLE_INVENTORY_ENABLED}] ini: diff --git a/lib/ansible/plugins/inventory/__init__.py b/lib/ansible/plugins/inventory/__init__.py index 2dd9eeefd9b..3cd8f43f519 100644 --- a/lib/ansible/plugins/inventory/__init__.py +++ b/lib/ansible/plugins/inventory/__init__.py @@ -25,6 +25,7 @@ import re import string from ansible.errors import AnsibleError, AnsibleParserError +from ansible.parsing.utils.addresses import parse_address from ansible.plugins import AnsiblePlugin from ansible.plugins.cache import InventoryFileCacheModule from ansible.module_utils._text import to_bytes, to_native @@ -231,6 +232,31 @@ class BaseInventoryPlugin(AnsiblePlugin): if k in data: self._options[k] = data.pop(k) + def _expand_hostpattern(self, hostpattern): + ''' + Takes a single host pattern and returns a list of hostnames and an + optional port number that applies to all of them. + ''' + # Can the given hostpattern be parsed as a host with an optional port + # specification? + + try: + (pattern, port) = parse_address(hostpattern, allow_ranges=True) + except Exception: + # not a recognizable host pattern + pattern = hostpattern + port = None + + # Once we have separated the pattern, we expand it into list of one or + # more hostnames, depending on whether it contains any [x:y] ranges. + + if detect_range(pattern): + hostnames = expand_hostname_range(pattern) + else: + hostnames = [pattern] + + return (hostnames, port) + def clear_cache(self): pass diff --git a/lib/ansible/plugins/inventory/advanced_host_list.py b/lib/ansible/plugins/inventory/advanced_host_list.py index 5e4711f9b9f..270394776d6 100644 --- a/lib/ansible/plugins/inventory/advanced_host_list.py +++ b/lib/ansible/plugins/inventory/advanced_host_list.py @@ -25,8 +25,7 @@ import os from ansible.errors import AnsibleError, AnsibleParserError from ansible.module_utils._text import to_bytes, to_native -from ansible.parsing.utils.addresses import parse_address -from ansible.plugins.inventory import BaseInventoryPlugin, detect_range, expand_hostname_range +from ansible.plugins.inventory import BaseInventoryPlugin class InventoryModule(BaseInventoryPlugin): @@ -62,28 +61,3 @@ class InventoryModule(BaseInventoryPlugin): self.inventory.add_host(host, group='ungrouped', port=port) except Exception as e: raise AnsibleParserError("Invalid data from string, could not parse: %s" % to_native(e)) - - def _expand_hostpattern(self, hostpattern): - ''' - Takes a single host pattern and returns a list of hostnames and an - optional port number that applies to all of them. - ''' - # Can the given hostpattern be parsed as a host with an optional port - # specification? - - try: - (pattern, port) = parse_address(hostpattern, allow_ranges=True) - except: - # not a recognizable host pattern - pattern = hostpattern - port = None - - # Once we have separated the pattern, we expand it into list of one or - # more hostnames, depending on whether it contains any [x:y] ranges. - - if detect_range(pattern): - hostnames = expand_hostname_range(pattern) - else: - hostnames = [pattern] - - return (hostnames, port) diff --git a/lib/ansible/plugins/inventory/ini.py b/lib/ansible/plugins/inventory/ini.py index 93afcf1898f..fe7f4cccf4f 100644 --- a/lib/ansible/plugins/inventory/ini.py +++ b/lib/ansible/plugins/inventory/ini.py @@ -73,8 +73,7 @@ EXAMPLES = ''' import ast import re -from ansible.plugins.inventory import BaseFileInventoryPlugin, detect_range, expand_hostname_range -from ansible.parsing.utils.addresses import parse_address +from ansible.plugins.inventory import BaseFileInventoryPlugin from ansible.errors import AnsibleError, AnsibleParserError from ansible.module_utils._text import to_bytes, to_text @@ -311,32 +310,6 @@ class InventoryModule(BaseFileInventoryPlugin): return hostnames, port, variables - def _expand_hostpattern(self, hostpattern): - ''' - Takes a single host pattern and returns a list of hostnames and an - optional port number that applies to all of them. - ''' - - # Can the given hostpattern be parsed as a host with an optional port - # specification? - - try: - (pattern, port) = parse_address(hostpattern, allow_ranges=True) - except Exception: - # not a recognizable host pattern - pattern = hostpattern - port = None - - # Once we have separated the pattern, we expand it into list of one or - # more hostnames, depending on whether it contains any [x:y] ranges. - - if detect_range(pattern): - hostnames = expand_hostname_range(pattern) - else: - hostnames = [pattern] - - return (hostnames, port) - @staticmethod def _parse_value(v): ''' diff --git a/lib/ansible/plugins/inventory/toml.py b/lib/ansible/plugins/inventory/toml.py new file mode 100644 index 00000000000..e215973b56b --- /dev/null +++ b/lib/ansible/plugins/inventory/toml.py @@ -0,0 +1,268 @@ +# Copyright (c) 2018 Matt Martz +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'core'} + +DOCUMENTATION = ''' + inventory: toml + version_added: "2.8" + short_description: Uses a specific TOML file as an inventory source. + description: + - TOML based inventory format + - File MUST have a valid '.toml' file extension + notes: + - Requires the 'toml' python library +''' + +EXAMPLES = ''' +example1: | + [all.vars] + has_java = false + + [web] + children = [ + "apache", + "nginx" + ] + vars = { http_port = 8080, myvar = 23 } + + [web.hosts] + host1 = {} + host2 = { ansible_port = 222 } + + [apache.hosts] + tomcat1 = {} + tomcat2 = { myvar = 34 } + tomcat3 = { mysecret = "03#pa33w0rd" } + + [nginx.hosts] + jenkins1 = {} + + [nginx.vars] + has_java = true + +example2: | + [all.vars] + has_java = false + + [web] + children = [ + "apache", + "nginx" + ] + + [web.vars] + http_port = 8080 + myvar = 23 + + [web.hosts.host1] + [web.hosts.host2] + ansible_port = 222 + + [apache.hosts.tomcat1] + + [apache.hosts.tomcat2] + myvar = 34 + + [apache.hosts.tomcat3] + mysecret = "03#pa33w0rd" + + [nginx.hosts.jenkins1] + + [nginx.vars] + has_java = true + +example3: | + [ungrouped.hosts] + host1 = {} + host2 = { ansible_host = "127.0.0.1", ansible_port = 44 } + host3 = { ansible_host = "127.0.0.1", ansible_port = 45 } + + [g1.hosts] + host4 = {} + + [g2.hosts] + host4 = {} +''' + +import os + +from functools import partial + +from ansible.errors import AnsibleFileNotFound, AnsibleParserError +from ansible.module_utils._text import to_bytes, to_native +from ansible.module_utils.common._collections_compat import MutableMapping, MutableSequence +from ansible.module_utils.six import string_types, text_type +from ansible.parsing.yaml.objects import AnsibleSequence, AnsibleUnicode +from ansible.plugins.inventory import BaseFileInventoryPlugin + +try: + import toml + HAS_TOML = True +except ImportError: + HAS_TOML = False + +try: + from __main__ import display +except ImportError: + from ansible.utils.display import Display + display = Display() + + +WARNING_MSG = ( + 'The TOML inventory format is marked as preview, which means that it is not guaranteed to have a backwards ' + 'compatible interface.' +) + + +if HAS_TOML and hasattr(toml, 'TomlEncoder'): + class AnsibleTomlEncoder(toml.TomlEncoder): + def __init__(self, *args, **kwargs): + super(AnsibleTomlEncoder, self).__init__(*args, **kwargs) + # Map our custom YAML object types to dump_funcs from ``toml`` + self.dump_funcs.update({ + AnsibleSequence: self.dump_funcs.get(list), + AnsibleUnicode: self.dump_funcs.get(str), + }) + display.warning(WARNING_MSG) + toml_dumps = partial(toml.dumps, encoder=AnsibleTomlEncoder()) +else: + def toml_dumps(data): + display.warning(WARNING_MSG) + return toml.dumps(convert_yaml_objects_to_native(data)) + + +def convert_yaml_objects_to_native(obj): + """Older versions of the ``toml`` python library, don't have a pluggable + way to tell the encoder about custom types, so we need to ensure objects + that we pass are native types. + + Only used on ``toml<0.10.0`` where ``toml.TomlEncoder`` is missing. + + This function recurses an object and ensures we cast any of the types from + ``ansible.parsing.yaml.objects`` into their native types, effectively cleansing + the data before we hand it over to ``toml`` + + This function doesn't directly check for the types from ``ansible.parsing.yaml.objects`` + but instead checks for the types those objects inherit from, to offer more flexibility. + """ + if isinstance(obj, dict): + return dict((k, convert_yaml_objects_to_native(v)) for k, v in obj.items()) + elif isinstance(obj, list): + return [convert_yaml_objects_to_native(v) for v in obj] + elif isinstance(obj, text_type): + return text_type(obj) + else: + return obj + + +class InventoryModule(BaseFileInventoryPlugin): + NAME = 'toml' + + def _parse_group(self, group, group_data): + if not isinstance(group_data, (MutableMapping, type(None))): + self.display.warning("Skipping '%s' as this is not a valid group definition" % group) + return + + self.inventory.add_group(group) + if group_data is None: + return + + for key, data in group_data.items(): + if key == 'vars': + if not isinstance(data, MutableMapping): + raise AnsibleParserError( + 'Invalid "vars" entry for "%s" group, requires a dict, found "%s" instead.' % + (group, type(data)) + ) + for var, value in data.items(): + self.inventory.set_variable(group, var, value) + + elif key == 'children': + if not isinstance(data, MutableSequence): + raise AnsibleParserError( + 'Invalid "children" entry for "%s" group, requires a list, found "%s" instead.' % + (group, type(data)) + ) + for subgroup in data: + self._parse_group(subgroup, {}) + self.inventory.add_child(group, subgroup) + + elif key == 'hosts': + if not isinstance(data, MutableMapping): + raise AnsibleParserError( + 'Invalid "hosts" entry for "%s" group, requires a dict, found "%s" instead.' % + (group, type(data)) + ) + for host_pattern, value in data.items(): + hosts, port = self._expand_hostpattern(host_pattern) + self._populate_host_vars(hosts, value, group, port) + else: + self.display.warning( + 'Skipping unexpected key "%s" in group "%s", only "vars", "children" and "hosts" are valid' % + (key, group) + ) + + def _load_file(self, file_name): + if not file_name or not isinstance(file_name, string_types): + raise AnsibleParserError("Invalid filename: '%s'" % to_native(file_name)) + + b_file_name = to_bytes(self.loader.path_dwim(file_name)) + if not self.loader.path_exists(b_file_name): + raise AnsibleFileNotFound("Unable to retrieve file contents", file_name=file_name) + + try: + with open(b_file_name, 'r') as f: + return toml.load(f) + except toml.TomlDecodeError as e: + raise AnsibleParserError( + 'TOML file (%s) is invalid: %s' % (file_name, to_native(e)), + orig_exc=e + ) + except (IOError, OSError) as e: + raise AnsibleParserError( + "An error occurred while trying to read the file '%s': %s" % (file_name, to_native(e)), + orig_exc=e + ) + except Exception as e: + raise AnsibleParserError( + "An unexpected error occurred while parsing the file '%s': %s" % (file_name, to_native(e)), + orig_exc=e + ) + + def parse(self, inventory, loader, path, cache=True): + ''' parses the inventory file ''' + if not HAS_TOML: + raise AnsibleParserError( + 'The TOML inventory plugin requires the python "toml" library' + ) + + display.warning(WARNING_MSG) + + super(InventoryModule, self).parse(inventory, loader, path) + self.set_options() + + try: + data = self._load_file(path) + except Exception as e: + raise AnsibleParserError(e) + + if not data: + raise AnsibleParserError('Parsed empty TOML file') + elif data.get('plugin'): + raise AnsibleParserError('Plugin configuration TOML file, not TOML inventory') + + for group_name in data: + self._parse_group(group_name, data[group_name]) + + def verify_file(self, path): + if super(InventoryModule, self).verify_file(path): + file_name, ext = os.path.splitext(path) + if ext == '.toml': + return True + return False diff --git a/lib/ansible/plugins/inventory/yaml.py b/lib/ansible/plugins/inventory/yaml.py index 2f535ec2628..f4b66e1197a 100644 --- a/lib/ansible/plugins/inventory/yaml.py +++ b/lib/ansible/plugins/inventory/yaml.py @@ -64,8 +64,7 @@ from ansible.errors import AnsibleParserError from ansible.module_utils.six import string_types from ansible.module_utils._text import to_native from ansible.module_utils.common._collections_compat import MutableMapping -from ansible.parsing.utils.addresses import parse_address -from ansible.plugins.inventory import BaseFileInventoryPlugin, detect_range, expand_hostname_range +from ansible.plugins.inventory import BaseFileInventoryPlugin class InventoryModule(BaseFileInventoryPlugin): @@ -156,28 +155,3 @@ class InventoryModule(BaseFileInventoryPlugin): (hostnames, port) = self._expand_hostpattern(host_pattern) return hostnames, port - - def _expand_hostpattern(self, hostpattern): - ''' - Takes a single host pattern and returns a list of hostnames and an - optional port number that applies to all of them. - ''' - # Can the given hostpattern be parsed as a host with an optional port - # specification? - - try: - (pattern, port) = parse_address(hostpattern, allow_ranges=True) - except Exception: - # not a recognizable host pattern - pattern = hostpattern - port = None - - # Once we have separated the pattern, we expand it into list of one or - # more hostnames, depending on whether it contains any [x:y] ranges. - - if detect_range(pattern): - hostnames = expand_hostname_range(pattern) - else: - hostnames = [pattern] - - return (hostnames, port)