Add toml inventory plugin (#41593)

* First pass at a toml inventory

* Make EXAMPLES yaml

* Remove unnecessary comment

* Small formatting changes

* Add ansible-inventory option to list as TOML

* TOML inventory improvements, to allow a more simple inventory, specifically related to children

* changelog

* Simplify logic

* Dedupe _expand_hostpattern, making it available to all inventory plugins

* Don't make the TOML inventory dependent on the YAML inventory

* Quote IP address values

* Add more TOML examples

* Further cleanups

* Enable the toml inventory to run by default

* Create toml specific dumper

* 2.8

* Clean up imports

* No toml pygments lexer

* Don't raise an exception early when toml isn't present, and move toml to the end, since it requires an external dep

* Require toml>=0.10.0

* Further clean up of empty data

* Don't require toml>=0.10.0, but prefer it, add code for fallback in older versions

* Ensure we actually pass an encoder to toml.dumps

* Simplify recursive data converter

* Appease tests, since we haven't limited controller testing to 2.7+

* Update docstring for convert_yaml_objects_to_native

* remove outdated catching of AttributeError

* We don't need to catch ImportError when import ansible.plugins.inventory.toml

* Add note about what self.dump_funcs.update is doing

* Address some things

* A little extra comment

* Fix toml availability check

* Don't create an intermediate list

* Require toml file extension

* Add metadata

* Remove TOML docs from intro_inventory to prevent people from getting the wrong idea

* It's in defaults, remove note

* core supported, indicate very clearly that this is preview status

changelogs/fragments/toml-inventory.yaml

@@ -0,0 +1,2 @@
+minor_changes:
+- inventory - added new TOML inventory plugin (https://github.com/ansible/ansible/pull/41593)

lib/ansible/cli/inventory.py

@@ -94,6 +94,8 @@ class InventoryCLI(CLI):
         # graph
         self.parser.add_option("-y", "--yaml", action="store_true", default=False, dest='yaml',
                                help='Use YAML format instead of default JSON, ignored for --graph')
+        self.parser.add_option('--toml', action='store_true', default=False, dest='toml',
+                               help='Use TOML format instead of default JSON, ignored for --graph')
         self.parser.add_option("--vars", action="store_true", default=False, dest='show_vars',
                                help='Add vars to graph display, ignored unless used with --graph')
 
@@ -176,6 +178,8 @@ class InventoryCLI(CLI):
             top = self._get_group('all')
             if self.options.yaml:
                 results = self.yaml_inventory(top)
+            elif self.options.toml:
+                results = self.toml_inventory(top)
             else:
                 results = self.json_inventory(top)
             results = self.dump(results)
@@ -193,6 +197,13 @@ class InventoryCLI(CLI):
             import yaml
             from ansible.parsing.yaml.dumper import AnsibleDumper
             results = yaml.dump(stuff, Dumper=AnsibleDumper, default_flow_style=False)
+        elif self.options.toml:
+            from ansible.plugins.inventory.toml import toml_dumps, HAS_TOML
+            if not HAS_TOML:
+                raise AnsibleError(
+                    'The python "toml" library is required when using the TOML output format'
+                )
+            results = toml_dumps(stuff)
         else:
             import json
             from ansible.parsing.ajson import AnsibleJSONEncoder
@@ -385,3 +396,45 @@ class InventoryCLI(CLI):
             return results
 
         return format_group(top)
+
+    def toml_inventory(self, top):
+        seen = set()
+        has_ungrouped = bool(next(g.hosts for g in top.child_groups if g.name == 'ungrouped'))
+
+        def format_group(group):
+            results = {}
+            results[group.name] = {}
+
+            results[group.name]['children'] = []
+            for subgroup in sorted(group.child_groups, key=attrgetter('name')):
+                if subgroup.name == 'ungrouped' and not has_ungrouped:
+                    continue
+                if group.name != 'all':
+                    results[group.name]['children'].append(subgroup.name)
+                results.update(format_group(subgroup))
+
+            if group.name != 'all':
+                for host in sorted(group.hosts, key=attrgetter('name')):
+                    if host.name not in seen:
+                        seen.add(host.name)
+                        host_vars = self._get_host_variables(host=host)
+                        self._remove_internal(host_vars)
+                    else:
+                        host_vars = {}
+                    try:
+                        results[group.name]['hosts'][host.name] = host_vars
+                    except KeyError:
+                        results[group.name]['hosts'] = {host.name: host_vars}
+
+            if self.options.export:
+                results[group.name]['vars'] = self._get_group_variables(group)
+
+            self._remove_empty(results[group.name])
+            if not results[group.name]:
+                del results[group.name]
+
+            return results
+
+        results = format_group(top)
+
+        return results

lib/ansible/config/base.yml

@@ -1408,7 +1408,7 @@ INVENTORY_ANY_UNPARSED_IS_FAILED:
   version_added: "2.7"
 INVENTORY_ENABLED:
   name: Active Inventory plugins
-  default: ['host_list', 'script', 'yaml', 'ini', 'auto']
+  default: ['host_list', 'script', 'yaml', 'ini', 'toml', 'auto']
   description: List of enabled inventory plugins, it also determines the order in which they are used.
   env: [{name: ANSIBLE_INVENTORY_ENABLED}]
   ini:

lib/ansible/plugins/inventory/__init__.py

@@ -25,6 +25,7 @@ import re
 import string
 
 from ansible.errors import AnsibleError, AnsibleParserError
+from ansible.parsing.utils.addresses import parse_address
 from ansible.plugins import AnsiblePlugin
 from ansible.plugins.cache import InventoryFileCacheModule
 from ansible.module_utils._text import to_bytes, to_native
@@ -231,6 +232,31 @@ class BaseInventoryPlugin(AnsiblePlugin):
             if k in data:
                 self._options[k] = data.pop(k)
 
+    def _expand_hostpattern(self, hostpattern):
+        '''
+        Takes a single host pattern and returns a list of hostnames and an
+        optional port number that applies to all of them.
+        '''
+        # Can the given hostpattern be parsed as a host with an optional port
+        # specification?
+
+        try:
+            (pattern, port) = parse_address(hostpattern, allow_ranges=True)
+        except Exception:
+            # not a recognizable host pattern
+            pattern = hostpattern
+            port = None
+
+        # Once we have separated the pattern, we expand it into list of one or
+        # more hostnames, depending on whether it contains any [x:y] ranges.
+
+        if detect_range(pattern):
+            hostnames = expand_hostname_range(pattern)
+        else:
+            hostnames = [pattern]
+
+        return (hostnames, port)
+
     def clear_cache(self):
         pass
 

lib/ansible/plugins/inventory/advanced_host_list.py

@@ -25,8 +25,7 @@ import os
 
 from ansible.errors import AnsibleError, AnsibleParserError
 from ansible.module_utils._text import to_bytes, to_native
-from ansible.parsing.utils.addresses import parse_address
+from ansible.plugins.inventory import BaseInventoryPlugin
-from ansible.plugins.inventory import BaseInventoryPlugin, detect_range, expand_hostname_range
 
 
 class InventoryModule(BaseInventoryPlugin):
@@ -62,28 +61,3 @@ class InventoryModule(BaseInventoryPlugin):
                             self.inventory.add_host(host, group='ungrouped', port=port)
         except Exception as e:
             raise AnsibleParserError("Invalid data from string, could not parse: %s" % to_native(e))
-
-    def _expand_hostpattern(self, hostpattern):
-        '''
-        Takes a single host pattern and returns a list of hostnames and an
-        optional port number that applies to all of them.
-        '''
-        # Can the given hostpattern be parsed as a host with an optional port
-        # specification?
-
-        try:
-            (pattern, port) = parse_address(hostpattern, allow_ranges=True)
-        except:
-            # not a recognizable host pattern
-            pattern = hostpattern
-            port = None
-
-        # Once we have separated the pattern, we expand it into list of one or
-        # more hostnames, depending on whether it contains any [x:y] ranges.
-
-        if detect_range(pattern):
-            hostnames = expand_hostname_range(pattern)
-        else:
-            hostnames = [pattern]
-
-        return (hostnames, port)

lib/ansible/plugins/inventory/ini.py

@@ -73,8 +73,7 @@ EXAMPLES = '''
 import ast
 import re
 
-from ansible.plugins.inventory import BaseFileInventoryPlugin, detect_range, expand_hostname_range
+from ansible.plugins.inventory import BaseFileInventoryPlugin
-from ansible.parsing.utils.addresses import parse_address
 
 from ansible.errors import AnsibleError, AnsibleParserError
 from ansible.module_utils._text import to_bytes, to_text
@@ -311,32 +310,6 @@ class InventoryModule(BaseFileInventoryPlugin):
 
         return hostnames, port, variables
 
-    def _expand_hostpattern(self, hostpattern):
-        '''
-        Takes a single host pattern and returns a list of hostnames and an
-        optional port number that applies to all of them.
-        '''
-
-        # Can the given hostpattern be parsed as a host with an optional port
-        # specification?
-
-        try:
-            (pattern, port) = parse_address(hostpattern, allow_ranges=True)
-        except Exception:
-            # not a recognizable host pattern
-            pattern = hostpattern
-            port = None
-
-        # Once we have separated the pattern, we expand it into list of one or
-        # more hostnames, depending on whether it contains any [x:y] ranges.
-
-        if detect_range(pattern):
-            hostnames = expand_hostname_range(pattern)
-        else:
-            hostnames = [pattern]
-
-        return (hostnames, port)
-
     @staticmethod
     def _parse_value(v):
         '''

lib/ansible/plugins/inventory/toml.py

@@ -0,0 +1,268 @@
+# Copyright (c) 2018 Matt Martz <matt@sivel.net>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+ANSIBLE_METADATA = {'metadata_version': '1.1',
+                    'status': ['preview'],
+                    'supported_by': 'core'}
+
+DOCUMENTATION = '''
+    inventory: toml
+    version_added: "2.8"
+    short_description: Uses a specific TOML file as an inventory source.
+    description:
+        - TOML based inventory format
+        - File MUST have a valid '.toml' file extension
+    notes:
+        - Requires the 'toml' python library
+'''
+
+EXAMPLES = '''
+example1: |
+    [all.vars]
+    has_java = false
+
+    [web]
+    children = [
+        "apache",
+        "nginx"
+    ]
+    vars = { http_port = 8080, myvar = 23 }
+
+    [web.hosts]
+    host1 = {}
+    host2 = { ansible_port = 222 }
+
+    [apache.hosts]
+    tomcat1 = {}
+    tomcat2 = { myvar = 34 }
+    tomcat3 = { mysecret = "03#pa33w0rd" }
+
+    [nginx.hosts]
+    jenkins1 = {}
+
+    [nginx.vars]
+    has_java = true
+
+example2: |
+    [all.vars]
+    has_java = false
+
+    [web]
+    children = [
+        "apache",
+        "nginx"
+    ]
+
+    [web.vars]
+    http_port = 8080
+    myvar = 23
+
+    [web.hosts.host1]
+    [web.hosts.host2]
+    ansible_port = 222
+
+    [apache.hosts.tomcat1]
+
+    [apache.hosts.tomcat2]
+    myvar = 34
+
+    [apache.hosts.tomcat3]
+    mysecret = "03#pa33w0rd"
+
+    [nginx.hosts.jenkins1]
+
+    [nginx.vars]
+    has_java = true
+
+example3: |
+    [ungrouped.hosts]
+    host1 = {}
+    host2 = { ansible_host = "127.0.0.1", ansible_port = 44 }
+    host3 = { ansible_host = "127.0.0.1", ansible_port = 45 }
+
+    [g1.hosts]
+    host4 = {}
+
+    [g2.hosts]
+    host4 = {}
+'''
+
+import os
+
+from functools import partial
+
+from ansible.errors import AnsibleFileNotFound, AnsibleParserError
+from ansible.module_utils._text import to_bytes, to_native
+from ansible.module_utils.common._collections_compat import MutableMapping, MutableSequence
+from ansible.module_utils.six import string_types, text_type
+from ansible.parsing.yaml.objects import AnsibleSequence, AnsibleUnicode
+from ansible.plugins.inventory import BaseFileInventoryPlugin
+
+try:
+    import toml
+    HAS_TOML = True
+except ImportError:
+    HAS_TOML = False
+
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
+
+WARNING_MSG = (
+    'The TOML inventory format is marked as preview, which means that it is not guaranteed to have a backwards '
+    'compatible interface.'
+)
+
+
+if HAS_TOML and hasattr(toml, 'TomlEncoder'):
+    class AnsibleTomlEncoder(toml.TomlEncoder):
+        def __init__(self, *args, **kwargs):
+            super(AnsibleTomlEncoder, self).__init__(*args, **kwargs)
+            # Map our custom YAML object types to dump_funcs from ``toml``
+            self.dump_funcs.update({
+                AnsibleSequence: self.dump_funcs.get(list),
+                AnsibleUnicode: self.dump_funcs.get(str),
+            })
+            display.warning(WARNING_MSG)
+    toml_dumps = partial(toml.dumps, encoder=AnsibleTomlEncoder())
+else:
+    def toml_dumps(data):
+        display.warning(WARNING_MSG)
+        return toml.dumps(convert_yaml_objects_to_native(data))
+
+
+def convert_yaml_objects_to_native(obj):
+    """Older versions of the ``toml`` python library, don't have a pluggable
+    way to tell the encoder about custom types, so we need to ensure objects
+    that we pass are native types.
+
+    Only used on ``toml<0.10.0`` where ``toml.TomlEncoder`` is missing.
+
+    This function recurses an object and ensures we cast any of the types from
+    ``ansible.parsing.yaml.objects`` into their native types, effectively cleansing
+    the data before we hand it over to ``toml``
+
+    This function doesn't directly check for the types from ``ansible.parsing.yaml.objects``
+    but instead checks for the types those objects inherit from, to offer more flexibility.
+    """
+    if isinstance(obj, dict):
+        return dict((k, convert_yaml_objects_to_native(v)) for k, v in obj.items())
+    elif isinstance(obj, list):
+        return [convert_yaml_objects_to_native(v) for v in obj]
+    elif isinstance(obj, text_type):
+        return text_type(obj)
+    else:
+        return obj
+
+
+class InventoryModule(BaseFileInventoryPlugin):
+    NAME = 'toml'
+
+    def _parse_group(self, group, group_data):
+        if not isinstance(group_data, (MutableMapping, type(None))):
+            self.display.warning("Skipping '%s' as this is not a valid group definition" % group)
+            return
+
+        self.inventory.add_group(group)
+        if group_data is None:
+            return
+
+        for key, data in group_data.items():
+            if key == 'vars':
+                if not isinstance(data, MutableMapping):
+                    raise AnsibleParserError(
+                        'Invalid "vars" entry for "%s" group, requires a dict, found "%s" instead.' %
+                        (group, type(data))
+                    )
+                for var, value in data.items():
+                    self.inventory.set_variable(group, var, value)
+
+            elif key == 'children':
+                if not isinstance(data, MutableSequence):
+                    raise AnsibleParserError(
+                        'Invalid "children" entry for "%s" group, requires a list, found "%s" instead.' %
+                        (group, type(data))
+                    )
+                for subgroup in data:
+                    self._parse_group(subgroup, {})
+                    self.inventory.add_child(group, subgroup)
+
+            elif key == 'hosts':
+                if not isinstance(data, MutableMapping):
+                    raise AnsibleParserError(
+                        'Invalid "hosts" entry for "%s" group, requires a dict, found "%s" instead.' %
+                        (group, type(data))
+                    )
+                for host_pattern, value in data.items():
+                    hosts, port = self._expand_hostpattern(host_pattern)
+                    self._populate_host_vars(hosts, value, group, port)
+            else:
+                self.display.warning(
+                    'Skipping unexpected key "%s" in group "%s", only "vars", "children" and "hosts" are valid' %
+                    (key, group)
+                )
+
+    def _load_file(self, file_name):
+        if not file_name or not isinstance(file_name, string_types):
+            raise AnsibleParserError("Invalid filename: '%s'" % to_native(file_name))
+
+        b_file_name = to_bytes(self.loader.path_dwim(file_name))
+        if not self.loader.path_exists(b_file_name):
+            raise AnsibleFileNotFound("Unable to retrieve file contents", file_name=file_name)
+
+        try:
+            with open(b_file_name, 'r') as f:
+                return toml.load(f)
+        except toml.TomlDecodeError as e:
+            raise AnsibleParserError(
+                'TOML file (%s) is invalid: %s' % (file_name, to_native(e)),
+                orig_exc=e
+            )
+        except (IOError, OSError) as e:
+            raise AnsibleParserError(
+                "An error occurred while trying to read the file '%s': %s" % (file_name, to_native(e)),
+                orig_exc=e
+            )
+        except Exception as e:
+            raise AnsibleParserError(
+                "An unexpected error occurred while parsing the file '%s': %s" % (file_name, to_native(e)),
+                orig_exc=e
+            )
+
+    def parse(self, inventory, loader, path, cache=True):
+        ''' parses the inventory file '''
+        if not HAS_TOML:
+            raise AnsibleParserError(
+                'The TOML inventory plugin requires the python "toml" library'
+            )
+
+        display.warning(WARNING_MSG)
+
+        super(InventoryModule, self).parse(inventory, loader, path)
+        self.set_options()
+
+        try:
+            data = self._load_file(path)
+        except Exception as e:
+            raise AnsibleParserError(e)
+
+        if not data:
+            raise AnsibleParserError('Parsed empty TOML file')
+        elif data.get('plugin'):
+            raise AnsibleParserError('Plugin configuration TOML file, not TOML inventory')
+
+        for group_name in data:
+            self._parse_group(group_name, data[group_name])
+
+    def verify_file(self, path):
+        if super(InventoryModule, self).verify_file(path):
+            file_name, ext = os.path.splitext(path)
+            if ext == '.toml':
+                return True
+        return False

lib/ansible/plugins/inventory/yaml.py

@@ -64,8 +64,7 @@ from ansible.errors import AnsibleParserError
 from ansible.module_utils.six import string_types
 from ansible.module_utils._text import to_native
 from ansible.module_utils.common._collections_compat import MutableMapping
-from ansible.parsing.utils.addresses import parse_address
+from ansible.plugins.inventory import BaseFileInventoryPlugin
-from ansible.plugins.inventory import BaseFileInventoryPlugin, detect_range, expand_hostname_range
 
 
 class InventoryModule(BaseFileInventoryPlugin):
@@ -156,28 +155,3 @@ class InventoryModule(BaseFileInventoryPlugin):
         (hostnames, port) = self._expand_hostpattern(host_pattern)
 
         return hostnames, port
-
-    def _expand_hostpattern(self, hostpattern):
-        '''
-        Takes a single host pattern and returns a list of hostnames and an
-        optional port number that applies to all of them.
-        '''
-        # Can the given hostpattern be parsed as a host with an optional port
-        # specification?
-
-        try:
-            (pattern, port) = parse_address(hostpattern, allow_ranges=True)
-        except Exception:
-            # not a recognizable host pattern
-            pattern = hostpattern
-            port = None
-
-        # Once we have separated the pattern, we expand it into list of one or
-        # more hostnames, depending on whether it contains any [x:y] ranges.
-
-        if detect_range(pattern):
-            hostnames = expand_hostname_range(pattern)
-        else:
-            hostnames = [pattern]
-
-        return (hostnames, port)