allow constructed to use vars plugin (#73418)

Allow constructed to optionally use vars plugin data

* mostly for those looking to leverage group_vars/ and host_vars/
* limited to already processed sources

changelogs/fragments/constructed_vars_plugins.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - The constructed inventory plugin has new option to force using vars plugins on previouslly processed inventory sources.

lib/ansible/inventory/data.py

@@ -51,6 +51,7 @@ class InventoryData(object):
         self.localhost = None
 
         self.current_source = None
+        self.processed_sources = []
 
         # Always create the 'all' and 'ungrouped' groups,
         for group in ('all', 'ungrouped'):
@@ -64,6 +65,7 @@ class InventoryData(object):
             'hosts': self.hosts,
             'local': self.localhost,
             'source': self.current_source,
+            'processed_sources': self.processed_sources
         }
         return data
 
@@ -73,6 +75,7 @@ class InventoryData(object):
         self.groups = data.get('groups')
         self.localhost = data.get('local')
         self.current_source = data.get('source')
+        self.processed_sources = data.get('processed_sources')
 
     def _create_implicit_localhost(self, pattern):
 

lib/ansible/inventory/manager.py

@@ -307,7 +307,9 @@ class InventoryManager(object):
                 else:
                     display.vvv("%s declined parsing %s as it did not pass its verify_file() method" % (plugin_name, source))
 
-        if not parsed:
+        if parsed:
+            self._inventory.processed_sources.append(self._inventory.current_source)
+        else:
             # only warn/error if NOT using the default or using it and the file is present
             # TODO: handle 'non file' inventorya and detect vs hardcode default
             if source != '/etc/ansible/hosts' or os.path.exists(source):

lib/ansible/plugins/inventory/__init__.py

@@ -381,11 +381,12 @@ class Constructable(object):
                     continue
                 self.inventory.set_variable(host, varname, composite)
 
-    def _add_host_to_composed_groups(self, groups, variables, host, strict=False):
+    def _add_host_to_composed_groups(self, groups, variables, host, strict=False, fetch_hostvars=True):
         ''' helper to create complex groups for plugins based on jinja2 conditionals, hosts that meet the conditional are added to group'''
         # process each 'group entry'
         if groups and isinstance(groups, dict):
-            variables = combine_vars(variables, self.inventory.get_host(host).get_vars())
+            if fetch_hostvars:
+                variables = combine_vars(variables, self.inventory.get_host(host).get_vars())
             self.templar.available_variables = variables
             for group_name in groups:
                 conditional = "{%% if %s %%} True {%% else %%} False {%% endif %%}" % groups[group_name]
@@ -403,13 +404,14 @@ class Constructable(object):
                     # add host to group
                     self.inventory.add_child(group_name, host)
 
-    def _add_host_to_keyed_groups(self, keys, variables, host, strict=False):
+    def _add_host_to_keyed_groups(self, keys, variables, host, strict=False, fetch_hostvars=True):
         ''' helper to create groups for plugins based on variable values and add the corresponding hosts to it'''
         if keys and isinstance(keys, list):
             for keyed in keys:
                 if keyed and isinstance(keyed, dict):
 
-                    variables = combine_vars(variables, self.inventory.get_host(host).get_vars())
+                    if fetch_hostvars:
+                        variables = combine_vars(variables, self.inventory.get_host(host).get_vars())
                     try:
                         key = self._compose(keyed.get('key'), variables)
                     except Exception as e:

lib/ansible/plugins/inventory/constructed.py

@@ -19,6 +19,16 @@ DOCUMENTATION = '''
             description: token that ensures this is a source file for the 'constructed' plugin.
             required: True
             choices: ['constructed']
+        use_vars_plugins:
+            description:
+                - Normally, for performance reasons, vars plugins get executed after the inventory sources complete the base inventory,
+                  this option allows for getting vars related to hosts/groups from those plugins.
+                - The host_group_vars (enabled by default) 'vars plugin' is the one responsible for reading host_vars/ and group_vars/ directories.
+                - This will not execute plugins that are not supposed to execute at the 'inventory' stage, see vars plugins docs for details.
+            required: false
+            default: false
+            type: boolean
+            version_added: '2.11'
     extends_documentation_fragment:
       - constructed
 '''
@@ -70,12 +80,13 @@ EXAMPLES = r'''
 import os
 
 from ansible import constants as C
-from ansible.errors import AnsibleParserError
+from ansible.errors import AnsibleParserError, AnsibleOptionsError
 from ansible.inventory.helpers import get_group_vars
 from ansible.plugins.inventory import BaseInventoryPlugin, Constructable
 from ansible.module_utils._text import to_native
 from ansible.utils.vars import combine_vars
 from ansible.vars.fact_cache import FactCache
+from ansible.vars.plugins import get_vars_from_inventory_sources
 
 
 class InventoryModule(BaseInventoryPlugin, Constructable):
@@ -100,6 +111,28 @@ class InventoryModule(BaseInventoryPlugin, Constructable):
 
         return valid
 
+    def get_all_host_vars(self, host, loader, sources):
+        ''' requires host object '''
+        return combine_vars(self.host_groupvars(host, loader, sources), self.host_vars(host, loader, sources))
+
+    def host_groupvars(self, host, loader, sources):
+        ''' requires host object '''
+        gvars = get_group_vars(host.get_groups())
+
+        if self.get_option('use_vars_plugins'):
+            gvars = combine_vars(gvars, get_vars_from_inventory_sources(loader, sources, host.get_groups(), 'all'))
+
+        return gvars
+
+    def host_vars(self, host, loader, sources):
+        ''' requires host object '''
+        hvars = host.get_vars()
+
+        if self.get_option('use_vars_plugins'):
+            hvars = combine_vars(hvars, get_vars_from_inventory_sources(loader, sources, [host], 'all'))
+
+        return hvars
+
     def parse(self, inventory, loader, path, cache=False):
         ''' parses the inventory file '''
 
@@ -107,6 +140,13 @@ class InventoryModule(BaseInventoryPlugin, Constructable):
 
         self._read_config_data(path)
 
+        sources = []
+        try:
+            sources = inventory.processed_sources
+        except AttributeError:
+            if self.get_option('use_vars_plugins'):
+                raise AnsibleOptionsError("The option use_vars_plugins requires ansible >= 2.11.")
+
         strict = self.get_option('strict')
         fact_cache = FactCache()
         try:
@@ -114,7 +154,7 @@ class InventoryModule(BaseInventoryPlugin, Constructable):
             for host in inventory.hosts:
 
                 # get available variables to templar
-                hostvars = combine_vars(get_group_vars(inventory.hosts[host].get_groups()), inventory.hosts[host].get_vars())
+                hostvars = self.get_all_host_vars(inventory.hosts[host], loader, sources)
                 if host in fact_cache:  # adds facts if cache is active
                     hostvars = combine_vars(hostvars, fact_cache[host])
 
@@ -122,15 +162,15 @@ class InventoryModule(BaseInventoryPlugin, Constructable):
                 self._set_composite_vars(self.get_option('compose'), hostvars, host, strict=strict)
 
                 # refetch host vars in case new ones have been created above
-                hostvars = combine_vars(get_group_vars(inventory.hosts[host].get_groups()), inventory.hosts[host].get_vars())
+                hostvars = self.get_all_host_vars(inventory.hosts[host], loader, sources)
                 if host in self._cache:  # adds facts if cache is active
                     hostvars = combine_vars(hostvars, self._cache[host])
 
                 # constructed groups based on conditionals
-                self._add_host_to_composed_groups(self.get_option('groups'), hostvars, host, strict=strict)
+                self._add_host_to_composed_groups(self.get_option('groups'), hostvars, host, strict=strict, fetch_hostvars=False)
 
                 # constructed groups based variable values
-                self._add_host_to_keyed_groups(self.get_option('keyed_groups'), hostvars, host, strict=strict)
+                self._add_host_to_keyed_groups(self.get_option('keyed_groups'), hostvars, host, strict=strict, fetch_hostvars=False)
 
         except Exception as e:
-            raise AnsibleParserError("failed to parse %s: %s " % (to_native(path), to_native(e)))
+            raise AnsibleParserError("failed to parse %s: %s " % (to_native(path), to_native(e)), orig_exc=e)

test/integration/targets/inventory_constructed/invs/1/group_vars/stuff.yml

@@ -0,0 +1 @@
+iamdefined: group4testing

test/integration/targets/inventory_constructed/invs/1/host_vars/testing.yml

@@ -0,0 +1 @@
+hola: lola

test/integration/targets/inventory_constructed/invs/1/one.yml

@@ -0,0 +1,5 @@
+all:
+  children:
+    stuff:
+      hosts:
+        testing:

test/integration/targets/inventory_constructed/invs/2/constructed.yml

@@ -0,0 +1,7 @@
+plugin: constructed
+use_vars_plugins: true
+keyed_groups:
+  - key: iamdefined
+    prefix: c
+  - key: hola
+    prefix: c

test/integration/targets/inventory_constructed/runme.sh

@@ -23,3 +23,10 @@ grep '@key0separatorvalue0' out.txt
 grep '@prefix_hostvalue1' out.txt
 grep '@prefix_item0' out.txt
 grep '@prefix_key0_value0' out.txt
+
+
+# test using use_vars_plugins
+ansible-inventory -i invs/1/one.yml -i invs/2/constructed.yml --graph | tee out.txt
+
+grep '@c_lola' out.txt
+grep '@c_group4testing' out.txt