Refactor vars_plugins (host/group_vars)

Split out parsing of vars files to per host and per group
parsing, instead of reparsing all groups for each host. This enhances
performance.

Extend vars_plugins' API with two new methods:
* get host variables: only parses host_vars
* get group variables: only parses group_vars for specific group
The initial run method is still used for backward compatibility.

Parse all vars_plugins at inventory initialisation, instead of
per host when touched first by runner. Here we can also loop through
all groups once easily, then parse them.
This also centralizes all parsing in the inventory constructor.

	modified:   bin/ansible
	modified:   bin/ansible-playbook
	modified:   lib/ansible/inventory/__init__.py
	modified:   lib/ansible/inventory/vars_plugins/group_vars.py

bin/ansible

@@ -136,7 +136,7 @@ class Cli(object):
             if not options.ask_vault_pass:
                 vault_pass = tmp_vault_pass
 
-        inventory_manager = inventory.Inventory(options.inventory)
+        inventory_manager = inventory.Inventory(options.inventory, vault_password=vault_pass)
         if options.subset:
             inventory_manager.subset(options.subset)
         hosts = inventory_manager.list_hosts(pattern)

bin/ansible-playbook

@@ -97,11 +97,6 @@ def main(args):
     if (options.ask_vault_pass and options.vault_password_file):
             parser.error("--ask-vault-pass and --vault-password-file are mutually exclusive")
 
-    inventory = ansible.inventory.Inventory(options.inventory)
-    inventory.subset(options.subset)
-    if len(inventory.list_hosts()) == 0:
-        raise errors.AnsibleError("provided hosts list is empty")
-
     sshpass = None
     sudopass = None
     su_pass = None
@@ -155,6 +150,11 @@ def main(args):
         if not (os.path.isfile(playbook) or stat.S_ISFIFO(os.stat(playbook).st_mode)):
             raise errors.AnsibleError("the playbook: %s does not appear to be a file" % playbook)
 
+    inventory = ansible.inventory.Inventory(options.inventory, vault_password=vault_pass)
+    inventory.subset(options.subset)
+    if len(inventory.list_hosts()) == 0:
+        raise errors.AnsibleError("provided hosts list is empty")
+
     # run all playbooks specified on the command line
     for playbook in args:
 

lib/ansible/inventory/__init__.py

@@ -39,13 +39,14 @@ class Inventory(object):
 
     __slots__ = [ 'host_list', 'groups', '_restriction', '_also_restriction', '_subset', 
                   'parser', '_vars_per_host', '_vars_per_group', '_hosts_cache', '_groups_list',
-                  '_pattern_cache', '_vars_plugins', '_playbook_basedir']
+                  '_pattern_cache', '_vault_password', '_vars_plugins', '_playbook_basedir']
 
-    def __init__(self, host_list=C.DEFAULT_HOST_LIST):
+    def __init__(self, host_list=C.DEFAULT_HOST_LIST, vault_password=None):
 
         # the host file file, or script path, or list of hosts
         # if a list, inventory data will NOT be loaded
         self.host_list = host_list
+        self._vault_password=vault_password
 
         # caching to avoid repeated calculations, particularly with
         # external inventory scripts.
@@ -140,6 +141,14 @@ class Inventory(object):
 
         self._vars_plugins = [ x for x in utils.plugins.vars_loader.all(self) ]
 
+        # get group vars from vars plugins
+        for group in self.groups:
+            group.vars = utils.combine_vars(group.vars, self.get_group_variables(group.name, self._vault_password))
+
+        # get host vars from vars plugins
+        for host in self.get_hosts():
+            host.vars = utils.combine_vars(host.vars, self.get_variables(host.name, self._vault_password))
+
 
     def _match(self, str, pattern_str):
         if pattern_str.startswith('~'):
@@ -370,16 +379,25 @@ class Inventory(object):
                 return group
         return None
 
-    def get_group_variables(self, groupname):
+    def get_group_variables(self, groupname, vault_password=None):
         if groupname not in self._vars_per_group:
-            self._vars_per_group[groupname] = self._get_group_variables(groupname)
+            self._vars_per_group[groupname] = self._get_group_variables(groupname, vault_password=vault_password)
         return self._vars_per_group[groupname]
 
-    def _get_group_variables(self, groupname):
+    def _get_group_variables(self, groupname, vault_password=None):
+
         group = self.get_group(groupname)
         if group is None:
             raise Exception("group not found: %s" % groupname)
-        return group.get_variables()
+
+        vars = {}
+        vars_results = [ plugin.get_group_vars(group, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_group_vars')]
+        for updated in vars_results:
+            if updated is not None:
+                vars.update(updated)
+
+        vars.update(group.get_variables())
+        return vars
 
     def get_variables(self, hostname, vault_password=None):
         if hostname not in self._vars_per_host:
@@ -393,14 +411,27 @@ class Inventory(object):
             raise errors.AnsibleError("host not found: %s" % hostname)
 
         vars = {}
-        vars_results = [ plugin.run(host, vault_password=vault_password) for plugin in self._vars_plugins ] 
+
+        # plugin.get_host_vars retrieves just vars for specific host
+        vars_results = [ plugin.get_host_vars(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_host_vars')]
+        for updated in vars_results:
+            if updated is not None:
+                vars = utils.combine_vars(vars, updated)
+
+        # plugin.run retrieves all vars (also from groups) for host
+        vars_results = [ plugin.run(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'run')]
         for updated in vars_results:
             if updated is not None:
                 vars = utils.combine_vars(vars, updated)
 
         vars = utils.combine_vars(vars, host.get_variables())
+
+        # still need to check InventoryParser per host vars
+        # which actually means InventoryScript per host,
+        # which is not performant
         if self.parser is not None:
             vars = utils.combine_vars(vars, self.parser.get_host_variables(host))
+
         return vars
 
     def add_group(self, group):

lib/ansible/inventory/vars_plugins/group_vars.py

@@ -143,28 +143,33 @@ class VarsModule(object):
         """ constructor """
 
         self.inventory = inventory
+        self.inventory_basedir = inventory.basedir()
+        # There's no playbook initialized yet:
+        self.pb_basedir = None
 
-    def run(self, host, vault_password=None):
 
-        """ main body of the plugin, does actual loading """
+    def get_host_vars(self, host, vault_password=None):
+        return self._get_vars(host=host, group=None, vault_password=vault_password)
 
-        inventory = self.inventory
-        basedir = inventory.playbook_basedir()
-        if basedir is not None: 
-            basedir = os.path.abspath(basedir)
-        self.pb_basedir = basedir
 
-        # sort groups by depth so deepest groups can override the less deep ones
-        groupz = sorted(inventory.groups_for_host(host.name), key=lambda g: g.depth)
-        groups = [ g.name for g in groupz ]
-        inventory_basedir = inventory.basedir()
+    def get_group_vars(self, group, vault_password=None):
+        return self._get_vars(host=None, group=group, vault_password=vault_password)
+
+
+    def _get_vars(self, host=None, group=None, vault_password=None):
+        """ main body of the plugin, does actual loading"""
+
+        if self.pb_basedir is None:
+            pb_basedir = self.inventory.playbook_basedir()
+            if pb_basedir is not None:
+                pb_basedir = os.path.abspath(pb_basedir)
+                self.pb_basedir = pb_basedir
 
         results = {}
         scan_pass = 0
 
         # look in both the inventory base directory and the playbook base directory
-        for basedir in [ inventory_basedir, self.pb_basedir ]:
-
+        for basedir in [self.inventory_basedir, self.pb_basedir ]:
 
             # this can happen from particular API usages, particularly if not run
             # from /usr/bin/ansible-playbook
@@ -178,17 +183,18 @@ class VarsModule(object):
                 continue
 
             # save work of second scan if the directories are the same
-            if inventory_basedir == self.pb_basedir and scan_pass != 1:
+            if self.inventory_basedir == self.pb_basedir and scan_pass != 1:
                 continue
 
-            # load vars in dir/group_vars/name_of_group
-            for group in groups:
-                base_path = os.path.join(basedir, "group_vars/%s" % group)
+            if group and host is None:
+                # load vars in dir/group_vars/name_of_group
+                base_path = os.path.join(basedir, "group_vars/%s" % group.name)
                 results = _load_vars(base_path, results, vault_password=vault_password)
 
-            # same for hostvars in dir/host_vars/name_of_host
-            base_path = os.path.join(basedir, "host_vars/%s" % host.name)
-            results = _load_vars(base_path, results, vault_password=vault_password)
+            elif host and group is None:
+                # same for hostvars in dir/host_vars/name_of_host
+                base_path = os.path.join(basedir, "host_vars/%s" % host.name)
+                results = _load_vars(base_path, results, vault_password=vault_password)
 
         # all done, results is a dictionary of variables for this particular host.
         return results