From f6a55a35525a6ca0b56ad856f2ab11bbd226c1b4 Mon Sep 17 00:00:00 2001 From: Serge van Ginderachter Date: Mon, 17 Mar 2014 18:05:30 +0100 Subject: [PATCH] Refactor vars_plugins (host/group_vars) Split out parsing of vars files to per host and per group parsing, instead of reparsing all groups for each host. This enhances performance. Extend vars_plugins' API with two new methods: * get host variables: only parses host_vars * get group variables: only parses group_vars for specific group The initial run method is still used for backward compatibility. Parse all vars_plugins at inventory initialisation, instead of per host when touched first by runner. Here we can also loop through all groups once easily, then parse them. This also centralizes all parsing in the inventory constructor. modified: bin/ansible modified: bin/ansible-playbook modified: lib/ansible/inventory/__init__.py modified: lib/ansible/inventory/vars_plugins/group_vars.py --- bin/ansible | 2 +- bin/ansible-playbook | 10 ++-- lib/ansible/inventory/__init__.py | 45 +++++++++++++++--- .../inventory/vars_plugins/group_vars.py | 46 +++++++++++-------- 4 files changed, 70 insertions(+), 33 deletions(-) diff --git a/bin/ansible b/bin/ansible index 1e2540fafb7..7e767b2f7db 100755 --- a/bin/ansible +++ b/bin/ansible @@ -136,7 +136,7 @@ class Cli(object): if not options.ask_vault_pass: vault_pass = tmp_vault_pass - inventory_manager = inventory.Inventory(options.inventory) + inventory_manager = inventory.Inventory(options.inventory, vault_password=vault_pass) if options.subset: inventory_manager.subset(options.subset) hosts = inventory_manager.list_hosts(pattern) diff --git a/bin/ansible-playbook b/bin/ansible-playbook index f54c17d7aa7..d91e2d94847 100755 --- a/bin/ansible-playbook +++ b/bin/ansible-playbook @@ -97,11 +97,6 @@ def main(args): if (options.ask_vault_pass and options.vault_password_file): parser.error("--ask-vault-pass and --vault-password-file are mutually exclusive") - inventory = ansible.inventory.Inventory(options.inventory) - inventory.subset(options.subset) - if len(inventory.list_hosts()) == 0: - raise errors.AnsibleError("provided hosts list is empty") - sshpass = None sudopass = None su_pass = None @@ -155,6 +150,11 @@ def main(args): if not (os.path.isfile(playbook) or stat.S_ISFIFO(os.stat(playbook).st_mode)): raise errors.AnsibleError("the playbook: %s does not appear to be a file" % playbook) + inventory = ansible.inventory.Inventory(options.inventory, vault_password=vault_pass) + inventory.subset(options.subset) + if len(inventory.list_hosts()) == 0: + raise errors.AnsibleError("provided hosts list is empty") + # run all playbooks specified on the command line for playbook in args: diff --git a/lib/ansible/inventory/__init__.py b/lib/ansible/inventory/__init__.py index a8cca8faaf2..06acf87e484 100644 --- a/lib/ansible/inventory/__init__.py +++ b/lib/ansible/inventory/__init__.py @@ -39,13 +39,14 @@ class Inventory(object): __slots__ = [ 'host_list', 'groups', '_restriction', '_also_restriction', '_subset', 'parser', '_vars_per_host', '_vars_per_group', '_hosts_cache', '_groups_list', - '_pattern_cache', '_vars_plugins', '_playbook_basedir'] + '_pattern_cache', '_vault_password', '_vars_plugins', '_playbook_basedir'] - def __init__(self, host_list=C.DEFAULT_HOST_LIST): + def __init__(self, host_list=C.DEFAULT_HOST_LIST, vault_password=None): # the host file file, or script path, or list of hosts # if a list, inventory data will NOT be loaded self.host_list = host_list + self._vault_password=vault_password # caching to avoid repeated calculations, particularly with # external inventory scripts. @@ -140,6 +141,14 @@ class Inventory(object): self._vars_plugins = [ x for x in utils.plugins.vars_loader.all(self) ] + # get group vars from vars plugins + for group in self.groups: + group.vars = utils.combine_vars(group.vars, self.get_group_variables(group.name, self._vault_password)) + + # get host vars from vars plugins + for host in self.get_hosts(): + host.vars = utils.combine_vars(host.vars, self.get_variables(host.name, self._vault_password)) + def _match(self, str, pattern_str): if pattern_str.startswith('~'): @@ -370,16 +379,25 @@ class Inventory(object): return group return None - def get_group_variables(self, groupname): + def get_group_variables(self, groupname, vault_password=None): if groupname not in self._vars_per_group: - self._vars_per_group[groupname] = self._get_group_variables(groupname) + self._vars_per_group[groupname] = self._get_group_variables(groupname, vault_password=vault_password) return self._vars_per_group[groupname] - def _get_group_variables(self, groupname): + def _get_group_variables(self, groupname, vault_password=None): + group = self.get_group(groupname) if group is None: raise Exception("group not found: %s" % groupname) - return group.get_variables() + + vars = {} + vars_results = [ plugin.get_group_vars(group, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_group_vars')] + for updated in vars_results: + if updated is not None: + vars.update(updated) + + vars.update(group.get_variables()) + return vars def get_variables(self, hostname, vault_password=None): if hostname not in self._vars_per_host: @@ -393,14 +411,27 @@ class Inventory(object): raise errors.AnsibleError("host not found: %s" % hostname) vars = {} - vars_results = [ plugin.run(host, vault_password=vault_password) for plugin in self._vars_plugins ] + + # plugin.get_host_vars retrieves just vars for specific host + vars_results = [ plugin.get_host_vars(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_host_vars')] + for updated in vars_results: + if updated is not None: + vars = utils.combine_vars(vars, updated) + + # plugin.run retrieves all vars (also from groups) for host + vars_results = [ plugin.run(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'run')] for updated in vars_results: if updated is not None: vars = utils.combine_vars(vars, updated) vars = utils.combine_vars(vars, host.get_variables()) + + # still need to check InventoryParser per host vars + # which actually means InventoryScript per host, + # which is not performant if self.parser is not None: vars = utils.combine_vars(vars, self.parser.get_host_variables(host)) + return vars def add_group(self, group): diff --git a/lib/ansible/inventory/vars_plugins/group_vars.py b/lib/ansible/inventory/vars_plugins/group_vars.py index 93edceeecb5..2e8c7511220 100644 --- a/lib/ansible/inventory/vars_plugins/group_vars.py +++ b/lib/ansible/inventory/vars_plugins/group_vars.py @@ -143,28 +143,33 @@ class VarsModule(object): """ constructor """ self.inventory = inventory + self.inventory_basedir = inventory.basedir() + # There's no playbook initialized yet: + self.pb_basedir = None - def run(self, host, vault_password=None): - """ main body of the plugin, does actual loading """ + def get_host_vars(self, host, vault_password=None): + return self._get_vars(host=host, group=None, vault_password=vault_password) - inventory = self.inventory - basedir = inventory.playbook_basedir() - if basedir is not None: - basedir = os.path.abspath(basedir) - self.pb_basedir = basedir - # sort groups by depth so deepest groups can override the less deep ones - groupz = sorted(inventory.groups_for_host(host.name), key=lambda g: g.depth) - groups = [ g.name for g in groupz ] - inventory_basedir = inventory.basedir() + def get_group_vars(self, group, vault_password=None): + return self._get_vars(host=None, group=group, vault_password=vault_password) + + + def _get_vars(self, host=None, group=None, vault_password=None): + """ main body of the plugin, does actual loading""" + + if self.pb_basedir is None: + pb_basedir = self.inventory.playbook_basedir() + if pb_basedir is not None: + pb_basedir = os.path.abspath(pb_basedir) + self.pb_basedir = pb_basedir results = {} scan_pass = 0 # look in both the inventory base directory and the playbook base directory - for basedir in [ inventory_basedir, self.pb_basedir ]: - + for basedir in [self.inventory_basedir, self.pb_basedir ]: # this can happen from particular API usages, particularly if not run # from /usr/bin/ansible-playbook @@ -178,17 +183,18 @@ class VarsModule(object): continue # save work of second scan if the directories are the same - if inventory_basedir == self.pb_basedir and scan_pass != 1: + if self.inventory_basedir == self.pb_basedir and scan_pass != 1: continue - # load vars in dir/group_vars/name_of_group - for group in groups: - base_path = os.path.join(basedir, "group_vars/%s" % group) + if group and host is None: + # load vars in dir/group_vars/name_of_group + base_path = os.path.join(basedir, "group_vars/%s" % group.name) results = _load_vars(base_path, results, vault_password=vault_password) - # same for hostvars in dir/host_vars/name_of_host - base_path = os.path.join(basedir, "host_vars/%s" % host.name) - results = _load_vars(base_path, results, vault_password=vault_password) + elif host and group is None: + # same for hostvars in dir/host_vars/name_of_host + base_path = os.path.join(basedir, "host_vars/%s" % host.name) + results = _load_vars(base_path, results, vault_password=vault_password) # all done, results is a dictionary of variables for this particular host. return results