Fix the way host and group vars are loaded

* In the VariableManager, we were not properly tracking if a file
  had already been loaded, so we continuously append data to the end
  of the list there for host and group vars, meaning large sets of data
  are duplicated multiple times
* In the inventory, we were merging the host/group vars with the vars
  local to the host needlessly, as the VariableManager already handles that.
  This leads to needless duplication of the data and makes combining the
  vars in VariableManager take even longer.
pull/16114/head
James Cammarata 9 years ago
parent f0023610e8
commit 273a965dcc

@ -573,9 +573,6 @@ class Inventory(object):
if self.parser is not None: if self.parser is not None:
vars = combine_vars(vars, self.parser.get_host_variables(host)) vars = combine_vars(vars, self.parser.get_host_variables(host))
# Read host_vars/ files
vars = combine_vars(vars, self.get_host_vars(host))
return vars return vars
def add_group(self, group): def add_group(self, group):
@ -697,16 +694,14 @@ class Inventory(object):
if found_group_vars: if found_group_vars:
self._group_vars_files = self._group_vars_files.union(found_group_vars) self._group_vars_files = self._group_vars_files.union(found_group_vars)
for group in self.groups.values(): for group in self.groups.values():
#group.vars = combine_vars(group.vars, self.get_group_vars(group, new_pb_basedir=True)) self.get_group_vars(group)
group.vars = combine_vars(group.vars, self.get_group_vars(group))
found_host_vars = self._find_host_vars_files(self._playbook_basedir) found_host_vars = self._find_host_vars_files(self._playbook_basedir)
if found_host_vars: if found_host_vars:
self._host_vars_files = self._find_host_vars_files(self._playbook_basedir) self._host_vars_files = self._find_host_vars_files(self._playbook_basedir)
# get host vars from host_vars/ files # get host vars from host_vars/ files
for host in self.get_hosts(): for host in self.get_hosts():
#host.vars = combine_vars(host.vars, self.get_host_vars(host, new_pb_basedir=True)) self.get_host_vars(host)
host.vars = combine_vars(host.vars, self.get_host_vars(host))
# invalidate cache # invalidate cache
self._vars_per_host = {} self._vars_per_host = {}
self._vars_per_group = {} self._vars_per_group = {}
@ -778,11 +773,11 @@ class Inventory(object):
if host is None and any(map(lambda ext: group.name + ext in self._group_vars_files, C.YAML_FILENAME_EXTENSIONS)): if host is None and any(map(lambda ext: group.name + ext in self._group_vars_files, C.YAML_FILENAME_EXTENSIONS)):
# load vars in dir/group_vars/name_of_group # load vars in dir/group_vars/name_of_group
base_path = to_unicode(os.path.abspath(os.path.join(to_bytes(basedir), b"group_vars/" + to_bytes(group.name))), errors='strict') base_path = to_unicode(os.path.abspath(os.path.join(to_bytes(basedir), b"group_vars/" + to_bytes(group.name))), errors='strict')
results = combine_vars(results, self._variable_manager.add_group_vars_file(base_path, self._loader)) self._variable_manager.add_group_vars_file(base_path, self._loader)
elif group is None and any(map(lambda ext: host.name + ext in self._host_vars_files, C.YAML_FILENAME_EXTENSIONS)): elif group is None and any(map(lambda ext: host.name + ext in self._host_vars_files, C.YAML_FILENAME_EXTENSIONS)):
# same for hostvars in dir/host_vars/name_of_host # same for hostvars in dir/host_vars/name_of_host
base_path = to_unicode(os.path.abspath(os.path.join(to_bytes(basedir), b"host_vars/" + to_bytes(host.name))), errors='strict') base_path = to_unicode(os.path.abspath(os.path.join(to_bytes(basedir), b"host_vars/" + to_bytes(host.name))), errors='strict')
results = combine_vars(results, self._variable_manager.add_host_vars_file(base_path, self._loader)) self._variable_manager.add_host_vars_file(base_path, self._loader)
# all done, results is a dictionary of variables for this particular host. # all done, results is a dictionary of variables for this particular host.
return results return results

@ -53,6 +53,11 @@ except ImportError:
VARIABLE_CACHE = dict() VARIABLE_CACHE = dict()
HOSTVARS_CACHE = dict() HOSTVARS_CACHE = dict()
class AnsibleInventoryVarsData(dict):
def __init__(self, *args, **kwargs):
super(AnsibleInventoryVarsData, self).__init__(*args, **kwargs)
self.path = None
def preprocess_vars(a): def preprocess_vars(a):
''' '''
Ensures that vars contained in the parameter passed in are Ensures that vars contained in the parameter passed in are
@ -549,8 +554,11 @@ class VariableManager:
if loader.path_exists(path): if loader.path_exists(path):
data = loader.load_from_file(path) data = loader.load_from_file(path)
name = self._get_inventory_basename(path) rval = AnsibleInventoryVarsData()
return (name, data) rval.path = path
if data is not None:
rval.update(data)
return rval
def add_host_vars_file(self, path, loader): def add_host_vars_file(self, path, loader):
''' '''
@ -559,14 +567,21 @@ class VariableManager:
the extension, for matching against a given inventory host name the extension, for matching against a given inventory host name
''' '''
(name, data) = self._load_inventory_file(path, loader) name = self._get_inventory_basename(path)
if data: if name not in self._host_vars_files:
if name not in self._host_vars_files: self._host_vars_files[name] = []
self._host_vars_files[name] = []
self._host_vars_files[name].append(data) for entry in self._host_vars_files[name]:
return data if entry.path == path:
data = entry
break
else: else:
return dict() data = self._load_inventory_file(path, loader)
if data:
self._host_vars_files[name].append(data)
return data
def add_group_vars_file(self, path, loader): def add_group_vars_file(self, path, loader):
''' '''
@ -575,14 +590,20 @@ class VariableManager:
the extension, for matching against a given inventory host name the extension, for matching against a given inventory host name
''' '''
(name, data) = self._load_inventory_file(path, loader) name = self._get_inventory_basename(path)
if data: if name not in self._group_vars_files:
if name not in self._group_vars_files: self._group_vars_files[name] = []
self._group_vars_files[name] = []
self._group_vars_files[name].append(data) for entry in self._group_vars_files[name]:
return data if entry.path == path:
data = entry
break
else: else:
return dict() data = self._load_inventory_file(path, loader)
if data:
self._group_vars_files[name].append(data)
return data
def clear_facts(self, hostname): def clear_facts(self, hostname):
''' '''

Loading…
Cancel
Save