From d3eaa1b79e76e4a3967d34c18f95ff3a14cca68c Mon Sep 17 00:00:00 2001 From: Serge van Ginderachter Date: Thu, 20 Mar 2014 01:40:06 +0100 Subject: [PATCH] InventoryDir: refactor logic Make sure all hosts and groups are unique objects and that those are referenced uniquely everywhere. Also fixes test_dir_inventory unit tests which were broken after previous patches. modified: lib/ansible/inventory/dir.py --- lib/ansible/inventory/dir.py | 171 +++++++++++++++++++++++++++++------ test/units/TestInventory.py | 2 +- 2 files changed, 146 insertions(+), 27 deletions(-) diff --git a/lib/ansible/inventory/dir.py b/lib/ansible/inventory/dir.py index 74a8eacba9d..f2c32a063e1 100644 --- a/lib/ansible/inventory/dir.py +++ b/lib/ansible/inventory/dir.py @@ -1,4 +1,5 @@ # (c) 2013, Daniel Hokka Zakrisson +# (c) 2014, Serge van Ginderachter # # This file is part of Ansible # @@ -36,9 +37,9 @@ class InventoryDirectory(object): self.parsers = [] self.hosts = {} self.groups = {} - + for i in self.names: - + if i.endswith("~") or i.endswith(".orig") or i.endswith(".bak"): continue if i.endswith(".ini"): @@ -62,31 +63,149 @@ class InventoryDirectory(object): else: parser = InventoryParser(filename=fullpath) self.parsers.append(parser) - # This takes a lot of code because we can't directly use any of the objects, as they have to blend - for name, group in parser.groups.iteritems(): - if name not in self.groups: - self.groups[name] = group - else: - # group is already there, copy variables - for k, v in group.get_variables().iteritems(): - self.groups[name].set_variable(k, v) + + # retrieve all groups and hosts form the parser and add them to + # self, don't look at group lists yet, to avoid + # recursion trouble, but just make sure all objects exist in self + newgroups = parser.groups.values() + for group in newgroups: for host in group.hosts: - if host.name not in self.hosts: - self.hosts[host.name] = host - else: - # host is already there, copy variables - # note: depth numbers on duplicates may be bogus - for k, v in host.vars.iteritems(): - self.hosts[host.name].set_variable(k, v) - # host can already exist from other source (same name, - # different object): - if host.name not in [h.name for h in self.groups[name].hosts]: - self.groups[name].add_host(self.hosts[host.name]) - - # This needs to be a second loop to ensure all the parent groups exist - for name, group in parser.groups.iteritems(): - for parent in group.parent_groups: - self.groups[parent.name].add_child_group(self.groups[name]) + self._add_host(host) + for group in newgroups: + self._add_group(group) + + # now check the objects lists so they contain only objects from + # self; membership data in groups is already fine (except all & + # ungrouped, see later), but might still reference objects not in self + for group in self.groups.values(): + # iterate on a copy of the lists, as those lists get changed in + # the loop + # list with group's child group objects: + for child in group.child_groups[:]: + if child != self.groups[child.name]: + group.child_groups.remove(child) + group.child_groups.append(self.groups[child.name]) + # list with group's parent group objects: + for parent in group.parent_groups[:]: + if parent != self.groups[parent.name]: + group.parent_groups.remove(parent) + group.parent_groups.append(self.groups[parent.name]) + # list with group's host objects: + for host in group.hosts[:]: + if host != self.hosts[host.name]: + group.hosts.remove(host) + group.hosts.append(self.hosts[host.name]) + # also check here that the group that contains host, is + # also contained in the host's group list + if group not in self.hosts[host.name].groups: + self.hosts[host.name].groups.append(group) + + # extra checks on special groups all and ungrouped + # remove hosts from 'ungrouped' if they became member of other groups + if 'ungrouped' in self.groups: + ungrouped = self.groups['ungrouped'] + # loop on a copy of ungrouped hosts, as we want to change that list + for host in ungrouped.hosts[:]: + if len(host.groups) > 1: + host.groups.remove(ungrouped) + ungrouped.hosts.remove(host) + + # remove hosts from 'all' if they became member of other groups + # all should only contain direct children, not grandchildren + # direct children should have dept == 1 + if 'all' in self.groups: + allgroup = self.groups['all' ] + # loop on a copy of all's child groups, as we want to change that list + for group in allgroup.child_groups[:]: + # groups might once have beeen added to all, and later be added + # to another group: we need to remove the link wit all then + if len(group.parent_groups) > 1: + # real children of all have just 1 parent, all + # this one has more, so not a direct child of all anymore + group.parent_groups.remove(allgroup) + allgroup.child_groups.remove(group) + elif allgroup not in group.parent_groups: + # this group was once added to all, but doesn't list it as + # a parent any more; the info in the group is the correct + # info + allgroup.child_groups.remove(group) + + + def _add_group(self, group): + """ Merge an existing group or add a new one; + Track parent and child groups, and hosts of the new one """ + + if group.name not in self.groups: + # it's brand new, add him! + self.groups[group.name] = group + if self.groups[group.name] != group: + # different object, merge + self._merge_groups(self.groups[group.name], group) + + def _add_host(self, host): + if host.name not in self.hosts: + # Papa's got a brand new host + self.hosts[host.name] = host + if self.hosts[host.name] != host: + # different object, merge + self._merge_hosts(self.hosts[host.name], host) + + def _merge_groups(self, group, newgroup): + """ Merge all of instance newgroup into group, + update parent/child relationships + group lists may still contain group objects that exist in self with + same name, but was instanciated as a different object in some other + inventory parser; these are handled later """ + + # name + if group.name != newgroup.name: + raise errors.AnsibleError("Cannot merge group %s with %s" % (group.name, newgroup.name)) + + # depth + group.depth = max([group.depth, newgroup.depth]) + + # hosts list (host objects are by now already added to self.hosts) + for host in newgroup.hosts: + grouphosts = dict([(h.name, h) for h in group.hosts]) + if host.name in grouphosts: + # same host name but different object, merge + self._merge_hosts(grouphosts[host.name], host) + else: + # new membership + group.add_host(self.hosts[host.name]) + + # group child membership relation + for newchild in newgroup.child_groups: + # dict with existing child groups: + childgroups = dict([(g.name, g) for g in group.child_groups]) + # check if child of new group is already known as a child + if newchild.name not in childgroups: + self.groups[group.name].add_child_group(newchild) + + # group parent membership relation + for newparent in newgroup.parent_groups: + # dict with existing parent groups: + parentgroups = dict([(g.name, g) for g in group.parent_groups]) + # check if parent of new group is already known as a parent + if newparent.name not in parentgroups: + if newparent.name not in self.groups: + # group does not exist yet in self, import him + self.groups[newparent.name] = newparent + # group now exists but not yet as a parent here + self.groups[newparent.name].add_child_group(group) + + # variables + group.vars = utils.combine_vars(group.vars, newgroup.vars) + + def _merge_hosts(self,host, newhost): + """ Merge all of instance newhost into host """ + + # name + if host.name != newhost.name: + raise errors.AnsibleError("Cannot merge host %s with %s" % (host.name, newhost.name)) + + # variables + host.vars = utils.combine_vars(host.vars, newhost.vars) def get_host_variables(self, host): """ Gets additional host variables from all inventories """ diff --git a/test/units/TestInventory.py b/test/units/TestInventory.py index 4aae739a233..7f20af6bb64 100644 --- a/test/units/TestInventory.py +++ b/test/units/TestInventory.py @@ -425,7 +425,7 @@ class TestInventory(unittest.TestCase): expected_vars = {'inventory_hostname': 'zeus', 'inventory_hostname_short': 'zeus', - 'group_names': ['greek', 'major-god', 'ungrouped'], + 'group_names': ['greek', 'major-god'], 'var_a': '3#4'} print "HOST VARS=%s" % host_vars