From b8865314ec196cb6b46b05cad0db3d3774205e8b Mon Sep 17 00:00:00 2001 From: Will West Date: Sat, 17 Jan 2015 11:49:13 -0600 Subject: [PATCH] improve ansible start up time for very large inventory first off, we add an oddly slow basic test of 10k item inventory Before: ``` Ran 229 tests in 13.214s OK real 0m13.403s user 0m12.106s sys 0m1.155s ``` After: ``` Ran 230 tests in 21.328s OK real 0m21.516s user 0m20.099s sys 0m1.275s ``` since that seems like a bit long for the test to add to runtime, lets profile `python -m cProfile -s time ./bin/ansible all -i test/units/inventory_test_data/huge_range --list-hosts` Before: ``` 1272607 function calls (1259689 primitive calls) in 8.497 seconds Ordered by: internal time ncalls tottime percall cumtime percall filename:lineno(function) 10000 4.393 0.000 4.396 0.000 __init__.py:395(_get_host) 20000 2.695 0.000 2.697 0.000 __init__.py:341(__append_host_to_results) 40369 0.113 0.000 0.113 0.000 {posix.lstat} 50006 0.102 0.000 0.153 0.000 __init__.py:1490(combine_vars) 40008 0.089 0.000 0.202 0.000 __init__.py:1546(_load_vars_from_path) 20195 0.088 0.000 0.088 0.000 {posix.stat} 10011 0.087 0.000 0.087 0.000 {posix.getcwd} ``` The top two lines are promising optimization targets - populate Inventory's host cache more in _get_host, as we are looping over all the groups anyways. - eliminate duplicate check of whether we've already included a host in the construction around __append_host_to_results we can infer presence of a host in the results list implies the presence of its name in the hostnames set, allowing us to only to the less expensive of the two checks After: ``` 1252610 function calls (1239692 primitive calls) in 1.320 seconds Ordered by: internal time ncalls tottime percall cumtime percall filename:lineno(function) 40369 0.105 0.000 0.105 0.000 {posix.lstat} 50006 0.094 0.000 0.141 0.000 __init__.py:1490(combine_vars) 40008 0.081 0.000 0.184 0.000 __init__.py:1546(_load_vars_from_path) 10011 0.080 0.000 0.080 0.000 {posix.getcwd} 20195 0.074 0.000 0.074 0.000 {posix.stat} 10002 0.069 0.000 0.261 0.000 __init__.py:1517(load_vars) ``` --- lib/ansible/inventory/__init__.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/ansible/inventory/__init__.py b/lib/ansible/inventory/__init__.py index dd9fb92db31..60c7eae67ba 100644 --- a/lib/ansible/inventory/__init__.py +++ b/lib/ansible/inventory/__init__.py @@ -351,7 +351,7 @@ class Inventory(object): pattern = pattern.replace("!","").replace("&", "") def __append_host_to_results(host): - if host not in results and host.name not in hostnames: + if host.name not in hostnames: hostnames.add(host.name) results.append(host) @@ -413,12 +413,13 @@ class Inventory(object): if host.name in self.LOCALHOST_ALIASES: return host return self._create_implicit_localhost(hostname) - else: - for group in self.groups: - for host in group.get_hosts(): - if hostname == host.name: - return host - return None + matching_host = None + for group in self.groups: + for host in group.get_hosts(): + if hostname == host.name: + matching_host = host + self._hosts_cache[host.name] = host + return matching_host def get_group(self, groupname): for group in self.groups: