diff --git a/lib/ansible/inventory/manager.py b/lib/ansible/inventory/manager.py index 46ead3131c8..54eac59a1d1 100644 --- a/lib/ansible/inventory/manager.py +++ b/lib/ansible/inventory/manager.py @@ -142,7 +142,7 @@ class InventoryManager(object): self._sources = sources # get to work! - self.parse_sources() + self.parse_sources(cache=True) @property def localhost(self): @@ -234,7 +234,7 @@ class InventoryManager(object): # recursively deal with directory entries fullpath = os.path.join(b_source, i) - parsed_this_one = self.parse_source(to_native(fullpath)) + parsed_this_one = self.parse_source(to_native(fullpath), cache=cache) display.debug(u'parsed %s as %s' % (fullpath, parsed_this_one)) if not parsed: parsed = parsed_this_one diff --git a/lib/ansible/plugins/cache/__init__.py b/lib/ansible/plugins/cache/__init__.py index 0008e982d20..76aed247834 100644 --- a/lib/ansible/plugins/cache/__init__.py +++ b/lib/ansible/plugins/cache/__init__.py @@ -79,12 +79,24 @@ class BaseFileCacheModule(BaseCacheModule): self.plugin_name = self.__module__.split('.')[-1] self._timeout = float(C.CACHE_PLUGIN_TIMEOUT) self._cache = {} - self._cache_dir = None + self._cache_dir = self._get_cache_connection(C.CACHE_PLUGIN_CONNECTION) + self._set_inventory_cache_override(**kwargs) + self.validate_cache_connection() - if C.CACHE_PLUGIN_CONNECTION: - # expects a dir path - self._cache_dir = os.path.expanduser(os.path.expandvars(C.CACHE_PLUGIN_CONNECTION)) + def _get_cache_connection(self, source): + if source: + try: + return os.path.expanduser(os.path.expandvars(source)) + except TypeError: + pass + + def _set_inventory_cache_override(self, **kwargs): + if kwargs.get('cache_timeout'): + self._timeout = kwargs.get('cache_timeout') + if kwargs.get('cache_connection'): + self._cache_dir = self._get_cache_connection(kwargs.get('cache_connection')) + def validate_cache_connection(self): if not self._cache_dir: raise AnsibleError("error, '%s' cache plugin requires the 'fact_caching_connection' config option " "to be set (to a writeable directory path)" % self.plugin_name) @@ -141,7 +153,7 @@ class BaseFileCacheModule(BaseCacheModule): def has_expired(self, key): if self._timeout == 0: - return False + return True cachefile = "%s/%s" % (self._cache_dir, key) try: @@ -284,3 +296,52 @@ class FactCache(MutableMapping): host_cache = self._plugin.get(key) host_cache.update(value) self._plugin.set(key, host_cache) + + +class InventoryFileCacheModule(BaseFileCacheModule): + """ + A caching module backed by file based storage. + """ + def __init__(self, plugin_name, timeout, cache_dir): + + self.plugin_name = plugin_name + self._timeout = timeout + self._cache = {} + self._cache_dir = self._get_cache_connection(cache_dir) + self.validate_cache_connection() + self._plugin = self.get_plugin(plugin_name) + + def validate_cache_connection(self): + try: + super(InventoryFileCacheModule, self).validate_cache_connection() + except AnsibleError as e: + cache_connection_set = False + else: + cache_connection_set = True + + if not cache_connection_set: + raise AnsibleError("error, '%s' inventory cache plugin requires the one of the following to be set:\n" + "ansible.cfg:\n[default]: fact_caching_connection,\n[inventory]: cache_connection;\n" + "Environment:\nANSIBLE_INVENTORY_CACHE_CONNECTION,\nANSIBLE_CACHE_PLUGIN_CONNECTION." + "to be set to a writeable directory path" % self.plugin_name) + + def get(self, cache_key): + + if not self.contains(cache_key): + # Check if cache file exists + raise KeyError + + return super(InventoryFileCacheModule, self).get(cache_key) + + def get_plugin(self, plugin_name): + plugin = cache_loader.get(plugin_name, cache_connection=self._cache_dir, cache_timeout=self._timeout) + if not plugin: + raise AnsibleError('Unable to load the facts cache plugin (%s).' % (plugin_name)) + self._cache = {} + return plugin + + def _load(self, path): + return self._plugin._load(path) + + def _dump(self, value, path): + return self._plugin._dump(value, path) diff --git a/lib/ansible/plugins/inventory/__init__.py b/lib/ansible/plugins/inventory/__init__.py index 06898736dbf..33288dc510f 100644 --- a/lib/ansible/plugins/inventory/__init__.py +++ b/lib/ansible/plugins/inventory/__init__.py @@ -28,6 +28,7 @@ from collections import MutableMapping from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError from ansible.plugins import AnsiblePlugin +from ansible.plugins.cache import InventoryFileCacheModule from ansible.module_utils._text import to_bytes, to_native from ansible.module_utils.parsing.convert_bool import boolean from ansible.module_utils.six import string_types @@ -142,6 +143,7 @@ class BaseInventoryPlugin(AnsiblePlugin): self._options = {} self.inventory = None self.display = display + self.cache = None def parse(self, inventory, loader, path, cache=True): ''' Populates self.groups from the given data. Raises an error on any parse failure. ''' @@ -185,9 +187,16 @@ class BaseInventoryPlugin(AnsiblePlugin): raise AnsibleParserError('inventory source has invalid structure, it should be a dictionary, got: %s' % type(config)) self.set_options(direct=config) + if self._options.get('cache'): + self._set_cache_options(self._options) return config + def _set_cache_options(self, options): + self.cache = InventoryFileCacheModule(plugin_name=options.get('cache_plugin'), + timeout=options.get('cache_timeout'), + cache_dir=options.get('cache_connection')) + def _consume_options(self, data): ''' update existing options from file data''' @@ -213,6 +222,9 @@ class Cacheable(object): _cache = {} + def get_cache_key(self, path): + return "{0}_{1}_{2}".format(self.NAME, self._get_cache_prefix(path), self._get_config_identifier(path)) + def _get_cache_prefix(self, path): ''' create predictable unique prefix for plugin/inventory ''' @@ -226,6 +238,11 @@ class Cacheable(object): return 's_'.join([d1[:5], d2[:5]]) + def _get_config_identifier(self, path): + ''' create predictable config-specific prefix for plugin/inventory ''' + + return hashlib.md5(path.encode()).hexdigest() + def clear_cache(self): self._cache = {} diff --git a/lib/ansible/plugins/inventory/virtualbox.py b/lib/ansible/plugins/inventory/virtualbox.py index bac45bfcbc6..f9b5cbf0491 100644 --- a/lib/ansible/plugins/inventory/virtualbox.py +++ b/lib/ansible/plugins/inventory/virtualbox.py @@ -14,6 +14,7 @@ DOCUMENTATION = ''' - The inventory_hostname is always the 'Name' of the virtualbox instance. extends_documentation_fragment: - constructed + - inventory_cache options: running_only: description: toggles showing all vms vs only those currently running @@ -91,7 +92,29 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): # constructed groups based on conditionals self._add_host_to_composed_groups(self.get_option('groups'), hostvars, host) - def _populate_from_source(self, source_data): + def _populate_from_cache(self, source_data): + hostvars = source_data.pop('_meta', {}).get('hostvars', {}) + for group in source_data: + if group == 'all': + continue + else: + self.inventory.add_group(group) + hosts = source_data[group].get('hosts', []) + for host in hosts: + self._populate_host_vars([host], hostvars.get(host, {}), group) + self.inventory.add_child('all', group) + if not source_data: + for host in hostvars: + self.inventory.add_host(host) + self._populate_host_vars([host], hostvars.get(host, {})) + + def _populate_from_source(self, source_data, using_current_cache=False): + if using_current_cache: + self._populate_from_cache(source_data) + return source_data + + cacheable_results = {'_meta': {'hostvars': {}}} + hostvars = {} prevkey = pref_k = '' current_host = None @@ -100,6 +123,9 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): netinfo = self.get_option('network_info_path') for line in source_data: + line = to_text(line) + if ':' not in line: + continue try: k, v = line.split(':', 1) except: @@ -127,8 +153,11 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): elif k == 'Groups': for group in v.split('/'): if group: + if group not in cacheable_results: + cacheable_results[group] = {'hosts': []} self.inventory.add_group(group) self.inventory.add_child(group, current_host) + cacheable_results[group]['hosts'].append(current_host) continue else: @@ -141,10 +170,32 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): else: if v != '': hostvars[current_host][pref_k] = v + if self._ungrouped_host(current_host, cacheable_results): + if 'ungrouped' not in cacheable_results: + cacheable_results['ungrouped'] = {'hosts': []} + cacheable_results['ungrouped']['hosts'].append(current_host) prevkey = pref_k self._set_variables(hostvars) + for host in hostvars: + h = self.inventory.get_host(host) + cacheable_results['_meta']['hostvars'][h.name] = h.vars + + return cacheable_results + + def _ungrouped_host(self, host, inventory): + def find_host(host, inventory): + for k, v in inventory.items(): + if k == '_meta': + continue + if isinstance(v, dict): + yield self._ungrouped_host(host, v) + elif isinstance(v, list): + yield host not in v + yield True + + return all([found_host for found_host in find_host(host, inventory)]) def verify_file(self, path): @@ -158,7 +209,7 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): super(InventoryModule, self).parse(inventory, loader, path) - cache_key = self._get_cache_prefix(path) + cache_key = self.get_cache_key(path) config_data = self._read_config_data(path) @@ -166,11 +217,15 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): self._consume_options(config_data) source_data = None - if cache and cache_key in self._cache: + if cache: + cache = self._options.get('cache') + + update_cache = False + if cache: try: - source_data = self._cache[cache_key] + source_data = self.cache.get(cache_key) except KeyError: - pass + update_cache = True if not source_data: b_pwfile = to_bytes(self.get_option('settings_password_file'), errors='surrogate_or_strict') @@ -192,7 +247,10 @@ class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable): except Exception as e: AnsibleParserError(to_native(e)) - source_data = p.stdout.read() - self._cache[cache_key] = to_text(source_data, errors='surrogate_or_strict') + source_data = p.stdout.read().splitlines() + + using_current_cache = cache and not update_cache + cacheable_results = self._populate_from_source(source_data, using_current_cache) - self._populate_from_source(source_data.splitlines()) + if update_cache: + self.cache.set(cache_key, cacheable_results) diff --git a/lib/ansible/plugins/strategy/__init__.py b/lib/ansible/plugins/strategy/__init__.py index 7b5d3348a8e..d57bdcad695 100644 --- a/lib/ansible/plugins/strategy/__init__.py +++ b/lib/ansible/plugins/strategy/__init__.py @@ -179,6 +179,7 @@ class StrategyBase: self._final_q = tqm._final_q self._step = getattr(tqm._options, 'step', False) self._diff = getattr(tqm._options, 'diff', False) + self.flush_cache = getattr(tqm._options, 'flush_cache', False) # the task cache is a dictionary of tuples of (host.name, task._uuid) # used to find the original task object of in-flight tasks and to store @@ -956,7 +957,7 @@ class StrategyBase: elif meta_action == 'flush_handlers': self.run_handlers(iterator, play_context) msg = "ran handlers" - elif meta_action == 'refresh_inventory': + elif meta_action == 'refresh_inventory' or self.flush_cache: self._inventory.refresh_inventory() msg = "inventory successfully refreshed" elif meta_action == 'clear_facts':