From 49d88cef9c5cef2dde0424c6cbe3fb5a237cecd4 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Sat, 18 Jul 2015 15:24:44 -0400 Subject: [PATCH 1/2] meta: refresh_inventory and several inventory fixes --- lib/ansible/constants.py | 3 +- lib/ansible/inventory/__init__.py | 84 +++++++++------------- lib/ansible/inventory/dir.py | 65 +++++++++++++++-- lib/ansible/plugins/inventory/README.md | 1 + lib/ansible/plugins/inventory/directory.py | 5 +- lib/ansible/plugins/inventory/ini.py | 5 +- lib/ansible/plugins/inventory/script.py | 31 ++++++++ lib/ansible/plugins/strategies/__init__.py | 18 +++++ lib/ansible/plugins/strategies/linear.py | 11 +-- 9 files changed, 153 insertions(+), 70 deletions(-) create mode 100644 lib/ansible/plugins/inventory/README.md create mode 100644 lib/ansible/plugins/inventory/script.py diff --git a/lib/ansible/constants.py b/lib/ansible/constants.py index f43869d2cda..37030276e8e 100644 --- a/lib/ansible/constants.py +++ b/lib/ansible/constants.py @@ -143,6 +143,7 @@ DEFAULT_EXECUTABLE = get_config(p, DEFAULTS, 'executable', 'ANSIBLE_EXECU DEFAULT_GATHERING = get_config(p, DEFAULTS, 'gathering', 'ANSIBLE_GATHERING', 'implicit').lower() DEFAULT_LOG_PATH = shell_expand_path(get_config(p, DEFAULTS, 'log_path', 'ANSIBLE_LOG_PATH', '')) DEFAULT_FORCE_HANDLERS = get_config(p, DEFAULTS, 'force_handlers', 'ANSIBLE_FORCE_HANDLERS', False, boolean=True) +DEFAULT_INVENTORY_IGNORE = get_config(p, DEFAULTS, 'inventory_ignore_extensions', 'ANSIBLE_INVENTORY_IGNORE', ["~", ".orig", ".bak", ".ini", ".cfg", ".retry", ".pyc", ".pyo"], islist=True) # selinux DEFAULT_SELINUX_SPECIAL_FS = get_config(p, 'selinux', 'special_context_filesystems', None, 'fuse, nfs, vboxsf, ramfs', islist=True) @@ -197,7 +198,7 @@ HOST_KEY_CHECKING = get_config(p, DEFAULTS, 'host_key_checking', ' SYSTEM_WARNINGS = get_config(p, DEFAULTS, 'system_warnings', 'ANSIBLE_SYSTEM_WARNINGS', True, boolean=True) DEPRECATION_WARNINGS = get_config(p, DEFAULTS, 'deprecation_warnings', 'ANSIBLE_DEPRECATION_WARNINGS', True, boolean=True) DEFAULT_CALLABLE_WHITELIST = get_config(p, DEFAULTS, 'callable_whitelist', 'ANSIBLE_CALLABLE_WHITELIST', [], islist=True) -COMMAND_WARNINGS = get_config(p, DEFAULTS, 'command_warnings', 'ANSIBLE_COMMAND_WARNINGS', False, boolean=True) +COMMAND_WARNINGS = get_config(p, DEFAULTS, 'command_warnings', 'ANSIBLE_COMMAND_WARNINGS', True, boolean=True) DEFAULT_LOAD_CALLBACK_PLUGINS = get_config(p, DEFAULTS, 'bin_ansible_callbacks', 'ANSIBLE_LOAD_CALLBACK_PLUGINS', False, boolean=True) DEFAULT_CALLBACK_WHITELIST = get_config(p, DEFAULTS, 'callback_whitelist', 'ANSIBLE_CALLBACK_WHITELIST', [], islist=True) RETRY_FILES_ENABLED = get_config(p, DEFAULTS, 'retry_files_enabled', 'ANSIBLE_RETRY_FILES_ENABLED', True, boolean=True) diff --git a/lib/ansible/inventory/__init__.py b/lib/ansible/inventory/__init__.py index 0edb4dc3862..b738c77b175 100644 --- a/lib/ansible/inventory/__init__.py +++ b/lib/ansible/inventory/__init__.py @@ -26,15 +26,12 @@ import re import stat from ansible import constants as C -from ansible import errors +from ansible.errors import AnsibleError -from ansible.inventory.ini import InventoryParser -from ansible.inventory.script import InventoryScript -from ansible.inventory.dir import InventoryDirectory +from ansible.inventory.dir import InventoryDirectory, get_file_parser from ansible.inventory.group import Group from ansible.inventory.host import Host from ansible.plugins import vars_loader -from ansible.utils.path import is_executable from ansible.utils.vars import combine_vars class Inventory(object): @@ -63,6 +60,7 @@ class Inventory(object): self._hosts_cache = {} self._groups_list = {} self._pattern_cache = {} + self._vars_plugins = [] # to be set by calling set_playbook_basedir by playbook code self._playbook_basedir = None @@ -75,6 +73,10 @@ class Inventory(object): self._also_restriction = None self._subset = None + self.parse_inventory(host_list) + + def parse_inventory(self, host_list): + if isinstance(host_list, basestring): if "," in host_list: host_list = host_list.split(",") @@ -102,51 +104,22 @@ class Inventory(object): else: all.add_host(Host(x)) elif os.path.exists(host_list): + #TODO: switch this to a plugin loader and a 'condition' per plugin on which it should be tried, restoring 'inventory pllugins' if os.path.isdir(host_list): # Ensure basedir is inside the directory - self.host_list = os.path.join(self.host_list, "") + host_list = os.path.join(self.host_list, "") self.parser = InventoryDirectory(loader=self._loader, filename=host_list) - self.groups = self.parser.groups.values() else: - # check to see if the specified file starts with a - # shebang (#!/), so if an error is raised by the parser - # class we can show a more apropos error - shebang_present = False - try: - with open(host_list, "r") as inv_file: - first_line = inv_file.readline() - if first_line.startswith("#!"): - shebang_present = True - except IOError: - pass - - if is_executable(host_list): - try: - self.parser = InventoryScript(loader=self._loader, filename=host_list) - self.groups = self.parser.groups.values() - except errors.AnsibleError: - if not shebang_present: - raise errors.AnsibleError("The file %s is marked as executable, but failed to execute correctly. " % host_list + \ - "If this is not supposed to be an executable script, correct this with `chmod -x %s`." % host_list) - else: - raise - else: - try: - self.parser = InventoryParser(filename=host_list) - self.groups = self.parser.groups.values() - except errors.AnsibleError: - if shebang_present: - raise errors.AnsibleError("The file %s looks like it should be an executable inventory script, but is not marked executable. " % host_list + \ - "Perhaps you want to correct this with `chmod +x %s`?" % host_list) - else: - raise + self.parser = get_file_parser(hostsfile, self._loader) + vars_loader.add_directory(self.basedir(), with_subdir=True) - vars_loader.add_directory(self.basedir(), with_subdir=True) - else: - raise errors.AnsibleError("Unable to find an inventory file (%s), " - "specify one with -i ?" % host_list) + if self.parser: + self.groups = self.parser.groups.values() + else: + # should never happen, but JIC + raise AnsibleError("Unable to parse %s as an inventory source" % host_list) - self._vars_plugins = [ x for x in vars_loader.all(self) ] + self._vars_plugins = [ x for x in vars_loader.all(self) ] # FIXME: shouldn't be required, since the group/host vars file # management will be done in VariableManager @@ -166,7 +139,7 @@ class Inventory(object): else: return fnmatch.fnmatch(str, pattern_str) except Exception, e: - raise errors.AnsibleError('invalid host pattern: %s' % pattern_str) + raise AnsibleError('invalid host pattern: %s' % pattern_str) def _match_list(self, items, item_attr, pattern_str): results = [] @@ -176,7 +149,7 @@ class Inventory(object): else: pattern = re.compile(pattern_str[1:]) except Exception, e: - raise errors.AnsibleError('invalid host pattern: %s' % pattern_str) + raise AnsibleError('invalid host pattern: %s' % pattern_str) for item in items: if pattern.match(getattr(item, item_attr)): @@ -286,7 +259,7 @@ class Inventory(object): first = int(first) if last: if first < 0: - raise errors.AnsibleError("invalid range: negative indices cannot be used as the first item in a range") + raise AnsibleError("invalid range: negative indices cannot be used as the first item in a range") last = int(last) else: last = first @@ -324,7 +297,7 @@ class Inventory(object): else: return [ hosts[left] ] except IndexError: - raise errors.AnsibleError("no hosts matching the pattern '%s' were found" % pat) + raise AnsibleError("no hosts matching the pattern '%s' were found" % pat) def _create_implicit_localhost(self, pattern): new_host = Host(pattern) @@ -467,7 +440,7 @@ class Inventory(object): host = self.get_host(hostname) if host is None: - raise errors.AnsibleError("host not found: %s" % hostname) + raise AnsibleError("host not found: %s" % hostname) vars = {} @@ -499,7 +472,7 @@ class Inventory(object): self.groups.append(group) self._groups_list = None # invalidate internal cache else: - raise errors.AnsibleError("group already in inventory: %s" % group.name) + raise AnsibleError("group already in inventory: %s" % group.name) def list_hosts(self, pattern="all"): @@ -670,3 +643,14 @@ class Inventory(object): # all done, results is a dictionary of variables for this particular host. return results + def refresh_inventory(self): + + self.clear_pattern_cache() + + self._hosts_cache = {} + self._vars_per_host = {} + self._vars_per_group = {} + self._groups_list = {} + self.groups = [] + + self.parse_inventory(self.host_list) diff --git a/lib/ansible/inventory/dir.py b/lib/ansible/inventory/dir.py index 735f32d62c3..e456a950d41 100644 --- a/lib/ansible/inventory/dir.py +++ b/lib/ansible/inventory/dir.py @@ -27,11 +27,58 @@ from ansible.errors import AnsibleError from ansible.inventory.host import Host from ansible.inventory.group import Group -from ansible.inventory.ini import InventoryParser -from ansible.inventory.script import InventoryScript -from ansible.utils.path import is_executable from ansible.utils.vars import combine_vars +from ansible.utils.path import is_executable +from ansible.inventory.ini import InventoryParser as InventoryINIParser +from ansible.inventory.script import InventoryScript + +__all__ = ['get_file_parser'] + +def get_file_parser(hostsfile, loader): + # check to see if the specified file starts with a + # shebang (#!/), so if an error is raised by the parser + # class we can show a more apropos error + + shebang_present = False + processed = False + myerr = [] + parser = None + + try: + inv_file = open(hostsfile) + first_line = inv_file.readlines()[0] + inv_file.close() + if first_line.startswith('#!'): + shebang_present = True + except: + pass + + if is_executable(hostsfile): + try: + parser = InventoryScript(loader=loader, filename=hostsfile) + processed = True + except Exception as e: + myerr.append("The file %s is marked as executable, but failed to execute correctly. " % hostsfile + \ + "If this is not supposed to be an executable script, correct this with `chmod -x %s`." % hostsfile) + myerr.append(str(e)) + + if not processed: + try: + parser = InventoryINIParser(filename=hostsfile) + processed = True + except Exception as e: + if shebang_present and not is_executable(hostsfile): + myerr.append("The file %s looks like it should be an executable inventory script, but is not marked executable. " % hostsfile + \ + "Perhaps you want to correct this with `chmod +x %s`?" % hostsfile) + else: + myerr.append(str(e)) + + if not processed and myerr: + raise AnsibleError( '\n'.join(myerr) ) + + return parser + class InventoryDirectory(object): ''' Host inventory parser for ansible using a directory of inventories. ''' @@ -48,7 +95,7 @@ class InventoryDirectory(object): for i in self.names: # Skip files that end with certain extensions or characters - if any(i.endswith(ext) for ext in ("~", ".orig", ".bak", ".ini", ".cfg", ".retry", ".pyc", ".pyo")): + if any(i.endswith(ext) for ext in C.DEFAULT_INVENTORY_IGNORE): continue # Skip hidden files if i.startswith('.') and not i.startswith('./'): @@ -59,10 +106,14 @@ class InventoryDirectory(object): fullpath = os.path.join(self.directory, i) if os.path.isdir(fullpath): parser = InventoryDirectory(loader=loader, filename=fullpath) - elif is_executable(fullpath): - parser = InventoryScript(loader=loader, filename=fullpath) else: - parser = InventoryParser(filename=fullpath) + parser = get_file_parser(fullpath, loader) + if parser is None: + #FIXME: needs to use display + import warnings + warnings.warning("Could not find parser for %s, skipping" % fullpath) + continue + self.parsers.append(parser) # retrieve all groups and hosts form the parser and add them to diff --git a/lib/ansible/plugins/inventory/README.md b/lib/ansible/plugins/inventory/README.md new file mode 100644 index 00000000000..1988f9adc6b --- /dev/null +++ b/lib/ansible/plugins/inventory/README.md @@ -0,0 +1 @@ +These are not currently in use, but this is what the future of inventory will become after 2.0 diff --git a/lib/ansible/plugins/inventory/directory.py b/lib/ansible/plugins/inventory/directory.py index a75ad44ea6c..2a2a3c31933 100644 --- a/lib/ansible/plugins/inventory/directory.py +++ b/lib/ansible/plugins/inventory/directory.py @@ -22,11 +22,14 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type import os +from ansible import constants as C from . aggregate import InventoryAggregateParser class InventoryDirectoryParser(InventoryAggregateParser): + CONDITION="is_dir(%s)" + def __init__(self, inven_directory): directory = inven_directory names = os.listdir(inven_directory) @@ -35,7 +38,7 @@ class InventoryDirectoryParser(InventoryAggregateParser): # Clean up the list of filenames for filename in names: # Skip files that end with certain extensions or characters - if any(filename.endswith(ext) for ext in ("~", ".orig", ".bak", ".ini", ".retry", ".pyc", ".pyo")): + if any(filename.endswith(ext) for ext in C.DEFAULT_INVENTORY_IGNORE): continue # Skip hidden files if filename.startswith('.') and not filename.startswith('.{0}'.format(os.path.sep)): diff --git a/lib/ansible/plugins/inventory/ini.py b/lib/ansible/plugins/inventory/ini.py index e185c1a7858..7c6c3aa95a8 100644 --- a/lib/ansible/plugins/inventory/ini.py +++ b/lib/ansible/plugins/inventory/ini.py @@ -23,10 +23,13 @@ __metaclass__ = type import os +from ansible import constants as C from . import InventoryParser class InventoryIniParser(InventoryAggregateParser): + CONDITION="is_file(%s)" + def __init__(self, inven_directory): directory = inven_directory names = os.listdir(inven_directory) @@ -35,7 +38,7 @@ class InventoryIniParser(InventoryAggregateParser): # Clean up the list of filenames for filename in names: # Skip files that end with certain extensions or characters - if any(filename.endswith(ext) for ext in ("~", ".orig", ".bak", ".ini", ".retry", ".pyc", ".pyo")): + if any(filename.endswith(ext) for ext in C.DEFAULT_INVENTORY_IGNORE): continue # Skip hidden files if filename.startswith('.') and not filename.startswith('.{0}'.format(os.path.sep)): diff --git a/lib/ansible/plugins/inventory/script.py b/lib/ansible/plugins/inventory/script.py new file mode 100644 index 00000000000..1346bf35622 --- /dev/null +++ b/lib/ansible/plugins/inventory/script.py @@ -0,0 +1,31 @@ +# (c) 2012-2014, Michael DeHaan +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +############################################# + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import os + +from ansible import constants as C +from . import InventoryParser + +class InventoryScriptParser(InventoryParser): + + CONDITION="is_file(%s) and is_executable(%s)" diff --git a/lib/ansible/plugins/strategies/__init__.py b/lib/ansible/plugins/strategies/__init__.py index 0728750bdec..9a7bd2db3a5 100644 --- a/lib/ansible/plugins/strategies/__init__.py +++ b/lib/ansible/plugins/strategies/__init__.py @@ -506,3 +506,21 @@ class StrategyBase: self._display.banner(msg) return ret + + def _execute_meta(self, task, play_context, iterator): + + # meta tasks store their args in the _raw_params field of args, + # since they do not use k=v pairs, so get that + meta_action = task.args.get('_raw_params') + + if meta_action == 'noop': + # FIXME: issue a callback for the noop here? + pass + elif meta_action == 'flush_handlers': + self.run_handlers(iterator, play_context) + elif meta_action == 'refresh_inventory': + self._inventory.refresh_inventory() + #elif meta_action == 'reset_connection': + # connection_info.connection.close() + else: + raise AnsibleError("invalid meta action requested: %s" % meta_action, obj=task._ds) diff --git a/lib/ansible/plugins/strategies/linear.py b/lib/ansible/plugins/strategies/linear.py index 1a05d184301..4b84884c6cb 100644 --- a/lib/ansible/plugins/strategies/linear.py +++ b/lib/ansible/plugins/strategies/linear.py @@ -178,16 +178,7 @@ class StrategyModule(StrategyBase): continue if task.action == 'meta': - # meta tasks store their args in the _raw_params field of args, - # since they do not use k=v pairs, so get that - meta_action = task.args.get('_raw_params') - if meta_action == 'noop': - # FIXME: issue a callback for the noop here? - continue - elif meta_action == 'flush_handlers': - self.run_handlers(iterator, play_context) - else: - raise AnsibleError("invalid meta action requested: %s" % meta_action, obj=task._ds) + self._execute_meta(task, play_context, iterator) else: # handle step if needed, skip meta actions as they are used internally if self._step and choose_step: From d1222291473075ac9486875bb132aa0cb4d3704d Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Sat, 18 Jul 2015 15:27:08 -0400 Subject: [PATCH 2/2] added info about refresh_inventory to changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b6d2fc0442..57213d2a16b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ Major Changes: * template code now retains types for bools and numbers instead of turning them into strings. If you need the old behaviour, quote the value and it will get passed around as a string * Consolidated code from modules using urllib2 to normalize features, TLS and SNI support + * Consiidated code from modules using urllib2 to normalize features, TLS and SNI support + * added meta: refresh_inventory to force rereading the inventory in a play Deprecated Modules (new ones in parens): * ec2_ami_search (ec2_ami_find)