meta: refresh_inventory and several inventory fixes

lib/ansible/constants.py

@@ -143,6 +143,7 @@ DEFAULT_EXECUTABLE        = get_config(p, DEFAULTS, 'executable', 'ANSIBLE_EXECU
 DEFAULT_GATHERING         = get_config(p, DEFAULTS, 'gathering', 'ANSIBLE_GATHERING', 'implicit').lower()
 DEFAULT_LOG_PATH          = shell_expand_path(get_config(p, DEFAULTS, 'log_path',           'ANSIBLE_LOG_PATH', ''))
 DEFAULT_FORCE_HANDLERS    = get_config(p, DEFAULTS, 'force_handlers', 'ANSIBLE_FORCE_HANDLERS', False, boolean=True)
+DEFAULT_INVENTORY_IGNORE  = get_config(p, DEFAULTS, 'inventory_ignore_extensions', 'ANSIBLE_INVENTORY_IGNORE', ["~", ".orig", ".bak", ".ini", ".cfg", ".retry", ".pyc", ".pyo"], islist=True)
 
 # selinux
 DEFAULT_SELINUX_SPECIAL_FS = get_config(p, 'selinux', 'special_context_filesystems', None, 'fuse, nfs, vboxsf, ramfs', islist=True)
@@ -197,7 +198,7 @@ HOST_KEY_CHECKING              = get_config(p, DEFAULTS, 'host_key_checking',  '
 SYSTEM_WARNINGS                = get_config(p, DEFAULTS, 'system_warnings', 'ANSIBLE_SYSTEM_WARNINGS', True, boolean=True)
 DEPRECATION_WARNINGS           = get_config(p, DEFAULTS, 'deprecation_warnings', 'ANSIBLE_DEPRECATION_WARNINGS', True, boolean=True)
 DEFAULT_CALLABLE_WHITELIST     = get_config(p, DEFAULTS, 'callable_whitelist', 'ANSIBLE_CALLABLE_WHITELIST', [], islist=True)
-COMMAND_WARNINGS               = get_config(p, DEFAULTS, 'command_warnings', 'ANSIBLE_COMMAND_WARNINGS', False, boolean=True)
+COMMAND_WARNINGS               = get_config(p, DEFAULTS, 'command_warnings', 'ANSIBLE_COMMAND_WARNINGS', True, boolean=True)
 DEFAULT_LOAD_CALLBACK_PLUGINS  = get_config(p, DEFAULTS, 'bin_ansible_callbacks', 'ANSIBLE_LOAD_CALLBACK_PLUGINS', False, boolean=True)
 DEFAULT_CALLBACK_WHITELIST     = get_config(p, DEFAULTS, 'callback_whitelist', 'ANSIBLE_CALLBACK_WHITELIST', [], islist=True)
 RETRY_FILES_ENABLED            = get_config(p, DEFAULTS, 'retry_files_enabled', 'ANSIBLE_RETRY_FILES_ENABLED', True, boolean=True)

lib/ansible/inventory/__init__.py

@@ -26,15 +26,12 @@ import re
 import stat
 
 from ansible import constants as C
-from ansible import errors
+from ansible.errors import AnsibleError
 
-from ansible.inventory.ini import InventoryParser
+from ansible.inventory.dir import InventoryDirectory, get_file_parser
-from ansible.inventory.script import InventoryScript
-from ansible.inventory.dir import InventoryDirectory
 from ansible.inventory.group import Group
 from ansible.inventory.host import Host
 from ansible.plugins import vars_loader
-from ansible.utils.path import is_executable
 from ansible.utils.vars import combine_vars
 
 class Inventory(object):
@@ -63,6 +60,7 @@ class Inventory(object):
         self._hosts_cache    = {}
         self._groups_list    = {}
         self._pattern_cache  = {}
+        self._vars_plugins   = []
 
         # to be set by calling set_playbook_basedir by playbook code
         self._playbook_basedir = None
@@ -75,6 +73,10 @@ class Inventory(object):
         self._also_restriction = None
         self._subset = None
 
+        self.parse_inventory(host_list)
+
+    def parse_inventory(self, host_list):
+
         if isinstance(host_list, basestring):
             if "," in host_list:
                 host_list = host_list.split(",")
@@ -102,51 +104,22 @@ class Inventory(object):
                     else:
                         all.add_host(Host(x))
         elif os.path.exists(host_list):
+            #TODO: switch this to a plugin loader and a 'condition' per plugin on which it should be tried, restoring 'inventory pllugins'
             if os.path.isdir(host_list):
                 # Ensure basedir is inside the directory
-                self.host_list = os.path.join(self.host_list, "")
+                host_list = os.path.join(self.host_list, "")
                 self.parser = InventoryDirectory(loader=self._loader, filename=host_list)
-                self.groups = self.parser.groups.values()
             else:
-                # check to see if the specified file starts with a
+                self.parser = get_file_parser(hostsfile, self._loader)
-                # shebang (#!/), so if an error is raised by the parser
+                vars_loader.add_directory(self.basedir(), with_subdir=True)
-                # class we can show a more apropos error
-                shebang_present = False
-                try:
-                    with open(host_list, "r") as inv_file:
-                        first_line = inv_file.readline()
-                        if first_line.startswith("#!"):
-                            shebang_present = True
-                except IOError:
-                    pass
-
-                if is_executable(host_list):
-                    try:
-                        self.parser = InventoryScript(loader=self._loader, filename=host_list)
-                        self.groups = self.parser.groups.values()
-                    except errors.AnsibleError:
-                        if not shebang_present:
-                            raise errors.AnsibleError("The file %s is marked as executable, but failed to execute correctly. " % host_list + \
-                                                      "If this is not supposed to be an executable script, correct this with `chmod -x %s`." % host_list)
-                        else:
-                            raise
-                else:
-                    try:
-                        self.parser = InventoryParser(filename=host_list)
-                        self.groups = self.parser.groups.values()
-                    except errors.AnsibleError:
-                        if shebang_present:
-                            raise errors.AnsibleError("The file %s looks like it should be an executable inventory script, but is not marked executable. " % host_list + \
-                                                      "Perhaps you want to correct this with `chmod +x %s`?" % host_list)
-                        else:
-                            raise
 
-            vars_loader.add_directory(self.basedir(), with_subdir=True)
+            if self.parser:
-        else:
+                self.groups = self.parser.groups.values()
-            raise errors.AnsibleError("Unable to find an inventory file (%s), "
+            else:
-                                      "specify one with -i ?" % host_list)
+                # should never happen, but JIC
+                raise AnsibleError("Unable to parse %s as an inventory source" % host_list)
 
-        self._vars_plugins = [ x for x in vars_loader.all(self) ]
+            self._vars_plugins = [ x for x in vars_loader.all(self) ]
 
         # FIXME: shouldn't be required, since the group/host vars file
         #        management will be done in VariableManager
@@ -166,7 +139,7 @@ class Inventory(object):
             else:
                 return fnmatch.fnmatch(str, pattern_str)
         except Exception, e:
-            raise errors.AnsibleError('invalid host pattern: %s' % pattern_str)
+            raise AnsibleError('invalid host pattern: %s' % pattern_str)
 
     def _match_list(self, items, item_attr, pattern_str):
         results = []
@@ -176,7 +149,7 @@ class Inventory(object):
             else:
                 pattern = re.compile(pattern_str[1:])
         except Exception, e:
-            raise errors.AnsibleError('invalid host pattern: %s' % pattern_str)
+            raise AnsibleError('invalid host pattern: %s' % pattern_str)
 
         for item in items:
             if pattern.match(getattr(item, item_attr)):
@@ -286,7 +259,7 @@ class Inventory(object):
             first = int(first)
             if last:
                 if first < 0:
-                    raise errors.AnsibleError("invalid range: negative indices cannot be used as the first item in a range")
+                    raise AnsibleError("invalid range: negative indices cannot be used as the first item in a range")
                 last = int(last)
             else:
                 last = first
@@ -324,7 +297,7 @@ class Inventory(object):
             else:
                 return [ hosts[left] ]
         except IndexError:
-            raise errors.AnsibleError("no hosts matching the pattern '%s' were found" % pat)
+            raise AnsibleError("no hosts matching the pattern '%s' were found" % pat)
 
     def _create_implicit_localhost(self, pattern):
         new_host = Host(pattern)
@@ -467,7 +440,7 @@ class Inventory(object):
 
         host = self.get_host(hostname)
         if host is None:
-            raise errors.AnsibleError("host not found: %s" % hostname)
+            raise AnsibleError("host not found: %s" % hostname)
 
         vars = {}
 
@@ -499,7 +472,7 @@ class Inventory(object):
             self.groups.append(group)
             self._groups_list = None  # invalidate internal cache 
         else:
-            raise errors.AnsibleError("group already in inventory: %s" % group.name)
+            raise AnsibleError("group already in inventory: %s" % group.name)
 
     def list_hosts(self, pattern="all"):
 
@@ -670,3 +643,14 @@ class Inventory(object):
         # all done, results is a dictionary of variables for this particular host.
         return results
 
+    def refresh_inventory(self):
+
+        self.clear_pattern_cache()
+
+        self._hosts_cache    = {}
+        self._vars_per_host  = {}
+        self._vars_per_group = {}
+        self._groups_list    = {}
+        self.groups = []
+
+        self.parse_inventory(self.host_list)

lib/ansible/inventory/dir.py

@@ -27,11 +27,58 @@ from ansible.errors import AnsibleError
 
 from ansible.inventory.host import Host
 from ansible.inventory.group import Group
-from ansible.inventory.ini import InventoryParser
-from ansible.inventory.script import InventoryScript
-from ansible.utils.path import is_executable
 from ansible.utils.vars import combine_vars
 
+from ansible.utils.path import is_executable
+from ansible.inventory.ini import InventoryParser as InventoryINIParser
+from ansible.inventory.script import InventoryScript
+
+__all__ = ['get_file_parser']
+
+def get_file_parser(hostsfile, loader):
+    # check to see if the specified file starts with a
+    # shebang (#!/), so if an error is raised by the parser
+    # class we can show a more apropos error
+
+    shebang_present = False
+    processed = False
+    myerr = []
+    parser = None
+
+    try:
+        inv_file = open(hostsfile)
+        first_line = inv_file.readlines()[0]
+        inv_file.close()
+        if first_line.startswith('#!'):
+            shebang_present = True
+    except:
+        pass
+
+    if is_executable(hostsfile):
+        try:
+            parser = InventoryScript(loader=loader, filename=hostsfile)
+            processed = True
+        except Exception as e:
+            myerr.append("The file %s is marked as executable, but failed to execute correctly. " % hostsfile + \
+                            "If this is not supposed to be an executable script, correct this with `chmod -x %s`." % hostsfile)
+            myerr.append(str(e))
+
+    if not processed:
+        try:
+            parser = InventoryINIParser(filename=hostsfile)
+            processed = True
+        except Exception as e:
+            if shebang_present and not is_executable(hostsfile):
+                myerr.append("The file %s looks like it should be an executable inventory script, but is not marked executable. " % hostsfile + \
+                              "Perhaps you want to correct this with `chmod +x %s`?" % hostsfile)
+            else:
+                myerr.append(str(e))
+
+    if not processed and myerr:
+        raise AnsibleError( '\n'.join(myerr) )
+
+    return parser
+
 class InventoryDirectory(object):
     ''' Host inventory parser for ansible using a directory of inventories. '''
 
@@ -48,7 +95,7 @@ class InventoryDirectory(object):
         for i in self.names:
 
             # Skip files that end with certain extensions or characters
-            if any(i.endswith(ext) for ext in ("~", ".orig", ".bak", ".ini", ".cfg", ".retry", ".pyc", ".pyo")):
+            if any(i.endswith(ext) for ext in C.DEFAULT_INVENTORY_IGNORE):
                 continue
             # Skip hidden files
             if i.startswith('.') and not i.startswith('./'):
@@ -59,10 +106,14 @@ class InventoryDirectory(object):
             fullpath = os.path.join(self.directory, i)
             if os.path.isdir(fullpath):
                 parser = InventoryDirectory(loader=loader, filename=fullpath)
-            elif is_executable(fullpath):
-                parser = InventoryScript(loader=loader, filename=fullpath)
             else:
-                parser = InventoryParser(filename=fullpath)
+                parser = get_file_parser(fullpath, loader)
+                if parser is None:
+                    #FIXME: needs to use display
+                    import warnings
+                    warnings.warning("Could not find parser for %s, skipping" % fullpath)
+                    continue
+
             self.parsers.append(parser)
 
             # retrieve all groups and hosts form the parser and add them to

lib/ansible/plugins/inventory/README.md

@@ -0,0 +1 @@
+These are not currently in use, but this is what the future of inventory will become after 2.0

lib/ansible/plugins/inventory/directory.py

@@ -22,11 +22,14 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import os
+from ansible import constants as C
 
 from . aggregate import InventoryAggregateParser
 
 class InventoryDirectoryParser(InventoryAggregateParser):
 
+    CONDITION="is_dir(%s)"
+
     def __init__(self, inven_directory):
         directory = inven_directory
         names = os.listdir(inven_directory)
@@ -35,7 +38,7 @@ class InventoryDirectoryParser(InventoryAggregateParser):
         # Clean up the list of filenames
         for filename in names:
             # Skip files that end with certain extensions or characters
-            if any(filename.endswith(ext) for ext in ("~", ".orig", ".bak", ".ini", ".retry", ".pyc", ".pyo")):
+            if any(filename.endswith(ext) for ext in C.DEFAULT_INVENTORY_IGNORE):
                 continue
             # Skip hidden files
             if filename.startswith('.') and not filename.startswith('.{0}'.format(os.path.sep)):

lib/ansible/plugins/inventory/ini.py

@@ -23,10 +23,13 @@ __metaclass__ = type
 
 import os
 
+from ansible import constants as C
 from . import InventoryParser
 
 class InventoryIniParser(InventoryAggregateParser):
 
+    CONDITION="is_file(%s)"
+
     def __init__(self, inven_directory):
         directory = inven_directory
         names = os.listdir(inven_directory)
@@ -35,7 +38,7 @@ class InventoryIniParser(InventoryAggregateParser):
         # Clean up the list of filenames
         for filename in names:
             # Skip files that end with certain extensions or characters
-            if any(filename.endswith(ext) for ext in ("~", ".orig", ".bak", ".ini", ".retry", ".pyc", ".pyo")):
+            if any(filename.endswith(ext) for ext in C.DEFAULT_INVENTORY_IGNORE):
                 continue
             # Skip hidden files
             if filename.startswith('.') and not filename.startswith('.{0}'.format(os.path.sep)):

lib/ansible/plugins/inventory/script.py

@@ -0,0 +1,31 @@
+# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+#############################################
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+
+from ansible import constants as C
+from . import InventoryParser
+
+class InventoryScriptParser(InventoryParser):
+
+    CONDITION="is_file(%s) and is_executable(%s)"

lib/ansible/plugins/strategies/__init__.py

@@ -506,3 +506,21 @@ class StrategyBase:
         self._display.banner(msg)
 
         return ret
+
+    def _execute_meta(self, task, play_context, iterator):
+
+        # meta tasks store their args in the _raw_params field of args,
+        # since they do not use k=v pairs, so get that
+        meta_action = task.args.get('_raw_params')
+
+        if meta_action == 'noop':
+            # FIXME: issue a callback for the noop here?
+            pass
+        elif meta_action == 'flush_handlers':
+            self.run_handlers(iterator, play_context)
+        elif meta_action == 'refresh_inventory':
+            self._inventory.refresh_inventory()
+        #elif meta_action == 'reset_connection':
+        #    connection_info.connection.close()
+        else:
+            raise AnsibleError("invalid meta action requested: %s" % meta_action, obj=task._ds)

lib/ansible/plugins/strategies/linear.py

@@ -178,16 +178,7 @@ class StrategyModule(StrategyBase):
                             continue
 
                     if task.action == 'meta':
-                        # meta tasks store their args in the _raw_params field of args,
+                        self._execute_meta(task, play_context, iterator)
-                        # since they do not use k=v pairs, so get that
-                        meta_action = task.args.get('_raw_params')
-                        if meta_action == 'noop':
-                            # FIXME: issue a callback for the noop here?
-                            continue
-                        elif meta_action == 'flush_handlers':
-                            self.run_handlers(iterator, play_context)
-                        else:
-                            raise AnsibleError("invalid meta action requested: %s" % meta_action, obj=task._ds)
                     else:
                         # handle step if needed, skip meta actions as they are used internally
                         if self._step and choose_step: