Merge pull request #11643 from bcoca/meta_inventory_and_fixes

Meta inventory and fixes
pull/11954/head
Brian Coca 9 years ago
commit fbc7224066

@ -19,6 +19,8 @@ Major Changes:
* template code now retains types for bools and numbers instead of turning them into strings. * template code now retains types for bools and numbers instead of turning them into strings.
If you need the old behaviour, quote the value and it will get passed around as a string If you need the old behaviour, quote the value and it will get passed around as a string
* Consolidated code from modules using urllib2 to normalize features, TLS and SNI support * Consolidated code from modules using urllib2 to normalize features, TLS and SNI support
* Consiidated code from modules using urllib2 to normalize features, TLS and SNI support
* added meta: refresh_inventory to force rereading the inventory in a play
Deprecated Modules (new ones in parens): Deprecated Modules (new ones in parens):
* ec2_ami_search (ec2_ami_find) * ec2_ami_search (ec2_ami_find)

@ -143,6 +143,7 @@ DEFAULT_EXECUTABLE = get_config(p, DEFAULTS, 'executable', 'ANSIBLE_EXECU
DEFAULT_GATHERING = get_config(p, DEFAULTS, 'gathering', 'ANSIBLE_GATHERING', 'implicit').lower() DEFAULT_GATHERING = get_config(p, DEFAULTS, 'gathering', 'ANSIBLE_GATHERING', 'implicit').lower()
DEFAULT_LOG_PATH = shell_expand_path(get_config(p, DEFAULTS, 'log_path', 'ANSIBLE_LOG_PATH', '')) DEFAULT_LOG_PATH = shell_expand_path(get_config(p, DEFAULTS, 'log_path', 'ANSIBLE_LOG_PATH', ''))
DEFAULT_FORCE_HANDLERS = get_config(p, DEFAULTS, 'force_handlers', 'ANSIBLE_FORCE_HANDLERS', False, boolean=True) DEFAULT_FORCE_HANDLERS = get_config(p, DEFAULTS, 'force_handlers', 'ANSIBLE_FORCE_HANDLERS', False, boolean=True)
DEFAULT_INVENTORY_IGNORE = get_config(p, DEFAULTS, 'inventory_ignore_extensions', 'ANSIBLE_INVENTORY_IGNORE', ["~", ".orig", ".bak", ".ini", ".cfg", ".retry", ".pyc", ".pyo"], islist=True)
# selinux # selinux
DEFAULT_SELINUX_SPECIAL_FS = get_config(p, 'selinux', 'special_context_filesystems', None, 'fuse, nfs, vboxsf, ramfs', islist=True) DEFAULT_SELINUX_SPECIAL_FS = get_config(p, 'selinux', 'special_context_filesystems', None, 'fuse, nfs, vboxsf, ramfs', islist=True)
@ -197,7 +198,7 @@ HOST_KEY_CHECKING = get_config(p, DEFAULTS, 'host_key_checking', '
SYSTEM_WARNINGS = get_config(p, DEFAULTS, 'system_warnings', 'ANSIBLE_SYSTEM_WARNINGS', True, boolean=True) SYSTEM_WARNINGS = get_config(p, DEFAULTS, 'system_warnings', 'ANSIBLE_SYSTEM_WARNINGS', True, boolean=True)
DEPRECATION_WARNINGS = get_config(p, DEFAULTS, 'deprecation_warnings', 'ANSIBLE_DEPRECATION_WARNINGS', True, boolean=True) DEPRECATION_WARNINGS = get_config(p, DEFAULTS, 'deprecation_warnings', 'ANSIBLE_DEPRECATION_WARNINGS', True, boolean=True)
DEFAULT_CALLABLE_WHITELIST = get_config(p, DEFAULTS, 'callable_whitelist', 'ANSIBLE_CALLABLE_WHITELIST', [], islist=True) DEFAULT_CALLABLE_WHITELIST = get_config(p, DEFAULTS, 'callable_whitelist', 'ANSIBLE_CALLABLE_WHITELIST', [], islist=True)
COMMAND_WARNINGS = get_config(p, DEFAULTS, 'command_warnings', 'ANSIBLE_COMMAND_WARNINGS', False, boolean=True) COMMAND_WARNINGS = get_config(p, DEFAULTS, 'command_warnings', 'ANSIBLE_COMMAND_WARNINGS', True, boolean=True)
DEFAULT_LOAD_CALLBACK_PLUGINS = get_config(p, DEFAULTS, 'bin_ansible_callbacks', 'ANSIBLE_LOAD_CALLBACK_PLUGINS', False, boolean=True) DEFAULT_LOAD_CALLBACK_PLUGINS = get_config(p, DEFAULTS, 'bin_ansible_callbacks', 'ANSIBLE_LOAD_CALLBACK_PLUGINS', False, boolean=True)
DEFAULT_CALLBACK_WHITELIST = get_config(p, DEFAULTS, 'callback_whitelist', 'ANSIBLE_CALLBACK_WHITELIST', [], islist=True) DEFAULT_CALLBACK_WHITELIST = get_config(p, DEFAULTS, 'callback_whitelist', 'ANSIBLE_CALLBACK_WHITELIST', [], islist=True)
RETRY_FILES_ENABLED = get_config(p, DEFAULTS, 'retry_files_enabled', 'ANSIBLE_RETRY_FILES_ENABLED', True, boolean=True) RETRY_FILES_ENABLED = get_config(p, DEFAULTS, 'retry_files_enabled', 'ANSIBLE_RETRY_FILES_ENABLED', True, boolean=True)

@ -26,15 +26,12 @@ import re
import stat import stat
from ansible import constants as C from ansible import constants as C
from ansible import errors from ansible.errors import AnsibleError
from ansible.inventory.ini import InventoryParser from ansible.inventory.dir import InventoryDirectory, get_file_parser
from ansible.inventory.script import InventoryScript
from ansible.inventory.dir import InventoryDirectory
from ansible.inventory.group import Group from ansible.inventory.group import Group
from ansible.inventory.host import Host from ansible.inventory.host import Host
from ansible.plugins import vars_loader from ansible.plugins import vars_loader
from ansible.utils.path import is_executable
from ansible.utils.vars import combine_vars from ansible.utils.vars import combine_vars
class Inventory(object): class Inventory(object):
@ -63,6 +60,7 @@ class Inventory(object):
self._hosts_cache = {} self._hosts_cache = {}
self._groups_list = {} self._groups_list = {}
self._pattern_cache = {} self._pattern_cache = {}
self._vars_plugins = []
# to be set by calling set_playbook_basedir by playbook code # to be set by calling set_playbook_basedir by playbook code
self._playbook_basedir = None self._playbook_basedir = None
@ -75,6 +73,10 @@ class Inventory(object):
self._also_restriction = None self._also_restriction = None
self._subset = None self._subset = None
self.parse_inventory(host_list)
def parse_inventory(self, host_list):
if isinstance(host_list, basestring): if isinstance(host_list, basestring):
if "," in host_list: if "," in host_list:
host_list = host_list.split(",") host_list = host_list.split(",")
@ -102,49 +104,20 @@ class Inventory(object):
else: else:
all.add_host(Host(x)) all.add_host(Host(x))
elif os.path.exists(host_list): elif os.path.exists(host_list):
#TODO: switch this to a plugin loader and a 'condition' per plugin on which it should be tried, restoring 'inventory pllugins'
if os.path.isdir(host_list): if os.path.isdir(host_list):
# Ensure basedir is inside the directory # Ensure basedir is inside the directory
self.host_list = os.path.join(self.host_list, "") host_list = os.path.join(self.host_list, "")
self.parser = InventoryDirectory(loader=self._loader, filename=host_list) self.parser = InventoryDirectory(loader=self._loader, filename=host_list)
self.groups = self.parser.groups.values()
else:
# check to see if the specified file starts with a
# shebang (#!/), so if an error is raised by the parser
# class we can show a more apropos error
shebang_present = False
try:
with open(host_list, "r") as inv_file:
first_line = inv_file.readline()
if first_line.startswith("#!"):
shebang_present = True
except IOError:
pass
if is_executable(host_list):
try:
self.parser = InventoryScript(loader=self._loader, filename=host_list)
self.groups = self.parser.groups.values()
except errors.AnsibleError:
if not shebang_present:
raise errors.AnsibleError("The file %s is marked as executable, but failed to execute correctly. " % host_list + \
"If this is not supposed to be an executable script, correct this with `chmod -x %s`." % host_list)
else:
raise
else:
try:
self.parser = InventoryParser(filename=host_list)
self.groups = self.parser.groups.values()
except errors.AnsibleError:
if shebang_present:
raise errors.AnsibleError("The file %s looks like it should be an executable inventory script, but is not marked executable. " % host_list + \
"Perhaps you want to correct this with `chmod +x %s`?" % host_list)
else: else:
raise self.parser = get_file_parser(hostsfile, self._loader)
vars_loader.add_directory(self.basedir(), with_subdir=True) vars_loader.add_directory(self.basedir(), with_subdir=True)
if self.parser:
self.groups = self.parser.groups.values()
else: else:
raise errors.AnsibleError("Unable to find an inventory file (%s), " # should never happen, but JIC
"specify one with -i ?" % host_list) raise AnsibleError("Unable to parse %s as an inventory source" % host_list)
self._vars_plugins = [ x for x in vars_loader.all(self) ] self._vars_plugins = [ x for x in vars_loader.all(self) ]
@ -166,7 +139,7 @@ class Inventory(object):
else: else:
return fnmatch.fnmatch(str, pattern_str) return fnmatch.fnmatch(str, pattern_str)
except Exception, e: except Exception, e:
raise errors.AnsibleError('invalid host pattern: %s' % pattern_str) raise AnsibleError('invalid host pattern: %s' % pattern_str)
def _match_list(self, items, item_attr, pattern_str): def _match_list(self, items, item_attr, pattern_str):
results = [] results = []
@ -176,7 +149,7 @@ class Inventory(object):
else: else:
pattern = re.compile(pattern_str[1:]) pattern = re.compile(pattern_str[1:])
except Exception, e: except Exception, e:
raise errors.AnsibleError('invalid host pattern: %s' % pattern_str) raise AnsibleError('invalid host pattern: %s' % pattern_str)
for item in items: for item in items:
if pattern.match(getattr(item, item_attr)): if pattern.match(getattr(item, item_attr)):
@ -286,7 +259,7 @@ class Inventory(object):
first = int(first) first = int(first)
if last: if last:
if first < 0: if first < 0:
raise errors.AnsibleError("invalid range: negative indices cannot be used as the first item in a range") raise AnsibleError("invalid range: negative indices cannot be used as the first item in a range")
last = int(last) last = int(last)
else: else:
last = first last = first
@ -324,7 +297,7 @@ class Inventory(object):
else: else:
return [ hosts[left] ] return [ hosts[left] ]
except IndexError: except IndexError:
raise errors.AnsibleError("no hosts matching the pattern '%s' were found" % pat) raise AnsibleError("no hosts matching the pattern '%s' were found" % pat)
def _create_implicit_localhost(self, pattern): def _create_implicit_localhost(self, pattern):
new_host = Host(pattern) new_host = Host(pattern)
@ -467,7 +440,7 @@ class Inventory(object):
host = self.get_host(hostname) host = self.get_host(hostname)
if host is None: if host is None:
raise errors.AnsibleError("host not found: %s" % hostname) raise AnsibleError("host not found: %s" % hostname)
vars = {} vars = {}
@ -499,7 +472,7 @@ class Inventory(object):
self.groups.append(group) self.groups.append(group)
self._groups_list = None # invalidate internal cache self._groups_list = None # invalidate internal cache
else: else:
raise errors.AnsibleError("group already in inventory: %s" % group.name) raise AnsibleError("group already in inventory: %s" % group.name)
def list_hosts(self, pattern="all"): def list_hosts(self, pattern="all"):
@ -670,3 +643,14 @@ class Inventory(object):
# all done, results is a dictionary of variables for this particular host. # all done, results is a dictionary of variables for this particular host.
return results return results
def refresh_inventory(self):
self.clear_pattern_cache()
self._hosts_cache = {}
self._vars_per_host = {}
self._vars_per_group = {}
self._groups_list = {}
self.groups = []
self.parse_inventory(self.host_list)

@ -27,11 +27,58 @@ from ansible.errors import AnsibleError
from ansible.inventory.host import Host from ansible.inventory.host import Host
from ansible.inventory.group import Group from ansible.inventory.group import Group
from ansible.inventory.ini import InventoryParser
from ansible.inventory.script import InventoryScript
from ansible.utils.path import is_executable
from ansible.utils.vars import combine_vars from ansible.utils.vars import combine_vars
from ansible.utils.path import is_executable
from ansible.inventory.ini import InventoryParser as InventoryINIParser
from ansible.inventory.script import InventoryScript
__all__ = ['get_file_parser']
def get_file_parser(hostsfile, loader):
# check to see if the specified file starts with a
# shebang (#!/), so if an error is raised by the parser
# class we can show a more apropos error
shebang_present = False
processed = False
myerr = []
parser = None
try:
inv_file = open(hostsfile)
first_line = inv_file.readlines()[0]
inv_file.close()
if first_line.startswith('#!'):
shebang_present = True
except:
pass
if is_executable(hostsfile):
try:
parser = InventoryScript(loader=loader, filename=hostsfile)
processed = True
except Exception as e:
myerr.append("The file %s is marked as executable, but failed to execute correctly. " % hostsfile + \
"If this is not supposed to be an executable script, correct this with `chmod -x %s`." % hostsfile)
myerr.append(str(e))
if not processed:
try:
parser = InventoryINIParser(filename=hostsfile)
processed = True
except Exception as e:
if shebang_present and not is_executable(hostsfile):
myerr.append("The file %s looks like it should be an executable inventory script, but is not marked executable. " % hostsfile + \
"Perhaps you want to correct this with `chmod +x %s`?" % hostsfile)
else:
myerr.append(str(e))
if not processed and myerr:
raise AnsibleError( '\n'.join(myerr) )
return parser
class InventoryDirectory(object): class InventoryDirectory(object):
''' Host inventory parser for ansible using a directory of inventories. ''' ''' Host inventory parser for ansible using a directory of inventories. '''
@ -48,7 +95,7 @@ class InventoryDirectory(object):
for i in self.names: for i in self.names:
# Skip files that end with certain extensions or characters # Skip files that end with certain extensions or characters
if any(i.endswith(ext) for ext in ("~", ".orig", ".bak", ".ini", ".cfg", ".retry", ".pyc", ".pyo")): if any(i.endswith(ext) for ext in C.DEFAULT_INVENTORY_IGNORE):
continue continue
# Skip hidden files # Skip hidden files
if i.startswith('.') and not i.startswith('./'): if i.startswith('.') and not i.startswith('./'):
@ -59,10 +106,14 @@ class InventoryDirectory(object):
fullpath = os.path.join(self.directory, i) fullpath = os.path.join(self.directory, i)
if os.path.isdir(fullpath): if os.path.isdir(fullpath):
parser = InventoryDirectory(loader=loader, filename=fullpath) parser = InventoryDirectory(loader=loader, filename=fullpath)
elif is_executable(fullpath):
parser = InventoryScript(loader=loader, filename=fullpath)
else: else:
parser = InventoryParser(filename=fullpath) parser = get_file_parser(fullpath, loader)
if parser is None:
#FIXME: needs to use display
import warnings
warnings.warning("Could not find parser for %s, skipping" % fullpath)
continue
self.parsers.append(parser) self.parsers.append(parser)
# retrieve all groups and hosts form the parser and add them to # retrieve all groups and hosts form the parser and add them to

@ -0,0 +1 @@
These are not currently in use, but this is what the future of inventory will become after 2.0

@ -22,11 +22,14 @@ from __future__ import (absolute_import, division, print_function)
__metaclass__ = type __metaclass__ = type
import os import os
from ansible import constants as C
from . aggregate import InventoryAggregateParser from . aggregate import InventoryAggregateParser
class InventoryDirectoryParser(InventoryAggregateParser): class InventoryDirectoryParser(InventoryAggregateParser):
CONDITION="is_dir(%s)"
def __init__(self, inven_directory): def __init__(self, inven_directory):
directory = inven_directory directory = inven_directory
names = os.listdir(inven_directory) names = os.listdir(inven_directory)
@ -35,7 +38,7 @@ class InventoryDirectoryParser(InventoryAggregateParser):
# Clean up the list of filenames # Clean up the list of filenames
for filename in names: for filename in names:
# Skip files that end with certain extensions or characters # Skip files that end with certain extensions or characters
if any(filename.endswith(ext) for ext in ("~", ".orig", ".bak", ".ini", ".retry", ".pyc", ".pyo")): if any(filename.endswith(ext) for ext in C.DEFAULT_INVENTORY_IGNORE):
continue continue
# Skip hidden files # Skip hidden files
if filename.startswith('.') and not filename.startswith('.{0}'.format(os.path.sep)): if filename.startswith('.') and not filename.startswith('.{0}'.format(os.path.sep)):

@ -23,10 +23,13 @@ __metaclass__ = type
import os import os
from ansible import constants as C
from . import InventoryParser from . import InventoryParser
class InventoryIniParser(InventoryAggregateParser): class InventoryIniParser(InventoryAggregateParser):
CONDITION="is_file(%s)"
def __init__(self, inven_directory): def __init__(self, inven_directory):
directory = inven_directory directory = inven_directory
names = os.listdir(inven_directory) names = os.listdir(inven_directory)
@ -35,7 +38,7 @@ class InventoryIniParser(InventoryAggregateParser):
# Clean up the list of filenames # Clean up the list of filenames
for filename in names: for filename in names:
# Skip files that end with certain extensions or characters # Skip files that end with certain extensions or characters
if any(filename.endswith(ext) for ext in ("~", ".orig", ".bak", ".ini", ".retry", ".pyc", ".pyo")): if any(filename.endswith(ext) for ext in C.DEFAULT_INVENTORY_IGNORE):
continue continue
# Skip hidden files # Skip hidden files
if filename.startswith('.') and not filename.startswith('.{0}'.format(os.path.sep)): if filename.startswith('.') and not filename.startswith('.{0}'.format(os.path.sep)):

@ -0,0 +1,31 @@
# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
#############################################
# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import os
from ansible import constants as C
from . import InventoryParser
class InventoryScriptParser(InventoryParser):
CONDITION="is_file(%s) and is_executable(%s)"

@ -506,3 +506,21 @@ class StrategyBase:
self._display.banner(msg) self._display.banner(msg)
return ret return ret
def _execute_meta(self, task, play_context, iterator):
# meta tasks store their args in the _raw_params field of args,
# since they do not use k=v pairs, so get that
meta_action = task.args.get('_raw_params')
if meta_action == 'noop':
# FIXME: issue a callback for the noop here?
pass
elif meta_action == 'flush_handlers':
self.run_handlers(iterator, play_context)
elif meta_action == 'refresh_inventory':
self._inventory.refresh_inventory()
#elif meta_action == 'reset_connection':
# connection_info.connection.close()
else:
raise AnsibleError("invalid meta action requested: %s" % meta_action, obj=task._ds)

@ -178,16 +178,7 @@ class StrategyModule(StrategyBase):
continue continue
if task.action == 'meta': if task.action == 'meta':
# meta tasks store their args in the _raw_params field of args, self._execute_meta(task, play_context, iterator)
# since they do not use k=v pairs, so get that
meta_action = task.args.get('_raw_params')
if meta_action == 'noop':
# FIXME: issue a callback for the noop here?
continue
elif meta_action == 'flush_handlers':
self.run_handlers(iterator, play_context)
else:
raise AnsibleError("invalid meta action requested: %s" % meta_action, obj=task._ds)
else: else:
# handle step if needed, skip meta actions as they are used internally # handle step if needed, skip meta actions as they are used internally
if self._step and choose_step: if self._step and choose_step:

Loading…
Cancel
Save