From c5cae87ecadc099800ef38870943a2c242a3498b Mon Sep 17 00:00:00 2001 From: Jeroen Hoekx Date: Fri, 13 Apr 2012 14:39:54 +0200 Subject: [PATCH 1/8] Refactor inventory code out of Runner. This introduces the Inventory class. Playbook uses the internals of Runner to limit the number of hosts to poll asynchronously. To accomodate this, Inventory can be restricted to specific hosts. --- bin/ansible | 8 +- lib/ansible/inventory.py | 189 +++++++++++++++++++++++++++++++++++++++ lib/ansible/playbook.py | 62 +++++++------ lib/ansible/runner.py | 158 +++----------------------------- 4 files changed, 241 insertions(+), 176 deletions(-) create mode 100644 lib/ansible/inventory.py diff --git a/bin/ansible b/bin/ansible index f254eaf3ff9..844004ecc4d 100755 --- a/bin/ansible +++ b/bin/ansible @@ -98,11 +98,11 @@ class Cli(object): # ---------------------------------------------- - def get_polling_runner(self, old_runner, hosts, jid): + def get_polling_runner(self, old_runner, jid): return ansible.runner.Runner( module_name='async_status', module_path=old_runner.module_path, module_args="jid=%s" % jid, remote_user=old_runner.remote_user, - remote_pass=old_runner.remote_pass, host_list=hosts, + remote_pass=old_runner.remote_pass, inventory=old_runner.inventory, timeout=old_runner.timeout, forks=old_runner.forks, remote_port=old_runner.remote_port, pattern='*', callbacks=self.silent_callbacks, verbose=True, @@ -138,8 +138,10 @@ class Cli(object): clock = options.seconds while (clock >= 0): - polling_runner = self.get_polling_runner(runner, poll_hosts, jid) + runner.inventory.restrict_to(poll_hosts) + polling_runner = self.get_polling_runner(runner, jid) poll_results = polling_runner.run() + runner.inventory.lift_restrictions() if poll_results is None: break for (host, host_result) in poll_results['contacted'].iteritems(): diff --git a/lib/ansible/inventory.py b/lib/ansible/inventory.py new file mode 100644 index 00000000000..448ef372b04 --- /dev/null +++ b/lib/ansible/inventory.py @@ -0,0 +1,189 @@ +# (c) 2012, Michael DeHaan +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +############################################# + +import fnmatch +import os +import subprocess + +import constants as C +from ansible import errors +from ansible import utils + +class Inventory(object): + """ Host inventory for ansible. + + The inventory is either a simple text file with systems and [groups] of + systems, or a script that will be called with --list or --host. + """ + + def __init__(self, host_list=C.DEFAULT_HOST_LIST, extra_vars=None): + + self._restriction = None + + if type(host_list) == list: + self.host_list = host_list + self.groups = dict(ungrouped=host_list) + self._is_script = False + return + + inventory_file = os.path.expanduser(host_list) + if not os.path.exists(inventory_file): + raise errors.AnsibleFileNotFound("inventory file not found: %s" % host_list) + + self.inventory_file = os.path.abspath(inventory_file) + + if os.access(self.inventory_file, os.X_OK): + self.host_list, self.groups = self._parse_from_script(extra_vars) + self._is_script = True + else: + self.host_list, self.groups = self._parse_from_file() + self._is_script = False + + # ***************************************************** + # Public API + + def list_hosts(self, pattern="all"): + """ Return a list of hosts [matching the pattern] """ + if self._restriction is None: + host_list = self.host_list + else: + host_list = [ h for h in self.host_list if h in self._restriction ] + return [ h for h in host_list if self._matches(h, pattern) ] + + def restrict_to(self, restriction): + """ Restrict list operations to the hosts given in restriction """ + if type(restriction)!=list: + restriction = [ restriction ] + + self._restriction = restriction + + def lift_restriction(self): + """ Do not restrict list operations """ + self._restriction = None + + def get_variables(self, host, extra_vars=None): + """ Return the variables associated with this host. """ + + if not self._is_script: + return {} + + return self._get_variables_from_script(host, extra_vars) + + # ***************************************************** + + def _parse_from_file(self): + ''' parse a textual host file ''' + + results = [] + groups = dict(ungrouped=[]) + lines = file(self.inventory_file).read().split("\n") + group_name = 'ungrouped' + for item in lines: + item = item.lstrip().rstrip() + if item.startswith("#"): + # ignore commented out lines + pass + elif item.startswith("["): + # looks like a group + group_name = item.replace("[","").replace("]","").lstrip().rstrip() + groups[group_name] = [] + elif item != "": + # looks like a regular host + groups[group_name].append(item) + if not item in results: + results.append(item) + return (results, groups) + + # ***************************************************** + + def _parse_from_script(self, extra_vars=None): + ''' evaluate a script that returns list of hosts by groups ''' + + results = [] + groups = dict(ungrouped=[]) + + cmd = [self.inventory_file, '--list'] + + if extra_vars: + cmd.extend(['--extra-vars', extra_vars]) + + cmd = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) + out, err = cmd.communicate() + rc = cmd.returncode + if rc: + raise errors.AnsibleError("%s: %s" % self.inventory_file, err) + + try: + groups = utils.json_loads(out) + except: + raise errors.AnsibleError("invalid JSON response from script: %s" % self.inventory_file) + + for (groupname, hostlist) in groups.iteritems(): + for host in hostlist: + if host not in results: + results.append(host) + return (results, groups) + + # ***************************************************** + + def _get_variables_from_script(self, host, extra_vars=None): + ''' support per system variabes from external variable scripts, see web docs ''' + + cmd = [self.inventory_file, '--host', host] + + if extra_vars: + cmd.extend(['--extra-vars', extra_vars]) + + cmd = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False + ) + out, err = cmd.communicate() + + variables = {} + try: + variables = utils.json_loads(out) + except: + raise errors.AnsibleError("%s returned invalid result when called with hostname %s" % ( + self.inventory_file, + host + )) + return variables + + def _matches(self, host_name, pattern): + ''' returns if a hostname is matched by the pattern ''' + + # a pattern is in fnmatch format but more than one pattern + # can be strung together with semicolons. ex: + # atlanta-web*.example.com;dc-web*.example.com + + if host_name == '': + return False + pattern = pattern.replace(";",":") + subpatterns = pattern.split(":") + for subpattern in subpatterns: + if subpattern == 'all': + return True + if fnmatch.fnmatch(host_name, subpattern): + return True + elif subpattern in self.groups: + if host_name in self.groups[subpattern]: + return True + return False diff --git a/lib/ansible/playbook.py b/lib/ansible/playbook.py index ba5115d02d9..c3e195e0409 100755 --- a/lib/ansible/playbook.py +++ b/lib/ansible/playbook.py @@ -17,6 +17,7 @@ ############################################# +import ansible.inventory import ansible.runner import ansible.constants as C from ansible import utils @@ -68,7 +69,6 @@ class PlayBook(object): if playbook is None or callbacks is None or runner_callbacks is None or stats is None: raise Exception('missing required arguments') - self.host_list = host_list self.module_path = module_path self.forks = forks self.timeout = timeout @@ -88,9 +88,13 @@ class PlayBook(object): self.basedir = os.path.dirname(playbook) self.playbook = self._parse_playbook(playbook) - self.host_list, self.groups = ansible.runner.Runner.parse_hosts( - host_list, override_hosts=self.override_hosts, extra_vars=self.extra_vars) - + if override_hosts is not None: + if type(override_hosts) != list: + raise errors.AnsibleError("override hosts must be a list") + self.inventory = ansible.inventory.Inventory(override_hosts) + else: + self.inventory = ansible.inventory.Inventory(host_list) + # ***************************************************** def _get_vars(self, play, dirname): @@ -233,7 +237,6 @@ class PlayBook(object): def _async_poll(self, runner, hosts, async_seconds, async_poll_interval, only_if): ''' launch an async job, if poll_interval is set, wait for completion ''' - runner.host_list = hosts runner.background = async_seconds results = runner.run() self.stats.compute(results, poll=True) @@ -257,7 +260,7 @@ class PlayBook(object): return results clock = async_seconds - runner.host_list = self.hosts_to_poll(results) + host_list = self.hosts_to_poll(results) poll_results = results while (clock >= 0): @@ -267,11 +270,13 @@ class PlayBook(object): runner.module_name = 'async_status' runner.background = 0 runner.pattern = '*' + self.inventory.restrict_to(host_list) poll_results = runner.run() self.stats.compute(poll_results, poll=True) - runner.host_list = self.hosts_to_poll(poll_results) + host_list = self.hosts_to_poll(poll_results) + self.inventory.lift_restriction() - if len(runner.host_list) == 0: + if len(host_list) == 0: break if poll_results is None: break @@ -298,15 +303,16 @@ class PlayBook(object): # ***************************************************** - def _run_module(self, pattern, host_list, module, args, vars, remote_user, + def _run_module(self, pattern, module, args, vars, remote_user, async_seconds, async_poll_interval, only_if, sudo, transport): ''' run a particular module step in a playbook ''' - hosts = [ h for h in host_list if (h not in self.stats.failures) and (h not in self.stats.dark)] + hosts = [ h for h in self.inventory.list_hosts() if (h not in self.stats.failures) and (h not in self.stats.dark)] + self.inventory.restrict_to(hosts) runner = ansible.runner.Runner( - pattern=pattern, groups=self.groups, module_name=module, - module_args=args, host_list=hosts, forks=self.forks, + pattern=pattern, inventory=self.inventory, module_name=module, + module_args=args, forks=self.forks, remote_pass=self.remote_pass, module_path=self.module_path, timeout=self.timeout, remote_user=remote_user, remote_port=self.remote_port, module_vars=vars, @@ -317,13 +323,16 @@ class PlayBook(object): ) if async_seconds == 0: - return runner.run() + results = runner.run() else: - return self._async_poll(runner, hosts, async_seconds, async_poll_interval, only_if) + results = self._async_poll(runner, hosts, async_seconds, async_poll_interval, only_if) + + self.inventory.lift_restriction() + return results # ***************************************************** - def _run_task(self, pattern=None, host_list=None, task=None, + def _run_task(self, pattern=None, task=None, remote_user=None, handlers=None, conditional=False, sudo=False, transport=None): ''' run a single task in the playbook and recursively run any subtasks. ''' @@ -354,7 +363,7 @@ class PlayBook(object): # load up an appropriate ansible runner to # run the task in parallel - results = self._run_module(pattern, host_list, module_name, + results = self._run_module(pattern, module_name, module_args, module_vars, remote_user, async_seconds, async_poll_interval, only_if, sudo, transport) @@ -406,7 +415,7 @@ class PlayBook(object): # ***************************************************** - def _do_conditional_imports(self, vars_files, host_list): + def _do_conditional_imports(self, vars_files): ''' handle the vars_files section, which can contain variables ''' # FIXME: save parsed variable results in memory to avoid excessive re-reading/parsing @@ -417,7 +426,7 @@ class PlayBook(object): if type(vars_files) != list: raise errors.AnsibleError("vars_files must be a list") - for host in host_list: + for host in self.inventory.list_hosts(): cache_vars = SETUP_CACHE.get(host,{}) SETUP_CACHE[host] = cache_vars for filename in vars_files: @@ -460,16 +469,18 @@ class PlayBook(object): if vars_files is not None: self.callbacks.on_setup_secondary() - self._do_conditional_imports(vars_files, self.host_list) + self._do_conditional_imports(vars_files) else: self.callbacks.on_setup_primary() - host_list = [ h for h in self.host_list if not (h in self.stats.failures or h in self.stats.dark) ] + host_list = [ h for h in self.inventory.list_hosts(pattern) + if not (h in self.stats.failures or h in self.stats.dark) ] + self.inventory.restrict_to(host_list) # push any variables down to the system setup_results = ansible.runner.Runner( - pattern=pattern, groups=self.groups, module_name='setup', - module_args=vars, host_list=host_list, + pattern=pattern, module_name='setup', + module_args=vars, inventory=self.inventory, forks=self.forks, module_path=self.module_path, timeout=self.timeout, remote_user=user, remote_pass=self.remote_pass, remote_port=self.remote_port, @@ -479,6 +490,8 @@ class PlayBook(object): ).run() self.stats.compute(setup_results, setup=True) + self.inventory.lift_restriction() + # now for each result, load into the setup cache so we can # let runner template out future commands setup_ok = setup_results.get('contacted', {}) @@ -494,7 +507,6 @@ class PlayBook(object): SETUP_CACHE[h].update(extra_vars) except: SETUP_CACHE[h] = extra_vars - return host_list # ***************************************************** @@ -530,7 +542,6 @@ class PlayBook(object): for task in tasks: self._run_task( pattern=pattern, - host_list=self.host_list, task=task, handlers=handlers, remote_user=user, @@ -547,16 +558,17 @@ class PlayBook(object): for task in handlers: triggered_by = task.get('run', None) if type(triggered_by) == list: + self.inventory.restrict_to(triggered_by) self._run_task( pattern=pattern, task=task, handlers=[], - host_list=triggered_by, conditional=True, remote_user=user, sudo=sudo, transport=transport ) + self.inventory.lift_restriction() # end of execution for this particular pattern. Multiple patterns # can be in a single playbook file diff --git a/lib/ansible/runner.py b/lib/ansible/runner.py index 418a069ba8c..50c4029a45d 100755 --- a/lib/ansible/runner.py +++ b/lib/ansible/runner.py @@ -18,7 +18,6 @@ ################################################ -import fnmatch import multiprocessing import signal import os @@ -27,10 +26,10 @@ import Queue import random import traceback import tempfile -import subprocess import ansible.constants as C import ansible.connection +import ansible.inventory from ansible import utils from ansible import errors from ansible import callbacks as ans_callbacks @@ -67,8 +66,6 @@ def _executor_hook(job_queue, result_queue): class Runner(object): - _external_variable_script = None - def __init__(self, host_list=C.DEFAULT_HOST_LIST, module_path=C.DEFAULT_MODULE_PATH, module_name=C.DEFAULT_MODULE_NAME, module_args=C.DEFAULT_MODULE_ARGS, forks=C.DEFAULT_FORKS, timeout=C.DEFAULT_TIMEOUT, pattern=C.DEFAULT_PATTERN, @@ -76,7 +73,7 @@ class Runner(object): sudo_pass=C.DEFAULT_SUDO_PASS, remote_port=C.DEFAULT_REMOTE_PORT, background=0, basedir=None, setup_cache=None, transport=C.DEFAULT_TRANSPORT, conditional='True', groups={}, callbacks=None, verbose=False, - debug=False, sudo=False, extra_vars=None, module_vars=None): + debug=False, sudo=False, extra_vars=None, module_vars=None, inventory=None): if setup_cache is None: setup_cache = {} @@ -92,11 +89,10 @@ class Runner(object): self.transport = transport self.connector = ansible.connection.Connection(self, self.transport) - if type(host_list) == str: - self.host_list, self.groups = self.parse_hosts(host_list) + if inventory is None: + self.inventory = ansible.inventory.Inventory(host_list, extra_vars) else: - self.host_list = host_list - self.groups = groups + self.inventory = inventory self.setup_cache = setup_cache self.conditional = conditional @@ -130,106 +126,6 @@ class Runner(object): # ***************************************************** - @classmethod - def parse_hosts_from_regular_file(cls, host_list): - ''' parse a textual host file ''' - - results = [] - groups = dict(ungrouped=[]) - lines = file(host_list).read().split("\n") - group_name = 'ungrouped' - for item in lines: - item = item.lstrip().rstrip() - if item.startswith("#"): - # ignore commented out lines - pass - elif item.startswith("["): - # looks like a group - group_name = item.replace("[","").replace("]","").lstrip().rstrip() - groups[group_name] = [] - elif item != "": - # looks like a regular host - groups[group_name].append(item) - if not item in results: - results.append(item) - return (results, groups) - - # ***************************************************** - - @classmethod - def parse_hosts_from_script(cls, host_list, extra_vars): - ''' evaluate a script that returns list of hosts by groups ''' - - results = [] - groups = dict(ungrouped=[]) - host_list = os.path.abspath(host_list) - cls._external_variable_script = host_list - cmd = [host_list, '--list'] - if extra_vars: - cmd.extend(['--extra-vars', extra_vars]) - cmd = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) - out, err = cmd.communicate() - rc = cmd.returncode - if rc: - raise errors.AnsibleError("%s: %s" % (host_list, err)) - try: - groups = utils.json_loads(out) - except: - raise errors.AnsibleError("invalid JSON response from script: %s" % host_list) - for (groupname, hostlist) in groups.iteritems(): - for host in hostlist: - if host not in results: - results.append(host) - return (results, groups) - - # ***************************************************** - - @classmethod - def parse_hosts(cls, host_list, override_hosts=None, extra_vars=None): - ''' parse the host inventory file, returns (hosts, groups) ''' - - if override_hosts is not None: - if type(override_hosts) != list: - raise errors.AnsibleError("override hosts must be a list") - return (override_hosts, dict(ungrouped=override_hosts)) - - if type(host_list) == list: - raise Exception("function can only be called on inventory files") - - host_list = os.path.expanduser(host_list) - if not os.path.exists(host_list): - raise errors.AnsibleFileNotFound("inventory file not found: %s" % host_list) - - if not os.access(host_list, os.X_OK): - return Runner.parse_hosts_from_regular_file(host_list) - else: - return Runner.parse_hosts_from_script(host_list, extra_vars) - - # ***************************************************** - - def _matches(self, host_name, pattern): - ''' returns if a hostname is matched by the pattern ''' - - # a pattern is in fnmatch format but more than one pattern - # can be strung together with semicolons. ex: - # atlanta-web*.example.com;dc-web*.example.com - - if host_name == '': - return False - pattern = pattern.replace(";",":") - subpatterns = pattern.split(":") - for subpattern in subpatterns: - if subpattern == 'all': - return True - if fnmatch.fnmatch(host_name, subpattern): - return True - elif subpattern in self.groups: - if host_name in self.groups[subpattern]: - return True - return False - - # ***************************************************** - def _connect(self, host): ''' connects to a host, returns (is_successful, connection_object OR traceback_string) ''' @@ -296,34 +192,6 @@ class Runner(object): # ***************************************************** - def _add_variables_from_script(self, conn, inject): - ''' support per system variabes from external variable scripts, see web docs ''' - - host = conn.host - - cmd = [Runner._external_variable_script, '--host', host] - if self.extra_vars: - cmd.extend(['--extra-vars', self.extra_vars]) - - cmd = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=False - ) - out, err = cmd.communicate() - inject2 = {} - try: - inject2 = utils.json_loads(out) - except: - raise errors.AnsibleError("%s returned invalid result when called with hostname %s" % ( - Runner._external_variable_script, - host - )) - # store injected variables in the templates - inject.update(inject2) - - # ***************************************************** - def _add_setup_vars(self, inject, args): ''' setup module variables need special handling ''' @@ -377,8 +245,9 @@ class Runner(object): if not eval(conditional): return [ utils.smjson(dict(skipped=True)), None, 'skipped' ] - if Runner._external_variable_script is not None: - self._add_variables_from_script(conn, inject) + host_variables = self.inventory.get_variables(conn.host, self.extra_vars) + inject.update(host_variables) + if self.module_name == 'setup': args = self._add_setup_vars(inject, args) args = self._add_setup_metadata(args) @@ -692,13 +561,6 @@ class Runner(object): # ***************************************************** - def _match_hosts(self, pattern): - ''' return all matched hosts fitting a pattern ''' - - return [ h for h in self.host_list if self._matches(h, pattern) ] - - # ***************************************************** - def _parallel_exec(self, hosts): ''' handles mulitprocessing when more than 1 fork is required ''' @@ -745,7 +607,7 @@ class Runner(object): results2["dark"][host] = result # hosts which were contacted but never got a chance to return - for host in self._match_hosts(self.pattern): + for host in self.inventory.list_hosts(self.pattern): if not (host in results2['dark'] or host in results2['contacted']): results2["dark"][host] = {} @@ -757,7 +619,7 @@ class Runner(object): ''' xfer & run module on all matched hosts ''' # find hosts that match the pattern - hosts = self._match_hosts(self.pattern) + hosts = self.inventory.list_hosts(self.pattern) if len(hosts) == 0: self.callbacks.on_no_hosts() return dict(contacted={}, dark={}) From 195e6d617b67c70043db7812fc2346d7a226c730 Mon Sep 17 00:00:00 2001 From: Jeroen Hoekx Date: Fri, 13 Apr 2012 20:50:30 +0200 Subject: [PATCH 2/8] Add tests for Inventory class. --- test/TestInventory.py | 143 ++++++++++++++++++++++++++++++++++++++++++ test/inventory_api.py | 39 ++++++++++++ test/simple_hosts | 12 ++++ 3 files changed, 194 insertions(+) create mode 100644 test/TestInventory.py create mode 100644 test/inventory_api.py create mode 100644 test/simple_hosts diff --git a/test/TestInventory.py b/test/TestInventory.py new file mode 100644 index 00000000000..b0e3c62ccf5 --- /dev/null +++ b/test/TestInventory.py @@ -0,0 +1,143 @@ +import os +import unittest + +from ansible.inventory import Inventory + +class TestInventory(unittest.TestCase): + + def setUp(self): + self.cwd = os.getcwd() + self.test_dir = os.path.join(self.cwd, 'test') + + self.inventory_file = os.path.join(self.test_dir, 'simple_hosts') + self.inventory_script = os.path.join(self.test_dir, 'inventory_api.py') + + os.chmod(self.inventory_script, 0755) + + def tearDown(self): + os.chmod(self.inventory_script, 0644) + + ### Simple inventory format tests + + def simple_inventory(self): + return Inventory( self.inventory_file ) + + def script_inventory(self): + return Inventory( self.inventory_script ) + + def test_simple(self): + inventory = self.simple_inventory() + hosts = inventory.list_hosts() + + expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki'] + assert hosts == expected_hosts + + def test_simple_all(self): + inventory = self.simple_inventory() + hosts = inventory.list_hosts('all') + + expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki'] + assert hosts == expected_hosts + + def test_simple_norse(self): + inventory = self.simple_inventory() + hosts = inventory.list_hosts("norse") + + expected_hosts=['thor', 'odin', 'loki'] + assert hosts == expected_hosts + + def test_simple_combined(self): + inventory = self.simple_inventory() + hosts = inventory.list_hosts("norse:greek") + + expected_hosts=['zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki'] + assert hosts == expected_hosts + + def test_simple_restrict(self): + inventory = self.simple_inventory() + + restricted_hosts = ['hera', 'poseidon', 'thor'] + expected_hosts=['zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki'] + + inventory.restrict_to(restricted_hosts) + hosts = inventory.list_hosts("norse:greek") + + assert hosts == restricted_hosts + + inventory.lift_restriction() + hosts = inventory.list_hosts("norse:greek") + + assert hosts == expected_hosts + + def test_simple_vars(self): + inventory = self.simple_inventory() + vars = inventory.get_variables('thor') + + assert vars == {} + + def test_simple_extra_vars(self): + inventory = self.simple_inventory() + vars = inventory.get_variables('thor', 'a=5') + + assert vars == {} + + ### Inventory API tests + + def test_script(self): + inventory = self.script_inventory() + hosts = inventory.list_hosts() + + expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki'] + + print "Expected: %s"%(expected_hosts) + print "Got : %s"%(hosts) + assert sorted(hosts) == sorted(expected_hosts) + + def test_script_all(self): + inventory = self.script_inventory() + hosts = inventory.list_hosts('all') + + expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki'] + assert sorted(hosts) == sorted(expected_hosts) + + def test_script_norse(self): + inventory = self.script_inventory() + hosts = inventory.list_hosts("norse") + + expected_hosts=['thor', 'odin', 'loki'] + assert sorted(hosts) == sorted(expected_hosts) + + def test_script_combined(self): + inventory = self.script_inventory() + hosts = inventory.list_hosts("norse:greek") + + expected_hosts=['zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki'] + assert sorted(hosts) == sorted(expected_hosts) + + def test_script_restrict(self): + inventory = self.script_inventory() + + restricted_hosts = ['hera', 'poseidon', 'thor'] + expected_hosts=['zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki'] + + inventory.restrict_to(restricted_hosts) + hosts = inventory.list_hosts("norse:greek") + + assert sorted(hosts) == sorted(restricted_hosts) + + inventory.lift_restriction() + hosts = inventory.list_hosts("norse:greek") + + assert sorted(hosts) == sorted(expected_hosts) + + def test_script_vars(self): + inventory = self.script_inventory() + vars = inventory.get_variables('thor') + + assert vars == {"hammer":True} + + def test_script_extra_vars(self): + inventory = self.script_inventory() + vars = inventory.get_variables('thor', 'simple=yes') + + assert vars == {"hammer":True, "simple": "yes"} \ No newline at end of file diff --git a/test/inventory_api.py b/test/inventory_api.py new file mode 100644 index 00000000000..bcde15bd3c7 --- /dev/null +++ b/test/inventory_api.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +import json +import sys + +from optparse import OptionParser + +parser = OptionParser() +parser.add_option('-l', '--list', default=False, dest="list_hosts", action="store_true") +parser.add_option('-H', '--host', default=None, dest="host") +parser.add_option('-e', '--extra-vars', default=None, dest="extra") + +options, args = parser.parse_args() + +systems = { + "ungouped": [ "jupiter", "saturn" ], + "greek": [ "zeus", "hera", "poseidon" ], + "norse": [ "thor", "odin", "loki" ] +} + +variables = { + "thor": { + "hammer": True + } +} + +if options.list_hosts == True: + print json.dumps(systems) + sys.exit(0) + +if options.host is not None: + if options.extra: + k,v = options.extra.split("=") + variables[options.host][k] = v + print json.dumps(variables[options.host]) + sys.exit(0) + +parser.print_help() +sys.exit(1) \ No newline at end of file diff --git a/test/simple_hosts b/test/simple_hosts new file mode 100644 index 00000000000..14312af92d8 --- /dev/null +++ b/test/simple_hosts @@ -0,0 +1,12 @@ +jupiter +saturn + +[greek] +zeus +hera +poseidon + +[norse] +thor +odin +loki From 746f1b92ae0111f7deedbe1518955138a51ee737 Mon Sep 17 00:00:00 2001 From: Jeroen Hoekx Date: Sat, 14 Apr 2012 09:29:14 +0200 Subject: [PATCH 3/8] Reimplement the class method on Runner. --- lib/ansible/runner.py | 11 +++++++++++ test/TestInventory.py | 26 +++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/lib/ansible/runner.py b/lib/ansible/runner.py index 50c4029a45d..0df79873474 100755 --- a/lib/ansible/runner.py +++ b/lib/ansible/runner.py @@ -123,6 +123,17 @@ class Runner(object): self._tmp_paths = {} random.seed() + # ***************************************************** + + @classmethod + def parse_hosts(cls, host_list, override_hosts=None, extra_vars=None): + ''' parse the host inventory file, returns (hosts, groups) ''' + if override_hosts is None: + inventory = ansible.inventory.Inventory(host_list, extra_vars) + else: + inventory = ansible.inventory.Inventory(override_hosts) + + return inventory.host_list, inventory.groups # ***************************************************** diff --git a/test/TestInventory.py b/test/TestInventory.py index b0e3c62ccf5..ba89bdc3a9c 100644 --- a/test/TestInventory.py +++ b/test/TestInventory.py @@ -2,6 +2,7 @@ import os import unittest from ansible.inventory import Inventory +from ansible.runner import Runner class TestInventory(unittest.TestCase): @@ -140,4 +141,27 @@ class TestInventory(unittest.TestCase): inventory = self.script_inventory() vars = inventory.get_variables('thor', 'simple=yes') - assert vars == {"hammer":True, "simple": "yes"} \ No newline at end of file + assert vars == {"hammer":True, "simple": "yes"} + + ### Test Runner class method + + def test_class_method(self): + hosts, groups = Runner.parse_hosts(self.inventory_file) + + expected_hosts = ['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki'] + assert hosts == expected_hosts + + expected_groups= { + 'ungrouped': ['jupiter', 'saturn'], + 'greek': ['zeus', 'hera', 'poseidon'], + 'norse': ['thor', 'odin', 'loki'] + } + assert groups == expected_groups + + def test_class_override(self): + override_hosts = ['thor', 'odin'] + hosts, groups = Runner.parse_hosts(self.inventory_file, override_hosts) + + assert hosts == override_hosts + + assert groups == { 'ungrouped': override_hosts } From f04041b37dd13d31b0a8d690a1b66da8e9244ce7 Mon Sep 17 00:00:00 2001 From: Jeroen Hoekx Date: Sat, 14 Apr 2012 12:15:57 +0200 Subject: [PATCH 4/8] Ignore port numbers in simple inventory format --- lib/ansible/inventory.py | 3 +++ test/simple_hosts | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/ansible/inventory.py b/lib/ansible/inventory.py index 448ef372b04..429b4c95d9f 100644 --- a/lib/ansible/inventory.py +++ b/lib/ansible/inventory.py @@ -105,6 +105,9 @@ class Inventory(object): groups[group_name] = [] elif item != "": # looks like a regular host + if ":" in item: + # a port was specified + item, port = item.split(":") groups[group_name].append(item) if not item in results: results.append(item) diff --git a/test/simple_hosts b/test/simple_hosts index 14312af92d8..6a4e297b4fb 100644 --- a/test/simple_hosts +++ b/test/simple_hosts @@ -3,7 +3,7 @@ saturn [greek] zeus -hera +hera:3000 poseidon [norse] From 54f452616079824e804885c43f97d63fe06a1aba Mon Sep 17 00:00:00 2001 From: Jeroen Hoekx Date: Sat, 14 Apr 2012 13:12:32 +0200 Subject: [PATCH 5/8] Export SSH port number as host variable. --- lib/ansible/inventory.py | 14 ++++++++++++++ test/TestInventory.py | 6 ++++++ 2 files changed, 20 insertions(+) diff --git a/lib/ansible/inventory.py b/lib/ansible/inventory.py index 429b4c95d9f..3f97460e51f 100644 --- a/lib/ansible/inventory.py +++ b/lib/ansible/inventory.py @@ -35,6 +35,7 @@ class Inventory(object): def __init__(self, host_list=C.DEFAULT_HOST_LIST, extra_vars=None): self._restriction = None + self._variables = {} if type(host_list) == list: self.host_list = host_list @@ -80,6 +81,9 @@ class Inventory(object): def get_variables(self, host, extra_vars=None): """ Return the variables associated with this host. """ + if host in self._variables: + return self._variables[host] + if not self._is_script: return {} @@ -108,6 +112,11 @@ class Inventory(object): if ":" in item: # a port was specified item, port = item.split(":") + try: + port = int(port) + except ValueError: + raise errors.AnsibleError("SSH port for %s in inventory (%s) should be numerical."%(item, port)) + self._set_variable(item, "ansible_ssh_port", port) groups[group_name].append(item) if not item in results: results.append(item) @@ -170,6 +179,11 @@ class Inventory(object): )) return variables + def _set_variable(self, host, key, value): + if not host in self._variables: + self._variables[host] = {} + self._variables[host][key] = value + def _matches(self, host_name, pattern): ''' returns if a hostname is matched by the pattern ''' diff --git a/test/TestInventory.py b/test/TestInventory.py index ba89bdc3a9c..2f86f507283 100644 --- a/test/TestInventory.py +++ b/test/TestInventory.py @@ -82,6 +82,12 @@ class TestInventory(unittest.TestCase): assert vars == {} + def test_simple_port(self): + inventory = self.simple_inventory() + vars = inventory.get_variables('hera') + + assert vars == {'ansible_ssh_port': 3000} + ### Inventory API tests def test_script(self): From 3a24aa9a70cf625aa9c9eff545f31c4fafeaa7bf Mon Sep 17 00:00:00 2001 From: Jeroen Hoekx Date: Sat, 14 Apr 2012 15:45:24 +0200 Subject: [PATCH 6/8] Add YAML inventory format. See test/yaml_hosts for an example. Hosts can be part of multiple groups. Groups can also have variables, inherited by the hosts. There is no variable scope, last variable seen wins. --- lib/ansible/inventory.py | 72 ++++++++++++++++++++++++++++++++++++++ test/TestInventory.py | 74 ++++++++++++++++++++++++++++++++++++++++ test/yaml_hosts | 26 ++++++++++++++ 3 files changed, 172 insertions(+) create mode 100644 test/yaml_hosts diff --git a/lib/ansible/inventory.py b/lib/ansible/inventory.py index 3f97460e51f..c3678d64a5e 100644 --- a/lib/ansible/inventory.py +++ b/lib/ansible/inventory.py @@ -97,6 +97,8 @@ class Inventory(object): results = [] groups = dict(ungrouped=[]) lines = file(self.inventory_file).read().split("\n") + if "---" in lines: + return self._parse_yaml() group_name = 'ungrouped' for item in lines: item = item.lstrip().rstrip() @@ -154,6 +156,76 @@ class Inventory(object): # ***************************************************** + def _parse_yaml(self): + """ Load the inventory from a yaml file. + + returns hosts and groups""" + data = utils.parse_yaml_from_file(self.inventory_file) + + if type(data) != list: + raise errors.AnsibleError("YAML inventory should be a list.") + + hosts = [] + groups = {} + + for item in data: + if type(item) == dict: + if "group" in item: + group_name = item["group"] + + group_vars = [] + if "vars" in item: + group_vars = item["vars"] + + group_hosts = [] + if "hosts" in item: + for host in item["hosts"]: + host_name = self._parse_yaml_host(host, group_vars) + group_hosts.append(host_name) + + groups[group_name] = group_hosts + hosts.extend(group_hosts) + + # or a host + elif "host" in item: + host_name = self._parse_yaml_host(item) + hosts.append(host_name) + else: + host_name = self._parse_yaml_host(item) + hosts.append(host_name) + + # filter duplicate hosts + output_hosts = [] + for host in hosts: + if host not in output_hosts: + output_hosts.append(host) + + return output_hosts, groups + + def _parse_yaml_host(self, item, variables=[]): + def set_variables(host, variables): + for variable in variables: + if len(variable) != 1: + raise AnsibleError("Only one item expected in %s"%(variable)) + k, v = variable.items()[0] + self._set_variable(host, k, v) + + if type(item) in [str, unicode]: + set_variables(item, variables) + return item + elif type(item) == dict: + if "host" in item: + host_name = item["host"] + set_variables(host_name, variables) + + if "vars" in item: + set_variables(host_name, item["vars"]) + + return host_name + else: + raise AnsibleError("Unknown item in inventory: %s"%(item)) + + def _get_variables_from_script(self, host, extra_vars=None): ''' support per system variabes from external variable scripts, see web docs ''' diff --git a/test/TestInventory.py b/test/TestInventory.py index 2f86f507283..a0b0b74d16d 100644 --- a/test/TestInventory.py +++ b/test/TestInventory.py @@ -12,6 +12,7 @@ class TestInventory(unittest.TestCase): self.inventory_file = os.path.join(self.test_dir, 'simple_hosts') self.inventory_script = os.path.join(self.test_dir, 'inventory_api.py') + self.inventory_yaml = os.path.join(self.test_dir, 'yaml_hosts') os.chmod(self.inventory_script, 0755) @@ -26,6 +27,9 @@ class TestInventory(unittest.TestCase): def script_inventory(self): return Inventory( self.inventory_script ) + def yaml_inventory(self): + return Inventory( self.inventory_yaml ) + def test_simple(self): inventory = self.simple_inventory() hosts = inventory.list_hosts() @@ -149,6 +153,76 @@ class TestInventory(unittest.TestCase): assert vars == {"hammer":True, "simple": "yes"} + ### Tests for yaml inventory file + + def test_yaml(self): + inventory = self.yaml_inventory() + hosts = inventory.list_hosts() + print hosts + expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki'] + assert hosts == expected_hosts + + def test_yaml_all(self): + inventory = self.yaml_inventory() + hosts = inventory.list_hosts('all') + + expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki'] + assert hosts == expected_hosts + + def test_yaml_norse(self): + inventory = self.yaml_inventory() + hosts = inventory.list_hosts("norse") + + expected_hosts=['thor', 'odin', 'loki'] + assert hosts == expected_hosts + + def test_yaml_combined(self): + inventory = self.yaml_inventory() + hosts = inventory.list_hosts("norse:greek") + + expected_hosts=['zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki'] + assert hosts == expected_hosts + + def test_yaml_restrict(self): + inventory = self.yaml_inventory() + + restricted_hosts = ['hera', 'poseidon', 'thor'] + expected_hosts=['zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki'] + + inventory.restrict_to(restricted_hosts) + hosts = inventory.list_hosts("norse:greek") + + assert hosts == restricted_hosts + + inventory.lift_restriction() + hosts = inventory.list_hosts("norse:greek") + + assert hosts == expected_hosts + + def test_yaml_vars(self): + inventory = self.yaml_inventory() + vars = inventory.get_variables('thor') + + assert vars == {"hammer":True} + + def test_yaml_host_vars(self): + inventory = self.yaml_inventory() + vars = inventory.get_variables('saturn') + + assert vars == {"moon":"titan"} + + def test_yaml_extra_vars(self): + inventory = self.yaml_inventory() + vars = inventory.get_variables('thor', 'a=5') + + assert vars == {"hammer":True} + + def test_yaml_port(self): + inventory = self.yaml_inventory() + vars = inventory.get_variables('hera') + + assert vars == {'ansible_ssh_port': 3000} + ### Test Runner class method def test_class_method(self): diff --git a/test/yaml_hosts b/test/yaml_hosts new file mode 100644 index 00000000000..95f278b91a0 --- /dev/null +++ b/test/yaml_hosts @@ -0,0 +1,26 @@ +--- + +- jupiter +- host: saturn + vars: + - moon: titan + +- group: greek + hosts: + - zeus + - hera + - poseidon + vars: + - ansible_ssh_port: 3000 + +- group: norse + hosts: + - host: thor + vars: + - hammer: True + - odin + - loki + +- group: multiple + hosts: + - saturn From 961ccdb2f4768392d9a1396c73a6b317b6b48f60 Mon Sep 17 00:00:00 2001 From: Jeroen Hoekx Date: Mon, 16 Apr 2012 10:55:08 +0200 Subject: [PATCH 7/8] List hosts in no group in the ungrouped group. --- lib/ansible/inventory.py | 17 ++++++++++++++++- test/TestInventory.py | 14 ++++++++++++++ test/yaml_hosts | 2 ++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/lib/ansible/inventory.py b/lib/ansible/inventory.py index c3678d64a5e..46a5edb589a 100644 --- a/lib/ansible/inventory.py +++ b/lib/ansible/inventory.py @@ -168,6 +168,8 @@ class Inventory(object): hosts = [] groups = {} + ungrouped = [] + for item in data: if type(item) == dict: if "group" in item: @@ -186,13 +188,14 @@ class Inventory(object): groups[group_name] = group_hosts hosts.extend(group_hosts) - # or a host elif "host" in item: host_name = self._parse_yaml_host(item) hosts.append(host_name) + ungrouped.append(host_name) else: host_name = self._parse_yaml_host(item) hosts.append(host_name) + ungrouped.append(host_name) # filter duplicate hosts output_hosts = [] @@ -200,6 +203,18 @@ class Inventory(object): if host not in output_hosts: output_hosts.append(host) + if len(ungrouped) > 0 : + # hosts can be defined top-level, but also in a group + really_ungrouped = [] + for host in ungrouped: + already_grouped = False + for name, group_hosts in groups.items(): + if host in group_hosts: + already_grouped = True + if not already_grouped: + really_ungrouped.append(host) + groups["ungrouped"] = really_ungrouped + return output_hosts, groups def _parse_yaml_host(self, item, variables=[]): diff --git a/test/TestInventory.py b/test/TestInventory.py index a0b0b74d16d..21cccf5fe89 100644 --- a/test/TestInventory.py +++ b/test/TestInventory.py @@ -51,6 +51,13 @@ class TestInventory(unittest.TestCase): expected_hosts=['thor', 'odin', 'loki'] assert hosts == expected_hosts + def test_simple_ungrouped(self): + inventory = self.simple_inventory() + hosts = inventory.list_hosts("ungrouped") + + expected_hosts=['jupiter', 'saturn'] + assert hosts == expected_hosts + def test_simple_combined(self): inventory = self.simple_inventory() hosts = inventory.list_hosts("norse:greek") @@ -176,6 +183,13 @@ class TestInventory(unittest.TestCase): expected_hosts=['thor', 'odin', 'loki'] assert hosts == expected_hosts + def test_simple_ungrouped(self): + inventory = self.yaml_inventory() + hosts = inventory.list_hosts("ungrouped") + + expected_hosts=['jupiter'] + assert hosts == expected_hosts + def test_yaml_combined(self): inventory = self.yaml_inventory() hosts = inventory.list_hosts("norse:greek") diff --git a/test/yaml_hosts b/test/yaml_hosts index 95f278b91a0..7568ff4bda2 100644 --- a/test/yaml_hosts +++ b/test/yaml_hosts @@ -5,6 +5,8 @@ vars: - moon: titan +- zeus + - group: greek hosts: - zeus From 8c3206c99fa940ce58fdcb90e9aa4e0942c32914 Mon Sep 17 00:00:00 2001 From: Jeroen Hoekx Date: Mon, 16 Apr 2012 10:59:34 +0200 Subject: [PATCH 8/8] Return a copy of the host variables. --- lib/ansible/inventory.py | 2 +- test/TestInventory.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/ansible/inventory.py b/lib/ansible/inventory.py index 46a5edb589a..05f8001fb12 100644 --- a/lib/ansible/inventory.py +++ b/lib/ansible/inventory.py @@ -82,7 +82,7 @@ class Inventory(object): """ Return the variables associated with this host. """ if host in self._variables: - return self._variables[host] + return self._variables[host].copy() if not self._is_script: return {} diff --git a/test/TestInventory.py b/test/TestInventory.py index 21cccf5fe89..4f48db14f65 100644 --- a/test/TestInventory.py +++ b/test/TestInventory.py @@ -219,6 +219,15 @@ class TestInventory(unittest.TestCase): assert vars == {"hammer":True} + def test_yaml_change_vars(self): + inventory = self.yaml_inventory() + vars = inventory.get_variables('thor') + + vars["hammer"] = False + + vars = inventory.get_variables('thor') + assert vars == {"hammer":True} + def test_yaml_host_vars(self): inventory = self.yaml_inventory() vars = inventory.get_variables('saturn')