From e3b2521f016052a37029737db53bcff9294d53c8 Mon Sep 17 00:00:00 2001 From: Chin Fang Date: Tue, 24 Jul 2012 12:38:52 -0700 Subject: [PATCH 1/2] Added a host expansion feature to ansible's inventory parsing --- CHANGELOG.md | 5 +++++ Makefile | 2 +- docs/man/man1/ansible.1 | 14 +++++++++++++- examples/hosts | 8 ++++++++ lib/ansible/inventory/ini.py | 35 ++++++++++++++++++++++++++++------- test/TestInventory.py | 36 +++++++++++++++++++++++++++++------- test/simple_hosts | 3 +++ 7 files changed, 87 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 909ad4d0325..3ce0cc250f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ Ansible Changes By Release 0.6 "Cabo" ------------ pending +* inventory file can use a line of the form base[beg:end]tail to define a + set of hosts, where [beg:end] defines a numerical range. 'beg' can be a + a string padded with zero(s) to the left. If so provided, it acts as + a formatting hint during hostname expansion. The hint must be confirmed + by having an 'end' that has the same length as 'beg' * groups variable available as a hash to return the hosts in each group name * fetch module now does not fail a system when requesting file paths (ex: logs) that don't exist * apt module now takes an optional install-recommends=yes|no (default yes) diff --git a/Makefile b/Makefile index b685863def2..ad872502ef5 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # useful targets: # make sdist ---------------- produce a tarball # make rpm ----------------- produce RPMs -# make debian --------------- produce a dpkg (FIXME?) +# make deb ------------------ produce a DEB # make docs ----------------- rebuild the manpages (results are checked in) # make tests ---------------- run the tests # make pyflakes, make pep8 -- source code checks diff --git a/docs/man/man1/ansible.1 b/docs/man/man1/ansible.1 index efe491345ca..d785f7c242c 100644 --- a/docs/man/man1/ansible.1 +++ b/docs/man/man1/ansible.1 @@ -140,7 +140,19 @@ Connection type to use\&. Possible options are .RE .SH "INVENTORY" .sp -Ansible stores the hosts it can potentially operate on in an inventory file\&. The syntax is one host per line\&. Groups headers are allowed and are included on their own line, enclosed in square brackets\&. +Ansible stores the hosts it can potentially operate on in an inventory +file\&. The syntax is one host per line\&. Optionally, ansible can use a +line of the form base[beg:end]tail to define a set of hosts, where +[beg:end] defines a numerical range. If 'beg' is left out, it +defaults to 0\&. An example: mail[1:6].example.com, where 'head' +is 'mail', 'beg' is 1, 'end' is 6, and 'tail' is '.example.com'\&. In +addition, 'beg' can be a a string padded with zero(s) to the left. If so +provided, it acts as a formatting hint during hostname expansion. The usage +must be confirmed by having an 'end' that has the same length as 'beg', +else an exception is raised. An example: mail[001:003].example.com is to be +expanded to mail001.example.com, mail002.example.com, and +mail003.example.com\&. Groups headers are allowed and are included on their +own line, enclosed in square brackets\&. .SH "FILES" .sp /etc/ansible/hosts \(em Default inventory file diff --git a/examples/hosts b/examples/hosts index f58b26ed9e9..a351674ac90 100644 --- a/examples/hosts +++ b/examples/hosts @@ -16,6 +16,8 @@ bikeshed.org bastion.secure.bikeshed.org 192.168.100.1 192.168.100.10 +# An example for host expansion that uses the default 'beg' and an 'end' +mail[:5].example.com # Ex 2: A collection of hosts belonging to the 'webservers' group [webservers] @@ -26,6 +28,9 @@ wheel.colors.com 192.168.1.110 # Your personal website also runs a webserver: myserver.com +# An example for host expansion that uses both a 'beg' and an 'end', with +# the 'beg' acting as a formatting hint during host name expansion +www[001:006].example.com # Ex 3: A collection of database servers in the 'dbservers' group [dbservers] @@ -35,3 +40,6 @@ db02.intranet.mydomain.net 10.25.1.57 # Perhaps you serve a db off your personal server too: myserver.com +# An example for host expansion that uses a regular 'beg' and a regular +# 'end' +db-[99:101]-node.example.com diff --git a/lib/ansible/inventory/ini.py b/lib/ansible/inventory/ini.py index 63260e3c3a9..9398a591b7b 100644 --- a/lib/ansible/inventory/ini.py +++ b/lib/ansible/inventory/ini.py @@ -24,6 +24,8 @@ import subprocess import ansible.constants as C from ansible.inventory.host import Host from ansible.inventory.group import Group +from ansible.inventory.expand_hosts import detect_range +from ansible.inventory.expand_hosts import expand_hostname_range from ansible import errors from ansible import utils @@ -80,21 +82,40 @@ class InventoryParser(object): continue hostname = tokens[0] port = C.DEFAULT_REMOTE_PORT - if hostname.find(":") != -1: - tokens2 = hostname.split(":") - hostname = tokens2[0] - port = tokens2[1] + # Two cases to check: + # 0. A hostname that contains a range pesudo-code and a port + # 1. A hostname that contains just a port + if (hostname.find("[") != -1 and + hostname.find("]") != -1 and + hostname.find(":") != -1 and + (hostname.rindex("]") < hostname.rindex(":")) or + (hostname.find("]") == -1 and hostname.find(":") != -1)): + tokens2 = hostname.rsplit(":", 1) + hostname = tokens2[0] + port = tokens2[1] + host = None + _all_hosts = [] if hostname in self.hosts: host = self.hosts[hostname] + _all_hosts.append(host) else: - host = Host(name=hostname, port=port) - self.hosts[hostname] = host + if detect_range(hostname): + _hosts = expand_hostname_range(hostname) + for _ in _hosts: + host = Host(name=_, port=port) + self.hosts[_] = host + _all_hosts.append(host) + else: + host = Host(name=hostname, port=port) + self.hosts[hostname] = host + _all_hosts.append(host) if len(tokens) > 1: for t in tokens[1:]: (k,v) = t.split("=") host.set_variable(k,v) - self.groups[active_group_name].add_host(host) + for _ in _all_hosts: + self.groups[active_group_name].add_host(_) # [southeast:children] # atlanta diff --git a/test/TestInventory.py b/test/TestInventory.py index 93bc27280c5..559d5f2bfbd 100644 --- a/test/TestInventory.py +++ b/test/TestInventory.py @@ -44,14 +44,24 @@ class TestInventory(unittest.TestCase): inventory = self.simple_inventory() hosts = inventory.list_hosts() - expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki'] + expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', + 'cerberus001','cerberus002','cerberus003', + 'cottus99', 'cottus100', + 'poseidon', 'thor', 'odin', 'loki', + 'thrudgelmir0', 'thrudgelmir1', 'thrudgelmir2', + 'thrudgelmir3', 'thrudgelmir4', 'thrudgelmir5'] assert sorted(hosts) == sorted(expected_hosts) def test_simple_all(self): inventory = self.simple_inventory() hosts = inventory.list_hosts('all') - expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki'] + expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', + 'cerberus001','cerberus002','cerberus003', + 'cottus99', 'cottus100', + 'poseidon', 'thor', 'odin', 'loki', + 'thrudgelmir0', 'thrudgelmir1', 'thrudgelmir2', + 'thrudgelmir3', 'thrudgelmir4', 'thrudgelmir5'] assert sorted(hosts) == sorted(expected_hosts) def test_simple_norse(self): @@ -65,21 +75,29 @@ class TestInventory(unittest.TestCase): inventory = self.simple_inventory() hosts = inventory.list_hosts("ungrouped") - expected_hosts=['jupiter', 'saturn'] + expected_hosts=['jupiter', 'saturn', + 'thrudgelmir0', 'thrudgelmir1', 'thrudgelmir2', + 'thrudgelmir3', 'thrudgelmir4', 'thrudgelmir5'] assert sorted(hosts) == sorted(expected_hosts) def test_simple_combined(self): inventory = self.simple_inventory() hosts = inventory.list_hosts("norse:greek") - expected_hosts=['zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki'] + expected_hosts=['zeus', 'hera', 'poseidon', + 'cerberus001','cerberus002','cerberus003', + 'cottus99','cottus100', + 'thor', 'odin', 'loki'] assert sorted(hosts) == sorted(expected_hosts) def test_simple_restrict(self): inventory = self.simple_inventory() restricted_hosts = ['hera', 'poseidon', 'thor'] - expected_hosts=['zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki'] + expected_hosts=['zeus', 'hera', 'poseidon', + 'cerberus001','cerberus002','cerberus003', + 'cottus99', 'cottus100', + 'thor', 'odin', 'loki'] inventory.restrict_to(restricted_hosts) hosts = inventory.list_hosts("norse:greek") @@ -99,11 +117,15 @@ class TestInventory(unittest.TestCase): inventory = self.simple_inventory() hosts = inventory.list_hosts("all:!greek") - expected_hosts=['jupiter', 'saturn', 'thor', 'odin', 'loki'] + expected_hosts=['jupiter', 'saturn', 'thor', 'odin', 'loki', + 'thrudgelmir0', 'thrudgelmir1', 'thrudgelmir2', + 'thrudgelmir3', 'thrudgelmir4', 'thrudgelmir5'] assert sorted(hosts) == sorted(expected_hosts) hosts = inventory.list_hosts("all:!norse:!greek") - expected_hosts=['jupiter', 'saturn'] + expected_hosts=['jupiter', 'saturn', + 'thrudgelmir0', 'thrudgelmir1', 'thrudgelmir2', + 'thrudgelmir3', 'thrudgelmir4', 'thrudgelmir5'] assert sorted(hosts) == sorted(expected_hosts) def test_simple_vars(self): diff --git a/test/simple_hosts b/test/simple_hosts index 6a4e297b4fb..8f1bd55a5f4 100644 --- a/test/simple_hosts +++ b/test/simple_hosts @@ -1,10 +1,13 @@ jupiter saturn +thrudgelmir[:6] [greek] zeus hera:3000 poseidon +cerberus[001:004] +cottus[99:101] [norse] thor From 5315dd146d61a69e909a16d98bf3639e47ba180b Mon Sep 17 00:00:00 2001 From: Chin Fang Date: Tue, 24 Jul 2012 12:43:35 -0700 Subject: [PATCH 2/2] Added lib/ansible/inventory/expand_hosts.py --- lib/ansible/inventory/expand_hosts.py | 94 +++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 lib/ansible/inventory/expand_hosts.py diff --git a/lib/ansible/inventory/expand_hosts.py b/lib/ansible/inventory/expand_hosts.py new file mode 100644 index 00000000000..e7a5ff37f61 --- /dev/null +++ b/lib/ansible/inventory/expand_hosts.py @@ -0,0 +1,94 @@ +# (c) 2012, Zettar Inc. +# Written by Chin Fang +# +# This file is part of Ansible +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see . +# +''' +This module is for enhancing ansible's inventory parsing capability such +that it can deal with hostnames specified using a simple pattern in the +form of [beg:end], example: [1:5] where if beg is not specified, it +defaults to 0. + +If beg is given and is left-zero-padded, e.g. '001', it is taken as a +formatting hint when the range is expanded. e.g. [001:010] is to be +expanded into 001, 002 ...009, 010. + +Note that when beg is specified with left zero padding, then the length of +end must be the same as that of beg, else a exception is raised. +''' + +def detect_range(line = None): + ''' + A helper function that checks a given host line to see if it contains + a range pattern descibed in the docstring above. + + Returnes True if the given line contains a pattern, else False. + ''' + if (not line.startswith("[") and + line.find("[") != -1 and + line.find(":") != -1 and + line.find("]") != -1 and + line.index("[") < line.index(":") < line.index("]")): + return True + else: + return False + +def expand_hostname_range(line = None): + ''' + A helper function that expands a given line that contains a pattern + specified in top docstring, and returns a list that consists of the + expanded version. + + The '[' and ']' characters are used to maintain the pseudo-code + appearance. They are replaced in this function with '|' to ease + string splitting. + + References: http://ansible.github.com/patterns.html#hosts-and-groups + ''' + all_hosts = [] + if line: + # A hostname such as db[1:6]-node is considered to consists + # three parts: + # head: 'db' + # nrange: [1:6]; range() is a built-in. Can't use the name + # tail: '-node' + + (head, nrange, tail) = line.replace('[','|').replace(']','|').split('|') + bounds = nrange.split(":") + if len(bounds) != 2: + raise ValueError("host range incorrectly specified!") + beg = bounds[0] + end = bounds[1] + if not beg: + beg = "0" + if not end: + raise ValueError("host range end value missing!") + if beg[0] == '0' and len(beg) > 1: + rlen = len(beg) # range length formatting hint + else: + rlen = None + if rlen > 1 and rlen != len(end): + raise ValueError("host range format incorrectly specified!") + + for _ in range(int(beg), int(end)): + if rlen: + rseq = str(_).zfill(rlen) # range sequence + else: + rseq = str(_) + hname = ''.join((head, rseq, tail)) + all_hosts.append(hname) + + return all_hosts