From 2a2d866c0d5214e763ad774fd2ce9453f3e830d3 Mon Sep 17 00:00:00 2001 From: Josh Preston Date: Thu, 18 Aug 2016 17:37:31 -0400 Subject: [PATCH] add cloudforms inventory script (#17037) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add cloudforms inventory script based on the foreman inventory script, features: * cached results (default 600 seconds) * paginated host results (default 100 hosts) * ssl verification (default True) * arguments to flush cache and run in debug mode * suggested rework * removed second cache / dict with duplicate info * added purge_actions configuration option to remove the actions from a host (defaults to False) * added prefer_ip_address configuration option so give the option of using ip address instead of name (defaults to True) * removed self variables — just use the arguments directly * added --pretty command line option to pretty print results * renamed _resolve_params to _resolve_host * implement suggestions * removed not used import * added warnings to help debug connection issues * renamed self.cache to self.hosts for clarity * now will use the first ip address as ansible_ssh_host * flipped default for prefer_ip_address config option to false - preserve name, and specify ansible_ssh_host as ip address * added checks and warnings to configuration options, sane defaults for all except required: ** `url` - the first part of the cloudforms server url (https://cfme.example.com) ** `username` - the cloudforms username to log in with ** `password` - the password for the cloudforms user specified * removed redundant call to fetch host information (since we’re paging results, no need to split the calls) * added warning for unexpected responses from CloudForms * debug for returned sting now prints the string instead of forcing to JSON * removed no longer needed methods to fetch host information * using ‘key in list’ instead of ‘list.has_key(key)’ * correctly formatted groups and allowed nested groups * now create groups for `location`, `type` and `vendor`, with appropriate sub-groups and children * made to_safe honor config option to clean group names for ansible consumption * remove prefer_ip_address configuration option no longer needed since we will specify `ansible_ssh_host` as the returned ip address. * removed dns_name no longer needed, will preserve `host[name]` as name in Ansible. * purge actions from hostvars changed purge_actions to True * flake8 suggestion for whitespace * fix undefined r variable in warning output use the correct ret variable * Default purge_actions to True We probably don’t need them, but it is configurable, so just default to remove them. * Add configuration option to nest cloudforms tags disabled by default, the nest_tags option will expand cloudforms tags into a nested group/subgroup structure. Otherwise, it will use the whole tag name. * added purging the actions removed in previous clean up in error. * fixed undefined variable specified the correct variable for logging. --- contrib/inventory/cloudforms.ini | 33 +++ contrib/inventory/cloudforms.py | 460 +++++++++++++++++++++++++++++++ 2 files changed, 493 insertions(+) create mode 100644 contrib/inventory/cloudforms.ini create mode 100755 contrib/inventory/cloudforms.py diff --git a/contrib/inventory/cloudforms.ini b/contrib/inventory/cloudforms.ini new file mode 100644 index 00000000000..142514a65ed --- /dev/null +++ b/contrib/inventory/cloudforms.ini @@ -0,0 +1,33 @@ +[cloudforms] + +# the version of CloudForms ; currently not used, but tested with +version = 4.1 + +# This should be the hostname of the CloudForms server +url = https://cfme.example.com + +# This will more than likely need to be a local CloudForms username +username = + +# The password for said username +password = + +# True = verify SSL certificate / False = trust anything +ssl_verify = True + +# limit the number of vms returned per request +limit = 100 + +# purge the CloudForms actions from hosts +purge_actions = True + +# Clean up group names (from tags and other groupings so Ansible doesn't complain) +clean_group_keys = True + +# Explode tags into nested groups / subgroups +nest_tags = False + +[cache] + +# Maximum time to trust the cache in seconds +max_age = 600 diff --git a/contrib/inventory/cloudforms.py b/contrib/inventory/cloudforms.py new file mode 100755 index 00000000000..79b8a4bc239 --- /dev/null +++ b/contrib/inventory/cloudforms.py @@ -0,0 +1,460 @@ +#!/usr/bin/python +# vim: set fileencoding=utf-8 : +# +# Copyright (C) 2016 Guido Günther +# +# This script is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with it. If not, see . +# +# This is loosely based on the foreman inventory script +# -- Josh Preston +# + +import argparse +import ConfigParser +import os +import re +from time import time +import requests +from requests.auth import HTTPBasicAuth +import warnings + +try: + import json +except ImportError: + import simplejson as json + + +class CloudFormsInventory(object): + def __init__(self): + """ + Main execution path + """ + self.inventory = dict() # A list of groups and the hosts in that group + self.hosts = dict() # Details about hosts in the inventory + + # Parse CLI arguments + self.parse_cli_args() + + # Read settings + self.read_settings() + + # Cache + if self.args.refresh_cache or not self.is_cache_valid(): + self.update_cache() + else: + self.load_inventory_from_cache() + self.load_hosts_from_cache() + + data_to_print = "" + + # Data to print + if self.args.host: + if self.args.debug: + print "Fetching host [%s]" % self.args.host + data_to_print += self.get_host_info(self.args.host) + else: + self.inventory['_meta'] = {'hostvars': {}} + for hostname in self.hosts: + self.inventory['_meta']['hostvars'][hostname] = { + 'cloudforms': self.hosts[hostname], + } + # include the ansible_ssh_host in the top level + if 'ansible_ssh_host' in self.hosts[hostname]: + self.inventory['_meta']['hostvars'][hostname]['ansible_ssh_host'] = self.hosts[hostname]['ansible_ssh_host'] + + data_to_print += self.json_format_dict(self.inventory, self.args.pretty) + + print(data_to_print) + + def is_cache_valid(self): + """ + Determines if the cache files have expired, or if it is still valid + """ + if self.args.debug: + print "Determining if cache [%s] is still valid (< %s seconds old)" % (self.cache_path_hosts, self.cache_max_age) + + if os.path.isfile(self.cache_path_hosts): + mod_time = os.path.getmtime(self.cache_path_hosts) + current_time = time() + if (mod_time + self.cache_max_age) > current_time: + if os.path.isfile(self.cache_path_inventory): + if self.args.debug: + print "Cache is still valid!" + return True + + if self.args.debug: + print "Cache is stale or does not exist." + + return False + + def read_settings(self): + """ + Reads the settings from the cloudforms.ini file + """ + config = ConfigParser.SafeConfigParser() + config_paths = [ + os.path.dirname(os.path.realpath(__file__)) + '/cloudforms.ini', + "/etc/ansible/cloudforms.ini", + ] + + env_value = os.environ.get('CLOUDFORMS_INI_PATH') + if env_value is not None: + config_paths.append(os.path.expanduser(os.path.expandvars(env_value))) + + if self.args.debug: + for config_path in config_paths: + print "Reading from configuration file [%s]" % config_path + + config.read(config_paths) + + # CloudForms API related + if config.has_option('cloudforms', 'url'): + self.cloudforms_url = config.get('cloudforms', 'url') + else: + self.cloudforms_url = None + + if not self.cloudforms_url: + warnings.warn("No url specified, expected something like 'https://cfme.example.com'") + + if config.has_option('cloudforms', 'username'): + self.cloudforms_username = config.get('cloudforms', 'username') + else: + self.cloudforms_username = None + + if not self.cloudforms_username: + warnings.warn("No username specified, you need to specify a CloudForms username.") + + if config.has_option('cloudforms', 'password'): + self.cloudforms_pw = config.get('cloudforms', 'password') + else: + self.cloudforms_pw = None + + if not self.cloudforms_pw: + warnings.warn("No password specified, you need to specify a password for the CloudForms user.") + + if config.has_option('cloudforms', 'ssl_verify'): + self.cloudforms_ssl_verify = config.getboolean('cloudforms', 'ssl_verify') + else: + self.cloudforms_ssl_verify = True + + if config.has_option('cloudforms', 'version'): + self.cloudforms_version = config.get('cloudforms', 'version') + else: + self.cloudforms_version = None + + if config.has_option('cloudforms', 'limit'): + self.cloudforms_limit = config.getint('cloudforms', 'limit') + else: + self.cloudforms_limit = 100 + + if config.has_option('cloudforms', 'purge_actions'): + self.cloudforms_purge_actions = config.getboolean('cloudforms', 'purge_actions') + else: + self.cloudforms_purge_actions = True + + if config.has_option('cloudforms', 'clean_group_keys'): + self.cloudforms_clean_group_keys = config.getboolean('cloudforms', 'clean_group_keys') + else: + self.cloudforms_clean_group_keys = True + + if config.has_option('cloudforms', 'nest_tags'): + self.cloudforms_nest_tags = config.getboolean('cloudforms', 'nest_tags') + else: + self.cloudforms_nest_tags = False + + # Ansible related + try: + group_patterns = config.get('ansible', 'group_patterns') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + group_patterns = "[]" + + self.group_patterns = eval(group_patterns) + + # Cache related + try: + cache_path = os.path.expanduser(config.get('cache', 'path')) + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + cache_path = '.' + (script, ext) = os.path.splitext(os.path.basename(__file__)) + self.cache_path_hosts = cache_path + "/%s.hosts" % script + self.cache_path_inventory = cache_path + "/%s.inventory" % script + self.cache_max_age = config.getint('cache', 'max_age') + + if self.args.debug: + print "CloudForms settings:" + print "cloudforms_url = %s" % self.cloudforms_url + print "cloudforms_username = %s" % self.cloudforms_username + print "cloudforms_pw = %s" % self.cloudforms_pw + print "cloudforms_ssl_verify = %s" % self.cloudforms_ssl_verify + print "cloudforms_version = %s" % self.cloudforms_version + print "cloudforms_limit = %s" % self.cloudforms_limit + print "cloudforms_purge_actions = %s" % self.cloudforms_purge_actions + print "Cache settings:" + print "cache_max_age = %s" % self.cache_max_age + print "cache_path_hosts = %s" % self.cache_path_hosts + print "cache_path_inventory = %s" % self.cache_path_inventory + + def parse_cli_args(self): + """ + Command line argument processing + """ + parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on CloudForms managed VMs') + parser.add_argument('--list', action='store_true', default=True, help='List instances (default: True)') + parser.add_argument('--host', action='store', help='Get all the variables about a specific instance') + parser.add_argument('--pretty', action='store_true', default=False, help='Pretty print JSON output (default: False)') + parser.add_argument('--refresh-cache', action='store_true', default=False, + help='Force refresh of cache by making API requests to CloudForms (default: False - use cache files)') + parser.add_argument('--debug', action='store_true', default=False, help='Show debug output while running (default: False)') + self.args = parser.parse_args() + + def _get_json(self, url): + """ + Make a request and return the JSON + """ + results = [] + + ret = requests.get(url, + auth=HTTPBasicAuth(self.cloudforms_username, self.cloudforms_pw), + verify=self.cloudforms_ssl_verify) + + ret.raise_for_status() + + try: + results = json.loads(ret.text) + except ValueError: + warnings.warn("Unexpected response from {0} ({1}): {2}".format(self.cloudforms_url, ret.status_code, ret.reason)) + results = {} + + if self.args.debug: + print "=======================================================================" + print "=======================================================================" + print "=======================================================================" + print ret.text + print "=======================================================================" + print "=======================================================================" + print "=======================================================================" + + return results + + def _get_hosts(self): + """ + Get all hosts by paging through the results + """ + limit = self.cloudforms_limit + + page = 0 + last_page = False + + results = [] + + while not last_page: + offset = page * limit + ret = self._get_json("%s/api/vms?offset=%s&limit=%s&expand=resources,tags,hosts,&attributes=ipaddresses" % (self.cloudforms_url, offset, limit)) + results += ret['resources'] + if ret['subcount'] < limit: + last_page = True + page += 1 + + return results + + def update_cache(self): + """ + Make calls to cloudforms and save the output in a cache + """ + self.groups = dict() + self.hosts = dict() + + if self.args.debug: + print "Updating cache..." + + for host in self._get_hosts(): + # Ignore VMs that are not powered on + if host['power_state'] != 'on': + if self.args.debug: + print "Skipping %s because power_state = %s" % (host['name'], host['power_state']) + continue + + # purge actions + if self.cloudforms_purge_actions and 'actions' in host: + del host['actions'] + + # Create ansible groups for tags + if 'tags' in host: + + # Create top-level group + if 'tags' not in self.inventory: + self.inventory['tags'] = dict(children=[], vars={}, hosts=[]) + + if not self.cloudforms_nest_tags: + # don't expand tags, just use them in a safe way + for group in host['tags']: + # Add sub-group, as a child of top-level + safe_key = self.to_safe(group['name']) + if safe_key: + if self.args.debug: + print "Adding sub-group '%s' to parent 'tags'" % safe_key + + if safe_key not in self.inventory['tags']['children']: + self.push(self.inventory['tags'], 'children', safe_key) + + self.push(self.inventory, safe_key, host['name']) + + if self.args.debug: + print "Found tag [%s] for host which will be mapped to [%s]" % (group['name'], safe_key) + else: + # expand the tags into nested groups / sub-groups + # Create nested groups for tags + safe_parent_tag_name = 'tags' + for tag in host['tags']: + tag_hierarchy = tag['name'][1:].split('/') + + if self.args.debug: + print "Working on list %s" % tag_hierarchy + + for tag_name in tag_hierarchy: + if self.args.debug: + print "Working on tag_name = %s" % tag_name + + safe_tag_name = self.to_safe(tag_name) + if self.args.debug: + print "Using sanitized name %s" % safe_tag_name + + # Create sub-group + if safe_tag_name not in self.inventory: + self.inventory[safe_tag_name] = dict(children=[], vars={}, hosts=[]) + + # Add sub-group, as a child of top-level + if safe_parent_tag_name: + if self.args.debug: + print "Adding sub-group '%s' to parent '%s'" % (safe_tag_name, safe_parent_tag_name) + + if safe_tag_name not in self.inventory[safe_parent_tag_name]['children']: + self.push(self.inventory[safe_parent_tag_name], 'children', safe_tag_name) + + # Make sure the next one uses this one as it's parent + safe_parent_tag_name = safe_tag_name + + # Add the host to the last tag + self.push(self.inventory[safe_parent_tag_name], 'hosts', host['name']) + + # Set ansible_ssh_host to the first available ip address + if 'ipaddresses' in host and host['ipaddresses'] and isinstance(host['ipaddresses'], list): + host['ansible_ssh_host'] = host['ipaddresses'][0] + + # Create additional groups + for key in ('location', 'type', 'vendor'): + safe_key = self.to_safe(host[key]) + + # Create top-level group + if key not in self.inventory: + self.inventory[key] = dict(children=[], vars={}, hosts=[]) + + # Create sub-group + if safe_key not in self.inventory: + self.inventory[safe_key] = dict(children=[], vars={}, hosts=[]) + + # Add sub-group, as a child of top-level + if safe_key not in self.inventory[key]['children']: + self.push(self.inventory[key], 'children', safe_key) + + if key in host: + # Add host to sub-group + self.push(self.inventory[safe_key], 'hosts', host['name']) + + self.hosts[host['name']] = host + self.push(self.inventory, 'all', host['name']) + + if self.args.debug: + print "Saving cached data" + + self.write_to_cache(self.hosts, self.cache_path_hosts) + self.write_to_cache(self.inventory, self.cache_path_inventory) + + def get_host_info(self, host): + """ + Get variables about a specific host + """ + if not self.hosts or len(self.hosts) == 0: + # Need to load cache from cache + self.load_hosts_from_cache() + + if host not in self.hosts: + if self.args.debug: + print "[%s] not found in cache." % host + + # try updating the cache + self.update_cache() + + if host not in self.hosts: + if self.args.debug: + print "[%s] does not exist after cache update." % host + # host might not exist anymore + return self.json_format_dict({}, self.args.pretty) + + return self.json_format_dict(self.hosts[host], self.args.pretty) + + def push(self, d, k, v): + """ + Safely puts a new entry onto an array. + """ + if k in d: + d[k].append(v) + else: + d[k] = [v] + + def load_inventory_from_cache(self): + """ + Reads the inventory from the cache file sets self.inventory + """ + cache = open(self.cache_path_inventory, 'r') + json_inventory = cache.read() + self.inventory = json.loads(json_inventory) + + def load_hosts_from_cache(self): + """ + Reads the cache from the cache file sets self.hosts + """ + cache = open(self.cache_path_hosts, 'r') + json_cache = cache.read() + self.hosts = json.loads(json_cache) + + def write_to_cache(self, data, filename): + """ + Writes data in JSON format to a file + """ + json_data = self.json_format_dict(data, True) + cache = open(filename, 'w') + cache.write(json_data) + cache.close() + + def to_safe(self, word): + """ + Converts 'bad' characters in a string to underscores so they can be used as Ansible groups + """ + if self.cloudforms_clean_group_keys: + regex = "[^A-Za-z0-9\_]" + return re.sub(regex, "_", word.replace(" ", "")) + else: + return word + + def json_format_dict(self, data, pretty=False): + """ + Converts a dict to a JSON object and dumps it as a formatted string + """ + if pretty: + return json.dumps(data, sort_keys=True, indent=2) + else: + return json.dumps(data) + +CloudFormsInventory()