From db709488bdb3ef2457ab8dba9fe32f95aec6c1d1 Mon Sep 17 00:00:00 2001 From: Samir Jha Date: Tue, 3 Dec 2019 17:11:40 -0500 Subject: [PATCH] Update foreman inventory to use foreman's inventory report (#62438) --- contrib/inventory/foreman.ini | 34 +++ contrib/inventory/foreman.py | 207 +++++++++++++++++- .../targets/inventory_foreman_script/runme.sh | 1 + 3 files changed, 241 insertions(+), 1 deletion(-) diff --git a/contrib/inventory/foreman.ini b/contrib/inventory/foreman.ini index e3946daf023..d1579638483 100644 --- a/contrib/inventory/foreman.ini +++ b/contrib/inventory/foreman.ini @@ -123,6 +123,12 @@ user = foreman password = secret ssl_verify = True +# Foreman 1.24 introduces a new reports API to improve performance of the inventory script. +# Note: This requires foreman_ansible plugin installed. +# Set to False if you want to use the old API. Defaults to True. + +use_reports_api = True + # Retrieve only hosts from the organization "Web Engineering". # host_filters = organization="Web Engineering" @@ -130,6 +136,34 @@ ssl_verify = True # also in the host collection "Apache Servers". # host_filters = organization="Web Engineering" and host_collection="Apache Servers" +# Foreman Inventory report related configuration options. +# Configs that default to True : +# want_organization , want_location, want_ipv4, want_host_group, want_subnet, want_smart_proxies, want_facts +# Configs that default to False : +# want_ipv6, want_subnet_v6, want_content_facet_attributes, want_host_params + +[report] +# want_organization = True +# want_location = True +# want_ipv4 = True +# want_ipv6 = False +# want_host_group = True +# want_subnet = True +# want_subnet_v6 = False +# want_smart_proxies = True +# want_content_facet_attributes = False +# want_host_params = False + +# use this config to determine if facts are to be fetched in the report and stored on the hosts. +# want_facts = False + +# Upon receiving a request to return inventory report, Foreman schedules a report generation job. +# The script then polls the report_data endpoint repeatedly to check if the job is complete and retrieves data +# poll_interval allows to define the polling interval between 2 calls to the report_data endpoint while polling. +# Defaults to 10 seconds + +# poll_interval = 10 + [ansible] group_patterns = ["{app}-{tier}-{color}", "{app}-{color}", diff --git a/contrib/inventory/foreman.py b/contrib/inventory/foreman.py index e3f98f360a8..343cf26c9d1 100755 --- a/contrib/inventory/foreman.py +++ b/contrib/inventory/foreman.py @@ -28,7 +28,7 @@ import copy import os import re import sys -from time import time +from time import time, sleep from collections import defaultdict from distutils.version import LooseVersion, StrictVersion @@ -87,6 +87,72 @@ class ForemanInventory(object): print("Error parsing configuration: %s" % e, file=sys.stderr) return False + # Inventory Report Related + try: + self.foreman_use_reports_api = config.getboolean('foreman', 'use_reports_api') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + self.foreman_use_reports_api = True + + try: + self.want_organization = config.getboolean('report', 'want_organization') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + self.want_organization = True + + try: + self.want_location = config.getboolean('report', 'want_location') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + self.want_location = True + + try: + self.want_IPv4 = config.getboolean('report', 'want_ipv4') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + self.want_IPv4 = True + + try: + self.want_IPv6 = config.getboolean('report', 'want_ipv6') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + self.want_IPv6 = False + + try: + self.want_host_group = config.getboolean('report', 'want_host_group') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + self.want_host_group = True + + try: + self.want_host_params = config.getboolean('report', 'want_host_params') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + self.want_host_params = False + + try: + self.want_subnet = config.getboolean('report', 'want_subnet') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + self.want_subnet = True + + try: + self.want_subnet_v6 = config.getboolean('report', 'want_subnet_v6') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + self.want_subnet_v6 = False + + try: + self.want_smart_proxies = config.getboolean('report', 'want_smart_proxies') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + self.want_smart_proxies = True + + try: + self.want_content_facet_attributes = config.getboolean('report', 'want_content_facet_attributes') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + self.want_content_facet_attributes = False + + try: + self.report_want_facts = config.getboolean('report', 'want_facts') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + self.report_want_facts = True + + try: + self.poll_interval = config.getint('report', 'poll_interval') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + self.poll_interval = 10 + # Ansible related try: group_patterns = config.get('ansible', 'group_patterns') @@ -105,6 +171,8 @@ class ForemanInventory(object): except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): self.want_facts = True + self.want_facts = self.want_facts and self.report_want_facts + try: self.want_hostcollections = config.getboolean('ansible', 'want_hostcollections') except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): @@ -198,6 +266,52 @@ class ForemanInventory(object): break return results + def _use_inventory_report(self): + if not self.foreman_use_reports_api: + return False + status_url = "%s/api/v2/status" % self.foreman_url + result = self._get_json(status_url) + foreman_version = (LooseVersion(result.get('version')) >= LooseVersion('1.24.0')) + return foreman_version + + def _fetch_params(self): + options, params = ("no", "yes"), dict() + params["Organization"] = options[self.want_organization] + params["Location"] = options[self.want_location] + params["IPv4"] = options[self.want_IPv4] + params["IPv6"] = options[self.want_IPv6] + params["Facts"] = options[self.want_facts] + params["Host Group"] = options[self.want_host_group] + params["Host Collections"] = options[self.want_hostcollections] + params["Subnet"] = options[self.want_subnet] + params["Subnet v6"] = options[self.want_subnet_v6] + params["Smart Proxies"] = options[self.want_smart_proxies] + params["Content Attributes"] = options[self.want_content_facet_attributes] + params["Host Parameters"] = options[self.want_host_params] + if self.host_filters: + params["Hosts"] = self.host_filters + return params + + def _post_request(self): + url = "%s/ansible/api/v2/ansible_inventories/schedule" % self.foreman_url + session = self._get_session() + params = {'input_values': self._fetch_params()} + ret = session.post(url, json=params) + if not ret: + raise Exception("Error scheduling inventory report on foreman. Please check foreman logs!") + url = "{0}/{1}".format(self.foreman_url, ret.json().get('data_url')) + response = session.get(url) + while response: + if response.status_code != 204: + break + else: + sleep(self.poll_interval) + response = session.get(url) + if not response: + raise Exception("Error receiving inventory report from foreman. Please check foreman logs!") + else: + return response.json() + def _get_hosts(self): url = "%s/api/v2/hosts" % self.foreman_url @@ -271,6 +385,97 @@ class ForemanInventory(object): def update_cache(self, scan_only_new_hosts=False): """Make calls to foreman and save the output in a cache""" + use_inventory_report = self._use_inventory_report() + if use_inventory_report: + self._update_cache_inventory(scan_only_new_hosts) + else: + self._update_cache_host_api(scan_only_new_hosts) + + def _update_cache_inventory(self, scan_only_new_hosts): + self.groups = dict() + self.hosts = dict() + try: + inventory_report_response = self._post_request() + except Exception: + self._update_cache_host_api(scan_only_new_hosts) + return + host_data = json.loads(inventory_report_response) + for host in host_data: + if not(host) or (host["name"] in self.cache.keys() and scan_only_new_hosts): + continue + dns_name = host['name'] + + host_params = host.pop('host_parameters', {}) + fact_list = host.pop('facts', {}) + content_facet_attributes = host.get('content_attributes', {}) or {} + + # Create ansible groups for hostgroup + group = 'host_group' + val = host.get(group) + if val: + safe_key = self.to_safe('%s%s_%s' % ( + to_text(self.group_prefix), + group, + to_text(val).lower() + )) + self.inventory[safe_key].append(dns_name) + + # Create ansible groups for environment, location and organization + for group in ['environment', 'location', 'organization']: + val = host.get('%s' % group) + if val: + safe_key = self.to_safe('%s%s_%s' % ( + to_text(self.group_prefix), + group, + to_text(val).lower() + )) + self.inventory[safe_key].append(dns_name) + + for group in ['lifecycle_environment', 'content_view']: + val = content_facet_attributes.get('%s_name' % group) + if val: + safe_key = self.to_safe('%s%s_%s' % ( + to_text(self.group_prefix), + group, + to_text(val).lower() + )) + self.inventory[safe_key].append(dns_name) + + params = host_params + + # Ansible groups by parameters in host groups and Foreman host + # attributes. + groupby = dict() + for k, v in params.items(): + groupby[k] = self.to_safe(to_text(v)) + + # The name of the ansible groups is given by group_patterns: + for pattern in self.group_patterns: + try: + key = pattern.format(**groupby) + self.inventory[key].append(dns_name) + except KeyError: + pass # Host not part of this group + + if self.want_hostcollections: + hostcollections = host.get('host_collections') + + if hostcollections: + # Create Ansible groups for host collections + for hostcollection in hostcollections: + safe_key = self.to_safe('%shostcollection_%s' % (self.group_prefix, hostcollection.lower())) + self.inventory[safe_key].append(dns_name) + + self.hostcollections[dns_name] = hostcollections + + self.cache[dns_name] = host + self.params[dns_name] = params + self.facts[dns_name] = fact_list + self.inventory['all'].append(dns_name) + self._write_cache() + + def _update_cache_host_api(self, scan_only_new_hosts): + """Make calls to foreman and save the output in a cache""" self.groups = dict() self.hosts = dict() diff --git a/test/integration/targets/inventory_foreman_script/runme.sh b/test/integration/targets/inventory_foreman_script/runme.sh index 72add9e7b41..a9c94fbe7de 100755 --- a/test/integration/targets/inventory_foreman_script/runme.sh +++ b/test/integration/targets/inventory_foreman_script/runme.sh @@ -17,6 +17,7 @@ url = http://${FOREMAN_HOST}:${FOREMAN_PORT} user = ansible-tester password = secure ssl_verify = False +use_reports_api = False FOREMAN_INI # use ansible to validate the return data