foreman.py: create Ansible groups from Satellite 6 host collections (#25269)

* create Ansible groups from host collections

* fix paging logic in _get_json

* replace Satellite with Foreman

* improve comment for environment variables

* enable host collections by default

* use requests params instead of urllib.quote_plus

* disable host collections by default

* change organization filter

* clean up load_cache methods

* remove usage of function defaults

* replace environment variables with host_filters

contrib/inventory/foreman.ini

@@ -68,6 +68,21 @@
 #
 #     foreman_hostgroup_myapp_webtier_datacenter1
 #
+# If the parameter want_hostcollections is set to true, the
+# collections each host is in are created as Ansible groups with a
+# foreman_hostcollection prefix, all lowercase and problematic
+# parameters removed. So e.g. the Foreman host collection
+#
+#     Patch Window Thursday
+#
+# would turn into the Ansible group:
+#
+#     foreman_hostcollection_patchwindowthursday
+#
+# If the parameter host_filters is set, it will be used as the
+# "search" parameter for the /api/v2/hosts call. This can be used to
+# restrict the list of returned host, as shown below.
+#
 # Furthermore Ansible groups can be created on the fly using the
 # *group_patterns* variable in *foreman.ini* so that you can build up
 # hierarchies using parameters on the hostgroup and host variables.
@@ -108,15 +123,28 @@ user = foreman
 password = secret
 ssl_verify = True
 
+# Retrieve only hosts from the organization "Web Engineering".
+# host_filters = organization="Web Engineering"
+
+# Retrieve only hosts from the organization "Web Engineering" that are
+# also in the host collection "Apache Servers".
+# host_filters = organization="Web Engineering" and host_collection="Apache Servers"
+
 [ansible]
 group_patterns = ["{app}-{tier}-{color}",
 	          "{app}-{color}",
 	          "{app}",
 		  "{tier}"]
 group_prefix = foreman_
+
 # Whether to fetch facts from Foreman and store them on the host
 want_facts = True
 
+# Whether to create Ansible groups for host collections. Only tested
+# with Katello (Red Hat Satellite). Disabled by default to not break
+# the script for stand-alone Foreman.
+want_hostcollections = False
+
 [cache]
 path = .
 max_age = 60

contrib/inventory/foreman.py

@@ -64,6 +64,7 @@ class ForemanInventory(object):
         self.params = dict()  # Params of each host
         self.facts = dict()   # Facts of each host
         self.hostgroups = dict()  # host groups
+        self.hostcollections = dict()  # host collections
         self.session = None   # Requests session
         self.config_paths = [
             "/etc/ansible/foreman.ini",
@@ -107,6 +108,16 @@ class ForemanInventory(object):
         except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
             self.want_facts = True
 
+        try:
+            self.want_hostcollections = config.getboolean('ansible', 'want_hostcollections')
+        except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
+            self.want_hostcollections = False
+
+        try:
+            self.host_filters = config.get('foreman', 'host_filters')
+        except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
+            self.host_filters = None
+
         # Cache related
         try:
             cache_path = os.path.expanduser(config.get('cache', 'path'))
@@ -117,10 +128,12 @@ class ForemanInventory(object):
         self.cache_path_inventory = cache_path + "/%s.index" % script
         self.cache_path_params = cache_path + "/%s.params" % script
         self.cache_path_facts = cache_path + "/%s.facts" % script
+        self.cache_path_hostcollections = cache_path + "/%s.hostcollections" % script
         try:
             self.cache_max_age = config.getint('cache', 'max_age')
         except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
             self.cache_max_age = 60
+
         return True
 
     def parse_cli_args(self):
@@ -140,12 +153,17 @@ class ForemanInventory(object):
             self.session.verify = self.foreman_ssl_verify
         return self.session
 
-    def _get_json(self, url, ignore_errors=None):
+    def _get_json(self, url, ignore_errors=None, params=None):
+        if params is None:
+            params = {}
+        params['per_page'] = 250
+
         page = 1
         results = []
         s = self._get_session()
         while True:
-            ret = s.get(url, params={'page': page, 'per_page': 250})
+            params['page'] = page
+            ret = s.get(url, params=params)
             if ignore_errors and ret.status_code in ignore_errors:
                 break
             ret.raise_for_status()
@@ -158,7 +176,7 @@ class ForemanInventory(object):
                 return json['results']
             # List of all hosts is returned paginaged
             results = results + json['results']
-            if len(results) >= json['total']:
+            if len(results) >= json['subtotal']:
                 break
             page += 1
             if len(json['results']) == 0:
@@ -169,20 +187,27 @@ class ForemanInventory(object):
         return results
 
     def _get_hosts(self):
-        return self._get_json("%s/api/v2/hosts" % self.foreman_url)
+        url = "%s/api/v2/hosts" % self.foreman_url
+
+        params = {}
+        if self.host_filters:
+            params['search'] = self.host_filters
+
+        return self._get_json(url, params=params)
 
-    def _get_all_params_by_id(self, hid):
+    def _get_host_data_by_id(self, hid):
         url = "%s/api/v2/hosts/%s" % (self.foreman_url, hid)
-        ret = self._get_json(url, [404])
+        return self._get_json(url)
-        if ret == []:
-            ret = {}
-        return ret.get('all_parameters', {})
 
-    def _resolve_params(self, host):
+    def _get_facts_by_id(self, hid):
-        """Fetch host params and convert to dict"""
+        url = "%s/api/v2/hosts/%s/facts" % (self.foreman_url, hid)
+        return self._get_json(url)
+
+    def _resolve_params(self, host_params):
+        """Convert host params to dict"""
         params = {}
 
-        for param in self._get_all_params_by_id(host['id']):
+        for param in host_params:
             name = param['name']
             params[name] = param['value']
 
@@ -218,6 +243,7 @@ class ForemanInventory(object):
         self.write_to_cache(self.inventory, self.cache_path_inventory)
         self.write_to_cache(self.params, self.cache_path_params)
         self.write_to_cache(self.facts, self.cache_path_facts)
+        self.write_to_cache(self.hostcollections, self.cache_path_hostcollections)
 
     def to_safe(self, word):
         '''Converts 'bad' characters in a string to underscores
@@ -238,6 +264,9 @@ class ForemanInventory(object):
         for host in self._get_hosts():
             dns_name = host['name']
 
+            host_data = self._get_host_data_by_id(host['id'])
+            host_params = host_data.get('all_parameters', {})
+
             # Create ansible groups for hostgroup
             group = 'hostgroup'
             val = host.get('%s_title' % group) or host.get('%s_name' % group)
@@ -258,7 +287,7 @@ class ForemanInventory(object):
                     safe_key = self.to_safe('%s%s_%s' % (self.group_prefix, group, val.lower()))
                     self.inventory[safe_key].append(dns_name)
 
-            params = self._resolve_params(host)
+            params = self._resolve_params(host_params)
 
             # Ansible groups by parameters in host groups and Foreman host
             # attributes.
@@ -274,6 +303,17 @@ class ForemanInventory(object):
                 except KeyError:
                     pass  # Host not part of this group
 
+            if self.want_hostcollections:
+                hostcollections = host_data.get('host_collections')
+
+                if hostcollections:
+                    # Create Ansible groups for host collections
+                    for hostcollection in hostcollections:
+                        safe_key = self.to_safe('%shostcollection_%s' % (self.group_prefix, hostcollection['name'].lower()))
+                        self.inventory[safe_key].append(dns_name)
+
+                self.hostcollections[dns_name] = hostcollections
+
             self.cache[dns_name] = host
             self.params[dns_name] = params
             self.facts[dns_name] = self._get_facts(host)
@@ -295,31 +335,36 @@ class ForemanInventory(object):
     def load_inventory_from_cache(self):
         """Read the index from the cache file sets self.index"""
 
-        cache = open(self.cache_path_inventory, 'r')
+        with open(self.cache_path_inventory, 'r') as fp:
-        json_inventory = cache.read()
+            self.inventory = json.load(fp)
-        self.inventory = json.loads(json_inventory)
 
     def load_params_from_cache(self):
         """Read the index from the cache file sets self.index"""
 
-        cache = open(self.cache_path_params, 'r')
+        with open(self.cache_path_params, 'r') as fp:
-        json_params = cache.read()
+            self.params = json.load(fp)
-        self.params = json.loads(json_params)
 
     def load_facts_from_cache(self):
         """Read the index from the cache file sets self.facts"""
+
         if not self.want_facts:
             return
-        cache = open(self.cache_path_facts, 'r')
+        with open(self.cache_path_facts, 'r') as fp:
-        json_facts = cache.read()
+            self.facts = json.load(fp)
-        self.facts = json.loads(json_facts)
+
+    def load_hostcollections_from_cache(self):
+        """Read the index from the cache file sets self.hostcollections"""
+
+        if not self.want_hostcollections:
+            return
+        with open(self.cache_path_hostcollections, 'r') as fp:
+            self.hostcollections = json.load(fp)
 
     def load_cache_from_cache(self):
         """Read the cache from the cache file sets self.cache"""
 
-        cache = open(self.cache_path_cache, 'r')
+        with open(self.cache_path_cache, 'r') as fp:
-        json_cache = cache.read()
+            self.cache = json.load(fp)
-        self.cache = json.loads(json_cache)
 
     def get_inventory(self):
         if self.args.refresh_cache or not self.is_cache_valid():
@@ -328,6 +373,7 @@ class ForemanInventory(object):
             self.load_inventory_from_cache()
             self.load_params_from_cache()
             self.load_facts_from_cache()
+            self.load_hostcollections_from_cache()
             self.load_cache_from_cache()
 
     def get_host_info(self):