vmware_inventory: permit to group by custom field & customize skip_keys (#20125)

* vmware_inventory: permit to group by custom field

This permits to create instances, affect some custom fields like EC2 tags and then retrieve groups from custom fields like EC2 inventory

* vmware_inventory: Customize skip_keys & add resourceconfig to skip_keys

Verify if customfield is a str before processing custom fields for a host

contrib/inventory/vmware_inventory.ini

@@ -42,6 +42,12 @@ password=vmware
 #lower_var_keys=True
 
 
+# Don't retrieve and process some VMware attribute keys
+# Default values permit to sanitize inventory meta and to improve a little bit
+# performance by removing non-common group attributes.
+#skip_keys = declaredalarmstate,disabledmethod,dynamicproperty,dynamictype,environmentbrowser,managedby,parent,childtype,resourceconfig
+
+
 # Host alias for objects in the inventory. VMWare allows duplicate VM names
 # so they can not be considered unique. Use this setting to alter the alias
 # returned for the hosts. Any atributes for the guest can be used to build
@@ -72,6 +78,17 @@ password=vmware
 # comma delimited to create as many groups as necessary
 #groupby_patterns={{ guest.guestid }},{{ 'templates' if config.template else 'guests'}}
 
+# Group by custom fields will use VMware custom fields to generate hostgroups
+# based on {{ custom_field_group_prefix }} + field_name + _ + field_value
+# Set groupby_custom_field to True will enable this feature
+# If custom field value is comma separated, multiple groups are created.
+# Warning: This required max_object_level to be set to 2 or greater.
+#groupby_custom_field = False
+
+# You can customize prefix used by custom field hostgroups generation here.
+# vmware_tag_ prefix is the default and consistent with ec2_tag_
+#custom_field_group_prefix = 'vmware_tag_'
+
 # The script attempts to recurse into virtualmachine objects and serialize
 # all available data. The serialization is comprehensive but slow. If the
 # vcenter environment is large and the desired properties are known, create

contrib/inventory/vmware_inventory.py

@@ -86,6 +86,7 @@ class VMWareInventory(object):
     password = None
     validate_certs = True
     host_filters = []
+    skip_keys = []
     groupby_patterns = []
 
     if sys.version_info > (3, 0):
@@ -95,20 +96,14 @@ class VMWareInventory(object):
     iter_types = [dict, list]
 
     bad_types = ['Array', 'disabledMethod', 'declaredAlarmState']
-    skip_keys = ['declaredalarmstate',
-                 'disabledmethod',
-                 'dynamicproperty',
-                 'dynamictype',
-                 'environmentbrowser',
-                 'managedby',
-                 'parent',
-                 'childtype']
 
     vimTableMaxDepth = {
         "vim.HostSystem": 2,
         "vim.VirtualMachine": 2,
     }
 
+    custom_fields = {}
+
     # translation table for attributes to fetch for known vim types
     if not HAS_PYVMOMI:
         vimTable = {}
@@ -178,12 +173,10 @@ class VMWareInventory(object):
 
         ''' Get instances and cache the data '''
 
-        instances = self.get_instances()
+        self.inventory = self.instances_to_inventory(self.get_instances())
-        self.instances = instances
+        self.write_to_cache(self.inventory)
-        self.inventory = self.instances_to_inventory(instances)
-        self.write_to_cache(self.inventory, self.cache_path_cache)
 
-    def write_to_cache(self, data, cache_path):
+    def write_to_cache(self, data):
 
         ''' Dump inventory to json file '''
 
@@ -218,11 +211,22 @@ class VMWareInventory(object):
             'cache_path': '~/.ansible/tmp',
             'cache_max_age': 3600,
             'max_object_level': 1,
+            'skip_keys': 'declaredalarmstate,'
+                         'disabledmethod,'
+                         'dynamicproperty,'
+                         'dynamictype,'
+                         'environmentbrowser,'
+                         'managedby,'
+                         'parent,'
+                         'childtype,'
+                         'resourceconfig',
             'alias_pattern': '{{ config.name + "_" + config.uuid }}',
             'host_pattern': '{{ guest.ipaddress }}',
             'host_filters': '{{ guest.gueststate == "running" }}',
             'groupby_patterns': '{{ guest.guestid }},{{ "templates" if config.template else "guests"}}',
-            'lower_var_keys': True}
+            'lower_var_keys': True,
+            'custom_field_group_prefix': 'vmware_tag_',
+            'groupby_custom_field': False}
         }
 
         if six.PY3:
@@ -274,7 +278,8 @@ class VMWareInventory(object):
             else:
                 self.lowerkeys = False
         self.debugl('lower keys is %s' % self.lowerkeys)
-
+        self.skip_keys = list(config.get('vmware', 'skip_keys').split(','))
+        self.debugl('skip keys is %s' % self.skip_keys)
         self.host_filters = list(config.get('vmware', 'host_filters').split(','))
         self.debugl('host filters are %s' % self.host_filters)
         self.groupby_patterns = list(config.get('vmware', 'groupby_patterns').split(','))
@@ -321,8 +326,7 @@ class VMWareInventory(object):
             context.verify_mode = ssl.CERT_NONE
             kwargs['sslContext'] = context
 
-        instances = self._get_instances(kwargs)
+        return self._get_instances(kwargs)
-        return instances
 
     def _get_instances(self, inkwargs):
 
@@ -365,6 +369,13 @@ class VMWareInventory(object):
                 ifacts = self.facts_from_vobj(instance)
             instance_tuples.append((instance, ifacts))
         self.debugl('facts collected for all instances')
+
+        cfm = content.customFieldsManager
+        if cfm is not None and cfm.field:
+            for f in cfm.field:
+                if f.managedObjectType == vim.VirtualMachine:
+                    self.custom_fields[f.key] = f.name;
+            self.debugl('%d custom fieds collected' % len(self.custom_fields))
         return instance_tuples
 
     def instances_to_inventory(self, instances):
@@ -452,6 +463,33 @@ class VMWareInventory(object):
                 if k not in inventory[v]['hosts']:
                     inventory[v]['hosts'].append(k)
 
+        if self.config.get('vmware', 'groupby_custom_field'):
+            for k, v in inventory['_meta']['hostvars'].items():
+                if 'customvalue' in v:
+                    for tv in v['customvalue']:
+                        if not isinstance(tv['value'], str) and not isinstance(tv['value'], unicode):
+                            continue
+
+                        newkey = None
+                        field_name = self.custom_fields[tv['key']] if tv['key'] in self.custom_fields else tv['key']
+                        values = []
+                        keylist = map(lambda x: x.strip(), tv['value'].split(','))
+                        for kl in keylist:
+                            try:
+                                newkey = self.config.get('vmware', 'custom_field_group_prefix') + field_name + '_' + kl
+                                newkey = newkey.strip()
+                            except Exception as e:
+                                self.debugl(e)
+                            values.append(newkey)
+                        for tag in values:
+                            if not tag:
+                                continue
+                            if tag not in inventory:
+                                inventory[tag] = {}
+                                inventory[tag]['hosts'] = []
+                            if k not in inventory[tag]['hosts']:
+                                inventory[tag]['hosts'].append(k)
+
         return inventory
 
     def create_template_mapping(self, inventory, pattern, dtype='string'):