nxos_telemetry replaced state (#62368)

lib/ansible/module_utils/network/nxos/config/telemetry/telemetry.py

@@ -18,8 +18,8 @@ from ansible.module_utils.network.common.utils import to_list
 from ansible.module_utils.network.nxos.facts.facts import Facts
 from ansible.module_utils.network.nxos.cmdref.telemetry.telemetry import TMS_GLOBAL, TMS_DESTGROUP, TMS_SENSORGROUP, TMS_SUBSCRIPTION
 from ansible.module_utils.network.nxos.utils.telemetry.telemetry import normalize_data, remove_duplicate_context
-from ansible.module_utils.network.nxos.utils.telemetry.telemetry import valiate_input, get_setval_path
-from ansible.module_utils.network.nxos.utils.telemetry.telemetry import get_module_params_subsection
+from ansible.module_utils.network.nxos.utils.telemetry.telemetry import valiate_input, get_setval_path, massage_data
+from ansible.module_utils.network.nxos.utils.telemetry.telemetry import get_module_params_subsection, remove_duplicate_commands
 from ansible.module_utils.network.nxos.utils.utils import normalize_interface
 from ansible.module_utils.network.nxos.nxos import NxosCmdRef
 
@@ -43,7 +43,6 @@ class Telemetry(ConfigBase):
 
     def get_telemetry_facts(self):
         """ Get the 'facts' (the current configuration)
-
         :rtype: A dictionary
         :returns: The current configuration as a dictionary
         """
@@ -68,9 +67,6 @@ class Telemetry(ConfigBase):
         state = self._module.params['state']
         if 'overridden' in state:
             self._module.fail_json(msg='State <overridden> is invalid for this module.')
-        if 'replaced' in state:
-            self._module.fail_json(msg='State: <replaced> not yet supported')
-
         # When state is 'deleted', the module_params should not contain data
         # under the 'config' key
         if 'deleted' in state and self._module.params.get('config'):
@@ -132,6 +128,10 @@ class Telemetry(ConfigBase):
         # and does not require any processing using NxosCmdRef objects.
         if state == 'deleted':
             return self._state_deleted(want, have)
+        elif state == 'replaced':
+            if want == have:
+                return []
+            return self._state_replaced(want, have)
 
         # Save off module params
         ALL_MP = self._module.params['config']
@@ -142,7 +142,7 @@ class Telemetry(ConfigBase):
         cmd_ref['TMS_SENSORGROUP'] = {}
         cmd_ref['TMS_SUBSCRIPTION'] = {}
 
-        # Get Telemetry Global Data
+        # Build Telemetry Global NxosCmdRef Object
         cmd_ref['TMS_GLOBAL']['ref'] = []
         self._module.params['config'] = get_module_params_subsection(ALL_MP, 'TMS_GLOBAL')
         cmd_ref['TMS_GLOBAL']['ref'].append(NxosCmdRef(self._module, TMS_GLOBAL))
@@ -152,94 +152,319 @@ class Telemetry(ConfigBase):
         ref.get_playvals()
         device_cache = ref.cache_existing
 
-        # Get Telemetry Destination Group Data
-        if want.get('destination_groups'):
-            td = {'name': 'destination_groups', 'type': 'TMS_DESTGROUP',
-                  'obj': TMS_DESTGROUP, 'cmd': 'destination-group {0}'}
-            cmd_ref[td['type']]['ref'] = []
-            saved_ids = []
-            for playvals in want[td['name']]:
-                valiate_input(playvals, td['name'], self._module)
-                if playvals['id'] in saved_ids:
-                    continue
-                saved_ids.append(playvals['id'])
-                resource_key = td['cmd'].format(playvals['id'])
-                # Only build the NxosCmdRef object for the destination group module parameters.
-                self._module.params['config'] = get_module_params_subsection(ALL_MP, td['type'], playvals['id'])
-                cmd_ref[td['type']]['ref'].append(NxosCmdRef(self._module, td['obj']))
-                ref = cmd_ref[td['type']]['ref'][-1]
-                ref.set_context([resource_key])
-                ref.get_existing(device_cache)
-                ref.get_playvals()
-                normalize_data(ref)
-
-        # Get Telemetry Sensor Group Data
-        if want.get('sensor_groups'):
-            td = {'name': 'sensor_groups', 'type': 'TMS_SENSORGROUP',
-                  'obj': TMS_SENSORGROUP, 'cmd': 'sensor-group {0}'}
+        def build_cmdref_objects(td):
             cmd_ref[td['type']]['ref'] = []
             saved_ids = []
-            for playvals in want[td['name']]:
-                valiate_input(playvals, td['name'], self._module)
-                if playvals['id'] in saved_ids:
-                    continue
-                saved_ids.append(playvals['id'])
-                resource_key = td['cmd'].format(playvals['id'])
-                # Only build the NxosCmdRef object for the sensor group module parameters.
-                self._module.params['config'] = get_module_params_subsection(ALL_MP, td['type'], playvals['id'])
-                cmd_ref[td['type']]['ref'].append(NxosCmdRef(self._module, td['obj']))
-                ref = cmd_ref[td['type']]['ref'][-1]
-                ref.set_context([resource_key])
-                if get_setval_path(self._module):
-                    # Sensor group path setting can contain optional values.
-                    # Call get_setval_path helper function to process any
-                    # optional setval keys.
-                    ref._ref['path']['setval'] = get_setval_path(self._module)
-                ref.get_existing(device_cache)
-                ref.get_playvals()
-
-        # Get Telemetry Subscription Data
-        if want.get('subscriptions'):
-            td = {'name': 'subscriptions', 'type': 'TMS_SUBSCRIPTION',
-                  'obj': TMS_SUBSCRIPTION, 'cmd': 'subscription {0}'}
-            cmd_ref[td['type']]['ref'] = []
-            saved_ids = []
-            for playvals in want[td['name']]:
-                valiate_input(playvals, td['name'], self._module)
-                if playvals['id'] in saved_ids:
-                    continue
-                saved_ids.append(playvals['id'])
-                resource_key = td['cmd'].format(playvals['id'])
-                # Only build the NxosCmdRef object for the subscription module parameters.
-                self._module.params['config'] = get_module_params_subsection(ALL_MP, td['type'], playvals['id'])
-                cmd_ref[td['type']]['ref'].append(NxosCmdRef(self._module, td['obj']))
-                ref = cmd_ref[td['type']]['ref'][-1]
-                ref.set_context([resource_key])
-                ref.get_existing(device_cache)
-                ref.get_playvals()
-
-        if state == 'overridden':
-            if want == have:
-                return []
-            commands = self._state_overridden(cmd_ref, want, have)
-        elif state == 'merged':
+            if want.get(td['name']):
+                for playvals in want[td['name']]:
+                    valiate_input(playvals, td['name'], self._module)
+                    if playvals['id'] in saved_ids:
+                        continue
+                    saved_ids.append(playvals['id'])
+                    resource_key = td['cmd'].format(playvals['id'])
+                    # Only build the NxosCmdRef object for the td['name'] module parameters.
+                    self._module.params['config'] = get_module_params_subsection(ALL_MP, td['type'], playvals['id'])
+                    cmd_ref[td['type']]['ref'].append(NxosCmdRef(self._module, td['obj']))
+                    ref = cmd_ref[td['type']]['ref'][-1]
+                    ref.set_context([resource_key])
+                    if td['type'] == 'TMS_SENSORGROUP' and get_setval_path(self._module):
+                        # Sensor group path setting can contain optional values.
+                        # Call get_setval_path helper function to process any
+                        # optional setval keys.
+                        ref._ref['path']['setval'] = get_setval_path(self._module)
+                    ref.get_existing(device_cache)
+                    ref.get_playvals()
+                    if td['type'] == 'TMS_DESTGROUP':
+                        normalize_data(ref)
+
+        # Build Telemetry Destination Group NxosCmdRef Objects
+        td = {'name': 'destination_groups', 'type': 'TMS_DESTGROUP',
+              'obj': TMS_DESTGROUP, 'cmd': 'destination-group {0}'}
+        build_cmdref_objects(td)
+
+        # Build Telemetry Sensor Group NxosCmdRef Objects
+        td = {'name': 'sensor_groups', 'type': 'TMS_SENSORGROUP',
+              'obj': TMS_SENSORGROUP, 'cmd': 'sensor-group {0}'}
+        build_cmdref_objects(td)
+
+        # Build Telemetry Subscription NxosCmdRef Objects
+        td = {'name': 'subscriptions', 'type': 'TMS_SUBSCRIPTION',
+              'obj': TMS_SUBSCRIPTION, 'cmd': 'subscription {0}'}
+        build_cmdref_objects(td)
+
+        if state == 'merged':
             if want == have:
                 return []
             commands = self._state_merged(cmd_ref)
-        elif state == 'replaced':
-            if want == have:
-                return []
-            commands = self._state_replaced(cmd_ref)
         return commands
 
     @staticmethod
-    def _state_replaced(cmd_ref):
+    def _state_replaced(want, have):
         """ The command generator when state is replaced
         :rtype: A list
         :returns: the commands necessary to migrate the current configuration
                   to the desired configuration
         """
         commands = []
+        massaged_have = massage_data(have)
+        massaged_want = massage_data(want)
+
+        ref = {}
+        ref['tms_global'] = NxosCmdRef([], TMS_GLOBAL, ref_only=True)
+        ref['tms_destgroup'] = NxosCmdRef([], TMS_DESTGROUP, ref_only=True)
+        ref['tms_sensorgroup'] = NxosCmdRef([], TMS_SENSORGROUP, ref_only=True)
+        ref['tms_subscription'] = NxosCmdRef([], TMS_SUBSCRIPTION, ref_only=True)
+
+        # Order matters for state replaced.
+        # First remove all subscriptions, followed by sensor-groups and destination-groups.
+        # Second add all destination-groups, followed by sensor-groups and subscriptions
+        add = {'TMS_GLOBAL': [], 'TMS_DESTGROUP': [], 'TMS_SENSORGROUP': [], 'TMS_SUBSCRIPTION': []}
+        delete = {'TMS_DESTGROUP': [], 'TMS_SENSORGROUP': [], 'TMS_SUBSCRIPTION': []}
+
+        # Process Telemetry Global Want and Have Values
+        # Possible states:
+        # - want and have are (set) (equal: no action, not equal: replace with want)
+        # - want (set) have (not set) (add want)
+        # - want (not set) have (set) (delete have)
+        # - want (not set) have (not set) (no action)
+        # global_ctx = ref['tms_global']._ref['_template']['context']
+        # property_ctx = ref['tms_global']._ref['certificate'].get('context')
+        # setval = ref['tms_global']._ref['certificate']['setval']
+        #
+        all_global_properties = ['certificate', 'compression', 'source_interface', 'vrf']
+        dest_profile_properties = ['compression', 'source_interface', 'vrf']
+        dest_profile_remote_commands = []
+        for property in all_global_properties:
+            cmd = None
+            global_ctx = ref['tms_global']._ref['_template']['context']
+            property_ctx = ref['tms_global']._ref[property].get('context')
+            setval = ref['tms_global']._ref[property]['setval']
+            kind = ref['tms_global']._ref[property]['kind']
+            if want.get(property) is not None:
+                if have.get(property) is not None:
+                    if want.get(property) != have.get(property):
+                        if kind == 'dict':
+                            cmd = [setval.format(**want.get(property))]
+                        else:
+                            cmd = [setval.format(want.get(property))]
+                elif have.get(property) is None:
+                    if kind == 'dict':
+                        cmd = [setval.format(**want.get(property))]
+                    else:
+                        cmd = [setval.format(want.get(property))]
+            elif want.get(property) is None:
+                if have.get(property) is not None:
+                    if kind == 'dict':
+                        cmd = ['no ' + setval.format(**have.get(property))]
+                    else:
+                        cmd = ['no ' + setval.format(have.get(property))]
+                    if property in dest_profile_properties:
+                        dest_profile_remote_commands.extend(cmd)
+
+            if cmd is not None:
+                ctx = global_ctx
+                if property_ctx is not None:
+                    ctx.extend(property_ctx)
+                add['TMS_GLOBAL'].extend(ctx)
+                add['TMS_GLOBAL'].extend(cmd)
+
+        add['TMS_GLOBAL'] = remove_duplicate_commands(add['TMS_GLOBAL'])
+        # If all destination profile commands are being removed then just
+        # remove the config context instead.
+        if len(dest_profile_remote_commands) == 3:
+            for item in dest_profile_remote_commands:
+                add['TMS_GLOBAL'].remove(item)
+            add['TMS_GLOBAL'].remove('destination-profile')
+            add['TMS_GLOBAL'].extend(['no destination-profile'])
+
+        # Process Telemetry destination_group, sensor_group and subscription Want and Have Values
+        # Possible states:
+        # - want (not set) have (set) (delete have)
+        # - want and have are (set) (equal: no action, not equal: replace with want)
+        # - want (set) have (not set) (add want)
+        # - want (not set) have (not set) (no action)
+        tms_resources = ['TMS_DESTGROUP', 'TMS_SENSORGROUP', 'TMS_SUBSCRIPTION']
+        for resource in tms_resources:
+            if resource == 'TMS_DESTGROUP':
+                name = 'destination-group'
+                cmd_property = 'destination'
+                global_ctx = ref['tms_destgroup']._ref['_template']['context']
+                setval = ref['tms_destgroup']._ref['destination']['setval']
+                want_resources = massaged_want.get('destination_groups')
+                have_resources = massaged_have.get('destination_groups')
+            if resource == 'TMS_SENSORGROUP':
+                name = 'sensor-group'
+                global_ctx = ref['tms_sensorgroup']._ref['_template']['context']
+                setval = {}
+                setval['data_source'] = ref['tms_sensorgroup']._ref['data_source']['setval']
+                setval['path'] = ref['tms_sensorgroup']._ref['path']['setval']
+                want_resources = massaged_want.get('sensor_groups')
+                have_resources = massaged_have.get('sensor_groups')
+            if resource == 'TMS_SUBSCRIPTION':
+                name = 'subscription'
+                global_ctx = ref['tms_subscription']._ref['_template']['context']
+                setval = {}
+                setval['destination_group'] = ref['tms_subscription']._ref['destination_group']['setval']
+                setval['sensor_group'] = ref['tms_subscription']._ref['sensor_group']['setval']
+                want_resources = massaged_want.get('subscriptions')
+                have_resources = massaged_have.get('subscriptions')
+
+            if not want_resources and have_resources:
+                # want not and have not set so delete have
+                for key in have_resources.keys():
+                    remove_context = ['{0} {1} {2}'.format('no', name, key)]
+                    delete[resource].extend(global_ctx)
+                    if remove_context[0] not in delete[resource]:
+                        delete[resource].extend(remove_context)
+            else:
+                # want and have are set.
+                # process wants:
+                for want_key in want_resources.keys():
+                    if want_key not in have_resources.keys():
+                        # Want resource key not in have resource key so add it
+                        property_ctx = ['{0} {1}'.format(name, want_key)]
+                        for item in want_resources[want_key]:
+                            if resource == 'TMS_DESTGROUP':
+                                cmd = [setval.format(**item[cmd_property])]
+                                add[resource].extend(global_ctx)
+                                if property_ctx[0] not in add[resource]:
+                                    add[resource].extend(property_ctx)
+                                add[resource].extend(cmd)
+                            if resource == 'TMS_SENSORGROUP':
+                                cmd = {}
+                                if item.get('data_source'):
+                                    cmd['data_source'] = [setval['data_source'].format(item['data_source'])]
+                                if item.get('path'):
+                                    setval['path'] = get_setval_path(item.get('path'))
+                                    cmd['path'] = [setval['path'].format(**item['path'])]
+                                add[resource].extend(global_ctx)
+                                if property_ctx[0] not in add[resource]:
+                                    add[resource].extend(property_ctx)
+                                if cmd.get('data_source'):
+                                    add[resource].extend(cmd['data_source'])
+                                if cmd.get('path'):
+                                    add[resource].extend(cmd['path'])
+                            if resource == 'TMS_SUBSCRIPTION':
+                                cmd = {}
+                                if item.get('destination_group'):
+                                    cmd['destination_group'] = [setval['destination_group'].format(item['destination_group'])]
+                                if item.get('sensor_group'):
+                                    cmd['sensor_group'] = [setval['sensor_group'].format(**item['sensor_group'])]
+                                add[resource].extend(global_ctx)
+                                if property_ctx[0] not in add[resource]:
+                                    add[resource].extend(property_ctx)
+                                if cmd.get('destination_group'):
+                                    add[resource].extend(cmd['destination_group'])
+                                if cmd.get('sensor_group'):
+                                    add[resource].extend(cmd['sensor_group'])
+
+                    elif want_key in have_resources.keys():
+                        # Want resource key exists in have resource keys but we need to
+                        # inspect the individual items under the resource key
+                        # for differences
+                        for item in want_resources[want_key]:
+                            if item not in have_resources[want_key]:
+                                if item is None:
+                                    continue
+                                # item wanted but does not exist so add it
+                                property_ctx = ['{0} {1}'.format(name, want_key)]
+                                if resource == 'TMS_DESTGROUP':
+                                    cmd = [setval.format(**item[cmd_property])]
+                                    add[resource].extend(global_ctx)
+                                    if property_ctx[0] not in add[resource]:
+                                        add[resource].extend(property_ctx)
+                                    add[resource].extend(cmd)
+                                if resource == 'TMS_SENSORGROUP':
+                                    cmd = {}
+                                    if item.get('data_source'):
+                                        cmd['data_source'] = [setval['data_source'].format(item['data_source'])]
+                                    if item.get('path'):
+                                        setval['path'] = get_setval_path(item.get('path'))
+                                        cmd['path'] = [setval['path'].format(**item['path'])]
+                                    add[resource].extend(global_ctx)
+                                    if property_ctx[0] not in add[resource]:
+                                        add[resource].extend(property_ctx)
+                                    if cmd.get('data_source'):
+                                        add[resource].extend(cmd['data_source'])
+                                    if cmd.get('path'):
+                                        add[resource].extend(cmd['path'])
+                                if resource == 'TMS_SUBSCRIPTION':
+                                    cmd = {}
+                                    if item.get('destination_group'):
+                                        cmd['destination_group'] = [setval['destination_group'].format(item['destination_group'])]
+                                    if item.get('sensor_group'):
+                                        cmd['sensor_group'] = [setval['sensor_group'].format(**item['sensor_group'])]
+                                    add[resource].extend(global_ctx)
+                                    if property_ctx[0] not in add[resource]:
+                                        add[resource].extend(property_ctx)
+                                    if cmd.get('destination_group'):
+                                        add[resource].extend(cmd['destination_group'])
+                                    if cmd.get('sensor_group'):
+                                        add[resource].extend(cmd['sensor_group'])
+
+                # process haves:
+                for have_key in have_resources.keys():
+                    if have_key not in want_resources.keys():
+                        # Want resource key is not in have resource keys so remove it
+                        cmd = ['no ' + '{0} {1}'.format(name, have_key)]
+                        delete[resource].extend(global_ctx)
+                        delete[resource].extend(cmd)
+                    elif have_key in want_resources.keys():
+                        # Have resource key exists in want resource keys but we need to
+                        # inspect the individual items under the resource key
+                        # for differences
+                        for item in have_resources[have_key]:
+                            if item not in want_resources[have_key]:
+                                if item is None:
+                                    continue
+                                # have item not wanted so remove it
+                                property_ctx = ['{0} {1}'.format(name, have_key)]
+                                if resource == 'TMS_DESTGROUP':
+                                    cmd = ['no ' + setval.format(**item[cmd_property])]
+                                    delete[resource].extend(global_ctx)
+                                    if property_ctx[0] not in delete[resource]:
+                                        delete[resource].extend(property_ctx)
+                                    delete[resource].extend(cmd)
+                                if resource == 'TMS_SENSORGROUP':
+                                    cmd = {}
+                                    if item.get('data_source'):
+                                        cmd['data_source'] = ['no ' + setval['data_source'].format(item['data_source'])]
+                                    if item.get('path'):
+                                        setval['path'] = get_setval_path(item.get('path'))
+                                        cmd['path'] = ['no ' + setval['path'].format(**item['path'])]
+                                    delete[resource].extend(global_ctx)
+                                    if property_ctx[0] not in delete[resource]:
+                                        delete[resource].extend(property_ctx)
+                                    if cmd.get('data_source'):
+                                        delete[resource].extend(cmd['data_source'])
+                                    if cmd.get('path'):
+                                        delete[resource].extend(cmd['path'])
+                                if resource == 'TMS_SUBSCRIPTION':
+                                    cmd = {}
+                                    if item.get('destination_group'):
+                                        cmd['destination_group'] = ['no ' + setval['destination_group'].format(item['destination_group'])]
+                                    if item.get('sensor_group'):
+                                        cmd['sensor_group'] = ['no ' + setval['sensor_group'].format(**item['sensor_group'])]
+                                    delete[resource].extend(global_ctx)
+                                    if property_ctx[0] not in delete[resource]:
+                                        delete[resource].extend(property_ctx)
+                                    if cmd.get('destination_group'):
+                                        delete[resource].extend(cmd['destination_group'])
+                                    if cmd.get('sensor_group'):
+                                        delete[resource].extend(cmd['sensor_group'])
+
+            add[resource] = remove_duplicate_context(add[resource])
+            delete[resource] = remove_duplicate_context(delete[resource])
+
+        commands.extend(delete['TMS_SUBSCRIPTION'])
+        commands.extend(delete['TMS_SENSORGROUP'])
+        commands.extend(delete['TMS_DESTGROUP'])
+        commands.extend(add['TMS_DESTGROUP'])
+        commands.extend(add['TMS_SENSORGROUP'])
+        commands.extend(add['TMS_SUBSCRIPTION'])
+        commands.extend(add['TMS_GLOBAL'])
+        commands = remove_duplicate_context(commands)
+
         return commands
 
     @staticmethod

lib/ansible/module_utils/network/nxos/nxos.py

@@ -739,13 +739,14 @@ class NxosCmdRef:
           multiplier: 3
     """
 
-    def __init__(self, module, cmd_ref_str):
+    def __init__(self, module, cmd_ref_str, ref_only=False):
         """Initialize cmd_ref from yaml data."""
+
         self._module = module
         self._check_imports()
         self._yaml_load(cmd_ref_str)
         self.cache_existing = None
-        self.present_states = ['present', 'merged']
+        self.present_states = ['present', 'merged', 'replaced']
         self.absent_states = ['absent', 'deleted']
         ref = self._ref
 
@@ -754,10 +755,12 @@ class NxosCmdRef:
         ref['_proposed'] = []
         ref['_context'] = []
         ref['_resource_key'] = None
-        ref['_state'] = module.params.get('state', 'present')
-        self.feature_enable()
-        self.get_platform_defaults()
-        self.normalize_defaults()
+
+        if not ref_only:
+            ref['_state'] = module.params.get('state', 'present')
+            self.feature_enable()
+            self.get_platform_defaults()
+            self.normalize_defaults()
 
     def __getitem__(self, key=None):
         if key is None:
@@ -1116,6 +1119,8 @@ class NxosCmdRef:
 
             # Multiple Instances:
             if isinstance(existing, dict) and multiple:
+                item_found = False
+
                 for ekey, evalue in existing.items():
                     if isinstance(evalue, dict):
                         # Remove values set to string 'None' from dvalue
@@ -1148,7 +1153,8 @@ class NxosCmdRef:
 
         # Remove any duplicate commands before returning.
         # pylint: disable=unnecessary-lambda
-        return sorted(set(proposed), key=lambda x: proposed.index(x))
+        cmds = sorted(set(proposed), key=lambda x: proposed.index(x))
+        return cmds
 
 
 def nxosCmdRef_import_check():

lib/ansible/module_utils/network/nxos/utils/telemetry/telemetry.py

@@ -10,6 +10,7 @@ from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
 import re
+from copy import deepcopy
 
 
 def get_module_params_subsection(module_params, tms_config, resource_key=None):
@@ -155,7 +156,7 @@ def remove_duplicate_context(cmds):
         return remove_duplicate_context(cmds)
 
 
-def get_setval_path(module):
+def get_setval_path(module_or_path_data):
     ''' Build setval for path parameter based on playbook inputs
         Full Command:
           - path {name} depth {depth} query-condition {query_condition} filter-condition {filter_condition}
@@ -166,16 +167,84 @@ def get_setval_path(module):
           - query-condition {query_condition},
           - filter-condition {filter_condition}
     '''
-    path = module.params['config']['sensor_groups'][0].get('path')
+    if isinstance(module_or_path_data, dict):
+        path = module_or_path_data
+    else:
+        path = module_or_path_data.params['config']['sensor_groups'][0].get('path')
     if path is None:
         return path
 
     setval = 'path {name}'
     if 'depth' in path.keys():
-        setval = setval + ' depth {depth}'
+        if path.get('depth') != 'None':
+            setval = setval + ' depth {depth}'
     if 'query_condition' in path.keys():
-        setval = setval + ' query-condition {query_condition}'
+        if path.get('query_condition') != 'None':
+            setval = setval + ' query-condition {query_condition}'
     if 'filter_condition' in path.keys():
-        setval = setval + ' filter-condition {filter_condition}'
+        if path.get('filter_condition') != 'None':
+            setval = setval + ' filter-condition {filter_condition}'
 
     return setval
+
+
+def remove_duplicate_commands(commands_list):
+    # Remove any duplicate commands.
+    # pylint: disable=unnecessary-lambda
+    return sorted(set(commands_list), key=lambda x: commands_list.index(x))
+
+
+def massage_data(have_or_want):
+    # Massage non global into a data structure that is indexed by id and
+    # normalized for destination_groups, sensor_groups and subscriptions.
+    data = deepcopy(have_or_want)
+    massaged = {}
+    massaged['destination_groups'] = {}
+    massaged['sensor_groups'] = {}
+    massaged['subscriptions'] = {}
+    from pprint import pprint
+    for subgroup in ['destination_groups', 'sensor_groups', 'subscriptions']:
+        for item in data.get(subgroup, []):
+            id = str(item.get('id'))
+            if id not in massaged[subgroup].keys():
+                massaged[subgroup][id] = []
+            item.pop('id')
+            if not item:
+                item = None
+            else:
+                if item.get('destination'):
+                    if item.get('destination').get('port'):
+                        item['destination']['port'] = str(item['destination']['port'])
+                    if item.get('destination').get('protocol'):
+                        item['destination']['protocol'] = item['destination']['protocol'].lower()
+                    if item.get('destination').get('encoding'):
+                        item['destination']['encoding'] = item['destination']['encoding'].lower()
+                if item.get('path'):
+                    for key in ['filter_condition', 'query_condition', 'depth']:
+                        if item.get('path').get(key) == 'None':
+                            del item['path'][key]
+                    if item.get('path').get('depth') is not None:
+                        item['path']['depth'] = str(item['path']['depth'])
+                if item.get('destination_group'):
+                    item['destination_group'] = str(item['destination_group'])
+                if item.get('sensor_group'):
+                    if item.get('sensor_group').get('id'):
+                        item['sensor_group']['id'] = str(item['sensor_group']['id'])
+                    if item.get('sensor_group').get('sample_interval'):
+                        item['sensor_group']['sample_interval'] = str(item['sensor_group']['sample_interval'])
+                if item.get('destination_group') and item.get('sensor_group'):
+                    item_copy = deepcopy(item)
+                    del item_copy['sensor_group']
+                    del item['destination_group']
+                    massaged[subgroup][id].append(item_copy)
+                    massaged[subgroup][id].append(item)
+                    continue
+                if item.get('path') and item.get('data_source'):
+                    item_copy = deepcopy(item)
+                    del item_copy['data_source']
+                    del item['path']
+                    massaged[subgroup][id].append(item_copy)
+                    massaged[subgroup][id].append(item)
+                    continue
+            massaged[subgroup][id].append(item)
+    return massaged

test/integration/targets/nxos_telemetry/tests/common/replaced.yaml

@@ -0,0 +1,190 @@
+---
+- debug: msg="START connection={{ ansible_connection }} nxos_telemetry replaced sanity test"
+
+- set_fact: source_interface="Loopback55"
+  when: imagetag and (major_version is version_compare('9.1', 'ge'))
+
+- set_fact: command_list_length=27
+- set_fact: command_list_length=28
+  when: imagetag and (major_version is version_compare('9.1', 'ge'))
+
+- set_fact: dict_facts_length=6
+- set_fact: dict_facts_length=7
+  when: imagetag and (major_version is version_compare('9.1', 'ge'))
+
+- name: Setup - disable feature telemetry
+  nxos_feature: &setup_teardown
+    feature: telemetry
+    state: disabled
+  ignore_errors: yes
+
+- name: Setup - enable feature telemetry
+  nxos_feature:
+    feature: telemetry
+    state: enabled
+
+- name: Setup - add initial telemetry config
+  cli_config:
+    config: |
+      telemetry
+        certificate test_cert host.example.com
+        destination-profile
+          use-vrf blue
+          use-compression gzip
+        destination-group 2
+          ip address 192.168.0.1 port 50001 protocol gRPC encoding GPB
+          ip address 192.168.0.2 port 60001 protocol gRPC encoding GPB
+        destination-group 10
+          ip address 192.168.0.1 port 50001 protocol gRPC encoding GPB
+          ip address 192.168.0.2 port 60001 protocol gRPC encoding GPB
+          ip address 192.168.1.1 port 55 protocol HTTP encoding JSON
+          ip address 192.168.1.2 port 100 protocol gRPC encoding GPB
+        destination-group 99
+        sensor-group 2
+          data-source NX-API
+          path sys/bgp/inst depth unbounded query-condition foo filter-condition foo
+        sensor-group 8
+          data-source NX-API
+          path sys/bgp depth 0 query-condition foo filter-condition foo
+        sensor-group 55
+          data-source DME
+          path sys/bgp/inst/dom-default/peer-[10.10.10.11]/ent-[10.10.10.11] depth 0 query-condition foo filter-condition foo
+          path sys/ospf depth 0 query-condition foo filter-condition or(eq(ethpmPhysIf.operSt,"down"),eq(ethpmPhysIf.operSt,"up"))
+        sensor-group 77
+        subscription 44
+          dst-grp 2
+          dst-grp 10
+          snsr-grp 2 sample-interval 2000
+          snsr-grp 8 sample-interval 2000
+        subscription 55
+          dst-grp 10
+          snsr-grp 55 sample-interval 2000
+        subscription 99
+          dst-grp 2
+          dst-grp 99
+          snsr-grp 8 sample-interval 90000
+          snsr-grp 77 sample-interval 2000
+- name: Setup - add initial source-interface telemetry config
+  cli_config:
+    config: |
+      telemetry
+        destination-profile
+          source-interface loopback55
+  when: imagetag and (major_version is version_compare('9.1', 'ge'))
+
+- block:
+  - name: Gather Telemetry Facts Before Changes
+    nxos_facts: &facts
+      gather_subset:
+        - '!all'
+        - '!min'
+      gather_network_resources:
+        - telemetry
+
+  - name: Telemetry - replaced
+    nxos_telemetry: &replace
+      state: 'replaced'
+      config:
+        certificate:
+          key: /file_dir/new_server.key
+          hostname: newhost.example.com
+        vrf: management
+        compression: gzip
+        destination_groups:
+          - id: 2
+            destination:
+              ip: 192.168.0.1
+              port: 65001
+              protocol: grpc
+              encoding: gpb
+          - id: 2
+            destination:
+              ip: 192.168.0.3
+              port: 55001
+              protocol: grpc
+              encoding: gpb
+        sensor_groups:
+          - id: 100
+            data_source: NX-API
+            path:
+              name: sys/bgp/inst
+              depth: unbounded
+              query_condition: foo
+              filter_condition: foo
+        subscriptions:
+          - id: 99
+            destination_group: 2
+            sensor_group:
+              id: 100
+              sample_interval: 2000
+    register: result
+
+  - assert:
+      that:
+        - "result.changed == true"
+        - "result.before|length == {{ dict_facts_length }}"
+        - "result.before.certificate|length == 2"
+        - "result.before.destination_groups|length == 7"
+        - "result.before.sensor_groups|length == 8"
+        - "result.before.subscriptions|length == 10"
+        - "'telemetry' in result.commands"
+        - "'no subscription 55' in result.commands"
+        - "'subscription 99' in result.commands"
+        - "'no dst-grp 99' in result.commands"
+        - "'no snsr-grp 8 sample-interval 90000' in result.commands"
+        - "'no snsr-grp 77 sample-interval 2000' in result.commands"
+        - "'no subscription 44' in result.commands"
+        - "'no sensor-group 55' in result.commands"
+        - "'no sensor-group 8' in result.commands"
+        - "'no sensor-group 2' in result.commands"
+        - "'no sensor-group 77' in result.commands"
+        - "'no destination-group 99' in result.commands"
+        - "'no destination-group 10' in result.commands"
+        - "'destination-group 2' in result.commands"
+        - "'no ip address 192.168.0.1 port 50001 protocol grpc encoding gpb' in result.commands"
+        - "'no ip address 192.168.0.2 port 60001 protocol grpc encoding gpb' in result.commands"
+        - "'destination-group 2' in result.commands"
+        - "'ip address 192.168.0.1 port 65001 protocol grpc encoding gpb' in result.commands"
+        - "'ip address 192.168.0.3 port 55001 protocol grpc encoding gpb' in result.commands"
+        - "'sensor-group 100' in result.commands"
+        - "'path sys/bgp/inst depth unbounded query-condition foo filter-condition foo' in result.commands"
+        - "'data-source NX-API' in result.commands"
+        - "'subscription 99' in result.commands"
+        - "'snsr-grp 100 sample-interval 2000' in result.commands"
+        - "'certificate /file_dir/new_server.key newhost.example.com' in result.commands"
+        - "'destination-profile' in result.commands"
+        - "'use-vrf management' in result.commands"
+        - "result.commands|length == {{ command_list_length }}"
+
+  # Source interface may or may not be included based on the image version.
+  - assert:
+      that:
+        - "'no source-interface loopback55' in result.commands"
+    when: imagetag and (major_version is version_compare('9.1', 'ge'))
+
+  - assert:
+      that:
+        - "(ansible_facts.network_resources.telemetry|dict2items)|symmetric_difference(result.before|dict2items)|length == 0"
+
+  - name: Gather Telemetry Facts After Changes
+    nxos_facts: *facts
+
+  - assert:
+      that:
+        - "(ansible_facts.network_resources.telemetry|dict2items)|symmetric_difference(result.after|dict2items)|length == 0"
+
+  - name: Telemetry - replaced - idempotence
+    nxos_telemetry: *replace
+    register: result
+
+  - assert:
+      that:
+        - "result.changed == false"
+        - "result.commands|length == 0"
+
+  always:
+  - name: Teardown
+    nxos_feature: *setup_teardown
+    ignore_errors: yes
+
+  - debug: msg="END connection={{ ansible_connection }} nxos_telemetry replaced sanity test"

test/units/modules/network/nxos/test_nxos_telemetry.py

@@ -869,12 +869,16 @@ class TestNxosTelemetryModule(TestNxosModule):
                 ],
                 'subscriptions': [
                     {'id': 5,
-                     'destination_group': 55,
-                     'sensor_group': {'id': 1, 'sample_interval': 1000},
+                     'destination_group': 88,
+                     'sensor_group': {'id': 77, 'sample_interval': 1000},
+                     },
+                    {'id': 5,
+                     'destination_group': 99,
+                     'sensor_group': {'id': 77, 'sample_interval': 1000},
                      },
                     {'id': 88,
-                     'destination_group': 3,
-                     'sensor_group': {'id': 4, 'sample_interval': 2000},
+                     'destination_group': 99,
+                     'sensor_group': {'id': 99, 'sample_interval': 2000},
                      },
                 ],
             }
@@ -900,11 +904,12 @@ class TestNxosTelemetryModule(TestNxosModule):
             'data-source DME',
             'path sys/bgp depth 0 query-condition query_condition_xyz filter-condition filter_condition_xyz',
             'subscription 5',
-            'dst-grp 55',
-            'snsr-grp 1 sample-interval 1000',
+            'dst-grp 88',
+            'dst-grp 99',
+            'snsr-grp 77 sample-interval 1000',
             'subscription 88',
-            'dst-grp 3',
-            'snsr-grp 4 sample-interval 2000'
+            'dst-grp 99',
+            'snsr-grp 99 sample-interval 2000'
         ])
 
     def test_telemetry_deleted_input_validation_n9k(self):
@@ -948,6 +953,272 @@ class TestNxosTelemetryModule(TestNxosModule):
         ), ignore_provider_arg)
         self.execute_module(changed=False)
 
+    def test_tms_replaced1_n9k(self):
+        # Assumes feature telemetry is enabled
+        # Modify global config and remove everything else
+        self.execute_show_command.return_value = load_fixture('nxos_telemetry', 'N9K.cfg')
+        self.get_platform_shortname.return_value = 'N9K'
+        set_module_args(dict(
+            state='replaced',
+            config=dict(
+                certificate={'key': '/bootflash/sample.key', 'hostname': 'server.example.com'},
+                compression='gzip',
+                vrf='blue',
+            )
+        ), ignore_provider_arg)
+        self.execute_module(changed=True, commands=[
+            'telemetry',
+            'no subscription 3',
+            'no subscription 4',
+            'no subscription 5',
+            'no subscription 6',
+            'no subscription 7',
+            'no sensor-group 2',
+            'no sensor-group 55',
+            'no sensor-group 56',
+            'no destination-group 2',
+            'no destination-group 10',
+            'certificate /bootflash/sample.key server.example.com',
+            'destination-profile',
+            'no source-interface loopback55',
+            'use-vrf blue'
+        ])
+
+    def test_tms_replaced2_n9k(self):
+        # Assumes feature telemetry is enabled
+        # Remove/default all global config
+        # Modify destination-group 10, add 11 and 99, remove 2
+        # Modify sensor-group 55, 56
+        # remove all subscriptions
+        self.execute_show_command.return_value = load_fixture('nxos_telemetry', 'N9K.cfg')
+        self.get_platform_shortname.return_value = 'N9K'
+        set_module_args({
+            'state': 'replaced',
+            'config': {
+                'destination_groups': [
+                    {'id': 10,
+                     'destination': {'ip': '192.168.1.1', 'port': '5001', 'protocol': 'GRPC', 'encoding': 'GPB'},
+                     },
+                    {'id': 11,
+                     'destination': {'ip': '192.168.1.2', 'port': '6001', 'protocol': 'GRPC', 'encoding': 'GPB'},
+                     },
+                    {'id': 99,
+                     'destination': {'ip': '192.168.1.2', 'port': '6001', 'protocol': 'GRPC', 'encoding': 'GPB'},
+                     },
+                    {'id': '99',
+                     'destination': {'ip': '192.168.1.1', 'port': '5001', 'protocol': 'GRPC', 'encoding': 'GPB'},
+                     },
+                ],
+                'sensor_groups': [
+                    {'id': 55,
+                     'data_source': 'NX-API',
+                     'path': {'name': 'sys/bgp', 'depth': 0, 'query_condition': 'query_condition_xyz', 'filter_condition': 'filter_condition_xyz'},
+                     },
+                    {'id': '56',
+                     'data_source': 'NX-API',
+                     'path': {'name': 'sys/bgp', 'depth': 0, 'query_condition': 'query_condition_xyz', 'filter_condition': 'filter_condition_xyz'},
+                     },
+                ],
+            }
+        }, ignore_provider_arg)
+        self.execute_module(changed=True, commands=[
+            'telemetry',
+            'no subscription 3',
+            'no subscription 5',
+            'no subscription 4',
+            'no subscription 7',
+            'no subscription 6',
+            'sensor-group 56',
+            'no data-source DME',
+            'no path environment',
+            'no path interface',
+            'no path resources',
+            'no path vxlan',
+            'no sensor-group 2',
+            'destination-group 10',
+            'no ip address 192.168.0.1 port 50001 protocol grpc encoding gpb',
+            'no ip address 192.168.0.2 port 60001 protocol grpc encoding gpb',
+            'no destination-group 2',
+            'destination-group 11',
+            'ip address 192.168.1.2 port 6001 protocol grpc encoding gpb',
+            'destination-group 10',
+            'ip address 192.168.1.1 port 5001 protocol grpc encoding gpb',
+            'destination-group 99',
+            'ip address 192.168.1.2 port 6001 protocol grpc encoding gpb',
+            'ip address 192.168.1.1 port 5001 protocol grpc encoding gpb',
+            'sensor-group 55',
+            'data-source NX-API',
+            'path sys/bgp depth 0 query-condition query_condition_xyz filter-condition filter_condition_xyz',
+            'sensor-group 56',
+            'data-source NX-API',
+            'path sys/bgp depth 0 query-condition query_condition_xyz filter-condition filter_condition_xyz',
+            'no certificate /bootflash/server.key localhost',
+            'no destination-profile'
+        ])
+
+    def test_tms_replaced3_n9k(self):
+        # Assumes feature telemetry is enabled
+        # Modify vrf global config, remove default all other global config.
+        # destination-group 2 destination '192.168.0.1' idempotent
+        # destination-group 2 destination '192.168.0.2' remove
+        # remove all other destination-groups
+        # Modify sensor-group 55 and delete all others
+        # Modify subscription 7, add 10 and delete all others
+        self.execute_show_command.return_value = load_fixture('nxos_telemetry', 'N9K.cfg')
+        self.get_platform_shortname.return_value = 'N9K'
+        set_module_args({
+            'state': 'replaced',
+            'config': {
+                'vrf': 'blue',
+                'destination_groups': [
+                    {'id': 2,
+                     'destination': {'ip': '192.168.0.1', 'port': 50001, 'protocol': 'GRPC', 'encoding': 'GPB'},
+                     },
+                ],
+                'sensor_groups': [
+                    {'id': 55,
+                     'data_source': 'NX-API',
+                     'path': {'name': 'sys/bgp', 'depth': 0, 'query_condition': 'query_condition_xyz', 'filter_condition': 'filter_condition_xyz'},
+                     },
+                ],
+                'subscriptions': [
+                    {'id': 7,
+                     'destination_group': 10,
+                     'sensor_group': {'id': 55, 'sample_interval': 1000},
+                     },
+                    {'id': 10,
+                     'destination_group': 2,
+                     'sensor_group': {'id': 55, 'sample_interval': 1000},
+                     },
+                ],
+            }
+        }, ignore_provider_arg)
+        self.execute_module(changed=True, commands=[
+            'telemetry',
+            'no subscription 3',
+            'no subscription 5',
+            'no subscription 4',
+            'subscription 7',
+            'no snsr-grp 2 sample-interval 1000',
+            'no subscription 6',
+            'no sensor-group 56',
+            'no sensor-group 2',
+            'no destination-group 10',
+            'destination-group 2',
+            'no ip address 192.168.0.2 port 60001 protocol grpc encoding gpb',
+            'sensor-group 55',
+            'data-source NX-API',
+            'path sys/bgp depth 0 query-condition query_condition_xyz filter-condition filter_condition_xyz',
+            'subscription 10',
+            'dst-grp 2',
+            'snsr-grp 55 sample-interval 1000',
+            'subscription 7',
+            'snsr-grp 55 sample-interval 1000',
+            'no certificate /bootflash/server.key localhost',
+            'destination-profile',
+            'no use-compression gzip',
+            'no source-interface loopback55',
+            'use-vrf blue'
+        ])
+
+    def test_tms_replaced_idempotent_n9k(self):
+        # Assumes feature telemetry is enabled
+        # Modify vrf global config, remove default all other global config.
+        # destination-group 2 destination '192.168.0.1' idempotent
+        # destination-group 2 destination '192.168.0.2' remove
+        # remove all other destination-groups
+        # Modify sensor-group 55 and delete all others
+        # Modify subscription 7, add 10 and delete all others
+        self.execute_show_command.return_value = load_fixture('nxos_telemetry', 'N9K.cfg')
+        self.get_platform_shortname.return_value = 'N9K'
+        set_module_args({
+            'state': 'replaced',
+            'config': {
+                'certificate': {'key': '/bootflash/server.key', 'hostname': 'localhost'},
+                'compression': 'gzip',
+                'vrf': 'management',
+                'source_interface': 'loopback55',
+                'destination_groups': [
+                    {'id': 2,
+                     'destination': {'ip': '192.168.0.1', 'port': 50001, 'protocol': 'GRPC', 'encoding': 'GPB'},
+                     },
+                    {'id': 2,
+                     'destination': {'ip': '192.168.0.2', 'port': 60001, 'protocol': 'GRPC', 'encoding': 'GPB'},
+                     },
+                    {'id': 10,
+                     'destination': {'ip': '192.168.0.1', 'port': 50001, 'protocol': 'GRPC', 'encoding': 'GPB'},
+                     },
+                    {'id': 10,
+                     'destination': {'ip': '192.168.0.2', 'port': 60001, 'protocol': 'GRPC', 'encoding': 'GPB'},
+                     },
+                ],
+                'sensor_groups': [
+                    {'id': 2,
+                     'data_source': 'DME',
+                     'path': {'name': 'boo', 'depth': 0},
+                     },
+                    {'id': 2,
+                     'path': {'name': 'sys/ospf', 'depth': 0, 'query_condition': 'qc', 'filter_condition': 'fc'},
+                     },
+                    {'id': 2,
+                     'path': {'name': 'interfaces', 'depth': 0},
+                     },
+                    {'id': 2,
+                     'path': {'name': 'sys/bgp'},
+                     },
+                    {'id': 2,
+                     'path': {'name': 'sys/bgp/inst', 'depth': 0, 'query_condition': 'foo', 'filter_condition': 'foo'},
+                     },
+                    {'id': 2,
+                     'path': {'name': 'sys/bgp/inst/dom-default/peer-[10.10.10.11]/ent-[10.10.10.11]'},
+                     },
+                    {'id': 2,
+                     'path': {'name': 'sys/bgp/inst/dom-default/peer-[20.20.20.11]/ent-[20.20.20.11]'},
+                     },
+                    {'id': 2,
+                     'path': {'name': 'too', 'depth': 0, 'filter_condition': 'foo'},
+                     },
+                    {'id': 55},
+                    {'id': 56,
+                     'data_source': 'DME',
+                     },
+                    {'id': 56,
+                     'path': {'name': 'environment'},
+                     },
+                    {'id': 56,
+                     'path': {'name': 'interface'},
+                     },
+                    {'id': 56,
+                     'path': {'name': 'resources'},
+                     },
+                    {'id': 56,
+                     'path': {'name': 'vxlan'},
+                     },
+                ],
+                'subscriptions': [
+                    {'id': 3},
+                    {'id': 4,
+                     'destination_group': 2,
+                     'sensor_group': {'id': 2, 'sample_interval': 1000},
+                     },
+                    {'id': 5,
+                     'destination_group': 2,
+                     },
+                    {'id': 5,
+                     'sensor_group': {'id': 2, 'sample_interval': 1000},
+                     },
+                    {'id': 6,
+                     'destination_group': 10,
+                     },
+                    {'id': 7,
+                     'destination_group': 10,
+                     'sensor_group': {'id': 2, 'sample_interval': 1000},
+                     },
+                ],
+            }
+        }, ignore_provider_arg)
+        self.execute_module(changed=False, commands=[])
+
 
 def build_args(data, type, state=None, check_mode=None):
     if state is None: