nxos_vrf: N6K regression test failures (#63277)

* nxos_vrf: N6K regression test needs 'nv overlay evpn'

* Add delay for vrf creation

* Revert "Add delay for vrf creation"

This reverts commit 8afa5281ff4b3c5a1775fbd40c5acb7ab5ecdf68.

* Add vrf_error_check w/delay to module

* whitespace

* httpapi fixes

* /input/input_data/ - 'input' is a built-in

* shippable retry

lib/ansible/module_utils/network/nxos/nxos.py

@@ -603,6 +603,8 @@ class HttpApi:
             if opts.get('ignore_timeout') and code:
                 responses.append(code)
                 return responses
+            elif opts.get('catch_clierror') and '400' in code:
+                return [code, err]
             elif code and 'no graceful-restart' in err:
                 if 'ISSU/HA will be affected if Graceful Restart is disabled' in err:
                     msg = ['']

lib/ansible/modules/network/nxos/nxos_vrf.py

@@ -457,6 +457,19 @@ def check_declarative_intent_params(want, module, element_spec, result):
                     module.fail_json(msg="Interface %s not configured on vrf %s" % (i, w['name']))
 
 
+def vrf_error_check(module, commands, responses):
+    """Checks for VRF config errors and executes a retry in some circumstances.
+    """
+    pattern = 'ERROR: Deletion of VRF .* in progress'
+    if re.search(pattern, str(responses)):
+        # Allow delay/retry for VRF changes
+        time.sleep(15)
+        responses = load_config(module, commands, opts={'catch_clierror': True})
+        if re.search(pattern, str(responses)):
+            module.fail_json(msg='VRF config (and retry) failure: %s ' % responses)
+        module.warn('VRF config delayed by VRF deletion - passed on retry')
+
+
 def main():
     """ main entry point for module execution
     """
@@ -504,7 +517,8 @@ def main():
     result['commands'] = commands
 
     if commands and not module.check_mode:
-        load_config(module, commands)
+        responses = load_config(module, commands, opts={'catch_clierror': True})
+        vrf_error_check(module, commands, responses)
         result['changed'] = True
 
     check_declarative_intent_params(want, module, element_spec, result)

lib/ansible/plugins/httpapi/nxos.py

@@ -178,7 +178,11 @@ def handle_response(response):
     if response['ins_api'].get('outputs'):
         for output in to_list(response['ins_api']['outputs']['output']):
             if output['code'] != '200':
-                raise ConnectionError('%s: %s' % (output['input'], output['msg']), code=output['code'])
+                # Best effort messages: some API output keys may not exist on some platforms
+                input_data = output.get('input', '')
+                msg = output.get('msg', '')
+                clierror = output.get('clierror', '')
+                raise ConnectionError('%s: %s: %s' % (input_data, msg, clierror), code=output['code'])
             elif 'body' in output:
                 result = output['body']
                 if isinstance(result, dict):

test/integration/targets/nxos_vrf/tests/common/sanity.yaml

@@ -25,6 +25,14 @@
     state: enabled
   ignore_errors: yes
 
+- name: "Setup: Enable nv overlay evpn"
+  nxos_config:
+    commands:
+      - nv overlay evpn
+    provider: "{{ connection }}"
+  ignore_errors: yes
+  when: platform is match("N5K|N6K")
+
 - block:
   - name: Ensure ntc VRF exists on switch
     nxos_vrf: &configure
@@ -91,6 +99,14 @@
   - assert: *false
 
   always:
+  - name: "Setup: Disable nv overlay evpn"
+    nxos_config:
+      commands:
+        - no nv overlay evpn
+      provider: "{{ connection }}"
+    ignore_errors: yes
+    when: platform is match("N5K|N6K")
+
   - name: "Disable feature BGP"
     nxos_feature:
       feature: bgp