Add collection support with local for supported network platforms (#66302)

* Add collection support with local for supported network platforms

*  The legacy behavior of network action plugins using persistent
   the framework is to override the connection=local with the actual connection type (network_cli/netconf/httpapi) based
   on the value of transport option in provider.
*  After the actual connection plugin is identified try to load it from
   `ansible.netcommon` collection, if it fails load it from ansible
   core. This is done to work with Ansible 2.10 and also maintain backward compatibility for
   Ansible 2.9 version to ensure it works with network collections.
*  To support this need to pass collection_list value to connection
   plugins. As in case of connection=local it loads `persistent`
   connection within action plugin which in turn invokes `network_cli`
   or `netconf` connection. Similarly `network_cli` connection plugin invokes
   `terminal` and `cliconf plugins and `netconf` connection plugin invokes `netconf`
   `plugins` based on value on `ansible_network_os`.
*  Add deprecation warning for connection=local support and
   provider support for applicable platforms

* Fix CI failures

changelogs/fragments/supported_network_platforms_local_collection.yml

@@ -0,0 +1,5 @@
+bugfixes:
+- Add support for network connection=local with collections for network supported platforms
+
+deprecated_features:
+- Deprecate connection=local support for network platforms using persistent framework

lib/ansible/module_utils/network/eos/eos.py

@@ -58,7 +58,7 @@ eos_provider_spec = {
     'transport': dict(default='cli', choices=['cli', 'eapi'])
 }
 eos_argument_spec = {
-    'provider': dict(type='dict', options=eos_provider_spec),
+    'provider': dict(type='dict', options=eos_provider_spec, removed_in_version=2.14),
 }
 
 

lib/ansible/module_utils/network/ios/ios.py

@@ -45,7 +45,7 @@ ios_provider_spec = {
     'timeout': dict(type='int')
 }
 ios_argument_spec = {
-    'provider': dict(type='dict', options=ios_provider_spec),
+    'provider': dict(type='dict', options=ios_provider_spec, removed_in_version=2.14),
 }
 
 

lib/ansible/module_utils/network/iosxr/iosxr.py

@@ -81,7 +81,7 @@ iosxr_provider_spec = {
 }
 
 iosxr_argument_spec = {
-    'provider': dict(type='dict', options=iosxr_provider_spec)
+    'provider': dict(type='dict', options=iosxr_provider_spec, removed_in_version=2.14)
 }
 
 command_spec = {

lib/ansible/module_utils/network/junos/junos.py

@@ -55,7 +55,7 @@ junos_provider_spec = {
     'transport': dict(default='netconf', choices=['cli', 'netconf'])
 }
 junos_argument_spec = {
-    'provider': dict(type='dict', options=junos_provider_spec),
+    'provider': dict(type='dict', options=junos_provider_spec, removed_in_version=2.14),
 }
 
 

lib/ansible/module_utils/network/nxos/nxos.py

@@ -81,7 +81,7 @@ nxos_provider_spec = {
     'transport': dict(type='str', default='cli', choices=['cli', 'nxapi'])
 }
 nxos_argument_spec = {
-    'provider': dict(type='dict', options=nxos_provider_spec),
+    'provider': dict(type='dict', options=nxos_provider_spec, removed_in_version=2.14),
 }
 
 

lib/ansible/module_utils/network/vyos/vyos.py

@@ -44,7 +44,7 @@ vyos_provider_spec = {
     'timeout': dict(type='int'),
 }
 vyos_argument_spec = {
-    'provider': dict(type='dict', options=vyos_provider_spec),
+    'provider': dict(type='dict', options=vyos_provider_spec, removed_in_version=2.14),
 }
 
 

lib/ansible/plugins/action/eos.py

@@ -39,6 +39,7 @@ class ActionModule(ActionNetworkModule):
         module_name = self._task.action.split('.')[-1]
         self._config_module = True if module_name == 'eos_config' else False
         persistent_connection = self._play_context.connection.split('.')[-1]
+        warnings = []
 
         if persistent_connection in ('network_cli', 'httpapi'):
             provider = self._task.args.get('provider', {})
@@ -56,8 +57,8 @@ class ActionModule(ActionNetworkModule):
 
             if transport == 'cli':
                 pc = copy.deepcopy(self._play_context)
-                pc.connection = 'network_cli'
-                pc.network_os = 'eos'
+                pc.connection = 'ansible.netcommon.network_cli'
+                pc.network_os = 'arista.eos.eos'
                 pc.remote_addr = provider['host'] or self._play_context.remote_addr
                 pc.port = int(provider['port'] or self._play_context.port or 22)
                 pc.remote_user = provider['username'] or self._play_context.connection_user
@@ -68,8 +69,16 @@ class ActionModule(ActionNetworkModule):
                     pc.become_method = 'enable'
                 pc.become_pass = provider['auth_pass']
 
+                connection = self._shared_loader_obj.connection_loader.get('ansible.netcommon.persistent', pc, sys.stdin,
+                                                                           task_uuid=self._task._uuid)
+
+                # TODO: Remove below code after ansible minimal is cut out
+                if connection is None:
+                    pc.connection = 'network_cli'
+                    pc.network_os = 'eos'
+                    connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin, task_uuid=self._task._uuid)
+
                 display.vvv('using connection plugin %s (was local)' % pc.connection, pc.remote_addr)
-                connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin, task_uuid=self._task._uuid)
 
                 command_timeout = int(provider['timeout']) if provider['timeout'] else connection.get_option('persistent_command_timeout')
                 connection.set_options(direct={'persistent_command_timeout': command_timeout})
@@ -82,13 +91,21 @@ class ActionModule(ActionNetworkModule):
                                    'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
 
                 task_vars['ansible_socket'] = socket_path
-
+                warnings.append(['connection local support for this module is deprecated and will be removed in version 2.14,'
+                                 ' use connection %s' % pc.connection])
             else:
                 self._task.args['provider'] = ActionModule.eapi_implementation(provider, self._play_context)
+                warnings.append(['connection local support for this module is deprecated and will be removed in version 2.14,'
+                                 ' use connection either httpapi or ansible.netcommon.httpapi (whichever is applicable)'])
         else:
             return {'failed': True, 'msg': 'Connection type %s is not valid for this module' % self._play_context.connection}
 
         result = super(ActionModule, self).run(task_vars=task_vars)
+        if warnings:
+            if 'warnings' in result:
+                result['warnings'].extend(warnings)
+            else:
+                result['warnings'] = warnings
         return result
 
     @staticmethod

lib/ansible/plugins/action/ios.py

@@ -38,6 +38,7 @@ class ActionModule(ActionNetworkModule):
         module_name = self._task.action.split('.')[-1]
         self._config_module = True if module_name == 'ios_config' else False
         persistent_connection = self._play_context.connection.split('.')[-1]
+        warnings = []
 
         if persistent_connection == 'network_cli':
             provider = self._task.args.get('provider', {})
@@ -47,8 +48,8 @@ class ActionModule(ActionNetworkModule):
         elif self._play_context.connection == 'local':
             provider = load_provider(ios_provider_spec, self._task.args)
             pc = copy.deepcopy(self._play_context)
-            pc.connection = 'network_cli'
-            pc.network_os = 'ios'
+            pc.connection = 'ansible.netcommon.network_cli'
+            pc.network_os = 'cisco.ios.ios'
             pc.remote_addr = provider['host'] or self._play_context.remote_addr
             pc.port = int(provider['port'] or self._play_context.port or 22)
             pc.remote_user = provider['username'] or self._play_context.connection_user
@@ -59,8 +60,16 @@ class ActionModule(ActionNetworkModule):
                 pc.become_method = 'enable'
             pc.become_pass = provider['auth_pass']
 
+            connection = self._shared_loader_obj.connection_loader.get('ansible.netcommon.persistent', pc, sys.stdin,
+                                                                       task_uuid=self._task._uuid)
+
+            # TODO: Remove below code after ansible minimal is cut out
+            if connection is None:
+                pc.connection = 'network_cli'
+                pc.network_os = 'ios'
+                connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin, task_uuid=self._task._uuid)
+
             display.vvv('using connection plugin %s (was local)' % pc.connection, pc.remote_addr)
-            connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin, task_uuid=self._task._uuid)
 
             command_timeout = int(provider['timeout']) if provider['timeout'] else connection.get_option('persistent_command_timeout')
             connection.set_options(direct={'persistent_command_timeout': command_timeout})
@@ -73,8 +82,14 @@ class ActionModule(ActionNetworkModule):
                                'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
 
             task_vars['ansible_socket'] = socket_path
+            warnings.append(['connection local support for this module is deprecated and will be removed in version 2.14, use connection %s' % pc.connection])
         else:
             return {'failed': True, 'msg': 'Connection type %s is not valid for this module' % self._play_context.connection}
 
         result = super(ActionModule, self).run(task_vars=task_vars)
+        if warnings:
+            if 'warnings' in result:
+                result['warnings'].extend(warnings)
+            else:
+                result['warnings'] = warnings
         return result

lib/ansible/plugins/action/iosxr.py

@@ -39,27 +39,40 @@ class ActionModule(ActionNetworkModule):
         self._config_module = True if module_name == 'iosxr_config' else False
         force_cli = module_name in ('iosxr_netconf', 'iosxr_config', 'iosxr_command', 'iosxr_facts')
         persistent_connection = self._play_context.connection.split('.')[-1]
+        warnings = []
 
         if self._play_context.connection == 'local':
             provider = load_provider(iosxr_provider_spec, self._task.args)
             pc = copy.deepcopy(self._play_context)
+            pc.network_os = 'cisco.iosxr.iosxr'
             if force_cli or provider['transport'] == 'cli':
-                pc.connection = 'network_cli'
+                pc.connection = 'ansible.netcommon.network_cli'
                 pc.port = int(provider['port'] or self._play_context.port or 22)
             elif provider['transport'] == 'netconf':
-                pc.connection = 'netconf'
+                pc.connection = 'ansible.netcommon.netconf'
                 pc.port = int(provider['port'] or self._play_context.port or 830)
             else:
                 return {'failed': True, 'msg': 'Transport type %s is not valid for this module' % provider['transport']}
 
-            pc.network_os = 'iosxr'
             pc.remote_addr = provider['host'] or self._play_context.remote_addr
             pc.port = int(provider['port'] or self._play_context.port or 22)
             pc.remote_user = provider['username'] or self._play_context.connection_user
             pc.password = provider['password'] or self._play_context.password
 
+            connection = self._shared_loader_obj.connection_loader.get('ansible.netcommon.persistent', pc, sys.stdin,
+                                                                       task_uuid=self._task._uuid)
+
+            # TODO: Remove below code after ansible minimal is cut out
+            if connection is None:
+                pc.network_os = 'iosxr'
+                if pc.connection.split('.')[-1] == 'netconf':
+                    pc.connection = 'netconf'
+                else:
+                    pc.connection = 'network_cli'
+
+                connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin, task_uuid=self._task._uuid)
+
             display.vvv('using connection plugin %s (was local)' % pc.connection, pc.remote_addr)
-            connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin, task_uuid=self._task._uuid)
 
             command_timeout = int(provider['timeout']) if provider['timeout'] else connection.get_option('persistent_command_timeout')
             connection.set_options(direct={'persistent_command_timeout': command_timeout})
@@ -72,6 +85,7 @@ class ActionModule(ActionNetworkModule):
                                'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
 
             task_vars['ansible_socket'] = socket_path
+            warnings.append(['connection local support for this module is deprecated and will be removed in version 2.14, use connection %s' % pc.connection])
         elif persistent_connection in ('netconf', 'network_cli'):
             if force_cli and persistent_connection != 'network_cli':
                 return {'failed': True, 'msg': 'Connection type %s is not valid for module %s' %
@@ -84,4 +98,9 @@ class ActionModule(ActionNetworkModule):
             return {'failed': True, 'msg': 'Connection type %s is not valid for this module' % self._play_context.connection}
 
         result = super(ActionModule, self).run(task_vars=task_vars)
+        if warnings:
+            if 'warnings' in result:
+                result['warnings'].extend(warnings)
+            else:
+                result['warnings'] = warnings
         return result

lib/ansible/plugins/action/junos.py

@@ -40,11 +40,12 @@ class ActionModule(ActionNetworkModule):
         module_name = self._task.action.split('.')[-1]
         self._config_module = True if module_name == 'junos_config' else False
         persistent_connection = self._play_context.connection.split('.')[-1]
+        warnings = []
 
         if self._play_context.connection == 'local':
             provider = load_provider(junos_provider_spec, self._task.args)
             pc = copy.deepcopy(self._play_context)
-            pc.network_os = 'junos'
+            pc.network_os = 'junipernetworks.junos.junos'
             pc.remote_addr = provider['host'] or self._play_context.remote_addr
 
             if provider['transport'] == 'cli' and module_name not in CLI_SUPPORTED_MODULES:
@@ -53,18 +54,30 @@ class ActionModule(ActionNetworkModule):
                                                % (provider['transport'], module_name)}
 
             if module_name == 'junos_netconf' or (provider['transport'] == 'cli' and module_name == 'junos_command'):
-                pc.connection = 'network_cli'
+                pc.connection = 'ansible.netcommon.network_cli'
                 pc.port = int(provider['port'] or self._play_context.port or 22)
             else:
-                pc.connection = 'netconf'
+                pc.connection = 'ansible.netcommon.netconf'
                 pc.port = int(provider['port'] or self._play_context.port or 830)
 
             pc.remote_user = provider['username'] or self._play_context.connection_user
             pc.password = provider['password'] or self._play_context.password
             pc.private_key_file = provider['ssh_keyfile'] or self._play_context.private_key_file
 
+            connection = self._shared_loader_obj.connection_loader.get('ansible.netcommon.persistent', pc, sys.stdin,
+                                                                       task_uuid=self._task._uuid)
+
+            # TODO: Remove below code after ansible minimal is cut out
+            if connection is None:
+                pc.network_os = 'junos'
+                if pc.connection.split('.')[-1] == 'netconf':
+                    pc.connection = 'netconf'
+                else:
+                    pc.connection = 'network_cli'
+
+                connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin, task_uuid=self._task._uuid)
+
             display.vvv('using connection plugin %s (was local)' % pc.connection, pc.remote_addr)
-            connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin, task_uuid=self._task._uuid)
 
             command_timeout = int(provider['timeout']) if provider['timeout'] else connection.get_option('persistent_command_timeout')
             connection.set_options(direct={'persistent_command_timeout': command_timeout})
@@ -77,6 +90,7 @@ class ActionModule(ActionNetworkModule):
                                'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
 
             task_vars['ansible_socket'] = socket_path
+            warnings.append(['connection local support for this module is deprecated and will be removed in version 2.14, use connection %s' % pc.connection])
         elif persistent_connection in ('netconf', 'network_cli'):
             provider = self._task.args.get('provider', {})
             if any(provider.values()):
@@ -93,4 +107,9 @@ class ActionModule(ActionNetworkModule):
                                                % (self._play_context.connection, module_name)}
 
         result = super(ActionModule, self).run(task_vars=task_vars)
+        if warnings:
+            if 'warnings' in result:
+                result['warnings'].extend(warnings)
+            else:
+                result['warnings'] = warnings
         return result

lib/ansible/plugins/action/netconf.py

@@ -36,6 +36,7 @@ class ActionModule(ActionNetworkModule):
         module_name = self._task.action.split('.')[-1]
         self._config_module = True if module_name == 'netconf_config' else False
         persistent_connection = self._play_context.connection.split('.')[-1]
+        warnings = []
 
         if persistent_connection not in ['netconf', 'local'] and module_name == 'netconf_config':
             return {'failed': True, 'msg': 'Connection type %s is not valid for netconf_config module. '
@@ -47,15 +48,22 @@ class ActionModule(ActionNetworkModule):
         if self._play_context.connection == 'local' and module_name == 'netconf_config':
             args = self._task.args
             pc = copy.deepcopy(self._play_context)
-            pc.connection = 'netconf'
+            pc.connection = 'ansible.netcommon.netconf'
             pc.port = int(args.get('port') or self._play_context.port or 830)
 
             pc.remote_user = args.get('username') or self._play_context.connection_user
             pc.password = args.get('password') or self._play_context.password
             pc.private_key_file = args.get('ssh_keyfile') or self._play_context.private_key_file
 
+            connection = self._shared_loader_obj.connection_loader.get('ansible.netcommon.persistent', pc, sys.stdin,
+                                                                       task_uuid=self._task._uuid)
+
+            # TODO: Remove below code after ansible minimal is cut out
+            if connection is None:
+                pc.connection = 'netconf'
+                connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin, task_uuid=self._task._uuid)
+
             display.vvv('using connection plugin %s (was local)' % pc.connection, pc.remote_addr)
-            connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin, task_uuid=self._task._uuid)
 
             timeout = args.get('timeout')
             command_timeout = int(timeout) if timeout else connection.get_option('persistent_command_timeout')
@@ -71,5 +79,12 @@ class ActionModule(ActionNetworkModule):
                                'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
 
             task_vars['ansible_socket'] = socket_path
+            warnings.append(['connection local support for this module is deprecated and will be removed in version 2.14, use connection %s' % pc.connection])
 
-        return super(ActionModule, self).run(task_vars=task_vars)
+        result = super(ActionModule, self).run(task_vars=task_vars)
+        if warnings:
+            if 'warnings' in result:
+                result['warnings'].extend(warnings)
+            else:
+                result['warnings'] = warnings
+        return result

lib/ansible/plugins/action/nxos.py

@@ -40,6 +40,7 @@ class ActionModule(ActionNetworkModule):
         module_name = self._task.action.split('.')[-1]
         self._config_module = True if module_name == 'nxos_config' else False
         persistent_connection = self._play_context.connection.split('.')[-1]
+        warnings = []
 
         if (self._play_context.connection in ('httpapi', 'local') or self._task.args.get('provider', {}).get('transport') == 'nxapi') \
                 and module_name in ('nxos_file_copy', 'nxos_nxapi'):
@@ -88,8 +89,8 @@ class ActionModule(ActionNetworkModule):
 
             if transport == 'cli':
                 pc = copy.deepcopy(self._play_context)
-                pc.connection = 'network_cli'
-                pc.network_os = 'nxos'
+                pc.connection = 'ansible.netcommon.network_cli'
+                pc.network_os = 'cisco.nxos.nxos'
                 pc.remote_addr = provider['host'] or self._play_context.remote_addr
                 pc.port = int(provider['port'] or self._play_context.port or 22)
                 pc.remote_user = provider['username'] or self._play_context.connection_user
@@ -100,8 +101,16 @@ class ActionModule(ActionNetworkModule):
                     pc.become_method = 'enable'
                 pc.become_pass = provider['auth_pass']
 
+                connection = self._shared_loader_obj.connection_loader.get('ansible.netcommon.persistent', pc, sys.stdin,
+                                                                           task_uuid=self._task._uuid)
+
+                # TODO: Remove below code after ansible minimal is cut out
+                if connection is None:
+                    pc.connection = 'network_cli'
+                    pc.network_os = 'nxos'
+                    connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin, task_uuid=self._task._uuid)
+
                 display.vvv('using connection plugin %s (was local)' % pc.connection, pc.remote_addr)
-                connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin, task_uuid=self._task._uuid)
 
                 command_timeout = int(provider['timeout']) if provider['timeout'] else connection.get_option('persistent_command_timeout')
                 connection.set_options(direct={'persistent_command_timeout': command_timeout})
@@ -117,10 +126,17 @@ class ActionModule(ActionNetworkModule):
 
             else:
                 self._task.args['provider'] = ActionModule.nxapi_implementation(provider, self._play_context)
+                warnings.append(['connection local support for this module is deprecated and will be removed in version 2.14,'
+                                 ' use connection either httpapi or ansible.netcommon.httpapi (whichever is applicable)'])
         else:
             return {'failed': True, 'msg': 'Connection type %s is not valid for this module' % self._play_context.connection}
 
         result = super(ActionModule, self).run(task_vars=task_vars)
+        if warnings:
+            if 'warnings' in result:
+                result['warnings'].extend(warnings)
+            else:
+                result['warnings'] = warnings
         return result
 
     @staticmethod

lib/ansible/plugins/action/vyos.py

@@ -38,6 +38,7 @@ class ActionModule(ActionNetworkModule):
         module_name = self._task.action.split('.')[-1]
         self._config_module = True if module_name == 'vyos_config' else False
         persistent_connection = self._play_context.connection.split('.')[-1]
+        warnings = []
 
         if persistent_connection == 'network_cli':
             provider = self._task.args.get('provider', {})
@@ -47,16 +48,24 @@ class ActionModule(ActionNetworkModule):
         elif self._play_context.connection == 'local':
             provider = load_provider(vyos_provider_spec, self._task.args)
             pc = copy.deepcopy(self._play_context)
-            pc.connection = 'network_cli'
-            pc.network_os = 'vyos'
+            pc.connection = 'ansible.netcommon.network_cli'
+            pc.network_os = 'vyos.vyos.vyos'
             pc.remote_addr = provider['host'] or self._play_context.remote_addr
             pc.port = int(provider['port'] or self._play_context.port or 22)
             pc.remote_user = provider['username'] or self._play_context.connection_user
             pc.password = provider['password'] or self._play_context.password
             pc.private_key_file = provider['ssh_keyfile'] or self._play_context.private_key_file
 
+            connection = self._shared_loader_obj.connection_loader.get('ansible.netcommon.persistent', pc, sys.stdin,
+                                                                       task_uuid=self._task._uuid)
+
+            # TODO: Remove below code after ansible minimal is cut out
+            if connection is None:
+                pc.connection = 'network_cli'
+                pc.network_os = 'vyos'
+                connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin, task_uuid=self._task._uuid)
+
             display.vvv('using connection plugin %s (was local)' % pc.connection, pc.remote_addr)
-            connection = self._shared_loader_obj.connection_loader.get('persistent', pc, sys.stdin, task_uuid=self._task._uuid)
 
             command_timeout = int(provider['timeout']) if provider['timeout'] else connection.get_option('persistent_command_timeout')
             connection.set_options(direct={'persistent_command_timeout': command_timeout})
@@ -69,8 +78,14 @@ class ActionModule(ActionNetworkModule):
                                'https://docs.ansible.com/ansible/network_debug_troubleshooting.html#unable-to-open-shell'}
 
             task_vars['ansible_socket'] = socket_path
+            warnings.append(['connection local support for this module is deprecated and will be removed in version 2.14, use connection %s' % pc.connection])
         else:
             return {'failed': True, 'msg': 'Connection type %s is not valid for this module' % self._play_context.connection}
 
         result = super(ActionModule, self).run(task_vars=task_vars)
+        if warnings:
+            if 'warnings' in result:
+                result['warnings'].extend(warnings)
+            else:
+                result['warnings'] = warnings
         return result