Fix netconf plugin related to collections (#65718)

* Fix netconf plugin related to collections

Fixes #65655 (partly)

*  Make netconf plugins configurable so that the
   information of ncclient device handler
   for give platform resides in the platform
   specific netconf plugin.
*  If the device handler value in ncclient is
   different from the ansible_network_os value
   the right value of `ncclient_device_handler`
   should be set in the plugin documentation.

* Fix review comments

* Fix CI issue

* Fix review comment

(cherry picked from commit 1cfab26fab)

changelogs/fragments/netconf_plugin_device_handler.yml

@@ -0,0 +1,2 @@
+bugfixes:
+- Make netconf plugin configurable to set ncclient device handler name in netconf plugin (https://github.com/ansible/ansible/pull/65718)

docs/docsite/Makefile

@@ -27,7 +27,7 @@ ifdef PLUGINS
 	PLUGIN_ARGS = -l $(PLUGINS)
 endif
 
-DOC_PLUGINS ?= become cache callback cliconf connection httpapi inventory lookup shell strategy vars
+DOC_PLUGINS ?= become cache callback cliconf connection httpapi inventory lookup netconf shell strategy vars
 
 assertrst:
 ifndef rst

docs/docsite/rst/plugins/netconf.rst

@@ -0,0 +1,62 @@
+.. _netconf_plugins:
+
+Netconf Plugins
+===============
+
+.. contents::
+   :local:
+   :depth: 2
+
+Netconf plugins are abstactions over the Netconf interface to network devices. They provide a standard interface
+for Ansible to execute tasks on those network devices.
+
+These plugins generally correspond one-to-one to network device platforms. The appropriate netconf plugin will
+thus be automatically loaded based on the ``ansible_network_os`` variable. If the platform supports standard
+Netconf implementation as defined in the Netconf RFC specification the ``default`` netconf plugin will be used.
+In case if the platform supports propriety Netconf RPC's in that case the interface can be defined in platform
+specific netconf plugin.
+
+.. _enabling_netconf:
+
+Adding netconf plugins
+-------------------------
+
+You can extend Ansible to support other network devices by dropping a custom plugin into the ``netconf_plugins`` directory.
+
+.. _using_netconf:
+
+Using netconf plugins
+------------------------
+
+The netconf plugin to use is determined automatically from the ``ansible_network_os`` variable. There should be no reason to override this functionality.
+
+Most netconf plugins can operate without configuration. A few have additional options that can be set to impact how
+tasks are translated into netconf commands. A ncclient device specific handler name can be set in the netconf plugin
+or else the value of ``default`` is used as per ncclient device handler.
+
+
+Plugins are self-documenting. Each plugin should document its configuration options.
+
+.. _netconf_plugin_list:
+
+Plugin list
+-----------
+
+You can use ``ansible-doc -t netconf -l`` to see the list of available plugins.
+Use ``ansible-doc -t netconf <plugin name>`` to see detailed documentation and examples.
+
+
+.. toctree:: :maxdepth: 1
+    :glob:
+
+    netconf/*
+
+
+.. seealso::
+
+   :ref:`Ansible for Network Automation<network_guide>`
+       An overview of using Ansible to automate networking devices.
+   `User Mailing List <https://groups.google.com/group/ansible-devel>`_
+       Have a question?  Stop by the google group!
+   `irc.freenode.net <http://irc.freenode.net>`_
+       #ansible-network IRC chat channel

docs/docsite/rst/plugins/plugins.rst

@@ -22,6 +22,7 @@ This section covers the various types of plugins that are included with Ansible:
    httpapi
    inventory
    lookup
+   netconf
    shell
    strategy
    vars

lib/ansible/constants.py

@@ -102,7 +102,7 @@ DEFAULT_PASSWORD_CHARS = to_text(ascii_letters + digits + ".,:-_", errors='stric
 DEFAULT_REMOTE_PASS = None
 DEFAULT_SUBSET = None
 # FIXME: expand to other plugins, but never doc fragments
-CONFIGURABLE_PLUGINS = ('become', 'cache', 'callback', 'cliconf', 'connection', 'httpapi', 'inventory', 'lookup', 'shell')
+CONFIGURABLE_PLUGINS = ('become', 'cache', 'callback', 'cliconf', 'connection', 'httpapi', 'inventory', 'lookup', 'netconf', 'shell')
 # NOTE: always update the docs/docsite/Makefile to match
 DOCUMENTABLE_PLUGINS = CONFIGURABLE_PLUGINS + ('module', 'strategy', 'vars')
 IGNORE_FILES = ("COPYING", "CONTRIBUTING", "LICENSE", "README", "VERSION", "GUIDELINES")  # ignore during module search

lib/ansible/plugins/connection/netconf.py

@@ -204,14 +204,6 @@ except (ImportError, AttributeError) as err:  # paramiko and gssapi are incompat
 
 logging.getLogger('ncclient').setLevel(logging.INFO)
 
-NETWORK_OS_DEVICE_PARAM_MAP = {
-    "nxos": "nexus",
-    "ios": "default",
-    "dellos10": "default",
-    "sros": "alu",
-    "ce": "huawei"
-}
-
 
 class Connection(NetworkConnectionBase):
     """NetConf connections"""
@@ -226,17 +218,17 @@ class Connection(NetworkConnectionBase):
         # This will be used to trigger the the use of guess_network_os when connecting.
         self._network_os = self._network_os or 'auto'
 
-        netconf = netconf_loader.get(self._network_os, self)
+        self.netconf = netconf_loader.get(self._network_os, self)
-        if netconf:
+        if self.netconf:
-            self._sub_plugin = {'type': 'netconf', 'name': netconf._load_name, 'obj': netconf}
+            self._sub_plugin = {'type': 'netconf', 'name': self.netconf._load_name, 'obj': self.netconf}
             self.queue_message('vvvv', 'loaded netconf plugin %s from path %s for network_os %s' %
-                               (netconf._load_name, netconf._original_path, self._network_os))
+                               (self.netconf._load_name, self.netconf._original_path, self._network_os))
         else:
-            netconf = netconf_loader.get("default", self)
+            self.netconf = netconf_loader.get("default", self)
-            self._sub_plugin = {'type': 'netconf', 'name': 'default', 'obj': netconf}
+            self._sub_plugin = {'type': 'netconf', 'name': 'default', 'obj': self.netconf}
             self.queue_message('display', 'unable to load netconf plugin for network_os %s, falling back to default plugin' % self._network_os)
-        self.queue_message('log', 'network_os is set to %s' % self._network_os)
 
+        self.queue_message('log', 'network_os is set to %s' % self._network_os)
         self._manager = None
         self.key_filename = None
         self._ssh_config = None
@@ -305,8 +297,12 @@ class Connection(NetworkConnectionBase):
             # Network os not discovered. Set it to default
             self.queue_message('vvv', 'Unable to discover network_os. Falling back to default.')
             self._network_os = 'default'
-
+        try:
-        device_params = {'name': NETWORK_OS_DEVICE_PARAM_MAP.get(self._network_os) or self._network_os}
+            ncclient_device_handler = self.netconf.get_option('ncclient_device_handler')
+        except KeyError:
+            ncclient_device_handler = 'default'
+        self.queue_message('vvv', 'identified ncclient device handler: %s.' % ncclient_device_handler)
+        device_params = {'name': ncclient_device_handler}
 
         try:
             port = self._play_context.port or 830

lib/ansible/plugins/netconf/__init__.py

@@ -102,9 +102,11 @@ class NetconfBase(AnsiblePlugin):
             conn.load_configuration(config=[''set system ntp server 1.1.1.1''], action='set', format='text')
     """
 
-    __rpc__ = ['get_config', 'edit_config', 'get_capabilities', 'get']
+    __rpc__ = ['rpc', 'get_config', 'get', 'edit_config', 'validate', 'copy_config', 'dispatch', 'lock', 'unlock',
+               'discard_changes', 'commit', 'get_schema', 'delete_config', 'get_device_operations']
 
     def __init__(self, connection):
+        super(NetconfBase, self).__init__()
         self._connection = connection
 
     @property

lib/ansible/plugins/netconf/ce.py

@@ -19,6 +19,23 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+---
+netconf: ce
+short_description: Use ce netconf plugin to run netconf commands on Huawei Cloudengine platform
+description:
+  - This ce plugin provides low level abstraction apis for
+    sending and receiving netconf commands from Huawei Cloudengine network devices.
+version_added: "2.9"
+options:
+  ncclient_device_handler:
+    type: str
+    default: huawei
+    description:
+      - Specifies the ncclient device handler name for Huawei Cloudengine.
+        To identify the ncclient device handler name refer ncclient library documentation.
+"""
+
 import json
 import re
 
@@ -101,9 +118,8 @@ class Netconf(NetconfBase):
 
     def get_capabilities(self):
         result = dict()
-        result['rpc'] = self.get_base_rpc() + ['commit', 'discard_changes', 'validate', 'lock', 'unlock', 'copy_copy',
+        result['rpc'] = self.get_base_rpc() + ['execute_rpc', 'load_configuration', 'get_configuration', 'compare_configuration',
-                                               'execute_rpc', 'load_configuration', 'get_configuration', 'command',
+                                               'execute_action', 'halt', 'reboot', 'execute_nc_cli', 'dispatch_rpc']
-                                               'reboot', 'halt']
         result['network_api'] = 'netconf'
         result['device_info'] = self.get_device_info()
         result['server_capabilities'] = [c for c in self.m.server_capabilities]

lib/ansible/plugins/netconf/default.py

@@ -19,6 +19,24 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+---
+author: Ansible Networking Team
+netconf: default
+short_description: Use default netconf plugin to run standard netconf commands as per RFC
+description:
+  - This default plugin provides low level abstraction apis for
+    sending and receiving netconf commands as per Netconf RFC specification.
+version_added: "2.9"
+options:
+  ncclient_device_handler:
+    type: str
+    default: default
+    description:
+      - Specifies the ncclient device handler name for network os that support default netconf
+        implementation as per Netconf RFC specification. To identify the ncclient device handler
+        name refer ncclient library documentation.
+"""
 import json
 
 from ansible.module_utils._text import to_text, to_bytes
@@ -40,9 +58,7 @@ class Netconf(NetconfBase):
 
     def get_capabilities(self):
         result = dict()
-        result['rpc'] = self.get_base_rpc() + ['commit', 'discard_changes', 'validate', 'lock', 'unlock', 'copy_copy',
+        result['rpc'] = self.get_base_rpc()
-                                               'execute_rpc', 'load_configuration', 'get_configuration', 'command',
-                                               'reboot', 'halt']
         result['network_api'] = 'netconf'
         result['device_info'] = self.get_device_info()
         result['server_capabilities'] = [c for c in self.m.server_capabilities]

lib/ansible/plugins/netconf/iosxr.py

@@ -20,6 +20,24 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+---
+author: Ansible Networking Team
+netconf: iosxr
+short_description: Use iosxr netconf plugin to run netconf commands on Cisco IOSXR platform
+description:
+  - This iosxr plugin provides low level abstraction apis for
+    sending and receiving netconf commands from Cisco iosxr network devices.
+version_added: "2.9"
+options:
+  ncclient_device_handler:
+    type: str
+    default: iosxr
+    description:
+      - Specifies the ncclient device handler name for Cisco iosxr network os. To
+        identify the ncclient device handler name refer ncclient library documentation.
+"""
+
 import json
 import re
 import collections
@@ -75,7 +93,7 @@ class Netconf(NetconfBase):
 
     def get_capabilities(self):
         result = dict()
-        result['rpc'] = self.get_base_rpc() + ['commit', 'discard_changes', 'validate', 'lock', 'unlock', 'get-schema']
+        result['rpc'] = self.get_base_rpc()
         result['network_api'] = 'netconf'
         result['device_info'] = self.get_device_info()
         result['server_capabilities'] = [c for c in self.m.server_capabilities]

lib/ansible/plugins/netconf/junos.py

@@ -19,6 +19,24 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+---
+author: Ansible Networking Team
+netconf: junos
+short_description: Use junos netconf plugin to run netconf commands on Juniper JUNOS platform
+description:
+  - This junos plugin provides low level abstraction apis for
+    sending and receiving netconf commands from Juniper JUNOS network devices.
+version_added: "2.9"
+options:
+  ncclient_device_handler:
+    type: str
+    default: junos
+    description:
+      - Specifies the ncclient device handler name for Juniper junos network os. To
+        identify the ncclient device handler name refer ncclient library documentation.
+"""
+
 import json
 import re
 

lib/ansible/plugins/netconf/sros.py

@@ -19,6 +19,27 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+DOCUMENTATION = """
+---
+netconf: sros
+short_description: Use Nokia SROS netconf plugin to run netconf commands on Nokia SROS platform
+deprecated:
+    why: This plugin moved in 'nokia.sros' collection
+    removed_in: '2.13'
+    alternative: "Use the netconf plugin in 'nokia.sros' collection within Ansible galaxy"
+description:
+  - This sros plugin provides low level abstraction apis for
+    sending and receiving netconf commands from Nokia sros network devices.
+version_added: "2.9"
+options:
+  ncclient_device_handler:
+    type: str
+    default: default
+    description:
+      - Specifies the ncclient device handler name for Nokia sros network os. To
+        identify the ncclient device handler name refer ncclient library documentation.
+"""
+
 import json
 import re
 
@@ -60,7 +81,7 @@ class Netconf(NetconfBase):
 
     def get_capabilities(self):
         result = dict()
-        result['rpc'] = self.get_base_rpc() + ['commit', 'discard_changes', 'validate', 'lock', 'unlock']
+        result['rpc'] = self.get_base_rpc()
         result['network_api'] = 'netconf'
         result['device_info'] = self.get_device_info()
         result['server_capabilities'] = [c for c in self.m.server_capabilities]