adds support for using connection=netconf (#33400)

* adds support for using connection=netconf

This change updates the module to provide support for using
connection=netconf instead of connection=local.  If connection=netconf
is used, then the various connection arguments will be silently ignored.

* adds netconf plugin default

This adds a default implementation for netconf plugins if the network_os
is not specified.  The default plugin will implement only the standard
netconf rpcs

* fix up pep8 issues

lib/ansible/modules/network/netconf/netconf_config.py

@@ -26,6 +26,7 @@ description:
 notes:
     - This module supports devices with and without the candidate and
       confirmed-commit capabilities. It always use the safer feature.
+    - This module supports the use of connection=netconf
 version_added: "2.2"
 options:
   host:
@@ -101,6 +102,13 @@ requirements:
 '''
 
 EXAMPLES = '''
+- name: use lookup filter to provide xml configuration
+  netconf_config:
+    xml: "{{ lookup('file', './config.xml') }}"
+    host: 10.0.0.1
+    username: admin
+    password: admin
+
 - name: set ntp server in the device
   netconf_config:
     host: 10.0.0.1
@@ -150,6 +158,8 @@ server_capabilities:
 import traceback
 import xml.dom.minidom
 
+from xml.etree.ElementTree import fromstring, tostring
+
 try:
     import ncclient.manager
     HAS_NCCLIENT = True
@@ -158,23 +168,31 @@ except ImportError:
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils._text import to_native
+from ansible.module_utils.connection import Connection, ConnectionError
 
 
-def netconf_edit_config(m, xml, commit, retkwargs, datastore):
+def netconf_edit_config(m, xml, commit, retkwargs, datastore, capabilities, local_connection):
     m.lock(target=datastore)
     try:
         if datastore == "candidate":
             m.discard_changes()
+
         config_before = m.get_config(source=datastore)
         m.edit_config(target=datastore, config=xml)
         config_after = m.get_config(source=datastore)
-        changed = config_before.data_xml != config_after.data_xml
+
+        if local_connection:
+            changed = config_before.data_xml != config_after.data_xml
+        else:
+            changed = config_before != config_after
+
         if changed and commit and datastore == "candidate":
-            if ":confirmed-commit" in m.server_capabilities:
+            if ":confirmed-commit" in capabilities:
                 m.commit(confirmed=True)
                 m.commit()
             else:
                 m.commit()
+
         return changed
     finally:
         m.unlock(target=datastore)
@@ -188,22 +206,28 @@ def main():
 
     module = AnsibleModule(
         argument_spec=dict(
-            host=dict(type='str', required=True),
+            xml=dict(type='str', required=False),
+            src=dict(type='path', required=False),
+
+            datastore=dict(choices=['auto', 'candidate', 'running'], default='auto'),
+            save=dict(type='bool', default=False),
+
+            # connection arguments
+            host=dict(type='str'),
             port=dict(type='int', default=830),
+
+            username=dict(type='str', no_log=True),
+            password=dict(type='str', no_log=True),
+
             hostkey_verify=dict(type='bool', default=True),
-            allow_agent=dict(type='bool', default=True),
             look_for_keys=dict(type='bool', default=True),
-            datastore=dict(choices=['auto', 'candidate', 'running'], default='auto'),
+
-            save=dict(type='bool', default=False),
+            allow_agent=dict(type='bool', default=True),
-            username=dict(type='str', required=True, no_log=True),
-            password=dict(type='str', required=True, no_log=True),
-            xml=dict(type='str', required=False),
-            src=dict(type='path', required=False),
         ),
         mutually_exclusive=[('xml', 'src')]
     )
 
-    if not HAS_NCCLIENT:
+    if not module._socket_path and not HAS_NCCLIENT:
         module.fail_json(msg='could not import the python library '
                          'ncclient required by this module')
 
@@ -214,68 +238,82 @@ def main():
     else:
         module.fail_json(msg='Option src or xml must be provided')
 
-    try:
+    local_connection = module._socket_path is None
-        xml.dom.minidom.parseString(config_xml)
 
-    except Exception as e:
+    if not local_connection:
-        module.fail_json(msg='error parsing XML: %s' % to_native(e), exception=traceback.format_exc())
+        m = Connection(module._socket_path)
-
+        capabilities = module.from_json(m.get_capabilities())
-    nckwargs = dict(
+        server_capabilities = capabilities.get('server_capabilities')
-        host=module.params['host'],
-        port=module.params['port'],
-        hostkey_verify=module.params['hostkey_verify'],
-        allow_agent=module.params['allow_agent'],
-        look_for_keys=module.params['look_for_keys'],
-        username=module.params['username'],
-        password=module.params['password'],
-    )
 
-    try:
+    else:
-        m = ncclient.manager.connect(**nckwargs)
+        nckwargs = dict(
-    except ncclient.transport.errors.AuthenticationError:
+            host=module.params['host'],
-        module.fail_json(
+            port=module.params['port'],
-            msg='authentication failed while connecting to device'
+            hostkey_verify=module.params['hostkey_verify'],
+            allow_agent=module.params['allow_agent'],
+            look_for_keys=module.params['look_for_keys'],
+            username=module.params['username'],
+            password=module.params['password'],
         )
-    except Exception as e:
+
-        module.fail_json(msg='error connecting to the device: %s' % to_native(e), exception=traceback.format_exc())
+        try:
+            m = ncclient.manager.connect(**nckwargs)
+            server_capabilities = list(m.server_capabilities)
+        except ncclient.transport.errors.AuthenticationError:
+            module.fail_json(
+                msg='authentication failed while connecting to device'
+            )
+        except Exception as e:
+            module.fail_json(msg='error connecting to the device: %s' % to_native(e), exception=traceback.format_exc())
+
+        try:
+            xml.dom.minidom.parseString(config_xml)
+        except Exception as e:
+            module.fail_json(msg='error parsing XML: %s' % to_native(e), exception=traceback.format_exc())
 
     retkwargs = dict()
-    retkwargs['server_capabilities'] = list(m.server_capabilities)
+    retkwargs['server_capabilities'] = server_capabilities
+
+    server_capabilities = '\n'.join(server_capabilities)
 
     if module.params['datastore'] == 'candidate':
-        if ':candidate' in m.server_capabilities:
+        if ':candidate' in server_capabilities:
             datastore = 'candidate'
         else:
-            m.close_session()
+            if local_connection:
+                m.close_session()
             module.fail_json(
                 msg=':candidate is not supported by this netconf server'
             )
     elif module.params['datastore'] == 'running':
-        if ':writable-running' in m.server_capabilities:
+        if ':writable-running' in server_capabilities:
             datastore = 'running'
         else:
-            m.close_session()
+            if local_connection:
+                m.close_session()
             module.fail_json(
                 msg=':writable-running is not supported by this netconf server'
             )
     elif module.params['datastore'] == 'auto':
-        if ':candidate' in m.server_capabilities:
+        if ':candidate' in server_capabilities:
             datastore = 'candidate'
-        elif ':writable-running' in m.server_capabilities:
+        elif ':writable-running' in server_capabilities:
             datastore = 'running'
         else:
-            m.close_session()
+            if local_connection:
+                m.close_session()
             module.fail_json(
                 msg='neither :candidate nor :writable-running are supported by this netconf server'
             )
     else:
-        m.close_session()
+        if local_connection:
+            m.close_session()
         module.fail_json(
             msg=module.params['datastore'] + ' datastore is not supported by this ansible module'
         )
 
     if module.params['save']:
-        if ':startup' not in m.server_capabilities:
+        if ':startup' not in server_capabilities:
             module.fail_json(
                 msg='cannot copy <running/> to <startup/>, while :startup is not supported'
             )
@@ -287,13 +325,16 @@ def main():
             commit=True,
             retkwargs=retkwargs,
             datastore=datastore,
+            capabilities=server_capabilities,
+            local_connection=local_connection
         )
         if changed and module.params['save']:
             m.copy_config(source="running", target="startup")
     except Exception as e:
         module.fail_json(msg='error editing configuration: %s' % to_native(e), exception=traceback.format_exc())
     finally:
-        m.close_session()
+        if local_connection:
+            m.close_session()
 
     module.exit_json(changed=changed, **retkwargs)
 

lib/ansible/plugins/connection/netconf.py

@@ -238,8 +238,7 @@ class Connection(ConnectionBase):
                 if network_os:
                     display.display('discovered network_os %s' % network_os, log_only=True)
 
-        if not network_os:
+        device_params = {'name': (network_os or 'default')}
-            raise AnsibleConnectionFailure('Unable to automatically determine host network os. Please ansible_network_os value')
 
         ssh_config = os.getenv('ANSIBLE_NETCONF_SSH_CONFIG', False)
         if ssh_config in BOOLEANS_TRUE:
@@ -256,9 +255,9 @@ class Connection(ConnectionBase):
                 key_filename=str(key_filename),
                 hostkey_verify=C.HOST_KEY_CHECKING,
                 look_for_keys=C.PARAMIKO_LOOK_FOR_KEYS,
+                device_params=device_params,
                 allow_agent=self._play_context.allow_agent,
                 timeout=self._play_context.timeout,
-                device_params={'name': network_os},
                 ssh_config=ssh_config
             )
         except SSHUnknownHostError as exc:

lib/ansible/plugins/netconf/__init__.py

@@ -102,7 +102,7 @@ class NetconfBase(with_metaclass(ABCMeta, object)):
            :source: name of the configuration datastore being queried
            :filter: specifies the portion of the configuration to retrieve
            (by default entire configuration is retrieved)"""
-        pass
+        return self.m.get_config(*args, **kwargs).data_xml
 
     @ensure_connected
     def get(self, *args, **kwargs):
@@ -110,7 +110,7 @@ class NetconfBase(with_metaclass(ABCMeta, object)):
         *filter* specifies the portion of the configuration to retrieve
         (by default entire configuration is retrieved)
         """
-        pass
+        return self.m.get(*args, **kwargs).data_xml
 
     @ensure_connected
     def edit_config(self, *args, **kwargs):
@@ -124,7 +124,7 @@ class NetconfBase(with_metaclass(ABCMeta, object)):
             :error_option: if specified must be one of { `"stop-on-error"`, `"continue-on-error"`, `"rollback-on-error"` }
             The `"rollback-on-error"` *error_option* depends on the `:rollback-on-error` capability.
         """
-        pass
+        return self.m.edit_config(*args, **kwargs).xml
 
     @ensure_connected
     def validate(self, *args, **kwargs):
@@ -132,7 +132,7 @@ class NetconfBase(with_metaclass(ABCMeta, object)):
         :source: is the name of the configuration datastore being validated or `config`
         element containing the configuration subtree to be validated
         """
-        pass
+        return self.m.validate(*args, **kwargs).xml
 
     @ensure_connected
     def copy_config(self, *args, **kwargs):
@@ -141,27 +141,27 @@ class NetconfBase(with_metaclass(ABCMeta, object)):
         :source: is the name of the configuration datastore to use as the source of the
                  copy operation or `config` element containing the configuration subtree to copy
         :target: is the name of the configuration datastore to use as the destination of the copy operation"""
-        return self.m.copy_config(*args, **kwargs).data_xml
+        return self.m.copy_config(*args, **kwargs).xml
 
     @ensure_connected
     def lock(self, *args, **kwargs):
         """Allows the client to lock the configuration system of a device.
         *target* is the name of the configuration datastore to lock
         """
-        return self.m.lock(*args, **kwargs).data_xml
+        return self.m.lock(*args, **kwargs).xml
 
     @ensure_connected
     def unlock(self, *args, **kwargs):
         """Release a configuration lock, previously obtained with the lock operation.
         :target: is the name of the configuration datastore to unlock
         """
-        return self.m.unlock(*args, **kwargs).data_xml
+        return self.m.unlock(*args, **kwargs).xml
 
     @ensure_connected
     def discard_changes(self, *args, **kwargs):
         """Revert the candidate configuration to the currently running configuration.
         Any uncommitted changes are discarded."""
-        pass
+        return self.m.discard_changes(*args, **kwargs).xml
 
     @ensure_connected
     def commit(self, *args, **kwargs):
@@ -175,19 +175,19 @@ class NetconfBase(with_metaclass(ABCMeta, object)):
         :confirmed: whether this is a confirmed commit
         :timeout: specifies the confirm timeout in seconds
         """
-        pass
+        return self.m.commit(*args, **kwargs).xml
 
     @ensure_connected
     def validate(self, *args, **kwargs):
         """Validate the contents of the specified configuration.
            :source: name of configuration data store"""
-        return self.m.validate(*args, **kwargs).data_xml
+        return self.m.validate(*args, **kwargs).xml
 
     @ensure_connected
     def get_schema(self, *args, **kwargs):
         """Retrieves the required schema from the device
         """
-        return self.m.get_schema(*args, **kwargs)
+        return self.m.get_schema(*args, **kwargs).xml
 
     @ensure_connected
     def locked(self, *args, **kwargs):
@@ -220,4 +220,4 @@ class NetconfBase(with_metaclass(ABCMeta, object)):
         """Fetch file over scp from remote device"""
         pass
 
-# TODO Restore .data_xml, when ncclient supports it for all platforms
+# TODO Restore .xml, when ncclient supports it for all platforms

lib/ansible/plugins/netconf/default.py

@@ -0,0 +1,51 @@
+#
+# (c) 2017 Red Hat Inc.
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+
+from ansible.module_utils._text import to_text, to_bytes
+from ansible.plugins.netconf import NetconfBase
+
+
+class Netconf(NetconfBase):
+
+    def get_text(self, ele, tag):
+        try:
+            return to_text(ele.find(tag).text, errors='surrogate_then_replace').strip()
+        except AttributeError:
+            pass
+
+    def get_device_info(self):
+        device_info = dict()
+        device_info['network_os'] = 'default'
+        return device_info
+
+    def get_capabilities(self):
+        result = dict()
+        result['rpc'] = self.get_base_rpc() + ['commit', 'discard_changes', 'validate', 'lock', 'unlock', 'copy_copy',
+                                               'execute_rpc', 'load_configuration', 'get_configuration', 'command',
+                                               'reboot', 'halt']
+        result['network_api'] = 'netconf'
+        result['device_info'] = self.get_device_info()
+        result['server_capabilities'] = [c for c in self.m.server_capabilities]
+        result['client_capabilities'] = [c for c in self.m.client_capabilities]
+        result['session_id'] = self.m.session_id
+        return json.dumps(result)