From 86c945a6280b176017b1a6ccda6615b2e3095b3b Mon Sep 17 00:00:00 2001 From: Deepak Agrawal Date: Wed, 16 May 2018 14:38:43 +0530 Subject: [PATCH] network_put and network_get modules (#39592) * Initial commit * Socket Timeout and dest file handler * sftp handling * module name change as per review * multiple thread tmp file overwite problem * Integration test suite for network_put * add additional testcase for dest argument * fix pylint/pep8/modules warnings * add socket timeout for get_file * network_get module * pep8 issue on network_get * Review comments --- lib/ansible/modules/network/files/__init__.py | 0 .../modules/network/files/network_get.py | 70 ++++++++ .../modules/network/files/network_put.py | 71 ++++++++ lib/ansible/plugins/action/network_get.py | 130 ++++++++++++++ lib/ansible/plugins/action/network_put.py | 167 ++++++++++++++++++ lib/ansible/plugins/cliconf/__init__.py | 10 +- .../targets/ios_file/defaults/main.yaml | 2 + test/integration/targets/ios_file/ios1.cfg | 3 + .../targets/ios_file/tasks/cli.yaml | 16 ++ .../targets/ios_file/tasks/main.yaml | 2 + .../ios_file/tests/cli/network_get.yaml | 43 +++++ .../ios_file/tests/cli/network_put.yaml | 34 ++++ 12 files changed, 543 insertions(+), 5 deletions(-) create mode 100644 lib/ansible/modules/network/files/__init__.py create mode 100644 lib/ansible/modules/network/files/network_get.py create mode 100644 lib/ansible/modules/network/files/network_put.py create mode 100644 lib/ansible/plugins/action/network_get.py create mode 100644 lib/ansible/plugins/action/network_put.py create mode 100644 test/integration/targets/ios_file/defaults/main.yaml create mode 100644 test/integration/targets/ios_file/ios1.cfg create mode 100644 test/integration/targets/ios_file/tasks/cli.yaml create mode 100644 test/integration/targets/ios_file/tasks/main.yaml create mode 100644 test/integration/targets/ios_file/tests/cli/network_get.yaml create mode 100644 test/integration/targets/ios_file/tests/cli/network_put.yaml diff --git a/lib/ansible/modules/network/files/__init__.py b/lib/ansible/modules/network/files/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/ansible/modules/network/files/network_get.py b/lib/ansible/modules/network/files/network_get.py new file mode 100644 index 00000000000..14162fbdb22 --- /dev/null +++ b/lib/ansible/modules/network/files/network_get.py @@ -0,0 +1,70 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# (c) 2018, Ansible by Red Hat, inc +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'network'} + + +DOCUMENTATION = """ +--- +module: network_get +version_added: "2.6" +author: "Deepak Agrawal (@dagrawal)" +short_description: Copy files from a network device to Ansible Controller +description: + - This module provides functionlity to copy file from network device to + ansible controller. +options: + src: + description: + - Specifies the source file. The path to the source file can either be + the full path on the network device or a relative path as per path + supported by destination network device. + required: true + protocol: + description: + - Protocol used to transfer file. + default: scp + choices: ['scp', 'sftp'] + dest: + description: + - Specifies the destination file. The path to the destination file can + either be the full path on the Ansible control host or a relative + path from the playbook or role root directory. + default: + - Same filename as specified in src. The path will be playbook root + or role root directory if playbook is part of a role. + +requirements: + - "scp" + +notes: + - Some devices need specific configurations to be enabled before scp can work + These configuration should be pre-configued before using this module + e.g ios - C(ip scp server enable) + - User privileage to do scp on network device should be pre-configured + e.g. ios - need user privileage 15 by default for allowing scp + - Default destination of source file +""" + +EXAMPLES = """ +- name: copy file from the network device to ansible controller + network_get: + src: running_cfg_ios1.txt + +- name: copy file from ios to common location at /tmp + network_put: + src: running_cfg_sw1.txt + dest : /tmp/ios1.txt +""" + +RETURN = """ +""" diff --git a/lib/ansible/modules/network/files/network_put.py b/lib/ansible/modules/network/files/network_put.py new file mode 100644 index 00000000000..cf7fb09cebc --- /dev/null +++ b/lib/ansible/modules/network/files/network_put.py @@ -0,0 +1,71 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# (c) 2018, Ansible by Red Hat, inc +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'network'} + + +DOCUMENTATION = """ +--- +module: network_put +version_added: "2.6" +author: "Deepak Agrawal (@dagrawal)" +short_description: Copy files from Ansibe controller to a network device +description: + - This module provides functionlity to copy file from Ansible controller to + network devices. +options: + src: + description: + - Specifies the source file. The path to the source file can either be + the full path on the Ansible control host or a relative path from the + playbook or role root directory. + required: true + protocol: + description: + - Protocol used to transfer file. + default: scp + choices: ['scp', 'sftp'] + dest: + description: + - Specifies the destination file. The path to destination file can + either be the full path or relative path as supported by network_os. + default: + - Filename from src and at default directory of user shell on + network_os. + required: no + +requirements: + - "scp" + +notes: + - Some devices need specific configurations to be enabled before scp can work + These configuration should be pre-configued before using this module + e.g ios - C(ip scp server enable). + - User privileage to do scp on network device should be pre-configured + e.g. ios - need user privileage 15 by default for allowing scp. + - Default destination of source file. +""" + +EXAMPLES = """ +- name: copy file from ansible controller to a network device + network_put: + src: running_cfg_ios1.txt + +- name: copy file at root dir of flash in slot 3 of sw1(ios) + network_put: + src: running_cfg_sw1.txt + protocol: sftp + dest : flash3:/running_cfg_sw1.txt +""" + +RETURN = """ +""" diff --git a/lib/ansible/plugins/action/network_get.py b/lib/ansible/plugins/action/network_get.py new file mode 100644 index 00000000000..28ae183ca89 --- /dev/null +++ b/lib/ansible/plugins/action/network_get.py @@ -0,0 +1,130 @@ +# (c) 2018, Ansible Inc, +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import copy +import os +import time +import re + +from ansible.module_utils._text import to_text +from ansible.module_utils.connection import Connection +from ansible.errors import AnsibleError +from ansible.plugins.action import ActionBase +from ansible.module_utils.six.moves.urllib.parse import urlsplit + +try: + from __main__ import display +except ImportError: + from ansible.utils.display import Display + display = Display() + + +class ActionModule(ActionBase): + + def run(self, tmp=None, task_vars=None): + socket_path = None + play_context = copy.deepcopy(self._play_context) + play_context.network_os = self._get_network_os(task_vars) + + result = super(ActionModule, self).run(task_vars=task_vars) + + if play_context.connection != 'network_cli': + # It is supported only with network_cli + result['failed'] = True + result['msg'] = ('please use network_cli connection type for network_get module') + return result + + try: + src = self._task.args.get('src') + except KeyError as exc: + return {'failed': True, 'msg': 'missing required argument: %s' % exc} + + # Get destination file if specified + dest = self._task.args.get('dest') + + if dest is None: + dest = self._get_default_dest(src) + else: + dest = self._handle_dest_path(dest) + + # Get proto + proto = self._task.args.get('protocol') + if proto is None: + proto = 'scp' + + sock_timeout = play_context.timeout + + if socket_path is None: + socket_path = self._connection.socket_path + + conn = Connection(socket_path) + + try: + out = conn.get_file( + source=src, destination=dest, + proto=proto, timeout=sock_timeout + ) + except Exception as exc: + result['failed'] = True + result['msg'] = ('Exception received : %s' % exc) + + result['changed'] = True + result['destination'] = dest + return result + + def _handle_dest_path(self, dest): + working_path = self._get_working_path() + + if os.path.isabs(dest) or urlsplit('dest').scheme: + dst = dest + else: + dst = self._loader.path_dwim_relative(working_path, '', dest) + + return dst + + def _get_src_filename_from_path(self, src_path): + filename_list = re.split('/|:', src_path) + return filename_list[-1] + + def _get_working_path(self): + cwd = self._loader.get_basedir() + if self._task._role is not None: + cwd = self._task._role._role_path + return cwd + + def _get_default_dest(self, src_path): + dest_path = self._get_working_path() + src_fname = self._get_src_filename_from_path(src_path) + filename = '%s/%s' % (dest_path, src_fname) + return filename + + def _get_network_os(self, task_vars): + if 'network_os' in self._task.args and self._task.args['network_os']: + display.vvvv('Getting network OS from task argument') + network_os = self._task.args['network_os'] + elif self._play_context.network_os: + display.vvvv('Getting network OS from inventory') + network_os = self._play_context.network_os + elif 'network_os' in task_vars.get('ansible_facts', {}) and task_vars['ansible_facts']['network_os']: + display.vvvv('Getting network OS from fact') + network_os = task_vars['ansible_facts']['network_os'] + else: + raise AnsibleError('ansible_network_os must be specified on this host to use platform agnostic modules') + + return network_os diff --git a/lib/ansible/plugins/action/network_put.py b/lib/ansible/plugins/action/network_put.py new file mode 100644 index 00000000000..fc6678c6475 --- /dev/null +++ b/lib/ansible/plugins/action/network_put.py @@ -0,0 +1,167 @@ +# (c) 2018, Ansible Inc, +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import copy +import os +import time +import uuid + +from ansible.module_utils._text import to_text +from ansible.module_utils.connection import Connection +from ansible.errors import AnsibleError +from ansible.plugins.action import ActionBase +from ansible.module_utils.six.moves.urllib.parse import urlsplit + +try: + from __main__ import display +except ImportError: + from ansible.utils.display import Display + display = Display() + + +class ActionModule(ActionBase): + + def run(self, tmp=None, task_vars=None): + socket_path = None + play_context = copy.deepcopy(self._play_context) + play_context.network_os = self._get_network_os(task_vars) + + result = super(ActionModule, self).run(task_vars=task_vars) + + if play_context.connection != 'network_cli': + # It is supported only with network_cli + result['failed'] = True + result['msg'] = ('please use network_cli connection type for network_put module') + return result + + src_file_path_name = self._task.args.get('src') + + try: + self._handle_template() + except ValueError as exc: + return dict(failed=True, msg=to_text(exc)) + + try: + src = self._task.args.get('src') + except KeyError as exc: + return {'failed': True, 'msg': 'missing required argument: %s' % exc} + + # Get destination file if specified + dest = self._task.args.get('dest') + + # Get proto + proto = self._task.args.get('protocol') + if proto is None: + proto = 'scp' + + sock_timeout = play_context.timeout + + # Now src has resolved file write to disk in current diectory for scp + filename = str(uuid.uuid4()) + cwd = self._loader.get_basedir() + output_file = cwd + '/' + filename + with open(output_file, 'w') as f: + f.write(src) + + if socket_path is None: + socket_path = self._connection.socket_path + + conn = Connection(socket_path) + if dest is None: + dest = src_file_path_name + + try: + out = conn.copy_file( + source=output_file, destination=dest, + proto=proto, timeout=sock_timeout + ) + except Exception as exc: + if to_text(exc) == "No response from server": + if play_context.network_os == 'iosxr': + # IOSXR sometimes closes socket prematurely after completion + # of file transfer + result['msg'] = 'Warning: iosxr scp server pre close issue. Please check dest' + else: + result['failed'] = True + result['msg'] = ('Exception received : %s' % exc) + + # Cleanup tmp file expanded wih ansible vars + os.remove(output_file) + result['changed'] = True + return result + + def _get_working_path(self): + cwd = self._loader.get_basedir() + if self._task._role is not None: + cwd = self._task._role._role_path + return cwd + + def _handle_template(self): + src = self._task.args.get('src') + working_path = self._get_working_path() + + if os.path.isabs(src) or urlsplit('src').scheme: + source = src + else: + source = self._loader.path_dwim_relative(working_path, 'templates', src) + if not source: + source = self._loader.path_dwim_relative(working_path, src) + + if not os.path.exists(source): + raise ValueError('path specified in src not found') + + try: + with open(source, 'r') as f: + template_data = to_text(f.read()) + except IOError: + return dict(failed=True, msg='unable to load src file') + + # Create a template search path in the following order: + # [working_path, self_role_path, dependent_role_paths, dirname(source)] + searchpath = [working_path] + if self._task._role is not None: + searchpath.append(self._task._role._role_path) + if hasattr(self._task, "_block:"): + dep_chain = self._task._block.get_dep_chain() + if dep_chain is not None: + for role in dep_chain: + searchpath.append(role._role_path) + searchpath.append(os.path.dirname(source)) + self._templar.environment.loader.searchpath = searchpath + self._task.args['src'] = self._templar.template( + template_data, + convert_data=False + ) + + return dict(failed=False, msg='successfully loaded file') + + def _get_network_os(self, task_vars): + if 'network_os' in self._task.args and self._task.args['network_os']: + display.vvvv('Getting network OS from task argument') + network_os = self._task.args['network_os'] + elif self._play_context.network_os: + display.vvvv('Getting network OS from inventory') + network_os = self._play_context.network_os + elif 'network_os' in task_vars.get('ansible_facts', {}) and task_vars['ansible_facts']['network_os']: + display.vvvv('Getting network OS from fact') + network_os = task_vars['ansible_facts']['network_os'] + else: + raise AnsibleError('ansible_network_os must be specified on this host to use platform agnostic modules') + + return network_os diff --git a/lib/ansible/plugins/cliconf/__init__.py b/lib/ansible/plugins/cliconf/__init__.py index c46d2a76d1e..e613cf39ebf 100644 --- a/lib/ansible/plugins/cliconf/__init__.py +++ b/lib/ansible/plugins/cliconf/__init__.py @@ -178,25 +178,25 @@ class CliconfBase(with_metaclass(ABCMeta, object)): "Discard changes in candidate datastore" return self._connection.method_not_found("discard_changes is not supported by network_os %s" % self._play_context.network_os) - def copy_file(self, source=None, destination=None, proto='scp'): + def copy_file(self, source=None, destination=None, proto='scp', timeout=30): """Copies file over scp/sftp to remote device""" ssh = self._connection.paramiko_conn._connect_uncached() if proto == 'scp': if not HAS_SCP: self._connection.internal_error("Required library scp is not installed. Please install it using `pip install scp`") - with SCPClient(ssh.get_transport()) as scp: - scp.put(source, destination) + with SCPClient(ssh.get_transport(), socket_timeout=timeout) as scp: + out = scp.put(source, destination) elif proto == 'sftp': with ssh.open_sftp() as sftp: sftp.put(source, destination) - def get_file(self, source=None, destination=None, proto='scp'): + def get_file(self, source=None, destination=None, proto='scp', timeout=30): """Fetch file over scp/sftp from remote device""" ssh = self._connection.paramiko_conn._connect_uncached() if proto == 'scp': if not HAS_SCP: self._connection.internal_error("Required library scp is not installed. Please install it using `pip install scp`") - with SCPClient(ssh.get_transport()) as scp: + with SCPClient(ssh.get_transport(), socket_timeout=timeout) as scp: scp.get(source, destination) elif proto == 'sftp': with ssh.open_sftp() as sftp: diff --git a/test/integration/targets/ios_file/defaults/main.yaml b/test/integration/targets/ios_file/defaults/main.yaml new file mode 100644 index 00000000000..5f709c5aac1 --- /dev/null +++ b/test/integration/targets/ios_file/defaults/main.yaml @@ -0,0 +1,2 @@ +--- +testcase: "*" diff --git a/test/integration/targets/ios_file/ios1.cfg b/test/integration/targets/ios_file/ios1.cfg new file mode 100644 index 00000000000..120dd4cad75 --- /dev/null +++ b/test/integration/targets/ios_file/ios1.cfg @@ -0,0 +1,3 @@ +vlan 3 + name ank_vlan3 +! diff --git a/test/integration/targets/ios_file/tasks/cli.yaml b/test/integration/targets/ios_file/tasks/cli.yaml new file mode 100644 index 00000000000..ea5c8c3742f --- /dev/null +++ b/test/integration/targets/ios_file/tasks/cli.yaml @@ -0,0 +1,16 @@ +--- +- name: collect all cli test cases + find: + paths: "{{ role_path }}/tests/cli" + patterns: "{{ testcase }}.yaml" + register: test_cases + delegate_to: localhost + +- name: set test_items + set_fact: test_items="{{ test_cases.files | map(attribute='path') | list }}" + +- name: run test cases (connection=network_cli) + include: "{{ test_case_to_run }}" + with_items: "{{ test_items }}" + loop_control: + loop_var: test_case_to_run diff --git a/test/integration/targets/ios_file/tasks/main.yaml b/test/integration/targets/ios_file/tasks/main.yaml new file mode 100644 index 00000000000..415c99d8b12 --- /dev/null +++ b/test/integration/targets/ios_file/tasks/main.yaml @@ -0,0 +1,2 @@ +--- +- { include: cli.yaml, tags: ['cli'] } diff --git a/test/integration/targets/ios_file/tests/cli/network_get.yaml b/test/integration/targets/ios_file/tests/cli/network_get.yaml new file mode 100644 index 00000000000..34991ebebce --- /dev/null +++ b/test/integration/targets/ios_file/tests/cli/network_get.yaml @@ -0,0 +1,43 @@ +--- +- debug: msg="START ios cli/network_get.yaml on connection={{ ansible_connection }}" + +# Add minimal testcase to check args are passed correctly to +# implementation module and module run is successful. + +- name: setup + ios_config: + lines: + - ip ssh version 2 + - ip scp server enable + - username {{ ansible_ssh_user }} privilege 15 + match: none + +- name: setup (copy file to be fetched from device) + network_put: + src: ios1.cfg + register: result + +- assert: + that: + - result.changed == true + +- name: get the file from device with dest unspecified + network_get: + src: ios1.cfg + register: result + +- assert: + that: + - result.changed == true + +- name: get the file from device with relative destination + network_get: + src: ios1.cfg + dest: 'ios_{{ ansible_host }}.cfg' + register: result + +- assert: + that: + - result.changed == true + +- debug: msg="END ios cli/network_get.yaml on connection={{ ansible_connection }}" diff --git a/test/integration/targets/ios_file/tests/cli/network_put.yaml b/test/integration/targets/ios_file/tests/cli/network_put.yaml new file mode 100644 index 00000000000..29e838cd4b5 --- /dev/null +++ b/test/integration/targets/ios_file/tests/cli/network_put.yaml @@ -0,0 +1,34 @@ +--- +- debug: msg="START ios cli/network_put.yaml on connection={{ ansible_connection }}" + +# Add minimal testcase to check args are passed correctly to +# implementation module and module run is successful. + +- name: setup + ios_config: + lines: + - ip ssh version 2 + - ip scp server enable + - username {{ ansible_ssh_user }} privilege 15 + match: none + +- name: copy file from controller to ios + scp (Default) + network_put: + src: ios1.cfg + register: result + +- assert: + that: + - result.changed == true + +- name: copy file from controller to ios + dest specified + network_put: + src: ios1.cfg + dest: ios.cfg + register: result + +- assert: + that: + - result.changed == true + +- debug: msg="END ios cli/network_put.yaml on connection={{ ansible_connection }}"