archive
/
ansible
mirror of https://github.com/ansible/ansible.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Improved netapp module utility for E-Series. (#59527)

Browse Source 
Add header option to request method in NetAppESeriesModule
Add multipart formdata builder function
Fix issue with url port change
pull/58151/head
Nathan Swartz 6 years ago committed by Jake Jackson
parent a01ee2759d
commit 26fff6f5c3
2 changed files with 183 additions and 213 deletions
Show Stats Download Patch File Download Diff File

239
lib/ansible/module_utils/netapp.py
Unescape Escape View File

@ -29,8 +29,11 @@
import json
import os
import random
import mimetypes
from pprint import pformat
from ansible.module_utils import six
from ansible.module_utils.basic import AnsibleModule, missing_required_lib
from ansible.module_utils.six.moves.urllib.error import HTTPError, URLError
from ansible.module_utils.urls import open_url
@ -245,7 +248,8 @@ class NetAppESeriesModule(object):
"""
DEFAULT_TIMEOUT = 60
DEFAULT_SECURE_PORT = "8443"
DEFAULT_REST_API_PATH = "devmgr/v2"
DEFAULT_REST_API_PATH = "devmgr/v2/"
DEFAULT_REST_API_ABOUT_PATH = "devmgr/utils/about"
DEFAULT_HEADERS = {"Content-Type": "application/json", "Accept": "application/json",
"netapp-client-type": "Ansible-%s" % ansible_version}
HTTP_AGENT = "Ansible / %s" % ansible_version
@ -264,122 +268,65 @@ class NetAppESeriesModule(object):
args = self.module.params
self.web_services_version = web_services_version if web_services_version else "02.00.0000.0000"
self.url = args["api_url"]
self.ssid = args["ssid"]
self.url = args["api_url"]
self.log_requests = log_requests
self.creds = dict(url_username=args["api_username"],
url_password=args["api_password"],
validate_certs=args["validate_certs"])
self.log_requests = log_requests
self.is_embedded_mode = None
self.web_services_validate = None
self._tweak_url()
@property
def _about_url(self):
"""Generates the about url based on the supplied web services rest api url.
:raise AnsibleFailJson: raised when supplied web services rest api is an invalid url.
:return: proxy or embedded about url.
"""
about = list(urlparse(self.url))
about[2] = "devmgr/utils/about"
return urlunparse(about)
def _force_secure_url(self):
"""Modifies supplied web services rest api to use secure https.
raise: AnsibleFailJson: raised when the url already utilizes the secure protocol
"""
url_parts = list(urlparse(self.url))
if "https://" in self.url and ":8443" in self.url:
self.module.fail_json(msg="Secure HTTP protocol already used. URL path [%s]" % self.url)
url_parts[0] = "https"
url_parts[1] = "%s:8443" % url_parts[1].split(":")[0]
self.url = urlunparse(url_parts)
if not self.url.endswith("/"):
self.url += "/"
self.module.warn("forced use of the secure protocol: %s" % self.url)
self.is_embedded_mode = None
self.is_web_services_valid_cache = None
def _tweak_url(self):
"""Adjust the rest api url is necessary.
def _check_web_services_version(self):
"""Verify proxy or embedded web services meets minimum version required for module.
:raise AnsibleFailJson: raised when self.url fails to have a hostname or ipv4 address.
"""
# ensure the protocol is either http or https
if self.url.split("://")[0] not in ["https", "http"]:
The minimum required web services version is evaluated against version supplied through the web services rest
api. AnsibleFailJson exception will be raised when the minimum is not met or exceeded.
self.url = self.url.split("://")[1] if "://" in self.url else self.url
This helper function will update the supplied api url if secure http is not used for embedded web services
if ":8080" in self.url:
self.url = "http://%s" % self.url
else:
self.url = "https://%s" % self.url
:raise AnsibleFailJson: raised when the contacted api service does not meet the minimum required version.
"""
if not self.is_web_services_valid_cache:
# parse url and verify protocol, port and path are consistent with required web services rest api url.
url_parts = list(urlparse(self.url))
if url_parts[1] == "":
self.module.fail_json(msg="Failed to provide a valid hostname or IP address. URL [%s]." % self.url)
split_hostname = url_parts[1].split(":")
if url_parts[0] not in ["https", "http"] or (len(split_hostname) == 2 and split_hostname[1] != "8080"):
if len(split_hostname) == 2 and split_hostname[1] == "8080":
url_parts[0] = "http"
url_parts[1] = "%s:8080" % split_hostname[0]
else:
url_parts[0] = "https"
url_parts[1] = "%s:8443" % split_hostname[0]
elif len(split_hostname) == 1:
if url_parts[0] == "https":
url_parts[1] = "%s:8443" % split_hostname[0]
elif url_parts[0] == "http":
url_parts[1] = "%s:8080" % split_hostname[0]
if url_parts[2] == "" or url_parts[2] != self.DEFAULT_REST_API_PATH:
url_parts[2] = self.DEFAULT_REST_API_PATH
self.url = urlunparse(url_parts)
if not self.url.endswith("/"):
self.url += "/"
self.module.log("valid url: %s" % self.url)
if not url_parts[0] or not url_parts[1]:
self.module.fail_json(msg="Failed to provide valid API URL. Example: https://192.168.1.100:8443/devmgr/v2. URL [%s]." % self.url)
def _is_web_services_valid(self):
"""Verify proxy or embedded web services meets minimum version required for module.
if url_parts[0] not in ["http", "https"]:
self.module.fail_json(msg="Protocol must be http or https. URL [%s]." % self.url)
The minimum required web services version is evaluated against version supplied through the web services rest
api. AnsibleFailJson exception will be raised when the minimum is not met or exceeded.
self.url = "%s://%s/" % (url_parts[0], url_parts[1])
about_url = self.url + self.DEFAULT_REST_API_ABOUT_PATH
rc, data = request(about_url, timeout=self.DEFAULT_TIMEOUT, headers=self.DEFAULT_HEADERS, ignore_errors=True, **self.creds)
:raise AnsibleFailJson: raised when the contacted api service does not meet the minimum required version.
"""
if not self.web_services_validate:
self.is_embedded()
if rc != 200:
self.module.warn("Failed to retrieve web services about information! Retrying with secure ports. Array Id [%s]." % self.ssid)
self.url = "https://%s:8443/" % url_parts[1].split(":")[0]
about_url = self.url + self.DEFAULT_REST_API_ABOUT_PATH
try:
rc, data = request(self._about_url, timeout=self.DEFAULT_TIMEOUT,
headers=self.DEFAULT_HEADERS, **self.creds)
rc, data = request(about_url, timeout=self.DEFAULT_TIMEOUT, headers=self.DEFAULT_HEADERS, **self.creds)
except Exception as error:
self.module.fail_json(msg="Failed to retrieve the webservices about information! Array Id [%s]. Error [%s]."
% (self.ssid, to_native(error)))
major, minor, other, revision = data["version"].split(".")
minimum_major, minimum_minor, other, minimum_revision = self.web_services_version.split(".")
if not (major > minimum_major or
(major == minimum_major and minor > minimum_minor) or
(major == minimum_major and minor == minimum_minor and revision >= minimum_revision)):
self.module.fail_json(
msg="Web services version does not meet minimum version required. Current version: [%s]."
self.module.fail_json(msg="Web services version does not meet minimum version required. Current version: [%s]."
" Version required: [%s]." % (data["version"], self.web_services_version))
except Exception as error:
self.module.fail_json(msg="Failed to retrieve the webservices about information! Array Id [%s]."
" Error [%s]." % (self.ssid, to_native(error)))
self.module.warn("Web services rest api version met the minimum required version.")
self.web_services_validate = True
return self.web_services_validate
self.module.log("Web services rest api version met the minimum required version.")
self.is_web_services_valid_cache = True
def is_embedded(self, retry=True):
def is_embedded(self):
"""Determine whether web services server is the embedded web services.
If web services about endpoint fails based on an URLError then the request will be attempted again using
@ -388,59 +335,101 @@ class NetAppESeriesModule(object):
:raise AnsibleFailJson: raised when web services about endpoint failed to be contacted.
:return bool: whether contacted web services is running from storage array (embedded) or from a proxy.
"""
self._check_web_services_version()
if self.is_embedded_mode is None:
about_url = self.url + self.DEFAULT_REST_API_ABOUT_PATH
try:
rc, data = request(self._about_url, timeout=self.DEFAULT_TIMEOUT,
headers=self.DEFAULT_HEADERS, **self.creds)
rc, data = request(about_url, timeout=self.DEFAULT_TIMEOUT, headers=self.DEFAULT_HEADERS, **self.creds)
self.is_embedded_mode = not data["runningAsProxy"]
except URLError as error:
if not retry:
self.module.fail_json(msg="Failed to retrieve the webservices about information! Array Id [%s]."
" Error [%s]." % (self.ssid, to_native(error)))
self.module.warn("Failed to retrieve the webservices about information! Will retry using secure"
" http. Array Id [%s]." % self.ssid)
self._force_secure_url()
return self.is_embedded(retry=False)
except Exception as error:
self.module.fail_json(msg="Failed to retrieve the webservices about information! Array Id [%s]."
" Error [%s]." % (self.ssid, to_native(error)))
self.module.fail_json(msg="Failed to retrieve the webservices about information! Array Id [%s]. Error [%s]."
% (self.ssid, to_native(error)))
return self.is_embedded_mode
def request(self, path, data=None, method='GET', ignore_errors=False):
def request(self, path, data=None, method='GET', headers=None, ignore_errors=False):
"""Issue an HTTP request to a url, retrieving an optional JSON response.
:param str path: web services rest api endpoint path (Example: storage-systems/1/graph). Note that when the
full url path is specified then that will be used without supplying the protocol, hostname, port and rest path.
:param data: data required for the request (data may be json or any python structured data)
:param str method: request method such as GET, POST, DELETE.
:param dict headers: dictionary containing request headers.
:param bool ignore_errors: forces the request to ignore any raised exceptions.
"""
if self._is_web_services_valid():
url = list(urlparse(path.strip("/")))
if url[2] == "":
self.module.fail_json(msg="Web services rest api endpoint path must be specified. Path [%s]." % path)
# if either the protocol or hostname/port are missing then add them.
if url[0] == "" or url[1] == "":
url[0], url[1] = list(urlparse(self.url))[:2]
# add rest api path if the supplied path does not begin with it.
if not all([word in url[2].split("/")[:2] for word in self.DEFAULT_REST_API_PATH.split("/")]):
if not url[2].startswith("/"):
url[2] = "/" + url[2]
url[2] = self.DEFAULT_REST_API_PATH + url[2]
# ensure data is json formatted
if not isinstance(data, str):
self._check_web_services_version()
if headers is None:
headers = self.DEFAULT_HEADERS
if not isinstance(data, str) and headers["Content-Type"] == "application/json":
data = json.dumps(data)
if self.log_requests:
self.module.log(pformat(dict(url=urlunparse(url), data=data, method=method)))
if path.startswith("/"):
path = path[1:]
request_url = self.url + self.DEFAULT_REST_API_PATH + path
if self.log_requests or True:
self.module.log(pformat(dict(url=request_url, data=data, method=method)))
return request(url=request_url, data=data, method=method, headers=headers, use_proxy=True, force=False, last_mod_time=None,
timeout=self.DEFAULT_TIMEOUT, http_agent=self.HTTP_AGENT, force_basic_auth=True, ignore_errors=ignore_errors, **self.creds)
return request(url=urlunparse(url), data=data, method=method, headers=self.DEFAULT_HEADERS, use_proxy=True,
force=False, last_mod_time=None, timeout=self.DEFAULT_TIMEOUT, http_agent=self.HTTP_AGENT,
force_basic_auth=True, ignore_errors=ignore_errors, **self.creds)
def create_multipart_formdata(files, fields=None, send_8kb=False):
"""Create the data for a multipart/form request."""
boundary = "---------------------------" + "".join([str(random.randint(0, 9)) for x in range(27)])
data_parts = list()
data = None
if six.PY2: # Generate payload for Python 2
newline = "\r\n"
if fields is not None:
for key, value in fields:
data_parts.extend(["--%s" % boundary,
'Content-Disposition: form-data; name="%s"' % key,
"",
value])
for name, filename, path in files:
with open(path, "rb") as fh:
value = fh.read(8192) if send_8kb else fh.read()
data_parts.extend(["--%s" % boundary,
'Content-Disposition: form-data; name="%s"; filename="%s"' % (name, filename),
"Content-Type: %s" % (mimetypes.guess_type(path)[0] or "application/octet-stream"),
"",
value])
data_parts.extend(["--%s--" % boundary, ""])
data = newline.join(data_parts)
else:
newline = six.b("\r\n")
if fields is not None:
for key, value in fields:
data_parts.extend([six.b("--%s" % boundary),
six.b('Content-Disposition: form-data; name="%s"' % key),
six.b(""),
six.b(value)])
for name, filename, path in files:
with open(path, "rb") as fh:
value = fh.read(8192) if send_8kb else fh.read()
data_parts.extend([six.b("--%s" % boundary),
six.b('Content-Disposition: form-data; name="%s"; filename="%s"' % (name, filename)),
six.b("Content-Type: %s" % (mimetypes.guess_type(path)[0] or "application/octet-stream")),
six.b(""),
value])
data_parts.extend([six.b("--%s--" % boundary), b""])
data = newline.join(data_parts)
headers = {
"Content-Type": "multipart/form-data; boundary=%s" % boundary,
"Content-Length": str(len(data))}
return headers, data
def request(url, data=None, headers=None, method='GET', use_proxy=True,

121
test/units/module_utils/test_netapp.py
Unescape Escape View File

@ -1,13 +1,17 @@
# (c) 2018, NetApp Inc.
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from ansible.module_utils.six.moves.urllib.error import URLError
from ansible.module_utils.netapp import NetAppESeriesModule
from units.modules.utils import ModuleTestCase, set_module_args, AnsibleFailJson
__metaclass__ = type
from units.compat import mock
try:
from unittest.mock import patch, mock_open
except ImportError:
from mock import patch, mock_open
from ansible.module_utils.six.moves.urllib.error import URLError
from ansible.module_utils.netapp import NetAppESeriesModule, create_multipart_formdata
from units.modules.utils import AnsibleExitJson, AnsibleFailJson, ModuleTestCase, set_module_args
class StubNetAppESeriesModule(NetAppESeriesModule):
@ -28,44 +32,45 @@ class NetappTest(ModuleTestCase):
module_args.update(args)
set_module_args(module_args)
def test_about_url_pass(self):
"""Verify about_url property returns expected about url."""
test_set = [("http://localhost/devmgr/v2", "http://localhost:8080/devmgr/utils/about"),
("http://localhost:8443/devmgr/v2", "https://localhost:8443/devmgr/utils/about"),
("http://localhost:8443/devmgr/v2/", "https://localhost:8443/devmgr/utils/about"),
("http://localhost:443/something_else", "https://localhost:8443/devmgr/utils/about"),
("http://localhost:8443", "https://localhost:8443/devmgr/utils/about"),
("http://localhost", "http://localhost:8080/devmgr/utils/about")]
for url in test_set:
self._set_args({"api_url": url[0]})
base = StubNetAppESeriesModule()
self.assertTrue(base._about_url == url[1])
def test_is_embedded_embedded_pass(self):
"""Verify is_embedded successfully returns True when an embedded web service's rest api is inquired."""
self._set_args()
with mock.patch(self.REQ_FUNC, return_value=(200, {"runningAsProxy": False})):
with patch(self.REQ_FUNC, side_effect=[(200, {"version": "03.10.9000.0009"}),
(200, {"runningAsProxy": False})]):
base = StubNetAppESeriesModule()
self.assertTrue(base.is_embedded())
with mock.patch(self.REQ_FUNC, return_value=(200, {"runningAsProxy": True})):
with patch(self.REQ_FUNC, side_effect=[(200, {"version": "03.10.9000.0009"}),
(200, {"runningAsProxy": True})]):
base = StubNetAppESeriesModule()
self.assertFalse(base.is_embedded())
def test_check_web_services_version_pass(self):
"""Verify that an acceptable rest api version passes."""
def test_is_embedded_fail(self):
"""Verify exception is thrown when a web service's rest api fails to return about information."""
self._set_args()
with patch(self.REQ_FUNC, side_effect=[(200, {"version": "03.10.9000.0009"}), Exception()]):
with self.assertRaisesRegexp(AnsibleFailJson, r"Failed to retrieve the webservices about information!"):
base = StubNetAppESeriesModule()
base.is_embedded()
with patch(self.REQ_FUNC, side_effect=[(200, {"version": "03.10.9000.0009"}), URLError(""), Exception()]):
with self.assertRaisesRegexp(AnsibleFailJson, r"Failed to retrieve the webservices about information!"):
base = StubNetAppESeriesModule()
base.is_embedded()
def test_check_web_services_version_fail(self):
"""Verify that an unacceptable rest api version fails."""
minimum_required = "02.10.9000.0010"
test_set = ["03.9.9000.0010", "03.10.9000.0009", "02.11.9000.0009", "02.10.9000.0010"]
test_set = ["02.10.9000.0009", "02.09.9000.0010", "01.10.9000.0010"]
self._set_args()
base = StubNetAppESeriesModule()
base.web_services_version = minimum_required
base.is_embedded = lambda: True
for current_version in test_set:
with mock.patch(self.REQ_FUNC, return_value=(200, {"version": current_version})):
self.assertTrue(base._is_web_services_valid())
with patch(self.REQ_FUNC, return_value=(200, {"version": current_version})):
with self.assertRaisesRegexp(AnsibleFailJson, r"version does not meet minimum version required."):
base._check_web_services_version()
def test_check_web_services_version_fail(self):
def test_check_web_services_version_pass(self):
"""Verify that an unacceptable rest api version fails."""
minimum_required = "02.10.9000.0010"
test_set = ["02.10.9000.0009", "02.09.9000.0010", "01.10.9000.0010"]
@ -75,51 +80,27 @@ class NetappTest(ModuleTestCase):
base.web_services_version = minimum_required
base.is_embedded = lambda: True
for current_version in test_set:
with mock.patch(self.REQ_FUNC, return_value=(200, {"version": current_version})):
with patch(self.REQ_FUNC, return_value=(200, {"version": current_version})):
with self.assertRaisesRegexp(AnsibleFailJson, r"version does not meet minimum version required."):
base._is_web_services_valid()
base._check_web_services_version()
def test_is_embedded_fail(self):
"""Verify exception is thrown when a web service's rest api fails to return about information."""
self._set_args()
with mock.patch(self.REQ_FUNC, return_value=Exception()):
with self.assertRaisesRegexp(AnsibleFailJson, r"Failed to retrieve the webservices about information!"):
base = StubNetAppESeriesModule()
base.is_embedded()
with mock.patch(self.REQ_FUNC, side_effect=[URLError(""), Exception()]):
with self.assertRaisesRegexp(AnsibleFailJson, r"Failed to retrieve the webservices about information!"):
base = StubNetAppESeriesModule()
base.is_embedded()
def test_check_check_web_services_version_fail(self):
"""Verify exception is thrown when api url is invalid."""
invalid_url_forms = ["localhost:8080/devmgr/v2",
"http:///devmgr/v2"]
def test_tweak_url_pass(self):
"""Verify a range of valid netapp eseries rest api urls pass."""
test_set = [("http://localhost/devmgr/v2", "http://localhost:8080/devmgr/v2/"),
("localhost", "https://localhost:8443/devmgr/v2/"),
("localhost:8443/devmgr/v2", "https://localhost:8443/devmgr/v2/"),
("https://localhost/devmgr/v2", "https://localhost:8443/devmgr/v2/"),
("http://localhost:8443", "https://localhost:8443/devmgr/v2/"),
("http://localhost:/devmgr/v2", "https://localhost:8443/devmgr/v2/"),
("http://localhost:8080", "http://localhost:8080/devmgr/v2/"),
("http://localhost", "http://localhost:8080/devmgr/v2/"),
("localhost/devmgr/v2", "https://localhost:8443/devmgr/v2/"),
("localhost/devmgr", "https://localhost:8443/devmgr/v2/"),
("localhost/devmgr/v3", "https://localhost:8443/devmgr/v2/"),
("localhost/something", "https://localhost:8443/devmgr/v2/"),
("ftp://localhost", "https://localhost:8443/devmgr/v2/"),
("ftp://localhost:8080", "http://localhost:8080/devmgr/v2/"),
("ftp://localhost/devmgr/v2/", "https://localhost:8443/devmgr/v2/")]
for test in test_set:
self._set_args({"api_url": test[0]})
with mock.patch(self.REQ_FUNC, side_effect=[URLError(""), (200, {"runningAsProxy": False})]):
invalid_url_protocols = ["ssh://localhost:8080/devmgr/v2"]
for url in invalid_url_forms:
self._set_args({"api_url": url})
with patch(self.REQ_FUNC, return_value=(200, {"runningAsProxy": True})):
with self.assertRaisesRegexp(AnsibleFailJson, r"Failed to provide valid API URL."):
base = StubNetAppESeriesModule()
base._tweak_url()
self.assertTrue(base.url == test[1])
def test_check_url_missing_hostname_fail(self):
"""Verify exception is thrown when hostname or ip address is missing."""
self._set_args({"api_url": "http:///devmgr/v2"})
with mock.patch(self.REQ_FUNC, return_value=(200, {"runningAsProxy": True})):
with self.assertRaisesRegexp(AnsibleFailJson, r"Failed to provide a valid hostname or IP address."):
base._check_web_services_version()
for url in invalid_url_protocols:
self._set_args({"api_url": url})
with patch(self.REQ_FUNC, return_value=(200, {"runningAsProxy": True})):
with self.assertRaisesRegexp(AnsibleFailJson, r"Protocol must be http or https."):
base = StubNetAppESeriesModule()
base._tweak_url()
base._check_web_services_version()

Write Preview
Loading…
Cancel
Save