Improved netapp module utility for E-Series. (#59527)

Add header option to request method in NetAppESeriesModule
Add multipart formdata builder function
Fix issue with url port change
pull/58151/head
Nathan Swartz 6 years ago committed by Jake Jackson
parent a01ee2759d
commit 26fff6f5c3

@ -29,8 +29,11 @@
import json import json
import os import os
import random
import mimetypes
from pprint import pformat from pprint import pformat
from ansible.module_utils import six
from ansible.module_utils.basic import AnsibleModule, missing_required_lib from ansible.module_utils.basic import AnsibleModule, missing_required_lib
from ansible.module_utils.six.moves.urllib.error import HTTPError, URLError from ansible.module_utils.six.moves.urllib.error import HTTPError, URLError
from ansible.module_utils.urls import open_url from ansible.module_utils.urls import open_url
@ -245,7 +248,8 @@ class NetAppESeriesModule(object):
""" """
DEFAULT_TIMEOUT = 60 DEFAULT_TIMEOUT = 60
DEFAULT_SECURE_PORT = "8443" DEFAULT_SECURE_PORT = "8443"
DEFAULT_REST_API_PATH = "devmgr/v2" DEFAULT_REST_API_PATH = "devmgr/v2/"
DEFAULT_REST_API_ABOUT_PATH = "devmgr/utils/about"
DEFAULT_HEADERS = {"Content-Type": "application/json", "Accept": "application/json", DEFAULT_HEADERS = {"Content-Type": "application/json", "Accept": "application/json",
"netapp-client-type": "Ansible-%s" % ansible_version} "netapp-client-type": "Ansible-%s" % ansible_version}
HTTP_AGENT = "Ansible / %s" % ansible_version HTTP_AGENT = "Ansible / %s" % ansible_version
@ -264,122 +268,65 @@ class NetAppESeriesModule(object):
args = self.module.params args = self.module.params
self.web_services_version = web_services_version if web_services_version else "02.00.0000.0000" self.web_services_version = web_services_version if web_services_version else "02.00.0000.0000"
self.url = args["api_url"]
self.ssid = args["ssid"] self.ssid = args["ssid"]
self.url = args["api_url"]
self.log_requests = log_requests
self.creds = dict(url_username=args["api_username"], self.creds = dict(url_username=args["api_username"],
url_password=args["api_password"], url_password=args["api_password"],
validate_certs=args["validate_certs"]) validate_certs=args["validate_certs"])
self.log_requests = log_requests
self.is_embedded_mode = None
self.web_services_validate = None
self._tweak_url()
@property
def _about_url(self):
"""Generates the about url based on the supplied web services rest api url.
:raise AnsibleFailJson: raised when supplied web services rest api is an invalid url.
:return: proxy or embedded about url.
"""
about = list(urlparse(self.url))
about[2] = "devmgr/utils/about"
return urlunparse(about)
def _force_secure_url(self):
"""Modifies supplied web services rest api to use secure https.
raise: AnsibleFailJson: raised when the url already utilizes the secure protocol
"""
url_parts = list(urlparse(self.url))
if "https://" in self.url and ":8443" in self.url:
self.module.fail_json(msg="Secure HTTP protocol already used. URL path [%s]" % self.url)
url_parts[0] = "https"
url_parts[1] = "%s:8443" % url_parts[1].split(":")[0]
self.url = urlunparse(url_parts)
if not self.url.endswith("/"): if not self.url.endswith("/"):
self.url += "/" self.url += "/"
self.module.warn("forced use of the secure protocol: %s" % self.url) self.is_embedded_mode = None
self.is_web_services_valid_cache = None
def _tweak_url(self): def _check_web_services_version(self):
"""Adjust the rest api url is necessary. """Verify proxy or embedded web services meets minimum version required for module.
:raise AnsibleFailJson: raised when self.url fails to have a hostname or ipv4 address. The minimum required web services version is evaluated against version supplied through the web services rest
""" api. AnsibleFailJson exception will be raised when the minimum is not met or exceeded.
# ensure the protocol is either http or https
if self.url.split("://")[0] not in ["https", "http"]:
self.url = self.url.split("://")[1] if "://" in self.url else self.url This helper function will update the supplied api url if secure http is not used for embedded web services
if ":8080" in self.url: :raise AnsibleFailJson: raised when the contacted api service does not meet the minimum required version.
self.url = "http://%s" % self.url """
else: if not self.is_web_services_valid_cache:
self.url = "https://%s" % self.url
# parse url and verify protocol, port and path are consistent with required web services rest api url.
url_parts = list(urlparse(self.url)) url_parts = list(urlparse(self.url))
if url_parts[1] == "": if not url_parts[0] or not url_parts[1]:
self.module.fail_json(msg="Failed to provide a valid hostname or IP address. URL [%s]." % self.url) self.module.fail_json(msg="Failed to provide valid API URL. Example: https://192.168.1.100:8443/devmgr/v2. URL [%s]." % self.url)
split_hostname = url_parts[1].split(":")
if url_parts[0] not in ["https", "http"] or (len(split_hostname) == 2 and split_hostname[1] != "8080"):
if len(split_hostname) == 2 and split_hostname[1] == "8080":
url_parts[0] = "http"
url_parts[1] = "%s:8080" % split_hostname[0]
else:
url_parts[0] = "https"
url_parts[1] = "%s:8443" % split_hostname[0]
elif len(split_hostname) == 1:
if url_parts[0] == "https":
url_parts[1] = "%s:8443" % split_hostname[0]
elif url_parts[0] == "http":
url_parts[1] = "%s:8080" % split_hostname[0]
if url_parts[2] == "" or url_parts[2] != self.DEFAULT_REST_API_PATH:
url_parts[2] = self.DEFAULT_REST_API_PATH
self.url = urlunparse(url_parts)
if not self.url.endswith("/"):
self.url += "/"
self.module.log("valid url: %s" % self.url)
def _is_web_services_valid(self): if url_parts[0] not in ["http", "https"]:
"""Verify proxy or embedded web services meets minimum version required for module. self.module.fail_json(msg="Protocol must be http or https. URL [%s]." % self.url)
The minimum required web services version is evaluated against version supplied through the web services rest self.url = "%s://%s/" % (url_parts[0], url_parts[1])
api. AnsibleFailJson exception will be raised when the minimum is not met or exceeded. about_url = self.url + self.DEFAULT_REST_API_ABOUT_PATH
rc, data = request(about_url, timeout=self.DEFAULT_TIMEOUT, headers=self.DEFAULT_HEADERS, ignore_errors=True, **self.creds)
:raise AnsibleFailJson: raised when the contacted api service does not meet the minimum required version. if rc != 200:
""" self.module.warn("Failed to retrieve web services about information! Retrying with secure ports. Array Id [%s]." % self.ssid)
if not self.web_services_validate: self.url = "https://%s:8443/" % url_parts[1].split(":")[0]
self.is_embedded() about_url = self.url + self.DEFAULT_REST_API_ABOUT_PATH
try: try:
rc, data = request(self._about_url, timeout=self.DEFAULT_TIMEOUT, rc, data = request(about_url, timeout=self.DEFAULT_TIMEOUT, headers=self.DEFAULT_HEADERS, **self.creds)
headers=self.DEFAULT_HEADERS, **self.creds) except Exception as error:
self.module.fail_json(msg="Failed to retrieve the webservices about information! Array Id [%s]. Error [%s]."
% (self.ssid, to_native(error)))
major, minor, other, revision = data["version"].split(".") major, minor, other, revision = data["version"].split(".")
minimum_major, minimum_minor, other, minimum_revision = self.web_services_version.split(".") minimum_major, minimum_minor, other, minimum_revision = self.web_services_version.split(".")
if not (major > minimum_major or if not (major > minimum_major or
(major == minimum_major and minor > minimum_minor) or (major == minimum_major and minor > minimum_minor) or
(major == minimum_major and minor == minimum_minor and revision >= minimum_revision)): (major == minimum_major and minor == minimum_minor and revision >= minimum_revision)):
self.module.fail_json( self.module.fail_json(msg="Web services version does not meet minimum version required. Current version: [%s]."
msg="Web services version does not meet minimum version required. Current version: [%s]."
" Version required: [%s]." % (data["version"], self.web_services_version)) " Version required: [%s]." % (data["version"], self.web_services_version))
except Exception as error:
self.module.fail_json(msg="Failed to retrieve the webservices about information! Array Id [%s]."
" Error [%s]." % (self.ssid, to_native(error)))
self.module.warn("Web services rest api version met the minimum required version.")
self.web_services_validate = True
return self.web_services_validate self.module.log("Web services rest api version met the minimum required version.")
self.is_web_services_valid_cache = True
def is_embedded(self, retry=True): def is_embedded(self):
"""Determine whether web services server is the embedded web services. """Determine whether web services server is the embedded web services.
If web services about endpoint fails based on an URLError then the request will be attempted again using If web services about endpoint fails based on an URLError then the request will be attempted again using
@ -388,59 +335,101 @@ class NetAppESeriesModule(object):
:raise AnsibleFailJson: raised when web services about endpoint failed to be contacted. :raise AnsibleFailJson: raised when web services about endpoint failed to be contacted.
:return bool: whether contacted web services is running from storage array (embedded) or from a proxy. :return bool: whether contacted web services is running from storage array (embedded) or from a proxy.
""" """
self._check_web_services_version()
if self.is_embedded_mode is None: if self.is_embedded_mode is None:
about_url = self.url + self.DEFAULT_REST_API_ABOUT_PATH
try: try:
rc, data = request(self._about_url, timeout=self.DEFAULT_TIMEOUT, rc, data = request(about_url, timeout=self.DEFAULT_TIMEOUT, headers=self.DEFAULT_HEADERS, **self.creds)
headers=self.DEFAULT_HEADERS, **self.creds)
self.is_embedded_mode = not data["runningAsProxy"] self.is_embedded_mode = not data["runningAsProxy"]
except URLError as error:
if not retry:
self.module.fail_json(msg="Failed to retrieve the webservices about information! Array Id [%s]."
" Error [%s]." % (self.ssid, to_native(error)))
self.module.warn("Failed to retrieve the webservices about information! Will retry using secure"
" http. Array Id [%s]." % self.ssid)
self._force_secure_url()
return self.is_embedded(retry=False)
except Exception as error: except Exception as error:
self.module.fail_json(msg="Failed to retrieve the webservices about information! Array Id [%s]." self.module.fail_json(msg="Failed to retrieve the webservices about information! Array Id [%s]. Error [%s]."
" Error [%s]." % (self.ssid, to_native(error))) % (self.ssid, to_native(error)))
return self.is_embedded_mode return self.is_embedded_mode
def request(self, path, data=None, method='GET', ignore_errors=False): def request(self, path, data=None, method='GET', headers=None, ignore_errors=False):
"""Issue an HTTP request to a url, retrieving an optional JSON response. """Issue an HTTP request to a url, retrieving an optional JSON response.
:param str path: web services rest api endpoint path (Example: storage-systems/1/graph). Note that when the :param str path: web services rest api endpoint path (Example: storage-systems/1/graph). Note that when the
full url path is specified then that will be used without supplying the protocol, hostname, port and rest path. full url path is specified then that will be used without supplying the protocol, hostname, port and rest path.
:param data: data required for the request (data may be json or any python structured data) :param data: data required for the request (data may be json or any python structured data)
:param str method: request method such as GET, POST, DELETE. :param str method: request method such as GET, POST, DELETE.
:param dict headers: dictionary containing request headers.
:param bool ignore_errors: forces the request to ignore any raised exceptions. :param bool ignore_errors: forces the request to ignore any raised exceptions.
""" """
if self._is_web_services_valid(): self._check_web_services_version()
url = list(urlparse(path.strip("/")))
if url[2] == "": if headers is None:
self.module.fail_json(msg="Web services rest api endpoint path must be specified. Path [%s]." % path) headers = self.DEFAULT_HEADERS
# if either the protocol or hostname/port are missing then add them. if not isinstance(data, str) and headers["Content-Type"] == "application/json":
if url[0] == "" or url[1] == "":
url[0], url[1] = list(urlparse(self.url))[:2]
# add rest api path if the supplied path does not begin with it.
if not all([word in url[2].split("/")[:2] for word in self.DEFAULT_REST_API_PATH.split("/")]):
if not url[2].startswith("/"):
url[2] = "/" + url[2]
url[2] = self.DEFAULT_REST_API_PATH + url[2]
# ensure data is json formatted
if not isinstance(data, str):
data = json.dumps(data) data = json.dumps(data)
if self.log_requests: if path.startswith("/"):
self.module.log(pformat(dict(url=urlunparse(url), data=data, method=method))) path = path[1:]
request_url = self.url + self.DEFAULT_REST_API_PATH + path
if self.log_requests or True:
self.module.log(pformat(dict(url=request_url, data=data, method=method)))
return request(url=request_url, data=data, method=method, headers=headers, use_proxy=True, force=False, last_mod_time=None,
timeout=self.DEFAULT_TIMEOUT, http_agent=self.HTTP_AGENT, force_basic_auth=True, ignore_errors=ignore_errors, **self.creds)
return request(url=urlunparse(url), data=data, method=method, headers=self.DEFAULT_HEADERS, use_proxy=True,
force=False, last_mod_time=None, timeout=self.DEFAULT_TIMEOUT, http_agent=self.HTTP_AGENT, def create_multipart_formdata(files, fields=None, send_8kb=False):
force_basic_auth=True, ignore_errors=ignore_errors, **self.creds) """Create the data for a multipart/form request."""
boundary = "---------------------------" + "".join([str(random.randint(0, 9)) for x in range(27)])
data_parts = list()
data = None
if six.PY2: # Generate payload for Python 2
newline = "\r\n"
if fields is not None:
for key, value in fields:
data_parts.extend(["--%s" % boundary,
'Content-Disposition: form-data; name="%s"' % key,
"",
value])
for name, filename, path in files:
with open(path, "rb") as fh:
value = fh.read(8192) if send_8kb else fh.read()
data_parts.extend(["--%s" % boundary,
'Content-Disposition: form-data; name="%s"; filename="%s"' % (name, filename),
"Content-Type: %s" % (mimetypes.guess_type(path)[0] or "application/octet-stream"),
"",
value])
data_parts.extend(["--%s--" % boundary, ""])
data = newline.join(data_parts)
else:
newline = six.b("\r\n")
if fields is not None:
for key, value in fields:
data_parts.extend([six.b("--%s" % boundary),
six.b('Content-Disposition: form-data; name="%s"' % key),
six.b(""),
six.b(value)])
for name, filename, path in files:
with open(path, "rb") as fh:
value = fh.read(8192) if send_8kb else fh.read()
data_parts.extend([six.b("--%s" % boundary),
six.b('Content-Disposition: form-data; name="%s"; filename="%s"' % (name, filename)),
six.b("Content-Type: %s" % (mimetypes.guess_type(path)[0] or "application/octet-stream")),
six.b(""),
value])
data_parts.extend([six.b("--%s--" % boundary), b""])
data = newline.join(data_parts)
headers = {
"Content-Type": "multipart/form-data; boundary=%s" % boundary,
"Content-Length": str(len(data))}
return headers, data
def request(url, data=None, headers=None, method='GET', use_proxy=True, def request(url, data=None, headers=None, method='GET', use_proxy=True,

@ -1,13 +1,17 @@
# (c) 2018, NetApp Inc. # (c) 2018, NetApp Inc.
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from ansible.module_utils.six.moves.urllib.error import URLError
from ansible.module_utils.netapp import NetAppESeriesModule
from units.modules.utils import ModuleTestCase, set_module_args, AnsibleFailJson
__metaclass__ = type __metaclass__ = type
from units.compat import mock
try:
from unittest.mock import patch, mock_open
except ImportError:
from mock import patch, mock_open
from ansible.module_utils.six.moves.urllib.error import URLError
from ansible.module_utils.netapp import NetAppESeriesModule, create_multipart_formdata
from units.modules.utils import AnsibleExitJson, AnsibleFailJson, ModuleTestCase, set_module_args
class StubNetAppESeriesModule(NetAppESeriesModule): class StubNetAppESeriesModule(NetAppESeriesModule):
@ -28,44 +32,45 @@ class NetappTest(ModuleTestCase):
module_args.update(args) module_args.update(args)
set_module_args(module_args) set_module_args(module_args)
def test_about_url_pass(self):
"""Verify about_url property returns expected about url."""
test_set = [("http://localhost/devmgr/v2", "http://localhost:8080/devmgr/utils/about"),
("http://localhost:8443/devmgr/v2", "https://localhost:8443/devmgr/utils/about"),
("http://localhost:8443/devmgr/v2/", "https://localhost:8443/devmgr/utils/about"),
("http://localhost:443/something_else", "https://localhost:8443/devmgr/utils/about"),
("http://localhost:8443", "https://localhost:8443/devmgr/utils/about"),
("http://localhost", "http://localhost:8080/devmgr/utils/about")]
for url in test_set:
self._set_args({"api_url": url[0]})
base = StubNetAppESeriesModule()
self.assertTrue(base._about_url == url[1])
def test_is_embedded_embedded_pass(self): def test_is_embedded_embedded_pass(self):
"""Verify is_embedded successfully returns True when an embedded web service's rest api is inquired.""" """Verify is_embedded successfully returns True when an embedded web service's rest api is inquired."""
self._set_args() self._set_args()
with mock.patch(self.REQ_FUNC, return_value=(200, {"runningAsProxy": False})): with patch(self.REQ_FUNC, side_effect=[(200, {"version": "03.10.9000.0009"}),
(200, {"runningAsProxy": False})]):
base = StubNetAppESeriesModule() base = StubNetAppESeriesModule()
self.assertTrue(base.is_embedded()) self.assertTrue(base.is_embedded())
with mock.patch(self.REQ_FUNC, return_value=(200, {"runningAsProxy": True})): with patch(self.REQ_FUNC, side_effect=[(200, {"version": "03.10.9000.0009"}),
(200, {"runningAsProxy": True})]):
base = StubNetAppESeriesModule() base = StubNetAppESeriesModule()
self.assertFalse(base.is_embedded()) self.assertFalse(base.is_embedded())
def test_check_web_services_version_pass(self): def test_is_embedded_fail(self):
"""Verify that an acceptable rest api version passes.""" """Verify exception is thrown when a web service's rest api fails to return about information."""
self._set_args()
with patch(self.REQ_FUNC, side_effect=[(200, {"version": "03.10.9000.0009"}), Exception()]):
with self.assertRaisesRegexp(AnsibleFailJson, r"Failed to retrieve the webservices about information!"):
base = StubNetAppESeriesModule()
base.is_embedded()
with patch(self.REQ_FUNC, side_effect=[(200, {"version": "03.10.9000.0009"}), URLError(""), Exception()]):
with self.assertRaisesRegexp(AnsibleFailJson, r"Failed to retrieve the webservices about information!"):
base = StubNetAppESeriesModule()
base.is_embedded()
def test_check_web_services_version_fail(self):
"""Verify that an unacceptable rest api version fails."""
minimum_required = "02.10.9000.0010" minimum_required = "02.10.9000.0010"
test_set = ["03.9.9000.0010", "03.10.9000.0009", "02.11.9000.0009", "02.10.9000.0010"] test_set = ["02.10.9000.0009", "02.09.9000.0010", "01.10.9000.0010"]
self._set_args() self._set_args()
base = StubNetAppESeriesModule() base = StubNetAppESeriesModule()
base.web_services_version = minimum_required base.web_services_version = minimum_required
base.is_embedded = lambda: True base.is_embedded = lambda: True
for current_version in test_set: for current_version in test_set:
with mock.patch(self.REQ_FUNC, return_value=(200, {"version": current_version})): with patch(self.REQ_FUNC, return_value=(200, {"version": current_version})):
self.assertTrue(base._is_web_services_valid()) with self.assertRaisesRegexp(AnsibleFailJson, r"version does not meet minimum version required."):
base._check_web_services_version()
def test_check_web_services_version_fail(self): def test_check_web_services_version_pass(self):
"""Verify that an unacceptable rest api version fails.""" """Verify that an unacceptable rest api version fails."""
minimum_required = "02.10.9000.0010" minimum_required = "02.10.9000.0010"
test_set = ["02.10.9000.0009", "02.09.9000.0010", "01.10.9000.0010"] test_set = ["02.10.9000.0009", "02.09.9000.0010", "01.10.9000.0010"]
@ -75,51 +80,27 @@ class NetappTest(ModuleTestCase):
base.web_services_version = minimum_required base.web_services_version = minimum_required
base.is_embedded = lambda: True base.is_embedded = lambda: True
for current_version in test_set: for current_version in test_set:
with mock.patch(self.REQ_FUNC, return_value=(200, {"version": current_version})): with patch(self.REQ_FUNC, return_value=(200, {"version": current_version})):
with self.assertRaisesRegexp(AnsibleFailJson, r"version does not meet minimum version required."): with self.assertRaisesRegexp(AnsibleFailJson, r"version does not meet minimum version required."):
base._is_web_services_valid() base._check_web_services_version()
def test_is_embedded_fail(self): def test_check_check_web_services_version_fail(self):
"""Verify exception is thrown when a web service's rest api fails to return about information.""" """Verify exception is thrown when api url is invalid."""
self._set_args() invalid_url_forms = ["localhost:8080/devmgr/v2",
with mock.patch(self.REQ_FUNC, return_value=Exception()): "http:///devmgr/v2"]
with self.assertRaisesRegexp(AnsibleFailJson, r"Failed to retrieve the webservices about information!"):
base = StubNetAppESeriesModule()
base.is_embedded()
with mock.patch(self.REQ_FUNC, side_effect=[URLError(""), Exception()]):
with self.assertRaisesRegexp(AnsibleFailJson, r"Failed to retrieve the webservices about information!"):
base = StubNetAppESeriesModule()
base.is_embedded()
def test_tweak_url_pass(self): invalid_url_protocols = ["ssh://localhost:8080/devmgr/v2"]
"""Verify a range of valid netapp eseries rest api urls pass."""
test_set = [("http://localhost/devmgr/v2", "http://localhost:8080/devmgr/v2/"), for url in invalid_url_forms:
("localhost", "https://localhost:8443/devmgr/v2/"), self._set_args({"api_url": url})
("localhost:8443/devmgr/v2", "https://localhost:8443/devmgr/v2/"), with patch(self.REQ_FUNC, return_value=(200, {"runningAsProxy": True})):
("https://localhost/devmgr/v2", "https://localhost:8443/devmgr/v2/"), with self.assertRaisesRegexp(AnsibleFailJson, r"Failed to provide valid API URL."):
("http://localhost:8443", "https://localhost:8443/devmgr/v2/"),
("http://localhost:/devmgr/v2", "https://localhost:8443/devmgr/v2/"),
("http://localhost:8080", "http://localhost:8080/devmgr/v2/"),
("http://localhost", "http://localhost:8080/devmgr/v2/"),
("localhost/devmgr/v2", "https://localhost:8443/devmgr/v2/"),
("localhost/devmgr", "https://localhost:8443/devmgr/v2/"),
("localhost/devmgr/v3", "https://localhost:8443/devmgr/v2/"),
("localhost/something", "https://localhost:8443/devmgr/v2/"),
("ftp://localhost", "https://localhost:8443/devmgr/v2/"),
("ftp://localhost:8080", "http://localhost:8080/devmgr/v2/"),
("ftp://localhost/devmgr/v2/", "https://localhost:8443/devmgr/v2/")]
for test in test_set:
self._set_args({"api_url": test[0]})
with mock.patch(self.REQ_FUNC, side_effect=[URLError(""), (200, {"runningAsProxy": False})]):
base = StubNetAppESeriesModule() base = StubNetAppESeriesModule()
base._tweak_url() base._check_web_services_version()
self.assertTrue(base.url == test[1])
for url in invalid_url_protocols:
def test_check_url_missing_hostname_fail(self): self._set_args({"api_url": url})
"""Verify exception is thrown when hostname or ip address is missing.""" with patch(self.REQ_FUNC, return_value=(200, {"runningAsProxy": True})):
self._set_args({"api_url": "http:///devmgr/v2"}) with self.assertRaisesRegexp(AnsibleFailJson, r"Protocol must be http or https."):
with mock.patch(self.REQ_FUNC, return_value=(200, {"runningAsProxy": True})):
with self.assertRaisesRegexp(AnsibleFailJson, r"Failed to provide a valid hostname or IP address."):
base = StubNetAppESeriesModule() base = StubNetAppESeriesModule()
base._tweak_url() base._check_web_services_version()

Loading…
Cancel
Save