huawei: use new client (#55368)

pull/55795/head
zengchen 6 years ago committed by René Moser
parent a237e2dd6a
commit 1d49313dc4

@ -2,16 +2,9 @@
# Simplified BSD License (see licenses/simplified_bsd.txt or
# https://opensource.org/licenses/BSD-2-Clause)
import re
import traceback
REQUESTS_IMP_ERR = None
try:
import requests
HAS_REQUESTS = True
except ImportError:
REQUESTS_IMP_ERR = traceback.format_exc()
HAS_REQUESTS = False
THIRD_LIBRARIES_IMP_ERR = None
try:
from keystoneauth1.adapter import Adapter
@ -22,7 +15,8 @@ except ImportError:
THIRD_LIBRARIES_IMP_ERR = traceback.format_exc()
HAS_THIRD_LIBRARIES = False
from ansible.module_utils.basic import AnsibleModule, env_fallback, missing_required_lib
from ansible.module_utils.basic import (AnsibleModule, env_fallback,
missing_required_lib)
from ansible.module_utils._text import to_text
@ -41,10 +35,6 @@ def navigate_hash(source, path, default=None):
return result
class HwcRequestException(Exception):
pass
def remove_empty_from_dict(obj):
return _DictClean(
obj,
@ -57,7 +47,8 @@ def remove_nones_from_dict(obj):
def replace_resource_dict(item, value):
""" Handles the replacement of dicts with values -> the needed value for HWC API"""
""" Handles the replacement of dicts with values ->
the needed value for HWC API"""
if isinstance(item, list):
items = []
for i in item:
@ -84,112 +75,184 @@ def are_dicts_different(expect, actual):
return DictComparison(expect_vals) != DictComparison(actual_vals)
class HwcSession(object):
"""Handles all authentation and HTTP sessions for HWC API calls."""
class HwcClientException(Exception):
def __init__(self, code, message):
super(HwcClientException, self).__init__()
self._code = code
self._message = message
def __str__(self):
msg = " code=%s," % str(self._code) if self._code != 0 else ""
return "[HwcClientException]%s message=%s" % (
msg, self._message)
class HwcClientException404(HwcClientException):
def __init__(self, message):
super(HwcClientException404, self).__init__(404, message)
def __str__(self):
return "[HwcClientException404] message=%s" % self._message
def session_method_wrapper(f):
def _wrap(self, url, *args, **kwargs):
try:
url = self.endpoint + url
r = f(self, url, *args, **kwargs)
except Exception as ex:
raise HwcClientException(
0, "Sending request failed, error=%s" % ex)
result = None
if r.content:
try:
result = r.json()
except Exception as ex:
raise HwcClientException(
0, "Parsing response to json failed, error: %s" % ex)
code = r.status_code
if code not in [200, 201, 202, 203, 204, 205, 206, 207, 208, 226]:
msg = ""
for i in [['message'], ['error', 'message']]:
try:
msg = navigate_hash(result, i)
break
except Exception:
pass
else:
msg = str(result)
if code == 404:
raise HwcClientException404(msg)
raise HwcClientException(code, msg)
return result
return _wrap
class _ServiceClient(object):
def __init__(self, client, endpoint, product):
self._client = client
self._endpoint = endpoint
self._default_header = {
'User-Agent': "Huawei-Ansible-MM-%s" % product,
'Accept': 'application/json',
}
@property
def endpoint(self):
return self._endpoint
@session_method_wrapper
def get(self, url, body=None, header=None, timeout=None):
return self._client.get(url, json=body, timeout=timeout,
headers=self._header(header))
@session_method_wrapper
def post(self, url, body=None, header=None, timeout=None):
return self._client.post(url, json=body, timeout=timeout,
headers=self._header(header))
@session_method_wrapper
def delete(self, url, body=None, header=None, timeout=None):
return self._client.delete(url, json=body, timeout=timeout,
headers=self._header(header))
@session_method_wrapper
def put(self, url, body=None, header=None, timeout=None):
return self._client.put(url, json=body, timeout=timeout,
headers=self._header(header))
def _header(self, header):
if header and isinstance(header, dict):
for k, v in self._default_header.items():
if k not in header:
header[k] = v
else:
header = self._default_header
return header
class Config(object):
def __init__(self, module, product):
self.module = module
self.product = product
self._validate()
self._session = self._credentials()
self._adapter = Adapter(self._session)
self._project_client = None
self._domain_client = None
self._module = module
self._product = product
self._endpoints = {}
self._project_id = ""
def get(self, url, body=None):
try:
return self._adapter.get(
url, json=body,
headers=self._headers(), raise_exc=False)
except getattr(requests.exceptions, 'RequestException') as inst:
self.module.fail_json(msg=inst.message)
self._validate()
self._gen_provider_client()
@property
def module(self):
return self._module
def client(self, region, service_type, service_level):
c = self._project_client
if service_level == "domain":
c = self._domain_client
e = self._get_service_endpoint(c, service_type, region)
return _ServiceClient(c, e, self._product)
def _gen_provider_client(self):
m = self._module
p = {
"auth_url": m.params['identity_endpoint'],
"password": m.params['password'],
"username": m.params['user'],
"project_name": m.params['project'],
"user_domain_name": m.params['domain'],
"reauthenticate": True
}
def post(self, url, body=None):
try:
return self._adapter.post(
url, json=body,
headers=self._headers(), raise_exc=False)
except getattr(requests.exceptions, 'RequestException') as inst:
self.module.fail_json(msg=inst.message)
self._project_client = Adapter(
session.Session(auth=v3.Password(**p)),
raise_exc=False)
def delete(self, url, body=None):
try:
return self._adapter.delete(
url, json=body,
headers=self._headers(), raise_exc=False)
except getattr(requests.exceptions, 'RequestException') as inst:
self.module.fail_json(msg=inst.message)
p.pop("project_name")
self._domain_client = Adapter(
session.Session(auth=v3.Password(**p)),
raise_exc=False)
def put(self, url, body=None):
try:
return self._adapter.put(
url, json=body,
headers=self._headers(), raise_exc=False)
except getattr(requests.exceptions, 'RequestException') as inst:
self.module.fail_json(msg=inst.message)
def _get_service_endpoint(self, client, service_type, region):
k = "%s.%s" % (service_type, region if region else "")
def get_service_endpoint(self, service_type):
if self._endpoints.get(service_type):
return self._endpoints.get(service_type)
if k in self._endpoints:
return self._endpoints.get(k)
e = None
url = None
try:
e = self._session.get_endpoint_data(
service_type=service_type,
region_name=self.module.params['region']
)
except getattr(requests.exceptions, 'RequestException') as inst:
self.module.fail_json(msg=inst.message)
url = client.get_endpoint(service_type=service_type,
region_name=region, interface="public")
except Exception as ex:
raise HwcClientException(
0, "Getting endpoint failed, error=%s" % ex)
if not e or e.url == "":
self.module.fail_json(
msg="Can not find the endpoint for %s" % service_type)
if url == "":
raise HwcClientException(
0, "Can not find the enpoint for %s" % service_type)
url = e.url
if url[-1] != "/":
url += "/"
self._endpoints[service_type] = url
self._endpoints[k] = url
return url
def get_project_id(self):
if self._project_id:
return self._project_id
try:
pid = self._session.get_project_id()
self._project_id = pid
return pid
except getattr(requests.exceptions, 'RequestException') as inst:
self.module.fail_json(msg=inst.message)
def _validate(self):
if not HAS_REQUESTS:
self.module.fail_json(msg=missing_required_lib('requests'),
exception=REQUESTS_IMP_ERR)
if not HAS_THIRD_LIBRARIES:
self.module.fail_json(
msg=missing_required_lib('keystoneauth1'),
exception=THIRD_LIBRARIES_IMP_ERR)
def _credentials(self):
auth = v3.Password(
auth_url=self.module.params['identity_endpoint'],
password=self.module.params['password'],
username=self.module.params['user'],
user_domain_name=self.module.params['domain'],
project_name=self.module.params['project'],
reauthenticate=True
)
return session.Session(auth=auth)
def _headers(self):
return {
'User-Agent': "Huawei-Ansible-MM-%s" % self.product,
'Accept': 'application/json',
}
class HwcModule(AnsibleModule):
def __init__(self, *args, **kwargs):
@ -322,3 +385,30 @@ class _DictClean(object):
if self.keep_it(v1):
r.append(v1)
return r
def build_path(module, path, kv=None):
if kv is None:
kv = dict()
v = {}
for p in re.findall(r"{[^/]*}", path):
n = p[1:][:-1]
if n in kv:
v[n] = str(kv[n])
else:
if n in module.params:
v[n] = str(module.params.get(n))
else:
v[n] = ""
return path.format(**v)
def get_region(module):
if module.params['region']:
return module.params['region']
return module.params['project_name'].split("_")[0]

@ -110,12 +110,12 @@ RETURN = '''
# Imports
###############################################################################
from ansible.module_utils.hwc_utils import (HwcSession, HwcModule,
DictComparison, navigate_hash,
remove_nones_from_dict,
from ansible.module_utils.hwc_utils import (Config, HwcModule, get_region,
HwcClientException, navigate_hash,
HwcClientException404,
remove_nones_from_dict, build_path,
remove_empty_from_dict,
are_dicts_different)
import json
import re
import time
@ -135,18 +135,19 @@ def main():
),
supports_check_mode=True,
)
session = HwcSession(module, 'network')
config = Config(module, 'vpc')
state = module.params['state']
if (not module.params.get("id")) and module.params.get("name"):
module.params['id'] = get_id_by_name(session)
module.params['id'] = get_id_by_name(config)
fetch = None
link = self_link(session)
link = self_link(module)
# the link will include Nones if required format parameters are missed
if not re.search('/None/|/None$', link):
fetch = fetch_resource(session, link)
client = config.client(get_region(module), "vpc", "project")
fetch = fetch_resource(module, client, link)
if fetch:
fetch = fetch.get('vpc')
changed = False
@ -157,20 +158,20 @@ def main():
current_state = response_to_hash(module, fetch)
if are_dicts_different(expect, current_state):
if not module.check_mode:
fetch = update(session, self_link(session), [200])
fetch = update(config, self_link(module))
fetch = response_to_hash(module, fetch.get('vpc'))
changed = True
else:
fetch = current_state
else:
if not module.check_mode:
delete(session, self_link(session))
delete(config, self_link(module))
fetch = {}
changed = True
else:
if state == 'present':
if not module.check_mode:
fetch = create(session, collection(session), [200])
fetch = create(config, "vpcs")
fetch = response_to_hash(module, fetch.get('vpc'))
changed = True
else:
@ -181,72 +182,89 @@ def main():
module.exit_json(**fetch)
def create(session, link, success_codes=None):
if not success_codes:
success_codes = [201, 202]
module = session.module
r = return_if_object(module, session.post(link, resource_to_create(module)), success_codes)
def create(config, link):
module = config.module
client = config.client(get_region(module), "vpc", "project")
wait_done = wait_for_operation(session, 'create', r)
r = None
try:
r = client.post(link, resource_to_create(module))
except HwcClientException as ex:
msg = ("module(hwc_network_vpc): error creating "
"resource, error: %s" % str(ex))
module.fail_json(msg=msg)
url = resource_get_url(session, wait_done)
return fetch_resource(session, url)
wait_done = wait_for_operation(config, 'create', r)
v = navigate_hash(wait_done, ['vpc', 'id'])
url = build_path(module, 'vpcs/{op_id}', {'op_id': v})
return fetch_resource(module, client, url)
def update(session, link, success_codes=None):
if not success_codes:
success_codes = [201, 202]
module = session.module
r = return_if_object(module, session.put(link, resource_to_update(module)), success_codes)
wait_done = wait_for_operation(session, 'update', r)
def update(config, link):
module = config.module
client = config.client(get_region(module), "vpc", "project")
url = resource_get_url(session, wait_done)
return fetch_resource(session, url)
r = None
try:
r = client.put(link, resource_to_update(module))
except HwcClientException as ex:
msg = ("module(hwc_network_vpc): error updating "
"resource, error: %s" % str(ex))
module.fail_json(msg=msg)
wait_for_operation(config, 'update', r)
def delete(session, link, success_codes=None):
if not success_codes:
success_codes = [202, 204]
return_if_object(session.module, session.delete(link), success_codes, False)
return fetch_resource(module, client, link)
wait_for_delete(session, link)
def delete(config, link):
module = config.module
client = config.client(get_region(module), "vpc", "project")
def fetch_resource(session, link, success_codes=None):
if not success_codes:
success_codes = [200]
return return_if_object(session.module, session.get(link), success_codes)
try:
client.delete(link)
except HwcClientException as ex:
msg = ("module(hwc_network_vpc): error deleting "
"resource, error: %s" % str(ex))
module.fail_json(msg=msg)
wait_for_delete(module, client, link)
def link_wrapper(f):
def _wrapper(module, *args, **kwargs):
try:
return f(module, *args, **kwargs)
except KeyError as ex:
module.fail_json(
msg="Mapping keys(%s) are not found in generating link." % ex)
return _wrapper
def fetch_resource(module, client, link):
try:
return client.get(link)
except HwcClientException as ex:
msg = ("module(hwc_network_vpc): error fetching "
"resource, error: %s" % str(ex))
module.fail_json(msg=msg)
def get_id_by_name(session):
module = session.module
def get_id_by_name(config):
module = config.module
client = config.client(get_region(module), "vpc", "project")
name = module.params.get("name")
link = list_link(session, {'limit': 10, 'marker': '{marker}'})
link = "vpcs"
query_link = "?marker={marker}&limit=10"
link += query_link
not_format_keys = re.findall("={marker}", link)
none_values = re.findall("=None", link)
if not (not_format_keys or none_values):
r = fetch_resource(session, link)
r = None
try:
r = client.get(link)
except Exception:
pass
if r is None:
return ""
return None
r = r.get('vpcs', [])
ids = [
i.get('id') for i in r if i.get('name', '') == name
]
if not ids:
return ""
return None
elif len(ids) == 1:
return ids[0]
else:
@ -258,7 +276,11 @@ def get_id_by_name(session):
p = {'marker': ''}
ids = set()
while True:
r = fetch_resource(session, link.format(**p))
r = None
try:
r = client.get(link.format(**p))
except Exception:
pass
if r is None:
break
r = r.get('vpcs', [])
@ -273,68 +295,11 @@ def get_id_by_name(session):
p['marker'] = r[-1].get('id')
return ids.pop() if ids else ""
@link_wrapper
def list_link(session, extra_data=None):
url = "{endpoint}vpcs?limit={limit}&marker={marker}"
combined = session.module.params.copy()
if extra_data:
combined.update(extra_data)
combined['endpoint'] = session.get_service_endpoint('vpc')
return url.format(**combined)
@link_wrapper
def self_link(session):
url = "{endpoint}vpcs/{id}"
return ids.pop() if ids else None
combined = session.module.params.copy()
combined['endpoint'] = session.get_service_endpoint('vpc')
return url.format(**combined)
@link_wrapper
def collection(session):
url = "{endpoint}vpcs"
combined = session.module.params.copy()
combined['endpoint'] = session.get_service_endpoint('vpc')
return url.format(**combined)
def return_if_object(module, response, success_codes, has_content=True):
code = response.status_code
# If not found, return nothing.
if code == 404:
return None
success_codes = [200, 201, 202, 203, 204, 205, 206, 207, 208, 226]
# If no content, return nothing.
if code in success_codes and not has_content:
return None
result = None
try:
result = response.json()
except getattr(json.decoder, 'JSONDecodeError', ValueError) as inst:
module.fail_json(msg="Invalid JSON response with error: %s" % inst)
if code not in success_codes:
msg = navigate_hash(result, ['message'])
if msg:
module.fail_json(msg=msg)
else:
module.fail_json(msg="operation failed, return code=%d" % code)
return result
def self_link(module):
return build_path(module, "vpcs/{id}")
def resource_to_create(module):
@ -376,33 +341,11 @@ def response_to_hash(module, response):
}
@link_wrapper
def resource_get_url(session, wait_done):
combined = session.module.params.copy()
combined['op_id'] = navigate_hash(wait_done, ['vpc', 'id'])
url = 'vpcs/{op_id}'.format(**combined)
endpoint = session.get_service_endpoint('vpc')
return endpoint + url
@link_wrapper
def async_op_url(session, extra_data=None):
url = "{endpoint}vpcs/{op_id}"
combined = session.module.params.copy()
if extra_data:
combined.update(extra_data)
combined['endpoint'] = session.get_service_endpoint('vpc')
return url.format(**combined)
def wait_for_operation(session, op_type, op_result):
def wait_for_operation(config, op_type, op_result):
module = config.module
op_id = navigate_hash(op_result, ['vpc', 'id'])
url = async_op_url(session, {'op_id': op_id})
timeout = 60 * int(session.module.params['timeouts'][op_type].rstrip('m'))
url = build_path(module, "vpcs/{op_id}", {'op_id': op_id})
timeout = 60 * int(module.params['timeouts'][op_type].rstrip('m'))
states = {
'create': {
'allowed': ['CREATING', 'DONW', 'OK'],
@ -415,16 +358,17 @@ def wait_for_operation(session, op_type, op_result):
}
return wait_for_completion(url, timeout, states[op_type]['allowed'],
states[op_type]['complete'], session)
states[op_type]['complete'], config)
def wait_for_completion(op_uri, timeout, allowed_states,
complete_states, session):
module = session.module
complete_states, config):
module = config.module
client = config.client(get_region(module), "vpc", "project")
end = time.time() + timeout
while time.time() <= end:
try:
op_result = fetch_resource(session, op_uri)
op_result = fetch_resource(module, client, op_uri)
except Exception:
time.sleep(1.0)
continue
@ -448,20 +392,20 @@ def raise_if_errors(response, module):
module.fail_json(msg=navigate_hash(response, []))
def wait_for_delete(session, link):
def wait_for_delete(module, client, link):
end = time.time() + 60 * int(
session.module.params['timeouts']['delete'].rstrip('m'))
module.params['timeouts']['delete'].rstrip('m'))
while time.time() <= end:
try:
resp = session.get(link)
if resp.status_code == 404:
client.get(link)
except HwcClientException404:
return
except Exception:
pass
time.sleep(1.0)
session.module.fail_json(msg="Timeout to wait for deletion to be complete.")
module.fail_json(msg="Timeout to wait for deletion to be complete.")
class VpcRoutesArray(object):

@ -109,12 +109,11 @@ update_time:
# Imports
###############################################################################
from ansible.module_utils.hwc_utils import (HwcSession, HwcModule,
DictComparison, navigate_hash,
remove_nones_from_dict,
from ansible.module_utils.hwc_utils import (Config, HwcModule, build_path,
HwcClientException, navigate_hash,
remove_nones_from_dict, get_region,
remove_empty_from_dict,
are_dicts_different)
import json
import re
###############################################################################
@ -135,18 +134,19 @@ def main():
supports_check_mode=True,
)
session = HwcSession(module, "app")
config = Config(module, "smn")
state = module.params['state']
if not module.params.get("id"):
module.params['id'] = get_resource_id(session)
module.params['id'] = get_resource_id(config)
fetch = None
link = self_link(session)
link = self_link(module)
# the link will include Nones if required format parameters are missed
if not re.search('/None/|/None$', link):
fetch = fetch_resource(session, link)
client = config.client(get_region(module), "smn", "project")
fetch = fetch_resource(module, client, link)
changed = False
if fetch:
@ -155,20 +155,20 @@ def main():
current_state = response_to_hash(module, fetch)
if are_dicts_different(expect, current_state):
if not module.check_mode:
fetch = update(session)
fetch = update(config)
fetch = response_to_hash(module, fetch)
changed = True
else:
fetch = current_state
else:
if not module.check_mode:
delete(session)
delete(config)
fetch = {}
changed = True
else:
if state == 'present':
if not module.check_mode:
fetch = create(session)
fetch = create(config)
fetch = response_to_hash(module, fetch)
changed = True
else:
@ -179,96 +179,86 @@ def main():
module.exit_json(**fetch)
def create(session):
link = collection(session)
module = session.module
success_codes = [201, 202]
r = return_if_object(module,
session.post(link, create_resource_opts(module)),
success_codes)
def create(config):
module = config.module
client = config.client(get_region(module), "smn", "project")
return get_resource(session, r)
def update(session):
link = self_link(session)
success_codes = [201, 202]
module = session.module
r = return_if_object(module, session.put(link, update_resource_opts(module)), success_codes)
return get_resource(session, r)
link = "notifications/topics"
r = None
try:
r = client.post(link, create_resource_opts(module))
except HwcClientException as ex:
msg = ("module(hwc_smn_topic): error creating "
"resource, error: %s" % str(ex))
module.fail_json(msg=msg)
return get_resource(config, r)
def delete(session):
link = self_link(session)
success_codes = [202, 204]
return_if_object(session.module, session.delete(link), success_codes, False)
def update(config):
module = config.module
client = config.client(get_region(module), "smn", "project")
def link_wrapper(f):
def _wrapper(module, *args, **kwargs):
link = self_link(module)
try:
return f(module, *args, **kwargs)
except KeyError as ex:
module.fail_json(
msg="Mapping keys(%s) are not found in generating link." % ex)
client.put(link, update_resource_opts(module))
except HwcClientException as ex:
msg = ("module(hwc_smn_topic): error updating "
"resource, error: %s" % str(ex))
module.fail_json(msg=msg)
return _wrapper
return fetch_resource(module, client, link)
def return_if_object(module, response, success_codes, has_content=True):
code = response.status_code
def delete(config):
module = config.module
client = config.client(get_region(module), "smn", "project")
# If not found, return nothing.
if code == 404:
return None
link = self_link(module)
try:
client.delete(link)
except HwcClientException as ex:
msg = ("module(hwc_smn_topic): error deleting "
"resource, error: %s" % str(ex))
module.fail_json(msg=msg)
if not success_codes:
success_codes = [200, 201, 202, 203, 204, 205, 206, 207, 208, 226]
# If no content, return nothing.
if code in success_codes and not has_content:
return None
result = None
def fetch_resource(module, client, link):
try:
result = response.json()
except getattr(json.decoder, 'JSONDecodeError', ValueError) as inst:
module.fail_json(msg="Invalid JSON response with error: %s" % inst)
if code not in success_codes:
msg = navigate_hash(result, ['message'])
if msg:
return client.get(link)
except HwcClientException as ex:
msg = ("module(hwc_smn_topic): error fetching "
"resource, error: %s" % str(ex))
module.fail_json(msg=msg)
else:
module.fail_json(msg="operation failed, return code=%d" % code)
return result
def get_resource(config, result):
module = config.module
client = config.client(get_region(module), "smn", "project")
def fetch_resource(session, link, success_codes=None):
if not success_codes:
success_codes = [200]
return return_if_object(session.module, session.get(link), success_codes)
d = {'topic_urn': navigate_hash(result, ['topic_urn'])}
url = build_path(module, 'notifications/topics/{topic_urn}', d)
return fetch_resource(module, client, url)
def get_resource(session, result):
combined = session.module.params.copy()
combined['topic_urn'] = navigate_hash(result, ['topic_urn'])
url = 'notifications/topics/{topic_urn}'.format(**combined)
e = session.get_service_endpoint('compute')
url = e.replace("ecs", "smn") + url
return fetch_resource(session, url)
def get_resource_id(config):
module = config.module
client = config.client(get_region(module), "smn", "project")
link = "notifications/topics"
query_link = "?offset={offset}&limit=10"
link += query_link
def get_resource_id(session):
module = session.module
link = list_link(session, {'limit': 10, 'offset': '{offset}'})
p = {'offset': 0}
v = module.params.get('name')
ids = set()
while True:
r = fetch_resource(session, link.format(**p))
r = None
try:
r = client.get(link.format(**p))
except Exception:
pass
if r is None:
break
r = r.get('topics', [])
@ -285,42 +275,8 @@ def get_resource_id(session):
return ids.pop() if ids else None
@link_wrapper
def list_link(session, extra_data=None):
url = "{endpoint}notifications/topics?limit={limit}&offset={offset}"
combined = session.module.params.copy()
if extra_data:
combined.update(extra_data)
e = session.get_service_endpoint('compute')
combined['endpoint'] = e.replace("ecs", "smn")
return url.format(**combined)
@link_wrapper
def self_link(session):
url = "{endpoint}notifications/topics/{id}"
combined = session.module.params.copy()
e = session.get_service_endpoint('compute')
combined['endpoint'] = e.replace("ecs", "smn")
return url.format(**combined)
@link_wrapper
def collection(session):
url = "{endpoint}notifications/topics"
combined = session.module.params.copy()
e = session.get_service_endpoint('compute')
combined['endpoint'] = e.replace("ecs", "smn")
return url.format(**combined)
def self_link(module):
return build_path(module, "notifications/topics/{id}")
def create_resource_opts(module):

Loading…
Cancel
Save