From 26f551d1c37277e5522836d7a6cb98de42c50fb7 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Fri, 9 Feb 2018 14:26:42 -0800 Subject: [PATCH] initial cutover to API profiles (#35538) * hardcoded API profiles in azure_rm_common * changed azure_rm_securitygroup module to use api_profiles, dynamic models, kwargs on all SDK methods * changed azure_rm_containerinstance module to use api_profiles, dynamic models, kwargs on all SDK methods * fixed polling performance issue in azure_rm_securitygroup (default poll interval was 30s) --- lib/ansible/module_utils/azure_rm_common.py | 157 +++++++++++++----- .../cloud/azure/azure_rm_containerinstance.py | 67 ++++---- .../cloud/azure/azure_rm_securitygroup.py | 32 ++-- .../utils/module_docs_fragments/azure.py | 8 + packaging/requirements/requirements-azure.txt | 2 +- .../requirements/integration.cloud.azure.txt | 2 +- 6 files changed, 174 insertions(+), 94 deletions(-) diff --git a/lib/ansible/module_utils/azure_rm_common.py b/lib/ansible/module_utils/azure_rm_common.py index d420c001746..0c0d726e31b 100644 --- a/lib/ansible/module_utils/azure_rm_common.py +++ b/lib/ansible/module_utils/azure_rm_common.py @@ -1,25 +1,11 @@ # Copyright (c) 2016 Matt Davis, # Chris Houseknecht, # -# This file is part of Ansible -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) -import json import os import re -import sys +import types import copy import inspect import traceback @@ -47,7 +33,8 @@ AZURE_COMMON_ARGS = dict( ad_user=dict(type='str', no_log=True), password=dict(type='str', no_log=True), cloud_environment=dict(type='str'), - cert_validation_mode=dict(type='str', choices=['validate', 'ignore']) + cert_validation_mode=dict(type='str', choices=['validate', 'ignore']), + api_profile=dict(type='str', default='latest') # debug=dict(type='bool', default=False), ) @@ -63,6 +50,31 @@ AZURE_CREDENTIAL_ENV_MAPPING = dict( cert_validation_mode='AZURE_CERT_VALIDATION_MODE', ) +# FUTURE: this should come from the SDK or an external location. +# For now, we have to copy from azure-cli +AZURE_API_PROFILES = { + 'latest': { + 'ContainerInstanceManagementClient': '2018-02-01-preview', + 'ComputeManagementClient': dict( + default_api_version='2017-12-01', + resource_skus='2017-09-01', + disks='2017-03-30', + snapshots='2017-03-30', + virtual_machine_run_commands='2017-03-30' + ), + 'NetworkManagementClient': '2017-11-01', + 'ResourceManagementClient': '2017-05-10', + 'StorageManagementClient': '2017-10-01' + }, + + '2017-03-09-profile': { + 'ComputeManagementClient': '2016-03-30', + 'NetworkManagementClient': '2015-06-15', + 'ResourceManagementClient': '2016-02-01', + 'StorageManagementClient': '2016-01-01' + } +} + AZURE_TAG_ARGS = dict( tags=dict(type='dict'), append_tags=dict(type='bool', default=True), @@ -162,36 +174,36 @@ def format_resource_id(val, subscription_id, namespace, types, resource_group): type=types, subscription=subscription_id) if not is_valid_resource_id(val) else val +# FUTURE: either get this from the requirements file (if we can be sure it's always available at runtime) +# or generate the requirements files from this so we only have one source of truth to maintain... AZURE_PKG_VERSIONS = { - StorageManagementClient.__name__: { + 'StorageManagementClient': { 'package_name': 'storage', - 'expected_version': '1.5.0', - 'installed_version': storage_client_version + 'expected_version': '1.5.0' }, - ComputeManagementClient.__name__: { + 'ComputeManagementClient': { 'package_name': 'compute', - 'expected_version': '2.0.0', - 'installed_version': compute_client_version + 'expected_version': '2.0.0' + }, + 'ContainerInstanceManagementClient': { + 'package_name': 'containerinstance', + 'expected_version': '0.3.1' }, - NetworkManagementClient.__name__: { + 'NetworkManagementClient': { 'package_name': 'network', - 'expected_version': '1.3.0', - 'installed_version': network_client_version + 'expected_version': '1.3.0' }, - ResourceManagementClient.__name__: { + 'ResourceManagementClient': { 'package_name': 'resource', - 'expected_version': '1.1.0', - 'installed_version': resource_client_version + 'expected_version': '1.1.0' }, - DnsManagementClient.__name__: { + 'DnsManagementClient': { 'package_name': 'dns', - 'expected_version': '1.0.1', - 'installed_version': dns_client_version + 'expected_version': '1.0.1' }, - WebSiteManagementClient.__name__: { + 'WebSiteManagementClient': { 'package_name': 'web', - 'expected_version': '0.32.0', - 'installed_version': web_client_version + 'expected_version': '0.32.0' }, } if HAS_AZURE else {} @@ -250,6 +262,7 @@ class AzureRMModuleBase(object): self._containerservice_client = None self.check_mode = self.module.check_mode + self.api_profile = self.module.params.get('api_profile') self.facts_module = facts_module # self.debug = self.module.params.get('debug') @@ -337,10 +350,15 @@ class AzureRMModuleBase(object): package_version = AZURE_PKG_VERSIONS.get(client_type.__name__, None) if package_version is not None: client_name = package_version.get('package_name') - client_version = package_version.get('installed_version') + try: + client_module = importlib.import_module(client_type.__module__) + client_version = client_module.VERSION + except RuntimeError: + # can't get at the module version for some reason, just fail silently... + return expected_version = package_version.get('expected_version') if Version(client_version) < Version(expected_version): - self.fail("Installed {0} client version is {1}. The supported version is {2}. Try " + self.fail("Installed azure-mgmt-{0} client version is {1}. The supported version is {2}. Try " "`pip install ansible[azure]`".format(client_name, client_version, expected_version)) def exec_module(self, **kwargs): @@ -767,18 +785,65 @@ class AzureRMModuleBase(object): def _validation_ignore_callback(session, global_config, local_config, **kwargs): session.verify = False + def get_api_profile(self, client_type_name, api_profile_name): + profile_all_clients = AZURE_API_PROFILES.get(api_profile_name) + + if not profile_all_clients: + raise KeyError("unknown Azure API profile: {0}".format(api_profile_name)) + + profile_raw = profile_all_clients.get(client_type_name, None) + + if not profile_raw: + self.module.warn("Azure API profile {0} does not define an entry for {1}".format(api_profile_name, client_type_name)) + + if isinstance(profile_raw, dict): + if not profile_raw.get('default_api_version'): + raise KeyError("Azure API profile {0} does not define 'default_api_version'".format(api_profile_name)) + return profile_raw + + # wrap basic strings in a dict that just defines the default + return dict(default_api_version=profile_raw) + def get_mgmt_svc_client(self, client_type, base_url=None, api_version=None): self.log('Getting management service client {0}'.format(client_type.__name__)) self.check_client_version(client_type) - if api_version: - client = client_type(self.azure_credentials, - self.subscription_id, - api_version=api_version, - base_url=base_url) - else: - client = client_type(self.azure_credentials, - self.subscription_id, - base_url=base_url) + + client_argspec = inspect.getargspec(client_type.__init__) + + client_kwargs = dict(credentials=self.azure_credentials, subscription_id=self.subscription_id, base_url=base_url) + + api_profile_dict = {} + + if self.api_profile: + api_profile_dict = self.get_api_profile(client_type.__name__, self.api_profile) + + if not base_url: + # most things are resource_manager, don't make everyone specify + base_url = self._cloud_environment.endpoints.resource_manager + + # unversioned clients won't accept profile; only send it if necessary + # clients without a version specified in the profile will use the default + if api_profile_dict and 'profile' in client_argspec.args: + client_kwargs['profile'] = api_profile_dict + + # If the client doesn't accept api_version, it's unversioned. + # If it does, favor explicitly-specified api_version, fall back to api_profile + if 'api_version' in client_argspec.args: + profile_default_version = api_profile_dict.get('default_api_version', None) + if api_version or profile_default_version: + client_kwargs['api_version'] = api_version or profile_default_version + + client = client_type(**client_kwargs) + + # FUTURE: remove this once everything exposes models directly (eg, containerinstance) + try: + getattr(client, "models") + except AttributeError: + def _ansible_get_models(self, *arg, **kwarg): + return self._ansible_models + + setattr(client, '_ansible_models', importlib.import_module(client_type.__module__).models) + client.models = types.MethodType(_ansible_get_models, client) # Add user agent for Ansible client.config.add_user_agent(ANSIBLE_USER_AGENT) diff --git a/lib/ansible/modules/cloud/azure/azure_rm_containerinstance.py b/lib/ansible/modules/cloud/azure/azure_rm_containerinstance.py index 1d52bdbcf12..4022c1fba98 100644 --- a/lib/ansible/modules/cloud/azure/azure_rm_containerinstance.py +++ b/lib/ansible/modules/cloud/azure/azure_rm_containerinstance.py @@ -145,14 +145,6 @@ from ansible.module_utils.azure_rm_common import AzureRMModuleBase try: from msrestazure.azure_exceptions import CloudError - from azure.mgmt.containerinstance.models import (ContainerGroup, - Container, - ResourceRequirements, - ResourceRequests, - ImageRegistryCredential, - IpAddress, - Port, - ContainerPort) from azure.mgmt.containerinstance import ContainerInstanceManagementClient except ImportError: # This is handled in azure_rm_common @@ -264,7 +256,8 @@ class AzureRMContainerInstance(AzureRMModuleBase): self.containers = None self.results = dict(changed=False, state=dict()) - self.mgmt_client = None + self.client = None + self.cgmodels = None super(AzureRMContainerInstance, self).__init__(derived_arg_spec=self.module_arg_spec, supports_check_mode=True, @@ -280,8 +273,10 @@ class AzureRMContainerInstance(AzureRMModuleBase): response = None results = dict() - self.mgmt_client = self.get_mgmt_svc_client(ContainerInstanceManagementClient, - base_url=self._cloud_environment.endpoints.resource_manager) + self.client = self.get_mgmt_svc_client(ContainerInstanceManagementClient) + + # since this client hasn't been upgraded to expose models directly off the OperationClass, fish them out + self.cgmodels = self.client.container_groups.models resource_group = self.get_resource_group(self.resource_group) @@ -341,9 +336,9 @@ class AzureRMContainerInstance(AzureRMModuleBase): registry_credentials = None if self.registry_login_server is not None: - registry_credentials = [ImageRegistryCredential(server=self.registry_login_server, - username=self.registry_username, - password=self.registry_password)] + registry_credentials = [self.cgmodels.ImageRegistryCredential(server=self.registry_login_server, + username=self.registry_username, + password=self.registry_password)] ip_address = None @@ -352,8 +347,8 @@ class AzureRMContainerInstance(AzureRMModuleBase): if self.ports: ports = [] for port in self.ports: - ports.append(Port(port=port, protocol="TCP")) - ip_address = IpAddress(ports=ports, ip=self.ip_address) + ports.append(self.cgmodels.Port(port=port, protocol="TCP")) + ip_address = self.cgmodels.IpAddress(ports=ports, ip=self.ip_address) containers = [] @@ -367,22 +362,26 @@ class AzureRMContainerInstance(AzureRMModuleBase): port_list = container_def.get("ports") if port_list: for port in port_list: - ports.append(ContainerPort(port)) - - containers.append(Container(name=name, - image=image, - resources=ResourceRequirements(ResourceRequests(memory_in_gb=memory, cpu=cpu)), - ports=ports)) - - parameters = ContainerGroup(location=self.location, - containers=containers, - image_registry_credentials=registry_credentials, - restart_policy=None, - ip_address=ip_address, - os_type=self.os_type, - volumes=None) - - response = self.mgmt_client.container_groups.create_or_update(self.resource_group, self.name, parameters) + ports.append(self.cgmodels.ContainerPort(port=port)) + + containers.append(self.cgmodels.Container(name=name, + image=image, + resources=self.cgmodels.ResourceRequirements( + requests=self.cgmodels.ResourceRequests(memory_in_gb=memory, cpu=cpu) + ), + ports=ports)) + + parameters = self.cgmodels.ContainerGroup(location=self.location, + containers=containers, + image_registry_credentials=registry_credentials, + restart_policy=None, + ip_address=ip_address, + os_type=self.os_type, + volumes=None) + + response = self.client.container_groups.create_or_update(resource_group_name=self.resource_group, + container_group_name=self.name, + container_group=parameters) return response.as_dict() @@ -393,7 +392,7 @@ class AzureRMContainerInstance(AzureRMModuleBase): :return: True ''' self.log("Deleting the container instance {0}".format(self.name)) - response = self.mgmt_client.container_groups.delete(self.resource_group, self.name) + response = self.client.container_groups.delete(resource_group_name=self.resource_group, container_group_name=self.name) return True def get_containerinstance(self): @@ -405,7 +404,7 @@ class AzureRMContainerInstance(AzureRMModuleBase): self.log("Checking if the container instance {0} is present".format(self.name)) found = False try: - response = self.mgmt_client.container_groups.get(self.resource_group, self.name) + response = self.client.container_groups.get(resource_group_name=self.resource_group, container_group_name=self.name) found = True self.log("Response : {0}".format(response)) self.log("Container instance : {0} found".format(response.name)) diff --git a/lib/ansible/modules/cloud/azure/azure_rm_securitygroup.py b/lib/ansible/modules/cloud/azure/azure_rm_securitygroup.py index 2fd95049c8c..045640a06da 100644 --- a/lib/ansible/modules/cloud/azure/azure_rm_securitygroup.py +++ b/lib/ansible/modules/cloud/azure/azure_rm_securitygroup.py @@ -334,6 +334,7 @@ state: try: from msrestazure.azure_exceptions import CloudError + from azure.mgmt.network import NetworkManagementClient except ImportError: # This is handled in azure_rm_common pass @@ -369,7 +370,7 @@ def validate_rule(self, rule, rule_type=None): if not rule.get('access'): rule['access'] = 'Allow' - access_names = [member.value for member in self.network_models.SecurityRuleAccess] + access_names = [member.value for member in self.nsg_models.SecurityRuleAccess] if rule['access'] not in access_names: raise Exception("Rule access must be one of [{0}]".format(', '.join(access_names))) @@ -382,14 +383,14 @@ def validate_rule(self, rule, rule_type=None): if not rule.get('protocol'): rule['protocol'] = '*' - protocol_names = [member.value for member in self.network_models.SecurityRuleProtocol] + protocol_names = [member.value for member in self.nsg_models.SecurityRuleProtocol] if rule['protocol'] not in protocol_names: raise Exception("Rule protocol must be one of [{0}]".format(', '.join(protocol_names))) if not rule.get('direction'): rule['direction'] = 'Inbound' - direction_names = [member.value for member in self.network_models.SecurityRuleDirection] + direction_names = [member.value for member in self.nsg_models.SecurityRuleDirection] if rule['direction'] not in direction_names: raise Exception("Rule direction must be one of [{0}]".format(', '.join(direction_names))) @@ -439,7 +440,7 @@ def create_rule_instance(self, rule): :param rule: dict :return: SecurityRule ''' - return self.network_models.SecurityRule( + return self.nsg_models.SecurityRule( protocol=rule['protocol'], source_address_prefix=rule['source_address_prefix'], destination_address_prefix=rule['destination_address_prefix'], @@ -535,6 +536,8 @@ class AzureRMSecurityGroup(AzureRMModuleBase): self.rules = None self.state = None self.tags = None + self.client = None # type: azure.mgmt.network.NetworkManagementClient + self.nsg_models = None # type: azure.mgmt.network.models self.results = dict( changed=False, @@ -545,6 +548,11 @@ class AzureRMSecurityGroup(AzureRMModuleBase): supports_check_mode=True) def exec_module(self, **kwargs): + self.client = self.get_mgmt_svc_client(NetworkManagementClient) + # tighten up poll interval for security groups; default 30s is an eternity + # this value is still overridden by the response Retry-After header (which is set on the initial operation response to 10s) + self.client.config.long_running_operation_timeout = 3 + self.nsg_models = self.client.network_security_groups.models for key in list(self.module_arg_spec.keys()) + ['tags']: setattr(self, key, kwargs[key]) @@ -572,7 +580,7 @@ class AzureRMSecurityGroup(AzureRMModuleBase): self.fail("Error validating default rule {0} - {1}".format(rule, str(exc))) try: - nsg = self.network_client.network_security_groups.get(self.resource_group, self.name) + nsg = self.client.network_security_groups.get(self.resource_group, self.name) results = create_network_security_group_dict(nsg) self.log("Found security group:") self.log(results, pretty_print=True) @@ -582,7 +590,7 @@ class AzureRMSecurityGroup(AzureRMModuleBase): elif self.state == 'absent': self.log("CHANGED: security group found but state is 'absent'") changed = True - except CloudError: + except CloudError: # TODO: actually check for ResourceMissingError if self.state == 'present': self.log("CHANGED: security group not found and state is 'present'") changed = True @@ -640,7 +648,7 @@ class AzureRMSecurityGroup(AzureRMModuleBase): self.results['changed'] = changed self.results['state'] = results - if not self.check_mode: + if not self.check_mode and changed: self.results['state'] = self.create_or_update(results) elif self.state == 'present' and changed: @@ -681,7 +689,7 @@ class AzureRMSecurityGroup(AzureRMModuleBase): return self.results def create_or_update(self, results): - parameters = self.network_models.NetworkSecurityGroup() + parameters = self.nsg_models.NetworkSecurityGroup() if results.get('rules'): parameters.security_rules = [] for rule in results.get('rules'): @@ -694,9 +702,9 @@ class AzureRMSecurityGroup(AzureRMModuleBase): parameters.location = results.get('location') try: - poller = self.network_client.network_security_groups.create_or_update(self.resource_group, - self.name, - parameters) + poller = self.client.network_security_groups.create_or_update(resource_group_name=self.resource_group, + network_security_group_name=self.name, + parameters=parameters) result = self.get_poller_result(poller) except CloudError as exc: self.fail("Error creating/updating security group {0} - {1}".format(self.name, str(exc))) @@ -704,7 +712,7 @@ class AzureRMSecurityGroup(AzureRMModuleBase): def delete(self): try: - poller = self.network_client.network_security_groups.delete(self.resource_group, self.name) + poller = self.client.network_security_groups.delete(resource_group_name=self.resource_group, network_security_group_name=self.name) result = self.get_poller_result(poller) except CloudError as exc: raise Exception("Error deleting security group {0} - {1}".format(self.name, str(exc))) diff --git a/lib/ansible/utils/module_docs_fragments/azure.py b/lib/ansible/utils/module_docs_fragments/azure.py index 17177915bee..4659cec95cf 100644 --- a/lib/ansible/utils/module_docs_fragments/azure.py +++ b/lib/ansible/utils/module_docs_fragments/azure.py @@ -90,6 +90,14 @@ options: - env default: auto version_added: 2.5 + api_profile: + description: + - Selects an API profile to use when communicating with Azure services. Default value of C(latest) is appropriate for public clouds; + future values will allow use with Azure Stack. + choices: + - latest + default: latest + version_added: 2.5 requirements: - "python >= 2.7" - "azure >= 2.0.0" diff --git a/packaging/requirements/requirements-azure.txt b/packaging/requirements/requirements-azure.txt index eee639ad8ad..ce6691b7900 100644 --- a/packaging/requirements/requirements-azure.txt +++ b/packaging/requirements/requirements-azure.txt @@ -16,4 +16,4 @@ azure-mgmt-web>=0.32.0,<0.33 azure-mgmt-containerservice>=2.0.0,<3.0.0 azure-mgmt-containerregistry>=1.0.1 azure-mgmt-rdbms>=0.2.0rc1,<0.3.0 -azure-mgmt-containerinstance>=0.2.0,<0.3.0 +azure-mgmt-containerinstance>=0.3.1 diff --git a/test/runner/requirements/integration.cloud.azure.txt b/test/runner/requirements/integration.cloud.azure.txt index eee639ad8ad..ce6691b7900 100644 --- a/test/runner/requirements/integration.cloud.azure.txt +++ b/test/runner/requirements/integration.cloud.azure.txt @@ -16,4 +16,4 @@ azure-mgmt-web>=0.32.0,<0.33 azure-mgmt-containerservice>=2.0.0,<3.0.0 azure-mgmt-containerregistry>=1.0.1 azure-mgmt-rdbms>=0.2.0rc1,<0.3.0 -azure-mgmt-containerinstance>=0.2.0,<0.3.0 +azure-mgmt-containerinstance>=0.3.1