initial cutover to API profiles (#35538)

* hardcoded API profiles in azure_rm_common
* changed azure_rm_securitygroup module to use api_profiles, dynamic models, kwargs on all SDK methods
* changed azure_rm_containerinstance module to use api_profiles, dynamic models, kwargs on all SDK methods
* fixed polling performance issue in azure_rm_securitygroup (default poll interval was 30s)
(cherry picked from commit 26f551d1c3)

lib/ansible/module_utils/azure_rm_common.py

@@ -1,25 +1,11 @@
 # Copyright (c) 2016 Matt Davis, <mdavis@ansible.com>
 #                    Chris Houseknecht, <house@redhat.com>
 #
-# This file is part of Ansible
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
-import json
 import os
 import re
-import sys
+import types
 import copy
 import inspect
 import traceback
@@ -47,7 +33,8 @@ AZURE_COMMON_ARGS = dict(
     ad_user=dict(type='str', no_log=True),
     password=dict(type='str', no_log=True),
     cloud_environment=dict(type='str'),
-    cert_validation_mode=dict(type='str', choices=['validate', 'ignore'])
+    cert_validation_mode=dict(type='str', choices=['validate', 'ignore']),
+    api_profile=dict(type='str', default='latest')
     # debug=dict(type='bool', default=False),
 )
 
@@ -63,6 +50,31 @@ AZURE_CREDENTIAL_ENV_MAPPING = dict(
     cert_validation_mode='AZURE_CERT_VALIDATION_MODE',
 )
 
+# FUTURE: this should come from the SDK or an external location.
+# For now, we have to copy from azure-cli
+AZURE_API_PROFILES = {
+    'latest': {
+        'ContainerInstanceManagementClient': '2018-02-01-preview',
+        'ComputeManagementClient': dict(
+            default_api_version='2017-12-01',
+            resource_skus='2017-09-01',
+            disks='2017-03-30',
+            snapshots='2017-03-30',
+            virtual_machine_run_commands='2017-03-30'
+        ),
+        'NetworkManagementClient': '2017-11-01',
+        'ResourceManagementClient': '2017-05-10',
+        'StorageManagementClient': '2017-10-01'
+    },
+
+    '2017-03-09-profile': {
+        'ComputeManagementClient': '2016-03-30',
+        'NetworkManagementClient': '2015-06-15',
+        'ResourceManagementClient': '2016-02-01',
+        'StorageManagementClient': '2016-01-01'
+    }
+}
+
 AZURE_TAG_ARGS = dict(
     tags=dict(type='dict'),
     append_tags=dict(type='bool', default=True),
@@ -162,36 +174,36 @@ def format_resource_id(val, subscription_id, namespace, types, resource_group):
                        type=types,
                        subscription=subscription_id) if not is_valid_resource_id(val) else val
 
+# FUTURE: either get this from the requirements file (if we can be sure it's always available at runtime)
+# or generate the requirements files from this so we only have one source of truth to maintain...
 AZURE_PKG_VERSIONS = {
-    StorageManagementClient.__name__: {
+    'StorageManagementClient': {
         'package_name': 'storage',
-        'expected_version': '1.5.0',
+        'expected_version': '1.5.0'
-        'installed_version': storage_client_version
     },
-    ComputeManagementClient.__name__: {
+    'ComputeManagementClient': {
         'package_name': 'compute',
-        'expected_version': '2.0.0',
+        'expected_version': '2.0.0'
-        'installed_version': compute_client_version
+    },
+    'ContainerInstanceManagementClient': {
+        'package_name': 'containerinstance',
+        'expected_version': '0.3.1'
     },
-    NetworkManagementClient.__name__: {
+    'NetworkManagementClient': {
         'package_name': 'network',
-        'expected_version': '1.3.0',
+        'expected_version': '1.3.0'
-        'installed_version': network_client_version
     },
-    ResourceManagementClient.__name__: {
+    'ResourceManagementClient': {
         'package_name': 'resource',
-        'expected_version': '1.1.0',
+        'expected_version': '1.1.0'
-        'installed_version': resource_client_version
     },
-    DnsManagementClient.__name__: {
+    'DnsManagementClient': {
         'package_name': 'dns',
-        'expected_version': '1.0.1',
+        'expected_version': '1.0.1'
-        'installed_version': dns_client_version
     },
-    WebSiteManagementClient.__name__: {
+    'WebSiteManagementClient': {
         'package_name': 'web',
-        'expected_version': '0.32.0',
+        'expected_version': '0.32.0'
-        'installed_version': web_client_version
     },
 } if HAS_AZURE else {}
 
@@ -250,6 +262,7 @@ class AzureRMModuleBase(object):
         self._containerservice_client = None
 
         self.check_mode = self.module.check_mode
+        self.api_profile = self.module.params.get('api_profile')
         self.facts_module = facts_module
         # self.debug = self.module.params.get('debug')
 
@@ -337,10 +350,15 @@ class AzureRMModuleBase(object):
         package_version = AZURE_PKG_VERSIONS.get(client_type.__name__, None)
         if package_version is not None:
             client_name = package_version.get('package_name')
-            client_version = package_version.get('installed_version')
+            try:
+                client_module = importlib.import_module(client_type.__module__)
+                client_version = client_module.VERSION
+            except RuntimeError:
+                # can't get at the module version for some reason, just fail silently...
+                return
             expected_version = package_version.get('expected_version')
             if Version(client_version) < Version(expected_version):
-                self.fail("Installed {0} client version is {1}. The supported version is {2}. Try "
+                self.fail("Installed azure-mgmt-{0} client version is {1}. The supported version is {2}. Try "
                           "`pip install ansible[azure]`".format(client_name, client_version, expected_version))
 
     def exec_module(self, **kwargs):
@@ -767,18 +785,65 @@ class AzureRMModuleBase(object):
     def _validation_ignore_callback(session, global_config, local_config, **kwargs):
         session.verify = False
 
+    def get_api_profile(self, client_type_name, api_profile_name):
+        profile_all_clients = AZURE_API_PROFILES.get(api_profile_name)
+
+        if not profile_all_clients:
+            raise KeyError("unknown Azure API profile: {0}".format(api_profile_name))
+
+        profile_raw = profile_all_clients.get(client_type_name, None)
+
+        if not profile_raw:
+            self.module.warn("Azure API profile {0} does not define an entry for {1}".format(api_profile_name, client_type_name))
+
+        if isinstance(profile_raw, dict):
+            if not profile_raw.get('default_api_version'):
+                raise KeyError("Azure API profile {0} does not define 'default_api_version'".format(api_profile_name))
+            return profile_raw
+
+        # wrap basic strings in a dict that just defines the default
+        return dict(default_api_version=profile_raw)
+
     def get_mgmt_svc_client(self, client_type, base_url=None, api_version=None):
         self.log('Getting management service client {0}'.format(client_type.__name__))
         self.check_client_version(client_type)
-        if api_version:
+
-            client = client_type(self.azure_credentials,
+        client_argspec = inspect.getargspec(client_type.__init__)
-                                 self.subscription_id,
+
-                                 api_version=api_version,
+        client_kwargs = dict(credentials=self.azure_credentials, subscription_id=self.subscription_id, base_url=base_url)
-                                 base_url=base_url)
+
-        else:
+        api_profile_dict = {}
-            client = client_type(self.azure_credentials,
+
-                                 self.subscription_id,
+        if self.api_profile:
-                                 base_url=base_url)
+            api_profile_dict = self.get_api_profile(client_type.__name__, self.api_profile)
+
+        if not base_url:
+            # most things are resource_manager, don't make everyone specify
+            base_url = self._cloud_environment.endpoints.resource_manager
+
+        # unversioned clients won't accept profile; only send it if necessary
+        # clients without a version specified in the profile will use the default
+        if api_profile_dict and 'profile' in client_argspec.args:
+            client_kwargs['profile'] = api_profile_dict
+
+        # If the client doesn't accept api_version, it's unversioned.
+        # If it does, favor explicitly-specified api_version, fall back to api_profile
+        if 'api_version' in client_argspec.args:
+            profile_default_version = api_profile_dict.get('default_api_version', None)
+            if api_version or profile_default_version:
+                client_kwargs['api_version'] = api_version or profile_default_version
+
+        client = client_type(**client_kwargs)
+
+        # FUTURE: remove this once everything exposes models directly (eg, containerinstance)
+        try:
+            getattr(client, "models")
+        except AttributeError:
+            def _ansible_get_models(self, *arg, **kwarg):
+                return self._ansible_models
+
+            setattr(client, '_ansible_models', importlib.import_module(client_type.__module__).models)
+            client.models = types.MethodType(_ansible_get_models, client)
 
         # Add user agent for Ansible
         client.config.add_user_agent(ANSIBLE_USER_AGENT)

lib/ansible/modules/cloud/azure/azure_rm_containerinstance.py

@@ -145,14 +145,6 @@ from ansible.module_utils.azure_rm_common import AzureRMModuleBase
 
 try:
     from msrestazure.azure_exceptions import CloudError
-    from azure.mgmt.containerinstance.models import (ContainerGroup,
-                                                     Container,
-                                                     ResourceRequirements,
-                                                     ResourceRequests,
-                                                     ImageRegistryCredential,
-                                                     IpAddress,
-                                                     Port,
-                                                     ContainerPort)
     from azure.mgmt.containerinstance import ContainerInstanceManagementClient
 except ImportError:
     # This is handled in azure_rm_common
@@ -264,7 +256,8 @@ class AzureRMContainerInstance(AzureRMModuleBase):
         self.containers = None
 
         self.results = dict(changed=False, state=dict())
-        self.mgmt_client = None
+        self.client = None
+        self.cgmodels = None
 
         super(AzureRMContainerInstance, self).__init__(derived_arg_spec=self.module_arg_spec,
                                                        supports_check_mode=True,
@@ -280,8 +273,10 @@ class AzureRMContainerInstance(AzureRMModuleBase):
         response = None
         results = dict()
 
-        self.mgmt_client = self.get_mgmt_svc_client(ContainerInstanceManagementClient,
+        self.client = self.get_mgmt_svc_client(ContainerInstanceManagementClient)
-                                                    base_url=self._cloud_environment.endpoints.resource_manager)
+
+        # since this client hasn't been upgraded to expose models directly off the OperationClass, fish them out
+        self.cgmodels = self.client.container_groups.models
 
         resource_group = self.get_resource_group(self.resource_group)
 
@@ -341,9 +336,9 @@ class AzureRMContainerInstance(AzureRMModuleBase):
         registry_credentials = None
 
         if self.registry_login_server is not None:
-            registry_credentials = [ImageRegistryCredential(server=self.registry_login_server,
+            registry_credentials = [self.cgmodels.ImageRegistryCredential(server=self.registry_login_server,
-                                                            username=self.registry_username,
+                                                                          username=self.registry_username,
-                                                            password=self.registry_password)]
+                                                                          password=self.registry_password)]
 
         ip_address = None
 
@@ -352,8 +347,8 @@ class AzureRMContainerInstance(AzureRMModuleBase):
             if self.ports:
                 ports = []
                 for port in self.ports:
-                    ports.append(Port(port=port, protocol="TCP"))
+                    ports.append(self.cgmodels.Port(port=port, protocol="TCP"))
-                ip_address = IpAddress(ports=ports, ip=self.ip_address)
+                ip_address = self.cgmodels.IpAddress(ports=ports, ip=self.ip_address)
 
         containers = []
 
@@ -367,22 +362,26 @@ class AzureRMContainerInstance(AzureRMModuleBase):
             port_list = container_def.get("ports")
             if port_list:
                 for port in port_list:
-                    ports.append(ContainerPort(port))
+                    ports.append(self.cgmodels.ContainerPort(port=port))
-
+
-            containers.append(Container(name=name,
+            containers.append(self.cgmodels.Container(name=name,
-                                        image=image,
+                                                      image=image,
-                                        resources=ResourceRequirements(ResourceRequests(memory_in_gb=memory, cpu=cpu)),
+                                                      resources=self.cgmodels.ResourceRequirements(
-                                        ports=ports))
+                                                          requests=self.cgmodels.ResourceRequests(memory_in_gb=memory, cpu=cpu)
-
+                                                      ),
-        parameters = ContainerGroup(location=self.location,
+                                                      ports=ports))
-                                    containers=containers,
+
-                                    image_registry_credentials=registry_credentials,
+        parameters = self.cgmodels.ContainerGroup(location=self.location,
-                                    restart_policy=None,
+                                                  containers=containers,
-                                    ip_address=ip_address,
+                                                  image_registry_credentials=registry_credentials,
-                                    os_type=self.os_type,
+                                                  restart_policy=None,
-                                    volumes=None)
+                                                  ip_address=ip_address,
-
+                                                  os_type=self.os_type,
-        response = self.mgmt_client.container_groups.create_or_update(self.resource_group, self.name, parameters)
+                                                  volumes=None)
+
+        response = self.client.container_groups.create_or_update(resource_group_name=self.resource_group,
+                                                                 container_group_name=self.name,
+                                                                 container_group=parameters)
 
         return response.as_dict()
 
@@ -393,7 +392,7 @@ class AzureRMContainerInstance(AzureRMModuleBase):
         :return: True
         '''
         self.log("Deleting the container instance {0}".format(self.name))
-        response = self.mgmt_client.container_groups.delete(self.resource_group, self.name)
+        response = self.client.container_groups.delete(resource_group_name=self.resource_group, container_group_name=self.name)
         return True
 
     def get_containerinstance(self):
@@ -405,7 +404,7 @@ class AzureRMContainerInstance(AzureRMModuleBase):
         self.log("Checking if the container instance {0} is present".format(self.name))
         found = False
         try:
-            response = self.mgmt_client.container_groups.get(self.resource_group, self.name)
+            response = self.client.container_groups.get(resource_group_name=self.resource_group, container_group_name=self.name)
             found = True
             self.log("Response : {0}".format(response))
             self.log("Container instance : {0} found".format(response.name))

lib/ansible/modules/cloud/azure/azure_rm_securitygroup.py

@@ -334,6 +334,7 @@ state:
 
 try:
     from msrestazure.azure_exceptions import CloudError
+    from azure.mgmt.network import NetworkManagementClient
 except ImportError:
     # This is handled in azure_rm_common
     pass
@@ -369,7 +370,7 @@ def validate_rule(self, rule, rule_type=None):
     if not rule.get('access'):
         rule['access'] = 'Allow'
 
-    access_names = [member.value for member in self.network_models.SecurityRuleAccess]
+    access_names = [member.value for member in self.nsg_models.SecurityRuleAccess]
     if rule['access'] not in access_names:
         raise Exception("Rule access must be one of [{0}]".format(', '.join(access_names)))
 
@@ -382,14 +383,14 @@ def validate_rule(self, rule, rule_type=None):
     if not rule.get('protocol'):
         rule['protocol'] = '*'
 
-    protocol_names = [member.value for member in self.network_models.SecurityRuleProtocol]
+    protocol_names = [member.value for member in self.nsg_models.SecurityRuleProtocol]
     if rule['protocol'] not in protocol_names:
         raise Exception("Rule protocol must be one of [{0}]".format(', '.join(protocol_names)))
 
     if not rule.get('direction'):
         rule['direction'] = 'Inbound'
 
-    direction_names = [member.value for member in self.network_models.SecurityRuleDirection]
+    direction_names = [member.value for member in self.nsg_models.SecurityRuleDirection]
     if rule['direction'] not in direction_names:
         raise Exception("Rule direction must be one of [{0}]".format(', '.join(direction_names)))
 
@@ -439,7 +440,7 @@ def create_rule_instance(self, rule):
     :param rule: dict
     :return: SecurityRule
     '''
-    return self.network_models.SecurityRule(
+    return self.nsg_models.SecurityRule(
         protocol=rule['protocol'],
         source_address_prefix=rule['source_address_prefix'],
         destination_address_prefix=rule['destination_address_prefix'],
@@ -535,6 +536,8 @@ class AzureRMSecurityGroup(AzureRMModuleBase):
         self.rules = None
         self.state = None
         self.tags = None
+        self.client = None  # type: azure.mgmt.network.NetworkManagementClient
+        self.nsg_models = None  # type: azure.mgmt.network.models
 
         self.results = dict(
             changed=False,
@@ -545,6 +548,11 @@ class AzureRMSecurityGroup(AzureRMModuleBase):
                                                    supports_check_mode=True)
 
     def exec_module(self, **kwargs):
+        self.client = self.get_mgmt_svc_client(NetworkManagementClient)
+        # tighten up poll interval for security groups; default 30s is an eternity
+        # this value is still overridden by the response Retry-After header (which is set on the initial operation response to 10s)
+        self.client.config.long_running_operation_timeout = 3
+        self.nsg_models = self.client.network_security_groups.models
 
         for key in list(self.module_arg_spec.keys()) + ['tags']:
             setattr(self, key, kwargs[key])
@@ -572,7 +580,7 @@ class AzureRMSecurityGroup(AzureRMModuleBase):
                     self.fail("Error validating default rule {0} - {1}".format(rule, str(exc)))
 
         try:
-            nsg = self.network_client.network_security_groups.get(self.resource_group, self.name)
+            nsg = self.client.network_security_groups.get(self.resource_group, self.name)
             results = create_network_security_group_dict(nsg)
             self.log("Found security group:")
             self.log(results, pretty_print=True)
@@ -582,7 +590,7 @@ class AzureRMSecurityGroup(AzureRMModuleBase):
             elif self.state == 'absent':
                 self.log("CHANGED: security group found but state is 'absent'")
                 changed = True
-        except CloudError:
+        except CloudError:  # TODO: actually check for ResourceMissingError
             if self.state == 'present':
                 self.log("CHANGED: security group not found and state is 'present'")
                 changed = True
@@ -640,7 +648,7 @@ class AzureRMSecurityGroup(AzureRMModuleBase):
 
             self.results['changed'] = changed
             self.results['state'] = results
-            if not self.check_mode:
+            if not self.check_mode and changed:
                 self.results['state'] = self.create_or_update(results)
 
         elif self.state == 'present' and changed:
@@ -681,7 +689,7 @@ class AzureRMSecurityGroup(AzureRMModuleBase):
         return self.results
 
     def create_or_update(self, results):
-        parameters = self.network_models.NetworkSecurityGroup()
+        parameters = self.nsg_models.NetworkSecurityGroup()
         if results.get('rules'):
             parameters.security_rules = []
             for rule in results.get('rules'):
@@ -694,9 +702,9 @@ class AzureRMSecurityGroup(AzureRMModuleBase):
         parameters.location = results.get('location')
 
         try:
-            poller = self.network_client.network_security_groups.create_or_update(self.resource_group,
+            poller = self.client.network_security_groups.create_or_update(resource_group_name=self.resource_group,
-                                                                                  self.name,
+                                                                          network_security_group_name=self.name,
-                                                                                  parameters)
+                                                                          parameters=parameters)
             result = self.get_poller_result(poller)
         except CloudError as exc:
             self.fail("Error creating/updating security group {0} - {1}".format(self.name, str(exc)))
@@ -704,7 +712,7 @@ class AzureRMSecurityGroup(AzureRMModuleBase):
 
     def delete(self):
         try:
-            poller = self.network_client.network_security_groups.delete(self.resource_group, self.name)
+            poller = self.client.network_security_groups.delete(resource_group_name=self.resource_group, network_security_group_name=self.name)
             result = self.get_poller_result(poller)
         except CloudError as exc:
             raise Exception("Error deleting security group {0} - {1}".format(self.name, str(exc)))

lib/ansible/utils/module_docs_fragments/azure.py

@@ -90,6 +90,14 @@ options:
         - env
         default: auto
         version_added: 2.5
+    api_profile:
+        description:
+        - Selects an API profile to use when communicating with Azure services. Default value of C(latest) is appropriate for public clouds;
+          future values will allow use with Azure Stack.
+        choices:
+        - latest
+        default: latest
+        version_added: 2.5
 requirements:
     - "python >= 2.7"
     - "azure >= 2.0.0"

packaging/requirements/requirements-azure.txt

@@ -16,4 +16,4 @@ azure-mgmt-web>=0.32.0,<0.33
 azure-mgmt-containerservice>=2.0.0,<3.0.0
 azure-mgmt-containerregistry>=1.0.1
 azure-mgmt-rdbms>=0.2.0rc1,<0.3.0
-azure-mgmt-containerinstance>=0.2.0,<0.3.0
+azure-mgmt-containerinstance>=0.3.1

test/runner/requirements/integration.cloud.azure.txt

@@ -16,4 +16,4 @@ azure-mgmt-web>=0.32.0,<0.33
 azure-mgmt-containerservice>=2.0.0,<3.0.0
 azure-mgmt-containerregistry>=1.0.1
 azure-mgmt-rdbms>=0.2.0rc1,<0.3.0
-azure-mgmt-containerinstance>=0.2.0,<0.3.0
+azure-mgmt-containerinstance>=0.3.1