initial commit of azure_rm inventory plugin (#44944)

* crusty refactor of azure_rm to support auth from non-modules
pull/44958/head
Matt Davis 6 years ago committed by GitHub
parent c2fa0d2c4b
commit 2822fd8d9b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -277,7 +277,6 @@ class AzureRMModuleBase(object):
self.fail("Do you have azure>={1} installed? Try `pip install ansible[azure]`"
"- {0}".format(HAS_AZURE_EXC, AZURE_MIN_RELEASE))
self._cloud_environment = None
self._network_client = None
self._storage_client = None
self._resource_client = None
@ -285,116 +284,23 @@ class AzureRMModuleBase(object):
self._dns_client = None
self._web_client = None
self._marketplace_client = None
self._containerservice_client = None
self._sql_client = None
self._mysql_client = None
self._postgresql_client = None
self._containerregistry_client = None
self._containerinstance_client = None
self._containerservice_client = None
self._traffic_manager_management_client = None
self._monitor_client = None
self._adfs_authority_url = None
self._resource = None
self.check_mode = self.module.check_mode
self.api_profile = self.module.params.get('api_profile')
self.facts_module = facts_module
# self.debug = self.module.params.get('debug')
# authenticate
self.credentials = self._get_credentials(self.module.params)
if not self.credentials:
if HAS_AZURE_CLI_CORE:
self.fail("Failed to get credentials. Either pass as parameters, set environment variables, "
"define a profile in ~/.azure/credentials, or log in with Azure CLI (`az login`).")
else:
self.fail("Failed to get credentials. Either pass as parameters, set environment variables, "
"define a profile in ~/.azure/credentials, or install Azure CLI and log in (`az login`).")
# cert validation mode precedence: module-arg, credential profile, env, "validate"
self._cert_validation_mode = self.module.params['cert_validation_mode'] or self.credentials.get('cert_validation_mode') or \
os.environ.get('AZURE_CERT_VALIDATION_MODE') or 'validate'
if self._cert_validation_mode not in ['validate', 'ignore']:
self.fail('invalid cert_validation_mode: {0}'.format(self._cert_validation_mode))
# if cloud_environment specified, look up/build Cloud object
raw_cloud_env = self.credentials.get('cloud_environment')
if self.credentials.get('credentials') is not None and raw_cloud_env is not None:
self._cloud_environment = raw_cloud_env
elif not raw_cloud_env:
self._cloud_environment = azure_cloud.AZURE_PUBLIC_CLOUD # SDK default
else:
# try to look up "well-known" values via the name attribute on azure_cloud members
all_clouds = [x[1] for x in inspect.getmembers(azure_cloud) if isinstance(x[1], azure_cloud.Cloud)]
matched_clouds = [x for x in all_clouds if x.name == raw_cloud_env]
if len(matched_clouds) == 1:
self._cloud_environment = matched_clouds[0]
elif len(matched_clouds) > 1:
self.fail("Azure SDK failure: more than one cloud matched for cloud_environment name '{0}'".format(raw_cloud_env))
else:
if not urlparse.urlparse(raw_cloud_env).scheme:
self.fail("cloud_environment must be an endpoint discovery URL or one of {0}".format([x.name for x in all_clouds]))
try:
self._cloud_environment = azure_cloud.get_cloud_from_metadata_endpoint(raw_cloud_env)
except Exception as e:
self.fail("cloud_environment {0} could not be resolved: {1}".format(raw_cloud_env, e.message), exception=traceback.format_exc(e))
if self.credentials.get('subscription_id', None) is None and self.credentials.get('credentials') is None:
self.fail("Credentials did not include a subscription_id value.")
self.log("setting subscription_id")
self.subscription_id = self.credentials['subscription_id']
# get authentication authority
# for adfs, user could pass in authority or not.
# for others, use default authority from cloud environment
if self.credentials.get('adfs_authority_url') is None:
self._adfs_authority_url = self._cloud_environment.endpoints.active_directory
else:
self._adfs_authority_url = self.credentials.get('adfs_authority_url')
# get resource from cloud environment
self._resource = self._cloud_environment.endpoints.active_directory_resource_id
if self.credentials.get('credentials') is not None:
# AzureCLI credentials
self.azure_credentials = self.credentials['credentials']
elif self.credentials.get('client_id') is not None and \
self.credentials.get('secret') is not None and \
self.credentials.get('tenant') is not None:
self.azure_credentials = ServicePrincipalCredentials(client_id=self.credentials['client_id'],
secret=self.credentials['secret'],
tenant=self.credentials['tenant'],
cloud_environment=self._cloud_environment,
verify=self._cert_validation_mode == 'validate')
elif self.credentials.get('ad_user') is not None and \
self.credentials.get('password') is not None and \
self.credentials.get('client_id') is not None and \
self.credentials.get('tenant') is not None:
self.azure_credentials = self.acquire_token_with_username_password(
self._adfs_authority_url,
self._resource,
self.credentials['ad_user'],
self.credentials['password'],
self.credentials['client_id'],
self.credentials['tenant'])
elif self.credentials.get('ad_user') is not None and self.credentials.get('password') is not None:
tenant = self.credentials.get('tenant')
if not tenant:
tenant = 'common' # SDK default
self.azure_credentials = UserPassCredentials(self.credentials['ad_user'],
self.credentials['password'],
tenant=tenant,
cloud_environment=self._cloud_environment,
verify=self._cert_validation_mode == 'validate')
else:
self.fail("Failed to authenticate with provided credentials. Some attributes were missing. "
"Credentials must include client_id, secret and tenant or ad_user and password, or "
"ad_user, password, client_id, tenant and adfs_authority_url(optional) for ADFS authentication, or "
"be logged in using AzureCLI.")
# delegate auth to AzureRMAuth class (shared with all plugin types)
self.azure_auth = AzureRMAuth(fail_impl=self.fail, **self.module.params)
# common parameter validation
if self.module.params.get('tags'):
@ -404,17 +310,6 @@ class AzureRMModuleBase(object):
res = self.exec_module(**self.module.params)
self.module.exit_json(**res)
def acquire_token_with_username_password(self, authority, resource, username, password, client_id, tenant):
authority_uri = authority
if tenant is not None:
authority_uri = authority + '/' + tenant
context = AuthenticationContext(authority_uri)
token_response = context.acquire_token_with_username_password(resource, username, password, client_id)
return AADTokenCredentials(token_response)
def check_client_version(self, client_type):
# Ensure Azure modules are at least 2.0.0rc5.
package_version = AZURE_PKG_VERSIONS.get(client_type.__name__, None)
@ -541,138 +436,6 @@ class AzureRMModuleBase(object):
except Exception as exc:
self.fail("Error retrieving resource group {0} - {1}".format(resource_group, str(exc)))
def _get_profile(self, profile="default"):
path = expanduser("~/.azure/credentials")
try:
config = configparser.ConfigParser()
config.read(path)
except Exception as exc:
self.fail("Failed to access {0}. Check that the file exists and you have read "
"access. {1}".format(path, str(exc)))
credentials = dict()
for key in AZURE_CREDENTIAL_ENV_MAPPING:
try:
credentials[key] = config.get(profile, key, raw=True)
except:
pass
if credentials.get('subscription_id'):
return credentials
return None
def _get_msi_credentials(self, subscription_id_param=None):
credentials = MSIAuthentication()
subscription_id = subscription_id_param or os.environ.get(AZURE_CREDENTIAL_ENV_MAPPING['subscription_id'], None)
if not subscription_id:
try:
# use the first subscription of the MSI
subscription_client = SubscriptionClient(credentials)
subscription = next(subscription_client.subscriptions.list())
subscription_id = str(subscription.subscription_id)
except Exception as exc:
self.fail("Failed to get MSI token: {0}. "
"Please check whether your machine enabled MSI or grant access to any subscription.".format(str(exc)))
return {
'credentials': credentials,
'subscription_id': subscription_id
}
def _get_azure_cli_credentials(self):
credentials, subscription_id = get_azure_cli_credentials()
cloud_environment = get_cli_active_cloud()
cli_credentials = {
'credentials': credentials,
'subscription_id': subscription_id,
'cloud_environment': cloud_environment
}
return cli_credentials
def _get_env_credentials(self):
env_credentials = dict()
for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.items():
env_credentials[attribute] = os.environ.get(env_variable, None)
if env_credentials['profile']:
credentials = self._get_profile(env_credentials['profile'])
return credentials
if env_credentials.get('subscription_id') is not None:
return env_credentials
return None
def _get_credentials(self, params):
# Get authentication credentials.
self.log('Getting credentials')
arg_credentials = dict()
for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.items():
arg_credentials[attribute] = params.get(attribute, None)
auth_source = params.get('auth_source', None)
if not auth_source:
auth_source = os.environ.get('ANSIBLE_AZURE_AUTH_SOURCE', 'auto')
if auth_source == 'msi':
self.log('Retrieving credenitals from MSI')
return self._get_msi_credentials(arg_credentials['subscription_id'])
if auth_source == 'cli':
if not HAS_AZURE_CLI_CORE:
self.fail("Azure auth_source is `cli`, but azure-cli package is not available. Try `pip install azure-cli --upgrade`")
try:
self.log('Retrieving credentials from Azure CLI profile')
cli_credentials = self._get_azure_cli_credentials()
return cli_credentials
except CLIError as err:
self.fail("Azure CLI profile cannot be loaded - {0}".format(err))
if auth_source == 'env':
self.log('Retrieving credentials from environment')
env_credentials = self._get_env_credentials()
return env_credentials
if auth_source == 'credential_file':
self.log("Retrieving credentials from credential file")
profile = params.get('profile', 'default')
default_credentials = self._get_profile(profile)
return default_credentials
# auto, precedence: module parameters -> environment variables -> default profile in ~/.azure/credentials
# try module params
if arg_credentials['profile'] is not None:
self.log('Retrieving credentials with profile parameter.')
credentials = self._get_profile(arg_credentials['profile'])
return credentials
if arg_credentials['subscription_id']:
self.log('Received credentials from parameters.')
return arg_credentials
# try environment
env_credentials = self._get_env_credentials()
if env_credentials:
self.log('Received credentials from env.')
return env_credentials
# try default profile from ~./azure/credentials
default_credentials = self._get_profile()
if default_credentials:
self.log('Retrieved default profile credentials from ~/.azure/credentials.')
return default_credentials
try:
if HAS_AZURE_CLI_CORE:
self.log('Retrieving credentials from AzureCLI profile')
cli_credentials = self._get_azure_cli_credentials()
return cli_credentials
except CLIError as ce:
self.log('Error getting AzureCLI profile credentials - {0}'.format(ce))
return None
def parse_resource_to_dict(self, resource):
'''
Return a dict of the give resource, which contains name and resource group.
@ -947,9 +710,9 @@ class AzureRMModuleBase(object):
if not base_url:
# most things are resource_manager, don't make everyone specify
base_url = self._cloud_environment.endpoints.resource_manager
base_url = self.azure_auth._cloud_environment.endpoints.resource_manager
client_kwargs = dict(credentials=self.azure_credentials, subscription_id=self.subscription_id, base_url=base_url)
client_kwargs = dict(credentials=self.azure_auth.azure_credentials, subscription_id=self.azure_auth.subscription_id, base_url=base_url)
api_profile_dict = {}
@ -992,11 +755,24 @@ class AzureRMModuleBase(object):
if VSCODEEXT_USER_AGENT_KEY in os.environ:
client.config.add_user_agent(os.environ[VSCODEEXT_USER_AGENT_KEY])
if self._cert_validation_mode == 'ignore':
if self.azure_auth._cert_validation_mode == 'ignore':
client.config.session_configuration_callback = self._validation_ignore_callback
return client
# passthru methods to AzureAuth instance for backcompat
@property
def credentials(self):
return self.azure_auth.credentials
@property
def _cloud_environment(self):
return self.azure_auth._cloud_environment
@property
def subscription_id(self):
return self.azure_auth.subscription_id
@property
def storage_client(self):
self.log('Getting storage client...')
@ -1152,3 +928,281 @@ class AzureRMModuleBase(object):
self._monitor_client = self.get_mgmt_svc_client(MonitorManagementClient,
base_url=self._cloud_environment.endpoints.resource_manager)
return self._monitor_client
class AzureRMAuthException(Exception):
pass
class AzureRMAuth(object):
def __init__(self, auth_source='auto', profile=None, subscription_id=None, client_id=None, secret=None,
tenant=None, ad_user=None, password=None, cloud_environment='AzureCloud', cert_validation_mode='validate',
api_profile='latest', adfs_authority_url=None, fail_impl=None, **kwargs):
if fail_impl:
self._fail_impl = fail_impl
else:
self._fail_impl = self._default_fail_impl
self._cloud_environment = None
self._adfs_authority_url = None
# authenticate
self.credentials = self._get_credentials(
dict(auth_source=auth_source, profile=profile, subscription_id=subscription_id, client_id=client_id, secret=secret,
tenant=tenant, ad_user=ad_user, password=password, cloud_environment=cloud_environment,
cert_validation_mode=cert_validation_mode, api_profile=api_profile, adfs_authority_url=adfs_authority_url))
if not self.credentials:
if HAS_AZURE_CLI_CORE:
self.fail("Failed to get credentials. Either pass as parameters, set environment variables, "
"define a profile in ~/.azure/credentials, or log in with Azure CLI (`az login`).")
else:
self.fail("Failed to get credentials. Either pass as parameters, set environment variables, "
"define a profile in ~/.azure/credentials, or install Azure CLI and log in (`az login`).")
# cert validation mode precedence: module-arg, credential profile, env, "validate"
self._cert_validation_mode = cert_validation_mode or self.credentials.get('cert_validation_mode') or \
os.environ.get('AZURE_CERT_VALIDATION_MODE') or 'validate'
if self._cert_validation_mode not in ['validate', 'ignore']:
self.fail('invalid cert_validation_mode: {0}'.format(self._cert_validation_mode))
# if cloud_environment specified, look up/build Cloud object
raw_cloud_env = self.credentials.get('cloud_environment')
if self.credentials.get('credentials') is not None and raw_cloud_env is not None:
self._cloud_environment = raw_cloud_env
elif not raw_cloud_env:
self._cloud_environment = azure_cloud.AZURE_PUBLIC_CLOUD # SDK default
else:
# try to look up "well-known" values via the name attribute on azure_cloud members
all_clouds = [x[1] for x in inspect.getmembers(azure_cloud) if isinstance(x[1], azure_cloud.Cloud)]
matched_clouds = [x for x in all_clouds if x.name == raw_cloud_env]
if len(matched_clouds) == 1:
self._cloud_environment = matched_clouds[0]
elif len(matched_clouds) > 1:
self.fail("Azure SDK failure: more than one cloud matched for cloud_environment name '{0}'".format(raw_cloud_env))
else:
if not urlparse.urlparse(raw_cloud_env).scheme:
self.fail("cloud_environment must be an endpoint discovery URL or one of {0}".format([x.name for x in all_clouds]))
try:
self._cloud_environment = azure_cloud.get_cloud_from_metadata_endpoint(raw_cloud_env)
except Exception as e:
self.fail("cloud_environment {0} could not be resolved: {1}".format(raw_cloud_env, e.message), exception=traceback.format_exc(e))
if self.credentials.get('subscription_id', None) is None and self.credentials.get('credentials') is None:
self.fail("Credentials did not include a subscription_id value.")
self.log("setting subscription_id")
self.subscription_id = self.credentials['subscription_id']
# get authentication authority
# for adfs, user could pass in authority or not.
# for others, use default authority from cloud environment
if self.credentials.get('adfs_authority_url') is None:
self._adfs_authority_url = self._cloud_environment.endpoints.active_directory
else:
self._adfs_authority_url = self.credentials.get('adfs_authority_url')
# get resource from cloud environment
self._resource = self._cloud_environment.endpoints.active_directory_resource_id
if self.credentials.get('credentials') is not None:
# AzureCLI credentials
self.azure_credentials = self.credentials['credentials']
elif self.credentials.get('client_id') is not None and \
self.credentials.get('secret') is not None and \
self.credentials.get('tenant') is not None:
self.azure_credentials = ServicePrincipalCredentials(client_id=self.credentials['client_id'],
secret=self.credentials['secret'],
tenant=self.credentials['tenant'],
cloud_environment=self._cloud_environment,
verify=self._cert_validation_mode == 'validate')
elif self.credentials.get('ad_user') is not None and \
self.credentials.get('password') is not None and \
self.credentials.get('client_id') is not None and \
self.credentials.get('tenant') is not None:
self.azure_credentials = self.acquire_token_with_username_password(
self._adfs_authority_url,
self._resource,
self.credentials['ad_user'],
self.credentials['password'],
self.credentials['client_id'],
self.credentials['tenant'])
elif self.credentials.get('ad_user') is not None and self.credentials.get('password') is not None:
tenant = self.credentials.get('tenant')
if not tenant:
tenant = 'common' # SDK default
self.azure_credentials = UserPassCredentials(self.credentials['ad_user'],
self.credentials['password'],
tenant=tenant,
cloud_environment=self._cloud_environment,
verify=self._cert_validation_mode == 'validate')
else:
self.fail("Failed to authenticate with provided credentials. Some attributes were missing. "
"Credentials must include client_id, secret and tenant or ad_user and password, or "
"ad_user, password, client_id, tenant and adfs_authority_url(optional) for ADFS authentication, or "
"be logged in using AzureCLI.")
def fail(self, msg, exception=None, **kwargs):
self._fail_impl(msg)
def _default_fail_impl(self, msg, exception=None, **kwargs):
raise AzureRMAuthException(msg)
def _get_profile(self, profile="default"):
path = expanduser("~/.azure/credentials")
try:
config = configparser.ConfigParser()
config.read(path)
except Exception as exc:
self.fail("Failed to access {0}. Check that the file exists and you have read "
"access. {1}".format(path, str(exc)))
credentials = dict()
for key in AZURE_CREDENTIAL_ENV_MAPPING:
try:
credentials[key] = config.get(profile, key, raw=True)
except:
pass
if credentials.get('subscription_id'):
return credentials
return None
def _get_msi_credentials(self, subscription_id_param=None):
credentials = MSIAuthentication()
subscription_id = subscription_id_param or os.environ.get(AZURE_CREDENTIAL_ENV_MAPPING['subscription_id'], None)
if not subscription_id:
try:
# use the first subscription of the MSI
subscription_client = SubscriptionClient(credentials)
subscription = next(subscription_client.subscriptions.list())
subscription_id = str(subscription.subscription_id)
except Exception as exc:
self.fail("Failed to get MSI token: {0}. "
"Please check whether your machine enabled MSI or grant access to any subscription.".format(str(exc)))
return {
'credentials': credentials,
'subscription_id': subscription_id
}
def _get_azure_cli_credentials(self):
credentials, subscription_id = get_azure_cli_credentials()
cloud_environment = get_cli_active_cloud()
cli_credentials = {
'credentials': credentials,
'subscription_id': subscription_id,
'cloud_environment': cloud_environment
}
return cli_credentials
def _get_env_credentials(self):
env_credentials = dict()
for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.items():
env_credentials[attribute] = os.environ.get(env_variable, None)
if env_credentials['profile']:
credentials = self._get_profile(env_credentials['profile'])
return credentials
if env_credentials.get('subscription_id') is not None:
return env_credentials
return None
# TODO: use explicit kwargs instead of intermediate dict
def _get_credentials(self, params):
# Get authentication credentials.
self.log('Getting credentials')
arg_credentials = dict()
for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.items():
arg_credentials[attribute] = params.get(attribute, None)
auth_source = params.get('auth_source', None)
if not auth_source:
auth_source = os.environ.get('ANSIBLE_AZURE_AUTH_SOURCE', 'auto')
if auth_source == 'msi':
self.log('Retrieving credenitals from MSI')
return self._get_msi_credentials(arg_credentials['subscription_id'])
if auth_source == 'cli':
if not HAS_AZURE_CLI_CORE:
self.fail("Azure auth_source is `cli`, but azure-cli package is not available. Try `pip install azure-cli --upgrade`")
try:
self.log('Retrieving credentials from Azure CLI profile')
cli_credentials = self._get_azure_cli_credentials()
return cli_credentials
except CLIError as err:
self.fail("Azure CLI profile cannot be loaded - {0}".format(err))
if auth_source == 'env':
self.log('Retrieving credentials from environment')
env_credentials = self._get_env_credentials()
return env_credentials
if auth_source == 'credential_file':
self.log("Retrieving credentials from credential file")
profile = params.get('profile', 'default')
default_credentials = self._get_profile(profile)
return default_credentials
# auto, precedence: module parameters -> environment variables -> default profile in ~/.azure/credentials
# try module params
if arg_credentials['profile'] is not None:
self.log('Retrieving credentials with profile parameter.')
credentials = self._get_profile(arg_credentials['profile'])
return credentials
if arg_credentials['subscription_id']:
self.log('Received credentials from parameters.')
return arg_credentials
# try environment
env_credentials = self._get_env_credentials()
if env_credentials:
self.log('Received credentials from env.')
return env_credentials
# try default profile from ~./azure/credentials
default_credentials = self._get_profile()
if default_credentials:
self.log('Retrieved default profile credentials from ~/.azure/credentials.')
return default_credentials
try:
if HAS_AZURE_CLI_CORE:
self.log('Retrieving credentials from AzureCLI profile')
cli_credentials = self._get_azure_cli_credentials()
return cli_credentials
except CLIError as ce:
self.log('Error getting AzureCLI profile credentials - {0}'.format(ce))
return None
def acquire_token_with_username_password(self, authority, resource, username, password, client_id, tenant):
authority_uri = authority
if tenant is not None:
authority_uri = authority + '/' + tenant
context = AuthenticationContext(authority_uri)
token_response = context.acquire_token_with_username_password(resource, username, password, client_id)
return AADTokenCredentials(token_response)
def log(self, msg, pretty_print=False):
pass
# Use only during module development
# if self.debug:
# log_file = open('azure_rm.log', 'a')
# if pretty_print:
# log_file.write(json.dumps(msg, indent=4, sort_keys=True))
# else:
# log_file.write(msg + u'\n')

@ -286,7 +286,7 @@ class Constructable(object):
composite = self._compose(compose[varname], variables)
except Exception as e:
if strict:
raise AnsibleError("Could not set %s: %s" % (varname, to_native(e)))
raise AnsibleError("Could not set %s for host %s: %s" % (varname, host, to_native(e)))
continue
self.inventory.set_variable(host, varname, composite)

@ -0,0 +1,540 @@
# Copyright (c) 2018 Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
DOCUMENTATION = r'''
name: azure_rm
plugin_type: inventory
short_description: Azure Resource Manager inventory plugin
extends_documentation_fragment:
- azure
description:
- Query VM details from Azure Resource Manager
- Requires a YAML configuration file whose name ends with '.azure_rm.yaml'
- By default, sets C(ansible_host) to the first public IP address found (preferring the primary NIC). If no
public IPs are found, the first private IP (also preferring the primary NIC). The default may be overridden
via C(hostvar_expressions); see examples.
options:
plugin:
description: marks this as an instance of the 'azure_rm' plugin
required: true
choices: ['azure_rm']
include_vm_resource_groups:
description: A list of resource group names to search for virtual machines. '\*' will include all resource
groups in the subscription.
default: ['*']
include_vmss_resource_groups:
description: A list of resource group names to search for virtual machine scale sets (VMSSs). '\*' will
include all resource groups in the subscription.
default: []
fail_on_template_errors:
description: When false, template failures during group and filter processing are silently ignored (eg,
if a filter or group expression refers to an undefined host variable)
choices: [True, False]
default: True
keyed_groups:
description: Creates groups based on the value of a host variable. Requires a list of dictionaries,
defining C(key) (the source dictionary-typed variable), C(prefix) (the prefix to use for the new group
name), and optionally C(separator) (which defaults to C(_))
conditional_groups:
description: A mapping of group names to Jinja2 expressions. When the mapped expression is true, the host
is added to the named group.
hostvar_expressions:
description: A mapping of hostvar names to Jinja2 expressions. The value for each host is the result of the
Jinja2 expression (which may refer to any of the host's existing variables at the time this inventory
plugin runs).
exclude_host_filters:
description: Excludes hosts from the inventory with a list of Jinja2 conditional expressions. Each
expression in the list is evaluated for each host; when the expression is true, the host is excluded
from the inventory.
default: []
batch_fetch:
description: To improve performance, results are fetched using an unsupported batch API. Disabling
C(batch_fetch) uses a much slower serial fetch, resulting in many more round-trips. Generally only
useful for troubleshooting.
default: true
default_host_filters:
description: A default set of filters that is applied in addition to the conditions in
C(exclude_host_filters) to exclude powered-off and not-fully-provisioned hosts. Set this to a different
value or empty list if you need to include hosts in these states.
default: ['powerstate != "running"', 'provisioning_state != "succeeded"']
'''
EXAMPLES = '''
# The following host variables are always available:
# public_ipv4_addresses: all public IP addresses, with the primary IP config from the primary NIC first
# public_dns_hostnames: all public DNS hostnames, with the primary IP config from the primary NIC first
# private_ipv4_addresses: all private IP addressses, with the primary IP config from the primary NIC first
# id: the VM's Azure resource ID, eg /subscriptions/00000000-0000-0000-1111-1111aaaabb/resourceGroups/my_rg/providers/Microsoft.Compute/virtualMachines/my_vm
# location: the VM's Azure location, eg 'westus', 'eastus'
# name: the VM's resource name, eg 'myvm'
# powerstate: the VM's current power state, eg: 'running', 'stopped', 'deallocated'
# provisioning_state: the VM's current provisioning state, eg: 'succeeded'
# tags: dictionary of the VM's defined tag values
# resource_type: the VM's resource type, eg: 'Microsoft.Compute/virtualMachine', 'Microsoft.Compute/virtualMachineScaleSets/virtualMachines'
# vmid: the VM's internal SMBIOS ID, eg: '36bca69d-c365-4584-8c06-a62f4a1dc5d2'
# vmss: if the VM is a member of a scaleset (vmss), a dictionary including the id and name of the parent scaleset
# sample 'myazuresub.azure_rm.yaml'
# required for all azure_rm inventory plugin configs
plugin: azure_rm
# forces this plugin to use a CLI auth session instead of the automatic auth source selection (eg, prevents the
# presence of 'ANSIBLE_AZURE_RM_X' environment variables from overriding CLI auth)
auth_source: cli
# fetches VMs from an explicit list of resource groups instead of default all (- '*')
include_vm_resource_groups:
- myrg1
- myrg2
# fetches VMs from VMSSs in all resource groups (defaults to no VMSS fetch)
include_vmss_resource_groups:
- '*'
# places a host in the named group if the associated condition evaluates to true
conditional_groups:
# since this will be true for every host, every host sourced from this inventory plugin config will be in the
# group 'all_the_hosts'
all_the_hosts: true
# if the VM's "name" variable contains "dbserver", it will be placed in the 'db_hosts' group
db_hosts: "'dbserver' in name"
# adds variables to each host found by this inventory plugin, whose values are the result of the associated expression
hostvar_expressions:
my_host_var:
# A statically-valued expression has to be both single and double-quoted, or use escaped quotes, since the outer
# layer of quotes will be consumed by YAML. Without the second set of quotes, it interprets 'staticvalue' as a
# variable instead of a string literal.
some_statically_valued_var: "'staticvalue'"
# overrides the default ansible_host value with a custom Jinja2 expression, in this case, the first DNS hostname, or
# if none are found, the first public IP address.
ansible_host: (public_dns_hostnames + public_ipv4_addresses) | first
# places hosts in dynamically-created groups based on a variable value.
keyed_groups:
# places each host in a group named 'tag_(tag name)_(tag value)' for each tag on a VM.
- prefix: tag
key: tags
# places each host in a group named 'azure_loc_(location name)', depending on the VM's location
- prefix: azure_loc
key: location
# places host in a group named 'some_tag_X' using the value of the 'sometag' tag on a VM as X, and defaulting to the
# value 'none' (eg, the group 'some_tag_none') if the 'sometag' tag is not defined for a VM.
- prefix: some_tag
key: tags.sometag | default('none')
# excludes a host from the inventory when any of these expressions is true, can refer to any vars defined on the host
exclude_host_filters:
# excludes hosts in the eastus region
- location in ['eastus']
# excludes hosts that are powered off
- powerstate != 'running'
'''
# FUTURE: do we need a set of sane default filters, separate from the user-defineable ones?
# eg, powerstate==running, provisioning_state==succeeded
import hashlib
import json
import re
try:
from queue import Queue, Empty
except ImportError:
from Queue import Queue, Empty
from collections import namedtuple
from ansible import release
from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable
from ansible.module_utils.six import iteritems
from ansible.module_utils.azure_rm_common import AzureRMAuth
from ansible.errors import AnsibleParserError, AnsibleError
from ansible.module_utils.parsing.convert_bool import boolean
from ansible.module_utils._text import to_native
from itertools import chain
from msrest import ServiceClient, Serializer, Deserializer
from msrestazure import AzureConfiguration
from msrestazure.polling.arm_polling import ARMPolling
class AzureRMRestConfiguration(AzureConfiguration):
def __init__(self, credentials, subscription_id, base_url=None):
if credentials is None:
raise ValueError("Parameter 'credentials' must not be None.")
if subscription_id is None:
raise ValueError("Parameter 'subscription_id' must not be None.")
if not base_url:
base_url = 'https://management.azure.com'
super(AzureRMRestConfiguration, self).__init__(base_url)
self.add_user_agent('ansible-dynamic-inventory/{0}'.format(release.__version__))
self.credentials = credentials
self.subscription_id = subscription_id
UrlAction = namedtuple('UrlAction', ['url', 'api_version', 'handler', 'handler_args'])
# FUTURE: add Cacheable support once we have a sane serialization format
class InventoryModule(BaseInventoryPlugin, Constructable):
NAME = 'azure_rm'
def __init__(self):
super(InventoryModule, self).__init__()
self._serializer = Serializer()
self._deserializer = Deserializer()
self._hosts = []
self._filters = None
# FUTURE: use API profiles with defaults
self._compute_api_version = '2017-03-30'
self._network_api_version = '2015-06-15'
self._default_header_parameters = {'Content-Type': 'application/json; charset=utf-8'}
self._request_queue = Queue()
self.azure_auth = None
self._batch_fetch = False
def verify_file(self, path):
'''
:param loader: an ansible.parsing.dataloader.DataLoader object
:param path: the path to the inventory config file
:return the contents of the config file
'''
if super(InventoryModule, self).verify_file(path):
if re.match(r'.+\.azure_rm\.y(a)?ml$', path):
return True
# display.debug("azure_rm inventory filename must match '*.azure_rm.yml' or '*.azure_rm.yaml'")
return False
def parse(self, inventory, loader, path, cache=True):
super(InventoryModule, self).parse(inventory, loader, path)
self._read_config_data(path)
self._batch_fetch = self.get_option('batch_fetch')
self._filters = self.get_option('exclude_host_filters') + self.get_option('default_host_filters')
try:
self._credential_setup()
self._get_hosts()
except Exception as ex:
raise
def _credential_setup(self):
auth_options = dict(
auth_source=self.get_option('auth_source'),
profile=self.get_option('profile'),
subscription_id=self.get_option('subscription_id'),
client_id=self.get_option('client_id'),
secret=self.get_option('secret'),
tenant=self.get_option('tenant'),
ad_user=self.get_option('ad_user'),
password=self.get_option('password'),
cloud_environment=self.get_option('cloud_environment'),
cert_validation_mode=self.get_option('cert_validation_mode'),
api_profile=self.get_option('api_profile'),
adfs_authority_url=self.get_option('adfs_authority_url')
)
self.azure_auth = AzureRMAuth(**auth_options)
self._clientconfig = AzureRMRestConfiguration(self.azure_auth.azure_credentials, self.azure_auth.subscription_id,
self.azure_auth._cloud_environment.endpoints.resource_manager)
self._client = ServiceClient(self._clientconfig.credentials, self._clientconfig)
def _enqueue_get(self, url, api_version, handler, handler_args=None):
if not handler_args:
handler_args = {}
self._request_queue.put_nowait(UrlAction(url=url, api_version=api_version, handler=handler, handler_args=handler_args))
def _enqueue_vm_list(self, rg='*'):
if not rg or rg == '*':
url = '/subscriptions/{subscriptionId}/providers/Microsoft.Compute/virtualMachines'
else:
url = '/subscriptions/{subscriptionId}/resourceGroups/{rg}/providers/Microsoft.Compute/virtualMachines'
url = url.format(subscriptionId=self._clientconfig.subscription_id, rg=rg)
self._enqueue_get(url=url, api_version=self._compute_api_version, handler=self._on_vm_page_response)
def _enqueue_vmss_list(self, rg=None):
if not rg or rg == '*':
url = '/subscriptions/{subscriptionId}/providers/Microsoft.Compute/virtualMachineScaleSets'
else:
url = '/subscriptions/{subscriptionId}/resourceGroups/{rg}/providers/Microsoft.Compute/virtualMachineScaleSets'
url = url.format(subscriptionId=self._clientconfig.subscription_id, rg=rg)
self._enqueue_get(url=url, api_version=self._compute_api_version, handler=self._on_vmss_page_response)
def _get_hosts(self):
for vm_rg in self.get_option('include_vm_resource_groups'):
self._enqueue_vm_list(vm_rg)
for vmss_rg in self.get_option('include_vmss_resource_groups'):
self._enqueue_vmss_list(vmss_rg)
if self._batch_fetch:
self._process_queue_batch()
else:
self._process_queue_serial()
constructable_config_strict = boolean(self.get_option('fail_on_template_errors'))
constructable_config_compose = self.get_option('hostvar_expressions')
constructable_config_groups = self.get_option('conditional_groups')
constructable_config_keyed_groups = self.get_option('keyed_groups')
for h in self._hosts:
inventory_hostname = self._get_hostname(h)
if self._filter_host(inventory_hostname, h.hostvars):
continue
self.inventory.add_host(inventory_hostname)
# FUTURE: configurable default IP list? can already do this via hostvar_expressions
self.inventory.set_variable(inventory_hostname, "ansible_host",
next(chain(h.hostvars['public_ipv4_addresses'], h.hostvars['private_ipv4_addresses']), None))
for k, v in iteritems(h.hostvars):
# FUTURE: configurable hostvar prefix? Makes docs harder...
self.inventory.set_variable(inventory_hostname, k, v)
# constructable delegation
self._set_composite_vars(constructable_config_compose, h.hostvars, inventory_hostname, strict=constructable_config_strict)
self._add_host_to_composed_groups(constructable_config_groups, h.hostvars, inventory_hostname, strict=constructable_config_strict)
self._add_host_to_keyed_groups(constructable_config_keyed_groups, h.hostvars, inventory_hostname, strict=constructable_config_strict)
# FUTURE: fix underlying inventory stuff to allow us to quickly access known groupvars from reconciled host
def _filter_host(self, inventory_hostname, hostvars):
self.templar.set_available_variables(hostvars)
for condition in self._filters:
# FUTURE: should warn/fail if conditional doesn't return True or False
conditional = "{{% if {0} %}} True {{% else %}} False {{% endif %}}".format(condition)
try:
if boolean(self.templar.template(conditional)):
return True
except Exception as e:
if boolean(self.get_option('fail_on_template_errors')):
raise AnsibleParserError("Error evaluating filter condition '{0}' for host {1}: {2}".format(condition, inventory_hostname, to_native(e)))
continue
return False
def _get_hostname(self, host):
# FUTURE: configurable hostname sources
return host.default_inventory_hostname
def _process_queue_serial(self):
try:
while True:
item = self._request_queue.get_nowait()
resp = self.send_request(item.url, item.api_version)
item.handler(resp, **item.handler_args)
except Empty:
pass
def _on_vm_page_response(self, response, vmss=None):
next_link = response.get('nextLink')
if next_link:
self._enqueue_get(url=next_link, api_version=self._compute_api_version, handler=self._on_vm_page_response)
for h in response['value']:
# FUTURE: add direct VM filtering by tag here (performance optimization)?
self._hosts.append(AzureHost(h, self, vmss=vmss))
def _on_vmss_page_response(self, response):
next_link = response.get('nextLink')
if next_link:
self._enqueue_get(url=next_link, api_version=self._compute_api_version, handler=self._on_vmss_page_response)
# FUTURE: add direct VMSS filtering by tag here (performance optimization)?
for vmss in response['value']:
url = '{0}/virtualMachines'.format(vmss['id'])
# VMSS instances look close enough to regular VMs that we can share the handler impl...
self._enqueue_get(url=url, api_version=self._compute_api_version, handler=self._on_vm_page_response, handler_args=dict(vmss=vmss))
# use the undocumented /batch endpoint to bulk-send up to 500 requests in a single round-trip
#
def _process_queue_batch(self):
while True:
batch_requests = []
batch_item_index = 0
batch_response_handlers = []
try:
while batch_item_index < 500:
item = self._request_queue.get_nowait()
query_parameters = {'api-version': item.api_version}
req = self._client.get(item.url, query_parameters)
batch_requests.append(dict(httpMethod="GET", url=req.url))
batch_response_handlers.append(item)
batch_item_index += 1
except Empty:
pass
if not batch_requests:
break
batch_resp = self._send_batch(batch_requests)
for idx, r in enumerate(batch_resp['responses']):
status_code = r.get('httpStatusCode')
if status_code != 200:
# FUTURE: error-tolerant operation mode (eg, permissions)
raise AnsibleError("a batched request failed with status code {0}, url {1}".format(status_code, batch_requests[idx].get('url')))
item = batch_response_handlers[idx]
# FUTURE: store/handle errors from individual handlers
item.handler(r['content'], **item.handler_args)
def _send_batch(self, batched_requests):
url = '/batch'
query_parameters = {'api-version': '2015-11-01'}
body_obj = dict(requests=batched_requests)
body_content = self._serializer.body(body_obj, 'object')
request = self._client.post(url, query_parameters)
initial_response = self._client.send(request, self._default_header_parameters, body_content)
# FUTURE: configurable timeout?
poller = ARMPolling(timeout=2)
poller.initialize(client=self._client,
initial_response=initial_response,
deserialization_callback=lambda r: self._deserializer('object', r))
poller.run()
return poller.resource()
def send_request(self, url, api_version):
query_parameters = {'api-version': api_version}
req = self._client.get(url, query_parameters)
resp = self._client.send(req, self._default_header_parameters, stream=False)
resp.raise_for_status()
content = resp.content
return json.loads(content)
# VM list (all, N resource groups): VM -> InstanceView, N NICs, N PublicIPAddress)
# VMSS VMs (all SS, N specific SS, N resource groups?): SS -> VM -> InstanceView, N NICs, N PublicIPAddress)
class AzureHost(object):
_powerstate_regex = re.compile('^PowerState/(?P<powerstate>.+)$')
def __init__(self, vm_model, inventory_client, vmss=None):
self._inventory_client = inventory_client
self._vm_model = vm_model
self._vmss = vmss
self._instanceview = None
self._powerstate = "unknown"
self.nics = []
# Azure often doesn't provide a globally-unique filename, so use resource name + a chunk of ID hash
self.default_inventory_hostname = '{0}_{1}'.format(vm_model['name'], hashlib.sha1(vm_model['id']).hexdigest()[0:4])
self._hostvars = {}
inventory_client._enqueue_get(url="{0}/instanceView".format(vm_model['id']),
api_version=self._inventory_client._compute_api_version,
handler=self._on_instanceview_response)
nic_refs = vm_model['properties']['networkProfile']['networkInterfaces']
for nic in nic_refs:
# single-nic instances don't set primary, so figure it out...
is_primary = nic.get('properties', {}).get('primary', len(nic_refs) == 1)
inventory_client._enqueue_get(url=nic['id'], api_version=self._inventory_client._network_api_version,
handler=self._on_nic_response,
handler_args=dict(is_primary=is_primary))
@property
def hostvars(self):
if self._hostvars != {}:
return self._hostvars
new_hostvars = dict(
public_ipv4_addresses=[],
public_dns_hostnames=[],
private_ipv4_addresses=[],
id=self._vm_model['id'],
location=self._vm_model['location'],
name=self._vm_model['name'],
powerstate=self._powerstate,
provisioning_state=self._vm_model['properties']['provisioningState'].lower(),
tags=self._vm_model.get('tags', {}),
resource_type=self._vm_model.get('type', "unknown"),
vmid=self._vm_model['properties']['vmId'],
vmss=dict(
id=self._vmss['id'],
name=self._vmss['name'],
) if self._vmss else {}
)
# set nic-related values from the primary NIC first
for nic in sorted(self.nics, key=lambda n: n.is_primary, reverse=True):
# and from the primary IP config per NIC first
for ipc in sorted(nic._nic_model['properties']['ipConfigurations'], key=lambda i: i['properties']['primary'], reverse=True):
private_ip = ipc['properties'].get('privateIPAddress')
if private_ip:
new_hostvars['private_ipv4_addresses'].append(private_ip)
pip_id = ipc['properties'].get('publicIPAddress', {}).get('id')
if pip_id:
pip = nic.public_ips[pip_id]
new_hostvars['public_ipv4_addresses'].append(pip._pip_model['properties']['ipAddress'])
pip_fqdn = pip._pip_model['properties'].get('dnsSettings', {}).get('fqdn')
if pip_fqdn:
new_hostvars['public_dns_hostnames'].append(pip_fqdn)
self._hostvars = new_hostvars
return self._hostvars
def _on_instanceview_response(self, vm_instanceview_model):
self._instanceview = vm_instanceview_model
self._powerstate = next((self._powerstate_regex.match(s.get('code', '')).group('powerstate')
for s in vm_instanceview_model.get('statuses', []) if self._powerstate_regex.match(s.get('code', ''))), 'unknown')
def _on_nic_response(self, nic_model, is_primary=False):
nic = AzureNic(nic_model=nic_model, inventory_client=self._inventory_client, is_primary=is_primary)
self.nics.append(nic)
class AzureNic(object):
def __init__(self, nic_model, inventory_client, is_primary=False):
self._nic_model = nic_model
self.is_primary = is_primary
self._inventory_client = inventory_client
self.public_ips = {}
for ipc in nic_model['properties']['ipConfigurations']:
pip = ipc['properties'].get('publicIPAddress')
if pip:
self._inventory_client._enqueue_get(url=pip['id'], api_version=self._inventory_client._network_api_version, handler=self._on_pip_response)
def _on_pip_response(self, pip_model):
self.public_ips[pip_model['id']] = AzurePip(pip_model)
class AzurePip(object):
def __init__(self, pip_model):
self._pip_model = pip_model
Loading…
Cancel
Save