From caf8bbf3bdea2fb6ac70deb26f5820127e1d572d Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Fri, 11 Aug 2017 17:33:39 -0700 Subject: [PATCH] Add Azure support to ansible-test. (#28092) * Initial Azure cloud testing support. * Add missing Azure requirements. * Fix test handling of setup and requirements. * Update Azure cloud plugin. * Add setup_azure role for integration tests. * Update minimal Azure integration test sample. --- packaging/requirements/requirements-azure.txt | 2 + .../cloud-config-azure.yml.template | 31 +++ test/integration/inventory | 3 + .../targets/azure_rm_virtualnetwork/aliases | 2 + .../azure_rm_virtualnetwork/meta/main.yml | 2 + .../azure_rm_virtualnetwork/tasks/main.yml | 7 + .../targets/setup_azure/tasks/main.yml | 2 + test/runner/lib/classification.py | 21 +- test/runner/lib/cloud/azure.py | 194 ++++++++++++++++++ test/runner/lib/core_ci.py | 1 + test/runner/lib/http.py | 4 +- 11 files changed, 261 insertions(+), 8 deletions(-) create mode 100644 test/integration/cloud-config-azure.yml.template create mode 100644 test/integration/targets/azure_rm_virtualnetwork/aliases create mode 100644 test/integration/targets/azure_rm_virtualnetwork/meta/main.yml create mode 100644 test/integration/targets/azure_rm_virtualnetwork/tasks/main.yml create mode 100644 test/integration/targets/setup_azure/tasks/main.yml create mode 100644 test/runner/lib/cloud/azure.py diff --git a/packaging/requirements/requirements-azure.txt b/packaging/requirements/requirements-azure.txt index b28cdb57542..a586e99f2d2 100644 --- a/packaging/requirements/requirements-azure.txt +++ b/packaging/requirements/requirements-azure.txt @@ -1,3 +1,5 @@ +packaging +requests[security] azure-mgmt-compute>=2.0.0,<3 azure-mgmt-network>=1.3.0,<2 azure-mgmt-storage>=1.2.0,<2 diff --git a/test/integration/cloud-config-azure.yml.template b/test/integration/cloud-config-azure.yml.template new file mode 100644 index 00000000000..14bb06e7994 --- /dev/null +++ b/test/integration/cloud-config-azure.yml.template @@ -0,0 +1,31 @@ +# This is the configuration template for ansible-test Azure integration tests. +# +# You do not need this template if you are: +# +# 1) Running integration tests without using ansible-test. +# 2) Using the automatically provisioned Azure credentials in ansible-test. +# +# If you do not want to use the automatically provisioned temporary Azure credentials, +# fill in the values below and save this file without the .template extension. +# This will cause ansible-test to use the given configuration instead of temporary credentials. +# +# NOTE: Automatic provisioning of Azure credentials requires one of: +# 1) ansible-core-ci API key in ~/.ansible-core-ci.key +# 2) Sherlock URL (including API key) in ~/.ansible-sherlock-ci.cfg + +# Provide either Service Principal or Active Directory credentials below. + +# Service Principal +AZURE_CLIENT_ID= +AZURE_SECRET= +AZURE_SUBSCRIPTION_ID= +AZURE_TENANT= + +# Active Directory +AZURE_AD_USER= +AZURE_PASSWORD= +AZURE_SUBSCRIPTION_ID= + +# Resource Groups +RESOURCE_GROUP= +RESOURCE_GROUP_SECONDARY= diff --git a/test/integration/inventory b/test/integration/inventory index 68f129c8af3..a49843c5180 100644 --- a/test/integration/inventory +++ b/test/integration/inventory @@ -50,3 +50,6 @@ overridden_in_parent=2000 [amazon] localhost ansible_ssh_host=127.0.0.1 ansible_connection=local + +[azure] +localhost ansible_ssh_host=127.0.0.1 ansible_connection=local diff --git a/test/integration/targets/azure_rm_virtualnetwork/aliases b/test/integration/targets/azure_rm_virtualnetwork/aliases new file mode 100644 index 00000000000..b1cd4a5978c --- /dev/null +++ b/test/integration/targets/azure_rm_virtualnetwork/aliases @@ -0,0 +1,2 @@ +cloud/azure +destructive diff --git a/test/integration/targets/azure_rm_virtualnetwork/meta/main.yml b/test/integration/targets/azure_rm_virtualnetwork/meta/main.yml new file mode 100644 index 00000000000..95e1952f989 --- /dev/null +++ b/test/integration/targets/azure_rm_virtualnetwork/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: + - setup_azure diff --git a/test/integration/targets/azure_rm_virtualnetwork/tasks/main.yml b/test/integration/targets/azure_rm_virtualnetwork/tasks/main.yml new file mode 100644 index 00000000000..19a3eacb92e --- /dev/null +++ b/test/integration/targets/azure_rm_virtualnetwork/tasks/main.yml @@ -0,0 +1,7 @@ +- block: + - name: create a virtual network + azure_rm_virtualnetwork: + resource_group: '{{ resource_group }}' + name: test + address_prefixes_cidr: + - "10.1.0.0/16" diff --git a/test/integration/targets/setup_azure/tasks/main.yml b/test/integration/targets/setup_azure/tasks/main.yml new file mode 100644 index 00000000000..17132a497b7 --- /dev/null +++ b/test/integration/targets/setup_azure/tasks/main.yml @@ -0,0 +1,2 @@ +- pip: + requirements: '{{ role_path }}/../../../../packaging/requirements/requirements-azure.txt' diff --git a/test/runner/lib/classification.py b/test/runner/lib/classification.py index 8359121100f..a7b21f648d1 100644 --- a/test/runner/lib/classification.py +++ b/test/runner/lib/classification.py @@ -347,9 +347,20 @@ class PathMapper(object): if path.startswith('packaging/'): if path.startswith('packaging/requirements/'): - return { - 'integration': 'ansible', - } + if name.startswith('requirements-') and ext == '.txt': + component = name.split('-', 1)[1] + + candidates = ( + 'cloud/%s/' % component, + ) + + for candidate in candidates: + if candidate in self.integration_targets_by_alias: + return { + 'integration': candidate, + } + + return all_tests(self.args) # broad impact, run all tests return minimal @@ -467,9 +478,7 @@ class PathMapper(object): return all_tests(self.args) # test infrastructure, run all tests if path == 'setup.py': - return { - 'integration': 'ansible', - } + return all_tests(self.args) # broad impact, run all tests if path == '.yamllint': return { diff --git a/test/runner/lib/cloud/azure.py b/test/runner/lib/cloud/azure.py new file mode 100644 index 00000000000..97e34e29694 --- /dev/null +++ b/test/runner/lib/cloud/azure.py @@ -0,0 +1,194 @@ +"""Azure plugin for integration tests.""" +from __future__ import absolute_import, print_function + +import os + +from lib.util import ( + ApplicationError, + display, + is_shippable, +) + +from lib.cloud import ( + CloudProvider, + CloudEnvironment, +) + +from lib.http import ( + HttpClient, + urlparse, + urlunparse, + parse_qs, +) + +from lib.core_ci import ( + AnsibleCoreCI, +) + + +class AzureCloudProvider(CloudProvider): + """Azure cloud provider plugin. Sets up cloud resources before delegation.""" + SHERLOCK_CONFIG_PATH = os.path.expanduser('~/.ansible-sherlock-ci.cfg') + + def filter(self, targets, exclude): + """Filter out the cloud tests when the necessary config and resources are not available. + :type targets: tuple[TestTarget] + :type exclude: list[str] + """ + if os.path.isfile(self.config_static_path): + return + + aci = self._create_ansible_core_ci() + + if os.path.isfile(aci.ci_key): + return + + if os.path.isfile(self.SHERLOCK_CONFIG_PATH): + return + + if is_shippable(): + return + + super(AzureCloudProvider, self).filter(targets, exclude) + + def setup(self): + """Setup the cloud resource before delegation and register a cleanup callback.""" + super(AzureCloudProvider, self).setup() + + if not self._use_static_config(): + self._setup_dynamic() + + get_config(self.config_path) # check required variables + + def _setup_dynamic(self): + """Request Azure credentials through Sherlock.""" + display.info('Provisioning %s cloud environment.' % self.platform, verbosity=1) + + config = self._read_config_template() + response = {} + + if os.path.isfile(self.SHERLOCK_CONFIG_PATH): + with open(self.SHERLOCK_CONFIG_PATH, 'r') as sherlock_fd: + sherlock_uri = sherlock_fd.readline().strip() + '&rgcount=2' + + parts = urlparse(sherlock_uri) + query_string = parse_qs(parts.query) + base_uri = urlunparse(parts[:4] + ('', '')) + + if 'code' not in query_string: + example_uri = 'https://example.azurewebsites.net/api/sandbox-provisioning' + raise ApplicationError('The Sherlock URI must include the API key in the query string. Example: %s?code=xxx' % example_uri) + + display.info('Initializing azure/sherlock from: %s' % base_uri, verbosity=1) + + http = HttpClient(self.args) + result = http.get(sherlock_uri) + + display.info('Started azure/sherlock from: %s' % base_uri, verbosity=1) + + if not self.args.explain: + response = result.json() + else: + aci = self._create_ansible_core_ci() + + aci_result = aci.start() + + if not self.args.explain: + response = aci_result['azure'] + + if not self.args.explain: + values = dict( + AZURE_CLIENT_ID=response['clientId'], + AZURE_SECRET=response['clientSecret'], + AZURE_SUBSCRIPTION_ID=response['subscriptionId'], + AZURE_TENANT=response['tenantId'], + RESOURCE_GROUP=response['resourceGroupNames'][0], + RESOURCE_GROUP_SECONDARY=response['resourceGroupNames'][1], + ) + + config = '\n'.join('%s: %s' % (key, values[key]) for key in sorted(values)) + + self._write_config(config) + + def _create_ansible_core_ci(self): + """ + :rtype: AnsibleCoreCI + """ + return AnsibleCoreCI(self.args, 'azure', 'sherlock', persist=False, stage=self.args.remote_stage) + + +class AzureCloudEnvironment(CloudEnvironment): + """Azure cloud environment plugin. Updates integration test environment after delegation.""" + def configure_environment(self, env, cmd): + """ + :type env: dict[str, str] + :type cmd: list[str] + """ + config = get_config(self.config_path) + + cmd.append('-e') + cmd.append('resource_prefix=%s' % self.resource_prefix) + cmd.append('-e') + cmd.append('resource_group=%s' % config['RESOURCE_GROUP']) + cmd.append('-e') + cmd.append('resource_group_secondary=%s' % config['RESOURCE_GROUP_SECONDARY']) + + for key in config: + env[key] = config[key] + + def on_failure(self, target, tries): + """ + :type target: TestTarget + :type tries: int + """ + if not tries and self.managed: + display.notice('If %s failed due to permissions, the test policy may need to be updated. ' + 'For help, consult @mattclay or @gundalow on GitHub or #ansible-devel on IRC.' % target.name) + + @property + def inventory_hosts(self): + """ + :rtype: str | None + """ + return 'azure' + + +def get_config(config_path): + """ + :param config_path: str + :return: dict[str, str] + """ + with open(config_path, 'r') as config_fd: + lines = [line for line in config_fd.read().splitlines() if ':' in line and line.strip() and not line.strip().startswith('#')] + config = dict((kvp[0].strip(), kvp[1].strip()) for kvp in [line.split(':', 1) for line in lines]) + + rg_vars = ( + 'RESOURCE_GROUP', + 'RESOURCE_GROUP_SECONDARY', + ) + + sp_vars = ( + 'AZURE_CLIENT_ID', + 'AZURE_SECRET', + 'AZURE_SUBSCRIPTION_ID', + 'AZURE_TENANT', + ) + + ad_vars = ( + 'AZURE_AD_USER', + 'AZURE_PASSWORD', + 'AZURE_SUBSCRIPTION_ID', + ) + + rg_ok = all(var in config for var in rg_vars) + sp_ok = all(var in config for var in sp_vars) + ad_ok = all(var in config for var in ad_vars) + + if not rg_ok: + raise ApplicationError('Resource groups must be defined with: %s' % ', '.join(sorted(rg_vars))) + + if not sp_ok and not ad_ok: + raise ApplicationError('Credentials must be defined using either:\nService Principal: %s\nActive Directory: %s' % ( + ', '.join(sorted(sp_vars)), ', '.join(sorted(ad_vars)))) + + return config diff --git a/test/runner/lib/core_ci.py b/test/runner/lib/core_ci.py index d154254e72c..5a33452e6fd 100644 --- a/test/runner/lib/core_ci.py +++ b/test/runner/lib/core_ci.py @@ -56,6 +56,7 @@ class AnsibleCoreCI(object): aws_platforms = ( 'aws', + 'azure', 'windows', 'freebsd', 'rhel', diff --git a/test/runner/lib/http.py b/test/runner/lib/http.py index 9ce433127d6..53df1a6bc60 100644 --- a/test/runner/lib/http.py +++ b/test/runner/lib/http.py @@ -15,10 +15,10 @@ except ImportError: try: # noinspection PyCompatibility - from urlparse import urlparse + from urlparse import urlparse, urlunparse, parse_qs except ImportError: # noinspection PyCompatibility, PyUnresolvedReferences - from urllib.parse import urlparse # pylint: disable=locally-disabled, ungrouped-imports + from urllib.parse import urlparse, urlunparse, parse_qs # pylint: disable=locally-disabled, ungrouped-imports from lib.util import ( CommonConfig,