Overhaul ansible-test cloud test plugins. (#53044)

pull/53158/head
Matt Clay 6 years ago committed by GitHub
parent 9c644d9bcc
commit eeaff731de
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -11,11 +11,12 @@
#
# NOTE: Automatic provisioning of AWS credentials requires an ansible-core-ci API key.
[default]
aws_access_key: @ACCESS_KEY
aws_secret_key: @SECRET_KEY
security_token: @SECURITY_TOKEN
aws_region: @REGION
# aliases for backwards compatibility with older integration test playbooks
ec2_access_key: '{{ aws_access_key }}'
ec2_secret_key: '{{ aws_secret_key }}'
ec2_region: '{{ aws_region }}'
ec2_access_key: {{ aws_access_key }}
ec2_secret_key: {{ aws_secret_key }}
ec2_region: {{ aws_region }}

@ -13,6 +13,7 @@
# 1) ansible-core-ci API key in ~/.ansible-core-ci.key
# 2) Sherlock URL (including API key) in ~/.ansible-sherlock-ci.cfg
[default]
# Provide either Service Principal or Active Directory credentials below.
# Service Principal

@ -4,5 +4,6 @@
#
# 1) Running integration tests without using ansible-test.
#
[default]
cloudscale_api_token = @API_TOKEN

@ -11,7 +11,7 @@
#
# It is recommended that you DO NOT use this template unless you cannot use the simulator.
[cloudstack]
[default]
endpoint = http://@HOST:@PORT/client/api
key = @KEY
secret = @SECRET

@ -11,6 +11,7 @@
#
# It is recommended that you DO NOT use this template unless you cannot use the simulator.
[default]
gcp_project: @PROJECT
gcp_cred_file: @CRED_FILE
gcp_cred_kind: @CRED_KIND

@ -11,6 +11,7 @@
#
# It is recommended that you DO NOT use this template unless you cannot use the simulator.
[default]
gitlab_host: http://@HOST:@PORT
gitlab_login_token: @TOKEN
gitlab_runner_registration_token: @RUNNER_TOKEN

@ -12,7 +12,7 @@
# If you run with @FIXTURES enabled (true) then you can decide if you want to
# run in @REPLAY mode (true) or, record mode (false).
[default]
opennebula_url: @URL
opennebula_username: @USERNAME
opennebula_password: @PASSWORD

@ -11,7 +11,7 @@
#
# NOTE: Automatic provisioning of Tower credentials requires an ansible-core-ci API key.
[general]
[default]
version=@VERSION
host=@HOST
username=@USERNAME

@ -6,8 +6,8 @@
dns_prefix: "aks{{ resource_group | hash('md5') | truncate(10, True, '') }}"
kubernetes_version: 1.7.7
service_principal:
client_id: "{{ lookup('env', 'AZURE_CLIENT_ID') }}"
client_secret: "{{ lookup('env', 'AZURE_SECRET') }}"
client_id: "{{ azure_client_id }}"
client_secret: "{{ azure_secret }}"
linux_profile:
admin_username: azureuser
ssh_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDSPmiqkvDH1/+MDAVDZT8381aYqp73Odz8cnD5hegNhqtXajqtiH0umVg7HybX3wt1HjcrwKJovZURcIbbcDvzdH2bnYbF93T4OLXA0bIfuIp6M86x1iutFtXdpN3TTicINrmSXEE2Ydm51iMu77B08ZERjVaToya2F7vC+egfoPvibf7OLxE336a5tPCywavvNihQjL8sjgpDT5AAScjb3YqK/6VLeQ18Ggt8/ufINsYkb+9/Ji/3OcGFeflnDXq80vPUyF3u4iIylob6RSZenC38cXmQB05tRNxS1B6BXCjMRdy0v4pa7oKM2GA4ADKpNrr0RI9ed+peRFwmsclH test@ansible
@ -36,8 +36,8 @@
dns_prefix: "aks{{ resource_group | hash('md5') | truncate(10, True, '') }}"
kubernetes_version: 1.7.7
service_principal:
client_id: "{{ lookup('env', 'AZURE_CLIENT_ID') }}"
client_secret: "{{ lookup('env', 'AZURE_SECRET') }}"
client_id: "{{ azure_client_id }}"
client_secret: "{{ azure_secret }}"
linux_profile:
admin_username: azureuser
ssh_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDSPmiqkvDH1/+MDAVDZT8381aYqp73Odz8cnD5hegNhqtXajqtiH0umVg7HybX3wt1HjcrwKJovZURcIbbcDvzdH2bnYbF93T4OLXA0bIfuIp6M86x1iutFtXdpN3TTicINrmSXEE2Ydm51iMu77B08ZERjVaToya2F7vC+egfoPvibf7OLxE336a5tPCywavvNihQjL8sjgpDT5AAScjb3YqK/6VLeQ18Ggt8/ufINsYkb+9/Ji/3OcGFeflnDXq80vPUyF3u4iIylob6RSZenC38cXmQB05tRNxS1B6BXCjMRdy0v4pa7oKM2GA4ADKpNrr0RI9ed+peRFwmsclH test@ansible
@ -74,7 +74,7 @@
dns_prefix: "aks{{ resource_group | hash('md5') | truncate(10, True, '') }}"
kubernetes_version: 1.7.7
service_principal:
client_id: "{{ lookup('env', 'AZURE_CLIENT_ID') }}"
client_id: "{{ azure_client_id }}"
linux_profile:
admin_username: azureuser
ssh_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDSPmiqkvDH1/+MDAVDZT8381aYqp73Odz8cnD5hegNhqtXajqtiH0umVg7HybX3wt1HjcrwKJovZURcIbbcDvzdH2bnYbF93T4OLXA0bIfuIp6M86x1iutFtXdpN3TTicINrmSXEE2Ydm51iMu77B08ZERjVaToya2F7vC+egfoPvibf7OLxE336a5tPCywavvNihQjL8sjgpDT5AAScjb3YqK/6VLeQ18Ggt8/ufINsYkb+9/Ji/3OcGFeflnDXq80vPUyF3u4iIylob6RSZenC38cXmQB05tRNxS1B6BXCjMRdy0v4pa7oKM2GA4ADKpNrr0RI9ed+peRFwmsclH test@ansible
@ -98,7 +98,7 @@
dns_prefix: "aks{{ resource_group | hash('md5') | truncate(10, True, '') }}"
kubernetes_version: 1.8.1
service_principal:
client_id: "{{ lookup('env', 'AZURE_CLIENT_ID') }}"
client_id: "{{ azure_client_id }}"
linux_profile:
admin_username: azureuser
ssh_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDSPmiqkvDH1/+MDAVDZT8381aYqp73Odz8cnD5hegNhqtXajqtiH0umVg7HybX3wt1HjcrwKJovZURcIbbcDvzdH2bnYbF93T4OLXA0bIfuIp6M86x1iutFtXdpN3TTicINrmSXEE2Ydm51iMu77B08ZERjVaToya2F7vC+egfoPvibf7OLxE336a5tPCywavvNihQjL8sjgpDT5AAScjb3YqK/6VLeQ18Ggt8/ufINsYkb+9/Ji/3OcGFeflnDXq80vPUyF3u4iIylob6RSZenC38cXmQB05tRNxS1B6BXCjMRdy0v4pa7oKM2GA4ADKpNrr0RI9ed+peRFwmsclH test@ansible
@ -122,7 +122,7 @@
dns_prefix: "aks{{ resource_group | hash('md5') | truncate(10, True, '') }}"
kubernetes_version: 1.8.1
service_principal:
client_id: "{{ lookup('env', 'AZURE_CLIENT_ID') }}"
client_id: "{{ azure_client_id }}"
linux_profile:
admin_username: azureuser
ssh_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDSPmiqkvDH1/+MDAVDZT8381aYqp73Odz8cnD5hegNhqtXajqtiH0umVg7HybX3wt1HjcrwKJovZURcIbbcDvzdH2bnYbF93T4OLXA0bIfuIp6M86x1iutFtXdpN3TTicINrmSXEE2Ydm51iMu77B08ZERjVaToya2F7vC+egfoPvibf7OLxE336a5tPCywavvNihQjL8sjgpDT5AAScjb3YqK/6VLeQ18Ggt8/ufINsYkb+9/Ji/3OcGFeflnDXq80vPUyF3u4iIylob6RSZenC38cXmQB05tRNxS1B6BXCjMRdy0v4pa7oKM2GA4ADKpNrr0RI9ed+peRFwmsclH test@ansible

@ -1,15 +1,9 @@
- name: Prepare random number
set_fact:
rpfx: "{{ resource_group | hash('md5') | truncate(7, True, '') }}{{ 1000 | random }}"
tenant_id: "{{ lookup('env','AZURE_TENANT') }}"
tenant_id: "{{ azure_tenant }}"
run_once: yes
- name: set service principal info
set_fact:
azure_client_id: "{{ lookup('env','AZURE_CLIENT_ID') }}"
azure_secret: "{{ lookup('env','AZURE_SECRET') }}"
no_log: yes
- name: lookup service principal object id
set_fact:
object_id: "{{ lookup('azure_service_principal_attribute',

@ -4,7 +4,7 @@
url: 'https://api.cloudscale.ch/v1/volumes'
method: POST
headers:
Authorization: 'Bearer {{ lookup("env", "CLOUDSCALE_API_TOKEN") }}'
Authorization: 'Bearer {{ cloudscale_api_token }}'
body:
name: '{{ cloudscale_resource_prefix }}-duplicate'
size_gb: 50

@ -1,7 +0,0 @@
- name: create the provider credentials from the cloud script
set_fact:
nios_provider:
host: "{{ lookup('env', 'NIOS_HOST') }}"
username: admin
password: infoblox
- debug: var=nios_provider

@ -2,8 +2,6 @@
- name: Check that SSL is available
tower_organization:
name: Default
environment:
TOWER_HOST: "https://{{ lookup('env', 'TOWER_HOST') }}"
register: result
- name: Check we haven't changed anything
@ -14,7 +12,6 @@
tower_organization:
name: Default
environment:
TOWER_HOST: "https://{{ lookup('env', 'TOWER_HOST') }}"
TOWER_CERTIFICATE: /dev/null # force check failure
ignore_errors: true
register: check_ssl_is_used
@ -42,7 +39,6 @@
tower_organization:
name: Default
environment:
TOWER_HOST: "https://{{ lookup('env', 'TOWER_HOST') }}"
TOWER_CERTIFICATE: /dev/null # should not fail because verify_ssl is disabled
always:
- name: Delete ~/.tower_cli.cfg

@ -16,20 +16,20 @@
- name: Update the project (to clone the git repo)
uri:
url: "https://{{ lookup('env', 'TOWER_HOST') }}/api/v2/projects/{{ result.id }}/update/"
url: "https://{{ tower_host }}/api/v2/projects/{{ result.id }}/update/"
method: POST
user: "{{ lookup('env', 'TOWER_USERNAME') }}"
password: "{{ lookup('env', 'TOWER_PASSWORD') }}"
user: "{{ tower_username }}"
password: "{{ tower_password }}"
validate_certs: false
status_code: 202
force_basic_auth: true
- name: Wait for the project to be status=successful
uri:
url: "https://{{ lookup('env', 'TOWER_HOST') }}/api/v2/projects/{{ result.id }}/"
url: "https://{{ tower_host }}/api/v2/projects/{{ result.id }}/"
method: GET
user: "{{ lookup('env', 'TOWER_USERNAME') }}"
password: "{{ lookup('env', 'TOWER_PASSWORD') }}"
user: "{{ tower_username }}"
password: "{{ tower_password }}"
validate_certs: false
force_basic_auth: true
return_content: true

@ -6,7 +6,7 @@
- assert:
that:
- "result.tower_version == '{{ lookup('env', 'TOWER_VERSION') }}'"
- "result.tower_version == '{{ tower_version }}'"
- name: Make sure the default Default organization exists
tower_organization:

@ -1,8 +1,8 @@
- name: Fetch project_base_dir
uri:
url: "https://{{ lookup('env', 'TOWER_HOST') }}/api/v2/config/"
user: "{{ lookup('env', 'TOWER_USERNAME') }}"
password: "{{ lookup('env', 'TOWER_PASSWORD') }}"
url: "https://{{ tower_host}}/api/v2/config/"
user: "{{ tower_username }}"
password: "{{ tower_password }}"
validate_certs: false
return_content: true
register: awx_config

@ -1,30 +1,30 @@
- name: Get unified job template ID for Demo Job Template"
uri:
url: "https://{{ lookup('env', 'TOWER_HOST') }}/api/v2/unified_job_templates/?name=Demo+Job+Template"
url: "https://{{ tower_host }}/api/v2/unified_job_templates/?name=Demo+Job+Template"
method: GET
password: "{{ lookup('env', 'TOWER_PASSWORD') }}"
user: "{{ lookup('env', 'TOWER_USERNAME') }}"
password: "{{ tower_password }}"
user: "{{ tower_username }}"
validate_certs: False
register: unified_job
- name: Build workflow
uri:
url: "https://{{ lookup('env', 'TOWER_HOST') }}/api/v2/workflow_job_templates/"
url: "https://{{ tower_host }}/api/v2/workflow_job_templates/"
body:
name: "Success Template"
variables: "---"
extra_vars: ""
body_format: 'json'
method: 'POST'
password: "{{ lookup('env', 'TOWER_PASSWORD') }}"
password: "{{ tower_password }}"
status_code: 201
user: "{{ lookup('env', 'TOWER_USERNAME') }}"
user: "{{ tower_username }}"
validate_certs: False
register: workflow
- name: Add a node
uri:
url: "https://{{ lookup('env', 'TOWER_HOST') }}/api/v2/workflow_job_templates/{{ workflow.json.id }}/workflow_nodes/"
url: "https://{{ tower_host }}/api/v2/workflow_job_templates/{{ workflow.json.id }}/workflow_nodes/"
body:
credential: null
diff_mode: null
@ -38,15 +38,15 @@
verbosity: null
body_format: 'json'
method: 'POST'
password: "{{ lookup('env', 'TOWER_PASSWORD') }}"
password: "{{ tower_password }}"
status_code: 201
user: "{{ lookup('env', 'TOWER_USERNAME') }}"
user: "{{ tower_username }}"
validate_certs: False
register: node1
- name: Add a node
uri:
url: "https://{{ lookup('env', 'TOWER_HOST') }}/api/v2/workflow_job_templates/{{ workflow.json.id }}/workflow_nodes/"
url: "https://{{ tower_host }}/api/v2/workflow_job_templates/{{ workflow.json.id }}/workflow_nodes/"
body:
credential: null
diff_mode: null
@ -60,19 +60,19 @@
verbosity: null
body_format: 'json'
method: 'POST'
password: "{{ lookup('env', 'TOWER_PASSWORD') }}"
password: "{{ tower_password }}"
status_code: 201
user: "{{ lookup('env', 'TOWER_USERNAME') }}"
user: "{{ tower_username }}"
validate_certs: False
register: node2
- name: "Link nodes {{ node2.json.id }} to {{ node1.json.id }}"
uri:
url: "https://{{ lookup('env', 'TOWER_HOST') }}/api/v2/workflow_job_template_nodes/{{ node1.json.id }}/success_nodes/"
url: "https://{{ tower_host }}/api/v2/workflow_job_template_nodes/{{ node1.json.id }}/success_nodes/"
body: '{ "id": {{ node2.json.id }} }'
body_format: 'json'
method: 'POST'
password: "{{ lookup('env', 'TOWER_PASSWORD') }}"
password: "{{ tower_password }}"
status_code: 204
user: "{{ lookup('env', 'TOWER_USERNAME') }}"
user: "{{ tower_username }}"
validate_certs: False

@ -17,20 +17,20 @@
- name: Update the project (to clone the git repo)
uri:
url: "https://{{ lookup('env', 'TOWER_HOST') }}/api/v2/projects/{{ result.id }}/update/"
url: "https://{{ tower_host }}/api/v2/projects/{{ result.id }}/update/"
method: POST
user: "{{ lookup('env', 'TOWER_USERNAME') }}"
password: "{{ lookup('env', 'TOWER_PASSWORD') }}"
user: "{{ tower_username }}"
password: "{{ tower_password }}"
validate_certs: false
status_code: 202
force_basic_auth: true
- name: Wait for the project to be status=successful
uri:
url: "https://{{ lookup('env', 'TOWER_HOST') }}/api/v2/projects/{{ result.id }}/"
url: "https://{{ tower_host }}/api/v2/projects/{{ result.id }}/"
method: GET
user: "{{ lookup('env', 'TOWER_USERNAME') }}"
password: "{{ lookup('env', 'TOWER_PASSWORD') }}"
user: "{{ tower_username }}"
password: "{{ tower_password }}"
validate_certs: false
force_basic_auth: true
return_content: true

@ -253,7 +253,7 @@ class CloudProvider(CloudBase):
"""Base class for cloud provider plugins. Sets up cloud resources before delegation."""
TEST_DIR = 'test/integration'
def __init__(self, args, config_extension='.yml'):
def __init__(self, args, config_extension='.ini'):
"""
:type args: IntegrationConfig
:type config_extension: str
@ -390,10 +390,9 @@ class CloudEnvironment(CloudBase):
pass
@abc.abstractmethod
def configure_environment(self, env, cmd):
"""Configuration which should be done once for each test target.
:type env: dict[str, str]
:type cmd: list[str]
def get_environment_config(self):
"""
:rtype: CloudEnvironmentConfig
"""
pass
@ -404,9 +403,17 @@ class CloudEnvironment(CloudBase):
"""
pass
@property
def inventory_hosts(self):
class CloudEnvironmentConfig(object):
"""Configuration for the environment."""
def __init__(self, env_vars=None, ansible_vars=None, module_defaults=None, callback_plugins=None):
"""
:rtype: str | None
:type env_vars: dict[str, str] | None
:type ansible_vars: dict[str, any] | None
:type module_defaults: dict[str, dict[str, any]] | None
:type callback_plugins: list[str] | None
"""
return None
self.env_vars = env_vars
self.ansible_vars = ansible_vars
self.module_defaults = module_defaults
self.callback_plugins = callback_plugins

@ -7,6 +7,7 @@ import time
from lib.cloud import (
CloudProvider,
CloudEnvironment,
CloudEnvironmentConfig,
)
from lib.util import (
@ -37,7 +38,7 @@ class ACMEProvider(CloudProvider):
"""
:type args: TestConfig
"""
super(ACMEProvider, self).__init__(args, config_extension='.ini')
super(ACMEProvider, self).__init__(args)
# The simulator must be pinned to a specific version to guarantee CI passes with the version used.
if os.environ.get('ANSIBLE_ACME_CONTAINER'):
@ -175,12 +176,14 @@ class ACMEProvider(CloudProvider):
class ACMEEnvironment(CloudEnvironment):
"""ACME environment plugin. Updates integration test environment after delegation."""
def configure_environment(self, env, cmd):
def get_environment_config(self):
"""
:type env: dict[str, str]
:type cmd: list[str]
:rtype: CloudEnvironmentConfig
"""
ansible_vars = dict(
acme_host=self._get_cloud_config('acme_host'),
)
# Send the container IP down to the integration test(s)
cmd.append('-e')
cmd.append('acme_host=%s' % self._get_cloud_config('acme_host'))
return CloudEnvironmentConfig(
ansible_vars=ansible_vars,
)

@ -7,11 +7,13 @@ from lib.util import (
ApplicationError,
display,
is_shippable,
ConfigParser,
)
from lib.cloud import (
CloudProvider,
CloudEnvironment,
CloudEnvironmentConfig,
)
from lib.core_ci import (
@ -84,16 +86,22 @@ class AwsCloudProvider(CloudProvider):
class AwsCloudEnvironment(CloudEnvironment):
"""AWS cloud environment plugin. Updates integration test environment after delegation."""
def configure_environment(self, env, cmd):
def get_environment_config(self):
"""
:type env: dict[str, str]
:type cmd: list[str]
:rtype: CloudEnvironmentConfig
"""
cmd.append('-e')
cmd.append('@%s' % self.config_path)
parser = ConfigParser()
parser.read(self.config_path)
cmd.append('-e')
cmd.append('resource_prefix=%s' % self.resource_prefix)
ansible_vars = dict(
resource_prefix=self.resource_prefix,
)
ansible_vars.update(dict(parser.items('default')))
return CloudEnvironmentConfig(
ansible_vars=ansible_vars,
)
def on_failure(self, target, tries):
"""

@ -7,11 +7,13 @@ from lib.util import (
ApplicationError,
display,
is_shippable,
ConfigParser,
)
from lib.cloud import (
CloudProvider,
CloudEnvironment,
CloudEnvironmentConfig,
)
from lib.http import (
@ -124,6 +126,8 @@ class AzureCloudProvider(CloudProvider):
config = '\n'.join('%s: %s' % (key, values[key]) for key in sorted(values))
config = '[default]\n' + config
self._write_config(config)
def _create_ansible_core_ci(self):
@ -135,22 +139,22 @@ class AzureCloudProvider(CloudProvider):
class AzureCloudEnvironment(CloudEnvironment):
"""Azure cloud environment plugin. Updates integration test environment after delegation."""
def configure_environment(self, env, cmd):
def get_environment_config(self):
"""
:type env: dict[str, str]
:type cmd: list[str]
:rtype: CloudEnvironmentConfig
"""
config = get_config(self.config_path)
env_vars = get_config(self.config_path)
cmd.append('-e')
cmd.append('resource_prefix=%s' % self.resource_prefix)
cmd.append('-e')
cmd.append('resource_group=%s' % config['RESOURCE_GROUP'])
cmd.append('-e')
cmd.append('resource_group_secondary=%s' % config['RESOURCE_GROUP_SECONDARY'])
ansible_vars = dict(
resource_prefix=self.resource_prefix,
)
for key in config:
env[key] = config[key]
ansible_vars.update(dict((key.lower(), value) for key, value in env_vars.items()))
return CloudEnvironmentConfig(
env_vars=env_vars,
ansible_vars=ansible_vars,
)
def on_failure(self, target, tries):
"""
@ -167,9 +171,10 @@ def get_config(config_path):
:type config_path: str
:rtype: dict[str, str]
"""
with open(config_path, 'r') as config_fd:
lines = [line for line in config_fd.read().splitlines() if ':' in line and line.strip() and not line.strip().startswith('#')]
config = dict((kvp[0].strip(), kvp[1].strip()) for kvp in [line.split(':', 1) for line in lines])
parser = ConfigParser()
parser.read(config_path)
config = dict((key.upper(), value) for key, value in parser.items('default'))
rg_vars = (
'RESOURCE_GROUP',

@ -11,6 +11,7 @@ from os.path import isfile
from lib.cloud import (
CloudProvider,
CloudEnvironment,
CloudEnvironmentConfig,
)
from lib.util import ConfigParser, display
@ -25,7 +26,7 @@ class CloudscaleCloudProvider(CloudProvider):
"""
:type args: TestConfig
"""
super(CloudscaleCloudProvider, self).__init__(args, config_extension='.ini')
super(CloudscaleCloudProvider, self).__init__(args)
def filter(self, targets, exclude):
"""Filter out the cloud tests when the necessary config and resources are not available.
@ -38,6 +39,7 @@ class CloudscaleCloudProvider(CloudProvider):
super(CloudscaleCloudProvider, self).filter(targets, exclude)
def setup(self):
"""Setup the cloud resource before delegation and register a cleanup callback."""
super(CloudscaleCloudProvider, self).setup()
if isfile(self.config_static_path):
@ -46,28 +48,30 @@ class CloudscaleCloudProvider(CloudProvider):
verbosity=1)
self.config_path = self.config_static_path
self.managed = False
return True
return False
class CloudscaleCloudEnvironment(CloudEnvironment):
"""Cloudscale cloud environment plugin. Updates integration test environment
after delegation.
"""
def configure_environment(self, env, cmd):
def get_environment_config(self):
"""
:type env: dict[str, str]
:type cmd: list[str]
:rtype: CloudEnvironmentConfig
"""
parser = ConfigParser()
parser.read(self.config_path)
changes = dict(
env_vars = dict(
CLOUDSCALE_API_TOKEN=parser.get('default', 'cloudscale_api_token'),
)
env.update(changes)
ansible_vars = dict(
cloudscale_resource_prefix=self.resource_prefix,
)
ansible_vars.update(dict((key.lower(), value) for key, value in env_vars.items()))
cmd.append('-e')
cmd.append('cloudscale_resource_prefix=%s' % self.resource_prefix)
return CloudEnvironmentConfig(
env_vars=env_vars,
ansible_vars=ansible_vars,
)

@ -9,6 +9,7 @@ import time
from lib.cloud import (
CloudProvider,
CloudEnvironment,
CloudEnvironmentConfig,
)
from lib.util import (
@ -45,7 +46,7 @@ class CsCloudProvider(CloudProvider):
"""
:type args: TestConfig
"""
super(CsCloudProvider, self).__init__(args, config_extension='.ini')
super(CsCloudProvider, self).__init__(args)
# The simulator must be pinned to a specific version to guarantee CI passes with the version used.
self.image = 'quay.io/ansible/cloudstack-test-container:1.2.0'
@ -262,16 +263,27 @@ class CsCloudProvider(CloudProvider):
class CsCloudEnvironment(CloudEnvironment):
"""CloudStack cloud environment plugin. Updates integration test environment after delegation."""
def configure_environment(self, env, cmd):
def get_environment_config(self):
"""
:type env: dict[str, str]
:type cmd: list[str]
:rtype: CloudEnvironmentConfig
"""
changes = dict(
CLOUDSTACK_CONFIG=self.config_path,
parser = ConfigParser()
parser.read(self.config_path)
config = dict(parser.items('default'))
env_vars = dict(
CLOUDSTACK_ENDPOINT=config['endpoint'],
CLOUDSTACK_KEY=config['key'],
CLOUDSTACK_SECRET=config['secret'],
CLOUDSTACK_TIMEOUT=config['timeout'],
)
env.update(changes)
ansible_vars = dict(
cs_resource_prefix=self.resource_prefix,
)
cmd.append('-e')
cmd.append('cs_resource_prefix=%s' % self.resource_prefix)
return CloudEnvironmentConfig(
env_vars=env_vars,
ansible_vars=ansible_vars,
)

@ -7,6 +7,7 @@ import os
from . import (
CloudProvider,
CloudEnvironment,
CloudEnvironmentConfig,
)
from ..util import (
@ -175,13 +176,15 @@ class ForemanEnvironment(CloudEnvironment):
Updates integration test environment after delegation.
"""
def configure_environment(self, env, cmd):
def get_environment_config(self):
"""
:type env: dict[str, str]
:type cmd: list[str]
:rtype: CloudEnvironmentConfig
"""
env_vars = dict(
FOREMAN_HOST=self._get_cloud_config('FOREMAN_HOST'),
FOREMAN_PORT=self._get_cloud_config('FOREMAN_PORT'),
)
# Send the container IP down to the integration test(s)
env['FOREMAN_HOST'] = self._get_cloud_config('FOREMAN_HOST')
env['FOREMAN_PORT'] = self._get_cloud_config('FOREMAN_PORT')
return CloudEnvironmentConfig(
env_vars=env_vars,
)

@ -7,11 +7,13 @@ import os
from lib.util import (
display,
ConfigParser,
)
from lib.cloud import (
CloudProvider,
CloudEnvironment,
CloudEnvironmentConfig,
)
@ -41,22 +43,19 @@ class GcpCloudProvider(CloudProvider):
class GcpCloudEnvironment(CloudEnvironment):
"""GCP cloud environment plugin. Updates integration test environment after delegation."""
def configure_environment(self, env, cmd):
def get_environment_config(self):
"""
:type env: dict[str, str]
:type cmd: list[str]
:rtype: CloudEnvironmentConfig
"""
cmd.append('-e')
cmd.append('@%s' % self.config_path)
parser = ConfigParser()
parser.read(self.config_path)
cmd.append('-e')
cmd.append('resource_prefix=%s' % self.resource_prefix)
ansible_vars = dict(
resource_prefix=self.resource_prefix,
)
def on_failure(self, target, tries):
"""
:type target: TestTarget
:type tries: int
"""
if not tries and self.managed:
display.notice('%s failed' % target.name)
ansible_vars.update(dict(parser.items('default')))
return CloudEnvironmentConfig(
ansible_vars=ansible_vars,
)

@ -7,6 +7,7 @@ import os
from . import (
CloudProvider,
CloudEnvironment,
CloudEnvironmentConfig,
)
from ..util import (
@ -174,12 +175,18 @@ class NiosEnvironment(CloudEnvironment):
Updates integration test environment after delegation.
"""
def configure_environment(self, env, cmd):
def get_environment_config(self):
"""
:type env: dict[str, str]
:type cmd: list[str]
:rtype: CloudEnvironmentConfig
"""
ansible_vars = dict(
nios_provider=dict(
host=self._get_cloud_config('NIOS_HOST'),
username='admin',
password='infoblox',
),
)
# Send the container IP down to the integration test(s)
env['NIOS_HOST'] = self._get_cloud_config('NIOS_HOST')
return CloudEnvironmentConfig(
ansible_vars=ansible_vars,
)

@ -2,11 +2,13 @@
from lib.cloud import (
CloudProvider,
CloudEnvironment
CloudEnvironment,
CloudEnvironmentConfig,
)
from lib.util import (
display,
ConfigParser,
)
@ -43,14 +45,19 @@ class OpenNebulaCloudEnvironment(CloudEnvironment):
"""
Updates integration test environment after delegation. Will setup the config file as parameter.
"""
def configure_environment(self, env, cmd):
def get_environment_config(self):
"""
:type env: dict[str, str]
:type cmd: list[str]
:rtype: CloudEnvironmentConfig
"""
cmd.append('-e')
cmd.append('@%s' % self.config_path)
parser = ConfigParser()
parser.read(self.config_path)
ansible_vars = dict(
resource_prefix=self.resource_prefix,
)
cmd.append('-e')
cmd.append('resource_prefix=%s' % self.resource_prefix)
ansible_vars.update(dict(parser.items('default')))
return CloudEnvironmentConfig(
ansible_vars=ansible_vars,
)

@ -9,6 +9,7 @@ import time
from lib.cloud import (
CloudProvider,
CloudEnvironment,
CloudEnvironmentConfig,
)
from lib.util import (
@ -211,13 +212,14 @@ class OpenShiftCloudProvider(CloudProvider):
class OpenShiftCloudEnvironment(CloudEnvironment):
"""OpenShift cloud environment plugin. Updates integration test environment after delegation."""
def configure_environment(self, env, cmd):
def get_environment_config(self):
"""
:type env: dict[str, str]
:type cmd: list[str]
:rtype: CloudEnvironmentConfig
"""
changes = dict(
env_vars = dict(
K8S_AUTH_KUBECONFIG=self.config_path,
)
env.update(changes)
return CloudEnvironmentConfig(
env_vars=env_vars,
)

@ -16,6 +16,7 @@ from lib.util import (
from lib.cloud import (
CloudProvider,
CloudEnvironment,
CloudEnvironmentConfig,
)
from lib.core_ci import (
@ -29,7 +30,7 @@ class TowerCloudProvider(CloudProvider):
"""
:type args: TestConfig
"""
super(TowerCloudProvider, self).__init__(args, config_extension='.cfg')
super(TowerCloudProvider, self).__init__(args)
self.aci = None
self.version = ''
@ -162,14 +163,20 @@ class TowerCloudEnvironment(CloudEnvironment):
time.sleep(5)
def configure_environment(self, env, cmd):
"""Configuration which should be done once for each test target.
:type env: dict[str, str]
:type cmd: list[str]
def get_environment_config(self):
"""
:rtype: CloudEnvironmentConfig
"""
config = TowerConfig.parse(self.config_path)
env.update(config.environment)
env_vars = config.environment
ansible_vars = dict((key.lower(), value) for key, value in env_vars.items())
return CloudEnvironmentConfig(
env_vars=env_vars,
ansible_vars=ansible_vars,
)
class TowerConfig(object):
@ -213,7 +220,7 @@ class TowerConfig(object):
'password',
)
values = dict((k, parser.get('general', k)) for k in keys)
values = dict((k, parser.get('default', k)) for k in keys)
config = TowerConfig(values)
missing = [k for k in keys if not values.get(k)]

@ -6,6 +6,7 @@ import os
from lib.cloud import (
CloudProvider,
CloudEnvironment,
CloudEnvironmentConfig,
)
from lib.util import (
@ -30,7 +31,7 @@ class VcenterProvider(CloudProvider):
"""
:type args: TestConfig
"""
super(VcenterProvider, self).__init__(args, config_extension='.ini')
super(VcenterProvider, self).__init__(args)
# The simulator must be pinned to a specific version to guarantee CI passes with the version used.
if os.environ.get('ANSIBLE_VCSIM_CONTAINER'):
@ -144,13 +145,19 @@ class VcenterProvider(CloudProvider):
class VcenterEnvironment(CloudEnvironment):
"""VMware vcenter/esx environment plugin. Updates integration test environment after delegation."""
def configure_environment(self, env, cmd):
def get_environment_config(self):
"""
:type env: dict[str, str]
:type cmd: list[str]
:rtype: CloudEnvironmentConfig
"""
cmd.append('-e')
cmd.append('vcsim=%s' % self._get_cloud_config('vcenter_host'))
env_vars = dict(
VCENTER_HOST=self._get_cloud_config('vcenter_host'),
)
ansible_vars = dict(
vcsim=self._get_cloud_config('vcenter_host'),
)
# Send the container IP down to the integration test(s)
env['VCENTER_HOST'] = self._get_cloud_config('vcenter_host')
return CloudEnvironmentConfig(
env_vars=env_vars,
ansible_vars=ansible_vars,
)

@ -5,7 +5,8 @@ import os
from lib.cloud import (
CloudProvider,
CloudEnvironment
CloudEnvironment,
CloudEnvironmentConfig,
)
from lib.util import ConfigParser
@ -18,7 +19,7 @@ class VultrCloudProvider(CloudProvider):
"""
:type args: TestConfig
"""
super(VultrCloudProvider, self).__init__(args, config_extension='.ini')
super(VultrCloudProvider, self).__init__(args)
def filter(self, targets, exclude):
"""Filter out the cloud tests when the necessary config and resources are not available.
@ -31,32 +32,34 @@ class VultrCloudProvider(CloudProvider):
super(VultrCloudProvider, self).filter(targets, exclude)
def setup(self):
"""Setup the cloud resource before delegation and register a cleanup callback."""
super(VultrCloudProvider, self).setup()
if os.path.isfile(self.config_static_path):
self.config_path = self.config_static_path
self.managed = False
return True
return False
class VultrCloudEnvironment(CloudEnvironment):
"""
Updates integration test environment after delegation. Will setup the config file as parameter.
"""
def configure_environment(self, env, cmd):
def get_environment_config(self):
"""
:type env: dict[str, str]
:type cmd: list[str]
:rtype: CloudEnvironmentConfig
"""
parser = ConfigParser()
parser.read(self.config_path)
changes = dict(
env_vars = dict(
VULTR_API_KEY=parser.get('default', 'key'),
)
env.update(changes)
cmd.append('-e')
cmd.append('vultr_resource_prefix=%s' % self.resource_prefix)
ansible_vars = dict(
vultr_resource_prefix=self.resource_prefix,
)
return CloudEnvironmentConfig(
env_vars=env_vars,
ansible_vars=ansible_vars,
)

@ -34,6 +34,7 @@ from lib.cloud import (
cloud_init,
get_cloud_environment,
get_cloud_platforms,
CloudEnvironmentConfig,
)
from lib.util import (
@ -110,6 +111,7 @@ from lib.metadata import (
from lib.integration import (
integration_test_environment,
integration_test_config_file,
)
SUPPORTED_PYTHON_VERSIONS = (
@ -1107,14 +1109,14 @@ def run_setup_targets(args, test_dir, target_names, targets_dict, targets_execut
targets_executed.add(target_name)
def integration_environment(args, target, cmd, test_dir, inventory_path, ansible_config):
def integration_environment(args, target, test_dir, inventory_path, ansible_config, env_config):
"""
:type args: IntegrationConfig
:type target: IntegrationTarget
:type cmd: list[str]
:type test_dir: str
:type inventory_path: str
:type ansible_config: str | None
:type env_config: CloudEnvironmentConfig | None
:rtype: dict[str, str]
"""
env = ansible_environment(args, ansible_config=ansible_config)
@ -1124,9 +1126,11 @@ def integration_environment(args, target, cmd, test_dir, inventory_path, ansible
HTTPTESTER='1',
))
callback_plugins = ['junit'] + (env_config.callback_plugins or [] if env_config else [])
integration = dict(
JUNIT_OUTPUT_DIR=os.path.abspath('test/results/junit'),
ANSIBLE_CALLBACK_WHITELIST='junit',
ANSIBLE_CALLBACK_WHITELIST=','.join(sorted(set(callback_plugins))),
ANSIBLE_TEST_CI=args.metadata.ci_provider,
OUTPUT_DIR=test_dir,
INVENTORY_PATH=os.path.abspath(inventory_path),
@ -1143,11 +1147,6 @@ def integration_environment(args, target, cmd, test_dir, inventory_path, ansible
env.update(integration)
cloud_environment = get_cloud_environment(args, target)
if cloud_environment:
cloud_environment.configure_environment(env, cmd)
return env
@ -1160,15 +1159,30 @@ def command_integration_script(args, target, test_dir, inventory_path):
"""
display.info('Running %s integration test script' % target.name)
env_config = None
if isinstance(args, PosixIntegrationConfig):
cloud_environment = get_cloud_environment(args, target)
if cloud_environment:
env_config = cloud_environment.get_environment_config()
with integration_test_environment(args, target, inventory_path) as test_env:
cmd = ['./%s' % os.path.basename(target.script_path)]
if args.verbosity:
cmd.append('-' + ('v' * args.verbosity))
env = integration_environment(args, target, cmd, test_dir, test_env.inventory_path, test_env.ansible_config)
env = integration_environment(args, target, test_dir, test_env.inventory_path, test_env.ansible_config, env_config)
cwd = os.path.join(test_env.integration_dir, 'targets', target.name)
if env_config and env_config.env_vars:
env.update(env_config.env_vars)
with integration_test_config_file(args, env_config, test_env.integration_dir) as config_path:
if config_path:
cmd += ['-e', '@%s' % config_path]
intercept_command(args, cmd, target_name=target.name, env=env, cwd=cwd)
@ -1182,6 +1196,8 @@ def command_integration_role(args, target, start_at_task, test_dir, inventory_pa
"""
display.info('Running %s integration test role' % target.name)
env_config = None
if isinstance(args, WindowsIntegrationConfig):
hosts = 'windows'
gather_facts = False
@ -1195,22 +1211,35 @@ def command_integration_role(args, target, start_at_task, test_dir, inventory_pa
cloud_environment = get_cloud_environment(args, target)
if cloud_environment:
hosts = cloud_environment.inventory_hosts or hosts
playbook = '''
- hosts: %s
gather_facts: %s
roles:
- { role: %s }
''' % (hosts, gather_facts, target.name)
env_config = cloud_environment.get_environment_config()
with integration_test_environment(args, target, inventory_path) as test_env:
play = dict(
hosts=hosts,
gather_facts=gather_facts,
vars_files=[
test_env.vars_file,
],
roles=[
target.name,
],
)
if env_config:
play.update(dict(
vars=env_config.ansible_vars,
environment=env_config.env_vars,
module_defaults=env_config.module_defaults,
))
playbook = json.dumps([play], indent=4, sort_keys=True)
with named_temporary_file(args=args, directory=test_env.integration_dir, prefix='%s-' % target.name, suffix='.yml', content=playbook) as playbook_path:
filename = os.path.basename(playbook_path)
display.info('>>> Playbook: %s\n%s' % (filename, playbook.strip()), verbosity=3)
cmd = ['ansible-playbook', filename, '-i', test_env.inventory_path, '-e', '@%s' % test_env.vars_file]
cmd = ['ansible-playbook', filename, '-i', test_env.inventory_path]
if start_at_task:
cmd += ['--start-at-task', start_at_task]
@ -1231,7 +1260,7 @@ def command_integration_role(args, target, start_at_task, test_dir, inventory_pa
if args.verbosity:
cmd.append('-' + ('v' * args.verbosity))
env = integration_environment(args, target, cmd, test_dir, test_env.inventory_path, test_env.ansible_config)
env = integration_environment(args, target, test_dir, test_env.inventory_path, test_env.ansible_config, env_config)
cwd = test_env.integration_dir
env['ANSIBLE_ROLES_PATH'] = os.path.abspath(os.path.join(test_env.integration_dir, 'targets'))

@ -3,6 +3,7 @@
from __future__ import absolute_import, print_function
import contextlib
import json
import os
import shutil
import tempfile
@ -22,12 +23,17 @@ from lib.util import (
ApplicationError,
display,
make_dirs,
named_temporary_file,
)
from lib.cache import (
CommonCache,
)
from lib.cloud import (
CloudEnvironmentConfig,
)
def generate_dependency_map(integration_targets):
"""
@ -188,6 +194,36 @@ def integration_test_environment(args, target, inventory_path):
shutil.rmtree(temp_dir)
@contextlib.contextmanager
def integration_test_config_file(args, env_config, integration_dir):
"""
:type args: IntegrationConfig
:type env_config: CloudEnvironmentConfig
:type integration_dir: str
"""
if not env_config:
yield None
return
config_vars = (env_config.ansible_vars or {}).copy()
config_vars.update(dict(
ansible_test=dict(
environment=env_config.env_vars,
module_defaults=env_config.module_defaults,
)
))
config_file = json.dumps(config_vars, indent=4, sort_keys=True)
with named_temporary_file(args, 'config-file-', '.json', integration_dir, config_file) as path:
filename = os.path.relpath(path, integration_dir)
display.info('>>> Config File: %s\n%s' % (filename, config_file), verbosity=3)
yield path
class IntegrationEnvironment(object):
"""Details about the integration environment."""
def __init__(self, integration_dir, inventory_path, ansible_config, vars_file):

Loading…
Cancel
Save