From 4ba9f63afe578628f9d0be2b105bee3af83a86bf Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Wed, 18 Jan 2017 16:31:34 -0800 Subject: [PATCH] Use multiple AWS regions in ansible-test. (#20429) * Move ansible-test EnvironmentConfig to util. * Use multiple AWS regions in ansible-test. --- test/runner/lib/core_ci.py | 31 ++++++++++++++++---- test/runner/lib/cover.py | 18 ++++++++++-- test/runner/lib/delegation.py | 2 +- test/runner/lib/executor.py | 55 +---------------------------------- test/runner/lib/util.py | 54 ++++++++++++++++++++++++++++++++++ test/runner/test.py | 11 +++++++ 6 files changed, 108 insertions(+), 63 deletions(-) diff --git a/test/runner/lib/core_ci.py b/test/runner/lib/core_ci.py index 98756e22129..4812f55011d 100644 --- a/test/runner/lib/core_ci.py +++ b/test/runner/lib/core_ci.py @@ -19,17 +19,22 @@ from lib.util import ( ApplicationError, run_command, make_dirs, - CommonConfig, + EnvironmentConfig, display, is_shippable, ) +AWS_ENDPOINTS = { + 'us-east-1': 'https://14blg63h2i.execute-api.us-east-1.amazonaws.com', + 'us-east-2': 'https://g5xynwbk96.execute-api.us-east-2.amazonaws.com', +} + class AnsibleCoreCI(object): """Client for Ansible Core CI services.""" def __init__(self, args, platform, version, stage='prod', persist=True, name=None): """ - :type args: CommonConfig + :type args: EnvironmentConfig :type platform: str :type version: str :type stage: str @@ -44,6 +49,7 @@ class AnsibleCoreCI(object): self.connection = None self.instance_id = None self.name = name if name else '%s-%s' % (self.platform, self.version) + self.ci_key = os.path.expanduser('~/.ansible-core-ci.key') aws_platforms = ( 'windows', @@ -58,7 +64,22 @@ class AnsibleCoreCI(object): ) if self.platform in aws_platforms: - self.endpoint = 'https://14blg63h2i.execute-api.us-east-1.amazonaws.com' + if args.remote_aws_region: + # permit command-line override of region selection + region = args.remote_aws_region + # use a dedicated CI key when overriding the region selection + self.ci_key += '.%s' % args.remote_aws_region + elif is_shippable(): + # split Shippable jobs across multiple regions to maximize use of launch credits + if self.platform == 'freebsd': + region = 'us-east-2' + else: + region = 'us-east-1' + else: + # send all non-Shippable jobs to us-east-1 to reduce api key maintenance + region = 'us-east-1' + + self.endpoint = AWS_ENDPOINTS[region] if self.platform == 'windows': self.ssh_key = None @@ -117,7 +138,7 @@ class AnsibleCoreCI(object): def start_remote(self): """Start instance for remote development/testing.""" - with open(os.path.expanduser('~/.ansible-core-ci.key'), 'r') as key_fd: + with open(self.ci_key, 'r') as key_fd: auth_key = key_fd.read().strip() self._start(dict( @@ -347,7 +368,7 @@ class SshKey(object): """Container for SSH key used to connect to remote instances.""" def __init__(self, args): """ - :type args: CommonConfig + :type args: EnvironmentConfig """ tmp = os.path.expanduser('~/.ansible/test/') diff --git a/test/runner/lib/cover.py b/test/runner/lib/cover.py index 493ff72b933..f5f5bf321be 100644 --- a/test/runner/lib/cover.py +++ b/test/runner/lib/cover.py @@ -5,9 +5,21 @@ from __future__ import absolute_import, print_function import os import re -from lib.target import walk_module_targets -from lib.util import display, ApplicationError, run_command -from lib.executor import EnvironmentConfig, Delegate, install_command_requirements +from lib.target import ( + walk_module_targets, +) + +from lib.util import ( + display, + ApplicationError, + EnvironmentConfig, + run_command, +) + +from lib.executor import ( + Delegate, + install_command_requirements, +) COVERAGE_DIR = 'test/results/coverage' COVERAGE_FILE = os.path.join(COVERAGE_DIR, 'coverage') diff --git a/test/runner/lib/delegation.py b/test/runner/lib/delegation.py index 20b613978f9..c351e181f72 100644 --- a/test/runner/lib/delegation.py +++ b/test/runner/lib/delegation.py @@ -11,7 +11,6 @@ import lib.thread from lib.executor import ( SUPPORTED_PYTHON_VERSIONS, - EnvironmentConfig, IntegrationConfig, SubprocessError, ShellConfig, @@ -29,6 +28,7 @@ from lib.manage_ci import ( from lib.util import ( ApplicationError, + EnvironmentConfig, run_command, common_environment, display, diff --git a/test/runner/lib/executor.py b/test/runner/lib/executor.py index a6f14b8f7ea..fcbb428a399 100644 --- a/test/runner/lib/executor.py +++ b/test/runner/lib/executor.py @@ -5,7 +5,6 @@ from __future__ import absolute_import, print_function import glob import os import tempfile -import sys import time import textwrap import functools @@ -31,6 +30,7 @@ from lib.manage_ci import ( from lib.util import ( CommonConfig, + EnvironmentConfig, ApplicationWarning, ApplicationError, SubprocessError, @@ -1037,17 +1037,6 @@ def detect_changes_local(args): return sorted(names) -def docker_qualify_image(name): - """ - :type name: str - :rtype: str - """ - if not name or any((c in name) for c in ('/', ':')): - return name - - return 'ansible/ansible:%s' % name - - def get_integration_filter(args, targets): """ :type args: IntegrationConfig @@ -1192,48 +1181,6 @@ class SanityFunc(SanityTest): self.intercept = intercept -class EnvironmentConfig(CommonConfig): - """Configuration common to all commands which execute in an environment.""" - def __init__(self, args, command): - """ - :type args: any - """ - super(EnvironmentConfig, self).__init__(args) - - self.command = command - - self.local = args.local is True - - if args.tox is True or args.tox is False or args.tox is None: - self.tox = args.tox is True - self.tox_args = 0 - self.python = args.python if 'python' in args else None # type: str - else: - self.tox = True - self.tox_args = 1 - self.python = args.tox # type: str - - self.docker = docker_qualify_image(args.docker) # type: str - self.remote = args.remote # type: str - - self.docker_privileged = args.docker_privileged if 'docker_privileged' in args else False # type: bool - self.docker_util = docker_qualify_image(args.docker_util if 'docker_util' in args else '') # type: str - self.docker_pull = args.docker_pull if 'docker_pull' in args else False # type: bool - - self.tox_sitepackages = args.tox_sitepackages # type: bool - - self.remote_stage = args.remote_stage # type: str - - self.requirements = args.requirements # type: bool - - self.python_version = self.python or '.'.join(str(i) for i in sys.version_info[:2]) - - self.delegate = self.tox or self.docker or self.remote - - if self.delegate: - self.requirements = True - - class TestConfig(EnvironmentConfig): """Configuration common to all test commands.""" def __init__(self, args, command): diff --git a/test/runner/lib/util.py b/test/runner/lib/util.py index 4868a87634a..3a43148256c 100644 --- a/test/runner/lib/util.py +++ b/test/runner/lib/util.py @@ -413,4 +413,58 @@ class CommonConfig(object): self.verbosity = args.verbosity # type: int +class EnvironmentConfig(CommonConfig): + """Configuration common to all commands which execute in an environment.""" + def __init__(self, args, command): + """ + :type args: any + """ + super(EnvironmentConfig, self).__init__(args) + + self.command = command + + self.local = args.local is True + + if args.tox is True or args.tox is False or args.tox is None: + self.tox = args.tox is True + self.tox_args = 0 + self.python = args.python if 'python' in args else None # type: str + else: + self.tox = True + self.tox_args = 1 + self.python = args.tox # type: str + + self.docker = docker_qualify_image(args.docker) # type: str + self.remote = args.remote # type: str + + self.docker_privileged = args.docker_privileged if 'docker_privileged' in args else False # type: bool + self.docker_util = docker_qualify_image(args.docker_util if 'docker_util' in args else '') # type: str + self.docker_pull = args.docker_pull if 'docker_pull' in args else False # type: bool + + self.tox_sitepackages = args.tox_sitepackages # type: bool + + self.remote_stage = args.remote_stage # type: str + self.remote_aws_region = args.remote_aws_region # type: str + + self.requirements = args.requirements # type: bool + + self.python_version = self.python or '.'.join(str(i) for i in sys.version_info[:2]) + + self.delegate = self.tox or self.docker or self.remote + + if self.delegate: + self.requirements = True + + +def docker_qualify_image(name): + """ + :type name: str + :rtype: str + """ + if not name or any((c in name) for c in ('/', ':')): + return name + + return 'ansible/ansible:%s' % name + + display = Display() # pylint: disable=locally-disabled, invalid-name diff --git a/test/runner/test.py b/test/runner/test.py index e89b31ddfbc..ecb165875ef 100755 --- a/test/runner/test.py +++ b/test/runner/test.py @@ -51,6 +51,10 @@ from lib.target import ( walk_sanity_targets, ) +from lib.core_ci import ( + AWS_ENDPOINTS, +) + import lib.cover @@ -409,6 +413,7 @@ def add_environments(parser, tox_version=False, tox_only=False): docker=None, remote=None, remote_stage=None, + remote_aws_region=None, ) return @@ -433,6 +438,12 @@ def add_environments(parser, tox_version=False, tox_only=False): choices=['prod', 'dev'], default='prod') + remote.add_argument('--remote-aws-region', + metavar='REGION', + help='remote aws region to use: %(choices)s (default: auto)', + choices=sorted(AWS_ENDPOINTS), + default=None) + def add_extra_docker_options(parser, integration=True): """