Testing: Add support for CentOS Linux On Power platform (#68130)

* Testing: Add CentOS Linux On Power platform

* Add arch designation to remotes.

This avoids overloading the provider with the arch.

Also add a changelog entry.

Co-authored-by: Matt Clay <matt@mystile.com>
pull/68969/head
Yanis Guenane 4 years ago committed by GitHub
parent abb807e5dd
commit 46d82179d8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -0,0 +1,7 @@
minor_changes:
- ansible-test now supports the ``--remote power/centos/7`` platform option.
- ansible-test now supports skip aliases in the format ``skip/{platform}/{version}`` for the ``--remote`` option.
This is preferred over the older ``skip/{platform}{version}`` format which included no ``/`` between the platform and version.
- ansible-test now supports skip aliases in the format ``skip/{arch}/{platform}`` and ``skip/{arch}/{platform}/{version}`` where ``arch`` can be ``power``.
These aliases are only effective for the ``--remote`` option.
- ansible-test provides clearer error messages when failing to detect the provider to use with the ``--remote`` option.

@ -42,6 +42,7 @@ matrix:
- env: T=linux/opensuse15/1
- env: T=linux/ubuntu1604/1
- env: T=linux/ubuntu1804/1
- env: T=power/centos/7/1
- env: T=aix/7.2/2
- env: T=osx/10.11/2
@ -58,6 +59,7 @@ matrix:
- env: T=linux/opensuse15/2
- env: T=linux/ubuntu1604/2
- env: T=linux/ubuntu1804/2
- env: T=power/centos/7/2
- env: T=aix/7.2/3
- env: T=osx/10.11/3
@ -74,6 +76,7 @@ matrix:
- env: T=linux/opensuse15/3
- env: T=linux/ubuntu1604/3
- env: T=linux/ubuntu1804/3
- env: T=power/centos/7/3
- env: T=aix/7.2/4
- env: T=osx/10.11/4
@ -90,6 +93,7 @@ matrix:
- env: T=linux/opensuse15/4
- env: T=linux/ubuntu1604/4
- env: T=linux/ubuntu1804/4
- env: T=power/centos/7/4
- env: T=aix/7.2/5
- env: T=osx/10.11/5
@ -106,6 +110,7 @@ matrix:
- env: T=linux/opensuse15/5
- env: T=linux/ubuntu1604/5
- env: T=linux/ubuntu1804/5
- env: T=power/centos/7/5
- env: T=fallaxy/2.7/1
- env: T=fallaxy/3.6/1
@ -125,6 +130,7 @@ matrix:
- env: T=i/linux/opensuse15
- env: T=i/linux/ubuntu1604
- env: T=i/linux/ubuntu1804
- env: T=i/power/centos/7
- env: T=i/windows/2012
- env: T=i/windows/2012-R2

@ -1,3 +1,4 @@
shippable/posix/group3
needs/target/binary_modules
skip/aix
skip/power/centos

@ -1,5 +1,6 @@
destructive
shippable/posix/group4
skip/aix
skip/power/centos
skip/freebsd
skip/osx

@ -1,3 +1,4 @@
shippable/posix/incidental
destructive
skip/aix
skip/power/centos

@ -1,5 +1,6 @@
shippable/posix/incidental
skip/aix
skip/power/centos
skip/osx
skip/freebsd
destructive

@ -3,4 +3,5 @@ destructive
needs/target/incidental_setup_openssl
needs/file/test/lib/ansible_test/_data/requirements/constraints.txt
skip/aix
skip/power/centos
skip/python2.6

@ -1,5 +1,6 @@
shippable/posix/incidental
skip/aix
skip/power/centos
skip/osx
skip/freebsd
skip/rhel8.0

@ -1,3 +1,4 @@
destructive
shippable/posix/incidental
skip/aix
skip/power/centos

@ -1,3 +1,4 @@
shippable/posix/group1
destructive
skip/aix
skip/power/centos

@ -1,3 +1,4 @@
destructive
shippable/posix/group5
skip/aix
skip/power/centos

@ -1,6 +1,7 @@
setup/always/setup_passlib
shippable/posix/group2
skip/aix
skip/power/centos
skip/osx
destructive
needs/root

@ -2,3 +2,4 @@ needs/root
shippable/posix/group2
destructive
skip/aix
skip/power/centos

@ -1,5 +1,6 @@
destructive
shippable/posix/group4
skip/aix
skip/power/centos
skip/freebsd
skip/osx

@ -1,3 +1,4 @@
shippable/posix/group1
destructive
skip/aix
skip/power/centos

@ -5,3 +5,4 @@ rhel/7.6 python=2.7
rhel/7.8 python=2.7
rhel/8.1 python=3.6
aix/7.2 python=2.7 httptester=disabled temp-unicode=disabled pip-check=disabled
power/centos/7 python=2.7

@ -70,6 +70,21 @@ elif [ "${platform}" = "rhel" ]; then
install_pip
fi
elif [ "${platform}" = "centos" ]; then
while true; do
yum install -q -y \
gcc \
python-devel \
python-virtualenv \
python2-cryptography \
libffi-devel \
openssl-devel \
&& break
echo "Failed to install packages. Sleeping before trying again..."
sleep 10
done
install_pip
elif [ "${platform}" = "osx" ]; then
while true; do
pip install --disable-pip-version-check --quiet \

@ -950,7 +950,7 @@ def add_environments(parser, isolated_delegation=True):
remote.add_argument('--remote-provider',
metavar='PROVIDER',
help='remote provider to use: %(choices)s',
choices=['default', 'aws', 'azure', 'parallels'],
choices=['default', 'aws', 'azure', 'parallels', 'ibmvpc', 'ibmps'],
default='default')
remote.add_argument('--remote-aws-region',

@ -35,6 +35,29 @@ except AttributeError:
TIntegrationConfig = None # pylint: disable=invalid-name
class ParsedRemote:
"""A parsed version of a "remote" string."""
def __init__(self, arch, platform, version): # type: (t.Optional[str], str, str) -> None
self.arch = arch
self.platform = platform
self.version = version
@staticmethod
def parse(value): # type: (str) -> t.Optional['ParsedRemote']
"""Return a ParsedRemote from the given value or None if the syntax is invalid."""
parts = value.split('/')
if len(parts) == 2:
arch = None
platform, version = parts
elif len(parts) == 3:
arch, platform, version = parts
else:
return None
return ParsedRemote(arch, platform, version)
class EnvironmentConfig(CommonConfig):
"""Configuration common to all commands which execute in an environment."""
def __init__(self, args, command):
@ -54,6 +77,14 @@ class EnvironmentConfig(CommonConfig):
self.docker_raw = args.docker # type: str
self.remote = args.remote # type: str
if self.remote:
self.parsed_remote = ParsedRemote.parse(self.remote)
if not self.parsed_remote or not self.parsed_remote.platform or not self.parsed_remote.version:
raise ApplicationError('Unrecognized remote "%s" syntax. Use "platform/version" or "arch/platform/version".' % self.remote)
else:
self.parsed_remote = None
self.docker_privileged = args.docker_privileged if 'docker_privileged' in args else False # type: bool
self.docker_pull = args.docker_pull if 'docker_pull' in args else False # type: bool
self.docker_keep_git = args.docker_keep_git if 'docker_keep_git' in args else False # type: bool

@ -53,7 +53,7 @@ AWS_ENDPOINTS = {
class AnsibleCoreCI:
"""Client for Ansible Core CI services."""
def __init__(self, args, platform, version, stage='prod', persist=True, load=True, name=None, provider=None):
def __init__(self, args, platform, version, stage='prod', persist=True, load=True, provider=None, arch=None):
"""
:type args: EnvironmentConfig
:type platform: str
@ -61,9 +61,11 @@ class AnsibleCoreCI:
:type stage: str
:type persist: bool
:type load: bool
:type name: str
:type provider: str | None
:type arch: str | None
"""
self.args = args
self.arch = arch
self.platform = platform
self.version = version
self.stage = stage
@ -73,7 +75,12 @@ class AnsibleCoreCI:
self.endpoint = None
self.max_threshold = 1
self.retries = 3
self.name = name if name else '%s-%s' % (self.platform, self.version)
if self.arch:
self.name = '%s-%s-%s' % (self.arch, self.platform, self.version)
else:
self.name = '%s-%s' % (self.platform, self.version)
self.ci_key = os.path.expanduser('~/.ansible-core-ci.key')
self.resource = 'jobs'
@ -98,32 +105,56 @@ class AnsibleCoreCI:
'aix',
'ibmi',
),
ibmvpc=(
'centos arch=power', # avoid ibmvpc as default for no-arch centos to avoid making centos default to power
),
parallels=(
'osx',
),
vmware=(
'vmware'
'vmware',
),
)
# Currently ansible-core-ci has no concept of arch selection. This effectively means each provider only supports one arch.
# The list below identifies which platforms accept an arch, and which one. These platforms can only be used with the specified arch.
provider_arches = dict(
ibmvpc='power',
)
if provider:
# override default provider selection (not all combinations are valid)
self.provider = provider
else:
self.provider = None
for candidate in providers:
if platform in providers[candidate]:
choices = [
platform,
'%s arch=%s' % (platform, arch),
]
if any(choice in providers[candidate] for choice in choices):
# assign default provider based on platform
self.provider = candidate
break
for candidate in providers:
if '%s/%s' % (platform, version) in providers[candidate]:
# assign default provider based on platform and version
self.provider = candidate
break
# If a provider has been selected, make sure the correct arch (or none) has been selected.
if self.provider:
required_arch = provider_arches.get(self.provider)
if self.arch != required_arch:
if required_arch:
if self.arch:
raise ApplicationError('Provider "%s" requires the "%s" arch instead of "%s".' % (self.provider, required_arch, self.arch))
raise ApplicationError('Provider "%s" requires the "%s" arch.' % (self.provider, required_arch))
raise ApplicationError('Provider "%s" does not support specification of an arch.' % self.provider)
self.path = os.path.expanduser('~/.ansible/test/instances/%s-%s-%s' % (self.name, self.provider, self.stage))
if self.provider in ('aws', 'azure', 'ibmps'):
if self.provider in ('aws', 'azure', 'ibmps', 'ibmvpc'):
if self.provider != 'aws':
self.resource = self.provider
@ -167,7 +198,10 @@ class AnsibleCoreCI:
self.endpoints = ['https://access.ws.testing.ansible.com']
self.max_threshold = 1
else:
raise ApplicationError('Unsupported platform: %s' % platform)
if self.arch:
raise ApplicationError('Provider not detected for platform "%s" on arch "%s".' % (self.platform, self.arch))
raise ApplicationError('Provider not detected for platform "%s" with no arch specified.' % self.platform)
if persist and load and self._load():
try:

@ -389,12 +389,9 @@ def delegate_remote(args, exclude, require, integration_targets):
:type require: list[str]
:type integration_targets: tuple[IntegrationTarget]
"""
parts = args.remote.split('/', 1)
remote = args.parsed_remote
platform = parts[0]
version = parts[1]
core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage, provider=args.remote_provider)
core_ci = AnsibleCoreCI(args, remote.platform, remote.version, stage=args.remote_stage, provider=args.remote_provider, arch=remote.arch)
success = False
raw = False
@ -422,7 +419,7 @@ def delegate_remote(args, exclude, require, integration_targets):
python_version = get_python_version(args, get_remote_completion(), args.remote)
if platform == 'windows':
if remote.platform == 'windows':
# Windows doesn't need the ansible-test fluff, just run the SSH command
manage = ManageWindowsCI(core_ci)
manage.setup(python_version)
@ -457,7 +454,7 @@ def delegate_remote(args, exclude, require, integration_targets):
if isinstance(args, TestConfig):
if args.coverage and not args.coverage_label:
cmd += ['--coverage-label', 'remote-%s-%s' % (platform, version)]
cmd += ['--coverage-label', 'remote-%s-%s' % (remote.platform, remote.version)]
if isinstance(args, IntegrationConfig):
if not args.allow_destructive:
@ -479,7 +476,7 @@ def delegate_remote(args, exclude, require, integration_targets):
finally:
download = False
if platform != 'windows':
if remote.platform != 'windows':
download = True
if isinstance(args, ShellConfig):
@ -495,7 +492,7 @@ def delegate_remote(args, exclude, require, integration_targets):
# AIX cp and GNU cp provide different options, no way could be found to have a common
# pattern and achieve the same goal
cp_opts = '-hr' if platform in ['aix', 'ibmi'] else '-a'
cp_opts = '-hr' if remote.platform in ['aix', 'ibmi'] else '-a'
manage.ssh('rm -rf {0} && mkdir {0} && cp {1} {2}/* {0}/ && chmod -R a+r {0}'.format(remote_temp_path, cp_opts, remote_results_root))
manage.download(remote_temp_path, local_test_root)

@ -1834,27 +1834,29 @@ def get_integration_remote_filter(args, targets):
:type targets: tuple[IntegrationTarget]
:rtype: list[str]
"""
parts = args.remote.split('/', 1)
platform = parts[0]
remote = args.parsed_remote
exclude = []
common_integration_filter(args, targets, exclude)
skip = 'skip/%s/' % platform
skipped = [target.name for target in targets if skip in target.aliases]
if skipped:
exclude.append(skip)
display.warning('Excluding tests marked "%s" which are not supported on %s: %s'
% (skip.rstrip('/'), platform, ', '.join(skipped)))
skips = {
'skip/%s' % remote.platform: remote.platform,
'skip/%s/%s' % (remote.platform, remote.version): '%s %s' % (remote.platform, remote.version),
'skip/%s%s' % (remote.platform, remote.version): '%s %s' % (remote.platform, remote.version), # legacy syntax, use above format
}
skip = 'skip/%s/' % args.remote.replace('/', '')
skipped = [target.name for target in targets if skip in target.aliases]
if skipped:
exclude.append(skip)
display.warning('Excluding tests marked "%s" which are not supported on %s: %s'
% (skip.rstrip('/'), args.remote.replace('/', ' '), ', '.join(skipped)))
if remote.arch:
skips.update({
'skip/%s/%s' % (remote.arch, remote.platform): '%s on %s' % (remote.platform, remote.arch),
'skip/%s/%s/%s' % (remote.arch, remote.platform, remote.version): '%s %s on %s' % (remote.platform, remote.version, remote.arch),
})
for skip, description in skips.items():
skipped = [target.name for target in targets if skip in target.skips]
if skipped:
exclude.append(skip + '/')
display.warning('Excluding tests marked "%s" which are not supported on %s: %s' % (skip, description, ', '.join(skipped)))
python_version = get_python_version(args, get_remote_completion(), args.remote)

@ -213,7 +213,7 @@ class ManagePosixCI:
raise NotImplementedError('provider %s has not been implemented' % self.core_ci.provider)
elif self.core_ci.platform == 'osx':
self.become = ['sudo', '-in', 'PATH=/usr/local/bin:$PATH']
elif self.core_ci.platform == 'rhel':
elif self.core_ci.platform == 'rhel' or self.core_ci.platform == 'centos':
self.become = ['sudo', '-in', 'bash', '-c']
elif self.core_ci.platform in ['aix', 'ibmi']:
self.become = []

@ -638,6 +638,9 @@ class IntegrationTarget(CompletionTarget):
targets_relative_path = data_context().content.integration_targets_path
# Collect skip entries before group expansion to avoid registering more specific skip entries as less specific versions.
self.skips = tuple(g for g in groups if g.startswith('skip/'))
# Collect file paths before group expansion to avoid including the directories.
# Ignore references to test targets, as those must be defined using `needs/target/*` or other target references.
self.needs_file = tuple(sorted(set('/'.join(g.split('/')[2:]) for g in groups if

@ -0,0 +1,18 @@
#!/usr/bin/env bash
set -o pipefail -eux
declare -a args
IFS='/:' read -ra args <<< "$1"
platform="${args[1]}"
version="${args[2]}"
target="shippable/posix/incidental/"
stage="${S:-prod}"
provider="${P:-default}"
# shellcheck disable=SC2086
ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
--remote "power/${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}"

@ -0,0 +1,22 @@
#!/usr/bin/env bash
set -o pipefail -eux
declare -a args
IFS='/:' read -ra args <<< "$1"
platform="${args[1]}"
version="${args[2]}"
if [ "${#args[@]}" -gt 3 ]; then
target="shippable/posix/group${args[3]}/"
else
target="shippable/posix/"
fi
stage="${S:-prod}"
provider="${P:-default}"
# shellcheck disable=SC2086
ansible-test integration --color -v --retry-on-error "${target}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} \
--remote "power/${platform}/${version}" --remote-terminate always --remote-stage "${stage}" --remote-provider "${provider}"
Loading…
Cancel
Save