mirror of https://github.com/ansible/ansible.git
Replace old shippable.py with new check_matrix.py. (#60022)
This new script does not depend on ansible-test and provides much more robust job matrix testing. It is also run on every job in the matrix now, to detect issues with jobs being re-run after matrix changes are made.pull/60103/head
Matt Clay
5 years ago
committed by
GitHub
parent
41235ac05a
commit
d3da8e4a5b
@ -1,117 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# PYTHON_ARGCOMPLETE_OK
|
||||
"""Verify the current Shippable run has the required number of jobs."""
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
# noinspection PyCompatibility
|
||||
import argparse
|
||||
import errno
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
from lib.http import (
|
||||
HttpClient,
|
||||
)
|
||||
|
||||
from lib.util import (
|
||||
display,
|
||||
ApplicationError,
|
||||
ApplicationWarning,
|
||||
MissingEnvironmentVariable,
|
||||
)
|
||||
|
||||
from lib.util_common import (
|
||||
CommonConfig,
|
||||
)
|
||||
|
||||
|
||||
try:
|
||||
import argcomplete
|
||||
except ImportError:
|
||||
argcomplete = None
|
||||
|
||||
|
||||
def main():
|
||||
"""Main program function."""
|
||||
try:
|
||||
args = parse_args()
|
||||
args.debug = False
|
||||
args.truncate = False
|
||||
args.redact = False
|
||||
|
||||
config = CommonConfig(args, '')
|
||||
|
||||
display.verbosity = config.verbosity
|
||||
display.color = config.color
|
||||
|
||||
try:
|
||||
run_id = os.environ['SHIPPABLE_BUILD_ID']
|
||||
except KeyError as ex:
|
||||
raise MissingEnvironmentVariable(ex.args[0])
|
||||
|
||||
client = HttpClient(config)
|
||||
response = client.get('https://api.shippable.com/jobs?runIds=%s' % run_id)
|
||||
jobs = response.json()
|
||||
|
||||
if not isinstance(jobs, list):
|
||||
raise ApplicationError(json.dumps(jobs, indent=4, sort_keys=True))
|
||||
|
||||
if len(jobs) == 1:
|
||||
raise ApplicationError('Shippable run %s has only one job. Did you use the "Rebuild with SSH" option?' % run_id)
|
||||
except ApplicationWarning as ex:
|
||||
display.warning(str(ex))
|
||||
exit(0)
|
||||
except ApplicationError as ex:
|
||||
display.error(str(ex))
|
||||
exit(1)
|
||||
except KeyboardInterrupt:
|
||||
exit(2)
|
||||
except IOError as ex:
|
||||
if ex.errno == errno.EPIPE:
|
||||
exit(3)
|
||||
raise
|
||||
|
||||
|
||||
def parse_args():
|
||||
"""Parse command line arguments."""
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
parser.add_argument('-e', '--explain',
|
||||
action='store_true',
|
||||
help='explain commands that would be executed')
|
||||
|
||||
parser.add_argument('-v', '--verbose',
|
||||
dest='verbosity',
|
||||
action='count',
|
||||
default=0,
|
||||
help='display more output')
|
||||
|
||||
parser.add_argument('--color',
|
||||
metavar='COLOR',
|
||||
nargs='?',
|
||||
help='generate color output: %(choices)s',
|
||||
choices=('yes', 'no', 'auto'),
|
||||
const='yes',
|
||||
default='auto')
|
||||
|
||||
if argcomplete:
|
||||
argcomplete.autocomplete(parser)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.color == 'yes':
|
||||
args.color = True
|
||||
elif args.color == 'no':
|
||||
args.color = False
|
||||
elif 'SHIPPABLE' in os.environ:
|
||||
args.color = True
|
||||
else:
|
||||
args.color = sys.stdout.isatty()
|
||||
|
||||
return args
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python
|
||||
"""Verify the currently executing Shippable test matrix matches the one defined in the "shippable.yml" file."""
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
|
||||
try:
|
||||
from typing import NoReturn
|
||||
except ImportError:
|
||||
NoReturn = None
|
||||
|
||||
try:
|
||||
# noinspection PyCompatibility
|
||||
from urllib2 import urlopen # pylint: disable=ansible-bad-import-from
|
||||
except ImportError:
|
||||
# noinspection PyCompatibility
|
||||
from urllib.request import urlopen
|
||||
|
||||
|
||||
def main(): # type: () -> None
|
||||
"""Main entry point."""
|
||||
with open('shippable.yml', 'rb') as yaml_file:
|
||||
yaml = yaml_file.read().decode('utf-8').splitlines()
|
||||
|
||||
defined_matrix = [match.group(1) for match in [re.search(r'^ *- env: T=(.*)$', line) for line in yaml] if match and match.group(1) != 'none']
|
||||
|
||||
if not defined_matrix:
|
||||
fail('No matrix entries found in the "shippable.yml" file.',
|
||||
'Did you modify the "shippable.yml" file?')
|
||||
|
||||
run_id = os.environ['SHIPPABLE_BUILD_ID']
|
||||
sleep = 1
|
||||
jobs = []
|
||||
|
||||
for attempts_remaining in range(4, -1, -1):
|
||||
try:
|
||||
jobs = json.loads(urlopen('https://api.shippable.com/jobs?runIds=%s' % run_id).read())
|
||||
|
||||
if not isinstance(jobs, list):
|
||||
raise Exception('Shippable run %s data is not a list.' % run_id)
|
||||
|
||||
break
|
||||
except Exception as ex:
|
||||
if not attempts_remaining:
|
||||
fail('Unable to retrieve Shippable run %s matrix.' % run_id,
|
||||
str(ex))
|
||||
|
||||
sys.stderr.write('Unable to retrieve Shippable run %s matrix: %s\n' % (run_id, ex))
|
||||
sys.stderr.write('Trying again in %d seconds...\n' % sleep)
|
||||
time.sleep(sleep)
|
||||
sleep *= 2
|
||||
|
||||
if len(jobs) != len(defined_matrix):
|
||||
if len(jobs) == 1:
|
||||
hint = '\n\nMake sure you do not use the "Rebuild with SSH" option.'
|
||||
else:
|
||||
hint = ''
|
||||
|
||||
fail('Shippable run %s has %d jobs instead of the expected %d jobs.' % (run_id, len(jobs), len(defined_matrix)),
|
||||
'Try re-running the entire matrix.%s' % hint)
|
||||
|
||||
actual_matrix = dict((job.get('jobNumber'), dict(tuple(line.split('=', 1)) for line in job.get('env', [])).get('T', '')) for job in jobs)
|
||||
errors = [(job_number, test, actual_matrix.get(job_number)) for job_number, test in enumerate(defined_matrix, 1) if actual_matrix.get(job_number) != test]
|
||||
|
||||
if len(errors):
|
||||
error_summary = '\n'.join('Job %s expected "%s" but found "%s" instead.' % (job_number, expected, actual) for job_number, expected, actual in errors)
|
||||
|
||||
fail('Shippable run %s has a job matrix mismatch.' % run_id,
|
||||
'Try re-running the entire matrix.\n\n%s' % error_summary)
|
||||
|
||||
|
||||
def fail(message, output): # type: (str, str) -> NoReturn
|
||||
# Include a leading newline to improve readability on Shippable "Tests" tab.
|
||||
# Without this, the first line becomes indented.
|
||||
output = '\n' + output.strip()
|
||||
|
||||
timestamp = datetime.datetime.utcnow().replace(microsecond=0).isoformat()
|
||||
|
||||
# hack to avoid requiring junit-xml, which isn't pre-installed on Shippable outside our test containers
|
||||
xml = '''
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<testsuites disabled="0" errors="1" failures="0" tests="1" time="0.0">
|
||||
\t<testsuite disabled="0" errors="1" failures="0" file="None" log="None" name="ansible-test" skipped="0" tests="1" time="0" timestamp="%s" url="None">
|
||||
\t\t<testcase classname="timeout" name="timeout">
|
||||
\t\t\t<error message="%s" type="error">%s</error>
|
||||
\t\t</testcase>
|
||||
\t</testsuite>
|
||||
</testsuites>
|
||||
''' % (timestamp, message, output)
|
||||
|
||||
with open('test/results/junit/check-matrix.xml', 'w') as junit_fd:
|
||||
junit_fd.write(xml.lstrip())
|
||||
|
||||
sys.stderr.write(message + '\n')
|
||||
sys.stderr.write(output + '\n')
|
||||
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Reference in New Issue