diff --git a/test/runner/shippable.py b/test/runner/shippable.py deleted file mode 100755 index 3882b62220a..00000000000 --- a/test/runner/shippable.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python -# PYTHON_ARGCOMPLETE_OK -"""Verify the current Shippable run has the required number of jobs.""" -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - -# noinspection PyCompatibility -import argparse -import errno -import json -import os -import sys - -from lib.http import ( - HttpClient, -) - -from lib.util import ( - display, - ApplicationError, - ApplicationWarning, - MissingEnvironmentVariable, -) - -from lib.util_common import ( - CommonConfig, -) - - -try: - import argcomplete -except ImportError: - argcomplete = None - - -def main(): - """Main program function.""" - try: - args = parse_args() - args.debug = False - args.truncate = False - args.redact = False - - config = CommonConfig(args, '') - - display.verbosity = config.verbosity - display.color = config.color - - try: - run_id = os.environ['SHIPPABLE_BUILD_ID'] - except KeyError as ex: - raise MissingEnvironmentVariable(ex.args[0]) - - client = HttpClient(config) - response = client.get('https://api.shippable.com/jobs?runIds=%s' % run_id) - jobs = response.json() - - if not isinstance(jobs, list): - raise ApplicationError(json.dumps(jobs, indent=4, sort_keys=True)) - - if len(jobs) == 1: - raise ApplicationError('Shippable run %s has only one job. Did you use the "Rebuild with SSH" option?' % run_id) - except ApplicationWarning as ex: - display.warning(str(ex)) - exit(0) - except ApplicationError as ex: - display.error(str(ex)) - exit(1) - except KeyboardInterrupt: - exit(2) - except IOError as ex: - if ex.errno == errno.EPIPE: - exit(3) - raise - - -def parse_args(): - """Parse command line arguments.""" - parser = argparse.ArgumentParser() - - parser.add_argument('-e', '--explain', - action='store_true', - help='explain commands that would be executed') - - parser.add_argument('-v', '--verbose', - dest='verbosity', - action='count', - default=0, - help='display more output') - - parser.add_argument('--color', - metavar='COLOR', - nargs='?', - help='generate color output: %(choices)s', - choices=('yes', 'no', 'auto'), - const='yes', - default='auto') - - if argcomplete: - argcomplete.autocomplete(parser) - - args = parser.parse_args() - - if args.color == 'yes': - args.color = True - elif args.color == 'no': - args.color = False - elif 'SHIPPABLE' in os.environ: - args.color = True - else: - args.color = sys.stdout.isatty() - - return args - - -if __name__ == '__main__': - main() diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index ecf66e8bfce..47b90930057 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -7970,4 +7970,5 @@ test/units/utils/test_helpers.py future-import-boilerplate test/units/utils/test_helpers.py metaclass-boilerplate test/units/utils/test_shlex.py future-import-boilerplate test/units/utils/test_shlex.py metaclass-boilerplate +test/utils/shippable/check_matrix.py replace-urlopen test/utils/shippable/timing.py shebang diff --git a/test/utils/shippable/check_matrix.py b/test/utils/shippable/check_matrix.py new file mode 100755 index 00000000000..99e4ea88ffb --- /dev/null +++ b/test/utils/shippable/check_matrix.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +"""Verify the currently executing Shippable test matrix matches the one defined in the "shippable.yml" file.""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import datetime +import json +import os +import re +import sys +import time + +try: + from typing import NoReturn +except ImportError: + NoReturn = None + +try: + # noinspection PyCompatibility + from urllib2 import urlopen # pylint: disable=ansible-bad-import-from +except ImportError: + # noinspection PyCompatibility + from urllib.request import urlopen + + +def main(): # type: () -> None + """Main entry point.""" + with open('shippable.yml', 'rb') as yaml_file: + yaml = yaml_file.read().decode('utf-8').splitlines() + + defined_matrix = [match.group(1) for match in [re.search(r'^ *- env: T=(.*)$', line) for line in yaml] if match and match.group(1) != 'none'] + + if not defined_matrix: + fail('No matrix entries found in the "shippable.yml" file.', + 'Did you modify the "shippable.yml" file?') + + run_id = os.environ['SHIPPABLE_BUILD_ID'] + sleep = 1 + jobs = [] + + for attempts_remaining in range(4, -1, -1): + try: + jobs = json.loads(urlopen('https://api.shippable.com/jobs?runIds=%s' % run_id).read()) + + if not isinstance(jobs, list): + raise Exception('Shippable run %s data is not a list.' % run_id) + + break + except Exception as ex: + if not attempts_remaining: + fail('Unable to retrieve Shippable run %s matrix.' % run_id, + str(ex)) + + sys.stderr.write('Unable to retrieve Shippable run %s matrix: %s\n' % (run_id, ex)) + sys.stderr.write('Trying again in %d seconds...\n' % sleep) + time.sleep(sleep) + sleep *= 2 + + if len(jobs) != len(defined_matrix): + if len(jobs) == 1: + hint = '\n\nMake sure you do not use the "Rebuild with SSH" option.' + else: + hint = '' + + fail('Shippable run %s has %d jobs instead of the expected %d jobs.' % (run_id, len(jobs), len(defined_matrix)), + 'Try re-running the entire matrix.%s' % hint) + + actual_matrix = dict((job.get('jobNumber'), dict(tuple(line.split('=', 1)) for line in job.get('env', [])).get('T', '')) for job in jobs) + errors = [(job_number, test, actual_matrix.get(job_number)) for job_number, test in enumerate(defined_matrix, 1) if actual_matrix.get(job_number) != test] + + if len(errors): + error_summary = '\n'.join('Job %s expected "%s" but found "%s" instead.' % (job_number, expected, actual) for job_number, expected, actual in errors) + + fail('Shippable run %s has a job matrix mismatch.' % run_id, + 'Try re-running the entire matrix.\n\n%s' % error_summary) + + +def fail(message, output): # type: (str, str) -> NoReturn + # Include a leading newline to improve readability on Shippable "Tests" tab. + # Without this, the first line becomes indented. + output = '\n' + output.strip() + + timestamp = datetime.datetime.utcnow().replace(microsecond=0).isoformat() + + # hack to avoid requiring junit-xml, which isn't pre-installed on Shippable outside our test containers + xml = ''' + + +\t +\t\t +\t\t\t%s +\t\t +\t + +''' % (timestamp, message, output) + + with open('test/results/junit/check-matrix.xml', 'w') as junit_fd: + junit_fd.write(xml.lstrip()) + + sys.stderr.write(message + '\n') + sys.stderr.write(output + '\n') + + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/test/utils/shippable/sanity.sh b/test/utils/shippable/sanity.sh index 09d859b3639..8a05ee20969 100755 --- a/test/utils/shippable/sanity.sh +++ b/test/utils/shippable/sanity.sh @@ -7,8 +7,6 @@ IFS='/:' read -ra args <<< "$1" group="${args[1]}" -shippable.py - if [ "${BASE_BRANCH:-}" ]; then base_branch="origin/${BASE_BRANCH}" else diff --git a/test/utils/shippable/shippable.sh b/test/utils/shippable/shippable.sh index 3571d3de4a4..79aa5d08e92 100755 --- a/test/utils/shippable/shippable.sh +++ b/test/utils/shippable/shippable.sh @@ -133,4 +133,5 @@ fi ansible-test env --dump --show --timeout "${timeout}" --color -v +"test/utils/shippable/check_matrix.py" "test/utils/shippable/${script}.sh" "${test}"