archive
/
ansible
mirror of https://github.com/ansible/ansible.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Retrofit shippable scripts to work with AZP

Browse Source 
Co-authored-by: Matt Clay <matt@mystile.com>
pull/74495/head
Alexander Sowitzki 4 years ago committed by Matt Clay
parent 57d661e96f
commit 26214788ee
5 changed files with 205 additions and 415 deletions
Show Stats Download Patch File Download Diff File

9
hacking/shippable/README.md
Unescape Escape View File

@ -4,10 +4,11 @@
This directory contains the following scripts:
- download.py - Download results from Shippable.
- get_recent_coverage_runs.py - Retrieve Shippable URLs of recent coverage test runs.
- incidental.py - Report on incidental code coverage using data from Shippable.
- run.py - Start new runs on Shippable.
- download.py - Download results from CI.
- get_recent_coverage_runs.py - Retrieve CI URLs of recent coverage test runs.
- incidental.py - Report on incidental code coverage using data from CI.
- run.py - Start new runs on CI.
- rebalance.py - Re-balance CI group(s) from a downloaded results directory.
## Incidental Code Coverage

393
hacking/shippable/download.py
Unescape Escape View File

@ -18,6 +18,7 @@
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
"""CLI tool for downloading results from Shippable CI runs."""
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
@ -27,6 +28,8 @@ import json
import os
import re
import sys
import io
import zipfile
import requests
@ -35,22 +38,33 @@ try:
except ImportError:
argcomplete = None
# Following changes should be made to improve the overall style:
# TODO use new style formatting method.
# TODO use requests session.
# TODO type hints.
# TODO pathlib.
def main():
"""Main program body."""
args = parse_args()
download_run(args)
def run_id_arg(arg):
m = re.fullmatch(r"(?:https:\/\/dev\.azure\.com\/ansible\/ansible\/_build\/results\?buildId=)?(\d+)", arg)
if not m:
raise ValueError("run does not seems to be a URI or an ID")
return m.group(1)
def parse_args():
"""Parse and return args."""
api_key = get_api_key()
parser = argparse.ArgumentParser(description='Download results from a Shippable run.')
parser = argparse.ArgumentParser(description='Download results from a CI run.')
parser.add_argument('run_id',
metavar='RUN',
help='shippable run id, run url or run name formatted as: account/project/run_number')
parser.add_argument('run', metavar='RUN', type=run_id_arg, help='AZP run id or URI')
parser.add_argument('-v', '--verbose',
dest='verbose',
@ -62,27 +76,15 @@ def parse_args():
action='store_true',
help='show what would be downloaded without downloading')
parser.add_argument('--key',
dest='api_key',
default=api_key,
required=api_key is None,
help='api key for accessing Shippable')
parser.add_argument('--console-logs',
action='store_true',
help='download console logs')
parser.add_argument('--test-results',
action='store_true',
help='download test results')
parser.add_argument('-p', '--pipeline-id', type=int, default=20, help='pipeline to download the job from')
parser.add_argument('--coverage-results',
parser.add_argument('--artifacts',
action='store_true',
help='download code coverage results')
help='download artifacts')
parser.add_argument('--job-metadata',
parser.add_argument('--console-logs',
action='store_true',
help='download job metadata')
help='download console logs')
parser.add_argument('--run-metadata',
action='store_true',
@ -92,35 +94,30 @@ def parse_args():
action='store_true',
help='download everything')
parser.add_argument('--job-number',
metavar='N',
action='append',
type=int,
help='limit downloads to the given job number')
parser.add_argument('--match-artifact-name',
default=re.compile('.*'),
type=re.compile,
help='only download artifacts which names match this regex')
parser.add_argument('--match-job-name',
default=re.compile('.*'),
type=re.compile,
help='only download artifacts from jobs which names match this regex')
if argcomplete:
argcomplete.autocomplete(parser)
args = parser.parse_args()
old_runs_prefix = 'https://app.shippable.com/runs/'
if args.run_id.startswith(old_runs_prefix):
args.run_id = args.run_id[len(old_runs_prefix):]
if args.all:
args.console_logs = True
args.test_results = True
args.coverage_results = True
args.job_metadata = True
args.artifacts = True
args.run_metadata = True
args.console_logs = True
selections = (
args.console_logs,
args.test_results,
args.coverage_results,
args.job_metadata,
args.artifacts,
args.run_metadata,
args.console_logs
)
if not any(selections):
@ -130,256 +127,100 @@ def parse_args():
def download_run(args):
"""Download a Shippable run."""
headers = dict(
Authorization='apiToken %s' % args.api_key,
)
match = re.search(
r'^https://app.shippable.com/github/(?P<account>[^/]+)/(?P<project>[^/]+)/runs/(?P<run_number>[0-9]+)(?:/summary|(/(?P<job_number>[0-9]+)))?$',
args.run_id)
if not match:
match = re.search(r'^(?P<account>[^/]+)/(?P<project>[^/]+)/(?P<run_number>[0-9]+)$', args.run_id)
"""Download a run."""
if match:
account = match.group('account')
project = match.group('project')
run_number = int(match.group('run_number'))
job_number = int(match.group('job_number')) if match.group('job_number') else None
output_dir = '%s' % args.run
if job_number:
if args.job_number:
sys.exit('ERROR: job number found in url and specified with --job-number')
args.job_number = [job_number]
url = 'https://api.shippable.com/projects'
response = requests.get(url, dict(projectFullNames='%s/%s' % (account, project)), headers=headers)
if response.status_code != 200:
raise Exception(response.content)
project_id = response.json()[0]['id']
url = 'https://api.shippable.com/runs?projectIds=%s&runNumbers=%s' % (project_id, run_number)
response = requests.get(url, headers=headers)
if response.status_code != 200:
raise Exception(response.content)
run = [run for run in response.json() if run['runNumber'] == run_number][0]
args.run_id = run['id']
elif re.search('^[a-f0-9]+$', args.run_id):
url = 'https://api.shippable.com/runs/%s' % args.run_id
response = requests.get(url, headers=headers)
if response.status_code != 200:
raise Exception(response.content)
run = response.json()
account = run['subscriptionOrgName']
project = run['projectName']
run_number = run['runNumber']
else:
sys.exit('ERROR: invalid run: %s' % args.run_id)
output_dir = '%s/%s/%s' % (account, project, run_number)
if not args.test:
if not os.path.exists(output_dir):
os.makedirs(output_dir)
if not args.test and not os.path.exists(output_dir):
os.makedirs(output_dir)
if args.run_metadata:
run_url = 'https://dev.azure.com/ansible/ansible/_apis/pipelines/%s/runs/%s?api-version=6.0-preview.1' % (args.pipeline_id, args.run)
run_info_response = requests.get(run_url)
run_info_response.raise_for_status()
run = run_info_response.json()
path = os.path.join(output_dir, 'run.json')
contents = json.dumps(run, sort_keys=True, indent=4)
if args.verbose or args.test:
if args.verbose:
print(path)
if not args.test:
with open(path, 'w') as metadata_fd:
metadata_fd.write(contents)
download_run_recursive(args, headers, output_dir, run, True)
def download_run_recursive(args, headers, output_dir, run, is_given=False):
# Notes:
# - The /runs response tells us if we need to eventually go up another layer
# or not (if we are a re-run attempt or not).
# - Given a run id, /jobs will tell us all the jobs in that run, and whether
# or not we can pull results from them.
#
# When we initially run (i.e., in download_run), we'll have a /runs output
# which we can use to get a /jobs output. Using the /jobs output, we filter
# on the jobs we need to fetch (usually only successful ones unless we are
# processing the initial/given run and not one of its parent runs) and
# download them accordingly.
#
# Lastly, we check if the run we are currently processing has another
# parent (reRunBatchId). If it does, we pull that /runs result and
# recurse using it to start the process over again.
response = requests.get('https://api.shippable.com/jobs?runIds=%s' % run['id'], headers=headers)
if response.status_code != 200:
raise Exception(response.content)
jobs = sorted(response.json(), key=lambda job: int(job['jobNumber']))
if is_given:
needed_jobs = [j for j in jobs if j['isConsoleArchived']]
else:
needed_jobs = [j for j in jobs if j['isConsoleArchived'] and j['statusCode'] == 30]
if not args.test:
if not os.path.exists(output_dir):
os.makedirs(output_dir)
download_jobs(args, needed_jobs, headers, output_dir)
rerun_batch_id = run.get('reRunBatchId')
if rerun_batch_id:
print('Downloading previous run: %s' % rerun_batch_id)
response = requests.get('https://api.shippable.com/runs/%s' % rerun_batch_id, headers=headers)
if response.status_code != 200:
raise Exception(response.content)
run = response.json()
download_run_recursive(args, headers, output_dir, run)
def download_jobs(args, jobs, headers, output_dir):
"""Download Shippable jobs."""
for j in jobs:
job_id = j['id']
job_number = j['jobNumber']
if args.job_number and job_number not in args.job_number:
continue
if args.job_metadata:
path = os.path.join(output_dir, '%s/job.json' % job_number)
contents = json.dumps(j, sort_keys=True, indent=4).encode('utf-8')
if args.verbose or args.test:
print(path)
if not args.test:
directory = os.path.dirname(path)
if not os.path.exists(directory):
os.makedirs(directory)
with open(path, 'wb') as metadata_fd:
metadata_fd.write(contents)
if args.console_logs:
path = os.path.join(output_dir, '%s/console.log' % job_number)
url = 'https://api.shippable.com/jobs/%s/consoles?download=true' % job_id
download(args, headers, path, url, is_json=False)
if args.test_results:
path = os.path.join(output_dir, '%s/test.json' % job_number)
url = 'https://api.shippable.com/jobs/%s/jobTestReports' % job_id
download(args, headers, path, url)
extract_contents(args, path, os.path.join(output_dir, '%s/test' % job_number))
if args.coverage_results:
path = os.path.join(output_dir, '%s/coverage.json' % job_number)
url = 'https://api.shippable.com/jobs/%s/jobCoverageReports' % job_id
download(args, headers, path, url)
extract_contents(args, path, os.path.join(output_dir, '%s/coverage' % job_number))
def extract_contents(args, path, output_dir):
"""
:type args: any
:type path: str
:type output_dir: str
"""
if not args.test:
if not os.path.exists(path):
return
with open(path, 'r') as json_fd:
items = json.load(json_fd)
for item in items:
contents = item['contents'].encode('utf-8')
path = output_dir + '/' + re.sub('^/*', '', item['path'])
directory = os.path.dirname(path)
if not os.path.exists(directory):
os.makedirs(directory)
if args.verbose:
print(path)
if path.endswith('.json'):
contents = json.dumps(json.loads(contents), sort_keys=True, indent=4).encode('utf-8')
if not os.path.exists(path):
with open(path, 'wb') as output_fd:
output_fd.write(contents)
def download(args, headers, path, url, is_json=True):
"""
:type args: any
:type headers: dict[str, str]
:type path: str
:type url: str
:type is_json: bool
"""
if args.verbose or args.test:
print(path)
if os.path.exists(path):
return
if not args.test:
response = requests.get(url, headers=headers)
if response.status_code != 200:
path += '.error'
if is_json:
content = json.dumps(response.json(), sort_keys=True, indent=4).encode(response.encoding)
timeline_response = requests.get('https://dev.azure.com/ansible/ansible/_apis/build/builds/%s/timeline?api-version=6.0' % args.run)
timeline_response.raise_for_status()
timeline = timeline_response.json()
roots = set()
by_id = {}
children_of = {}
parent_of = {}
for r in timeline['records']:
thisId = r['id']
parentId = r['parentId']
by_id[thisId] = r
if parentId is None:
roots.add(thisId)
else:
content = response.content
directory = os.path.dirname(path)
if not os.path.exists(directory):
os.makedirs(directory)
with open(path, 'wb') as content_fd:
content_fd.write(content)
def get_api_key():
"""
rtype: str
"""
key = os.environ.get('SHIPPABLE_KEY', None)
if key:
return key
path = os.path.join(os.environ['HOME'], '.shippable.key')
try:
with open(path, 'r') as key_fd:
return key_fd.read().strip()
except IOError:
return None
parent_of[thisId] = parentId
children_of[parentId] = children_of.get(parentId, []) + [thisId]
allowed = set()
def allow_recursive(ei):
allowed.add(ei)
for ci in children_of.get(ei, []):
allow_recursive(ci)
for ri in roots:
r = by_id[ri]
allowed.add(ri)
for ci in children_of.get(r['id'], []):
c = by_id[ci]
if not args.match_job_name.match("%s %s" % (r['name'], c['name'])):
continue
allow_recursive(c['id'])
if args.artifacts:
artifact_list_url = 'https://dev.azure.com/ansible/ansible/_apis/build/builds/%s/artifacts?api-version=6.0' % args.run
artifact_list_response = requests.get(artifact_list_url)
artifact_list_response.raise_for_status()
for artifact in artifact_list_response.json()['value']:
if artifact['source'] not in allowed or not args.match_artifact_name.match(artifact['name']):
continue
if args.verbose:
print('%s/%s' % (output_dir, artifact['name']))
if not args.test:
response = requests.get(artifact['resource']['downloadUrl'])
response.raise_for_status()
archive = zipfile.ZipFile(io.BytesIO(response.content))
archive.extractall(path=output_dir)
if args.console_logs:
for r in timeline['records']:
if not r['log'] or r['id'] not in allowed or not args.match_artifact_name.match(r['name']):
continue
names = []
parent_id = r['id']
while parent_id is not None:
p = by_id[parent_id]
name = p['name']
if name not in names:
names = [name] + names
parent_id = parent_of.get(p['id'], None)
path = " ".join(names)
log_path = os.path.join(output_dir, '%s.log' % path)
if args.verbose:
print(log_path)
if not args.test:
log = requests.get(r['log']['url'])
log.raise_for_status()
open(log_path, 'wb').write(log.content)
if __name__ == '__main__':

63
hacking/shippable/get_recent_coverage_runs.py
Unescape Escape View File

@ -23,31 +23,48 @@ __metaclass__ = type
from ansible.utils.color import stringc
import requests
import sys
import datetime
# Following changes should be made to improve the overall style:
# TODO use argparse for arguments.
# TODO use new style formatting method.
# TODO use requests session.
# TODO type hints.
BRANCH = 'devel'
PIPELINE_ID = 20
MAX_AGE = datetime.timedelta(hours=24)
if len(sys.argv) > 1:
BRANCH = sys.argv[1]
def get_coverage_runs():
response = requests.get(
'https://api.shippable.com/runs?projectIds=573f79d02a8192902e20e34b'
'&branch=%s&limit=1000' % BRANCH)
if response.status_code != 200:
raise Exception(response.content)
list_response = requests.get("https://dev.azure.com/ansible/ansible/_apis/pipelines/%s/runs?api-version=6.0-preview.1" % PIPELINE_ID)
list_response.raise_for_status()
runs = response.json()
runs = list_response.json()
coverage_runs = []
criteria = ['COMPLETE="yes"', 'COVERAGE="yes"']
for run_summary in runs["value"][0:1000]:
run_response = requests.get(run_summary['url'])
run_response.raise_for_status()
run = run_response.json()
for run in runs:
injected_vars = run.get('cleanRunYml', {}).get('env', {}).get('injected')
if not injected_vars:
if run['resources']['repositories']['self']['refName'] != 'refs/heads/%s' % BRANCH:
continue
if all(criterion in injected_vars for criterion in criteria):
if 'finishedDate' in run_summary:
age = datetime.datetime.now() - datetime.datetime.strptime(run['finishedDate'].split(".")[0], "%Y-%m-%dT%H:%M:%S")
if age > MAX_AGE:
break
artifact_response = requests.get("https://dev.azure.com/ansible/ansible/_apis/build/builds/%s/artifacts?api-version=6.0" % run['id'])
artifact_response.raise_for_status()
artifacts = artifact_response.json()['value']
if any([a["name"].startswith("Coverage") for a in artifacts]):
# TODO wrongfully skipped if all jobs failed.
coverage_runs.append(run)
return coverage_runs
@ -57,29 +74,29 @@ def pretty_coverage_runs(runs):
ended = []
in_progress = []
for run in runs:
if run.get('endedAt'):
if run.get('finishedDate'):
ended.append(run)
else:
in_progress.append(run)
for run in sorted(ended, key=lambda x: x['endedAt']):
if run['statusCode'] == 30:
print('🙂 [%s] https://app.shippable.com/github/ansible/ansible/runs/%s (%s)' % (
for run in sorted(ended, key=lambda x: x['finishedDate']):
if run['result'] == "succeeded":
print('🙂 [%s] https://dev.azure.com/ansible/ansible/_build/results?buildId=%s (%s)' % (
stringc('PASS', 'green'),
run['runNumber'],
run['endedAt']))
run['id'],
run['finishedDate']))
else:
print('😢 [%s] https://app.shippable.com/github/ansible/ansible/runs/%s (%s)' % (
print('😢 [%s] https://dev.azure.com/ansible/ansible/_build/results?buildId=%s (%s)' % (
stringc('FAIL', 'red'),
run['runNumber'],
run['endedAt']))
run['id'],
run['finishedDate']))
if in_progress:
print('The following runs are ongoing:')
for run in in_progress:
print('🤔 [%s] https://app.shippable.com/github/ansible/ansible/runs/%s' % (
print('🤔 [%s] https://dev.azure.com/ansible/ansible/_build/results?buildId=%s' % (
stringc('FATE', 'yellow'),
run['runNumber']))
run['id']))
def main():

37
hacking/shippable/incidental.py
Unescape Escape View File

@ -37,6 +37,11 @@ try:
except ImportError:
argcomplete = None
# Following changes should be made to improve the overall style:
# TODO use new style formatting method.
# TODO type hints.
# TODO pathlib.
def main():
"""Main program body."""
@ -132,14 +137,9 @@ def incidental_report(args):
raise ApplicationError('%s: commit not found: %s\n'
'make sure your source repository is up-to-date' % (git.path, coverage_data.result_sha))
if coverage_data.status_code != 30:
check_failed(args, 'results from Shippable indicate tests did not pass (status code: %d)\n'
're-run until passing, then download the latest results and re-run the report using those results' % coverage_data.status_code)
if coverage_data.missing_jobs or coverage_data.extra_jobs:
check_failed(args, 'unexpected results from Shippable -- missing jobs: %s, extra jobs: %s\n'
'make sure the tests were successful and the all results were downloaded\n' % (
sorted(coverage_data.missing_jobs), sorted(coverage_data.extra_jobs)))
if coverage_data.result != "succeeded":
check_failed(args, 'results indicate tests did not pass (result: %s)\n'
're-run until passing, then download the latest results and re-run the report using those results' % coverage_data.result)
if not coverage_data.paths:
raise ApplicationError('no coverage data found\n'
@ -280,26 +280,13 @@ class CoverageData:
with open(os.path.join(result_path, 'run.json')) as run_file:
run = json.load(run_file)
self.org_name = run['subscriptionOrgName']
self.project_name = run['projectName']
self.result_sha = run['commitSha']
self.status_code = run['statusCode']
self.result_sha = run["resources"]["repositories"]["self"]["version"]
self.result = run['result']
self.github_base_url = 'https://github.com/%s/%s/blob/%s/' % (self.org_name, self.project_name, self.result_sha)
self.github_base_url = 'https://github.com/ansible/ansible/blob/%s/' % self.result_sha
# locate available results
self.paths = sorted(glob.glob(os.path.join(result_path, '*', 'test', 'testresults', 'coverage-analyze-targets.json')))
# make sure the test matrix is complete
matrix_include = run['cleanRunYml']['matrix']['include']
matrix_jobs = list((idx, dict(tuple(item.split('=', 1)) for item in value['env'])) for idx, value in enumerate(matrix_include, start=1))
sanity_job_numbers = set(idx for idx, env in matrix_jobs if env['T'].startswith('sanity/'))
units_job_numbers = set(idx for idx, env in matrix_jobs if env['T'].startswith('units/'))
expected_job_numbers = set(idx for idx, env in matrix_jobs)
actual_job_numbers = set(int(os.path.relpath(path, result_path).split(os.path.sep)[0]) for path in self.paths)
self.missing_jobs = expected_job_numbers - actual_job_numbers - sanity_job_numbers - units_job_numbers
self.extra_jobs = actual_job_numbers - expected_job_numbers - sanity_job_numbers - units_job_numbers
self.paths = sorted(glob.glob(os.path.join(result_path, '*', 'coverage-analyze-targets.json')))
class Git:

114
hacking/shippable/run.py
Unescape Escape View File

@ -17,7 +17,9 @@
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
"""CLI tool for starting new Shippable CI runs."""
"""CLI tool for starting new CI runs."""
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
@ -25,45 +27,42 @@ __metaclass__ = type
import argparse
import json
import os
import sys
import requests
import requests.auth
try:
import argcomplete
except ImportError:
argcomplete = None
# TODO: Dev does not have a token for AZP, somebody please test this.
# Following changes should be made to improve the overall style:
# TODO use new style formatting method.
# TODO type hints.
def main():
"""Main program body."""
args = parse_args()
start_run(args)
def parse_args():
"""Parse and return args."""
api_key = get_api_key()
args = parse_args()
parser = argparse.ArgumentParser(description='Start a new Shippable run.')
key = os.environ.get('AZP_TOKEN', None)
if not key:
sys.stderr.write("please set you AZP token in AZP_TOKEN")
sys.exit(1)
parser.add_argument('project',
metavar='account/project',
help='Shippable account/project')
start_run(args, key)
target = parser.add_mutually_exclusive_group()
target.add_argument('--branch',
help='branch name')
def parse_args():
"""Parse and return args."""
target.add_argument('--run',
metavar='ID',
help='Shippable run ID')
parser = argparse.ArgumentParser(description='Start a new CI run.')
parser.add_argument('--key',
metavar='KEY',
default=api_key,
required=not api_key,
help='Shippable API key')
parser.add_argument('-p', '--pipeline-id', type=int, default=20, help='pipeline to download the job from')
parser.add_argument('--ref', help='git ref name to run on')
parser.add_argument('--env',
nargs=2,
@ -79,71 +78,16 @@ def parse_args():
return args
def start_run(args):
"""Start a new Shippable run."""
headers = dict(
Authorization='apiToken %s' % args.key,
)
# get project ID
data = dict(
projectFullNames=args.project,
)
url = 'https://api.shippable.com/projects'
response = requests.get(url, data, headers=headers)
if response.status_code != 200:
raise Exception(response.content)
result = response.json()
if len(result) != 1:
raise Exception(
'Received %d items instead of 1 looking for %s in:\n%s' % (
len(result),
args.project,
json.dumps(result, indent=4, sort_keys=True)))
project_id = response.json()[0]['id']
# new build
data = dict(
globalEnv=dict((kp[0], kp[1]) for kp in args.env or [])
)
if args.branch:
data['branchName'] = args.branch
elif args.run:
data['runId'] = args.run
url = 'https://api.shippable.com/projects/%s/newBuild' % project_id
response = requests.post(url, json=data, headers=headers)
if response.status_code != 200:
raise Exception("HTTP %s: %s\n%s" % (response.status_code, response.reason, response.content))
print(json.dumps(response.json(), indent=4, sort_keys=True))
def get_api_key():
"""
rtype: str
"""
key = os.environ.get('SHIPPABLE_KEY', None)
def start_run(args, key):
"""Start a new CI run."""
if key:
return key
url = "https://dev.azure.com/ansible/ansible/_apis/pipelines/%s/runs?api-version=6.0-preview.1" % args.pipeline_id
payload = {"resources": {"repositories": {"self": {"refName": args.ref}}}}
path = os.path.join(os.environ['HOME'], '.shippable.key')
resp = requests.post(url, auth=requests.auth.HTTPBasicAuth('user', key), data=payload)
resp.raise_for_status()
try:
with open(path, 'r') as key_fd:
return key_fd.read().strip()
except IOError:
return None
print(json.dumps(resp.json(), indent=4, sort_keys=True))
if __name__ == '__main__':

Write Preview
Loading…
Cancel
Save