Improve ansible-test HttpClient error handling. (#30301)

- Automatic retries on DNS lookup failures.
- Handle API errors in shippable.py.

test/runner/lib/http.py

@@ -6,6 +6,7 @@ Avoids use of urllib2 due to lack of SNI support.
 from __future__ import absolute_import, print_function
 
 import json
+import time
 
 try:
     from urllib import urlencode
@@ -24,6 +25,8 @@ from lib.util import (
     CommonConfig,
     ApplicationError,
     run_command,
+    SubprocessError,
+    display,
 )
 
 
@@ -83,7 +86,28 @@ class HttpClient(object):
 
         cmd += [url]
 
-        stdout, _ = run_command(self.args, cmd, capture=True, always=self.always, cmd_verbosity=2)
+        attempts = 0
+        max_attempts = 3
+        sleep_seconds = 3
+
+        # curl error codes which are safe to retry (request never sent to server)
+        retry_on_status = (
+            6,  # CURLE_COULDNT_RESOLVE_HOST
+        )
+
+        while True:
+            attempts += 1
+
+            try:
+                stdout, _ = run_command(self.args, cmd, capture=True, always=self.always, cmd_verbosity=2)
+                break
+            except SubprocessError as ex:
+                if ex.status in retry_on_status and attempts < max_attempts:
+                    display.warning(u'%s' % ex)
+                    time.sleep(sleep_seconds)
+                    continue
+
+                raise
 
         if self.args.explain and not self.always:
             return HttpResponse(method, url, 200, '')

test/runner/shippable.py

@@ -7,6 +7,7 @@ from __future__ import absolute_import, print_function
 # noinspection PyCompatibility
 import argparse
 import errno
+import json
 import os
 import sys
 
@@ -44,6 +45,9 @@ def main():
         response = client.get('https://api.shippable.com/jobs?runIds=%s' % run_id)
         jobs = response.json()
 
+        if not isinstance(jobs, list):
+            raise ApplicationError(json.dumps(jobs, indent=4, sort_keys=True))
+
         if len(jobs) == 1:
             raise ApplicationError('Shippable run %s has only one job. Did you use the "Rebuild with SSH" option?' % run_id)
     except ApplicationWarning as ex: