diff --git a/test/utils/shippable/ansible-core-ci b/test/utils/shippable/ansible-core-ci
index c0c25aaf220..44fd1df1058 100755
--- a/test/utils/shippable/ansible-core-ci
+++ b/test/utils/shippable/ansible-core-ci
@@ -359,7 +359,10 @@ class HttpResponse:
         self.response = response
 
     def json(self):
-        return json.loads(self.response)
+        try:
+            return json.loads(self.response)
+        except ValueError:
+            raise ValueError('Cannot parse response as JSON:\n%s' % self.response)
 
 
 requests = HttpRequest()