diff --git a/test/utils/shippable/docs.sh b/test/utils/shippable/docs.sh
index 9b5a6164f64..2858f87c997 100755
--- a/test/utils/shippable/docs.sh
+++ b/test/utils/shippable/docs.sh
@@ -42,10 +42,12 @@ pip list
 
 source hacking/env-setup
 
+docs_status=0
+
 PAGER=/bin/cat \
     ANSIBLE_DEPRECATION_WARNINGS=false \
     bin/ansible-doc -l \
-    2>/tmp/ansible-doc.err
+    2>/tmp/ansible-doc.err || docs_status=$?
 
 if [ -s /tmp/ansible-doc.err ]; then
     # report warnings as errors
@@ -53,3 +55,8 @@ if [ -s /tmp/ansible-doc.err ]; then
     cat /tmp/ansible-doc.err
     exit 1
 fi
+
+if [ "${docs_status}" -ne 0 ]; then
+    echo "Running 'ansible-doc -l' failed with no output on stderr and exit code: ${docs_status}"
+    exit 1
+fi