diff --git a/test/utils/run_tests.sh b/test/utils/run_tests.sh
index f7d5a878fc4..63105c29310 100755
--- a/test/utils/run_tests.sh
+++ b/test/utils/run_tests.sh
@@ -1,5 +1,15 @@
 #!/bin/sh
 
+if [ "${SHIPPABLE}" = "true" ]; then
+    echo "It appears this job is running on Shippable instead of Travis."
+    if [ "${IS_PULL_REQUEST}" = "true" ]; then
+        echo "Please rebase the branch used for this pull request."
+    else
+        echo "This branch needs to be updated to work with Shippable."
+    fi
+    exit 1
+fi
+
 set -e
 set -u
 set -x