diff --git a/hacking/tests/gen_distribution_version_testcase.py b/hacking/tests/gen_distribution_version_testcase.py
index 04141718829..775c30f8f94 100755
--- a/hacking/tests/gen_distribution_version_testcase.py
+++ b/hacking/tests/gen_distribution_version_testcase.py
@@ -12,6 +12,7 @@ import platform
 import os.path
 import subprocess
 import json
+import sys
 
 filelist = [
         '/etc/oracle-release',
@@ -46,7 +47,15 @@ dist = platform.dist()
 
 
 facts = ['distribution', 'distribution_version', 'distribution_release', 'distribution_major_version', 'os_family']
-ansible_out = subprocess.Popen(['ansible', 'localhost', '-m', 'setup'], stdout=subprocess.PIPE).communicate()[0]
+
+try:
+    ansible_out = subprocess.check_output(
+        ['ansible', 'localhost', '-m', 'setup'])
+except subprocess.CalledProcessError as e:
+    print("ERROR: ansible run failed, output was: \n")
+    print(e.output)
+    sys.exit(e.returncode)
+
 parsed = json.loads(ansible_out[ansible_out.index('{'):])
 ansible_facts = {}
 for fact in facts: