Add `--failure-ok` option to `ansible-test`. (#22623)

test/runner/lib/executor.py

@@ -712,8 +712,13 @@ def command_compile(args):
             failed.append('compile --python %s' % version)
 
     if failed:
-        raise ApplicationError('The %d compile test(s) listed below (out of %d) failed. See error output above for details.\n%s' % (
+        message = 'The %d compile test(s) listed below (out of %d) failed. See error output above for details.\n%s' % (
-            len(failed), total, '\n'.join(failed)))
+            len(failed), total, '\n'.join(failed))
+
+        if args.failure_ok:
+            display.error(message)
+        else:
+            raise ApplicationError(message)
 
 
 def compile_version(args, python_version, include, exclude):

test/runner/lib/sanity.py

@@ -114,8 +114,13 @@ def command_sanity(args):
                 failed.append(result.test + options)
 
     if failed:
-        raise ApplicationError('The %d sanity test(s) listed below (out of %d) failed. See error output above for details.\n%s' % (
+        message = 'The %d sanity test(s) listed below (out of %d) failed. See error output above for details.\n%s' % (
-            len(failed), total, '\n'.join(failed)))
+            len(failed), total, '\n'.join(failed))
+
+        if args.failure_ok:
+            display.error(message)
+        else:
+            raise ApplicationError(message)
 
 
 def command_sanity_code_smell(args, _, script):

test/runner/lib/test.py

@@ -36,6 +36,7 @@ class TestConfig(EnvironmentConfig):
 
         self.lint = args.lint if 'lint' in args else False  # type: bool
         self.junit = args.junit if 'junit' in args else False  # type: bool
+        self.failure_ok = args.failure_ok if 'failure_ok' in args else False  # type: bool
 
 
 class TestResult(object):
@@ -266,7 +267,7 @@ class TestFailure(TestResult):
         if args.explain:
             return
 
-        with open(path, 'wb') as bot_fd:
+        with open(path, 'w') as bot_fd:
             json.dump(bot_data, bot_fd, indent=4, sort_keys=True)
             bot_fd.write('\n')
 

test/runner/test.py

@@ -383,6 +383,11 @@ def add_lint(parser):
                         action='store_true',
                         help='write test failures to junit xml files')
 
+    parser.add_argument('--failure-ok',
+                        action='store_true',
+                        help='exit successfully on failed tests after saving results')
+
+
 
 def add_changes(parser, argparse):
     """

test/utils/shippable/other.sh

@@ -17,17 +17,11 @@ ln -sf x86_64-linux-gnu-gcc-4.9 /usr/bin/x86_64-linux-gnu-gcc
 
 retry.py pip install tox --disable-pip-version-check
 
-errors=0
+ansible-test compile --failure-ok --color -v --junit --requirements
+ansible-test sanity  --failure-ok --color -v --junit --tox --skip-test ansible-doc --python 2.7
+ansible-test sanity  --failure-ok --color -v --junit --tox --test ansible-doc --coverage
 
-set +e
+if find test/results/bot/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then
-
+    echo "One or more of the above ansible-test commands recorded at least one test failure."
-ansible-test compile --color -v --junit --requirements || ((errors++))
-ansible-test sanity --color -v --junit --tox --skip-test ansible-doc --python 2.7 || ((errors++))
-ansible-test sanity --color -v --junit --tox --test ansible-doc --coverage || ((errors++))
-
-set -e
-
-if [ ${errors} -gt 0 ]; then
-    echo "${errors} of the above ansible-test command(s) failed."
     exit 1
 fi