Count ignored failed tasks and successful tasks with expected failure… (#33282)

* Count ignored failed tasks and successful tasks with expected failure as failed tests.

* Make junit callback plugins changes backward compatible: add new task marker TOGGLE RESULT and new option fail_on_ignore

lib/ansible/plugins/callback/junit.py

@@ -15,6 +15,8 @@ DOCUMENTATION = '''
       - "Tasks show up in the report as follows:
         'ok': pass
         'failed' with 'EXPECTED FAILURE' in the task name: pass
+        'failed' with 'TOGGLE RESULT' in the task name: pass
+        'ok' with 'TOGGLE RESULT' in the task name: failure
         'failed' due to an exception: error
         'failed' for other reasons: failure
         'skipped': skipped"
@@ -37,6 +39,12 @@ DOCUMENTATION = '''
         description: Consider any tasks reporting "changed" as a junit test failure
         env:
           - name: JUNIT_FAIL_ON_CHANGE
+      fail_on_ignore:
+        name: JUnit fail on ignore
+        default: False
+        description: Consider failed tasks as a junit test failure even if ignore_on_error is set
+        env:
+          - name: JUNIT_FAIL_ON_IGNORE
     requirements:
       - whitelist in configuration
       - junit_xml (python lib)
@@ -73,6 +81,8 @@ class CallbackModule(CallbackBase):
     Tasks show up in the report as follows:
         'ok': pass
         'failed' with 'EXPECTED FAILURE' in the task name: pass
+        'failed' with 'TOGGLE RESULT' in the task name: pass
+        'ok' with 'TOGGLE RESULT' in the task name: failure
         'failed' due to an exception: error
         'failed' for other reasons: failure
         'skipped': skipped
@@ -84,6 +94,8 @@ class CallbackModule(CallbackBase):
                                      Default: False
         JUNIT_FAIL_ON_CHANGE (optional): Consider any tasks reporting "changed" as a junit test failure
                                      Default: False
+        JUNIT_FAIL_ON_IGNORE (optional): Consider failed tasks as a junit test failure even if ignore_on_error is set
+                                     Default: False
 
     Requires:
         junit_xml
@@ -101,6 +113,7 @@ class CallbackModule(CallbackBase):
         self._output_dir = os.getenv('JUNIT_OUTPUT_DIR', os.path.expanduser('~/.ansible.log'))
         self._task_class = os.getenv('JUNIT_TASK_CLASS', 'False').lower()
         self._fail_on_change = os.getenv('JUNIT_FAIL_ON_CHANGE', 'False').lower()
+        self._fail_on_ignore = os.getenv('JUNIT_FAIL_ON_IGNORE', 'False').lower()
         self._playbook_path = None
         self._playbook_name = None
         self._play_name = None
@@ -159,8 +172,14 @@ class CallbackModule(CallbackBase):
         if self._fail_on_change == 'true' and status == 'ok' and result._result.get('changed', False):
             status = 'failed'
 
+        # ignore failure if expected and toggle result if asked for
         if status == 'failed' and 'EXPECTED FAILURE' in task_data.name:
             status = 'ok'
+        elif 'TOGGLE RESULT' in task_data.name:
+            if status == 'failed':
+                status = 'ok'
+            elif status == 'ok':
+                status = 'failed'
 
         task_data.add_host(HostData(host_uuid, host_name, status, result))
 
@@ -248,7 +267,7 @@ class CallbackModule(CallbackBase):
         self._start_task(task)
 
     def v2_runner_on_failed(self, result, ignore_errors=False):
-        if ignore_errors:
+        if ignore_errors and self._fail_on_ignore != 'true':
             self._finish_task('ok', result)
         else:
             self._finish_task('failed', result)