Fix ansible-test code coverage collection. (#68502)

The last task in a play should now properly report code coverage.

This change should also eliminate empty coverage files, as well as incomplete coverage files resulting from early worker termination.

changelogs/fragments/ansible-test-coverage-incomplete.yml

@@ -0,0 +1,3 @@
+bugfixes:
+    - ansible-test now correctly collects code coverage on the last task in a play.
+      This should resolve issues with missing code coverage, empty coverage files and corrupted coverage files resulting from early worker termination.

lib/ansible/config/base.yml

@@ -1828,6 +1828,25 @@ TAGS_SKIP:
   ini:
   - {key: skip, section: tags}
   version_added: "2.5"
+WORKER_SHUTDOWN_POLL_COUNT:
+  name: Worker Shutdown Poll Count
+  default: 0
+  description:
+    - The maximum number of times to check Task Queue Manager worker processes to verify they have exited cleanly.
+    - After this limit is reached any worker processes still running will be terminated.
+    - This is for internal use only.
+  env: [{name: ANSIBLE_WORKER_SHUTDOWN_POLL_COUNT}]
+  type: integer
+  version_added: '2.10'
+WORKER_SHUTDOWN_POLL_DELAY:
+  name: Worker Shutdown Poll Delay
+  default: 0.1
+  description:
+    - The number of seconds to sleep between polling loops when checking Task Queue Manager worker processes to verify they have exited cleanly.
+    - This is for internal use only.
+  env: [{name: ANSIBLE_WORKER_SHUTDOWN_POLL_DELAY}]
+  type: float
+  version_added: '2.10'
 USE_PERSISTENT_CONNECTIONS:
   name: Persistence
   default: False

lib/ansible/executor/task_queue_manager.py

@@ -21,6 +21,7 @@ __metaclass__ = type
 
 import os
 import tempfile
+import time
 
 from ansible import constants as C
 from ansible import context
@@ -256,6 +257,15 @@ class TaskQueueManager:
 
     def _cleanup_processes(self):
         if hasattr(self, '_workers'):
+            for attempts_remaining in range(C.WORKER_SHUTDOWN_POLL_COUNT - 1, -1, -1):
+                if not any(worker_prc and worker_prc.is_alive() for worker_prc in self._workers):
+                    break
+
+                if attempts_remaining:
+                    time.sleep(C.WORKER_SHUTDOWN_POLL_DELAY)
+                else:
+                    display.warning('One or more worker processes are still running and will be terminated.')
+
             for worker_prc in self._workers:
                 if worker_prc and worker_prc.is_alive():
                     try:

test/lib/ansible_test/_internal/ansible_util.py

@@ -77,6 +77,11 @@ def ansible_environment(args, color=True, ansible_config=None):
         PYTHONPATH=get_ansible_python_path(),
         PAGER='/bin/cat',
         PATH=path,
+        # give TQM worker processes time to report code coverage results
+        # without this the last task in a play may write no coverage file, an empty file, or an incomplete file
+        # enabled even when not using code coverage to surface warnings when worker processes do not exit cleanly
+        ANSIBLE_WORKER_SHUTDOWN_POLL_COUNT='100',
+        ANSIBLE_WORKER_SHUTDOWN_POLL_DELAY='0.1',
     )
 
     if isinstance(args, IntegrationConfig) and args.coverage: