ansible-test - Fix integration test code coverage (#86100)

changelogs/fragments/ansible-test-integration-coverage.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - ansible-test - Restore code coverage reporting for Python code residing in integration tests.

test/integration/targets/ansible-test-coverage/ansible_collections/ns/col/plugins/module_utils/test_util.py

@@ -1,5 +1,4 @@
 from __future__ import annotations
 
 
-def test_coverage():
-    pass
+A_CONSTANT = True

test/integration/targets/ansible-test-coverage/ansible_collections/ns/col/tests/integration/targets/hello/aliases

@@ -0,0 +1 @@
+context/controller

test/integration/targets/ansible-test-coverage/ansible_collections/ns/col/tests/integration/targets/hello/runme.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eu
+
+python.py world.py

test/integration/targets/ansible-test-coverage/ansible_collections/ns/col/tests/integration/targets/hello/world.py

@@ -0,0 +1,3 @@
+from __future__ import annotations
+
+print("Hello World!")

test/integration/targets/ansible-test-coverage/ansible_collections/ns/col/tests/unit/test_something.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+
+def test_me() -> None:
+    pass

test/integration/targets/ansible-test-coverage/runme.sh

@@ -7,10 +7,31 @@ set -x
 # common args for all tests
 common=(--venv --color --truncate 0 "${@}")
 
-# run a lightweight test that generates code coverge output
-ansible-test sanity --test import "${common[@]}" --coverage
+# run a lightweight test that generates code coverage output
+ansible-test sanity --test import "${common[@]}" --coverage --python "${ANSIBLE_TEST_PYTHON_VERSION}"
+
+# run an integration test that generates code coverage for a Python file within the integration test
+ansible-test integration "${common[@]}" --coverage
+
+# run a unit test that generates code coverage using a venv
+ansible-test units "${common[@]}" --coverage --python "${ANSIBLE_TEST_PYTHON_VERSION}" --venv
 
 # report on code coverage in all supported formats
-ansible-test coverage report "${common[@]}"
+ansible-test coverage report "${common[@]}" | tee coverage.report
 ansible-test coverage html "${common[@]}"
 ansible-test coverage xml "${common[@]}"
+
+# ensure import test coverage was collected
+grep '^plugins/module_utils/test_util.py .* 100%$' coverage.report
+
+# ensure integration test coverage was collected
+grep '^tests/integration/targets/hello/world.py .* 100%$' coverage.report
+
+# ensure unit test coverage was collected
+grep '^tests/unit/test_something.py .* 100%$' coverage.report
+
+# ensure tests/output/ (from --venv) does not appear in the coverage report (except in the report filename)
+if grep "^tests/output/" coverage.report; then
+  echo "unexpected coverage output: tests/output/"
+  exit 1
+fi

test/lib/ansible_test/_internal/coverage_util.py

@@ -261,7 +261,7 @@ omit =
     */pyshared/*
     */pytest
     */AnsiballZ_*.py
-    */test/results/*
+    */test/results/.tmp/delegation/*
 """
 
     coverage_config = coverage_config.lstrip()
@@ -279,8 +279,8 @@ def generate_collection_coverage_config() -> str:
     ]
 
     omit_patterns = [
-        # {base}/ansible_collections/{ns}/{col}/tests/output/*
-        os.path.join(data_context().content.root, data_context().content.results_path, '*'),
+        # {base}/ansible_collections/{ns}/{col}/tests/output/.tmp/delegation/*
+        os.path.join(data_context().content.root, data_context().content.results_path, '.tmp/delegation/*'),
     ]
 
     include = textwrap.indent('\n'.join(include_patterns), ' ' * 4)