From 4e7403e5d39e3d720bdb2f127684c71d56542591 Mon Sep 17 00:00:00 2001 From: Jordan Borean Date: Fri, 22 Nov 2024 13:32:39 +1000 Subject: [PATCH] ansible-test - fix coverage for test modules Fixes the coverage path translation for modules located in integration test paths. Instead of trying to match by the unique temporary path name that the module is executed as, the reporting tool will translate it to the static path that the module is actually located under. --- .../ansible-test-coverage-test-files.yml | 4 +++ .../ansible-test-coverage-windows/aliases | 3 +++ .../ns/col/plugins/modules/win_collection.ps1 | 6 +++++ .../library/test_win_collection.ps1 | 6 +++++ .../targets/win_collection/tasks/main.yml | 5 ++++ .../ansible-test-coverage-windows/runme.sh | 20 ++++++++++++++ .../test-coverage.py | 27 +++++++++++++++++++ test/integration/targets/collection/setup.sh | 7 +++++ .../_internal/commands/coverage/__init__.py | 10 +++---- 9 files changed, 83 insertions(+), 5 deletions(-) create mode 100644 changelogs/fragments/ansible-test-coverage-test-files.yml create mode 100644 test/integration/targets/ansible-test-coverage-windows/aliases create mode 100644 test/integration/targets/ansible-test-coverage-windows/ansible_collections/ns/col/plugins/modules/win_collection.ps1 create mode 100644 test/integration/targets/ansible-test-coverage-windows/ansible_collections/ns/col/tests/integration/targets/win_collection/library/test_win_collection.ps1 create mode 100644 test/integration/targets/ansible-test-coverage-windows/ansible_collections/ns/col/tests/integration/targets/win_collection/tasks/main.yml create mode 100755 test/integration/targets/ansible-test-coverage-windows/runme.sh create mode 100644 test/integration/targets/ansible-test-coverage-windows/test-coverage.py diff --git a/changelogs/fragments/ansible-test-coverage-test-files.yml b/changelogs/fragments/ansible-test-coverage-test-files.yml new file mode 100644 index 00000000000..28b35e6cc38 --- /dev/null +++ b/changelogs/fragments/ansible-test-coverage-test-files.yml @@ -0,0 +1,4 @@ +bugfixes: + - >- + ansible-test - Fix up coverage reporting to properly translate the temporary path of integration test modules to + the expected static test module path. diff --git a/test/integration/targets/ansible-test-coverage-windows/aliases b/test/integration/targets/ansible-test-coverage-windows/aliases new file mode 100644 index 00000000000..b7a6b165419 --- /dev/null +++ b/test/integration/targets/ansible-test-coverage-windows/aliases @@ -0,0 +1,3 @@ +shippable/windows/group1 +windows +needs/target/collection diff --git a/test/integration/targets/ansible-test-coverage-windows/ansible_collections/ns/col/plugins/modules/win_collection.ps1 b/test/integration/targets/ansible-test-coverage-windows/ansible_collections/ns/col/plugins/modules/win_collection.ps1 new file mode 100644 index 00000000000..53b2f2da3b3 --- /dev/null +++ b/test/integration/targets/ansible-test-coverage-windows/ansible_collections/ns/col/plugins/modules/win_collection.ps1 @@ -0,0 +1,6 @@ +#!powershell + +#AnsibleRequires -CSharpUtil Ansible.Basic + +$module = [Ansible.Basic.AnsibleModule]::Create($args, @{}) +$module.ExitJson() diff --git a/test/integration/targets/ansible-test-coverage-windows/ansible_collections/ns/col/tests/integration/targets/win_collection/library/test_win_collection.ps1 b/test/integration/targets/ansible-test-coverage-windows/ansible_collections/ns/col/tests/integration/targets/win_collection/library/test_win_collection.ps1 new file mode 100644 index 00000000000..53b2f2da3b3 --- /dev/null +++ b/test/integration/targets/ansible-test-coverage-windows/ansible_collections/ns/col/tests/integration/targets/win_collection/library/test_win_collection.ps1 @@ -0,0 +1,6 @@ +#!powershell + +#AnsibleRequires -CSharpUtil Ansible.Basic + +$module = [Ansible.Basic.AnsibleModule]::Create($args, @{}) +$module.ExitJson() diff --git a/test/integration/targets/ansible-test-coverage-windows/ansible_collections/ns/col/tests/integration/targets/win_collection/tasks/main.yml b/test/integration/targets/ansible-test-coverage-windows/ansible_collections/ns/col/tests/integration/targets/win_collection/tasks/main.yml new file mode 100644 index 00000000000..6196b768c6b --- /dev/null +++ b/test/integration/targets/ansible-test-coverage-windows/ansible_collections/ns/col/tests/integration/targets/win_collection/tasks/main.yml @@ -0,0 +1,5 @@ +- name: run module in collection to test coverage for collection plugins + win_collection: + +- name: run module in library adjacent to test coverage for test plugins + test_win_collection: diff --git a/test/integration/targets/ansible-test-coverage-windows/runme.sh b/test/integration/targets/ansible-test-coverage-windows/runme.sh new file mode 100755 index 00000000000..70593e017a6 --- /dev/null +++ b/test/integration/targets/ansible-test-coverage-windows/runme.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +TEST_PATH="${PWD}/test-coverage.py" + +source ../collection/setup.sh +cp "${INVENTORY_PATH}" tests/integration/inventory.winrm + +set -x + +# common args for all tests +common=(--venv --color --truncate 0 "${@}") + +# run command that generates coverage data for Windows +ansible-test windows-integration win_collection "${common[@]}" --coverage + +# report on code coverage in all supported formats +ansible-test coverage report "${common[@]}" + +# test we covered the 2 files we expect to have been covered and their lines +python "${TEST_PATH}" diff --git a/test/integration/targets/ansible-test-coverage-windows/test-coverage.py b/test/integration/targets/ansible-test-coverage-windows/test-coverage.py new file mode 100644 index 00000000000..98dbca7437c --- /dev/null +++ b/test/integration/targets/ansible-test-coverage-windows/test-coverage.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import json +import os +import os.path + + +def main() -> None: + collection_root = os.getcwd() + print(f"Running windows-integration coverage test in '{collection_root}'") + + result_path = os.path.join(collection_root, "tests", "output", "coverage", "coverage-powershell") + module_path = os.path.join(collection_root, "plugins", "modules", "win_collection.ps1") + test_path = os.path.join(collection_root, "tests", "integration", "targets", "win_collection", "library", "test_win_collection.ps1") + with open(result_path, mode="rb") as fd: + data = json.load(fd) + + for path, result in data.items(): + print(f"Testing result for path '{path}' -> {result!r}") + assert path in [module_path, test_path], f"Found unexpected coverage result path '{path}'" + assert result == {'5': 1, '6': 1}, "Coverage result did not pick up a hit on lines 5 and 6" + + assert len(data) == 2, f"Expected coverage results for 2 files but got {len(data)}" + + +if __name__ == '__main__': + main() diff --git a/test/integration/targets/collection/setup.sh b/test/integration/targets/collection/setup.sh index 74466555e1c..346a21ca9a2 100755 --- a/test/integration/targets/collection/setup.sh +++ b/test/integration/targets/collection/setup.sh @@ -12,6 +12,9 @@ # # 3) Sanity tests which are multi-version require an ignore entry per Python version. # This script replicates these ignore entries for each supported Python version based on the ignored path. +# +# 4) Windows tests need access to the ansible.windows vendored collection. +# This script copies any of the existing collections in ANSIBLE_COLLECTIONS_PATH to the temporary directory. set -eu -o pipefail @@ -26,4 +29,8 @@ trap 'rm -rf "${WORK_DIR}"' EXIT cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}" cd "${WORK_DIR}/ansible_collections/ns/${COLLECTION_NAME:-col}" +if [ "${ANSIBLE_COLLECTIONS_PATH:+set}" = "set" ]; then + cp -aL "${ANSIBLE_COLLECTIONS_PATH}"/ansible_collections/* "${WORK_DIR}/ansible_collections" +fi + "${TEST_DIR}/../collection/update-ignore.py" diff --git a/test/lib/ansible_test/_internal/commands/coverage/__init__.py b/test/lib/ansible_test/_internal/commands/coverage/__init__.py index c4c5f09e825..6c6e8cdabaa 100644 --- a/test/lib/ansible_test/_internal/commands/coverage/__init__.py +++ b/test/lib/ansible_test/_internal/commands/coverage/__init__.py @@ -293,6 +293,11 @@ def sanitize_filename( new_name = re.sub('^.*/ansible_modlib.zip/ansible/', ansible_path, filename) display.info('%s -> %s' % (filename, new_name), verbosity=3) filename = new_name + elif integration_temp_path in filename: + # Rewrite the path of code running from an integration test temporary directory. + new_name = re.sub(r'^.*' + re.escape(integration_temp_path) + '[^/]+/', root_path, filename) + display.info('%s -> %s' % (filename, new_name), verbosity=3) + filename = new_name elif collection_search_re and collection_search_re.search(filename): new_name = os.path.abspath(collection_sub_re.sub('', filename)) display.info('%s -> %s' % (filename, new_name), verbosity=3) @@ -328,11 +333,6 @@ def sanitize_filename( new_name = re.sub('^(/.*?)?/root/ansible/', root_path, filename) display.info('%s -> %s' % (filename, new_name), verbosity=3) filename = new_name - elif integration_temp_path in filename: - # Rewrite the path of code running from an integration test temporary directory. - new_name = re.sub(r'^.*' + re.escape(integration_temp_path) + '[^/]+/', root_path, filename) - display.info('%s -> %s' % (filename, new_name), verbosity=3) - filename = new_name filename = os.path.abspath(filename) # make sure path is absolute (will be relative if previously exported)