[stable-2.9] Fix ansible-test coverage path handling. (#61528) (#61593)

* [stable-2.9] Fix ansible-test coverage path handling. (#61528)

* Fix ansible-test coverage path handling.
* Split CI unit tests into two groups.
(cherry picked from commit e4e5005640)

Co-authored-by: Matt Clay <matt@mystile.com>

* Add changelog fragment.

changelogs/fragments/ansible-test-powershell-coverage-fix.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - ansible-test now creates output directories as needed for powershell coverage output before generating reports

shippable.yml

@@ -14,12 +14,19 @@ matrix:
     - env: T=sanity/4
     - env: T=sanity/5
 
-    - env: T=units/2.6
-    - env: T=units/2.7
-    - env: T=units/3.5
-    - env: T=units/3.6
-    - env: T=units/3.7
-    - env: T=units/3.8
+    - env: T=units/2.6/1
+    - env: T=units/2.7/1
+    - env: T=units/3.5/1
+    - env: T=units/3.6/1
+    - env: T=units/3.7/1
+    - env: T=units/3.8/1
+
+    - env: T=units/2.6/2
+    - env: T=units/2.7/2
+    - env: T=units/3.5/2
+    - env: T=units/3.6/2
+    - env: T=units/3.7/2
+    - env: T=units/3.8/2
 
     - env: T=windows/2008/1
     - env: T=windows/2008-R2/1

test/lib/ansible_test/_internal/cover.py

@@ -31,13 +31,15 @@ from .util import (
     ApplicationError,
     common_environment,
     ANSIBLE_TEST_DATA_ROOT,
-    to_bytes,
     to_text,
+    make_dirs,
 )
 
 from .util_common import (
     intercept_command,
     ResultType,
+    write_text_test_results,
+    write_json_test_results,
 )
 
 from .config import (
@@ -68,7 +70,12 @@ def command_coverage_combine(args):
     :type args: CoverageConfig
     :rtype: list[str]
     """
-    return _command_coverage_combine_powershell(args) + _command_coverage_combine_python(args)
+    paths = _command_coverage_combine_powershell(args) + _command_coverage_combine_python(args)
+
+    for path in paths:
+        display.info('Generated combined output: %s' % path, verbosity=1)
+
+    return paths
 
 
 def _command_coverage_combine_python(args):
@@ -342,6 +349,7 @@ def command_coverage_html(args):
             continue
 
         dir_name = os.path.join(ResultType.REPORTS.path, os.path.basename(output_file))
+        make_dirs(dir_name)
         run_coverage(args, output_file, 'html', ['-i', '-d', dir_name])
 
         display.info('HTML report generated: file:///%s' % os.path.join(dir_name, 'index.html'))
@@ -354,7 +362,7 @@ def command_coverage_xml(args):
     output_files = command_coverage_combine(args)
 
     for output_file in output_files:
-        xml_name = os.path.join(ResultType.REPORTS.path, '%s.xml' % os.path.basename(output_file))
+        xml_name = '%s.xml' % os.path.basename(output_file)
         if output_file.endswith('-powershell'):
             report = _generage_powershell_xml(output_file)
 
@@ -362,10 +370,11 @@ def command_coverage_xml(args):
             reparsed = minidom.parseString(rough_string)
             pretty = reparsed.toprettyxml(indent='    ')
 
-            with open(xml_name, 'w') as xml_fd:
-                xml_fd.write(pretty)
+            write_text_test_results(ResultType.REPORTS, xml_name, pretty)
         else:
-            run_coverage(args, output_file, 'xml', ['-i', '-o', xml_name])
+            xml_path = os.path.join(ResultType.REPORTS.path, xml_name)
+            make_dirs(ResultType.REPORTS.path)
+            run_coverage(args, output_file, 'xml', ['-i', '-o', xml_path])
 
 
 def command_coverage_erase(args):
@@ -504,8 +513,6 @@ def _command_coverage_combine_powershell(args):
     invalid_path_count = 0
     invalid_path_chars = 0
 
-    coverage_file = os.path.join(ResultType.COVERAGE.path, COVERAGE_OUTPUT_FILE_NAME)
-
     for group in sorted(groups):
         coverage_data = groups[group]
 
@@ -528,11 +535,11 @@ def _command_coverage_combine_powershell(args):
                 coverage_data[source] = _default_stub_value(source_line_count)
 
         if not args.explain:
-            output_file = coverage_file + group + '-powershell'
-            with open(output_file, 'wb') as output_file_fd:
-                output_file_fd.write(to_bytes(json.dumps(coverage_data)))
+            output_file = COVERAGE_OUTPUT_FILE_NAME + group + '-powershell'
 
-            output_files.append(output_file)
+            write_json_test_results(ResultType.COVERAGE, output_file, coverage_data)
+
+            output_files.append(os.path.join(ResultType.COVERAGE.path, output_file))
 
     if invalid_path_count > 0:
         display.warning(

test/utils/shippable/units.sh

@@ -6,14 +6,55 @@ declare -a args
 IFS='/:' read -ra args <<< "$1"
 
 version="${args[1]}"
+group="${args[2]}"
 
 if [[ "${COVERAGE:-}" == "--coverage" ]]; then
-    timeout=99
+    timeout=60
 else
-    timeout=11
+    timeout=6
 fi
 
+group1=()
+group2=()
+
+# create two groups by putting long running network tests into one group
+# add or remove more network platforms as needed to balance the two groups
+
+networks=(
+    f5
+    fortimanager
+    fortios
+    ios
+    junos
+    netact
+    netscaler
+    netvisor
+    nos
+    nso
+    nuage
+    nxos
+    onyx
+    opx
+    ovs
+    radware
+    routeros
+    slxos
+    voss
+    vyos
+)
+
+for network in "${networks[@]}"; do
+    group1+=(--exclude "test/units/modules/network/${network}/")
+    group2+=("test/units/modules/network/${network}/")
+done
+
+case "${group}" in
+    1) options=("${group1[@]}") ;;
+    2) options=("${group2[@]}") ;;
+esac
+
 ansible-test env --timeout "${timeout}" --color -v
 
 # shellcheck disable=SC2086
 ansible-test units --color -v --docker default --python "${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+    "${options[@]}" \