Add --all and --stub coverage options.

ci_coverage
ci_complete

test/runner/lib/cover.py

@@ -7,6 +7,7 @@ import re
 
 from lib.target import (
     walk_module_targets,
+    walk_compile_targets,
 )
 
 from lib.util import (
@@ -44,6 +45,14 @@ def command_coverage_combine(args):
     counter = 0
     groups = {}
 
+    if args.all or args.stub:
+        sources = sorted(os.path.abspath(target.path) for target in walk_compile_targets())
+    else:
+        sources = []
+
+    if args.stub:
+        groups['=stub'] = dict((source, set()) for source in sources)
+
     for coverage_file in coverage_files:
         counter += 1
         display.info('[%4d/%4d] %s' % (counter, len(coverage_files), coverage_file), verbosity=2)
@@ -115,6 +124,9 @@ def command_coverage_combine(args):
 
             updated.add_arcs({filename: list(arc_data[filename])})
 
+        if args.all:
+            updated.add_arcs(dict((source, []) for source in sources))
+
         if not args.explain:
             output_file = COVERAGE_FILE + group
             updated.write_file(output_file)
@@ -130,7 +142,7 @@ def command_coverage_report(args):
     output_files = command_coverage_combine(args)
 
     for output_file in output_files:
-        if args.group_by:
+        if args.group_by or args.stub:
             display.info('>>> Coverage Group: %s' % ' '.join(os.path.basename(output_file).split('=')[1:]))
 
         env = common_environment()
@@ -238,3 +250,5 @@ class CoverageConfig(EnvironmentConfig):
         super(CoverageConfig, self).__init__(args, 'coverage')
 
         self.group_by = frozenset(args.group_by) if 'group_by' in args and args.group_by else set()  # type: frozenset[str]
+        self.all = args.all if 'all' in args else False  # type: bool
+        self.stub = args.stub if 'stub' in args else False  # type: bool

test/runner/test.py

@@ -516,6 +516,14 @@ def add_extra_coverage_options(parser):
                         choices=lib.cover.COVERAGE_GROUPS,
                         help='group output by: %s' % ', '.join(lib.cover.COVERAGE_GROUPS))
 
+    parser.add_argument('--all',
+                        action='store_true',
+                        help='include all python source files')
+
+    parser.add_argument('--stub',
+                        action='store_true',
+                        help='generate empty report of all python source files')
+
 
 def add_extra_docker_options(parser, integration=True):
     """

test/utils/shippable/shippable.sh

@@ -52,7 +52,15 @@ find lib/ansible/modules -type d -empty -print -delete
 function cleanup
 {
     if find test/results/coverage/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then
-        ansible-test coverage xml --color -v --requirements --group-by command --group-by version
+        # for complete on-demand coverage generate a report for all files with no coverage on the "other" job so we only have one copy
+        if [ "${COVERAGE}" ] && [ "${CHANGED}" == "" ] && [ "${TEST}" == "other" ]; then
+            stub="--stub"
+        else
+            stub=""
+        fi
+
+        # shellcheck disable=SC2086
+        ansible-test coverage xml --color -v --requirements --group-by command --group-by version ${stub:+"$stub"}
         cp -a test/results/reports/coverage=*.xml shippable/codecoverage/
 
         # upload coverage report to codecov.io only when using complete on-demand coverage