Enable codecov.io and add coverage grouping.

.coveragerc

@@ -20,5 +20,5 @@ data_file = test/results/coverage/coverage
 omit =
     */python*/dist-packages/*
     */python*/site-packages/*
-    */python*/distutils
+    */python*/distutils/*
     */pytest

.gitignore

@@ -68,8 +68,8 @@ packaging/release/ansible_release
 /.cache/
 /test/results/coverage/*=coverage.*
 /test/results/coverage/coverage*
-/test/results/reports/coverage.xml
+/test/results/reports/coverage*.xml
-/test/results/reports/coverage/
+/test/results/reports/coverage*/
 /test/results/bot/*.json
 /test/results/junit/*.xml
 /test/results/logs/*.log

test/runner/lib/cover.py

@@ -14,6 +14,7 @@ from lib.util import (
     ApplicationError,
     EnvironmentConfig,
     run_command,
+    common_environment,
 )
 
 from lib.executor import (
@@ -23,11 +24,13 @@ from lib.executor import (
 
 COVERAGE_DIR = 'test/results/coverage'
 COVERAGE_FILE = os.path.join(COVERAGE_DIR, 'coverage')
+COVERAGE_GROUPS = ('command', 'target', 'environment', 'version')
 
 
 def command_coverage_combine(args):
     """Patch paths in coverage files and merge into a single file.
     :type args: CoverageConfig
+    :rtype: list[str]
     """
     coverage = initialize_coverage(args)
 
@@ -35,12 +38,11 @@ def command_coverage_combine(args):
 
     coverage_files = [os.path.join(COVERAGE_DIR, f) for f in os.listdir(COVERAGE_DIR) if '=coverage.' in f]
 
-    arc_data = {}
-
     ansible_path = os.path.abspath('lib/ansible/') + '/'
     root_path = os.getcwd() + '/'
 
     counter = 0
+    groups = {}
 
     for coverage_file in coverage_files:
         counter += 1
@@ -48,6 +50,12 @@ def command_coverage_combine(args):
 
         original = coverage.CoverageData()
 
+        group = get_coverage_group(args, coverage_file)
+
+        if group is None:
+            display.warning('Unexpected name for coverage file: %s' % coverage_file)
+            continue
+
         if os.path.getsize(coverage_file) == 0:
             display.warning('Empty coverage file: %s' % coverage_file)
             continue
@@ -83,46 +91,77 @@ def command_coverage_combine(args):
                 display.info('%s -> %s' % (filename, new_name), verbosity=3)
                 filename = new_name
 
+            if group not in groups:
+                groups[group] = {}
+
+            arc_data = groups[group]
+
             if filename not in arc_data:
                 arc_data[filename] = set()
 
             arc_data[filename].update(arcs)
 
-    updated = coverage.CoverageData()
+    output_files = []
 
-    for filename in arc_data:
+    for group in sorted(groups):
-        if not os.path.isfile(filename):
+        arc_data = groups[group]
-            display.warning('Invalid coverage path: %s' % filename)
-            continue
 
-        updated.add_arcs({filename: list(arc_data[filename])})
+        updated = coverage.CoverageData()
 
-    if not args.explain:
+        for filename in arc_data:
-        updated.write_file(COVERAGE_FILE)
+            if not os.path.isfile(filename):
+                display.warning('Invalid coverage path: %s' % filename)
+                continue
+
+            updated.add_arcs({filename: list(arc_data[filename])})
+
+        if not args.explain:
+            output_file = COVERAGE_FILE + group
+            updated.write_file(output_file)
+            output_files.append(output_file)
+
+    return sorted(output_files)
 
 
 def command_coverage_report(args):
     """
     :type args: CoverageConfig
     """
-    command_coverage_combine(args)
+    output_files = command_coverage_combine(args)
-    run_command(args, ['coverage', 'report'])
+
+    for output_file in output_files:
+        if args.group_by:
+            display.info('>>> Coverage Group: %s' % ' '.join(os.path.basename(output_file).split('=')[1:]))
+
+        env = common_environment()
+        env.update(dict(COVERAGE_FILE=output_file))
+        run_command(args, env=env, cmd=['coverage', 'report'])
 
 
 def command_coverage_html(args):
     """
     :type args: CoverageConfig
     """
-    command_coverage_combine(args)
+    output_files = command_coverage_combine(args)
-    run_command(args, ['coverage', 'html', '-d', 'test/results/reports/coverage'])
+
+    for output_file in output_files:
+        dir_name = 'test/results/reports/%s' % os.path.basename(output_file)
+        env = common_environment()
+        env.update(dict(COVERAGE_FILE=output_file))
+        run_command(args, env=env, cmd=['coverage', 'html', '-d', dir_name])
 
 
 def command_coverage_xml(args):
     """
     :type args: CoverageConfig
     """
-    command_coverage_combine(args)
+    output_files = command_coverage_combine(args)
-    run_command(args, ['coverage', 'xml', '-o', 'test/results/reports/coverage.xml'])
+
+    for output_file in output_files:
+        xml_name = 'test/results/reports/%s.xml' % os.path.basename(output_file)
+        env = common_environment()
+        env.update(dict(COVERAGE_FILE=output_file))
+        run_command(args, env=env, cmd=['coverage', 'xml', '-o', xml_name])
 
 
 def command_coverage_erase(args):
@@ -163,6 +202,33 @@ def initialize_coverage(args):
     return coverage
 
 
+def get_coverage_group(args, coverage_file):
+    """
+    :type args: CoverageConfig
+    :type coverage_file: str
+    :rtype: str
+    """
+    parts = os.path.basename(coverage_file).split('=', 4)
+
+    if len(parts) != 5 or not parts[4].startswith('coverage.'):
+        return None
+
+    names = dict(
+        command=parts[0],
+        target=parts[1],
+        environment=parts[2],
+        version=parts[3],
+    )
+
+    group = ''
+
+    for part in COVERAGE_GROUPS:
+        if part in args.group_by:
+            group += '=%s' % names[part]
+
+    return group
+
+
 class CoverageConfig(EnvironmentConfig):
     """Configuration for the coverage command."""
     def __init__(self, args):
@@ -170,3 +236,5 @@ class CoverageConfig(EnvironmentConfig):
         :type args: any
         """
         super(CoverageConfig, self).__init__(args, 'coverage')
+
+        self.group_by = frozenset(args.group_by) if 'group_by' in args and args.group_by else set()  # type: frozenset[str]

test/runner/lib/executor.py

@@ -858,7 +858,7 @@ def intercept_command(args, cmd, target_name, capture=False, env=None, data=None
     version = python_version or args.python_version
     interpreter = find_executable('python%s' % version)
     coverage_file = os.path.abspath(os.path.join(inject_path, '..', 'output', '%s=%s=%s=%s=coverage' % (
-        args.command, target_name, args.coverage_label or 'local-%s' % version, version)))
+        args.command, target_name, args.coverage_label or 'local-%s' % version, 'python-%s' % version)))
 
     env['PATH'] = inject_path + os.pathsep + env['PATH']
     env['ANSIBLE_TEST_PYTHON_VERSION'] = version

test/runner/test.py

@@ -337,6 +337,8 @@ def parse_args():
     coverage_combine.set_defaults(func=lib.cover.command_coverage_combine,
                                   config=lib.cover.CoverageConfig)
 
+    add_extra_coverage_options(coverage_combine)
+
     coverage_erase = coverage_subparsers.add_parser('erase',
                                                     parents=[coverage_common],
                                                     help='erase coverage data files')
@@ -351,6 +353,8 @@ def parse_args():
     coverage_report.set_defaults(func=lib.cover.command_coverage_report,
                                  config=lib.cover.CoverageConfig)
 
+    add_extra_coverage_options(coverage_report)
+
     coverage_html = coverage_subparsers.add_parser('html',
                                                    parents=[coverage_common],
                                                    help='generate html coverage report')
@@ -358,6 +362,8 @@ def parse_args():
     coverage_html.set_defaults(func=lib.cover.command_coverage_html,
                                config=lib.cover.CoverageConfig)
 
+    add_extra_coverage_options(coverage_html)
+
     coverage_xml = coverage_subparsers.add_parser('xml',
                                                   parents=[coverage_common],
                                                   help='generate xml coverage report')
@@ -365,6 +371,8 @@ def parse_args():
     coverage_xml.set_defaults(func=lib.cover.command_coverage_xml,
                               config=lib.cover.CoverageConfig)
 
+    add_extra_coverage_options(coverage_xml)
+
     if argcomplete:
         argcomplete.autocomplete(parser, always_complete_options=False, validator=lambda i, k: True)
 
@@ -498,6 +506,17 @@ def add_environments(parser, tox_version=False, tox_only=False):
                         default='never')
 
 
+def add_extra_coverage_options(parser):
+    """
+    :type parser: argparse.ArgumentParser
+    """
+    parser.add_argument('--group-by',
+                        metavar='GROUP',
+                        action='append',
+                        choices=lib.cover.COVERAGE_GROUPS,
+                        help='group output by: %s' % ', '.join(lib.cover.COVERAGE_GROUPS))
+
+
 def add_extra_docker_options(parser, integration=True):
     """
     :type parser: argparse.ArgumentParser

test/utils/shippable/shippable.sh

@@ -52,8 +52,30 @@ find lib/ansible/modules -type d -empty -print -delete
 function cleanup
 {
     if find test/results/coverage/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then
-        ansible-test coverage xml --color -v --requirements
+        ansible-test coverage xml --color -v --requirements --group-by command --group-by version
-        cp -a test/results/reports/coverage.xml shippable/codecoverage/coverage.xml
+        cp -a test/results/reports/coverage=*.xml shippable/codecoverage/
+
+        # upload coverage report to codecov.io only when using complete on-demand coverage
+        if [ "${COVERAGE}" ] && [ "${CHANGED}" == "" ]; then
+            for file in test/results/reports/coverage=*.xml; do
+                flags="${file##*/coverage=}"
+                flags="${flags%.xml}"
+                flags="${flags//=/,}"
+                flags="${flags//[^a-zA-Z0-9_,]/_}"
+
+                bash <(curl -s https://codecov.io/bash) \
+                    -f "${file}" \
+                    -F "${flags}" \
+                    -n "${TEST}" \
+                    -t 83cd8957-dc76-488c-9ada-210dcea51633 \
+                    -X coveragepy \
+                    -X gcov \
+                    -X fix \
+                    -X search \
+                    -X xcode \
+                || echo "Failed to upload code coverage report to codecov.io: ${file}"
+            done
+        fi
     fi
 
     rmdir shippable/testresults/