From 887ab35656b3aa235a38004f9889a09397dae9c5 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Mon, 11 Mar 2019 15:13:09 -0700 Subject: [PATCH] Add support for ansible-test --coverage-check. (#53573) --- lib/ansible/executor/module_common.py | 34 +++++++++++++++++++++------ test/runner/lib/cli.py | 4 ++++ test/runner/lib/config.py | 4 ++++ test/runner/lib/util.py | 3 +++ test/units/conftest.py | 5 ++++ test/utils/shippable/network.sh | 2 +- test/utils/shippable/shippable.sh | 8 +++---- test/utils/shippable/units.sh | 2 +- 8 files changed, 49 insertions(+), 13 deletions(-) diff --git a/lib/ansible/executor/module_common.py b/lib/ansible/executor/module_common.py index d4266464fa2..021ea5349b5 100644 --- a/lib/ansible/executor/module_common.py +++ b/lib/ansible/executor/module_common.py @@ -339,7 +339,12 @@ ANSIBALLZ_COVERAGE_TEMPLATE = ''' os.environ['COVERAGE_FILE'] = '%(coverage_output)s' import atexit - import coverage + + try: + import coverage + except ImportError: + print('{"msg": "Could not import `coverage` module.", "failed": true}') + sys.exit(1) cov = coverage.Coverage(config_file='%(coverage_config)s') @@ -352,6 +357,14 @@ ANSIBALLZ_COVERAGE_TEMPLATE = ''' cov.start() ''' +ANSIBALLZ_COVERAGE_CHECK_TEMPLATE = ''' + try: + imp.find_module('coverage') + except ImportError: + print('{"msg": "Could not find `coverage` module.", "failed": true}') + sys.exit(1) +''' + ANSIBALLZ_RLIMIT_TEMPLATE = ''' import resource @@ -829,12 +842,19 @@ def _find_module_utils(module_name, b_module_data, module_path, module_args, tas coverage_config = os.environ.get('_ANSIBLE_COVERAGE_CONFIG') if coverage_config: - # Enable code coverage analysis of the module. - # This feature is for internal testing and may change without notice. - coverage = ANSIBALLZ_COVERAGE_TEMPLATE % dict( - coverage_config=coverage_config, - coverage_output=os.environ['_ANSIBLE_COVERAGE_OUTPUT'] - ) + coverage_output = os.environ['_ANSIBLE_COVERAGE_OUTPUT'] + + if coverage_output: + # Enable code coverage analysis of the module. + # This feature is for internal testing and may change without notice. + coverage = ANSIBALLZ_COVERAGE_TEMPLATE % dict( + coverage_config=coverage_config, + coverage_output=coverage_output, + ) + else: + # Verify coverage is available without importing it. + # This will detect when a module would fail with coverage enabled with minimal overhead. + coverage = ANSIBALLZ_COVERAGE_CHECK_TEMPLATE else: coverage = '' diff --git a/test/runner/lib/cli.py b/test/runner/lib/cli.py index 6d6e5653b89..577495c4088 100644 --- a/test/runner/lib/cli.py +++ b/test/runner/lib/cli.py @@ -209,6 +209,10 @@ def parse_args(): default='', help='label to include in coverage output file names') + test.add_argument('--coverage-check', + action='store_true', + help='only verify code coverage can be enabled') + test.add_argument('--metadata', help=argparse.SUPPRESS) diff --git a/test/runner/lib/config.py b/test/runner/lib/config.py index a4c3ff09c84..324bcdccb9d 100644 --- a/test/runner/lib/config.py +++ b/test/runner/lib/config.py @@ -104,6 +104,7 @@ class TestConfig(EnvironmentConfig): self.coverage = args.coverage # type: bool self.coverage_label = args.coverage_label # type: str + self.coverage_check = args.coverage_check # type: bool self.include = args.include or [] # type: list [str] self.exclude = args.exclude or [] # type: list [str] self.require = args.require or [] # type: list [str] @@ -124,6 +125,9 @@ class TestConfig(EnvironmentConfig): self.metadata = Metadata.from_file(args.metadata) if args.metadata else Metadata() self.metadata_path = None + if self.coverage_check: + self.coverage = True + class ShellConfig(EnvironmentConfig): """Configuration for the shell command.""" diff --git a/test/runner/lib/util.py b/test/runner/lib/util.py index 75166fe352a..5893a6fcde0 100644 --- a/test/runner/lib/util.py +++ b/test/runner/lib/util.py @@ -211,6 +211,9 @@ def intercept_command(args, cmd, target_name, capture=False, env=None, data=None coverage_file = os.path.abspath(os.path.join(inject_path, '..', 'output', '%s=%s=%s=%s=coverage' % ( args.command, target_name, args.coverage_label or 'local-%s' % version, 'python-%s' % version))) + if args.coverage_check: + coverage_file = '' + env['PATH'] = inject_path + os.path.pathsep + env['PATH'] env['ANSIBLE_TEST_PYTHON_VERSION'] = version env['ANSIBLE_TEST_PYTHON_INTERPRETER'] = interpreter diff --git a/test/units/conftest.py b/test/units/conftest.py index c7163201d0e..0fb7955081f 100644 --- a/test/units/conftest.py +++ b/test/units/conftest.py @@ -29,6 +29,11 @@ def pytest_configure(): if not coverage_config: return + coverage_output = os.environ.get('_ANSIBLE_COVERAGE_OUTPUT') + + if not coverage_output: + return + cov = coverage.Coverage(config_file=coverage_config) coverage_instances.append(cov) else: diff --git a/test/utils/shippable/network.sh b/test/utils/shippable/network.sh index b63e7b3917a..51093feba4c 100755 --- a/test/utils/shippable/network.sh +++ b/test/utils/shippable/network.sh @@ -6,7 +6,7 @@ set -o pipefail -eux ansible-test network-integration --explain ${CHANGED:+"$CHANGED"} ${UNSTABLE:+"$UNSTABLE"} 2>&1 \ | { grep ' network-integration: .* (targeted)$' || true; } > /tmp/network.txt -if [ "${COVERAGE}" ]; then +if [ "${COVERAGE}" == "--coverage" ]; then # when on-demand coverage is enabled, force tests to run for all network platforms echo "coverage" > /tmp/network.txt fi diff --git a/test/utils/shippable/shippable.sh b/test/utils/shippable/shippable.sh index 22ff3c0cce1..8e90ab90951 100755 --- a/test/utils/shippable/shippable.sh +++ b/test/utils/shippable/shippable.sh @@ -46,7 +46,7 @@ elif [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]]; then export COVERAGE="--coverage" else # on-demand coverage reporting disabled (default behavior, always-on coverage reporting remains enabled) - export COVERAGE="" + export COVERAGE="--coverage-check" fi if [ -n "${COMPLETE:-}" ]; then @@ -75,7 +75,7 @@ function cleanup { if find test/results/coverage/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then # for complete on-demand coverage generate a report for all files with no coverage on the "other" job so we only have one copy - if [ "${COVERAGE}" ] && [ "${CHANGED}" == "" ] && [ "${test}" == "sanity/1" ]; then + if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ] && [ "${test}" == "sanity/1" ]; then stub="--stub" else stub="" @@ -86,7 +86,7 @@ function cleanup cp -a test/results/reports/coverage=*.xml shippable/codecoverage/ # upload coverage report to codecov.io only when using complete on-demand coverage - if [ "${COVERAGE}" ] && [ "${CHANGED}" == "" ]; then + if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ]; then for file in test/results/reports/coverage=*.xml; do flags="${file##*/coverage=}" flags="${flags%.xml}" @@ -116,7 +116,7 @@ function cleanup trap cleanup EXIT -if [[ "${COVERAGE:-}" ]]; then +if [[ "${COVERAGE:-}" == "--coverage" ]]; then timeout=60 else timeout=45 diff --git a/test/utils/shippable/units.sh b/test/utils/shippable/units.sh index 11d1a792cab..89712811c65 100755 --- a/test/utils/shippable/units.sh +++ b/test/utils/shippable/units.sh @@ -7,7 +7,7 @@ IFS='/:' read -ra args <<< "$1" version="${args[1]}" -if [[ "${COVERAGE:-}" ]]; then +if [[ "${COVERAGE:-}" == "--coverage" ]]; then timeout=90 else timeout=10