From 7b3f0b88701b08b05e9f850d329aaa6db98c9e4c Mon Sep 17 00:00:00 2001
From: Matt Clay <matt@mystile.com>
Date: Sun, 7 May 2017 16:32:19 +0800
Subject: [PATCH] Add support for on-demand coverage on Shippable.

Triggered by setting the COVERAGE environment var to a non-empty value.
Alternatively the string `ci_coverage` can be added to a commit message.
---
 test/utils/shippable/cloud.sh     | 2 +-
 test/utils/shippable/freebsd.sh   | 2 +-
 test/utils/shippable/linux.sh     | 2 +-
 test/utils/shippable/network.sh   | 4 ++--
 test/utils/shippable/osx.sh       | 3 ++-
 test/utils/shippable/other.sh     | 4 ++--
 test/utils/shippable/shippable.sh | 6 ++++++
 test/utils/shippable/windows.sh   | 4 ++--
 8 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/test/utils/shippable/cloud.sh b/test/utils/shippable/cloud.sh
index 37535cf5e7b..9d101db56a8 100755
--- a/test/utils/shippable/cloud.sh
+++ b/test/utils/shippable/cloud.sh
@@ -8,4 +8,4 @@ IFS='/:' read -ra args <<< "${TEST}"
 image="ansible/ansible:${args[1]}"
 target="posix/ci/cloud/"
 
-ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}"
+ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}" "${COVERAGE}"
diff --git a/test/utils/shippable/freebsd.sh b/test/utils/shippable/freebsd.sh
index 5af7eab052b..ded7ca1f582 100755
--- a/test/utils/shippable/freebsd.sh
+++ b/test/utils/shippable/freebsd.sh
@@ -9,4 +9,4 @@ platform="${args[0]}"
 version="${args[1]}"
 target="posix/ci/"
 
-ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --exclude "posix/ci/cloud/"
+ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --exclude "posix/ci/cloud/" "${COVERAGE}"
diff --git a/test/utils/shippable/linux.sh b/test/utils/shippable/linux.sh
index 1d98735c7e9..9375e0699ee 100755
--- a/test/utils/shippable/linux.sh
+++ b/test/utils/shippable/linux.sh
@@ -8,4 +8,4 @@ IFS='/:' read -ra args <<< "${TEST}"
 image="ansible/ansible:${args[1]}"
 target="posix/ci/group${args[2]}/"
 
-ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}"
+ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}" "${COVERAGE}"
diff --git a/test/utils/shippable/network.sh b/test/utils/shippable/network.sh
index 57892bc4b3f..41fb6cbe6a9 100755
--- a/test/utils/shippable/network.sh
+++ b/test/utils/shippable/network.sh
@@ -12,7 +12,7 @@ if [ -s /tmp/network.txt ]; then
 
     echo "Running network integration tests for multiple platforms concurrently."
 
-    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements \
+    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements "${COVERAGE}" \
         --platform vyos/1.1.0 \
         --platform ios/csr1000v \
 
@@ -20,6 +20,6 @@ else
     echo "No changes requiring integration tests specific to networking were detected."
     echo "Running network integration tests for a single platform only."
 
-    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements \
+    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements "${COVERAGE}" \
         --platform vyos/1.1.0
 fi
diff --git a/test/utils/shippable/osx.sh b/test/utils/shippable/osx.sh
index 5e6c18c91f7..abdc3db6e75 100755
--- a/test/utils/shippable/osx.sh
+++ b/test/utils/shippable/osx.sh
@@ -9,4 +9,5 @@ platform="${args[0]}"
 version="${args[1]}"
 target="posix/ci/"
 
-ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --remote-terminate success --exclude "posix/ci/cloud/"
+ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --remote-terminate success --exclude "posix/ci/cloud/" \
+    "${COVERAGE}"
diff --git a/test/utils/shippable/other.sh b/test/utils/shippable/other.sh
index 3af08119135..3ca5b6a53ca 100755
--- a/test/utils/shippable/other.sh
+++ b/test/utils/shippable/other.sh
@@ -10,8 +10,8 @@ retry.py pip install tox --disable-pip-version-check
 
 echo '{"verified": false, "results": []}' > test/results/bot/ansible-test-failure.json
 
-ansible-test compile --failure-ok --color -v --junit --requirements
-ansible-test sanity  --failure-ok --color -v --junit --tox --skip-test ansible-doc --python 3.5
+ansible-test compile --failure-ok --color -v --junit --requirements --coverage
+ansible-test sanity  --failure-ok --color -v --junit --tox --skip-test ansible-doc --python 3.5 --coverage
 ansible-test sanity  --failure-ok --color -v --junit --tox --test ansible-doc --coverage
 
 rm test/results/bot/ansible-test-failure.json
diff --git a/test/utils/shippable/shippable.sh b/test/utils/shippable/shippable.sh
index 4dde3766dc1..516c2dc8812 100755
--- a/test/utils/shippable/shippable.sh
+++ b/test/utils/shippable/shippable.sh
@@ -23,6 +23,12 @@ pip list --disable-pip-version-check
 
 export PATH="test/runner:${PATH}"
 export PYTHONIOENCODING='utf-8'
+export COVERAGE="${COVERAGE:-}"
+
+# run integration coverage if 'ci_coverage' is in the commit message or the COVERAGE var is non-empty
+if [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]] || [ -n "${COVERAGE}" ]; then
+    export COVERAGE="--coverage"
+fi
 
 # remove empty core/extras module directories from PRs created prior to the repo-merge
 find lib/ansible/modules -type d -empty -print -delete
diff --git a/test/utils/shippable/windows.sh b/test/utils/shippable/windows.sh
index 7ac724c8ce0..0c7a0dd8ccb 100755
--- a/test/utils/shippable/windows.sh
+++ b/test/utils/shippable/windows.sh
@@ -22,7 +22,7 @@ if [ -s /tmp/windows.txt ]; then
 
     target="windows/ci/"
 
-    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements \
+    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements "${COVERAGE}" \
         --windows 2008-SP2 \
         --windows 2008-R2_SP1 \
         --windows 2012-RTM \
@@ -34,6 +34,6 @@ else
 
     target="windows/ci/group${job}/"
 
-    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements \
+    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements "${COVERAGE}" \
         --windows 2012-R2_RTM
 fi