Add support for on-demand coverage on Shippable.

Triggered by setting the COVERAGE environment var to a non-empty value.
Alternatively the string `ci_coverage` can be added to a commit message.

test/utils/shippable/cloud.sh

@@ -8,4 +8,4 @@ IFS='/:' read -ra args <<< "${TEST}"
 image="ansible/ansible:${args[1]}"
 target="posix/ci/cloud/"
 
-ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}"
+ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}" "${COVERAGE}"

test/utils/shippable/freebsd.sh

@@ -9,4 +9,4 @@ platform="${args[0]}"
 version="${args[1]}"
 target="posix/ci/"
 
-ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --exclude "posix/ci/cloud/"
+ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --exclude "posix/ci/cloud/" "${COVERAGE}"

test/utils/shippable/linux.sh

@@ -8,4 +8,4 @@ IFS='/:' read -ra args <<< "${TEST}"
 image="ansible/ansible:${args[1]}"
 target="posix/ci/group${args[2]}/"
 
-ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}"
+ansible-test integration --color -v --retry-on-error "${target}" --docker "${image}" "${COVERAGE}"

test/utils/shippable/network.sh

@@ -12,7 +12,7 @@ if [ -s /tmp/network.txt ]; then
 
     echo "Running network integration tests for multiple platforms concurrently."
 
-    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements \
+    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements "${COVERAGE}" \
         --platform vyos/1.1.0 \
         --platform ios/csr1000v \
 
@@ -20,6 +20,6 @@ else
     echo "No changes requiring integration tests specific to networking were detected."
     echo "Running network integration tests for a single platform only."
 
-    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements \
+    ansible-test network-integration --color -v --retry-on-error "${target}" --requirements "${COVERAGE}" \
         --platform vyos/1.1.0
 fi

test/utils/shippable/osx.sh

@@ -9,4 +9,5 @@ platform="${args[0]}"
 version="${args[1]}"
 target="posix/ci/"
 
-ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --remote-terminate success --exclude "posix/ci/cloud/"
+ansible-test integration --color -v --retry-on-error "${target}" --remote "${platform}/${version}" --remote-terminate success --exclude "posix/ci/cloud/" \
+    "${COVERAGE}"

test/utils/shippable/other.sh

@@ -10,8 +10,8 @@ retry.py pip install tox --disable-pip-version-check
 
 echo '{"verified": false, "results": []}' > test/results/bot/ansible-test-failure.json
 
-ansible-test compile --failure-ok --color -v --junit --requirements
-ansible-test sanity  --failure-ok --color -v --junit --tox --skip-test ansible-doc --python 3.5
+ansible-test compile --failure-ok --color -v --junit --requirements --coverage
+ansible-test sanity  --failure-ok --color -v --junit --tox --skip-test ansible-doc --python 3.5 --coverage
 ansible-test sanity  --failure-ok --color -v --junit --tox --test ansible-doc --coverage
 
 rm test/results/bot/ansible-test-failure.json

test/utils/shippable/shippable.sh

@@ -23,6 +23,12 @@ pip list --disable-pip-version-check
 
 export PATH="test/runner:${PATH}"
 export PYTHONIOENCODING='utf-8'
+export COVERAGE="${COVERAGE:-}"
+
+# run integration coverage if 'ci_coverage' is in the commit message or the COVERAGE var is non-empty
+if [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]] || [ -n "${COVERAGE}" ]; then
+    export COVERAGE="--coverage"
+fi
 
 # remove empty core/extras module directories from PRs created prior to the repo-merge
 find lib/ansible/modules -type d -empty -print -delete

test/utils/shippable/windows.sh

@@ -22,7 +22,7 @@ if [ -s /tmp/windows.txt ]; then
 
     target="windows/ci/"
 
-    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements \
+    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements "${COVERAGE}" \
         --windows 2008-SP2 \
         --windows 2008-R2_SP1 \
         --windows 2012-RTM \
@@ -34,6 +34,6 @@ else
 
     target="windows/ci/group${job}/"
 
-    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements \
+    ansible-test windows-integration --color -v --retry-on-error "${target}" --requirements "${COVERAGE}" \
         --windows 2012-R2_RTM
 fi