quiet default ansible-doc integration test output (#79921)

* quiet default ansible-doc integration test output 

* typical non-verbose output was exceeding 27k lines per fun
* disables `set -x` unless `-v` is passed to the script (eg when ansible-test is called with `-v` or `--retry-on-error` adds it on the second try
* added simple progress echoes
* suppress some grep output

* only enable `set -x` for >= `-vvv`

* fix shellcheck default complaint

test/integration/targets/ansible-doc/runme.sh

@@ -1,36 +1,60 @@
 #!/usr/bin/env bash
 
-set -eux
+# always set sane error behaviors, enable execution tracing later if sufficient verbosity requested
+set -eu
+
+verbosity=0
+
+# default to silent output for naked grep; -vvv+ will adjust this
+export GREP_OPTS=-q
+
+# shell tracing output is very large from this script; only enable if >= -vvv was passed
+while getopts :v opt
+do  case "$opt" in
+      v) ((verbosity+=1)) ;;
+      *) ;;
+    esac
+done
+
+if (( verbosity >= 3 ));
+then
+  set -x;
+  export GREP_OPTS= ;
+fi
+
+echo "running playbook-backed docs tests"
 ansible-playbook test.yml -i inventory "$@"
 
 # test keyword docs
-ansible-doc -t keyword -l | grep 'vars_prompt: list of variables to prompt for.'
+ansible-doc -t keyword -l | grep $GREP_OPTS 'vars_prompt: list of variables to prompt for.'
-ansible-doc -t keyword vars_prompt | grep 'description: list of variables to prompt for.'
+ansible-doc -t keyword vars_prompt | grep $GREP_OPTS 'description: list of variables to prompt for.'
-ansible-doc -t keyword asldkfjaslidfhals 2>&1 | grep 'Skipping Invalid keyword'
+ansible-doc -t keyword asldkfjaslidfhals 2>&1 | grep $GREP_OPTS 'Skipping Invalid keyword'
 
 # collections testing
 (
 unset ANSIBLE_PLAYBOOK_DIR
 cd "$(dirname "$0")"
 
-# test module docs from collection
+
+echo "test fakemodule docs from collection"
 # we use sed to strip the module path from the first line
 current_out="$(ansible-doc --playbook-dir ./ testns.testcol.fakemodule | sed '1 s/\(^> TESTNS\.TESTCOL\.FAKEMODULE\).*(.*)$/\1/')"
 expected_out="$(sed '1 s/\(^> TESTNS\.TESTCOL\.FAKEMODULE\).*(.*)$/\1/' fakemodule.output)"
 test "$current_out" == "$expected_out"
 
+echo "test randommodule docs from collection"
 # we use sed to strip the module path from the first line
 current_out="$(ansible-doc --playbook-dir ./ testns.testcol.randommodule | sed '1 s/\(^> TESTNS\.TESTCOL\.RANDOMMODULE\).*(.*)$/\1/')"
 expected_out="$(sed '1 s/\(^> TESTNS\.TESTCOL\.RANDOMMODULE\).*(.*)$/\1/' randommodule-text.output)"
 test "$current_out" == "$expected_out"
 
-# ensure we do work with valid collection name for list
+echo "ensure we do work with valid collection name for list"
-ansible-doc --list testns.testcol --playbook-dir ./ 2>&1 | grep -v "Invalid collection name"
+ansible-doc --list testns.testcol --playbook-dir ./ 2>&1 | grep $GREP_OPTS -v "Invalid collection name"
 
-# ensure we dont break on invalid collection name for list
+echo "ensure we dont break on invalid collection name for list"
-ansible-doc --list testns.testcol.fakemodule  --playbook-dir ./ 2>&1 | grep "Invalid collection name"
+ansible-doc --list testns.testcol.fakemodule  --playbook-dir ./ 2>&1 | grep $GREP_OPTS "Invalid collection name"
 
-# test listing diff plugin types from collection
+echo "testing ansible-doc output for various plugin types"
 for ptype in cache inventory lookup vars filter module
 do
 	# each plugin type adds 1 from collection
@@ -50,20 +74,20 @@ do
                 elif [ "${ptype}" == "lookup" ]; then expected_names=("noop");
                 elif [ "${ptype}" == "vars" ]; then expected_names=("noop_vars_plugin"); fi
 	fi
-	# ensure we ONLY list from the collection
+	echo "testing collection-filtered list for plugin ${ptype}"
 	justcol=$(ansible-doc -l -t ${ptype} --playbook-dir ./ testns.testcol|wc -l)
 	test "$justcol" -eq "$expected"
 
-	# ensure the right names are displayed
+	echo "validate collection plugin name display for plugin ${ptype}"
 	list_result=$(ansible-doc -l -t ${ptype} --playbook-dir ./ testns.testcol)
 	metadata_result=$(ansible-doc --metadata-dump --no-fail-on-errors -t ${ptype} --playbook-dir ./ testns.testcol)
 	for name in "${expected_names[@]}"; do
-		echo "${list_result}" | grep "testns.testcol.${name}"
+		echo "${list_result}" | grep $GREP_OPTS "testns.testcol.${name}"
-		echo "${metadata_result}" | grep "testns.testcol.${name}"
+		echo "${metadata_result}" | grep $GREP_OPTS "testns.testcol.${name}"
 	done
 
 	# ensure we get error if passing invalid collection, much less any plugins
-	ansible-doc -l -t ${ptype} testns.testcol  2>&1 | grep "unable to locate collection"
+	ansible-doc -l -t ${ptype} bogus.boguscoll  2>&1 | grep $GREP_OPTS "unable to locate collection"
 
 	# TODO: do we want per namespace?
 	# ensure we get 1 plugins when restricting namespace
@@ -73,20 +97,23 @@ done
 
 #### test role functionality
 
-# Test role text output
+echo "testing role text output"
 # we use sed to strip the role path from the first line
 current_role_out="$(ansible-doc -t role -r ./roles test_role1 | sed '1 s/\(^> TEST_ROLE1\).*(.*)$/\1/')"
 expected_role_out="$(sed '1 s/\(^> TEST_ROLE1\).*(.*)$/\1/' fakerole.output)"
 test "$current_role_out" == "$expected_role_out"
 
+echo "testing multiple role entrypoints"
 # Two collection roles are defined, but only 1 has a role arg spec with 2 entry points
 output=$(ansible-doc -t role -l --playbook-dir . testns.testcol | wc -l)
 test "$output" -eq 2
 
+echo "testing standalone roles"
 # Include normal roles (no collection filter)
 output=$(ansible-doc -t role -l --playbook-dir . | wc -l)
 test "$output" -eq 3
 
+echo "testing role precedence"
 # Test that a role in the playbook dir with the same name as a role in the
 # 'roles' subdir of the playbook dir does not appear (lower precedence).
 output=$(ansible-doc -t role -l --playbook-dir . | grep -c "test_role1 from roles subdir")
@@ -94,7 +121,7 @@ test "$output" -eq 1
 output=$(ansible-doc -t role -l --playbook-dir . | grep -c "test_role1 from playbook dir" || true)
 test "$output" -eq 0
 
-# Test entry point filter
+echo "testing role entrypoint filter"
 current_role_out="$(ansible-doc -t role --playbook-dir . testns.testcol.testrole -e alternate| sed '1 s/\(^> TESTNS\.TESTCOL\.TESTROLE\).*(.*)$/\1/')"
 expected_role_out="$(sed '1 s/\(^> TESTNS\.TESTCOL\.TESTROLE\).*(.*)$/\1/' fakecollrole.output)"
 test "$current_role_out" == "$expected_role_out"
@@ -103,6 +130,7 @@ test "$current_role_out" == "$expected_role_out"
 
 #### test add_collection_to_versions_and_dates()
 
+echo "testing json output"
 current_out="$(ansible-doc --json --playbook-dir ./ testns.testcol.randommodule | sed 's/ *$//' | sed 's/ *"filename": "[^"]*",$//')"
 expected_out="$(sed 's/ *"filename": "[^"]*",$//' randommodule.output)"
 test "$current_out" == "$expected_out"
@@ -119,8 +147,9 @@ current_out="$(ansible-doc --json --playbook-dir ./ -t vars testns.testcol.noop_
 expected_out="$(sed 's/ *"filename": "[^"]*",$//' noop_vars_plugin.output)"
 test "$current_out" == "$expected_out"
 
+echo "testing metadata dump"
 # just ensure it runs
-ANSIBLE_LIBRARY='./nolibrary' ansible-doc --metadata-dump --playbook-dir /dev/null >/dev/null
+ANSIBLE_LIBRARY='./nolibrary' ansible-doc --metadata-dump --playbook-dir /dev/null 1>/dev/null 2>&1
 
 # create broken role argument spec
 mkdir -p broken-docs/collections/ansible_collections/testns/testcol/roles/testrole/meta
@@ -144,71 +173,72 @@ argument_specs:
 EOF
 
 # ensure that --metadata-dump does not fail when --no-fail-on-errors is supplied
-ANSIBLE_LIBRARY='./nolibrary' ansible-doc --metadata-dump --no-fail-on-errors --playbook-dir broken-docs testns.testcol >/dev/null
+ANSIBLE_LIBRARY='./nolibrary' ansible-doc --metadata-dump --no-fail-on-errors --playbook-dir broken-docs testns.testcol 1>/dev/null 2>&1
 
 # ensure that --metadata-dump does fail when --no-fail-on-errors is not supplied
 output=$(ANSIBLE_LIBRARY='./nolibrary' ansible-doc --metadata-dump --playbook-dir broken-docs testns.testcol 2>&1 | grep -c 'ERROR!' || true)
 test "${output}" -eq 1
 
-# ensure we list the 'legacy plugins'
+
+echo "testing legacy plugin listing"
 [ "$(ansible-doc -M ./library -l ansible.legacy |wc -l)"  -gt "0" ]
 
-# playbook dir should work the same
+echo "testing legacy plugin list via --playbook-dir"
 [ "$(ansible-doc -l ansible.legacy --playbook-dir ./|wc -l)" -gt "0" ]
 
-# see that we show undocumented when missing docs
+echo "testing undocumented plugin output"
 [ "$(ansible-doc -M ./library -l ansible.legacy |grep -c UNDOCUMENTED)" == "6" ]
 
-# ensure filtering works and does not include any 'test_' modules
+echo "testing filtering does not include any 'test_' modules"
 [ "$(ansible-doc -M ./library -l ansible.builtin |grep -c test_)" == 0 ]
 [ "$(ansible-doc --playbook-dir ./  -l ansible.builtin |grep -c test_)" == 0 ]
 
-# ensure filtering still shows modules
+echo "testing filtering still shows modules"
 count=$(ANSIBLE_LIBRARY='./nolibrary' ansible-doc -l ansible.builtin |wc -l)
 [ "${count}" -gt "0" ]
 [ "$(ansible-doc -M ./library -l ansible.builtin |wc -l)" == "${count}" ]
 [ "$(ansible-doc --playbook-dir ./ -l ansible.builtin |wc -l)" == "${count}" ]
 
 
-# produce 'sidecar' docs for test
+echo "testing sidecar docs for jinja plugins"
 [ "$(ansible-doc -t test --playbook-dir ./ testns.testcol.yolo| wc -l)" -gt "0" ]
 [ "$(ansible-doc -t filter --playbook-dir ./ donothing| wc -l)" -gt "0" ]
 [ "$(ansible-doc -t filter --playbook-dir ./ ansible.legacy.donothing| wc -l)" -gt "0" ]
 
-# no docs and no sidecar
+echo "testing no docs and no sidecar"
-ansible-doc -t filter --playbook-dir ./ nodocs 2>&1| grep -c 'missing documentation' || true
+ansible-doc -t filter --playbook-dir ./ nodocs 2>&1| grep $GREP_OPTS -c 'missing documentation' || true
 
-# produce 'sidecar' docs for module
+echo "testing sidecar docs for module"
 [ "$(ansible-doc -M ./library test_win_module| wc -l)" -gt "0" ]
 [ "$(ansible-doc --playbook-dir ./ test_win_module| wc -l)" -gt "0" ]
 
-# test 'double DOCUMENTATION' use
+echo "testing duplicate DOCUMENTATION"
 [ "$(ansible-doc --playbook-dir ./ double_doc| wc -l)" -gt "0" ]
 
-# don't break on module dir
+echo "testing don't break on module dir"
 ansible-doc --list --module-path ./modules > /dev/null
 
-# ensure we dedupe by fqcn and not base name
+echo "testing dedupe by fqcn and not base name"
 [ "$(ansible-doc -l -t filter --playbook-dir ./ |grep -c 'b64decode')" -eq "3" ]
 
-# ensure we don't show duplicates for plugins that only exist in ansible.builtin when listing ansible.legacy plugins
+echo "testing no duplicates for plugins that only exist in ansible.builtin when listing ansible.legacy plugins"
 [ "$(ansible-doc -l -t filter --playbook-dir ./ |grep -c 'b64encode')" -eq "1" ]
 
-# with playbook dir, legacy should override
+echo "testing with playbook dir, legacy should override"
-ansible-doc -t filter split --playbook-dir ./ |grep histerical
+ansible-doc -t filter split --playbook-dir ./ |grep $GREP_OPTS histerical
 
 pyc_src="$(pwd)/filter_plugins/other.py"
 pyc_1="$(pwd)/filter_plugins/split.pyc"
 pyc_2="$(pwd)/library/notaplugin.pyc"
 trap 'rm -rf "$pyc_1" "$pyc_2"' EXIT
 
-# test pyc files are not used as adjacent documentation
+echo "testing pyc files are not used as adjacent documentation"
 python -c "import py_compile; py_compile.compile('$pyc_src', cfile='$pyc_1')"
-ansible-doc -t filter split --playbook-dir ./ |grep histerical
+ansible-doc -t filter split --playbook-dir ./ |grep $GREP_OPTS histerical
 
-# test pyc files are not listed as plugins
+echo "testing pyc files are not listed as plugins"
 python -c "import py_compile; py_compile.compile('$pyc_src', cfile='$pyc_2')"
 test "$(ansible-doc -l -t module --playbook-dir ./ 2>&1 1>/dev/null |grep -c "notaplugin")" == 0
 
-# without playbook dir, builtin should return
+echo "testing without playbook dir, builtin should return"
-ansible-doc -t filter split |grep -v histerical
+ansible-doc -t filter split 2>&1 |grep $GREP_OPTS -v histerical