Initial ansible-test sanity docs. (#26775)

* Rename no-iterkeys test for consistency.

* Require docs for all ansible-test sanity tests.

* Initial ansible-test sanity docs.

* Fix capitalization of Python.

* Fix sanity code smell test false positives.

* Fix another code-smell false positive.

docs/bin/testing_formatter.sh

@@ -0,0 +1,12 @@
+#!/bin/bash -eu
+
+cat <<- EOF > ../docsite/rst/dev_guide/testing/sanity/index.rst
+Sanity Tests
+============
+
+The following sanity tests are available as \`\`--test\`\` options for \`\`ansible-test sanity\`\`:
+
+$(for test in $(../../test/runner/ansible-test sanity --list-tests); do echo "- :doc:\`${test} <${test}>\`"; done)
+
+This list is also available using \`\`ansible-test sanity --list-tests\`\`.
+EOF

docs/docsite/.gitignore

@@ -11,6 +11,7 @@ ansible*.xml
 .buildinfo
 objects.inv
 .doctrees
+rst/dev_guide/testing/sanity/index.rst
 rst/modules/*.rst
 rst/playbooks_keywords.rst
 *.min.css

docs/docsite/Makefile

@@ -1,6 +1,7 @@
 OS := $(shell uname -s)
 SITELIB = $(shell python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()"):
 FORMATTER=../bin/plugin_formatter.py
+TESTING_FORMATTER=../bin/testing_formatter.sh
 DUMPER=../bin/dump_keywords.py
 ifeq ($(shell echo $(OS) | egrep -ic 'Darwin|FreeBSD|OpenBSD|DragonFly'),1)
 CPUS ?= $(shell sysctl hw.ncpu|awk '{print $$2}')
@@ -17,7 +18,7 @@ all: docs
 
 docs: clean htmldocs
 
-htmldocs: keywords modules staticmin
+htmldocs: testing keywords modules staticmin
 	CPUS=$(CPUS) $(MAKE) -f Makefile.sphinx html
 
 webdocs: docs
@@ -51,6 +52,9 @@ keywords: $(FORMATTER) ../templates/playbooks_keywords.rst.j2
 modules: $(FORMATTER) ../templates/plugin.rst.j2
 	PYTHONPATH=../../lib $(FORMATTER) -t rst --template-dir=../templates --module-dir=../../lib/ansible/modules -o rst/
 
+testing:
+	$(TESTING_FORMATTER)
+
 staticmin:
 	cat _themes/srtd/static/css/theme.css | sed -e 's/^[ 	]*//g; s/[ 	]*$$//g; s/\([:{;,]\) /\1/g; s/ {/{/g; s/\/\*.*\*\///g; /^$$/d' | sed -e :a -e '$$!N; s/\n\(.\)/\1/; ta' > _themes/srtd/static/css/theme.min.css
 

docs/docsite/rst/dev_guide/testing/sanity/ansible-doc.rst

@@ -0,0 +1,4 @@
+Sanity Tests » ansible-doc
+==========================
+
+Verifies that ``ansible-doc`` can parse module documentation on all supported Python versions.

docs/docsite/rst/dev_guide/testing/sanity/ansible-var-precedence-check.rst

@@ -0,0 +1,4 @@
+Sanity Tests » ansible-var-precedence-check
+===========================================
+
+Check the order of precedence for Ansible variables against :doc:`variable_precedence`.

docs/docsite/rst/dev_guide/testing/sanity/boilerplate.rst

@@ -0,0 +1,11 @@
+Sanity Tests » boilerplate
+==========================
+
+Most Python files other than those under ``lib/ansible/modules/`` should include the following boilerplate:
+
+.. code-block:: python
+
+    from __future__ import (absolute_import, division, print_function)
+
+    __metaclass__ = type
+

docs/docsite/rst/dev_guide/testing/sanity/configure-remoting-ps1.rst

@@ -0,0 +1,5 @@
+Sanity Tests » configure-remoting-ps1
+=====================================
+
+The file ``examples/scripts/ConfigureRemotingForAnsible.ps1`` is required and must be a regular file.
+It is used by external automated processes and cannot be moved, renamed or replaced with a symbolic link.

docs/docsite/rst/dev_guide/testing/sanity/empty-init.rst

@@ -0,0 +1,7 @@
+Sanity Tests » empty-init
+=========================
+
+The ``__init__.py`` files under the following directories must be empty:
+
+- ``lib/ansible/modules/``
+- ``test/units/``

docs/docsite/rst/dev_guide/testing/sanity/import.rst

@@ -0,0 +1,5 @@
+Sanity Tests » import
+=====================
+
+All Python imports in ``lib/ansible/modules/`` and ``lib/ansible/module_utils/`` which are not from the Python standard library
+must be imported in a try/except ImportError block.

docs/docsite/rst/dev_guide/testing/sanity/integration-aliases.rst

@@ -0,0 +1,36 @@
+Sanity Tests » integration-aliases
+==================================
+
+Each integration test must have an ``aliases`` file to control test execution.
+
+If the tests cannot be run as part of CI (requires external services, unsupported dependencies, etc.),
+then they most likely belong in ``test/integration/roles/`` instead of ``test/integration/targets/``.
+In that case, do not add an ``aliases`` file. Instead, just relocate the tests.
+
+In some cases tests requiring external resources can be run as a part of CI.
+This is often true when those resources can be provided by a docker container.
+
+However, if you think that the tests should be able to be supported by CI, please discuss test
+organization with @mattclay or @gundalow on GitHub or #ansible-devel on IRC.
+
+If the tests can be run as part of CI, you'll need to add an appropriate CI alias, such as:
+
+- ``posix/ci/group1``
+- ``windows/ci/group2``
+
+The CI groups are used to balance tests across multiple jobs to minimize test run time.
+Using the relevant ``group1`` entry is fine in most cases. Groups can be changed later to redistribute tests.
+
+Aliases can also be used to express test requirements:
+
+- ``needs/privileged``
+- ``needs/root``
+- ``needs/ssh``
+
+Other aliases are used to skip tests under certain conditions:
+
+- ``skip/freebsd``
+- ``skip/osx``
+- ``skip/python3``
+
+Take a look at existing ``aliases`` files to see what aliases are available and how they're used.

docs/docsite/rst/dev_guide/testing/sanity/line-endings.rst

@@ -0,0 +1,4 @@
+Sanity Tests » line-endings
+===========================
+
+All files must use ``\n`` for line endings instead of ``\r\n``.

docs/docsite/rst/dev_guide/testing/sanity/no-basestring.rst

@@ -0,0 +1,4 @@
+Sanity Tests » no-basestring
+============================
+
+Do not use ``isinstance(s, basestring)``.

docs/docsite/rst/dev_guide/testing/sanity/no-dict-iteritems.rst

@@ -0,0 +1,16 @@
+Sanity Tests » no-dict-iteritems
+================================
+
+The ``dict.iteritems`` method has been removed in Python 3. There are two recommended alternatives:
+
+.. code-block:: python
+
+    for KEY, VALUE in DICT.items():
+       pass
+
+.. code-block:: python
+
+    from ansible.module_utils.six import iteritems
+
+    for KEY, VALUE in iteritems(DICT):
+        pass

docs/docsite/rst/dev_guide/testing/sanity/no-dict-iterkeys.rst

@@ -0,0 +1,9 @@
+Sanity Tests » no-dict-iterkeys
+===============================
+
+The ``dict.iterkeys`` method has been removed in Python 3. Use the following instead:
+
+.. code-block:: python
+
+    for KEY in DICT:
+        pass

docs/docsite/rst/dev_guide/testing/sanity/no-dict-itervalues.rst

@@ -0,0 +1,16 @@
+Sanity Tests » no-dict-itervalues
+=================================
+
+The ``dict.itervalues`` method has been removed in Python 3. There are two recommended alternatives:
+
+.. code-block:: python
+
+    for VALUE in DICT.values():
+       pass
+
+.. code-block:: python
+
+    from ansible.module_utils.six import itervalues
+
+    for VALUE in itervalues(DICT):
+        pass

docs/docsite/rst/dev_guide/testing/sanity/no-list-cmp.rst

@@ -0,0 +1,8 @@
+Sanity Tests » no-list-cmp
+==========================
+
+The ``cmp`` function has been removed in Python 3.
+
+See `When sorting, use key instead of cmp`_.
+
+.. _When sorting, use key instead of cmp: http://python3porting.com/preparing.html#when-sorting-use-key-instead-of-cmp

docs/docsite/rst/dev_guide/testing/sanity/pep8.rst

@@ -0,0 +1,6 @@
+Sanity Tests » pep8
+===================
+
+Python static analysis for PEP 8 style guideline compliance.
+
+See :doc:`testing_pep8` for more information.

docs/docsite/rst/dev_guide/testing/sanity/pylint-ansible-test.rst

@@ -0,0 +1,6 @@
+Sanity Tests » pylint-ansible-test
+==================================
+
+Python static analysis for common programming errors.
+
+A more strict set of rules applied to ``ansible-test``.

docs/docsite/rst/dev_guide/testing/sanity/pylint.rst

@@ -0,0 +1,4 @@
+Sanity Tests » pylint
+=====================
+
+Python static analysis for common programming errors.

docs/docsite/rst/dev_guide/testing/sanity/replace-urlopen.rst

@@ -0,0 +1,4 @@
+Sanity Tests » replace-urlopen
+==============================
+
+Use ``open_url`` from ``module_utils`` instead of ``urlopen``.

docs/docsite/rst/dev_guide/testing/sanity/required-and-default-attributes.rst

@@ -0,0 +1,5 @@
+Sanity Tests » required-and-default-attributes
+==============================================
+
+Use only one of ``default`` or ``required`` with ``FieldAttribute``.
+

docs/docsite/rst/dev_guide/testing/sanity/rstcheck.rst

@@ -0,0 +1,4 @@
+Sanity Tests » rstcheck
+=======================
+
+Check reStructuredText files for syntax and formatting issues.

docs/docsite/rst/dev_guide/testing/sanity/sanity-docs.rst

@@ -0,0 +1,4 @@
+Sanity Tests » sanity-docs
+==========================
+
+Documentation for each ``ansible-test sanity`` test is required.

docs/docsite/rst/dev_guide/testing/sanity/shebang.rst

@@ -0,0 +1,16 @@
+Sanity Tests » shebang
+======================
+
+Most executable files should only use one of the following shebangs:
+
+- ``#!/bin/sh``
+- ``#!/bin/bash``
+- ``#!/usr/bin/make``
+- ``#!/usr/bin/env python``
+- ``#!/usr/bin/env bash``
+
+NOTE: For ``#!/bin/bash``, any of the options ``eux`` may also be used, such as ``#!/bin/bash -eux``.
+
+This does not apply to Ansible modules, which should not be executable and must always use ``#!/usr/bin/python``.
+
+Some exceptions are permitted. Ask if you have questions.

docs/docsite/rst/dev_guide/testing/sanity/shellcheck.rst

@@ -0,0 +1,4 @@
+Sanity Tests » shellcheck
+=========================
+
+Static code analysis for shell scripts using the excellent `shellcheck <https://www.shellcheck.net/>`_ tool.

docs/docsite/rst/dev_guide/testing/sanity/test-constraints.rst

@@ -0,0 +1,4 @@
+Sanity Tests » test-constraints
+===============================
+
+Constraints for test requirements should be in ``test/runner/requirements/constraints.txt``.

docs/docsite/rst/dev_guide/testing/sanity/use-compat-six.rst

@@ -0,0 +1,4 @@
+Sanity Tests » use-compat-six
+=============================
+
+Use ``six`` from ``module_utils`` instead of ``six``.

docs/docsite/rst/dev_guide/testing/sanity/validate-modules.rst

@@ -0,0 +1,6 @@
+Sanity Tests » validate-modules
+===============================
+
+Analyze modules for common issues in code and documentation.
+
+See :doc:`testing_validate-modules` for more information.

docs/docsite/rst/dev_guide/testing/sanity/yamllint.rst

@@ -0,0 +1,4 @@
+Sanity Tests » yamllint
+=======================
+
+Check YAML files for syntax and formatting issues.

test/runner/lib/sanity.py

@@ -626,6 +626,34 @@ def command_sanity_rstcheck(args, targets):
     return SanitySuccess(test)
 
 
+# noinspection PyUnusedLocal
+def command_sanity_sanity_docs(args, targets):  # pylint: disable=locally-disabled, unused-argument
+    """
+    :type args: SanityConfig
+    :type targets: SanityTargets
+    :rtype: SanityResult
+    """
+    test = 'sanity-docs'
+
+    sanity_dir = 'docs/docsite/rst/dev_guide/testing/sanity'
+    sanity_docs = set(part[0] for part in (os.path.splitext(name) for name in os.listdir(sanity_dir)) if part[1] == '.rst')
+    sanity_tests = set(sanity_test.name for sanity_test in sanity_get_tests())
+
+    missing = sanity_tests - sanity_docs
+
+    results = []
+
+    results += [SanityMessage(
+        message='missing docs for ansible-test sanity --test %s' % r,
+        path=os.path.join(sanity_dir, '%s.rst' % r),
+    ) for r in sorted(missing)]
+
+    if results:
+        return SanityFailure(test, messages=results)
+
+    return SanitySuccess(test)
+
+
 def command_sanity_ansible_doc(args, targets, python_version):
     """
     :type args: SanityConfig
@@ -856,6 +884,7 @@ SANITY_TESTS = (
     SanityFunc('pylint', command_sanity_pylint, intercept=False),
     SanityFunc('yamllint', command_sanity_yamllint, intercept=False),
     SanityFunc('rstcheck', command_sanity_rstcheck, intercept=False),
+    SanityFunc('sanity-docs', command_sanity_sanity_docs, intercept=False),
     SanityFunc('validate-modules', command_sanity_validate_modules, intercept=False),
     SanityFunc('ansible-doc', command_sanity_ansible_doc),
     SanityFunc('import', command_sanity_import),

test/sanity/code-smell/no-basestring.sh

@@ -6,6 +6,8 @@ BASESTRING_USERS=$(grep -r basestring . \
     | grep isinstance \
     | grep -v \
     -e test/results/ \
+    -e docs/docsite/_build/ \
+    -e docs/docsite/rst/dev_guide/testing/sanity/ \
     -e lib/ansible/module_utils/six/_six.py \
     -e lib/ansible/modules/ \
     -e '^[^:]*:#'

test/sanity/code-smell/no-dict-iterkeys.sh

@@ -7,8 +7,11 @@ ITERKEYS_USERS=$(grep -r -I iterkeys . \
     --exclude-dir docsite \
     --exclude-dir results \
     | grep -v \
+    -e 'metadata-.*.json:' \
+    -e lib/ansible.egg-info/ \
     -e lib/ansible/module_utils/six/_six.py \
-    -e test/sanity/code-smell/no-iterkeys.sh \
+    -e docs/docsite/rst/dev_guide/testing/sanity/ \
+    -e test/sanity/code-smell/no-dict-iterkeys.sh \
     -e '^[^:]*:#'
     )
 

test/sanity/code-smell/no-list-cmp.sh

@@ -5,6 +5,8 @@ CMP_USERS=$(grep -rI ' cmp[^a-zA-Z0-9_:=]' . \
 	| grep -v \
 	-e lib/ansible/module_utils/six/_six.py \
 	-e .git \
+	-e docs/docsite/_build/ \
+	-e docs/docsite/rst/dev_guide/testing/sanity/ \
 	-e test/sanity/code-smell/no-list-cmp.sh
 	)