Initial ansible-test implementation. (#18556)

.coveragerc

@@ -1,4 +1,22 @@
-[report]
+# This configuration file is used for manual execution of coverage
+# as well as for tests run through ansible-test.
+
+[run]
+branch = True
+
+# Enable concurrency. This also enables parallel mode, which results in
+# multiple coverage files being created. Concurrency allows us to collect
+# results from multiple tests simultaneously, as well as supporting multiple
+# test runs, such as from integration tests.
+concurrency = multiprocessing
+
+# When running tests through ansible-test, this option is overridden by
+# the COVERAGE_FILE environment variable. This option is present for
+# convenience when running coverage manually from this directory.
+data_file = test/results/coverage/coverage
+
+# Don't collect or report code coverage from files matching these patterns.
 omit =
-    */python?.?/*
-    */site-packages/nose/*
+    */python*/dist-packages/*
+    */python*/site-packages/*
+    */pytest

.gitignore

@@ -52,7 +52,6 @@ credentials.yml
 # test output
 *.retry
 *.out
-.coverage
 .tox
 .cache
 results.xml
@@ -67,3 +66,11 @@ ansible.egg-info/
 /shippable/
 # Release directory
 packaging/release/ansible_release
+/.cache/
+/test/results/coverage/coverage*
+/test/results/reports/coverage.xml
+/test/results/reports/coverage/
+/test/results/junit/*.xml
+/test/results/logs/*.log
+/test/integration/inventory.remote
+/test/integration/inventory.winrm

shippable.yml

@@ -8,60 +8,51 @@ matrix:
   exclude:
     - env: TEST=none
   include:
-    - env: TEST=remote TARGET=all PLATFORM=osx VERSION=10.11
+    - env: TEST=other
+    - env: TEST=units
 
-    - env: TEST=remote TARGET=all PLATFORM=freebsd VERSION=10.3-STABLE PRIVILEGED=true
+    - env: TEST=osx/10.11
 
-    - env: TEST=remote TARGET=ci_win1 PLATFORM=windows VERSION=2012-R2_RTM
-    - env: TEST=remote TARGET=ci_win2 PLATFORM=windows VERSION=2012-R2_RTM
-    - env: TEST=remote TARGET=ci_win3 PLATFORM=windows VERSION=2012-R2_RTM
+    - env: TEST=freebsd/10.3-STABLE
 
-    - env: TEST=integration TARGET=destructive IMAGE=ansible/ansible:centos6
-    - env: TEST=integration TARGET=destructive IMAGE=ansible/ansible:centos7
-    - env: TEST=integration TARGET=destructive IMAGE=ansible/ansible:fedora-rawhide
-    - env: TEST=integration TARGET=destructive IMAGE=ansible/ansible:fedora23
-    - env: TEST=integration TARGET=destructive IMAGE=ansible/ansible:opensuseleap
-    - env: TEST=integration TARGET=destructive IMAGE=ansible/ansible:ubuntu1204 PRIVILEGED=true
-    - env: TEST=integration TARGET=destructive IMAGE=ansible/ansible:ubuntu1404 PRIVILEGED=true
-    - env: TEST=integration TARGET=destructive IMAGE=ansible/ansible:ubuntu1604
-    - env: TEST=integration TARGET=destructive IMAGE=ansible/ansible:ubuntu1604py3 PYTHON3=1
+    - env: TEST=windows/1
+    - env: TEST=windows/2
+    - env: TEST=windows/3
 
-    - env: TEST=integration TARGET=non_destructive IMAGE=ansible/ansible:centos6
-    - env: TEST=integration TARGET=non_destructive IMAGE=ansible/ansible:centos7 PRIVILEGED=true
-    - env: TEST=integration TARGET=non_destructive IMAGE=ansible/ansible:fedora-rawhide PRIVILEGED=true
-    - env: TEST=integration TARGET=non_destructive IMAGE=ansible/ansible:fedora23 PRIVILEGED=true
-    - env: TEST=integration TARGET=non_destructive IMAGE=ansible/ansible:opensuseleap PRIVILEGED=true
-    - env: TEST=integration TARGET=non_destructive IMAGE=ansible/ansible:ubuntu1204 PRIVILEGED=true
-    - env: TEST=integration TARGET=non_destructive IMAGE=ansible/ansible:ubuntu1404 PRIVILEGED=true
-    - env: TEST=integration TARGET=non_destructive IMAGE=ansible/ansible:ubuntu1604 PRIVILEGED=true
-    - env: TEST=integration TARGET=non_destructive IMAGE=ansible/ansible:ubuntu1604py3 PYTHON3=1 PRIVILEGED=true
+    - env: TEST=linux/centos6/1
+    - env: TEST=linux/centos7/1
+    - env: TEST=linux/fedora-rawhide/1
+    - env: TEST=linux/fedora23/1
+    - env: TEST=linux/opensuseleap/1
+    - env: TEST=linux/ubuntu1204/1
+    - env: TEST=linux/ubuntu1404/1
+    - env: TEST=linux/ubuntu1604/1
+    - env: TEST=linux/ubuntu1604py3/1
 
-    - env: TEST=integration TARGET=other IMAGE=ansible/ansible:centos6
-    - env: TEST=integration TARGET=other IMAGE=ansible/ansible:centos7
-    - env: TEST=integration TARGET=other IMAGE=ansible/ansible:fedora-rawhide
-    - env: TEST=integration TARGET=other IMAGE=ansible/ansible:fedora23
-    - env: TEST=integration TARGET=other IMAGE=ansible/ansible:opensuseleap
-    - env: TEST=integration TARGET=other IMAGE=ansible/ansible:ubuntu1204
-    - env: TEST=integration TARGET=other IMAGE=ansible/ansible:ubuntu1404
-    - env: TEST=integration TARGET=other IMAGE=ansible/ansible:ubuntu1604
-    - env: TEST=integration TARGET=other IMAGE=ansible/ansible:ubuntu1604py3 PYTHON3=1
+    - env: TEST=linux/centos6/2
+    - env: TEST=linux/centos7/2
+    - env: TEST=linux/fedora-rawhide/2
+    - env: TEST=linux/fedora23/2
+    - env: TEST=linux/opensuseleap/2
+    - env: TEST=linux/ubuntu1204/2
+    - env: TEST=linux/ubuntu1404/2
+    - env: TEST=linux/ubuntu1604/2
+    - env: TEST=linux/ubuntu1604py3/2
 
-    - env: TEST=sanity INSTALL_DEPS=1 TOXENV=py24
-      python: 2.7
-    - env: TEST=sanity INSTALL_DEPS=1 TOXENV=py26
-      python: 2.6
-    - env: TEST=sanity INSTALL_DEPS=1 TOXENV=py27
-      python: 2.7
-    - env: TEST=sanity INSTALL_DEPS=1 TOXENV=py35
-      python: 3.5
-
-    - env: TEST=code-smell INSTALL_DEPS=1
-      python: 2.7
+    - env: TEST=linux/centos6/3
+    - env: TEST=linux/centos7/3
+    - env: TEST=linux/fedora-rawhide/3
+    - env: TEST=linux/fedora23/3
+    - env: TEST=linux/opensuseleap/3
+    - env: TEST=linux/ubuntu1204/3
+    - env: TEST=linux/ubuntu1404/3
+    - env: TEST=linux/ubuntu1604/3
+    - env: TEST=linux/ubuntu1604py3/3
 build:
   pre_ci_boot:
     options: "--privileged=false --net=bridge"
   ci:
-    - test/utils/shippable/ci.sh
+    - test/utils/shippable/timing.sh test/utils/shippable/shippable.sh
 
 integrations:
   notifications:

test/compile/python2.4-skip.txt

@@ -0,0 +1,51 @@
+/contrib/
+/docsite/
+/hacking/
+/lib/ansible/cli/
+/lib/ansible/compat/
+/lib/ansible/config/
+/lib/ansible/constants.py
+/lib/ansible/errors/
+/lib/ansible/executor/
+/lib/ansible/galaxy/
+/lib/ansible/__init__.py
+/lib/ansible/inventory/
+/lib/ansible/modules/core/cloud/[^/]+/(?!(ec2_facts.py|_ec2_ami_search.py))
+/lib/ansible/modules/extras/cloud/
+/lib/ansible/modules/extras/clustering/consul.*.py
+/lib/ansible/modules/extras/clustering/znode.py
+/lib/ansible/modules/extras/database/influxdb/
+/lib/ansible/modules/extras/database/mssql/
+/lib/ansible/modules/extras/infrastructure/foreman/
+/lib/ansible/modules/extras/monitoring/zabbix.*.py
+/lib/ansible/modules/extras/network/f5/
+/lib/ansible/modules/extras/network/nmcli.py
+/lib/ansible/modules/extras/notification/pushbullet.py
+/lib/ansible/modules/extras/packaging/language/maven_artifact.py
+/lib/ansible/modules/extras/packaging/os/dnf.py
+/lib/ansible/modules/extras/packaging/os/layman.py
+/lib/ansible/modules/extras/remote_management/ipmi/
+/lib/ansible/modules/extras/univention/
+/lib/ansible/modules/extras/web_infrastructure/letsencrypt.py
+/lib/ansible/modules/__init__.py
+/lib/ansible/module_utils/a10.py
+/lib/ansible/module_utils/azure_rm_common.py
+/lib/ansible/module_utils/cloud.py
+/lib/ansible/module_utils/docker_common.py
+/lib/ansible/module_utils/ec2.py
+/lib/ansible/module_utils/gcdns.py
+/lib/ansible/module_utils/gce.py
+/lib/ansible/module_utils/gcp.py
+/lib/ansible/module_utils/lxd.py
+/lib/ansible/module_utils/openstack.py
+/lib/ansible/module_utils/rax.py
+/lib/ansible/module_utils/vca.py
+/lib/ansible/module_utils/vmware.py
+/lib/ansible/parsing/
+/lib/ansible/playbook/
+/lib/ansible/plugins/
+/lib/ansible/release.py
+/lib/ansible/template/
+/lib/ansible/utils/
+/lib/ansible/vars/
+/test/

test/compile/python2.6-skip.txt

@@ -0,0 +1,2 @@
+/contrib/inventory/vagrant.py
+/hacking/dump_playbook_attributes.py

test/compile/python3.5-skip.txt

@@ -0,0 +1,2 @@
+/examples/scripts/uptime.py
+/test/samples/multi.py

test/integration/targets/add_host/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/ansible/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/apache2_module/aliases

@@ -0,0 +1,2 @@
+destructive
+posix/ci/group1

test/integration/targets/apt/aliases

@@ -0,0 +1,2 @@
+destructive
+posix/ci/group1

test/integration/targets/apt_repository/aliases

@@ -0,0 +1,3 @@
+apt_key
+destructive
+posix/ci/group1

test/integration/targets/args/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/assemble/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/async/aliases

@@ -0,0 +1,3 @@
+async_status
+async_wrapper
+posix/ci/group2

test/integration/targets/async_extra_data/aliases

@@ -0,0 +1,4 @@
+needs/ssh
+posix/ci/group3
+skip/freebsd
+skip/osx

test/integration/targets/authorized_key/aliases

@@ -0,0 +1,2 @@
+needs/root
+posix/ci/group2

test/integration/targets/become/aliases

@@ -0,0 +1,2 @@
+destructive
+posix/ci/group1

test/integration/targets/binary/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/binary_modules_posix/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/binary_modules_winrm/aliases

@@ -0,0 +1 @@
+windows/ci/group2

test/integration/targets/blocks/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/changed_when/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/check_mode/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/command_shell/aliases

@@ -0,0 +1,3 @@
+command
+posix/ci/group2
+shell

test/integration/targets/conditionals/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/connection_chroot/aliases

@@ -0,0 +1,2 @@
+needs/root
+posix/ci/group3

test/integration/targets/connection_local/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/connection_paramiko_ssh/aliases

@@ -0,0 +1,2 @@
+needs/ssh
+posix/ci/group3

test/integration/targets/connection_ssh/aliases

@@ -0,0 +1,2 @@
+needs/ssh
+posix/ci/group3

test/integration/targets/connection_winrm/aliases

@@ -0,0 +1 @@
+windows/ci/group3

test/integration/targets/copy/aliases

@@ -0,0 +1,2 @@
+needs/root
+posix/ci/group2

test/integration/targets/delegate_to/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/dnf/aliases

@@ -0,0 +1,2 @@
+destructive
+posix/ci/group1

test/integration/targets/docker/aliases

@@ -0,0 +1,2 @@
+destructive
+posix/ci/group1

test/integration/targets/embedded_module/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/environment/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/facts_d/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/failed_when/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/fetch/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/file/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/filters/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/gathering_facts/aliases

@@ -0,0 +1,2 @@
+posix/ci/group3
+skip/osx

test/integration/targets/gem/aliases

@@ -0,0 +1,2 @@
+destructive
+posix/ci/group1

test/integration/targets/get_url/aliases

@@ -0,0 +1,4 @@
+destructive
+posix/ci/group1
+skip/freebsd
+skip/osx

test/integration/targets/git/aliases

@@ -0,0 +1,3 @@
+destructive
+posix/ci/group1
+skip/osx

test/integration/targets/group_by/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/handlers/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/hash/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/hg/aliases

@@ -0,0 +1,2 @@
+posix/ci/group2
+skip/python3

test/integration/targets/hosts_field/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/ignore_errors/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/include_vars/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/includes/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/iterators/aliases

@@ -0,0 +1,2 @@
+posix/ci/group2
+skip/osx

test/integration/targets/lineinfile/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/lookup_paths/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/lookup_properties/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/lookups/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/loops/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/mount/aliases

@@ -0,0 +1,3 @@
+needs/privileged
+needs/root
+posix/ci/group2

test/integration/targets/mysql_db/aliases

@@ -0,0 +1,4 @@
+destructive
+posix/ci/group1
+skip/freebsd
+skip/osx

test/integration/targets/mysql_user/aliases

@@ -0,0 +1,4 @@
+destructive
+posix/ci/group1
+skip/freebsd
+skip/osx

test/integration/targets/mysql_variables/aliases

@@ -0,0 +1,4 @@
+destructive
+posix/ci/group1
+skip/freebsd
+skip/osx

test/integration/targets/no_log/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/parsing/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/ping/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/pip/aliases

@@ -0,0 +1,2 @@
+destructive
+posix/ci/group1

test/integration/targets/postgresql/aliases

@@ -0,0 +1,8 @@
+destructive
+needs/privileged
+posix/ci/group1
+postgresql_db
+postgresql_privs
+postgresql_user
+skip/freebsd
+skip/osx

test/integration/targets/pull_limit_inventory/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/pull_no_127/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/pull_run/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/script/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/service/aliases

@@ -0,0 +1,6 @@
+destructive
+needs/privileged
+posix/ci/group1
+skip/freebsd
+skip/osx
+skip/python3

test/integration/targets/special_vars/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/stat/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/subversion/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/synchronize/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/tags/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/task_ordering/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/template/aliases

@@ -0,0 +1,2 @@
+needs/root
+posix/ci/group2

test/integration/targets/templating_settings/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/test_infra/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/unarchive/aliases

@@ -0,0 +1,2 @@
+needs/root
+posix/ci/group2

test/integration/targets/unicode/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/until/aliases

@@ -0,0 +1 @@
+posix/ci/group2

test/integration/targets/uri/aliases

@@ -0,0 +1,5 @@
+destructive
+posix/ci/group1
+skip/freebsd
+skip/osx
+skip/python3

test/integration/targets/var_blending/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/var_precedence/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/vault/aliases

@@ -0,0 +1 @@
+posix/ci/group3

test/integration/targets/win_async_wrapper/aliases

@@ -0,0 +1,2 @@
+async_status
+windows/ci/group3

test/integration/targets/win_command/aliases

@@ -0,0 +1 @@
+windows/ci/group3

test/integration/targets/win_copy/aliases

@@ -0,0 +1 @@
+windows/ci/group2

test/integration/targets/win_feature/aliases

@@ -0,0 +1 @@
+windows/ci/group3

test/integration/targets/win_fetch/aliases

@@ -0,0 +1 @@
+windows/ci/group1

test/integration/targets/win_fetch/tasks/main.yml

@@ -16,6 +16,10 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
+- name: define host-specific output_dir
+  set_fact:
+    output_dir: "{{ output_dir }}/{{ inventory_hostname }}"
+
 - name: clean out the test directory
   local_action: file name={{ output_dir|mandatory }} state=absent
   run_once: true