Split integration tests out from Makefile. (#17976)

test/integration/Makefile

@@ -25,154 +25,138 @@ EUID := $(shell id -u -r)
 
 UNAME := $(shell uname | tr '[:upper:]' '[:lower:]')
 
-all: setup other non_destructive destructive
+all: other non_destructive destructive
 
-other: test_test_infra parsing test_var_precedence unicode test_templating_settings environment test_connection test_async_conditional includes blocks pull check_mode test_hash test_handlers test_group_by test_vault test_tags test_lookup_paths no_log test_gathering_facts test_binary_modules test_hosts_field
+other: test_test_infra parsing test_var_precedence unicode test_templating_settings environment test_as includes blocks pull_run pull_no_127 pull_limit_inventory check_mode test_hash test_handlers test_group_by test_vault test_tags test_lookup_paths no_log test_gathering_facts test_binary_modules_posix test_hosts_field test_lookup_properties args
 
 test_test_infra:
-	# ensure fail/assert work locally and can stop execution with non-zero exit code
-	PB_OUT=$$(ansible-playbook -i inventory.local test_test_infra.yml) ; APB_RC=$$? ; echo "$$PB_OUT" ; echo "rc was $$APB_RC (must be non-zero)" ; [ $$APB_RC -ne 0 ] ; echo "ensure playbook output shows assert/fail works (True)" ; echo "$$PB_OUT" | grep "fail works (True)" || exit 1 ; echo "$$PB_OUT" | fgrep "assert works (True)" || exit 1
-	# ensure we work using all specified test args, overridden inventory, etc
-	PB_OUT=$$(ansible-playbook -i $(INVENTORY) test_test_infra.yml -e @$(VARS_FILE) $(CREDENTIALS_ARG) $(TEST_FLAGS)) ; APB_RC=$$? ; echo "$$PB_OUT" ; echo "rc was $$APB_RC (must be non-zero)" ; [ $$APB_RC -ne 0 ] ; echo "ensure playbook output shows assert/fail works (True)" ; echo "$$PB_OUT" | grep "fail works (True)" || exit 1 ; echo "$$PB_OUT" | fgrep "assert works (True)" || exit 1
+	(cd targets/test_infra && ./runme.sh $(TEST_FLAGS))
 
 setup:
 	rm -rf $(TEST_DIR)
 	mkdir -p $(TEST_DIR)
 
-parsing: setup
-	ansible-playbook bad_parsing.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -vvv $(TEST_FLAGS) --tags prepare,common,scenario5
-	ansible-playbook good_parsing.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
+parsing:
+	(cd targets/parsing && ./runme.sh $(TEST_FLAGS))
 
-includes: setup
-	ansible-playbook test_includes.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) $(TEST_FLAGS)
-
-pull:	pull_run pull_no_127 pull_limit_inventory
+includes:
+	(cd targets/includes && ./runme.sh $(TEST_FLAGS))
 
 pull_run:
-	ansible-pull -d $(MYTMPDIR) -U https://github.com/ansible-test-robinro/pull-integration-test.git $(TEST_FLAGS) | grep MAGICKEYWORD; \
-    RC=$$? ; \
-	rm -rf $(MYTMPDIR); \
-    exit $$RC
+	(cd targets/pull_run && ./runme.sh $(TEST_FLAGS))
 
-# test for https://github.com/ansible/ansible/issues/13681
 pull_no_127:
-	ansible-pull -d $(MYTMPDIR) -U https://github.com/ansible-test-robinro/pull-integration-test.git $(TEST_FLAGS) | grep -v 127\.0\.0\.1; \
-    RC=$$? ; \
-	rm -rf $(MYTMPDIR); \
-    exit $$RC
+	(cd targets/pull_no_127 && ./runme.sh $(TEST_FLAGS))
 
-# test for https://github.com/ansible/ansible/issues/13688
 pull_limit_inventory:
-	ansible-pull -d $(MYTMPDIR) -U https://github.com/ansible-test-robinro/pull-integration-test.git $(TEST_FLAGS); \
-    RC=$$? ; \
-	rm -rf $(MYTMPDIR); \
-    exit $$RC
-
+	(cd targets/pull_limit_inventory && ./runme.sh $(TEST_FLAGS))
 
-unicode: setup
-	ansible-playbook unicode.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) -v $(TEST_FLAGS) -e 'extra_var=café'
-	# Test the start-at-task flag #9571
-	ansible-playbook unicode.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) -v --start-at-task '*¶' -e 'start_at_task=True' $(TEST_FLAGS)
+unicode:
+	(cd targets/unicode && ./runme.sh $(TEST_FLAGS))
 
-test_templating_settings: setup
-	ansible-playbook test_templating_settings.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
+test_templating_settings:
+	(cd targets/templating_settings && ./runme.sh $(TEST_FLAGS))
 
-test_gathering_facts: setup
-	ansible-playbook test_gathering_facts.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) -v $(TEST_FLAGS)
+test_gathering_facts:
+	(cd targets/gathering_facts && ./runme.sh $(TEST_FLAGS))
 
-environment: setup
-	ansible-playbook test_environment.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) $(TEST_FLAGS)
+environment:
+	(cd targets/environment && ./runme.sh $(TEST_FLAGS))
 
 non_destructive: setup
 	ansible-playbook non_destructive.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
 
-# For our Docker images, which identify themselves with "ENV container=docker", automatically run the test_async target.
-# Otherwise, skip it, since we don't know if local ssh is available. You can always run the test_async target manually.
-ifeq ($(container),docker)
-test_async_conditional: test_async
-else
-test_async_conditional:
-endif
+# Tests which can be run when running as root and a local SSH daemon is available.
+test_root_ssh: test_connection_local test_connection_chroot test_connection_ssh test_connection_paramiko_ssh test_delegate_to
+
+# For tests running in our Docker containers (identified with "ENV container=docker").
+# These tests are run as root and have access to a local SSH daemon.
+test_as_docker: test_root_ssh test_async_extra_data
+
+# For tests running in our FreeBSD instances on EC2 (identified with "ENV container=freebsd").
+# These tests are run as root and have access to a local SSH daemon.
+test_as_freebsd: test_root_ssh
+
+# For tests running in our OS X instances at MacStadium (identified with "ENV container=osx").
+# These tests are run as root and have access to a local SSH daemon.
+test_as_osx: test_root_ssh
+
+# For tests running as root.
+# Login to a local SSH daemon assumed to be unavailable.
+test_as_root: test_connection_local test_connection_chroot
 
-# For our Docker images, which identify themselves with "ENV container=docker", use the test_docker inventory group.
-# Otherwise use the test_default inventory group, which runs fewer tests, but should work on any system.
+# For tests not running as root.
+# Login to a local SSH daemon assumed to be unavailable.
+test_as_non_root: test_connection_local
+
+# Determine which of the above environments we're running in.
 ifeq ($(container),docker)
-TEST_CONNECTION_FILTER := 'test_docker'
+TEST_AS_TARGET := test_as_docker
 else ifeq ($(container),freebsd)
-TEST_CONNECTION_FILTER := 'test_freebsd'
+TEST_AS_TARGET := test_as_freebsd
+else ifeq ($(container),osx)
+TEST_AS_TARGET := test_as_osx
+else ifeq ($(EUID),0)
+TEST_AS_TARGET := test_as_root
 else
-TEST_CONNECTION_FILTER := 'test_default'
+TEST_AS_TARGET := test_as_non_root
 endif
 
-# Skip connection plugins which require root when not running as root.
-ifneq ($(EUID),0)
-TEST_CONNECTION_FILTER += !chroot
-endif
+test_as: $(TEST_AS_TARGET)
+
+test_connection_local:
+	(cd targets/connection_local && ./runme.sh $(TEST_FLAGS))
+
+test_connection_chroot:
+	(cd targets/connection_chroot && ./runme.sh $(TEST_FLAGS))
+
+test_connection_docker:
+	(cd targets/connection_docker && ./runme.sh $(TEST_FLAGS))
+
+test_connection_libvirt_lxc:
+	(cd targets/connection_libvirt_lxc && ./runme.sh $(TEST_FLAGS))
+
+test_connection_jail:
+	(cd targets/connection_jail && ./runme.sh $(TEST_FLAGS))
 
-# Connection plugin test command to repeat with each locale setting.
-TEST_CONNECTION_CMD = $(1) ansible-playbook test_connection.yml -i test_connection.inventory -l '$(TEST_CONNECTION_FILTER)' $(TEST_FLAGS)
+test_connection_ssh:
+	(cd targets/connection_ssh && ./runme.sh $(TEST_FLAGS))
 
-test_connection: setup
-	$(call TEST_CONNECTION_CMD)
-	$(call TEST_CONNECTION_CMD, LC_ALL=C LANG=C)
+test_connection_paramiko_ssh:
+	(cd targets/connection_paramiko_ssh && ./runme.sh $(TEST_FLAGS))
 
-# Connection plugin test command to repeat with each locale setting. WinRM specific version.
-TEST_CONNECTION_WINRM_CMD = $(1) ansible-playbook test_connection_winrm.yml -i inventory.winrm $(TEST_FLAGS)
+test_connection_lxd:
+	(cd targets/connection_lxd && ./runme.sh $(TEST_FLAGS))
 
-test_connection_winrm: setup
-	$(call TEST_CONNECTION_WINRM_CMD)
-	$(call TEST_CONNECTION_WINRM_CMD, LC_ALL=C LANG=C)
+test_connection_lxc:
+	(cd targets/connection_lxc && ./runme.sh $(TEST_FLAGS))
+
+test_connection_winrm:
+	(cd targets/connection_winrm && ./runme.sh $(TEST_FLAGS))
 
 destructive: setup
 	ansible-playbook destructive.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
 
-check_mode: setup
-	ansible-playbook check_mode.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v --check $(TEST_FLAGS)
+check_mode:
+	(cd targets/check_mode && ./runme.sh $(TEST_FLAGS))
 
-test_group_by: setup
-	ansible-playbook test_group_by.yml -i inventory.group_by -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
+test_group_by:
+	(cd targets/group_by && ./runme.sh $(TEST_FLAGS))
 
 test_handlers:
-	ansible-playbook test_handlers.yml --tags scenario1 -i inventory.handlers -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
-	[ "$$(ansible-playbook test_handlers.yml --tags scenario2 -l A -i inventory.handlers -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) | egrep -o 'RUNNING HANDLER \[test_handlers : .*?]')" = "RUNNING HANDLER [test_handlers : test handler]" ]
-	# Not forcing, should only run on successful host
-	[ "$$(ansible-playbook test_force_handlers.yml --tags normal -i inventory.handlers -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) | egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_B" ]
-	# Forcing from command line
-	[ "$$(ansible-playbook test_force_handlers.yml --tags normal -i inventory.handlers --force-handlers -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) | egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_A CALLED_HANDLER_B" ]
-	# Forcing from command line, should only run later tasks on unfailed hosts
-	[ "$$(ansible-playbook test_force_handlers.yml --tags normal -i inventory.handlers --force-handlers -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) | egrep -o CALLED_TASK_. | sort | uniq | xargs)" = "CALLED_TASK_B CALLED_TASK_D CALLED_TASK_E" ]
-	# Forcing from command line, should call handlers even if all hosts fail
-	[ "$$(ansible-playbook test_force_handlers.yml --tags normal -i inventory.handlers --force-handlers -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v -e fail_all=yes $(TEST_FLAGS) | egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_A CALLED_HANDLER_B" ]
-	# Forcing from ansible.cfg
-	[ "$$(ANSIBLE_FORCE_HANDLERS=true ansible-playbook --tags normal test_force_handlers.yml -i inventory.handlers -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) | egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_A CALLED_HANDLER_B" ]
-	# Forcing true in play
-	[ "$$(ansible-playbook test_force_handlers.yml --tags force_true_in_play -i inventory.handlers -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) | egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_A CALLED_HANDLER_B" ]
-	# Forcing false in play, which overrides command line
-	[ "$$(ansible-playbook test_force_handlers.yml --force-handlers --tags force_false_in_play -i inventory.handlers -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) | egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_B" ]
+	(cd targets/handlers && ./runme.sh $(TEST_FLAGS))
 
 test_hash:
-	ANSIBLE_HASH_BEHAVIOUR=replace ansible-playbook test_hash.yml -i $(INVENTORY) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) -e '{"test_hash":{"extra_args":"this is an extra arg"}}'
-	ANSIBLE_HASH_BEHAVIOUR=merge ansible-playbook test_hash.yml -i $(INVENTORY) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) -e '{"test_hash":{"extra_args":"this is an extra arg"}}'
-
-test_var_precedence: setup
-	ansible-playbook test_var_precedence.yml -i $(INVENTORY) $(CREDENTIALS_ARG) $(TEST_FLAGS) -v -e outputdir=$(TEST_DIR) -e 'extra_var=extra_var' -e 'extra_var_override=extra_var_override'
-
-test_vault: setup
-	ansible-playbook test_vault.yml -i $(INVENTORY) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) --vault-password-file $(VAULT_PASSWORD_FILE) --list-tasks -e outputdir=$(TEST_DIR) -e @$(VARS_FILE)
-	ansible-playbook test_vault.yml -i $(INVENTORY) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) --vault-password-file $(VAULT_PASSWORD_FILE) --list-hosts -e outputdir=$(TEST_DIR)  -e @$(VARS_FILE)
-	ansible-playbook test_vault.yml -i $(INVENTORY) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) --vault-password-file $(VAULT_PASSWORD_FILE) --syntax-check -e outputdir=$(TEST_DIR)  -e @$(VARS_FILE)
-	ansible-playbook test_vault.yml -i $(INVENTORY) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) --vault-password-file $(VAULT_PASSWORD_FILE) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE)
-	ansible-playbook test_vault_embedded.yml -i $(INVENTORY) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) --vault-password-file $(VAULT_PASSWORD_FILE) --syntax-check -e outputdir=$(TEST_DIR)  -e @$(VARS_FILE)
-	ansible-playbook test_vault_embedded.yml -i $(INVENTORY) $(CREDENTIALS_ARG) -v $(TEST_FLAGS) --vault-password-file $(VAULT_PASSWORD_FILE) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE)
-
-# test_delegate_to does not work unless we have permission to ssh to localhost.
-# Would take some more effort on our test systems to implement that -- probably
-# the test node should create an ssh public-private key pair that allows the
-# root user on a node to ssh to itself.  Until then, this is not in make all.
-# Have to run it manually.  Ordinary users should be able to run this test as
-# long as they have permissions to login to their local machine via ssh.
-test_delegate_to: setup
-	ansible-playbook test_delegate_to.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
+	(cd targets/hash && ./runme.sh $(TEST_FLAGS))
+
+test_var_precedence:
+	(cd targets/var_precedence && ./runme.sh $(TEST_FLAGS))
+
+test_vault:
+	(cd targets/vault && ./runme.sh $(TEST_FLAGS))
+
+test_delegate_to:
+	(cd targets/delegate_to && ./runme.sh $(TEST_FLAGS))
 
 # Split Windows CI targets to support parallel execution.
 # Targets should be balanced to have similar run times.
@@ -183,43 +167,20 @@ ci_win3: test_win_group3 test_connection_winrm
 
 test_winrm: test_win_group1 test_win_group2 test_win_group3
 
-test_win_group1: setup
+test_win_group1:
 	ansible-playbook test_win_group1.yml -i inventory.winrm -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
 
-test_win_group2: setup
+test_win_group2:
 	ansible-playbook test_win_group2.yml -i inventory.winrm -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
 
-test_win_group3: setup
+test_win_group3:
 	ansible-playbook test_win_group3.yml -i inventory.winrm -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
 
-test_tags: setup
-	# Run these using en_US.UTF-8 because list-tasks is a user output function and so it tailors its output to the user's locale.  For unicode tags, this means replacing non-ascii chars with "?"
-	# Run everything by default
-	[ "$$(LC_ALL=en_US.UTF-8 ansible-playbook --list-tasks test_tags.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v | fgrep Task_with | xargs)" = "Task_with_tag TAGS: [tag] Task_with_always_tag TAGS: [always] Task_with_unicode_tag TAGS: [くらとみ] Task_with_list_of_tags TAGS: [café, press] Task_without_tag TAGS: []" ]
-	# Run the exact tags, and always
-	[ "$$(LC_ALL=en_US.UTF-8 ansible-playbook --list-tasks --tags tag test_tags.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v | fgrep Task_with | xargs)" = "Task_with_tag TAGS: [tag] Task_with_always_tag TAGS: [always]" ]
-	# Skip one tag
-	[ "$$(LC_ALL=en_US.UTF-8 ansible-playbook --list-tasks --skip-tags tag test_tags.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v | fgrep Task_with | xargs)" = "Task_with_always_tag TAGS: [always] Task_with_unicode_tag TAGS: [くらとみ] Task_with_list_of_tags TAGS: [café, press] Task_without_tag TAGS: []" ]
-	# Skip a unicode tag
-	[ "$$(LC_ALL=en_US.UTF-8 ansible-playbook --list-tasks --skip-tags くらとみ test_tags.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v | fgrep Task_with | xargs)" = "Task_with_tag TAGS: [tag] Task_with_always_tag TAGS: [always] Task_with_list_of_tags TAGS: [café, press] Task_without_tag TAGS: []" ]
-	# Run just a unicode tag and always
-	[ "$$(LC_ALL=en_US.UTF-8 ansible-playbook --list-tasks --tags くらとみ test_tags.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v | fgrep Task_with | xargs)" = "Task_with_always_tag TAGS: [always] Task_with_unicode_tag TAGS: [くらとみ]" ]
-	# Run a tag from a list of tags and always
-	[ "$$(LC_ALL=en_US.UTF-8 ansible-playbook --list-tasks --tags café test_tags.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v | fgrep Task_with | xargs)" = "Task_with_always_tag TAGS: [always] Task_with_list_of_tags TAGS: [café, press]" ]
-
-blocks: setup
-	# remove old output log
-	rm -f block_test.out
-	# run the test and check to make sure the right number of completions was logged
-	ansible-playbook -vv $(TEST_FLAGS) -e outputdir=$(TEST_DIR) test_blocks/main.yml | tee block_test.out
-	env python -c 'import sys, re; sys.stdout.write(re.sub("\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]", "", sys.stdin.read()))' <block_test.out >block_test_wo_colors.out
-	[ "$$(grep 'TEST COMPLETE' block_test.out | wc -l | sed 's/ *//')" = "$$(egrep '^[0-9]+ plays in' block_test_wo_colors.out | cut -f1 -d' ')" ]
-	# cleanup the output log again, to make sure the test is clean
-	rm -f block_test.out block_test_wo_colors.out
-	# run test with free strategy and again count the completions
-	ansible-playbook -vv $(TEST_FLAGS) -e outputdir=$(TEST_DIR) test_blocks/main.yml -e test_strategy=free | tee block_test.out
-	env python -c 'import sys, re; sys.stdout.write(re.sub("\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]", "", sys.stdin.read()))' <block_test.out >block_test_wo_colors.out
-	[ "$$(grep 'TEST COMPLETE' block_test.out | wc -l | sed 's/ *//')" = "$$(egrep '^[0-9]+ plays in' block_test_wo_colors.out | cut -f1 -d' ')" ]
+test_tags:
+	(cd targets/tags && ./runme.sh $(TEST_FLAGS))
+
+blocks:
+	(cd targets/blocks && ./runme.sh $(TEST_FLAGS))
 
 cloud: amazon rackspace azure
 
@@ -328,65 +289,26 @@ test_galaxy_git: setup
     rm -rf $$mytmpdir ; \
     exit $$RC
 
-test_lookup_paths: setup
-	ansible-playbook lookup_paths/play.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) -v $(TEST_FLAGS)
+test_lookup_paths:
+	(cd targets/lookup_paths && ./runme.sh $(TEST_FLAGS))
 
-no_log: setup
-	# This test expects 7 loggable vars and 0 non loggable ones, if either mismatches it fails, run the ansible-playbook command to debug
-	[ "$$(ansible-playbook no_log_local.yml -i $(INVENTORY) -e outputdir=$(TEST_DIR) $(TEST_FLAGS) -vvvvv | awk --source 'BEGIN { logme = 0; nolog = 0; } /LOG_ME/ { logme += 1;} /DO_NOT_LOG/ { nolog += 1;} END { printf "%d/%d", logme, nolog; }')" = "26/0" ]
+no_log:
+	(cd targets/no_log && ./runme.sh $(TEST_FLAGS))
 
-test_binary_modules_winrm: INVENTORY = inventory.winrm
-test_binary_modules_winrm: test_binary_modules
+test_binary_modules_winrm:
+	(cd targets/binary_modules_winrm && ./runme.sh $(TEST_FLAGS))
 
-test_binary_modules:
-	ANSIBLE_HOST_KEY_CHECKING=false ansible-playbook download_binary_modules.yml -i $(INVENTORY) -v $(TEST_FLAGS)
-	ANSIBLE_HOST_KEY_CHECKING=false ansible-playbook test_binary_modules.yml -i $(INVENTORY) -v $(TEST_FLAGS)
+test_binary_modules_posix:
+	(cd targets/binary_modules_posix && ./runme.sh $(TEST_FLAGS))
 
-binary_modules:
-	# Compiled versions of these binary modules are available at the url below.
-	# This avoids a dependency on go and keeps the binaries out of our git repository.
-	# https://ansible-ci-files.s3.amazonaws.com/test/integration/roles/test_binary_modules/
-	cd library; \
-	GOOS=linux   GOARCH=amd64 go build -o helloworld_linux       helloworld.go; \
-	GOOS=windows GOARCH=amd64 go build -o helloworld_win32nt.exe helloworld.go; \
-	GOOS=darwin  GOARCH=amd64 go build -o helloworld_darwin      helloworld.go; \
-	GOOS=freebsd GOARCH=amd64 go build -o helloworld_freebsd     helloworld.go
+test_async_extra_data:
+	(cd targets/async_extra_data && ./runme.sh $(TEST_FLAGS))
 
-clean_binary_modules:
-	rm library/helloworld_*
+test_hosts_field:
+	(cd targets/hosts_field && ./runme.sh $(TEST_FLAGS))
 
-test_async:
-	# Verify that extra data before module JSON output during async call is ignored.
-	ANSIBLE_DEBUG=0 LC_ALL=bogus ansible-playbook test_async.yml -i localhost, -e ansible_connection=ssh -v $(TEST_FLAGS)
-	# Verify that the warning exists by examining debug output.
-	ANSIBLE_DEBUG=1 LC_ALL=bogus ansible-playbook test_async.yml -i localhost, -e ansible_connection=ssh -v $(TEST_FLAGS) \
-	| grep 'bash: warning: setlocale: LC_ALL: cannot change locale (bogus)' > /dev/null
+test_lookup_properties:
+	(cd targets/lookup_properties && ./runme.sh $(TEST_FLAGS))
 
-test_hosts_field:
-	# Hosts in playbook has a list of strings consisting solely of digits
-	ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e'target_kv=42' -e'{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" -t string_digit_host_in_list -v $(TEST_FLAGS) | tee test_hosts_field.out
-	grep 'Running on 42' test_hosts_field.out 2>&1
-	test `grep -c 'ok=1' test_hosts_field.out` = 1
-	# Hosts taken from kv extra_var on the CLI
-	ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e'target_kv=42' -e'{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" -t hosts_from_kv_string -v $(TEST_FLAGS) | tee test_hosts_field.out
-	grep 'Running on 42' test_hosts_field.out 2>&1
-	test `grep -c 'ok=1' test_hosts_field.out` = 1
-	# hosts is taken from an all digit json extra_vars string on the CLI
-	ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e'target_kv=42' -e'{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" -t hosts_from_cli_json_string -v $(TEST_FLAGS) | tee test_hosts_field.out
-	grep 'Running on 42' test_hosts_field.out 2>&1
-	test `grep -c 'ok=1' test_hosts_field.out` = 1
-	# hosts is taken from a json list in extra_vars on the CLI
-	ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e'target_kv=42' -e'{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" -t hosts_from_cli_json_list -v $(TEST_FLAGS) | tee test_hosts_field.out
-	grep 'Running on 42' test_hosts_field.out 2>&1
-	grep 'Running on localhost' test_hosts_field.out 2>&1
-	test `grep -c 'ok=1' test_hosts_field.out` = 2
-	# hosts is taken from a json string in an extra_vars file
-	ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e'target_kv=42' -e'{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" -t hosts_from_json_file_string -v $(TEST_FLAGS) | tee test_hosts_field.out
-	grep 'Running on 42' test_hosts_field.out 2>&1
-	test `grep -c 'ok=1' test_hosts_field.out` = 1
-	# hosts is taken from a json list in an extra_vars file
-	ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e'target_kv=42' -e'{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" -t hosts_from_json_file_list -v $(TEST_FLAGS) | tee test_hosts_field.out
-	grep 'Running on 42' test_hosts_field.out 2>&1
-	grep 'Running on localhost' test_hosts_field.out 2>&1
-	test `grep -c 'ok=1' test_hosts_field.out` = 2
-	rm test_hosts_field.out
+args:
+	(cd targets/args && ./runme.sh $(TEST_FLAGS))

test/integration/all.yml

@@ -1,6 +0,0 @@
-- include: test_setup.yml
-- include: non_destructive.yml
-- include: destructive.yml
-- include: rackspace.yml
-- include: amazon.yml
-- include: good_parsing.yml

test/integration/non_destructive.yml

@@ -19,7 +19,6 @@
     - { role: test_lookups, tags: test_lookups }
     - { role: test_changed_when, tags: test_changed_when }
     - { role: test_failed_when, tags: test_failed_when }
-    - { role: test_handlers, tags: test_handlers }
     - { role: test_until, tags: test_until }
     - { role: test_copy, tags: test_copy }
     - { role: test_stat, tags: test_stat }
@@ -44,3 +43,4 @@
     - { role: test_binary, tags: test_binary }
     - { role: test_loops, tags: test_loops }
     - { role: test_mount, tags: [test_mount, needs_root, needs_privileged]}
+    - { role: test_include_vars, tags: test_include_vars }

test/integration/roles/test_bad_parsing/meta/main.yml

@@ -1,2 +0,0 @@
-dependencies:
-  - { role: prepare_tests }

test/integration/roles/test_check_mode/meta/main.yml

@@ -1,3 +0,0 @@
-dependencies: 
-  - prepare_tests
-

test/integration/roles/test_handlers_meta/meta/main.yml

@@ -1,3 +0,0 @@
-dependencies: 
-  - prepare_tests
-

test/integration/targets/args/runme.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -eu
+
+echo "arg[#]: $#"
+echo "arg[0]: $0"
+
+i=0
+for arg in "$@"; do
+    i=$((i+1))
+    echo "arg[$i]: ${arg}"
+done

test/integration/targets/async_extra_data/runme.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# Verify that extra data before module JSON output during async call is ignored.
+ANSIBLE_DEBUG=0 LC_ALL=bogus ansible-playbook test_async.yml -i localhost, -e ansible_connection=ssh -v "$@"
+# Verify that the warning exists by examining debug output.
+ANSIBLE_DEBUG=1 LC_ALL=bogus ansible-playbook test_async.yml -i localhost, -e ansible_connection=ssh -v "$@" \
+    | grep 'bash: warning: setlocale: LC_ALL: cannot change locale (bogus)' > /dev/null

test/integration/targets/binary_modules/Makefile

@@ -0,0 +1,14 @@
+.PHONY: all clean
+
+all:
+	# Compiled versions of these binary modules are available at the url below.
+	# This avoids a dependency on go and keeps the binaries out of our git repository.
+	# https://ansible-ci-files.s3.amazonaws.com/test/integration/roles/test_binary_modules/
+	cd library; \
+	GOOS=linux   GOARCH=amd64 go build -o helloworld_linux       helloworld.go; \
+	GOOS=windows GOARCH=amd64 go build -o helloworld_win32nt.exe helloworld.go; \
+	GOOS=darwin  GOARCH=amd64 go build -o helloworld_darwin      helloworld.go; \
+	GOOS=freebsd GOARCH=amd64 go build -o helloworld_freebsd     helloworld.go
+
+clean:
+	rm -f library/helloworld_*

test/integration/targets/binary_modules/test.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -eux
+
+[ -f "${INVENTORY}" ]
+
+ANSIBLE_HOST_KEY_CHECKING=false ansible-playbook download_binary_modules.yml -i "${INVENTORY}" -v "$@"
+ANSIBLE_HOST_KEY_CHECKING=false ansible-playbook test_binary_modules.yml     -i "${INVENTORY}" -v "$@"

test/integration/targets/binary_modules_posix/runme.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -eux
+
+cd ../binary_modules
+INVENTORY=../../inventory ./test.sh "$@"

test/integration/targets/binary_modules_winrm/runme.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -eux
+
+cd ../binary_modules
+INVENTORY=../../inventory.winrm ./test.sh "$@"

test/integration/targets/blocks/runme.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# remove old output log
+rm -f block_test.out
+# run the test and check to make sure the right number of completions was logged
+ansible-playbook -vv main.yml -i ../../inventory "$@" | tee block_test.out
+env python -c \
+    'import sys, re; sys.stdout.write(re.sub("\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]", "", sys.stdin.read()))' \
+    <block_test.out >block_test_wo_colors.out
+[ "$(grep -c 'TEST COMPLETE' block_test.out)" = "$(egrep '^[0-9]+ plays in' block_test_wo_colors.out | cut -f1 -d' ')" ]
+# cleanup the output log again, to make sure the test is clean
+rm -f block_test.out block_test_wo_colors.out
+# run test with free strategy and again count the completions
+ansible-playbook -vv main.yml -i ../../inventory -e test_strategy=free "$@" | tee block_test.out
+env python -c \
+    'import sys, re; sys.stdout.write(re.sub("\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]", "", sys.stdin.read()))' \
+    <block_test.out >block_test_wo_colors.out
+[ "$(grep -c 'TEST COMPLETE' block_test.out)" = "$(egrep '^[0-9]+ plays in' block_test_wo_colors.out | cut -f1 -d' ')" ]

test/integration/targets/check_mode/check_mode.yml

@@ -1,4 +1,6 @@
 - hosts: testhost
+  vars:
+  - output_dir: .
   roles:
   - { role: test_always_run, tags: test_always_run }
   - { role: test_check_mode, tags: test_check_mode }

test/integration/targets/check_mode/roles/test_check_mode/tasks/main.yml

@@ -39,6 +39,10 @@
   template: src=foo.j2 dest={{output_dir}}/checkmode_foo.templated2 mode=0644
   register: template_result2
 
+- name: remove templated file
+  file: path={{output_dir}}/checkmode_foo.templated2 state=absent
+  check_mode: no
+
 - name: verify that the file was not changed
   assert: 
     that: 

test/integration/targets/check_mode/runme.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook check_mode.yml -i ../../inventory -v --check "$@"

test/integration/targets/connection/test.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -eux
+
+[ -f "${INVENTORY}" ]
+
+# Run connection tests with both the default and C locale.
+
+                ansible-playbook test_connection.yml -i "${INVENTORY}" "$@"
+LC_ALL=C LANG=C ansible-playbook test_connection.yml -i "${INVENTORY}" "$@"

test/integration/targets/connection/test_connection.yml

@@ -1,4 +1,4 @@
-- hosts: all
+- hosts: "{{ hosts }}"
   gather_facts: no
   serial: 1
   tasks:
@@ -17,24 +17,24 @@
   ### copy local file with unicode filename and content
 
   - name: create local file with unicode filename and content
-    local_action: lineinfile dest=/tmp/ansible-local-汉语/汉语.txt create=true line=汉语
+    local_action: lineinfile dest={{ local_tmp }}-汉语/汉语.txt create=true line=汉语
   - name: remove remote file with unicode filename and content
-    file: path=/tmp/ansible-remote-汉语/汉语.txt state=absent
+    action: "{{ action_prefix }}file path={{ remote_tmp }}-汉语/汉语.txt state=absent"
   - name: create remote directory with unicode name
-    file: path=/tmp/ansible-remote-汉语 state=directory
+    action: "{{ action_prefix }}file path={{ remote_tmp }}-汉语 state=directory"
   - name: copy local file with unicode filename and content
-    copy: src=/tmp/ansible-local-汉语/汉语.txt dest=/tmp/ansible-remote-汉语/汉语.txt
+    action: "{{ action_prefix }}copy src={{ local_tmp }}-汉语/汉语.txt dest={{ remote_tmp }}-汉语/汉语.txt"
 
   ### fetch remote file with unicode filename and content
 
   - name: remove local file with unicode filename and content
-    local_action: file path=/tmp/ansible-local-汉语/汉语.txt state=absent
+    local_action: file path={{ local_tmp }}-汉语/汉语.txt state=absent
   - name: fetch remote file with unicode filename and content
-    fetch: src=/tmp/ansible-remote-汉语/汉语.txt dest=/tmp/ansible-local-汉语/汉语.txt fail_on_missing=true validate_checksum=true flat=true
+    fetch: src={{ remote_tmp }}-汉语/汉语.txt dest={{ local_tmp }}-汉语/汉语.txt fail_on_missing=true validate_checksum=true flat=true
 
   ### remove local and remote temp files
 
   - name: remove local temp file
-    local_action: file path=/tmp/ansible-local-汉语 state=absent
+    local_action: file path={{ local_tmp }}-汉语 state=absent
   - name: remove remote temp file
-    file: path=/tmp/ansible-remote-汉语 state=absent
+    action: "{{ action_prefix }}file path={{ remote_tmp }}-汉语 state=absent"

test/integration/targets/connection_chroot/runme.sh

@@ -0,0 +1 @@
+../connection_posix/test.sh

test/integration/targets/connection_chroot/test_connection.inventory

@@ -0,0 +1,6 @@
+[chroot]
+chroot-pipelining    ansible_ssh_pipelining=true
+chroot-no-pipelining ansible_ssh_pipelining=false
+[chroot:vars]
+ansible_host=/
+ansible_connection=chroot

test/integration/targets/connection_docker/runme.sh

@@ -0,0 +1 @@
+../connection_posix/test.sh

test/integration/targets/connection_docker/test_connection.inventory

@@ -0,0 +1,6 @@
+[docker]
+docker-pipelining    ansible_ssh_pipelining=true
+docker-no-pipelining ansible_ssh_pipelining=false
+[docker:vars]
+ansible_host=ubuntu-latest
+ansible_connection=docker

test/integration/targets/connection_jail/runme.sh

@@ -0,0 +1 @@
+../connection_posix/test.sh

test/integration/targets/connection_jail/test_connection.inventory

@@ -0,0 +1,7 @@
+[jail]
+jail-pipelining    ansible_ssh_pipelining=true
+jail-no-pipelining ansible_ssh_pipelining=false
+[jail:vars]
+ansible_host=freebsd_10_2
+ansible_connection=jail
+ansible_python_interpreter=/usr/local/bin/python

test/integration/targets/connection_libvirt_lxc/runme.sh

@@ -0,0 +1 @@
+../connection_posix/test.sh

test/integration/targets/connection_libvirt_lxc/test_connection.inventory

@@ -0,0 +1,6 @@
+[libvirt_lxc]
+libvirt_lxc-pipelining    ansible_ssh_pipelining=true
+libvirt_lxc-no-pipelining ansible_ssh_pipelining=false
+[libvirt_lxc:vars]
+ansible_host=lv-ubuntu-wily-amd64
+ansible_connection=libvirt_lxc

test/integration/targets/connection_local/runme.sh

@@ -0,0 +1 @@
+../connection_posix/test.sh

test/integration/targets/connection_local/test_connection.inventory

@@ -0,0 +1,6 @@
+[local]
+local-pipelining    ansible_ssh_pipelining=true
+local-no-pipelining ansible_ssh_pipelining=false
+[local:vars]
+ansible_host=localhost
+ansible_connection=local

test/integration/targets/connection_lxc/runme.sh

@@ -0,0 +1 @@
+../connection_posix/test.sh

test/integration/targets/connection_lxc/test_connection.inventory

@@ -0,0 +1,17 @@
+[lxc]
+lxc-pipelining    ansible_ssh_pipelining=true
+lxc-no-pipelining ansible_ssh_pipelining=false
+[lxc:vars]
+# 1. install lxc
+# 2. install python2-lxc
+# $ pip install git+https://github.com/lxc/python2-lxc.git
+# 3. create container:
+# $ sudo lxc-create -t download -n centos-7-amd64 -- -d centos -r 7 -a amd64
+# 4. start container:
+# $ sudo lxc-start -n centos-7-amd64 -d
+# 5. run test:
+# $ sudo -E make test_connection_lxc
+# 6. stop container
+# $ sudo lxc-stop -n centos-7-amd64
+ansible_host=centos-7-amd64
+ansible_connection=lxc

test/integration/targets/connection_lxd/runme.sh

@@ -0,0 +1 @@
+../connection_posix/test.sh

test/integration/targets/connection_lxd/test_connection.inventory

@@ -0,0 +1,6 @@
+[lxd]
+lxd-pipelining    ansible_ssh_pipelining=true
+lxd-no-pipelining ansible_ssh_pipelining=false
+[lxd:vars]
+ansible_host=centos-7-amd64
+ansible_connection=lxd

test/integration/targets/connection_paramiko_ssh/runme.sh

@@ -0,0 +1 @@
+../connection_posix/test.sh

test/integration/targets/connection_paramiko_ssh/test_connection.inventory

@@ -0,0 +1,6 @@
+[paramiko_ssh]
+paramiko_ssh-pipelining    ansible_ssh_pipelining=true
+paramiko_ssh-no-pipelining ansible_ssh_pipelining=false
+[paramiko_ssh:vars]
+ansible_host=localhost
+ansible_connection=paramiko_ssh

test/integration/targets/connection_posix/test.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# Connection tests for POSIX platforms use this script by linking to it from the appropriate 'connection_' target dir.
+# The name of the inventory group to test is extracted from the directory name following the 'connection_' prefix.
+
+group=$(python -c \
+    "from os import path; print(path.basename(path.abspath(path.dirname('$0'))).replace('connection_', ''))")
+
+cd ../connection
+
+INVENTORY="../connection_${group}/test_connection.inventory" ./test.sh \
+    -e hosts="${group}" \
+    -e action_prefix= \
+    -e local_tmp=/tmp/ansible-local \
+    -e remote_tmp=/tmp/ansible-remote \
+    "$@"

test/integration/targets/connection_ssh/runme.sh

@@ -0,0 +1 @@
+../connection_posix/test.sh

test/integration/targets/connection_ssh/test_connection.inventory

@@ -0,0 +1,6 @@
+[ssh]
+ssh-pipelining    ansible_ssh_pipelining=true
+ssh-no-pipelining ansible_ssh_pipelining=false
+[ssh:vars]
+ansible_host=localhost
+ansible_connection=ssh

test/integration/targets/connection_winrm/runme.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -eux
+
+cd ../connection
+
+INVENTORY=../../inventory.winrm ./test.sh \
+    -e hosts=winrm \
+    -e action_prefix=win_ \
+    -e local_tmp=/tmp/ansible-local \
+    -e remote_tmp=c:/windows/temp/ansible-remote \
+    "$@"

test/integration/targets/delegate_to/roles/test_template/templates/foo.j2

@@ -0,0 +1,3 @@
+{{ templated_var }}
+
+{{ templated_dict | to_nice_json }}

test/integration/targets/delegate_to/runme.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ANSIBLE_HOST_KEY_CHECKING=false ansible-playbook test_delegate_to.yml -i ../../inventory -v "$@"

test/integration/targets/delegate_to/test_delegate_to.yml

@@ -1,9 +1,9 @@
 - hosts: testhost3
-  roles:
-    - { role: prepare_tests }
   vars:
     - template_role: ./roles/test_template
+    - output_dir: "{{ playbook_dir }}"
     - templated_var: foo
+    - templated_dict: { 'hello': 'world' }
   tasks:
     - name: Test no delegate_to
       setup:
@@ -38,7 +38,7 @@
       delegate_to: testhost4
 
     - name: Test file works with delegate_to and a host not in inventory
-      file: path={{ output_dir }}/test_follow_link mode=0644 state=touch
+      file: path={{ output_dir }}/tmp.txt mode=0644 state=touch
       delegate_to: 127.0.0.254
 
     - name: Test template works with delegate_to and a host in inventory
@@ -48,3 +48,9 @@
     - name: Test template works with delegate_to and a host not in inventory
       template: src={{ template_role }}/templates/foo.j2 dest={{ output_dir }}/foo.txt
       delegate_to: 127.0.0.254
+
+    - name: remove test file
+      file: path={{ output_dir }}/foo.txt state=absent
+
+    - name: remove test file
+      file: path={{ output_dir }}/tmp.txt state=absent

test/integration/targets/environment/runme.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook test_environment.yml -i ../../inventory "$@"

test/integration/targets/environment/test_environment.yml

@@ -2,8 +2,6 @@
   vars:
     - test1:
         key1: val1
-  roles:
-    - { role: prepare_tests }
   tasks:
     - name: check that envvar does not exist
       shell: echo $key1

test/integration/targets/gathering_facts/runme.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook test_gathering_facts.yml -i ../../inventory -v "$@"

test/integration/targets/group_by/runme.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook test_group_by.yml -i inventory.group_by -v "$@"

test/integration/targets/handlers/runme.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook test_handlers.yml -i inventory.handlers -v "$@" --tags scenario1
+
+[ "$(ansible-playbook test_handlers.yml -i inventory.handlers -v "$@" --tags scenario2 -l A \
+| egrep -o 'RUNNING HANDLER \[test_handlers : .*?]')" = "RUNNING HANDLER [test_handlers : test handler]" ]
+
+# Not forcing, should only run on successful host
+[ "$(ansible-playbook test_force_handlers.yml -i inventory.handlers -v "$@" --tags normal \
+| egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_B" ]
+
+# Forcing from command line
+[ "$(ansible-playbook test_force_handlers.yml -i inventory.handlers -v "$@" --tags normal --force-handlers \
+| egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_A CALLED_HANDLER_B" ]
+
+# Forcing from command line, should only run later tasks on unfailed hosts
+[ "$(ansible-playbook test_force_handlers.yml -i inventory.handlers -v "$@" --tags normal --force-handlers \
+| egrep -o CALLED_TASK_. | sort | uniq | xargs)" = "CALLED_TASK_B CALLED_TASK_D CALLED_TASK_E" ]
+
+# Forcing from command line, should call handlers even if all hosts fail
+[ "$(ansible-playbook test_force_handlers.yml -i inventory.handlers -v "$@" --tags normal --force-handlers -e fail_all=yes \
+| egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_A CALLED_HANDLER_B" ]
+
+# Forcing from ansible.cfg
+[ "$(ANSIBLE_FORCE_HANDLERS=true ansible-playbook test_force_handlers.yml -i inventory.handlers -v "$@" --tags normal \
+| egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_A CALLED_HANDLER_B" ]
+
+# Forcing true in play
+[ "$(ansible-playbook test_force_handlers.yml -i inventory.handlers -v "$@" --tags force_true_in_play \
+| egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_A CALLED_HANDLER_B" ]
+
+# Forcing false in play, which overrides command line
+[ "$(ansible-playbook test_force_handlers.yml -i inventory.handlers -v "$@" --tags force_false_in_play --force-handlers \
+| egrep -o CALLED_HANDLER_. | sort | uniq | xargs)" = "CALLED_HANDLER_B" ]

test/integration/targets/hash/runme.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -eux
+
+JSON_ARG='{"test_hash":{"extra_args":"this is an extra arg"}}'
+
+ANSIBLE_HASH_BEHAVIOUR=replace ansible-playbook test_hash.yml -i ../../inventory -v "$@" -e "${JSON_ARG}"
+ANSIBLE_HASH_BEHAVIOUR=merge   ansible-playbook test_hash.yml -i ../../inventory -v "$@" -e "${JSON_ARG}"

test/integration/targets/hosts_field/inventory.hosts_field

@@ -1,2 +1 @@
 42 ansible_host=127.0.0.42 ansible_connection=local
-

test/integration/targets/hosts_field/runme.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# Hosts in playbook has a list of strings consisting solely of digits
+ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e 'target_kv=42' \
+    -e '{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" \
+    -t string_digit_host_in_list -v "$@" | tee test_hosts_field.out
+grep 'Running on 42' test_hosts_field.out 2>&1
+test "$(grep -c 'ok=1' test_hosts_field.out)" = 1
+
+# Hosts taken from kv extra_var on the CLI
+ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e 'target_kv=42' \
+    -e '{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" \
+    -t hosts_from_kv_string -v "$@" | tee test_hosts_field.out
+grep 'Running on 42' test_hosts_field.out 2>&1
+test "$(grep -c 'ok=1' test_hosts_field.out)" = 1
+
+# hosts is taken from an all digit json extra_vars string on the CLI
+ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e 'target_kv=42' \
+    -e '{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" \
+    -t hosts_from_cli_json_string -v "$@" | tee test_hosts_field.out
+grep 'Running on 42' test_hosts_field.out 2>&1
+test "$(grep -c 'ok=1' test_hosts_field.out)" = 1
+
+# hosts is taken from a json list in extra_vars on the CLI
+ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e 'target_kv=42' \
+    -e '{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" \
+    -t hosts_from_cli_json_list -v "$@" | tee test_hosts_field.out
+grep 'Running on 42' test_hosts_field.out 2>&1
+grep 'Running on localhost' test_hosts_field.out 2>&1
+test "$(grep -c 'ok=1' test_hosts_field.out)" = 2
+
+# hosts is taken from a json string in an extra_vars file
+ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e 'target_kv=42' \
+    -e '{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" \
+    -t hosts_from_json_file_string -v "$@" | tee test_hosts_field.out
+grep 'Running on 42' test_hosts_field.out 2>&1
+test "$(grep -c 'ok=1' test_hosts_field.out)" = 1
+
+# hosts is taken from a json list in an extra_vars file
+ansible-playbook test_hosts_field.yml -i inventory.hosts_field -e 'target_kv=42' \
+    -e '{ "target_json_cli": "42", "target_json_cli_list": ["42", "localhost"] }' -e "@test_hosts_field.json" \
+    -t hosts_from_json_file_list -v "$@" | tee test_hosts_field.out
+grep 'Running on 42' test_hosts_field.out 2>&1
+grep 'Running on localhost' test_hosts_field.out 2>&1
+test "$(grep -c 'ok=1' test_hosts_field.out)" = 2
+
+rm test_hosts_field.out

test/integration/targets/includes/runme.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook test_includes.yml -i ../../inventory "$@"

test/integration/targets/lookup_paths/runme.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eux
+
+ansible-playbook play.yml -i ../../inventory -v "$@"