Add integration tests for test plugins. (#70576)

test/integration/targets/test_core/inventory

@@ -0,0 +1 @@
+unreachable ansible_connection=ssh ansible_host=127.0.0.1 ansible_port=1011  # IANA Reserved port

test/integration/targets/test_core/runme.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eu
+
+ANSIBLE_ROLES_PATH=../ ansible-playbook --vault-password-file vault-password runme.yml -i inventory "${@}"

test/integration/targets/test_core/runme.yml

@@ -0,0 +1,4 @@
+- hosts: localhost
+  gather_facts: no
+  roles:
+    - test_core

test/integration/targets/test_core/tasks/main.yml

@@ -1,3 +1,254 @@
+- name: Failure
+  set_fact:
+    hello: world
+  failed_when: true
+  ignore_errors: yes
+  register: intentional_failure
+
+- name: Success
+  set_fact:
+    hello: world
+  register: intentional_success
+
+- name: Try failure test on non-dictionary
+  set_fact:
+    hello: "{{ 'nope' is failure }}"
+  ignore_errors: yes
+  register: misuse_of_failure
+
+- name: Assert failure tests work
+  assert:
+    that:
+      - intentional_failure is failed  # old name
+      - intentional_failure is failure
+      - intentional_success is not failure
+      - misuse_of_failure is failed
+
+- name: Assert successful tests work
+  assert:
+    that:
+      - intentional_success is succeeded  # old name
+      - intentional_success is success  # old name
+      - intentional_success is successful
+      - intentional_failure is not successful
+
+- name: Try reachable host
+  command: id
+  register: reachable_host
+
+- name: Try unreachable host
+  command: id
+  delegate_to: unreachable
+  ignore_unreachable: yes
+  ignore_errors: yes
+  register: unreachable_host
+
+- name: Try reachable test on non-dictionary
+  set_fact:
+    hello: "{{ 'nope' is reachable }}"
+  ignore_errors: yes
+  register: misuse_of_reachable
+
+- name: Assert reachable tests work
+  assert:
+    that:
+      - misuse_of_reachable is failed
+      - reachable_host is reachable
+      - unreachable_host is not reachable
+
+- name: Try unreachable test on non-dictionary
+  set_fact:
+    hello: "{{ 'nope' is unreachable }}"
+  ignore_errors: yes
+  register: misuse_of_unreachable
+
+- name: Assert unreachable tests work
+  assert:
+    that:
+      - misuse_of_unreachable is failed
+      - reachable_host is not unreachable
+      - unreachable_host is unreachable
+
+- name: Make changes
+  file:
+    path: dir_for_changed
+    state: directory
+  register: directory_created
+
+- name: Make no changes
+  file:
+    path: dir_for_changed
+    state: directory
+  register: directory_unchanged
+
+- name: Try changed test on non-dictionary
+  set_fact:
+    hello: "{{ 'nope' is changed }}"
+  ignore_errors: yes
+  register: misuse_of_changed
+
+# providing artificial task results since there are no modules in ansible-base that provide a 'results' list instead of 'changed'
+- name: Prepare artificial task results
+  set_fact:
+    results_all_changed:
+      results:
+        - changed: true
+        - changed: true
+    results_some_changed:
+      results:
+        - changed: true
+        - changed: false
+    results_none_changed:
+      results:
+        - changed: false
+        - changed: false
+    results_missing_changed: {}
+
+- name: Assert changed tests work
+  assert:
+    that:
+      - directory_created is changed
+      - directory_unchanged is not changed
+      - misuse_of_changed is failed
+      - results_all_changed is changed
+      - results_some_changed is changed
+      - results_none_changed is not changed
+      - results_missing_changed is not changed
+
+- name: Skip me
+  set_fact:
+    hello: world
+  when: false
+  register: skipped_task
+
+- name: Don't skip me
+  set_fact:
+    hello: world
+  register: executed_task
+
+- name: Try skipped test on non-dictionary
+  set_fact:
+    hello: "{{ 'nope' is skipped }}"
+  ignore_errors: yes
+  register: misuse_of_skipped
+
+- name: Assert skipped tests work
+  assert:
+    that:
+      - skipped_task is skipped
+      - executed_task is not skipped
+      - misuse_of_skipped is failure
+
+- name: Not an async task
+  set_fact:
+    hello: world
+  register: non_async_task
+
+- name: Complete an async task
+  command: id
+  async: 10
+  poll: 1
+  register: async_completed
+
+- name: Start an async task without waiting for completion
+  shell: sleep 3
+  async: 10
+  poll: 0
+  register: async_incomplete
+
+- name: Try finished test on non-dictionary
+  set_fact:
+    hello: "{{ 'nope' is finished }}"
+  ignore_errors: yes
+  register: misuse_of_finished
+
+- name: Assert finished tests work (warning expected)
+  assert:
+    that:
+      - non_async_task is finished
+      - misuse_of_finished is failed
+      - async_completed is finished
+      - async_incomplete is not finished
+
+- name: Try started test on non-dictionary
+  set_fact:
+    hello: "{{ 'nope' is started }}"
+  ignore_errors: yes
+  register: misuse_of_started
+
+- name: Assert started tests work (warning expected)
+  assert:
+    that:
+      - non_async_task is started
+      - misuse_of_started is failed
+      - async_completed is started
+      - async_incomplete is started
+
+- name: Assert match tests work
+  assert:
+    that:
+      - "'hello' is match('h.ll.')"
+      - "'hello' is not match('.ll.')"
+
+- name: Assert search tests work
+  assert:
+    that:
+      - "'hello' is search('.l')"
+      - "'hello' is not search('nope')"
+
+- name: Assert regex tests work
+  assert:
+    that:
+      - "'hello' is regex('.l')"
+      - "'hello' is regex('.L', ignorecase=true)"
+      - "'hello\nAnsible' is regex('^Ansible', multiline=true)"
+      - "'hello' is not regex('.L')"
+      - "'hello\nAnsible' is not regex('^Ansible')"
+
+- name: Try version tests with bad operator
+  set_fact:
+    result: "{{ '1.0' is version('1.0', 'equals') }}"
+  ignore_errors: yes
+  register: version_bad_operator
+
+- name: Try version tests with bad value
+  set_fact:
+    result: "{{ '1.0' is version('nope', '==', true) }}"
+  ignore_errors: yes
+  register: version_bad_value
+
+- name: Assert version tests work
+  assert:
+    that:
+      - "'1.0' is version_compare('1.0', '==')"  # old name
+      - "'1.0' is version('1.0', '==')"
+      - "'1.0' is version('2.0', '!=')"
+      - "'1.0' is version('2.0', '<')"
+      - "'2.0' is version('1.0', '>')"
+      - "'1.0' is version('1.0', '<=')"
+      - "'1.0' is version('1.0', '>=')"
+      - "'1.0' is version_compare('1.0', '==', true)"  # old name
+      - "'1.0' is version('1.0', '==', true)"
+      - "'1.0' is version('2.0', '!=', true)"
+      - "'1.0' is version('2.0', '<', true)"
+      - "'2.0' is version('1.0', '>', true)"
+      - "'1.0' is version('1.0', '<=', true)"
+      - "'1.0' is version('1.0', '>=', true)"
+      - version_bad_operator is failed
+      - version_bad_value is failed
+
+- name: Assert any tests work
+  assert:
+    that:
+      - "[true, false] is any"
+      - "[false] is not any"
+
+- name: Assert all tests work
+  assert:
+    that:
+      - "[true] is all"
+      - "[true, false] is not all"
+
 - name: Assert truthy tests work
   assert:
     that:
@@ -13,6 +264,7 @@
       - '[] is not truthy'
       - '"on" is truthy(convert_bool=True)'
       - '"off" is not truthy(convert_bool=True)'
+      - '"fred" is truthy(convert_bool=True)'
       - '{} is not truthy'
       - '{"key": "value"} is truthy'      
 
@@ -33,3 +285,19 @@
       - '"off" is falsy(convert_bool=True)'
       - '{} is falsy'
       - '{"key": "value"} is not falsy'
+
+- name: Create vaulted variable for vault_encrypted test
+  set_fact:
+    vaulted_value: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          35323961353038346165643738646465376139363061353835303739663538343266303232326635
+          3365353662646236356665323135633630656238316530640a663362363763633436373439663031
+          33663433383037396438656464636433653837376361313638366362333037323961316364363363
+          3835616438623261650a636164376534376661393134326662326362323131373964313961623365
+          3833
+
+- name: Assert vault_encrypted tests work
+  assert:
+    that:
+      - vaulted_value is vault_encrypted
+      - inventory_hostname is not vault_encrypted

test/integration/targets/test_core/vault-password

@@ -0,0 +1 @@
+test-vault-password

test/integration/targets/test_files/aliases

@@ -0,0 +1,2 @@
+shippable/posix/group5
+skip/python2.6  # tests are controller only, and we no longer support Python 2.6 on the controller

test/integration/targets/test_files/tasks/main.yml

@@ -0,0 +1,60 @@
+- name: Create a broken symbolic link
+  file:
+    src: does_not_exist
+    dest: link_to_nonexistent_file
+    state: link
+    force: yes
+    follow: no
+
+- name: Assert directory tests work
+  assert:
+    that:
+      - "'.' is is_dir"  # old name
+      - "'.' is directory"
+      - "'does_not_exist' is not directory"
+
+- name: Assert file tests work
+  assert:
+    that:
+      - "(role_path + '/aliases') is is_file"  # old name
+      - "(role_path + '/aliases') is file"
+      - "'does_not_exist' is not file"
+
+- name: Assert link tests work
+  assert:
+    that:
+      - "'link_to_nonexistent_file' is link"
+      - "'.' is not link"
+
+- name: Assert exists tests work
+  assert:
+    that:
+      - "(role_path + '/aliases') is exists"
+      - "'link_to_nonexistent_file' is not exists"
+
+- name: Assert link_exists tests work
+  assert:
+    that:
+      - "'link_to_nonexistent_file' is link_exists"
+      - "'does_not_exist' is not link_exists"
+
+- name: Assert abs tests work
+  assert:
+    that:
+      - "'/' is is_abs"  # old name
+      - "'/' is abs"
+      - "'../' is not abs"
+
+- name: Assert same_file tests work
+  assert:
+    that:
+      - "'/' is is_same_file('/')"  # old name
+      - "'/' is same_file('/')"
+      - "'/' is not same_file(role_path + '/aliases')"
+
+- name: Assert mount tests work
+  assert:
+    that:
+      - "'/' is is_mount"  # old name
+      - "'/' is mount"
+      - "'/does_not_exist' is not mount"

test/integration/targets/test_mathstuff/aliases

@@ -0,0 +1,2 @@
+shippable/posix/group5
+skip/python2.6  # tests are controller only, and we no longer support Python 2.6 on the controller

test/integration/targets/test_mathstuff/tasks/main.yml

@@ -0,0 +1,38 @@
+- name: Get Jinja2 version
+  set_fact:
+    jinja2_version: >-
+                    {{ lookup('pipe', '{{ ansible_playbook_python }} -c "import jinja2; print(jinja2.__version__)"') }}
+
+- name: Assert subset tests work
+  assert:
+    that:
+      - "[1] is issubset([1, 2])"  # old name
+      - "[1] is subset([1, 2])"
+      - "[1] is not subset([2])"
+
+- name: Assert superset tests work
+  assert:
+    that:
+      - "[1, 2] is issuperset([1])"  # old name
+      - "[1, 2] is superset([1])"
+      - "[2] is not superset([1])"
+
+- name: Assert contains tests work
+  assert:
+    that:
+      - "[1] is contains(1)"
+      - "[1] is not contains(2)"
+
+- name: Assert nan tests work
+  assert:
+    that:
+      - "'bad' is not nan"
+      - "1.1 | float is not nan"
+
+# Jinja2 versions prior to 2.10 will traceback when using: 'nan' | float
+- name: Assert nan tests work (Jinja2 2.10+)
+  assert:
+    that:
+      - "'nan' | float is isnan"  # old name
+      - "'nan' | float is nan"
+  when: jinja2_version is version('2.10', '>=')