issue #164: convert "examples" into actual tests

- Add new Travis mode, "ansible_tests.sh" that runs
  integrations/all.yml. Slowly build this up over time to cover more of
  the existing junk.

- Add basic assertions on the output of the existing runner__* files.

- Wire up 2.4.3/2.5.0 jobs in Travis.

.travis.yml

@@ -12,6 +12,8 @@ python:
 env:
 - MODE=mitogen
 - MODE=debops
+- MODE=ansible ANSIBLE_VERSION=2.4.3.0
+- MODE=ansible ANSIBLE_VERSION=2.5.0
 
 install:
 - pip install -r dev_requirements.txt

.travis/ansible_tests.sh

@@ -0,0 +1,45 @@
+#!/bin/bash -ex
+# Run tests/ansible/integration/all.yml under Ansible and Ansible-Mitogen
+
+TRAVIS_BUILD_DIR="${TRAVIS_BUILD_DIR:-`pwd`}"
+TMPDIR="/tmp/ansible-tests-$$"
+ANSIBLE_VERSION="${ANSIBLE_VERSION:-2.4.3.0}"
+
+function on_exit()
+{
+    rm -rf "$TMPDIR"
+    docker kill target || true
+}
+
+trap on_exit EXIT
+mkdir "$TMPDIR"
+
+
+echo travis_fold:start:docker_setup
+docker run --rm --detach --name=target d2mw/mitogen-test /bin/sleep 86400
+echo travis_fold:end:docker_setup
+
+
+echo travis_fold:start:job_setup
+pip install -U ansible==${ANSIBLE_VERSION}"
+cd ${TRAVIS_BUILD_DIR}/tests/ansible
+
+cat >> ${TMPDIR}/hosts <<-EOF
+localhost
+target ansible_connection=docker ansible_python_interpreter=/usr/bin/python2.7
+EOF
+echo travis_fold:end:job_setup
+
+
+echo travis_fold:start:mitogen_linear
+ANSIBLE_STRATEGY=mitogen_linear /usr/bin/time ansible-playbook \
+    integration/all.yml \
+    -i "${TMPDIR}/hosts"
+echo travis_fold:end:mitogen_linear
+
+
+echo travis_fold:start:vanilla_ansible
+/usr/bin/time ansible-playbook \
+    integration/all.yml \
+    -i "${TMPDIR}/hosts"
+echo travis_fold:end:vanilla_ansible

tests/ansible/integration/action__low_level_execute_command.yml

@@ -2,6 +2,10 @@
 
 - hosts: all
   tasks:
+    - name: integration/action__low_level_execute_command.yml
+      assert:
+        that: true
+
     # "echo -en" to test we actually hit bash shell too.
     - name: Run raw module without sudo
       raw: 'echo -en $((1 + 1))'

tests/ansible/integration/runner__builtin_command_module.yml

@@ -1,5 +1,16 @@
 - hosts: all
+  gather_facts: true
   tasks:
-    - name: builtin_command_module
+    - name: integration/runner__builtin_command_module.yml
       command: hostname
-      with_sequence: start=1 end={{end|default(100)}}
+      with_sequence: start=1 end={{end|default(1)}}
+      register: out
+
+    - assert:
+        that: |
+          out.changed and
+          out.results[0].changed and
+          out.results[0].cmd == ['hostname'] and
+          out.results[0].item == '1' and
+          out.results[0].rc == 0 and
+          (out.results[0].stdout == ansible_nodename)

tests/ansible/integration/runner__custom_bash_old_style_module.yml

@@ -1,5 +1,13 @@
 - hosts: all
   tasks:
-    - custom_bash_old_style_module:
+    - name: integration/runner__custom_bash_old_style_module.yml
+      custom_bash_old_style_module:
         foo: true
-      with_sequence: start=1 end={{end|default(100)}}
+      with_sequence: start=1 end={{end|default(1)}}
+      register: out
+
+    - assert:
+        that: |
+          (not out.changed) and
+          (not out.results[0].changed) and
+          out.results[0].msg == 'Here is my input'

tests/ansible/integration/runner__custom_bash_want_json_module.yml

@@ -1,5 +1,13 @@
 - hosts: all
   tasks:
-    - custom_bash_want_json_module:
+    - name: integration/runner__custom_bash_want_json_module.yml
+      custom_bash_want_json_module:
         foo: true
-      with_sequence: start=1 end={{end|default(100)}}
+      with_sequence: start=1 end={{end|default(1)}}
+      register: out
+
+    - assert:
+        that: |
+          (not out.changed) and
+          (not out.results[0].changed) and
+          out.results[0].msg == 'Here is my input'

tests/ansible/integration/runner__custom_binary_producing_json.yml

@@ -1,5 +1,13 @@
 - hosts: all
   tasks:
-    - custom_binary_producing_json:
+    - name: integration/runner__custom_binary_producing_json.yml
+      custom_binary_producing_json:
         foo: true
-      with_sequence: start=1 end={{end|default(100)}}
+      with_sequence: start=1 end={{end|default(1)}}
+      register: out
+
+    - assert:
+        that: |
+          out.changed and
+          out.results[0].changed and
+          out.results[0].msg == 'Hello, world.'

tests/ansible/integration/runner__custom_binary_producing_junk.yml

@@ -1,6 +1,15 @@
 - hosts: all
   tasks:
-    - custom_binary_producing_junk:
+    - name: integration/runner__custom_binary_producing_junk.yml
+      custom_binary_producing_junk:
         foo: true
-      with_sequence: start=1 end={{end|default(100)}}
+      with_sequence: start=1 end={{end|default(1)}}
       ignore_errors: true
+      register: out
+
+    - assert:
+        that: |
+          out.failed and
+          out.results[0].failed and
+          out.results[0].msg == 'MODULE FAILURE' and
+          out.results[0].rc == 0

tests/ansible/integration/runner__custom_binary_single_null.yml

@@ -1,6 +1,15 @@
 - hosts: all
   tasks:
-    - custom_binary_single_null:
+    - name: integration/runner__custom_binary_single_null.yml
+      custom_binary_single_null:
         foo: true
-      with_sequence: start=1 end={{end|default(100)}}
+      with_sequence: start=1 end={{end|default(1)}}
       ignore_errors: true
+      register: out
+
+    - assert:
+        that: |
+          out.failed and
+          out.results[0].failed and
+          out.results[0].msg == 'MODULE FAILURE' and
+          out.results[0].rc == 126

tests/ansible/integration/runner__custom_perl_json_args_module.yml

@@ -1,5 +1,14 @@
 - hosts: all
   tasks:
-    - custom_perl_json_args_module:
+    - name: integration/runner__custom_perl_json_args_module.yml
+      custom_perl_json_args_module:
         foo: true
-      with_sequence: start=1 end={{end|default(100)}}
+      with_sequence: start=1 end={{end|default(1)}}
+      register: out
+
+    - assert:
+        that: |
+          (not out.changed) and
+          (not out.results[0].changed) and
+          out.results[0].input[0].foo and
+          out.results[0].message == 'I am a perl script! Here is my input.'

tests/ansible/integration/runner__custom_perl_want_json_module.yml

@@ -1,5 +1,14 @@
 - hosts: all
   tasks:
-    - custom_perl_want_json_module:
+    - name: integration/runner__custom_perl_want_json_module.yml
+      custom_perl_want_json_module:
         foo: true
-      with_sequence: start=1 end={{end|default(100)}}
+      with_sequence: start=1 end={{end|default(1)}}
+      register: out
+
+    - assert:
+        that: |
+          (not out.changed) and
+          (not out.results[0].changed) and
+          out.results[0].input[0].foo and
+          out.results[0].message == 'I am a want JSON perl script! Here is my input.'

tests/ansible/integration/runner__custom_python_json_args_module.yml

@@ -1,5 +1,14 @@
 - hosts: all
   tasks:
-    - custom_python_json_args_module:
+    - name: integration/runner__custom_python_json_args_module.yml
+      custom_python_json_args_module:
         foo: true
-      with_sequence: start=1 end={{end|default(100)}}
+      with_sequence: start=1 end={{end|default(1)}}
+      register: out
+
+    - assert:
+        that: |
+          (not out.changed) and
+          (not out.results[0].changed) and
+          out.results[0].input[0].foo and
+          out.results[0].msg == 'Here is my input'

tests/ansible/integration/runner__custom_python_new_style_module.yml

@@ -1,5 +1,14 @@
 - hosts: all
   tasks:
-    - custom_python_new_style_module:
+    - name: integration/runner__custom_python_new_style_module.yml
+      custom_python_new_style_module:
         foo: true
-      with_sequence: start=1 end={{end|default(100)}}
+      with_sequence: start=1 end={{end|default(1)}}
+      register: out
+
+    - assert:
+        that: |
+          (not out.changed) and
+          (not out.results[0].changed) and
+          out.results[0].input[0].ANSIBLE_MODULE_ARGS.foo and
+          out.results[0].msg == 'Here is my input'

tests/ansible/integration/runner__custom_python_want_json_module.yml

@@ -1,5 +1,14 @@
 - hosts: all
   tasks:
-    - custom_python_want_json_module:
+    - name: integration/runner__custom_python_want_json_module.yml
+      custom_python_want_json_module:
         foo: true
-      with_sequence: start=1 end={{end|default(100)}}
+      with_sequence: start=1 end={{end|default(1)}}
+      register: out
+
+    - assert:
+        that: |
+          (not out.changed) and
+          (not out.results[0].changed) and
+          out.results[0].input[0].foo and
+          out.results[0].msg == 'Here is my input'