Use httptester docker image for http tests (#15811)

* Use httptester docker image for http tests

* When not running with an httptester linked container, use public test sites

test/integration/destructive.yml

@@ -21,4 +21,5 @@
     - { role: test_zypper, tags: test_zypper}
     - { role: test_zypper_repository, tags: test_zypper_repository}
     - { role: test_uri, tags: test_uri }
+    - { role: test_get_url, tags: test_get_url }
     - { role: test_apache2_module, tags: test_apache2_module }

test/integration/non_destructive.yml

@@ -38,7 +38,6 @@
     - { role: test_command_shell, tags: test_command_shell }
     - { role: test_script, tags: test_script }
     - { role: test_authorized_key, tags: test_authorized_key }
-    - { role: test_get_url, tags: test_get_url }
     - { role: test_embedded_module, tags: test_embedded_module }
     - { role: test_add_host, tags: test_add_host }
     - { role: test_binary, tags: test_binary }

test/integration/roles/prepare_http_tests/defaults/main.yml

@@ -0,0 +1,3 @@
+badssl_host: wrong.host.badssl.com
+httpbin_host: httpbin.org
+sni_host: sni.velox.ch

test/integration/roles/prepare_http_tests/tasks/main.yml

@@ -0,0 +1,35 @@
+# The docker --link functionality gives us an ENV var we can key off of to see if we have access to
+# the httptester container
+- set_fact:
+    has_httptester: "{{ lookup('env', 'ANSIBLE.HTTP.TESTS_PORT_80_TCP_ADDR') != '' }}"
+
+# If we are running with access to a httptester container, grab it's cacert and install it
+- block:
+    # Override hostname defaults with httptester linked names
+    - include_vars: httptester.yml
+
+    - name: RedHat - Enable the dynamic CA configuration feature
+      command: update-ca-trust force-enable
+      when: ansible_os_family == 'RedHat'
+
+    - name: RedHat - Retrieve test cacert
+      get_url:
+        url: "http://ansible.http.tests/cacert.pem"
+        dest: "/etc/pki/ca-trust/source/anchors/ansible.pem"
+      when: ansible_os_family == 'RedHat'
+
+    - name: Debian - Retrieve test cacert
+      get_url:
+        url: "http://ansible.http.tests/cacert.pem"
+        dest: "/usr/local/share/ca-certificates/ansible.crt"
+      when: ansible_os_family == 'Debian'
+
+    - name: Redhat - Update ca trust
+      command: update-ca-trust extract
+      when: ansible_os_family == 'RedHat'
+
+    - name: Debian - Update ca certificates
+      command: update-ca-certificates
+      when: ansible_os_family == 'Debian'
+
+  when: has_httptester|bool

test/integration/roles/prepare_http_tests/vars/httptester.yml

@@ -0,0 +1,4 @@
+# these are fake hostnames provided by docker link for the httptester container
+badssl_host: fail.ansible.http.tests
+httpbin_host: ansible.http.tests
+sni_host: sni1.ansible.http.tests

test/integration/roles/test_get_url/meta/main.yml

@@ -1,3 +1,4 @@
 dependencies:
   - prepare_tests
+  - prepare_http_tests
 

test/integration/roles/test_get_url/tasks/main.yml

@@ -66,7 +66,7 @@
         - result.failed
 
 - name: test https fetch
-  get_url: url="https://raw.githubusercontent.com/ansible/ansible/devel/README.md" dest={{output_dir}}/get_url.txt force=yes
+  get_url: url="https://{{ httpbin_host }}/get" dest={{output_dir}}/get_url.txt force=yes
   register: result
 
 - name: assert the get_url call was successful
@@ -77,16 +77,10 @@
 
 - name: test https fetch to a site with mismatched hostname and certificate
   get_url:
-    url: "https://www.kennethreitz.org/"
+    url: "https://{{ badssl_host }}/"
     dest: "{{ output_dir }}/shouldnotexist.html"
   ignore_errors: True
   register: result
-  # kennethreitz having trouble staying up.  Eventually need to install our own
-  # certs & web server to test this... also need to install and test it with
-  # a proxy so the complications are inevitable
-  until: "'read operation timed out' not in result.msg"
-  retries: 30
-  delay: 10
 
 - stat:
     path: "{{ output_dir }}/shouldnotexist.html"
@@ -101,16 +95,13 @@
 
 - name: test https fetch to a site with mismatched hostname and certificate and validate_certs=no
   get_url:
-    url: "https://www.kennethreitz.org/"
-    dest: "{{ output_dir }}/kreitz.html"
+    url: "https://{{ badssl_host }}/"
+    dest: "{{ output_dir }}/get_url_no_validate.html"
     validate_certs: no
   register: result
-  until: "'read operation timed out' not in result.msg"
-  retries: 30
-  delay: 10
 
 - stat:
-    path: "{{ output_dir }}/kreitz.html"
+    path: "{{ output_dir }}/get_url_no_validate.html"
   register: stat_result
 
 - name: Assert that the file was downloaded
@@ -119,48 +110,44 @@
       - "result.changed == true"
       - "stat_result.stat.exists == true"
 
-# At the moment, AWS can't make an https request to velox.ch... connection
-# timed out.  So we'll use a different test until/unless the problem is resolved
-## SNI Tests
-## SNI is only built into the stdlib from python-2.7.9 onwards
-#- name: Test that SNI works
-#  get_url:
-#    # A test site that returns a page with information on what SNI information
-#    # the client sent.  A failure would have the string: did not send a TLS server name indication extension
-#    url: 'https://foo.sni.velox.ch/'
-#    dest: "{{ output_dir }}/sni.html"
-#  register: get_url_result
-#  ignore_errors: True
-#
-#- command: "grep 'sent the following TLS server name indication extension' {{ output_dir}}/sni.html"
-#  register: data_result
-#  when: "{{ python_has_ssl_context }}"
-#
-#- debug: var=get_url_result
-#- name: Assert that SNI works with this python version
-#  assert:
-#    that:
-#      - 'data_result.rc == 0'
-#      - '"failed" not in get_url_result'
-#  when: "{{ python_has_ssl_context }}"
-#
-## If the client doesn't support SNI then get_url should have failed with a certificate mismatch
-#- name: Assert that hostname verification failed because SNI is not supported on this version of python
-#  assert:
-#    that:
-#      - 'get_url_result["failed"]'
-#  when: "{{ not python_has_ssl_context }}"
+# SNI Tests
+# SNI is only built into the stdlib from python-2.7.9 onwards
+- name: Test that SNI works
+  get_url:
+    url: 'https://{{ sni_host }}/'
+    dest: "{{ output_dir }}/sni.html"
+  register: get_url_result
+  ignore_errors: True
+
+- command: "grep '{{ sni_host }}' {{ output_dir}}/sni.html"
+  register: data_result
+  when: "{{ python_has_ssl_context }}"
+
+- debug: var=get_url_result
+- name: Assert that SNI works with this python version
+  assert:
+    that:
+      - 'data_result.rc == 0'
+      - '"failed" not in get_url_result'
+  when: "{{ python_has_ssl_context }}"
+
+# If the client doesn't support SNI then get_url should have failed with a certificate mismatch
+- name: Assert that hostname verification failed because SNI is not supported on this version of python
+  assert:
+    that:
+      - 'get_url_result["failed"]'
+  when: "{{ not python_has_ssl_context }}"
 
 # These tests are just side effects of how the site is hosted.  It's not
 # specifically a test site.  So the tests may break due to the hosting changing
 - name: Test that SNI works
   get_url:
-    url: 'https://www.mnot.net/blog/2014/05/09/if_you_can_read_this_youre_sniing'
+    url: 'https://{{ sni_host }}/'
     dest: "{{ output_dir }}/sni.html"
   register: get_url_result
   ignore_errors: True
 
-- command: "grep '<h2>If You Can Read This, You.re SNIing</h2>' {{ output_dir}}/sni.html"
+- command: "grep '{{ sni_host }}' {{ output_dir}}/sni.html"
   register: data_result
   when: "{{ python_has_ssl_context }}"
 
@@ -182,12 +169,12 @@
 
 - name: Test get_url with redirect
   get_url:
-    url: 'http://httpbin.org/redirect/6'
+    url: 'http://{{ httpbin_host }}/redirect/6'
     dest: "{{ output_dir }}/redirect.json"
 
 - name: Test that setting file modes work
   get_url:
-    url: 'http://httpbin.org/'
+    url: 'http://{{ httpbin_host }}/'
     dest: '{{ output_dir }}/test'
     mode: '0707'
   register: result
@@ -204,7 +191,7 @@
 
 - name: Test that setting file modes on an already downlaoded file work
   get_url:
-    url: 'http://httpbin.org/'
+    url: 'http://{{ httpbin_host }}/'
     dest: '{{ output_dir }}/test'
     mode: '0070'
   register: result

test/integration/roles/test_lookups/tasks/main.yml

@@ -84,7 +84,7 @@
 # ENV LOOKUP
 
 - name: get first environment var name
-  shell: env | head -n1 | cut -d\= -f1
+  shell: env | fgrep -v '.' | head -n1 | cut -d\= -f1
   register: known_var_name
 
 - name: get first environment var value

test/integration/roles/test_uri/meta/main.yml

@@ -1,2 +1,3 @@
 dependencies:
   - prepare_tests
+  - prepare_http_tests

test/integration/roles/test_uri/tasks/main.yml

@@ -94,16 +94,10 @@
 
 - name: test https fetch to a site with mismatched hostname and certificate
   uri:
-    url: "https://www.kennethreitz.org/"
+    url: "https://{{ badssl_host }}/"
     dest: "{{ output_dir }}/shouldnotexist.html"
   ignore_errors: True
   register: result
-  # kennethreitz having trouble staying up.  Eventually need to install our own
-  # certs & web server to test this... also need to install and test it with
-  # a proxy so the complications are inevitable
-  until: "'read operation timed out' not in result.msg"
-  retries: 30
-  delay: 10
 
 - stat:
     path: "{{ output_dir }}/shouldnotexist.html"
@@ -123,13 +117,10 @@
 
 - name: test https fetch to a site with mismatched hostname and certificate and validate_certs=no
   uri:
-    url: "https://www.kennethreitz.org/"
+    url: "https://{{ badssl_host }}/"
     dest: "{{ output_dir }}/kreitz.html"
     validate_certs: no
   register: result
-  until: "'read operation timed out' not in result.msg"
-  retries: 30
-  delay: 10
 
 - stat:
     path: "{{ output_dir }}/kreitz.html"
@@ -143,7 +134,7 @@
 
 - name: test redirect without follow_redirects
   uri:
-    url: 'http://httpbin.org/redirect/2'
+    url: 'http://{{ httpbin_host }}/redirect/2'
     follow_redirects: 'none'
     status_code: 302
   register: result
@@ -151,21 +142,21 @@
 - name: Assert location header
   assert:
     that:
-      - 'result.location|default("") == "http://httpbin.org/relative-redirect/1"'
+      - 'result.location|default("") == "http://{{ httpbin_host }}/relative-redirect/1"'
 
 - name: Check SSL with redirect
   uri:
-    url: 'https://httpbin.org/redirect/2'
+    url: 'https://{{ httpbin_host }}/redirect/2'
   register: result
 
 - name: Assert SSL with redirect
   assert:
     that:
-      - 'result.url|default("") == "https://httpbin.org/get"'
+      - 'result.url|default("") == "https://{{ httpbin_host }}/get"'
 
 - name: redirect to bad SSL site
   uri:
-    url: 'http://wrong.host.badssl.com'
+    url: 'http://{{ badssl_host }}'
   register: result
   ignore_errors: true
 
@@ -173,30 +164,30 @@
   assert:
     that:
       - result|failed
-      - '"wrong.host.badssl.com" in result.msg'
+      - 'badssl_host in result.msg'
 
 - name: test basic auth
   uri:
-    url: 'http://httpbin.org/basic-auth/user/passwd'
+    url: 'http://{{ httpbin_host }}/basic-auth/user/passwd'
     user: user
     password: passwd
 
 - name: test basic forced auth
   uri:
-    url: 'http://httpbin.org/hidden-basic-auth/user/passwd'
+    url: 'http://{{ httpbin_host }}/hidden-basic-auth/user/passwd'
     force_basic_auth: true
     user: user
     password: passwd
 
 - name: test PUT
   uri:
-    url: 'http://httpbin.org/put'
+    url: 'http://{{ httpbin_host }}/put'
     method: PUT
     body: 'foo=bar'
 
 - name: test OPTIONS
   uri:
-    url: 'http://httpbin.org/'
+    url: 'http://{{ httpbin_host }}/'
     method: OPTIONS
   register: result
 
@@ -217,7 +208,7 @@
 # test run.
 - name: Test that SNI succeeds on python versions that have SNI
   uri:
-    url: "{{ SNI_URI }}"
+    url: 'https://{{ sni_host }}/'
     return_content: true
   when: ansible_python.has_sslcontext
   register: result
@@ -226,12 +217,12 @@
   assert:
     that:
       - result|success
-    - "\"<h2>If You Can Read This, You're SNIing</h2>\" in result.content"
+      - 'sni_host == result.content'
   when: ansible_python.has_sslcontext
 
 - name: Verify SNI verification fails on old python without urllib3 contrib
   uri:
-    url: '{{ SNI_URI }}'
+    url: 'https://{{ sni_host }}'
   ignore_errors: true
   when: not ansible_python.has_sslcontext
   register: result
@@ -257,7 +248,7 @@
 
 - name: Verify SNI verificaiton succeeds on old python with urllib3 contrib
   uri:
-    url: '{{ SNI_URI }}'
+    url: 'https://{{ sni_host }}'
     return_content: true
   when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
   register: result
@@ -266,8 +257,7 @@
   assert:
     that:
       - result|success
-      #- '"Great! Your client" in result.content'
-      - "\"<h2>If You Can Read This, You're SNIing</h2>\" in result.content"
+      - 'sni_host == result.content'
   when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
 
 - name: Uninstall ndg-httpsclient and urllib3
@@ -287,7 +277,7 @@
 
 - name: validate the status_codes are correct
   uri:
-    url: https://httpbin.org/status/202
+    url: "https://{{ httpbin_host }}/status/202"
     status_code: 202
     method: POST
     body: foo

test/utils/run_tests.sh

@@ -4,6 +4,8 @@ set -e
 set -u
 set -x
 
+LINKS="--link=httptester:ansible.http.tests --link=httptester:sni1.ansible.http.tests --link=httptester:sni2.ansible.http.tests --link=httptester:fail.ansible.http.tests"
+
 if [ "${TARGET}" = "sanity" ]; then
     ./test/code-smell/replace-urlopen.sh .
     ./test/code-smell/use-compat-six.sh lib
@@ -12,9 +14,13 @@ if [ "${TARGET}" = "sanity" ]; then
     if test x"$TOXENV" != x'py24' ; then tox ; fi
     if test x"$TOXENV" = x'py24' ; then python2.4 -V && python2.4 -m compileall -fq -x 'module_utils/(a10|rax|openstack|ec2|gce|docker_common|azure_rm_common).py' lib/ansible/module_utils ; fi
 else
+    if [ ! -e /tmp/cid_httptester ]; then
+        docker pull sivel/httptester
+        docker run -d --name=httptester sivel/httptester > /tmp/cid_httptester
+    fi
     export C_NAME="testAbull_$$_$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 8 | head -n 1)"
     docker pull ansible/ansible:${TARGET}
-    docker run -d --volume="${PWD}:/root/ansible:Z"  --name "${C_NAME}" ${TARGET_OPTIONS:=''} ansible/ansible:${TARGET} > /tmp/cid_${TARGET}
+    docker run -d --volume="${PWD}:/root/ansible:Z" $LINKS --name "${C_NAME}" ${TARGET_OPTIONS:=''} ansible/ansible:${TARGET} > /tmp/cid_${TARGET}
     docker exec -ti $(cat /tmp/cid_${TARGET}) /bin/sh -c "export TEST_FLAGS='${TEST_FLAGS:-''}'; cd /root/ansible; . hacking/env-setup; (cd test/integration; LC_ALL=en_US.utf-8 make ${MAKE_TARGET:-})"
     docker kill $(cat /tmp/cid_${TARGET})