Urls client cert auth (#18141)

* Build HTTPSClientAuthHandler more similarly to how HTTPSHandler works

* Add docs for new client cert authentication

* Support older versions of python

* Simplify logic

* Initial support for client certs in urls.py

* Add an extra test

* Add a get_url test for client cert auth

* Add additional test for client cert auth, with validation and ssl mismatch

* Skip assert when http tester not available

* Update version_added for new options

lib/ansible/module_utils/urls.py

@@ -411,6 +411,33 @@ if hasattr(httplib, 'HTTPSConnection') and hasattr(urllib_request, 'HTTPSHandler
         https_request = AbstractHTTPHandler.do_request_
 
 
+class HTTPSClientAuthHandler(urllib_request.HTTPSHandler):
+    '''Handles client authentication via cert/key
+
+    This is a fairly lightweight extension on HTTPSHandler, and can be used
+    in place of HTTPSHandler
+    '''
+
+    def __init__(self, client_cert=None, client_key=None, **kwargs):
+        urllib_request.HTTPSHandler.__init__(self, **kwargs)
+        self.client_cert = client_cert
+        self.client_key = client_key
+
+    def https_open(self, req):
+        return self.do_open(self._build_https_connection, req)
+
+    def _build_https_connection(self, host, **kwargs):
+        kwargs.update({
+            'cert_file': self.client_cert,
+            'key_file': self.client_key,
+        })
+        try:
+            kwargs['context'] = self._context
+        except AttributeError:
+            pass
+        return httplib.HTTPSConnection(host, **kwargs)
+
+
 def generic_urlparse(parts):
     '''
     Returns a dictionary of url parts as parsed by urlparse,
@@ -796,7 +823,8 @@ def maybe_add_ssl_handler(url, validate_certs):
 def open_url(url, data=None, headers=None, method=None, use_proxy=True,
              force=False, last_mod_time=None, timeout=10, validate_certs=True,
              url_username=None, url_password=None, http_agent=None,
-             force_basic_auth=False, follow_redirects='urllib2'):
+             force_basic_auth=False, follow_redirects='urllib2',
+             client_cert=None, client_key=None):
     '''
     Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3)
 
@@ -875,7 +903,12 @@ def open_url(url, data=None, headers=None, method=None, use_proxy=True,
         context.options |= ssl.OP_NO_SSLv3
         context.verify_mode = ssl.CERT_NONE
         context.check_hostname = False
-        handlers.append(urllib_request.HTTPSHandler(context=context))
+        handlers.append(HTTPSClientAuthHandler(client_cert=client_cert,
+                                               client_key=client_key,
+                                               context=context))
+    elif client_cert:
+        handlers.append(HTTPSClientAuthHandler(client_cert=client_cert,
+                                               client_key=client_key))
 
     # pre-2.6 versions of python cannot use the custom https
     # handler, since the socket class is lacking create_connection.
@@ -952,7 +985,8 @@ def url_argument_spec():
         url_username=dict(required=False),
         url_password=dict(required=False, no_log=True),
         force_basic_auth=dict(required=False, type='bool', default='no'),
-
+        client_cert=dict(required=False, type='path', default=None),
+        client_key=dict(required=False, type='path', default=None),
     )
 
 
@@ -1001,6 +1035,9 @@ def fetch_url(module, url, data=None, headers=None, method=None,
 
     follow_redirects = module.params.get('follow_redirects', 'urllib2')
 
+    client_cert = module.params.get('client_cert')
+    client_key = module.params.get('client_key')
+
     r = None
     info = dict(url=url)
     try:
@@ -1008,7 +1045,8 @@ def fetch_url(module, url, data=None, headers=None, method=None,
                      use_proxy=use_proxy, force=force, last_mod_time=last_mod_time, timeout=timeout,
                      validate_certs=validate_certs, url_username=username,
                      url_password=password, http_agent=http_agent, force_basic_auth=force_basic_auth,
-                     follow_redirects=follow_redirects)
+                     follow_redirects=follow_redirects, client_cert=client_cert,
+                     client_key=client_key)
         info.update(r.info())
         info.update(dict(msg="OK (%s bytes)" % r.headers.get('Content-Length', 'unknown'), url=r.geturl(), status=r.code))
     except NoSSLError:

lib/ansible/modules/network/basics/get_url.py

@@ -156,6 +156,22 @@ options:
     required: false
     choices: [ "yes", "no" ]
     default: "no"
+  client_cert:
+    required: false
+    default: null
+    description:
+      - PEM formatted certificate chain file to be used for SSL client
+        authentication. This file can also include the key as well, and if
+        the key is included, I(client_key) is not required
+    version_added: 2.4
+  client_key:
+    required: false
+    default: null
+    description:
+      - PEM formatted file that contains your private key to be used for SSL
+        client authentication. If I(client_cert) contains both the certificate
+        and key, this option is not required.
+    version_added: 2.4
   others:
     description:
       - all arguments accepted by the M(file) module also work here

lib/ansible/modules/network/basics/uri.py

@@ -156,6 +156,22 @@ options:
     default: 'yes'
     choices: ['yes', 'no']
     version_added: '1.9.2'
+  client_cert:
+    required: false
+    default: null
+    description:
+      - PEM formatted certificate chain file to be used for SSL client
+        authentication. This file can also include the key as well, and if
+        the key is included, I(client_key) is not required
+    version_added: 2.4
+  client_key:
+    required: false
+    default: null
+    description:
+      - PEM formatted file that contains your private key to be used for SSL
+        client authentication. If I(client_cert) contains both the certificate
+        and key, this option is not required.
+    version_added: 2.4
 notes:
   - The dependency on httplib2 was removed in Ansible 2.1
 author: "Romeo Theriault (@romeotheriault)"

test/integration/targets/get_url/tasks/main.yml

@@ -211,3 +211,18 @@
   get_url: 
     url: https://{{ httpbin_host }}
     dest: "{{ output_dir }}"
+
+
+- name: Test client cert auth, with certs
+  get_url:
+    url: "https://ansible.http.tests/ssl_client_verify"
+    client_cert: "{{ output_dir }}/client.pem"
+    client_key: "{{ output_dir }}/client.key"
+    dest: "{{ output_dir }}/ssl_client_verify"
+  when: has_httptester
+
+- name: Assert that the ssl_client_verify file contains the correct content
+  assert:
+    that:
+      - 'lookup("file", "{{ output_dir }}/ssl_client_verify") == "ansible.http.tests:SUCCESS"'
+  when: has_httptester

test/integration/targets/prepare_http_tests/tasks/main.yml

@@ -18,6 +18,14 @@
         dest: "/etc/pki/ca-trust/source/anchors/ansible.pem"
       when: ansible_os_family == 'RedHat'
 
+    - name: Get client cert/key
+      get_url:
+        url: "http://ansible.http.tests/{{ item }}"
+        dest: "{{ output_dir }}/{{ item }}"
+      with_items:
+        - client.pem
+        - client.key
+
     - name: Suse - Retrieve test cacert
       get_url:
         url: "http://ansible.http.tests/cacert.pem"

test/integration/targets/uri/tasks/main.yml

@@ -332,3 +332,44 @@
     return_content: true
   register: result
   failed_when: result.json.headers['Content-Type'] != 'text/json'
+
+- name: Test client cert auth, no certs
+  uri:
+    url: "https://ansible.http.tests/ssl_client_verify"
+    status_code: 200
+    return_content: true
+  register: result
+  failed_when: result.content != "ansible.http.tests:NONE"
+  when: has_httptester
+
+- name: Test client cert auth, with certs
+  uri:
+    url: "https://ansible.http.tests/ssl_client_verify"
+    client_cert: "{{ output_dir }}/client.pem"
+    client_key: "{{ output_dir }}/client.key"
+    return_content: true
+  register: result
+  failed_when: result.content != "ansible.http.tests:SUCCESS"
+  when: has_httptester
+
+- name: Test client cert auth, with no validation
+  uri:
+    url: "https://fail.ansible.http.tests/ssl_client_verify"
+    client_cert: "{{ output_dir }}/client.pem"
+    client_key: "{{ output_dir }}/client.key"
+    return_content: true
+    validate_certs: no
+  register: result
+  failed_when: result.content != "ansible.http.tests:SUCCESS"
+  when: has_httptester
+
+- name: Test client cert auth, with validation and ssl mismatch
+  uri:
+    url: "https://fail.ansible.http.tests/ssl_client_verify"
+    client_cert: "{{ output_dir }}/client.pem"
+    client_key: "{{ output_dir }}/client.key"
+    return_content: true
+    validate_certs: yes
+  register: result
+  failed_when: not result|failed
+  when: has_httptester