Expose `unredirected_headers` to `uri` and `get_url` modules (#75308)

changelogs/fragments/expose-unredirected-headers.yml

@@ -0,0 +1,5 @@
+minor_changes:
+- >
+  ``uri``/``get_url`` - Expose ``unredirected_headers`` to modules to allow user control
+bugfixes:
+- urls - Fix logic in matching ``unredirected_headers`` to perform case insensitive matching

lib/ansible/module_utils/urls.py

@@ -1436,9 +1436,9 @@ class Request:
             request.add_header('If-Modified-Since', tstamp)
 
         # user defined headers now, which may override things we've set above
-        unredirected_headers = unredirected_headers or []
+        unredirected_headers = [h.lower() for h in (unredirected_headers or [])]
         for header in headers:
-            if header in unredirected_headers:
+            if header.lower() in unredirected_headers:
                 request.add_unredirected_header(header, headers[header])
             else:
                 request.add_header(header, headers[header])
@@ -1689,7 +1689,7 @@ def url_argument_spec():
 
 def fetch_url(module, url, data=None, headers=None, method=None,
               use_proxy=True, force=False, last_mod_time=None, timeout=10,
-              use_gssapi=False, unix_socket=None, ca_path=None, cookies=None):
+              use_gssapi=False, unix_socket=None, ca_path=None, cookies=None, unredirected_headers=None):
     """Sends a request via HTTP(S) or FTP (needs the module as parameter)
 
     :arg module: The AnsibleModule (used to get username, password etc. (s.b.).
@@ -1706,6 +1706,8 @@ def fetch_url(module, url, data=None, headers=None, method=None,
     :kwarg unix_socket: (optional) String of file system path to unix socket file to use when establishing
         connection to the provided url
     :kwarg ca_path: (optional) String of file system path to CA cert bundle to use
+    :kwarg cookies: (optional) CookieJar object to send with the request
+    :kwarg unredirected_headers: (optional) A list of headers to not attach on a redirected request
 
     :returns: A tuple of (**response**, **info**). Use ``response.read()`` to read the data.
         The **info** contains the 'status' and other meta data. When a HttpError (status >= 400)
@@ -1758,7 +1760,7 @@ def fetch_url(module, url, data=None, headers=None, method=None,
                      url_password=password, http_agent=http_agent, force_basic_auth=force_basic_auth,
                      follow_redirects=follow_redirects, client_cert=client_cert,
                      client_key=client_key, cookies=cookies, use_gssapi=use_gssapi,
-                     unix_socket=unix_socket, ca_path=ca_path)
+                     unix_socket=unix_socket, ca_path=ca_path, unredirected_headers=unredirected_headers)
         # Lowercase keys, to conform to py2 behavior, so that py3 and py2 are predictable
         info.update(dict((k.lower(), v) for k, v in r.info().items()))
 
@@ -1831,7 +1833,8 @@ def fetch_url(module, url, data=None, headers=None, method=None,
 
 
 def fetch_file(module, url, data=None, headers=None, method=None,
-               use_proxy=True, force=False, last_mod_time=None, timeout=10):
+               use_proxy=True, force=False, last_mod_time=None, timeout=10,
+               unredirected_headers=None):
     '''Download and save a file via HTTP(S) or FTP (needs the module as parameter).
     This is basically a wrapper around fetch_url().
 
@@ -1845,6 +1848,7 @@ def fetch_file(module, url, data=None, headers=None, method=None,
     :kwarg boolean force: If True: Do not get a cached copy (Default: False)
     :kwarg last_mod_time: Default: None
     :kwarg int timeout:   Default: 10
+    :kwarg unredirected_headers: (optional) A list of headers to not attach on a redirected request
 
     :returns: A string, the path to the downloaded file.
     '''
@@ -1854,7 +1858,8 @@ def fetch_file(module, url, data=None, headers=None, method=None,
     fetch_temp_file = tempfile.NamedTemporaryFile(dir=module.tmpdir, prefix=file_name, suffix=file_ext, delete=False)
     module.add_cleanup_file(fetch_temp_file.name)
     try:
-        rsp, info = fetch_url(module, url, data, headers, method, use_proxy, force, last_mod_time, timeout)
+        rsp, info = fetch_url(module, url, data, headers, method, use_proxy, force, last_mod_time, timeout,
+                              unredirected_headers=unredirected_headers)
         if not rsp:
             module.fail_json(msg="Failure downloading %s, %s" % (url, info['msg']))
         data = rsp.read(bufsize)

lib/ansible/modules/get_url.py

@@ -165,6 +165,15 @@ options:
       - Header to identify as, generally appears in web server logs.
     type: str
     default: ansible-httpget
+  unredirected_headers:
+    description:
+      - A list of header names that will not be sent on subsequent redirected requests. This list is case
+        insensitive. By default all headers will be redirected. In some cases it may be beneficial to list
+        headers such as C(Authorization) here to avoid potential credential exposure.
+    default: []
+    type: list
+    elements: str
+    version_added: '2.12'
   use_gssapi:
     description:
       - Use GSSAPI to perform the authentication, typically this is for Kerberos or Kerberos through Negotiate
@@ -357,7 +366,7 @@ def url_filename(url):
     return fn
 
 
-def url_get(module, url, dest, use_proxy, last_mod_time, force, timeout=10, headers=None, tmp_dest='', method='GET'):
+def url_get(module, url, dest, use_proxy, last_mod_time, force, timeout=10, headers=None, tmp_dest='', method='GET', unredirected_headers=None):
     """
     Download data from the url and store in a temporary file.
 
@@ -365,7 +374,8 @@ def url_get(module, url, dest, use_proxy, last_mod_time, force, timeout=10, head
     """
 
     start = datetime.datetime.utcnow()
-    rsp, info = fetch_url(module, url, use_proxy=use_proxy, force=force, last_mod_time=last_mod_time, timeout=timeout, headers=headers, method=method)
+    rsp, info = fetch_url(module, url, use_proxy=use_proxy, force=force, last_mod_time=last_mod_time, timeout=timeout, headers=headers, method=method,
+                          unredirected_headers=unredirected_headers)
     elapsed = (datetime.datetime.utcnow() - start).seconds
 
     if info['status'] == 304:
@@ -450,6 +460,7 @@ def main():
         timeout=dict(type='int', default=10),
         headers=dict(type='dict'),
         tmp_dest=dict(type='path'),
+        unredirected_headers=dict(type='list', elements='str', default=[]),
     )
 
     module = AnsibleModule(
@@ -478,6 +489,7 @@ def main():
     timeout = module.params['timeout']
     headers = module.params['headers']
     tmp_dest = module.params['tmp_dest']
+    unredirected_headers = module.params['unredirected_headers']
 
     result = dict(
         changed=False,
@@ -505,7 +517,8 @@ def main():
         if is_url(checksum):
             checksum_url = checksum
             # download checksum file to checksum_tmpsrc
-            checksum_tmpsrc, checksum_info = url_get(module, checksum_url, dest, use_proxy, last_mod_time, force, timeout, headers, tmp_dest)
+            checksum_tmpsrc, checksum_info = url_get(module, checksum_url, dest, use_proxy, last_mod_time, force, timeout, headers, tmp_dest,
+                                                     unredirected_headers=unredirected_headers)
             with open(checksum_tmpsrc) as f:
                 lines = [line.rstrip('\n') for line in f]
             os.remove(checksum_tmpsrc)
@@ -576,7 +589,7 @@ def main():
     # download to tmpsrc
     start = datetime.datetime.utcnow()
     method = 'HEAD' if module.check_mode else 'GET'
-    tmpsrc, info = url_get(module, url, dest, use_proxy, last_mod_time, force, timeout, headers, tmp_dest, method)
+    tmpsrc, info = url_get(module, url, dest, use_proxy, last_mod_time, force, timeout, headers, tmp_dest, method, unredirected_headers=unredirected_headers)
     result['elapsed'] = (datetime.datetime.utcnow() - start).seconds
     result['src'] = tmpsrc
 

lib/ansible/modules/uri.py

@@ -181,6 +181,15 @@ options:
       - Header to identify as, generally appears in web server logs.
     type: str
     default: ansible-httpget
+  unredirected_headers:
+    description:
+      - A list of header names that will not be sent on subsequent redirected requests. This list is case
+        insensitive. By default all headers will be redirected. In some cases it may be beneficial to list
+        headers such as C(Authorization) here to avoid potential credential exposure.
+    default: []
+    type: list
+    elements: str
+    version_added: '2.12'
   use_gssapi:
     description:
       - Use GSSAPI to perform the authentication, typically this is for Kerberos or Kerberos through Negotiate
@@ -553,7 +562,7 @@ def form_urlencoded(body):
     return body
 
 
-def uri(module, url, dest, body, body_format, method, headers, socket_timeout, ca_path):
+def uri(module, url, dest, body, body_format, method, headers, socket_timeout, ca_path, unredirected_headers):
     # is dest is set and is a directory, let's check if we get redirected and
     # set the filename from that url
     redirected = False
@@ -598,7 +607,7 @@ def uri(module, url, dest, body, body_format, method, headers, socket_timeout, c
 
     resp, info = fetch_url(module, url, data=data, headers=headers,
                            method=method, timeout=socket_timeout, unix_socket=module.params['unix_socket'],
-                           ca_path=ca_path,
+                           ca_path=ca_path, unredirected_headers=unredirected_headers,
                            **kwargs)
 
     try:
@@ -642,6 +651,7 @@ def main():
         unix_socket=dict(type='path'),
         remote_src=dict(type='bool', default=False),
         ca_path=dict(type='path', default=None),
+        unredirected_headers=dict(type='list', elements='str', default=[]),
     )
 
     module = AnsibleModule(
@@ -666,6 +676,7 @@ def main():
     socket_timeout = module.params['timeout']
     ca_path = module.params['ca_path']
     dict_headers = module.params['headers']
+    unredirected_headers = module.params['unredirected_headers']
 
     if not re.match('^[A-Z]+$', method):
         module.fail_json(msg="Parameter 'method' needs to be a single word in uppercase, like GET or POST.")
@@ -708,7 +719,7 @@ def main():
     # Make the request
     start = datetime.datetime.utcnow()
     resp, content, dest = uri(module, url, dest, body, body_format, method,
-                              dict_headers, socket_timeout, ca_path)
+                              dict_headers, socket_timeout, ca_path, unredirected_headers)
     resp['elapsed'] = (datetime.datetime.utcnow() - start).seconds
     resp['status'] = int(resp['status'])
     resp['changed'] = False

test/integration/targets/get_url/tasks/main.yml

@@ -579,6 +579,35 @@
       - '(result.content | b64decode) == "ansible.http.tests:SUCCESS"'
   when: has_httptester
 
+- name: test unredirected_headers
+  get_url:
+    url: 'https://{{ httpbin_host }}/redirect-to?status_code=301&url=/basic-auth/user/passwd'
+    username: user
+    password: passwd
+    force_basic_auth: true
+    unredirected_headers:
+      - authorization
+    dest: "{{ remote_tmp_dir }}/doesnt_matter"
+  ignore_errors: true
+  register: unredirected_headers
+
+- name: test unredirected_headers
+  get_url:
+    url: 'https://{{ httpbin_host }}/redirect-to?status_code=301&url=/basic-auth/user/passwd'
+    username: user
+    password: passwd
+    force_basic_auth: true
+    dest: "{{ remote_tmp_dir }}/doesnt_matter"
+  register: redirected_headers
+
+- name: ensure unredirected_headers caused auth to fail
+  assert:
+    that:
+      - unredirected_headers is failed
+      - unredirected_headers.status_code == 401
+      - redirected_headers is successful
+      - redirected_headers.status_code == 200
+
 - name: Test use_gssapi=True
   include_tasks:
     file: use_gssapi.yml

test/integration/targets/uri/tasks/main.yml

@@ -228,6 +228,33 @@
     headers:
       Cookie: "fake=fake_value"
 
+- name: test unredirected_headers
+  uri:
+    url: 'https://{{ httpbin_host }}/redirect-to?status_code=301&url=/basic-auth/user/passwd'
+    user: user
+    password: passwd
+    force_basic_auth: true
+    unredirected_headers:
+      - authorization
+  ignore_errors: true
+  register: unredirected_headers
+
+- name: test omitting unredirected headers
+  uri:
+    url: 'https://{{ httpbin_host }}/redirect-to?status_code=301&url=/basic-auth/user/passwd'
+    user: user
+    password: passwd
+    force_basic_auth: true
+  register: redirected_headers
+
+- name: ensure unredirected_headers caused auth to fail
+  assert:
+    that:
+      - unredirected_headers is failed
+      - unredirected_headers.status == 401
+      - redirected_headers is successful
+      - redirected_headers.status == 200
+
 - name: test PUT
   uri:
     url: 'https://{{ httpbin_host }}/put'

test/units/module_utils/urls/test_fetch_url.py

@@ -67,7 +67,7 @@ def test_fetch_url(open_url_mock, fake_ansible_module):
     open_url_mock.assert_called_once_with('http://ansible.com/', client_cert=None, client_key=None, cookies=kwargs['cookies'], data=None,
                                           follow_redirects='urllib2', force=False, force_basic_auth='', headers=None,
                                           http_agent='ansible-httpget', last_mod_time=None, method=None, timeout=10, url_password='', url_username='',
-                                          use_proxy=True, validate_certs=True, use_gssapi=False, unix_socket=None, ca_path=None)
+                                          use_proxy=True, validate_certs=True, use_gssapi=False, unix_socket=None, ca_path=None, unredirected_headers=None)
 
 
 def test_fetch_url_params(open_url_mock, fake_ansible_module):
@@ -89,7 +89,7 @@ def test_fetch_url_params(open_url_mock, fake_ansible_module):
     open_url_mock.assert_called_once_with('http://ansible.com/', client_cert='client.pem', client_key='client.key', cookies=kwargs['cookies'], data=None,
                                           follow_redirects='all', force=False, force_basic_auth=True, headers=None,
                                           http_agent='ansible-test', last_mod_time=None, method=None, timeout=10, url_password='passwd', url_username='user',
-                                          use_proxy=True, validate_certs=False, use_gssapi=False, unix_socket=None, ca_path=None)
+                                          use_proxy=True, validate_certs=False, use_gssapi=False, unix_socket=None, ca_path=None, unredirected_headers=None)
 
 
 def test_fetch_url_cookies(mocker, fake_ansible_module):