diff --git a/changelogs/fragments/expose-unredirected-headers.yml b/changelogs/fragments/expose-unredirected-headers.yml new file mode 100644 index 00000000000..202cdb37374 --- /dev/null +++ b/changelogs/fragments/expose-unredirected-headers.yml @@ -0,0 +1,5 @@ +minor_changes: +- > + ``uri``/``get_url`` - Expose ``unredirected_headers`` to modules to allow user control +bugfixes: +- urls - Fix logic in matching ``unredirected_headers`` to perform case insensitive matching diff --git a/lib/ansible/module_utils/urls.py b/lib/ansible/module_utils/urls.py index 632387f6728..df707a21c21 100644 --- a/lib/ansible/module_utils/urls.py +++ b/lib/ansible/module_utils/urls.py @@ -1436,9 +1436,9 @@ class Request: request.add_header('If-Modified-Since', tstamp) # user defined headers now, which may override things we've set above - unredirected_headers = unredirected_headers or [] + unredirected_headers = [h.lower() for h in (unredirected_headers or [])] for header in headers: - if header in unredirected_headers: + if header.lower() in unredirected_headers: request.add_unredirected_header(header, headers[header]) else: request.add_header(header, headers[header]) @@ -1689,7 +1689,7 @@ def url_argument_spec(): def fetch_url(module, url, data=None, headers=None, method=None, use_proxy=True, force=False, last_mod_time=None, timeout=10, - use_gssapi=False, unix_socket=None, ca_path=None, cookies=None): + use_gssapi=False, unix_socket=None, ca_path=None, cookies=None, unredirected_headers=None): """Sends a request via HTTP(S) or FTP (needs the module as parameter) :arg module: The AnsibleModule (used to get username, password etc. (s.b.). @@ -1706,6 +1706,8 @@ def fetch_url(module, url, data=None, headers=None, method=None, :kwarg unix_socket: (optional) String of file system path to unix socket file to use when establishing connection to the provided url :kwarg ca_path: (optional) String of file system path to CA cert bundle to use + :kwarg cookies: (optional) CookieJar object to send with the request + :kwarg unredirected_headers: (optional) A list of headers to not attach on a redirected request :returns: A tuple of (**response**, **info**). Use ``response.read()`` to read the data. The **info** contains the 'status' and other meta data. When a HttpError (status >= 400) @@ -1758,7 +1760,7 @@ def fetch_url(module, url, data=None, headers=None, method=None, url_password=password, http_agent=http_agent, force_basic_auth=force_basic_auth, follow_redirects=follow_redirects, client_cert=client_cert, client_key=client_key, cookies=cookies, use_gssapi=use_gssapi, - unix_socket=unix_socket, ca_path=ca_path) + unix_socket=unix_socket, ca_path=ca_path, unredirected_headers=unredirected_headers) # Lowercase keys, to conform to py2 behavior, so that py3 and py2 are predictable info.update(dict((k.lower(), v) for k, v in r.info().items())) @@ -1831,7 +1833,8 @@ def fetch_url(module, url, data=None, headers=None, method=None, def fetch_file(module, url, data=None, headers=None, method=None, - use_proxy=True, force=False, last_mod_time=None, timeout=10): + use_proxy=True, force=False, last_mod_time=None, timeout=10, + unredirected_headers=None): '''Download and save a file via HTTP(S) or FTP (needs the module as parameter). This is basically a wrapper around fetch_url(). @@ -1845,6 +1848,7 @@ def fetch_file(module, url, data=None, headers=None, method=None, :kwarg boolean force: If True: Do not get a cached copy (Default: False) :kwarg last_mod_time: Default: None :kwarg int timeout: Default: 10 + :kwarg unredirected_headers: (optional) A list of headers to not attach on a redirected request :returns: A string, the path to the downloaded file. ''' @@ -1854,7 +1858,8 @@ def fetch_file(module, url, data=None, headers=None, method=None, fetch_temp_file = tempfile.NamedTemporaryFile(dir=module.tmpdir, prefix=file_name, suffix=file_ext, delete=False) module.add_cleanup_file(fetch_temp_file.name) try: - rsp, info = fetch_url(module, url, data, headers, method, use_proxy, force, last_mod_time, timeout) + rsp, info = fetch_url(module, url, data, headers, method, use_proxy, force, last_mod_time, timeout, + unredirected_headers=unredirected_headers) if not rsp: module.fail_json(msg="Failure downloading %s, %s" % (url, info['msg'])) data = rsp.read(bufsize) diff --git a/lib/ansible/modules/get_url.py b/lib/ansible/modules/get_url.py index 985fa6a642b..4f6404f4651 100644 --- a/lib/ansible/modules/get_url.py +++ b/lib/ansible/modules/get_url.py @@ -165,6 +165,15 @@ options: - Header to identify as, generally appears in web server logs. type: str default: ansible-httpget + unredirected_headers: + description: + - A list of header names that will not be sent on subsequent redirected requests. This list is case + insensitive. By default all headers will be redirected. In some cases it may be beneficial to list + headers such as C(Authorization) here to avoid potential credential exposure. + default: [] + type: list + elements: str + version_added: '2.12' use_gssapi: description: - Use GSSAPI to perform the authentication, typically this is for Kerberos or Kerberos through Negotiate @@ -357,7 +366,7 @@ def url_filename(url): return fn -def url_get(module, url, dest, use_proxy, last_mod_time, force, timeout=10, headers=None, tmp_dest='', method='GET'): +def url_get(module, url, dest, use_proxy, last_mod_time, force, timeout=10, headers=None, tmp_dest='', method='GET', unredirected_headers=None): """ Download data from the url and store in a temporary file. @@ -365,7 +374,8 @@ def url_get(module, url, dest, use_proxy, last_mod_time, force, timeout=10, head """ start = datetime.datetime.utcnow() - rsp, info = fetch_url(module, url, use_proxy=use_proxy, force=force, last_mod_time=last_mod_time, timeout=timeout, headers=headers, method=method) + rsp, info = fetch_url(module, url, use_proxy=use_proxy, force=force, last_mod_time=last_mod_time, timeout=timeout, headers=headers, method=method, + unredirected_headers=unredirected_headers) elapsed = (datetime.datetime.utcnow() - start).seconds if info['status'] == 304: @@ -450,6 +460,7 @@ def main(): timeout=dict(type='int', default=10), headers=dict(type='dict'), tmp_dest=dict(type='path'), + unredirected_headers=dict(type='list', elements='str', default=[]), ) module = AnsibleModule( @@ -478,6 +489,7 @@ def main(): timeout = module.params['timeout'] headers = module.params['headers'] tmp_dest = module.params['tmp_dest'] + unredirected_headers = module.params['unredirected_headers'] result = dict( changed=False, @@ -505,7 +517,8 @@ def main(): if is_url(checksum): checksum_url = checksum # download checksum file to checksum_tmpsrc - checksum_tmpsrc, checksum_info = url_get(module, checksum_url, dest, use_proxy, last_mod_time, force, timeout, headers, tmp_dest) + checksum_tmpsrc, checksum_info = url_get(module, checksum_url, dest, use_proxy, last_mod_time, force, timeout, headers, tmp_dest, + unredirected_headers=unredirected_headers) with open(checksum_tmpsrc) as f: lines = [line.rstrip('\n') for line in f] os.remove(checksum_tmpsrc) @@ -576,7 +589,7 @@ def main(): # download to tmpsrc start = datetime.datetime.utcnow() method = 'HEAD' if module.check_mode else 'GET' - tmpsrc, info = url_get(module, url, dest, use_proxy, last_mod_time, force, timeout, headers, tmp_dest, method) + tmpsrc, info = url_get(module, url, dest, use_proxy, last_mod_time, force, timeout, headers, tmp_dest, method, unredirected_headers=unredirected_headers) result['elapsed'] = (datetime.datetime.utcnow() - start).seconds result['src'] = tmpsrc diff --git a/lib/ansible/modules/uri.py b/lib/ansible/modules/uri.py index c21c45f213c..81a971c0d24 100644 --- a/lib/ansible/modules/uri.py +++ b/lib/ansible/modules/uri.py @@ -181,6 +181,15 @@ options: - Header to identify as, generally appears in web server logs. type: str default: ansible-httpget + unredirected_headers: + description: + - A list of header names that will not be sent on subsequent redirected requests. This list is case + insensitive. By default all headers will be redirected. In some cases it may be beneficial to list + headers such as C(Authorization) here to avoid potential credential exposure. + default: [] + type: list + elements: str + version_added: '2.12' use_gssapi: description: - Use GSSAPI to perform the authentication, typically this is for Kerberos or Kerberos through Negotiate @@ -553,7 +562,7 @@ def form_urlencoded(body): return body -def uri(module, url, dest, body, body_format, method, headers, socket_timeout, ca_path): +def uri(module, url, dest, body, body_format, method, headers, socket_timeout, ca_path, unredirected_headers): # is dest is set and is a directory, let's check if we get redirected and # set the filename from that url redirected = False @@ -598,7 +607,7 @@ def uri(module, url, dest, body, body_format, method, headers, socket_timeout, c resp, info = fetch_url(module, url, data=data, headers=headers, method=method, timeout=socket_timeout, unix_socket=module.params['unix_socket'], - ca_path=ca_path, + ca_path=ca_path, unredirected_headers=unredirected_headers, **kwargs) try: @@ -642,6 +651,7 @@ def main(): unix_socket=dict(type='path'), remote_src=dict(type='bool', default=False), ca_path=dict(type='path', default=None), + unredirected_headers=dict(type='list', elements='str', default=[]), ) module = AnsibleModule( @@ -666,6 +676,7 @@ def main(): socket_timeout = module.params['timeout'] ca_path = module.params['ca_path'] dict_headers = module.params['headers'] + unredirected_headers = module.params['unredirected_headers'] if not re.match('^[A-Z]+$', method): module.fail_json(msg="Parameter 'method' needs to be a single word in uppercase, like GET or POST.") @@ -708,7 +719,7 @@ def main(): # Make the request start = datetime.datetime.utcnow() resp, content, dest = uri(module, url, dest, body, body_format, method, - dict_headers, socket_timeout, ca_path) + dict_headers, socket_timeout, ca_path, unredirected_headers) resp['elapsed'] = (datetime.datetime.utcnow() - start).seconds resp['status'] = int(resp['status']) resp['changed'] = False diff --git a/test/integration/targets/get_url/tasks/main.yml b/test/integration/targets/get_url/tasks/main.yml index 32da1d51183..b5a9c7e5ba2 100644 --- a/test/integration/targets/get_url/tasks/main.yml +++ b/test/integration/targets/get_url/tasks/main.yml @@ -579,6 +579,35 @@ - '(result.content | b64decode) == "ansible.http.tests:SUCCESS"' when: has_httptester +- name: test unredirected_headers + get_url: + url: 'https://{{ httpbin_host }}/redirect-to?status_code=301&url=/basic-auth/user/passwd' + username: user + password: passwd + force_basic_auth: true + unredirected_headers: + - authorization + dest: "{{ remote_tmp_dir }}/doesnt_matter" + ignore_errors: true + register: unredirected_headers + +- name: test unredirected_headers + get_url: + url: 'https://{{ httpbin_host }}/redirect-to?status_code=301&url=/basic-auth/user/passwd' + username: user + password: passwd + force_basic_auth: true + dest: "{{ remote_tmp_dir }}/doesnt_matter" + register: redirected_headers + +- name: ensure unredirected_headers caused auth to fail + assert: + that: + - unredirected_headers is failed + - unredirected_headers.status_code == 401 + - redirected_headers is successful + - redirected_headers.status_code == 200 + - name: Test use_gssapi=True include_tasks: file: use_gssapi.yml diff --git a/test/integration/targets/uri/tasks/main.yml b/test/integration/targets/uri/tasks/main.yml index e1cacb36929..c6ba67224e3 100644 --- a/test/integration/targets/uri/tasks/main.yml +++ b/test/integration/targets/uri/tasks/main.yml @@ -228,6 +228,33 @@ headers: Cookie: "fake=fake_value" +- name: test unredirected_headers + uri: + url: 'https://{{ httpbin_host }}/redirect-to?status_code=301&url=/basic-auth/user/passwd' + user: user + password: passwd + force_basic_auth: true + unredirected_headers: + - authorization + ignore_errors: true + register: unredirected_headers + +- name: test omitting unredirected headers + uri: + url: 'https://{{ httpbin_host }}/redirect-to?status_code=301&url=/basic-auth/user/passwd' + user: user + password: passwd + force_basic_auth: true + register: redirected_headers + +- name: ensure unredirected_headers caused auth to fail + assert: + that: + - unredirected_headers is failed + - unredirected_headers.status == 401 + - redirected_headers is successful + - redirected_headers.status == 200 + - name: test PUT uri: url: 'https://{{ httpbin_host }}/put' diff --git a/test/units/module_utils/urls/test_fetch_url.py b/test/units/module_utils/urls/test_fetch_url.py index 9cac2a35e4b..4869bb0f46b 100644 --- a/test/units/module_utils/urls/test_fetch_url.py +++ b/test/units/module_utils/urls/test_fetch_url.py @@ -67,7 +67,7 @@ def test_fetch_url(open_url_mock, fake_ansible_module): open_url_mock.assert_called_once_with('http://ansible.com/', client_cert=None, client_key=None, cookies=kwargs['cookies'], data=None, follow_redirects='urllib2', force=False, force_basic_auth='', headers=None, http_agent='ansible-httpget', last_mod_time=None, method=None, timeout=10, url_password='', url_username='', - use_proxy=True, validate_certs=True, use_gssapi=False, unix_socket=None, ca_path=None) + use_proxy=True, validate_certs=True, use_gssapi=False, unix_socket=None, ca_path=None, unredirected_headers=None) def test_fetch_url_params(open_url_mock, fake_ansible_module): @@ -89,7 +89,7 @@ def test_fetch_url_params(open_url_mock, fake_ansible_module): open_url_mock.assert_called_once_with('http://ansible.com/', client_cert='client.pem', client_key='client.key', cookies=kwargs['cookies'], data=None, follow_redirects='all', force=False, force_basic_auth=True, headers=None, http_agent='ansible-test', last_mod_time=None, method=None, timeout=10, url_password='passwd', url_username='user', - use_proxy=True, validate_certs=False, use_gssapi=False, unix_socket=None, ca_path=None) + use_proxy=True, validate_certs=False, use_gssapi=False, unix_socket=None, ca_path=None, unredirected_headers=None) def test_fetch_url_cookies(mocker, fake_ansible_module):