From 0fba72ce3cc52a9ab9cc1b84ed787f9b2d29f236 Mon Sep 17 00:00:00 2001 From: Dag Wieers Date: Thu, 17 May 2018 19:18:18 +0200 Subject: [PATCH] uri: Add form-urlencoded support to body_format (#37188) * uri: Add form-urlencoded support to body_format This PR adds form-urlencoded support so the user does not need to take care of correctly encode input and have the same convenience as using JSON. This fixes #37182 * Various fixes * Undo documentation improvements No longer my problem * Fix the remaining review comments --- lib/ansible/modules/net_tools/basics/uri.py | 119 ++++++++++++++------ test/integration/targets/uri/tasks/main.yml | 56 +++++++++ 2 files changed, 141 insertions(+), 34 deletions(-) diff --git a/lib/ansible/modules/net_tools/basics/uri.py b/lib/ansible/modules/net_tools/basics/uri.py index b40cfb211e1..e7ccdefaf50 100644 --- a/lib/ansible/modules/net_tools/basics/uri.py +++ b/lib/ansible/modules/net_tools/basics/uri.py @@ -1,7 +1,7 @@ #!/usr/bin/python # -*- coding: utf-8 -*- -# (c) 2013, Romeo Theriault +# Copyright: (c) 2013, Romeo Theriault # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) from __future__ import absolute_import, division, print_function @@ -40,14 +40,15 @@ options: description: - The body of the http request/response to the web service. If C(body_format) is set to 'json' it will take an already formatted JSON string or convert a data structure - into JSON. + into JSON. If C(body_format) is set to 'form-urlencoded' it will convert a dictionary + or list of tuples into an 'application/x-www-form-urlencoded' string. (Added in v2.7) body_format: description: - - The serialization format of the body. When set to json, encodes the + - The serialization format of the body. When set to C(json) or C(form-urlencoded), encodes the body argument, if needed, and automatically sets the Content-Type header accordingly. As of C(2.3) it is possible to override the `Content-Type` header, when - set to json via the I(headers) option. - choices: [ "raw", "json" ] + set to C(json) or C(form-urlencoded) via the I(headers) option. + choices: [ form-urlencoded, json, raw ] default: raw version_added: "2.0" method: @@ -79,8 +80,8 @@ options: any redirects. Note that C(yes) and C(no) choices are accepted for backwards compatibility, where C(yes) is the equivalent of C(all) and C(no) is the equivalent of C(safe). C(yes) and C(no) are deprecated and will be removed in some future version of Ansible. - choices: [ all, none, safe ] - default: "safe" + choices: [ all, 'none', safe ] + default: safe creates: description: - A filename, when it already exists, this step will not be run. @@ -89,8 +90,8 @@ options: - A filename, when it does not exist, this step will not be run. status_code: description: - - A valid, numeric, HTTP status code that signifies success of the - request. Can also be comma separated list of status codes. + - A list of valid, numeric, HTTP status codes that signifies success of the + request. default: 200 timeout: description: @@ -107,7 +108,7 @@ options: description: - Add custom HTTP headers to a request in the format of a YAML hash. As of C(2.3) supplying C(Content-Type) here will override the header - generated by supplying C(json) for I(body_format). + generated by supplying C(json) or C(form-urlencoded) for I(body_format). version_added: '2.1' others: description: @@ -150,12 +151,8 @@ EXAMPLES = r''' - uri: url: http://www.example.com return_content: yes - register: webpage - -- name: Fail if AWESOME is not in the page content - fail: - when: "'AWESOME' not in webpage.content" - + register: this + failed_when: "'AWESOME' not in this.content" - name: Create a JIRA issue uri: @@ -174,10 +171,24 @@ EXAMPLES = r''' - uri: url: https://your.form.based.auth.example.com/index.php method: POST - body: "name=your_username&password=your_password&enter=Sign%20in" + body_format: form-urlencoded + body: + name: your_username + password: your_password + enter: Sign in + status_code: 302 + register: login + +# Same, but now using a list of tuples +- uri: + url: https://your.form.based.auth.example.com/index.php + method: POST + body_format: form-urlencoded + body: + - [ name, your_username ] + - [ password, your_password ] + - [ enter, Sign in ] status_code: 302 - headers: - Content-Type: "application/x-www-form-urlencoded" register: login - uri: @@ -185,17 +196,16 @@ EXAMPLES = r''' method: GET return_content: yes headers: - Cookie: "{{login.set_cookie}}" + Cookie: "{{ login.set_cookie }}" - name: Queue build of a project in Jenkins uri: - url: "http://{{ jenkins.host }}/job/{{ jenkins.job }}/build?token={{ jenkins.token }}" + url: http://{{ jenkins.host }}/job/{{ jenkins.job }}/build?token={{ jenkins.token }} method: GET user: "{{ jenkins.user }}" password: "{{ jenkins.password }}" force_basic_auth: yes status_code: 201 - ''' RETURN = r''' @@ -230,9 +240,10 @@ import shutil import tempfile import traceback - +from collections import Mapping, Sequence from ansible.module_utils.basic import AnsibleModule -import ansible.module_utils.six as six +from ansible.module_utils.six import iteritems, string_types +from ansible.module_utils.six.moves.urllib.parse import urlencode, urlsplit from ansible.module_utils._text import to_native, to_text from ansible.module_utils.urls import fetch_url, url_argument_spec @@ -290,7 +301,7 @@ def write_file(module, url, dest, content): def url_filename(url): - fn = os.path.basename(six.moves.urllib.parse.urlsplit(url)[2]) + fn = os.path.basename(urlsplit(url)[2]) if fn == '': return 'index.html' return fn @@ -305,7 +316,7 @@ def absolute_location(url, location): return location elif location.startswith('/'): - parts = six.moves.urllib.parse.urlsplit(url) + parts = urlsplit(url) base = url.replace(parts[2], '') return '%s%s' % (base, location) @@ -317,6 +328,39 @@ def absolute_location(url, location): return location +def kv_list(data): + ''' Convert data into a list of key-value tuples ''' + if data is None: + return None + + if isinstance(data, Sequence): + return list(data) + + if isinstance(data, Mapping): + return list(data.items()) + + raise TypeError('cannot form-urlencode body, expect list or dict') + + +def form_urlencoded(body): + ''' Convert data into a form-urlencoded string ''' + if isinstance(body, string_types): + return body + + if isinstance(body, (Mapping, Sequence)): + result = [] + # Turn a list of lists into a list of tupples that urlencode accepts + for key, values in kv_list(body): + if isinstance(values, string_types) or not isinstance(values, (Mapping, Sequence)): + values = [values] + for value in values: + if value is not None: + result.append((to_text(key), to_text(value))) + return urlencode(result, doseq=True) + + return body + + def uri(module, url, dest, body, body_format, method, headers, socket_timeout): # is dest is set and is a directory, let's check if we get redirected and # set the filename from that url @@ -373,9 +417,9 @@ def main(): url_username=dict(type='str', aliases=['user']), url_password=dict(type='str', aliases=['password'], no_log=True), body=dict(type='raw'), - body_format=dict(type='str', default='raw', choices=['raw', 'json']), + body_format=dict(type='str', default='raw', choices=['form-urlencoded', 'json', 'raw']), method=dict(type='str', default='GET', choices=['GET', 'POST', 'PUT', 'HEAD', 'DELETE', 'OPTIONS', 'PATCH', 'TRACE', 'CONNECT', 'REFRESH']), - return_content=dict(type='bool', default='no'), + return_content=dict(type='bool', default=False), follow_redirects=dict(type='str', default='safe', choices=['all', 'no', 'none', 'safe', 'urllib2', 'yes']), creates=dict(type='path'), removes=dict(type='path'), @@ -406,16 +450,23 @@ def main(): if body_format == 'json': # Encode the body unless its a string, then assume it is pre-formatted JSON - if not isinstance(body, six.string_types): + if not isinstance(body, string_types): body = json.dumps(body) - lower_header_keys = [key.lower() for key in dict_headers] - if 'content-type' not in lower_header_keys: + if 'content-type' not in [header.lower() for header in dict_headers]: dict_headers['Content-Type'] = 'application/json' + elif body_format == 'form-urlencoded': + if not isinstance(body, string_types): + try: + body = form_urlencoded(body) + except ValueError as e: + module.fail_json(msg='failed to parse body as form_urlencoded: %s' % to_native(e)) + if 'content-type' not in [header.lower() for header in dict_headers]: + dict_headers['Content-Type'] = 'application/x-www-form-urlencoded' # TODO: Deprecated section. Remove in Ansible 2.9 # Grab all the http headers. Need this hack since passing multi-values is # currently a bit ugly. (e.g. headers='{"Content-Type":"application/json"}') - for key, value in six.iteritems(module.params): + for key, value in iteritems(module.params): if key.startswith("HEADER_"): module.deprecate('Supplying headers via HEADER_* is deprecated. Please use `headers` to' ' supply headers for the request', version='2.9') @@ -432,7 +483,7 @@ def main(): if removes is not None: # do not run the command if the line contains removes=filename - # and the filename do not exists. This allows idempotence + # and the filename does not exist. This allows idempotence # of uri executions. if not os.path.exists(removes): module.exit_json(stdout="skipped, since '%s' does not exist" % removes, changed=False, rc=0) @@ -463,7 +514,7 @@ def main(): # In python3, the headers are title cased. Lowercase them to be # compatible with the python2 behaviour. uresp = {} - for key, value in six.iteritems(resp): + for key, value in iteritems(resp): ukey = key.replace("-", "_").lower() uresp[ukey] = value diff --git a/test/integration/targets/uri/tasks/main.yml b/test/integration/targets/uri/tasks/main.yml index 78e8c357418..69f62f87023 100644 --- a/test/integration/targets/uri/tasks/main.yml +++ b/test/integration/targets/uri/tasks/main.yml @@ -334,6 +334,62 @@ register: result failed_when: result.json.headers['Content-Type'] != 'text/json' +- name: Validate body_format form-urlencoded using dicts works + uri: + url: https://{{ httpbin_host }}/post + method: POST + body: + user: foo + password: bar!#@ |&82$M + submit: Sign in + body_format: form-urlencoded + return_content: yes + register: result + +- name: Assert form-urlencoded dict input + assert: + that: + - result is successful + - result.json.headers['Content-Type'] == 'application/x-www-form-urlencoded' + - result.json.form.password == 'bar!#@ |&82$M' + +- name: Validate body_format form-urlencoded using lists works + uri: + url: https://{{ httpbin_host }}/post + method: POST + body: + - [ user, foo ] + - [ password, bar!#@ |&82$M ] + - [ submit, Sign in ] + body_format: form-urlencoded + return_content: yes + register: result + +- name: Assert form-urlencoded list input + assert: + that: + - result is successful + - result.json.headers['Content-Type'] == 'application/x-www-form-urlencoded' + - result.json.form.password == 'bar!#@ |&82$M' + +- name: Validate body_format form-urlencoded of invalid input fails + uri: + url: https://{{ httpbin_host }}/post + method: POST + body: + - foo + - bar: baz + body_format: form-urlencoded + return_content: yes + register: result + ignore_errors: yes + +- name: Assert invalid input fails + assert: + that: + - result is failure + - "'failed to parse body as form_urlencoded: too many values to unpack' in result.msg" + - name: Test client cert auth, no certs uri: url: "https://ansible.http.tests/ssl_client_verify"