Create urlsplit filter (#28537)

* Improve tests for uri filter

* Create URL Split docs

* Add urlsplit filter

* Py3 compatibility

* Use helper method and eliminate query options

* Add options, cleanup output, fix tests

* Update docs

* Add parenthesis to boilerplate import

* Add debug task to tests

* Use exclude option to filter returned values

* Filter out additional option for Python 3

docs/docsite/rst/playbooks_filters.rst

@@ -332,7 +332,7 @@ output, use the ``parse_cli`` filter::
 
 The ``parse_cli`` filter will load the spec file and pass the command output
 through, it returning JSON output.  The spec file is a YAML yaml that defines
-how to parse the CLI output.  
+how to parse the CLI output.
 
 The spec file should be valid formatted YAML.  It defines how to parse the CLI
 output and return JSON data.  Below is an example of a valid spec file that
@@ -357,8 +357,8 @@ will parse the output from the ``show vlan`` command.::
 The spec file above will return a JSON data structure that is a list of hashes
 with the parsed VLAN information.
 
-The same command could be parsed into a hash by using the key and values 
+The same command could be parsed into a hash by using the key and values
-directives.  Here is an example of how to parse the output into a hash 
+directives.  Here is an example of how to parse the output into a hash
 value using the same ``show vlan`` command.::
 
     ---
@@ -379,7 +379,7 @@ value using the same ``show vlan`` command.::
       state_static:
         value: present
 
-Another common use case for parsing CLI commands is to break a large command 
+Another common use case for parsing CLI commands is to break a large command
 into blocks that can parsed.  This can be done using the ``start_block`` and
 ``end_block`` directives to break the command into blocks that can be parsed.::
 
@@ -594,6 +594,56 @@ which will produce this output:
 
 .. _other_useful_filters:
 
+URL Split Filter
+`````````````````
+
+.. versionadded:: 2.4
+
+The ``urlsplit`` filter extracts the fragment, hostname, netloc, password, path, port, query, scheme, and username from an URL. With no arguments, returns a dictionary of all the fields::
+
+    {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#frament" | urlsplit('hostname') }}
+    # => 'www.acme.com'
+
+    {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#frament" | urlsplit('netloc') }}
+    # => 'user:password@www.acme.com:9000'
+
+    {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#frament" | urlsplit('username') }}
+    # => 'user'
+
+    {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#frament" | urlsplit('password') }}
+    # => 'password'
+
+    {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#frament" | urlsplit('path') }}
+    # => '/dir/index.html'
+
+    {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#frament" | urlsplit('port') }}
+    # => '9000'
+
+    {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#frament" | urlsplit('scheme') }}
+    # => 'http'
+
+    {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#frament" | urlsplit('query') }}
+    # => 'query=term'
+
+    {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#frament" | urlsplit('fragment') }}
+    # => 'fragment'
+
+    {{ "http://user:password@www.acme.com:9000/dir/index.html?query=term#frament" | urlsplit }}
+    # =>
+    #   {
+    #       "fragment": "fragment",
+    #       "hostname": "www.acme.com",
+    #       "netloc": "user:password@www.acme.com:9000",
+    #       "password": "password",
+    #       "path": "/dir/index.html",
+    #       "port": 9000,
+    #       "query": "query=term",
+    #       "scheme": "http",
+    #       "username": "user"
+    #   }
+
+
+
 Other Useful Filters
 ````````````````````
 
@@ -681,7 +731,7 @@ To replace text in a string with regex, use the "regex_replace" filter::
 
     # convert "localhost:80" to "localhost, 80" using named groups
     {{ 'localhost:80' | regex_replace('^(?P<host>.+):(?P<port>\\d+)$', '\\g<host>, \\g<port>') }}
-    
+
     # convert "localhost:80" to "localhost"
     {{ 'localhost:80' | regex_replace(':80') }}
 
@@ -717,26 +767,26 @@ To get permutations of a list::
     - name: give me largest permutations (order matters)
       debug: msg="{{ [1,2,3,4,5]|permutations|list }}"
 
-    - name: give me permutations of sets of 3
+    - name: give me permutations of sets of three
       debug: msg="{{ [1,2,3,4,5]|permutations(3)|list }}"
 
 Combinations always require a set size::
 
-    - name: give me combinations for sets of 2
+    - name: give me combinations for sets of two
       debug: msg="{{ [1,2,3,4,5]|combinations(2)|list }}"
 
 
 To get a list combining the elements of other lists use ``zip``::
 
-    - name: give me list combo of 2 lists 
+    - name: give me list combo of two lists
       debug: msg="{{ [1,2,3,4,5]|zip(['a','b','c','d','e','f'])|list }}"
 
-    - name: give me shortest combo of 2 lists
+    - name: give me shortest combo of two lists
       debug: msg="{{ [1,2,3]|zip(['a','b','c','d','e','f'])|list }}"
 
 To always exhaust all list use ``zip_longest``::
 
-    - name: give me longest combo of 3 lists , fill with X
+    - name: give me longest combo of three lists , fill with X
       debug: msg="{{ [1,2,3]|zip_longest(['a','b','c','d','e','f'], [21, 22, 23], fillvalue='X')|list }}"
 
 

lib/ansible/plugins/filter/urlsplit.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2017 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+ANSIBLE_METADATA = {
+    'metadata_version': '1.1',
+    'status': ['preview'],
+    'supported_by': 'community'
+}
+
+
+from ansible.errors import AnsibleFilterError
+from ansible.module_utils.six.moves.urllib.parse import urlsplit
+from ansible.utils import helpers
+
+
+def split_url(value, query='', alias='urlsplit'):
+
+    results = helpers.object_to_dict(urlsplit(value), exclude=['count', 'index', 'geturl', 'encode'])
+
+    # If a query is supplied, make sure it's valid then return the results.
+    # If no option is supplied, return the entire dictionary.
+    if query:
+        if query not in results:
+            raise AnsibleFilterError(alias + ': unknown URL component: %s' % query)
+        return results[query]
+    else:
+        return results
+
+
+# ---- Ansible filters ----
+class FilterModule(object):
+    ''' URI filter '''
+
+    def filters(self):
+        return {
+            'urlsplit': split_url
+        }

test/integration/targets/filters/tasks/main.yml

@@ -20,23 +20,29 @@
   shell: echo hi
   register: some_registered_var
 
-- debug: var=some_registered_var
+- debug:
+    var: some_registered_var
 
 - name: Verify that we workaround a py26 json bug
-  template: src=py26json.j2 dest={{output_dir}}/py26json.templated mode=0644
+  template:
+    src: py26json.j2
+    dest: "{{ output_dir }}/py26json.templated"
+    mode: 0644
 
 - name: 9851 - Verify that we don't trigger https://github.com/ansible/ansible/issues/9851
   copy:
-    content: " [{{item|to_nice_json}}]"
+    content: " [{{ item | to_nice_json }}]"
-    dest: "{{output_dir}}/9851.out"
+    dest: "{{ output_dir }}/9851.out"
   with_items:
   - {"k": "Quotes \"'\n"}
 
 - name: 9851 - copy known good output into place
-  copy: src=9851.txt dest={{output_dir}}/9851.txt
+  copy:
+    src: 9851.txt
+    dest: "{{ output_dir }}/9851.txt"
 
 - name: 9851 - Compare generated json to known good
-  shell: diff -w {{output_dir}}/9851.out {{output_dir}}/9851.txt
+  shell: diff -w {{ output_dir }}/9851.out {{ output_dir }}/9851.txt
   register: diff_result_9851
 
 - name: 9851 - verify generated file matches known good
@@ -45,72 +51,78 @@
         - 'diff_result_9851.stdout == ""'
 
 - name: fill in a basic template
-  template: src=foo.j2 dest={{output_dir}}/foo.templated mode=0644
+  template:
+    src: foo.j2
+    dest: "{{ output_dir }}/foo.templated"
+    mode: 0644
   register: template_result
 
 - name: copy known good into place
-  copy: src=foo.txt dest={{output_dir}}/foo.txt
+  copy:
+    src: foo.txt
+    dest: "{{ output_dir }}/foo.txt"
 
 - name: compare templated file to known good
-  shell: diff -w {{output_dir}}/foo.templated {{output_dir}}/foo.txt
+  shell: diff -w {{ output_dir }}/foo.templated {{ output_dir }}/foo.txt
   register: diff_result
 
 - name: verify templated file matches known good
   assert:
     that:
-        - 'diff_result.stdout == ""'
+      - 'diff_result.stdout == ""'
 
 - name: Verify human_readable
   tags: "human_readable"
   assert:
     that:
-        - '"1.00 Bytes" == 1|human_readable'
+      - '"1.00 Bytes" == 1|human_readable'
-        - '"1.00 bits" == 1|human_readable(isbits=True)'
+      - '"1.00 bits" == 1|human_readable(isbits=True)'
-        - '"10.00 KB" == 10240|human_readable'
+      - '"10.00 KB" == 10240|human_readable'
-        - '"97.66 MB" == 102400000|human_readable'
+      - '"97.66 MB" == 102400000|human_readable'
-        - '"0.10 GB" == 102400000|human_readable(unit="G")'
+      - '"0.10 GB" == 102400000|human_readable(unit="G")'
-        - '"0.10 Gb" == 102400000|human_readable(isbits=True, unit="G")'
+      - '"0.10 Gb" == 102400000|human_readable(isbits=True, unit="G")'
 
 - name: Verify human_to_bytes
   tags: "human_to_bytes"
   assert:
     that:
-        - "{{'0'|human_to_bytes}}        == 0"
+      - "{{'0'|human_to_bytes}}        == 0"
-        - "{{'0.1'|human_to_bytes}}      == 0"
+      - "{{'0.1'|human_to_bytes}}      == 0"
-        - "{{'0.9'|human_to_bytes}}      == 1"
+      - "{{'0.9'|human_to_bytes}}      == 1"
-        - "{{'1'|human_to_bytes}}        == 1"
+      - "{{'1'|human_to_bytes}}        == 1"
-        - "{{'10.00 KB'|human_to_bytes}} == 10240"
+      - "{{'10.00 KB'|human_to_bytes}} == 10240"
-        - "{{   '11 MB'|human_to_bytes}} == 11534336"
+      - "{{   '11 MB'|human_to_bytes}} == 11534336"
-        - "{{  '1.1 GB'|human_to_bytes}} == 1181116006"
+      - "{{  '1.1 GB'|human_to_bytes}} == 1181116006"
-        - "{{'10.00 Kb'|human_to_bytes(isbits=True)}} == 10240"
+      - "{{'10.00 Kb'|human_to_bytes(isbits=True)}} == 10240"
 
 - name: Verify human_to_bytes (bad string)
-  tags: "human_to_bytes"
+  set_fact:
-  set_fact: bad_string="{{'10.00 foo'|human_to_bytes}}"
+    bad_string: "{{ '10.00 foo' | human_to_bytes }}"
   ignore_errors: yes
-  register: _
+  tags: human_to_bytes
+  register: _human_bytes_test
 
 - name: Verify human_to_bytes (bad string)
-  tags: "human_to_bytes"
+  tags: human_to_bytes
   assert:
-    that: "{{_.failed}}"
+    that: "{{_human_bytes_test.failed}}"
 
 - name: Test extract
   assert:
     that:
-        - '"c" == 2 | extract(["a", "b", "c"])'
+      - '"c" == 2 | extract(["a", "b", "c"])'
-        - '"b" == 1 | extract(["a", "b", "c"])'
+      - '"b" == 1 | extract(["a", "b", "c"])'
-        - '"a" == 0 | extract(["a", "b", "c"])'
+      - '"a" == 0 | extract(["a", "b", "c"])'
 
 - name: Container lookups with extract
   assert:
     that:
-        - "'x' == [0]|map('extract',['x','y'])|list|first"
+      - "'x' == [0]|map('extract',['x','y'])|list|first"
-        - "'y' == [1]|map('extract',['x','y'])|list|first"
+      - "'y' == [1]|map('extract',['x','y'])|list|first"
-        - "42 == ['x']|map('extract',{'x':42,'y':31})|list|first"
+      - "42 == ['x']|map('extract',{'x':42,'y':31})|list|first"
-        - "31 == ['x','y']|map('extract',{'x':42,'y':31})|list|last"
+      - "31 == ['x','y']|map('extract',{'x':42,'y':31})|list|last"
-        - "'local' == ['localhost']|map('extract',hostvars,'ansible_connection')|list|first"
+      - "'local' == ['localhost']|map('extract',hostvars,'ansible_connection')|list|first"
-        - "'local' == ['localhost']|map('extract',hostvars,['ansible_connection'])|list|first"
+      - "'local' == ['localhost']|map('extract',hostvars,['ansible_connection'])|list|first"
   # map was added to jinja2 in version 2.7
   when: "{{ ( lookup('pipe', '{{ ansible_python[\"executable\"] }} -c \"import jinja2; print(jinja2.__version__)\"') |
               version_compare('2.7', '>=') ) }}"
@@ -120,22 +132,40 @@
     that:
       - "users | json_query('[*].hosts[].host') == ['host_a', 'host_b', 'host_c', 'host_d']"
 
-- name: "20379 - set_fact app_var_git_branch "
+- name: Test hash filter
-  set_fact:
+  assert:
-    app_var_git_branch: multi-deployment-400-743
+    that:
+      - '"{{ "hash" | hash("sha1") }}" == "2346ad27d7568ba9896f1b7da6b5991251debdf2"'
+      - '"{{ "café" | hash("sha1") }}" == "f424452a9673918c6f09b0cdd35b20be8e6ae7d7"'
 
-- name: "20379 - trigger a error in jmespath via json_query filter to test error handling"
+- debug:
-  debug:
+    var: "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit"
-    msg: "{{ example_20379 | json_query('ApplicationVersions[].VersionLabel[] | [?starts_with(@, `multi`)]') }}"
+    verbosity: 1
-  ignore_errors: true
+  tags: debug
 
-- name: "20379 - Test errors related to https://github.com/ansible/ansible/issues/20379"
+- name: Test urlsplit filter
   assert:
-    that: "example_20379 | json_query('ApplicationVersions[].VersionLabel[] | [?starts_with(@, '+app_var_git_branch+')] | [2:]') == multisdfsdf"
+    that:
-  ignore_errors: true
+      - "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit('fragment') == 'fragment'"
+      - "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit('hostname') == 'www.acme.com'"
+      - "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit('netloc') == 'mary:MySecret@www.acme.com:9000'"
+      - "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit('path') == '/dir/index.html'"
+      - "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit('port') == 9000"
+      - "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit('query') == 'query=term'"
+      - "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit('scheme') == 'http'"
+      - "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit('username') == 'mary'"
+      - "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit('password') == 'MySecret'"
+      - "'http://mary:MySecret@www.acme.com:9000/dir/index.html?query=term#fragment' | urlsplit == { 'fragment': 'fragment', 'hostname': 'www.acme.com', 'netloc': 'mary:MySecret@www.acme.com:9000', 'password': 'MySecret', 'path': '/dir/index.html', 'port': 9000, 'query': 'query=term', 'scheme': 'http', 'username': 'mary' }"
+
+
+- name: Test urlsplit filter bad argument
+  debug:
+    var: "'http://www.acme.com:9000/dir/index.html' | urlsplit('bad_filter')"
+  register: _bad_urlsplit_filter
+  ignore_errors: yes
 
-- name: Test hash filter
+- name: Verify urlsplit filter showed an error message
   assert:
     that:
-      - '"{{ "hash" | hash("sha1") }}" == "2346ad27d7568ba9896f1b7da6b5991251debdf2"'
+      - _bad_urlsplit_filter | failed
-      - '"{{ "café" | hash("sha1") }}" == "f424452a9673918c6f09b0cdd35b20be8e6ae7d7"'
+      - "'unknown URL component' in _bad_urlsplit_filter.msg"