Remove py2 support from urls.py (#81880)

changelogs/fragments/urls-no-py2.yml

@@ -0,0 +1,2 @@
+major_changes:
+- urls.py - Removed support for Python 2

lib/ansible/plugins/lookup/url.py

@@ -87,7 +87,7 @@ options:
         - section: url_lookup
           key: force_basic_auth
   follow_redirects:
-    description: String of urllib2, all/yes, safe, none to determine how redirects are followed, see RedirectHandlerFactory for more information
+    description: String of urllib2, all/yes, safe, none to determine how redirects are followed
     type: string
     version_added: "2.10"
     default: 'urllib2'
@@ -98,6 +98,13 @@ options:
     ini:
         - section: url_lookup
           key: follow_redirects
+    choices:
+        - urllib2
+        - all
+        - 'yes'
+        - safe
+        - none
+        - 'no'
   use_gssapi:
     description:
     - Use GSSAPI handler of requests

test/integration/targets/unarchive/tasks/test_download.yml

@@ -6,19 +6,9 @@
 
 - name: Test TLS download
   block:
-  - name: Install packages to make TLS connections work on CentOS 6
-    pip:
-      name:
-        - urllib3==1.10.2
-        - ndg_httpsclient==0.4.4
-        - pyOpenSSL==16.2.0
-      state: present
-    when:
-      - ansible_facts.distribution == 'CentOS'
-      - not ansible_facts.python.has_sslcontext
   - name: unarchive a tar from an URL
     unarchive:
-      src: "https://releases.ansible.com/ansible/ansible-latest.tar.gz"
+      src: "https://ansible-ci-files.s3.amazonaws.com/test/integration/targets/apt/echo-hello-source.tar.gz"
       dest: "{{ remote_tmp_dir }}/test-unarchive-tar-gz"
       mode: "0700"
       remote_src: yes
@@ -28,16 +18,6 @@
       that:
         - "unarchive13.changed == true"
   always:
-    - name: Uninstall CentOS 6 TLS connections packages
-      pip:
-        name:
-          - urllib3
-          - ndg_httpsclient
-          - pyOpenSSL
-        state: absent
-      when:
-        - ansible_facts.distribution == 'CentOS'
-        - not ansible_facts.python.has_sslcontext
     - name: remove our tar.gz unarchive destination
       file:
         path: '{{ remote_tmp_dir }}/test-unarchive-tar-gz'

test/integration/targets/uri/tasks/install-socat-and-test-unix-socket.yml

@@ -0,0 +1,25 @@
+- when: ansible_facts.distribution not in ['MacOSX']
+  block:
+    - name: install socat
+      package:
+        name: socat
+        state: present
+      register: socat_install_package
+      when: ansible_facts.distribution not in ['Alpine']
+
+    - name: install socat for alpine
+      command: apk add --virtual .socat socat
+      register: socat_install_alpine
+      when: ansible_facts.distribution == 'Alpine'
+
+    - include_tasks: unix-socket.yml
+  always:
+    - name: uninstall socat
+      package:
+        name: socat
+        state: absent
+      when: socat_install_package|default({}) is changed
+
+    - name: uninstall socat for alpine
+      command: apk del .socat
+      when: socat_install_alpine|default({}) is changed and 'Installing socat' in socat_install_alpine.stdout

test/integration/targets/uri/tasks/main.yml

@@ -321,12 +321,6 @@
     url: 'https://{{ httpbin_host }}/get'
     use_proxy: no
 
-# Ubuntu12.04 doesn't have python-urllib3, this makes handling required dependencies a pain across all variations
-# We'll use this to just skip 12.04 on those tests.  We should be sufficiently covered with other OSes and versions
-- name: Set fact if running on Ubuntu 12.04
-  set_fact:
-    is_ubuntu_precise: "{{ ansible_distribution == 'Ubuntu' and ansible_distribution_release == 'precise' }}"
-
 - name: Test that SNI succeeds on python versions that have SNI
   uri:
     url: 'https://{{ sni_host }}/'
@@ -341,128 +335,6 @@
       - 'sni_host in result.content'
   when: ansible_python.has_sslcontext
 
-- name: Verify SNI verification fails on old python without urllib3 contrib
-  uri:
-    url: 'https://{{ sni_host }}'
-  ignore_errors: true
-  when: not ansible_python.has_sslcontext
-  register: result
-
-- name: Assert SNI verification fails on old python
-  assert:
-    that:
-      - result is failed
-  when: result is not skipped
-
-- name: check if urllib3 is installed as an OS package
-  package:
-    name: "{{ uri_os_packages[ansible_os_family].urllib3 }}"
-  check_mode: yes
-  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool and uri_os_packages[ansible_os_family].urllib3|default
-  register: urllib3
-
-- name: uninstall conflicting urllib3 pip package
-  pip:
-    name: urllib3
-    state: absent
-  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool and uri_os_packages[ansible_os_family].urllib3|default and urllib3.changed
-
-- name: install OS packages that are needed for SNI on old python
-  package:
-    name: "{{ item }}"
-  with_items: "{{ uri_os_packages[ansible_os_family].step1 | default([]) }}"
-  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
-
-- name: install python modules for Older Python SNI verification
-  pip:
-    name: "{{ item }}"
-  with_items:
-    - ndg-httpsclient
-  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
-
-- name: Verify SNI verification succeeds on old python with urllib3 contrib
-  uri:
-    url: 'https://{{ sni_host }}'
-    return_content: true
-  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
-  register: result
-
-- name: Assert SNI verification succeeds on old python
-  assert:
-    that:
-      - result is successful
-      - 'sni_host in result.content'
-  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
-
-- name: Uninstall ndg-httpsclient
-  pip:
-    name: "{{ item }}"
-    state: absent
-  with_items:
-    - ndg-httpsclient
-  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
-
-- name: uninstall OS packages that are needed for SNI on old python
-  package:
-    name: "{{ item }}"
-    state: absent
-  with_items: "{{ uri_os_packages[ansible_os_family].step1 | default([]) }}"
-  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
-
-- name: install OS packages that are needed for building cryptography
-  package:
-    name: "{{ item }}"
-  with_items: "{{ uri_os_packages[ansible_os_family].step2 | default([]) }}"
-  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
-
-- name: create constraints path
-  set_fact:
-    remote_constraints: "{{ remote_tmp_dir }}/constraints.txt"
-  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
-
-- name: create constraints file
-  copy:
-    content: |
-      cryptography == 2.1.4
-      idna == 2.5
-      pyopenssl == 17.5.0
-      six == 1.13.0
-      urllib3 == 1.23
-    dest: "{{ remote_constraints }}"
-  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
-
-- name: install urllib3 and pyopenssl via pip
-  pip:
-    name: "{{ item }}"
-    extra_args: "-c {{ remote_constraints }}"
-  with_items:
-    - urllib3
-    - PyOpenSSL
-  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
-
-- name: Verify SNI verification succeeds on old python with pip urllib3 contrib
-  uri:
-    url: 'https://{{ sni_host }}'
-    return_content: true
-  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
-  register: result
-
-- name: Assert SNI verification succeeds on old python with pip urllib3 contrib
-  assert:
-    that:
-      - result is successful
-      - 'sni_host in result.content'
-  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
-
-- name: Uninstall urllib3 and PyOpenSSL
-  pip:
-    name: "{{ item }}"
-    state: absent
-  with_items:
-    - urllib3
-    - PyOpenSSL
-  when: not ansible_python.has_sslcontext and not is_ubuntu_precise|bool
-
 - name: validate the status_codes are correct
   uri:
     url: "https://{{ httpbin_host }}/status/202"
@@ -766,6 +638,30 @@
     dest: "{{ remote_tmp_dir }}/output"
     state: absent
 
+- name: Test download root to dir without content-disposition
+  uri:
+    url: "https://{{ httpbin_host }}/"
+    dest: "{{ remote_tmp_dir }}"
+  register: get_root_no_filename
+
+- name: Test downloading to dir without content-disposition
+  uri:
+    url: "https://{{ httpbin_host }}/response-headers"
+    dest: "{{ remote_tmp_dir }}"
+  register: get_dir_no_filename
+
+- name: Test downloading to dir with content-disposition
+  uri:
+    url: 'https://{{ httpbin_host }}/response-headers?Content-Disposition=attachment%3B%20filename%3D%22filename.json%22'
+    dest: "{{ remote_tmp_dir }}"
+  register: get_dir_filename
+
+- assert:
+    that:
+      - get_root_no_filename.path == remote_tmp_dir ~ "/index.html"
+      - get_dir_no_filename.path == remote_tmp_dir ~ "/response-headers"
+      - get_dir_filename.path == remote_tmp_dir ~ "/filename.json"
+
 - name: Test follow_redirects=none
   import_tasks: redirect-none.yml
 
@@ -798,3 +694,6 @@
 
 - name: Test use_netrc.yml
   import_tasks: use_netrc.yml
+
+- name: Test unix socket
+  import_tasks: install-socat-and-test-unix-socket.yml

test/integration/targets/uri/tasks/unix-socket.yml

@@ -0,0 +1,36 @@
+- name: Bind socat to socket
+  command: socat UNIX-LISTEN:{{ remote_tmp_dir }}/{{ item.name }}.sock,fork,reuseaddr TCP4:{{ httpbin_host }}:{{ item.port }}
+  loop:
+    - port: 80
+      name: http
+    - port: 443
+      name: https
+  async: 10
+  poll: 0
+
+- name: Test http connection to unix socket
+  uri:
+    url: http://localhost/get
+    unix_socket: '{{ remote_tmp_dir }}/http.sock'
+  register: unix_socket_http
+
+- name: Test https connection to unix socket with valdiate_certs=false
+  uri:
+    url: https://localhost/get
+    unix_socket: '{{ remote_tmp_dir }}/https.sock'
+    # Ignore ssl verification since we list the host as localhost
+    # to ensure we really are connecting over the socket
+    validate_certs: false
+  register: unix_socket_https_no_validate
+
+- name: Test https connection to unix socket
+  uri:
+    url: https://{{ httpbin_host }}/get
+    unix_socket: '{{ remote_tmp_dir }}/https.sock'
+  register: unix_socket_https
+
+- assert:
+    that:
+      - unix_socket_http.json is defined
+      - unix_socket_https_no_validate.json is defined
+      - unix_socket_https.json is defined

test/integration/targets/uri/vars/main.yml

@@ -1,20 +0,0 @@
-uri_os_packages:
-  RedHat:
-    urllib3: python-urllib3
-    step1:
-      - python-pyasn1
-      - pyOpenSSL
-      - python-urllib3
-    step2:
-      - libffi-devel
-      - openssl-devel
-      - python-devel
-  Debian:
-    step1:
-      - python-pyasn1
-      - python-openssl
-      - python-urllib3
-    step2:
-      - libffi-dev
-      - libssl-dev
-      - python-dev

test/units/module_utils/urls/test_RedirectHandlerFactory.py

@@ -5,15 +5,18 @@
 from __future__ import annotations
 
 
-from ansible.module_utils.urls import HAS_SSLCONTEXT, RedirectHandlerFactory, urllib_request, urllib_error
-from ansible.module_utils.six import StringIO
+import io
+import urllib.request
+import urllib.error
+
+from ansible.module_utils.urls import HTTPRedirectHandler
 
 import pytest
 
 
 @pytest.fixture
 def urllib_req():
-    req = urllib_request.Request(
+    req = urllib.request.Request(
         'https://ansible.com/'
     )
     return req
@@ -21,20 +24,20 @@ def urllib_req():
 
 @pytest.fixture
 def request_body():
-    return StringIO('TESTS')
+    return io.StringIO('TESTS')
 
 
 def test_no_redirs(urllib_req, request_body):
-    handler = RedirectHandlerFactory('none', False)
+    handler = HTTPRedirectHandler('none')
     inst = handler()
-    with pytest.raises(urllib_error.HTTPError):
+    with pytest.raises(urllib.error.HTTPError):
         inst.redirect_request(urllib_req, request_body, 301, '301 Moved Permanently', {}, 'https://docs.ansible.com/')
 
 
 def test_urllib2_redir(urllib_req, request_body, mocker):
-    redir_request_mock = mocker.patch('ansible.module_utils.urls.urllib_request.HTTPRedirectHandler.redirect_request')
+    redir_request_mock = mocker.patch('ansible.module_utils.urls.urllib.request.HTTPRedirectHandler.redirect_request')
 
-    handler = RedirectHandlerFactory('urllib2', False)
+    handler = HTTPRedirectHandler('urllib2')
     inst = handler()
     inst.redirect_request(urllib_req, request_body, 301, '301 Moved Permanently', {}, 'https://docs.ansible.com/')
 
@@ -42,30 +45,30 @@ def test_urllib2_redir(urllib_req, request_body, mocker):
 
 
 def test_all_redir(urllib_req, request_body, mocker):
-    req_mock = mocker.patch('ansible.module_utils.urls.RequestWithMethod')
-    handler = RedirectHandlerFactory('all', False)
+    req_mock = mocker.patch('ansible.module_utils.urls.urllib.request.Request')
+    handler = HTTPRedirectHandler('all')
     inst = handler()
     inst.redirect_request(urllib_req, request_body, 301, '301 Moved Permanently', {}, 'https://docs.ansible.com/')
     req_mock.assert_called_once_with('https://docs.ansible.com/', data=None, headers={}, method='GET', origin_req_host='ansible.com', unverifiable=True)
 
 
 def test_all_redir_post(request_body, mocker):
-    handler = RedirectHandlerFactory('all', False)
+    handler = HTTPRedirectHandler('all')
     inst = handler()
 
-    req = urllib_request.Request(
+    req = urllib.request.Request(
         'https://ansible.com/',
         'POST'
     )
 
-    req_mock = mocker.patch('ansible.module_utils.urls.RequestWithMethod')
+    req_mock = mocker.patch('ansible.module_utils.urls.urllib.request.Request')
     inst.redirect_request(req, request_body, 301, '301 Moved Permanently', {}, 'https://docs.ansible.com/')
     req_mock.assert_called_once_with('https://docs.ansible.com/', data=None, headers={}, method='GET', origin_req_host='ansible.com', unverifiable=True)
 
 
 def test_redir_headers_removal(urllib_req, request_body, mocker):
-    req_mock = mocker.patch('ansible.module_utils.urls.RequestWithMethod')
-    handler = RedirectHandlerFactory('all', False)
+    req_mock = mocker.patch('ansible.module_utils.urls.urllib.request.Request')
+    handler = HTTPRedirectHandler('all')
     inst = handler()
 
     urllib_req.headers = {
@@ -80,8 +83,8 @@ def test_redir_headers_removal(urllib_req, request_body, mocker):
 
 
 def test_redir_url_spaces(urllib_req, request_body, mocker):
-    req_mock = mocker.patch('ansible.module_utils.urls.RequestWithMethod')
-    handler = RedirectHandlerFactory('all', False)
+    req_mock = mocker.patch('ansible.module_utils.urls.urllib.request.Request')
+    handler = HTTPRedirectHandler('all')
     inst = handler()
 
     inst.redirect_request(urllib_req, request_body, 301, '301 Moved Permanently', {}, 'https://docs.ansible.com/foo bar')
@@ -91,8 +94,8 @@ def test_redir_url_spaces(urllib_req, request_body, mocker):
 
 
 def test_redir_safe(urllib_req, request_body, mocker):
-    req_mock = mocker.patch('ansible.module_utils.urls.RequestWithMethod')
-    handler = RedirectHandlerFactory('safe', False)
+    req_mock = mocker.patch('ansible.module_utils.urls.urllib.request.Request')
+    handler = HTTPRedirectHandler('safe')
     inst = handler()
     inst.redirect_request(urllib_req, request_body, 301, '301 Moved Permanently', {}, 'https://docs.ansible.com/')
 
@@ -100,38 +103,29 @@ def test_redir_safe(urllib_req, request_body, mocker):
 
 
 def test_redir_safe_not_safe(request_body):
-    handler = RedirectHandlerFactory('safe', False)
+    handler = HTTPRedirectHandler('safe')
     inst = handler()
 
-    req = urllib_request.Request(
+    req = urllib.request.Request(
         'https://ansible.com/',
         'POST'
     )
 
-    with pytest.raises(urllib_error.HTTPError):
+    with pytest.raises(urllib.error.HTTPError):
         inst.redirect_request(req, request_body, 301, '301 Moved Permanently', {}, 'https://docs.ansible.com/')
 
 
 def test_redir_no_error_on_invalid(urllib_req, request_body):
-    handler = RedirectHandlerFactory('invalid', False)
+    handler = HTTPRedirectHandler('invalid')
     inst = handler()
 
-    with pytest.raises(urllib_error.HTTPError):
+    with pytest.raises(urllib.error.HTTPError):
         inst.redirect_request(urllib_req, request_body, 301, '301 Moved Permanently', {}, 'https://docs.ansible.com/')
 
 
-def test_redir_validate_certs(urllib_req, request_body, mocker):
-    opener_mock = mocker.patch('ansible.module_utils.urls.urllib_request._opener')
-    handler = RedirectHandlerFactory('all', True)
-    inst = handler()
-    inst.redirect_request(urllib_req, request_body, 301, '301 Moved Permanently', {}, 'https://docs.ansible.com/')
-
-    assert opener_mock.add_handler.call_count == int(not HAS_SSLCONTEXT)
-
-
 def test_redir_http_error_308_urllib2(urllib_req, request_body, mocker):
-    redir_mock = mocker.patch.object(urllib_request.HTTPRedirectHandler, 'redirect_request')
-    handler = RedirectHandlerFactory('urllib2', False)
+    redir_mock = mocker.patch.object(urllib.request.HTTPRedirectHandler, 'redirect_request')
+    handler = HTTPRedirectHandler('urllib2')
     inst = handler()
 
     inst.redirect_request(urllib_req, request_body, 308, '308 Permanent Redirect', {}, 'https://docs.ansible.com/')

test/units/module_utils/urls/test_Request.py

@@ -6,27 +6,28 @@ from __future__ import annotations
 
 import datetime
 import os
+import urllib.request
+import http.client
 
-from ansible.module_utils.urls import (Request, open_url, urllib_request, HAS_SSLCONTEXT, cookiejar, RequestWithMethod,
-                                       UnixHTTPHandler, UnixHTTPSConnection, httplib)
-from ansible.module_utils.urls import SSLValidationHandler, HTTPSClientAuthHandler, RedirectHandlerFactory
+from ansible.module_utils.urls import (Request, open_url, cookiejar,
+                                       UnixHTTPHandler, UnixHTTPSConnection)
+from ansible.module_utils.urls import HTTPRedirectHandler
 
 import pytest
 from units.compat.mock import call
 
 
-if HAS_SSLCONTEXT:
-    import ssl
+import ssl
 
 
 @pytest.fixture
 def urlopen_mock(mocker):
-    return mocker.patch('ansible.module_utils.urls.urllib_request.urlopen')
+    return mocker.patch('ansible.module_utils.urls.urllib.request.urlopen')
 
 
 @pytest.fixture
 def install_opener_mock(mocker):
-    return mocker.patch('ansible.module_utils.urls.urllib_request.install_opener')
+    return mocker.patch('ansible.module_utils.urls.urllib.request.install_opener')
 
 
 def test_Request_fallback(urlopen_mock, install_opener_mock, mocker):
@@ -77,10 +78,11 @@ def test_Request_fallback(urlopen_mock, install_opener_mock, mocker):
         call(None, True),  # auto_decompress
         call(None, ['ECDHE-RSA-AES128-SHA256']),  # ciphers
         call(None, True),  # use_netrc
+        call(None, None),  # context
     ]
     fallback_mock.assert_has_calls(calls)
 
-    assert fallback_mock.call_count == 18  # All but headers use fallback
+    assert fallback_mock.call_count == 19  # All but headers use fallback
 
     args = urlopen_mock.call_args[0]
     assert args[1] is None  # data, this is handled in the Request not urlopen
@@ -111,39 +113,18 @@ def test_Request_open(urlopen_mock, install_opener_mock):
     opener = install_opener_mock.call_args[0][0]
     handlers = opener.handlers
 
-    if not HAS_SSLCONTEXT:
-        expected_handlers = (
-            SSLValidationHandler,
-            RedirectHandlerFactory(),  # factory, get handler
-        )
-    else:
-        expected_handlers = (
-            RedirectHandlerFactory(),  # factory, get handler
-        )
+    expected_handlers = (
+        HTTPRedirectHandler(),
+    )
 
     found_handlers = []
     for handler in handlers:
-        if isinstance(handler, SSLValidationHandler) or handler.__class__.__name__ == 'RedirectHandler':
+        if handler.__class__.__name__ == 'HTTPRedirectHandler':
             found_handlers.append(handler)
 
     assert len(found_handlers) == len(expected_handlers)
 
 
-def test_Request_open_http(urlopen_mock, install_opener_mock):
-    r = Request().open('GET', 'http://ansible.com/')
-    args = urlopen_mock.call_args[0]
-
-    opener = install_opener_mock.call_args[0][0]
-    handlers = opener.handlers
-
-    found_handlers = []
-    for handler in handlers:
-        if isinstance(handler, SSLValidationHandler):
-            found_handlers.append(handler)
-
-    assert len(found_handlers) == 0
-
-
 def test_Request_open_unix_socket(urlopen_mock, install_opener_mock):
     r = Request().open('GET', 'http://ansible.com/', unix_socket='/foo/bar/baz.sock')
     args = urlopen_mock.call_args[0]
@@ -159,7 +140,9 @@ def test_Request_open_unix_socket(urlopen_mock, install_opener_mock):
     assert len(found_handlers) == 1
 
 
-def test_Request_open_https_unix_socket(urlopen_mock, install_opener_mock):
+def test_Request_open_https_unix_socket(urlopen_mock, install_opener_mock, mocker):
+    do_open = mocker.patch.object(urllib.request.HTTPSHandler, 'do_open')
+
     r = Request().open('GET', 'https://ansible.com/', unix_socket='/foo/bar/baz.sock')
     args = urlopen_mock.call_args[0]
 
@@ -168,13 +151,15 @@ def test_Request_open_https_unix_socket(urlopen_mock, install_opener_mock):
 
     found_handlers = []
     for handler in handlers:
-        if isinstance(handler, HTTPSClientAuthHandler):
+        if isinstance(handler, urllib.request.HTTPSHandler):
             found_handlers.append(handler)
 
     assert len(found_handlers) == 1
 
-    inst = found_handlers[0]._build_https_connection('foo')
-    assert isinstance(inst, UnixHTTPSConnection)
+    found_handlers[0].https_open(None)
+    args = do_open.call_args[0]
+    cls = args[0]
+    assert isinstance(cls, UnixHTTPSConnection)
 
 
 def test_Request_open_ftp(urlopen_mock, install_opener_mock, mocker):
@@ -198,8 +183,8 @@ def test_Request_open_username(urlopen_mock, install_opener_mock):
     handlers = opener.handlers
 
     expected_handlers = (
-        urllib_request.HTTPBasicAuthHandler,
-        urllib_request.HTTPDigestAuthHandler,
+        urllib.request.HTTPBasicAuthHandler,
+        urllib.request.HTTPDigestAuthHandler,
     )
 
     found_handlers = []
@@ -217,8 +202,8 @@ def test_Request_open_username_in_url(urlopen_mock, install_opener_mock):
     handlers = opener.handlers
 
     expected_handlers = (
-        urllib_request.HTTPBasicAuthHandler,
-        urllib_request.HTTPDigestAuthHandler,
+        urllib.request.HTTPBasicAuthHandler,
+        urllib.request.HTTPDigestAuthHandler,
     )
 
     found_handlers = []
@@ -235,8 +220,8 @@ def test_Request_open_username_force_basic(urlopen_mock, install_opener_mock):
     handlers = opener.handlers
 
     expected_handlers = (
-        urllib_request.HTTPBasicAuthHandler,
-        urllib_request.HTTPDigestAuthHandler,
+        urllib.request.HTTPBasicAuthHandler,
+        urllib.request.HTTPDigestAuthHandler,
     )
 
     found_handlers = []
@@ -261,8 +246,8 @@ def test_Request_open_auth_in_netloc(urlopen_mock, install_opener_mock):
     handlers = opener.handlers
 
     expected_handlers = (
-        urllib_request.HTTPBasicAuthHandler,
-        urllib_request.HTTPDigestAuthHandler,
+        urllib.request.HTTPBasicAuthHandler,
+        urllib.request.HTTPDigestAuthHandler,
     )
 
     found_handlers = []
@@ -295,21 +280,22 @@ def test_Request_open_netrc(urlopen_mock, install_opener_mock, monkeypatch):
 
 
 def test_Request_open_no_proxy(urlopen_mock, install_opener_mock, mocker):
-    build_opener_mock = mocker.patch('ansible.module_utils.urls.urllib_request.build_opener')
+    build_opener_mock = mocker.patch('ansible.module_utils.urls.urllib.request.build_opener')
 
     r = Request().open('GET', 'http://ansible.com/', use_proxy=False)
 
     handlers = build_opener_mock.call_args[0]
     found_handlers = []
     for handler in handlers:
-        if isinstance(handler, urllib_request.ProxyHandler):
+        if isinstance(handler, urllib.request.ProxyHandler):
             found_handlers.append(handler)
 
     assert len(found_handlers) == 1
 
 
-@pytest.mark.skipif(not HAS_SSLCONTEXT, reason="requires SSLContext")
-def test_Request_open_no_validate_certs(urlopen_mock, install_opener_mock):
+def test_Request_open_no_validate_certs(urlopen_mock, install_opener_mock, mocker):
+    do_open = mocker.patch.object(urllib.request.HTTPSHandler, 'do_open')
+
     r = Request().open('GET', 'https://ansible.com/', validate_certs=False)
 
     opener = install_opener_mock.call_args[0][0]
@@ -317,14 +303,16 @@ def test_Request_open_no_validate_certs(urlopen_mock, install_opener_mock):
 
     ssl_handler = None
     for handler in handlers:
-        if isinstance(handler, HTTPSClientAuthHandler):
+        if isinstance(handler, urllib.request.HTTPSHandler):
             ssl_handler = handler
             break
 
     assert ssl_handler is not None
 
-    inst = ssl_handler._build_https_connection('foo')
-    assert isinstance(inst, httplib.HTTPSConnection)
+    ssl_handler.https_open(None)
+    args = do_open.call_args[0]
+    cls = args[0]
+    assert cls is http.client.HTTPSConnection
 
     context = ssl_handler._context
     # Differs by Python version
@@ -336,7 +324,9 @@ def test_Request_open_no_validate_certs(urlopen_mock, install_opener_mock):
     assert context.check_hostname is False
 
 
-def test_Request_open_client_cert(urlopen_mock, install_opener_mock):
+def test_Request_open_client_cert(urlopen_mock, install_opener_mock, mocker):
+    load_cert_chain = mocker.patch.object(ssl.SSLContext, 'load_cert_chain')
+
     here = os.path.dirname(__file__)
 
     client_cert = os.path.join(here, 'fixtures/client.pem')
@@ -349,16 +339,13 @@ def test_Request_open_client_cert(urlopen_mock, install_opener_mock):
 
     ssl_handler = None
     for handler in handlers:
-        if isinstance(handler, HTTPSClientAuthHandler):
+        if isinstance(handler, urllib.request.HTTPSHandler):
             ssl_handler = handler
             break
 
     assert ssl_handler is not None
 
-    assert ssl_handler.client_cert == client_cert
-    assert ssl_handler.client_key == client_key
-
-    ssl_handler._build_https_connection('ansible.com')
+    load_cert_chain.assert_called_once_with(client_cert, keyfile=client_key)
 
 
 def test_Request_open_cookies(urlopen_mock, install_opener_mock):
@@ -369,7 +356,7 @@ def test_Request_open_cookies(urlopen_mock, install_opener_mock):
 
     cookies_handler = None
     for handler in handlers:
-        if isinstance(handler, urllib_request.HTTPCookieProcessor):
+        if isinstance(handler, urllib.request.HTTPCookieProcessor):
             cookies_handler = handler
             break
 
@@ -387,15 +374,6 @@ def test_Request_open_invalid_method(urlopen_mock, install_opener_mock):
     # assert r.status == 504
 
 
-def test_Request_open_custom_method(urlopen_mock, install_opener_mock):
-    r = Request().open('DELETE', 'https://ansible.com/')
-
-    args = urlopen_mock.call_args[0]
-    req = args[0]
-
-    assert isinstance(req, RequestWithMethod)
-
-
 def test_Request_open_user_agent(urlopen_mock, install_opener_mock):
     r = Request().open('GET', 'https://ansible.com/', http_agent='ansible-tests')
 

test/units/module_utils/urls/test_RequestWithMethod.py

@@ -1,21 +0,0 @@
-# -*- coding: utf-8 -*-
-# (c) 2018 Matt Martz <matt@sivel.net>
-# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-
-from __future__ import annotations
-
-from ansible.module_utils.urls import RequestWithMethod
-
-
-def test_RequestWithMethod():
-    get = RequestWithMethod('https://ansible.com/', 'GET')
-    assert get.get_method() == 'GET'
-
-    post = RequestWithMethod('https://ansible.com/', 'POST', data='foo', headers={'Bar': 'baz'})
-    assert post.get_method() == 'POST'
-    assert post.get_full_url() == 'https://ansible.com/'
-    assert post.data == 'foo'
-    assert post.headers == {'Bar': 'baz'}
-
-    none = RequestWithMethod('https://ansible.com/', '')
-    assert none.get_method() == 'GET'

test/units/module_utils/urls/test_fetch_url.py

@@ -4,13 +4,14 @@
 
 from __future__ import annotations
 
+import io
 import socket
 import sys
+import http.client
+import urllib.error
+from http.cookiejar import Cookie
 
-from ansible.module_utils.six import StringIO
-from ansible.module_utils.six.moves.http_cookiejar import Cookie
-from ansible.module_utils.six.moves.http_client import HTTPMessage
-from ansible.module_utils.urls import fetch_url, urllib_error, ConnectionError, NoSSLError, httplib
+from ansible.module_utils.urls import fetch_url, ConnectionError
 
 import pytest
 from units.compat.mock import MagicMock
@@ -52,13 +53,6 @@ class FakeAnsibleModule:
         raise FailJson(*args, **kwargs)
 
 
-def test_fetch_url_no_urlparse(mocker, fake_ansible_module):
-    mocker.patch('ansible.module_utils.urls.HAS_URLPARSE', new=False)
-
-    with pytest.raises(FailJson):
-        fetch_url(fake_ansible_module, 'http://ansible.com/')
-
-
 def test_fetch_url(open_url_mock, fake_ansible_module):
     r, info = fetch_url(fake_ansible_module, 'http://ansible.com/')
 
@@ -98,13 +92,8 @@ def test_fetch_url_cookies(mocker, fake_ansible_module):
     def make_cookies(*args, **kwargs):
         cookies = kwargs['cookies']
         r = MagicMock()
-        try:
-            r.headers = HTTPMessage()
-            add_header = r.headers.add_header
-        except TypeError:
-            # PY2
-            r.headers = HTTPMessage(StringIO())
-            add_header = r.headers.addheader
+        r.headers = http.client.HTTPMessage()
+        add_header = r.headers.add_header
         r.info.return_value = r.headers
         for name, value in (('Foo', 'bar'), ('Baz', 'qux')):
             cookie = Cookie(
@@ -150,26 +139,6 @@ def test_fetch_url_cookies(mocker, fake_ansible_module):
     assert info['set-cookie'] == 'Foo=bar, Baz=qux'
 
 
-def test_fetch_url_nossl(open_url_mock, fake_ansible_module, mocker):
-    mocker.patch('ansible.module_utils.urls.get_distribution', return_value='notredhat')
-
-    open_url_mock.side_effect = NoSSLError
-    with pytest.raises(FailJson) as excinfo:
-        fetch_url(fake_ansible_module, 'http://ansible.com/')
-
-    assert 'python-ssl' not in excinfo.value.kwargs['msg']
-
-    mocker.patch('ansible.module_utils.urls.get_distribution', return_value='redhat')
-
-    open_url_mock.side_effect = NoSSLError
-    with pytest.raises(FailJson) as excinfo:
-        fetch_url(fake_ansible_module, 'http://ansible.com/')
-
-    assert 'python-ssl' in excinfo.value.kwargs['msg']
-    assert 'http://ansible.com/' == excinfo.value.kwargs['url']
-    assert excinfo.value.kwargs['status'] == -1
-
-
 def test_fetch_url_connectionerror(open_url_mock, fake_ansible_module):
     open_url_mock.side_effect = ConnectionError('TESTS')
     with pytest.raises(FailJson) as excinfo:
@@ -189,12 +158,12 @@ def test_fetch_url_connectionerror(open_url_mock, fake_ansible_module):
 
 
 def test_fetch_url_httperror(open_url_mock, fake_ansible_module):
-    open_url_mock.side_effect = urllib_error.HTTPError(
+    open_url_mock.side_effect = urllib.error.HTTPError(
         'http://ansible.com/',
         500,
         'Internal Server Error',
         {'Content-Type': 'application/json'},
-        StringIO('TESTS')
+        io.StringIO('TESTS')
     )
 
     r, info = fetch_url(fake_ansible_module, 'http://ansible.com/')
@@ -204,7 +173,7 @@ def test_fetch_url_httperror(open_url_mock, fake_ansible_module):
 
 
 def test_fetch_url_urlerror(open_url_mock, fake_ansible_module):
-    open_url_mock.side_effect = urllib_error.URLError('TESTS')
+    open_url_mock.side_effect = urllib.error.URLError('TESTS')
     r, info = fetch_url(fake_ansible_module, 'http://ansible.com/')
     assert info == {'msg': 'Request failed: <urlopen error TESTS>', 'status': -1, 'url': 'http://ansible.com/'}
 
@@ -224,6 +193,6 @@ def test_fetch_url_exception(open_url_mock, fake_ansible_module):
 
 
 def test_fetch_url_badstatusline(open_url_mock, fake_ansible_module):
-    open_url_mock.side_effect = httplib.BadStatusLine('TESTS')
+    open_url_mock.side_effect = http.client.BadStatusLine('TESTS')
     r, info = fetch_url(fake_ansible_module, 'http://ansible.com/')
     assert info == {'msg': 'Connection failure: connection was closed before a valid response was received: TESTS', 'status': -1, 'url': 'http://ansible.com/'}

test/units/module_utils/urls/test_generic_urlparse.py

@@ -25,32 +25,3 @@ def test_generic_urlparse_netloc():
     assert generic_parts.hostname == 'ansible.com'
     assert generic_parts.port == 443
     assert urlunparse(generic_parts.as_list()) == url
-
-
-def test_generic_urlparse_no_netloc():
-    url = 'https://user:passwd@ansible.com:443/blog'
-    parts = list(urlparse(url))
-    generic_parts = generic_urlparse(parts)
-    assert generic_parts.hostname == 'ansible.com'
-    assert generic_parts.port == 443
-    assert generic_parts.username == 'user'
-    assert generic_parts.password == 'passwd'
-    assert urlunparse(generic_parts.as_list()) == url
-
-
-def test_generic_urlparse_no_netloc_no_auth():
-    url = 'https://ansible.com:443/blog'
-    parts = list(urlparse(url))
-    generic_parts = generic_urlparse(parts)
-    assert generic_parts.username is None
-    assert generic_parts.password is None
-
-
-def test_generic_urlparse_no_netloc_no_host():
-    url = '/blog'
-    parts = list(urlparse(url))
-    generic_parts = generic_urlparse(parts)
-    assert generic_parts.username is None
-    assert generic_parts.password is None
-    assert generic_parts.port is None
-    assert generic_parts.hostname == ''

test/units/module_utils/urls/test_gzip.py

@@ -7,14 +7,8 @@ from __future__ import annotations
 import gzip
 import io
 import sys
+import http.client
 
-try:
-    from urllib.response import addinfourl
-except ImportError:
-    from urllib import addinfourl
-
-from ansible.module_utils.six import PY3
-from ansible.module_utils.six.moves import http_client
 from ansible.module_utils.urls import GzipDecodedReader, Request
 
 import pytest
@@ -37,7 +31,7 @@ class Sock(io.BytesIO):
 
 @pytest.fixture
 def urlopen_mock(mocker):
-    return mocker.patch('ansible.module_utils.urls.urllib_request.urlopen')
+    return mocker.patch('ansible.module_utils.urls.urllib.request.urlopen')
 
 
 JSON_DATA = b'{"foo": "bar", "baz": "qux", "sandwich": "ham", "tech_level": "pickle", "pop": "corn", "ansible": "awesome"}'
@@ -62,22 +56,13 @@ Content-Length: 100
 
 
 def test_Request_open_gzip(urlopen_mock):
-    h = http_client.HTTPResponse(
+    h = http.client.HTTPResponse(
         Sock(GZIP_RESP),
         method='GET',
     )
     h.begin()
 
-    if PY3:
-        urlopen_mock.return_value = h
-    else:
-        urlopen_mock.return_value = addinfourl(
-            h.fp,
-            h.msg,
-            'http://ansible.com/',
-            h.status,
-        )
-        urlopen_mock.return_value.msg = h.reason
+    urlopen_mock.return_value = h
 
     r = Request().open('GET', 'https://ansible.com/')
     assert isinstance(r.fp, GzipDecodedReader)
@@ -85,22 +70,13 @@ def test_Request_open_gzip(urlopen_mock):
 
 
 def test_Request_open_not_gzip(urlopen_mock):
-    h = http_client.HTTPResponse(
+    h = http.client.HTTPResponse(
         Sock(RESP),
         method='GET',
     )
     h.begin()
 
-    if PY3:
-        urlopen_mock.return_value = h
-    else:
-        urlopen_mock.return_value = addinfourl(
-            h.fp,
-            h.msg,
-            'http://ansible.com/',
-            h.status,
-        )
-        urlopen_mock.return_value.msg = h.reason
+    urlopen_mock.return_value = h
 
     r = Request().open('GET', 'https://ansible.com/')
     assert not isinstance(r.fp, GzipDecodedReader)
@@ -108,22 +84,13 @@ def test_Request_open_not_gzip(urlopen_mock):
 
 
 def test_Request_open_decompress_false(urlopen_mock):
-    h = http_client.HTTPResponse(
+    h = http.client.HTTPResponse(
         Sock(RESP),
         method='GET',
     )
     h.begin()
 
-    if PY3:
-        urlopen_mock.return_value = h
-    else:
-        urlopen_mock.return_value = addinfourl(
-            h.fp,
-            h.msg,
-            'http://ansible.com/',
-            h.status,
-        )
-        urlopen_mock.return_value.msg = h.reason
+    urlopen_mock.return_value = h
 
     r = Request().open('GET', 'https://ansible.com/', decompress=False)
     assert not isinstance(r.fp, GzipDecodedReader)
@@ -141,10 +108,7 @@ def test_GzipDecodedReader_no_gzip(monkeypatch, mocker):
             raise ImportError
         return orig_import(*args)
 
-    if PY3:
-        mocker.patch('builtins.__import__', _import)
-    else:
-        mocker.patch('__builtin__.__import__', _import)
+    mocker.patch('builtins.__import__', _import)
 
     mod = __import__('ansible.module_utils.urls').module_utils.urls
     assert mod.HAS_GZIP is False

test/units/module_utils/urls/test_urls.py

@@ -5,85 +5,6 @@
 from __future__ import annotations
 
 from ansible.module_utils import urls
-from ansible.module_utils.common.text.converters import to_native
-
-import pytest
-
-
-def test_build_ssl_validation_error(mocker):
-    mocker.patch.object(urls, 'HAS_SSLCONTEXT', new=False)
-    mocker.patch.object(urls, 'HAS_URLLIB3_PYOPENSSLCONTEXT', new=False)
-    mocker.patch.object(urls, 'HAS_URLLIB3_SSL_WRAP_SOCKET', new=False)
-    with pytest.raises(urls.SSLValidationError) as excinfo:
-        urls.build_ssl_validation_error('hostname', 'port', 'paths', exc=None)
-
-    assert 'python >= 2.7.9' in to_native(excinfo.value)
-    assert 'the python executable used' in to_native(excinfo.value)
-    assert 'urllib3' in to_native(excinfo.value)
-    assert 'python >= 2.6' in to_native(excinfo.value)
-    assert 'validate_certs=False' in to_native(excinfo.value)
-
-    mocker.patch.object(urls, 'HAS_SSLCONTEXT', new=True)
-    with pytest.raises(urls.SSLValidationError) as excinfo:
-        urls.build_ssl_validation_error('hostname', 'port', 'paths', exc=None)
-
-    assert 'validate_certs=False' in to_native(excinfo.value)
-
-    mocker.patch.object(urls, 'HAS_SSLCONTEXT', new=False)
-    mocker.patch.object(urls, 'HAS_URLLIB3_PYOPENSSLCONTEXT', new=True)
-    mocker.patch.object(urls, 'HAS_URLLIB3_SSL_WRAP_SOCKET', new=True)
-
-    mocker.patch.object(urls, 'HAS_SSLCONTEXT', new=True)
-    with pytest.raises(urls.SSLValidationError) as excinfo:
-        urls.build_ssl_validation_error('hostname', 'port', 'paths', exc=None)
-
-    assert 'urllib3' not in to_native(excinfo.value)
-
-    with pytest.raises(urls.SSLValidationError) as excinfo:
-        urls.build_ssl_validation_error('hostname', 'port', 'paths', exc='BOOM')
-
-    assert 'BOOM' in to_native(excinfo.value)
-
-
-def test_maybe_add_ssl_handler(mocker):
-    mocker.patch.object(urls, 'HAS_SSL', new=False)
-    with pytest.raises(urls.NoSSLError):
-        urls.maybe_add_ssl_handler('https://ansible.com/', True)
-
-    mocker.patch.object(urls, 'HAS_SSL', new=True)
-    url = 'https://user:passwd@ansible.com/'
-    handler = urls.maybe_add_ssl_handler(url, True)
-    assert handler.hostname == 'ansible.com'
-    assert handler.port == 443
-
-    url = 'https://ansible.com:4433/'
-    handler = urls.maybe_add_ssl_handler(url, True)
-    assert handler.hostname == 'ansible.com'
-    assert handler.port == 4433
-
-    url = 'https://user:passwd@ansible.com:4433/'
-    handler = urls.maybe_add_ssl_handler(url, True)
-    assert handler.hostname == 'ansible.com'
-    assert handler.port == 4433
-
-    url = 'https://ansible.com/'
-    handler = urls.maybe_add_ssl_handler(url, True)
-    assert handler.hostname == 'ansible.com'
-    assert handler.port == 443
-
-    url = 'http://ansible.com/'
-    handler = urls.maybe_add_ssl_handler(url, True)
-    assert handler is None
-
-    url = 'https://[2a00:16d8:0:7::205]:4443/'
-    handler = urls.maybe_add_ssl_handler(url, True)
-    assert handler.hostname == '2a00:16d8:0:7::205'
-    assert handler.port == 4443
-
-    url = 'https://[2a00:16d8:0:7::205]/'
-    handler = urls.maybe_add_ssl_handler(url, True)
-    assert handler.hostname == '2a00:16d8:0:7::205'
-    assert handler.port == 443
 
 
 def test_basic_auth_header():
@@ -102,7 +23,7 @@ def test_ParseResultDottedDict():
 
 def test_unix_socket_patch_httpconnection_connect(mocker):
     unix_conn = mocker.patch.object(urls.UnixHTTPConnection, 'connect')
-    conn = urls.httplib.HTTPConnection('ansible.com')
+    conn = urls.http.client.HTTPConnection('ansible.com')
     with urls.unix_socket_patch_httpconnection_connect():
         conn.connect()
     assert unix_conn.call_count == 1