diff --git a/changelogs/fragments/uri-unix-socket.yml b/changelogs/fragments/uri-unix-socket.yml new file mode 100644 index 00000000000..ca695edd987 --- /dev/null +++ b/changelogs/fragments/uri-unix-socket.yml @@ -0,0 +1,2 @@ +minor_changes: +- uri/urls - Support unix domain sockets (https://github.com/ansible/ansible/pull/43560) diff --git a/lib/ansible/module_utils/urls.py b/lib/ansible/module_utils/urls.py index 418479ba4a3..cc75918d343 100644 --- a/lib/ansible/module_utils/urls.py +++ b/lib/ansible/module_utils/urls.py @@ -42,6 +42,8 @@ import sys import tempfile import traceback +from contextlib import contextmanager + try: import httplib except ImportError: @@ -362,10 +364,11 @@ if hasattr(httplib, 'HTTPSConnection') and hasattr(urllib_request, 'HTTPSHandler in place of HTTPSHandler ''' - def __init__(self, client_cert=None, client_key=None, **kwargs): + def __init__(self, client_cert=None, client_key=None, unix_socket=None, **kwargs): urllib_request.HTTPSHandler.__init__(self, **kwargs) self.client_cert = client_cert self.client_key = client_key + self._unix_socket = unix_socket def https_open(self, req): return self.do_open(self._build_https_connection, req) @@ -379,9 +382,69 @@ if hasattr(httplib, 'HTTPSConnection') and hasattr(urllib_request, 'HTTPSHandler kwargs['context'] = self._context except AttributeError: pass + if self._unix_socket: + return UnixHTTPSConnection(self._unix_socket)(host, **kwargs) return httplib.HTTPSConnection(host, **kwargs) +@contextmanager +def unix_socket_patch_httpconnection_connect(): + '''Monkey patch ``httplib.HTTPConnection.connect`` to be ``UnixHTTPConnection.connect`` + so that when calling ``super(UnixHTTPSConnection, self).connect()`` we get the + correct behavior of creating self.sock for the unix socket + ''' + _connect = httplib.HTTPConnection.connect + httplib.HTTPConnection.connect = UnixHTTPConnection.connect + yield + httplib.HTTPConnection.connect = _connect + + +class UnixHTTPSConnection(httplib.HTTPSConnection): + def __init__(self, unix_socket): + self._unix_socket = unix_socket + + def connect(self): + # This method exists simply to ensure we monkeypatch + # httplib.HTTPConnection.connect to call UnixHTTPConnection.connect + with unix_socket_patch_httpconnection_connect(): + super(UnixHTTPSConnection, self).connect() + + def __call__(self, *args, **kwargs): + httplib.HTTPSConnection.__init__(self, *args, **kwargs) + return self + + +class UnixHTTPConnection(httplib.HTTPConnection): + '''Handles http requests to a unix socket file''' + + def __init__(self, unix_socket): + self._unix_socket = unix_socket + + def connect(self): + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + self.sock.connect(self._unix_socket) + except OSError as e: + raise OSError('Invalid Socket File (%s): %s' % (self._unix_socket, e)) + if self.timeout is not socket._GLOBAL_DEFAULT_TIMEOUT: + self.sock.settimeout(self.timeout) + + def __call__(self, *args, **kwargs): + httplib.HTTPConnection.__init__(self, *args, **kwargs) + return self + + +class UnixHTTPHandler(urllib_request.HTTPHandler): + '''Handler for Unix urls''' + + def __init__(self, unix_socket, **kwargs): + urllib_request.HTTPHandler.__init__(self, **kwargs) + self._unix_socket = unix_socket + + def http_open(self, req): + return self.do_open(UnixHTTPConnection(self._unix_socket), req) + + class ParseResultDottedDict(dict): ''' A dict that acts similarly to the ParseResult named tuple from urllib @@ -854,7 +917,7 @@ def rfc2822_date_string(timetuple, zone='-0000'): class Request: def __init__(self, headers=None, use_proxy=True, force=False, timeout=10, validate_certs=True, url_username=None, url_password=None, http_agent=None, force_basic_auth=False, - follow_redirects='urllib2', client_cert=None, client_key=None, cookies=None): + follow_redirects='urllib2', client_cert=None, client_key=None, cookies=None, unix_socket=None): """This class works somewhat similarly to the ``Session`` class of from requests by defining a cookiejar that an be used across requests as well as cascaded defaults that can apply to repeated requests @@ -875,7 +938,7 @@ class Request: self.headers = headers or {} if not isinstance(self.headers, dict): - raise ValueError("headers must be a dict") + raise ValueError("headers must be a dict: %r" % self.headers) self.use_proxy = use_proxy self.force = force self.timeout = timeout @@ -887,6 +950,7 @@ class Request: self.follow_redirects = follow_redirects self.client_cert = client_cert self.client_key = client_key + self.unix_socket = unix_socket if isinstance(cookies, cookiejar.CookieJar): self.cookies = cookies else: @@ -901,7 +965,8 @@ class Request: force=None, last_mod_time=None, timeout=None, validate_certs=None, url_username=None, url_password=None, http_agent=None, force_basic_auth=None, follow_redirects=None, - client_cert=None, client_key=None, cookies=None, use_gssapi=False): + client_cert=None, client_key=None, cookies=None, use_gssapi=False, + unix_socket=None): """ Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3) @@ -936,6 +1001,8 @@ class Request: :kwarg cookies: (optional) CookieJar object to send with the request :kwarg use_gssapi: (optional) Use GSSAPI handler of requests. + :kwarg unix_socket: (optional) String of file system path to unix socket file to use when establishing + connection to the provided url :returns: HTTPResponse """ @@ -959,8 +1026,13 @@ class Request: client_cert = self._fallback(client_cert, self.client_cert) client_key = self._fallback(client_key, self.client_key) cookies = self._fallback(cookies, self.cookies) + unix_socket = self._fallback(unix_socket, self.unix_socket) handlers = [] + + if unix_socket: + handlers.append(UnixHTTPHandler(unix_socket)) + ssl_handler = maybe_add_ssl_handler(url, validate_certs) if ssl_handler: handlers.append(ssl_handler) @@ -1033,10 +1105,12 @@ class Request: context.check_hostname = False handlers.append(HTTPSClientAuthHandler(client_cert=client_cert, client_key=client_key, - context=context)) - elif client_cert: + context=context, + unix_socket=unix_socket)) + elif client_cert or unix_socket: handlers.append(HTTPSClientAuthHandler(client_cert=client_cert, - client_key=client_key)) + client_key=client_key, + unix_socket=unix_socket)) # pre-2.6 versions of python cannot use the custom https # handler, since the socket class is lacking create_connection. @@ -1164,7 +1238,7 @@ def open_url(url, data=None, headers=None, method=None, use_proxy=True, url_username=None, url_password=None, http_agent=None, force_basic_auth=False, follow_redirects='urllib2', client_cert=None, client_key=None, cookies=None, - use_gssapi=False): + use_gssapi=False, unix_socket=None): ''' Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3) @@ -1176,7 +1250,7 @@ def open_url(url, data=None, headers=None, method=None, use_proxy=True, url_username=url_username, url_password=url_password, http_agent=http_agent, force_basic_auth=force_basic_auth, follow_redirects=follow_redirects, client_cert=client_cert, client_key=client_key, cookies=cookies, - use_gssapi=use_gssapi) + use_gssapi=use_gssapi, unix_socket=unix_socket) # @@ -1212,7 +1286,7 @@ def url_argument_spec(): def fetch_url(module, url, data=None, headers=None, method=None, use_proxy=True, force=False, last_mod_time=None, timeout=10, - use_gssapi=False): + use_gssapi=False, unix_socket=None): """Sends a request via HTTP(S) or FTP (needs the module as parameter) :arg module: The AnsibleModule (used to get username, password etc. (s.b.). @@ -1226,6 +1300,8 @@ def fetch_url(module, url, data=None, headers=None, method=None, :kwarg last_mod_time: Default: None :kwarg int timeout: Default: 10 :kwarg boolean use_gssapi: Default: False + :kwarg unix_socket: (optional) String of file system path to unix socket file to use when establishing + connection to the provided url :returns: A tuple of (**response**, **info**). Use ``response.read()`` to read the data. The **info** contains the 'status' and other meta data. When a HttpError (status > 400) @@ -1275,7 +1351,8 @@ def fetch_url(module, url, data=None, headers=None, method=None, validate_certs=validate_certs, url_username=username, url_password=password, http_agent=http_agent, force_basic_auth=force_basic_auth, follow_redirects=follow_redirects, client_cert=client_cert, - client_key=client_key, cookies=cookies, use_gssapi=use_gssapi) + client_key=client_key, cookies=cookies, use_gssapi=use_gssapi, + unix_socket=unix_socket) # Lowercase keys, to conform to py2 behavior, so that py3 and py2 are predictable info.update(dict((k.lower(), v) for k, v in r.info().items())) diff --git a/lib/ansible/modules/net_tools/basics/uri.py b/lib/ansible/modules/net_tools/basics/uri.py index e1c77dc0a0c..a160b97a561 100644 --- a/lib/ansible/modules/net_tools/basics/uri.py +++ b/lib/ansible/modules/net_tools/basics/uri.py @@ -170,6 +170,10 @@ options: - If C(no), it will not use a proxy, even if one is defined in an environment variable on the target hosts. type: bool default: yes + unix_socket: + description: + - Path to Unix domain socket to use for connection + version_added: '2.8' notes: - The dependency on httplib2 was removed in Ansible 2.1. - The module returns all the HTTP headers in lower-case. @@ -460,7 +464,7 @@ def uri(module, url, dest, body, body_format, method, headers, socket_timeout): _, redir_info = fetch_url(module, url, data=body, headers=headers, method=method, - timeout=socket_timeout) + timeout=socket_timeout, unix_socket=module.params['unix_socket']) # if we are redirected, update the url with the location header, # and update dest with the new url filename if redir_info['status'] in (301, 302, 303, 307): @@ -475,7 +479,8 @@ def uri(module, url, dest, body, body_format, method, headers, socket_timeout): module.params['follow_redirects'] = follow_redirects resp, info = fetch_url(module, url, data=data, headers=headers, - method=method, timeout=socket_timeout, **kwargs) + method=method, timeout=socket_timeout, unix_socket=module.params['unix_socket'], + **kwargs) try: content = resp.read() @@ -515,6 +520,7 @@ def main(): status_code=dict(type='list', default=[200]), timeout=dict(type='int', default=30), headers=dict(type='dict', default={}), + unix_socket=dict(type='path'), ) module = AnsibleModule( diff --git a/test/units/module_utils/urls/test_Request.py b/test/units/module_utils/urls/test_Request.py index f8ca7939901..9b69edd3785 100644 --- a/test/units/module_utils/urls/test_Request.py +++ b/test/units/module_utils/urls/test_Request.py @@ -8,7 +8,8 @@ __metaclass__ = type import datetime import os -from ansible.module_utils.urls import Request, open_url, urllib_request, HAS_SSLCONTEXT, cookiejar, ConnectionError, RequestWithMethod +from ansible.module_utils.urls import (Request, open_url, urllib_request, HAS_SSLCONTEXT, cookiejar, ConnectionError, RequestWithMethod, + UnixHTTPHandler, UnixHTTPSConnection, httplib) from ansible.module_utils.urls import SSLValidationHandler, HTTPSClientAuthHandler, RedirectHandlerFactory import pytest @@ -45,6 +46,7 @@ def test_Request_fallback(urlopen_mock, install_opener_mock, mocker): client_cert='/tmp/client.pem', client_key='/tmp/client.key', cookies=cookies, + unix_socket='/foo/bar/baz.sock', ) fallback_mock = mocker.spy(request, '_fallback') @@ -63,10 +65,11 @@ def test_Request_fallback(urlopen_mock, install_opener_mock, mocker): call(None, '/tmp/client.pem'), # client_cert call(None, '/tmp/client.key'), # client_key call(None, cookies), # cookies + call(None, '/foo/bar/baz.sock'), # unix_socket ] fallback_mock.assert_has_calls(calls) - assert fallback_mock.call_count == 12 # All but headers use fallback + assert fallback_mock.call_count == 13 # All but headers use fallback args = urlopen_mock.call_args[0] assert args[1] is None # data, this is handled in the Request not urlopen @@ -125,6 +128,39 @@ def test_Request_open_http(urlopen_mock, install_opener_mock): assert len(found_handlers) == 0 +def test_Request_open_unix_socket(urlopen_mock, install_opener_mock): + r = Request().open('GET', 'http://ansible.com/', unix_socket='/foo/bar/baz.sock') + args = urlopen_mock.call_args[0] + + opener = install_opener_mock.call_args[0][0] + handlers = opener.handlers + + found_handlers = [] + for handler in handlers: + if isinstance(handler, UnixHTTPHandler): + found_handlers.append(handler) + + assert len(found_handlers) == 1 + + +def test_Request_open_https_unix_socket(urlopen_mock, install_opener_mock): + r = Request().open('GET', 'https://ansible.com/', unix_socket='/foo/bar/baz.sock') + args = urlopen_mock.call_args[0] + + opener = install_opener_mock.call_args[0][0] + handlers = opener.handlers + + found_handlers = [] + for handler in handlers: + if isinstance(handler, HTTPSClientAuthHandler): + found_handlers.append(handler) + + assert len(found_handlers) == 1 + + inst = found_handlers[0]._build_https_connection('foo') + assert isinstance(inst, UnixHTTPSConnection) + + def test_Request_open_ftp(urlopen_mock, install_opener_mock, mocker): mocker.patch('ansible.module_utils.urls.ParseResultDottedDict.as_list', side_effect=AssertionError) @@ -270,6 +306,10 @@ def test_Request_open_no_validate_certs(urlopen_mock, install_opener_mock): break assert ssl_handler is not None + + inst = ssl_handler._build_https_connection('foo') + assert isinstance(inst, httplib.HTTPSConnection) + context = ssl_handler._context assert context.protocol == ssl.PROTOCOL_SSLv23 if ssl.OP_NO_SSLv2: @@ -399,4 +439,5 @@ def test_open_url(urlopen_mock, install_opener_mock, mocker): force=False, last_mod_time=None, timeout=10, validate_certs=True, url_username=None, url_password=None, http_agent=None, force_basic_auth=False, follow_redirects='urllib2', - client_cert=None, client_key=None, cookies=None, use_gssapi=False) + client_cert=None, client_key=None, cookies=None, use_gssapi=False, + unix_socket=None) diff --git a/test/units/module_utils/urls/test_fetch_url.py b/test/units/module_utils/urls/test_fetch_url.py index f983b1e8be8..0af03e74fb4 100644 --- a/test/units/module_utils/urls/test_fetch_url.py +++ b/test/units/module_utils/urls/test_fetch_url.py @@ -67,7 +67,7 @@ def test_fetch_url(open_url_mock, fake_ansible_module): open_url_mock.assert_called_once_with('http://ansible.com/', client_cert=None, client_key=None, cookies=kwargs['cookies'], data=None, follow_redirects='urllib2', force=False, force_basic_auth='', headers=None, http_agent='ansible-httpget', last_mod_time=None, method=None, timeout=10, url_password='', url_username='', - use_proxy=True, validate_certs=True, use_gssapi=False) + use_proxy=True, validate_certs=True, use_gssapi=False, unix_socket=None) def test_fetch_url_params(open_url_mock, fake_ansible_module): @@ -89,7 +89,7 @@ def test_fetch_url_params(open_url_mock, fake_ansible_module): open_url_mock.assert_called_once_with('http://ansible.com/', client_cert='client.pem', client_key='client.key', cookies=kwargs['cookies'], data=None, follow_redirects='all', force=False, force_basic_auth=True, headers=None, http_agent='ansible-test', last_mod_time=None, method=None, timeout=10, url_password='passwd', url_username='user', - use_proxy=True, validate_certs=False, use_gssapi=False) + use_proxy=True, validate_certs=False, use_gssapi=False, unix_socket=None) def test_fetch_url_cookies(mocker, fake_ansible_module): diff --git a/test/units/module_utils/urls/test_urls.py b/test/units/module_utils/urls/test_urls.py index 8f2b721b405..3dfe28e0cb2 100644 --- a/test/units/module_utils/urls/test_urls.py +++ b/test/units/module_utils/urls/test_urls.py @@ -89,3 +89,11 @@ def test_ParseResultDottedDict(): assert parts[0] == dotted_parts.scheme assert dotted_parts.as_list() == list(parts) + + +def test_unix_socket_patch_httpconnection_connect(mocker): + unix_conn = mocker.patch.object(urls.UnixHTTPConnection, 'connect') + conn = urls.httplib.HTTPConnection('ansible.com') + with urls.unix_socket_patch_httpconnection_connect(): + conn.connect() + assert unix_conn.call_count == 1