Support unix socket with urls/uri (#43560)

* First pass at allowing unix socket with urls/uri. See #42341

* Only insert handler as needed

* Fix and add tests

* Add HTTPS functionality for unix sockets

* Additional test fixes

* Create context manager for monkey patching HTTPConnection.connect, de-dupe code, raise better errors

* doc

* Add a few more tests

* Fix __call__

* Remove unused import

* Patch HTTPConnection.connect with the functionality we want, instead of duplicating code and disabling

* Fix var name

* Remove unused var

* Add changelog fragment

* Update uri docs

* Fix rebase indentation issue
pull/52505/head
Matt Martz 6 years ago committed by GitHub
parent 5b28cd65f0
commit 77bfcad33a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -0,0 +1,2 @@
minor_changes:
- uri/urls - Support unix domain sockets (https://github.com/ansible/ansible/pull/43560)

@ -42,6 +42,8 @@ import sys
import tempfile
import traceback
from contextlib import contextmanager
try:
import httplib
except ImportError:
@ -362,10 +364,11 @@ if hasattr(httplib, 'HTTPSConnection') and hasattr(urllib_request, 'HTTPSHandler
in place of HTTPSHandler
'''
def __init__(self, client_cert=None, client_key=None, **kwargs):
def __init__(self, client_cert=None, client_key=None, unix_socket=None, **kwargs):
urllib_request.HTTPSHandler.__init__(self, **kwargs)
self.client_cert = client_cert
self.client_key = client_key
self._unix_socket = unix_socket
def https_open(self, req):
return self.do_open(self._build_https_connection, req)
@ -379,9 +382,69 @@ if hasattr(httplib, 'HTTPSConnection') and hasattr(urllib_request, 'HTTPSHandler
kwargs['context'] = self._context
except AttributeError:
pass
if self._unix_socket:
return UnixHTTPSConnection(self._unix_socket)(host, **kwargs)
return httplib.HTTPSConnection(host, **kwargs)
@contextmanager
def unix_socket_patch_httpconnection_connect():
'''Monkey patch ``httplib.HTTPConnection.connect`` to be ``UnixHTTPConnection.connect``
so that when calling ``super(UnixHTTPSConnection, self).connect()`` we get the
correct behavior of creating self.sock for the unix socket
'''
_connect = httplib.HTTPConnection.connect
httplib.HTTPConnection.connect = UnixHTTPConnection.connect
yield
httplib.HTTPConnection.connect = _connect
class UnixHTTPSConnection(httplib.HTTPSConnection):
def __init__(self, unix_socket):
self._unix_socket = unix_socket
def connect(self):
# This method exists simply to ensure we monkeypatch
# httplib.HTTPConnection.connect to call UnixHTTPConnection.connect
with unix_socket_patch_httpconnection_connect():
super(UnixHTTPSConnection, self).connect()
def __call__(self, *args, **kwargs):
httplib.HTTPSConnection.__init__(self, *args, **kwargs)
return self
class UnixHTTPConnection(httplib.HTTPConnection):
'''Handles http requests to a unix socket file'''
def __init__(self, unix_socket):
self._unix_socket = unix_socket
def connect(self):
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try:
self.sock.connect(self._unix_socket)
except OSError as e:
raise OSError('Invalid Socket File (%s): %s' % (self._unix_socket, e))
if self.timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
self.sock.settimeout(self.timeout)
def __call__(self, *args, **kwargs):
httplib.HTTPConnection.__init__(self, *args, **kwargs)
return self
class UnixHTTPHandler(urllib_request.HTTPHandler):
'''Handler for Unix urls'''
def __init__(self, unix_socket, **kwargs):
urllib_request.HTTPHandler.__init__(self, **kwargs)
self._unix_socket = unix_socket
def http_open(self, req):
return self.do_open(UnixHTTPConnection(self._unix_socket), req)
class ParseResultDottedDict(dict):
'''
A dict that acts similarly to the ParseResult named tuple from urllib
@ -854,7 +917,7 @@ def rfc2822_date_string(timetuple, zone='-0000'):
class Request:
def __init__(self, headers=None, use_proxy=True, force=False, timeout=10, validate_certs=True,
url_username=None, url_password=None, http_agent=None, force_basic_auth=False,
follow_redirects='urllib2', client_cert=None, client_key=None, cookies=None):
follow_redirects='urllib2', client_cert=None, client_key=None, cookies=None, unix_socket=None):
"""This class works somewhat similarly to the ``Session`` class of from requests
by defining a cookiejar that an be used across requests as well as cascaded defaults that
can apply to repeated requests
@ -875,7 +938,7 @@ class Request:
self.headers = headers or {}
if not isinstance(self.headers, dict):
raise ValueError("headers must be a dict")
raise ValueError("headers must be a dict: %r" % self.headers)
self.use_proxy = use_proxy
self.force = force
self.timeout = timeout
@ -887,6 +950,7 @@ class Request:
self.follow_redirects = follow_redirects
self.client_cert = client_cert
self.client_key = client_key
self.unix_socket = unix_socket
if isinstance(cookies, cookiejar.CookieJar):
self.cookies = cookies
else:
@ -901,7 +965,8 @@ class Request:
force=None, last_mod_time=None, timeout=None, validate_certs=None,
url_username=None, url_password=None, http_agent=None,
force_basic_auth=None, follow_redirects=None,
client_cert=None, client_key=None, cookies=None, use_gssapi=False):
client_cert=None, client_key=None, cookies=None, use_gssapi=False,
unix_socket=None):
"""
Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3)
@ -936,6 +1001,8 @@ class Request:
:kwarg cookies: (optional) CookieJar object to send with the
request
:kwarg use_gssapi: (optional) Use GSSAPI handler of requests.
:kwarg unix_socket: (optional) String of file system path to unix socket file to use when establishing
connection to the provided url
:returns: HTTPResponse
"""
@ -959,8 +1026,13 @@ class Request:
client_cert = self._fallback(client_cert, self.client_cert)
client_key = self._fallback(client_key, self.client_key)
cookies = self._fallback(cookies, self.cookies)
unix_socket = self._fallback(unix_socket, self.unix_socket)
handlers = []
if unix_socket:
handlers.append(UnixHTTPHandler(unix_socket))
ssl_handler = maybe_add_ssl_handler(url, validate_certs)
if ssl_handler:
handlers.append(ssl_handler)
@ -1033,10 +1105,12 @@ class Request:
context.check_hostname = False
handlers.append(HTTPSClientAuthHandler(client_cert=client_cert,
client_key=client_key,
context=context))
elif client_cert:
context=context,
unix_socket=unix_socket))
elif client_cert or unix_socket:
handlers.append(HTTPSClientAuthHandler(client_cert=client_cert,
client_key=client_key))
client_key=client_key,
unix_socket=unix_socket))
# pre-2.6 versions of python cannot use the custom https
# handler, since the socket class is lacking create_connection.
@ -1164,7 +1238,7 @@ def open_url(url, data=None, headers=None, method=None, use_proxy=True,
url_username=None, url_password=None, http_agent=None,
force_basic_auth=False, follow_redirects='urllib2',
client_cert=None, client_key=None, cookies=None,
use_gssapi=False):
use_gssapi=False, unix_socket=None):
'''
Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3)
@ -1176,7 +1250,7 @@ def open_url(url, data=None, headers=None, method=None, use_proxy=True,
url_username=url_username, url_password=url_password, http_agent=http_agent,
force_basic_auth=force_basic_auth, follow_redirects=follow_redirects,
client_cert=client_cert, client_key=client_key, cookies=cookies,
use_gssapi=use_gssapi)
use_gssapi=use_gssapi, unix_socket=unix_socket)
#
@ -1212,7 +1286,7 @@ def url_argument_spec():
def fetch_url(module, url, data=None, headers=None, method=None,
use_proxy=True, force=False, last_mod_time=None, timeout=10,
use_gssapi=False):
use_gssapi=False, unix_socket=None):
"""Sends a request via HTTP(S) or FTP (needs the module as parameter)
:arg module: The AnsibleModule (used to get username, password etc. (s.b.).
@ -1226,6 +1300,8 @@ def fetch_url(module, url, data=None, headers=None, method=None,
:kwarg last_mod_time: Default: None
:kwarg int timeout: Default: 10
:kwarg boolean use_gssapi: Default: False
:kwarg unix_socket: (optional) String of file system path to unix socket file to use when establishing
connection to the provided url
:returns: A tuple of (**response**, **info**). Use ``response.read()`` to read the data.
The **info** contains the 'status' and other meta data. When a HttpError (status > 400)
@ -1275,7 +1351,8 @@ def fetch_url(module, url, data=None, headers=None, method=None,
validate_certs=validate_certs, url_username=username,
url_password=password, http_agent=http_agent, force_basic_auth=force_basic_auth,
follow_redirects=follow_redirects, client_cert=client_cert,
client_key=client_key, cookies=cookies, use_gssapi=use_gssapi)
client_key=client_key, cookies=cookies, use_gssapi=use_gssapi,
unix_socket=unix_socket)
# Lowercase keys, to conform to py2 behavior, so that py3 and py2 are predictable
info.update(dict((k.lower(), v) for k, v in r.info().items()))

@ -170,6 +170,10 @@ options:
- If C(no), it will not use a proxy, even if one is defined in an environment variable on the target hosts.
type: bool
default: yes
unix_socket:
description:
- Path to Unix domain socket to use for connection
version_added: '2.8'
notes:
- The dependency on httplib2 was removed in Ansible 2.1.
- The module returns all the HTTP headers in lower-case.
@ -460,7 +464,7 @@ def uri(module, url, dest, body, body_format, method, headers, socket_timeout):
_, redir_info = fetch_url(module, url, data=body,
headers=headers,
method=method,
timeout=socket_timeout)
timeout=socket_timeout, unix_socket=module.params['unix_socket'])
# if we are redirected, update the url with the location header,
# and update dest with the new url filename
if redir_info['status'] in (301, 302, 303, 307):
@ -475,7 +479,8 @@ def uri(module, url, dest, body, body_format, method, headers, socket_timeout):
module.params['follow_redirects'] = follow_redirects
resp, info = fetch_url(module, url, data=data, headers=headers,
method=method, timeout=socket_timeout, **kwargs)
method=method, timeout=socket_timeout, unix_socket=module.params['unix_socket'],
**kwargs)
try:
content = resp.read()
@ -515,6 +520,7 @@ def main():
status_code=dict(type='list', default=[200]),
timeout=dict(type='int', default=30),
headers=dict(type='dict', default={}),
unix_socket=dict(type='path'),
)
module = AnsibleModule(

@ -8,7 +8,8 @@ __metaclass__ = type
import datetime
import os
from ansible.module_utils.urls import Request, open_url, urllib_request, HAS_SSLCONTEXT, cookiejar, ConnectionError, RequestWithMethod
from ansible.module_utils.urls import (Request, open_url, urllib_request, HAS_SSLCONTEXT, cookiejar, ConnectionError, RequestWithMethod,
UnixHTTPHandler, UnixHTTPSConnection, httplib)
from ansible.module_utils.urls import SSLValidationHandler, HTTPSClientAuthHandler, RedirectHandlerFactory
import pytest
@ -45,6 +46,7 @@ def test_Request_fallback(urlopen_mock, install_opener_mock, mocker):
client_cert='/tmp/client.pem',
client_key='/tmp/client.key',
cookies=cookies,
unix_socket='/foo/bar/baz.sock',
)
fallback_mock = mocker.spy(request, '_fallback')
@ -63,10 +65,11 @@ def test_Request_fallback(urlopen_mock, install_opener_mock, mocker):
call(None, '/tmp/client.pem'), # client_cert
call(None, '/tmp/client.key'), # client_key
call(None, cookies), # cookies
call(None, '/foo/bar/baz.sock'), # unix_socket
]
fallback_mock.assert_has_calls(calls)
assert fallback_mock.call_count == 12 # All but headers use fallback
assert fallback_mock.call_count == 13 # All but headers use fallback
args = urlopen_mock.call_args[0]
assert args[1] is None # data, this is handled in the Request not urlopen
@ -125,6 +128,39 @@ def test_Request_open_http(urlopen_mock, install_opener_mock):
assert len(found_handlers) == 0
def test_Request_open_unix_socket(urlopen_mock, install_opener_mock):
r = Request().open('GET', 'http://ansible.com/', unix_socket='/foo/bar/baz.sock')
args = urlopen_mock.call_args[0]
opener = install_opener_mock.call_args[0][0]
handlers = opener.handlers
found_handlers = []
for handler in handlers:
if isinstance(handler, UnixHTTPHandler):
found_handlers.append(handler)
assert len(found_handlers) == 1
def test_Request_open_https_unix_socket(urlopen_mock, install_opener_mock):
r = Request().open('GET', 'https://ansible.com/', unix_socket='/foo/bar/baz.sock')
args = urlopen_mock.call_args[0]
opener = install_opener_mock.call_args[0][0]
handlers = opener.handlers
found_handlers = []
for handler in handlers:
if isinstance(handler, HTTPSClientAuthHandler):
found_handlers.append(handler)
assert len(found_handlers) == 1
inst = found_handlers[0]._build_https_connection('foo')
assert isinstance(inst, UnixHTTPSConnection)
def test_Request_open_ftp(urlopen_mock, install_opener_mock, mocker):
mocker.patch('ansible.module_utils.urls.ParseResultDottedDict.as_list', side_effect=AssertionError)
@ -270,6 +306,10 @@ def test_Request_open_no_validate_certs(urlopen_mock, install_opener_mock):
break
assert ssl_handler is not None
inst = ssl_handler._build_https_connection('foo')
assert isinstance(inst, httplib.HTTPSConnection)
context = ssl_handler._context
assert context.protocol == ssl.PROTOCOL_SSLv23
if ssl.OP_NO_SSLv2:
@ -399,4 +439,5 @@ def test_open_url(urlopen_mock, install_opener_mock, mocker):
force=False, last_mod_time=None, timeout=10, validate_certs=True,
url_username=None, url_password=None, http_agent=None,
force_basic_auth=False, follow_redirects='urllib2',
client_cert=None, client_key=None, cookies=None, use_gssapi=False)
client_cert=None, client_key=None, cookies=None, use_gssapi=False,
unix_socket=None)

@ -67,7 +67,7 @@ def test_fetch_url(open_url_mock, fake_ansible_module):
open_url_mock.assert_called_once_with('http://ansible.com/', client_cert=None, client_key=None, cookies=kwargs['cookies'], data=None,
follow_redirects='urllib2', force=False, force_basic_auth='', headers=None,
http_agent='ansible-httpget', last_mod_time=None, method=None, timeout=10, url_password='', url_username='',
use_proxy=True, validate_certs=True, use_gssapi=False)
use_proxy=True, validate_certs=True, use_gssapi=False, unix_socket=None)
def test_fetch_url_params(open_url_mock, fake_ansible_module):
@ -89,7 +89,7 @@ def test_fetch_url_params(open_url_mock, fake_ansible_module):
open_url_mock.assert_called_once_with('http://ansible.com/', client_cert='client.pem', client_key='client.key', cookies=kwargs['cookies'], data=None,
follow_redirects='all', force=False, force_basic_auth=True, headers=None,
http_agent='ansible-test', last_mod_time=None, method=None, timeout=10, url_password='passwd', url_username='user',
use_proxy=True, validate_certs=False, use_gssapi=False)
use_proxy=True, validate_certs=False, use_gssapi=False, unix_socket=None)
def test_fetch_url_cookies(mocker, fake_ansible_module):

@ -89,3 +89,11 @@ def test_ParseResultDottedDict():
assert parts[0] == dotted_parts.scheme
assert dotted_parts.as_list() == list(parts)
def test_unix_socket_patch_httpconnection_connect(mocker):
unix_conn = mocker.patch.object(urls.UnixHTTPConnection, 'connect')
conn = urls.httplib.HTTPConnection('ansible.com')
with urls.unix_socket_patch_httpconnection_connect():
conn.connect()
assert unix_conn.call_count == 1

Loading…
Cancel
Save