From d4930e6692fb2ae8e0aa118052ee6cce370feda6 Mon Sep 17 00:00:00 2001 From: Matt Martz Date: Fri, 1 Jun 2018 11:44:20 -0500 Subject: [PATCH] Add requests.Session like class (#37622) * Adds requests.Session like class * py2 syntax fix * Add a few examples to the Request docstrings * Add helper methods and docs * Fix test failures * Switch tests to test Request instead of open_url, add simple open_url test to validate funcitonality * Fix filename in replace-urlopen code smell test --- lib/ansible/module_utils/urls.py | 425 +++++++++++++----- test/sanity/code-smell/replace-urlopen.py | 2 +- .../{test_open_url.py => test_Request.py} | 169 +++++-- 3 files changed, 432 insertions(+), 164 deletions(-) rename test/units/module_utils/urls/{test_open_url.py => test_Request.py} (54%) diff --git a/lib/ansible/module_utils/urls.py b/lib/ansible/module_utils/urls.py index c0658d67a22..1b02a568987 100644 --- a/lib/ansible/module_utils/urls.py +++ b/lib/ansible/module_utils/urls.py @@ -816,147 +816,328 @@ def maybe_add_ssl_handler(url, validate_certs): return SSLValidationHandler(hostname, port) -def open_url(url, data=None, headers=None, method=None, use_proxy=True, - force=False, last_mod_time=None, timeout=10, validate_certs=True, +class Request: + def __init__(self, headers=None, use_proxy=True, force=False, timeout=10, validate_certs=True, + url_username=None, url_password=None, http_agent=None, force_basic_auth=False, + follow_redirects='urllib2', client_cert=None, client_key=None, cookies=None): + """This class works somewhat similarly to the ``Session`` class of from requests + by defining a cookiejar that an be used across requests as well as cascaded defaults that + can apply to repeated requests + + For documentation of params, see ``Request.open`` + + >>> from ansible.module_utils.urls import Request + >>> r = Request() + >>> r.open('GET', 'http://httpbin.org/cookies/set?k1=v1').read() + '{\n "cookies": {\n "k1": "v1"\n }\n}\n' + >>> r = Request(url_username='user', url_password='passwd') + >>> r.open('GET', 'http://httpbin.org/basic-auth/user/passwd').read() + '{\n "authenticated": true, \n "user": "user"\n}\n' + >>> r = Request(headers=dict(foo='bar')) + >>> r.open('GET', 'http://httpbin.org/get', headers=dict(baz='qux')).read() + + """ + + self.headers = headers or {} + if not isinstance(self.headers, dict): + raise ValueError("headers must be a dict") + self.use_proxy = use_proxy + self.force = force + self.timeout = timeout + self.validate_certs = validate_certs + self.url_username = url_username + self.url_password = url_password + self.http_agent = http_agent + self.force_basic_auth = force_basic_auth + self.follow_redirects = follow_redirects + self.client_cert = client_cert + self.client_key = client_key + if isinstance(cookies, cookiejar.CookieJar): + self.cookies = cookies + else: + self.cookies = cookiejar.CookieJar() + + def _fallback(self, value, fallback): + if value is None: + return fallback + return value + + def open(self, method, url, data=None, headers=None, use_proxy=None, + force=None, last_mod_time=None, timeout=None, validate_certs=None, url_username=None, url_password=None, http_agent=None, - force_basic_auth=False, follow_redirects='urllib2', + force_basic_auth=None, follow_redirects=None, client_cert=None, client_key=None, cookies=None): - ''' - Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3) + """ + Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3) + + Does not require the module environment + + Returns :class:`HTTPResponse` object. + + :arg method: method for the request + :arg url: URL to request + + :kwarg data: (optional) bytes, or file-like object to send + in the body of the request + :kwarg headers: (optional) Dictionary of HTTP Headers to send with the + request + :kwarg use_proxy: (optional) Boolean of whether or not to use proxy + :kwarg force: (optional) Boolean of whether or not to set `cache-control: no-cache` header + :kwarg last_mod_time: (optional) Datetime object to use when setting If-Modified-Since header + :kwarg timeout: (optional) How long to wait for the server to send + data before giving up, as a float + :kwarg validate_certs: (optional) Booleani that controls whether we verify + the server's TLS certificate + :kwarg url_username: (optional) String of the user to use when authenticating + :kwarg url_password: (optional) String of the password to use when authenticating + :kwarg http_agent: (optional) String of the User-Agent to use in the request + :kwarg force_basic_auth: (optional) Boolean determining if auth header should be sent in the initial request + :kwarg follow_redirects: (optional) String of urllib2, all/yes, safe, none to determine how redirects are + followed, see RedirectHandlerFactory for more information + :kwarg client_cert: (optional) PEM formatted certificate chain file to be used for SSL client authentication. + This file can also include the key as well, and if the key is included, client_key is not required + :kwarg client_key: (optional) PEM formatted file that contains your private key to be used for SSL client + authentication. If client_cert contains both the certificate and key, this option is not required + :kwarg cookies: (optional) CookieJar object to send with the + request + :returns: HTTPResponse + """ - Does not require the module environment - ''' - handlers = [] - ssl_handler = maybe_add_ssl_handler(url, validate_certs) - if ssl_handler: - handlers.append(ssl_handler) - - parsed = generic_urlparse(urlparse(url)) - if parsed.scheme != 'ftp': - username = url_username + method = method.upper() if headers is None: headers = {} + elif not isinstance(headers, dict): + raise ValueError("headers must be a dict") + headers = dict(self.headers, **headers) + + use_proxy = self._fallback(use_proxy, self.use_proxy) + force = self._fallback(force, self.force) + timeout = self._fallback(timeout, self.timeout) + validate_certs = self._fallback(validate_certs, self.validate_certs) + url_username = self._fallback(url_username, self.url_username) + url_password = self._fallback(url_password, self.url_password) + http_agent = self._fallback(http_agent, self.http_agent) + force_basic_auth = self._fallback(force_basic_auth, self.force_basic_auth) + follow_redirects = self._fallback(follow_redirects, self.follow_redirects) + client_cert = self._fallback(client_cert, self.client_cert) + client_key = self._fallback(client_key, self.client_key) + cookies = self._fallback(cookies, self.cookies) + + handlers = [] + ssl_handler = maybe_add_ssl_handler(url, validate_certs) + if ssl_handler: + handlers.append(ssl_handler) + + parsed = generic_urlparse(urlparse(url)) + if parsed.scheme != 'ftp': + username = url_username + + if username: + password = url_password + netloc = parsed.netloc + elif '@' in parsed.netloc: + credentials, netloc = parsed.netloc.split('@', 1) + if ':' in credentials: + username, password = credentials.split(':', 1) + else: + username = credentials + password = '' - if username: - password = url_password - netloc = parsed.netloc - elif '@' in parsed.netloc: - credentials, netloc = parsed.netloc.split('@', 1) - if ':' in credentials: - username, password = credentials.split(':', 1) - else: - username = credentials - password = '' + parsed_list = parsed.as_list() + parsed_list[1] = netloc - parsed_list = parsed.as_list() - parsed_list[1] = netloc + # reconstruct url without credentials + url = urlunparse(parsed_list) - # reconstruct url without credentials - url = urlunparse(parsed_list) + if username and not force_basic_auth: + passman = urllib_request.HTTPPasswordMgrWithDefaultRealm() - if username and not force_basic_auth: - passman = urllib_request.HTTPPasswordMgrWithDefaultRealm() + # this creates a password manager + passman.add_password(None, netloc, username, password) - # this creates a password manager - passman.add_password(None, netloc, username, password) + # because we have put None at the start it will always + # use this username/password combination for urls + # for which `theurl` is a super-url + authhandler = urllib_request.HTTPBasicAuthHandler(passman) + digest_authhandler = urllib_request.HTTPDigestAuthHandler(passman) - # because we have put None at the start it will always - # use this username/password combination for urls - # for which `theurl` is a super-url - authhandler = urllib_request.HTTPBasicAuthHandler(passman) - digest_authhandler = urllib_request.HTTPDigestAuthHandler(passman) + # create the AuthHandler + handlers.append(authhandler) + handlers.append(digest_authhandler) - # create the AuthHandler - handlers.append(authhandler) - handlers.append(digest_authhandler) + elif username and force_basic_auth: + headers["Authorization"] = basic_auth_header(username, password) - elif username and force_basic_auth: - headers["Authorization"] = basic_auth_header(username, password) + else: + try: + rc = netrc.netrc(os.environ.get('NETRC')) + login = rc.authenticators(parsed.hostname) + except IOError: + login = None - else: - try: - rc = netrc.netrc(os.environ.get('NETRC')) - login = rc.authenticators(parsed.hostname) - except IOError: - login = None - - if login: - username, _, password = login - if username and password: - headers["Authorization"] = basic_auth_header(username, password) - - if not use_proxy: - proxyhandler = urllib_request.ProxyHandler({}) - handlers.append(proxyhandler) - - if HAS_SSLCONTEXT and not validate_certs: - # In 2.7.9, the default context validates certificates - context = SSLContext(ssl.PROTOCOL_SSLv23) - if ssl.OP_NO_SSLv2: - context.options |= ssl.OP_NO_SSLv2 - context.options |= ssl.OP_NO_SSLv3 - context.verify_mode = ssl.CERT_NONE - context.check_hostname = False - handlers.append(HTTPSClientAuthHandler(client_cert=client_cert, - client_key=client_key, - context=context)) - elif client_cert: - handlers.append(HTTPSClientAuthHandler(client_cert=client_cert, - client_key=client_key)) - - # pre-2.6 versions of python cannot use the custom https - # handler, since the socket class is lacking create_connection. - # Some python builds lack HTTPS support. - if hasattr(socket, 'create_connection') and CustomHTTPSHandler: - handlers.append(CustomHTTPSHandler) - - handlers.append(RedirectHandlerFactory(follow_redirects, validate_certs)) - - # add some nicer cookie handling - if cookies is not None: - handlers.append(urllib_request.HTTPCookieProcessor(cookies)) - - opener = urllib_request.build_opener(*handlers) - urllib_request.install_opener(opener) - - data = to_bytes(data, nonstring='passthru') - if method: - if method.upper() not in ('OPTIONS', 'GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE', 'CONNECT', 'PATCH'): - raise ConnectionError('invalid HTTP request method; %s' % method.upper()) - request = RequestWithMethod(url, method.upper(), data) - else: - request = urllib_request.Request(url, data) - - # add the custom agent header, to help prevent issues - # with sites that block the default urllib agent string - if http_agent: - request.add_header('User-agent', http_agent) - - # Cache control - # Either we directly force a cache refresh - if force: - request.add_header('cache-control', 'no-cache') - # or we do it if the original is more recent than our copy - elif last_mod_time: - tstamp = last_mod_time.strftime('%a, %d %b %Y %H:%M:%S +0000') - request.add_header('If-Modified-Since', tstamp) - - # user defined headers now, which may override things we've set above - if headers: - if not isinstance(headers, dict): - raise ValueError("headers provided to open_url() must be a dict") + if login: + username, _, password = login + if username and password: + headers["Authorization"] = basic_auth_header(username, password) + + if not use_proxy: + proxyhandler = urllib_request.ProxyHandler({}) + handlers.append(proxyhandler) + + if HAS_SSLCONTEXT and not validate_certs: + # In 2.7.9, the default context validates certificates + context = SSLContext(ssl.PROTOCOL_SSLv23) + if ssl.OP_NO_SSLv2: + context.options |= ssl.OP_NO_SSLv2 + context.options |= ssl.OP_NO_SSLv3 + context.verify_mode = ssl.CERT_NONE + context.check_hostname = False + handlers.append(HTTPSClientAuthHandler(client_cert=client_cert, + client_key=client_key, + context=context)) + elif client_cert: + handlers.append(HTTPSClientAuthHandler(client_cert=client_cert, + client_key=client_key)) + + # pre-2.6 versions of python cannot use the custom https + # handler, since the socket class is lacking create_connection. + # Some python builds lack HTTPS support. + if hasattr(socket, 'create_connection') and CustomHTTPSHandler: + handlers.append(CustomHTTPSHandler) + + handlers.append(RedirectHandlerFactory(follow_redirects, validate_certs)) + + # add some nicer cookie handling + if cookies is not None: + handlers.append(urllib_request.HTTPCookieProcessor(cookies)) + + opener = urllib_request.build_opener(*handlers) + urllib_request.install_opener(opener) + + data = to_bytes(data, nonstring='passthru') + if method not in ('OPTIONS', 'GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE', 'CONNECT', 'PATCH'): + raise ConnectionError('invalid HTTP request method; %s' % method) + request = RequestWithMethod(url, method, data) + + # add the custom agent header, to help prevent issues + # with sites that block the default urllib agent string + if http_agent: + request.add_header('User-agent', http_agent) + + # Cache control + # Either we directly force a cache refresh + if force: + request.add_header('cache-control', 'no-cache') + # or we do it if the original is more recent than our copy + elif last_mod_time: + tstamp = last_mod_time.strftime('%a, %d %b %Y %H:%M:%S +0000') + request.add_header('If-Modified-Since', tstamp) + + # user defined headers now, which may override things we've set above for header in headers: request.add_header(header, headers[header]) - urlopen_args = [request, None] - if sys.version_info >= (2, 6, 0): - # urlopen in python prior to 2.6.0 did not - # have a timeout parameter - urlopen_args.append(timeout) + urlopen_args = [request, None] + if sys.version_info >= (2, 6, 0): + # urlopen in python prior to 2.6.0 did not + # have a timeout parameter + urlopen_args.append(timeout) + + r = urllib_request.urlopen(*urlopen_args) + return r + + def get(self, url, **kwargs): + r"""Sends a GET request. Returns :class:`HTTPResponse` object. + + :arg url: URL to request + :kwarg \*\*kwargs: Optional arguments that ``open`` takes. + :returns: HTTPResponse + """ + + return self.open('GET', url, **kwargs) + + def options(self, url, **kwargs): + r"""Sends a OPTIONS request. Returns :class:`HTTPResponse` object. + + :arg url: URL to request + :kwarg \*\*kwargs: Optional arguments that ``open`` takes. + :returns: HTTPResponse + """ + + return self.open('OPTIONS', url, **kwargs) + + def head(self, url, **kwargs): + r"""Sends a HEAD request. Returns :class:`HTTPResponse` object. + + :arg url: URL to request + :kwarg \*\*kwargs: Optional arguments that ``open`` takes. + :returns: HTTPResponse + """ + + return self.open('HEAD', url, **kwargs) + + def post(self, url, data=None, **kwargs): + r"""Sends a POST request. Returns :class:`HTTPResponse` object. + + :arg url: URL to request. + :kwarg data: (optional) bytes, or file-like object to send in the body of the request. + :kwarg \*\*kwargs: Optional arguments that ``open`` takes. + :returns: HTTPResponse + """ + + return self.open('POST', url, data=data, **kwargs) + + def put(self, url, data=None, **kwargs): + r"""Sends a PUT request. Returns :class:`HTTPResponse` object. + + :arg url: URL to request. + :kwarg data: (optional) bytes, or file-like object to send in the body of the request. + :kwarg \*\*kwargs: Optional arguments that ``open`` takes. + :returns: HTTPResponse + """ + + return self.open('PUT', url, data=data, **kwargs) + + def patch(self, url, data=None, **kwargs): + r"""Sends a PATCH request. Returns :class:`HTTPResponse` object. + + :arg url: URL to request. + :kwarg data: (optional) bytes, or file-like object to send in the body of the request. + :kwarg \*\*kwargs: Optional arguments that ``open`` takes. + :returns: HTTPResponse + """ + + return self.open('PATCH', url, data=data, **kwargs) + + def delete(self, url, **kwargs): + r"""Sends a DELETE request. Returns :class:`HTTPResponse` object. + + :arg url: URL to request + :kwargs \*\*kwargs: Optional arguments that ``open`` takes. + :returns: HTTPResponse + """ + + return self.open('DELETE', url, **kwargs) + + +def open_url(url, data=None, headers=None, method=None, use_proxy=True, + force=False, last_mod_time=None, timeout=10, validate_certs=True, + url_username=None, url_password=None, http_agent=None, + force_basic_auth=False, follow_redirects='urllib2', + client_cert=None, client_key=None, cookies=None): + ''' + Sends a request via HTTP(S) or FTP using urllib2 (Python2) or urllib (Python3) + + Does not require the module environment + ''' + method = method or 'GET' + return Request().open(method, url, data=data, headers=headers, use_proxy=use_proxy, + force=force, last_mod_time=last_mod_time, timeout=timeout, validate_certs=validate_certs, + url_username=url_username, url_password=url_password, http_agent=http_agent, + force_basic_auth=force_basic_auth, follow_redirects=follow_redirects, + client_cert=client_cert, client_key=client_key, cookies=cookies) - r = urllib_request.urlopen(*urlopen_args) - return r # # Module-related functions diff --git a/test/sanity/code-smell/replace-urlopen.py b/test/sanity/code-smell/replace-urlopen.py index 6ce8ec52be0..677f9c88a9c 100755 --- a/test/sanity/code-smell/replace-urlopen.py +++ b/test/sanity/code-smell/replace-urlopen.py @@ -10,7 +10,7 @@ def main(): 'test/sanity/code-smell/%s' % os.path.basename(__file__), 'lib/ansible/module_utils/six/__init__.py', 'lib/ansible/module_utils/urls.py', - 'test/units/module_utils/urls/test_open_url.py', + 'test/units/module_utils/urls/test_Request.py', 'test/units/module_utils/urls/test_fetch_url.py', ]) diff --git a/test/units/module_utils/urls/test_open_url.py b/test/units/module_utils/urls/test_Request.py similarity index 54% rename from test/units/module_utils/urls/test_open_url.py rename to test/units/module_utils/urls/test_Request.py index 9185b62a6f1..e6b23b5b7c0 100644 --- a/test/units/module_utils/urls/test_open_url.py +++ b/test/units/module_utils/urls/test_Request.py @@ -8,10 +8,11 @@ __metaclass__ = type import datetime import os -from ansible.module_utils.urls import open_url, urllib_request, HAS_SSLCONTEXT, cookiejar, ConnectionError, RequestWithMethod +from ansible.module_utils.urls import Request, open_url, urllib_request, HAS_SSLCONTEXT, cookiejar, ConnectionError, RequestWithMethod from ansible.module_utils.urls import SSLValidationHandler, HTTPSClientAuthHandler, RedirectHandlerFactory import pytest +from mock import call if HAS_SSLCONTEXT: @@ -28,8 +29,62 @@ def install_opener_mock(mocker): return mocker.patch('ansible.module_utils.urls.urllib_request.install_opener') -def test_open_url(urlopen_mock, install_opener_mock): - r = open_url('https://ansible.com/') +def test_Request_fallback(urlopen_mock, install_opener_mock, mocker): + cookies = cookiejar.CookieJar() + request = Request( + headers={'foo': 'bar'}, + use_proxy=False, + force=True, + timeout=100, + validate_certs=False, + url_username='user', + url_password='passwd', + http_agent='ansible-tests', + force_basic_auth=True, + follow_redirects='all', + client_cert='/tmp/client.pem', + client_key='/tmp/client.key', + cookies=cookies, + ) + fallback_mock = mocker.spy(request, '_fallback') + + r = request.open('GET', 'https://ansible.com') + + calls = [ + call(None, False), # use_proxy + call(None, True), # force + call(None, 100), # timeout + call(None, False), # validate_certs + call(None, 'user'), # url_username + call(None, 'passwd'), # url_password + call(None, 'ansible-tests'), # http_agent + call(None, True), # force_basic_auth + call(None, 'all'), # follow_redirects + call(None, '/tmp/client.pem'), # client_cert + call(None, '/tmp/client.key'), # client_key + call(None, cookies), # cookies + ] + fallback_mock.assert_has_calls(calls) + + assert fallback_mock.call_count == 12 # All but headers use fallback + + args = urlopen_mock.call_args[0] + assert args[1] is None # data, this is handled in the Request not urlopen + assert args[2] == 100 # timeout + + req = args[0] + assert req.headers == { + 'Authorization': b'Basic dXNlcjpwYXNzd2Q=', + 'Cache-control': 'no-cache', + 'Foo': 'bar', + 'User-agent': 'ansible-tests' + } + assert req.data is None + assert req.get_method() == 'GET' + + +def test_Request_open(urlopen_mock, install_opener_mock): + r = Request().open('GET', 'https://ansible.com/') args = urlopen_mock.call_args[0] assert args[1] is None # data, this is handled in the Request not urlopen assert args[2] == 10 # timeout @@ -55,8 +110,8 @@ def test_open_url(urlopen_mock, install_opener_mock): assert len(found_handlers) == 2 -def test_open_url_http(urlopen_mock, install_opener_mock): - r = open_url('http://ansible.com/') +def test_Request_open_http(urlopen_mock, install_opener_mock): + r = Request().open('GET', 'http://ansible.com/') args = urlopen_mock.call_args[0] opener = install_opener_mock.call_args[0][0] @@ -70,22 +125,22 @@ def test_open_url_http(urlopen_mock, install_opener_mock): assert len(found_handlers) == 0 -def test_open_url_ftp(urlopen_mock, install_opener_mock, mocker): +def test_Request_open_ftp(urlopen_mock, install_opener_mock, mocker): mocker.patch('ansible.module_utils.urls.ParseResultDottedDict.as_list', side_effect=AssertionError) # Using ftp scheme should prevent the AssertionError side effect to fire - r = open_url('ftp://foo@ansible.com/') + r = Request().open('GET', 'ftp://foo@ansible.com/') -def test_open_url_headers(urlopen_mock, install_opener_mock): - r = open_url('http://ansible.com/', headers={'Foo': 'bar'}) +def test_Request_open_headers(urlopen_mock, install_opener_mock): + r = Request().open('GET', 'http://ansible.com/', headers={'Foo': 'bar'}) args = urlopen_mock.call_args[0] req = args[0] assert req.headers == {'Foo': 'bar'} -def test_open_url_username(urlopen_mock, install_opener_mock): - r = open_url('http://ansible.com/', url_username='user') +def test_Request_open_username(urlopen_mock, install_opener_mock): + r = Request().open('GET', 'http://ansible.com/', url_username='user') opener = install_opener_mock.call_args[0][0] handlers = opener.handlers @@ -103,8 +158,8 @@ def test_open_url_username(urlopen_mock, install_opener_mock): assert found_handlers[0].passwd.passwd[None] == {(('ansible.com', '/'),): ('user', None)} -def test_open_url_username_in_url(urlopen_mock, install_opener_mock): - r = open_url('http://user2@ansible.com/') +def test_Request_open_username_in_url(urlopen_mock, install_opener_mock): + r = Request().open('GET', 'http://user2@ansible.com/') opener = install_opener_mock.call_args[0][0] handlers = opener.handlers @@ -121,8 +176,8 @@ def test_open_url_username_in_url(urlopen_mock, install_opener_mock): assert found_handlers[0].passwd.passwd[None] == {(('ansible.com', '/'),): ('user2', '')} -def test_open_url_username_force_basic(urlopen_mock, install_opener_mock): - r = open_url('http://ansible.com/', url_username='user', url_password='passwd', force_basic_auth=True) +def test_Request_open_username_force_basic(urlopen_mock, install_opener_mock): + r = Request().open('GET', 'http://ansible.com/', url_username='user', url_password='passwd', force_basic_auth=True) opener = install_opener_mock.call_args[0][0] handlers = opener.handlers @@ -144,8 +199,8 @@ def test_open_url_username_force_basic(urlopen_mock, install_opener_mock): assert req.headers.get('Authorization') == b'Basic dXNlcjpwYXNzd2Q=' -def test_open_url_auth_in_netloc(urlopen_mock, install_opener_mock): - r = open_url('http://user:passwd@ansible.com/') +def test_Request_open_auth_in_netloc(urlopen_mock, install_opener_mock): + r = Request().open('GET', 'http://user:passwd@ansible.com/') args = urlopen_mock.call_args[0] req = args[0] assert req.get_full_url() == 'http://ansible.com/' @@ -166,31 +221,31 @@ def test_open_url_auth_in_netloc(urlopen_mock, install_opener_mock): assert len(found_handlers) == 2 -def test_open_url_netrc(urlopen_mock, install_opener_mock, monkeypatch): +def test_Request_open_netrc(urlopen_mock, install_opener_mock, monkeypatch): here = os.path.dirname(__file__) monkeypatch.setenv('NETRC', os.path.join(here, 'fixtures/netrc')) - r = open_url('http://ansible.com/') + r = Request().open('GET', 'http://ansible.com/') args = urlopen_mock.call_args[0] req = args[0] assert req.headers.get('Authorization') == b'Basic dXNlcjpwYXNzd2Q=' - r = open_url('http://foo.ansible.com/') + r = Request().open('GET', 'http://foo.ansible.com/') args = urlopen_mock.call_args[0] req = args[0] assert 'Authorization' not in req.headers monkeypatch.setenv('NETRC', os.path.join(here, 'fixtures/netrc.nonexistant')) - r = open_url('http://ansible.com/') + r = Request().open('GET', 'http://ansible.com/') args = urlopen_mock.call_args[0] req = args[0] assert 'Authorization' not in req.headers -def test_open_url_no_proxy(urlopen_mock, install_opener_mock, mocker): +def test_Request_open_no_proxy(urlopen_mock, install_opener_mock, mocker): build_opener_mock = mocker.patch('ansible.module_utils.urls.urllib_request.build_opener') - r = open_url('http://ansible.com/', use_proxy=False) + r = Request().open('GET', 'http://ansible.com/', use_proxy=False) handlers = build_opener_mock.call_args[0] found_handlers = [] @@ -202,8 +257,8 @@ def test_open_url_no_proxy(urlopen_mock, install_opener_mock, mocker): @pytest.mark.skipif(not HAS_SSLCONTEXT, reason="requires SSLContext") -def test_open_url_no_validate_certs(urlopen_mock, install_opener_mock): - r = open_url('https://ansible.com/', validate_certs=False) +def test_Request_open_no_validate_certs(urlopen_mock, install_opener_mock): + r = Request().open('GET', 'https://ansible.com/', validate_certs=False) opener = install_opener_mock.call_args[0][0] handlers = opener.handlers @@ -224,13 +279,13 @@ def test_open_url_no_validate_certs(urlopen_mock, install_opener_mock): assert context.check_hostname is False -def test_open_url_client_cert(urlopen_mock, install_opener_mock): +def test_Request_open_client_cert(urlopen_mock, install_opener_mock): here = os.path.dirname(__file__) client_cert = os.path.join(here, 'fixtures/client.pem') client_key = os.path.join(here, 'fixtures/client.key') - r = open_url('https://ansible.com/', client_cert=client_cert, client_key=client_key) + r = Request().open('GET', 'https://ansible.com/', client_cert=client_cert, client_key=client_key) opener = install_opener_mock.call_args[0][0] handlers = opener.handlers @@ -252,8 +307,8 @@ def test_open_url_client_cert(urlopen_mock, install_opener_mock): assert https_connection.cert_file == client_cert -def test_open_url_cookies(urlopen_mock, install_opener_mock): - r = open_url('https://ansible.com/', cookies=cookiejar.CookieJar()) +def test_Request_open_cookies(urlopen_mock, install_opener_mock): + r = Request().open('GET', 'https://ansible.com/', cookies=cookiejar.CookieJar()) opener = install_opener_mock.call_args[0][0] handlers = opener.handlers @@ -267,13 +322,13 @@ def test_open_url_cookies(urlopen_mock, install_opener_mock): assert cookies_handler is not None -def test_open_url_invalid_method(urlopen_mock, install_opener_mock): +def test_Request_open_invalid_method(urlopen_mock, install_opener_mock): with pytest.raises(ConnectionError): - r = open_url('https://ansible.com/', method='BOGUS') + r = Request().open('BOGUS', 'https://ansible.com/') -def test_open_url_custom_method(urlopen_mock, install_opener_mock): - r = open_url('https://ansible.com/', method='DELETE') +def test_Request_open_custom_method(urlopen_mock, install_opener_mock): + r = Request().open('DELETE', 'https://ansible.com/') args = urlopen_mock.call_args[0] req = args[0] @@ -281,8 +336,8 @@ def test_open_url_custom_method(urlopen_mock, install_opener_mock): assert isinstance(req, RequestWithMethod) -def test_open_url_user_agent(urlopen_mock, install_opener_mock): - r = open_url('https://ansible.com/', http_agent='ansible-tests') +def test_Request_open_user_agent(urlopen_mock, install_opener_mock): + r = Request().open('GET', 'https://ansible.com/', http_agent='ansible-tests') args = urlopen_mock.call_args[0] req = args[0] @@ -290,8 +345,8 @@ def test_open_url_user_agent(urlopen_mock, install_opener_mock): assert req.headers.get('User-agent') == 'ansible-tests' -def test_open_url_force(urlopen_mock, install_opener_mock): - r = open_url('https://ansible.com/', force=True, last_mod_time=datetime.datetime.now()) +def test_Request_open_force(urlopen_mock, install_opener_mock): + r = Request().open('GET', 'https://ansible.com/', force=True, last_mod_time=datetime.datetime.now()) args = urlopen_mock.call_args[0] req = args[0] @@ -300,9 +355,9 @@ def test_open_url_force(urlopen_mock, install_opener_mock): assert 'If-modified-since' not in req.headers -def test_open_url_last_mod(urlopen_mock, install_opener_mock): +def test_Request_open_last_mod(urlopen_mock, install_opener_mock): now = datetime.datetime.now() - r = open_url('https://ansible.com/', last_mod_time=now) + r = Request().open('GET', 'https://ansible.com/', last_mod_time=now) args = urlopen_mock.call_args[0] req = args[0] @@ -310,6 +365,38 @@ def test_open_url_last_mod(urlopen_mock, install_opener_mock): assert req.headers.get('If-modified-since') == now.strftime('%a, %d %b %Y %H:%M:%S +0000') -def test_open_url_headers_not_dict(urlopen_mock, install_opener_mock): +def test_Request_open_headers_not_dict(urlopen_mock, install_opener_mock): + with pytest.raises(ValueError): + Request().open('GET', 'https://ansible.com/', headers=['bob']) + + +def test_Request_init_headers_not_dict(urlopen_mock, install_opener_mock): with pytest.raises(ValueError): - r = open_url('https://ansible.com/', headers=['bob']) + Request(headers=['bob']) + + +@pytest.mark.parametrize('method,kwargs', [ + ('get', {}), + ('options', {}), + ('head', {}), + ('post', {'data': None}), + ('put', {'data': None}), + ('patch', {'data': None}), + ('delete', {}), +]) +def test_methods(method, kwargs, mocker): + expected = method.upper() + open_mock = mocker.patch('ansible.module_utils.urls.Request.open') + request = Request() + getattr(request, method)('https://ansible.com') + open_mock.assert_called_once_with(expected, 'https://ansible.com', **kwargs) + + +def test_open_url(urlopen_mock, install_opener_mock, mocker): + req_mock = mocker.patch('ansible.module_utils.urls.Request.open') + open_url('https://ansible.com/') + req_mock.assert_called_once_with('GET', 'https://ansible.com/', data=None, headers=None, use_proxy=True, + force=False, last_mod_time=None, timeout=10, validate_certs=True, + url_username=None, url_password=None, http_agent=None, + force_basic_auth=False, follow_redirects='urllib2', + client_cert=None, client_key=None, cookies=None)