From 4161d78a94cf91f56370645dd54dda6a4b0ebdeb Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Fri, 12 Jun 2015 12:24:23 -0700 Subject: [PATCH] Split the fetch_url() function into fetch_url and open_url(). open_url() is suitable for use outside of a module environment. Will let us use open_url to do SSL cert verification in other, non-module code. --- lib/ansible/module_utils/urls.py | 186 ++++++++++++++++++------------- 1 file changed, 110 insertions(+), 76 deletions(-) diff --git a/lib/ansible/module_utils/urls.py b/lib/ansible/module_utils/urls.py index 18317e86aeb..2725980fcb5 100644 --- a/lib/ansible/module_utils/urls.py +++ b/lib/ansible/module_utils/urls.py @@ -26,12 +26,6 @@ # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -try: - import urllib - HAS_URLLIB = True -except: - HAS_URLLIB = False - try: import urllib2 HAS_URLLIB2 = True @@ -62,7 +56,9 @@ except ImportError: import httplib import os import re +import sys import socket +import platform import tempfile @@ -89,6 +85,27 @@ zKPZsZ2miVGclicJHzm5q080b1p/sZtuKIEZk6vZqEg= -----END CERTIFICATE----- """ +# +# Exceptions +# + +class ConnectionError(Exception): + """Failed to connect to the server""" + pass + +class ProxyError(ConnectionError): + """Failure to connect because of a proxy""" + pass + +class SSLValidationError(ConnectionError): + """Failure to connect due to SSL validation failing""" + pass + +class NoSSLError(SSLValidationError): + """Needed to connect to an HTTPS url but no ssl library available to verify the certificate""" + pass + + class CustomHTTPSConnection(httplib.HTTPSConnection): def connect(self): "Connect to a host on a given (SSL) port." @@ -153,7 +170,7 @@ def generic_urlparse(parts): username, password = auth.split(':', 1) generic_parts['username'] = username generic_parts['password'] = password - generic_parts['hostname'] = hostnme + generic_parts['hostname'] = hostname generic_parts['port'] = port except: generic_parts['username'] = None @@ -189,8 +206,7 @@ class SSLValidationHandler(urllib2.BaseHandler): ''' CONNECT_COMMAND = "CONNECT %s:%s HTTP/1.0\r\nConnection: close\r\n" - def __init__(self, module, hostname, port): - self.module = module + def __init__(self, hostname, port): self.hostname = hostname self.port = port @@ -200,23 +216,22 @@ class SSLValidationHandler(urllib2.BaseHandler): ca_certs = [] paths_checked = [] - platform = get_platform() - distribution = get_distribution() + system = platform.system() # build a list of paths to check for .crt/.pem files # based on the platform type paths_checked.append('/etc/ssl/certs') - if platform == 'Linux': + if system == 'Linux': paths_checked.append('/etc/pki/ca-trust/extracted/pem') paths_checked.append('/etc/pki/tls/certs') paths_checked.append('/usr/share/ca-certificates/cacert.org') - elif platform == 'FreeBSD': + elif system == 'FreeBSD': paths_checked.append('/usr/local/share/certs') - elif platform == 'OpenBSD': + elif system == 'OpenBSD': paths_checked.append('/etc/ssl') - elif platform == 'NetBSD': + elif system == 'NetBSD': ca_certs.append('/etc/openssl/certs') - elif platform == 'SunOS': + elif system == 'SunOS': paths_checked.append('/opt/local/etc/openssl/certs') # fall back to a user-deployed cert in a standard @@ -226,7 +241,7 @@ class SSLValidationHandler(urllib2.BaseHandler): tmp_fd, tmp_path = tempfile.mkstemp() # Write the dummy ca cert if we are running on Mac OS X - if platform == 'Darwin': + if system == 'Darwin': os.write(tmp_fd, DUMMY_CA_CERT) # Default Homebrew path for OpenSSL certs paths_checked.append('/usr/local/etc/openssl') @@ -259,7 +274,7 @@ class SSLValidationHandler(urllib2.BaseHandler): if int(resp_code) not in valid_codes: raise Exception except: - self.module.fail_json(msg='Connection to proxy failed') + raise ProxyError('Connection to proxy failed') def detect_no_proxy(self, url): ''' @@ -304,7 +319,7 @@ class SSLValidationHandler(urllib2.BaseHandler): ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED) match_hostname(ssl_s.getpeercert(), self.hostname) else: - self.module.fail_json(msg='Unsupported proxy scheme: %s. Currently ansible only supports HTTP proxies.' % proxy_parts.get('scheme')) + raise ProxyError('Unsupported proxy scheme: %s. Currently ansible only supports HTTP proxies.' % proxy_parts.get('scheme')) else: s.connect((self.hostname, self.port)) ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED) @@ -315,15 +330,14 @@ class SSLValidationHandler(urllib2.BaseHandler): except (ssl.SSLError, socket.error), e: # fail if we tried all of the certs but none worked if 'connection refused' in str(e).lower(): - self.module.fail_json(msg='Failed to connect to %s:%s.' % (self.hostname, self.port)) + raise ConnectionError('Failed to connect to %s:%s.' % (self.hostname, self.port)) else: - self.module.fail_json( - msg='Failed to validate the SSL certificate for %s:%s. ' % (self.hostname, self.port) + \ - 'Use validate_certs=no or make sure your managed systems have a valid CA certificate installed. ' + \ - 'Paths checked for this platform: %s' % ", ".join(paths_checked) + raise SSLValidationError('Failed to validate the SSL certificate for %s:%s. ' + 'Use validate_certs=False (insecure) or make sure your managed systems have a valid CA certificate installed. ' + 'Paths checked for this platform: %s' % (self.hostname, self.port, ", ".join(paths_checked)) ) except CertificateError: - self.module.fail_json(msg="SSL Certificate does not belong to %s. Make sure the url has a certificate that belongs to it or use validate_certs=no (insecure)" % self.hostname) + raise SSLValidationError("SSL Certificate does not belong to %s. Make sure the url has a certificate that belongs to it or use validate_certs=False (insecure)" % self.hostname) try: # cleanup the temp file created, don't worry @@ -336,55 +350,23 @@ class SSLValidationHandler(urllib2.BaseHandler): https_request = http_request - -def url_argument_spec(): - ''' - Creates an argument spec that can be used with any module - that will be requesting content via urllib/urllib2 - ''' - return dict( - url = dict(), - force = dict(default='no', aliases=['thirsty'], type='bool'), - http_agent = dict(default='ansible-httpget'), - use_proxy = dict(default='yes', type='bool'), - validate_certs = dict(default='yes', type='bool'), - url_username = dict(required=False), - url_password = dict(required=False), - ) - - -def fetch_url(module, url, data=None, headers=None, method=None, - use_proxy=True, force=False, last_mod_time=None, timeout=10): +# Rewrite of fetch_url to not require the module environment +def open_url(url, data=None, headers=None, method=None, use_proxy=True, + force=False, last_mod_time=None, timeout=10, validate_certs=True, + url_username=None, url_password=None, http_agent=None): ''' Fetches a file from an HTTP/FTP server using urllib2 ''' - - if not HAS_URLLIB: - module.fail_json(msg='urllib is not installed') - if not HAS_URLLIB2: - module.fail_json(msg='urllib2 is not installed') - elif not HAS_URLPARSE: - module.fail_json(msg='urlparse is not installed') - - r = None handlers = [] - info = dict(url=url) - - distribution = get_distribution() - # Get validate_certs from the module params - validate_certs = module.params.get('validate_certs', True) # FIXME: change the following to use the generic_urlparse function # to remove the indexed references for 'parsed' parsed = urlparse.urlparse(url) if parsed[0] == 'https' and validate_certs: if not HAS_SSL: - if distribution == 'Redhat': - module.fail_json(msg='SSL validation is not available in your version of python. You can use validate_certs=no, however this is unsafe and not recommended. You can also install python-ssl from EPEL') - else: - module.fail_json(msg='SSL validation is not available in your version of python. You can use validate_certs=no, however this is unsafe and not recommended') + raise NoSSLError('SSL validation is not available in your version of python. You can use validate_certs=False, however this is unsafe and not recommended') if not HAS_MATCH_HOSTNAME: - module.fail_json(msg='Available SSL validation does not check that the certificate matches the hostname. You can install backports.ssl_match_hostname or update your managed machine to python-2.7.9 or newer. You could also use validate_certs=no, however this is unsafe and not recommended') + raise SSLValidationError('Available SSL validation does not check that the certificate matches the hostname. You can install backports.ssl_match_hostname or update your managed machine to python-2.7.9 or newer. You could also use validate_certs=False, however this is unsafe and not recommended') # do the cert validation netloc = parsed[1] @@ -398,13 +380,14 @@ def fetch_url(module, url, data=None, headers=None, method=None, port = 443 # create the SSL validation handler and # add it to the list of handlers - ssl_handler = SSLValidationHandler(module, hostname, port) + ssl_handler = SSLValidationHandler(hostname, port) handlers.append(ssl_handler) if parsed[0] != 'ftp': - username = module.params.get('url_username', '') + username = url_username + if username: - password = module.params.get('url_password', '') + password = url_password netloc = parsed[1] elif '@' in parsed[1]: credentials, netloc = parsed[1].split('@', 1) @@ -448,14 +431,14 @@ def fetch_url(module, url, data=None, headers=None, method=None, if method: if method.upper() not in ('OPTIONS','GET','HEAD','POST','PUT','DELETE','TRACE','CONNECT'): - module.fail_json(msg='invalid HTTP request method; %s' % method.upper()) + raise ConnectionError('invalid HTTP request method; %s' % method.upper()) request = RequestWithMethod(url, method.upper(), data) else: request = urllib2.Request(url, data) # add the custom agent header, to help prevent issues # with sites that block the default urllib agent string - request.add_header('User-agent', module.params.get('http_agent')) + request.add_header('User-agent', http_agent) # if we're ok with getting a 304, set the timestamp in the # header, otherwise make sure we don't get a cached copy @@ -468,20 +451,72 @@ def fetch_url(module, url, data=None, headers=None, method=None, # user defined headers now, which may override things we've set above if headers: if not isinstance(headers, dict): - module.fail_json("headers provided to fetch_url() must be a dict") + raise ValueError("headers provided to fetch_url() must be a dict") for header in headers: request.add_header(header, headers[header]) + if sys.version_info < (2,6,0): + # urlopen in python prior to 2.6.0 did not + # have a timeout parameter + r = urllib2.urlopen(request, None) + else: + r = urllib2.urlopen(request, None, timeout) + + return r + +# +# Module-related functions +# + +def url_argument_spec(): + ''' + Creates an argument spec that can be used with any module + that will be requesting content via urllib/urllib2 + ''' + return dict( + url = dict(), + force = dict(default='no', aliases=['thirsty'], type='bool'), + http_agent = dict(default='ansible-httpget'), + use_proxy = dict(default='yes', type='bool'), + validate_certs = dict(default='yes', type='bool'), + url_username = dict(required=False), + url_password = dict(required=False), + ) + +def fetch_url(module, url, data=None, headers=None, method=None, + use_proxy=True, force=False, last_mod_time=None, timeout=10): + ''' + Fetches a file from an HTTP/FTP server using urllib2. Requires the module environment + ''' + + if not HAS_URLLIB2: + module.fail_json(msg='urllib2 is not installed') + elif not HAS_URLPARSE: + module.fail_json(msg='urlparse is not installed') + + # Get validate_certs from the module params + validate_certs = module.params.get('validate_certs', True) + + username = module.params.get('url_username', '') + password = module.params.get('url_password', '') + http_agent = module.params.get('http_agent', None) + + r = None + info = dict(url=url) try: - if sys.version_info < (2,6,0): - # urlopen in python prior to 2.6.0 did not - # have a timeout parameter - r = urllib2.urlopen(request, None) - else: - r = urllib2.urlopen(request, None, timeout) + r = open_url(url, data=None, headers=None, method=None, + use_proxy=True, force=False, last_mod_time=None, timeout=10, + validate_certs=validate_certs, url_username=username, + url_password=password, http_agent=http_agent) info.update(r.info()) info['url'] = r.geturl() # The URL goes in too, because of redirects. info.update(dict(msg="OK (%s bytes)" % r.headers.get('Content-Length', 'unknown'), status=200)) + except NoSSLError, e: + distribution = get_distribution() + if distribution.lower() == 'redhat': + module.fail_json(msg='%s. You can also install python-ssl from EPEL' % str(e)) + except (ConnectionError, ValueError), e: + module.fail_json(msg=str(e)) except urllib2.HTTPError, e: info.update(dict(msg=str(e), status=e.code)) except urllib2.URLError, e: @@ -493,4 +528,3 @@ def fetch_url(module, url, data=None, headers=None, method=None, info.update(dict(msg="An unknown error occurred: %s" % str(e), status=-1)) return r, info -