Split the fetch_url() function into fetch_url and open_url().

open_url() is suitable for use outside of a module environment.  Will
let us use open_url to do SSL cert verification in other, non-module
code.

lib/ansible/module_utils/urls.py

@@ -26,12 +26,6 @@
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-try:
-    import urllib
-    HAS_URLLIB = True
-except:
-    HAS_URLLIB = False
-
 try:
     import urllib2
     HAS_URLLIB2 = True
@@ -62,7 +56,9 @@ except ImportError:
 import httplib
 import os
 import re
+import sys
 import socket
+import platform
 import tempfile
 
 
@@ -89,6 +85,27 @@ zKPZsZ2miVGclicJHzm5q080b1p/sZtuKIEZk6vZqEg=
 -----END CERTIFICATE-----
 """
 
+#
+# Exceptions
+#
+
+class ConnectionError(Exception):
+    """Failed to connect to the server"""
+    pass
+
+class ProxyError(ConnectionError):
+    """Failure to connect because of a proxy"""
+    pass
+
+class SSLValidationError(ConnectionError):
+    """Failure to connect due to SSL validation failing"""
+    pass
+
+class NoSSLError(SSLValidationError):
+    """Needed to connect to an HTTPS url but no ssl library available to verify the certificate"""
+    pass
+
+
 class CustomHTTPSConnection(httplib.HTTPSConnection):
     def connect(self):
         "Connect to a host on a given (SSL) port."
@@ -153,7 +170,7 @@ def generic_urlparse(parts):
                 username, password = auth.split(':', 1)
             generic_parts['username'] = username
             generic_parts['password'] = password
-            generic_parts['hostname'] = hostnme
+            generic_parts['hostname'] = hostname
             generic_parts['port']     = port
         except:
             generic_parts['username'] = None
@@ -189,8 +206,7 @@ class SSLValidationHandler(urllib2.BaseHandler):
     '''
     CONNECT_COMMAND = "CONNECT %s:%s HTTP/1.0\r\nConnection: close\r\n"
 
-    def __init__(self, module, hostname, port):
+    def __init__(self, hostname, port):
-        self.module = module
         self.hostname = hostname
         self.port = port
 
@@ -200,23 +216,22 @@ class SSLValidationHandler(urllib2.BaseHandler):
 
         ca_certs = []
         paths_checked = []
-        platform = get_platform()
-        distribution = get_distribution()
 
+        system = platform.system()
         # build a list of paths to check for .crt/.pem files
         # based on the platform type
         paths_checked.append('/etc/ssl/certs')
-        if platform == 'Linux':
+        if system == 'Linux':
             paths_checked.append('/etc/pki/ca-trust/extracted/pem')
             paths_checked.append('/etc/pki/tls/certs')
             paths_checked.append('/usr/share/ca-certificates/cacert.org')
-        elif platform == 'FreeBSD':
+        elif system == 'FreeBSD':
             paths_checked.append('/usr/local/share/certs')
-        elif platform == 'OpenBSD':
+        elif system == 'OpenBSD':
             paths_checked.append('/etc/ssl')
-        elif platform == 'NetBSD':
+        elif system == 'NetBSD':
             ca_certs.append('/etc/openssl/certs')
-        elif platform == 'SunOS':
+        elif system == 'SunOS':
             paths_checked.append('/opt/local/etc/openssl/certs')
 
         # fall back to a user-deployed cert in a standard
@@ -226,7 +241,7 @@ class SSLValidationHandler(urllib2.BaseHandler):
         tmp_fd, tmp_path = tempfile.mkstemp()
 
         # Write the dummy ca cert if we are running on Mac OS X
-        if platform == 'Darwin':
+        if system == 'Darwin':
             os.write(tmp_fd, DUMMY_CA_CERT)
             # Default Homebrew path for OpenSSL certs 
             paths_checked.append('/usr/local/etc/openssl')
@@ -259,7 +274,7 @@ class SSLValidationHandler(urllib2.BaseHandler):
             if int(resp_code) not in valid_codes:
                 raise Exception
         except:
-            self.module.fail_json(msg='Connection to proxy failed')
+            raise ProxyError('Connection to proxy failed')
 
     def detect_no_proxy(self, url):
         '''
@@ -304,7 +319,7 @@ class SSLValidationHandler(urllib2.BaseHandler):
                     ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED)
                     match_hostname(ssl_s.getpeercert(), self.hostname)
                 else:
-                    self.module.fail_json(msg='Unsupported proxy scheme: %s. Currently ansible only supports HTTP proxies.' % proxy_parts.get('scheme'))
+                    raise ProxyError('Unsupported proxy scheme: %s. Currently ansible only supports HTTP proxies.' % proxy_parts.get('scheme'))
             else:
                 s.connect((self.hostname, self.port))
                 ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED)
@@ -315,15 +330,14 @@ class SSLValidationHandler(urllib2.BaseHandler):
         except (ssl.SSLError, socket.error), e:
             # fail if we tried all of the certs but none worked
             if 'connection refused' in str(e).lower():
-                self.module.fail_json(msg='Failed to connect to %s:%s.' % (self.hostname, self.port))
+                raise ConnectionError('Failed to connect to %s:%s.' % (self.hostname, self.port))
             else:
-                self.module.fail_json(
+                raise SSLValidationError('Failed to validate the SSL certificate for %s:%s. '
-                    msg='Failed to validate the SSL certificate for %s:%s. ' % (self.hostname, self.port) + \
+                    'Use validate_certs=False (insecure) or make sure your managed systems have a valid CA certificate installed. '
-                    'Use validate_certs=no or make sure your managed systems have a valid CA certificate installed. ' + \
+                    'Paths checked for this platform: %s' % (self.hostname, self.port, ", ".join(paths_checked))
-                    'Paths checked for this platform: %s' % ", ".join(paths_checked)
                 )
         except CertificateError:
-            self.module.fail_json(msg="SSL Certificate does not belong to %s.  Make sure the url has a certificate that belongs to it or use validate_certs=no (insecure)" % self.hostname)
+            raise SSLValidationError("SSL Certificate does not belong to %s.  Make sure the url has a certificate that belongs to it or use validate_certs=False (insecure)" % self.hostname)
 
         try:
             # cleanup the temp file created, don't worry
@@ -336,55 +350,23 @@ class SSLValidationHandler(urllib2.BaseHandler):
 
     https_request = http_request
 
-
+# Rewrite of fetch_url to not require the module environment
-def url_argument_spec():
+def open_url(url, data=None, headers=None, method=None, use_proxy=True,
-    '''
+        force=False, last_mod_time=None, timeout=10, validate_certs=True,
-    Creates an argument spec that can be used with any module
+        url_username=None, url_password=None, http_agent=None):
-    that will be requesting content via urllib/urllib2
-    '''
-    return dict(
-        url = dict(),
-        force = dict(default='no', aliases=['thirsty'], type='bool'),
-        http_agent = dict(default='ansible-httpget'),
-        use_proxy = dict(default='yes', type='bool'),
-        validate_certs = dict(default='yes', type='bool'),
-        url_username = dict(required=False),
-        url_password = dict(required=False),
-    )
-
-
-def fetch_url(module, url, data=None, headers=None, method=None, 
-              use_proxy=True, force=False, last_mod_time=None, timeout=10):
     '''
     Fetches a file from an HTTP/FTP server using urllib2
     '''
-
-    if not HAS_URLLIB:
-        module.fail_json(msg='urllib is not installed')
-    if not HAS_URLLIB2:
-        module.fail_json(msg='urllib2 is not installed')
-    elif not HAS_URLPARSE:
-        module.fail_json(msg='urlparse is not installed')
-
-    r = None
     handlers = []
-    info = dict(url=url)
-
-    distribution = get_distribution()
-    # Get validate_certs from the module params
-    validate_certs = module.params.get('validate_certs', True)
 
     # FIXME: change the following to use the generic_urlparse function
     #        to remove the indexed references for 'parsed'
     parsed = urlparse.urlparse(url)
     if parsed[0] == 'https' and validate_certs:
         if not HAS_SSL:
-            if distribution == 'Redhat':
+            raise NoSSLError('SSL validation is not available in your version of python. You can use validate_certs=False, however this is unsafe and not recommended')
-                module.fail_json(msg='SSL validation is not available in your version of python. You can use validate_certs=no, however this is unsafe and not recommended. You can also install python-ssl from EPEL')
-            else:
-                module.fail_json(msg='SSL validation is not available in your version of python. You can use validate_certs=no, however this is unsafe and not recommended')
         if not HAS_MATCH_HOSTNAME:
-            module.fail_json(msg='Available SSL validation does not check that the certificate matches the hostname.  You can install backports.ssl_match_hostname or update your managed machine to python-2.7.9 or newer.  You could also use validate_certs=no, however this is unsafe and not recommended')
+            raise SSLValidationError('Available SSL validation does not check that the certificate matches the hostname.  You can install backports.ssl_match_hostname or update your managed machine to python-2.7.9 or newer.  You could also use validate_certs=False, however this is unsafe and not recommended')
 
         # do the cert validation
         netloc = parsed[1]
@@ -398,13 +380,14 @@ def fetch_url(module, url, data=None, headers=None, method=None,
             port = 443
         # create the SSL validation handler and
         # add it to the list of handlers
-        ssl_handler = SSLValidationHandler(module, hostname, port)
+        ssl_handler = SSLValidationHandler(hostname, port)
         handlers.append(ssl_handler)
 
     if parsed[0] != 'ftp':
-        username = module.params.get('url_username', '')
+        username = url_username
+
         if username:
-            password = module.params.get('url_password', '')
+            password = url_password
             netloc = parsed[1]
         elif '@' in parsed[1]:
             credentials, netloc = parsed[1].split('@', 1)
@@ -448,14 +431,14 @@ def fetch_url(module, url, data=None, headers=None, method=None,
 
     if method:
         if method.upper() not in ('OPTIONS','GET','HEAD','POST','PUT','DELETE','TRACE','CONNECT'):
-            module.fail_json(msg='invalid HTTP request method; %s' % method.upper())
+            raise ConnectionError('invalid HTTP request method; %s' % method.upper())
         request = RequestWithMethod(url, method.upper(), data)
     else:
         request = urllib2.Request(url, data)
 
     # add the custom agent header, to help prevent issues 
     # with sites that block the default urllib agent string 
-    request.add_header('User-agent', module.params.get('http_agent'))
+    request.add_header('User-agent', http_agent)
 
     # if we're ok with getting a 304, set the timestamp in the 
     # header, otherwise make sure we don't get a cached copy
@@ -468,20 +451,72 @@ def fetch_url(module, url, data=None, headers=None, method=None,
     # user defined headers now, which may override things we've set above
     if headers:
         if not isinstance(headers, dict):
-            module.fail_json("headers provided to fetch_url() must be a dict")
+            raise ValueError("headers provided to fetch_url() must be a dict")
         for header in headers:
             request.add_header(header, headers[header])
 
+    if sys.version_info < (2,6,0):
+        # urlopen in python prior to 2.6.0 did not
+        # have a timeout parameter
+        r = urllib2.urlopen(request, None)
+    else:
+        r = urllib2.urlopen(request, None, timeout)
+
+    return r
+
+#
+# Module-related functions
+#
+
+def url_argument_spec():
+    '''
+    Creates an argument spec that can be used with any module
+    that will be requesting content via urllib/urllib2
+    '''
+    return dict(
+        url = dict(),
+        force = dict(default='no', aliases=['thirsty'], type='bool'),
+        http_agent = dict(default='ansible-httpget'),
+        use_proxy = dict(default='yes', type='bool'),
+        validate_certs = dict(default='yes', type='bool'),
+        url_username = dict(required=False),
+        url_password = dict(required=False),
+    )
+
+def fetch_url(module, url, data=None, headers=None, method=None, 
+              use_proxy=True, force=False, last_mod_time=None, timeout=10):
+    '''
+    Fetches a file from an HTTP/FTP server using urllib2.  Requires the module environment
+    '''
+
+    if not HAS_URLLIB2:
+        module.fail_json(msg='urllib2 is not installed')
+    elif not HAS_URLPARSE:
+        module.fail_json(msg='urlparse is not installed')
+
+    # Get validate_certs from the module params
+    validate_certs = module.params.get('validate_certs', True)
+
+    username = module.params.get('url_username', '')
+    password = module.params.get('url_password', '')
+    http_agent = module.params.get('http_agent', None)
+
+    r = None
+    info = dict(url=url)
     try:
-        if sys.version_info < (2,6,0):
+        r = open_url(url, data=None, headers=None, method=None,
-            # urlopen in python prior to 2.6.0 did not
+                use_proxy=True, force=False, last_mod_time=None, timeout=10,
-            # have a timeout parameter
+                validate_certs=validate_certs, url_username=username,
-            r = urllib2.urlopen(request, None)
+                url_password=password, http_agent=http_agent)
-        else:
-            r = urllib2.urlopen(request, None, timeout)
         info.update(r.info())
         info['url'] = r.geturl()  # The URL goes in too, because of redirects.
         info.update(dict(msg="OK (%s bytes)" % r.headers.get('Content-Length', 'unknown'), status=200))
+    except NoSSLError, e:
+        distribution = get_distribution()
+        if distribution.lower() == 'redhat':
+            module.fail_json(msg='%s. You can also install python-ssl from EPEL' % str(e))
+    except (ConnectionError, ValueError), e:
+        module.fail_json(msg=str(e))
     except urllib2.HTTPError, e:
         info.update(dict(msg=str(e), status=e.code))
     except urllib2.URLError, e:
@@ -493,4 +528,3 @@ def fetch_url(module, url, data=None, headers=None, method=None,
         info.update(dict(msg="An unknown error occurred: %s" % str(e), status=-1))
 
     return r, info
-