Fix url lookup for python 3 (#17295)

* Use six instead of urllib2, for python 3 compat

* Open the certificate file using binary mode

On python3, os.write requires 'bytes'. Also avoid
using a too broad exception, since the issue was hard
to spot due to it.

* Do not add the header User-agent if not set

Python3 module do raise a exception if a header is
not a string-like object, and the default value is None.

lib/ansible/module_utils/urls.py

@@ -115,6 +115,7 @@ except ImportError:
 
 import ansible.module_utils.six.moves.urllib.request as urllib_request
 import ansible.module_utils.six.moves.urllib.error as urllib_error
+from ansible.module_utils.six import b
 
 try:
     # python3
@@ -606,11 +607,11 @@ class SSLValidationHandler(urllib_request.BaseHandler):
                     full_path = os.path.join(path, f)
                     if os.path.isfile(full_path) and os.path.splitext(f)[1] in ('.crt','.pem'):
                         try:
-                            cert_file = open(full_path, 'r')
+                            cert_file = open(full_path, 'rb')
                             os.write(tmp_fd, cert_file.read())
-                            os.write(tmp_fd, '\n')
+                            os.write(tmp_fd, b('\n'))
                             cert_file.close()
-                        except:
+                        except (OSError, IOError):
                             pass
 
         return (tmp_path, paths_checked)
@@ -844,6 +845,7 @@ def open_url(url, data=None, headers=None, method=None, use_proxy=True,
 
     # add the custom agent header, to help prevent issues
     # with sites that block the default urllib agent string
+    if http_agent:
         request.add_header('User-agent', http_agent)
 
     # if we're ok with getting a 304, set the timestamp in the

lib/ansible/plugins/lookup/url.py

@@ -17,12 +17,12 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import urllib2
 
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
 from ansible.module_utils.urls import open_url, ConnectionError, SSLValidationError
 from ansible.utils.unicode import to_unicode
+from ansible.compat.six.moves.urllib.error import HTTPError, URLError
 
 try:
     from __main__ import display
@@ -42,9 +42,9 @@ class LookupModule(LookupBase):
             display.vvvv("url lookup connecting to %s" % term)
             try:
                 response = open_url(term, validate_certs=validate_certs)
-            except urllib2.HTTPError as e:
+            except HTTPError as e:
                 raise AnsibleError("Received HTTP error for %s : %s" % (term, str(e)))
-            except urllib2.URLError as e:
+            except URLError as e:
                 raise AnsibleError("Failed lookup url for %s : %s" % (term, str(e)))
             except SSLValidationError as e:
                 raise AnsibleError("Error validating the server's certificate for %s: %s" % (term, str(e)))