openss: fix various test and Python 3 issues (#47188)

changelogs/fragments/openssl-python3.yaml

@@ -0,0 +1,3 @@
+bugfixes:
+- openssl_csr - fix byte encoding issue on Python 3
+- openssl_pkcs12 - fix byte encoding issue on Python 3

lib/ansible/modules/crypto/openssl_csr.py

@@ -465,7 +465,7 @@ class CertificateSigningRequest(crypto_utils.OpenSSLObject):
             return _check_keyUsage_(extensions, b'basicConstraints', self.basicConstraints, self.basicConstraints_critical)
 
         def _check_ocspMustStaple(extensions):
-            oms_ext = [ext for ext in extensions if ext.get_short_name() == MUST_STAPLE_NAME and str(ext) == MUST_STAPLE_VALUE]
+            oms_ext = [ext for ext in extensions if to_bytes(ext.get_short_name()) == MUST_STAPLE_NAME and to_bytes(ext) == MUST_STAPLE_VALUE]
             if OpenSSL.SSL.OPENSSL_VERSION_NUMBER < 0x10100000:
                 # Older versions of libssl don't know about OCSP Must Staple
                 oms_ext.extend([ext for ext in extensions if ext.get_short_name() == b'UNDEF' and ext.get_data() == b'\x30\x03\x02\x01\x05'])

lib/ansible/modules/crypto/openssl_pkcs12.py

@@ -150,7 +150,7 @@ else:
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils import crypto as crypto_utils
-from ansible.module_utils._text import to_native
+from ansible.module_utils._text import to_bytes, to_native
 
 
 class PkcsError(crypto_utils.OpenSSLObjectError):
@@ -231,7 +231,7 @@ class Pkcs(crypto_utils.OpenSSLObject):
                                         self.certificate_path))
 
         if self.friendly_name:
-            self.pkcs12.set_friendlyname(self.friendly_name)
+            self.pkcs12.set_friendlyname(to_bytes(self.friendly_name))
 
         if self.privatekey_path:
             self.pkcs12.set_privatekey(crypto_utils.load_privatekey(
@@ -266,7 +266,7 @@ class Pkcs(crypto_utils.OpenSSLObject):
             pkcs12_file = os.open(self.path,
                                   os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
                                   self.mode)
-            os.write(pkcs12_file, '%s%s' % (pkey, crt))
+            os.write(pkcs12_file, b'%s%s' % (pkey, crt))
             os.close(pkcs12_file)
 
         except IOError as exc:

test/integration/targets/openssl_privatekey/tests/validate.yml

@@ -1,5 +1,5 @@
 - name: Validate privatekey1 (test - RSA key with size 4096 bits)
-  shell: "openssl rsa -noout -text -in {{ output_dir }}/privatekey1.pem  | grep Private | sed 's/Private-Key: (\\(.*\\) bit)/\\1/'"
+  shell: "openssl rsa -noout -text -in {{ output_dir }}/privatekey1.pem  | grep Private | sed 's/\\(RSA\\s\\)*Private-Key: (\\(.*\\) bit.*)/\\2/'"
   register: privatekey1
 
 - name: Validate privatekey1 (assert - RSA key with size 4096 bits)
@@ -9,7 +9,7 @@
 
 
 - name: Validate privatekey2 (test - RSA key with size 2048 bits)
-  shell: "openssl rsa -noout -text -in {{ output_dir }}/privatekey2.pem  | grep Private | sed 's/Private-Key: (\\(.*\\) bit)/\\1/'"
+  shell: "openssl rsa -noout -text -in {{ output_dir }}/privatekey2.pem  | grep Private | sed 's/\\(RSA\\s\\)*Private-Key: (\\(.*\\) bit.*)/\\2/'"
   register: privatekey2
 
 - name: Validate privatekey2 (assert - RSA key with size 2048 bits)
@@ -19,7 +19,7 @@
 
 
 - name: Validate privatekey3 (test - DSA key with size 4096 bits)
-  shell: "openssl dsa -noout -text -in {{ output_dir }}/privatekey3.pem  | grep Private | sed 's/Private-Key: (\\(.*\\) bit)/\\1/'"
+  shell: "openssl dsa -noout -text -in {{ output_dir }}/privatekey3.pem  | grep Private | sed 's/\\(RSA\\s\\)*Private-Key: (\\(.*\\) bit.*)/\\2/'"
   register: privatekey3
 
 - name: Validate privatekey3 (assert - DSA key with size 4096 bits)
@@ -40,7 +40,7 @@
 
 
 - name: Validate privatekey5 (test - Passphrase protected key + idempotence)
-  shell: "openssl rsa -noout -text -in {{ output_dir }}/privatekey5.pem -passin pass:ansible  | grep Private | sed 's/Private-Key: (\\(.*\\) bit)/\\1/'"
+  shell: "openssl rsa -noout -text -in {{ output_dir }}/privatekey5.pem -passin pass:ansible  | grep Private | sed 's/\\(RSA\\s\\)*Private-Key: (\\(.*\\) bit.*)/\\2/'"
   register: privatekey5
   # Current version of OS/X that runs in the CI (10.11) does not have an up to date version of the OpenSSL library
   # leading to this test to fail when run in the CI. However, this test has been run for 10.12 and has returned succesfully.
@@ -59,7 +59,7 @@
 
 
 - name: Validate privatekey6 (test - Passphrase protected key with non ascii character)
-  shell: "openssl rsa -noout -text -in {{ output_dir }}/privatekey6.pem -passin pass:ànsïblé | grep Private | sed 's/Private-Key: (\\(.*\\) bit)/\\1/'"
+  shell: "openssl rsa -noout -text -in {{ output_dir }}/privatekey6.pem -passin pass:ànsïblé | grep Private | sed 's/\\(RSA\\s\\)*Private-Key: (\\(.*\\) bit.*)/\\2/'"
   register: privatekey6
   when: openssl_version.stdout is version('0.9.8zh', '>=')