Merge pull request #17590 from abadger/vault-fixes

Vault fixes

CHANGELOG.md

@@ -169,6 +169,22 @@ Ansible Changes By Release
 * removed previously deprecated ';' as host list separator.
 * Only check if the default ssh client supports ControlPersist once instead of once for each host + task combination.
 
+
+###For custom front ends using the API:
+* ansible.parsing.vault:
+  * VaultLib.is_encrypted() has been deprecated.  It will be removed in 2.4.
+    Use ansible.parsing.vault.is_encrypted() instead
+  * VaultFile has been removed. This unfinished code was never used inside of
+    Ansible.  The feature it was intended to support has now been implemented
+    without using this.
+  * VaultAES, the older, insecure encrypted format that debuted in Ansible-1.5
+    and was relaced by VaultAES256 less than a week later, now has a deprecation
+    warning.  **It will be removed in 2.3**.  In the unlikely event that you
+    wrote a vault file in that 1 week window and have never modified the file
+    since (ansible-vault automatically re-encrypts the file using VaultAES256
+    whenever it is written to but not read), run ``ansible-vault rekey
+    [filename]`` to move to VaultAES256.
+
 ## 2.1.2 "The Song Remains the Same"
 
 ###Deprecations:

lib/ansible/parsing/vault/__init__.py

@@ -1,4 +1,6 @@
 # (c) 2014, James Tanner <tanner.jc@gmail.com>
+# (c) 2016, Adrian Likins <alikins@redhat.com>
+# (c) 2016, Toshio Kuratomi <tkuratomi@ansible.com>
 #
 # Ansible is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -25,7 +27,6 @@ import tempfile
 import random
 from io import BytesIO
 from subprocess import call
-from ansible.errors import AnsibleError
 from hashlib import sha256
 from binascii import hexlify
 from binascii import unhexlify
@@ -61,7 +62,9 @@ try:
 except ImportError:
     HAS_AES = False
 
-from ansible.compat.six import PY3
+from ansible.compat.six import PY3, binary_type
+from ansible.compat.six.moves import zip
+from ansible.errors import AnsibleError
 from ansible.module_utils._text import to_bytes, to_text
 
 try:
@@ -84,14 +87,13 @@ except Exception as e:
     import traceback
     display.debug("Traceback from import of cryptography was {0}".format(traceback.format_exc()))
 
-
 HAS_ANY_PBKDF2HMAC = HAS_PBKDF2 or HAS_PBKDF2HMAC
 
+
 CRYPTO_UPGRADE = "ansible-vault requires a newer version of pycrypto than the one installed on your platform." \
     " You may fix this with OS-specific commands such as: yum install python-devel; rpm -e --nodeps python-crypto; pip install pycrypto"
 
 b_HEADER = b'$ANSIBLE_VAULT'
-HEADER = '$ANSIBLE_VAULT'
 CIPHER_WHITELIST = frozenset((u'AES', u'AES256'))
 CIPHER_WRITE_WHITELIST = frozenset((u'AES256',))
 # See also CIPHER_MAPPING at the bottom of the file which maps cipher strings
@@ -108,32 +110,56 @@ class AnsibleVaultError(AnsibleError):
     pass
 
 
-def is_encrypted(b_data):
+def is_encrypted(data):
     """ Test if this is vault encrypted data blob
 
-    :arg data: a python2 str or a python3 'bytes' to test whether it is
-        recognized as vault encrypted data
+    :arg data: a byte or text string to test whether it is recognized as vault
+        encrypted data
     :returns: True if it is recognized.  Otherwise, False.
     """
+    try:
+        # Make sure we have a byte string and that it only contains ascii
+        # bytes.
+        b_data = to_bytes(to_text(data, encoding='ascii', errors='strict', nonstring='strict'), encoding='ascii', errors='strict')
+    except (UnicodeError, TypeError):
+        # The vault format is pure ascii so if we failed to encode to bytes
+        # via ascii we know that this is not vault data.
+        # Similarly, if it's not a string, it's not vault data
+        return False
+
     if b_data.startswith(b_HEADER):
         return True
     return False
 
 
-def is_encrypted_file(file_obj):
+def is_encrypted_file(file_obj, start_pos=0, count=-1):
     """Test if the contents of a file obj are a vault encrypted data blob.
 
-    The data read from the file_obj is expected to be bytestrings (py2 'str' or
-    python3 'bytes'). This more or less expects 'utf-8' encoding.
-
     :arg file_obj: A file object that will be read from.
-    :returns: True if the file is a vault file. Otherwise, False.
+    :kwarg start_pos: A byte offset in the file to start reading the header
+        from.  Defaults to 0, the beginning of the file.
+    :kwarg count: Read up to this number of bytes from the file to determine
+        if it looks like encrypted vault data.  The default is -1, read to the
+        end of file.
+    :returns: True if the file looks like a vault file. Otherwise, False.
     """
     # read the header and reset the file stream to where it started
     current_position = file_obj.tell()
-    b_header_part = file_obj.read(len(b_HEADER))
-    file_obj.seek(current_position)
-    return is_encrypted(b_header_part)
+    try:
+        file_obj.seek(start_pos)
+        vaulttext = file_obj.read(count)
+        try:
+            b_vaulttext = to_bytes(to_text(vaulttext, encoding='ascii', errors='strict'), encoding='ascii', errors='strict')
+        except (UnicodeError, TypeError):
+            # At present, vault files contain only ascii characters.  The encoding is utf-8
+            # without BOM (for future expansion).  If the header does not
+            # decode as ascii then we know we do not have proper vault
+            # encrypted data.
+            return False
+    finally:
+        file_obj.seek(current_position)
+
+    return is_encrypted(b_vaulttext)
 
 
 class VaultLib:
@@ -143,11 +169,11 @@ class VaultLib:
         self.cipher_name = None
         self.b_version = b'1.1'
 
-    # really b_data, but for compat
-    def is_encrypted(self, data):
+    @staticmethod
+    def is_encrypted(data):
         """ Test if this is vault encrypted data
 
-        :arg data: a python2 utf-8 string or a python3 'bytes' to test whether it is
+        :arg data: a byte or text string or a python3 to test for whether it is
             recognized as vault encrypted data
         :returns: True if it is recognized.  Otherwise, False.
         """
@@ -155,36 +181,29 @@ class VaultLib:
         # This could in the future, check to see if the data is a vault blob and
         # is encrypted with a key associated with this vault
         # instead of just checking the format.
+        display.deprecated(u'vault.VaultLib.is_encrypted is deprecated.  Use vault.is_encrypted instead', version='2.4')
         return is_encrypted(data)
 
-    def is_encrypted_file(self, file_obj):
+    @staticmethod
+    def is_encrypted_file(file_obj):
+        display.deprecated(u'vault.VaultLib.is_encrypted_file is deprecated.  Use vault.is_encrypted_file instead', version='2.4')
         return is_encrypted_file(file_obj)
 
-    def encrypt(self, data):
+    def encrypt(self, plaintext):
         """Vault encrypt a piece of data.
 
-        :arg data: a PY2 unicode string or PY3 string to encrypt.
+        :arg plaintext: a text or byte string to encrypt.
         :returns: a utf-8 encoded byte str of encrypted data.  The string
             contains a header identifying this as vault encrypted data and
             formatted to newline terminated lines of 80 characters.  This is
             suitable for dumping as is to a vault file.
 
-        The unicode or string passed in as data will encoded to UTF-8 before
-        encryption. If the a already encoded string or PY2 bytestring needs to
-        be encrypted, use encrypt_bytestring().
+        If the string passed in is a text string, it will be encoded to UTF-8
+        before encryption.
         """
-        plaintext = data
-        plaintext_bytes = plaintext.encode('utf-8')
-
-        return self.encrypt_bytestring(plaintext_bytes)
-
-    def encrypt_bytestring(self, plaintext_bytes):
-        '''Encrypt a PY2 bytestring.
-
-        Like encrypt(), except plaintext_bytes is not encoded to UTF-8
-        before encryption.'''
+        b_plaintext = to_bytes(plaintext, errors='surrogate_or_strict')
 
-        if self.is_encrypted(plaintext_bytes):
+        if is_encrypted(b_plaintext):
             raise AnsibleError("input is already encrypted")
 
         if not self.cipher_name or self.cipher_name not in CIPHER_WRITE_WHITELIST:
@@ -197,32 +216,35 @@ class VaultLib:
         this_cipher = Cipher()
 
         # encrypt data
-        ciphertext_bytes = this_cipher.encrypt(plaintext_bytes, self.b_password)
+        b_ciphertext = this_cipher.encrypt(b_plaintext, self.b_password)
 
         # format the data for output to the file
-        ciphertext_envelope = self._format_output(ciphertext_bytes)
-        return ciphertext_envelope
+        b_vaulttext = self._format_output(b_ciphertext)
+        return b_vaulttext
 
-    def decrypt(self, data, filename=None):
+    def decrypt(self, vaulttext, filename=None):
         """Decrypt a piece of vault encrypted data.
 
-        :arg data: a string to decrypt.  Since vault encrypted data is an
+        :arg vaulttext: a string to decrypt.  Since vault encrypted data is an
             ascii text format this can be either a byte str or unicode string.
+        :kwarg filename: a filename that the data came from.  This is only
+            used to make better error messages in case the data cannot be
+            decrypted.
         :returns: a byte string containing the decrypted data
         """
-        b_data = to_bytes(data, errors='strict', encoding='utf-8')
+        b_vaulttext = to_bytes(vaulttext, errors='strict', encoding='utf-8')
 
         if self.b_password is None:
             raise AnsibleError("A vault password must be specified to decrypt data")
 
-        if not self.is_encrypted(b_data):
+        if not is_encrypted(b_vaulttext):
             msg = "input is not vault encrypted data"
             if filename:
                 msg += "%s is not a vault encrypted file" % filename
             raise AnsibleError(msg)
 
         # clean out header
-        b_data = self._split_header(b_data)
+        b_vaulttext = self._split_header(b_vaulttext)
 
         # create the cipher object
         cipher_class_name = u'Vault{0}'.format(self.cipher_name)
@@ -233,20 +255,20 @@ class VaultLib:
         else:
             raise AnsibleError("{0} cipher could not be found".format(self.cipher_name))
 
-        # try to unencrypt data
-        b_data = this_cipher.decrypt(b_data, self.b_password)
-        if b_data is None:
+        # try to unencrypt vaulttext
+        b_plaintext = this_cipher.decrypt(b_vaulttext, self.b_password)
+        if b_plaintext is None:
             msg = "Decryption failed"
             if filename:
                 msg += " on %s" % filename
             raise AnsibleError(msg)
 
-        return b_data
+        return b_plaintext
 
-    def _format_output(self, b_data):
+    def _format_output(self, b_ciphertext):
         """ Add header and format to 80 columns
 
-            :arg b_data: the encrypted and hexlified data as a byte string
+            :arg b_vaulttext: the encrypted and hexlified data as a byte string
             :returns: a byte str that should be dumped into a file.  It's
                 formatted to 80 char columns and has the header prepended
         """
@@ -254,38 +276,37 @@ class VaultLib:
         if not self.cipher_name:
             raise AnsibleError("the cipher must be set before adding a header")
 
-        b_header = HEADER.encode('utf-8')
-        header = b';'.join([b_header, self.b_version,
+        header = b';'.join([b_HEADER, self.b_version,
                         to_bytes(self.cipher_name,'utf-8', errors='strict')])
-        tmpdata = [header]
-        tmpdata += [b_data[i:i + 80] for i in range(0, len(b_data), 80)]
-        tmpdata += [b'']
-        tmpdata = b'\n'.join(tmpdata)
+        b_vaulttext = [header]
+        b_vaulttext += [b_ciphertext[i:i + 80] for i in range(0, len(b_ciphertext), 80)]
+        b_vaulttext += [b'']
+        b_vaulttext = b'\n'.join(b_vaulttext)
 
-        return tmpdata
+        return b_vaulttext
 
-    def _split_header(self, b_data):
-        """Retrieve information about the Vault and  clean the data
+    def _split_header(self, b_vaulttext):
+        """Retrieve information about the Vault and clean the data
 
         When data is saved, it has a header prepended and is formatted into 80
         character lines.  This method extracts the information from the header
         and then removes the header and the inserted newlines.  The string returned
         is suitable for processing by the Cipher classes.
 
-        :arg b_data: byte str containing the data from a save file
+        :arg b_vaulttext: byte str containing the data from a save file
         :returns: a byte str suitable for passing to a Cipher class's
             decrypt() function.
         """
         # used by decrypt
 
-        tmpdata = b_data.split(b'\n')
-        tmpheader = tmpdata[0].strip().split(b';')
+        b_tmpdata = b_vaulttext.split(b'\n')
+        b_tmpheader = b_tmpdata[0].strip().split(b';')
 
-        self.b_version = tmpheader[1].strip()
-        self.cipher_name = to_text(tmpheader[2].strip())
-        clean_data = b''.join(tmpdata[1:])
+        self.b_version = b_tmpheader[1].strip()
+        self.cipher_name = to_text(b_tmpheader[2].strip())
+        b_ciphertext = b''.join(b_tmpdata[1:])
 
-        return clean_data
+        return b_ciphertext
 
 
 class VaultEditor:
@@ -400,7 +421,7 @@ class VaultEditor:
         # A file to be encrypted into a vaultfile could be any encoding
         # so treat the contents as a byte string.
         plaintext = self.read_data(filename)
-        ciphertext = self.vault.encrypt_bytestring(plaintext)
+        ciphertext = self.vault.encrypt(plaintext)
         self.write_data(ciphertext, output_file or filename)
 
     def decrypt_file(self, filename, output_file=None):
@@ -533,47 +554,6 @@ class VaultEditor:
         return editor
 
 
-class VaultFile(object):
-
-    def __init__(self, password, filename):
-        self.password = password
-
-        self.filename = filename
-        if not os.path.isfile(self.filename):
-            raise AnsibleError("%s does not exist" % self.filename)
-        try:
-            self.filehandle = open(filename, "rb")
-        except Exception as e:
-            raise AnsibleError("Could not open %s: %s" % (self.filename, str(e)))
-
-        _, self.tmpfile = tempfile.mkstemp()
-
-    # TODO:
-    # __del__ can be problematic in python... For this use case, make
-    # VaultFile a context manager instead (implement __enter__ and __exit__)
-    def __del__(self):
-        self.filehandle.close()
-        os.unlink(self.tmpfile)
-
-    def is_encrypted(self):
-        return is_encrypted_file(self.filehandle)
-
-    def get_decrypted(self):
-        check_prereqs()
-
-        if self.is_encrypted():
-            tmpdata = self.filehandle.read()
-            this_vault = VaultLib(self.password)
-            dec_data = this_vault.decrypt(tmpdata)
-            if dec_data is None:
-                raise AnsibleError("Failed to decrypt: %s" % self.filename)
-            else:
-                self.tmpfile.write(dec_data)
-                return self.tmpfile
-        else:
-            return self.filename
-
-
 ########################################
 #               CIPHERS                #
 ########################################
@@ -591,76 +571,84 @@ class VaultAES:
         if not HAS_AES:
             raise AnsibleError(CRYPTO_UPGRADE)
 
-    def aes_derive_key_and_iv(self, password, salt, key_length, iv_length):
+    def _aes_derive_key_and_iv(self, b_password, b_salt, key_length, iv_length):
 
         """ Create a key and an initialization vector """
 
-        d = d_i = b''
-        while len(d) < key_length + iv_length:
-            text = b''.join([d_i, password, salt])
-            d_i = to_bytes(md5(text).digest(), errors='strict')
-            d += d_i
+        b_d = b_di = b''
+        while len(b_d) < key_length + iv_length:
+            b_text = b''.join([b_di, b_password, b_salt])
+            b_di = to_bytes(md5(b_text).digest(), errors='strict')
+            b_d += b_di
 
-        key = d[:key_length]
-        iv = d[key_length:key_length+iv_length]
+        b_key = b_d[:key_length]
+        b_iv = b_d[key_length:key_length+iv_length]
 
-        return key, iv
+        return b_key, b_iv
 
-    def encrypt(self, data, password, key_length=32):
+    def encrypt(self, b_plaintext, b_password, key_length=32):
 
         """ Read plaintext data from in_file and write encrypted to out_file """
 
         raise AnsibleError("Encryption disabled for deprecated VaultAES class")
 
-    def decrypt(self, data, password, key_length=32):
+    def decrypt(self, b_vaulttext, b_password, key_length=32):
 
-        """ Read encrypted data from in_file and write decrypted to out_file """
+        """ Decrypt the given data and return it
+        :arg b_data: A byte string containing the encrypted data
+        :arg b_password: A byte string containing the encryption password
+        :arg key_length: Length of the key
+        :returns: A byte string containing the decrypted data
+        """
 
+        display.deprecated(u'The VaultAES format is insecure and has been'
+                ' deprecated since Ansible-1.5.  Use vault rekey FILENAME to'
+                ' switch to the newer VaultAES256 format', version='2.3')
         # http://stackoverflow.com/a/14989032
 
-        data = unhexlify(data)
+        b_ciphertext = unhexlify(b_vaulttext)
 
-        in_file = BytesIO(data)
+        in_file = BytesIO(b_ciphertext)
         in_file.seek(0)
         out_file = BytesIO()
 
         bs = AES.block_size
-        tmpsalt = in_file.read(bs)
-        salt = tmpsalt[len(b'Salted__'):]
-        key, iv = self.aes_derive_key_and_iv(password, salt, key_length, bs)
-        cipher = AES.new(key, AES.MODE_CBC, iv)
-        next_chunk = b''
+        b_tmpsalt = in_file.read(bs)
+        b_salt = b_tmpsalt[len(b'Salted__'):]
+        b_key, b_iv = self._aes_derive_key_and_iv(b_password, b_salt, key_length, bs)
+        cipher = AES.new(b_key, AES.MODE_CBC, b_iv)
+        b_next_chunk = b''
         finished = False
 
         while not finished:
-            chunk, next_chunk = next_chunk, cipher.decrypt(in_file.read(1024 * bs))
-            if len(next_chunk) == 0:
+            b_chunk, b_next_chunk = b_next_chunk, cipher.decrypt(in_file.read(1024 * bs))
+            if len(b_next_chunk) == 0:
                 if PY3:
-                    padding_length = chunk[-1]
+                    padding_length = b_chunk[-1]
                 else:
-                    padding_length = ord(chunk[-1])
+                    padding_length = ord(b_chunk[-1])
 
-                chunk = chunk[:-padding_length]
+                b_chunk = b_chunk[:-padding_length]
                 finished = True
 
-            out_file.write(chunk)
+            out_file.write(b_chunk)
             out_file.flush()
 
         # reset the stream pointer to the beginning
         out_file.seek(0)
-        out_data = out_file.read()
+        b_out_data = out_file.read()
         out_file.close()
 
         # split out sha and verify decryption
-        split_data = out_data.split(b"\n", 1)
-        this_sha = split_data[0]
-        this_data = split_data[1]
-        test_sha = to_bytes(sha256(this_data).hexdigest())
+        b_split_data = b_out_data.split(b"\n", 1)
+        b_this_sha = b_split_data[0]
+        b_plaintext = b_split_data[1]
+        b_test_sha = to_bytes(sha256(b_plaintext).hexdigest())
 
-        if this_sha != test_sha:
+        if b_this_sha != b_test_sha:
             raise AnsibleError("Decryption failed")
 
-        return this_data
+        return b_plaintext
 
 
 class VaultAES256:
@@ -678,17 +666,19 @@ class VaultAES256:
 
         check_prereqs()
 
-    def create_key(self, password, salt, keylength, ivlength):
+    @staticmethod
+    def _create_key(b_password, b_salt, keylength, ivlength):
         hash_function = SHA256
 
         # make two keys and one iv
         pbkdf2_prf = lambda p, s: HMAC.new(p, s, hash_function).digest()
 
-        derivedkey = PBKDF2(password, salt, dkLen=(2 * keylength) + ivlength,
+        b_derivedkey = PBKDF2(b_password, b_salt, dkLen=(2 * keylength) + ivlength,
                             count=10000, prf=pbkdf2_prf)
-        return derivedkey
+        return b_derivedkey
 
-    def gen_key_initctr(self, password, salt):
+    @classmethod
+    def _gen_key_initctr(cls, b_password, b_salt):
         # 16 for AES 128, 32 for AES256
         keylength = 32
 
@@ -700,79 +690,80 @@ class VaultAES256:
             kdf = PBKDF2HMAC(
                 algorithm=c_SHA256(),
                 length=2 * keylength + ivlength,
-                salt=salt,
+                salt=b_salt,
                 iterations=10000,
                 backend=backend)
-            derivedkey = kdf.derive(password)
+            b_derivedkey = kdf.derive(b_password)
         else:
-            derivedkey = self.create_key(password, salt, keylength, ivlength)
+            b_derivedkey = cls._create_key(b_password, b_salt, keylength, ivlength)
 
-        key1 = derivedkey[:keylength]
-        key2 = derivedkey[keylength:(keylength * 2)]
-        iv = derivedkey[(keylength * 2):(keylength * 2) + ivlength]
+        b_key1 = b_derivedkey[:keylength]
+        b_key2 = b_derivedkey[keylength:(keylength * 2)]
+        b_iv = b_derivedkey[(keylength * 2):(keylength * 2) + ivlength]
 
-        return key1, key2, hexlify(iv)
+        return b_key1, b_key2, hexlify(b_iv)
 
-    def encrypt(self, data, password):
-        salt = os.urandom(32)
-        key1, key2, iv = self.gen_key_initctr(password, salt)
+    def encrypt(self, b_plaintext, b_password):
+        b_salt = os.urandom(32)
+        b_key1, b_key2, b_iv = self._gen_key_initctr(b_password, b_salt)
 
         # PKCS#7 PAD DATA http://tools.ietf.org/html/rfc5652#section-6.3
         bs = AES.block_size
-        padding_length = (bs - len(data) % bs) or bs
-        data += to_bytes(padding_length * chr(padding_length), encoding='ascii', errors='strict')
+        padding_length = (bs - len(b_plaintext) % bs) or bs
+        b_plaintext += to_bytes(padding_length * chr(padding_length), encoding='ascii', errors='strict')
 
         # COUNTER.new PARAMETERS
         # 1) nbits (integer) - Length of the counter, in bits.
-        # 2) initial_value (integer) - initial value of the counter. "iv" from gen_key_initctr
+        # 2) initial_value (integer) - initial value of the counter. "iv" from _gen_key_initctr
 
-        ctr = Counter.new(128, initial_value=int(iv, 16))
+        ctr = Counter.new(128, initial_value=int(b_iv, 16))
 
         # AES.new PARAMETERS
-        # 1) AES key, must be either 16, 24, or 32 bytes long -- "key" from gen_key_initctr
+        # 1) AES key, must be either 16, 24, or 32 bytes long -- "key" from _gen_key_initctr
         # 2) MODE_CTR, is the recommended mode
         # 3) counter=<CounterObject>
 
-        cipher = AES.new(key1, AES.MODE_CTR, counter=ctr)
+        cipher = AES.new(b_key1, AES.MODE_CTR, counter=ctr)
 
         # ENCRYPT PADDED DATA
-        cryptedData = cipher.encrypt(data)
+        b_ciphertext = cipher.encrypt(b_plaintext)
 
         # COMBINE SALT, DIGEST AND DATA
-        hmac = HMAC.new(key2, cryptedData, SHA256)
-        message = b'\n'.join([hexlify(salt), to_bytes(hmac.hexdigest()), hexlify(cryptedData)])
-        message = hexlify(message)
-        return message
+        hmac = HMAC.new(b_key2, b_ciphertext, SHA256)
+        b_vaulttext = b'\n'.join([hexlify(b_salt), to_bytes(hmac.hexdigest()), hexlify(b_ciphertext)])
+        b_vaulttext = hexlify(b_vaulttext)
+        return b_vaulttext
 
-    def decrypt(self, data, password):
+    def decrypt(self, b_vaulttext, b_password):
         # SPLIT SALT, DIGEST, AND DATA
-        data = unhexlify(data)
-        salt, cryptedHmac, cryptedData = data.split(b"\n", 2)
-        salt = unhexlify(salt)
-        cryptedData = unhexlify(cryptedData)
-        key1, key2, iv = self.gen_key_initctr(password, salt)
+        b_vaulttext = unhexlify(b_vaulttext)
+        b_salt, b_cryptedHmac, b_ciphertext = b_vaulttext.split(b"\n", 2)
+        b_salt = unhexlify(b_salt)
+        b_ciphertext = unhexlify(b_ciphertext)
+        b_key1, b_key2, b_iv = self._gen_key_initctr(b_password, b_salt)
 
         # EXIT EARLY IF DIGEST DOESN'T MATCH
-        hmacDecrypt = HMAC.new(key2, cryptedData, SHA256)
-        if not self.is_equal(cryptedHmac, to_bytes(hmacDecrypt.hexdigest())):
+        hmacDecrypt = HMAC.new(b_key2, b_ciphertext, SHA256)
+        if not self._is_equal(b_cryptedHmac, to_bytes(hmacDecrypt.hexdigest())):
             return None
         # SET THE COUNTER AND THE CIPHER
-        ctr = Counter.new(128, initial_value=int(iv, 16))
-        cipher = AES.new(key1, AES.MODE_CTR, counter=ctr)
+        ctr = Counter.new(128, initial_value=int(b_iv, 16))
+        cipher = AES.new(b_key1, AES.MODE_CTR, counter=ctr)
 
         # DECRYPT PADDED DATA
-        decryptedData = cipher.decrypt(cryptedData)
+        b_plaintext = cipher.decrypt(b_ciphertext)
 
         # UNPAD DATA
-        try:
-            padding_length = ord(decryptedData[-1])
-        except TypeError:
-            padding_length = decryptedData[-1]
+        if PY3:
+            padding_length = b_plaintext[-1]
+        else:
+            padding_length = ord(b_plaintext[-1])
 
-        decryptedData = decryptedData[:-padding_length]
-        return decryptedData
+        b_plaintext = b_plaintext[:-padding_length]
+        return b_plaintext
 
-    def is_equal(self, a, b):
+    @staticmethod
+    def _is_equal(b_a, b_b):
         """
         Comparing 2 byte arrrays in constant time
         to avoid timing attacks.
@@ -780,16 +771,19 @@ class VaultAES256:
         It would be nice if there was a library for this but
         hey.
         """
+        if not (isinstance(b_a, binary_type) and isinstance(b_b, binary_type)):
+            raise TypeError('_is_equal can only be used to compare two byte strings')
+
         # http://codahale.com/a-lesson-in-timing-attacks/
-        if len(a) != len(b):
+        if len(b_a) != len(b_b):
             return False
 
         result = 0
-        for x, y in zip(a, b):
+        for b_x, b_y in zip(b_a, b_b):
             if PY3:
-                result |= x ^ y
+                result |= b_x ^ b_y
             else:
-                result |= ord(x) ^ ord(y)
+                result |= ord(b_x) ^ ord(b_y)
         return result == 0
 
 

test/units/cli/test_galaxy.py

@@ -24,11 +24,9 @@ import shutil
 import tarfile
 import tempfile
 
-from mock import patch
-
 from ansible.compat.six import PY3
 from ansible.compat.tests import unittest
-from mock import patch, call
+from ansible.compat.tests.mock import call, patch
 
 import ansible
 from ansible.errors import AnsibleError, AnsibleOptionsError

test/units/module_utils/test_database.py

@@ -1,5 +1,4 @@
 import collections
-import mock
 import os
 import re
 
@@ -24,6 +23,8 @@ except ImportError:
             else: excName = str(expected)
             raise AssertionError("%s not raised" % excName)
 
+from ansible.compat.tests import mock
+
 from ansible.module_utils.database import (
     pg_quote_identifier,
     SQLParseError,

test/units/modules/core/test_apt.py

@@ -1,9 +1,9 @@
 import collections
-import mock
 import os
 import sys
-from ansible.compat.tests import unittest
 
+from ansible.compat.tests import mock
+from ansible.compat.tests import unittest
 
 try:
     from ansible.modules.core.packaging.os.apt import (

test/units/parsing/vault/test_vault.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
+# (c) 2016, Toshio Kuratomi <tkuratomi@ansible.com>
 #
 # This file is part of Ansible
 #
@@ -34,7 +35,7 @@ from ansible.compat.tests import unittest
 from ansible import errors
 from ansible.parsing.vault import VaultLib
 from ansible.parsing import vault
-from ansible.module_utils._text import to_bytes
+from ansible.module_utils._text import to_bytes, to_text
 
 
 # Counter import fails for 2.0.1, requires >= 2.6.1 from pip
@@ -60,241 +61,254 @@ except ImportError:
 
 
 class TestVaultIsEncrypted(unittest.TestCase):
-    def test_utf8_not_encrypted(self):
-        b_data = "foobar".encode('utf8')
+    def test_bytes_not_encrypted(self):
+        b_data = b"foobar"
         self.assertFalse(vault.is_encrypted(b_data))
 
-    def test_utf8_encrypted(self):
-        data = u"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible")
-        b_data = data.encode('utf8')
+    def test_bytes_encrypted(self):
+        b_data = b"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible")
         self.assertTrue(vault.is_encrypted(b_data))
 
-    def test_bytes_not_encrypted(self):
-        b_data = b"foobar"
+    def test_text_not_encrypted(self):
+        b_data = to_text(b"foobar")
         self.assertFalse(vault.is_encrypted(b_data))
 
-    def test_bytes_encrypted(self):
-        b_data = b"$ANSIBLE_VAULT;9.9;TEST\n%s" + hexlify(b"ansible")
+    def test_text_encrypted(self):
+        b_data = to_text(b"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible"))
         self.assertTrue(vault.is_encrypted(b_data))
 
-    def test_unicode_not_encrypted_py3(self):
-        if not six.PY3:
-            raise SkipTest()
-        data = u"ァ ア ィ イ ゥ ウ ェ エ ォ オ カ ガ キ ギ ク グ ケ "
-        self.assertRaises(TypeError, vault.is_encrypted, data)
-
-    def test_unicode_not_encrypted_py2(self):
-        if six.PY3:
-            raise SkipTest()
-        data = u"ァ ア ィ イ ゥ ウ ェ エ ォ オ カ ガ キ ギ ク グ ケ "
-        # py2 will take a unicode string, but that should always fails
+    def test_invalid_text_not_ascii(self):
+        data = u"$ANSIBLE_VAULT;9.9;TEST\n%s"% u"ァ ア ィ イ ゥ ウ ェ エ ォ オ カ ガ キ ギ ク グ ケ "
         self.assertFalse(vault.is_encrypted(data))
 
-    def test_unicode_is_encrypted_py3(self):
-        if not six.PY3:
-            raise SkipTest()
-        data = "$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible")
-        # should still be a type error
-        self.assertRaises(TypeError, vault.is_encrypted, data)
-
-    def test_unicode_is_encrypted_py2(self):
-        if six.PY3:
-            raise SkipTest()
-        data = u"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible")
-        # THis works, but arguably shouldn't...
-        self.assertTrue(vault.is_encrypted(data))
+    def test_invalid_bytes_not_ascii(self):
+        data = u"$ANSIBLE_VAULT;9.9;TEST\n%s"% u"ァ ア ィ イ ゥ ウ ェ エ ォ オ カ ガ キ ギ ク グ ケ "
+        b_data = to_bytes(data, encoding='utf-8')
+        self.assertFalse(vault.is_encrypted(b_data))
 
 
 class TestVaultIsEncryptedFile(unittest.TestCase):
-    def test_utf8_not_encrypted(self):
-        b_data = "foobar".encode('utf8')
+    def test_binary_file_handle_not_encrypted(self):
+        b_data = b"foobar"
         b_data_fo = io.BytesIO(b_data)
         self.assertFalse(vault.is_encrypted_file(b_data_fo))
 
-    def test_utf8_encrypted(self):
-        data = u"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible")
-        b_data = data.encode('utf8')
+    def test_text_file_handle_not_encrypted(self):
+        data = u"foobar"
+        data_fo = io.StringIO(data)
+        self.assertFalse(vault.is_encrypted_file(data_fo))
+
+    def test_binary_file_handle_encrypted(self):
+        b_data = b"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible")
         b_data_fo = io.BytesIO(b_data)
         self.assertTrue(vault.is_encrypted_file(b_data_fo))
 
-    def test_bytes_not_encrypted(self):
-        b_data = b"foobar"
+    def test_text_file_handle_encrypted(self):
+        data = u"$ANSIBLE_VAULT;9.9;TEST\n%s" % to_text(hexlify(b"ansible"))
+        data_fo = io.StringIO(data)
+        self.assertTrue(vault.is_encrypted_file(data_fo))
+
+    def test_binary_file_handle_invalid(self):
+        data = u"$ANSIBLE_VAULT;9.9;TEST\n%s" % u"ァ ア ィ イ ゥ ウ ェ エ ォ オ カ ガ キ ギ ク グ ケ "
+        b_data = to_bytes(data)
         b_data_fo = io.BytesIO(b_data)
         self.assertFalse(vault.is_encrypted_file(b_data_fo))
 
-    def test_bytes_encrypted(self):
-        b_data = b"$ANSIBLE_VAULT;9.9;TEST\n%s" + hexlify(b"ansible")
+    def test_text_file_handle_invalid(self):
+        data = u"$ANSIBLE_VAULT;9.9;TEST\n%s" % u"ァ ア ィ イ ゥ ウ ェ エ ォ オ カ ガ キ ギ ク グ ケ "
+        data_fo = io.StringIO(data)
+        self.assertFalse(vault.is_encrypted_file(data_fo))
+
+    def test_file_already_read_from_finds_header(self):
+        b_data = b"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible\ntesting\nfile pos")
         b_data_fo = io.BytesIO(b_data)
+        b_data_fo.read(42)  # Arbitrary number
         self.assertTrue(vault.is_encrypted_file(b_data_fo))
 
+    def test_file_already_read_from_saves_file_pos(self):
+        b_data = b"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible\ntesting\nfile pos")
+        b_data_fo = io.BytesIO(b_data)
+        b_data_fo.read(69)  # Arbitrary number
+        vault.is_encrypted_file(b_data_fo)
+        self.assertEqual(b_data_fo.tell(), 69)
+
+    def test_file_with_offset(self):
+        b_data = b"JUNK$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible\ntesting\nfile pos")
+        b_data_fo = io.BytesIO(b_data)
+        self.assertTrue(vault.is_encrypted_file(b_data_fo, start_pos=4))
+
+    def test_file_with_count(self):
+        b_data = b"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible\ntesting\nfile pos")
+        vault_length = len(b_data)
+        b_data = b_data + u'ァ ア'.encode('utf-8')
+        b_data_fo = io.BytesIO(b_data)
+        self.assertTrue(vault.is_encrypted_file(b_data_fo, count=vault_length))
+
+    def test_file_with_offset_and_count(self):
+        b_data = b"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible\ntesting\nfile pos")
+        vault_length = len(b_data)
+        b_data = b'JUNK' + b_data + u'ァ ア'.encode('utf-8')
+        b_data_fo = io.BytesIO(b_data)
+        self.assertTrue(vault.is_encrypted_file(b_data_fo, start_pos=4, count=vault_length))
+
 
 class TestVaultCipherAes256(unittest.TestCase):
+    def setUp(self):
+        self.vault_cipher = vault.VaultAES256()
+
     def test(self):
-        vault_cipher = vault.VaultAES256()
-        self.assertIsInstance(vault_cipher, vault.VaultAES256)
+        self.assertIsInstance(self.vault_cipher, vault.VaultAES256)
 
     # TODO: tag these as slow tests
     def test_create_key(self):
-        vault_cipher = vault.VaultAES256()
-        password = 'hunter42'
+        b_password = b'hunter42'
         b_salt = os.urandom(32)
-        b_key = vault_cipher.create_key(password=password, salt=b_salt, keylength=32, ivlength=16)
+        b_key = self.vault_cipher._create_key(b_password, b_salt, keylength=32, ivlength=16)
         self.assertIsInstance(b_key, six.binary_type)
 
     def test_create_key_known(self):
-        vault_cipher = vault.VaultAES256()
-        password = 'hunter42'
+        b_password = b'hunter42'
 
         # A fixed salt
         b_salt = b'q' * 32  # q is the most random letter.
-        b_key = vault_cipher.create_key(password=password, salt=b_salt, keylength=32, ivlength=16)
+        b_key = self.vault_cipher._create_key(b_password, b_salt, keylength=32, ivlength=16)
         self.assertIsInstance(b_key, six.binary_type)
 
         # verify we get the same answer
         # we could potentially run a few iterations of this and time it to see if it's roughly constant time
         #  and or that it exceeds some minimal time, but that would likely cause unreliable fails, esp in CI
-        b_key_2 = vault_cipher.create_key(password=password, salt=b_salt, keylength=32, ivlength=16)
+        b_key_2 = self.vault_cipher._create_key(b_password, b_salt, keylength=32, ivlength=16)
         self.assertIsInstance(b_key, six.binary_type)
         self.assertEqual(b_key, b_key_2)
 
     def test_is_equal_is_equal(self):
-        vault_cipher = vault.VaultAES256()
-        res = vault_cipher.is_equal(b'abcdefghijklmnopqrstuvwxyz', b'abcdefghijklmnopqrstuvwxyz')
-        self.assertTrue(res)
+        self.assertTrue(self.vault_cipher._is_equal(b'abcdefghijklmnopqrstuvwxyz', b'abcdefghijklmnopqrstuvwxyz'))
 
     def test_is_equal_unequal_length(self):
-        vault_cipher = vault.VaultAES256()
-        res = vault_cipher.is_equal(b'abcdefghijklmnopqrstuvwxyz', b'abcdefghijklmnopqrstuvwx and sometimes y')
-        self.assertFalse(res)
+        self.assertFalse(self.vault_cipher._is_equal(b'abcdefghijklmnopqrstuvwxyz', b'abcdefghijklmnopqrstuvwx and sometimes y'))
 
     def test_is_equal_not_equal(self):
-        vault_cipher = vault.VaultAES256()
-        res = vault_cipher.is_equal(b'abcdefghijklmnopqrstuvwxyz', b'AbcdefghijKlmnopQrstuvwxZ')
-        self.assertFalse(res)
+        self.assertFalse(self.vault_cipher._is_equal(b'abcdefghijklmnopqrstuvwxyz', b'AbcdefghijKlmnopQrstuvwxZ'))
 
     def test_is_equal_empty(self):
-        vault_cipher = vault.VaultAES256()
-        res = vault_cipher.is_equal(b'', b'')
-        self.assertTrue(res)
-
-    # NOTE: I'm not really sure what the method should do if it doesn't get bytes,
-    #       but this at least sees if it explodes (maybe it should?)
-    def test_is_equal_unicode_py3(self):
-        if not six.PY3:
-            raise SkipTest
-        vault_cipher = vault.VaultAES256()
-        self.assertRaises(TypeError, vault_cipher.is_equal,
-                          u'私はガラスを食べられます。それは私を傷つけません。',
-                          u'私はガラスを食べられます。それは私を傷つけません。')
+        self.assertTrue(self.vault_cipher._is_equal(b'', b''))
 
-    def test_is_equal_unicode_py2(self):
-        if not six.PY2:
-            raise SkipTest
-        vault_cipher = vault.VaultAES256()
-        res = vault_cipher.is_equal(u'私はガラスを食べられます。それは私を傷つけません。',
-                                    u'私はガラスを食べられます。それは私を傷つけません。')
-        self.assertTrue(res)
+    def test_is_equal_non_ascii_equal(self):
+        utf8_data = to_bytes(u'私はガラスを食べられます。それは私を傷つけません。')
+        self.assertTrue(self.vault_cipher._is_equal(utf8_data, utf8_data))
 
-    def test_is_equal_unicode_different(self):
-        vault_cipher = vault.VaultAES256()
-        res = vault_cipher.is_equal(u'私はガラスを食べられます。それは私を傷つけません。',
-                                    u'Pot să mănânc sticlă și ea nu mă rănește.')
-        self.assertFalse(res)
+    def test_is_equal_non_ascii_unequal(self):
+        utf8_data = to_bytes(u'私はガラスを食べられます。それは私を傷つけません。')
+        utf8_data2 = to_bytes(u'Pot să mănânc sticlă și ea nu mă rănește.')
 
+        # Test for the len optimization path
+        self.assertFalse(self.vault_cipher._is_equal(utf8_data, utf8_data2))
+        # Test for the slower, char by char comparison path
+        self.assertFalse(self.vault_cipher._is_equal(utf8_data, utf8_data[:-1] + b'P'))
 
-class TestVaultLib(unittest.TestCase):
+    def test_is_equal_non_bytes(self):
+        """ Anything not a byte string should raise a TypeError """
+        self.assertRaises(TypeError, self.vault_cipher._is_equal, u"One fish", b"two fish")
+        self.assertRaises(TypeError, self.vault_cipher._is_equal, b"One fish", u"two fish")
+        self.assertRaises(TypeError, self.vault_cipher._is_equal, 1, b"red fish")
+        self.assertRaises(TypeError, self.vault_cipher._is_equal, b"blue fish", 2)
 
-    def test_methods_exist(self):
-        v = VaultLib('ansible')
-        slots = ['is_encrypted',
-                 'encrypt',
-                 'decrypt',
-                 '_format_output',
-                 '_split_header',]
-        for slot in slots:
-            assert hasattr(v, slot), "VaultLib is missing the %s method" % slot
+
+class TestVaultLib(unittest.TestCase):
+    def setUp(self):
+        self.v = VaultLib('test-vault-password')
 
     def test_encrypt(self):
-        v = VaultLib(password='the_unit_test_password')
-        plaintext = u'Some text to encrypt.'
-        ciphertext = v.encrypt(plaintext)
+        plaintext = u'Some text to encrypt in a café'
+        b_vaulttext = self.v.encrypt(plaintext)
 
-        self.assertIsInstance(ciphertext, (bytes, str))
-        # TODO: assert something...
+        self.assertIsInstance(b_vaulttext, six.binary_type)
 
-    def test_is_encrypted(self):
-        v = VaultLib(None)
-        assert not v.is_encrypted("foobar".encode('utf-8')), "encryption check on plaintext failed"
-        data = u"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible")
-        assert v.is_encrypted(data.encode('utf-8')), "encryption check on headered text failed"
+        b_header = b'$ANSIBLE_VAULT;1.1;AES256\n'
+        self.assertEqual(b_vaulttext[:len(b_header)], b_header)
 
-    def test_is_encrypted_bytes(self):
-        v = VaultLib(None)
-        assert not v.is_encrypted(b"foobar"), "encryption check on plaintext failed"
-        data = b"$ANSIBLE_VAULT;9.9;TEST\n%s" + hexlify(b"ansible")
-        assert v.is_encrypted(data), "encryption check on headered text failed"
+    def test_encrypt_bytes(self):
+
+        plaintext = to_bytes(u'Some text to encrypt in a café')
+        b_vaulttext = self.v.encrypt(plaintext)
+
+        self.assertIsInstance(b_vaulttext, six.binary_type)
+
+        b_header = b'$ANSIBLE_VAULT;1.1;AES256\n'
+        self.assertEqual(b_vaulttext[:len(b_header)], b_header)
+
+    def test_is_encrypted(self):
+        self.assertFalse(self.v.is_encrypted(b"foobar"), msg="encryption check on plaintext yielded false positive")
+        b_data = b"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible")
+        self.assertTrue(self.v.is_encrypted(b_data), msg="encryption check on headered text failed")
 
     def test_format_output(self):
-        v = VaultLib('ansible')
-        v.cipher_name = "TEST"
-        sensitive_data = b"ansible"
-        data = v._format_output(sensitive_data)
-        lines = data.split(b'\n')
-        assert len(lines) > 1, "failed to properly add header"
-        header = to_bytes(lines[0])
-        assert header.endswith(b';TEST'), "header does end with cipher name"
-        header_parts = header.split(b';')
-        assert len(header_parts) == 3, "header has the wrong number of parts"
-        assert header_parts[0] == b'$ANSIBLE_VAULT', "header does not start with $ANSIBLE_VAULT"
-        assert header_parts[1] == v.b_version, "header version is incorrect"
-        assert header_parts[2] == b'TEST', "header does end with cipher name"
+        self.v.cipher_name = "TEST"
+        b_ciphertext = b"ansible"
+        b_vaulttext = self.v._format_output(b_ciphertext)
+        b_lines = b_vaulttext.split(b'\n')
+        self.assertGreater(len(b_lines), 1, msg="failed to properly add header")
+
+        b_header = b_lines[0]
+        self.assertTrue(b_header.endswith(b';TEST'), msg="header does not end with cipher name")
+
+        b_header_parts = b_header.split(b';')
+        self.assertEqual(len(b_header_parts), 3, msg="header has the wrong number of parts")
+        self.assertEqual(b_header_parts[0], b'$ANSIBLE_VAULT', msg="header does not start with $ANSIBLE_VAULT")
+        self.assertEqual(b_header_parts[1], self.v.b_version, msg="header version is incorrect")
+        self.assertEqual(b_header_parts[2], b'TEST', msg="header does not end with cipher name")
 
     def test_split_header(self):
-        v = VaultLib('ansible')
-        data = b"$ANSIBLE_VAULT;9.9;TEST\nansible"
-        rdata = v._split_header(data)
-        lines = rdata.split(b'\n')
-        assert lines[0] == b"ansible"
-        assert v.cipher_name == 'TEST', "cipher name was not set"
-        assert v.b_version == b"9.9"
+        b_vaulttext = b"$ANSIBLE_VAULT;9.9;TEST\nansible"
+        b_ciphertext = self.v._split_header(b_vaulttext)
+        b_lines = b_ciphertext.split(b'\n')
+        self.assertEqual(b_lines[0], b"ansible", msg="Payload was not properly split from the header")
+        self.assertEqual(self.v.cipher_name, u'TEST', msg="cipher name was not properly set")
+        self.assertEqual(self.v.b_version, b"9.9", msg="version was not properly set")
 
     def test_encrypt_decrypt_aes(self):
         if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
             raise SkipTest
-        v = VaultLib('ansible')
-        v.cipher_name = u'AES'
+        self.v.cipher_name = u'AES'
+        self.v.b_password = b'ansible'
         # AES encryption code has been removed, so this is old output for
         # AES-encrypted 'foobar' with password 'ansible'.
-        enc_data = b'$ANSIBLE_VAULT;1.1;AES\n53616c7465645f5fc107ce1ef4d7b455e038a13b053225776458052f8f8f332d554809d3f150bfa3\nfe3db930508b65e0ff5947e4386b79af8ab094017629590ef6ba486814cf70f8e4ab0ed0c7d2587e\n786a5a15efeb787e1958cbdd480d076c\n'
-        dec_data = v.decrypt(enc_data)
-        assert dec_data == b"foobar", "decryption failed"
+        b_vaulttext = b'''$ANSIBLE_VAULT;1.1;AES
+53616c7465645f5fc107ce1ef4d7b455e038a13b053225776458052f8f8f332d554809d3f150bfa3
+fe3db930508b65e0ff5947e4386b79af8ab094017629590ef6ba486814cf70f8e4ab0ed0c7d2587e
+786a5a15efeb787e1958cbdd480d076c
+'''
+        b_plaintext = self.v.decrypt(b_vaulttext)
+        self.assertEqual(b_plaintext, b"foobar", msg="decryption failed")
 
     def test_encrypt_decrypt_aes256(self):
         if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
             raise SkipTest
-        v = VaultLib('ansible')
-        v.cipher_name = 'AES256'
-        plaintext = "foobar"
-        enc_data = v.encrypt(plaintext)
-        dec_data = v.decrypt(enc_data)
-        assert enc_data != b"foobar", "encryption failed"
-        assert dec_data == b"foobar", "decryption failed"
+        self.v.cipher_name = u'AES256'
+        plaintext = u"foobar"
+        b_vaulttext = self.v.encrypt(plaintext)
+        b_plaintext = self.v.decrypt(b_vaulttext)
+        self.assertNotEqual(b_vaulttext, b"foobar", msg="encryption failed")
+        self.assertEqual(b_plaintext, b"foobar", msg="decryption failed")
 
     def test_encrypt_decrypt_aes256_existing_vault(self):
         if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
             raise SkipTest
-        v = VaultLib('test-vault-password')
-        v.cipher_name = 'AES256'
-        plaintext = b"Setec Astronomy"
-        enc_data = '''$ANSIBLE_VAULT;1.1;AES256
+        self.v.cipher_name = u'AES256'
+        b_orig_plaintext = b"Setec Astronomy"
+        vaulttext = u'''$ANSIBLE_VAULT;1.1;AES256
 33363965326261303234626463623963633531343539616138316433353830356566396130353436
 3562643163366231316662386565383735653432386435610a306664636137376132643732393835
 63383038383730306639353234326630666539346233376330303938323639306661313032396437
 6233623062366136310a633866373936313238333730653739323461656662303864663666653563
 3138'''
 
-        dec_data = v.decrypt(enc_data)
-        assert dec_data == plaintext, "decryption failed"
+        b_plaintext = self.v.decrypt(vaulttext)
+        self.assertEqual(b_plaintext, b_plaintext, msg="decryption failed")
+
+        b_vaulttext = to_bytes(vaulttext, encoding='ascii', errors='strict')
+        b_plaintext = self.v.decrypt(b_vaulttext)
+        self.assertEqual(b_plaintext, b_orig_plaintext, msg="decryption failed")
 
     def test_encrypt_decrypt_aes256_bad_hmac(self):
         # FIXME This test isn't working quite yet.
@@ -302,8 +316,7 @@ class TestVaultLib(unittest.TestCase):
 
         if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
             raise SkipTest
-        v = VaultLib('test-vault-password')
-        v.cipher_name = 'AES256'
+        self.v.cipher_name = 'AES256'
         # plaintext = "Setec Astronomy"
         enc_data = '''$ANSIBLE_VAULT;1.1;AES256
 33363965326261303234626463623963633531343539616138316433353830356566396130353436
@@ -312,7 +325,7 @@ class TestVaultLib(unittest.TestCase):
 6233623062366136310a633866373936313238333730653739323461656662303864663666653563
 3138'''
         b_data = to_bytes(enc_data, errors='strict', encoding='utf-8')
-        b_data = v._split_header(b_data)
+        b_data = self.v._split_header(b_data)
         foo = binascii.unhexlify(b_data)
         lines = foo.splitlines()
         # line 0 is salt, line 1 is hmac, line 2+ is ciphertext
@@ -331,31 +344,33 @@ class TestVaultLib(unittest.TestCase):
         b_ciphertext_data = binascii.hexlify(b_ciphertext)
         b_payload = b'\n'.join([b_salt, b_hmac, b_ciphertext_data])
         # reformat
-        b_invalid_ciphertext = v._format_output(b_payload)
+        b_invalid_ciphertext = self.v._format_output(b_payload)
 
         # assert we throw an error
-        v.decrypt(b_invalid_ciphertext)
+        self.v.decrypt(b_invalid_ciphertext)
 
     def test_encrypt_encrypted(self):
         if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
             raise SkipTest
-        v = VaultLib('ansible')
-        v.cipher_name = 'AES'
-        data = "$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(six.b("ansible"))
-        self.assertRaises(errors.AnsibleError, v.encrypt, data,)
+        self.v.cipher_name = u'AES'
+        b_vaulttext = b"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible")
+        vaulttext = to_text(b_vaulttext, errors='strict')
+        self.assertRaises(errors.AnsibleError, self.v.encrypt, b_vaulttext)
+        self.assertRaises(errors.AnsibleError, self.v.encrypt, vaulttext)
 
     def test_decrypt_decrypted(self):
         if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
             raise SkipTest
-        v = VaultLib('ansible')
-        data = "ansible"
-        self.assertRaises(errors.AnsibleError, v.decrypt, data)
+        plaintext = u"ansible"
+        self.assertRaises(errors.AnsibleError, self.v.decrypt, plaintext)
+
+        b_plaintext = b"ansible"
+        self.assertRaises(errors.AnsibleError, self.v.decrypt, b_plaintext)
 
     def test_cipher_not_set(self):
         # not setting the cipher should default to AES256
         if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
             raise SkipTest
-        v = VaultLib('ansible')
-        data = "ansible"
-        v.encrypt(data)
-        self.assertEquals(v.cipher_name, "AES256")
+        plaintext = u"ansible"
+        self.v.encrypt(plaintext)
+        self.assertEquals(self.v.cipher_name, "AES256")

test/units/parsing/vault/test_vault_editor.py

@@ -104,12 +104,8 @@ class TestVaultEditor(unittest.TestCase):
 
         self.assertTrue(os.path.exists(tmp_file.name))
 
-    @unittest.skipIf(sys.version_info[0] >= 3, "VaultAES still needs to be ported to Python 3")
     def test_decrypt_1_0(self):
-        """
-        Skip testing decrypting 1.0 files if we don't have access to AES, KDF or
-        Counter, or we are running on python3 since VaultAES hasn't been backported.
-        """
+        # Skip testing decrypting 1.0 files if we don't have access to AES, KDF or Counter.
         if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
             raise SkipTest
 
@@ -163,12 +159,8 @@ class TestVaultEditor(unittest.TestCase):
         assert error_hit is False, "error decrypting 1.0 file"
         assert fdata.strip() == "foo", "incorrect decryption of 1.0 file: %s" % fdata.strip()
 
-    @unittest.skipIf(sys.version_info[0] >= 3, "VaultAES still needs to be ported to Python 3")
     def test_rekey_migration(self):
-        """
-        Skip testing rekeying files if we don't have access to AES, KDF or
-        Counter, or we are running on python3 since VaultAES hasn't been backported.
-        """
+        # Skip testing rekeying files if we don't have access to AES, KDF or Counter.
         if not HAS_AES or not HAS_COUNTER or not HAS_PBKDF2:
             raise SkipTest
 
@@ -205,4 +197,4 @@ class TestVaultEditor(unittest.TestCase):
 
         assert vl.cipher_name == "AES256", "wrong cipher name set after rekey: %s" % vl.cipher_name
         assert error_hit is False, "error decrypting migrated 1.0 file"
-        assert dec_data.strip() == "foo", "incorrect decryption of rekeyed/migrated file: %s" % dec_data
+        assert dec_data.strip() == b"foo", "incorrect decryption of rekeyed/migrated file: %s" % dec_data

test/units/plugins/connections/test_connection.py

@@ -21,8 +21,7 @@ __metaclass__ = type
 
 from io import StringIO
 
-import mock
-
+from ansible.compat.tests import mock
 from ansible.compat.tests import unittest
 from ansible.errors import AnsibleError
 from ansible.playbook.play_context import PlayContext