Merge pull request #14797 from mattclay/unicode-fixes

Use to_bytes on filenames in filesystem calls.

lib/ansible/parsing/dataloader.py

@@ -118,7 +118,7 @@ class DataLoader():
 
     def path_exists(self, path):
         path = self.path_dwim(path)
-        return os.path.exists(to_bytes(path))
+        return os.path.exists(to_bytes(path, errors='strict'))
 
     def is_file(self, path):
         path = self.path_dwim(path)

lib/ansible/plugins/connection/chroot.py

@@ -127,7 +127,7 @@ class Connection(ConnectionBase):
 
         out_path = pipes.quote(self._prefix_login_path(out_path))
         try:
-            with open(in_path, 'rb') as in_file:
+            with open(to_bytes(in_path, errors='strict'), 'rb') as in_file:
                 try:
                     p = self._buffered_exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), stdin=in_file)
                 except OSError:
@@ -153,7 +153,7 @@ class Connection(ConnectionBase):
         except OSError:
             raise AnsibleError("chroot connection requires dd command in the chroot")
 
-        with open(out_path, 'wb+') as out_file:
+        with open(to_bytes(out_path, errors='strict'), 'wb+') as out_file:
             try:
                 chunk = p.stdout.read(BUFSIZE)
                 while chunk:

lib/ansible/plugins/connection/local.py

@@ -127,10 +127,10 @@ class Connection(ConnectionBase):
         super(Connection, self).put_file(in_path, out_path)
 
         display.vvv(u"{0} PUT {1} TO {2}".format(self._play_context.remote_addr, in_path, out_path))
-        if not os.path.exists(in_path):
+        if not os.path.exists(to_bytes(in_path, errors='strict')):
             raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_str(in_path)))
         try:
-            shutil.copyfile(in_path, out_path)
+            shutil.copyfile(to_bytes(in_path, errors='strict'), to_bytes(out_path, errors='strict'))
         except shutil.Error:
             raise AnsibleError("failed to copy: {0} and {1} are the same".format(to_str(in_path), to_str(out_path)))
         except IOError as e:

lib/ansible/plugins/connection/paramiko_ssh.py

@@ -43,6 +43,7 @@ from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound
 from ansible.plugins.connection import ConnectionBase
 from ansible.utils.path import makedirs_safe
+from ansible.utils.unicode import to_bytes
 
 try:
     from __main__ import display
@@ -322,7 +323,7 @@ class Connection(ConnectionBase):
 
         display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr)
 
-        if not os.path.exists(in_path):
+        if not os.path.exists(to_bytes(in_path, errors='strict')):
             raise AnsibleFileNotFound("file or module does not exist: %s" % in_path)
 
         try:
@@ -331,7 +332,7 @@ class Connection(ConnectionBase):
             raise AnsibleError("failed to open a SFTP connection (%s)" % e)
 
         try:
-            self.sftp.put(in_path, out_path)
+            self.sftp.put(to_bytes(in_path, errors='strict'), to_bytes(out_path, errors='strict'))
         except IOError:
             raise AnsibleError("failed to transfer file to %s" % out_path)
 
@@ -357,7 +358,7 @@ class Connection(ConnectionBase):
             raise AnsibleError("failed to open a SFTP connection (%s)", e)
 
         try:
-            self.sftp.get(in_path, out_path)
+            self.sftp.get(to_bytes(in_path, errors='strict'), to_bytes(out_path, errors='strict'))
         except IOError:
             raise AnsibleError("failed to transfer file from %s" % in_path)
 

lib/ansible/plugins/connection/ssh.py

@@ -621,7 +621,7 @@ class Connection(ConnectionBase):
         super(Connection, self).put_file(in_path, out_path)
 
         display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self.host)
-        if not os.path.exists(in_path):
+        if not os.path.exists(to_bytes(in_path, errors='strict')):
             raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_str(in_path)))
 
         # scp and sftp require square brackets for IPv6 addresses, but

lib/ansible/utils/hashing.py

@@ -21,6 +21,7 @@ __metaclass__ = type
 
 import os
 from ansible.errors import AnsibleError
+from ansible.utils.unicode import to_bytes
 
 # Note, sha1 is the only hash algorithm compatible with python2.4 and with
 # FIPS-140 mode (as of 11-2014)
@@ -54,12 +55,12 @@ def secure_hash_s(data, hash_func=sha1):
 def secure_hash(filename, hash_func=sha1):
     ''' Return a secure hash hex digest of local file, None if file is not present or a directory. '''
 
-    if not os.path.exists(filename) or os.path.isdir(filename):
+    if not os.path.exists(to_bytes(filename, errors='strict')) or os.path.isdir(to_bytes(filename, errors='strict')):
         return None
     digest = hash_func()
     blocksize = 64 * 1024
     try:
-        infile = open(filename, 'rb')
+        infile = open(to_bytes(filename, errors='strict'), 'rb')
         block = infile.read(blocksize)
         while block:
             digest.update(block)