Merge pull request #14875 from mattclay/plugins-unicode

Fix unicode handling in connection plugins.

lib/ansible/plugins/action/fetch.py

@@ -25,6 +25,7 @@ from ansible.plugins.action import ActionBase
 from ansible.utils.boolean import boolean
 from ansible.utils.hashing import checksum, checksum_s, md5, secure_hash
 from ansible.utils.path import makedirs_safe
+from ansible.utils.unicode import to_bytes
 
 
 class ActionModule(ActionBase):
@@ -158,7 +159,7 @@ class ActionModule(ActionBase):
                 self._connection.fetch_file(source, dest)
             else:
                 try:
-                    f = open(dest, 'w')
+                    f = open(to_bytes(dest, errors='strict'), 'w')
                     f.write(remote_data)
                     f.close()
                 except (IOError, OSError) as e:

lib/ansible/plugins/connection/chroot.py

@@ -91,7 +91,7 @@ class Connection(ConnectionBase):
         local_cmd = [self.chroot_cmd, self.chroot, executable, '-c', cmd]
 
         display.vvv("EXEC %s" % (local_cmd), host=self.chroot)
-        local_cmd = map(to_bytes, local_cmd)
+        local_cmd = [to_bytes(i, errors='strict') for i in local_cmd]
         p = subprocess.Popen(local_cmd, shell=False, stdin=stdin,
                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 

lib/ansible/plugins/connection/docker.py

@@ -127,7 +127,7 @@ class Connection(ConnectionBase):
         local_cmd = [self.docker_cmd, "exec", '-i', self._play_context.remote_addr, executable, '-c', cmd]
 
         display.vvv("EXEC %s" % (local_cmd,), host=self._play_context.remote_addr)
-        local_cmd = map(to_bytes, local_cmd)
+        local_cmd = [to_bytes(i, errors='strict') for i in local_cmd]
         p = subprocess.Popen(local_cmd, shell=False, stdin=subprocess.PIPE,
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
@@ -154,14 +154,14 @@ class Connection(ConnectionBase):
         display.vvv("PUT %s TO %s" % (in_path, out_path), host=self._play_context.remote_addr)
 
         out_path = self._prefix_login_path(out_path)
-        if not os.path.exists(in_path):
+        if not os.path.exists(to_bytes(in_path, errors='strict')):
             raise AnsibleFileNotFound(
                 "file or module does not exist: %s" % in_path)
 
         if self.can_copy_bothways:
             # only docker >= 1.8.1 can do this natively
             args = [ self.docker_cmd, "cp", in_path, "%s:%s" % (self._play_context.remote_addr, out_path) ]
-            args = map(to_bytes, args)
+            args = [to_bytes(i, errors='strict') for i in args]
             p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
             stdout, stderr = p.communicate()
             if p.returncode != 0:
@@ -173,8 +173,8 @@ class Connection(ConnectionBase):
             executable = C.DEFAULT_EXECUTABLE.split()[0] if C.DEFAULT_EXECUTABLE else '/bin/sh'
             args = [self.docker_cmd, "exec", "-i", self._play_context.remote_addr, executable, "-c",
                     "dd of=%s bs=%s" % (out_path, BUFSIZE)]
-            args = map(to_bytes, args)
+            args = [to_bytes(i, errors='strict') for i in args]
-            with open(in_path, 'rb') as in_file:
+            with open(to_bytes(in_path, errors='strict'), 'rb') as in_file:
                 try:
                     p = subprocess.Popen(args, stdin=in_file,
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -196,7 +196,7 @@ class Connection(ConnectionBase):
         out_dir = os.path.dirname(out_path)
 
         args = [self.docker_cmd, "cp", "%s:%s" % (self._play_context.remote_addr, in_path), out_dir]
-        args = map(to_bytes, args)
+        args = [to_bytes(i, errors='strict') for i in args]
 
         p = subprocess.Popen(args, stdin=subprocess.PIPE,
                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -205,7 +205,7 @@ class Connection(ConnectionBase):
         # Rename if needed
         actual_out_path = os.path.join(out_dir, os.path.basename(in_path))
         if actual_out_path != out_path:
-            os.rename(actual_out_path, out_path)
+            os.rename(to_bytes(actual_out_path, errors='strict'), to_bytes(out_path, errors='strict'))
 
     def close(self):
         """ Terminate the connection. Nothing to do for Docker"""

lib/ansible/plugins/connection/jail.py

@@ -111,7 +111,7 @@ class Connection(ConnectionBase):
         local_cmd = [self.jexec_cmd, self.jail, executable, '-c', cmd]
 
         display.vvv("EXEC %s" % (local_cmd,), host=self.jail)
-        local_cmd = map(to_bytes, local_cmd)
+        local_cmd = [to_bytes(i, errors='strict') for i in local_cmd]
         p = subprocess.Popen(local_cmd, shell=False, stdin=stdin,
                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
@@ -154,7 +154,7 @@ class Connection(ConnectionBase):
 
         out_path = pipes.quote(self._prefix_login_path(out_path))
         try:
-            with open(in_path, 'rb') as in_file:
+            with open(to_bytes(in_path, errors='strict'), 'rb') as in_file:
                 try:
                     p = self._buffered_exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), stdin=in_file)
                 except OSError:
@@ -180,7 +180,7 @@ class Connection(ConnectionBase):
         except OSError:
             raise AnsibleError("jail connection requires dd command in the jail")
 
-        with open(out_path, 'wb+') as out_file:
+        with open(to_bytes(out_path, errors='strict'), 'wb+') as out_file:
             try:
                 chunk = p.stdout.read(BUFSIZE)
                 while chunk:

lib/ansible/plugins/connection/libvirt_lxc.py

@@ -91,7 +91,7 @@ class Connection(ConnectionBase):
         local_cmd = [self.virsh, '-q', '-c', 'lxc:///', 'lxc-enter-namespace', self.lxc, '--', executable , '-c', cmd]
 
         display.vvv("EXEC %s" % (local_cmd,), host=self.lxc)
-        local_cmd = map(to_bytes, local_cmd)
+        local_cmd = [to_bytes(i, errors='strict') for i in local_cmd]
         p = subprocess.Popen(local_cmd, shell=False, stdin=stdin,
                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
@@ -127,7 +127,7 @@ class Connection(ConnectionBase):
 
         out_path = pipes.quote(self._prefix_login_path(out_path))
         try:
-            with open(in_path, 'rb') as in_file:
+            with open(to_bytes(in_path, errors='strict'), 'rb') as in_file:
                 try:
                     p = self._buffered_exec_command('dd of=%s bs=%s' % (out_path, BUFSIZE), stdin=in_file)
                 except OSError:
@@ -153,7 +153,7 @@ class Connection(ConnectionBase):
         except OSError:
             raise AnsibleError("chroot connection requires dd command in the chroot")
 
-        with open(out_path, 'wb+') as out_file:
+        with open(to_bytes(out_path, errors='strict'), 'wb+') as out_file:
             try:
                 chunk = p.stdout.read(BUFSIZE)
                 while chunk:

test/integration/test_connection.inventory

@@ -12,6 +12,28 @@ chroot-no-pipelining ansible_ssh_pipelining=false
 ansible_host=/
 ansible_connection=chroot
 
+[docker]
+docker-pipelining    ansible_ssh_pipelining=true
+docker-no-pipelining ansible_ssh_pipelining=false
+[docker:vars]
+ansible_host=ubuntu-latest
+ansible_connection=docker
+
+[libvirt_lxc]
+libvirt_lxc-pipelining    ansible_ssh_pipelining=true
+libvirt_lxc-no-pipelining ansible_ssh_pipelining=false
+[libvirt_lxc:vars]
+ansible_host=lv-ubuntu-wily-amd64
+ansible_connection=libvirt_lxc
+
+[jail]
+jail-pipelining    ansible_ssh_pipelining=true
+jail-no-pipelining ansible_ssh_pipelining=false
+[jail:vars]
+ansible_host=freebsd_10_2
+ansible_connection=jail
+ansible_python_interpreter=/usr/local/bin/python
+
 [ssh]
 ssh-pipelining    ansible_ssh_pipelining=true
 ssh-no-pipelining ansible_ssh_pipelining=false
@@ -27,5 +49,8 @@ ansible_host=localhost
 ansible_connection=paramiko_ssh
 
 [skip-during-build:children]
+docker
+libvirt_lxc
+jail
 ssh
 paramiko_ssh