From 46903c80faaf2a73056e7b9fbd52085291b4931f Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Wed, 6 Jan 2016 15:18:22 -0800 Subject: [PATCH] More fixes for unicode handling in the connection plugins. Tested that ssh, docker, local, lxc-libvirt, chroot all work with the updated unicode integration test. --- lib/ansible/inventory/script.py | 5 ++-- lib/ansible/plugins/connection/docker.py | 4 +-- lib/ansible/plugins/connection/local.py | 14 +++++------ lib/ansible/plugins/connection/ssh.py | 32 +++++++++++++----------- lib/ansible/plugins/connection/winrm.py | 2 +- lib/ansible/plugins/shell/sh.py | 4 +-- test/integration/unicode.yml | 30 ++++++++++++++++++++++ 7 files changed, 62 insertions(+), 29 deletions(-) diff --git a/lib/ansible/inventory/script.py b/lib/ansible/inventory/script.py index 6dfb1d2af08..cdfe676bcd2 100644 --- a/lib/ansible/inventory/script.py +++ b/lib/ansible/inventory/script.py @@ -31,6 +31,7 @@ from ansible.errors import AnsibleError from ansible.inventory.host import Host from ansible.inventory.group import Group from ansible.module_utils.basic import json_dict_bytes_to_unicode +from ansible.utils.unicode import to_str class InventoryScript: @@ -72,11 +73,11 @@ class InventoryScript: self.raw = self._loader.load(self.data) except Exception as e: sys.stderr.write(err + "\n") - raise AnsibleError("failed to parse executable inventory script results from {0}: {1}".format(self.filename, str(e))) + raise AnsibleError("failed to parse executable inventory script results from {0}: {1}".format(to_str(self.filename_, to_str(e))) if not isinstance(self.raw, Mapping): sys.stderr.write(err + "\n") - raise AnsibleError("failed to parse executable inventory script results from {0}: data needs to be formatted as a json dict".format(self.filename)) + raise AnsibleError("failed to parse executable inventory script results from {0}: data needs to be formatted as a json dict".format(to_str(self.filename))) self.raw = json_dict_bytes_to_unicode(self.raw) diff --git a/lib/ansible/plugins/connection/docker.py b/lib/ansible/plugins/connection/docker.py index ce556a1431b..130317f24aa 100644 --- a/lib/ansible/plugins/connection/docker.py +++ b/lib/ansible/plugins/connection/docker.py @@ -113,7 +113,7 @@ class Connection(ConnectionBase): """ Connect to the container. Nothing to do """ super(Connection, self)._connect() if not self._connected: - display.vvv("ESTABLISH DOCKER CONNECTION FOR USER: {0}".format( + display.vvv(u"ESTABLISH DOCKER CONNECTION FOR USER: {0}".format( self._play_context.remote_user, host=self._play_context.remote_addr) ) self._connected = True @@ -172,7 +172,7 @@ class Connection(ConnectionBase): # running containers, so we use docker exec to implement this executable = C.DEFAULT_EXECUTABLE.split()[0] if C.DEFAULT_EXECUTABLE else '/bin/sh' args = [self.docker_cmd, "exec", "-i", self._play_context.remote_addr, executable, "-c", - "dd of={0} bs={1}".format(out_path, BUFSIZE)] + "dd of=%s bs=%s" % (out_path, BUFSIZE)] args = map(to_bytes, args) with open(in_path, 'rb') as in_file: try: diff --git a/lib/ansible/plugins/connection/local.py b/lib/ansible/plugins/connection/local.py index 29b1e9a5ca2..5004c3698db 100644 --- a/lib/ansible/plugins/connection/local.py +++ b/lib/ansible/plugins/connection/local.py @@ -31,7 +31,7 @@ import ansible.constants as C from ansible.errors import AnsibleError, AnsibleFileNotFound from ansible.plugins.connection import ConnectionBase -from ansible.utils.unicode import to_bytes +from ansible.utils.unicode import to_bytes, to_str try: from __main__ import display @@ -57,7 +57,7 @@ class Connection(ConnectionBase): self._play_context.remote_user = getpass.getuser() if not self._connected: - display.vvv("ESTABLISH LOCAL CONNECTION FOR USER: {0}".format(self._play_context.remote_user, host=self._play_context.remote_addr)) + display.vvv(u"ESTABLISH LOCAL CONNECTION FOR USER: {0}".format(self._play_context.remote_user, host=self._play_context.remote_addr)) self._connected = True return self @@ -126,22 +126,22 @@ class Connection(ConnectionBase): super(Connection, self).put_file(in_path, out_path) - display.vvv("{0} PUT {1} TO {2}".format(self._play_context.remote_addr, in_path, out_path)) + display.vvv(u"{0} PUT {1} TO {2}".format(self._play_context.remote_addr, in_path, out_path)) if not os.path.exists(in_path): - raise AnsibleFileNotFound("file or module does not exist: {0}".format(in_path)) + raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_str(in_path))) try: shutil.copyfile(in_path, out_path) except shutil.Error: - raise AnsibleError("failed to copy: {0} and {1} are the same".format(in_path, out_path)) + raise AnsibleError("failed to copy: {0} and {1} are the same".format(to_str(in_path), to_str(out_path))) except IOError as e: - raise AnsibleError("failed to transfer file to {0}: {1}".format(out_path, e)) + raise AnsibleError("failed to transfer file to {0}: {1}".format(to_str(out_path), to_str(e))) def fetch_file(self, in_path, out_path): ''' fetch a file from local to local -- for copatibility ''' super(Connection, self).fetch_file(in_path, out_path) - display.vvv("{0} FETCH {1} TO {2}".format(self._play_context.remote_addr, in_path, out_path)) + display.vvv(u"{0} FETCH {1} TO {2}".format(self._play_context.remote_addr, in_path, out_path)) self.put_file(in_path, out_path) def close(self): diff --git a/lib/ansible/plugins/connection/ssh.py b/lib/ansible/plugins/connection/ssh.py index 074f6aaa8ae..0a0b2bb04bc 100644 --- a/lib/ansible/plugins/connection/ssh.py +++ b/lib/ansible/plugins/connection/ssh.py @@ -32,7 +32,7 @@ from ansible import constants as C from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound from ansible.plugins.connection import ConnectionBase from ansible.utils.path import unfrackpath, makedirs_safe -from ansible.utils.unicode import to_bytes, to_unicode +from ansible.utils.unicode import to_bytes, to_unicode, to_str from ansible.compat.six import text_type, binary_type try: @@ -197,7 +197,7 @@ class Connection(ConnectionBase): if user: self._add_args( "ANSIBLE_REMOTE_USER/remote_user/ansible_user/user/-u set", - ("-o", "User={0}".format(self._play_context.remote_user)) + ("-o", "User={0}".format(to_bytes(self._play_context.remote_user))) ) self._add_args( @@ -231,7 +231,7 @@ class Connection(ConnectionBase): raise AnsibleError("Cannot write to ControlPath %s" % cpdir) args = ("-o", "ControlPath={0}".format( - C.ANSIBLE_SSH_CONTROL_PATH % dict(directory=cpdir)) + to_bytes(C.ANSIBLE_SSH_CONTROL_PATH % dict(directory=cpdir))) ) self._add_args("found only ControlPersist; added ControlPath", args) @@ -320,7 +320,7 @@ class Connection(ConnectionBase): Starts the command and communicates with it until it ends. ''' - display_cmd = map(pipes.quote, cmd) + display_cmd = map(to_unicode, map(pipes.quote, cmd)) display.vvv(u'SSH: EXEC {0}'.format(u' '.join(display_cmd)), host=self.host) # Start the given command. If we don't need to pipeline data, we can try @@ -354,7 +354,7 @@ class Connection(ConnectionBase): if self._play_context.password: os.close(self.sshpass_pipe[0]) - os.write(self.sshpass_pipe[1], "{0}\n".format(self._play_context.password)) + os.write(self.sshpass_pipe[1], "{0}\n".format(to_bytes(self._play_context.password))) os.close(self.sshpass_pipe[1]) ## SSH state machine @@ -562,7 +562,7 @@ class Connection(ConnectionBase): super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable) - display.vvv("ESTABLISH SSH CONNECTION FOR USER: {0}".format(self._play_context.remote_user), host=self._play_context.remote_addr) + display.vvv(u"ESTABLISH SSH CONNECTION FOR USER: {0}".format(self._play_context.remote_user), host=self._play_context.remote_addr) # we can only use tty when we are not pipelining the modules. piping # data into /usr/bin/python inside a tty automatically invokes the @@ -630,44 +630,46 @@ class Connection(ConnectionBase): super(Connection, self).put_file(in_path, out_path) - display.vvv("PUT {0} TO {1}".format(in_path, out_path), host=self.host) + display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self.host) if not os.path.exists(in_path): - raise AnsibleFileNotFound("file or module does not exist: {0}".format(in_path)) + raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_str(in_path))) # scp and sftp require square brackets for IPv6 addresses, but # accept them for hostnames and IPv4 addresses too. host = '[%s]' % self.host if C.DEFAULT_SCP_IF_SSH: - cmd = self._build_command('scp', in_path, '{0}:{1}'.format(host, pipes.quote(out_path))) + cmd = self._build_command('scp', in_path, u'{0}:{1}'.format(host, pipes.quote(out_path))) in_data = None else: - cmd = self._build_command('sftp', host) - in_data = "put {0} {1}\n".format(pipes.quote(in_path), pipes.quote(out_path)) + cmd = self._build_command('sftp', to_bytes(host)) + in_data = u"put {0} {1}\n".format(pipes.quote(in_path), pipes.quote(out_path)) + in_data = to_bytes(in_data, nonstring='passthru') (returncode, stdout, stderr) = self._run(cmd, in_data) if returncode != 0: - raise AnsibleError("failed to transfer file to {0}:\n{1}\n{2}".format(out_path, stdout, stderr)) + raise AnsibleError("failed to transfer file to {0}:\n{1}\n{2}".format(to_str(out_path), to_str(stdout), to_str(stderr))) def fetch_file(self, in_path, out_path): ''' fetch a file from remote to local ''' super(Connection, self).fetch_file(in_path, out_path) - display.vvv("FETCH {0} TO {1}".format(in_path, out_path), host=self.host) + display.vvv(u"FETCH {0} TO {1}".format(in_path, out_path), host=self.host) # scp and sftp require square brackets for IPv6 addresses, but # accept them for hostnames and IPv4 addresses too. host = '[%s]' % self.host if C.DEFAULT_SCP_IF_SSH: - cmd = self._build_command('scp', '{0}:{1}'.format(host, pipes.quote(in_path)), out_path) + cmd = self._build_command('scp', u'{0}:{1}'.format(host, pipes.quote(in_path)), out_path) in_data = None else: cmd = self._build_command('sftp', host) - in_data = "get {0} {1}\n".format(pipes.quote(in_path), pipes.quote(out_path)) + in_data = u"get {0} {1}\n".format(pipes.quote(in_path), pipes.quote(out_path)) + in_data = to_bytes(in_data, nonstring='passthru') (returncode, stdout, stderr) = self._run(cmd, in_data) if returncode != 0: diff --git a/lib/ansible/plugins/connection/winrm.py b/lib/ansible/plugins/connection/winrm.py index ef863529b97..622231675d0 100644 --- a/lib/ansible/plugins/connection/winrm.py +++ b/lib/ansible/plugins/connection/winrm.py @@ -318,7 +318,7 @@ class Connection(ConnectionBase): local_sha1 = secure_hash(in_path) if not remote_sha1 == local_sha1: - raise AnsibleError("Remote sha1 hash {0} does not match local hash {1}".format(remote_sha1, local_sha1)) + raise AnsibleError("Remote sha1 hash {0} does not match local hash {1}".format(to_str(remote_sha1), to_str(local_sha1))) def fetch_file(self, in_path, out_path): diff --git a/lib/ansible/plugins/shell/sh.py b/lib/ansible/plugins/shell/sh.py index 7fbfa819ef1..8b20338a603 100644 --- a/lib/ansible/plugins/shell/sh.py +++ b/lib/ansible/plugins/shell/sh.py @@ -136,8 +136,8 @@ class ShellModule(object): shell_escaped_path = pipes.quote(path) test = "rc=flag; [ -r %(p)s ] %(shell_or)s rc=2; [ -f %(p)s ] %(shell_or)s rc=1; [ -d %(p)s ] %(shell_and)s rc=3; %(i)s -V 2>/dev/null %(shell_or)s rc=4; [ x\"$rc\" != \"xflag\" ] %(shell_and)s echo \"${rc} \"%(p)s %(shell_and)s exit 0" % dict(p=shell_escaped_path, i=python_interp, shell_and=self._SHELL_AND, shell_or=self._SHELL_OR) csums = [ - "({0} -c 'import hashlib; BLOCKSIZE = 65536; hasher = hashlib.sha1();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL), # Python > 2.4 (including python3) - "({0} -c 'import sha; BLOCKSIZE = 65536; hasher = sha.sha();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL), # Python == 2.4 + u"({0} -c 'import hashlib; BLOCKSIZE = 65536; hasher = hashlib.sha1();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL), # Python > 2.4 (including python3) + u"({0} -c 'import sha; BLOCKSIZE = 65536; hasher = sha.sha();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL), # Python == 2.4 ] cmd = (" %s " % self._SHELL_OR).join(csums) diff --git a/test/integration/unicode.yml b/test/integration/unicode.yml index f38bf8f5e86..74d5772264c 100644 --- a/test/integration/unicode.yml +++ b/test/integration/unicode.yml @@ -93,6 +93,36 @@ that: - "'Zażółć' in results.stdout_lines" + - name: Clean a temp directory + file: + path: /var/tmp/ansible_test_unicode_get_put + state: absent + + - name: Create a temp directory + file: + path: /var/tmp/ansible_test_unicode_get_put + state: directory + + - name: Create a file with a non-ascii filename + file: + path: /var/tmp/ansible_test_unicode_get_put/Zażółć + state: touch + delegate_to: localhost + + - name: Put with unicode filename + copy: + src: /var/tmp/ansible_test_unicode_get_put/Zażółć + dest: /var/tmp/ansible_test_unicode_get_put/Zażółć2 + + - name: Fetch with unicode filename + fetch: + src: /var/tmp/ansible_test_unicode_get_put/Zażółć2 + dest: /var/tmp/ansible_test_unicode_get_put/ + + - name: Clean a temp directory + file: + path: /var/tmp/ansible_test_unicode_get_put + state: absent - name: 'A play for hosts in group: ĪīĬĭ' hosts: 'ĪīĬĭ'