Wait until later to convert to byte strings for output

Also some pre-emptive python3 compat and a code simplification

lib/ansible/executor/playbook_executor.py

@@ -186,21 +186,21 @@ class PlaybookExecutor:
         for h in hosts:
             t = self._tqm._stats.summarize(h)
 
-            self._display.display("%s : %s %s %s %s" % (
+            self._display.display(u"%s : %s %s %s %s" % (
                 hostcolor(h, t),
-                colorize('ok', t['ok'], 'green'),
-                colorize('changed', t['changed'], 'yellow'),
-                colorize('unreachable', t['unreachable'], 'red'),
-                colorize('failed', t['failures'], 'red')),
+                colorize(u'ok', t['ok'], 'green'),
+                colorize(u'changed', t['changed'], 'yellow'),
+                colorize(u'unreachable', t['unreachable'], 'red'),
+                colorize(u'failed', t['failures'], 'red')),
                 screen_only=True
             )
 
-            self._display.display("%s : %s %s %s %s" % (
+            self._display.display(u"%s : %s %s %s %s" % (
                 hostcolor(h, t, False),
-                colorize('ok', t['ok'], None),
-                colorize('changed', t['changed'], None),
-                colorize('unreachable', t['unreachable'], None),
-                colorize('failed', t['failures'], None)),
+                colorize(u'ok', t['ok'], None),
+                colorize(u'changed', t['changed'], None),
+                colorize(u'unreachable', t['unreachable'], None),
+                colorize(u'failed', t['failures'], None)),
                 log_only=True
             )
 

lib/ansible/utils/color.py

@@ -54,22 +54,22 @@ if C.ANSIBLE_FORCE_COLOR:
 # http://nezzen.net/2008/06/23/colored-text-in-python-using-ansi-escape-sequences/
 
 codeCodes = {
-    'black':     '0;30', 'bright gray':    '0;37',
-    'blue':      '0;34', 'white':          '1;37',
-    'green':     '0;32', 'bright blue':    '1;34',
-    'cyan':      '0;36', 'bright green':   '1;32',
-    'red':       '0;31', 'bright cyan':    '1;36',
-    'purple':    '0;35', 'bright red':     '1;31',
-    'yellow':    '0;33', 'bright purple':  '1;35',
-    'dark gray': '1;30', 'bright yellow':  '1;33',
-    'normal':    '0'
+    'black':     u'0;30', 'bright gray':    u'0;37',
+    'blue':      u'0;34', 'white':          u'1;37',
+    'green':     u'0;32', 'bright blue':    u'1;34',
+    'cyan':      u'0;36', 'bright green':   u'1;32',
+    'red':       u'0;31', 'bright cyan':    u'1;36',
+    'purple':    u'0;35', 'bright red':     u'1;31',
+    'yellow':    u'0;33', 'bright purple':  u'1;35',
+    'dark gray': u'1;30', 'bright yellow':  u'1;33',
+    'normal':    u'0'
 }
 
 def stringc(text, color):
     """String in color."""
 
     if ANSIBLE_COLOR:
-        return "\033["+codeCodes[color]+"m"+text+"\033[0m"
+        return u"\033[%sm%s\033[0m" % (codeCodes[color], text)
     else:
         return text
 
@@ -78,17 +78,17 @@ def stringc(text, color):
 def colorize(lead, num, color):
     """ Print 'lead' = 'num' in 'color' """
     if num != 0 and ANSIBLE_COLOR and color is not None:
-        return "%s%s%-15s" % (stringc(lead, color), stringc("=", color), stringc(str(num), color))
+        return u"%s%s%-15s" % (stringc(lead, color), stringc("=", color), stringc(str(num), color))
     else:
-        return "%s=%-4s" % (lead, str(num))
+        return u"%s=%-4s" % (lead, str(num))
 
 def hostcolor(host, stats, color=True):
     if ANSIBLE_COLOR and color:
         if stats['failures'] != 0 or stats['unreachable'] != 0:
-            return "%-37s" % stringc(host, 'red')
+            return u"%-37s" % stringc(host, 'red')
         elif stats['changed'] != 0:
-            return "%-37s" % stringc(host, 'yellow')
+            return u"%-37s" % stringc(host, 'yellow')
         else:
-            return "%-37s" % stringc(host, 'green')
-    return "%-26s" % host
+            return u"%-37s" % stringc(host, 'green')
+    return u"%-26s" % host
 

lib/ansible/utils/display.py

@@ -35,8 +35,14 @@ from multiprocessing import Lock
 from ansible import constants as C
 from ansible.errors import AnsibleError
 from ansible.utils.color import stringc
-from ansible.utils.unicode import to_bytes
+from ansible.utils.unicode import to_bytes, to_unicode
 
+try:
+    # Python 2
+    input = raw_input
+except NameError:
+    # Python 3
+    pass
 
 
 # These are module level as we currently fork and serialize the whole process and locks in the objects don't play well with that
@@ -96,30 +102,50 @@ class Display:
                 self.noncow = random.choice(cows)
 
     def display(self, msg, color=None, stderr=False, screen_only=False, log_only=False):
+        """ Display a message to the user
+
+        Note: msg *must* be a unicode string to prevent UnicodeError tracebacks.
+        """ 
 
         # FIXME: this needs to be implemented
         #msg = utils.sanitize_output(msg)
-        msg2 = self._safe_output(msg, stderr=stderr)
         if color:
-            msg2 = stringc(msg, color)
+            msg = stringc(msg, color)
 
         if not log_only:
-            b_msg2 = to_bytes(msg2)
+            if not msg.endswith(u'\n'):
+                msg2 = msg + u'\n'
+            else:
+                msg2 = msg
+
+            msg2 = to_bytes(msg2, encoding=self._output_encoding(stderr=stderr))
+            if sys.version_info >= (3,):
+                # Convert back to text string on python3
+                # We first convert to a byte string so that we get rid of
+                # characters that are invalid in the user's locale
+                msg2 = to_unicode(msg2, self._output_encoding(stderr=stderr))
+
             if not stderr:
-                print(b_msg2)
+                sys.stdout.write(msg2)
                 sys.stdout.flush()
             else:
-                print(b_msg2, file=sys.stderr)
+                sys.stderr.write(msg2)
                 sys.stderr.flush()
 
         if logger and not screen_only:
-            while msg.startswith("\n"):
-                msg = msg.replace("\n","")
-            b_msg = to_bytes(msg)
+            msg2 = msg.lstrip(u'\n')
+
+            msg2 = to_bytes(msg2)
+            if sys.version_info >= (3,):
+                # Convert back to text string on python3
+                # We first convert to a byte string so that we get rid of
+                # characters that are invalid in the user's locale
+                msg2 = to_unicode(msg2, self._output_encoding(stderr=stderr))
+
             if color == 'red':
-                logger.error(b_msg)
+                logger.error(msg2)
             else:
-                logger.info(b_msg)
+                logger.info(msg2)
 
     def vv(self, msg, host=None):
         return self.verbose(msg, host=host, caplevel=1)
@@ -230,21 +256,20 @@ class Display:
             self.display(new_msg, color='red', stderr=True)
             self._errors[new_msg] = 1
 
+    @staticmethod
     def prompt(self, msg):
-
-        return raw_input(self._safe_output(msg))
-
-    def _safe_output(self, msg, stderr=False):
-
-        encoding='utf-8'
-        if not stderr and sys.stdout.encoding:
-            encoding = sys.stdout.encoding
-        elif stderr and sys.stderr.encoding:
-            encoding = sys.stderr.encoding
-
-        msg = to_bytes(msg, encoding)
-
-        return msg
+        prompt_string = to_bytes(msg, encoding=self._output_encoding())
+        if sys.version_info >= (3,):
+            # Convert back into text on python3.  We do this double conversion
+            # to get rid of characters that are illegal in the user's locale
+            prompt_string = to_unicode(prompt_string)
+        return input(prompt_string)
+
+    @staticmethod
+    def _output_encoding(stderr=False):
+        if stderr:
+            return sys.stderr.encoding or 'utf-8'
+        return sys.stout.encoding or 'utf-8'
 
     def _set_column_width(self):
         if os.isatty(0):
@@ -252,4 +277,3 @@ class Display:
         else:
             tty_size = 0
         self.columns = max(79, tty_size)
-